From da9780f28f20d3a0cb2ee326b562b9e4f6cdbafe Mon Sep 17 00:00:00 2001
From: Kai Reinhard <K.Reinhard@micromata.de>
Date: Mon, 07 Jan 2019 21:26:33 +0000
Subject: [PATCH] Serialization via FST lib.
---
borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFilelistCache.java | 112 +++++++++++++++----------------------------------------
1 files changed, 31 insertions(+), 81 deletions(-)
diff --git a/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFilelistCache.java b/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFilelistCache.java
index a2f924e..964d839 100644
--- a/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFilelistCache.java
+++ b/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFilelistCache.java
@@ -9,9 +9,10 @@
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
+import org.nustaq.serialization.FSTConfiguration;
+import org.nustaq.serialization.FSTObjectInput;
+import org.nustaq.serialization.FSTObjectOutput;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,6 +40,14 @@
// For avoiding concurrent writing of same files (e. g. after the user has pressed a button twice).
private Set<File> savingFiles = new HashSet<>();
private Recents recents = new Recents(MAX_NUMBER_OF_RECENT_ENTRIES);
+ final FSTConfiguration conf = FSTConfiguration.createDefaultConfiguration();
+
+ ArchiveFilelistCache(File cacheDir, int cacheArchiveContentMaxDiscSizeMB) {
+ this.cacheDir = cacheDir;
+ this.cacheArchiveContentMaxDiscSizeMB = cacheArchiveContentMaxDiscSizeMB;
+ conf.registerClass(BorgFilesystemItem.class);
+ conf.setShareReferences(false);
+ }
public void save(BorgRepoConfig repoConfig, Archive archive, List<BorgFilesystemItem> filesystemItems) {
if (CollectionUtils.isEmpty(filesystemItems)) {
@@ -56,12 +65,13 @@
Collections.sort(filesystemItems); // Sort by path.
}
log.info("Saving archive content as file list: " + file.getAbsolutePath());
- try (ObjectOutputStream outputStream = new ObjectOutputStream(new BufferedOutputStream(new GzipCompressorOutputStream(new FileOutputStream(file))))) {
- outputStream.writeObject(filesystemItems.size());
+ try (FSTObjectOutput outputStream
+ = new FSTObjectOutput(new BufferedOutputStream(new GzipCompressorOutputStream(new FileOutputStream(file))), conf)) {
+ outputStream.writeObject(filesystemItems.size(), Integer.class);
Iterator<BorgFilesystemItem> it = filesystemItems.iterator();
while (it.hasNext()) {
BorgFilesystemItem item = it.next();
- outputStream.writeObject(item);
+ outputStream.writeObject(item, BorgFilesystemItem.class);
}
outputStream.writeObject("EOF");
} catch (IOException ex) {
@@ -72,8 +82,6 @@
savingFiles.remove(file);
}
}
- compactPathes(filesystemItems);
- recents.add(new RecentEntry(archive, filesystemItems));
log.info("Saving done.");
}
@@ -109,10 +117,6 @@
* @return
*/
public List<BorgFilesystemItem> load(BorgRepoConfig repoConfig, Archive archive, FileSystemFilter filter) {
- RecentEntry recent = recents.getRecent(archive);
- if (recent != null) {
- return filter(recent.filesystemItems, filter);
- }
File file = getFile(repoConfig, archive);
if (!file.exists()) {
return null;
@@ -140,9 +144,6 @@
log.error("File '" + file.getAbsolutePath() + "' doesn't exist. Can't get archive content files.");
return null;
}
- if (archive != null) {
- recents.removeOldestEntry();
- }
log.info("Loading archive content as file list from: " + file.getAbsolutePath());
try {
// Set last modified time of file:
@@ -151,38 +152,30 @@
log.error("Can't set lastModifiedTime on file '" + file.getAbsolutePath() + "'. Pruning old cache files may not work.");
}
List<BorgFilesystemItem> list = new ArrayList<>();
- try (ObjectInputStream inputStream = new ObjectInputStream(new BufferedInputStream(new GzipCompressorInputStream(new FileInputStream(file))))) {
- Object obj = inputStream.readObject();
- if (!(obj instanceof Integer)) {
- log.error("Can't load archive content. Integer expected, but received: " + obj.getClass());
- return null;
- }
- int size = (Integer) obj;
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+/* try {
+ IOUtils.copy(new BufferedInputStream(new GzipCompressorInputStream(new FileInputStream(file))), out);
+ } catch (IOException ex) {
+
+ }*/
+ byte[] bytes = out.toByteArray();
+ out = null;
+ try (FSTObjectInput inputStream = new FSTObjectInput(new BufferedInputStream(new GzipCompressorInputStream(new FileInputStream(file))), conf)) {
+ int size = (Integer) inputStream.readObject(Integer.class);
int fileNumber = -1;
for (int i = 0; i < size; i++) {
++fileNumber;
- obj = inputStream.readObject();
- if (obj instanceof BorgFilesystemItem) {
- BorgFilesystemItem item = (BorgFilesystemItem) obj;
- item.setFileNumber(fileNumber);
- if (filter == null || filter.matches(item)) {
- list.add(item);
- if (filter != null && filter.isFinished()) break;
- }
- } else {
- log.error("Can't load archive content. FilesystemItem expected, but received: "
- + (obj != null ? obj.getClass() : "null")
- + " at position " + i + ".");
- return null;
+ BorgFilesystemItem item = (BorgFilesystemItem) inputStream.readObject(BorgFilesystemItem.class);
+ item.setFileNumber(fileNumber);
+ if (filter == null || filter.matches(item)) {
+ list.add(item);
+ if (filter != null && filter.isFinished()) break;
}
}
- } catch (IOException | ClassNotFoundException ex) {
+ } catch (Exception ex) {
log.error("Error while reading file list '" + file.getAbsolutePath() + "': " + ex.getMessage(), ex);
}
Collections.sort(list); // Sort by path (if archive list order wasn't correct).
- if (archive != null) {
- recents.add(new RecentEntry(archive, list));
- }
log.info("Loading done.");
return filter(list, filter);
}
@@ -286,11 +279,6 @@
true));
}
- ArchiveFilelistCache(File cacheDir, int cacheArchiveContentMaxDiscSizeMB) {
- this.cacheDir = cacheDir;
- this.cacheArchiveContentMaxDiscSizeMB = cacheArchiveContentMaxDiscSizeMB;
- }
-
private boolean isCacheFile(File file) {
return file.getName().startsWith(CACHE_ARCHIVE_LISTS_BASENAME);
}
@@ -348,43 +336,5 @@
this.filesystemItems = filesystemItems;
}
}
-
- static void compactPathes(List<BorgFilesystemItem> items) {
- long origSize = 0;
- long compactSize = 0;
- String currentDir = null;
- for (BorgFilesystemItem item : items) {
- String path = item.getPath();
- origSize += path.length();
- if (currentDir != null && path.startsWith(currentDir)) {
- String compactPath = "#" + path.substring(currentDir.length());
- item.setPath(compactPath);
- }
- if ("d".equals(item.getType())) {
- currentDir = path;
- } else {
- currentDir = FilenameUtils.getPath(path);
- }
- compactSize += item.getPath().length();
- //log.info(StringUtils.rightPad(path, 40) + " -> " + item.getPath());
- }
- log.info("Compact pathes: " + FileUtils.byteCountToDisplaySize(origSize) + " -> " + FileUtils.byteCountToDisplaySize(compactSize));
- }
-
- static void expandPathes(List<BorgFilesystemItem> items) {
- String currentDir = null;
- for (BorgFilesystemItem item : items) {
- String path = item.getPath();
- if (path.startsWith("#")) {
- item.setPath(currentDir + path.substring(1));
- }
- if ("d".equals(item.getType())) {
- currentDir = item.getPath();
- } else {
- currentDir = FilenameUtils.getPath(item.getPath());
- }
- log.info(StringUtils.rightPad(path, 40) + " -> " + item.getPath());
- }
- }
}
--
Gitblit v1.10.0