From da9780f28f20d3a0cb2ee326b562b9e4f6cdbafe Mon Sep 17 00:00:00 2001
From: Kai Reinhard <K.Reinhard@micromata.de>
Date: Mon, 07 Jan 2019 21:26:33 +0000
Subject: [PATCH] Serialization via FST lib.
---
borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFilelistCache.java | 112 ++++++++++---------------------------
borgbutler-core/build.gradle | 3
borgbutler-core/src/test/java/de/micromata/borgbutler/cache/ArchiveFilelistCacheTest.java | 39 -------------
3 files changed, 33 insertions(+), 121 deletions(-)
diff --git a/borgbutler-core/build.gradle b/borgbutler-core/build.gradle
index cb8f89f..e78a6a7 100644
--- a/borgbutler-core/build.gradle
+++ b/borgbutler-core/build.gradle
@@ -12,7 +12,8 @@
compile group: 'org.apache.commons', name: 'commons-collections4', version: '4.2'
compile group: 'org.apache.commons', name: 'commons-compress', version: '1.18'
compile group: 'org.apache.commons', name: 'commons-jcs-core', version: '2.2.1'
-
+ // https://mvnrepository.com/artifact/de.ruedigermoeller/fst
+ compile group: 'de.ruedigermoeller', name: 'fst', version: '2.57' // Serialization (faster than Java built-in)
compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.9.6'
compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: '2.9.6'
diff --git a/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFilelistCache.java b/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFilelistCache.java
index a2f924e..964d839 100644
--- a/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFilelistCache.java
+++ b/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFilelistCache.java
@@ -9,9 +9,10 @@
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
+import org.nustaq.serialization.FSTConfiguration;
+import org.nustaq.serialization.FSTObjectInput;
+import org.nustaq.serialization.FSTObjectOutput;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,6 +40,14 @@
// For avoiding concurrent writing of same files (e. g. after the user has pressed a button twice).
private Set<File> savingFiles = new HashSet<>();
private Recents recents = new Recents(MAX_NUMBER_OF_RECENT_ENTRIES);
+ final FSTConfiguration conf = FSTConfiguration.createDefaultConfiguration();
+
+ ArchiveFilelistCache(File cacheDir, int cacheArchiveContentMaxDiscSizeMB) {
+ this.cacheDir = cacheDir;
+ this.cacheArchiveContentMaxDiscSizeMB = cacheArchiveContentMaxDiscSizeMB;
+ conf.registerClass(BorgFilesystemItem.class);
+ conf.setShareReferences(false);
+ }
public void save(BorgRepoConfig repoConfig, Archive archive, List<BorgFilesystemItem> filesystemItems) {
if (CollectionUtils.isEmpty(filesystemItems)) {
@@ -56,12 +65,13 @@
Collections.sort(filesystemItems); // Sort by path.
}
log.info("Saving archive content as file list: " + file.getAbsolutePath());
- try (ObjectOutputStream outputStream = new ObjectOutputStream(new BufferedOutputStream(new GzipCompressorOutputStream(new FileOutputStream(file))))) {
- outputStream.writeObject(filesystemItems.size());
+ try (FSTObjectOutput outputStream
+ = new FSTObjectOutput(new BufferedOutputStream(new GzipCompressorOutputStream(new FileOutputStream(file))), conf)) {
+ outputStream.writeObject(filesystemItems.size(), Integer.class);
Iterator<BorgFilesystemItem> it = filesystemItems.iterator();
while (it.hasNext()) {
BorgFilesystemItem item = it.next();
- outputStream.writeObject(item);
+ outputStream.writeObject(item, BorgFilesystemItem.class);
}
outputStream.writeObject("EOF");
} catch (IOException ex) {
@@ -72,8 +82,6 @@
savingFiles.remove(file);
}
}
- compactPathes(filesystemItems);
- recents.add(new RecentEntry(archive, filesystemItems));
log.info("Saving done.");
}
@@ -109,10 +117,6 @@
* @return
*/
public List<BorgFilesystemItem> load(BorgRepoConfig repoConfig, Archive archive, FileSystemFilter filter) {
- RecentEntry recent = recents.getRecent(archive);
- if (recent != null) {
- return filter(recent.filesystemItems, filter);
- }
File file = getFile(repoConfig, archive);
if (!file.exists()) {
return null;
@@ -140,9 +144,6 @@
log.error("File '" + file.getAbsolutePath() + "' doesn't exist. Can't get archive content files.");
return null;
}
- if (archive != null) {
- recents.removeOldestEntry();
- }
log.info("Loading archive content as file list from: " + file.getAbsolutePath());
try {
// Set last modified time of file:
@@ -151,38 +152,30 @@
log.error("Can't set lastModifiedTime on file '" + file.getAbsolutePath() + "'. Pruning old cache files may not work.");
}
List<BorgFilesystemItem> list = new ArrayList<>();
- try (ObjectInputStream inputStream = new ObjectInputStream(new BufferedInputStream(new GzipCompressorInputStream(new FileInputStream(file))))) {
- Object obj = inputStream.readObject();
- if (!(obj instanceof Integer)) {
- log.error("Can't load archive content. Integer expected, but received: " + obj.getClass());
- return null;
- }
- int size = (Integer) obj;
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+/* try {
+ IOUtils.copy(new BufferedInputStream(new GzipCompressorInputStream(new FileInputStream(file))), out);
+ } catch (IOException ex) {
+
+ }*/
+ byte[] bytes = out.toByteArray();
+ out = null;
+ try (FSTObjectInput inputStream = new FSTObjectInput(new BufferedInputStream(new GzipCompressorInputStream(new FileInputStream(file))), conf)) {
+ int size = (Integer) inputStream.readObject(Integer.class);
int fileNumber = -1;
for (int i = 0; i < size; i++) {
++fileNumber;
- obj = inputStream.readObject();
- if (obj instanceof BorgFilesystemItem) {
- BorgFilesystemItem item = (BorgFilesystemItem) obj;
- item.setFileNumber(fileNumber);
- if (filter == null || filter.matches(item)) {
- list.add(item);
- if (filter != null && filter.isFinished()) break;
- }
- } else {
- log.error("Can't load archive content. FilesystemItem expected, but received: "
- + (obj != null ? obj.getClass() : "null")
- + " at position " + i + ".");
- return null;
+ BorgFilesystemItem item = (BorgFilesystemItem) inputStream.readObject(BorgFilesystemItem.class);
+ item.setFileNumber(fileNumber);
+ if (filter == null || filter.matches(item)) {
+ list.add(item);
+ if (filter != null && filter.isFinished()) break;
}
}
- } catch (IOException | ClassNotFoundException ex) {
+ } catch (Exception ex) {
log.error("Error while reading file list '" + file.getAbsolutePath() + "': " + ex.getMessage(), ex);
}
Collections.sort(list); // Sort by path (if archive list order wasn't correct).
- if (archive != null) {
- recents.add(new RecentEntry(archive, list));
- }
log.info("Loading done.");
return filter(list, filter);
}
@@ -286,11 +279,6 @@
true));
}
- ArchiveFilelistCache(File cacheDir, int cacheArchiveContentMaxDiscSizeMB) {
- this.cacheDir = cacheDir;
- this.cacheArchiveContentMaxDiscSizeMB = cacheArchiveContentMaxDiscSizeMB;
- }
-
private boolean isCacheFile(File file) {
return file.getName().startsWith(CACHE_ARCHIVE_LISTS_BASENAME);
}
@@ -348,43 +336,5 @@
this.filesystemItems = filesystemItems;
}
}
-
- static void compactPathes(List<BorgFilesystemItem> items) {
- long origSize = 0;
- long compactSize = 0;
- String currentDir = null;
- for (BorgFilesystemItem item : items) {
- String path = item.getPath();
- origSize += path.length();
- if (currentDir != null && path.startsWith(currentDir)) {
- String compactPath = "#" + path.substring(currentDir.length());
- item.setPath(compactPath);
- }
- if ("d".equals(item.getType())) {
- currentDir = path;
- } else {
- currentDir = FilenameUtils.getPath(path);
- }
- compactSize += item.getPath().length();
- //log.info(StringUtils.rightPad(path, 40) + " -> " + item.getPath());
- }
- log.info("Compact pathes: " + FileUtils.byteCountToDisplaySize(origSize) + " -> " + FileUtils.byteCountToDisplaySize(compactSize));
- }
-
- static void expandPathes(List<BorgFilesystemItem> items) {
- String currentDir = null;
- for (BorgFilesystemItem item : items) {
- String path = item.getPath();
- if (path.startsWith("#")) {
- item.setPath(currentDir + path.substring(1));
- }
- if ("d".equals(item.getType())) {
- currentDir = item.getPath();
- } else {
- currentDir = FilenameUtils.getPath(item.getPath());
- }
- log.info(StringUtils.rightPad(path, 40) + " -> " + item.getPath());
- }
- }
}
diff --git a/borgbutler-core/src/test/java/de/micromata/borgbutler/cache/ArchiveFilelistCacheTest.java b/borgbutler-core/src/test/java/de/micromata/borgbutler/cache/ArchiveFilelistCacheTest.java
index 5ff965c..7568e77 100644
--- a/borgbutler-core/src/test/java/de/micromata/borgbutler/cache/ArchiveFilelistCacheTest.java
+++ b/borgbutler-core/src/test/java/de/micromata/borgbutler/cache/ArchiveFilelistCacheTest.java
@@ -125,45 +125,6 @@
cache.removeAllCacheFiles();
}
- @Test
- void reducePath() {
- List<BorgFilesystemItem> list = new ArrayList<>();
- String[] items = {
- "d home",
- "d home/kai",
- "- home/kai/abc.txt",
- "d home/kai/Documents",
- "- home/kai/Documents/test1.doc",
- "- home/kai/Documents/test2.doc",
- "- home/kai/image.doc",
- "- home/kai/Files/tmp/a.txt",
- "- home/kai/Files/tmp/b.txt",
- "- home/kai/Java.pdf",
- "- home/kai/Movies/a.mov",
- "- home/pete/txt.mov",
- "- home/steve/1/2/3/4/5/6/7/test.txt",
- "- home/test.txt",
- "- home/xaver/1/2/3/4/5/6/7/test.txt",
- "- opt/local/test.txt",
- "- opt/local/test2.txt",
- };
- for (String path : items) {
- BorgFilesystemItem item = new BorgFilesystemItem().setPath(path.substring(2));
- if (path.startsWith("d")) {
- item.setType("d");
- } else {
- item.setType("-");
- }
- list.add(item);
- }
- ArchiveFilelistCache.compactPathes(list);
- ArchiveFilelistCache.expandPathes(list);
- for (int i = 0; i < list.size(); i++) {
- assertEquals(items[i].substring(2), list.get(i).getPath());
- }
-
- }
-
private List<BorgFilesystemItem> createList(int number) throws Exception {
List<BorgFilesystemItem> list = new ArrayList<>();
for (int i = 0; i < 1000000; i++) {
--
Gitblit v1.10.0