From 219ec32448572da39d629dfcd9c37ec362378ffd Mon Sep 17 00:00:00 2001
From: Kai Reinhard <K.Reinhard@micromata.de>
Date: Mon, 07 Jan 2019 22:50:08 +0000
Subject: [PATCH] Stores recent gz files in MemoryCache (doesn't improve performance on ssd drives).
---
borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFilelistCache.java | 112 ++++++++++++---------------
borgbutler-core/src/main/java/de/micromata/borgbutler/cache/memory/MemoryCache.java | 60 +++++++++++++++
borgbutler-core/src/main/java/de/micromata/borgbutler/cache/memory/MemoryCacheObject.java | 30 +++++++
3 files changed, 140 insertions(+), 62 deletions(-)
diff --git a/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFilelistCache.java b/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFilelistCache.java
index 964d839..1bdebee 100644
--- a/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFilelistCache.java
+++ b/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFilelistCache.java
@@ -1,5 +1,7 @@
package de.micromata.borgbutler.cache;
+import de.micromata.borgbutler.cache.memory.MemoryCache;
+import de.micromata.borgbutler.cache.memory.MemoryCacheObject;
import de.micromata.borgbutler.config.BorgRepoConfig;
import de.micromata.borgbutler.data.Archive;
import de.micromata.borgbutler.data.FileSystemFilter;
@@ -9,6 +11,8 @@
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.output.ByteArrayOutputStream;
import org.apache.commons.lang3.StringUtils;
import org.nustaq.serialization.FSTConfiguration;
import org.nustaq.serialization.FSTObjectInput;
@@ -33,19 +37,19 @@
private static Logger log = LoggerFactory.getLogger(ArchiveFilelistCache.class);
private static final String CACHE_ARCHIVE_LISTS_BASENAME = "archive-content-";
private static final String CACHE_FILE_GZIP_EXTENSION = ".gz";
- private static final int MAX_NUMBER_OF_RECENT_ENTRIES = 2;
+ private static final int MAX_SIZE_MEMORY_CACHE = 30 * 1024 * 1024; // 50 MB
private File cacheDir;
private int cacheArchiveContentMaxDiscSizeMB;
private long FILES_EXPIRE_TIME = 7 * 24 * 3660 * 1000; // Expires after 7 days.
// For avoiding concurrent writing of same files (e. g. after the user has pressed a button twice).
private Set<File> savingFiles = new HashSet<>();
- private Recents recents = new Recents(MAX_NUMBER_OF_RECENT_ENTRIES);
+ private MemoryCache<Archive, RecentEntry> recents = new MemoryCache<>(MAX_SIZE_MEMORY_CACHE);
final FSTConfiguration conf = FSTConfiguration.createDefaultConfiguration();
ArchiveFilelistCache(File cacheDir, int cacheArchiveContentMaxDiscSizeMB) {
this.cacheDir = cacheDir;
this.cacheArchiveContentMaxDiscSizeMB = cacheArchiveContentMaxDiscSizeMB;
- conf.registerClass(BorgFilesystemItem.class);
+ conf.registerClass(Integer.class, BorgFilesystemItem.class);
conf.setShareReferences(false);
}
@@ -65,18 +69,33 @@
Collections.sort(filesystemItems); // Sort by path.
}
log.info("Saving archive content as file list: " + file.getAbsolutePath());
+ boolean ok = false;
+
+ int fileNumber = -1;
try (FSTObjectOutput outputStream
= new FSTObjectOutput(new BufferedOutputStream(new GzipCompressorOutputStream(new FileOutputStream(file))), conf)) {
outputStream.writeObject(filesystemItems.size(), Integer.class);
Iterator<BorgFilesystemItem> it = filesystemItems.iterator();
while (it.hasNext()) {
BorgFilesystemItem item = it.next();
+ item.setFileNumber(++fileNumber);
outputStream.writeObject(item, BorgFilesystemItem.class);
}
outputStream.writeObject("EOF");
+ ok = true;
} catch (IOException ex) {
log.error("Error while writing file list '" + file.getAbsolutePath() + "': " + ex.getMessage(), ex);
}
+ if (ok) {
+ // Storing current read gz archive in memory cache (recents):
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try {
+ FileUtils.copyFile(file, baos);
+ recents.add(new RecentEntry(archive, baos.toByteArray()));
+ } catch (IOException ex) {
+ log.error("Error while writing gz archive to memory cache: " + ex.getMessage(), ex);
+ }
+ }
} finally {
synchronized (savingFiles) {
savingFiles.remove(file);
@@ -151,22 +170,26 @@
} catch (IOException ex) {
log.error("Can't set lastModifiedTime on file '" + file.getAbsolutePath() + "'. Pruning old cache files may not work.");
}
+ RecentEntry recentEntry = recents.getRecent(archive);
+ byte[] bytes = null;
+ if (recentEntry != null) {
+ bytes = recentEntry.serializedGz;
+ } else {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try {
+ FileUtils.copyFile(file, baos);
+ bytes = baos.toByteArray();
+ recents.add(new RecentEntry(archive, bytes));
+ } catch (IOException ex) {
+ log.error("Error while restoring file: " + file.getAbsolutePath() + ": " + ex.getMessage(), ex);
+ return null;
+ }
+ }
List<BorgFilesystemItem> list = new ArrayList<>();
- ByteArrayOutputStream out = new ByteArrayOutputStream();
-/* try {
- IOUtils.copy(new BufferedInputStream(new GzipCompressorInputStream(new FileInputStream(file))), out);
- } catch (IOException ex) {
-
- }*/
- byte[] bytes = out.toByteArray();
- out = null;
- try (FSTObjectInput inputStream = new FSTObjectInput(new BufferedInputStream(new GzipCompressorInputStream(new FileInputStream(file))), conf)) {
+ try (FSTObjectInput inputStream = new FSTObjectInput(new BufferedInputStream(new GzipCompressorInputStream(new ByteArrayInputStream(bytes))), conf)) {
int size = (Integer) inputStream.readObject(Integer.class);
- int fileNumber = -1;
for (int i = 0; i < size; i++) {
- ++fileNumber;
BorgFilesystemItem item = (BorgFilesystemItem) inputStream.readObject(BorgFilesystemItem.class);
- item.setFileNumber(fileNumber);
if (filter == null || filter.matches(item)) {
list.add(item);
if (filter != null && filter.isFinished()) break;
@@ -175,7 +198,7 @@
} catch (Exception ex) {
log.error("Error while reading file list '" + file.getAbsolutePath() + "': " + ex.getMessage(), ex);
}
- Collections.sort(list); // Sort by path (if archive list order wasn't correct).
+
log.info("Loading done.");
return filter(list, filter);
}
@@ -283,57 +306,22 @@
return file.getName().startsWith(CACHE_ARCHIVE_LISTS_BASENAME);
}
- private class Recents {
- private RecentEntry[] recents;
- private int pos = 0;
- private int size;
+ private class RecentEntry extends MemoryCacheObject<Archive> {
+ private byte[] serializedGz;
- private Recents(int size) {
- this.size = size;
- recents = new RecentEntry[size];
+ @Override
+ protected boolean matches(Archive identifier) {
+ return StringUtils.equals(this.getIdentifier().getId(), identifier.getId());
}
- private void add(RecentEntry entry) {
- synchronized (this) {
- log.info("Add recent at position #" + pos + ": " + entry.archive.getName());
- recents[pos++] = entry;
- if (pos >= size) pos = 0;
- }
+ @Override
+ protected int getSize() {
+ return serializedGz != null ? serializedGz.length : 0;
}
- private RecentEntry getRecent(Archive archive) {
- synchronized (this) {
- for (RecentEntry entry : recents) {
- if (entry != null && entry.matches(archive)) {
- return entry;
- }
- }
- }
- log.info("No recent entry found for archive: " + archive.getName());
- return null;
- }
-
- private void removeOldestEntry() {
- int oldestEntry = pos + 1 >= size ? 0 : pos + 1;
- recents[oldestEntry] = null;
- log.info("Remove oldest entry #" + oldestEntry + ": " + recents[oldestEntry]);
- }
- }
-
- private class RecentEntry {
- private Archive archive;
- private List<BorgFilesystemItem> filesystemItems;
-
- private boolean matches(Archive archive) {
- if (this.archive == null || archive == null) {
- return false;
- }
- return StringUtils.equals(this.archive.getId(), archive.getId());
- }
-
- private RecentEntry(Archive archive, List<BorgFilesystemItem> filesystemItems) {
- this.archive = archive;
- this.filesystemItems = filesystemItems;
+ private RecentEntry(Archive archive, byte[] serializedGz) {
+ super(archive);
+ this.serializedGz = serializedGz;
}
}
}
diff --git a/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/memory/MemoryCache.java b/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/memory/MemoryCache.java
new file mode 100644
index 0000000..8b1ba37
--- /dev/null
+++ b/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/memory/MemoryCache.java
@@ -0,0 +1,60 @@
+package de.micromata.borgbutler.cache.memory;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class MemoryCache<I, M extends MemoryCacheObject<I>> {
+ private static Logger log = LoggerFactory.getLogger(MemoryCache.class);
+ private long maxMemorySize;
+ private List<M> recents = new ArrayList<>();
+
+ public MemoryCache(long maxMemorySize) {
+ this.maxMemorySize = maxMemorySize;
+ }
+
+ public M getRecent(I identifier) {
+ synchronized (recents) {
+ for (M entry : recents) {
+ if (entry._matches(identifier)) {
+ log.debug("Getting recent entry: " + entry.getIdentifier());
+ return entry;
+ }
+ }
+ }
+ return null;
+ }
+
+ public void add(M newEntry) {
+ if (newEntry.getSize() > maxMemorySize) {
+ // Object to large for storing in memory.
+ return;
+ }
+ synchronized (recents) {
+ recents.add(newEntry);
+ log.debug("Add new recent entry: " + newEntry.getIdentifier());
+ int size;
+ while (true) {
+ size = 0;
+ MemoryCacheObject<I> oldest = null;
+ for (MemoryCacheObject<I> entry : recents) {
+ if (oldest == null || entry.lastAcess < oldest.lastAcess) {
+ oldest = entry;
+ }
+ size += entry.getSize();
+ }
+ if (oldest == null) {
+ break;
+ }
+ if (size >= maxMemorySize) {
+ log.debug("Removing oldest recent entry: " + oldest.getIdentifier());
+ recents.remove(oldest);
+ } else {
+ break;
+ }
+ }
+ }
+ }
+}
diff --git a/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/memory/MemoryCacheObject.java b/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/memory/MemoryCacheObject.java
new file mode 100644
index 0000000..cbb802c
--- /dev/null
+++ b/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/memory/MemoryCacheObject.java
@@ -0,0 +1,30 @@
+package de.micromata.borgbutler.cache.memory;
+
+import lombok.Getter;
+
+public abstract class MemoryCacheObject<I> {
+ @Getter
+ private I identifier;
+ long lastAcess;
+
+ abstract protected boolean matches(I identifier);
+
+ abstract protected int getSize();
+
+ boolean _matches(I identifier) {
+ if (this.identifier == null || identifier == null) {
+ return false;
+ }
+ if (matches(identifier)) {
+ lastAcess = System.currentTimeMillis();
+ return true;
+ }
+ return false;
+ }
+
+ public MemoryCacheObject(I identifier) {
+ this.identifier = identifier;
+ lastAcess = System.currentTimeMillis();
+ }
+
+}
--
Gitblit v1.10.0