From a19451799f5a65a5b65cfd8f419d3f3e8699ff63 Mon Sep 17 00:00:00 2001
From: Kai Reinhard <K.Reinhard@micromata.de>
Date: Mon, 10 Dec 2018 23:09:03 +0000
Subject: [PATCH] Expiry
---
borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFilelistCache.java | 75 ++++++++++++++++++++++++-
borgbutler-core/src/main/java/de/micromata/borgbutler/config/Configuration.java | 6 +-
borgbutler-core/src/test/java/de/micromata/borgbutler/cache/ArchiveFilelistCacheTest.java | 70 +++++++++++++++++++----
README.adoc | 2
borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ButlerCache.java | 4 +
5 files changed, 136 insertions(+), 21 deletions(-)
diff --git a/README.adoc b/README.adoc
index d1e57fc..9c9decf 100644
--- a/README.adoc
+++ b/README.adoc
@@ -14,7 +14,7 @@
~/.borgbutler.json
----
{
- "cache_max_disc_size_mb" : 1000,
+ "cache_archive_content_max_disc_size_mb" : 1000,
"repo-configs" : [ {
"name" : "My-Laptop-Local-Backup",
"repo" : "/Volumes/backup/my-laptop-backup",
diff --git a/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFilelistCache.java b/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFilelistCache.java
index 2fcd07d..4ac4219 100644
--- a/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFilelistCache.java
+++ b/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFilelistCache.java
@@ -12,13 +12,21 @@
import org.slf4j.LoggerFactory;
import java.io.*;
+import java.nio.file.Files;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.attribute.FileTime;
import java.util.List;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
class ArchiveFilelistCache {
private static Logger log = LoggerFactory.getLogger(ArchiveFilelistCache.class);
private static final String CACHE_ARCHIVE_LISTS_BASENAME = "archive-content-";
private static final String CACHE_FILE_GZIP_EXTENSION = "gz";
private File cacheDir;
+ private int cacheArchiveContentMaxDiscSizeMB;
+ private long FILES_EXPIRE_TIME = 5 * 24 * 3660 * 1000; // Expires after 5 days.
@Getter
private Archive archive;
@@ -80,10 +88,70 @@
*/
public void cleanUp() {
File[] files = cacheDir.listFiles();
+ long currentMillis = System.currentTimeMillis();
+ for (File file : files) {
+ try {
+ if (!file.exists() || !isCacheFile(file)) continue;
+ // Get last access time of file:
+ FileTime time = Files.readAttributes(file.toPath(), BasicFileAttributes.class).lastAccessTime();
+ if (currentMillis - FILES_EXPIRE_TIME > time.toMillis()) {
+ log.info("Delete old cache file (last access " + time + " older than 5 days): " + file.getAbsolutePath());
+ file.delete();
+ }
+ } catch (IOException ex) {
+ log.error("Can't get last accesstime from cache files (ignore file '" + file.getAbsolutePath() + "'): " + ex.getMessage(), ex);
+ }
+ }
+ int sizeInMB = getCacheDiskSizeInMB(files);
+ if (sizeInMB > cacheArchiveContentMaxDiscSizeMB) {
+ log.info("Maximum size of cache files exceeded (" + sizeInMB + "MB > " + cacheArchiveContentMaxDiscSizeMB
+ + "MB). Deleting the old ones (with the oldest access)...");
+ }
+ SortedMap<FileTime, File> sortedFiles = new TreeMap<>();
+ for (File file : files) {
+ if (!file.exists() || !isCacheFile(file)) continue;
+ try {
+ // Get last access time of file:
+ FileTime time = Files.readAttributes(file.toPath(), BasicFileAttributes.class).lastAccessTime();
+ sortedFiles.put(time, file);
+ } catch (IOException ex) {
+ log.error("Can't get last accesstime from cache files (ignore file '" + file.getAbsolutePath() + "'): " + ex.getMessage(), ex);
+ }
+ }
+ for (Map.Entry<FileTime, File> entry : sortedFiles.entrySet()) {
+ FileTime time = entry.getKey();
+ File file = entry.getValue();
+ if (!file.exists() || !isCacheFile(file)) continue;
+ log.info("Deleting cache file (last access " + time + "): " + file.getAbsolutePath());
+ file.delete();
+ int newSizeInMB = getCacheDiskSizeInMB(files);
+ if (newSizeInMB < cacheArchiveContentMaxDiscSizeMB) {
+ log.info("New cache size is " + newSizeInMB + "MB. (" + (sizeInMB - newSizeInMB) + "MB deleted.)");
+ break;
+ }
+ }
+ }
+
+ public int getCacheDiskSizeInMB() {
+ return getCacheDiskSizeInMB(cacheDir.listFiles());
+ }
+
+ private int getCacheDiskSizeInMB(File[] files) {
+ int sizeInMB = 0;
+ for (File file : files) {
+ if (!file.exists()) continue;
+ if (!isCacheFile(file)) continue;
+ sizeInMB += (int) (file.length() / 1048576); // In MB
+ }
+ return sizeInMB;
+ }
+
+ public void removeAllCacheFiles() {
+ File[] files = cacheDir.listFiles();
for (File file : files) {
if (isCacheFile(file)) {
- log.info("Processing cache file: " + file.getAbsolutePath());
- //file.delete();
+ log.info("Deleting cache file: " + file.getAbsolutePath());
+ file.delete();
}
}
}
@@ -93,8 +161,9 @@
+ "-" + repoConfig.getRepo() + "-" + archive.getArchive() + ".gz", true));
}
- ArchiveFilelistCache(File cacheDir) {
+ ArchiveFilelistCache(File cacheDir, int cacheArchiveContentMaxDiscSizeMB) {
this.cacheDir = cacheDir;
+ this.cacheArchiveContentMaxDiscSizeMB = cacheArchiveContentMaxDiscSizeMB;
}
private boolean isCacheFile(File file) {
diff --git a/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ButlerCache.java b/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ButlerCache.java
index fb1e288..6f093c4 100644
--- a/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ButlerCache.java
+++ b/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ButlerCache.java
@@ -2,6 +2,7 @@
import de.micromata.borgbutler.BorgCommands;
import de.micromata.borgbutler.config.BorgRepoConfig;
+import de.micromata.borgbutler.config.Configuration;
import de.micromata.borgbutler.config.ConfigurationHandler;
import de.micromata.borgbutler.json.borg.*;
import org.apache.commons.collections4.CollectionUtils;
@@ -103,9 +104,10 @@
}
private ButlerCache() {
+ Configuration configuration = ConfigurationHandler.getConfiguration();
this.jcsCache = JCSCache.getInstance();
this.repoInfoCacheAccess = jcsCache.getJCSCache("repoInfo");
this.repoListCacheAccess = jcsCache.getJCSCache("repoList");
- this.archiveFilelistCache = new ArchiveFilelistCache(getCacheDir());
+ this.archiveFilelistCache = new ArchiveFilelistCache(getCacheDir(), configuration.getCacheArchiveContentMaxDiscSizeMB());
}
}
diff --git a/borgbutler-core/src/main/java/de/micromata/borgbutler/config/Configuration.java b/borgbutler-core/src/main/java/de/micromata/borgbutler/config/Configuration.java
index 67868a4..f1c9f04 100644
--- a/borgbutler-core/src/main/java/de/micromata/borgbutler/config/Configuration.java
+++ b/borgbutler-core/src/main/java/de/micromata/borgbutler/config/Configuration.java
@@ -10,11 +10,11 @@
@Getter
private String borgCommand = "borg";
/**
- * Default is 200 MB (approximately).
+ * Default is 100 MB (approximately).
*/
@Getter
- @JsonProperty("cache_max_disc_size_mb")
- private int cacheMaxDiscSizeMB = 200;
+ @JsonProperty("cache_archive_content_max_disc_size_mb")
+ private int cacheArchiveContentMaxDiscSizeMB = 100;
@Getter
@JsonProperty("repo-configs")
diff --git a/borgbutler-core/src/test/java/de/micromata/borgbutler/cache/ArchiveFilelistCacheTest.java b/borgbutler-core/src/test/java/de/micromata/borgbutler/cache/ArchiveFilelistCacheTest.java
index 9a85eec..129a0d2 100644
--- a/borgbutler-core/src/test/java/de/micromata/borgbutler/cache/ArchiveFilelistCacheTest.java
+++ b/borgbutler-core/src/test/java/de/micromata/borgbutler/cache/ArchiveFilelistCacheTest.java
@@ -8,7 +8,11 @@
import org.slf4j.LoggerFactory;
import java.io.File;
+import java.io.IOException;
import java.lang.reflect.Field;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.FileTime;
import java.util.ArrayList;
import java.util.List;
@@ -21,15 +25,12 @@
@Test
void readWriteTest() throws Exception {
- List<FilesystemItem> list = new ArrayList<>();
- for (int i = 0; i < 1000000; i++) {
- list.add(create(i));
- }
- ArchiveFilelistCache cache = new ArchiveFilelistCache(new File("out"));
+ List<FilesystemItem> list = createList(1000000);
+ ArchiveFilelistCache cache = new ArchiveFilelistCache(new File("out"), 100);
+ cache.removeAllCacheFiles();
BorgRepoConfig repoConfig = new BorgRepoConfig();
repoConfig.setRepo("repo");
- Archive archive = new Archive();
- set(archive, "archive", "archive-2018-12-10");
+ Archive archive = createArchive("2018-12-10");
log.info("Saving " + list.size() + " items to out dir.");
cache.save(repoConfig, archive, list);
log.info("Saving done.");
@@ -40,23 +41,53 @@
for (int i = 0; i < filesystemItems.length; i++) {
assertEquals(list.get(i).getPath(), filesystemItems[i].getPath());
}
+ cache.removeAllCacheFiles();
}
@Test
void readWriteEmptyTest() throws Exception {
List<FilesystemItem> list = new ArrayList<>();
- ArchiveFilelistCache cache = new ArchiveFilelistCache(new File("out"));
+ ArchiveFilelistCache cache = new ArchiveFilelistCache(new File("out"), 100);
+ cache.removeAllCacheFiles();
BorgRepoConfig repoConfig = new BorgRepoConfig();
repoConfig.setRepo("repo");
- Archive archive = new Archive();
- set(archive, "archive", "archive-2018-12-09");
- if (cache.getFile(repoConfig, archive).exists()) {
- cache.getFile(repoConfig, archive).delete();
- }
+ Archive archive = createArchive("2018-12-09");
assertNull(cache.load(repoConfig, archive));
cache.save(repoConfig, archive, list);
FilesystemItem[] filesystemItems = cache.load(repoConfig, archive);
assertNull(cache.load(repoConfig, archive));
+ cache.removeAllCacheFiles();
+ }
+
+ @Test
+ void cleanUpTest() throws Exception {
+ List<FilesystemItem> list = createList(1000000);
+ ArchiveFilelistCache cache = new ArchiveFilelistCache(new File("out"), 5);
+ cache.removeAllCacheFiles();
+ BorgRepoConfig repoConfig = new BorgRepoConfig();
+ repoConfig.setRepo("repo");
+
+ long millis = System.currentTimeMillis();
+
+ Archive archive = createArchive("2018-11-20");
+ cache.save(repoConfig, archive, list);
+ setLastAccessTime(cache.getFile(repoConfig, archive), millis - 10 * 3600000); // Fake lastAccessTime - 10 h
+
+ archive = createArchive("2018-11-21");
+ cache.save(repoConfig, archive, list);
+ setLastAccessTime(cache.getFile(repoConfig, archive), millis - 60000); // Fake lastAccessTime - 1min
+
+ cache.save(repoConfig, archive, list);
+ cache.cleanUp();
+ cache.removeAllCacheFiles();
+ }
+
+ private List<FilesystemItem> createList(int number) throws Exception {
+ List<FilesystemItem> list = new ArrayList<>();
+ for (int i = 0; i < 1000000; i++) {
+ list.add(create(i));
+ }
+ return list;
}
private FilesystemItem create(int i) throws Exception {
@@ -73,4 +104,17 @@
f1.set(obj, value);
return this;
}
+
+ private Archive createArchive(String time) throws Exception {
+ Archive archive = new Archive();
+ set(archive, "archive", "archive-" + time);
+ set(archive, "time", time);
+ return archive;
+ }
+
+ private void setLastAccessTime(File file, long accessTime) throws IOException {
+ Path path = file.toPath();
+ FileTime fileTime = FileTime.fromMillis(accessTime);
+ Files.setAttribute(path, "lastAccessTime", fileTime);
+ }
}
--
Gitblit v1.10.0