From 5f559bf4282731412429c7b8cbc4104a9f1b6dff Mon Sep 17 00:00:00 2001
From: Kai Reinhard <K.Reinhard@micromata.de>
Date: Tue, 11 Dec 2018 06:46:42 +0000
Subject: [PATCH] Pruning of expired and oldest files after cache size exceeded works now.
---
borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFilelistCache.java | 38 +++++++++++++-----
borgbutler-core/src/test/java/de/micromata/borgbutler/cache/ArchiveFilelistCacheTest.java | 59 +++++++++++++++++++++++++----
2 files changed, 77 insertions(+), 20 deletions(-)
diff --git a/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFilelistCache.java b/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFilelistCache.java
index 4ac4219..9b355cc 100644
--- a/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFilelistCache.java
+++ b/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFilelistCache.java
@@ -26,7 +26,7 @@
private static final String CACHE_FILE_GZIP_EXTENSION = "gz";
private File cacheDir;
private int cacheArchiveContentMaxDiscSizeMB;
- private long FILES_EXPIRE_TIME = 5 * 24 * 3660 * 1000; // Expires after 5 days.
+ private long FILES_EXPIRE_TIME = 7 * 24 * 3660 * 1000; // Expires after 7 days.
@Getter
private Archive archive;
@@ -50,6 +50,13 @@
log.info("Saving done.");
}
+ /**
+ * Will load and touch the archive file if exist. The file will be touched (last modified time will be set to now)
+ * for pruning oldest cache files. The last modified time will be the time of the last usage.
+ * @param repoConfig
+ * @param archive
+ * @return
+ */
public FilesystemItem[] load(BorgRepoConfig repoConfig, Archive archive) {
File file = getFile(repoConfig, archive);
if (!file.exists()) {
@@ -57,6 +64,12 @@
}
log.info("Loading archive content as file list from: " + file.getAbsolutePath());
FilesystemItem[] list = null;
+ try {
+ // Set last modified time of file:
+ Files.setAttribute(file.toPath(), "lastModifiedTime", FileTime.fromMillis(System.currentTimeMillis()));
+ } catch (IOException ex) {
+ log.error("Can't set lastModifiedTime on file '" + file.getAbsolutePath() + "'. Pruning old cache files may not work.");
+ }
try (ObjectInputStream inputStream = new ObjectInputStream(new BufferedInputStream(new GzipCompressorInputStream(new FileInputStream(file))))) {
Object obj = inputStream.readObject();
if (!(obj instanceof Integer)) {
@@ -84,7 +97,7 @@
/**
* Deletes archive contents older than 7 days and deletes the oldest archive contents if the max cache size is
- * exceeded.
+ * exceeded. The last modified time of a file is equals to the last usage by {@link #load(BorgRepoConfig, Archive)}.
*/
public void cleanUp() {
File[] files = cacheDir.listFiles();
@@ -92,37 +105,40 @@
for (File file : files) {
try {
if (!file.exists() || !isCacheFile(file)) continue;
- // Get last access time of file:
- FileTime time = Files.readAttributes(file.toPath(), BasicFileAttributes.class).lastAccessTime();
+ // Get last modified time of file:
+ FileTime time = Files.readAttributes(file.toPath(), BasicFileAttributes.class).lastModifiedTime();
if (currentMillis - FILES_EXPIRE_TIME > time.toMillis()) {
- log.info("Delete old cache file (last access " + time + " older than 5 days): " + file.getAbsolutePath());
+ log.info("Delete expired cache file (last usage " + time + " older than 7 days): " + file.getAbsolutePath());
file.delete();
}
} catch (IOException ex) {
- log.error("Can't get last accesstime from cache files (ignore file '" + file.getAbsolutePath() + "'): " + ex.getMessage(), ex);
+ log.error("Can't get last modified time from cache files (ignore file '" + file.getAbsolutePath() + "'): " + ex.getMessage(), ex);
}
}
int sizeInMB = getCacheDiskSizeInMB(files);
if (sizeInMB > cacheArchiveContentMaxDiscSizeMB) {
log.info("Maximum size of cache files exceeded (" + sizeInMB + "MB > " + cacheArchiveContentMaxDiscSizeMB
- + "MB). Deleting the old ones (with the oldest access)...");
+ + "MB). Deleting the old ones (with the oldest usage)...");
+ } else {
+ // Nothing to clean up anymore.
+ return;
}
SortedMap<FileTime, File> sortedFiles = new TreeMap<>();
for (File file : files) {
if (!file.exists() || !isCacheFile(file)) continue;
try {
- // Get last access time of file:
- FileTime time = Files.readAttributes(file.toPath(), BasicFileAttributes.class).lastAccessTime();
+ // Get last modified time of file:
+ FileTime time = Files.readAttributes(file.toPath(), BasicFileAttributes.class).lastModifiedTime();
sortedFiles.put(time, file);
} catch (IOException ex) {
- log.error("Can't get last accesstime from cache files (ignore file '" + file.getAbsolutePath() + "'): " + ex.getMessage(), ex);
+ log.error("Can't get last modified time from cache files (ignore file '" + file.getAbsolutePath() + "'): " + ex.getMessage(), ex);
}
}
for (Map.Entry<FileTime, File> entry : sortedFiles.entrySet()) {
FileTime time = entry.getKey();
File file = entry.getValue();
if (!file.exists() || !isCacheFile(file)) continue;
- log.info("Deleting cache file (last access " + time + "): " + file.getAbsolutePath());
+ log.info("Deleting cache file (last usage " + time + "): " + file.getAbsolutePath());
file.delete();
int newSizeInMB = getCacheDiskSizeInMB(files);
if (newSizeInMB < cacheArchiveContentMaxDiscSizeMB) {
diff --git a/borgbutler-core/src/test/java/de/micromata/borgbutler/cache/ArchiveFilelistCacheTest.java b/borgbutler-core/src/test/java/de/micromata/borgbutler/cache/ArchiveFilelistCacheTest.java
index 129a0d2..84ac637 100644
--- a/borgbutler-core/src/test/java/de/micromata/borgbutler/cache/ArchiveFilelistCacheTest.java
+++ b/borgbutler-core/src/test/java/de/micromata/borgbutler/cache/ArchiveFilelistCacheTest.java
@@ -16,8 +16,7 @@
import java.util.ArrayList;
import java.util.List;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.*;
public class ArchiveFilelistCacheTest {
@@ -60,9 +59,9 @@
}
@Test
- void cleanUpTest() throws Exception {
+ void cleanUpMaximumSizeTest() throws Exception {
List<FilesystemItem> list = createList(1000000);
- ArchiveFilelistCache cache = new ArchiveFilelistCache(new File("out"), 5);
+ ArchiveFilelistCache cache = new ArchiveFilelistCache(new File("out"), 3);
cache.removeAllCacheFiles();
BorgRepoConfig repoConfig = new BorgRepoConfig();
repoConfig.setRepo("repo");
@@ -71,14 +70,56 @@
Archive archive = createArchive("2018-11-20");
cache.save(repoConfig, archive, list);
- setLastAccessTime(cache.getFile(repoConfig, archive), millis - 10 * 3600000); // Fake lastAccessTime - 10 h
+ File oldestFile = cache.getFile(repoConfig, archive);
+ setLastModificationTime(oldestFile, millis - 10 * 3600000); // Fake lastModifiedTime - 10 h
archive = createArchive("2018-11-21");
cache.save(repoConfig, archive, list);
- setLastAccessTime(cache.getFile(repoConfig, archive), millis - 60000); // Fake lastAccessTime - 1min
+ File newestFile = cache.getFile(repoConfig, archive);
+ setLastModificationTime(newestFile, millis - 60000); // Fake lastModifiedTime - 1min
+ archive = createArchive("2018-11-22");
cache.save(repoConfig, archive, list);
+ File file = cache.getFile(repoConfig, archive);
+ setLastModificationTime(file, millis - 3600000); // Fake lastModifiedTime - 1 hour
+
+ assertTrue(oldestFile.exists());
+ assertTrue(newestFile.exists());
+ assertTrue(file.exists());
+
cache.cleanUp();
+ assertFalse(oldestFile.exists());
+ assertFalse(file.exists());
+ assertTrue(newestFile.exists());
+ cache.removeAllCacheFiles();
+ }
+
+ @Test
+ void cleanUpExpiredTest() throws Exception {
+ List<FilesystemItem> list = createList(1000);
+ ArchiveFilelistCache cache = new ArchiveFilelistCache(new File("out"), 3);
+ cache.removeAllCacheFiles();
+ BorgRepoConfig repoConfig = new BorgRepoConfig();
+ repoConfig.setRepo("repo");
+
+ long millis = System.currentTimeMillis();
+
+ Archive archive = createArchive("2018-10-20");
+ cache.save(repoConfig, archive, list);
+ File notExpiredFile = cache.getFile(repoConfig, archive);
+ setLastModificationTime(notExpiredFile, millis - 6 * 24 * 3600000); // Fake lastModifiedTime - 10 h
+
+ archive = createArchive("2018-10-21");
+ cache.save(repoConfig, archive, list);
+ File expiredFile = cache.getFile(repoConfig, archive);
+ setLastModificationTime(expiredFile, millis - 8 * 24 * 3600000); // Fake lastModifiedTime - 10 h
+
+ assertTrue(expiredFile.exists());
+ assertTrue(notExpiredFile.exists());
+
+ cache.cleanUp();
+ assertFalse(expiredFile.exists());
+ assertTrue(notExpiredFile.exists());
cache.removeAllCacheFiles();
}
@@ -112,9 +153,9 @@
return archive;
}
- private void setLastAccessTime(File file, long accessTime) throws IOException {
+ private void setLastModificationTime(File file, long lastModificationTime) throws IOException {
Path path = file.toPath();
- FileTime fileTime = FileTime.fromMillis(accessTime);
- Files.setAttribute(path, "lastAccessTime", fileTime);
+ FileTime fileTime = FileTime.fromMillis(lastModificationTime);
+ Files.setAttribute(path, "lastModifiedTime", fileTime);
}
}
--
Gitblit v1.10.0