mirror of https://github.com/micromata/borgbackup-butler.git
borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFilelistCache.java
@@ -26,7 +26,7 @@ private static final String CACHE_FILE_GZIP_EXTENSION = "gz"; private File cacheDir; private int cacheArchiveContentMaxDiscSizeMB; private long FILES_EXPIRE_TIME = 5 * 24 * 3660 * 1000; // Expires after 5 days. private long FILES_EXPIRE_TIME = 7 * 24 * 3660 * 1000; // Expires after 7 days. @Getter private Archive archive; @@ -50,6 +50,13 @@ log.info("Saving done."); } /** * Will load and touch the archive file if exist. The file will be touched (last modified time will be set to now) * for pruning oldest cache files. The last modified time will be the time of the last usage. * @param repoConfig * @param archive * @return */ public FilesystemItem[] load(BorgRepoConfig repoConfig, Archive archive) { File file = getFile(repoConfig, archive); if (!file.exists()) { @@ -57,6 +64,12 @@ } log.info("Loading archive content as file list from: " + file.getAbsolutePath()); FilesystemItem[] list = null; try { // Set last modified time of file: Files.setAttribute(file.toPath(), "lastModifiedTime", FileTime.fromMillis(System.currentTimeMillis())); } catch (IOException ex) { log.error("Can't set lastModifiedTime on file '" + file.getAbsolutePath() + "'. Pruning old cache files may not work."); } try (ObjectInputStream inputStream = new ObjectInputStream(new BufferedInputStream(new GzipCompressorInputStream(new FileInputStream(file))))) { Object obj = inputStream.readObject(); if (!(obj instanceof Integer)) { @@ -84,7 +97,7 @@ /** * Deletes archive contents older than 7 days and deletes the oldest archive contents if the max cache size is * exceeded. * exceeded. The last modified time of a file is equals to the last usage by {@link #load(BorgRepoConfig, Archive)}. */ public void cleanUp() { File[] files = cacheDir.listFiles(); @@ -92,37 +105,40 @@ for (File file : files) { try { if (!file.exists() || !isCacheFile(file)) continue; // Get last access time of file: FileTime time = Files.readAttributes(file.toPath(), BasicFileAttributes.class).lastAccessTime(); // Get last modified time of file: FileTime time = Files.readAttributes(file.toPath(), BasicFileAttributes.class).lastModifiedTime(); if (currentMillis - FILES_EXPIRE_TIME > time.toMillis()) { log.info("Delete old cache file (last access " + time + " older than 5 days): " + file.getAbsolutePath()); log.info("Delete expired cache file (last usage " + time + " older than 7 days): " + file.getAbsolutePath()); file.delete(); } } catch (IOException ex) { log.error("Can't get last accesstime from cache files (ignore file '" + file.getAbsolutePath() + "'): " + ex.getMessage(), ex); log.error("Can't get last modified time from cache files (ignore file '" + file.getAbsolutePath() + "'): " + ex.getMessage(), ex); } } int sizeInMB = getCacheDiskSizeInMB(files); if (sizeInMB > cacheArchiveContentMaxDiscSizeMB) { log.info("Maximum size of cache files exceeded (" + sizeInMB + "MB > " + cacheArchiveContentMaxDiscSizeMB + "MB). Deleting the old ones (with the oldest access)..."); + "MB). Deleting the old ones (with the oldest usage)..."); } else { // Nothing to clean up anymore. return; } SortedMap<FileTime, File> sortedFiles = new TreeMap<>(); for (File file : files) { if (!file.exists() || !isCacheFile(file)) continue; try { // Get last access time of file: FileTime time = Files.readAttributes(file.toPath(), BasicFileAttributes.class).lastAccessTime(); // Get last modified time of file: FileTime time = Files.readAttributes(file.toPath(), BasicFileAttributes.class).lastModifiedTime(); sortedFiles.put(time, file); } catch (IOException ex) { log.error("Can't get last accesstime from cache files (ignore file '" + file.getAbsolutePath() + "'): " + ex.getMessage(), ex); log.error("Can't get last modified time from cache files (ignore file '" + file.getAbsolutePath() + "'): " + ex.getMessage(), ex); } } for (Map.Entry<FileTime, File> entry : sortedFiles.entrySet()) { FileTime time = entry.getKey(); File file = entry.getValue(); if (!file.exists() || !isCacheFile(file)) continue; log.info("Deleting cache file (last access " + time + "): " + file.getAbsolutePath()); log.info("Deleting cache file (last usage " + time + "): " + file.getAbsolutePath()); file.delete(); int newSizeInMB = getCacheDiskSizeInMB(files); if (newSizeInMB < cacheArchiveContentMaxDiscSizeMB) { borgbutler-core/src/test/java/de/micromata/borgbutler/cache/ArchiveFilelistCacheTest.java
@@ -16,8 +16,7 @@ import java.util.ArrayList; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.*; public class ArchiveFilelistCacheTest { @@ -60,9 +59,9 @@ } @Test void cleanUpTest() throws Exception { void cleanUpMaximumSizeTest() throws Exception { List<FilesystemItem> list = createList(1000000); ArchiveFilelistCache cache = new ArchiveFilelistCache(new File("out"), 5); ArchiveFilelistCache cache = new ArchiveFilelistCache(new File("out"), 3); cache.removeAllCacheFiles(); BorgRepoConfig repoConfig = new BorgRepoConfig(); repoConfig.setRepo("repo"); @@ -71,14 +70,56 @@ Archive archive = createArchive("2018-11-20"); cache.save(repoConfig, archive, list); setLastAccessTime(cache.getFile(repoConfig, archive), millis - 10 * 3600000); // Fake lastAccessTime - 10 h File oldestFile = cache.getFile(repoConfig, archive); setLastModificationTime(oldestFile, millis - 10 * 3600000); // Fake lastModifiedTime - 10 h archive = createArchive("2018-11-21"); cache.save(repoConfig, archive, list); setLastAccessTime(cache.getFile(repoConfig, archive), millis - 60000); // Fake lastAccessTime - 1min File newestFile = cache.getFile(repoConfig, archive); setLastModificationTime(newestFile, millis - 60000); // Fake lastModifiedTime - 1min archive = createArchive("2018-11-22"); cache.save(repoConfig, archive, list); File file = cache.getFile(repoConfig, archive); setLastModificationTime(file, millis - 3600000); // Fake lastModifiedTime - 1 hour assertTrue(oldestFile.exists()); assertTrue(newestFile.exists()); assertTrue(file.exists()); cache.cleanUp(); assertFalse(oldestFile.exists()); assertFalse(file.exists()); assertTrue(newestFile.exists()); cache.removeAllCacheFiles(); } @Test void cleanUpExpiredTest() throws Exception { List<FilesystemItem> list = createList(1000); ArchiveFilelistCache cache = new ArchiveFilelistCache(new File("out"), 3); cache.removeAllCacheFiles(); BorgRepoConfig repoConfig = new BorgRepoConfig(); repoConfig.setRepo("repo"); long millis = System.currentTimeMillis(); Archive archive = createArchive("2018-10-20"); cache.save(repoConfig, archive, list); File notExpiredFile = cache.getFile(repoConfig, archive); setLastModificationTime(notExpiredFile, millis - 6 * 24 * 3600000); // Fake lastModifiedTime - 10 h archive = createArchive("2018-10-21"); cache.save(repoConfig, archive, list); File expiredFile = cache.getFile(repoConfig, archive); setLastModificationTime(expiredFile, millis - 8 * 24 * 3600000); // Fake lastModifiedTime - 10 h assertTrue(expiredFile.exists()); assertTrue(notExpiredFile.exists()); cache.cleanUp(); assertFalse(expiredFile.exists()); assertTrue(notExpiredFile.exists()); cache.removeAllCacheFiles(); } @@ -112,9 +153,9 @@ return archive; } private void setLastAccessTime(File file, long accessTime) throws IOException { private void setLastModificationTime(File file, long lastModificationTime) throws IOException { Path path = file.toPath(); FileTime fileTime = FileTime.fromMillis(accessTime); Files.setAttribute(path, "lastAccessTime", fileTime); FileTime fileTime = FileTime.fromMillis(lastModificationTime); Files.setAttribute(path, "lastModifiedTime", fileTime); } }
