package de.micromata.borgbutler.cache; import de.micromata.borgbutler.BorgCommandResult; import de.micromata.borgbutler.BorgCommands; import de.micromata.borgbutler.config.BorgRepoConfig; import de.micromata.borgbutler.config.Configuration; import de.micromata.borgbutler.config.ConfigurationHandler; import de.micromata.borgbutler.data.Archive; import de.micromata.borgbutler.data.ArchiveShortInfo; import de.micromata.borgbutler.data.FileSystemFilter; import de.micromata.borgbutler.data.Repository; import de.micromata.borgbutler.json.borg.BorgFilesystemItem; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.jcs.JCS; import org.apache.commons.jcs.access.CacheAccess; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.util.ArrayList; import java.util.Iterator; import java.util.List; /** * Caches all borg objects such as repository information, archive and archive content. Therefore a fluent work * is possible. Without caching, working with BorgButler especially for remote backups would be a time consuming mess. *
* For most objects JCS is used. For file lists (up to million of files) an own implementation is used because JCS * isn't recommended for caching a very large number of objects or very large objects. */ public class ButlerCache { private Logger log = LoggerFactory.getLogger(ButlerCache.class); public static final String CACHE_DIR_NAME = "cache"; private static ButlerCache instance = new ButlerCache(); private JCSCache jcsCache; private CacheAccess repoCacheAccess; private ArchiveFilelistCache archiveFilelistCache; private int notYetLoadedIdCounter = 1; public static ButlerCache getInstance() { return instance; } /** * @param idOrName * @return Repository. */ public Repository getRepository(String idOrName) { BorgRepoConfig repoConfig = ConfigurationHandler.getConfiguration().getRepoConfig(idOrName); if (repoConfig != null) { return getRepository(repoConfig); } List repositories = getAllRepositories(); if (CollectionUtils.isNotEmpty(repositories)) { for (Repository repository : repositories) { if (StringUtils.equals(idOrName, repository.getName()) || StringUtils.equals(idOrName, repository.getId())) { return repository; } } } log.warn("Repo with id or name '" + idOrName + "' not found."); return null; } /** * @param repoConfig * @return Repository. */ private Repository getRepository(BorgRepoConfig repoConfig) { Repository repository = null; try { repository = repoCacheAccess.get(repoConfig.getRepo()); } catch (Exception ex) { log.warn("Error while deserializing repository (maybe data format was changed). Reloading..."); } if (repository == null || repository.getLocation() == null) { BorgCommandResult result = BorgCommands.info(repoConfig); repository = result.getObject(); if (repository != null) { repoCacheAccess.put(repoConfig.getRepo(), repository); } } if (repository == null) { log.warn("Repo with name '" + repoConfig.getRepo() + "' not found."); } else { repoConfig.setId(repository.getId()); } return repository; } /** * @return the list of all repositories. */ public List getAllRepositories() { List repositories = new ArrayList<>(); for (BorgRepoConfig repoConfig : ConfigurationHandler.getConfiguration().getAllRepoConfigs()) { if (repoConfig == null) { // Occurs sometimes for demo repos. continue; } Repository repository = repoCacheAccess.get(repoConfig.getRepo()); if (repository == null) { if (repoConfig.getId() == null) { // Temporary id: repoConfig.setId("not_yet_loaded_" + notYetLoadedIdCounter++); } repository = new Repository(); repository.setDisplayName(repoConfig.getDisplayName()); repository.setName(repoConfig.getRepo()); repository.setId(repoConfig.getId()); } else if (repoConfig.getId() == null) { // On initial call, the repo id is not assigned to BorgRepoConfig for cached repositories: repoConfig.setId(repository.getId()); } repositories.add(repository); } return repositories; } public void clearAllCaches() { clearRepoCacheAccess(); log.info("Clearing cache with file lists of archives..."); this.archiveFilelistCache.removeAllCacheFiles(); } public void clearRepoCacheAccess() { log.info("Clearing repositories cache..."); this.repoCacheAccess.clear(); } public void clearRepoCacheAccess(String repo) { if (this.repoCacheAccess.get(repo) != null) { log.info("Clearing repository cache '" + repo + "'..."); this.repoCacheAccess.remove(repo); } } public void clearRepoCacheAccess(Repository repository) { clearRepoCacheAccess(repository.getName()); } /** * @param idOrName * @return The repository (ensures that the list of archives is loaded). */ public Repository getRepositoryArchives(String idOrName) { Repository repository = getRepository(idOrName); if (repository == null) { return null; } if (repository.isArchivesLoaded()) { updateArchivesCacheStatusAndShortInfos(repository); return repository; } BorgRepoConfig repoConfig = ConfigurationHandler.getConfiguration().getRepoConfig(repository.getName()); BorgCommands.list(repoConfig, repository); updateArchivesCacheStatusAndShortInfos(repository); return repository; } public Archive getArchive(String repoName, String archiveIdOrName) { return getArchive(repoName, archiveIdOrName, false); } /** * @param repoIdOrName * @param archiveIdOrName * @param forceReload If true, any cache value will be ignored. Default is false. * @return */ public Archive getArchive(String repoIdOrName, String archiveIdOrName, boolean forceReload) { BorgRepoConfig repoConfig = ConfigurationHandler.getConfiguration().getRepoConfig(repoIdOrName); if (repoConfig == null) { log.error("Can't find configured repo '" + repoIdOrName + "'."); return null; } return getArchive(repoConfig, archiveIdOrName, forceReload); } public Archive getArchive(BorgRepoConfig repoConfig, String archiveIdOrName, boolean forceReload) { Repository repository = getRepositoryArchives(repoConfig.getRepo()); if (repository == null) { log.error("Repository '" + repoConfig.getRepo() + "' not found."); return null; } Archive archive = null; if (CollectionUtils.isEmpty(repository.getArchives())) { log.warn("Repository '" + repoConfig.getRepo() + "' doesn't contain archives."); } else { for (Archive arch : repository.getArchives()) { if (StringUtils.equals(arch.getName(), archiveIdOrName) || StringUtils.equals(arch.getId(), archiveIdOrName)) { archive = arch; break; } } } if (archive == null) { log.error("Archive with id or name '" + archiveIdOrName + "' not found for repo '" + repoConfig.getRepo() + "'."); return null; } if (!forceReload && archive.hasInfoData()) { // borg info archive was already called. updateArchivesCacheStatusAndShortInfos(repository); return archive; } BorgCommands.info(repoConfig, archive, repository); updateArchivesCacheStatusAndShortInfos(repository); return archive; } public Archive getArchive(String archiveId) { for (Repository repository : getAllRepositories()) { if (repository.getArchives() != null) { for (Archive archive : repository.getArchives()) { if (StringUtils.equals(archiveId, archive.getId())) { return archive; } } } } log.error("Archive with id '" + archiveId + "' not found. May-be not yet loaded into the cache."); return null; } /** * Updates for all archives of the given repository the cache status ({@link Archive#isFileListAlreadyCached()} and * updates also the list of {@link ArchiveShortInfo} for all archives of the given repository. * * @param repository */ private void updateArchivesCacheStatusAndShortInfos(Repository repository) { if (repository == null || repository.getArchives() == null) { return; } List archiveInfoList = new ArrayList<>(); for (Archive archive : repository.getArchives()) { archive.setFileListAlreadyCached(archiveFilelistCache.contains(repository, archive)); archiveInfoList.add(new ArchiveShortInfo(archive)); } for (Archive archive : repository.getArchives()) { // ArchiveInfoList for comparing current archives with one of all other archives. archive.setArchiveShortInfoList(archiveInfoList); } } /** * @param archiveId * @param forceLoad If false, the file list will only get if not yet loaded. * @param filter If not null, the result will be filtered. * @return */ public List getArchiveContent(String archiveId, boolean forceLoad, FileSystemFilter filter) { Archive archive = getArchive(archiveId); if (archive == null) { log.error("Can't find archive with id '" + archiveId + "'. May-be it doesn't exist or the archives of the target repository aren't yet loaded."); return null; } BorgRepoConfig repoConfig = ConfigurationHandler.getConfiguration().getRepoConfig(archive.getRepoId()); return getArchiveContent(repoConfig, archive, forceLoad, filter); } /** * @param repoConfig * @param archive * @return */ public List getArchiveContent(BorgRepoConfig repoConfig, Archive archive) { return getArchiveContent(repoConfig, archive, true, null); } /** * @param repoConfig * @param archive * @param filter If given, only the items matching this filter are returned.. * @return */ public List getArchiveContent(BorgRepoConfig repoConfig, Archive archive, boolean forceLoad, FileSystemFilter filter) { if (archive == null || StringUtils.isBlank(archive.getName())) { return null; } synchronized (archive) { List items = archiveFilelistCache.load(repoConfig, archive, filter); if (items == null && forceLoad) { List list = BorgCommands.listArchiveContent(repoConfig, archive); if (CollectionUtils.isNotEmpty(list)) { archiveFilelistCache.save(repoConfig, archive, list); items = new ArrayList<>(); Iterator it = list.iterator(); // Don't use for-each (ConcurrentModificationException) while (it.hasNext()) { BorgFilesystemItem item = it.next(); if (filter == null || filter.matches(item)) { items.add(item); if (filter != null && filter.isFinished()) break; } } if (filter != null) { items = filter.reduce(items); } } } if (items == null && forceLoad) { log.warn("Repo::archiv with name '" + archive.getBorgIdentifier() + "' not found or job was cancelled."); } return items; } } public List getArchiveContent(File file) { return archiveFilelistCache.load(file, null); } public void deleteCachedArchiveContent(String repoIdOrName, String archiveIdOrName) { Repository repository = getRepository(repoIdOrName); if (repository == null) { log.warn("Can't delete archive content cache file, repository not found: " + repoIdOrName); return; } Archive archive = getArchive(repoIdOrName, archiveIdOrName); if (archive == null) { log.warn("Can't delete archive content cache file, archive not found: " + repoIdOrName); return; } archiveFilelistCache.deleteCachFile(repository, archive); } public void shutdown() { JCS.shutdown(); } public File getCacheDir() { return jcsCache.getCacheDir(); } private ButlerCache() { Configuration configuration = ConfigurationHandler.getConfiguration(); this.jcsCache = JCSCache.getInstance(); this.repoCacheAccess = jcsCache.getJCSCache("repositories"); this.archiveFilelistCache = new ArchiveFilelistCache(getCacheDir(), configuration.getMaxArchiveContentCacheCapacityMb()); // Assign the repo ids to the repo config objects: getAllRepositories(); } }