borgbutler-core/src/main/java/de/micromata/borgbutler/BorgCommands.java
@@ -15,7 +15,6 @@ import org.apache.commons.exec.PumpStreamHandler; import org.apache.commons.exec.environment.EnvironmentUtils; import org.apache.commons.io.output.ByteArrayOutputStream; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,69 +58,63 @@ /** * Executes borg list repository. * The given repository will be cloned and archives will be added. * The field {@link Repository#getLastModified()} of masterRepository will be updated. * The given repository will be used and archives will be added. * * @param repoConfig The repo config associated to the masterRepository. Needed for the borg call. * @param masterRepository Repository without archives. * @return Parsed repo config returned by Borg command including archives. * @param repository Repository without archives, archives will be loaded. */ public static Repository list(BorgRepoConfig repoConfig, Repository masterRepository) { String json = execute(repoConfig, "list", masterRepository.getName(), "--json"); public static void list(BorgRepoConfig repoConfig, Repository repository) { String json = execute(repoConfig, "list", repository.getName(), "--json"); if (json == null) { log.error("Can't load archives from repo '" + masterRepository.getName() + "'."); return null; log.error("Can't load archives from repo '" + repository.getName() + "'."); return; } BorgRepoList repoList = JsonUtils.fromJson(BorgRepoList.class, json); if (repoList == null) { log.error("Can't load archives from repo '" + masterRepository.getName() + "'."); return null; if (repoList == null || CollectionUtils.isEmpty(repoList.getArchives())) { log.error("Can't load archives from repo '" + repository.getName() + "'."); return; } masterRepository.setLastModified(DateUtils.format(repoList.getRepository().getLastModified())) repository.setLastModified(DateUtils.format(repoList.getRepository().getLastModified())) .setLastCacheRefresh(DateUtils.format(LocalDateTime.now())); Repository repository = ObjectUtils.clone(masterRepository) .addAll(repoList.getArchives()); if (repository.getArchives() != null) { for (BorgArchive archive : repository.getArchives()) { // Reformat Borg date strings. archive.setStart(DateUtils.format(archive.getStart())); archive.setTime(DateUtils.format(archive.getTime())); for (BorgArchive borgArchive : repoList.getArchives()) { Archive archive = new Archive() .setName(borgArchive.getArchive()) .setId(borgArchive.getId()) .setStart(DateUtils.format(borgArchive.getStart())) .setTime(DateUtils.format(borgArchive.getTime())) .setRepoId(repository.getId()) .setRepoName(repository.getName()); repository.add(archive); } } return repository; } /** * Executes borg info repository::archive. * The given repository will be cloned and assigned to the returned archive. * The given repository will be modified. * The field {@link Repository#getLastModified()} of masterRepository will be updated. * * @param repoConfig The repo config associated to the masterRepository. Needed for the borg call. * @param archiveName The name of the archive returned by {@link BorgArchive#getArchive()} * @param masterRepository Repository without archives. * @return Parsed repo config returned by Borg command including archives. * @param repoConfig The repo config associated to the repository. Needed for the borg call. * @param archive The archive to update. * @param repository Repository without archives. */ public static Archive info(BorgRepoConfig repoConfig, String archiveName, Repository masterRepository) { String archiveFullname = repoConfig.getRepo() + "::" + archiveName; public static void info(BorgRepoConfig repoConfig, Archive archive, Repository repository) { String archiveFullname = repoConfig.getRepo() + "::" + archive.getName(); String json = execute(repoConfig, "info", archiveFullname, "--json"); if (json == null) { return null; return; } BorgArchiveInfo archiveInfo = JsonUtils.fromJson(BorgArchiveInfo.class, json); if (archiveInfo == null) { log.error("Archive '" + archiveFullname + "' not found."); return null; return; } masterRepository.setLastModified(DateUtils.format(archiveInfo.getRepository().getLastModified())) repository.setLastModified(DateUtils.format(archiveInfo.getRepository().getLastModified())) .setLastCacheRefresh(DateUtils.format(LocalDateTime.now())); Repository repository = ObjectUtils.clone(masterRepository); Archive archive = new Archive() .setCache(archiveInfo.getCache()) .setEncryption(archiveInfo.getEncryption()) .setRepository(repository); archive.setCache(archiveInfo.getCache()) .setEncryption(archiveInfo.getEncryption()); if (CollectionUtils.isEmpty(archiveInfo.getArchives())) { log.error("The returned borg archive contains no archive infos: " + json); return null; return; } if (archiveInfo.getArchives().size() > 1 ) { log.warn("Archive '" + archiveFullname + "' contains more than one archives!? (Using only first.)"); @@ -131,8 +124,8 @@ .setChunkerParams(borgArchive2.getChunkerParams()) .setCommandLine(borgArchive2.getCommandLine()) .setComment(borgArchive2.getComment()) .setStats(borgArchive2.getStats()); return archive; .setStats(borgArchive2.getStats()) .setUsername(borgArchive2.getUsername()); } public static List<BorgFilesystemItem> listArchiveContent(BorgRepoConfig repoConfig, String archiveId) { borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFilelistCache.java
@@ -1,6 +1,7 @@ package de.micromata.borgbutler.cache; import de.micromata.borgbutler.config.BorgRepoConfig; import de.micromata.borgbutler.data.Archive; import de.micromata.borgbutler.json.borg.BorgArchive; import de.micromata.borgbutler.json.borg.BorgFilesystemItem; import de.micromata.borgbutler.utils.ReplaceUtils; @@ -29,10 +30,10 @@ private long FILES_EXPIRE_TIME = 7 * 24 * 3660 * 1000; // Expires after 7 days. @Getter private BorgArchive archive; private Archive archive; private List<BorgFilesystemItem> content; public void save(BorgRepoConfig repoConfig, BorgArchive archive, List<BorgFilesystemItem> filesystemItems) { public void save(BorgRepoConfig repoConfig, Archive archive, List<BorgFilesystemItem> filesystemItems) { File file = getFile(repoConfig, archive); if (CollectionUtils.isEmpty(filesystemItems)) { return; @@ -57,7 +58,7 @@ * @param archive * @return */ public BorgFilesystemItem[] load(BorgRepoConfig repoConfig, BorgArchive archive) { public BorgFilesystemItem[] load(BorgRepoConfig repoConfig, Archive archive) { File file = getFile(repoConfig, archive); if (!file.exists()) { return null; @@ -181,9 +182,9 @@ } } File getFile(BorgRepoConfig repoConfig, BorgArchive archive) { File getFile(BorgRepoConfig repoConfig, Archive archive) { return new File(cacheDir, ReplaceUtils.encodeFilename(CACHE_ARCHIVE_LISTS_BASENAME + archive.getTime() + "-" + repoConfig.getRepo() + "-" + archive.getArchive() + ".gz", true)); + "-" + repoConfig.getRepo() + "-" + archive.getName() + ".gz", true)); } ArchiveFilelistCache(File cacheDir, int cacheArchiveContentMaxDiscSizeMB) { borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ButlerCache.java
@@ -6,7 +6,6 @@ import de.micromata.borgbutler.config.ConfigurationHandler; import de.micromata.borgbutler.data.Archive; import de.micromata.borgbutler.data.Repository; import de.micromata.borgbutler.json.borg.BorgArchive; import de.micromata.borgbutler.json.borg.BorgFilesystemItem; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.jcs.JCS; @@ -26,8 +25,6 @@ private JCSCache jcsCache; private CacheAccess<String, Repository> repoCacheAccess; private CacheAccess<String, Repository> repoArchivesCacheAccess; private CacheAccess<String, Archive> archivesCacheAccess; private ArchiveFilelistCache archiveFilelistCache; public static ButlerCache getInstance() { @@ -36,7 +33,7 @@ /** * @param idOrName * @return Repository without list of archives. * @return Repository. */ public Repository getRepository(String idOrName) { BorgRepoConfig repoConfig = ConfigurationHandler.getConfiguration().getRepoConfig(idOrName); @@ -57,7 +54,7 @@ /** * @param repoConfig * @return Repository without list of archives. * @return Repository. */ private Repository getRepository(BorgRepoConfig repoConfig) { Repository repository = repoCacheAccess.get(repoConfig.getRepo()); @@ -74,7 +71,7 @@ } /** * @return the list of all repositories without the list of archives. * @return the list of all repositories. */ public List<Repository> getAllRepositories() { List<Repository> repositories = new ArrayList<>(); @@ -90,11 +87,8 @@ public void clearAllCaches() { clearRepoCacheAccess(); clearRepoArchicesCacheAccess(); log.info("Clearing cache with file lists of archives..."); this.archiveFilelistCache.removeAllCacheFiles(); log.info("Clearing archives cache..."); this.archivesCacheAccess.clear(); } public void clearRepoCacheAccess() { @@ -102,28 +96,20 @@ this.repoCacheAccess.clear(); } public void clearRepoArchicesCacheAccess() { log.info("Clearing repositories cache (with included archives)..."); this.repoArchivesCacheAccess.clear(); } /** * @param idOrName * @return The repository including all archives. * @return The repository (ensures that the list of archives is loaded). */ public Repository getRepositoryArchives(String idOrName) { Repository masterRepository = getRepository(idOrName); if (masterRepository == null) { Repository repository = getRepository(idOrName); if (repository == null) { return null; } Repository repository = repoArchivesCacheAccess.get(masterRepository.getName()); if (repository != null) { if (repository.isArchivesLoaded()) { return repository; } BorgRepoConfig repoConfig = ConfigurationHandler.getConfiguration().getRepoConfig(masterRepository.getName()); repository = BorgCommands.list(repoConfig, masterRepository); if (repository == null) return null; repoArchivesCacheAccess.put(repository.getName(), repository); BorgRepoConfig repoConfig = ConfigurationHandler.getConfiguration().getRepoConfig(repository.getName()); BorgCommands.list(repoConfig, repository); return repository; } @@ -147,50 +133,50 @@ } public Archive getArchive(BorgRepoConfig repoConfig, String archiveIdOrName, boolean forceReload) { Repository masterRepository = getRepositoryArchives(repoConfig.getRepo()); if (masterRepository == null) { Repository repository = getRepositoryArchives(repoConfig.getRepo()); if (repository == null) { log.error("Repository '" + repoConfig.getRepo() + "' not found."); return null; } String archiveName = archiveIdOrName; if (CollectionUtils.isEmpty(masterRepository.getArchives())) { Archive archive = null; if (CollectionUtils.isEmpty(repository.getArchives())) { log.warn("Repository '" + repoConfig.getRepo() + "' doesn't contain archives."); } else { for (BorgArchive borgArchive : masterRepository.getArchives()) { if (StringUtils.equals(borgArchive.getArchive(), archiveIdOrName) || StringUtils.equals(borgArchive.getId(), archiveIdOrName)) { archiveName = borgArchive.getArchive(); for (Archive arch : repository.getArchives()) { if (StringUtils.equals(arch.getName(), archiveIdOrName) || StringUtils.equals(arch.getId(), archiveIdOrName)) { archive = arch; break; } } } String archiveFullname = repoConfig.getRepo() + "::" + archiveName; if (!forceReload) { Archive archive = this.archivesCacheAccess.get(archiveFullname); if (archive != null) { if (archive == null) { log.error("Archive with id or name '" + archiveIdOrName + "' not found for repo '" + repoConfig.getRepo() + "'."); return null; } if (!forceReload && archive.hasInfoData()) { // borg info archive was already called. return archive; } } Archive archive = BorgCommands.info(repoConfig, archiveName, masterRepository); if (archive != null) this.archivesCacheAccess.put(archiveFullname, archive); BorgCommands.info(repoConfig, archive, repository); return archive; } public BorgFilesystemItem[] getArchiveContent(BorgRepoConfig repoConfig, BorgArchive archive) { if (archive == null || StringUtils.isBlank(archive.getArchive())) { public BorgFilesystemItem[] getArchiveContent(BorgRepoConfig repoConfig, Archive archive) { if (archive == null || StringUtils.isBlank(archive.getName())) { return null; } BorgFilesystemItem[] items = archiveFilelistCache.load(repoConfig, archive); if (items == null) { List<BorgFilesystemItem> list = BorgCommands.listArchiveContent(repoConfig, archive.getArchive()); List<BorgFilesystemItem> list = BorgCommands.listArchiveContent(repoConfig, archive.getName()); if (CollectionUtils.isNotEmpty(list)) { archiveFilelistCache.save(repoConfig, archive, list); items = list.toArray(new BorgFilesystemItem[0]); } } if (items == null) { log.warn("Repo::archiv with name '" + repoConfig.getRepo() + "::" + archive.getArchive() + "' not found."); log.warn("Repo::archiv with name '" + archive.getBorgIdentifier() + "' not found."); } return items; } @@ -211,8 +197,6 @@ Configuration configuration = ConfigurationHandler.getConfiguration(); this.jcsCache = JCSCache.getInstance(); this.repoCacheAccess = jcsCache.getJCSCache("repositories"); this.repoArchivesCacheAccess = jcsCache.getJCSCache("repositoriesArchives"); this.archivesCacheAccess = jcsCache.getJCSCache("archives"); this.archiveFilelistCache = new ArchiveFilelistCache(getCacheDir(), configuration.getCacheArchiveContentMaxDiscSizeMB()); } } borgbutler-core/src/main/java/de/micromata/borgbutler/data/Archive.java
@@ -5,16 +5,32 @@ import de.micromata.borgbutler.json.borg.BorgEncryption; import lombok.Getter; import lombok.Setter; import org.apache.commons.lang3.StringUtils; import java.io.Serializable; /** * */ public class Archive implements Serializable { public class Archive implements Serializable, Comparable<Archive> { /** * For convenience purposes for the client. */ @Getter @Setter private Repository repository; private String repoName; /** * For convenience purposes for the client. */ @Getter @Setter private String repoId; @Getter @Setter private String name; @Getter @Setter private String id; @Getter @Setter private BorgCache cache; @@ -39,6 +55,40 @@ private String start; @Getter @Setter private String time; @Getter @Setter private BorgArchiveStats stats; @Getter @Setter private String username; /** * * @return repoName::archiveName */ public String getBorgIdentifier() { return repoName + "::" + name; } /** * Is <tt>borg info repo::archive</tt> already called for this archive? * * @return true, if borg info was called, otherwise false. */ public boolean hasInfoData() { return commandLine != null && commandLine.length > 0; } /** * In reverse order, compares times. * * @param o * @return */ @Override public int compareTo(Archive o) { // Reverse order: return StringUtils.compare(o.time, this.time); } } borgbutler-core/src/main/java/de/micromata/borgbutler/data/Repository.java
@@ -1,21 +1,20 @@ package de.micromata.borgbutler.data; import de.micromata.borgbutler.config.BorgRepoConfig; import de.micromata.borgbutler.json.borg.BorgArchive; import de.micromata.borgbutler.json.borg.BorgCache; import de.micromata.borgbutler.json.borg.BorgEncryption; import lombok.Getter; import lombok.Setter; import org.apache.commons.collections4.CollectionUtils; import java.io.Serializable; import java.util.Collection; import java.util.SortedSet; import java.util.TreeSet; /** * Part of Borg json objects to refer objects to repositories. */ public class Repository implements Serializable, Cloneable { public class Repository implements Serializable { private static final long serialVersionUID = 1278802519434516280L; /** * The repo configured for borg. @@ -68,18 +67,21 @@ */ @Getter @Setter private SortedSet<BorgArchive> archives; private SortedSet<Archive> archives; public Repository addAll(Collection<BorgArchive> archives) { public Repository add(Archive archive) { if (this.archives == null) { this.archives = new TreeSet<>(); } this.archives.addAll(archives); this.archives.add(archive); return this; } @Override public Object clone() throws CloneNotSupportedException { return super.clone(); /** * Is <tt>borg list repo</tt> already called? * @return */ public boolean isArchivesLoaded() { return CollectionUtils.isNotEmpty(archives); } } borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/BorgArchive.java
@@ -3,14 +3,13 @@ import de.micromata.borgbutler.json.JsonUtils; import lombok.Getter; import lombok.Setter; import org.apache.commons.lang3.StringUtils; import java.io.Serializable; /** * This object is given by <tt>borg list repo</tt>. */ public class BorgArchive implements Serializable, Comparable<BorgArchive> { public class BorgArchive implements Serializable { private static final long serialVersionUID = -7872260170265536732L; @Getter private String archive; @@ -27,17 +26,6 @@ @Setter private String time; /** * In reverse order, compares times. * @param o * @return */ @Override public int compareTo(BorgArchive o) { // Reverse order: return StringUtils.compare(o.time, this.time); } public String toString() { return JsonUtils.toJson(this, true); } borgbutler-core/src/test/java/de/micromata/borgbutler/cache/ArchiveFilelistCacheTest.java
@@ -1,7 +1,7 @@ package de.micromata.borgbutler.cache; import de.micromata.borgbutler.config.BorgRepoConfig; import de.micromata.borgbutler.json.borg.BorgArchive; import de.micromata.borgbutler.data.Archive; import de.micromata.borgbutler.json.borg.BorgFilesystemItem; import org.junit.jupiter.api.Test; import org.slf4j.Logger; @@ -29,7 +29,7 @@ cache.removeAllCacheFiles(); BorgRepoConfig repoConfig = new BorgRepoConfig(); repoConfig.setRepo("repo"); BorgArchive archive = createArchive("2018-12-10"); Archive archive = createArchive("2018-12-10"); log.info("Saving " + list.size() + " items to out dir."); cache.save(repoConfig, archive, list); log.info("Saving done."); @@ -50,7 +50,7 @@ cache.removeAllCacheFiles(); BorgRepoConfig repoConfig = new BorgRepoConfig(); repoConfig.setRepo("repo"); BorgArchive archive = createArchive("2018-12-09"); Archive archive = createArchive("2018-12-09"); assertNull(cache.load(repoConfig, archive)); cache.save(repoConfig, archive, list); BorgFilesystemItem[] filesystemItems = cache.load(repoConfig, archive); @@ -68,7 +68,7 @@ long millis = System.currentTimeMillis(); BorgArchive archive = createArchive("2018-11-20"); Archive archive = createArchive("2018-11-20"); cache.save(repoConfig, archive, list); File oldestFile = cache.getFile(repoConfig, archive); setLastModificationTime(oldestFile, millis - 10 * 3600000); // Fake lastModifiedTime - 10 h @@ -104,7 +104,7 @@ long millis = System.currentTimeMillis(); BorgArchive archive = createArchive("2018-10-20"); Archive archive = createArchive("2018-10-20"); cache.save(repoConfig, archive, list); File notExpiredFile = cache.getFile(repoConfig, archive); setLastModificationTime(notExpiredFile, millis - 6 * 24 * 3600000); // Fake lastModifiedTime - 10 h @@ -146,8 +146,8 @@ return this; } private BorgArchive createArchive(String time) throws Exception { BorgArchive archive = new BorgArchive(); private Archive createArchive(String time) throws Exception { Archive archive = new Archive(); set(archive, "archive", "archive-" + time); set(archive, "time", time); return archive; borgbutler-core/src/test/java/de/micromata/borgbutler/cache/CacheTest.java
@@ -5,7 +5,6 @@ import de.micromata.borgbutler.config.ConfigurationHandler; import de.micromata.borgbutler.data.Archive; import de.micromata.borgbutler.data.Repository; import de.micromata.borgbutler.json.borg.BorgArchive; import de.micromata.borgbutler.json.borg.BorgFilesystemItem; import org.apache.commons.collections4.CollectionUtils; import org.junit.jupiter.api.Test; @@ -43,24 +42,24 @@ assertEquals(config.getRepoConfigs().size(), ButlerCache.getInstance().getAllRepositories().size()); } List<BorgRepoConfig> repoConfigs = ConfigurationHandler.getConfiguration().getRepoConfigs(); BorgArchive borgArchive = null; Archive archive = null; BorgRepoConfig repoConfig = null; if (CollectionUtils.isNotEmpty(repoConfigs)) { repoConfig = repoConfigs.get(0); Repository rerepositoryoList = ButlerCache.getInstance().getRepositoryArchives(repoConfig.getRepo()); if (rerepositoryoList != null && CollectionUtils.isNotEmpty(rerepositoryoList.getArchives())) { borgArchive = rerepositoryoList.getArchives().first(); archive = rerepositoryoList.getArchives().first(); } } { if (borgArchive != null) { Archive archive = ButlerCache.getInstance().getArchive(repoConfig.getRepo(), borgArchive.getName()); assertNotNull(archive); archive = ButlerCache.getInstance().getArchive(repoConfig.getRepo(), borgArchive.getId()); assertNotNull(archive); BorgFilesystemItem[] content = ButlerCache.getInstance().getArchiveContent(repoConfig, borgArchive); if (archive != null) { Archive archive2 = ButlerCache.getInstance().getArchive(repoConfig.getRepo(), archive.getName()); assertNotNull(archive2); archive = ButlerCache.getInstance().getArchive(repoConfig.getRepo(), archive.getId()); assertNotNull(archive2); BorgFilesystemItem[] content = ButlerCache.getInstance().getArchiveContent(repoConfig, archive2); log.info("Number of items (content) of archive: " + content.length); content = ButlerCache.getInstance().getArchiveContent(repoConfig, borgArchive); content = ButlerCache.getInstance().getArchiveContent(repoConfig, archive2); log.info("Number of items (content) of archive: " + content.length); } } borgbutler-server/src/main/java/de/micromata/borgbutler/server/rest/ReposRest.java
@@ -49,7 +49,7 @@ public String getRepoArchiveList(@QueryParam("id") String id, @QueryParam("force") boolean force, @QueryParam("prettyPrinter") boolean prettyPrinter) { if (force) { ButlerCache.getInstance().clearRepoArchicesCacheAccess(); ButlerCache.getInstance().clearRepoCacheAccess(); } Repository repository = ButlerCache.getInstance().getRepositoryArchives(id); return JsonUtils.toJson(repository, prettyPrinter); @@ -86,7 +86,6 @@ public String getList(@QueryParam("force") boolean force, @QueryParam("prettyPrinter") boolean prettyPrinter) { if (force) { ButlerCache.getInstance().clearRepoCacheAccess(); ButlerCache.getInstance().clearRepoArchicesCacheAccess(); } List<Repository> repositories = ButlerCache.getInstance().getAllRepositories(); if (CollectionUtils.isEmpty(repositories)) { borgbutler-webapp/src/components/views/repos/ArchiveView.jsx
@@ -38,7 +38,7 @@ .then(json => { this.setState({ isFetching: false, repo: json archive: json }) }) .catch(() => this.setState({isFetching: false, failed: true})); @@ -46,7 +46,7 @@ render = () => { let content = undefined; const repo = this.state.repo; const archive = this.state.archive; let pageHeader = ''; if (this.state.isFetching) { @@ -62,7 +62,7 @@ />; } else if (this.state.repo) { pageHeader = <React.Fragment> {repo.displayName} {archive.id} <div className={'btn btn-outline-primary refresh-button-right'} onClick={this.fetchArchive.bind(this, true)} borgbutler-webapp/src/components/views/repos/RepoArchiveListView.jsx
@@ -112,7 +112,7 @@ // Return the element. Also pass key return ( <tr key={archive.id}> <td><Link to={`/archives/${repo.id}/${archive.id}`}>{archive.archive}</Link></td> <td><Link to={`/archives/${repo.id}/${archive.id}`}>{archive.name}</Link></td> <td>{archive.time}</td> <td>{archive.id}</td> </tr>);