From 7da4760af26519051b1accd840c9199a11f7491a Mon Sep 17 00:00:00 2001
From: Kai Reinhard <K.Reinhard@micromata.de>
Date: Mon, 10 Dec 2018 16:53:15 +0000
Subject: [PATCH] JCS...
---
borgbutler-core/src/main/java/de/micromata/borgbutler/cache/JCSCache.java | 27 +-----
borgbutler-server/src/main/java/de/micromata/borgbutler/server/Main.java | 2
borgbutler-core/src/main/java/de/micromata/borgbutler/BorgCommands.java | 52 +++++++-----
borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/RepoInfo.java | 3
borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/RepoList.java | 3
borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/ArchiveInfo.java | 2
borgbutler-core/src/main/resources/jcs-basic-config.properties | 31 +++----
borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ButlerCache.java | 42 +++++++++-
borgbutler-core/src/test/java/de/micromata/borgbutler/cache/CacheTest.java | 19 +++-
9 files changed, 108 insertions(+), 73 deletions(-)
diff --git a/borgbutler-core/src/main/java/de/micromata/borgbutler/BorgCommands.java b/borgbutler-core/src/main/java/de/micromata/borgbutler/BorgCommands.java
index 1ee5061..af5cd4a 100644
--- a/borgbutler-core/src/main/java/de/micromata/borgbutler/BorgCommands.java
+++ b/borgbutler-core/src/main/java/de/micromata/borgbutler/BorgCommands.java
@@ -14,10 +14,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.StringReader;
+import java.io.*;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -42,6 +39,16 @@
return repoInfo;
}
+ public static RepoList list(BorgRepoConfig repoConfig) {
+ String json = execute(repoConfig, "list", repoConfig.getRepo(), "--json");
+ if (json == null) {
+ return null;
+ }
+ RepoList repoList = JsonUtils.fromJson(RepoList.class, json);
+ repoList.setOriginalJson(json);
+ return repoList;
+ }
+
/**
* Executes borg info archive
*
@@ -54,30 +61,29 @@
if (json == null) {
return null;
}
- ArchiveInfo archiveList = JsonUtils.fromJson(ArchiveInfo.class, json);
- archiveList.setOriginalJson(json);
- return archiveList;
+ ArchiveInfo archiveInfo = JsonUtils.fromJson(ArchiveInfo.class, json);
+ archiveInfo.setOriginalJson(json);
+ return archiveInfo;
}
- public static RepoList list(BorgRepoConfig repoConfig) {
- String json = execute(repoConfig, "list", repoConfig.getRepo(), "--json");
- if (json == null) {
- return null;
- }
- RepoList repoList = JsonUtils.fromJson(RepoList.class, json);
- repoList.setOriginalJson(json);
- return repoList;
- }
-
- public static List<FilesystemItem> list(BorgRepoConfig repoConfig, Archive archive) {
+ public static List<FilesystemItem> listArchiveContent(BorgRepoConfig repoConfig, Archive archive) {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
- execute(outputStream, repoConfig, "list", repoConfig.getRepo() + "::" + archive.getArchive(),
- "--json-lines");
- String response = outputStream.toString(Definitions.STD_CHARSET);
+ //execute(outputStream, repoConfig, "list", repoConfig.getRepo() + "::" + archive.getArchive(),
+ // "--json-lines");
+ //String response = outputStream.toString(Definitions.STD_CHARSET);
+ String response = null;
try {
- IOUtils.copy(new StringReader(response), new FileWriter("response.json"));
+ File file = new File("/Users/kai/tmp/response.json");
+ if (file.exists()) {
+ log.info("******** Reading test file....");
+ StringWriter writer = new StringWriter();
+ IOUtils.copy(new FileReader(file), writer);
+ response = writer.toString();
+ } else {
+ log.info("******** Writing test file....");
+ IOUtils.copy(new StringReader(response), new FileWriter(file));
+ }
} catch (IOException ex) {
-
}
List<FilesystemItem> content = new ArrayList<>();
try (Scanner scanner = new Scanner(response)) {
diff --git a/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ButlerCache.java b/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ButlerCache.java
index b3ae45e..0b1a425 100644
--- a/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ButlerCache.java
+++ b/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ButlerCache.java
@@ -5,6 +5,7 @@
import de.micromata.borgbutler.config.BorgRepoConfig;
import de.micromata.borgbutler.config.ConfigurationHandler;
import de.micromata.borgbutler.json.borg.*;
+import org.apache.commons.jcs.JCS;
import org.apache.commons.jcs.access.CacheAccess;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
@@ -22,6 +23,7 @@
private JCSCache jcsCache = JCSCache.getInstance();
private CacheAccess<String, RepoInfo> repoInfoCacheAccess;
private CacheAccess<String, RepoList> repoListCacheAccess;
+ private CacheAccess<String, List<FilesystemItem>> archiveContentCacheAccess;
@JsonIgnore
private File cacheDir;
@@ -65,25 +67,53 @@
public RepoList getRepoList(String idOrName) {
BorgRepoConfig repoConfig = ConfigurationHandler.getConfiguration().getRepoConfig(idOrName);
- ArchiveInfo archiveInfo = BorgCommands.info(repoConfig, repoConfig.getRepo());
- RepoList repoList = BorgCommands.list(repoConfig);
- return null;
+ //ArchiveInfo archiveInfo = BorgCommands.info(repoConfig, repoConfig.getRepo());
+ RepoList repoList = repoListCacheAccess.get(repoConfig.getRepo());
+ if (repoList == null) {
+ repoList = BorgCommands.list(repoConfig);
+ repoListCacheAccess.put(repoConfig.getRepo(), repoList);
+ }
+ if (repoList == null) {
+ log.warn("Repo with name '" + idOrName + "' not found.");
+ }
+ return repoList;
}
- public List<FilesystemItem> getArchiveContent_(BorgRepoConfig repoConfig, Archive archive) {
+ public List<FilesystemItem> getArchiveContent(BorgRepoConfig repoConfig, Archive archive) {
if (archive == null || StringUtils.isBlank(archive.getArchive())) {
return null;
}
- List<FilesystemItem> content = BorgCommands.list(repoConfig, archive);
+ String repoArchiveId = getRepoArchiveId(repoConfig.getRepo(), archive.getId());
+ List<FilesystemItem> content = archiveContentCacheAccess.get(repoArchiveId);
+ if (content == null) {
+ content = BorgCommands.listArchiveContent(repoConfig, archive);
+ archiveContentCacheAccess.put(repoArchiveId, content);
+ archiveContentCacheAccess.getStatistics();
+ }
+ log.info("archiveContentCacheAccess.stats: " + this.archiveContentCacheAccess.getStats());
+ if (content == null) {
+ log.warn("Repo::archiv with name '" + repoConfig.getRepo() + "::" + archive.getArchive() + "' not found.");
+ }
return content;
}
+ public String getRepoArchiveId(String repo, String archiveId) {
+ return repo + "::" + archiveId;
+ }
+
+ public void shutdown() {
+ log.info("archiveContentCacheAccess.stats: " + this.archiveContentCacheAccess.getStats());
+ JCS.shutdown();
+ }
+
private ButlerCache() {
cacheDir = new File(ConfigurationHandler.getInstance().getWorkingDir(), CACHE_DIR_NAME);
if (!cacheDir.exists()) {
log.info("Creating cache dir: " + cacheDir.getAbsolutePath());
cacheDir.mkdir();
}
- this.repoInfoCacheAccess = jcsCache.getJCSCache();
+ this.repoInfoCacheAccess = jcsCache.getJCSCache("repoInfo");
+ this.repoListCacheAccess = jcsCache.getJCSCache("repoList");
+ this.archiveContentCacheAccess = jcsCache.getJCSCache("archiveContent");
}
}
diff --git a/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/JCSCache.java b/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/JCSCache.java
index 5f4fa4d..9dbadb6 100644
--- a/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/JCSCache.java
+++ b/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/JCSCache.java
@@ -18,28 +18,12 @@
private static final String CONFIG_FILE = "jcs-basic-config.properties";
public static final String CACHE_DIR_NAME = "cache";
- public enum Region {DEFAULT, ARCHIVE_CONTENT}
-
public static JCSCache getInstance() {
return instance;
}
- /**
- * @param <K>
- * @param <V>
- * @return JCS cache for default region.
- */
- public <K, V> CacheAccess<K, V> getJCSCache() {
- return JCS.getInstance("default");
- }
-
- public <K, V> CacheAccess<K, V> getJCSCache(Region region) {
- switch (region) {
- case ARCHIVE_CONTENT:
- return JCS.getInstance("default");
- default:
- return getJCSCache();
- }
+ public <K, V> CacheAccess<K, V> getJCSCache(String region) {
+ return JCS.getInstance(region);
}
private JCSCache() {
@@ -57,9 +41,10 @@
log.error("Error while loading jcs config file '" + CONFIG_FILE + "': " + ex.getMessage(), ex);
}
props.setProperty("jcs.auxiliary.DC.attributes.DiskPath", cacheDir.getAbsolutePath());
- props.setProperty("jcs.auxiliary.DC2.attributes.DiskPath", cacheDir.getAbsolutePath());
- int cacheMaxDiscSizeMB = configuration.getCacheMaxDiscSizeMB();
- props.setProperty("jcs.auxiliary.DC2.attributes.MaxKeySize", String.valueOf(cacheMaxDiscSizeMB * 1000));
+ //props.setProperty("jcs.auxiliary.DC2.attributes.DiskPath", cacheDir.getAbsolutePath());
+ //int cacheMaxDiscSizeMB = configuration.getCacheMaxDiscSizeMB();
+ //log.info("Using cache size for archive contents: " + cacheMaxDiscSizeMB + "MB.");
+ //props.setProperty("jcs.auxiliary.DC2.attributes.MaxKeySize", String.valueOf(cacheMaxDiscSizeMB * 1000));
JCS.setConfigProperties(props);
}
}
diff --git a/borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/ArchiveInfo.java b/borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/ArchiveInfo.java
index f8164fc..671c908 100644
--- a/borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/ArchiveInfo.java
+++ b/borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/ArchiveInfo.java
@@ -8,7 +8,7 @@
import java.util.List;
/**
- * Result of <tt>borg list repo</tt>.
+ * Result of <tt>borg info repo::archive</tt>.
*/
public class ArchiveInfo extends RepositoryMatcher implements Serializable {
private static final long serialVersionUID = -4200553322856662346L;
diff --git a/borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/RepoInfo.java b/borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/RepoInfo.java
index f660f6c..9578834 100644
--- a/borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/RepoInfo.java
+++ b/borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/RepoInfo.java
@@ -5,6 +5,9 @@
import java.io.Serializable;
+/**
+ * Result of borg info repo
+ */
public class RepoInfo extends RepositoryMatcher implements Serializable {
private static final long serialVersionUID = -1588038325129799400L;
@Getter
diff --git a/borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/RepoList.java b/borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/RepoList.java
index 25ba5a1..a8151c1 100644
--- a/borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/RepoList.java
+++ b/borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/RepoList.java
@@ -5,6 +5,9 @@
import java.io.Serializable;
import java.util.List;
+/**
+ * Result of borg list repo
+ */
public class RepoList extends RepositoryMatcher implements Serializable {
private static final long serialVersionUID = 1006757749929526034L;
@Getter
diff --git a/borgbutler-core/src/main/resources/jcs-basic-config.properties b/borgbutler-core/src/main/resources/jcs-basic-config.properties
index c3db1b0..f126769 100644
--- a/borgbutler-core/src/main/resources/jcs-basic-config.properties
+++ b/borgbutler-core/src/main/resources/jcs-basic-config.properties
@@ -13,31 +13,30 @@
# PRE-DEFINED CACHE REGIONS
# For file system content (over million of entries for one borg backup archive.
-jcs.region.archiveContent=DC2
-jcs.region.archiveContent.cacheattributes=org.apache.commons.jcs.engine.CompositeCacheAttributes
-jcs.region.archiveContent.cacheattributes.MaxObjects=1000
-jcs.region.archiveContent.cacheattributes.MemoryCacheName=org.apache.commons.jcs.engine.memory.lru.LRUMemoryCache
-jcs.region.archiveContent.cacheattributes.UseMemoryShrinker=true
-jcs.region.archiveContent.cacheattributes.MaxMemoryIdleTimeSeconds=3600
-jcs.region.archiveContent.cacheattributes.ShrinkerIntervalSeconds=60
+#jcs.region.archiveContent=DC2
+#jcs.region.archiveContent.cacheattributes=org.apache.commons.jcs.engine.CompositeCacheAttributes
+#jcs.region.archiveContent.cacheattributes.MaxObjects=1000
+#jcs.region.archiveContent.cacheattributes.MemoryCacheName=org.apache.commons.jcs.engine.memory.lru.LRUMemoryCache
+#jcs.region.archiveContent.cacheattributes.UseMemoryShrinker=true
+#jcs.region.archiveContent.cacheattributes.MaxMemoryIdleTimeSeconds=3600
+#jcs.region.archiveContent.cacheattributes.ShrinkerIntervalSeconds=60
# Only a few archive contents should be spooled (saved to disc):
-jcs.region.archiveContent.cacheattributes.MaxSpoolPerRun=5
-jcs.region.archiveContent.elementattributes=org.apache.commons.jcs.engine.ElementAttributes
-jcs.region.archiveContent.elementattributes.IsEternal=false
+#jcs.region.archiveContent.cacheattributes.MaxSpoolPerRun=5
+#jcs.region.archiveContent.elementattributes=org.apache.commons.jcs.engine.ElementAttributes
+#jcs.region.archiveContent.elementattributes.IsEternal=false
# AVAILABLE AUXILIARY CACHES
jcs.auxiliary.DC=org.apache.commons.jcs.auxiliary.disk.indexed.IndexedDiskCacheFactory
jcs.auxiliary.DC.attributes=org.apache.commons.jcs.auxiliary.disk.indexed.IndexedDiskCacheAttributes
-jcs.auxiliary.DC.attributes.maxKeySize=1000
+jcs.auxiliary.DC.attributes.MaxKeySize=1000
jcs.auxiliary.DC.attributes.OptimizeOnShutdown=true
# jcs.auxiliary.DC.attributes.DiskPath=${user.dir}/.borgbutler/cache # Will be configured in ~/.borgbutler.json
-jcs.auxiliary.DC2=org.apache.commons.jcs.auxiliary.disk.indexed.IndexedDiskCacheFactory
-jcs.auxiliary.DC2.attributes=org.apache.commons.jcs.auxiliary.disk.indexed.IndexedDiskCacheAttributes
-jcs.auxiliary.DC2.attributes.maxKeySize=10
-jcs.auxiliary.DC2.attributes.OptimizeOnShutdown=true
-jcs.auxiliary.DC2.attributes.DiskLimitType=SIZE
+#jcs.auxiliary.DC2=org.apache.commons.jcs.auxiliary.disk.indexed.IndexedDiskCacheFactory
+#jcs.auxiliary.DC2.attributes=org.apache.commons.jcs.auxiliary.disk.indexed.IndexedDiskCacheAttributes
+#jcs.auxiliary.DC2.attributes.OptimizeOnShutdown=true
+#jcs.auxiliary.DC2.attributes.DiskLimitType=SIZE
# jcs.auxiliary.DC2.attributes.MaxKeySize=10000 (KB): Will be configured in ~/.borgbutler.json!
# jcs.auxiliary.DC2.attributes.DiskPath=${user.dir}/.borgbutler/cache # Will be configured in ~/.borgbutler.json
diff --git a/borgbutler-core/src/test/java/de/micromata/borgbutler/cache/CacheTest.java b/borgbutler-core/src/test/java/de/micromata/borgbutler/cache/CacheTest.java
index 4e07239..ebf96e3 100644
--- a/borgbutler-core/src/test/java/de/micromata/borgbutler/cache/CacheTest.java
+++ b/borgbutler-core/src/test/java/de/micromata/borgbutler/cache/CacheTest.java
@@ -3,11 +3,17 @@
import de.micromata.borgbutler.config.BorgRepoConfig;
import de.micromata.borgbutler.config.Configuration;
import de.micromata.borgbutler.config.ConfigurationHandler;
+import de.micromata.borgbutler.json.borg.Archive;
+import de.micromata.borgbutler.json.borg.FilesystemItem;
import de.micromata.borgbutler.json.borg.RepoInfo;
+import de.micromata.borgbutler.json.borg.RepoList;
+import org.apache.commons.collections4.CollectionUtils;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.List;
+
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -30,9 +36,9 @@
}
assertEquals(config.getRepoConfigs().size(), ButlerCache.getInstance().getAllRepositories().size());
}
-/* {
+ {
for (BorgRepoConfig repoConfig : ConfigurationHandler.getConfiguration().getRepoConfigs()) {
- RepoList repoList = ButlerCache.getInstance().getRepoListCache().get(repoConfig, repoConfig.getRepo());
+ RepoList repoList = ButlerCache.getInstance().getRepoList(repoConfig.getRepo());
}
assertEquals(config.getRepoConfigs().size(), ButlerCache.getInstance().getAllRepositories().size());
}
@@ -41,11 +47,11 @@
BorgRepoConfig repoConfig = null;
if (CollectionUtils.isNotEmpty(repoConfigs)) {
repoConfig = repoConfigs.get(0);
- RepoList repoList = ButlerCache.getInstance().getRepoListCache().get(repoConfig, repoConfig.getRepo());
+ RepoList repoList = ButlerCache.getInstance().getRepoList(repoConfig.getRepo());
if (repoList != null && CollectionUtils.isNotEmpty(repoList.getArchives())) {
archive = repoList.getArchives().get(0);
}
- }*/
+ }
{/*
List<BorgRepoConfig> repoConfigs = ConfigurationHandler.getConfiguration().getRepoConfigs();
if (CollectionUtils.isNotEmpty(repoConfigs)) {
@@ -61,13 +67,14 @@
}
}*/
}
- /*
{
if (archive != null) {
List<FilesystemItem> content = ButlerCache.getInstance().getArchiveContent(repoConfig, archive);
log.info("Number of items (content) of archive: " + content.size());
content = ButlerCache.getInstance().getArchiveContent(repoConfig, archive);
+ log.info("Number of items (content) of archive: " + content.size());
}
- }*/
+ }
+ ButlerCache.getInstance().shutdown();
}
}
diff --git a/borgbutler-server/src/main/java/de/micromata/borgbutler/server/Main.java b/borgbutler-server/src/main/java/de/micromata/borgbutler/server/Main.java
index 2bcd91d..bee8c24 100644
--- a/borgbutler-server/src/main/java/de/micromata/borgbutler/server/Main.java
+++ b/borgbutler-server/src/main/java/de/micromata/borgbutler/server/Main.java
@@ -1,5 +1,6 @@
package de.micromata.borgbutler.server;
+import de.micromata.borgbutler.cache.ButlerCache;
import de.micromata.borgbutler.server.jetty.JettyServer;
import de.micromata.borgbutler.server.user.SingleUserManager;
import de.micromata.borgbutler.server.user.UserManager;
@@ -104,6 +105,7 @@
}
log.info("Shutting down BorgButler web server...");
server.stop();
+ ButlerCache.getInstance().shutdown();
}
private static void printHelp(Options options) {
--
Gitblit v1.10.0