From e9c74d4f90ae553035ab2edd81e336a8d205b3d2 Mon Sep 17 00:00:00 2001
From: Kai Reinhard <K.Reinhard@micromata.de>
Date: Sun, 09 Dec 2018 15:47:48 +0000
Subject: [PATCH] ...
---
borgbutler-core/src/main/java/de/micromata/borgbutler/cache/AbstractCache.java | 4
borgbutler-core/src/main/java/de/micromata/borgbutler/BorgCommands.java | 51 +++++++---
borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/Archive.java | 2
borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/RepoList.java | 2
borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFileListCache.java | 53 ++++++++++
borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ButlerCache.java | 30 ++++-
borgbutler-core/src/main/java/de/micromata/borgbutler/utils/ReplaceUtils.java | 91 ++++++++++++++++++
borgbutler-core/src/test/java/de/micromata/borgbutler/cache/CacheTest.java | 11 +-
8 files changed, 212 insertions(+), 32 deletions(-)
diff --git a/borgbutler-core/src/main/java/de/micromata/borgbutler/BorgCommands.java b/borgbutler-core/src/main/java/de/micromata/borgbutler/BorgCommands.java
index 7388b2d..6185d2a 100644
--- a/borgbutler-core/src/main/java/de/micromata/borgbutler/BorgCommands.java
+++ b/borgbutler-core/src/main/java/de/micromata/borgbutler/BorgCommands.java
@@ -5,18 +5,23 @@
import de.micromata.borgbutler.config.ConfigurationHandler;
import de.micromata.borgbutler.config.Definitions;
import de.micromata.borgbutler.json.JsonUtils;
-import de.micromata.borgbutler.json.borg.ArchiveInfo;
-import de.micromata.borgbutler.json.borg.RepoInfo;
-import de.micromata.borgbutler.json.borg.RepoList;
+import de.micromata.borgbutler.json.borg.*;
import org.apache.commons.exec.*;
import org.apache.commons.exec.environment.EnvironmentUtils;
+import org.apache.commons.io.IOUtils;
import org.apache.commons.io.output.ByteArrayOutputStream;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.FileWriter;
import java.io.IOException;
+import java.io.OutputStream;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Map;
+import java.util.Scanner;
public class BorgCommands {
private static Logger log = LoggerFactory.getLogger(BorgCommands.class);
@@ -64,18 +69,35 @@
return repoList;
}
- public static String list(BorgRepoConfig repoConfig, String archive) {
- String json = execute(repoConfig, "list", repoConfig.getRepo() + "::" + archive,
+ public static List<FilesystemItem> list(BorgRepoConfig repoConfig, Archive archive) {
+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+ execute(outputStream, repoConfig, "list", repoConfig.getRepo() + "::" + archive.getArchive(),
"--json-lines");
- if (json == null) {
- return null;
+ String response = outputStream.toString(Definitions.STD_CHARSET);
+ try {
+ IOUtils.copy(new StringReader(response), new FileWriter("response.json"));
+ }catch (IOException ex) {
+
}
- // RepoList repoList = JsonUtils.fromJson(RepoList.class, json);
- // repoList.setOriginalJson(json);
- return json;
+ List<FilesystemItem> content = new ArrayList<>();
+ try (Scanner scanner = new Scanner(response)) {
+ while (scanner.hasNextLine()) {
+ String json = scanner.nextLine();
+ FilesystemItem item = JsonUtils.fromJson(FilesystemItem.class, json);
+ content.add(item);
+ }
+ }
+ return content;
}
private static String execute(BorgRepoConfig repoConfig, String command, String repoOrArchive, String... args) {
+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+ execute(outputStream, repoConfig, command, repoOrArchive, args);
+ String json = outputStream.toString(Definitions.STD_CHARSET);
+ return json;
+ }
+
+ private static void execute(OutputStream outputStream, BorgRepoConfig repoConfig, String command, String repoOrArchive, String... args) {
CommandLine cmdLine = new CommandLine(ConfigurationHandler.getConfiguration().getBorgCommand());
cmdLine.addArgument(command);
for (String arg : args) {
@@ -86,7 +108,6 @@
//executor.setExitValue(2);
//ExecuteWatchdog watchdog = new ExecuteWatchdog(60000);
//executor.setWatchdog(watchdog);
- ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
ExecuteResultHandler handler = new DefaultExecuteResultHandler();
PumpStreamHandler streamHandler = new PumpStreamHandler(outputStream);
executor.setStreamHandler(streamHandler);
@@ -96,14 +117,12 @@
executor.execute(cmdLine, getEnvironment(repoConfig));
} catch (Exception ex) {
log.error("Error while creating environment for borg call '" + borgCall + "': " + ex.getMessage(), ex);
- String response = outputStream.toString(Definitions.STD_CHARSET);
- log.error("Response: " + response);
- return null;
+ log.error("Response: " + StringUtils.abbreviate(outputStream.toString(), 10000));
+ return;
}
- String json = outputStream.toString(Definitions.STD_CHARSET);
- return json;
}
+
private static Map<String, String> getEnvironment(BorgRepoConfig repoConfig) throws IOException {
Configuration config = ConfigurationHandler.getConfiguration();
Map<String, String> env = EnvironmentUtils.getProcEnvironment();
diff --git a/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/AbstractCache.java b/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/AbstractCache.java
index b31e7e0..7cb2908 100644
--- a/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/AbstractCache.java
+++ b/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/AbstractCache.java
@@ -129,8 +129,8 @@
this.compress = zip;
String filename = CACHE_FILE_PREFIX + cacheFilename + "." + CACHE_FILE_EXTENSION;
if (this.compress)
- filename = filename + CACHE_FILE_EXTENSION;
- cacheFile = new File(cacheDir, CACHE_FILE_PREFIX + cacheFilename + "." + CACHE_FILE_EXTENSION);
+ filename = filename + CACHE_FILE_ZIP_EXTENSION;
+ cacheFile = new File(cacheDir, filename);
this.state = STATE.INITIAL;
}
diff --git a/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFileListCache.java b/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFileListCache.java
new file mode 100644
index 0000000..fbe2d34
--- /dev/null
+++ b/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFileListCache.java
@@ -0,0 +1,53 @@
+package de.micromata.borgbutler.cache;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import de.micromata.borgbutler.BorgCommands;
+import de.micromata.borgbutler.config.BorgRepoConfig;
+import de.micromata.borgbutler.json.borg.Archive;
+import de.micromata.borgbutler.json.borg.FilesystemItem;
+import de.micromata.borgbutler.utils.ReplaceUtils;
+import lombok.Getter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.util.List;
+
+class ArchiveFileListCache extends AbstractCache {
+ private static Logger log = LoggerFactory.getLogger(ArchiveFileListCache.class);
+ public static final String CACHE_ARCHIVE_LISTS_BASENAME = "archive-content-";
+
+ @JsonIgnore
+ @Getter
+ private Archive archive;
+ @JsonProperty
+ private List<FilesystemItem> content;
+
+ public List<FilesystemItem> getContent(BorgRepoConfig repoConfig) {
+ if (content == null) {
+ read();
+ }
+ if (content == null) {
+ this.content = BorgCommands.list(repoConfig, archive);
+ save();
+ }
+ return content;
+ }
+
+ protected void update(AbstractCache readCache) {
+ this.content = ((ArchiveFileListCache) readCache).content;
+ }
+
+ /**
+ * Needed by jackson for deserialization.
+ */
+ ArchiveFileListCache() {
+ }
+
+ ArchiveFileListCache(File cacheDir, BorgRepoConfig repoConfig, Archive archive) {
+ super(cacheDir, ReplaceUtils.encodeFilename(CACHE_ARCHIVE_LISTS_BASENAME + repoConfig.getName() + "-" + archive.getArchive(),
+ true), true);
+ this.archive = archive;
+ }
+}
diff --git a/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ButlerCache.java b/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ButlerCache.java
index 1be4dca..00ecb3b 100644
--- a/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ButlerCache.java
+++ b/borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ButlerCache.java
@@ -1,8 +1,12 @@
package de.micromata.borgbutler.cache;
import com.fasterxml.jackson.annotation.JsonIgnore;
+import de.micromata.borgbutler.config.BorgRepoConfig;
import de.micromata.borgbutler.config.ConfigurationHandler;
+import de.micromata.borgbutler.json.borg.Archive;
+import de.micromata.borgbutler.json.borg.FilesystemItem;
import lombok.Getter;
+import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -21,6 +25,7 @@
private RepoListCache repoListCache;
private ArchiveListCache archiveListCache;
private List<AbstractElementsCache> caches;
+ private List<ArchiveFileListCache> archiveFileListCaches;
@JsonIgnore
private File cacheDir;
@@ -29,18 +34,30 @@
return instance;
}
- public void read() {
- for (AbstractElementsCache cache : caches) {
- cache.read();
- }
- }
-
public void save() {
for (AbstractElementsCache cache : caches) {
cache.save();
}
}
+ public List<FilesystemItem> getArchiveContent(BorgRepoConfig repoConfig, Archive archive) {
+ if (archive == null || StringUtils.isBlank(archive.getArchive())) {
+ return null;
+ }
+ ArchiveFileListCache cache = null;
+ for (ArchiveFileListCache existingCache : archiveFileListCaches) {
+ if (archive.equals(existingCache.getArchive())) {
+ // Cache is already known:
+ cache = existingCache;
+ break;
+ }
+ }
+ if (cache == null) {
+ cache = new ArchiveFileListCache(cacheDir, repoConfig, archive);
+ }
+ return cache.getContent(repoConfig);
+ }
+
/**
* Removes all cache files and clears all caches.
*/
@@ -67,5 +84,6 @@
caches.add(repoInfoCache = new RepoInfoCache(cacheDir));
caches.add(repoListCache = new RepoListCache(cacheDir));
caches.add(archiveListCache = new ArchiveListCache(cacheDir));
+ archiveFileListCaches = new ArrayList<>();
}
}
diff --git a/borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/Archive1.java b/borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/Archive.java
similarity index 96%
rename from borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/Archive1.java
rename to borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/Archive.java
index 65a4982..6eee7de 100644
--- a/borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/Archive1.java
+++ b/borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/Archive.java
@@ -8,7 +8,7 @@
/**
* This object is given by <tt>borg list repo</tt>.
*/
-public class Archive1 {
+public class Archive {
@Getter
private String archive;
@Getter
diff --git a/borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/RepoList.java b/borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/RepoList.java
index 51d1454..d9993da 100644
--- a/borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/RepoList.java
+++ b/borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/RepoList.java
@@ -6,7 +6,7 @@
public class RepoList extends RepositoryMatcher {
@Getter
- private List<Archive1> archives;
+ private List<Archive> archives;
@Getter
private Encryption encryption;
diff --git a/borgbutler-core/src/main/java/de/micromata/borgbutler/utils/ReplaceUtils.java b/borgbutler-core/src/main/java/de/micromata/borgbutler/utils/ReplaceUtils.java
new file mode 100644
index 0000000..c7e1952
--- /dev/null
+++ b/borgbutler-core/src/main/java/de/micromata/borgbutler/utils/ReplaceUtils.java
@@ -0,0 +1,91 @@
+package de.micromata.borgbutler.utils;
+
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class ReplaceUtils {
+ public static final String ALLOWED_FILENAME_CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-";
+ public static final String PRESERVED_FILENAME_CHARS = "\"*/:<>?\\|";
+ public static final char FILENAME_REPLACE_CHAR = '_';
+
+ private static Map<Character, String> umlautReplacementMap;
+
+ static {
+ umlautReplacementMap = new HashMap<>();
+ umlautReplacementMap.put('Ä', "Ae");
+ umlautReplacementMap.put('Ö', "Oe");
+ umlautReplacementMap.put('Ü', "Ue");
+ umlautReplacementMap.put('ä', "ae");
+ umlautReplacementMap.put('ö', "oe");
+ umlautReplacementMap.put('ü', "ue");
+ umlautReplacementMap.put('ß', "ss");
+ }
+
+ /**
+ * Preserved characters (Windows): 0x00-0x1F 0x7F " * / : < > ? \ |
+ * Preserved characters (Mac OS): ':'
+ * Preserved characters (Unix): '/'
+ * Max length: 255
+ *
+ * @param filename
+ * @param reducedCharsOnly if true, only {@link #ALLOWED_FILENAME_CHARS} are allowed and German Umlaute are replaced
+ * 'Ä'->'Ae' etc. If not, all characters excluding {@link #PRESERVED_FILENAME_CHARS} are allowed and
+ * all white spaces will be replaced by ' ' char.
+ * @return
+ */
+
+ public static String encodeFilename(String filename, boolean reducedCharsOnly) {
+ if (StringUtils.isEmpty(filename)) {
+ return "file";
+ }
+ if (reducedCharsOnly) {
+ filename = replaceGermanUmlauteAndAccents(filename);
+ }
+ StringBuilder sb = new StringBuilder();
+ char[] charArray = filename.toCharArray();
+ for (int i = 0; i < charArray.length; i++) {
+ char ch = charArray[i];
+ if (reducedCharsOnly) {
+ if (ALLOWED_FILENAME_CHARS.indexOf(ch) >= 0) {
+ sb.append(ch);
+ } else {
+ sb.append(FILENAME_REPLACE_CHAR);
+ }
+ } else {
+ if (ch <= 31 || ch == 127) { // Not 0x00-0x1F and not 0x7F
+ sb.append(FILENAME_REPLACE_CHAR);
+ } else if (PRESERVED_FILENAME_CHARS.indexOf(ch) >= 0) {
+ sb.append(FILENAME_REPLACE_CHAR);
+ } else if (Character.isWhitespace(ch)) {
+ sb.append(' ');
+ } else {
+ sb.append(ch);
+ }
+ }
+ }
+ String result = sb.toString();
+ if (result.length() > 255) {
+ return result.substring(0, 255);
+ }
+ return result;
+ }
+
+ public static String replaceGermanUmlauteAndAccents(String text) {
+ if (text == null) {
+ return null;
+ }
+ StringBuilder sb = new StringBuilder();
+ char[] charArray = text.toCharArray();
+ for (int i = 0; i < charArray.length; i++) {
+ char ch = charArray[i];
+ if (umlautReplacementMap.containsKey(ch)) {
+ sb.append(umlautReplacementMap.get(ch));
+ } else {
+ sb.append(ch);
+ }
+ }
+ return StringUtils.stripAccents(sb.toString());
+ }
+}
diff --git a/borgbutler-core/src/test/java/de/micromata/borgbutler/cache/CacheTest.java b/borgbutler-core/src/test/java/de/micromata/borgbutler/cache/CacheTest.java
index 9fe3e29..647d5ac 100644
--- a/borgbutler-core/src/test/java/de/micromata/borgbutler/cache/CacheTest.java
+++ b/borgbutler-core/src/test/java/de/micromata/borgbutler/cache/CacheTest.java
@@ -3,7 +3,7 @@
import de.micromata.borgbutler.config.BorgRepoConfig;
import de.micromata.borgbutler.config.Configuration;
import de.micromata.borgbutler.config.ConfigurationHandler;
-import de.micromata.borgbutler.json.borg.Archive1;
+import de.micromata.borgbutler.json.borg.Archive;
import de.micromata.borgbutler.json.borg.RepoInfo;
import de.micromata.borgbutler.json.borg.RepoList;
import org.apache.commons.collections4.CollectionUtils;
@@ -43,7 +43,7 @@
assertEquals(config.getRepoConfigs().size(), ButlerCache.getInstance().getRepoInfoCache().getElements().size());
}
List<BorgRepoConfig> repoConfigs = ConfigurationHandler.getConfiguration().getRepoConfigs();
- Archive1 archive = null;
+ Archive archive = null;
BorgRepoConfig repoConfig = null;
if (CollectionUtils.isNotEmpty(repoConfigs)) {
repoConfig = repoConfigs.get(0);
@@ -67,11 +67,10 @@
}
}*/
}
- {/*
+ {
if (archive != null) {
- String json = BorgCommands.list(repoConfig, archive.getArchive());
- log.info(json);
- }*/
+ ButlerCache.getInstance().getArchiveContent(repoConfig, archive);
+ }
}
butlerCache.save();
}
--
Gitblit v1.10.0