JCS...

Kai Reinhard

10.53.2018 7da4760af26519051b1accd840c9199a11f7491a

JCS...

 borgbutler-core/src/main/java/de/micromata/borgbutler/BorgCommands.java

@@ -14,10 +14,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.FileWriter;
import java.io.IOException;
import java.io.OutputStream;
import java.io.StringReader;
import java.io.*;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -42,6 +39,16 @@
        return repoInfo;
    }

    public static RepoList list(BorgRepoConfig repoConfig) {
        String json = execute(repoConfig, "list", repoConfig.getRepo(), "--json");
        if (json == null) {
            return null;
        }
        RepoList repoList = JsonUtils.fromJson(RepoList.class, json);
        repoList.setOriginalJson(json);
        return repoList;
    }

    /**
     * Executes borg info archive
     *
@@ -54,30 +61,29 @@
        if (json == null) {
            return null;
        }
        ArchiveInfo archiveList = JsonUtils.fromJson(ArchiveInfo.class, json);
        archiveList.setOriginalJson(json);
        return archiveList;
        ArchiveInfo archiveInfo = JsonUtils.fromJson(ArchiveInfo.class, json);
        archiveInfo.setOriginalJson(json);
        return archiveInfo;
    }

    public static RepoList list(BorgRepoConfig repoConfig) {
        String json = execute(repoConfig, "list", repoConfig.getRepo(), "--json");
        if (json == null) {
            return null;
        }
        RepoList repoList = JsonUtils.fromJson(RepoList.class, json);
        repoList.setOriginalJson(json);
        return repoList;
    }

    public static List<FilesystemItem> list(BorgRepoConfig repoConfig, Archive archive) {
    public static List<FilesystemItem> listArchiveContent(BorgRepoConfig repoConfig, Archive archive) {
        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
        execute(outputStream, repoConfig, "list", repoConfig.getRepo() + "::" + archive.getArchive(),
                "--json-lines");
        String response = outputStream.toString(Definitions.STD_CHARSET);
        //execute(outputStream, repoConfig, "list", repoConfig.getRepo() + "::" + archive.getArchive(),
        //        "--json-lines");
        //String response = outputStream.toString(Definitions.STD_CHARSET);
        String response = null;
        try {
            IOUtils.copy(new StringReader(response), new FileWriter("response.json"));
            File file = new File("/Users/kai/tmp/response.json");
            if (file.exists()) {
                log.info("******** Reading test file....");
                StringWriter writer = new StringWriter();
                IOUtils.copy(new FileReader(file), writer);
                response = writer.toString();
            } else {
                log.info("******** Writing test file....");
                IOUtils.copy(new StringReader(response), new FileWriter(file));
            }
        } catch (IOException ex) {

        }
        List<FilesystemItem> content = new ArrayList<>();
        try (Scanner scanner = new Scanner(response)) {

 borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ButlerCache.java

@@ -5,6 +5,7 @@
import de.micromata.borgbutler.config.BorgRepoConfig;
import de.micromata.borgbutler.config.ConfigurationHandler;
import de.micromata.borgbutler.json.borg.*;
import org.apache.commons.jcs.JCS;
import org.apache.commons.jcs.access.CacheAccess;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
@@ -22,6 +23,7 @@
    private JCSCache jcsCache = JCSCache.getInstance();
    private CacheAccess<String, RepoInfo> repoInfoCacheAccess;
    private CacheAccess<String, RepoList> repoListCacheAccess;
    private CacheAccess<String, List<FilesystemItem>> archiveContentCacheAccess;

    @JsonIgnore
    private File cacheDir;
@@ -65,25 +67,53 @@

    public RepoList getRepoList(String idOrName) {
        BorgRepoConfig repoConfig = ConfigurationHandler.getConfiguration().getRepoConfig(idOrName);
        ArchiveInfo archiveInfo = BorgCommands.info(repoConfig, repoConfig.getRepo());
        RepoList repoList = BorgCommands.list(repoConfig);
        return null;
        //ArchiveInfo archiveInfo = BorgCommands.info(repoConfig, repoConfig.getRepo());
        RepoList repoList = repoListCacheAccess.get(repoConfig.getRepo());
        if (repoList == null) {
            repoList = BorgCommands.list(repoConfig);
            repoListCacheAccess.put(repoConfig.getRepo(), repoList);
        }
        if (repoList == null) {
            log.warn("Repo with name '" + idOrName + "' not found.");
        }
        return repoList;
    }

    public List<FilesystemItem> getArchiveContent_(BorgRepoConfig repoConfig, Archive archive) {
    public List<FilesystemItem> getArchiveContent(BorgRepoConfig repoConfig, Archive archive) {
        if (archive == null || StringUtils.isBlank(archive.getArchive())) {
            return null;
        }
        List<FilesystemItem> content = BorgCommands.list(repoConfig, archive);
        String repoArchiveId = getRepoArchiveId(repoConfig.getRepo(), archive.getId());
        List<FilesystemItem> content = archiveContentCacheAccess.get(repoArchiveId);
        if (content == null) {
            content = BorgCommands.listArchiveContent(repoConfig, archive);
            archiveContentCacheAccess.put(repoArchiveId, content);
            archiveContentCacheAccess.getStatistics();
        }
        log.info("archiveContentCacheAccess.stats: " + this.archiveContentCacheAccess.getStats());
        if (content == null) {
            log.warn("Repo::archiv with name '" + repoConfig.getRepo() + "::" + archive.getArchive() + "' not found.");
        }
        return content;
    }

    public String getRepoArchiveId(String repo, String archiveId) {
        return repo + "::" + archiveId;
    }

    public void shutdown() {
        log.info("archiveContentCacheAccess.stats: " + this.archiveContentCacheAccess.getStats());
        JCS.shutdown();
    }

    private ButlerCache() {
        cacheDir = new File(ConfigurationHandler.getInstance().getWorkingDir(), CACHE_DIR_NAME);
        if (!cacheDir.exists()) {
            log.info("Creating cache dir: " + cacheDir.getAbsolutePath());
            cacheDir.mkdir();
        }
        this.repoInfoCacheAccess = jcsCache.getJCSCache();
        this.repoInfoCacheAccess = jcsCache.getJCSCache("repoInfo");
        this.repoListCacheAccess = jcsCache.getJCSCache("repoList");
        this.archiveContentCacheAccess = jcsCache.getJCSCache("archiveContent");
    }
}

 borgbutler-core/src/main/java/de/micromata/borgbutler/cache/JCSCache.java

@@ -18,28 +18,12 @@
    private static final String CONFIG_FILE = "jcs-basic-config.properties";
    public static final String CACHE_DIR_NAME = "cache";

    public enum Region {DEFAULT, ARCHIVE_CONTENT}

    public static JCSCache getInstance() {
        return instance;
    }

    /**
     * @param <K>
     * @param <V>
     * @return JCS cache for default region.
     */
    public <K, V> CacheAccess<K, V> getJCSCache() {
        return JCS.getInstance("default");
    }

    public <K, V> CacheAccess<K, V> getJCSCache(Region region) {
        switch (region) {
            case ARCHIVE_CONTENT:
                return JCS.getInstance("default");
            default:
                return getJCSCache();
        }
    public <K, V> CacheAccess<K, V> getJCSCache(String region) {
        return JCS.getInstance(region);
    }

    private JCSCache() {
@@ -57,9 +41,10 @@
            log.error("Error while loading jcs config file '" + CONFIG_FILE + "': " + ex.getMessage(), ex);
        }
        props.setProperty("jcs.auxiliary.DC.attributes.DiskPath", cacheDir.getAbsolutePath());
        props.setProperty("jcs.auxiliary.DC2.attributes.DiskPath", cacheDir.getAbsolutePath());
        int cacheMaxDiscSizeMB = configuration.getCacheMaxDiscSizeMB();
        props.setProperty("jcs.auxiliary.DC2.attributes.MaxKeySize", String.valueOf(cacheMaxDiscSizeMB * 1000));
        //props.setProperty("jcs.auxiliary.DC2.attributes.DiskPath", cacheDir.getAbsolutePath());
        //int cacheMaxDiscSizeMB = configuration.getCacheMaxDiscSizeMB();
        //log.info("Using cache size for archive contents: " + cacheMaxDiscSizeMB + "MB.");
        //props.setProperty("jcs.auxiliary.DC2.attributes.MaxKeySize", String.valueOf(cacheMaxDiscSizeMB * 1000));
        JCS.setConfigProperties(props);
    }
}

 borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/ArchiveInfo.java

@@ -8,7 +8,7 @@
import java.util.List;

/**
 * Result of <tt>borg list repo</tt>.
 * Result of <tt>borg info repo::archive</tt>.
 */
public class ArchiveInfo extends RepositoryMatcher implements Serializable {
    private static final long serialVersionUID = -4200553322856662346L;

 borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/RepoInfo.java

@@ -5,6 +5,9 @@

import java.io.Serializable;

/**
 * Result of borg info repo
 */
public class RepoInfo extends RepositoryMatcher implements Serializable {
    private static final long serialVersionUID = -1588038325129799400L;
    @Getter

 borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/RepoList.java

@@ -5,6 +5,9 @@
import java.io.Serializable;
import java.util.List;

/**
 * Result of borg list repo
 */
public class RepoList extends RepositoryMatcher implements Serializable {
    private static final long serialVersionUID = 1006757749929526034L;
    @Getter

 borgbutler-core/src/main/resources/jcs-basic-config.properties

@@ -13,31 +13,30 @@
# PRE-DEFINED CACHE REGIONS

# For file system content (over million of entries for one borg backup archive.
jcs.region.archiveContent=DC2
jcs.region.archiveContent.cacheattributes=org.apache.commons.jcs.engine.CompositeCacheAttributes
jcs.region.archiveContent.cacheattributes.MaxObjects=1000
jcs.region.archiveContent.cacheattributes.MemoryCacheName=org.apache.commons.jcs.engine.memory.lru.LRUMemoryCache
jcs.region.archiveContent.cacheattributes.UseMemoryShrinker=true
jcs.region.archiveContent.cacheattributes.MaxMemoryIdleTimeSeconds=3600
jcs.region.archiveContent.cacheattributes.ShrinkerIntervalSeconds=60
#jcs.region.archiveContent=DC2
#jcs.region.archiveContent.cacheattributes=org.apache.commons.jcs.engine.CompositeCacheAttributes
#jcs.region.archiveContent.cacheattributes.MaxObjects=1000
#jcs.region.archiveContent.cacheattributes.MemoryCacheName=org.apache.commons.jcs.engine.memory.lru.LRUMemoryCache
#jcs.region.archiveContent.cacheattributes.UseMemoryShrinker=true
#jcs.region.archiveContent.cacheattributes.MaxMemoryIdleTimeSeconds=3600
#jcs.region.archiveContent.cacheattributes.ShrinkerIntervalSeconds=60
# Only a few archive contents should be spooled (saved to disc):
jcs.region.archiveContent.cacheattributes.MaxSpoolPerRun=5
jcs.region.archiveContent.elementattributes=org.apache.commons.jcs.engine.ElementAttributes
jcs.region.archiveContent.elementattributes.IsEternal=false
#jcs.region.archiveContent.cacheattributes.MaxSpoolPerRun=5
#jcs.region.archiveContent.elementattributes=org.apache.commons.jcs.engine.ElementAttributes
#jcs.region.archiveContent.elementattributes.IsEternal=false


# AVAILABLE AUXILIARY CACHES

jcs.auxiliary.DC=org.apache.commons.jcs.auxiliary.disk.indexed.IndexedDiskCacheFactory
jcs.auxiliary.DC.attributes=org.apache.commons.jcs.auxiliary.disk.indexed.IndexedDiskCacheAttributes
jcs.auxiliary.DC.attributes.maxKeySize=1000
jcs.auxiliary.DC.attributes.MaxKeySize=1000
jcs.auxiliary.DC.attributes.OptimizeOnShutdown=true
# jcs.auxiliary.DC.attributes.DiskPath=${user.dir}/.borgbutler/cache # Will be configured in ~/.borgbutler.json

jcs.auxiliary.DC2=org.apache.commons.jcs.auxiliary.disk.indexed.IndexedDiskCacheFactory
jcs.auxiliary.DC2.attributes=org.apache.commons.jcs.auxiliary.disk.indexed.IndexedDiskCacheAttributes
jcs.auxiliary.DC2.attributes.maxKeySize=10
jcs.auxiliary.DC2.attributes.OptimizeOnShutdown=true
jcs.auxiliary.DC2.attributes.DiskLimitType=SIZE
#jcs.auxiliary.DC2=org.apache.commons.jcs.auxiliary.disk.indexed.IndexedDiskCacheFactory
#jcs.auxiliary.DC2.attributes=org.apache.commons.jcs.auxiliary.disk.indexed.IndexedDiskCacheAttributes
#jcs.auxiliary.DC2.attributes.OptimizeOnShutdown=true
#jcs.auxiliary.DC2.attributes.DiskLimitType=SIZE
# jcs.auxiliary.DC2.attributes.MaxKeySize=10000 (KB): Will be configured in ~/.borgbutler.json!
# jcs.auxiliary.DC2.attributes.DiskPath=${user.dir}/.borgbutler/cache # Will be configured in ~/.borgbutler.json

 borgbutler-core/src/test/java/de/micromata/borgbutler/cache/CacheTest.java

@@ -3,11 +3,17 @@
import de.micromata.borgbutler.config.BorgRepoConfig;
import de.micromata.borgbutler.config.Configuration;
import de.micromata.borgbutler.config.ConfigurationHandler;
import de.micromata.borgbutler.json.borg.Archive;
import de.micromata.borgbutler.json.borg.FilesystemItem;
import de.micromata.borgbutler.json.borg.RepoInfo;
import de.micromata.borgbutler.json.borg.RepoList;
import org.apache.commons.collections4.CollectionUtils;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.List;

import static org.junit.jupiter.api.Assertions.assertEquals;


@@ -30,9 +36,9 @@
            }
            assertEquals(config.getRepoConfigs().size(), ButlerCache.getInstance().getAllRepositories().size());
        }
/*        {
        {
            for (BorgRepoConfig repoConfig : ConfigurationHandler.getConfiguration().getRepoConfigs()) {
                RepoList repoList = ButlerCache.getInstance().getRepoListCache().get(repoConfig, repoConfig.getRepo());
                RepoList repoList = ButlerCache.getInstance().getRepoList(repoConfig.getRepo());
            }
            assertEquals(config.getRepoConfigs().size(), ButlerCache.getInstance().getAllRepositories().size());
        }
@@ -41,11 +47,11 @@
        BorgRepoConfig repoConfig = null;
        if (CollectionUtils.isNotEmpty(repoConfigs)) {
            repoConfig = repoConfigs.get(0);
            RepoList repoList = ButlerCache.getInstance().getRepoListCache().get(repoConfig, repoConfig.getRepo());
            RepoList repoList = ButlerCache.getInstance().getRepoList(repoConfig.getRepo());
            if (repoList != null && CollectionUtils.isNotEmpty(repoList.getArchives())) {
                archive = repoList.getArchives().get(0);
            }
        }*/
        }
        {/*
            List<BorgRepoConfig> repoConfigs = ConfigurationHandler.getConfiguration().getRepoConfigs();
            if (CollectionUtils.isNotEmpty(repoConfigs)) {
@@ -61,13 +67,14 @@
                }
            }*/
        }
        /*
        {
            if (archive != null) {
                List<FilesystemItem> content = ButlerCache.getInstance().getArchiveContent(repoConfig, archive);
                log.info("Number of items (content) of archive: " + content.size());
                content = ButlerCache.getInstance().getArchiveContent(repoConfig, archive);
                log.info("Number of items (content) of archive: " + content.size());
            }
        }*/
        }
        ButlerCache.getInstance().shutdown();
    }
}

 borgbutler-server/src/main/java/de/micromata/borgbutler/server/Main.java

@@ -1,5 +1,6 @@
package de.micromata.borgbutler.server;

import de.micromata.borgbutler.cache.ButlerCache;
import de.micromata.borgbutler.server.jetty.JettyServer;
import de.micromata.borgbutler.server.user.SingleUserManager;
import de.micromata.borgbutler.server.user.UserManager;
@@ -104,6 +105,7 @@
        }
        log.info("Shutting down BorgButler web server...");
        server.stop();
        ButlerCache.getInstance().shutdown();
    }

    private static void printHelp(Options options) {