...

Kai Reinhard

15.55.2018 df12a3a8864d167fb300dd09e645dda430d24164

...

 borgbutler-core/src/main/java/de/micromata/borgbutler/BorgCommands.java

@@ -4,10 +4,12 @@
import de.micromata.borgbutler.config.Configuration;
import de.micromata.borgbutler.config.ConfigurationHandler;
import de.micromata.borgbutler.config.Definitions;
import de.micromata.borgbutler.data.Archive;
import de.micromata.borgbutler.data.Repository;
import de.micromata.borgbutler.json.JsonUtils;
import de.micromata.borgbutler.json.borg.*;
import de.micromata.borgbutler.utils.DateUtils;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.exec.CommandLine;
import org.apache.commons.exec.DefaultExecutor;
import org.apache.commons.exec.PumpStreamHandler;
@@ -60,6 +62,7 @@
     * The given repository will be cloned and archives will be added.
     * The field {@link Repository#getLastModified()} of masterRepository will be updated.
     *
     * @param repoConfig       The repo config associated to the masterRepository. Needed for the borg call.
     * @param masterRepository Repository without archives.
     * @return Parsed repo config returned by Borg command including archives.
     */
@@ -89,24 +92,52 @@
    }

    /**
     * Executes borg info archive
     * Executes borg info repository::archive.
     * The given repository will be cloned and assigned to the returned archive.
     * The field {@link Repository#getLastModified()} of masterRepository will be updated.
     *
     * @param repoConfig
     * @param archive
     * @return
     * @param repoConfig       The repo config associated to the masterRepository. Needed for the borg call.
     * @param archiveName      The name of the archive returned by {@link BorgArchive#getArchive()}
     * @param masterRepository Repository without archives.
     * @return Parsed repo config returned by Borg command including archives.
     */
    public static BorgArchiveInfo info(BorgRepoConfig repoConfig, String archive) {
        String json = execute(repoConfig, "info", repoConfig.getRepo() + "::" + archive, "--json");
    public static Archive info(BorgRepoConfig repoConfig, String archiveName, Repository masterRepository) {
        String archiveFullname = repoConfig.getRepo() + "::" + archiveName;
        String json = execute(repoConfig, "info", archiveFullname, "--json");
        if (json == null) {
            return null;
        }
        BorgArchiveInfo archiveInfo = JsonUtils.fromJson(BorgArchiveInfo.class, json);
        return archiveInfo;
        if (archiveInfo == null) {
            log.error("Archive '" + archiveFullname + "' not found.");
            return null;
        }
        masterRepository.setLastModified(DateUtils.format(archiveInfo.getRepository().getLastModified()))
                .setLastCacheRefresh(DateUtils.format(LocalDateTime.now()));
        Repository repository = ObjectUtils.clone(masterRepository);
        Archive archive = new Archive()
                .setCache(archiveInfo.getCache())
                .setEncryption(archiveInfo.getEncryption())
                .setRepository(repository);
        if (CollectionUtils.isEmpty(archiveInfo.getArchives())) {
            log.error("The returned borg archive contains no archive infos: " + json);
            return null;
        }
        if (archiveInfo.getArchives().size() > 1   ) {
            log.warn("Archive '" + archiveFullname + "' contains more than one archives!? (Using only first.)");
        }
        BorgArchive2 borgArchive2 = archiveInfo.getArchives().get(0);
        archive.setStart(DateUtils.format(borgArchive2.getStart()))
                .setChunkerParams(borgArchive2.getChunkerParams())
                .setCommandLine(borgArchive2.getCommandLine())
                .setComment(borgArchive2.getComment())
                .setStats(borgArchive2.getStats());
        return archive;
    }

    public static List<BorgFilesystemItem> listArchiveContent(BorgRepoConfig repoConfig, BorgArchive archive) {
    public static List<BorgFilesystemItem> listArchiveContent(BorgRepoConfig repoConfig, String archiveId) {
        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
        execute(outputStream, repoConfig, "list", repoConfig.getRepo() + "::" + archive.getArchive(),
        execute(outputStream, repoConfig, "list", repoConfig.getRepo() + "::" + archiveId,
                "--json-lines");
        String response = outputStream.toString(Definitions.STD_CHARSET);
        List<BorgFilesystemItem> content = new ArrayList<>();

 borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ButlerCache.java

@@ -4,6 +4,7 @@
import de.micromata.borgbutler.config.BorgRepoConfig;
import de.micromata.borgbutler.config.Configuration;
import de.micromata.borgbutler.config.ConfigurationHandler;
import de.micromata.borgbutler.data.Archive;
import de.micromata.borgbutler.data.Repository;
import de.micromata.borgbutler.json.borg.BorgArchive;
import de.micromata.borgbutler.json.borg.BorgFilesystemItem;
@@ -26,6 +27,7 @@
    private JCSCache jcsCache;
    private CacheAccess<String, Repository> repoCacheAccess;
    private CacheAccess<String, Repository> repoArchivesCacheAccess;
    private CacheAccess<String, Archive> archivesCacheAccess;
    private ArchiveFilelistCache archiveFilelistCache;

    public static ButlerCache getInstance() {
@@ -89,6 +91,8 @@
        clearRepoArchicesCacheAccess();
        log.info("Clearing cache with file lists of archives...");
        this.archiveFilelistCache.removeAllCacheFiles();
        log.info("Clearing archives cache...");
        this.archivesCacheAccess.clear();
    }

    public void clearRepoCacheAccess() {
@@ -121,13 +125,63 @@
        return repository;
    }

    public Archive getArchive(String repoName, String archiveIdOrName) {
        return getArchive(repoName, archiveIdOrName, false);
    }

    /**
     * @param repoName
     * @param archiveIdOrName
     * @param forceReload     If true, any cache value will be ignored. Default is false.
     * @return
     */
    public Archive getArchive(String repoName, String archiveIdOrName, boolean forceReload) {
        BorgRepoConfig repoConfig = ConfigurationHandler.getConfiguration().getRepoConfig(repoName);
        if (repoConfig == null) {
            log.error("Can't find configured repo '" + repoName + "'.");
            return null;
        }
        return getArchive(repoConfig, archiveIdOrName, forceReload);
    }

    public Archive getArchive(BorgRepoConfig repoConfig, String archiveIdOrName, boolean forceReload) {
        Repository masterRepository = getRepositoryArchives(repoConfig.getRepo());
        if (masterRepository == null) {
            log.error("Repository '" + repoConfig.getRepo() + "' not found.");
            return null;
        }
        String archiveName = archiveIdOrName;
        if (CollectionUtils.isEmpty(masterRepository.getArchives())) {
            log.warn("Repository '" + repoConfig.getRepo() + "' doesn't contain archives.");
        } else {
            for (BorgArchive borgArchive : masterRepository.getArchives()) {
                if (StringUtils.equals(borgArchive.getArchive(), archiveIdOrName)
                        || StringUtils.equals(borgArchive.getId(), archiveIdOrName)) {
                    archiveName = borgArchive.getArchive();
                    break;
                }
            }
        }
        String archiveFullname = repoConfig.getRepo() + "::" + archiveName;
        if (!forceReload) {
            Archive archive = this.archivesCacheAccess.get(archiveFullname);
            if (archive != null) {
                return archive;
            }
        }
        Archive archive = BorgCommands.info(repoConfig, archiveName, masterRepository);
        if (archive != null)
            this.archivesCacheAccess.put(archiveFullname, archive);
        return archive;
    }

    public BorgFilesystemItem[] getArchiveContent(BorgRepoConfig repoConfig, BorgArchive archive) {
        if (archive == null || StringUtils.isBlank(archive.getArchive())) {
            return null;
        }
        BorgFilesystemItem[] items = archiveFilelistCache.load(repoConfig, archive);
        if (items == null) {
            List<BorgFilesystemItem> list = BorgCommands.listArchiveContent(repoConfig, archive);
            List<BorgFilesystemItem> list = BorgCommands.listArchiveContent(repoConfig, archive.getArchive());
            if (CollectionUtils.isNotEmpty(list)) {
                archiveFilelistCache.save(repoConfig, archive, list);
                items = list.toArray(new BorgFilesystemItem[0]);
@@ -156,6 +210,7 @@
        this.jcsCache = JCSCache.getInstance();
        this.repoCacheAccess = jcsCache.getJCSCache("repositories");
        this.repoArchivesCacheAccess = jcsCache.getJCSCache("repositoriesArchives");
        this.archivesCacheAccess = jcsCache.getJCSCache("archives");
        this.archiveFilelistCache = new ArchiveFilelistCache(getCacheDir(), configuration.getCacheArchiveContentMaxDiscSizeMB());
    }
}

 borgbutler-core/src/main/java/de/micromata/borgbutler/data/Archive.java

New file
@@ -0,0 +1,44 @@
package de.micromata.borgbutler.data;

import de.micromata.borgbutler.json.borg.BorgArchiveStats;
import de.micromata.borgbutler.json.borg.BorgCache;
import de.micromata.borgbutler.json.borg.BorgEncryption;
import lombok.Getter;
import lombok.Setter;

import java.io.Serializable;

/**
 *
 */
public class Archive implements Serializable {
    @Getter
    @Setter
    private Repository repository;
    @Getter
    @Setter
    private BorgCache cache;
    @Getter
    @Setter
    private BorgEncryption encryption;

    @Getter
    @Setter
    private int[] chunkerParams;
    /**
     * The command line used for creating this archive: borg create --filter...
     */
    @Getter
    @Setter
    private String[] commandLine;
    @Getter
    @Setter
    private String comment;
    @Getter
    @Setter
    private String start;
    @Getter
    @Setter
    private BorgArchiveStats stats;

}

 borgbutler-core/src/test/java/de/micromata/borgbutler/cache/CacheTest.java

@@ -3,6 +3,7 @@
import de.micromata.borgbutler.config.BorgRepoConfig;
import de.micromata.borgbutler.config.Configuration;
import de.micromata.borgbutler.config.ConfigurationHandler;
import de.micromata.borgbutler.data.Archive;
import de.micromata.borgbutler.data.Repository;
import de.micromata.borgbutler.json.borg.BorgArchive;
import de.micromata.borgbutler.json.borg.BorgFilesystemItem;
@@ -14,6 +15,7 @@
import java.util.List;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;


public class CacheTest {
@@ -41,20 +43,24 @@
            assertEquals(config.getRepoConfigs().size(), ButlerCache.getInstance().getAllRepositories().size());
        }
        List<BorgRepoConfig> repoConfigs = ConfigurationHandler.getConfiguration().getRepoConfigs();
        BorgArchive archive = null;
        BorgArchive borgArchive = null;
        BorgRepoConfig repoConfig = null;
        if (CollectionUtils.isNotEmpty(repoConfigs)) {
            repoConfig = repoConfigs.get(0);
            Repository rerepositoryoList = ButlerCache.getInstance().getRepositoryArchives(repoConfig.getRepo());
            if (rerepositoryoList != null && CollectionUtils.isNotEmpty(rerepositoryoList.getArchives())) {
                archive = rerepositoryoList.getArchives().get(0);
                borgArchive = rerepositoryoList.getArchives().first();
            }
        }
        {
            if (archive != null) {
                BorgFilesystemItem[] content = ButlerCache.getInstance().getArchiveContent(repoConfig, archive);
            if (borgArchive != null) {
                Archive archive = ButlerCache.getInstance().getArchive(repoConfig.getRepo(), borgArchive.getName());
                assertNotNull(archive);
                archive = ButlerCache.getInstance().getArchive(repoConfig.getRepo(), borgArchive.getId());
                assertNotNull(archive);
                BorgFilesystemItem[] content = ButlerCache.getInstance().getArchiveContent(repoConfig, borgArchive);
                log.info("Number of items (content) of archive: " + content.length);
                content = ButlerCache.getInstance().getArchiveContent(repoConfig, archive);
                content = ButlerCache.getInstance().getArchiveContent(repoConfig, borgArchive);
                log.info("Number of items (content) of archive: " + content.length);
            }
        }

 borgbutler-server/src/main/java/de/micromata/borgbutler/server/rest/ReposRest.java

@@ -1,6 +1,7 @@
package de.micromata.borgbutler.server.rest;

import de.micromata.borgbutler.cache.ButlerCache;
import de.micromata.borgbutler.data.Archive;
import de.micromata.borgbutler.data.Repository;
import de.micromata.borgbutler.json.JsonUtils;
import de.micromata.borgbutler.json.borg.BorgRepository;
@@ -55,6 +56,24 @@
    }

    @GET
    @Path("archive")
    @Produces(MediaType.APPLICATION_JSON)
    /**
     *
     * @param repo Name of repository ({@link Repository#getName()}.
     * @param archive Id or name of archive.
     * @param prettyPrinter If true then the json output will be in pretty format.
     * @return Repository (including list of archives) as json string.
     * @see JsonUtils#toJson(Object, boolean)
     */
    public String getArchive(@QueryParam("repo") String repoName,
                             @QueryParam("archive") String archiveIdOrName, @QueryParam("force") boolean force,
                             @QueryParam("prettyPrinter") boolean prettyPrinter) {
        Archive archive = ButlerCache.getInstance().getArchive(repoName, archiveIdOrName, force);
        return JsonUtils.toJson(archive, prettyPrinter);
    }

    @GET
    @Path("list")
    @Produces(MediaType.APPLICATION_JSON)
    /**

 borgbutler-webapp/src/components/views/repos/ArchiveView.jsx

New file
@@ -0,0 +1,101 @@
import React from 'react'
import {Table} from 'reactstrap';
import {PageHeader} from '../../general/BootstrapComponents';
import {getRestServiceUrl} from '../../../utilities/global';
import ErrorAlert from '../../general/ErrorAlert';
import {IconRefresh} from "../../general/IconComponents";

class ArchiveView extends React.Component {

    state = {
        repo: this.props.match.params.repoId,
        archive: this.props.match.params.archiveId,
        isFetching: false,
        activeTab: '1',
    };

    componentDidMount = () => {
        this.fetchArchive();
    };


    fetchArchive = (force) => {
        this.setState({
            isFetching: true,
            failed: false
        });
        fetch(getRestServiceUrl('repos/archive', {
            repo: this.state.repo,
            archive: this.state.archive,
            force: force
        }), {
            method: 'GET',
            headers: {
                'Accept': 'application/json'
            }
        })
            .then(response => response.json())
            .then(json => {
                this.setState({
                    isFetching: false,
                    repo: json
                })
            })
            .catch(() => this.setState({isFetching: false, failed: true}));
    };

    render = () => {
        let content = undefined;
        const repo = this.state.repo;
        let pageHeader = '';

        if (this.state.isFetching) {
            content = <i>Loading...</i>;
        } else if (this.state.failed) {
            content = <ErrorAlert
                title={'Cannot load Repositories'}
                description={'Something went wrong during contacting the rest api.'}
                action={{
                    handleClick: this.fetchArchive,
                    title: 'Try again'
                }}
            />;
        } else if (this.state.repo) {
            pageHeader = <React.Fragment>
                {repo.displayName}
                <div
                    className={'btn btn-outline-primary refresh-button-right'}
                    onClick={this.fetchArchive.bind(this, true)}
                >
                    <IconRefresh/>
                </div>
            </React.Fragment>;
            content = <React.Fragment>
                <Table hover>
                    <tbody>
                    <tr>
                        <th>Archive</th>
                        <th>Time</th>
                        <th>Id</th>
                    </tr>
                    </tbody>
                </Table>
            </React.Fragment>;

        }
        return <React.Fragment>
            <PageHeader>
                {pageHeader}
            </PageHeader>
            {content}
        </React.Fragment>;
    };

    constructor(props) {
        super(props);

        this.fetchArchive = this.fetchArchive.bind(this);
    }
}

export default ArchiveView;

 borgbutler-webapp/src/components/views/repos/RepoArchiveListView.jsx

@@ -49,6 +49,13 @@
        })
    };

    redirectToArchive = (archive) => {
        //console.log("Archive: " + archive.id + ", repo: " + this.state.repo.id);
        console.log("context: " + this.context.history);

        this.context.history.pushState(null, '/archives');
        //this.router.transitionTo('/archives/dfsafds'); // , {repoId: this.state.repo.id}, {archiveId: archive.id}
    }

    render = () => {
        let content = undefined;
@@ -103,7 +110,7 @@
                            {repo.archives.map((archive) => {
                                // Return the element. Also pass key
                                return (
                                    <tr key={archive.id}>
                                    <tr key={archive.id} onClick={() => this.redirectToArchive(archive)}>
                                        <td>{archive.archive}</td>
                                        <td>{archive.time}</td>
                                        <td>{archive.id}</td>
@@ -192,6 +199,7 @@

        this.fetchRepo = this.fetchRepo.bind(this);
        this.toggleTab = this.toggleTab.bind(this);
        this.redirectToArchive = this.redirectToArchive.bind(this);
    }
}


 borgbutler-webapp/src/containers/WebApp.jsx

@@ -7,6 +7,7 @@
import Start from '../components/views/Start';
import RepoListView from '../components/views/repos/RepoListView';
import RepoArchiveListView from '../components/views/repos/RepoArchiveListView';
import ArchiveView from '../components/views/repos/ArchiveView';
import ConfigurationPage from '../components/views/config/ConfigurationPage';
import RestServices from '../components/views/develop/RestServices';
import {isDevelopmentMode} from '../utilities/global';
@@ -52,6 +53,7 @@
                                ))
                            }
                            <Route path={'/repoArchives/:id'} component={RepoArchiveListView}/>
                            <Route path={'/archives/:repoId/:archiveId'} component={ArchiveView}/>
                        </Switch>
                    </div>
                    <Footer versionInfo={this.props.version}/>