Queuing of borg commands started.

Kai Reinhard

19.41.2018 414fa1102f661ff23913c1b51207ad0479f99bfb

 borgbutler-core/src/main/java/de/micromata/borgbutler/BorgCommands.java

@@ -1,33 +1,26 @@
package de.micromata.borgbutler;

import de.micromata.borgbutler.config.BorgRepoConfig;
import de.micromata.borgbutler.config.Configuration;
import de.micromata.borgbutler.config.ConfigurationHandler;
import de.micromata.borgbutler.config.Definitions;
import de.micromata.borgbutler.data.Archive;
import de.micromata.borgbutler.data.ArchiveShortInfo;
import de.micromata.borgbutler.data.Repository;
import de.micromata.borgbutler.json.JsonUtils;
import de.micromata.borgbutler.json.borg.*;
import de.micromata.borgbutler.utils.DateUtils;
import de.micromata.borgbutler.utils.ReplaceUtils;
import lombok.Setter;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.exec.CommandLine;
import org.apache.commons.exec.DefaultExecutor;
import org.apache.commons.exec.PumpStreamHandler;
import org.apache.commons.exec.environment.EnvironmentUtils;
import org.apache.commons.io.output.ByteArrayOutputStream;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.naming.Context;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.time.LocalDateTime;
import java.util.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Scanner;

public class BorgCommands {
    private static Logger log = LoggerFactory.getLogger(BorgCommands.class);
@@ -39,7 +32,7 @@
     * @return Parsed repo config returned by Borg command (without archives).
     */
    public static Repository info(BorgRepoConfig repoConfig) {
        Context context = new Context().setRepoConfig(repoConfig).setCommand("info").setParams("--json");
        BorgCommand context = new BorgCommand().setRepoConfig(repoConfig).setCommand("info").setParams("--json");
        String json = execute(context);
        if (json == null) {
            return null;
@@ -67,8 +60,8 @@
     * @param repository Repository without archives, archives will be loaded.
     */
    public static void list(BorgRepoConfig repoConfig, Repository repository) {
        Context context = new Context().setRepoConfig(repoConfig).setCommand("list").setParams("--json");
        String json = execute(context);
        BorgCommand command = new BorgCommand().setRepoConfig(repoConfig).setCommand("list").setParams("--json");
        String json = execute(command);
        if (json == null) {
            log.error("Can't load archives from repo '" + repository.getName() + "'.");
            return;
@@ -103,15 +96,15 @@
     * @param repository Repository without archives.
     */
    public static void info(BorgRepoConfig repoConfig, Archive archive, Repository repository) {
        Context context = new Context().setRepoConfig(repoConfig).setCommand("info").setArchive(archive.getName())
        BorgCommand command = new BorgCommand().setRepoConfig(repoConfig).setCommand("info").setArchive(archive.getName())
                .setParams("--json");
        String json = execute(context);
        String json = execute(command);
        if (json == null) {
            return;
        }
        BorgArchiveInfo archiveInfo = JsonUtils.fromJson(BorgArchiveInfo.class, json);
        if (archiveInfo == null) {
            log.error("Archive '" + context.getRepoArchive() + "' not found.");
            log.error("Archive '" + command.getRepoArchive() + "' not found.");
            return;
        }
        repository.setLastModified(DateUtils.format(archiveInfo.getRepository().getLastModified()))
@@ -123,7 +116,7 @@
            return;
        }
        if (archiveInfo.getArchives().size() > 1) {
            log.warn("Archive '" + context.getRepoArchive() + "' contains more than one archives!? (Using only first.)");
            log.warn("Archive '" + command.getRepoArchive() + "' contains more than one archives!? (Using only first.)");
        }
        BorgArchive2 borgArchive = archiveInfo.getArchives().get(0);
        archive.setStart(DateUtils.format(borgArchive.getStart()))
@@ -140,10 +133,10 @@
    }

    public static List<BorgFilesystemItem> listArchiveContent(BorgRepoConfig repoConfig, String archive) {
        Context context = new Context().setRepoConfig(repoConfig).setCommand("list").setArchive(archive)
        BorgCommand command = new BorgCommand().setRepoConfig(repoConfig).setCommand("list").setArchive(archive)
                .setParams("--json-lines");
        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
        execute(outputStream, context);
        execute(outputStream, command);
        String response = outputStream.toString(Definitions.STD_CHARSET);
        List<BorgFilesystemItem> content = new ArrayList<>();
        try (Scanner scanner = new Scanner(response)) {
@@ -173,105 +166,17 @@
        if (!restoreDir.exists()) {
            restoreDir.mkdirs();
        }
        Context context = new Context().setWorkingDir(restoreDir).setRepoConfig(repoConfig)
        BorgCommand command = new BorgCommand().setWorkingDir(restoreDir).setRepoConfig(repoConfig)
                .setCommand("extract").setArchive(archive).setArgs(path);
        execute(context);
        execute(command);
        return restoreDir;
    }

    private static String execute(Context context) {
        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
        execute(outputStream, context);
        String json = outputStream.toString(Definitions.STD_CHARSET);
        return json;
    private static String execute(BorgCommand command) {
        return BorgExecutorQueue.getInstance().execute(command);
    }

    private static void execute(OutputStream outputStream, Context context) {
        CommandLine cmdLine = new CommandLine(ConfigurationHandler.getConfiguration().getBorgCommand());
        cmdLine.addArgument(context.command);
        if (context.params != null) {
            for (String param : context.params) {
                if (param != null)
                    cmdLine.addArgument(param);
            }
        }
        cmdLine.addArgument(context.getRepoArchive());
        if (context.args != null) {
            for (String arg : context.args) {
                if (arg != null)
                    cmdLine.addArgument(arg);
            }
        }
        DefaultExecutor executor = new DefaultExecutor();
        if (context.workingDir != null) {
            executor.setWorkingDirectory(context.workingDir);
        }
        //executor.setExitValue(2);
        //ExecuteWatchdog watchdog = new ExecuteWatchdog(60000);
        //executor.setWatchdog(watchdog);
        //  ExecuteResultHandler handler = new DefaultExecuteResultHandler();
        PumpStreamHandler streamHandler = new PumpStreamHandler(outputStream);
        executor.setStreamHandler(streamHandler);
        String borgCall = cmdLine.getExecutable() + " " + StringUtils.join(cmdLine.getArguments(), " ");
        log.info("Executing '" + borgCall + "'...");
        try {
            executor.execute(cmdLine, getEnvironment(context.repoConfig));
        } catch (Exception ex) {
            log.error("Error while creating environment for borg call '" + borgCall + "': " + ex.getMessage(), ex);
            log.error("Response: " + StringUtils.abbreviate(outputStream.toString(), 10000));
            return;
        }
    }


    private static Map<String, String> getEnvironment(BorgRepoConfig repoConfig) throws IOException {
        Configuration config = ConfigurationHandler.getConfiguration();
        Map<String, String> env = EnvironmentUtils.getProcEnvironment();
        addEnvironmentVariable(env, "BORG_REPO", repoConfig.getRepo());
        addEnvironmentVariable(env, "BORG_RSH", repoConfig.getRsh());
        addEnvironmentVariable(env, "BORG_PASSPHRASE", repoConfig.getPassphrase());
        String passcommand = repoConfig.getPasswordCommand();
        if (StringUtils.isNotBlank(passcommand)) {
            // For MacOS BORG_PASSCOMMAND="security find-generic-password -a $USER -s borg-passphrase -w"
            passcommand = passcommand.replace("$USER", System.getProperty("user.name"));
            addEnvironmentVariable(env, "BORG_PASSCOMMAND", passcommand);
        }
        return env;
    }

    private static void addEnvironmentVariable(Map<String, String> env, String name, String value) {
        if (StringUtils.isNotBlank(value)) {
            EnvironmentUtils.addVariableToEnvironment(env, name + "=" + value);
        }
    }

    private static class Context {
        @Setter
        File workingDir;
        String[] args;
        String[] params;
        @Setter
        BorgRepoConfig repoConfig;
        @Setter
        String command;
        @Setter
        String archive;

        Context setArgs(String... args) {
            this.args = args;
            return this;
        }

        Context setParams(String... params) {
            this.params = params;
            return this;
        }

        String getRepoArchive() {
            if (archive == null) {
                return repoConfig.getRepo();
            }
            return repoConfig.getRepo() + "::" + archive;
        }
    private static void execute(OutputStream outputStream, BorgCommand command) {
        BorgExecutorQueue.getInstance().execute(outputStream, command);
    }
}