List archive content: Creating file list on the fly (for saving memory).

Kai Reinhard

30.47.2018 3a8b25a696e7e7e26ec655275d0af50ace95f2db

List archive content: Creating file list on the fly (for saving memory).

 borgbutler-core/src/main/java/de/micromata/borgbutler/BorgCommands.java

@@ -19,7 +19,6 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Scanner;

/**
 * Creates and executes  borg commands by calling system's borg application (Borg must be installed).
@@ -174,18 +173,19 @@
                .setParams("--json-lines")
                .setDescription("Loading list of files of archive '" + archive.getName() + "' of repo '" + repoConfig.getDisplayName() + "'.");
        List<BorgFilesystemItem> content = new ArrayList<>();
        JobResult<String> jobResult = execute(command).getResult();
        if (jobResult.getStatus() != JobResult.Status.OK) {
            return content;
        }
        String result = jobResult.getResultObject();
        try (Scanner scanner = new Scanner(result)) {
            while (scanner.hasNextLine()) {
                String json = scanner.nextLine();
                BorgFilesystemItem item = JsonUtils.fromJson(BorgFilesystemItem.class, json);
        // The returned job might be an already queued or running one!
        BorgJob job = BorgQueueExecutor.getInstance().execute(new BorgJob(command) {
            @Override
            protected void processStdOutLine(String line, int level) {
                BorgFilesystemItem item = JsonUtils.fromJson(BorgFilesystemItem.class, line);
                item.setMtime(DateUtils.format(item.getMtime()));
                content.add(item);
            }
        });

        JobResult<String> jobResult = job.getResult();
        if (jobResult.getStatus() != JobResult.Status.OK) {
            return content;
        }
        Collections.sort(content); // Sort by path.
        return content;

 borgbutler-core/src/main/java/de/micromata/borgbutler/BorgJob.java

@@ -3,6 +3,7 @@
import de.micromata.borgbutler.config.BorgRepoConfig;
import de.micromata.borgbutler.config.ConfigurationHandler;
import de.micromata.borgbutler.jobs.AbstractCommandLineJob;
import lombok.Getter;
import org.apache.commons.exec.CommandLine;
import org.apache.commons.exec.environment.EnvironmentUtils;
import org.apache.commons.lang3.StringUtils;
@@ -18,6 +19,7 @@
 */
public class BorgJob extends AbstractCommandLineJob<String> {
    private Logger log = LoggerFactory.getLogger(BorgJob.class);
    @Getter
    private BorgCommand command;

    public BorgJob(BorgCommand command) {

 borgbutler-core/src/main/java/de/micromata/borgbutler/BorgQueueExecutor.java

@@ -37,7 +37,11 @@

    public BorgJob execute(BorgCommand command) {
        BorgJob job = new BorgJob(command);
        return (BorgJob)getQueue(command.getRepoConfig()).append(job);
        return execute(job);
    }

    public BorgJob execute(BorgJob job) {
        return (BorgJob)getQueue(job.getCommand().getRepoConfig()).append(job);
    }

    private BorgQueueExecutor() {

 borgbutler-core/src/main/java/de/micromata/borgbutler/jobs/AbstractCommandLineJob.java

@@ -67,24 +67,12 @@
        PumpStreamHandler streamHandler = new PumpStreamHandler(new LogOutputStream() {
            @Override
            protected void processLine(String line, int level) {
                //log.info(line);
                try {
                    outputStream.write(line.getBytes());
                    outputStream.write("\n".getBytes());
                } catch (IOException ex) {
                    log.error(ex.getMessage(), ex);
                }
                processStdOutLine(line, level);
            }
        }, new LogOutputStream() {
            @Override
            protected void processLine(String line, int logLevel) {
                //log.error(line);
                try {
                    errorOutputStream.write(line.getBytes());
                    errorOutputStream.write("\n".getBytes());
                } catch (IOException ex) {
                    log.error(ex.getMessage(), ex);
                }
            protected void processLine(String line, int level) {
                processStdErrLine(line, level);
            }
        });
        executor.setStreamHandler(streamHandler);
@@ -108,6 +96,26 @@
        return result;
    }

    protected void processStdOutLine(String line, int level) {
        //log.info(line);
        try {
            outputStream.write(line.getBytes());
            outputStream.write("\n".getBytes());
        } catch (IOException ex) {
            log.error(ex.getMessage(), ex);
        }
    }

    protected void processStdErrLine(String line, int level) {
        //log.info(line);
        try {
            errorOutputStream.write(line.getBytes());
            errorOutputStream.write("\n".getBytes());
        } catch (IOException ex) {
            log.error(ex.getMessage(), ex);
        }
    }

    @Override
    protected void cancelRunningProcess() {
        if (watchdog != null) {