Job monitor...

Kai Reinhard

05.03.2019 add986cff9c8e1a00428122a0d5515e6bd00120f

Job monitor...

 borgbutler-core/src/main/java/de/micromata/borgbutler/BorgCommands.java

@@ -172,7 +172,7 @@
                .setParams("--json-lines")
                .setDescription("Loading list of files of archive '" + archive.getName() + "' of repo '" + repoConfig.getDisplayName() + "'.");
        // The returned job might be an already queued or running one!
        final ProgressMessage progressMessage = new ProgressMessage()
        final ProgressInfo progressInfo = new ProgressInfo()
                .setMessage("Getting file list...")
                .setCurrent(0);
        BorgJob<List<BorgFilesystemItem>> job = BorgQueueExecutor.getInstance().execute(new BorgJob<List<BorgFilesystemItem>>(command) {
@@ -181,7 +181,7 @@
                BorgFilesystemItem item = JsonUtils.fromJson(BorgFilesystemItem.class, line);
                item.setMtime(DateUtils.format(item.getMtime()));
                payload.add(item);
                setProgressMessage(progressMessage.incrementCurrent());
                setProgressInfo(progressInfo.incrementCurrent());
            }
        });
        job.payload = new ArrayList<>();

 borgbutler-core/src/main/java/de/micromata/borgbutler/BorgJob.java

@@ -5,7 +5,7 @@
import de.micromata.borgbutler.data.Archive;
import de.micromata.borgbutler.jobs.AbstractCommandLineJob;
import de.micromata.borgbutler.json.JsonUtils;
import de.micromata.borgbutler.json.borg.ProgressMessage;
import de.micromata.borgbutler.json.borg.ProgressInfo;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.Setter;
@@ -34,7 +34,7 @@

    @Getter
    @Setter(AccessLevel.PROTECTED)
    private ProgressMessage progressMessage;
    private ProgressInfo progressInfo;

    public BorgJob(BorgCommand command) {
        this.command = command;
@@ -73,9 +73,9 @@
    protected void processStdErrLine(String line, int level) {
        try {
            if (StringUtils.startsWith(line, "{\"message")) {
                ProgressMessage message = JsonUtils.fromJson(ProgressMessage.class, line);
                ProgressInfo message = JsonUtils.fromJson(ProgressInfo.class, line);
                if (message != null) {
                    progressMessage = message;
                    progressInfo = message;
                    return;
                }
            }
@@ -115,8 +115,8 @@
        clone.setStatus(getStatus());
        clone.setWorkingDirectory(getWorkingDirectory());
        clone.setDescription(getDescription());
        if (progressMessage != null) {
            clone.setProgressMessage(progressMessage.clone());
        if (progressInfo != null) {
            clone.setProgressInfo(progressInfo.clone());
        }
        return clone;
    }

 borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/ProgressInfo.java

File was renamed from borgbutler-core/src/main/java/de/micromata/borgbutler/json/borg/ProgressMessage.java
@@ -7,7 +7,7 @@
 * Output of borg option <tt>--progress</tt>.
 * See https://borgbackup.readthedocs.io/en/stable/internals/frontends.html,
 */
public class ProgressMessage implements Cloneable {
public class ProgressInfo implements Cloneable {
    // {"message": "Calculating statistics...   0%", "current": 1, "total": 2497, "info": null, "operation": 1, "msgid": null, "type": "progress_percent", "finished": false, "time": 1546640510.116256}
    /**
     * e. g. Calculating statistics...   5%
@@ -22,6 +22,7 @@
    @Setter
    private long current;
    @Getter
    @Setter
    private long total;
    /**
     * Array that describes the current item, may be null, contents depend on msgid.
@@ -48,16 +49,16 @@
    @Getter
    private double time;

    public ProgressMessage incrementCurrent() {
    public ProgressInfo incrementCurrent() {
        ++current;
        return this;
    }

    @Override
    public ProgressMessage clone() {
        ProgressMessage clone = null;
    public ProgressInfo clone() {
        ProgressInfo clone = null;
        try {
            clone = (ProgressMessage) super.clone();
            clone = (ProgressInfo) super.clone();
        } catch (CloneNotSupportedException ex) {
            throw new UnsupportedOperationException(this.getClass().getCanonicalName() + " isn't cloneable: " + ex.getMessage(), ex);
        }

 borgbutler-server/src/main/java/de/micromata/borgbutler/server/rest/JobsRest.java

@@ -4,7 +4,9 @@
import de.micromata.borgbutler.BorgQueueExecutor;
import de.micromata.borgbutler.cache.ButlerCache;
import de.micromata.borgbutler.data.Repository;
import de.micromata.borgbutler.jobs.AbstractJob;
import de.micromata.borgbutler.json.JsonUtils;
import de.micromata.borgbutler.json.borg.ProgressInfo;
import de.micromata.borgbutler.server.rest.queue.JsonJob;
import de.micromata.borgbutler.server.rest.queue.JsonJobQueue;
import org.apache.commons.collections4.CollectionUtils;
@@ -23,15 +25,20 @@
public class JobsRest {
    private static Logger log = LoggerFactory.getLogger(JobsRest.class);

    private static List<JsonJobQueue> testList;

    @GET
    @Produces(MediaType.APPLICATION_JSON)
    /**
     *
     * @param testMode If true, then a test job list is created.
     * @param prettyPrinter If true then the json output will be in pretty format.
     * @return Job queues as json string.
     * @see JsonUtils#toJson(Object, boolean)
     */
    public String getJobs(@QueryParam("prettyPrinter") boolean prettyPrinter) {
    public String getJobs(@QueryParam("testMode") boolean testMode, @QueryParam("prettyPrinter") boolean prettyPrinter) {
        if (testMode) {
            return returnTestList(prettyPrinter);
        }
        BorgQueueExecutor borgQueueExecutor = BorgQueueExecutor.getInstance();
        List<JsonJobQueue> queueList = new ArrayList<>();
        for (String repo : borgQueueExecutor.getRepos()) {
@@ -51,4 +58,69 @@
        }
        return JsonUtils.toJson(queueList, prettyPrinter);
    }

    private String returnTestList(boolean prettyPrinter) {
        if (testList == null) {
            testList = new ArrayList<>();
            JsonJobQueue queue = new JsonJobQueue().setRepo("My Computer");
            addTestJob(queue, "Calculating statistics... ",
                    "Loading info of archive 'my-computer-2018-12-05T23:10:33' of repo 'My-Computer-Cloud'.", 0, 1000);
            addTestJob(queue, null,
                    "Loading list of files of archive 'my-computer-2018-12-05T23:10:33' of repo 'My-Computer-Cloud'.", 0, 0);
            testList.add(queue);

            queue = new JsonJobQueue().setRepo("My Server");
            addTestJob(queue, "Getting file list...",
                    "Loading list of files of archive 'my-server-2018-12-05T23:10:33' of repo 'My-Server-Cloud'.", 0, 0);
            addTestJob(queue, null,
                    "Loading info of archive 'my-server-2018-12-05T23:10:33' of repo 'My-Server-Cloud'.", 0, 1000);
            testList.add(queue);
        } else {
            for (JsonJobQueue jobQueue : testList) {
                for (JsonJob job : jobQueue.getJobs()) {
                    if (job.getStatus() != AbstractJob.Status.RUNNING) continue;
                    if (job.getProgressText().startsWith("Calculating")) {
                        long current = job.getProgressInfo().getCurrent();
                        current += Math.random() * 100;
                        if (current > 1000) {
                            current = 0; // Reset to beginning.
                        }
                        job.getProgressInfo().setCurrent(current);
                        job.getProgressInfo().setMessage("Calculating statistics...  " + Math.round(current / 10) + "%");
                    } else {
                        long current = job.getProgressInfo().getCurrent();
                        current += Math.random() * 10000;
                        job.getProgressInfo().setCurrent(current);
                    }
                    job.buildProgressText();
                }
            }
        }
        return JsonUtils.toJson(testList, prettyPrinter);
    }


    private JsonJob addTestJob(JsonJobQueue queue, String message, String description, long current, long total) {
        ProgressInfo msg = new ProgressInfo()
                .setMessage(message)
                .setCurrent(current)
                .setTotal(total);
        JsonJob job = new JsonJob()
                .setProgressInfo(msg)
                .setDescription(description)
                .setStatus(AbstractJob.Status.QUEUED)
                .setCommandLineAsString(description);
        job.buildProgressText();
        if (message != null) {
            job.setStatus(AbstractJob.Status.RUNNING);

        } else {
            job.setStatus(AbstractJob.Status.QUEUED);
        }
        if (queue.getJobs() == null) {
            queue.setJobs(new ArrayList<>());
        }
        queue.getJobs().add(job);
        return job;
    }
}

 borgbutler-server/src/main/java/de/micromata/borgbutler/server/rest/queue/JsonJob.java

@@ -2,7 +2,7 @@

import de.micromata.borgbutler.BorgJob;
import de.micromata.borgbutler.jobs.AbstractJob;
import de.micromata.borgbutler.json.borg.ProgressMessage;
import de.micromata.borgbutler.json.borg.ProgressInfo;
import de.micromata.borgbutler.server.user.UserUtils;
import lombok.Getter;
import lombok.Setter;
@@ -18,14 +18,16 @@
    @Setter
    private String title;
    @Getter
    @Setter
    private String description;
    @Getter
    @Setter
    private String progressText;
    @Getter
    @Setter
    private ProgressInfo progressInfo;
    @Getter
    private ProgressMessage progressMessage;
    @Getter
    @Setter
    private String commandLineAsString;

    public JsonJob() {
@@ -35,35 +37,40 @@
        this.cancellationRequested = borgJob.isCancellationRequested();
        this.status = borgJob.getStatus();
        this.title = borgJob.getTitle();
        ProgressMessage progressMessage = borgJob.getProgressMessage();
        if (progressMessage != null) {
            this.progressMessage = progressMessage;
            this.progressText = progressMessageToString();
        ProgressInfo progressInfo = borgJob.getProgressInfo();
        if (progressInfo != null) {
            this.progressInfo = progressInfo;
            buildProgressText();
        }
        this.commandLineAsString = borgJob.getCommandLineAsString();
        this.description = borgJob.getDescription();
    }

    public String progressMessageToString() {
        if (progressMessage == null) {
    /**
     * Builds and sets progressText from the progressInfo object if given.
     * @return progressText
     */
    public String buildProgressText() {
        if (progressInfo == null) {
            return "";
        }
        StringBuilder sb = new StringBuilder();
        if (progressMessage.getMessage()!= null) {
            sb.append(progressMessage.getMessage());
        if (progressInfo.getMessage() != null) {
            sb.append(progressInfo.getMessage());
        }
        if (progressMessage.getCurrent() > 0) {
            sb.append(" (").append(UserUtils.formatNumber(progressMessage.getCurrent()));
            if (progressMessage.getTotal() > 0) {
                sb.append("/").append(UserUtils.formatNumber(progressMessage.getTotal()));
        if (progressInfo.getCurrent() > 0) {
            sb.append(" (").append(UserUtils.formatNumber(progressInfo.getCurrent()));
            if (progressInfo.getTotal() > 0) {
                sb.append("/").append(UserUtils.formatNumber(progressInfo.getTotal()));
            }
            sb.append(")");
        }
        if (progressMessage.isFinished()) {
        if (progressInfo.isFinished()) {
            sb.append(" (finished)");
        }
        sb.append(".");
        return sb.toString();
        progressText = sb.toString();
        return progressText;
    }

}

 borgbutler-webapp/src/components/views/jobs/JobMonitorPanel.jsx

@@ -1,24 +1,38 @@
import React from 'react';
import {getRestServiceUrl} from "../../../utilities/global";
import {Button} from 'reactstrap';
import {getRestServiceUrl, isDevelopmentMode} from "../../../utilities/global";
import JobQueue from "./JobQueue";
import ErrorAlert from "../archives/ArchiveView";

class JobMonitorPanel extends React.Component {
    state = {
        isFetching: false
        isFetching: false,
        testMode: false
    };

    componentDidMount = () => {
        this.fetchArchive();
        this.interval = setInterval(() => this.fetchArchive(), 2000);
    };

    componentWillUnmount() {
        clearInterval(this.interval);
    }

    fetchArchive = (force) => {
    toggleTestMode() {
        this.setState({
            testMode: !this.state.testMode
        });
    }

    fetchArchive = () => {
        this.setState({
            isFetching: true,
            failed: false
        });
        fetch(getRestServiceUrl('jobs'), {
        fetch(getRestServiceUrl('jobs', {
            testMode: this.state.testMode
        }), {
            method: 'GET',
            headers: {
                'Accept': 'application/json'
@@ -32,7 +46,7 @@
                this.setState({
                    isFetching: false,
                    queues
                })
                });
            })
            .catch(() => this.setState({isFetching: false, failed: true}));
    };
@@ -52,13 +66,21 @@
                }}
            />;
        } else if (this.state.queues) {
            content = <React.Fragment>
                {this.state.queues
                    .map((queue) => <JobQueue
                        queue={queue}
                        key={queue.repo}
                    />)}
            </React.Fragment>;
            if (this.state.queues.length > 0) {
                content = <React.Fragment>
                    {this.state.queues
                        .map((queue) => <JobQueue
                            queue={queue}
                            key={queue.repo}
                        />)}
                </React.Fragment>;
            } else if (isDevelopmentMode()) {
                content = <React.Fragment>No jobs are running or queued.<br/><br/>
                    <Button color="primary" onClick={this.toggleTestMode}>Test mode</Button>
                </React.Fragment>
            } else {
                content = <React.Fragment>No jobs are running or queued.</React.Fragment>
            }

        }
        return <React.Fragment>
@@ -70,6 +92,7 @@
        super(props);

        this.fetchArchive = this.fetchArchive.bind(this);
        this.toggleTestMode = this.toggleTestMode.bind(this);
    }
}


 borgbutler-webapp/src/components/views/jobs/JobQueue.jsx

@@ -1,16 +1,36 @@
import React from 'react';
import { Collapse, Button, CardBody, Card } from 'reactstrap';
import Job from "./Job";

function JobQueue({queue}) {
    return (
        <div>
            <h2>{queue.repo}</h2>
            {queue.jobs
                .map((job, index) => <Job
                    job={job}
                    key={job.commandLineAsString}
                />)}
        </div>
    )
class JobQueue extends React.Component {
    constructor(props) {
        super(props);
        this.toggle = this.toggle.bind(this);
        this.state = {collapse: true};
    }

    toggle() {
        this.setState({collapse: !this.state.collapse});
    }

    render() {
        return (
            <div>
                <Button color="primary" onClick={this.toggle} style={{ marginBottom: '1rem' }}>{this.props.queue.repo}</Button>
                <Collapse isOpen={this.state.collapse}>
                    <Card>
                        <CardBody>
                            {this.props.queue.jobs
                                .map((job, index) => <Job
                                    job={job}
                                    key={job.commandLineAsString}
                                />)}
                        </CardBody>
                    </Card>
                </Collapse>
            </div>
        );
    }
}

export default JobQueue;