Serialization via FST lib.

Kai Reinhard

07.26.2019 da9780f28f20d3a0cb2ee326b562b9e4f6cdbafe

Serialization via FST lib.

 borgbutler-core/build.gradle

@@ -12,7 +12,8 @@
    compile group: 'org.apache.commons', name: 'commons-collections4', version: '4.2'
    compile group: 'org.apache.commons', name: 'commons-compress', version: '1.18'
    compile group: 'org.apache.commons', name: 'commons-jcs-core', version: '2.2.1'

    // https://mvnrepository.com/artifact/de.ruedigermoeller/fst
    compile group: 'de.ruedigermoeller', name: 'fst', version: '2.57' // Serialization (faster than Java built-in)

    compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.9.6'
    compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: '2.9.6'

 borgbutler-core/src/main/java/de/micromata/borgbutler/cache/ArchiveFilelistCache.java

@@ -9,9 +9,10 @@
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
import org.nustaq.serialization.FSTConfiguration;
import org.nustaq.serialization.FSTObjectInput;
import org.nustaq.serialization.FSTObjectOutput;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

@@ -39,6 +40,14 @@
    // For avoiding concurrent writing of same files (e. g. after the user has pressed a button twice).
    private Set<File> savingFiles = new HashSet<>();
    private Recents recents = new Recents(MAX_NUMBER_OF_RECENT_ENTRIES);
    final FSTConfiguration conf = FSTConfiguration.createDefaultConfiguration();

    ArchiveFilelistCache(File cacheDir, int cacheArchiveContentMaxDiscSizeMB) {
        this.cacheDir = cacheDir;
        this.cacheArchiveContentMaxDiscSizeMB = cacheArchiveContentMaxDiscSizeMB;
        conf.registerClass(BorgFilesystemItem.class);
        conf.setShareReferences(false);
    }

    public void save(BorgRepoConfig repoConfig, Archive archive, List<BorgFilesystemItem> filesystemItems) {
        if (CollectionUtils.isEmpty(filesystemItems)) {
@@ -56,12 +65,13 @@
                Collections.sort(filesystemItems); // Sort by path.
            }
            log.info("Saving archive content as file list: " + file.getAbsolutePath());
            try (ObjectOutputStream outputStream = new ObjectOutputStream(new BufferedOutputStream(new GzipCompressorOutputStream(new FileOutputStream(file))))) {
                outputStream.writeObject(filesystemItems.size());
            try (FSTObjectOutput outputStream
                         = new FSTObjectOutput(new BufferedOutputStream(new GzipCompressorOutputStream(new FileOutputStream(file))), conf)) {
                outputStream.writeObject(filesystemItems.size(), Integer.class);
                Iterator<BorgFilesystemItem> it = filesystemItems.iterator();
                while (it.hasNext()) {
                    BorgFilesystemItem item = it.next();
                    outputStream.writeObject(item);
                    outputStream.writeObject(item, BorgFilesystemItem.class);
                }
                outputStream.writeObject("EOF");
            } catch (IOException ex) {
@@ -72,8 +82,6 @@
                savingFiles.remove(file);
            }
        }
        compactPathes(filesystemItems);
        recents.add(new RecentEntry(archive, filesystemItems));
        log.info("Saving done.");
    }

@@ -109,10 +117,6 @@
     * @return
     */
    public List<BorgFilesystemItem> load(BorgRepoConfig repoConfig, Archive archive, FileSystemFilter filter) {
        RecentEntry recent = recents.getRecent(archive);
        if (recent != null) {
            return filter(recent.filesystemItems, filter);
        }
        File file = getFile(repoConfig, archive);
        if (!file.exists()) {
            return null;
@@ -140,9 +144,6 @@
            log.error("File '" + file.getAbsolutePath() + "' doesn't exist. Can't get archive content files.");
            return null;
        }
        if (archive != null) {
            recents.removeOldestEntry();
        }
        log.info("Loading archive content as file list from: " + file.getAbsolutePath());
        try {
            // Set last modified time of file:
@@ -151,38 +152,30 @@
            log.error("Can't set lastModifiedTime on file '" + file.getAbsolutePath() + "'. Pruning old cache files may not work.");
        }
        List<BorgFilesystemItem> list = new ArrayList<>();
        try (ObjectInputStream inputStream = new ObjectInputStream(new BufferedInputStream(new GzipCompressorInputStream(new FileInputStream(file))))) {
            Object obj = inputStream.readObject();
            if (!(obj instanceof Integer)) {
                log.error("Can't load archive content. Integer expected, but received: " + obj.getClass());
                return null;
            }
            int size = (Integer) obj;
        ByteArrayOutputStream out = new ByteArrayOutputStream();
/*        try {
            IOUtils.copy(new BufferedInputStream(new GzipCompressorInputStream(new FileInputStream(file))), out);
        } catch (IOException ex) {

        }*/
        byte[] bytes = out.toByteArray();
        out = null;
        try (FSTObjectInput inputStream = new FSTObjectInput(new BufferedInputStream(new GzipCompressorInputStream(new FileInputStream(file))), conf)) {
            int size = (Integer) inputStream.readObject(Integer.class);
            int fileNumber = -1;
            for (int i = 0; i < size; i++) {
                ++fileNumber;
                obj = inputStream.readObject();
                if (obj instanceof BorgFilesystemItem) {
                    BorgFilesystemItem item = (BorgFilesystemItem) obj;
                    item.setFileNumber(fileNumber);
                    if (filter == null || filter.matches(item)) {
                        list.add(item);
                        if (filter != null && filter.isFinished()) break;
                    }
                } else {
                    log.error("Can't load archive content. FilesystemItem expected, but received: "
                            + (obj != null ? obj.getClass() : "null")
                            + " at position " + i + ".");
                    return null;
                BorgFilesystemItem item = (BorgFilesystemItem) inputStream.readObject(BorgFilesystemItem.class);
                item.setFileNumber(fileNumber);
                if (filter == null || filter.matches(item)) {
                    list.add(item);
                    if (filter != null && filter.isFinished()) break;
                }
            }
        } catch (IOException | ClassNotFoundException ex) {
        } catch (Exception ex) {
            log.error("Error while reading file list '" + file.getAbsolutePath() + "': " + ex.getMessage(), ex);
        }
        Collections.sort(list); // Sort by path (if archive list order wasn't correct).
        if (archive != null) {
            recents.add(new RecentEntry(archive, list));
        }
        log.info("Loading done.");
        return filter(list, filter);
    }
@@ -286,11 +279,6 @@
                true));
    }

    ArchiveFilelistCache(File cacheDir, int cacheArchiveContentMaxDiscSizeMB) {
        this.cacheDir = cacheDir;
        this.cacheArchiveContentMaxDiscSizeMB = cacheArchiveContentMaxDiscSizeMB;
    }

    private boolean isCacheFile(File file) {
        return file.getName().startsWith(CACHE_ARCHIVE_LISTS_BASENAME);
    }
@@ -348,43 +336,5 @@
            this.filesystemItems = filesystemItems;
        }
    }

    static void compactPathes(List<BorgFilesystemItem> items) {
        long origSize = 0;
        long compactSize = 0;
        String currentDir = null;
        for (BorgFilesystemItem item : items) {
            String path = item.getPath();
            origSize += path.length();
            if (currentDir != null && path.startsWith(currentDir)) {
                String compactPath = "#" + path.substring(currentDir.length());
                item.setPath(compactPath);
            }
            if ("d".equals(item.getType())) {
                currentDir = path;
            } else {
                currentDir = FilenameUtils.getPath(path);
            }
            compactSize += item.getPath().length();
            //log.info(StringUtils.rightPad(path, 40) + " -> " + item.getPath());
        }
        log.info("Compact pathes: " + FileUtils.byteCountToDisplaySize(origSize) + " -> " + FileUtils.byteCountToDisplaySize(compactSize));
    }

    static void expandPathes(List<BorgFilesystemItem> items) {
        String currentDir = null;
        for (BorgFilesystemItem item : items) {
            String path = item.getPath();
            if (path.startsWith("#")) {
                item.setPath(currentDir + path.substring(1));
            }
            if ("d".equals(item.getType())) {
                currentDir = item.getPath();
            } else {
                currentDir = FilenameUtils.getPath(item.getPath());
            }
            log.info(StringUtils.rightPad(path, 40) + " -> " + item.getPath());
        }
    }
}


 borgbutler-core/src/test/java/de/micromata/borgbutler/cache/ArchiveFilelistCacheTest.java

@@ -125,45 +125,6 @@
        cache.removeAllCacheFiles();
    }

    @Test
    void reducePath() {
        List<BorgFilesystemItem> list = new ArrayList<>();
        String[] items = {
                "d home",
                "d home/kai",
                "- home/kai/abc.txt",
                "d home/kai/Documents",
                "- home/kai/Documents/test1.doc",
                "- home/kai/Documents/test2.doc",
                "- home/kai/image.doc",
                "- home/kai/Files/tmp/a.txt",
                "- home/kai/Files/tmp/b.txt",
                "- home/kai/Java.pdf",
                "- home/kai/Movies/a.mov",
                "- home/pete/txt.mov",
                "- home/steve/1/2/3/4/5/6/7/test.txt",
                "- home/test.txt",
                "- home/xaver/1/2/3/4/5/6/7/test.txt",
                "- opt/local/test.txt",
                "- opt/local/test2.txt",
        };
        for (String path : items) {
            BorgFilesystemItem item = new BorgFilesystemItem().setPath(path.substring(2));
            if (path.startsWith("d")) {
                item.setType("d");
            } else {
                item.setType("-");
            }
            list.add(item);
        }
        ArchiveFilelistCache.compactPathes(list);
        ArchiveFilelistCache.expandPathes(list);
        for (int i = 0; i < list.size(); i++) {
            assertEquals(items[i].substring(2), list.get(i).getPath());
        }

    }

    private List<BorgFilesystemItem> createList(int number) throws Exception {
        List<BorgFilesystemItem> list = new ArrayList<>();
        for (int i = 0; i < 1000000; i++) {