From bdddb3b6920b573ed2bb069f9eed21b91e9651a3 Mon Sep 17 00:00:00 2001
From: Valery Kharseko <vharseko@3a-systems.ru>
Date: Thu, 09 Jul 2026 09:31:05 +0000
Subject: [PATCH] [#692] Restore partial import semantics for include/exclude branches (#689)
---
opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java | 236 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 files changed, 234 insertions(+), 2 deletions(-)
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java
index 9f0b828..139076a 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java
@@ -13,7 +13,7 @@
*
* Portions Copyright 2014 The Apache Software Foundation
* Copyright 2015-2016 ForgeRock AS.
- * Portions Copyright 2023-2025 3A Systems, LLC
+ * Portions Copyright 2023-2026 3A Systems, LLC
*/
package org.opends.server.backends.pluggable;
@@ -102,6 +102,7 @@
import org.opends.server.backends.pluggable.DN2ID.TreeVisitor;
import org.opends.server.backends.pluggable.ImportLDIFReader.EntryInformation;
import org.opends.server.backends.pluggable.OnDiskMergeImporter.BufferPool.MemoryBuffer;
+import org.opends.server.backends.pluggable.spi.AccessMode;
import org.opends.server.backends.pluggable.spi.Cursor;
import org.opends.server.backends.pluggable.spi.Importer;
import org.opends.server.backends.pluggable.spi.ReadOperation;
@@ -118,6 +119,8 @@
import org.opends.server.types.DirectoryException;
import org.opends.server.types.Entry;
import org.opends.server.types.InitializationException;
+import org.opends.server.types.ExistingFileBehavior;
+import org.opends.server.types.LDIFExportConfig;
import org.opends.server.types.LDIFImportConfig;
import org.opends.server.types.LDIFImportResult;
@@ -156,6 +159,8 @@
private static final String PHASE1_IMPORTER_THREAD_NAME = "PHASE1-IMPORTER-%d";
+ private static final String PHASE1_MIGRATOR_THREAD_NAME = "PHASE1-MIGRATOR-%d";
+
private static final String PHASE2_IMPORTER_THREAD_NAME = "PHASE2-IMPORTER-%d";
private static final String SORTER_THREAD_NAME = "PHASE1-SORTER-%d";
@@ -190,6 +195,23 @@
{
logger.info(NOTE_IMPORT_STARTING, DirectoryServer.getVersionString(), BUILD_ID, REVISION);
+ // A partial import (include and/or exclude branches) only replaces the
+ // imported branches: existing entries outside of the imported scope must
+ // be preserved. Snapshot them into temporary LDIF files before the
+ // import deletes the entry containers, so that they can be fed through
+ // the import pipeline together with the user-provided LDIF file.
+ final List<File> migrationFiles;
+ try
+ {
+ migrationFiles = exportEntriesToPreserve(importConfig);
+ }
+ catch (Exception e)
+ {
+ throw new ExecutionException(e);
+ }
+
+ try
+ {
final long startTime = System.currentTimeMillis();
final int maxThreadCount = importConfig.getThreadCount() == 0
? getDefaultNumberOfThread()
@@ -212,7 +234,34 @@
final AbstractTwoPhaseImportStrategy importStrategy =
new ExternalSortAndImportStrategy(entryContainers, dbStorage, tempDir, bufferPool, sorter);
importer = new OnDiskMergeImporter(PHASE2_IMPORTER_THREAD_NAME, importStrategy);
- importer.doImport(source);
+ if (migrationFiles.isEmpty())
+ {
+ importer.doImport(source);
+ }
+ else
+ {
+ final List<String> migrationPaths = new ArrayList<>(migrationFiles.size());
+ for (File migrationFile : migrationFiles)
+ {
+ migrationPaths.add(migrationFile.getAbsolutePath());
+ }
+ final LDIFImportConfig migrationConfig = new LDIFImportConfig(migrationPaths);
+ try
+ {
+ // Entries come straight from this backend: no filtering and no
+ // schema validation must be applied to them.
+ migrationConfig.setValidateSchema(false);
+ try (final LDIFReaderSource migrationSource =
+ new LDIFReaderSource(rootContainer, migrationConfig, PHASE1_MIGRATOR_THREAD_NAME, threadCount))
+ {
+ importer.doImport(new CompositeSource(Arrays.asList((Source) migrationSource, source)));
+ }
+ }
+ finally
+ {
+ migrationConfig.close();
+ }
+ }
}
finally
{
@@ -246,6 +295,150 @@
{
throw new ExecutionException(e);
}
+ }
+ finally
+ {
+ for (File migrationFile : migrationFiles)
+ {
+ migrationFile.delete();
+ }
+ }
+ }
+
+ /**
+ * Exports the entries which must survive a partial import to temporary
+ * LDIF files, mimicking the migration performed by the previous generation
+ * importer: when include and/or exclude branches are configured, only the
+ * imported branches are replaced, so existing entries outside of the
+ * include branches, as well as existing entries below the exclude branches
+ * of an imported branch, are re-imported from the snapshot files.
+ *
+ * @return the temporary LDIF files containing the entries to preserve
+ */
+ private List<File> exportEntriesToPreserve(LDIFImportConfig importConfig) throws Exception
+ {
+ if (importConfig.clearBackend())
+ {
+ return Collections.emptyList();
+ }
+ final Set<DN> includeBranches = importConfig.getIncludeBranches();
+ final Set<DN> excludeBranches = importConfig.getExcludeBranches();
+ if (includeBranches.isEmpty() && excludeBranches.isEmpty())
+ {
+ return Collections.emptyList();
+ }
+
+ final List<File> migrationFiles = new ArrayList<>();
+ boolean storageOpen = false;
+ try
+ {
+ for (EntryContainer entryContainer : rootContainer.getEntryContainers())
+ {
+ final DN baseDN = entryContainer.getBaseDN();
+ if (excludeBranches.contains(baseDN))
+ {
+ // This entire base DN is excluded: the import does not touch it.
+ continue;
+ }
+
+ final List<DN> includesUnderBase = new ArrayList<>();
+ for (DN includeBranch : includeBranches)
+ {
+ if (includeBranch.isSubordinateOrEqualTo(baseDN))
+ {
+ includesUnderBase.add(includeBranch);
+ }
+ }
+ if (!includeBranches.isEmpty() && includesUnderBase.isEmpty())
+ {
+ // Nothing is imported under this base DN: the import does not touch it.
+ continue;
+ }
+
+ // Everything under this base DN which is not below an include branch
+ // is replaced by the import, unless the base DN itself is included.
+ final boolean wholeBaseReplaced = includesUnderBase.isEmpty() || includesUnderBase.contains(baseDN);
+ if (!wholeBaseReplaced)
+ {
+ if (!storageOpen)
+ {
+ rootContainer.getStorage().open(AccessMode.READ_ONLY);
+ storageOpen = true;
+ }
+ logger.info(NOTE_IMPORT_MIGRATION_START, "existing", baseDN);
+ migrationFiles.add(
+ exportBranches(Collections.singletonList(baseDN), new ArrayList<>(includesUnderBase)));
+ }
+
+ // Existing entries below exclude branches located inside the
+ // replaced scope must be preserved as well.
+ final List<DN> excludedToPreserve = new ArrayList<>();
+ for (DN excludeBranch : excludeBranches)
+ {
+ if (excludeBranch.isSubordinateOrEqualTo(baseDN)
+ && (wholeBaseReplaced || isUnderAnyOf(excludeBranch, includesUnderBase)))
+ {
+ excludedToPreserve.add(excludeBranch);
+ }
+ }
+ if (!excludedToPreserve.isEmpty())
+ {
+ if (!storageOpen)
+ {
+ rootContainer.getStorage().open(AccessMode.READ_ONLY);
+ storageOpen = true;
+ }
+ logger.info(NOTE_IMPORT_MIGRATION_START, "excluded", baseDN);
+ migrationFiles.add(exportBranches(excludedToPreserve, Collections.<DN> emptyList()));
+ }
+ }
+ return migrationFiles;
+ }
+ catch (Exception e)
+ {
+ for (File migrationFile : migrationFiles)
+ {
+ migrationFile.delete();
+ }
+ throw e;
+ }
+ finally
+ {
+ if (storageOpen)
+ {
+ rootContainer.getStorage().close();
+ }
+ }
+ }
+
+ private static boolean isUnderAnyOf(DN dn, List<DN> branches)
+ {
+ for (DN branch : branches)
+ {
+ if (dn.isSubordinateOrEqualTo(branch))
+ {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private File exportBranches(List<DN> includeBranches, List<DN> excludeBranches) throws Exception
+ {
+ final File migrationFile = File.createTempFile("import-migration-", ".ldif");
+ final LDIFExportConfig exportConfig =
+ new LDIFExportConfig(migrationFile.getAbsolutePath(), ExistingFileBehavior.OVERWRITE);
+ exportConfig.setIncludeBranches(includeBranches);
+ exportConfig.setExcludeBranches(excludeBranches);
+ try
+ {
+ new ExportJob(exportConfig).exportLDIF(rootContainer);
+ }
+ finally
+ {
+ exportConfig.close();
+ }
+ return migrationFile;
}
private static int getDefaultNumberOfThread()
@@ -644,6 +837,45 @@
boolean isCancelled();
}
+ /** Chains {@link Source}s, processing them sequentially. */
+ private static final class CompositeSource implements Source
+ {
+ private final List<Source> sources;
+
+ CompositeSource(final List<Source> sources)
+ {
+ this.sources = sources;
+ }
+
+ @Override
+ public void processAllEntries(EntryProcessor processor) throws InterruptedException, ExecutionException
+ {
+ for (Source source : sources)
+ {
+ source.processAllEntries(processor);
+ }
+ }
+
+ @Override
+ public boolean isCancelled()
+ {
+ for (Source source : sources)
+ {
+ if (source.isCancelled())
+ {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public void close()
+ {
+ // The chained sources are closed by their respective owners.
+ }
+ }
+
/** Extract LDAP {@link Entry}s from an LDIF file. */
private static final class LDIFReaderSource implements Source
{
--
Gitblit v1.10.0