From 329d0afabe7be59137b61f6fa95ca052a4f98a67 Mon Sep 17 00:00:00 2001
From: Valery Kharseko <vharseko@3a-systems.ru>
Date: Wed, 23 Sep 2026 12:37:49 +0000
Subject: [PATCH] [#1059] Give no key up on import or rebuild under an index-entry-limit of 0 (#1060)
---
opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java | 15 ++
opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/OnDiskMergeImporterTest.java | 112 ++++++++++++++++++
opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/BackendStatTest.java | 43 +++++++
opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/PluggableBackendImplTestCase.java | 120 +++++++++++++++++++
opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/BackendStat.java | 11 +
5 files changed, 297 insertions(+), 4 deletions(-)
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/BackendStat.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/BackendStat.java
index dbee294..4ee981c 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/BackendStat.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/BackendStat.java
@@ -1100,6 +1100,15 @@
}
}
+ /**
+ * Whether a key holding this many entries has come near the entry limit of its index. An
+ * index-entry-limit of 0 is no limit at all, and no key is near it.
+ */
+ static boolean nearLimit(long size, long entryLimit)
+ {
+ return entryLimit > 0 && size >= entryLimit * 0.8;
+ }
+
private void appendIndexStats(final TableBuilder builder, EntryContainer ec, final Index index,
final Map<Index, StringBuilder> undefinedKeys)
{
@@ -1136,7 +1145,7 @@
if (entryIDSet.isDefined())
{
- if (entryIDSet.size() >= entryLimit * 0.8)
+ if (nearLimit(entryIDSet.size(), entryLimit))
{
if (entryIDSet.size() >= entryLimit * 0.95)
{
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java
index efb49c5..cd8bb92 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java
@@ -3553,6 +3553,17 @@
}
/**
+ * The number of entry IDs at which the collectors give a key up. An index-entry-limit of 0 is no
+ * limit at all ("For no limit, use 0 for the value"), as the live index takes it; compared as a
+ * count it would have every key given up.
+ */
+ private static int entryLimitOf(DefaultIndex index)
+ {
+ final int indexEntryLimit = index.getIndexEntryLimit();
+ return indexEntryLimit > 0 ? indexEntryLimit : Integer.MAX_VALUE;
+ }
+
+ /**
* {@link Collector} that accepts encoded {@link EntryIDSet} objects and
* produces a {@link ByteString} representing the merged {@link EntryIDSet}.
*/
@@ -3564,7 +3575,7 @@
EntryIDsCollector(DefaultIndex index)
{
this.index = index;
- this.indexLimit = index.getIndexEntryLimit();
+ this.indexLimit = entryLimitOf(index);
}
@Override
@@ -3638,7 +3649,7 @@
EntryIDSetsCollector(DefaultIndex index)
{
this.index = index;
- this.indexLimit = index.getIndexEntryLimit();
+ this.indexLimit = entryLimitOf(index);
}
@Override
diff --git a/opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/BackendStatTest.java b/opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/BackendStatTest.java
new file mode 100644
index 0000000..69321a1
--- /dev/null
+++ b/opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/BackendStatTest.java
@@ -0,0 +1,43 @@
+/*
+ * The contents of this file are subject to the terms of the Common Development and
+ * Distribution License (the License). You may not use this file except in compliance with the
+ * License.
+ *
+ * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
+ * specific language governing permission and limitations under the License.
+ *
+ * When distributing Covered Software, include this CDDL Header Notice in each file and include
+ * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
+ * Header, with the fields enclosed by brackets [] replaced by your own identifying
+ * information: "Portions copyright [year] [name of copyright owner]".
+ *
+ * Copyright 2026 3A Systems, LLC.
+ */
+package org.opends.server.backends.pluggable;
+
+import static org.assertj.core.api.Assertions.*;
+
+import org.opends.server.DirectoryServerTestCase;
+import org.testng.annotations.Test;
+
+/** The figures {@code backendstat show-index-status} works out for each key of an index. */
+@SuppressWarnings("javadoc")
+@Test(groups = { "precommit", "pluggablebackend", "unit" }, sequential = true)
+public class BackendStatTest extends DirectoryServerTestCase
+{
+ /** A key is reported as near its limit from 80% of the limit on. */
+ @Test
+ public void testAKeyIsNearItsLimitFromEightyPercentOn()
+ {
+ assertThat(BackendStat.nearLimit(79, 100)).isFalse();
+ assertThat(BackendStat.nearLimit(80, 100)).isTrue();
+ }
+
+ /** An index-entry-limit of 0 is no limit at all, and no key is near it (#1059). */
+ @Test
+ public void testNoKeyIsNearNoLimit()
+ {
+ assertThat(BackendStat.nearLimit(1, 0)).isFalse();
+ assertThat(BackendStat.nearLimit(Integer.MAX_VALUE, 0)).isFalse();
+ }
+}
diff --git a/opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/OnDiskMergeImporterTest.java b/opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/OnDiskMergeImporterTest.java
index 214c9a00..d617f35 100644
--- a/opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/OnDiskMergeImporterTest.java
+++ b/opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/OnDiskMergeImporterTest.java
@@ -12,6 +12,7 @@
* information: "Portions Copyright [year] [name of copyright owner]".
*
* Copyright 2015-2016 ForgeRock AS.
+ * Portions Copyright 2026 3A Systems, LLC.
*/
package org.opends.server.backends.pluggable;
@@ -71,6 +72,7 @@
import org.opends.server.backends.pluggable.OnDiskMergeImporter.Collector;
import org.opends.server.backends.pluggable.OnDiskMergeImporter.DnValidationCursorDecorator;
import org.opends.server.backends.pluggable.OnDiskMergeImporter.EntryIDSetsCollector;
+import org.opends.server.backends.pluggable.OnDiskMergeImporter.EntryIDsCollector;
import org.opends.server.backends.pluggable.OnDiskMergeImporter.ExternalSortChunk;
import org.opends.server.backends.pluggable.OnDiskMergeImporter.ExternalSortChunk.CollectorCursor;
import org.opends.server.backends.pluggable.OnDiskMergeImporter.ExternalSortChunk.CompositeCursor;
@@ -470,6 +472,116 @@
assertThat(toPairs(result)).containsExactlyElementsOf(toPairs(expected));
}
+ /**
+ * An index-entry-limit of 0 is no limit at all ("For no limit, use 0 for the value"): the phase-two
+ * collector gives no key up under it, however many chunks hold the key. A key which a chunk had
+ * already given up stays undefined - nothing can put its entries back (#1059).
+ */
+ @Test
+ public void testEntryIDSetCollectorGivesNoKeyUpUnderNoLimit()
+ {
+ final MeteredCursor<String, ByteString> source = cursorOf(
+ Pair.of("key1", EntryIDSet.CODEC_V2.encode(newDefinedSet(2))),
+ Pair.of("key1", EntryIDSet.CODEC_V2.encode(newDefinedSet(1))),
+
+ Pair.of("key2", EntryIDSet.CODEC_V2.encode(newDefinedSet(1))),
+
+ Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(1))),
+ Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(2))),
+ Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(3))),
+ Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(4))),
+ Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(5))),
+ Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(6))),
+ Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(7))),
+ Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(8))),
+ Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(9))),
+ Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(10))),
+
+ Pair.of("key4", EntryIDSet.CODEC_V2.encode(newDefinedSet(10))),
+ Pair.of("key4", EntryIDSet.CODEC_V2.encode(newUndefinedSet())));
+
+ final SequentialCursor<String, ByteString> expected = cursorOf(
+ Pair.of("key1", EntryIDSet.CODEC_V2.encode(newDefinedSet(1, 2))),
+ Pair.of("key2", EntryIDSet.CODEC_V2.encode(newDefinedSet(1))),
+ Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(1, 2, 3, 4, 5, 6, 7, 8, 9, 10))),
+ Pair.of("key4", EntryIDSet.CODEC_V2.encode(newUndefinedSet())));
+
+ final SequentialCursor<String, ByteString> result =
+ new CollectorCursor<>(source, new EntryIDSetsCollector(new DummyIndex(0)));
+
+ assertThat(toPairs(result)).containsExactlyElementsOf(toPairs(expected));
+ }
+
+ /** The phase-one collector gives a key up once it holds as many entry IDs as the limit allows. */
+ @Test
+ public void testEntryIDsCollector()
+ {
+ final MeteredCursor<String, ByteString> source = cursorOf(
+ Pair.of("key1", entryID(2)),
+ Pair.of("key1", entryID(1)),
+
+ Pair.of("key2", entryID(1)),
+
+ Pair.of("key3", entryID(1)),
+ Pair.of("key3", entryID(2)),
+ Pair.of("key3", entryID(3)),
+ Pair.of("key3", entryID(4)),
+ Pair.of("key3", entryID(5)),
+ Pair.of("key3", entryID(6)),
+ Pair.of("key3", entryID(7)),
+ Pair.of("key3", entryID(8)),
+ Pair.of("key3", entryID(9)),
+ Pair.of("key3", entryID(10)));
+
+ final SequentialCursor<String, ByteString> expected = cursorOf(
+ Pair.of("key1", EntryIDSet.CODEC_V2.encode(newDefinedSet(1, 2))),
+ Pair.of("key2", EntryIDSet.CODEC_V2.encode(newDefinedSet(1))),
+ Pair.of("key3", EntryIDSet.CODEC_V2.encode(newUndefinedSet())));
+
+ final SequentialCursor<String, ByteString> result =
+ new CollectorCursor<>(source, new EntryIDsCollector(new DummyIndex(10)));
+
+ assertThat(toPairs(result)).containsExactlyElementsOf(toPairs(expected));
+ }
+
+ /** Under an index-entry-limit of 0 the phase-one collector gives no key up either (#1059). */
+ @Test
+ public void testEntryIDsCollectorGivesNoKeyUpUnderNoLimit()
+ {
+ final MeteredCursor<String, ByteString> source = cursorOf(
+ Pair.of("key1", entryID(2)),
+ Pair.of("key1", entryID(1)),
+
+ Pair.of("key2", entryID(1)),
+
+ Pair.of("key3", entryID(1)),
+ Pair.of("key3", entryID(2)),
+ Pair.of("key3", entryID(3)),
+ Pair.of("key3", entryID(4)),
+ Pair.of("key3", entryID(5)),
+ Pair.of("key3", entryID(6)),
+ Pair.of("key3", entryID(7)),
+ Pair.of("key3", entryID(8)),
+ Pair.of("key3", entryID(9)),
+ Pair.of("key3", entryID(10)));
+
+ final SequentialCursor<String, ByteString> expected = cursorOf(
+ Pair.of("key1", EntryIDSet.CODEC_V2.encode(newDefinedSet(1, 2))),
+ Pair.of("key2", EntryIDSet.CODEC_V2.encode(newDefinedSet(1))),
+ Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(1, 2, 3, 4, 5, 6, 7, 8, 9, 10))));
+
+ final SequentialCursor<String, ByteString> result =
+ new CollectorCursor<>(source, new EntryIDsCollector(new DummyIndex(0)));
+
+ assertThat(toPairs(result)).containsExactlyElementsOf(toPairs(expected));
+ }
+
+ /** An entry ID the way phase one buffers it: the bare ID, not an {@link EntryIDSet}. */
+ private static ByteString entryID(long id)
+ {
+ return new EntryID(id).toByteString();
+ }
+
@Test
public void testUniqueValueCollectorAcceptUniqueValues()
{
diff --git a/opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/PluggableBackendImplTestCase.java b/opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/PluggableBackendImplTestCase.java
index ae0c5a7..475a64e 100644
--- a/opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/PluggableBackendImplTestCase.java
+++ b/opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/PluggableBackendImplTestCase.java
@@ -94,6 +94,7 @@
public abstract class PluggableBackendImplTestCase<C extends PluggableBackendCfg> extends DirectoryServerTestCase
{
private BackendImpl<C> backend;
+ private C backendCfg;
private List<Entry> topEntries;
private List<Entry> entries;
private List<Entry> workEntries;
@@ -144,7 +145,7 @@
testBaseDN = DN.valueOf("dc=test,dc=com");
- C backendCfg = createBackendCfg();
+ backendCfg = createBackendCfg();
when(backendCfg.dn()).thenReturn(testBaseDN);
when(backendCfg.getBaseDN()).thenReturn(newTreeSet(testBaseDN));
when(backendCfg.listBackendIndexes()).thenReturn(backendIndexes.keySet().toArray(new String[0]));
@@ -1229,6 +1230,123 @@
assertThat(backend.verifyBackend(config)).isEqualTo(0);
}
+ /**
+ * An index-entry-limit of 0 is no limit at all ("For no limit, use 0 for the value"), and the live
+ * index honours it. The importer compared it as the smallest limit there is and wrote every key of
+ * an import undefined: the index was then trusted, and every search through it unindexed (#1059).
+ */
+ @Test
+ public void testImportUnderNoIndexEntryLimitKeepsEveryKey() throws Exception
+ {
+ final BackendIndexCfg snIndexCfg = backendCfg.getBackendIndex("sn");
+ when(snIndexCfg.getIndexEntryLimit()).thenReturn(0);
+ try
+ {
+ final byte[] ldif = exportLDIF();
+ backend.finalizeBackend();
+ importLDIF(ldif);
+ backend.openBackend();
+
+ final Map<String, Boolean> keys = indexKeys("sn");
+ assertThat(keys).as("the keys the sn indexes hold").isNotEmpty();
+ assertThat(keys).as("a key the import gave up under no limit").doesNotContainValue(false);
+ }
+ finally
+ {
+ when(snIndexCfg.getIndexEntryLimit()).thenReturn(4000);
+ reopenBackend();
+ }
+ }
+
+ /** A rebuild writes its keys the way an import does: under no limit it gives none of them up either. */
+ @Test
+ public void testRebuildUnderNoIndexEntryLimitKeepsEveryKey() throws Exception
+ {
+ final BackendIndexCfg snIndexCfg = backendCfg.getBackendIndex("sn");
+ when(snIndexCfg.getIndexEntryLimit()).thenReturn(0);
+ try
+ {
+ final RebuildConfig rebuildConfig = new RebuildConfig();
+ rebuildConfig.setBaseDN(testBaseDN);
+ rebuildConfig.addRebuildIndex("sn");
+ backend.closeBackend();
+ backend.rebuildBackend(rebuildConfig, TestCaseUtils.getServerContext());
+ backend.openBackend();
+
+ final Map<String, Boolean> keys = indexKeys("sn");
+ assertThat(keys).as("the keys the sn indexes hold").isNotEmpty();
+ assertThat(keys).as("a key the rebuild gave up under no limit").doesNotContainValue(false);
+ }
+ finally
+ {
+ when(snIndexCfg.getIndexEntryLimit()).thenReturn(4000);
+ reopenBackend();
+ }
+ }
+
+ private byte[] exportLDIF() throws Exception
+ {
+ final ByteArrayOutputStream ldif = new ByteArrayOutputStream();
+ try (LDIFExportConfig exportConfig = new LDIFExportConfig(ldif))
+ {
+ exportConfig.setIncludeOperationalAttributes(true);
+ backend.exportLDIF(exportConfig);
+ }
+ return ldif.toByteArray();
+ }
+
+ /** Imports the LDIF into the cleared backend, which the caller has finalized and opens again afterwards. */
+ private void importLDIF(byte[] ldif) throws Exception
+ {
+ final ByteArrayOutputStream rejectedEntries = new ByteArrayOutputStream();
+ try (LDIFImportConfig importConfig = new LDIFImportConfig(new ByteArrayInputStream(ldif)))
+ {
+ importConfig.setClearBackend(true);
+ importConfig.writeRejectedEntries(rejectedEntries);
+ importConfig.setIncludeBranches(Collections.singleton(testBaseDN));
+ importConfig.setThreadCount(0);
+ backend.importLDIF(importConfig, TestCaseUtils.getServerContext());
+ }
+ assertEquals(rejectedEntries.size(), 0, "No entries should be rejected. Content was:\n" + rejectedEntries);
+ }
+
+ /** Every key of every index of the attribute, and whether the index still holds its entries. */
+ private Map<String, Boolean> indexKeys(String attributeName) throws Exception
+ {
+ final AttributeType attributeType = TestCaseUtils.getServerContext().getSchema().getAttributeType(attributeName);
+ final EntryContainer entryContainer = backend.getRootContainer().getEntryContainer(testBaseDN);
+ final AttributeIndex attributeIndex = entryContainer.getAttributeIndex(attributeType);
+ return backend.getRootContainer().getStorage().read(new ReadOperation<Map<String, Boolean>>()
+ {
+ @Override
+ public Map<String, Boolean> run(ReadableTransaction txn) throws Exception
+ {
+ final Map<String, Boolean> keys = new TreeMap<>();
+ for (AttributeIndex.MatchingRuleIndex index : attributeIndex.getNameToIndexes().values())
+ {
+ try (Cursor<ByteString, EntryIDSet> cursor = index.openCursor(txn))
+ {
+ while (cursor.next())
+ {
+ keys.put(index.getName() + " " + cursor.getKey().toHexString(), cursor.getValue().isDefined());
+ }
+ }
+ }
+ return keys;
+ }
+ });
+ }
+
+ /** Opens the backend afresh, so that its indexes hold the configuration the other tests expect. */
+ private void reopenBackend() throws Exception
+ {
+ if (backend.getRootContainer() != null)
+ {
+ backend.finalizeBackend();
+ }
+ backend.openBackend();
+ }
+
@Test
public void testVerifyID2ChildrenCount() throws Exception
{
--
Gitblit v1.10.0