mirror of https://github.com/OpenIdentityPlatform/OpenDJ.git
opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/BackendStat.java
@@ -1100,6 +1100,15 @@ } } /** * Whether a key holding this many entries has come near the entry limit of its index. An * index-entry-limit of 0 is no limit at all, and no key is near it. */ static boolean nearLimit(long size, long entryLimit) { return entryLimit > 0 && size >= entryLimit * 0.8; } private void appendIndexStats(final TableBuilder builder, EntryContainer ec, final Index index, final Map<Index, StringBuilder> undefinedKeys) { @@ -1136,7 +1145,7 @@ if (entryIDSet.isDefined()) { if (entryIDSet.size() >= entryLimit * 0.8) if (nearLimit(entryIDSet.size(), entryLimit)) { if (entryIDSet.size() >= entryLimit * 0.95) { opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java
@@ -3553,6 +3553,17 @@ } /** * The number of entry IDs at which the collectors give a key up. An index-entry-limit of 0 is no * limit at all ("For no limit, use 0 for the value"), as the live index takes it; compared as a * count it would have every key given up. */ private static int entryLimitOf(DefaultIndex index) { final int indexEntryLimit = index.getIndexEntryLimit(); return indexEntryLimit > 0 ? indexEntryLimit : Integer.MAX_VALUE; } /** * {@link Collector} that accepts encoded {@link EntryIDSet} objects and * produces a {@link ByteString} representing the merged {@link EntryIDSet}. */ @@ -3564,7 +3575,7 @@ EntryIDsCollector(DefaultIndex index) { this.index = index; this.indexLimit = index.getIndexEntryLimit(); this.indexLimit = entryLimitOf(index); } @Override @@ -3638,7 +3649,7 @@ EntryIDSetsCollector(DefaultIndex index) { this.index = index; this.indexLimit = index.getIndexEntryLimit(); this.indexLimit = entryLimitOf(index); } @Override opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/BackendStatTest.java
New file @@ -0,0 +1,43 @@ /* * The contents of this file are subject to the terms of the Common Development and * Distribution License (the License). You may not use this file except in compliance with the * License. * * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the * specific language governing permission and limitations under the License. * * When distributing Covered Software, include this CDDL Header Notice in each file and include * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL * Header, with the fields enclosed by brackets [] replaced by your own identifying * information: "Portions copyright [year] [name of copyright owner]". * * Copyright 2026 3A Systems, LLC. */ package org.opends.server.backends.pluggable; import static org.assertj.core.api.Assertions.*; import org.opends.server.DirectoryServerTestCase; import org.testng.annotations.Test; /** The figures {@code backendstat show-index-status} works out for each key of an index. */ @SuppressWarnings("javadoc") @Test(groups = { "precommit", "pluggablebackend", "unit" }, sequential = true) public class BackendStatTest extends DirectoryServerTestCase { /** A key is reported as near its limit from 80% of the limit on. */ @Test public void testAKeyIsNearItsLimitFromEightyPercentOn() { assertThat(BackendStat.nearLimit(79, 100)).isFalse(); assertThat(BackendStat.nearLimit(80, 100)).isTrue(); } /** An index-entry-limit of 0 is no limit at all, and no key is near it (#1059). */ @Test public void testNoKeyIsNearNoLimit() { assertThat(BackendStat.nearLimit(1, 0)).isFalse(); assertThat(BackendStat.nearLimit(Integer.MAX_VALUE, 0)).isFalse(); } } opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/OnDiskMergeImporterTest.java
@@ -12,6 +12,7 @@ * information: "Portions Copyright [year] [name of copyright owner]". * * Copyright 2015-2016 ForgeRock AS. * Portions Copyright 2026 3A Systems, LLC. */ package org.opends.server.backends.pluggable; @@ -71,6 +72,7 @@ import org.opends.server.backends.pluggable.OnDiskMergeImporter.Collector; import org.opends.server.backends.pluggable.OnDiskMergeImporter.DnValidationCursorDecorator; import org.opends.server.backends.pluggable.OnDiskMergeImporter.EntryIDSetsCollector; import org.opends.server.backends.pluggable.OnDiskMergeImporter.EntryIDsCollector; import org.opends.server.backends.pluggable.OnDiskMergeImporter.ExternalSortChunk; import org.opends.server.backends.pluggable.OnDiskMergeImporter.ExternalSortChunk.CollectorCursor; import org.opends.server.backends.pluggable.OnDiskMergeImporter.ExternalSortChunk.CompositeCursor; @@ -470,6 +472,116 @@ assertThat(toPairs(result)).containsExactlyElementsOf(toPairs(expected)); } /** * An index-entry-limit of 0 is no limit at all ("For no limit, use 0 for the value"): the phase-two * collector gives no key up under it, however many chunks hold the key. A key which a chunk had * already given up stays undefined - nothing can put its entries back (#1059). */ @Test public void testEntryIDSetCollectorGivesNoKeyUpUnderNoLimit() { final MeteredCursor<String, ByteString> source = cursorOf( Pair.of("key1", EntryIDSet.CODEC_V2.encode(newDefinedSet(2))), Pair.of("key1", EntryIDSet.CODEC_V2.encode(newDefinedSet(1))), Pair.of("key2", EntryIDSet.CODEC_V2.encode(newDefinedSet(1))), Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(1))), Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(2))), Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(3))), Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(4))), Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(5))), Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(6))), Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(7))), Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(8))), Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(9))), Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(10))), Pair.of("key4", EntryIDSet.CODEC_V2.encode(newDefinedSet(10))), Pair.of("key4", EntryIDSet.CODEC_V2.encode(newUndefinedSet()))); final SequentialCursor<String, ByteString> expected = cursorOf( Pair.of("key1", EntryIDSet.CODEC_V2.encode(newDefinedSet(1, 2))), Pair.of("key2", EntryIDSet.CODEC_V2.encode(newDefinedSet(1))), Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(1, 2, 3, 4, 5, 6, 7, 8, 9, 10))), Pair.of("key4", EntryIDSet.CODEC_V2.encode(newUndefinedSet()))); final SequentialCursor<String, ByteString> result = new CollectorCursor<>(source, new EntryIDSetsCollector(new DummyIndex(0))); assertThat(toPairs(result)).containsExactlyElementsOf(toPairs(expected)); } /** The phase-one collector gives a key up once it holds as many entry IDs as the limit allows. */ @Test public void testEntryIDsCollector() { final MeteredCursor<String, ByteString> source = cursorOf( Pair.of("key1", entryID(2)), Pair.of("key1", entryID(1)), Pair.of("key2", entryID(1)), Pair.of("key3", entryID(1)), Pair.of("key3", entryID(2)), Pair.of("key3", entryID(3)), Pair.of("key3", entryID(4)), Pair.of("key3", entryID(5)), Pair.of("key3", entryID(6)), Pair.of("key3", entryID(7)), Pair.of("key3", entryID(8)), Pair.of("key3", entryID(9)), Pair.of("key3", entryID(10))); final SequentialCursor<String, ByteString> expected = cursorOf( Pair.of("key1", EntryIDSet.CODEC_V2.encode(newDefinedSet(1, 2))), Pair.of("key2", EntryIDSet.CODEC_V2.encode(newDefinedSet(1))), Pair.of("key3", EntryIDSet.CODEC_V2.encode(newUndefinedSet()))); final SequentialCursor<String, ByteString> result = new CollectorCursor<>(source, new EntryIDsCollector(new DummyIndex(10))); assertThat(toPairs(result)).containsExactlyElementsOf(toPairs(expected)); } /** Under an index-entry-limit of 0 the phase-one collector gives no key up either (#1059). */ @Test public void testEntryIDsCollectorGivesNoKeyUpUnderNoLimit() { final MeteredCursor<String, ByteString> source = cursorOf( Pair.of("key1", entryID(2)), Pair.of("key1", entryID(1)), Pair.of("key2", entryID(1)), Pair.of("key3", entryID(1)), Pair.of("key3", entryID(2)), Pair.of("key3", entryID(3)), Pair.of("key3", entryID(4)), Pair.of("key3", entryID(5)), Pair.of("key3", entryID(6)), Pair.of("key3", entryID(7)), Pair.of("key3", entryID(8)), Pair.of("key3", entryID(9)), Pair.of("key3", entryID(10))); final SequentialCursor<String, ByteString> expected = cursorOf( Pair.of("key1", EntryIDSet.CODEC_V2.encode(newDefinedSet(1, 2))), Pair.of("key2", EntryIDSet.CODEC_V2.encode(newDefinedSet(1))), Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)))); final SequentialCursor<String, ByteString> result = new CollectorCursor<>(source, new EntryIDsCollector(new DummyIndex(0))); assertThat(toPairs(result)).containsExactlyElementsOf(toPairs(expected)); } /** An entry ID the way phase one buffers it: the bare ID, not an {@link EntryIDSet}. */ private static ByteString entryID(long id) { return new EntryID(id).toByteString(); } @Test public void testUniqueValueCollectorAcceptUniqueValues() { opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/PluggableBackendImplTestCase.java
@@ -94,6 +94,7 @@ public abstract class PluggableBackendImplTestCase<C extends PluggableBackendCfg> extends DirectoryServerTestCase { private BackendImpl<C> backend; private C backendCfg; private List<Entry> topEntries; private List<Entry> entries; private List<Entry> workEntries; @@ -144,7 +145,7 @@ testBaseDN = DN.valueOf("dc=test,dc=com"); C backendCfg = createBackendCfg(); backendCfg = createBackendCfg(); when(backendCfg.dn()).thenReturn(testBaseDN); when(backendCfg.getBaseDN()).thenReturn(newTreeSet(testBaseDN)); when(backendCfg.listBackendIndexes()).thenReturn(backendIndexes.keySet().toArray(new String[0])); @@ -1229,6 +1230,123 @@ assertThat(backend.verifyBackend(config)).isEqualTo(0); } /** * An index-entry-limit of 0 is no limit at all ("For no limit, use 0 for the value"), and the live * index honours it. The importer compared it as the smallest limit there is and wrote every key of * an import undefined: the index was then trusted, and every search through it unindexed (#1059). */ @Test public void testImportUnderNoIndexEntryLimitKeepsEveryKey() throws Exception { final BackendIndexCfg snIndexCfg = backendCfg.getBackendIndex("sn"); when(snIndexCfg.getIndexEntryLimit()).thenReturn(0); try { final byte[] ldif = exportLDIF(); backend.finalizeBackend(); importLDIF(ldif); backend.openBackend(); final Map<String, Boolean> keys = indexKeys("sn"); assertThat(keys).as("the keys the sn indexes hold").isNotEmpty(); assertThat(keys).as("a key the import gave up under no limit").doesNotContainValue(false); } finally { when(snIndexCfg.getIndexEntryLimit()).thenReturn(4000); reopenBackend(); } } /** A rebuild writes its keys the way an import does: under no limit it gives none of them up either. */ @Test public void testRebuildUnderNoIndexEntryLimitKeepsEveryKey() throws Exception { final BackendIndexCfg snIndexCfg = backendCfg.getBackendIndex("sn"); when(snIndexCfg.getIndexEntryLimit()).thenReturn(0); try { final RebuildConfig rebuildConfig = new RebuildConfig(); rebuildConfig.setBaseDN(testBaseDN); rebuildConfig.addRebuildIndex("sn"); backend.closeBackend(); backend.rebuildBackend(rebuildConfig, TestCaseUtils.getServerContext()); backend.openBackend(); final Map<String, Boolean> keys = indexKeys("sn"); assertThat(keys).as("the keys the sn indexes hold").isNotEmpty(); assertThat(keys).as("a key the rebuild gave up under no limit").doesNotContainValue(false); } finally { when(snIndexCfg.getIndexEntryLimit()).thenReturn(4000); reopenBackend(); } } private byte[] exportLDIF() throws Exception { final ByteArrayOutputStream ldif = new ByteArrayOutputStream(); try (LDIFExportConfig exportConfig = new LDIFExportConfig(ldif)) { exportConfig.setIncludeOperationalAttributes(true); backend.exportLDIF(exportConfig); } return ldif.toByteArray(); } /** Imports the LDIF into the cleared backend, which the caller has finalized and opens again afterwards. */ private void importLDIF(byte[] ldif) throws Exception { final ByteArrayOutputStream rejectedEntries = new ByteArrayOutputStream(); try (LDIFImportConfig importConfig = new LDIFImportConfig(new ByteArrayInputStream(ldif))) { importConfig.setClearBackend(true); importConfig.writeRejectedEntries(rejectedEntries); importConfig.setIncludeBranches(Collections.singleton(testBaseDN)); importConfig.setThreadCount(0); backend.importLDIF(importConfig, TestCaseUtils.getServerContext()); } assertEquals(rejectedEntries.size(), 0, "No entries should be rejected. Content was:\n" + rejectedEntries); } /** Every key of every index of the attribute, and whether the index still holds its entries. */ private Map<String, Boolean> indexKeys(String attributeName) throws Exception { final AttributeType attributeType = TestCaseUtils.getServerContext().getSchema().getAttributeType(attributeName); final EntryContainer entryContainer = backend.getRootContainer().getEntryContainer(testBaseDN); final AttributeIndex attributeIndex = entryContainer.getAttributeIndex(attributeType); return backend.getRootContainer().getStorage().read(new ReadOperation<Map<String, Boolean>>() { @Override public Map<String, Boolean> run(ReadableTransaction txn) throws Exception { final Map<String, Boolean> keys = new TreeMap<>(); for (AttributeIndex.MatchingRuleIndex index : attributeIndex.getNameToIndexes().values()) { try (Cursor<ByteString, EntryIDSet> cursor = index.openCursor(txn)) { while (cursor.next()) { keys.put(index.getName() + " " + cursor.getKey().toHexString(), cursor.getValue().isDefined()); } } } return keys; } }); } /** Opens the backend afresh, so that its indexes hold the configuration the other tests expect. */ private void reopenBackend() throws Exception { if (backend.getRootContainer() != null) { backend.finalizeBackend(); } backend.openBackend(); } @Test public void testVerifyID2ChildrenCount() throws Exception {
