/* * The contents of this file are subject to the terms of the Common Development and * Distribution License (the License). You may not use this file except in compliance with the * License. * * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the * specific language governing permission and limitations under the License. * * When distributing Covered Software, include this CDDL Header Notice in each file and include * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL * Header, with the fields enclosed by brackets [] replaced by your own identifying * information: "Portions copyright [year] [name of copyright owner]". * * Copyright 2026 3A Systems, LLC. */ package org.opends.server.backends.pluggable; import static org.assertj.core.api.Assertions.assertThat; import static org.forgerock.opendj.config.ConfigurationMock.mockCfg; import static org.mockito.Mockito.any; import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.opends.server.util.CollectionUtils.newTreeSet; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.SortedSet; import org.forgerock.opendj.config.server.ConfigException; import org.forgerock.opendj.config.server.ConfigurationChangeListener; import org.forgerock.opendj.ldap.ByteSequence; import org.forgerock.opendj.ldap.ByteString; import org.forgerock.opendj.ldap.DN; import org.forgerock.opendj.ldap.schema.AttributeType; import org.forgerock.opendj.server.config.meta.BackendIndexCfgDefn.IndexType; import org.forgerock.opendj.server.config.server.BackendIndexCfg; import org.forgerock.opendj.server.config.server.PDBBackendCfg; import org.forgerock.opendj.server.config.server.PluggableBackendCfg; import org.mockito.ArgumentCaptor; import org.opends.server.DirectoryServerTestCase; import org.opends.server.TestCaseUtils; import org.opends.server.backends.pdb.PDBStorage; import org.opends.server.backends.pluggable.spi.AccessMode; import org.opends.server.backends.pluggable.spi.Cursor; import org.opends.server.backends.pluggable.spi.Importer; import org.opends.server.backends.pluggable.spi.ReadOperation; import org.opends.server.backends.pluggable.spi.Storage; import org.opends.server.backends.pluggable.spi.StorageRuntimeException; import org.opends.server.backends.pluggable.spi.StorageStatus; import org.opends.server.backends.pluggable.spi.TreeName; import org.opends.server.backends.pluggable.spi.UpdateFunction; import org.opends.server.backends.pluggable.spi.WriteOperation; import org.opends.server.backends.pluggable.spi.WriteableTransaction; import org.opends.server.core.ServerContext; import org.opends.server.types.BackupConfig; import org.opends.server.types.BackupDirectory; import org.opends.server.types.DirectoryException; import org.opends.server.types.RestoreConfig; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import com.persistit.exception.RollbackException; /** * Tests that {@link RootContainer#open(AccessMode)} survives a replay of its {@link WriteOperation}. * {@link Storage#write(WriteOperation)} may replay the operation after a transaction conflict, and * {@code RootContainer.open} opens and registers the entry container of every base DN inside a * single one of them, so every side effect that write performs must either be transactional or be * idempotent - see OpenDJ issue #896. *

* The conflict is raised from inside the operation as the {@link RollbackException} PersistIt itself * raises, so that the replay is driven by {@code PDBStorage.write}'s own retry loop rather than by a * second call to it, as {@link ReplayedConfigChangeTest} does for the sibling path. */ @SuppressWarnings("javadoc") @Test(groups = { "precommit", "pluggablebackend" }, sequential = true) public class ReplayedOpenTest extends DirectoryServerTestCase { private static final String BACKEND_ID = "ReplayedOpenTest"; /** Opened and registered first, since the base DNs are opened in the order of the sorted set. */ private static final DN FIRST = DN.valueOf("dc=b896a,dc=com"); /** Opened while the first one is already registered, which is where a conflict reaches the bug. */ private static final DN SECOND = DN.valueOf("dc=b896b,dc=com"); private ServerContext serverContext; private AttributeType cnType; /** The configuration of the cn index, which the attribute index of every base DN registers with. */ private BackendIndexCfg indexCfg; @BeforeClass public void startServer() throws Exception { TestCaseUtils.startServer(); serverContext = TestCaseUtils.getServerContext(); cnType = serverContext.getSchema().getAttributeType("cn"); } /** * These tests open a backend which is designed to fail, and a failing one can leave a base DN * behind in the server wide registry, where it would outlive the test and break the next one. */ @AfterMethod public void deregisterLeftoverBaseDNs() { for (DN baseDN : new DN[] { FIRST, SECOND }) { try { serverContext.getBackendConfigManager().deregisterBaseDN(baseDN); } catch (Exception alreadyGone) { // Which is what the test should have left behind. } } } /** * The case the report is written from: the conflict is raised while the second base DN is being * opened, so the replay meets the first one already registered by the attempt it replaces. */ @Test public void openIsReplayableWhenTheTransactionConflictsWhileTheSecondBaseDNIsOpened() throws Exception { final ReplayingBackend backend = newBackend(newTreeSet(FIRST, SECOND)); boolean opened = false; try { backend.storage.conflictAtTreesOf(SECOND, 1); backend.openBackend(); opened = true; assertThat(backend.storage.attempts()).isEqualTo(2); final RootContainer rootContainer = backend.getRootContainer(); assertThat(rootContainer.getBaseDNs()).containsOnly(FIRST, SECOND); // Each base DN is answered with a container of its own, rather than with the one an ancestor // registered, and the trees of both are there to be read. for (DN baseDN : new DN[] { FIRST, SECOND }) { final EntryContainer ec = rootContainer.getEntryContainer(baseDN); assertThat((Object) ec.getBaseDN()).isEqualTo(baseDN); assertThat(rootContainer.getStorage().listTrees()).containsAll(treesOf(ec)); } // What the attempt which was replaced opened is off the backend configuration, and the live // pair is on it: SECOND of that attempt was given up by its own close(), raised from // EntryContainer.open's own catch, and FIRST by the replay, which begins by unregistering and // closing what the attempt it replaces had registered. Counting the removals would say neither // which containers were given back nor that the adds balance: one of the two takes off a // listener SECOND had never added, the conflict being raised at its id2entry, before // EntryContainer.open registers anything. assertThat(entryContainersRegisteredOn(backend.configuredWith)) .containsOnly(rootContainer.getEntryContainer(FIRST), rootContainer.getEntryContainer(SECOND)); // The same balance one level down: a container given back without the listener each of its // indexes registered would satisfy the assertion above. assertThat(indexesRegisteredOn(indexCfg)) .containsOnly(rootContainer.getEntryContainer(FIRST).getAttributeIndex(cnType), rootContainer.getEntryContainer(SECOND).getAttributeIndex(cnType)); } finally { close(backend, opened); } } /** * A conflict raised once the operation has run to completion, before the commit, replays an * operation which ran to completion, so every base DN of the backend is registered by the * attempt the replay replaces. One base DN would be enough to reach that, and * {@code FailedBackendOpenTest.aReplayedOpenLeavesOneSetOfEntryContainers} drives the same * conflict with one; two are used here so that the case differs from the one above only in where * the conflict was raised, and so that the give-back has more than one container to walk. */ @Test public void openIsReplayableWhenTheTransactionConflictsAtCommitTime() throws Exception { final ReplayingBackend backend = newBackend(newTreeSet(FIRST, SECOND)); boolean opened = false; try { backend.storage.conflictAtCommit(1); backend.openBackend(); opened = true; assertThat(backend.storage.attempts()).isEqualTo(2); final RootContainer rootContainer = backend.getRootContainer(); assertThat(rootContainer.getBaseDNs()).containsOnly(FIRST, SECOND); // The two entry containers the rolled back attempt opened registered five configuration // listeners each, and one more per index, which only their close() takes back, so the replay // has to give both up before it opens another pair: what is left of the two attempts is the // live pair alone. assertThat(entryContainersRegisteredOn(backend.configuredWith)) .containsOnly(rootContainer.getEntryContainer(FIRST), rootContainer.getEntryContainer(SECOND)); assertThat(indexesRegisteredOn(indexCfg)) .containsOnly(rootContainer.getEntryContainer(FIRST).getAttributeIndex(cnType), rootContainer.getEntryContainer(SECOND).getAttributeIndex(cnType)); } finally { close(backend, opened); } } /** * A failure the storage engine does not replay ends the open, and {@code RootContainer}'s own * give-back is the only one there is: {@code BackendImpl.newRootContainer} drops the root * container without closing it, so what the attempt had opened by then - both entry containers, * the root container's own listener and the storage - is handed back by * {@code RootContainer.giveUpAfterFailedOpen} or not at all. */ @Test public void anOpenWhichIsNotReplayedGivesUpTheEntryContainersItOpened() throws Exception { final ReplayingBackend backend = newBackend(newTreeSet(FIRST, SECOND)); try { backend.storage.failWithoutReplay(); try { backend.openBackend(); throw new AssertionError("the open was expected to fail"); } catch (Exception expected) { // The failure itself is the caller's business; what it left behind is this test's. } // The failure is the one the engines do not replay, so the open ends at the first attempt: // a replay would open a second pair of containers over the first. assertThat(backend.storage.attempts()).isEqualTo(1); // The two entry containers the attempt opened, and the root container's own listener, which // goes with them since #993: nothing closes a root container which did not open. verify(backend.configuredWith, times(3)).removePluggableChangeListener(any()); } finally { close(backend, false); } } private static Set treesOf(EntryContainer ec) { final Set names = new HashSet<>(); for (Tree tree : ec.listTrees()) { names.add(tree.getName()); } return names; } /** * Closes a backend which opened; for one which did not, closes its storage a second time, in case * the give-back of the failed open did not reach it - a no-op under {@code PDBStorage.close}'s * {@code db != null} guard when it did, and a volume left open fails every following test with a * {@code StorageInUseException} rather than with what actually broke. */ private static void close(ReplayingBackend backend, boolean opened) { if (opened) { backend.finalizeBackend(); } else { backend.storage.close(); } } /** * The entry containers still registered as listeners of the backend configuration, by identity. *

* A replayed open registers a container per base DN twice and gives the first set back, so what * the case is about is which containers are left rather than how many calls were made: a removal * says nothing on its own - an entry container whose open failed before it had registered * anything is taken off all the same, by the {@code close()} of {@code EntryContainer.open}'s own * catch. */ private static List entryContainersRegisteredOn(PluggableBackendCfg cfg) { final ArgumentCaptor> added = captorFor(ConfigurationChangeListener.class); verify(cfg, atLeast(0)).addPluggableChangeListener(added.capture()); final ArgumentCaptor> removed = captorFor(ConfigurationChangeListener.class); verify(cfg, atLeast(0)).removePluggableChangeListener(removed.capture()); return balanceOf(EntryContainer.class, added.getAllValues(), removed.getAllValues()); } /** The attribute indexes still registered as listeners of an index configuration, by identity. */ private static List indexesRegisteredOn(BackendIndexCfg cfg) { final ArgumentCaptor> added = captorFor(ConfigurationChangeListener.class); verify(cfg, atLeast(0)).addChangeListener(added.capture()); final ArgumentCaptor> removed = captorFor(ConfigurationChangeListener.class); verify(cfg, atLeast(0)).removeChangeListener(removed.capture()); return balanceOf(AttributeIndex.class, added.getAllValues(), removed.getAllValues()); } /** * What was added to a configuration and not taken off it again, of one kind of listener: the root * container and the two configuration managers of every entry container listen to the backend * configuration as well, and neither is what these cases are about. Identity, since none of these * classes overrides {@code equals()}. */ private static List balanceOf(Class kind, List added, List removed) { final List registered = new ArrayList<>(); for (Object listener : added) { if (kind.isInstance(listener)) { registered.add(kind.cast(listener)); } } for (Object listener : removed) { if (kind.isInstance(listener)) { registered.remove(kind.cast(listener)); } } return registered; } @SuppressWarnings({ "unchecked", "rawtypes" }) private static ArgumentCaptor captorFor(Class listenerClass) { return (ArgumentCaptor) ArgumentCaptor.forClass((Class) listenerClass); } private ReplayingBackend newBackend(SortedSet baseDNs) throws Exception { final ReplayingBackend backend = new ReplayingBackend(); backend.setBackendID(BACKEND_ID); backend.configuredWith = backendCfg(baseDNs); backend.configureBackend(backend.configuredWith, serverContext); // Start from a pristine on-disk state so that a previous run cannot mask the defect. backend.storage.removeStorageFiles(); return backend; } private PDBBackendCfg backendCfg(SortedSet baseDNs) throws ConfigException { final PDBBackendCfg cfg = mockCfg(PDBBackendCfg.class); when(cfg.dn()).thenReturn(DN.valueOf("ds-cfg-backend-id=" + BACKEND_ID + ",cn=Backends,cn=config")); when(cfg.getBackendId()).thenReturn(BACKEND_ID); when(cfg.getDBDirectory()).thenReturn(BACKEND_ID); when(cfg.getDBDirectoryPermissions()).thenReturn("755"); when(cfg.getDBCacheSize()).thenReturn(0L); when(cfg.getDBCachePercent()).thenReturn(20); when(cfg.getBaseDN()).thenReturn(baseDNs); when(cfg.listBackendIndexes()).thenReturn(new String[] { "cn" }); when(cfg.listBackendVLVIndexes()).thenReturn(new String[0]); indexCfg = mock(BackendIndexCfg.class); when(indexCfg.getIndexType()).thenReturn(newTreeSet(IndexType.EQUALITY)); when(indexCfg.getAttribute()).thenReturn(cnType); when(indexCfg.getIndexEntryLimit()).thenReturn(4000); when(indexCfg.getSubstringLength()).thenReturn(6); when(cfg.getBackendIndex("cn")).thenReturn(indexCfg); return cfg; } /** A backend whose storage makes the next write operation conflict, and so be replayed. */ private static final class ReplayingBackend extends BackendImpl { private ReplayingStorage storage; /** The configuration the entry containers register their listeners with. */ private PDBBackendCfg configuredWith; @Override protected Storage configureStorage(PDBBackendCfg cfg, ServerContext serverContext) throws ConfigException { storage = new ReplayingStorage(new PDBStorage(cfg, serverContext)); return storage; } } /** A failure which no storage engine replays, unlike {@link RollbackException}. */ private static final class UnreplayableFailure extends Exception { private static final long serialVersionUID = 1L; } /** * Decorates a {@link Storage} so that the next {@link Storage#write(WriteOperation)} conflicts a * given number of times before it is let through. The conflict is raised from within the single * {@code write} the delegate is asked for, so the delegate's own retry loop performs the replay. */ private static final class ReplayingStorage implements Storage { /** Where the conflict is raised, which decides how much of the operation has run. */ private enum ConflictPoint { /** As the first tree of a given base DN is opened, with the base DNs before it registered. */ TREES_OF_BASE_DN, /** Once the operation has run to completion, before the commit. */ COMMIT, /** Once the operation has run to completion, as a failure which is not replayed at all. */ NO_REPLAY } private final Storage delegate; private ConflictPoint conflictPoint; private String conflictingPrefix; private int conflictsLeft; private int attempts; ReplayingStorage(Storage delegate) { this.delegate = delegate; } void conflictAtTreesOf(DN baseDN, int conflicts) { conflictingPrefix = baseDN.toNormalizedUrlSafeString(); arm(ConflictPoint.TREES_OF_BASE_DN, conflicts); } void conflictAtCommit(int conflicts) { arm(ConflictPoint.COMMIT, conflicts); } void failWithoutReplay() { arm(ConflictPoint.NO_REPLAY, 1); } private void arm(ConflictPoint where, int conflicts) { conflictPoint = where; conflictsLeft = conflicts; attempts = 0; } /** How many times the armed operation was run, the first attempt included. */ int attempts() { return attempts; } @Override public void write(final WriteOperation writeOperation) throws Exception { final ConflictPoint armed = conflictPoint; if (armed == null) { delegate.write(writeOperation); return; } conflictPoint = null; // A single call, so that the replay is the delegate's own and keeps whatever the delegate // holds for the duration of a write, rather than starting afresh as a second call would. delegate.write(new WriteOperation() { @Override public void run(WriteableTransaction txn) throws Exception { attempts++; if (conflictsLeft-- <= 0) { writeOperation.run(txn); return; } if (armed == ConflictPoint.TREES_OF_BASE_DN) { writeOperation.run(new ConflictingAtTreesOf(txn, conflictingPrefix)); return; } writeOperation.run(txn); if (armed == ConflictPoint.NO_REPLAY) { throw new UnreplayableFailure(); } throw new RollbackException(); } }); } @Override public Importer startImport() throws ConfigException { return delegate.startImport(); } @Override public void open(AccessMode accessMode) throws Exception { delegate.open(accessMode); } @Override public T read(ReadOperation readOperation) throws Exception { return delegate.read(readOperation); } @Override public void removeStorageFiles() { delegate.removeStorageFiles(); } @Override public StorageStatus getStorageStatus() { return delegate.getStorageStatus(); } @Override public boolean supportsBackupAndRestore() { return delegate.supportsBackupAndRestore(); } @Override public void createBackup(BackupConfig backupConfig) throws DirectoryException { delegate.createBackup(backupConfig); } @Override public void removeBackup(BackupDirectory backupDirectory, String backupID) throws DirectoryException { delegate.removeBackup(backupDirectory, backupID); } @Override public void restoreBackup(RestoreConfig restoreConfig) throws DirectoryException { delegate.restoreBackup(restoreConfig); } @Override public Set listTrees() { return delegate.listTrees(); } @Override public void close() { delegate.close(); } } /** * A transaction which conflicts as the first tree of one base DN is opened, and delegates * everything the operation did before that. This is where an entry container which is being opened * meets a write-write conflict: PDBStorage wraps the {@link RollbackException} PersistIt raises at * the store into a {@link StorageRuntimeException}, which is what {@code EntryContainer.open}'s own * catch unwinds on, so the base DNs opened before this one have been registered and the one being * opened has not. */ private static final class ConflictingAtTreesOf implements WriteableTransaction { private final WriteableTransaction delegate; private final String conflictingPrefix; ConflictingAtTreesOf(WriteableTransaction delegate, String conflictingPrefix) { this.delegate = delegate; this.conflictingPrefix = conflictingPrefix; } @Override public void openTree(TreeName name, boolean createOnDemand) { if (conflictingPrefix.equals(name.getBaseDN())) { // What PDBStorage delivers: the engine's RollbackException wrapped, which // EntryContainer.open's catch(StorageRuntimeException) unwinds on. throw new StorageRuntimeException(new RollbackException()); } delegate.openTree(name, createOnDemand); } @Override public void deleteTree(TreeName name) { delegate.deleteTree(name); } @Override public void put(TreeName treeName, ByteSequence key, ByteSequence value) { delegate.put(treeName, key, value); } @Override public boolean update(TreeName treeName, ByteSequence key, UpdateFunction f) { return delegate.update(treeName, key, f); } @Override public boolean delete(TreeName treeName, ByteSequence key) { return delegate.delete(treeName, key); } @Override public ByteString read(TreeName treeName, ByteSequence key) { return delegate.read(treeName, key); } @Override public Cursor openCursor(TreeName treeName) { return delegate.openCursor(treeName); } @Override public long getRecordCount(TreeName treeName) { return delegate.getRecordCount(treeName); } @Override public boolean treeExists(TreeName treeName) { return delegate.treeExists(treeName); } } }