/*
|
* The contents of this file are subject to the terms of the Common Development and
|
* Distribution License (the License). You may not use this file except in compliance with the
|
* License.
|
*
|
* You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
|
* specific language governing permission and limitations under the License.
|
*
|
* When distributing Covered Software, include this CDDL Header Notice in each file and include
|
* the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
|
* Header, with the fields enclosed by brackets [] replaced by your own identifying
|
* information: "Portions copyright [year] [name of copyright owner]".
|
*
|
* Copyright 2026 3A Systems, LLC.
|
*/
|
package org.opends.server.backends.pluggable;
|
|
import static org.assertj.core.api.Assertions.assertThat;
|
import static org.forgerock.opendj.config.ConfigurationMock.mockCfg;
|
import static org.mockito.Mockito.any;
|
import static org.mockito.Mockito.atLeast;
|
import static org.mockito.Mockito.mock;
|
import static org.mockito.Mockito.times;
|
import static org.mockito.Mockito.verify;
|
import static org.mockito.Mockito.when;
|
import static org.opends.server.util.CollectionUtils.newTreeSet;
|
|
import java.util.ArrayList;
|
import java.util.HashSet;
|
import java.util.List;
|
import java.util.Set;
|
import java.util.SortedSet;
|
|
import org.forgerock.opendj.config.server.ConfigException;
|
import org.forgerock.opendj.config.server.ConfigurationChangeListener;
|
import org.forgerock.opendj.ldap.ByteSequence;
|
import org.forgerock.opendj.ldap.ByteString;
|
import org.forgerock.opendj.ldap.DN;
|
import org.forgerock.opendj.ldap.schema.AttributeType;
|
import org.forgerock.opendj.server.config.meta.BackendIndexCfgDefn.IndexType;
|
import org.forgerock.opendj.server.config.server.BackendIndexCfg;
|
import org.forgerock.opendj.server.config.server.PDBBackendCfg;
|
import org.forgerock.opendj.server.config.server.PluggableBackendCfg;
|
import org.mockito.ArgumentCaptor;
|
import org.opends.server.DirectoryServerTestCase;
|
import org.opends.server.TestCaseUtils;
|
import org.opends.server.backends.pdb.PDBStorage;
|
import org.opends.server.backends.pluggable.spi.AccessMode;
|
import org.opends.server.backends.pluggable.spi.Cursor;
|
import org.opends.server.backends.pluggable.spi.Importer;
|
import org.opends.server.backends.pluggable.spi.ReadOperation;
|
import org.opends.server.backends.pluggable.spi.Storage;
|
import org.opends.server.backends.pluggable.spi.StorageRuntimeException;
|
import org.opends.server.backends.pluggable.spi.StorageStatus;
|
import org.opends.server.backends.pluggable.spi.TreeName;
|
import org.opends.server.backends.pluggable.spi.UpdateFunction;
|
import org.opends.server.backends.pluggable.spi.WriteOperation;
|
import org.opends.server.backends.pluggable.spi.WriteableTransaction;
|
import org.opends.server.core.ServerContext;
|
import org.opends.server.types.BackupConfig;
|
import org.opends.server.types.BackupDirectory;
|
import org.opends.server.types.DirectoryException;
|
import org.opends.server.types.RestoreConfig;
|
import org.testng.annotations.AfterMethod;
|
import org.testng.annotations.BeforeClass;
|
import org.testng.annotations.Test;
|
|
import com.persistit.exception.RollbackException;
|
|
/**
|
* Tests that {@link RootContainer#open(AccessMode)} survives a replay of its {@link WriteOperation}.
|
* {@link Storage#write(WriteOperation)} may replay the operation after a transaction conflict, and
|
* {@code RootContainer.open} opens and registers the entry container of every base DN inside a
|
* single one of them, so every side effect that write performs must either be transactional or be
|
* idempotent - see OpenDJ issue #896.
|
* <p>
|
* The conflict is raised from inside the operation as the {@link RollbackException} PersistIt itself
|
* raises, so that the replay is driven by {@code PDBStorage.write}'s own retry loop rather than by a
|
* second call to it, as {@link ReplayedConfigChangeTest} does for the sibling path.
|
*/
|
@SuppressWarnings("javadoc")
|
@Test(groups = { "precommit", "pluggablebackend" }, sequential = true)
|
public class ReplayedOpenTest extends DirectoryServerTestCase
|
{
|
private static final String BACKEND_ID = "ReplayedOpenTest";
|
/** Opened and registered first, since the base DNs are opened in the order of the sorted set. */
|
private static final DN FIRST = DN.valueOf("dc=b896a,dc=com");
|
/** Opened while the first one is already registered, which is where a conflict reaches the bug. */
|
private static final DN SECOND = DN.valueOf("dc=b896b,dc=com");
|
|
private ServerContext serverContext;
|
private AttributeType cnType;
|
/** The configuration of the cn index, which the attribute index of every base DN registers with. */
|
private BackendIndexCfg indexCfg;
|
|
@BeforeClass
|
public void startServer() throws Exception
|
{
|
TestCaseUtils.startServer();
|
serverContext = TestCaseUtils.getServerContext();
|
cnType = serverContext.getSchema().getAttributeType("cn");
|
}
|
|
/**
|
* These tests open a backend which is designed to fail, and a failing one can leave a base DN
|
* behind in the server wide registry, where it would outlive the test and break the next one.
|
*/
|
@AfterMethod
|
public void deregisterLeftoverBaseDNs()
|
{
|
for (DN baseDN : new DN[] { FIRST, SECOND })
|
{
|
try
|
{
|
serverContext.getBackendConfigManager().deregisterBaseDN(baseDN);
|
}
|
catch (Exception alreadyGone)
|
{
|
// Which is what the test should have left behind.
|
}
|
}
|
}
|
|
/**
|
* The case the report is written from: the conflict is raised while the second base DN is being
|
* opened, so the replay meets the first one already registered by the attempt it replaces.
|
*/
|
@Test
|
public void openIsReplayableWhenTheTransactionConflictsWhileTheSecondBaseDNIsOpened() throws Exception
|
{
|
final ReplayingBackend backend = newBackend(newTreeSet(FIRST, SECOND));
|
boolean opened = false;
|
try
|
{
|
backend.storage.conflictAtTreesOf(SECOND, 1);
|
backend.openBackend();
|
opened = true;
|
|
assertThat(backend.storage.attempts()).isEqualTo(2);
|
final RootContainer rootContainer = backend.getRootContainer();
|
assertThat(rootContainer.getBaseDNs()).containsOnly(FIRST, SECOND);
|
// Each base DN is answered with a container of its own, rather than with the one an ancestor
|
// registered, and the trees of both are there to be read.
|
for (DN baseDN : new DN[] { FIRST, SECOND })
|
{
|
final EntryContainer ec = rootContainer.getEntryContainer(baseDN);
|
assertThat((Object) ec.getBaseDN()).isEqualTo(baseDN);
|
assertThat(rootContainer.getStorage().listTrees()).containsAll(treesOf(ec));
|
}
|
// What the attempt which was replaced opened is off the backend configuration, and the live
|
// pair is on it: SECOND of that attempt was given up by its own close(), raised from
|
// EntryContainer.open's own catch, and FIRST by the replay, which begins by unregistering and
|
// closing what the attempt it replaces had registered. Counting the removals would say neither
|
// which containers were given back nor that the adds balance: one of the two takes off a
|
// listener SECOND had never added, the conflict being raised at its id2entry, before
|
// EntryContainer.open registers anything.
|
assertThat(entryContainersRegisteredOn(backend.configuredWith))
|
.containsOnly(rootContainer.getEntryContainer(FIRST), rootContainer.getEntryContainer(SECOND));
|
// The same balance one level down: a container given back without the listener each of its
|
// indexes registered would satisfy the assertion above.
|
assertThat(indexesRegisteredOn(indexCfg))
|
.containsOnly(rootContainer.getEntryContainer(FIRST).getAttributeIndex(cnType),
|
rootContainer.getEntryContainer(SECOND).getAttributeIndex(cnType));
|
}
|
finally
|
{
|
close(backend, opened);
|
}
|
}
|
|
/**
|
* A conflict raised once the operation has run to completion, before the commit, replays an
|
* operation which ran to completion, so every base DN of the backend is registered by the
|
* attempt the replay replaces. One base DN would be enough to reach that, and
|
* {@code FailedBackendOpenTest.aReplayedOpenLeavesOneSetOfEntryContainers} drives the same
|
* conflict with one; two are used here so that the case differs from the one above only in where
|
* the conflict was raised, and so that the give-back has more than one container to walk.
|
*/
|
@Test
|
public void openIsReplayableWhenTheTransactionConflictsAtCommitTime() throws Exception
|
{
|
final ReplayingBackend backend = newBackend(newTreeSet(FIRST, SECOND));
|
boolean opened = false;
|
try
|
{
|
backend.storage.conflictAtCommit(1);
|
backend.openBackend();
|
opened = true;
|
|
assertThat(backend.storage.attempts()).isEqualTo(2);
|
final RootContainer rootContainer = backend.getRootContainer();
|
assertThat(rootContainer.getBaseDNs()).containsOnly(FIRST, SECOND);
|
// The two entry containers the rolled back attempt opened registered five configuration
|
// listeners each, and one more per index, which only their close() takes back, so the replay
|
// has to give both up before it opens another pair: what is left of the two attempts is the
|
// live pair alone.
|
assertThat(entryContainersRegisteredOn(backend.configuredWith))
|
.containsOnly(rootContainer.getEntryContainer(FIRST), rootContainer.getEntryContainer(SECOND));
|
assertThat(indexesRegisteredOn(indexCfg))
|
.containsOnly(rootContainer.getEntryContainer(FIRST).getAttributeIndex(cnType),
|
rootContainer.getEntryContainer(SECOND).getAttributeIndex(cnType));
|
}
|
finally
|
{
|
close(backend, opened);
|
}
|
}
|
|
/**
|
* A failure the storage engine does not replay ends the open, and {@code RootContainer}'s own
|
* give-back is the only one there is: {@code BackendImpl.newRootContainer} drops the root
|
* container without closing it, so what the attempt had opened by then - both entry containers,
|
* the root container's own listener and the storage - is handed back by
|
* {@code RootContainer.giveUpAfterFailedOpen} or not at all.
|
*/
|
@Test
|
public void anOpenWhichIsNotReplayedGivesUpTheEntryContainersItOpened() throws Exception
|
{
|
final ReplayingBackend backend = newBackend(newTreeSet(FIRST, SECOND));
|
try
|
{
|
backend.storage.failWithoutReplay();
|
|
try
|
{
|
backend.openBackend();
|
throw new AssertionError("the open was expected to fail");
|
}
|
catch (Exception expected)
|
{
|
// The failure itself is the caller's business; what it left behind is this test's.
|
}
|
|
// The failure is the one the engines do not replay, so the open ends at the first attempt:
|
// a replay would open a second pair of containers over the first.
|
assertThat(backend.storage.attempts()).isEqualTo(1);
|
// The two entry containers the attempt opened, and the root container's own listener, which
|
// goes with them since #993: nothing closes a root container which did not open.
|
verify(backend.configuredWith, times(3)).removePluggableChangeListener(any());
|
}
|
finally
|
{
|
close(backend, false);
|
}
|
}
|
|
private static Set<TreeName> treesOf(EntryContainer ec)
|
{
|
final Set<TreeName> names = new HashSet<>();
|
for (Tree tree : ec.listTrees())
|
{
|
names.add(tree.getName());
|
}
|
return names;
|
}
|
|
/**
|
* Closes a backend which opened; for one which did not, closes its storage a second time, in case
|
* the give-back of the failed open did not reach it - a no-op under {@code PDBStorage.close}'s
|
* {@code db != null} guard when it did, and a volume left open fails every following test with a
|
* {@code StorageInUseException} rather than with what actually broke.
|
*/
|
private static void close(ReplayingBackend backend, boolean opened)
|
{
|
if (opened)
|
{
|
backend.finalizeBackend();
|
}
|
else
|
{
|
backend.storage.close();
|
}
|
}
|
|
/**
|
* The entry containers still registered as listeners of the backend configuration, by identity.
|
* <p>
|
* A replayed open registers a container per base DN twice and gives the first set back, so what
|
* the case is about is which containers are left rather than how many calls were made: a removal
|
* says nothing on its own - an entry container whose open failed before it had registered
|
* anything is taken off all the same, by the {@code close()} of {@code EntryContainer.open}'s own
|
* catch.
|
*/
|
private static List<EntryContainer> entryContainersRegisteredOn(PluggableBackendCfg cfg)
|
{
|
final ArgumentCaptor<ConfigurationChangeListener<PluggableBackendCfg>> added =
|
captorFor(ConfigurationChangeListener.class);
|
verify(cfg, atLeast(0)).addPluggableChangeListener(added.capture());
|
final ArgumentCaptor<ConfigurationChangeListener<PluggableBackendCfg>> removed =
|
captorFor(ConfigurationChangeListener.class);
|
verify(cfg, atLeast(0)).removePluggableChangeListener(removed.capture());
|
return balanceOf(EntryContainer.class, added.getAllValues(), removed.getAllValues());
|
}
|
|
/** The attribute indexes still registered as listeners of an index configuration, by identity. */
|
private static List<AttributeIndex> indexesRegisteredOn(BackendIndexCfg cfg)
|
{
|
final ArgumentCaptor<ConfigurationChangeListener<BackendIndexCfg>> added =
|
captorFor(ConfigurationChangeListener.class);
|
verify(cfg, atLeast(0)).addChangeListener(added.capture());
|
final ArgumentCaptor<ConfigurationChangeListener<BackendIndexCfg>> removed =
|
captorFor(ConfigurationChangeListener.class);
|
verify(cfg, atLeast(0)).removeChangeListener(removed.capture());
|
return balanceOf(AttributeIndex.class, added.getAllValues(), removed.getAllValues());
|
}
|
|
/**
|
* What was added to a configuration and not taken off it again, of one kind of listener: the root
|
* container and the two configuration managers of every entry container listen to the backend
|
* configuration as well, and neither is what these cases are about. Identity, since none of these
|
* classes overrides {@code equals()}.
|
*/
|
private static <T> List<T> balanceOf(Class<T> kind, List<?> added, List<?> removed)
|
{
|
final List<T> registered = new ArrayList<>();
|
for (Object listener : added)
|
{
|
if (kind.isInstance(listener))
|
{
|
registered.add(kind.cast(listener));
|
}
|
}
|
for (Object listener : removed)
|
{
|
if (kind.isInstance(listener))
|
{
|
registered.remove(kind.cast(listener));
|
}
|
}
|
return registered;
|
}
|
|
@SuppressWarnings({ "unchecked", "rawtypes" })
|
private static <T> ArgumentCaptor<T> captorFor(Class<?> listenerClass)
|
{
|
return (ArgumentCaptor<T>) ArgumentCaptor.forClass((Class) listenerClass);
|
}
|
|
private ReplayingBackend newBackend(SortedSet<DN> baseDNs) throws Exception
|
{
|
final ReplayingBackend backend = new ReplayingBackend();
|
backend.setBackendID(BACKEND_ID);
|
backend.configuredWith = backendCfg(baseDNs);
|
backend.configureBackend(backend.configuredWith, serverContext);
|
// Start from a pristine on-disk state so that a previous run cannot mask the defect.
|
backend.storage.removeStorageFiles();
|
return backend;
|
}
|
|
private PDBBackendCfg backendCfg(SortedSet<DN> baseDNs) throws ConfigException
|
{
|
final PDBBackendCfg cfg = mockCfg(PDBBackendCfg.class);
|
when(cfg.dn()).thenReturn(DN.valueOf("ds-cfg-backend-id=" + BACKEND_ID + ",cn=Backends,cn=config"));
|
when(cfg.getBackendId()).thenReturn(BACKEND_ID);
|
when(cfg.getDBDirectory()).thenReturn(BACKEND_ID);
|
when(cfg.getDBDirectoryPermissions()).thenReturn("755");
|
when(cfg.getDBCacheSize()).thenReturn(0L);
|
when(cfg.getDBCachePercent()).thenReturn(20);
|
when(cfg.getBaseDN()).thenReturn(baseDNs);
|
when(cfg.listBackendIndexes()).thenReturn(new String[] { "cn" });
|
when(cfg.listBackendVLVIndexes()).thenReturn(new String[0]);
|
|
indexCfg = mock(BackendIndexCfg.class);
|
when(indexCfg.getIndexType()).thenReturn(newTreeSet(IndexType.EQUALITY));
|
when(indexCfg.getAttribute()).thenReturn(cnType);
|
when(indexCfg.getIndexEntryLimit()).thenReturn(4000);
|
when(indexCfg.getSubstringLength()).thenReturn(6);
|
when(cfg.getBackendIndex("cn")).thenReturn(indexCfg);
|
return cfg;
|
}
|
|
/** A backend whose storage makes the next write operation conflict, and so be replayed. */
|
private static final class ReplayingBackend extends BackendImpl<PDBBackendCfg>
|
{
|
private ReplayingStorage storage;
|
/** The configuration the entry containers register their listeners with. */
|
private PDBBackendCfg configuredWith;
|
|
@Override
|
protected Storage configureStorage(PDBBackendCfg cfg, ServerContext serverContext) throws ConfigException
|
{
|
storage = new ReplayingStorage(new PDBStorage(cfg, serverContext));
|
return storage;
|
}
|
}
|
|
/** A failure which no storage engine replays, unlike {@link RollbackException}. */
|
private static final class UnreplayableFailure extends Exception
|
{
|
private static final long serialVersionUID = 1L;
|
}
|
|
/**
|
* Decorates a {@link Storage} so that the next {@link Storage#write(WriteOperation)} conflicts a
|
* given number of times before it is let through. The conflict is raised from within the single
|
* {@code write} the delegate is asked for, so the delegate's own retry loop performs the replay.
|
*/
|
private static final class ReplayingStorage implements Storage
|
{
|
/** Where the conflict is raised, which decides how much of the operation has run. */
|
private enum ConflictPoint
|
{
|
/** As the first tree of a given base DN is opened, with the base DNs before it registered. */
|
TREES_OF_BASE_DN,
|
/** Once the operation has run to completion, before the commit. */
|
COMMIT,
|
/** Once the operation has run to completion, as a failure which is not replayed at all. */
|
NO_REPLAY
|
}
|
|
private final Storage delegate;
|
private ConflictPoint conflictPoint;
|
private String conflictingPrefix;
|
private int conflictsLeft;
|
private int attempts;
|
|
ReplayingStorage(Storage delegate)
|
{
|
this.delegate = delegate;
|
}
|
|
void conflictAtTreesOf(DN baseDN, int conflicts)
|
{
|
conflictingPrefix = baseDN.toNormalizedUrlSafeString();
|
arm(ConflictPoint.TREES_OF_BASE_DN, conflicts);
|
}
|
|
void conflictAtCommit(int conflicts)
|
{
|
arm(ConflictPoint.COMMIT, conflicts);
|
}
|
|
void failWithoutReplay()
|
{
|
arm(ConflictPoint.NO_REPLAY, 1);
|
}
|
|
private void arm(ConflictPoint where, int conflicts)
|
{
|
conflictPoint = where;
|
conflictsLeft = conflicts;
|
attempts = 0;
|
}
|
|
/** How many times the armed operation was run, the first attempt included. */
|
int attempts()
|
{
|
return attempts;
|
}
|
|
@Override
|
public void write(final WriteOperation writeOperation) throws Exception
|
{
|
final ConflictPoint armed = conflictPoint;
|
if (armed == null)
|
{
|
delegate.write(writeOperation);
|
return;
|
}
|
conflictPoint = null;
|
// A single call, so that the replay is the delegate's own and keeps whatever the delegate
|
// holds for the duration of a write, rather than starting afresh as a second call would.
|
delegate.write(new WriteOperation()
|
{
|
@Override
|
public void run(WriteableTransaction txn) throws Exception
|
{
|
attempts++;
|
if (conflictsLeft-- <= 0)
|
{
|
writeOperation.run(txn);
|
return;
|
}
|
if (armed == ConflictPoint.TREES_OF_BASE_DN)
|
{
|
writeOperation.run(new ConflictingAtTreesOf(txn, conflictingPrefix));
|
return;
|
}
|
writeOperation.run(txn);
|
if (armed == ConflictPoint.NO_REPLAY)
|
{
|
throw new UnreplayableFailure();
|
}
|
throw new RollbackException();
|
}
|
});
|
}
|
|
@Override
|
public Importer startImport() throws ConfigException
|
{
|
return delegate.startImport();
|
}
|
|
@Override
|
public void open(AccessMode accessMode) throws Exception
|
{
|
delegate.open(accessMode);
|
}
|
|
@Override
|
public <T> T read(ReadOperation<T> readOperation) throws Exception
|
{
|
return delegate.read(readOperation);
|
}
|
|
@Override
|
public void removeStorageFiles()
|
{
|
delegate.removeStorageFiles();
|
}
|
|
@Override
|
public StorageStatus getStorageStatus()
|
{
|
return delegate.getStorageStatus();
|
}
|
|
@Override
|
public boolean supportsBackupAndRestore()
|
{
|
return delegate.supportsBackupAndRestore();
|
}
|
|
@Override
|
public void createBackup(BackupConfig backupConfig) throws DirectoryException
|
{
|
delegate.createBackup(backupConfig);
|
}
|
|
@Override
|
public void removeBackup(BackupDirectory backupDirectory, String backupID) throws DirectoryException
|
{
|
delegate.removeBackup(backupDirectory, backupID);
|
}
|
|
@Override
|
public void restoreBackup(RestoreConfig restoreConfig) throws DirectoryException
|
{
|
delegate.restoreBackup(restoreConfig);
|
}
|
|
@Override
|
public Set<TreeName> listTrees()
|
{
|
return delegate.listTrees();
|
}
|
|
@Override
|
public void close()
|
{
|
delegate.close();
|
}
|
}
|
|
/**
|
* A transaction which conflicts as the first tree of one base DN is opened, and delegates
|
* everything the operation did before that. This is where an entry container which is being opened
|
* meets a write-write conflict: PDBStorage wraps the {@link RollbackException} PersistIt raises at
|
* the store into a {@link StorageRuntimeException}, which is what {@code EntryContainer.open}'s own
|
* catch unwinds on, so the base DNs opened before this one have been registered and the one being
|
* opened has not.
|
*/
|
private static final class ConflictingAtTreesOf implements WriteableTransaction
|
{
|
private final WriteableTransaction delegate;
|
private final String conflictingPrefix;
|
|
ConflictingAtTreesOf(WriteableTransaction delegate, String conflictingPrefix)
|
{
|
this.delegate = delegate;
|
this.conflictingPrefix = conflictingPrefix;
|
}
|
|
@Override
|
public void openTree(TreeName name, boolean createOnDemand)
|
{
|
if (conflictingPrefix.equals(name.getBaseDN()))
|
{
|
// What PDBStorage delivers: the engine's RollbackException wrapped, which
|
// EntryContainer.open's catch(StorageRuntimeException) unwinds on.
|
throw new StorageRuntimeException(new RollbackException());
|
}
|
delegate.openTree(name, createOnDemand);
|
}
|
|
@Override
|
public void deleteTree(TreeName name)
|
{
|
delegate.deleteTree(name);
|
}
|
|
@Override
|
public void put(TreeName treeName, ByteSequence key, ByteSequence value)
|
{
|
delegate.put(treeName, key, value);
|
}
|
|
@Override
|
public boolean update(TreeName treeName, ByteSequence key, UpdateFunction f)
|
{
|
return delegate.update(treeName, key, f);
|
}
|
|
@Override
|
public boolean delete(TreeName treeName, ByteSequence key)
|
{
|
return delegate.delete(treeName, key);
|
}
|
|
@Override
|
public ByteString read(TreeName treeName, ByteSequence key)
|
{
|
return delegate.read(treeName, key);
|
}
|
|
@Override
|
public Cursor<ByteString, ByteString> openCursor(TreeName treeName)
|
{
|
return delegate.openCursor(treeName);
|
}
|
|
@Override
|
public long getRecordCount(TreeName treeName)
|
{
|
return delegate.getRecordCount(treeName);
|
}
|
|
@Override
|
public boolean treeExists(TreeName treeName)
|
{
|
return delegate.treeExists(treeName);
|
}
|
}
|
}
|
