From 2c22d96fcf2a6c9323d331e1197337fec239bc88 Mon Sep 17 00:00:00 2001
From: Maxim Thomas <maxim.thomas@gmail.com>
Date: Wed, 09 Sep 2026 07:21:53 +0000
Subject: [PATCH] [#907] Change the base DNs of a pluggable backend outside the write the storage replays (#914)
---
opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/ReplayedConfigChangeTest.java | 967 ++++++++++++++++++++++++++++++++++++++++++
opendj-server-legacy/src/messages/org/opends/messages/backend.properties | 18
opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/EntryContainer.java | 7
opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/BackendImpl.java | 352 +++++++++++++--
4 files changed, 1,293 insertions(+), 51 deletions(-)
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/BackendImpl.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/BackendImpl.java
index 03cd930..d837419 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/BackendImpl.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/BackendImpl.java
@@ -18,11 +18,13 @@
package org.opends.server.backends.pluggable;
import static org.forgerock.util.Reject.*;
+import static org.forgerock.util.Utils.closeSilently;
import static org.opends.messages.BackendMessages.*;
import static org.opends.server.util.ServerConstants.*;
import static org.opends.server.util.StaticUtils.*;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
@@ -50,9 +52,11 @@
import org.opends.server.backends.pluggable.spi.Storage;
import org.opends.server.backends.pluggable.spi.StorageInUseException;
import org.opends.server.backends.pluggable.spi.StorageRuntimeException;
+import org.opends.server.backends.pluggable.spi.TreeName;
import org.opends.server.backends.pluggable.spi.WriteOperation;
import org.opends.server.backends.pluggable.spi.WriteableTransaction;
import org.opends.server.core.AddOperation;
+import org.opends.server.core.BackendConfigManager;
import org.opends.server.core.DeleteOperation;
import org.opends.server.core.DirectoryServer;
import org.opends.server.core.ModifyDNOperation;
@@ -845,83 +849,333 @@
return true;
}
+ /**
+ * {@inheritDoc}
+ * <p>
+ * {@link Storage#write(WriteOperation)} replays its operation after a transaction conflict, so
+ * the operation below is confined to work a rollback undoes: the trees are deleted and opened
+ * there, while the registries, which no rollback reaches, are updated once the write has
+ * committed. Getting this the wrong way round leaves the change half applied, and its replay
+ * reports the missing half rather than the conflict that caused it.
+ * <p>
+ * What makes the operation replayable is that the base DNs to remove and to add are worked out
+ * once, ahead of the write, so that no attempt can see different work to do than the attempt it
+ * is replacing.
+ */
@Override
public ConfigChangeResult applyConfigurationChange(final PluggableBackendCfg newCfg)
{
final ConfigChangeResult ccr = new ConfigChangeResult();
- try
+ // Read once: importLDIF, rebuildBackend, exportLDIF and verifyBackend all assign this field
+ // and null it out again, and this method now goes on using it past the commit.
+ final RootContainer rc = rootContainer;
+ if (rc == null)
{
- if(rootContainer != null)
+ return ccr;
+ }
+
+ final SortedSet<DN> newBaseDNs = newCfg.getBaseDN();
+ // Ask the root container what this backend holds rather than the configuration it was last
+ // given: a base DN which an earlier, failed change left behind is work to do, and a
+ // configuration which was never applied is not. RootContainer.getBaseDNs() is a live view of
+ // the registered containers, so take a copy of it before anything registers one.
+ final Set<DN> currentBaseDNs = new HashSet<>(rc.getBaseDNs());
+ final List<EntryContainer> deleted = new ArrayList<>();
+ for (DN baseDN : currentBaseDNs)
+ {
+ if (!newBaseDNs.contains(baseDN))
{
- rootContainer.getStorage().write(new WriteOperation()
+ final EntryContainer ec = rc.getEntryContainer(baseDN);
+ // Answered exactly, never with an ancestor's container: getEntryContainer walks up the DN
+ // until it finds one, which is how an entry is routed to the base DN above it, and one
+ // backend holds hierarchically related base DNs whenever a registry which refused one left
+ // its container behind. Deleting the trees an ancestor answered with is deleting the trees
+ // of a base DN this backend is still serving.
+ if (ec == null || !baseDN.equals(ec.getBaseDN()))
{
- @Override
- public void run(WriteableTransaction txn) throws Exception
- {
- SortedSet<DN> newBaseDNs = newCfg.getBaseDN();
-
- // Check for changes to the base DNs.
- removeDeletedBaseDNs(newBaseDNs, txn);
- if (!createNewBaseDNs(newBaseDNs, ccr, txn))
- {
- return;
- }
-
- baseDNs = new HashSet<>(newBaseDNs);
-
- // Put the new configuration in place.
- cfg = newCfg;
- }
- });
+ // Unregistered since the copy above was taken, which is what closing the root container
+ // leaves behind: importLDIF, rebuildBackend and exportLDIF all do that, as does a backend
+ // being disabled. There is nothing to delete and nothing to say about the rest of the
+ // change, so none of it is attempted - and a result is returned rather than an exception,
+ // which is what the administration framework is owed whatever happens.
+ ccr.setResultCode(serverContext.getCoreConfigManager().getServerErrorResultCode());
+ ccr.addMessage(ERR_BACKEND_BASEDN_NO_LONGER_HELD.get(getBackendID(), baseDN));
+ return ccr;
+ }
+ deleted.add(ec);
}
}
- catch (Exception e)
+ final List<DN> added = new ArrayList<>();
+ for (DN baseDN : newBaseDNs)
{
- ccr.setResultCode(serverContext.getCoreConfigManager().getServerErrorResultCode());
- ccr.addMessage(LocalizableMessage.raw(stackTraceToSingleLineString(e)));
+ if (!currentBaseDNs.contains(baseDN))
+ {
+ added.add(baseDN);
+ }
+ }
+ if (deleted.isEmpty() && added.isEmpty())
+ {
+ // The common case - index-entry-limit, db-cache-percent, preload-time-limit and the rest,
+ // which the entry containers apply through their own listeners. There is no storage work to
+ // do, so no transaction is opened to commit nothing.
+ baseDNs = new HashSet<>(newBaseDNs);
+ cfg = newCfg;
+ return ccr;
+ }
+ // Opened by the write operation, registered only once it has committed.
+ final List<EntryContainer> created = new ArrayList<>();
+ try
+ {
+ try
+ {
+ changeBaseDNTrees(rc, deleted, added, created);
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+
+ ccr.setResultCode(serverContext.getCoreConfigManager().getServerErrorResultCode());
+ // The failure alone never says which base DNs the change was about, so name them.
+ //
+ // Only persistit and je roll the whole write back, leaving nothing at all applied and
+ // neither registry to touch. The jdbc backend does not, on any of its engines: its
+ // commitStatement() issues the statement and commits it, and commitsBeforeDdl() decides
+ // only which side of the statement the attempt stops being replayable on, never whether a
+ // completed "create table" or "drop table" survives the rollback of the write around it.
+ // Neither does cassandra, which has no transaction to roll back.
+ // giveUpBaseDNsWhoseTreesAreGone below reads what is actually left rather than trusting
+ // either answer.
+ ccr.addMessage(ERR_BACKEND_CANNOT_CHANGE_BASEDNS.get(
+ getBackendID(), baseDNsOf(deleted), added, stackTraceToSingleLineString(e)));
+ // Read before the entry containers are closed: what a container holds is what says which
+ // trees belong to it.
+ giveUpBaseDNsWhoseTreesAreGone(rc, deleted, ccr);
+ closeSilently(created);
+ return ccr;
+ }
+
+ // The change is durable from here on, so every base DN is seen through even if one fails.
+ for (EntryContainer ec : deleted)
+ {
+ deregisterDeletedBaseDN(rc, ec, ccr);
+ }
+ registerNewBaseDNs(rc, created, ccr);
+
+ // Put the new configuration in place.
+ cfg = newCfg;
+ }
+ finally
+ {
+ // What the root container ended up holding, not what was asked for: a base DN whose
+ // registration failed is not one this backend serves, and getBaseDNs() is what the monitors,
+ // isIndexed() and closeBackend() are answered from. Taken on the way out of every path which
+ // reached the write, the failed ones included, so that the two never disagree. The change
+ // which had no storage work to do sets it from the new configuration above; the one which
+ // found a base DN this backend no longer holds leaves it alone, since the root container it
+ // would be read from is being closed underneath it.
+ baseDNs = new HashSet<>(rc.getBaseDNs());
}
return ccr;
}
- private void removeDeletedBaseDNs(SortedSet<DN> newBaseDNs, WriteableTransaction txn) throws DirectoryException
+ /**
+ * Deletes the trees of the base DNs being removed and opens the ones being added, as the single
+ * write operation a storage engine may replay.
+ * <p>
+ * The trees of a removed base DN are deleted while it is still registered, so its entry container
+ * is held exclusively for as long as the write runs, retries included, as
+ * {@link RootContainer#close()}, EntryContainer's index delete listener and AttributeIndex all do.
+ * That keeps out the operations which arrive during that window; an operation which had taken hold
+ * of the container before the lock still ends up in a closed one once it is released, as it did
+ * before this ordering.
+ * <p>
+ * The locks are given up with the write and are never held into the registry work which follows it.
+ * {@link BackendConfigManager} guards its registry with a single lock which the server already
+ * takes in the opposite order - {@code shutdownLocalBackends}, a backend being disabled and
+ * {@code applyConfigurationDelete} all hold it while finalizing a backend, which closes its root
+ * container and locks every entry container in turn. Holding the container lock into
+ * {@code deregisterBaseDN} would deadlock a base DN change against a shutdown, with no timeout on
+ * either side.
+ */
+ private void changeBaseDNTrees(final RootContainer rc, final List<EntryContainer> deleted,
+ final List<DN> added, final List<EntryContainer> created) throws Exception
{
- for (DN baseDN : cfg.getBaseDN())
+ for (EntryContainer ec : deleted)
{
- if (!newBaseDNs.contains(baseDN))
+ // Taken outside the try, because EntryContainer.lock() has no throwing path - the write side
+ // of a ReentrantReadWriteLock, then a drain which swallows the interrupt - so every one of
+ // them is held by the time it is entered, and unlocking what was never locked cannot happen.
+ ec.lock();
+ }
+ try
+ {
+ rc.getStorage().write(new WriteOperation()
{
- // The base DN was deleted.
- serverContext.getBackendConfigManager().deregisterBaseDN(baseDN);
- EntryContainer ec = rootContainer.unregisterEntryContainer(baseDN);
- ec.close();
- ec.delete(txn);
+ @Override
+ public void run(WriteableTransaction txn) throws Exception
+ {
+ // Give up what a previous, rolled back attempt had opened: its trees are gone, and its
+ // entry containers still hold the configuration listeners they registered.
+ closeSilently(created);
+ created.clear();
+
+ // Opening the added base DNs comes first, so that the failure this operation is most
+ // likely to meet is met while everything is still there to roll back to. Once a tree
+ // has been deleted, a storage engine which does not undo that has nothing to give
+ // back.
+ for (DN baseDN : added)
+ {
+ created.add(rc.openEntryContainer(baseDN, txn, AccessMode.READ_WRITE));
+ }
+ for (EntryContainer ec : deleted)
+ {
+ ec.delete(txn);
+ }
+ }
+ });
+ }
+ finally
+ {
+ for (EntryContainer ec : deleted)
+ {
+ ec.unlock();
}
}
}
- private boolean createNewBaseDNs(Set<DN> newBaseDNs, ConfigChangeResult ccr, WriteableTransaction txn)
+ /**
+ * Gives up the base DNs whose trees the failed write took with it. There is anything to give up
+ * only on an engine which does not roll the write back whole: mysql and oracle commit their DDL of
+ * their own accord, cassandra has no transaction at all, and the jdbc backend commits after each
+ * statement on every engine it supports. A base DN kept registered without its trees answers every
+ * operation with a storage error, where its removal was meant to leave a plain "no such entry";
+ * one whose trees the rollback put back is left exactly as it was.
+ * <p>
+ * The trees the same write created for a base DN which is not being added after all are left where
+ * they are, and this is the only place which could have taken them back. The configuration naming
+ * that base DN was stored before this listener was called - {@code
+ * ConfigurationHandler.replaceEntry} writes the entry, and only then notifies its change listeners
+ * - and the failure does not take it back, so {@link RootContainer#open} opens that base DN again
+ * from it the next time this backend is opened, adopting the trees which survived and creating the
+ * ones which did not. Deleting them here would take away the trees of a base DN the stored
+ * configuration still asks this backend to serve, and would reach only the base DNs whose opening
+ * succeeded anyway: one which failed while being opened never became an entry container, and
+ * nothing but its own trees names them.
+ */
+ private void giveUpBaseDNsWhoseTreesAreGone(RootContainer rc, List<EntryContainer> deleted, ConfigChangeResult ccr)
{
- for (DN baseDN : newBaseDNs)
+ if (deleted.isEmpty())
{
- if (!rootContainer.getBaseDNs().contains(baseDN))
+ return;
+ }
+ final Set<TreeName> storedTrees;
+ try
+ {
+ storedTrees = rc.getStorage().listTrees();
+ }
+ catch (Exception e)
+ {
+ // Nothing can be said about what survived, so nothing is given up on the strength of it - and
+ // that is the case where the failure itself says least about what this backend is left
+ // serving, so it is said outright rather than left to a bare admin action.
+ logger.traceException(e);
+ ccr.setAdminActionRequired(true);
+ ccr.addMessage(ERR_BACKEND_CANNOT_LIST_TREES_AFTER_BASEDN_CHANGE.get(
+ getBackendID(), stackTraceToSingleLineString(e)));
+ return;
+ }
+ for (EntryContainer ec : deleted)
+ {
+ if (!storedTrees.containsAll(treeNamesOf(ec)))
{
- try
- {
- // The base DN was added.
- EntryContainer ec = rootContainer.openEntryContainer(baseDN, txn, AccessMode.READ_WRITE);
- rootContainer.registerEntryContainer(baseDN, ec);
- serverContext.getBackendConfigManager().registerBaseDN(baseDN, this, false);
- }
- catch (Exception e)
- {
- logger.traceException(e);
+ ccr.setAdminActionRequired(true);
+ deregisterDeletedBaseDN(rc, ec, ccr);
+ }
+ }
+ }
- ccr.setResultCode(serverContext.getCoreConfigManager().getServerErrorResultCode());
- ccr.addMessage(ERR_BACKEND_CANNOT_REGISTER_BASEDN.get(baseDN, e));
- return false;
+ private static Set<TreeName> treeNamesOf(EntryContainer ec)
+ {
+ final Set<TreeName> names = new HashSet<>();
+ for (Tree tree : ec.listTrees())
+ {
+ names.add(tree.getName());
+ }
+ return names;
+ }
+
+ private void deregisterDeletedBaseDN(RootContainer rc, EntryContainer ec, ConfigChangeResult ccr)
+ {
+ final DN baseDN = ec.getBaseDN();
+ final BackendConfigManager backendConfigManager = serverContext.getBackendConfigManager();
+ try
+ {
+ backendConfigManager.deregisterBaseDN(baseDN);
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+
+ if (backendConfigManager.getLocalBackendWithBaseDN(baseDN) == this)
+ {
+ // deregisterBaseDN puts its new registry in place only once it has succeeded, so this base
+ // DN is still routed here. Leave the entry container registered: closeBackend() reclaims a
+ // base DN through rootContainer.getBaseDNs(), and one taken out of there would stay claimed
+ // by a backend which no longer holds it until the server is restarted. That is the opposite
+ // of what deregisterBaseDNsWhoseTreesAreGone does, and for the opposite reason: there the
+ // registry has already stopped routing to the base DN, so keeping the container only leaves
+ // a storage error where a "no such entry" was meant to be, while here the registry is still
+ // routing to it and dropping the container is what would leave that error behind.
+ ccr.setResultCode(serverContext.getCoreConfigManager().getServerErrorResultCode());
+ ccr.setAdminActionRequired(true);
+ ccr.addMessage(ERR_BACKEND_CANNOT_DEREGISTER_BASEDN.get(baseDN, stackTraceToSingleLineString(e)));
+ return;
+ }
+ // It is not registered here, which is what an earlier change whose registerBaseDN failed
+ // leaves behind. Nothing routes to it, so there is nothing to hold on to.
+ }
+ rc.unregisterEntryContainer(baseDN);
+ closeSilently(ec);
+ }
+
+ private void registerNewBaseDNs(RootContainer rc, List<EntryContainer> created, ConfigChangeResult ccr)
+ {
+ for (EntryContainer ec : created)
+ {
+ final DN baseDN = ec.getBaseDN();
+ boolean registered = false;
+ try
+ {
+ rc.registerEntryContainer(baseDN, ec);
+ registered = true;
+ serverContext.getBackendConfigManager().registerBaseDN(baseDN, this, false);
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+
+ ccr.setResultCode(serverContext.getCoreConfigManager().getServerErrorResultCode());
+ ccr.setAdminActionRequired(true);
+ ccr.addMessage(ERR_BACKEND_CANNOT_REGISTER_BASEDN.get(baseDN, stackTraceToSingleLineString(e)));
+ if (!registered)
+ {
+ // Nothing else can reclaim it: closeBackend() and RootContainer.close() both work from
+ // the registered containers, and this one keeps the configuration listeners its
+ // constructor registered for as long as it is alive.
+ closeSilently(ec);
}
}
}
- return true;
+ }
+
+ private static List<DN> baseDNsOf(List<EntryContainer> entryContainers)
+ {
+ final List<DN> baseDNs = new ArrayList<>(entryContainers.size());
+ for (EntryContainer ec : entryContainers)
+ {
+ baseDNs.add(ec.getBaseDN());
+ }
+ return baseDNs;
}
/**
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/EntryContainer.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/EntryContainer.java
index a7ae259..917b230 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/EntryContainer.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/EntryContainer.java
@@ -2423,8 +2423,11 @@
}
/**
- * Delete this entry container from disk. The entry container should be
- * closed before calling this method.
+ * Deletes this entry container from disk, that is, every tree {@link #listTrees()} enumerates.
+ * The entry container may be open or closed: the trees are taken from the attribute and VLV index
+ * maps, which {@link #close()} closes the indexes of but leaves populated, so the same set is
+ * deleted either way. A {@code close()} which cleared those maps would turn a call made after it
+ * into a partial deletion, silently. Either way the container is not to be used afterwards.
*
* @param txn a non null transaction
* @throws StorageRuntimeException If an error occurs while removing the entry container.
diff --git a/opendj-server-legacy/src/messages/org/opends/messages/backend.properties b/opendj-server-legacy/src/messages/org/opends/messages/backend.properties
index eebfe84..dbf5c55 100644
--- a/opendj-server-legacy/src/messages/org/opends/messages/backend.properties
+++ b/opendj-server-legacy/src/messages/org/opends/messages/backend.properties
@@ -1114,3 +1114,21 @@
ERR_COMPSCHEMA_CANNOT_MIGRATE_619=The compressed schema definitions of backend '%s' could not be migrated from \
the shared tree '%s' to '%s': %s. The backend cannot be opened, because its entries were encoded against the \
definitions that were not migrated and would decode as the wrong attributes
+ERR_BACKEND_CANNOT_DEREGISTER_BASEDN_620=An error occurred while attempting to deregister base DN %s \
+ from the Directory Server: %s
+ERR_BACKEND_CANNOT_CHANGE_BASEDNS_621=The base DNs of backend %s could not be changed (to remove: %s, \
+ to add: %s): %s. A storage engine which rolls the whole write back leaves the backend exactly as it \
+ was; on one which does not, the base DNs whose trees are gone have been given up. The base DNs being \
+ added are not being served, but the configuration which has been stored still names them, so the next \
+ time this backend is opened it opens them from that configuration, keeping whatever trees the failed \
+ change created for them and creating the ones it did not. A base DN being removed of which only some \
+ trees survived is given up with those trees still in the storage, where nothing names them afterwards; \
+ removing them means re-creating the backend
+ERR_BACKEND_CANNOT_LIST_TREES_AFTER_BASEDN_CHANGE_622=The base DN change of backend %s failed, and the \
+ trees which survived it could not be listed: %s. No base DN has been given up on the strength of that, \
+ so this backend may still be serving one whose trees are gone, which answers every operation against it \
+ with a storage error
+ERR_BACKEND_BASEDN_NO_LONGER_HELD_623=The base DNs of backend %s could not be changed: base DN %s is no \
+ longer one this backend holds, which is what closing its root container leaves behind - an LDIF import, \
+ an index rebuild, an LDIF export and the backend being disabled all do that. Nothing has been changed; \
+ submit the change again once that has finished
diff --git a/opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/ReplayedConfigChangeTest.java b/opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/ReplayedConfigChangeTest.java
new file mode 100644
index 0000000..19c1aaa
--- /dev/null
+++ b/opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/ReplayedConfigChangeTest.java
@@ -0,0 +1,967 @@
+/*
+ * The contents of this file are subject to the terms of the Common Development and
+ * Distribution License (the License). You may not use this file except in compliance with the
+ * License.
+ *
+ * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
+ * specific language governing permission and limitations under the License.
+ *
+ * When distributing Covered Software, include this CDDL Header Notice in each file and include
+ * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
+ * Header, with the fields enclosed by brackets [] replaced by your own identifying
+ * information: "Portions copyright [year] [name of copyright owner]".
+ *
+ * Copyright 2026 3A Systems, LLC.
+ */
+package org.opends.server.backends.pluggable;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.opends.messages.BackendMessages.ERR_BACKEND_BASEDN_NO_LONGER_HELD;
+import static org.opends.messages.BackendMessages.ERR_BACKEND_CANNOT_LIST_TREES_AFTER_BASEDN_CHANGE;
+import static org.opends.messages.BackendMessages.ERR_BACKEND_CANNOT_REGISTER_BASEDN;
+import static org.forgerock.opendj.config.ConfigurationMock.mockCfg;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.opends.server.backends.pluggable.State.IndexFlag.TRUSTED;
+import static org.opends.server.backends.pluggable.SuffixContainer.STATE_INDEX_NAME;
+import static org.opends.server.util.CollectionUtils.newTreeSet;
+import static org.forgerock.util.Utils.closeSilently;
+
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.forgerock.i18n.LocalizableMessage;
+import org.forgerock.opendj.config.server.ConfigChangeResult;
+import org.forgerock.opendj.config.server.ConfigException;
+import org.forgerock.opendj.ldap.ByteSequence;
+import org.forgerock.opendj.ldap.ByteString;
+import org.forgerock.opendj.ldap.DN;
+import org.forgerock.opendj.ldap.ResultCode;
+import org.forgerock.opendj.ldap.schema.AttributeType;
+import org.forgerock.opendj.server.config.meta.BackendIndexCfgDefn.IndexType;
+import org.forgerock.opendj.server.config.server.BackendIndexCfg;
+import org.forgerock.opendj.server.config.server.PDBBackendCfg;
+import org.opends.server.DirectoryServerTestCase;
+import org.opends.server.TestCaseUtils;
+import org.opends.server.backends.pdb.PDBStorage;
+import org.opends.server.backends.pluggable.State.IndexFlag;
+import org.opends.server.backends.pluggable.spi.AccessMode;
+import org.opends.server.backends.pluggable.spi.Cursor;
+import org.opends.server.backends.pluggable.spi.Importer;
+import org.opends.server.backends.pluggable.spi.ReadOperation;
+import org.opends.server.backends.pluggable.spi.Storage;
+import org.opends.server.backends.pluggable.spi.StorageRuntimeException;
+import org.opends.server.backends.pluggable.spi.StorageStatus;
+import org.opends.server.backends.pluggable.spi.TreeName;
+import org.opends.server.backends.pluggable.spi.UpdateFunction;
+import org.opends.server.backends.pluggable.spi.WriteOperation;
+import org.opends.server.backends.pluggable.spi.WriteableTransaction;
+import org.opends.server.core.ServerContext;
+import org.opends.server.types.BackupConfig;
+import org.opends.server.types.BackupDirectory;
+import org.opends.server.types.DirectoryException;
+import org.opends.server.types.RestoreConfig;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import com.persistit.exception.RollbackException;
+
+/**
+ * Tests that {@link BackendImpl#applyConfigurationChange} survives a replay of its
+ * {@link WriteOperation}. {@link Storage#write(WriteOperation)} may replay the operation after a
+ * transaction conflict, so every side effect it performs must either be transactional or be
+ * idempotent - see OpenDJ issue #907.
+ * <p>
+ * The conflict is raised from inside the operation as the {@link RollbackException} PersistIt
+ * itself raises, so that the replay is driven by {@code PDBStorage.write}'s own retry loop rather
+ * than by a second call to it. That loop keeps one storage implementation - and with it its cache
+ * of PersistIt exchanges - across every attempt, which a second call would not.
+ */
+@SuppressWarnings("javadoc")
+@Test(groups = { "precommit", "pluggablebackend" }, sequential = true)
+public class ReplayedConfigChangeTest extends DirectoryServerTestCase
+{
+ private static final String BACKEND_ID = "ReplayedConfigChangeTest";
+ private static final DN KEPT = DN.valueOf("dc=b907a,dc=com");
+ private static final DN REMOVED = DN.valueOf("dc=b907b,dc=com");
+ private static final DN ADDED = DN.valueOf("dc=b907c,dc=com");
+ /** Hierarchically related to {@link #KEPT}, which one backend is not allowed to serve as well. */
+ private static final DN UNREGISTRABLE = DN.valueOf("dc=b907d,dc=b907a,dc=com");
+
+ private ServerContext serverContext;
+ private AttributeType cnType;
+
+ @BeforeClass
+ public void startServer() throws Exception
+ {
+ TestCaseUtils.startServer();
+ serverContext = TestCaseUtils.getServerContext();
+ cnType = serverContext.getSchema().getAttributeType("cn");
+ }
+
+ /**
+ * These tests are designed to fail, and a failing one can leave a base DN behind in the server
+ * wide registry, where it would outlive the test and break the next one to use that DN.
+ */
+ @AfterMethod
+ public void deregisterLeftoverBaseDNs()
+ {
+ for (DN baseDN : new DN[] { KEPT, REMOVED, ADDED, UNREGISTRABLE })
+ {
+ try
+ {
+ serverContext.getBackendConfigManager().deregisterBaseDN(baseDN);
+ }
+ catch (Exception alreadyGone)
+ {
+ // Which is what the test should have left behind.
+ }
+ }
+ }
+
+ /**
+ * A base DN removal whose transaction conflicts before it touches the storage must be replayed
+ * without reporting a failure against the base DN it has already deregistered.
+ */
+ @Test
+ public void removalIsReplayableWhenTheTransactionConflictsBeforeAnyStorageAccess() throws Exception
+ {
+ final ReplayingBackend backend = openBackend(newTreeSet(KEPT, REMOVED));
+ try
+ {
+ final RootContainer rootContainer = backend.getRootContainer();
+ assertThat(rootContainer.getBaseDNs()).contains(REMOVED);
+
+ backend.storage.conflictAtFirstStorageAccess(1);
+ final ConfigChangeResult ccr = backend.applyConfigurationChange(backendCfg(newTreeSet(KEPT)));
+
+ assertThat(backend.storage.attempts()).isEqualTo(2);
+ assertThat(ccr.getMessages()).isEmpty();
+ assertThat(ccr.getResultCode()).isEqualTo(ResultCode.SUCCESS);
+ assertThat(rootContainer.getBaseDNs()).doesNotContain(REMOVED);
+ assertThat(backend.getBaseDNs()).doesNotContain(REMOVED);
+ assertThat(serverContext.getBackendConfigManager().getLocalBackendWithBaseDN(REMOVED)).isNull();
+ }
+ finally
+ {
+ backend.finalizeBackend();
+ }
+ }
+
+ /**
+ * A base DN addition whose transaction conflicts at commit time must be replayed, so that what
+ * the entry container it opens writes ends up committed rather than discarded by the rollback.
+ */
+ @Test
+ public void additionIsReplayableWhenTheTransactionConflictsAtCommitTime() throws Exception
+ {
+ final ReplayingBackend backend = openBackend(newTreeSet(KEPT));
+ try
+ {
+ final RootContainer rootContainer = backend.getRootContainer();
+ assertThat(rootContainer.getBaseDNs()).doesNotContain(ADDED);
+
+ backend.storage.conflictAtCommit(1);
+ final ConfigChangeResult ccr = backend.applyConfigurationChange(backendCfg(newTreeSet(KEPT, ADDED)));
+
+ assertThat(backend.storage.attempts()).isEqualTo(2);
+ assertThat(ccr.getMessages()).isEmpty();
+ assertThat(ccr.getResultCode()).isEqualTo(ResultCode.SUCCESS);
+ assertThat(rootContainer.getBaseDNs()).contains(ADDED);
+ assertThat(backend.getBaseDNs()).contains(ADDED);
+ assertThat(serverContext.getBackendConfigManager().getLocalBackendWithBaseDN(ADDED)).isSameAs(backend);
+ // Everything the newly opened entry container wrote belongs to the rolled back transaction,
+ // so the storage has to be asked, not the entry container which remembers writing it.
+ final EntryContainer ec = rootContainer.getEntryContainer(ADDED);
+ final TreeName cnIndex = ec.getAttributeIndex(cnType).getNameToIndexes().values().iterator().next().getName();
+ assertThat(rootContainer.getStorage().listTrees()).contains(cnIndex);
+ assertThat(persistedFlags(rootContainer, ec, cnIndex)).contains(TRUSTED);
+ // The entry container the rolled back attempt opened registered five configuration listeners,
+ // which only its close() takes back, so the replay has to give it up before opening another.
+ verify(backend.configuredWith, times(1)).removePluggableChangeListener(any());
+ }
+ finally
+ {
+ backend.finalizeBackend();
+ }
+ }
+
+ /**
+ * The trees of a removed base DN are deleted by the operation itself, so a replay deletes trees a
+ * rolled back attempt had already deleted. This is the case which reaches the storage, and it
+ * removes and adds a base DN at once because that is what an operator editing the configuration
+ * does.
+ */
+ @Test
+ public void aRemovalAndAnAdditionInOneChangeSurviveRepeatedReplay() throws Exception
+ {
+ final ReplayingBackend backend = openBackend(newTreeSet(KEPT, REMOVED));
+ try
+ {
+ final RootContainer rootContainer = backend.getRootContainer();
+ final Set<TreeName> removedTrees = treesOf(rootContainer.getEntryContainer(REMOVED));
+ assertThat(rootContainer.getStorage().listTrees()).containsAll(removedTrees);
+
+ // More than one conflict, because the contract is that the operation is replayed until it
+ // succeeds rather than that it survives a single replay.
+ backend.storage.conflictAtCommit(2);
+ final ConfigChangeResult ccr = backend.applyConfigurationChange(backendCfg(newTreeSet(KEPT, ADDED)));
+
+ assertThat(backend.storage.attempts()).isEqualTo(3);
+ assertThat(ccr.getMessages()).isEmpty();
+ assertThat(ccr.getResultCode()).isEqualTo(ResultCode.SUCCESS);
+ assertThat(rootContainer.getBaseDNs()).contains(KEPT, ADDED).doesNotContain(REMOVED);
+ assertThat(backend.getBaseDNs()).contains(KEPT, ADDED).doesNotContain(REMOVED);
+
+ final Set<TreeName> storedTrees = rootContainer.getStorage().listTrees();
+ assertThat(storedTrees).doesNotContainAnyElementsOf(removedTrees);
+ assertThat(storedTrees).containsAll(treesOf(rootContainer.getEntryContainer(ADDED)));
+ assertThat(storedTrees).containsAll(treesOf(rootContainer.getEntryContainer(KEPT)));
+ }
+ finally
+ {
+ backend.finalizeBackend();
+ }
+ }
+
+ /**
+ * A failure the storage engine does not replay must leave the backend as it was and say which
+ * base DNs the change was about, since the failure itself never names them.
+ */
+ @Test
+ public void aFailureWhichIsNotReplayedAppliesNothingAndNamesTheBaseDNs() throws Exception
+ {
+ final ReplayingBackend backend = openBackend(newTreeSet(KEPT, REMOVED));
+ try
+ {
+ final RootContainer rootContainer = backend.getRootContainer();
+ final Set<TreeName> removedTrees = treesOf(rootContainer.getEntryContainer(REMOVED));
+
+ backend.storage.failWithoutReplay();
+ final ConfigChangeResult ccr = backend.applyConfigurationChange(backendCfg(newTreeSet(KEPT, ADDED)));
+
+ assertThat(ccr.getResultCode()).isNotEqualTo(ResultCode.SUCCESS);
+ assertThat(ccr.getMessages().toString()).contains(REMOVED.toString()).contains(ADDED.toString());
+
+ // Nothing was registered, nothing was deregistered, and the rollback put the trees back.
+ assertThat(rootContainer.getBaseDNs()).contains(REMOVED).doesNotContain(ADDED);
+ assertThat(backend.getBaseDNs()).contains(REMOVED).doesNotContain(ADDED);
+ assertThat(serverContext.getBackendConfigManager().getLocalBackendWithBaseDN(REMOVED)).isSameAs(backend);
+ assertThat(serverContext.getBackendConfigManager().getLocalBackendWithBaseDN(ADDED)).isNull();
+ assertThat(rootContainer.getStorage().listTrees()).containsAll(removedTrees);
+ }
+ finally
+ {
+ backend.finalizeBackend();
+ }
+ }
+
+ /**
+ * A failure which the storage engine neither replays nor rolls back - the DDL of mysql and oracle
+ * commits of its own accord, and cassandra has no transaction at all - leaves the trees of a
+ * removed base DN gone. That base DN has to stop being reachable, or every operation against it
+ * meets a storage error rather than the "no such entry" its removal was meant to leave.
+ */
+ @Test
+ public void aFailureWhichIsNotRolledBackGivesUpTheBaseDNsWhoseTreesAreGone() throws Exception
+ {
+ final ReplayingBackend backend = openBackend(newTreeSet(KEPT, REMOVED));
+ try
+ {
+ final RootContainer rootContainer = backend.getRootContainer();
+ final Set<TreeName> removedTrees = treesOf(rootContainer.getEntryContainer(REMOVED));
+
+ backend.storage.failAfterCommit();
+ final ConfigChangeResult ccr = backend.applyConfigurationChange(backendCfg(newTreeSet(KEPT, ADDED)));
+
+ assertThat(ccr.getResultCode()).isNotEqualTo(ResultCode.SUCCESS);
+ assertThat(ccr.adminActionRequired()).isTrue();
+ assertThat(ccr.getMessages().toString()).contains(REMOVED.toString()).contains(ADDED.toString());
+
+ // The trees are gone, so the base DN is given up rather than left routed at them.
+ assertThat(rootContainer.getStorage().listTrees()).doesNotContainAnyElementsOf(removedTrees);
+ assertThat(rootContainer.getBaseDNs()).doesNotContain(REMOVED);
+ assertThat(backend.getBaseDNs()).doesNotContain(REMOVED);
+ assertThat(serverContext.getBackendConfigManager().getLocalBackendWithBaseDN(REMOVED)).isNull();
+
+ // The added base DN is not registered, since the change it belongs to failed.
+ assertThat(rootContainer.getBaseDNs()).doesNotContain(ADDED);
+ assertThat(backend.getBaseDNs()).doesNotContain(ADDED);
+ assertThat(serverContext.getBackendConfigManager().getLocalBackendWithBaseDN(ADDED)).isNull();
+ }
+ finally
+ {
+ backend.finalizeBackend();
+ }
+ }
+
+ /**
+ * The same failure leaves the trees it created for a base DN which is not being added after all
+ * exactly where they are. The configuration which names that base DN was stored before this
+ * listener was called - {@code ConfigurationHandler.replaceEntry} writes the entry, and only then
+ * notifies - and the failure does not take it back, so the next open of this backend opens that
+ * base DN again from it, adopting the trees which survived and creating the ones which did not.
+ * Deleting them here would take away the trees of a base DN the stored configuration still asks
+ * this backend to serve, and would buy nothing: that open re-creates them empty.
+ */
+ @Test
+ public void aFailureWhichIsNotRolledBackLeavesTheTreesItCreated() throws Exception
+ {
+ final ReplayingBackend backend = openBackend(newTreeSet(KEPT, REMOVED));
+ try
+ {
+ final RootContainer rootContainer = backend.getRootContainer();
+ final Set<TreeName> storedBefore = new HashSet<>(rootContainer.getStorage().listTrees());
+
+ backend.storage.failAfterCommit();
+ final ConfigChangeResult ccr = backend.applyConfigurationChange(backendCfg(newTreeSet(KEPT, ADDED)));
+
+ assertThat(ccr.getResultCode()).isNotEqualTo(ResultCode.SUCCESS);
+ assertThat(rootContainer.getBaseDNs()).doesNotContain(ADDED);
+ // Named by the failure, since nothing in the running server names them any more.
+ assertThat(ccr.getMessages().toString()).contains(ADDED.toString());
+
+ final Set<TreeName> left = new HashSet<>(rootContainer.getStorage().listTrees());
+ left.removeAll(storedBefore);
+ assertThat(left).as("the trees created for the base DN the stored configuration still names")
+ .isNotEmpty();
+ for (TreeName tree : left)
+ {
+ assertThat(tree.getBaseDN()).isEqualTo(ADDED.toNormalizedUrlSafeString());
+ }
+ }
+ finally
+ {
+ backend.finalizeBackend();
+ }
+ }
+
+ /**
+ * Whether anything survived a failure is read from the trees the storage still holds, so a backend
+ * which cannot be asked for them reconciles nothing at all. The operator has to be told that,
+ * since it is the case where the failure alone says least about what the backend is left serving.
+ */
+ @Test
+ public void aFailureWhoseSurvivingTreesCannotBeListedSaysSo() throws Exception
+ {
+ final ReplayingBackend backend = openBackend(newTreeSet(KEPT, REMOVED));
+ try
+ {
+ backend.storage.onListTrees(new Runnable()
+ {
+ @Override
+ public void run()
+ {
+ throw new StorageRuntimeException("the trees cannot be listed");
+ }
+ });
+
+ backend.storage.failAfterCommit();
+ final ConfigChangeResult ccr = backend.applyConfigurationChange(backendCfg(newTreeSet(KEPT)));
+ backend.storage.onListTrees(null);
+
+ assertThat(ccr.getResultCode()).isNotEqualTo(ResultCode.SUCCESS);
+ assertThat(ccr.adminActionRequired()).isTrue();
+ assertThat(ordinalsOf(ccr)).contains(ERR_BACKEND_CANNOT_LIST_TREES_AFTER_BASEDN_CHANGE.ordinal());
+ }
+ finally
+ {
+ backend.finalizeBackend();
+ }
+ }
+
+ /**
+ * A base DN this backend no longer holds must fail the change rather than the method: an entry
+ * container unregistered while the change was working out what to do leaves the root container
+ * with nothing to answer for that base DN, and the administration framework is owed a result
+ * whatever happens.
+ */
+ @Test
+ public void aBaseDNTheBackendNoLongerHoldsFailsTheChangeRatherThanTheMethod() throws Exception
+ {
+ final ReplayingBackend backend = openBackend(newTreeSet(KEPT, REMOVED));
+ try
+ {
+ final RootContainer rootContainer = backend.getRootContainer();
+ final PDBBackendCfg newCfg = backendCfg(newTreeSet(KEPT));
+ when(newCfg.getBaseDN()).thenReturn(new UnregisteringWhenAsked(rootContainer, REMOVED, newTreeSet(KEPT)));
+
+ final ConfigChangeResult ccr = backend.applyConfigurationChange(newCfg);
+
+ assertThat(ccr.getResultCode()).isNotEqualTo(ResultCode.SUCCESS);
+ assertThat(ordinalsOf(ccr)).contains(ERR_BACKEND_BASEDN_NO_LONGER_HELD.ordinal());
+ assertThat(ccr.getMessages().toString()).contains(REMOVED.toString());
+ // Nothing was applied, so the base DNs this backend serves are the ones it served before.
+ assertThat(backend.getBaseDNs()).contains(KEPT);
+ assertThat(rootContainer.getStorage().listTrees()).containsAll(treesOf(rootContainer.getEntryContainer(KEPT)));
+ }
+ finally
+ {
+ backend.finalizeBackend();
+ }
+ }
+
+ /**
+ * A base DN whose entry container is gone must not be answered with an ancestor's.
+ * {@link RootContainer#getEntryContainer} walks up the DN until it finds a container, which is how
+ * an entry is routed to the base DN above it; asked for a base DN the root container no longer
+ * holds, it hands back the container of the one it does. Deleting the trees of that container is
+ * deleting the trees of a base DN this backend is still serving.
+ * <p>
+ * Two base DNs of one backend are hierarchically related only after a registration the registry
+ * refused, which leaves the entry container behind in the root container - see
+ * {@link #aBaseDNWhichCannotBeRegisteredReportsWhereItFailed}.
+ */
+ @Test
+ public void anEntryContainerWhichIsGoneIsNotAnsweredWithItsParent() throws Exception
+ {
+ final ReplayingBackend backend = openBackend(newTreeSet(KEPT));
+ try
+ {
+ final RootContainer rootContainer = backend.getRootContainer();
+ // Refused by the registry, and so left in the root container underneath KEPT.
+ backend.applyConfigurationChange(backendCfg(newTreeSet(KEPT, UNREGISTRABLE)));
+ assertThat(rootContainer.getBaseDNs()).contains(KEPT, UNREGISTRABLE);
+ final Set<TreeName> keptTrees = treesOf(rootContainer.getEntryContainer(KEPT));
+
+ final PDBBackendCfg newCfg = backendCfg(newTreeSet(KEPT));
+ when(newCfg.getBaseDN()).thenReturn(new UnregisteringWhenAsked(rootContainer, UNREGISTRABLE, newTreeSet(KEPT)));
+ final ConfigChangeResult ccr = backend.applyConfigurationChange(newCfg);
+
+ // The base DN above the one which was gone is left alone, trees and routing both.
+ assertThat(rootContainer.getStorage().listTrees())
+ .as("the trees of the base DN above the one which was gone").containsAll(keptTrees);
+ assertThat(rootContainer.getBaseDNs()).contains(KEPT);
+ assertThat(serverContext.getBackendConfigManager().getLocalBackendWithBaseDN(KEPT)).isSameAs(backend);
+ // And the change says which base DN stopped it.
+ assertThat(ccr.getResultCode()).isNotEqualTo(ResultCode.SUCCESS);
+ assertThat(ordinalsOf(ccr)).contains(ERR_BACKEND_BASEDN_NO_LONGER_HELD.ordinal());
+ assertThat(ccr.getMessages().toString()).contains(UNREGISTRABLE.toString());
+ }
+ finally
+ {
+ backend.finalizeBackend();
+ }
+ }
+
+ /**
+ * A base DN the registry refuses is reported with the whole of what refused it. The registry
+ * raises the same message for several reasons and from more than one place, so the exception's own
+ * text does not say which of them happened; the frames it was raised on do.
+ */
+ @Test
+ public void aBaseDNWhichCannotBeRegisteredReportsWhereItFailed() throws Exception
+ {
+ final ReplayingBackend backend = openBackend(newTreeSet(KEPT));
+ try
+ {
+ final ConfigChangeResult ccr = backend.applyConfigurationChange(backendCfg(newTreeSet(KEPT, UNREGISTRABLE)));
+
+ assertThat(ccr.getResultCode()).isNotEqualTo(ResultCode.SUCCESS);
+ assertThat(ordinalsOf(ccr)).contains(ERR_BACKEND_CANNOT_REGISTER_BASEDN.ordinal());
+ assertThat(ccr.getMessages().toString())
+ .as("the reported cause never says where it was raised")
+ .contains("BackendConfigManager.java:");
+ }
+ finally
+ {
+ backend.finalizeBackend();
+ }
+ }
+
+ /**
+ * The entry container locks are held for the write which deletes the trees, and no longer:
+ * everything below the write reaches {@code BackendConfigManager}, whose single registry lock the
+ * server already takes in the opposite order - {@code shutdownLocalBackends} and a backend being
+ * disabled both hold it while closing a root container, which locks every entry container in turn.
+ * Holding both in this order would deadlock a base DN change against a shutdown, with no timeout
+ * on either side.
+ */
+ @Test
+ public void theRegistryIsNotTouchedWhileAnEntryContainerLockIsHeld() throws Exception
+ {
+ final ReplayingBackend backend = openBackend(newTreeSet(KEPT, REMOVED));
+ try
+ {
+ final RootContainer rootContainer = backend.getRootContainer();
+ final EntryContainer removed = rootContainer.getEntryContainer(REMOVED);
+ // Listing the surviving trees is the last thing the failure path does before it deregisters,
+ // so it is asked on the very thread, and at the very moment, the deadlock would be reached.
+ final boolean[] lockHeld = new boolean[] { false };
+ final boolean[] asked = new boolean[] { false };
+ backend.storage.onListTrees(new Runnable()
+ {
+ @Override
+ public void run()
+ {
+ asked[0] = true;
+ lockHeld[0] |= ((ReentrantReadWriteLock.WriteLock) removed.exclusiveLock).isHeldByCurrentThread();
+ }
+ });
+
+ backend.storage.failAfterCommit();
+ backend.applyConfigurationChange(backendCfg(newTreeSet(KEPT)));
+ backend.storage.onListTrees(null);
+
+ assertThat(asked).as("the failure path never listed the surviving trees").containsExactly(true);
+ assertThat(lockHeld).as("the entry container lock was still held").containsExactly(false);
+ }
+ finally
+ {
+ backend.finalizeBackend();
+ }
+ }
+
+ /**
+ * A configuration change which leaves the base DNs alone - every change to index-entry-limit,
+ * db-cache-percent and the rest - has no storage work to do, so it opens no transaction to
+ * commit nothing.
+ */
+ @Test
+ public void aChangeWhichLeavesTheBaseDNsAloneOpensNoTransaction() throws Exception
+ {
+ final ReplayingBackend backend = openBackend(newTreeSet(KEPT, REMOVED));
+ try
+ {
+ final int writesBefore = backend.storage.writes();
+ final ConfigChangeResult ccr = backend.applyConfigurationChange(backendCfg(newTreeSet(KEPT, REMOVED)));
+
+ assertThat(ccr.getResultCode()).isEqualTo(ResultCode.SUCCESS);
+ assertThat(ccr.getMessages()).isEmpty();
+ assertThat(backend.storage.writes()).isEqualTo(writesBefore);
+ assertThat(backend.getBaseDNs()).contains(KEPT, REMOVED);
+ }
+ finally
+ {
+ backend.finalizeBackend();
+ }
+ }
+
+ /** The messages a change result carries, by identity rather than by their formatted text. */
+ private static Set<Integer> ordinalsOf(ConfigChangeResult ccr)
+ {
+ final Set<Integer> ordinals = new HashSet<>();
+ for (LocalizableMessage message : ccr.getMessages())
+ {
+ ordinals.add(message.ordinal());
+ }
+ return ordinals;
+ }
+
+ private static Set<TreeName> treesOf(EntryContainer ec)
+ {
+ final Set<TreeName> names = new HashSet<>();
+ for (Tree tree : ec.listTrees())
+ {
+ names.add(tree.getName());
+ }
+ return names;
+ }
+
+ /** Reads back the flags an index was given when it was opened, as they are stored. */
+ private static EnumSet<IndexFlag> persistedFlags(RootContainer rootContainer, EntryContainer ec, TreeName index)
+ throws Exception
+ {
+ final State state = new State(new TreeName(ec.getTreePrefix(), STATE_INDEX_NAME));
+ return rootContainer.getStorage().read(txn -> state.getIndexFlags(txn, index));
+ }
+
+ private ReplayingBackend openBackend(SortedSet<DN> baseDNs) throws Exception
+ {
+ final ReplayingBackend backend = new ReplayingBackend();
+ backend.setBackendID(BACKEND_ID);
+ backend.configuredWith = backendCfg(baseDNs);
+ backend.configureBackend(backend.configuredWith, serverContext);
+ // Start from a pristine on-disk state so that a previous run cannot mask the defect.
+ backend.storage.removeStorageFiles();
+ try
+ {
+ backend.openBackend();
+ }
+ catch (Exception e)
+ {
+ // openBackend() opens the root container before it preloads, counts the entries, registers
+ // the base DNs and registers the monitor, so a failure in any of those leaves the volume open
+ // and the monitor registered. Every following test would then fail in openBackend() too, and
+ // the one which actually broke would be lost among them.
+ try
+ {
+ if (backend.getRootContainer() != null)
+ {
+ backend.finalizeBackend();
+ }
+ else
+ {
+ backend.storage.close();
+ }
+ }
+ catch (Exception cleanupFailure)
+ {
+ // openBackend() registers the root container monitor last of all, and closeBackend()
+ // deregisters it without a null check, so cleaning up after a failure before that throws a
+ // NullPointerException of its own. The failure being cleaned up after is the one worth
+ // reading.
+ e.addSuppressed(cleanupFailure);
+ }
+ throw e;
+ }
+ return backend;
+ }
+
+ private PDBBackendCfg backendCfg(SortedSet<DN> baseDNs) throws ConfigException
+ {
+ final PDBBackendCfg cfg = mockCfg(PDBBackendCfg.class);
+ when(cfg.dn()).thenReturn(DN.valueOf("ds-cfg-backend-id=" + BACKEND_ID + ",cn=Backends,cn=config"));
+ when(cfg.getBackendId()).thenReturn(BACKEND_ID);
+ when(cfg.getDBDirectory()).thenReturn(BACKEND_ID);
+ when(cfg.getDBDirectoryPermissions()).thenReturn("755");
+ when(cfg.getDBCacheSize()).thenReturn(0L);
+ when(cfg.getDBCachePercent()).thenReturn(20);
+ when(cfg.getBaseDN()).thenReturn(baseDNs);
+ when(cfg.listBackendIndexes()).thenReturn(new String[] { "cn" });
+ when(cfg.listBackendVLVIndexes()).thenReturn(new String[0]);
+
+ final BackendIndexCfg indexCfg = mock(BackendIndexCfg.class);
+ when(indexCfg.getIndexType()).thenReturn(newTreeSet(IndexType.EQUALITY));
+ when(indexCfg.getAttribute()).thenReturn(cnType);
+ when(indexCfg.getIndexEntryLimit()).thenReturn(4000);
+ when(indexCfg.getSubstringLength()).thenReturn(6);
+ when(cfg.getBackendIndex("cn")).thenReturn(indexCfg);
+ return cfg;
+ }
+
+ /** A backend whose storage makes the next write operation conflict, and so be replayed. */
+ private static final class ReplayingBackend extends BackendImpl<PDBBackendCfg>
+ {
+ private ReplayingStorage storage;
+ /** The configuration the entry containers register their listeners with. */
+ private PDBBackendCfg configuredWith;
+
+ @Override
+ protected Storage configureStorage(PDBBackendCfg cfg, ServerContext serverContext) throws ConfigException
+ {
+ storage = new ReplayingStorage(new PDBStorage(cfg, serverContext));
+ return storage;
+ }
+ }
+
+ /**
+ * The base DNs a change asks for, which unregisters an entry container the first time it is asked
+ * whether it holds one. {@link BackendImpl#applyConfigurationChange} copies the base DNs the root
+ * container holds and then looks each of their entry containers up, and asks this set in between;
+ * an importLDIF, a rebuildBackend, an exportLDIF or the backend being disabled closes the root
+ * container in that window and unregisters every one of them. Done here rather than raced for, so
+ * that the window is closed on the same thread every time.
+ */
+ private static final class UnregisteringWhenAsked extends TreeSet<DN>
+ {
+ private static final long serialVersionUID = 1L;
+
+ private final transient RootContainer rootContainer;
+ private final DN toUnregister;
+ private boolean unregistered;
+
+ UnregisteringWhenAsked(RootContainer rootContainer, DN toUnregister, SortedSet<DN> baseDNs)
+ {
+ super(baseDNs);
+ this.rootContainer = rootContainer;
+ this.toUnregister = toUnregister;
+ }
+
+ @Override
+ public boolean contains(Object baseDN)
+ {
+ if (!unregistered)
+ {
+ unregistered = true;
+ // Closed here because nothing else will: the root container closes the containers it
+ // holds, and this one has just been taken out of it, with its configuration listeners
+ // still registered.
+ closeSilently(rootContainer.unregisterEntryContainer(toUnregister));
+ }
+ return super.contains(baseDN);
+ }
+ }
+
+ /** A failure which no storage engine replays, unlike {@link RollbackException}. */
+ private static final class UnreplayableFailure extends Exception
+ {
+ private static final long serialVersionUID = 1L;
+ }
+
+ /**
+ * Decorates a {@link Storage} so that the next {@link Storage#write(WriteOperation)} conflicts a
+ * given number of times before it is let through. The conflict is raised from within the single
+ * {@code write} the delegate is asked for, so the delegate's own retry loop performs the replay.
+ */
+ private static final class ReplayingStorage implements Storage
+ {
+ /** Where the conflict is raised, which decides how much of the operation has run. */
+ private enum ConflictPoint
+ {
+ /** As soon as the operation first touches the transaction, before it has changed anything. */
+ FIRST_STORAGE_ACCESS,
+ /** Once the operation has run to completion, as a conflict reported by {@code commit()}. */
+ COMMIT,
+ /** Once the operation has run to completion, as a failure which is not replayed at all. */
+ NO_REPLAY,
+ /**
+ * Once the operation has committed, as a failure which is not replayed either: what an engine
+ * whose tree deletions do not belong to the transaction leaves behind.
+ */
+ NO_REPLAY_AFTER_COMMIT
+ }
+
+ private final Storage delegate;
+ private Runnable onListTrees;
+ private ConflictPoint conflictPoint;
+ private int conflictsLeft;
+ private int attempts;
+ private int writes;
+
+ ReplayingStorage(Storage delegate)
+ {
+ this.delegate = delegate;
+ }
+
+ void conflictAtFirstStorageAccess(int conflicts)
+ {
+ arm(ConflictPoint.FIRST_STORAGE_ACCESS, conflicts);
+ }
+
+ void conflictAtCommit(int conflicts)
+ {
+ arm(ConflictPoint.COMMIT, conflicts);
+ }
+
+ void failWithoutReplay()
+ {
+ arm(ConflictPoint.NO_REPLAY, 1);
+ }
+
+ void failAfterCommit()
+ {
+ arm(ConflictPoint.NO_REPLAY_AFTER_COMMIT, 1);
+ }
+
+ private void arm(ConflictPoint where, int conflicts)
+ {
+ conflictPoint = where;
+ conflictsLeft = conflicts;
+ attempts = 0;
+ }
+
+ /** How many times the armed operation was run, the first attempt included. */
+ int attempts()
+ {
+ return attempts;
+ }
+
+ /** How many write operations this storage was asked for, armed or not. */
+ int writes()
+ {
+ return writes;
+ }
+
+ @Override
+ public void write(final WriteOperation writeOperation) throws Exception
+ {
+ writes++;
+ final ConflictPoint armed = conflictPoint;
+ if (armed == null)
+ {
+ delegate.write(writeOperation);
+ return;
+ }
+ conflictPoint = null;
+ if (armed == ConflictPoint.NO_REPLAY_AFTER_COMMIT)
+ {
+ // Committed, then reported as a failure: the operation's work outlives the failure, as it
+ // does where the storage engine does not roll a tree deletion back.
+ delegate.write(new WriteOperation()
+ {
+ @Override
+ public void run(WriteableTransaction txn) throws Exception
+ {
+ attempts++;
+ writeOperation.run(txn);
+ }
+ });
+ throw new UnreplayableFailure();
+ }
+ // A single call, so that the replay is the delegate's own and keeps whatever the delegate
+ // holds for the duration of a write, rather than starting afresh as a second call would.
+ delegate.write(new WriteOperation()
+ {
+ @Override
+ public void run(WriteableTransaction txn) throws Exception
+ {
+ attempts++;
+ if (conflictsLeft-- <= 0)
+ {
+ writeOperation.run(txn);
+ return;
+ }
+ if (armed == ConflictPoint.FIRST_STORAGE_ACCESS)
+ {
+ writeOperation.run(new ConflictingTransaction());
+ return;
+ }
+ writeOperation.run(txn);
+ if (armed == ConflictPoint.NO_REPLAY)
+ {
+ throw new UnreplayableFailure();
+ }
+ throw new RollbackException();
+ }
+ });
+ }
+
+ @Override
+ public Importer startImport() throws ConfigException
+ {
+ return delegate.startImport();
+ }
+
+ @Override
+ public void open(AccessMode accessMode) throws Exception
+ {
+ delegate.open(accessMode);
+ }
+
+ @Override
+ public <T> T read(ReadOperation<T> readOperation) throws Exception
+ {
+ return delegate.read(readOperation);
+ }
+
+ @Override
+ public void removeStorageFiles()
+ {
+ delegate.removeStorageFiles();
+ }
+
+ @Override
+ public StorageStatus getStorageStatus()
+ {
+ return delegate.getStorageStatus();
+ }
+
+ @Override
+ public boolean supportsBackupAndRestore()
+ {
+ return delegate.supportsBackupAndRestore();
+ }
+
+ @Override
+ public void createBackup(BackupConfig backupConfig) throws DirectoryException
+ {
+ delegate.createBackup(backupConfig);
+ }
+
+ @Override
+ public void removeBackup(BackupDirectory backupDirectory, String backupID) throws DirectoryException
+ {
+ delegate.removeBackup(backupDirectory, backupID);
+ }
+
+ @Override
+ public void restoreBackup(RestoreConfig restoreConfig) throws DirectoryException
+ {
+ delegate.restoreBackup(restoreConfig);
+ }
+
+ /** Run whenever the trees which survived a failure are listed, and only then. */
+ void onListTrees(Runnable probe)
+ {
+ this.onListTrees = probe;
+ }
+
+ @Override
+ public Set<TreeName> listTrees()
+ {
+ if (onListTrees != null)
+ {
+ onListTrees.run();
+ }
+ return delegate.listTrees();
+ }
+
+ @Override
+ public void close()
+ {
+ delegate.close();
+ }
+ }
+
+ /** A transaction which conflicts as soon as it is used, without ever reaching the storage. */
+ private static final class ConflictingTransaction implements WriteableTransaction
+ {
+ private static RollbackException conflict()
+ {
+ return new RollbackException();
+ }
+
+ @Override
+ public void openTree(TreeName name, boolean createOnDemand)
+ {
+ throw conflict();
+ }
+
+ @Override
+ public void deleteTree(TreeName name)
+ {
+ throw conflict();
+ }
+
+ @Override
+ public void put(TreeName treeName, ByteSequence key, ByteSequence value)
+ {
+ throw conflict();
+ }
+
+ @Override
+ public boolean update(TreeName treeName, ByteSequence key, UpdateFunction f)
+ {
+ throw conflict();
+ }
+
+ @Override
+ public boolean delete(TreeName treeName, ByteSequence key)
+ {
+ throw conflict();
+ }
+
+ @Override
+ public ByteString read(TreeName treeName, ByteSequence key)
+ {
+ throw conflict();
+ }
+
+ @Override
+ public Cursor<ByteString, ByteString> openCursor(TreeName treeName)
+ {
+ throw conflict();
+ }
+
+ @Override
+ public long getRecordCount(TreeName treeName)
+ {
+ throw conflict();
+ }
+
+ @Override
+ public boolean treeExists(TreeName treeName)
+ {
+ throw conflict();
+ }
+ }
+}
--
Gitblit v1.10.0