From d0deb0709fc73a94d3d46ac76753933383861b00 Mon Sep 17 00:00:00 2001
From: Valery Kharseko <vharseko@3a-systems.ru>
Date: Thu, 10 Sep 2026 11:56:42 +0000
Subject: [PATCH] [#901] Make the replay retry budget of a replication domain configurable (#944)
---
opendj-server-legacy/src/main/java/org/opends/server/replication/plugin/RemotePendingChanges.java | 24 +++
opendj-maven-plugin/src/main/resources/config/xml/org/forgerock/opendj/server/config/ReplicationDomainConfiguration.xml | 41 +++++
opendj-server-legacy/src/messages/org/opends/messages/replication.properties | 4
opendj-server-legacy/src/test/java/org/opends/server/replication/plugin/DomainFakeCfg.java | 15 +
opendj-server-legacy/src/test/java/org/opends/server/replication/UpdateOperationTest.java | 186 +++++++++++++++++++++--
opendj-server-legacy/src/test/java/org/opends/server/replication/plugin/AssuredReplicationPluginTest.java | 42 +++--
opendj-server-legacy/src/main/java/org/opends/server/replication/plugin/LDAPReplicationDomain.java | 68 ++------
opendj-server-legacy/resource/schema/02-config.ldif | 11 +
opendj-server-legacy/src/test/java/org/opends/server/replication/ReplicationTestCase.java | 39 ++++
9 files changed, 343 insertions(+), 87 deletions(-)
diff --git a/opendj-maven-plugin/src/main/resources/config/xml/org/forgerock/opendj/server/config/ReplicationDomainConfiguration.xml b/opendj-maven-plugin/src/main/resources/config/xml/org/forgerock/opendj/server/config/ReplicationDomainConfiguration.xml
index 603e5d5..87d3b38 100644
--- a/opendj-maven-plugin/src/main/resources/config/xml/org/forgerock/opendj/server/config/ReplicationDomainConfiguration.xml
+++ b/opendj-maven-plugin/src/main/resources/config/xml/org/forgerock/opendj/server/config/ReplicationDomainConfiguration.xml
@@ -14,6 +14,7 @@
Copyright 2007-2010 Sun Microsystems, Inc.
Portions Copyright 2011-2015 ForgeRock AS.
+ Portions Copyright 2026 3A Systems, LLC.
! -->
<adm:managed-object name="replication-domain"
plural-name="replication-domains"
@@ -559,4 +560,44 @@
</ldap:attribute>
</adm:profile>
</adm:property>
+ <adm:property name="replay-give-up-delay">
+ <adm:synopsis>
+ Specifies how long this directory server retries the replay of a change which its
+ backend could not apply, before it gives up on that change and moves on to the
+ changes which follow it.
+ </adm:synopsis>
+ <adm:description>
+ The change is asked for again and retried for as long as this budget lasts, over as
+ many deliveries as it takes. The budget is a duration rather than a number of
+ attempts because a backend which is being imported into, rebuilt or restored serves
+ nothing for as long as that operation runs: raise this value before such a
+ maintenance operation when it is going to outlast the default. Once the budget is
+ spent, the change is recorded as replayed although it never was - this replica then
+ diverges from the rest of the topology, raises the
+ org.opends.server.replication.UnreplayedChange alert and has to be reinitialized. A
+ value of "unlimited" has this server retry the change for as long as it keeps
+ failing, which holds the replication of this domain back until an administrator
+ intervenes: a change which is retried on and on keeps one of the replay threads this
+ server shares between all of its domains busy waiting for it, and has a warning
+ logged for every delivery of it, so a domain left like this is paid for by the
+ healthy ones too. A value of 0 has this server give up on a change as soon as one
+ delivery of it failed, the attempts made in place within that delivery still being
+ taken. The budget is measured from the first failure of the change rather than from
+ the moment it was set, so lowering it gives up on the next failed delivery of a
+ change which has been failing for longer than the new value.
+ </adm:description>
+ <adm:default-behavior>
+ <adm:defined>
+ <adm:value>300000ms</adm:value>
+ </adm:defined>
+ </adm:default-behavior>
+ <adm:syntax>
+ <adm:duration base-unit="ms" lower-limit="0" allow-unlimited="true" />
+ </adm:syntax>
+ <adm:profile name="ldap">
+ <ldap:attribute>
+ <ldap:name>ds-cfg-replay-give-up-delay</ldap:name>
+ </ldap:attribute>
+ </adm:profile>
+ </adm:property>
</adm:managed-object>
diff --git a/opendj-server-legacy/resource/schema/02-config.ldif b/opendj-server-legacy/resource/schema/02-config.ldif
index 2f8cc52..029d64d 100644
--- a/opendj-server-legacy/resource/schema/02-config.ldif
+++ b/opendj-server-legacy/resource/schema/02-config.ldif
@@ -15,7 +15,7 @@
# Portions Copyright 2011 profiq, s.r.o.
# Portions Copyright 2012 Manuel Gaupp
# Portions copyright 2015 Edan Idzerda
-# Portions copyright 2023-2025 3A Systems LLC
+# Portions copyright 2023-2026 3A Systems LLC
# This file contains the attribute type and objectclass definitions for use
# with the Directory Server configuration.
@@ -4106,6 +4106,12 @@
EQUALITY caseIgnoreMatch
SYNTAX 1.3.6.1.4.1.1466.115.121.1.15
X-ORIGIN 'OpenDJ Directory Server' )
+attributeTypes: ( 1.3.6.1.4.1.60142.2.1.1.1
+ NAME 'ds-cfg-replay-give-up-delay'
+ EQUALITY caseIgnoreMatch
+ SYNTAX 1.3.6.1.4.1.1466.115.121.1.15
+ SINGLE-VALUE
+ X-ORIGIN 'OpenDJ Directory Server' )
objectClasses: ( 1.3.6.1.4.1.26027.1.2.1
NAME 'ds-cfg-access-control-handler'
SUP top
@@ -4642,7 +4648,8 @@
ds-cfg-changetime-heartbeat-interval $
ds-cfg-log-changenumber $
ds-cfg-initialization-window-size $
- ds-cfg-source-address )
+ ds-cfg-source-address $
+ ds-cfg-replay-give-up-delay )
X-ORIGIN 'OpenDS Directory Server' )
objectClasses: ( 1.3.6.1.4.1.26027.1.2.58
NAME 'ds-cfg-length-based-password-validator'
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/replication/plugin/LDAPReplicationDomain.java b/opendj-server-legacy/src/main/java/org/opends/server/replication/plugin/LDAPReplicationDomain.java
index 5680d30..add2d58 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/replication/plugin/LDAPReplicationDomain.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/replication/plugin/LDAPReplicationDomain.java
@@ -319,17 +319,6 @@
*/
public static final int IN_PLACE_REPLAY_ATTEMPTS = 10;
/**
- * How long the replay of a change is retried before this replica gives up on it and
- * moves on to the changes which follow it.
- * <p>
- * The budget is a duration rather than a number of attempts because what
- * {@link #isServerFailure(ResultCode, ResultCode)} reports is measured in minutes: a
- * backend which is being rebuilt, imported into or restored (OPENDJ-49) serves nothing
- * while it works, and a handful of attempts would have this replica give up on every
- * change of a maintenance window it only had to wait out.
- */
- private static final long REPLAY_GIVE_UP_DELAY_IN_MS = 300000;
- /**
* How long the session is left down before the change is asked for again, multiplied
* by the number of attempts already made: a backend which keeps failing must not be
* hammered with a session restart per failed change.
@@ -367,12 +356,6 @@
*/
private final AtomicInteger consecutiveSessionRestarts = new AtomicInteger();
/**
- * How long the replay of a change is retried before this replica gives up on it. Only
- * the tests, which can not wait out {@link #REPLAY_GIVE_UP_DELAY_IN_MS}, set another
- * value.
- */
- private volatile long replayGiveUpDelayInMs = REPLAY_GIVE_UP_DELAY_IN_MS;
- /**
* Serialises the session of this domain being stopped and started again: the replay
* thread which restarts it after a failed replay must not race the domain being
* disabled for an import or a restore, or it would bring a broker and a listener
@@ -3017,9 +3000,10 @@
* The change has deliberately been left out of the ServerState, so the replication
* server still owns it: restart the session so that it is sent again and replayed on
* a backend which has hopefully recovered in the meantime. Give up once its replay has
- * been failing for {@link #REPLAY_GIVE_UP_DELAY_IN_MS} and record it as replayed, so
- * that a change which can never be applied here does not stop this replica for good:
- * the administrator is told that this replica has diverged and must be reinitialized.
+ * been failing for the {@code replay-give-up-delay} of this domain and record it as
+ * replayed, so that a change which can never be applied here does not stop this replica
+ * for good: the administrator is told that this replica has diverged and must be
+ * reinitialized.
*
* @param csn
* the CSN of the change which could not be replayed
@@ -3052,7 +3036,20 @@
*/
return false;
}
- if (failure.getFailingForMs() >= replayGiveUpDelayInMs)
+ /*
+ * The budget is read from the configuration at every decision rather than kept in a
+ * field of its own: an administrator who raises it because a maintenance window is
+ * going to outlast it is not made to restart this server for that, and
+ * applyConfigurationChange() replaces the configuration object as a whole. A negative
+ * value is the "unlimited" of the duration syntax - this replica then keeps asking for
+ * the change rather than ever recording one it did not apply, which is the choice of
+ * an operator who would rather have the replication of this domain stop than have it
+ * diverge. The generated getter yields the value in the base unit the property is
+ * declared with, which is milliseconds here, so it is comparable to what the failure
+ * reports as it is.
+ */
+ final long giveUpDelayInMs = config.getReplayGiveUpDelay();
+ if (giveUpDelayInMs >= 0 && failure.getFailingForMs() >= giveUpDelayInMs)
{
final LocalizableMessage message = ERR_REPLAY_SKIPPING_CHANGE.get(
csn, getBaseDN(), failure.getFailingForMs(), failure.getAttempts());
@@ -3230,34 +3227,6 @@
}
/**
- * Returns how long the replay of a change is retried before this replica gives up on
- * it.
- * <p>
- * Only there for the tests, which set another value and put this one back.
- *
- * @return how long a change is retried, in milliseconds
- */
- @VisibleForTesting
- public long getReplayGiveUpDelay()
- {
- return replayGiveUpDelayInMs;
- }
-
- /**
- * Sets how long the replay of a change is retried before this replica gives up on it.
- * <p>
- * Only there for the tests, which can not wait out the {@link
- * #REPLAY_GIVE_UP_DELAY_IN_MS} a backend under maintenance is given.
- *
- * @param delayInMs how long a change is retried, in milliseconds
- */
- @VisibleForTesting
- public void setReplayGiveUpDelay(long delayInMs)
- {
- this.replayGiveUpDelayInMs = delayInMs;
- }
-
- /**
* Generate a new CSN and insert it in the pending list.
*
* @param operation
@@ -5138,6 +5107,7 @@
attributes.add("remote-pending-changes-size", remotePendingChanges.getQueueSize());
attributes.add("dependent-changes-size", remotePendingChanges.getDependentChangesSize());
attributes.add("changes-in-progress-size", remotePendingChanges.changesInProgressSize());
+ attributes.add("changes-with-failed-replay", remotePendingChanges.getFailingChangesSize());
}
/**
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/replication/plugin/RemotePendingChanges.java b/opendj-server-legacy/src/main/java/org/opends/server/replication/plugin/RemotePendingChanges.java
index 4eef2d8..1d54c51 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/replication/plugin/RemotePendingChanges.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/replication/plugin/RemotePendingChanges.java
@@ -393,6 +393,30 @@
}
/**
+ * Returns how many of the listed changes have a failed replay recorded against them.
+ * <p>
+ * A change is counted from its first failed replay until it leaves this map, whether it
+ * leaves it applied or given up on. It is what tells that the replay of this domain is
+ * stuck: a domain whose {@code replay-give-up-delay} is unlimited never gives up, so it
+ * never counts a failed change either, and the changes it keeps asking for are only
+ * visible here.
+ *
+ * @return the number of listed changes with a failed replay recorded against them
+ */
+ public int getFailingChangesSize()
+ {
+ pendingChangesReadLock.lock();
+ try
+ {
+ return failingChanges;
+ }
+ finally
+ {
+ pendingChangesReadLock.unlock();
+ }
+ }
+
+ /**
* Forgets every change listed here, without updating the ServerState.
* <p>
* Called when the domain is disabled: its ServerState is saved and cleared from
diff --git a/opendj-server-legacy/src/messages/org/opends/messages/replication.properties b/opendj-server-legacy/src/messages/org/opends/messages/replication.properties
index cde9f7a..a7500d9 100644
--- a/opendj-server-legacy/src/messages/org/opends/messages/replication.properties
+++ b/opendj-server-legacy/src/messages/org/opends/messages/replication.properties
@@ -620,7 +620,9 @@
session to the replication server so that it is sent again
ERR_REPLAY_SKIPPING_CHANGE_308=Could not replay change %s in domain "%s": its replay has been \
failing for %d ms over %d deliveries, each attempted several times in place. The change is being \
- skipped: this replica now diverges from the rest of the topology and must be reinitialized
+ skipped: this replica now diverges from the rest of the topology and must be reinitialized. \
+ Raise the replay-give-up-delay property of this domain to give a change longer before it is \
+ given up on
NOTE_REPLAY_ABANDONED_CHANGE_309=Could not replay change %s in domain "%s": the replay thread \
it was given to is stopping. The change has not been recorded as replayed and is given back to \
the replication server, which still owns it
diff --git a/opendj-server-legacy/src/test/java/org/opends/server/replication/ReplicationTestCase.java b/opendj-server-legacy/src/test/java/org/opends/server/replication/ReplicationTestCase.java
index bcedc42..9d5f9db 100644
--- a/opendj-server-legacy/src/test/java/org/opends/server/replication/ReplicationTestCase.java
+++ b/opendj-server-legacy/src/test/java/org/opends/server/replication/ReplicationTestCase.java
@@ -56,6 +56,7 @@
import org.opends.server.core.AddOperation;
import org.opends.server.core.DeleteOperation;
import org.opends.server.core.DirectoryServer;
+import org.opends.server.core.ModifyOperation;
import org.opends.server.protocols.internal.InternalClientConnection;
import org.opends.server.protocols.internal.InternalSearchOperation;
import org.opends.server.protocols.internal.SearchRequest;
@@ -859,6 +860,44 @@
return Requests.newModifyRequest(entryDN).addModification(modType, attrName, attrValue);
}
+ /**
+ * Applies a modification to the configuration entry of a replication domain, the way an
+ * administrator would, and checks that it was applied.
+ * <p>
+ * The domain reads its configuration for every decision it makes, so a property changed
+ * here takes effect on what the domain is doing right now.
+ *
+ * @param domainConfigDN
+ * the DN of the configuration entry of the domain
+ * @param modType
+ * the modification to apply, {@link ModificationType#DELETE} with no value
+ * taking an attribute away so that its property falls back to its default
+ * @param attrName
+ * the configuration attribute to modify
+ * @param values
+ * the values to set, none when the attribute is being deleted
+ */
+ protected static void modifyDomainConfig(
+ DN domainConfigDN, ModificationType modType, String attrName, String... values)
+ {
+ final ModifyRequest request = Requests.newModifyRequest(domainConfigDN)
+ .addModification(modType, attrName, (Object[]) values);
+ final ModifyOperation modOp =
+ InternalClientConnection.getRootConnection().processModify(request);
+ if (modType == DELETE && modOp.getResultCode() == NO_SUCH_ATTRIBUTE)
+ {
+ /*
+ * The attribute is already gone, which is what the delete was asking for. Said here
+ * rather than at the call sites because a delete is how a test puts a property back
+ * to its default in a finally: a cleanup which throws would replace the failure it
+ * is cleaning up after, and say nothing about it.
+ */
+ return;
+ }
+ assertEquals(modOp.getResultCode(), SUCCESS,
+ "Cannot " + modType + " " + attrName + " on " + domainConfigDN);
+ }
+
/** Utility method to create, run a task and check its result. */
protected void task(String task) throws Exception
{
diff --git a/opendj-server-legacy/src/test/java/org/opends/server/replication/UpdateOperationTest.java b/opendj-server-legacy/src/test/java/org/opends/server/replication/UpdateOperationTest.java
index 4a9df33..0571660 100644
--- a/opendj-server-legacy/src/test/java/org/opends/server/replication/UpdateOperationTest.java
+++ b/opendj-server-legacy/src/test/java/org/opends/server/replication/UpdateOperationTest.java
@@ -45,7 +45,6 @@
import org.forgerock.opendj.ldap.ModificationType;
import org.forgerock.opendj.ldap.ResultCode;
import org.forgerock.opendj.ldap.requests.ModifyDNRequest;
-import org.forgerock.opendj.ldap.requests.ModifyRequest;
import org.forgerock.opendj.ldap.schema.AttributeType;
import org.forgerock.opendj.server.config.server.ReplicationSynchronizationProviderCfg;
import org.opends.server.TestCaseUtils;
@@ -103,8 +102,12 @@
* How long a change is retried in the tests which check that this replica gives up on
* a change it can never apply: long enough for the change to be delivered again a
* couple of times, short enough not to make the test wait out a real backend outage.
+ * In the duration syntax of the {@code replay-give-up-delay} property.
*/
- private static final long TEST_GIVE_UP_DELAY_IN_MS = 2000;
+ private static final String TEST_GIVE_UP_DELAY = "2000ms";
+
+ /** The configuration attribute which carries the replay give-up budget of a domain. */
+ private static final String ATTR_REPLAY_GIVE_UP_DELAY = "ds-cfg-replay-give-up-delay";
/** An entry with a entryUUID. */
private Entry personWithUUIDEntry;
@@ -1442,14 +1445,13 @@
final long initialFailures = getMonitorAttrValue(baseDN, "replayed-updates-failed");
domain.resetUnreplayedChangeAlertThrottle();
final int initialAlerts = DummyAlertHandler.getAlertCount(ALERT_TYPE_REPLICATION_UNREPLAYED_CHANGE);
- final long giveUpDelay = domain.getReplayGiveUpDelay();
+ // A backend which is down for maintenance is waited out for minutes: this test can
+ // not, so the change is given up on after a couple of deliveries instead. The short
+ // circuit below is the server's, so it is registered inside the try which takes it
+ // back.
+ setReplayGiveUpDelay(TEST_GIVE_UP_DELAY);
try
{
- // A backend which is down for maintenance is waited out for minutes: this test
- // can not, so the change is given up on after a couple of deliveries instead.
- // Set inside the try which puts it back, like the short circuit below: both are
- // the domain's and the server's for as long as they are left behind.
- domain.setReplayGiveUpDelay(TEST_GIVE_UP_DELAY_IN_MS);
/*
* Fail the replay the way a storage failure does: the backend reports it with the
* server-error-result-code, 80 by default. The short circuit has to be set at the
@@ -1521,7 +1523,7 @@
finally
{
ShortCircuitPlugin.deregisterShortCircuit(OperationType.DELETE, "PreParse");
- domain.setReplayGiveUpDelay(giveUpDelay);
+ resetReplayGiveUpDelay();
}
}
finally
@@ -2016,11 +2018,10 @@
final LDAPReplicationDomain domain = MultimasterReplication.findDomain(baseDN, null);
final long initialFailures = getMonitorAttrValue(baseDN, "replayed-updates-failed");
- final long giveUpDelay = domain.getReplayGiveUpDelay();
+ // Two changes have to be given up on here, so the budget is shortened the same way.
+ setReplayGiveUpDelay(TEST_GIVE_UP_DELAY);
try
{
- // Both are put back by the finally below, so both are set inside the try.
- domain.setReplayGiveUpDelay(TEST_GIVE_UP_DELAY_IN_MS);
ShortCircuitPlugin.registerShortCircuit(
OperationType.DELETE, "PreParse", ResultCode.OTHER.intValue());
@@ -2058,7 +2059,128 @@
finally
{
ShortCircuitPlugin.deregisterShortCircuit(OperationType.DELETE, "PreParse");
- domain.setReplayGiveUpDelay(giveUpDelay);
+ resetReplayGiveUpDelay();
+ }
+ }
+ finally
+ {
+ broker.stop();
+ }
+ }
+
+ /**
+ * Test case for [Issue 901]: a replica whose replay give-up budget is unlimited keeps
+ * asking for a change it can not replay instead of ever recording it as replayed, and
+ * the budget is read from the configuration for every decision - so lowering it takes
+ * effect on the change which is failing right now, without this server being restarted.
+ */
+ @Test
+ public void anUnlimitedReplayGiveUpDelayIsNeverSpent() throws Exception
+ {
+ testSetUp("anUnlimitedReplayGiveUpDelayIsNeverSpent");
+ logger.error(LocalizableMessage.raw("Starting replication test : anUnlimitedReplayGiveUpDelayIsNeverSpent"));
+
+ final int serverId = 18;
+ ReplicationBroker broker =
+ openReplicationSession(baseDN, serverId, 100, replServerPort, 1000);
+ try
+ {
+ CSNGenerator gen = new CSNGenerator(serverId, 0);
+
+ Entry tmp = TestCaseUtils.addEntry(
+ "dn: uid=user.901," + baseDN,
+ "objectClass: top",
+ "objectClass: person",
+ "objectClass: organizationalPerson",
+ "objectClass: inetOrgPerson",
+ "uid: user.901",
+ "cn: Aaccf Amar",
+ "sn: Amar");
+ String uuid = getEntry(tmp.getName(), 1, true).parseAttribute("entryuuid").asString();
+
+ final LDAPReplicationDomain domain = MultimasterReplication.findDomain(baseDN, null);
+ final long initialFailures = getMonitorAttrValue(baseDN, "replayed-updates-failed");
+ // An operator who would rather have this domain stop than have it diverge: the
+ // change is retried for as long as it keeps failing.
+ setReplayGiveUpDelay("unlimited");
+ try
+ {
+ ShortCircuitPlugin.registerShortCircuit(
+ OperationType.DELETE, "PreParse", ResultCode.OTHER.intValue());
+
+ final CSN csn = gen.newCSN();
+ broker.publish(new DeleteMsg(tmp.getName(), csn, uuid));
+
+ /*
+ * One delivery burns IN_PLACE_REPLAY_ATTEMPTS short circuits before it is handed
+ * back and the session is restarted, so twice that many of them is a change which
+ * was delivered, given back and delivered again - the loop this replica is
+ * deliberately left in.
+ */
+ TestTimer timer = new TestTimer.Builder()
+ .maxSleep(60, SECONDS)
+ .sleepTimes(100, MILLISECONDS)
+ .toTimer();
+ timer.repeatUntilSuccess(new CallableVoid()
+ {
+ @Override
+ public void call() throws Exception
+ {
+ assertTrue(ShortCircuitPlugin.getShortCircuitCount(OperationType.DELETE, "PreParse")
+ > 2 * IN_PLACE_REPLAY_ATTEMPTS,
+ "the change was not asked for again while the budget was unlimited");
+ }
+ });
+ assertFalse(domain.getServerState().cover(csn),
+ "a replica which never gives up must not record a change it did not apply");
+ /*
+ * The monitor attribute is the only signal this domain has left: a budget which
+ * is never spent raises no alert and counts no failed replay, so the change this
+ * replica keeps asking for is visible here and nowhere else. It counts changes
+ * rather than deliveries, so the redeliveries above leave it at one.
+ */
+ assertMonitorAttrValueEventually(baseDN, "changes-with-failed-replay", 1,
+ "the change which keeps failing must be counted, once, as a failing change");
+ /*
+ * The change was delivered at least twice by now, so a counter which was bumped
+ * per delivery rather than per change would already be past the value which is
+ * being watched: what this asserts is carried by the value itself rather than by
+ * how long the window is, which is why the default number of samples is enough
+ * here - a window covering a redelivery would have to outlast a backoff which has
+ * been climbing since the first failure.
+ */
+ assertMonitorAttrValueStays(baseDN, "replayed-updates-failed", initialFailures,
+ "no change may be counted as failed while the budget is unlimited");
+
+ /*
+ * The budget is read for every decision, so the failure which comes next spends
+ * this one: the change this replica was holding on to is given up on without the
+ * server, or the domain, being restarted for the new value to be seen.
+ */
+ setReplayGiveUpDelay("0ms");
+ TestTimer giveUpTimer = new TestTimer.Builder()
+ .maxSleep(120, SECONDS)
+ .sleepTimes(200, MILLISECONDS)
+ .toTimer();
+ giveUpTimer.repeatUntilSuccess(new CallableVoid()
+ {
+ @Override
+ public void call() throws Exception
+ {
+ assertTrue(domain.getServerState().cover(csn),
+ "a budget which was lowered must be spent by the change which is failing");
+ }
+ });
+ assertMonitorAttrValueEventually(baseDN, "replayed-updates-failed", initialFailures + 1,
+ "the change which was given up on must be counted once");
+ assertMonitorAttrValueEventually(baseDN, "changes-with-failed-replay", 0,
+ "a change which was given up on leaves the pending changes and stops being counted");
+ assertNotNull(getEntry(tmp.getName(), 1, true), "the entry must not have been deleted");
+ }
+ finally
+ {
+ ShortCircuitPlugin.deregisterShortCircuit(OperationType.DELETE, "PreParse");
+ resetReplayGiveUpDelay();
}
}
finally
@@ -2543,6 +2665,39 @@
}
/**
+ * Sets how long the domain of this test retries the replay of a change before it gives
+ * up on it, the way an administrator would.
+ * <p>
+ * A backend under maintenance is waited out for minutes, which no test can afford: the
+ * budget is spent over a couple of deliveries instead. The domain reads the property for
+ * every decision it makes, so this takes effect on a change which is failing right now,
+ * and none of the values it takes stops or starts the session.
+ * <p>
+ * The domain outlives the test methods, so a test which shortens the budget puts it back
+ * with {@link #resetReplayGiveUpDelay()} in a finally, and calls this one before that
+ * try: the reset then only ever runs on an attribute which is there to be removed.
+ *
+ * @param delay
+ * the budget in the duration syntax of the property: {@code 2000ms},
+ * {@code 0ms} to give up as soon as one delivery of the change failed,
+ * {@code unlimited} never to give up on it
+ */
+ private void setReplayGiveUpDelay(String delay)
+ {
+ modifyDomainConfig(synchroServerEntry.getName(), REPLACE, ATTR_REPLAY_GIVE_UP_DELAY, delay);
+ }
+
+ /**
+ * Puts the configured replay give-up budget of the domain of this test back, that is the
+ * default of the property: the domain outlives the test methods, so a shortened budget
+ * which is left behind is the next test's too.
+ */
+ private void resetReplayGiveUpDelay()
+ {
+ modifyDomainConfig(synchroServerEntry.getName(), DELETE, ATTR_REPLAY_GIVE_UP_DELAY);
+ }
+
+ /**
* Enable or disable the receive status of a synchronization provider.
*
* @param syncConfigDN The DN of the synchronization provider configuration
@@ -2552,10 +2707,7 @@
*/
private static void setReceiveStatus(DN syncConfigDN, boolean enable)
{
- String attrValue = enable ? "TRUE" : "FALSE";
- ModifyRequest request = modifyRequest(syncConfigDN, REPLACE, "ds-cfg-receive-status", attrValue);
- ModifyOperation modOp = getRootConnection().processModify(request);
- assertEquals(modOp.getResultCode(), ResultCode.SUCCESS, "Cannot set receive status");
+ modifyDomainConfig(syncConfigDN, REPLACE, "ds-cfg-receive-status", enable ? "TRUE" : "FALSE");
}
/**
diff --git a/opendj-server-legacy/src/test/java/org/opends/server/replication/plugin/AssuredReplicationPluginTest.java b/opendj-server-legacy/src/test/java/org/opends/server/replication/plugin/AssuredReplicationPluginTest.java
index a657149..2b3f225 100644
--- a/opendj-server-legacy/src/test/java/org/opends/server/replication/plugin/AssuredReplicationPluginTest.java
+++ b/opendj-server-legacy/src/test/java/org/opends/server/replication/plugin/AssuredReplicationPluginTest.java
@@ -211,11 +211,17 @@
}
/**
- * Creates a domain using the passed assured settings.
+ * Creates a domain using the passed assured settings, plus the passed configuration
+ * lines, which are put on the entry the domain is created from.
* Returns the matching config entry added to the config backend.
+ * <p>
+ * A property the domain must have from the start belongs here rather than in a
+ * configuration change made afterwards: a change stops and starts the session when what
+ * it changes calls for it, and the FakeReplicationServer these domains talk to is not
+ * built to be reconnected to.
*/
private Entry createAssuredDomain(AssuredMode assuredMode, int safeDataLevel,
- long assuredTimeout) throws Exception
+ long assuredTimeout, String... extraConfigLdifLines) throws Exception
{
String baseDn = null;
switch (assuredMode)
@@ -242,6 +248,10 @@
// heartbeat = 10 min so no need to emulate heartbeat in fake RS: session
// not closed by client
"ds-cfg-changetime-heartbeat-interval: 0ms\n";
+ for (String extraLine : extraConfigLdifLines)
+ {
+ prefixLdif += extraLine + "\n";
+ }
String configEntryLdif = null;
switch (assuredMode)
@@ -1194,7 +1204,18 @@
true, testcase);
replicationServer.start(NO_READ);
- safeReadDomainCfgEntry = createAssuredDomain(AssuredMode.SAFE_READ_MODE, 0, TIMEOUT);
+ /*
+ * A change which keeps failing is asked for again over a restarted session until
+ * this replica gives up on it. That is right, and it is not what this test is
+ * about: the FakeReplicationServer is not built to be reconnected to, and the
+ * restarts would run on while the assertions and the teardown below take their
+ * course. A give-up delay of zero has the first failed delivery spend the whole
+ * budget, so the change is given up on where it is reported and no session is
+ * restarted: the ack of this delivery is published either way - it is sent before
+ * the give-up is decided - and the domain settles instead of reconnecting.
+ */
+ safeReadDomainCfgEntry = createAssuredDomain(AssuredMode.SAFE_READ_MODE, 0, TIMEOUT,
+ "ds-cfg-replay-give-up-delay: 0ms");
waitForConnectionToRs(testcase, replicationServer);
Entry entry = makeEntry(
@@ -1203,9 +1224,6 @@
"objectClass: organizationalUnit");
String parentUid = getEntryUUID(DN.valueOf(SAFE_READ_DN));
- final LDAPReplicationDomain domain =
- MultimasterReplication.findDomain(DN.valueOf(SAFE_READ_DN), null);
- final long giveUpDelay = domain.getReplayGiveUpDelay();
try
{
/*
@@ -1219,17 +1237,6 @@
*/
ShortCircuitPlugin.registerShortCircuit(
OperationType.ADD, "PreParse", ResultCode.OTHER.intValue());
- /*
- * A change which keeps failing is asked for again over a restarted session until
- * this replica gives up on it. That is right, and it is not what this test is
- * about: the FakeReplicationServer is not built to be reconnected to, and the
- * restarts would run on while the assertions and the teardown below take their
- * course. A give-up delay of zero has the first failure spend the whole budget, so
- * the change is given up on where it is reported and no session is restarted: the
- * ack of this delivery is published either way - it is sent before the give-up is
- * decided - and the domain settles instead of reconnecting.
- */
- domain.setReplayGiveUpDelay(0);
AckMsg ackMsg = replicationServer.sendAssuredAddMsg(entry, parentUid);
assertNull(DirectoryServer.getEntry(entry.getName()), "the entry must not have been added");
@@ -1258,7 +1265,6 @@
}
finally
{
- domain.setReplayGiveUpDelay(giveUpDelay);
ShortCircuitPlugin.deregisterShortCircuit(OperationType.ADD, "PreParse");
}
} finally
diff --git a/opendj-server-legacy/src/test/java/org/opends/server/replication/plugin/DomainFakeCfg.java b/opendj-server-legacy/src/test/java/org/opends/server/replication/plugin/DomainFakeCfg.java
index 43299a2..43993fc 100644
--- a/opendj-server-legacy/src/test/java/org/opends/server/replication/plugin/DomainFakeCfg.java
+++ b/opendj-server-legacy/src/test/java/org/opends/server/replication/plugin/DomainFakeCfg.java
@@ -13,6 +13,7 @@
*
* Copyright 2007-2010 Sun Microsystems, Inc.
* Portions Copyright 2011-2016 ForgeRock AS.
+ * Portions Copyright 2026 3A Systems, LLC.
*/
package org.opends.server.replication.plugin;
@@ -251,6 +252,20 @@
return assuredType;
}
+ /**
+ * Returns how long the replay of a change is retried before the domain gives up on it.
+ * <p>
+ * The default the property was given, spelled out here like the other values of this
+ * fake configuration: no test varies the budget on a domain built from one.
+ *
+ * @return the budget in milliseconds
+ */
+ @Override
+ public long getReplayGiveUpDelay()
+ {
+ return 300000;
+ }
+
@Override
public SortedSet<String> getReferralsUrl()
{
--
Gitblit v1.10.0