* These tests need no server: the changes are built by a CSNGenerator, which needs nothing but * a server id. */ @SuppressWarnings("javadoc") @Test(groups = { "precommit", "replication" }, sequential = true) public class PendingChangesTest extends DirectoryServerTestCase { private static final int SERVER_ID = 42; /** A peer replication server the collocated one relays the message to. */ private static final int RS_ID = 11; private static DN baseDN; @BeforeClass public static void classSetup() throws Exception { baseDN = DN.valueOf("dc=example,dc=com"); } @Test public void replicaOfflineMsgTheBrokerPublishedIsReportedAsSent() throws Exception { final ReplicationDomain domain = domainWhichPublishes(true); final PendingChanges pendingChanges = newPendingChanges(domain); final CSN offlineCSN = pendingChanges.putReplicaOfflineMsg(); assertNotNull(offlineCSN, "the message was published and must be reported as sent"); final UpdateMsg published = onlyMsgPublishedBy(domain); assertTrue(published instanceof ReplicaOfflineMsg, "published " + published); assertEquals(published.getCSN(), offlineCSN); } /** * The collocated replication server forwards the message as soon as it is on the wire, so an * announcement made after the publish is one the forward found nothing to clear: nothing will * ever remove it, and the shutdown waits out its whole grace period for a message which the * topology already has. *
* The forward is reported from inside publish(), which is where the message reaches the
* session, so the race is reproduced rather than waited for.
*/
@Test
public void theReplicaOfflineMsgIsAnnouncedBeforeItIsPublished() throws Exception
{
final DSRSShutdownSync shutdownSync = new DSRSShutdownSync();
final ReplicationDomain domain = mock(ReplicationDomain.class);
forwardWhilePublishing(domain, shutdownSync);
final PendingChanges pendingChanges = newPendingChanges(domain, shutdownSync);
pendingChanges.putReplicaOfflineMsg();
assertTrue(shutdownSync.canShutdown(baseDN),
"the message was forwarded, so nothing must hold the shutdown back any longer");
}
/**
* The announcement of a message the broker took stands until a peer forwards it: the
* withdrawal is for the message the broker refused, and an announcement taken back after a
* publish which succeeded would leave the shutdown nothing to wait for.
*/
@Test
public void theAnnouncementOfAPublishedMessageStandsUntilItIsForwarded() throws Exception
{
final DSRSShutdownSync shutdownSync = new DSRSShutdownSync();
final PendingChanges pendingChanges =
newPendingChanges(domainWhichPublishes(true), shutdownSync);
pendingChanges.putReplicaOfflineMsg();
assertFalse(shutdownSync.canShutdown(baseDN),
"the message went out and nobody has forwarded it yet, so the shutdown must wait for it");
}
/**
* The broker writes nothing when it has no usable session, when the changes which come before
* this one still have to be republished by the recovery, or when it is stopped in between - and
* what was not written must not be reported as sent: the shutdown of a collocated replication
* server waits out the whole grace period of a message it was told about and which never
* reached the wire.
*/
@Test
public void replicaOfflineMsgTheBrokerRefusedIsNotReportedAsSent() throws Exception
{
final ReplicationDomain domain = domainWhichPublishes(false);
final PendingChanges pendingChanges = newPendingChanges(domain);
assertNull(pendingChanges.putReplicaOfflineMsg(), "the broker refused the message");
assertTrue(onlyMsgPublishedBy(domain) instanceof ReplicaOfflineMsg, "it was attempted");
}
/**
* The announcement is made before the message is published, and the broker may refuse it
* once it is: an announcement which stayed would be one nobody will ever forward, and the
* shutdown would wait out its whole grace period for a message which never left. It is
* therefore withdrawn - and it is a withdrawal, not an announcement which was never made: the
* shutdown is held back while the broker holds the message.
*/
@Test
public void theAnnouncementOfAReplicaOfflineMsgTheBrokerRefusedIsWithdrawn() throws Exception
{
final DSRSShutdownSync shutdownSync = new DSRSShutdownSync();
final ReplicationDomain domain = mock(ReplicationDomain.class);
refuseWhilePublishing(domain, shutdownSync);
final PendingChanges pendingChanges = newPendingChanges(domain, shutdownSync);
assertNull(pendingChanges.putReplicaOfflineMsg(), "the broker refused the message");
assertTrue(shutdownSync.canShutdown(baseDN),
"the message never reached the wire, so nothing must hold the shutdown back");
}
/**
* The message carries the newest CSN of the replica, so a change which is still in flight
* holds it back, and the broker is never even asked to publish it.
*/
@Test
public void replicaOfflineMsgQueuedBehindAnUncommittedChangeIsNotReportedAsSent() throws Exception
{
final ReplicationDomain domain = domainWhichPublishes(true);
final PendingChanges pendingChanges = newPendingChanges(domain);
pendingChanges.putLocalOperation(newLocalOperation());
assertNull(pendingChanges.putReplicaOfflineMsg(), "nothing was published");
verify(domain, never()).publish(any(UpdateMsg.class));
}
/**
* A message which could not be published is given up on rather than left queued: the replica
* which could not announce itself offline is either shutting down, and the message dies with
* the process, or it is being disabled for an import or a configuration change - and once it
* comes back, announcing it offline on the session which follows would be a lie.
*/
@Test
public void replicaOfflineMsgWhichCouldNotBeSentIsNotPublishedLater() throws Exception
{
final ReplicationDomain domain = domainWhichPublishes(true);
final PendingChanges pendingChanges = newPendingChanges(domain);
final CSN changeCSN = pendingChanges.putLocalOperation(newLocalOperation());
assertNull(pendingChanges.putReplicaOfflineMsg(), "nothing was published");
// The change which held the message back completes.
pendingChanges.commitAndPushCommittedChanges(changeCSN, mock(LDAPUpdateMsg.class));
final UpdateMsg published = onlyMsgPublishedBy(domain);
assertTrue(published instanceof LDAPUpdateMsg, "published " + published);
}
/**
* The announcement follows the publication rather than the queueing, so a message which a
* change in flight holds back is not announced: neither while it waits, nor when the change
* which held it back completes and the message is given up on. Announcing it either time
* would leave the shutdown waiting out its whole grace period for a forward which cannot
* happen.
*/
@Test
public void theReplicaOfflineMsgHeldBackByAChangeInFlightIsNeverAnnounced() throws Exception
{
final DSRSShutdownSync shutdownSync = new DSRSShutdownSync();
final ReplicationDomain domain = domainWhichPublishes(true);
final PendingChanges pendingChanges = newPendingChanges(domain, shutdownSync);
final CSN inFlight = pendingChanges.putLocalOperation(newLocalOperation());
pendingChanges.putReplicaOfflineMsg();
assertTrue(shutdownSync.canShutdown(baseDN),
"the message is still queued behind a change in flight, and nothing was announced");
pendingChanges.commitAndPushCommittedChanges(inFlight, mock(LDAPUpdateMsg.class));
assertTrue(shutdownSync.canShutdown(baseDN),
"the message was given up on with the change which held it back, and never announced");
}
/**
* A change the broker refused leaves the pending changes all the same: the replica has done
* it, its ServerState says so, and it is by finding that state ahead of the one its
* replication server reports that the next session republishes the change from the historical
* information of its entry. Only the offline announcement, which is stored nowhere, needs the
* answer of the broker.
*/
@Test
public void changeTheBrokerRefusedStillLeavesThePendingChanges() throws Exception
{
final ReplicationDomain domain = domainWhichPublishes(false);
final PendingChanges pendingChanges = newPendingChanges(domain);
final CSN changeCSN = pendingChanges.putLocalOperation(newLocalOperation());
assertEquals(pendingChanges.size(), 1);
pendingChanges.commitAndPushCommittedChanges(changeCSN, mock(LDAPUpdateMsg.class));
assertTrue(onlyMsgPublishedBy(domain) instanceof LDAPUpdateMsg, "the change was published");
assertEquals(pendingChanges.size(), 0, "and is not queued for a second attempt");
}
/**
* The domain keeps the messages of its last local changes, in order, so that a recovery publishes
* them as they were done rather than rebuild them from the historical information, which holds
* nothing of a change a later one replaced (issue #1246).
*/
@Test
public void theMessagesOfTheLastLocalChangesAreKeptInOrder() throws Exception
{
final PendingChanges pendingChanges = newPendingChanges(domainWhichPublishes(true));
final LDAPUpdateMsg first = pushLocalChange(pendingChanges);
final LDAPUpdateMsg second = pushLocalChange(pendingChanges);
final LDAPUpdateMsg third = pushLocalChange(pendingChanges);
assertEquals(pendingChanges.recentChangesAfter(first.getCSN(), third.getCSN()).messages,
Arrays.asList(second, third));
assertEquals(pendingChanges.recentChangesAfter(first.getCSN(), second.getCSN()).messages,
Arrays.asList(second));
}
/**
* The messages kept go back to the CSN right before the first one: a recovery from there - from the
* change before it, done in the same millisecond - gets the first change as it was done, rather than
* rebuilt from the historical information.
*/
@Test
public void theMessagesKeptGoBackToRightBeforeTheFirstOne() throws Exception
{
final PendingChanges pendingChanges = newPendingChanges(domainWhichPublishes(true));
final LDAPUpdateMsg first = pushLocalChange(pendingChanges);
assertEquals(pendingChanges.recentChangesAfter(rightBefore(first), first.getCSN()).messages,
Arrays.asList(first));
}
/**
* The messages kept of a first change numbered 0 go back to the greatest sequence number of the
* millisecond before, which no CSN of the server comes after before that change: a recovery from there
* gets the change as it was done. A CSN which went back to the same millisecond would come after the
* change, and the recovery would rebuild it from the historical information instead.
*/
@Test
public void theMessagesKeptOfAChangeNumberedZeroGoBackToTheMillisecondBefore() throws Exception
{
final ServerState state = new ServerState();
// The generator goes on from the greatest sequence number, so its first change wraps to 0.
state.update(new CSN(0, Integer.MAX_VALUE, SERVER_ID));
final PendingChanges pendingChanges = new PendingChanges(new CSNGenerator(SERVER_ID, state),
domainWhichPublishes(true), new ShutdownSyncAnnouncer(new DSRSShutdownSync(), baseDN));
final LDAPUpdateMsg first = pushLocalChange(pendingChanges);
assertEquals(first.getCSN().getSeqnum(), 0);
final CSN justBefore = new CSN(first.getCSN().getTime() - 1, Integer.MAX_VALUE, SERVER_ID);
assertEquals(pendingChanges.recentChangesAfter(new CSN(0, 0, SERVER_ID), first.getCSN()).keptAfter,
justBefore);
assertEquals(pendingChanges.recentChangesAfter(justBefore, first.getCSN()).messages,
Arrays.asList(first));
}
/**
* Past as many changes as it keeps at most, the domain drops the oldest messages, and tells where
* the ones it kept begin: a recovery rebuilds the changes before that from the historical
* information.
*/
@Test
public void pastTheMostChangesKeptTheOldestAreDropped() throws Exception
{
final PendingChanges pendingChanges = newPendingChanges(domainWhichPublishes(true));
pendingChanges.setMaxRecentChanges(2);
final LDAPUpdateMsg first = pushLocalChange(pendingChanges);
final LDAPUpdateMsg second = pushLocalChange(pendingChanges);
final LDAPUpdateMsg third = pushLocalChange(pendingChanges);
assertEquals(pendingChanges.recentChangesAfter(first.getCSN(), third.getCSN()).messages,
Arrays.asList(second, third));
final PendingChanges.RecentChanges fromBefore =
pendingChanges.recentChangesAfter(new CSN(0, 0, SERVER_ID), third.getCSN());
assertNull(fromBefore.messages, "the messages kept do not go back that far");
assertEquals(fromBefore.keptAfter, first.getCSN());
}
/** Past as much of the heap as their messages take at most, the domain drops the oldest messages. */
@Test
public void pastTheMostBytesKeptTheOldestAreDropped() throws Exception
{
assertTrue(PendingChanges.MAX_RECENT_BYTES <= Runtime.getRuntime().maxMemory() / 16,
"a domain must not reserve more than a sixteenth of the heap for recent messages");
final PendingChanges pendingChanges = newPendingChanges(domainWhichPublishes(true));
final int half = sizeTaking(PendingChanges.MAX_RECENT_BYTES / 2);
final LDAPUpdateMsg first = pushLocalChange(pendingChanges, half);
final LDAPUpdateMsg second = pushLocalChange(pendingChanges, half);
assertEquals(pendingChanges.recentChangesAfter(rightBefore(first), second.getCSN()).messages,
Arrays.asList(first, second));
final LDAPUpdateMsg third = pushLocalChange(pendingChanges, 0);
assertEquals(pendingChanges.recentChangesAfter(first.getCSN(), third.getCSN()).messages,
Arrays.asList(second, third));
assertEquals(pendingChanges.recentChangesAfter(rightBefore(first), third.getCSN()).keptAfter, first.getCSN());
}
/**
* A change pushed again - the publish of the push before threw, and left it pending - is counted
* once: counted twice, it took more of the heap than it does, and was dropped though it was the only
* message kept.
*/
@Test
public void aChangePushedAgainIsCountedOnce() throws Exception
{
final ReplicationDomain domain = mock(ReplicationDomain.class);
when(domain.publish(any(UpdateMsg.class))).thenThrow(new IllegalStateException("the publish failed"))
.thenReturn(true);
final PendingChanges pendingChanges = newPendingChanges(domain);
final CSN csn = pendingChanges.putLocalOperation(newLocalOperation());
final LDAPUpdateMsg large = newMsg(csn, sizeTaking(PendingChanges.MAX_RECENT_BYTES * 3 / 5));
try
{
pendingChanges.commitAndPushCommittedChanges(csn, large);
fail("the publish threw");
}
catch (IllegalStateException expected)
{
// The change is still pending, and the next push pushes it again.
}
final LDAPUpdateMsg next = pushLocalChange(pendingChanges, 0);
assertEquals(pendingChanges.recentChangesAfter(rightBefore(large), next.getCSN()).messages,
Arrays.asList(large, next));
}
/**
* A change the pending changes leave to a recovery is kept as well: the recovery of a later
* session, whose replication server misses it, publishes it as it was done.
*/
@Test
public void aChangeLeftToARecoveryIsKeptAsWell() throws Exception
{
final ReplicationDomain domain = domainWhichPublishes(true);
when(domain.getServerState()).thenReturn(new ServerState());
final PendingChanges pendingChanges = newPendingChanges(domain);
final LDAPUpdateMsg published = pushLocalChange(pendingChanges);
pendingChanges.setRecovering(true);
final LDAPUpdateMsg kept = pushLocalChange(pendingChanges);
assertEquals(onlyMsgPublishedBy(domain), published);
assertEquals(pendingChanges.recentChangesAfter(published.getCSN(), kept.getCSN()).messages,
Arrays.asList(kept));
}
/**
* The messages kept are forgotten once the domain reads its state from data which may lack their
* changes, and none of the changes before is taken for kept: the state read back can be older than
* changes done since the domain started. The messages of the changes pushed from there are kept.
*/
@Test
public void theMessagesKeptAreForgotten() throws Exception
{
final ReplicationDomain domain = domainWhichPublishes(true);
final PendingChanges pendingChanges = newPendingChanges(domain);
final CSN first = pushLocalChange(pendingChanges).getCSN();
final CSN last = pushLocalChange(pendingChanges).getCSN();
// The state read back is older than the last change.
when(domain.getLastLocalChange()).thenReturn(first);
pendingChanges.forgetRecentChanges();
final PendingChanges.RecentChanges fromFirst = pendingChanges.recentChangesAfter(first, last);
assertNull(fromFirst.messages, "the messages were forgotten");
assertNull(fromFirst.keptAfter, "and the change after the last one of the state is not taken for kept");
final LDAPUpdateMsg next = pushLocalChange(pendingChanges);
final LDAPUpdateMsg afterNext = pushLocalChange(pendingChanges);
assertNull(pendingChanges.recentChangesAfter(first, afterNext.getCSN()).messages,
"the change forgotten is not taken for kept");
assertEquals(pendingChanges.recentChangesAfter(next.getCSN(), afterNext.getCSN()).messages,
Arrays.asList(afterNext));
}
@Test
public void aCommittedChangeHeldBehindAnotherIsNotKeptAfterTheDataWasReplaced() throws Exception
{
final PendingChanges pending = newPendingChanges(domainWhichPublishes(false));
final CSN first = pending.putLocalOperation(newLocalOperation());
final CSN second = pending.putLocalOperation(newLocalOperation());
final LDAPUpdateMsg secondMsg = newMsg(second, 0);
pending.commitAndPushCommittedChanges(second, secondMsg);
pending.forgetRecentChanges();
pending.commitAndPushCommittedChanges(first, newMsg(first, 0));
final LDAPUpdateMsg after = pushLocalChange(pending);
assertNull(pending.recentChangesAfter(first, after.getCSN()).messages,
"the change made on the replaced data must not be treated as kept");
assertEquals(pending.recentChangesAfter(rightBefore(after), after.getCSN()).messages, Arrays.asList(after));
}
@Test
public void anUncommittedChangeAtDataReloadIsNotKeptAfterItCommits() throws Exception
{
final PendingChanges pending = newPendingChanges(domainWhichPublishes(false));
final CSN old = pending.putLocalOperation(newLocalOperation());
pending.forgetRecentChanges();
pending.commitAndPushCommittedChanges(old, newMsg(old, 0));
final LDAPUpdateMsg after = pushLocalChange(pending);
assertNull(pending.recentChangesAfter(rightBefore(newMsg(old, 0)), after.getCSN()).messages,
"a change pending at reload must not be published from the recent messages");
assertEquals(pending.recentChangesAfter(rightBefore(after), after.getCSN()).messages, Arrays.asList(after));
}
/** Pushes a local change whose message takes no bytes, and returns its message. */
private LDAPUpdateMsg pushLocalChange(PendingChanges pendingChanges)
{
return pushLocalChange(pendingChanges, 0);
}
/** Pushes a local change whose message has the provided size, and returns its message. */
private LDAPUpdateMsg pushLocalChange(PendingChanges pendingChanges, int size)
{
final CSN csn = pendingChanges.putLocalOperation(newLocalOperation());
final LDAPUpdateMsg msg = newMsg(csn, size);
pendingChanges.commitAndPushCommittedChanges(csn, msg);
return msg;
}
private static LDAPUpdateMsg newMsg(CSN csn, int size)
{
final LDAPUpdateMsg msg = mock(LDAPUpdateMsg.class);
when(msg.getCSN()).thenReturn(csn);
when(msg.size()).thenReturn(size);
return msg;
}
/** Returns the size of a message which takes the provided part of the heap once kept. */
private static int sizeTaking(long footprint)
{
return (int) ((footprint - PendingChanges.MESSAGE_OVERHEAD) / 2);
}
/** Returns the CSN right before the one of the message: the change before it, in the same millisecond. */
private static CSN rightBefore(LDAPUpdateMsg msg)
{
return new CSN(msg.getCSN().getTime(), msg.getCSN().getSeqnum() - 1, SERVER_ID);
}
/**
* Reports the forward of the message from within the publish which puts it on the wire, after
* checking from there that the announcement is already in place, and publishes it. The forward
* is the one of a peer the message was never recorded as queued for, which is what a forward
* racing the announcement looks like.
*/
private void forwardWhilePublishing(
final ReplicationDomain domain, final DSRSShutdownSync shutdownSync)
{
doAnswer(invocation -> {
final UpdateMsg msg = (UpdateMsg) invocation.getArguments()[0];
if (msg instanceof ReplicaOfflineMsg)
{
assertFalse(shutdownSync.canShutdown(baseDN),
"the message must be announced before it is published");
shutdownSync.replicaOfflineMsgForwarded(baseDN, msg.getCSN(), RS_ID);
}
return true;
}).when(domain).publish(any(UpdateMsg.class));
}
/**
* Refuses to publish the message, the way a broker with no usable session does, after checking
* from within the publish that the announcement is already in place.
*/
private void refuseWhilePublishing(
final ReplicationDomain domain, final DSRSShutdownSync shutdownSync)
{
doAnswer(invocation -> {
assertFalse(shutdownSync.canShutdown(baseDN),
"the message must be announced before it is published");
return false;
}).when(domain).publish(any(UpdateMsg.class));
}
private PendingChanges newPendingChanges(ReplicationDomain domain)
{
return newPendingChanges(domain, new DSRSShutdownSync());
}
/**
* The pending changes of a replica whose domain announces itself through the shutdown sync,
* with the announcer LDAPReplicationDomain hands its own pending changes.
*/
private PendingChanges newPendingChanges(
final ReplicationDomain domain, final DSRSShutdownSync shutdownSync)
{
return new PendingChanges(new CSNGenerator(SERVER_ID, 0), domain,
new ShutdownSyncAnnouncer(shutdownSync, baseDN));
}
/** A domain whose broker accepts, or refuses, whatever it is given to publish. */
private ReplicationDomain domainWhichPublishes(boolean accepted)
{
final ReplicationDomain domain = mock(ReplicationDomain.class);
when(domain.publish(any(UpdateMsg.class))).thenReturn(accepted);
return domain;
}
/** A local operation, i.e. one this replica must publish to the other replicas. */
private PluginOperation newLocalOperation()
{
final PluginOperation operation = mock(PluginOperation.class);
when(operation.isSynchronizationOperation()).thenReturn(false);
return operation;
}
/**
* The single message the domain was asked to publish, failing the test if it published
* anything else: a message which is not sent and a message which is sent twice are both the
* kind of mistake these tests are about.
*/
private UpdateMsg onlyMsgPublishedBy(ReplicationDomain domain)
{
final ArgumentCaptor