/* * The contents of this file are subject to the terms of the Common Development and * Distribution License (the License). You may not use this file except in compliance with the * License. * * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the * specific language governing permission and limitations under the License. * * When distributing Covered Software, include this CDDL Header Notice in each file and include * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL * Header, with the fields enclosed by brackets [] replaced by your own identifying * information: "Portions Copyright [year] [name of copyright owner]". * * Copyright 2006-2008 Sun Microsystems, Inc. * Portions Copyright 2011-2016 ForgeRock AS. * Portions Copyright 2026 3A Systems, LLC. */ package org.opends.server.replication.plugin; import static org.opends.messages.ReplicationMessages.*; import static org.opends.server.util.StaticUtils.*; import java.util.concurrent.BlockingQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; import org.opends.server.api.DirectoryThread; import org.forgerock.i18n.slf4j.LocalizedLogger; import org.opends.server.replication.protocol.LDAPUpdateMsg; /** * Thread that is used to get message from the replication servers (stored * in the updates queue) and replay them in the current server. A configurable * number of this thread is created for the whole MultimasterReplication object * (i.e: these threads are shared across the ReplicationDomain objects for * replaying the updates they receive) */ public class ReplayThread extends DirectoryThread { private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass(); /** * The give-back a thread runs on every domain of this server on its way out, held here * rather than written where it is run: a method reference is linked, and its instance * made, where it is first run, and this one is first run on the way out of a thread - * which an OutOfMemoryError may be ending, on the road this give-back is there for. Made * when this class is loaded instead, on a thread which can allocate (issue #986). */ private static final Consumer GIVE_BACK_PARKED_CHANGES = LDAPReplicationDomain::giveBackChangesParkedByStoppingThread; private final BlockingQueue updateToReplayQueue; private final ReentrantLock switchQueueLock; private AtomicBoolean shutdown = new AtomicBoolean(false); private static int count; /** * Constructor for the ReplayThread. * * @param updateToReplayQueue The queue of update messages we have to replay * @param switchQueueLock lock to ensure moving updates from one queue to another is atomic */ public ReplayThread(BlockingQueue updateToReplayQueue, ReentrantLock switchQueueLock) { super("Replica replay thread " + count++); this.updateToReplayQueue = updateToReplayQueue; this.switchQueueLock = switchQueueLock; } /** * Shutdown this replay thread. */ public void shutdown() { shutdown.set(true); } /** * Run method for this class. */ @Override public void run() { if (logger.isTraceEnabled()) { logger.trace("Replication Replay thread starting."); } try { replayUntilStopped(); } finally { /* * The changes this thread parked as waiting for another change are handed out again * by getNextUpdate() alone, which every replay loop of a domain runs once it is done * with a change: a parked change is replayed by whichever thread clears the change it * was waiting for. A thread which is stopping is not on that road anymore, so what it * parked would be left owned by a thread which does not exist, while every redelivery * of a change a replay thread owns is refused as a duplicate: on a domain which then * goes quiet that change is where the ServerState of this replica, and every change * behind it from every master, stops (issue #986). * * Given back by the thread which owns them, so that the rule every road which reads * ownership follows holds on this one as well: a change is given back by the thread it * was handed to and by nobody else (issue #922). It is also the one place which sees * them all - the pool is shared by every domain of this server, while a replay knows * only the domain it was replaying for. * * The session which brings them back is asked for and left standing, in every domain * which got something back, and the state checkpointer of each of them runs it within * its tick: a thread on its way out is not held for a session - the threads of the * pool are stopped one after the other and joined, and each running a restart of its * own would have the configuration change which is stopping them wait for one restart * per thread - and a change delivered again before the pool which replaces this one * is up waits in the replay queue for it. A thread which an OutOfMemoryError is ending * gives back here what it parked in the domains it was not replaying for, on the same * terms; the change it was replaying, and what it had parked in that same domain, were * given back and asked for again on its way out of replay(). */ giveBackParkedChanges(); } if (logger.isTraceEnabled()) { logger.trace("Replication Replay thread stopping."); } } /** * Takes the deliveries of the domains of this server off the shared replay queue and * replays them, until this thread is stopped. */ private void replayUntilStopped() { while (!shutdown.get()) { try { if (switchQueueLock.tryLock(1L, TimeUnit.SECONDS)) { LDAPReplicationDomain domain; LDAPUpdateMsg updateMsg; try { if (shutdown.get()) { break; } UpdateToReplay updateToreplay = updateToReplayQueue.poll(1L, TimeUnit.SECONDS); if (updateToreplay == null) { continue; } // Find replication domain for that update message and mark it as "in progress" updateMsg = updateToreplay.getUpdateMessage(); domain = updateToreplay.getReplicationDomain(); if (!domain.markInProgress(updateMsg)) { /* * The domain restarted its session after a failed replay while this * message was waiting here, so it does not know about this change * anymore: the replication server sends it again over the new session. */ continue; } } finally { switchQueueLock.unlock(); } domain.replay(updateMsg, shutdown); } } catch (OutOfMemoryError e) { /* * The JVM is out of memory, which is not something to carry on replaying from: this * thread does not stay for the changes which follow. Nothing is reported here - the * uncaught exception handler of DirectoryThread is what says this thread is gone, * with an alert - and the change it was replaying has been given back, counted and * asked for again by the domain on its way out (issue #922). * * The other errors of the JVM are caught below: a StackOverflowError is gone once * the stack has unwound, and a thread which ends here is one nothing replaces. */ throw e; } catch (Throwable t) { /* * catch all exceptions happening so that the thread never dies even * in case of problems. * * An Error is not an Exception, so one raised here used to unwind run() and end * this thread. Nothing creates a replay thread to replace it - the pool is created * when the first domain of this server is - so the shared replay queue would have * one consumer fewer for every domain, for as long as the server is up, until it * has none left and replication stops (issue #923). */ logger.error(ERR_EXCEPTION_REPLAYING_REPLICATION_MESSAGE, stackTraceToSingleLineString(t)); } } } /** * Gives back the changes this thread parked as waiting for another change, in every * domain of this server. *

* A change which is given back stays listed and uncommitted, the way a change whose replay * failed does: it is not in the data, so it holds the ServerState of its domain back and * the changes which follow it keep waiting for it, until the delivery which takes it over * replays it. *

* Every domain gets its turn whatever one of them threw: what can throw here is an * allocation, on the way out of a thread an OutOfMemoryError may be ending - the iterator * over the domains, before any of them is reached, then for each of them the list of what * it released, made before anything is released, or the report of a change once it is, * and between two domains the list a second failure is recorded in under the first, which * the loop guards on its own - and the domains which follow would otherwise be left with * changes owned by a thread which does not exist anymore, the state this give-back is * for. A domain which threw past the release has asked for its restart already: the * request is made before the report. The first failure is thrown once the loop is over, * so that the uncaught exception handler of {@link DirectoryThread} writes the line and * raises the alert. */ private void giveBackParkedChanges() { MultimasterReplication.forEachDomain(GIVE_BACK_PARKED_CHANGES); } }