/*
* The contents of this file are subject to the terms of the Common Development and
* Distribution License (the License). You may not use this file except in compliance with the
* License.
*
* You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
* specific language governing permission and limitations under the License.
*
* When distributing Covered Software, include this CDDL Header Notice in each file and include
* the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
* Header, with the fields enclosed by brackets [] replaced by your own identifying
* information: "Portions copyright [year] [name of copyright owner]".
*
* Copyright 2026 3A Systems, LLC.
*/
package org.opends.server.replication.server;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.opends.messages.ReplicationMessages.ERR_DS_BADLY_DISCONNECTED;
import static org.opends.messages.ReplicationMessages.ERR_WRITER_UNEXPECTED_EXCEPTION;
import static org.opends.server.TestCaseUtils.TEST_ROOT_DN_STRING;
import java.io.Closeable;
import java.io.IOException;
import java.lang.reflect.Field;
import java.net.ServerSocket;
import java.net.Socket;
import java.net.SocketTimeoutException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import org.forgerock.opendj.ldap.DN;
import org.opends.server.replication.ReplicationTestCase;
import org.opends.server.replication.common.CSNGenerator;
import org.opends.server.replication.protocol.DeleteMsg;
import org.opends.server.replication.protocol.ProtocolVersion;
import org.opends.server.replication.protocol.ReplSessionSecurity;
import org.opends.server.replication.protocol.Session;
import org.opends.server.replication.protocol.UpdateMsg;
import org.opends.server.replication.service.DSRSShutdownSync;
import org.opends.server.util.StaticUtils;
import org.testng.annotations.Test;
/**
* A {@code ServerWriter} reports a write which fails on a session nothing on this side has closed
* as a peer badly disconnected - msgID 211 for a directory server, 181 for a replication server,
* both at ERROR - but not a write which fails on a session this side has closed itself:
* {@code ServerReader} closes the session once its read has ended, and a writer which had an
* update for that peer at that moment then fails on the closed socket. Under load that is every
* peer which leaves in order: a directory server stopped, or moving to another replication
* server, a replication server stopped - the restart rows of the replication convergence run
* (#1208) logged it for each of them. How the connection ended is the reader's to report.
*
* The writer runs on the thread of the test, against a session over the loopback and a handler
* which hands it one update: what is under test is only what the writer logs when the write of
* that update fails.
*/
@SuppressWarnings("javadoc")
public class ServerWriterDisconnectTest extends ReplicationTestCase
{
private static final int RS_ID = 9301;
private static final int SOCKET_TIMEOUT_MS = 5000;
/**
* The session runs its publisher thread, as {@code ServerHandler} starts it, and is closed as
* {@code ServerReader} closes it on the {@code StopMsg} of the peer. The update the writer takes
* next is written by {@code publish()} itself - the close took the session off the queueing
* branch - and fails on the closed socket. That is no disconnection of the peer.
*/
@Test
public void aWriteFailingOnASessionThisSideClosedIsNotReported() throws Exception
{
final int dsId = 9311;
try (ServerSocket listen = new ServerSocket(0))
{
final Session[] pair = connectSessionPair(listen);
final Session session = pair[1];
try
{
session.start();
session.waitForStartup();
session.close();
final List logged = runWriterWithOneUpdate(session, dsId, false);
assertThat(badlyDisconnected(logged, dsId))
.as("a write which failed on a session closed on this side was reported as a peer "
+ "badly disconnected")
.isEmpty();
}
finally
{
StaticUtils.close(pair);
}
}
}
/**
* The sockets of the session are closed under it, nothing on this side having closed the
* session, and its publisher thread is not running: the write fails on a connection which broke,
* and the writer reports it. A replication server session does not get here - see
* {@link #closeTheSocketsUnder(Session)} - this is what keeps the guard of the writer from being
* one that never reports.
*/
@Test
public void aWriteFailingOnASessionNothingClosedIsReported() throws Exception
{
final int dsId = 9312;
try (ServerSocket listen = new ServerSocket(0))
{
final Session[] pair = connectSessionPair(listen);
final Session session = pair[1];
try
{
closeTheSocketsUnder(session);
final List logged = runWriterWithOneUpdate(session, dsId, false);
assertThat(badlyDisconnected(logged, dsId))
.as("a write which failed on a connection which broke under the session was not "
+ "reported")
.hasSize(1);
}
finally
{
StaticUtils.close(pair);
}
}
}
/**
* The same broken connection while the handler is being shut down: what fails then is the end
* of a session the shutdown is taking down anyway, which {@code ServerReader} does not report
* either.
*/
@Test
public void aWriteFailingWhileTheHandlerShutsDownIsNotReported() throws Exception
{
final int dsId = 9313;
try (ServerSocket listen = new ServerSocket(0))
{
final Session[] pair = connectSessionPair(listen);
final Session session = pair[1];
try
{
closeTheSocketsUnder(session);
final List logged = runWriterWithOneUpdate(session, dsId, true);
assertThat(badlyDisconnected(logged, dsId))
.as("a write which failed while the handler was shutting down was reported as a peer "
+ "badly disconnected")
.isEmpty();
}
finally
{
StaticUtils.close(pair);
}
}
}
/**
* The peer goes away without a {@code StopMsg} - killed, older than protocol V4, or giving up a
* session which had failed - and the read of the session fails on it. {@code ServerReader} then
* reports the end of the connection itself and closes the session in its {@code finally}, as
* here: the close takes the branch of a session which failed and writes nothing. The update the
* writer takes next fails on the closed socket, and the writer reports nothing.
*/
@Test
public void aWriteFailingOnASessionClosedAfterItsReadFailedIsNotReported() throws Exception
{
final int dsId = 9314;
try (ServerSocket listen = new ServerSocket(0))
{
final Session[] pair = connectSessionPair(listen);
final Session session = pair[1];
try
{
session.start();
session.waitForStartup();
closeTheSocketsUnder(pair[0]);
// an end of the stream, or a reset when the peer had left something unread
assertThatThrownBy(session::receive)
.as("the read of a session whose peer went away")
.isInstanceOf(IOException.class)
.isNotInstanceOf(SocketTimeoutException.class);
session.close();
final List logged = runWriterWithOneUpdate(session, dsId, false);
assertThat(badlyDisconnected(logged, dsId))
.as("a write which failed on a session closed after its read failed was reported as "
+ "a peer badly disconnected")
.isEmpty();
}
finally
{
StaticUtils.close(pair);
}
}
}
/**
* Runs a writer for a directory server on the thread of the test, its handler handing it one
* update, and answers what was logged meanwhile. It checks that the writer got as far as the
* write, and that the write failed - a write which returned would have the writer take the next
* update - on the socket rather than on anything else.
*/
private List runWriterWithOneUpdate(final Session session, final int dsId,
final boolean shuttingDown) throws Exception
{
final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING);
final UpdateMsg update = new DeleteMsg(DN.valueOf("uid=writer," + TEST_ROOT_DN_STRING),
new CSNGenerator(dsId, 0).newCSN(), "00000000-0000-0000-0000-000000000000");
final DataServerHandler handler = mock(DataServerHandler.class);
/*
* The handler reaches getReplicationServerId() of the package-private MessageHandler through a
* bridge method, which the mock does not intercept: the replication server it reads is set.
*/
final ReplicationServer replicationServer = mock(ReplicationServer.class);
when(replicationServer.getServerId()).thenReturn(RS_ID);
final Field replicationServerField = MessageHandler.class.getDeclaredField("replicationServer");
replicationServerField.setAccessible(true);
replicationServerField.set(handler, replicationServer);
when(handler.take()).thenReturn(update, (UpdateMsg) null);
when(handler.isDataServer()).thenReturn(true);
when(handler.getServerId()).thenReturn(dsId);
when(handler.getProtocolVersion()).thenReturn(ProtocolVersion.getCurrentVersion());
when(handler.shuttingDown()).thenReturn(shuttingDown);
when(handler.getBadlyDisconnectedErrorMessage()).thenReturn(ERR_DS_BADLY_DISCONNECTED.get(
RS_ID, dsId, session.getReadableRemoteAddress(), baseDN));
final ReplicationServerDomain domain = mock(ReplicationServerDomain.class);
when(domain.getBaseDN()).thenReturn(baseDN);
/*
* A publisher of its own rather than the error log of the tests, which holds every record
* twice, each copy with a timestamp of its own: one record is then one line.
*/
final List logged = errorLogRecordsOf(() ->
{
new ServerWriter(session, handler, domain, new DSRSShutdownSync()).run();
return null;
});
verify(handler).countSentUpdate(update);
verify(handler).take();
verify(domain).stopServer(handler, false);
assertThat(withMsgID(logged, ERR_WRITER_UNEXPECTED_EXCEPTION.ordinal()))
.as("the write of the update failed on something else than the socket")
.isEmpty();
return logged;
}
private static List badlyDisconnected(final List logged, final int dsId)
{
final List records = withMsgID(logged, ERR_DS_BADLY_DISCONNECTED.ordinal());
records.removeIf(record -> !record.contains("DS(" + dsId + ")"));
return records;
}
/**
* Answers the records logged at ERROR with the provided ordinal of the replication messages. The
* category is matched as well: the messages of other categories share the ordinals, and another
* thread of the server may log one of them meanwhile. Every record is kept, none deduplicated:
* the timestamp resolves only to the second, so a message logged twice in a row is the same line
* twice.
*/
private static List withMsgID(final List logged, final int msgID)
{
final List records = new ArrayList<>();
for (final String record : logged)
{
if (record.contains("category=SYNC severity=ERROR msgID=" + msgID + " "))
{
records.add(record);
}
}
return records;
}
/**
* Closes the sockets a session writes through while leaving the session unaware of it, so that
* its next write fails. A session whose publisher thread is not running is then in the state a
* connection which broke leaves it in. A replication server session runs it - ServerHandler
* starts it - and is never in that state: its publish() queues, and the publisher records a
* write which failed without handing it to the writer, until a close takes the session off the
* queueing branch.
*
* The plain socket is closed first, as Session.close() does: the close of the SSL socket layered
* on it would otherwise wait out the timeout of the socket for a close_notify of the peer.
*/
private static void closeTheSocketsUnder(final Session session) throws Exception
{
for (final String name : new String[] { "plainSocket", "secureSocket" })
{
final Field socket = Session.class.getDeclaredField(name);
socket.setAccessible(true);
StaticUtils.close((Closeable) socket.get(session));
}
}
/**
* A connected pair of sessions over the loopback, the client end first, neither started. The
* replication server end is the second.
*/
private Session[] connectSessionPair(final ServerSocket listen) throws Exception
{
final ReplSessionSecurity security = getReplSessionSecurity();
final Socket clientSocket = new Socket("127.0.0.1", listen.getLocalPort());
clientSocket.setTcpNoDelay(true);
final ExecutorService executor = Executors.newSingleThreadExecutor();
try
{
final Future clientEnd = executor.submit(new Callable()
{
@Override
public Session call() throws Exception
{
return security.createClientSession(clientSocket, SOCKET_TIMEOUT_MS);
}
});
final Socket serverSocket = listen.accept();
serverSocket.setTcpNoDelay(true);
final Session serverEnd = security.createServerSession(serverSocket, SOCKET_TIMEOUT_MS);
try
{
return new Session[] { clientEnd.get(SOCKET_TIMEOUT_MS, TimeUnit.MILLISECONDS), serverEnd };
}
catch (Exception e)
{
// the factories close what they opened when they fail, but not the end already built
StaticUtils.close(serverEnd);
throw e;
}
}
finally
{
executor.shutdown();
}
}
}