/*
* The contents of this file are subject to the terms of the Common Development and
* Distribution License (the License). You may not use this file except in compliance with the
* License.
*
* You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
* specific language governing permission and limitations under the License.
*
* When distributing Covered Software, include this CDDL Header Notice in each file and include
* the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
* Header, with the fields enclosed by brackets [] replaced by your own identifying
* information: "Portions copyright [year] [name of copyright owner]".
*
* Copyright 2026 3A Systems, LLC.
*/
package org.opends.server.replication.server;
import static org.assertj.core.api.Assertions.assertThat;
import static org.opends.messages.ReplicationMessages.*;
import static org.opends.server.TestCaseUtils.*;
import static org.testng.Assert.*;
import java.io.IOException;
import java.io.OutputStream;
import java.lang.reflect.Field;
import java.net.ServerSocket;
import java.net.Socket;
import java.util.Collections;
import java.util.List;
import java.util.TreeSet;
import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import javax.net.ssl.SSLSocket;
import org.forgerock.i18n.LocalizableMessage;
import org.forgerock.opendj.ldap.DN;
import org.opends.server.TestCaseUtils;
import org.opends.server.replication.ReplicationTestCase;
import org.opends.server.replication.common.DSInfo;
import org.opends.server.replication.common.RSInfo;
import org.opends.server.replication.common.ServerState;
import org.opends.server.replication.protocol.ReplServerStartMsg;
import org.opends.server.replication.protocol.ReplSessionSecurity;
import org.opends.server.replication.protocol.ReplicationMsg;
import org.opends.server.replication.protocol.ServerStartMsg;
import org.opends.server.replication.protocol.Session;
import org.opends.server.replication.protocol.TopologyMsg;
import org.testng.annotations.Test;
/**
* Tests for issue #1241: the severity an aborted replication handshake is logged with.
*
* {@code abortStart} used to log the reason of every abort at ERROR, so a peer which
* restarts under load reads as an error of every server it was handshaking with: the
* listen thread which gives way to a cross connection
* ({@code WARN_TIMEOUT_WHEN_CROSS_CONNECTION}, a warning by its own name), and the
* handshake its peer left ({@code *_DISCONNECTED_DURING_HANDSHAKE}, where a peer which gave
* up on its own timeout logs that timeout at WARNING itself). Both are warnings now, in
* either phase of the handshake, and so is a peer which stays silent past the connection
* timeout: this server gives up on it as it does on a peer which went away.
*
* What this server rejects, a peer which breaks the protocol, a frame which does not decode
* and a frame too large to allocate are still errors. Each call site picks the severity of its
* own abort, so a case pins only the one it reaches: the cases which say so reach every
* {@code abortStart} of a reason in the handshakes of a replication server and of a data
* server but two, so that logging an abort as a warning is not a way to pass them. The two are
* the {@code catch (Exception)} of {@code startFromRemoteRS()}, which a peer reaches only
* through a failure of the JDK (a topology without the replication server which sent it fails
* on {@code get(0)}), and the {@code catch (DirectoryException)} of
* {@code startFromRemoteDS()}, which no peer reaches with a message.
*
* Each test drives one handler on a session pair of its own, as the listen thread or the
* connect thread of the server would, and reads the severity off the record of the reason.
* The peer which goes away aborts its end of the session rather than closing it:
* {@code Session.close()} publishes a {@code StopMsg}, which is a peer ending the handshake
* in order, and is aborted on silently.
*/
@SuppressWarnings("javadoc")
public class HandshakeAbortSeverityTest extends ReplicationTestCase
{
private static final int SOCKET_TIMEOUT_MS = 30000;
/** The connection timeout of a handshake whose peer stays silent: nothing comes before it. */
private static final int SILENT_PEER_TIMEOUT_MS = 500;
/** Longer than the longest wait of {@code lockDomainWithTimeout()}, which is 8 seconds. */
private static final long LOCK_TIMEOUT_MS = 30000;
private static final int REMOTE_DS_ID = 61;
private static final int REMOTE_RS_ID = 62;
private static final String REMOTE_RS_URL = "127.0.0.1:8989";
/** What {@code Session.receive()} reports of the frame {@link #sendTooLargeFrameHeader} announces. */
private static final LocalizableMessage PACKET_TOO_LARGE = LocalizableMessage.raw(
"Packet too large, can't allocate " + Integer.MAX_VALUE + " bytes.");
/**
* The listen thread waits a random 3 to 8 seconds for the domain lock, so that of two
* replication servers connecting to each other at the same time one gives way: the
* connect thread holding the lock here is the one which goes on.
*/
@Test
public void aCrossConnectionGivenWayToIsAWarning() throws Exception
{
final int rsId = 8261;
final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING);
runCase("handshakeAbortSeverityCrossDb", rsId, (replicationServer, remoteEnd, session) -> {
final ReplicationServerDomain domain =
replicationServer.getReplicationServerDomain(baseDN, true);
final ReplServerStartMsg startMsg =
(ReplServerStartMsg) sendThrough(remoteEnd, session, replServerStartMsg(baseDN));
final ReplicationServerHandler handler =
new ReplicationServerHandler(session, 100, replicationServer, 100);
// The lock is held by a thread of its own, as the connect thread holds it: the
// domain lock is reentrant, so the thread of the handshake would take it.
final ExecutorService lockHolder = Executors.newSingleThreadExecutor();
final List records;
try
{
final CountDownLatch locked = new CountDownLatch(1);
final CountDownLatch handshakeDone = new CountDownLatch(1);
final Future holding = lockHolder.submit(() -> {
domain.lock();
try
{
locked.countDown();
handshakeDone.await();
}
finally
{
domain.release();
}
return null;
});
assertTrue(locked.await(LOCK_TIMEOUT_MS, TimeUnit.MILLISECONDS),
"the domain lock should have been taken");
try
{
records = errorLogRecordsOf(() -> {
handler.startFromRemoteRS(startMsg);
return null;
});
}
finally
{
handshakeDone.countDown();
}
holding.get(LOCK_TIMEOUT_MS, TimeUnit.MILLISECONDS);
}
finally
{
lockHolder.shutdownNow();
}
assertThat(recordOf(records, WARN_TIMEOUT_WHEN_CROSS_CONNECTION.get(
baseDN, REMOTE_RS_ID, session.getReadableRemoteAddress(), rsId)))
.as("giving way to a cross connection is a warning, as its message says")
.contains("severity=WARNING");
assertThat(countRecordsOf(records,
WARN_RS_DISCONNECTED_DURING_HANDSHAKE.get(REMOTE_RS_ID, rsId).toString()))
.as("the handshake ends where this server gives way to the cross connection")
.isZero();
});
}
@Test
public void aDataServerGoneDuringTheHandshakeIsAWarning() throws Exception
{
final int rsId = 8262;
final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING);
runCase("handshakeAbortSeverityDSDb", rsId, (replicationServer, remoteEnd, session) -> {
final ServerStartMsg startMsg =
(ServerStartMsg) sendThrough(remoteEnd, session, serverStartMsg(baseDN));
remoteEnd.abort(new IOException("the directory server gave up on the handshake"));
final DataServerHandler handler =
new DataServerHandler(session, 100, replicationServer, 100);
final List records = errorLogRecordsOf(() -> {
handler.startFromRemoteDS(startMsg);
return null;
});
assertThat(recordOf(records, WARN_DS_DISCONNECTED_DURING_HANDSHAKE.get(REMOTE_DS_ID, rsId)))
.as("a directory server which went away during the handshake is a warning")
.contains("severity=WARNING");
assertAbortedHandshakeWentNoFurther(session);
});
}
/**
* A peer which stays connected and says nothing past the connection timeout is given up on
* as one which went away.
*/
@Test
public void aDataServerSilentDuringTheHandshakeIsAWarning() throws Exception
{
final int rsId = 8274;
final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING);
runCase("handshakeAbortSeveritySilentDSDb", rsId, (replicationServer, remoteEnd, session) -> {
final ServerStartMsg startMsg =
(ServerStartMsg) sendThrough(remoteEnd, session, serverStartMsg(baseDN));
session.setSoTimeout(SILENT_PEER_TIMEOUT_MS);
final DataServerHandler handler =
new DataServerHandler(session, 100, replicationServer, 100);
final List records = errorLogRecordsOf(() -> {
handler.startFromRemoteDS(startMsg);
return null;
});
assertThat(recordOf(records, WARN_DS_DISCONNECTED_DURING_HANDSHAKE.get(REMOTE_DS_ID, rsId)))
.as("a directory server silent past the connection timeout is a warning")
.contains("severity=WARNING");
assertAbortedHandshakeWentNoFurther(session);
});
}
/**
* A directory server which sends something else where its start session message is due
* breaks the protocol, and that is still an error.
*/
@Test
public void aDataServerWhichSendsNoStartSessionIsAnError() throws Exception
{
final int rsId = 8275;
final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING);
runCase("handshakeAbortSeverityNoStartSessionDb", rsId, (replicationServer, remoteEnd, session) -> {
final ServerStartMsg startMsg =
(ServerStartMsg) sendThrough(remoteEnd, session, serverStartMsg(baseDN));
// Published ahead of the start message it answers: the session reads it all the same.
remoteEnd.publish(serverStartMsg(baseDN));
final DataServerHandler handler =
new DataServerHandler(session, 100, replicationServer, 100);
final List records = errorLogRecordsOf(() -> {
handler.startFromRemoteDS(startMsg);
return null;
});
assertThat(recordOf(records, LocalizableMessage.raw("Protocol error: StartSessionMsg required.")))
.as("a directory server which breaks the protocol is an error")
.contains("severity=ERROR");
assertAbortedHandshakeWentNoFurther(session);
});
}
@Test
public void aReplicationServerGoneDuringTheHandshakeItStartedIsAWarning() throws Exception
{
final int rsId = 8263;
final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING);
runCase("handshakeAbortSeverityFromRSDb", rsId, (replicationServer, remoteEnd, session) -> {
final ReplServerStartMsg startMsg =
(ReplServerStartMsg) sendThrough(remoteEnd, session, replServerStartMsg(baseDN));
remoteEnd.abort(new IOException("the replication server gave up on the handshake"));
final ReplicationServerHandler handler =
new ReplicationServerHandler(session, 100, replicationServer, 100);
final List records = errorLogRecordsOf(() -> {
handler.startFromRemoteRS(startMsg);
return null;
});
assertThat(recordOf(records, WARN_RS_DISCONNECTED_DURING_HANDSHAKE.get(REMOTE_RS_ID, rsId)))
.as("a replication server which went away during the handshake it started is a warning")
.contains("severity=WARNING");
});
}
@Test
public void aReplicationServerGoneDuringTheHandshakeThisServerStartedIsAWarning() throws Exception
{
final int rsId = 8264;
final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING);
runCase("handshakeAbortSeverityToRSDb", rsId, (replicationServer, remoteEnd, session) -> {
remoteEnd.abort(new IOException("the replication server went away"));
final ReplicationServerHandler handler =
new ReplicationServerHandler(session, 100, replicationServer, 100);
final List records = errorLogRecordsOf(() -> {
assertFalse(handler.connect(baseDN, true), "the peer went away, so this is no session");
return null;
});
assertThat(recordOf(records,
WARN_RS_DISCONNECTED_DURING_HANDSHAKE.get(rsId, session.getReadableRemoteAddress())))
.as("a replication server which went away during the handshake this server started"
+ " is a warning")
.contains("severity=WARNING");
});
}
/**
* What the connect thread of a cross connection reads, when the listen thread of its peer
* waits for the domain lock longer than the connection timeout: up to 8 seconds against 5 by
* default.
*/
@Test
public void aReplicationServerSilentDuringTheHandshakeThisServerStartedIsAWarning()
throws Exception
{
final int rsId = 8276;
final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING);
runCase("handshakeAbortSeveritySilentToRSDb", rsId, (replicationServer, remoteEnd, session) -> {
session.setSoTimeout(SILENT_PEER_TIMEOUT_MS);
final ReplicationServerHandler handler =
new ReplicationServerHandler(session, 100, replicationServer, 100);
final List records = errorLogRecordsOf(() -> {
assertFalse(handler.connect(baseDN, true), "the peer did not answer, so this is no session");
return null;
});
assertThat(recordOf(records,
WARN_RS_DISCONNECTED_DURING_HANDSHAKE.get(rsId, session.getReadableRemoteAddress())))
.as("a replication server silent past the connection timeout is a warning")
.contains("severity=WARNING");
});
}
/**
* The second phase of the handshake reads the topology of the peer through a method of its
* own, which used to report every failure of the session as the error of a raw message.
*/
@Test
public void aReplicationServerGoneBeforeItsTopologyInTheHandshakeItStartedIsAWarning()
throws Exception
{
final int rsId = 8266;
final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING);
runCase("handshakeAbortSeverityFromRSTopoDb", rsId, (replicationServer, remoteEnd, session) -> {
final ReplServerStartMsg startMsg =
(ReplServerStartMsg) sendThrough(remoteEnd, session, replServerStartMsg(baseDN));
final ReplicationServerHandler handler =
new ReplicationServerHandler(session, 100, replicationServer, 100);
final List records = errorLogRecordsOf(() -> {
// Gone once it has read the start message of this server: that is sent, and the
// handshake waits for the topology of the peer.
goneAfterReading(1, remoteEnd, () -> {
handler.startFromRemoteRS(startMsg);
return null;
});
return null;
});
assertThat(recordOf(records, WARN_RS_DISCONNECTED_DURING_HANDSHAKE.get(REMOTE_RS_ID, rsId)))
.as("a replication server which went away before it sent its topology is a warning")
.contains("severity=WARNING");
});
}
/**
* The case above for a peer which stays silent: the second phase used to log the timeout
* as the error of the raw "Read timed out".
*/
@Test
public void aReplicationServerSilentBeforeItsTopologyInTheHandshakeItStartedIsAWarning()
throws Exception
{
final int rsId = 8277;
final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING);
runCase("handshakeAbortSeveritySilentFromRSTopoDb", rsId, (replicationServer, remoteEnd, session) -> {
final ReplServerStartMsg startMsg =
(ReplServerStartMsg) sendThrough(remoteEnd, session, replServerStartMsg(baseDN));
session.setSoTimeout(SILENT_PEER_TIMEOUT_MS);
final ReplicationServerHandler handler =
new ReplicationServerHandler(session, 100, replicationServer, 100);
final List records = errorLogRecordsOf(() -> {
handler.startFromRemoteRS(startMsg);
return null;
});
assertThat(recordOf(records, WARN_RS_DISCONNECTED_DURING_HANDSHAKE.get(REMOTE_RS_ID, rsId)))
.as("a replication server silent past the connection timeout before it sent its topology"
+ " is a warning")
.contains("severity=WARNING");
});
}
@Test
public void aReplicationServerGoneBeforeItsTopologyInTheHandshakeThisServerStartedIsAWarning()
throws Exception
{
final int rsId = 8267;
final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING);
runCase("handshakeAbortSeverityToRSTopoDb", rsId, (replicationServer, remoteEnd, session) -> {
// Published ahead of the start message it answers: the session reads it all the same.
remoteEnd.publish(replServerStartMsg(baseDN));
final ReplicationServerHandler handler =
new ReplicationServerHandler(session, 100, replicationServer, 100);
final List records = errorLogRecordsOf(() -> {
// Gone once it has read the start message and the topology of this server: both
// are sent, and the handshake waits for the topology of the peer.
assertFalse(goneAfterReading(2, remoteEnd, () -> handler.connect(baseDN, true)),
"the peer went away, so this is no session");
return null;
});
assertThat(recordOf(records,
WARN_RS_DISCONNECTED_DURING_HANDSHAKE.get(rsId, session.getReadableRemoteAddress())))
.as("a replication server which went away before it sent its topology is a warning")
.contains("severity=WARNING");
});
}
/**
* A peer which answers the start message of this server with something else breaks the
* protocol, and that is still an error: of the aborts, only the peer going away and the
* cross connection are warnings.
*/
@Test
public void aPeerWhichBreaksTheProtocolIsStillAnError() throws Exception
{
final int rsId = 8265;
final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING);
runCase("handshakeAbortSeverityProtocolDb", rsId, (replicationServer, remoteEnd, session) -> {
// Published ahead of the start message it answers: the session reads it all the same.
remoteEnd.publish(new TopologyMsg(
Collections. emptyList(), Collections. emptyList()));
final ReplicationServerHandler handler =
new ReplicationServerHandler(session, 100, replicationServer, 100);
final List records = errorLogRecordsOf(() -> {
assertFalse(handler.connect(baseDN, true), "the peer broke the protocol, so this is no session");
return null;
});
assertThat(recordOf(records, ERR_REPLICATION_PROTOCOL_MESSAGE_TYPE.get(
TopologyMsg.class.getCanonicalName(), "ReplServerStartMsg")))
.as("a peer which breaks the protocol is an error")
.contains("severity=ERROR");
});
}
/**
* The case above in the second phase, which is aborted from the {@code DirectoryException}
* arm of the handshake: a peer which sends its start message again where its topology is
* due.
*/
@Test
public void aPeerWhichSendsNoTopologyInTheHandshakeItStartedIsAnError() throws Exception
{
final int rsId = 8268;
final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING);
runCase("handshakeAbortSeverityFromRSNoTopoDb", rsId, (replicationServer, remoteEnd, session) -> {
final ReplServerStartMsg startMsg =
(ReplServerStartMsg) sendThrough(remoteEnd, session, replServerStartMsg(baseDN));
remoteEnd.publish(replServerStartMsg(baseDN));
final ReplicationServerHandler handler =
new ReplicationServerHandler(session, 100, replicationServer, 100);
final List records = errorLogRecordsOf(() -> {
handler.startFromRemoteRS(startMsg);
return null;
});
assertThat(recordOf(records, ERR_REPLICATION_PROTOCOL_MESSAGE_TYPE.get(
ReplServerStartMsg.class.getCanonicalName(), "TopologyMsg")))
.as("a peer which breaks the protocol is an error")
.contains("severity=ERROR");
});
}
@Test
public void aPeerWhichSendsNoTopologyInTheHandshakeThisServerStartedIsAnError() throws Exception
{
final int rsId = 8269;
final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING);
runCase("handshakeAbortSeverityToRSNoTopoDb", rsId, (replicationServer, remoteEnd, session) -> {
// Published ahead of the messages they answer: the session reads them all the same.
remoteEnd.publish(replServerStartMsg(baseDN));
remoteEnd.publish(replServerStartMsg(baseDN));
final ReplicationServerHandler handler =
new ReplicationServerHandler(session, 100, replicationServer, 100);
final List records = errorLogRecordsOf(() -> {
assertFalse(handler.connect(baseDN, true), "the peer broke the protocol, so this is no session");
return null;
});
assertThat(recordOf(records, ERR_REPLICATION_PROTOCOL_MESSAGE_TYPE.get(
ReplServerStartMsg.class.getCanonicalName(), "TopologyMsg")))
.as("a peer which breaks the protocol is an error")
.contains("severity=ERROR");
});
}
/**
* A rejection by this server: a peer which names the server id of a replication server the
* domain already holds a session with, at another address.
*/
@Test
public void aServerIdAnotherReplicationServerHoldsIsAnError() throws Exception
{
final int rsId = 8270;
final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING);
runCase("handshakeAbortSeverityDuplicateDb", rsId, (replicationServer, firstEnd, first) -> {
// The first peer completes its handshake, so that the domain holds it: its topology is
// published ahead of the start message of this server it answers.
final ReplServerStartMsg firstStartMsg =
(ReplServerStartMsg) sendThrough(firstEnd, first, replServerStartMsg(baseDN));
firstEnd.publish(new TopologyMsg(Collections. emptyList(),
Collections.singletonList(new RSInfo(REMOTE_RS_ID, REMOTE_RS_URL, -1, (byte) 1, 1))));
final ReplicationServerHandler registered =
new ReplicationServerHandler(first, 100, replicationServer, 100);
registered.startFromRemoteRS(firstStartMsg);
// The precondition of the case rather than an assumption.
assertThat(replicationServer.getReplicationServerDomain(baseDN).getConnectedRSs().get(REMOTE_RS_ID))
.as("the first peer should hold its session")
.isSameAs(registered);
try (ServerSocket listen = TestCaseUtils.bindFreePort())
{
listen.setSoTimeout(SOCKET_TIMEOUT_MS);
final Session[] sessionPair = connectSessionPair(listen, getReplSessionSecurity());
try (Session remoteEnd = sessionPair[0];
Session session = sessionPair[1])
{
final ReplServerStartMsg startMsg = (ReplServerStartMsg) sendThrough(remoteEnd, session,
replServerStartMsg(baseDN, "127.0.0.1:8990"));
final ReplicationServerHandler handler =
new ReplicationServerHandler(session, 100, replicationServer, 100);
final List records = errorLogRecordsOf(() -> {
handler.startFromRemoteRS(startMsg);
return null;
});
assertThat(recordOf(records, ERR_DUPLICATE_REPLICATION_SERVER_ID.get(
replicationServer.getMonitorInstanceName(), registered.getServerAddressURL(),
handler.getServerAddressURL(), REMOTE_RS_ID)))
.as("a peer this server rejects is an error")
.contains("severity=ERROR");
}
}
});
}
/**
* A frame too large to allocate is not the peer going away, though the session reports it
* as an {@code IOException}: either the peer breaks the protocol, a handshake message being
* nowhere near that size, or this server has no heap to spare it, and both are errors.
*/
@Test
public void aFrameTooLargeForTheStartMessageOfTheHandshakeThisServerStartedIsAnError()
throws Exception
{
final int rsId = 8271;
final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING);
runCase("handshakeAbortSeverityToRSFrameDb", rsId, (replicationServer, remoteEnd, session) -> {
sendTooLargeFrameHeader(remoteEnd);
final ReplicationServerHandler handler =
new ReplicationServerHandler(session, 100, replicationServer, 100);
final List records = errorLogRecordsOf(() -> {
assertFalse(handler.connect(baseDN, true), "the frame cannot be read, so this is no session");
return null;
});
assertTooLargeFrameIsAnError(records, session,
WARN_RS_DISCONNECTED_DURING_HANDSHAKE.get(rsId, session.getReadableRemoteAddress()));
});
}
/**
* A frame which does not decode fails in {@code Session.receive()} with neither an
* {@code IOException} nor a {@code DirectoryException}, and is aborted on from the
* {@code catch (Exception)} of the handshake, still as an error.
*/
@Test
public void aFrameWhichDoesNotDecodeInTheHandshakeThisServerStartedIsAnError()
throws Exception
{
final int rsId = 8278;
final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING);
runCase("handshakeAbortSeverityToRSUndecodableDb", rsId, (replicationServer, remoteEnd, session) -> {
// A frame of one byte, a message type which no message has.
sendRaw(remoteEnd, new byte[] { '0', '0', '0', '0', '0', '0', '0', '1', 0 });
final ReplicationServerHandler handler =
new ReplicationServerHandler(session, 100, replicationServer, 100);
final List records = errorLogRecordsOf(() -> {
assertFalse(handler.connect(baseDN, true), "the frame cannot be decoded, so this is no session");
return null;
});
assertThat(recordOf(records, LocalizableMessage.raw("received message with unknown type")))
.as("a frame which does not decode is an error")
.contains("severity=ERROR");
assertThat(countRecordsOf(records,
WARN_RS_DISCONNECTED_DURING_HANDSHAKE.get(rsId, session.getReadableRemoteAddress())
.toString()))
.as("a frame which does not decode is not the peer going away")
.isZero();
});
}
@Test
public void aFrameTooLargeForTheTopologyInTheHandshakeItStartedIsAnError() throws Exception
{
final int rsId = 8272;
final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING);
runCase("handshakeAbortSeverityFromRSFrameDb", rsId, (replicationServer, remoteEnd, session) -> {
final ReplServerStartMsg startMsg =
(ReplServerStartMsg) sendThrough(remoteEnd, session, replServerStartMsg(baseDN));
sendTooLargeFrameHeader(remoteEnd);
final ReplicationServerHandler handler =
new ReplicationServerHandler(session, 100, replicationServer, 100);
final List records = errorLogRecordsOf(() -> {
handler.startFromRemoteRS(startMsg);
return null;
});
assertTooLargeFrameIsAnError(records, session,
WARN_RS_DISCONNECTED_DURING_HANDSHAKE.get(REMOTE_RS_ID, rsId));
});
}
@Test
public void aFrameTooLargeForTheStartSessionOfADataServerIsAnError() throws Exception
{
final int rsId = 8273;
final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING);
runCase("handshakeAbortSeverityDSFrameDb", rsId, (replicationServer, remoteEnd, session) -> {
final ServerStartMsg startMsg =
(ServerStartMsg) sendThrough(remoteEnd, session, serverStartMsg(baseDN));
sendTooLargeFrameHeader(remoteEnd);
final DataServerHandler handler =
new DataServerHandler(session, 100, replicationServer, 100);
final List records = errorLogRecordsOf(() -> {
handler.startFromRemoteDS(startMsg);
return null;
});
assertTooLargeFrameIsAnError(records, session,
WARN_DS_DISCONNECTED_DURING_HANDSHAKE.get(REMOTE_DS_ID, rsId));
assertAbortedHandshakeWentNoFurther(session);
});
}
/** One case: the replication server under test, and a session pair to it. */
private interface HandshakeCase
{
void run(ReplicationServer replicationServer, Session remoteEnd, Session session)
throws Exception;
}
/**
* Runs the provided case on a replication server of its own and a session pair to it,
* which are torn down once it ran.
*/
private void runCase(String dbName, int rsId, HandshakeCase handshakeCase) throws Exception
{
ReplicationServer replicationServer = null;
try (ServerSocket listen = TestCaseUtils.bindFreePort())
{
listen.setSoTimeout(SOCKET_TIMEOUT_MS);
replicationServer = newReplicationServer(dbName, rsId);
final Session[] sessionPair =
connectSessionPair(listen, getReplSessionSecurity());
try (Session remoteEnd = sessionPair[0];
Session session = sessionPair[1])
{
handshakeCase.run(replicationServer, remoteEnd, session);
}
}
finally
{
removeQuietly(replicationServer);
}
}
/**
* Runs the provided handshake on a thread of its own, as the listen thread or the connect
* thread of the server would, while the remote end reads the provided number of messages
* and goes away: the handshake then fails on what it waits for next, rather than on a
* message it sends.
*/
private static T goneAfterReading(int messages, Session remoteEnd, Callable handshake)
throws Exception
{
final ExecutorService handshakeThread = Executors.newSingleThreadExecutor();
try
{
final Future done = handshakeThread.submit(handshake);
for (int i = 0; i < messages; i++)
{
remoteEnd.receive();
}
remoteEnd.abort(new IOException("the replication server went away"));
return done.get(SOCKET_TIMEOUT_MS, TimeUnit.MILLISECONDS);
}
finally
{
handshakeThread.shutdownNow();
}
}
/**
* Writes the length header of a frame of {@link Integer#MAX_VALUE} bytes from the remote
* end, and nothing after it: more than any array can hold, so the local end fails to
* allocate it whatever its heap. The session writes no frame it did not encode itself, so
* the header goes to the socket under it.
*/
private static void sendTooLargeFrameHeader(Session remoteEnd) throws Exception
{
sendRaw(remoteEnd, String.format("%08x", Integer.MAX_VALUE).getBytes());
}
/** Writes the provided bytes from the remote end to the socket under it, as they are. */
private static void sendRaw(Session remoteEnd, byte[] bytes) throws Exception
{
final Field secureSocket = Session.class.getDeclaredField("secureSocket");
secureSocket.setAccessible(true);
final OutputStream output = ((SSLSocket) secureSocket.get(remoteEnd)).getOutputStream();
output.write(bytes);
output.flush();
}
private static void assertTooLargeFrameIsAnError(List records, Session session,
LocalizableMessage disconnected)
{
final String record = recordOf(records, PACKET_TOO_LARGE);
assertThat(record)
.as("a frame too large to allocate is an error")
.contains("severity=ERROR");
assertThat(record)
.as("the record of a frame too large to allocate names the peer which sent it")
.contains("Peer: " + session.getReadableRemoteAddress());
assertThat(countRecordsOf(records, disconnected.toString()))
.as("a frame too large to allocate is not the peer going away")
.isZero();
}
/**
* Asserts that the handshake of a data server ended where it was aborted: only
* {@code finalizeStart()} starts the thread of the session, once the handler is registered
* with the domain. Read off the thread rather than off the domain, which the reader of a
* handler started on a closed session may have unregistered it from already.
*/
private static void assertAbortedHandshakeWentNoFurther(Session session)
{
assertThat(session.getState())
.as("a handshake which was aborted starts no session thread")
.isEqualTo(Thread.State.NEW);
}
private ReplicationServer newReplicationServer(String dbName, int rsId) throws Exception
{
return new ReplicationServer(new ReplServerFakeConfiguration(
TestCaseUtils.findFreePort(), dbName, 0, rsId, 0, 100, new TreeSet()));
}
private ServerStartMsg serverStartMsg(DN baseDN)
{
return new ServerStartMsg(REMOTE_DS_ID, "127.0.0.1:1389", baseDN, 100, 0,
new ServerState(), -1, true, (byte) 1);
}
private ReplServerStartMsg replServerStartMsg(DN baseDN)
{
return replServerStartMsg(baseDN, REMOTE_RS_URL);
}
/**
* Returns the start message of a fake peer replication server.
*
* A generation id of -1 leaves the one of the domain alone, and the session stays
* encrypted, so that neither end has to leave the SSL session for the handshake to go on.
*/
private ReplServerStartMsg replServerStartMsg(DN baseDN, String serverURL)
{
return new ReplServerStartMsg(REMOTE_RS_ID, serverURL, baseDN, 100,
new ServerState(), -1, true, (byte) 1, 5000);
}
/**
* Publishes the provided start message from the remote end and returns it as the local
* end reads it, which is the message the listen thread hands to the handler: it carries
* the protocol version of the session, which a message built here does not.
*/
private ReplicationMsg sendThrough(Session remoteEnd, Session session, ReplicationMsg startMsg)
throws Exception
{
remoteEnd.publish(startMsg);
return session.receive();
}
/** Teardown must never mask the primary assertion failure. */
private void removeQuietly(ReplicationServer replicationServer)
{
try
{
remove(replicationServer);
}
catch (Exception ignored)
{
}
}
/**
* Establishes a connected session pair over the given listen socket, as a
* remote server connecting to the RS would. The TLS negotiation performed by
* the session factories needs both ends handshaking at the same time, so the
* client end runs on its own thread.
*
* @return the two sessions: the remote (client) end first, then the local
* (server) end to hand to the handler under test
*/
private Session[] connectSessionPair(ServerSocket listenSocket,
final ReplSessionSecurity security) throws Exception
{
final Socket clientSocket =
new Socket("127.0.0.1", listenSocket.getLocalPort());
clientSocket.setTcpNoDelay(true);
final ExecutorService executor = Executors.newSingleThreadExecutor();
try
{
final Future clientEnd = executor.submit(
() -> security.createClientSession(clientSocket, SOCKET_TIMEOUT_MS));
final Socket serverSocket = listenSocket.accept();
serverSocket.setTcpNoDelay(true);
final Session serverEnd =
security.createServerSession(serverSocket, SOCKET_TIMEOUT_MS);
assertNotNull(serverEnd,
"could not create a session for the handler under test");
return new Session[] {
clientEnd.get(SOCKET_TIMEOUT_MS, TimeUnit.MILLISECONDS), serverEnd };
}
finally
{
executor.shutdown();
}
}
}