[#1287] Complete large TLS handshakes without blocking writes (#1290)

Valery Kharseko

2 hours ago 32d7c59f16eac665800e5f131b06c14e76fac248

 opendj-core/src/main/java/org/forgerock/opendj/ldap/LDAPClientContext.java

@@ -13,6 +13,7 @@
 *
 * Copyright 2010 Sun Microsystems, Inc.
 * Portions copyright 2012-2016 ForgeRock AS.
 * Portions Copyright 2026 3A Systems, LLC.
 */

package org.forgerock.opendj.ldap;
@@ -146,7 +147,8 @@
     *            Must be {@code true} if the TLS filter has to be installed as a consequence of a StartTLS request
     *            performed by a client. When {@code true} the TLS filter will be installed atomically after the first
     *            message sent to prevent race-condition.
     * @return {@code true} if the TLS filter has been enabled, {@code false} if it was already enabled.
     * @return {@code true} if the TLS filter has been enabled, or {@code false} if it was already enabled, an upgrade
     *         is pending, or the connection is closed.
     * @throws NullPointerException
     *             if sslEngine is null
     */

 opendj-doc-generated-ref/src/main/asciidoc/reference/appendix-log-messages.adoc

@@ -12,7 +12,7 @@
  information: "Portions copyright [year] [name of copyright owner]".
 
  Copyright 2017 ForgeRock AS.
  Portions Copyright 2024-2026 3A Systems LLC.
  Portions Copyright 2024-2026 3A Systems, LLC.
////

:figure-caption!:
@@ -12093,6 +12093,13 @@
+
Message: Use of the proxied authorization V2 control for user %s is not allowed: the account's password is expired.

[#log-ref-log-ref-ERR_LDAP_TLS_UPGRADE_UNAVAILABLE_1544]
ID: 1544::
Severity: ERROR

+
Message: StartTLS cannot be enabled because the connection is closed, TLS is already enabled, or a TLS upgrade is pending.

--



 opendj-grizzly/src/main/java/org/forgerock/opendj/grizzly/LDAPServerFilter.java

@@ -31,10 +31,13 @@
import java.util.List;
import java.util.Properties;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;

import javax.net.ssl.SSLEngine;
import javax.net.ssl.SSLEngineResult.HandshakeStatus;
import javax.net.ssl.SSLSession;
import javax.security.sasl.Sasl;
import javax.security.sasl.SaslServer;
@@ -65,6 +68,7 @@
import org.forgerock.util.promise.PromiseImpl;
import org.forgerock.util.promise.ResultHandler;
import org.forgerock.util.promise.RuntimeExceptionHandler;
import org.glassfish.grizzly.Buffer;
import org.glassfish.grizzly.Connection;
import org.glassfish.grizzly.EmptyCompletionHandler;
import org.glassfish.grizzly.filterchain.BaseFilter;
@@ -73,6 +77,7 @@
import org.glassfish.grizzly.filterchain.FilterChainBuilder;
import org.glassfish.grizzly.filterchain.FilterChainContext;
import org.glassfish.grizzly.filterchain.NextAction;
import org.glassfish.grizzly.ssl.SSLBaseFilter.HandshakeListener;
import org.glassfish.grizzly.ssl.SSLContextConfigurator;
import org.glassfish.grizzly.ssl.SSLFilter;
import org.glassfish.grizzly.ssl.SSLUtils;
@@ -124,6 +129,21 @@

    private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();

    /** One daemon timer for handshakes on all listeners; completed connections leave its queue immediately. */
    private static final ScheduledThreadPoolExecutor HANDSHAKE_TIMER = newHandshakeTimer();

    private static ScheduledThreadPoolExecutor newHandshakeTimer() {
        final ScheduledThreadPoolExecutor timer = new ScheduledThreadPoolExecutor(1, task -> {
            final Thread thread = new Thread(task, "OpenDJ LDAP TLS handshake timer");
            thread.setDaemon(true);
            return thread;
        });
        timer.setRemoveOnCancelPolicy(true);
        timer.setKeepAliveTime(30, TimeUnit.SECONDS);
        timer.allowCoreThreadTimeOut(true);
        return timer;
    }

    private final int maxConcurrentRequests;
    private final Options connectionOptions;

@@ -312,6 +332,10 @@

        private final Connection<?> connection;
        private volatile boolean isClosed;
        /** Guarded by this context, including the race between handshake completion and expiry. */
        private ScheduledFuture<?> handshakeDeadline;
        private long handshakeDeadlineGeneration;
        private boolean initialHandshakePending;
        private final List<LDAPClientContextEventListener> connectionEventListeners = new LinkedList<>();
        private GrizzlyBackpressureSubscription downstream;

@@ -339,6 +363,7 @@
        @Override
        public NextAction handleClose(final FilterChainContext ctx) {
            isClosed = true;
            cancelHandshakeDeadline();
            // downstream can be null if the connection is closed before handleAccept() completion.
            final GrizzlyBackpressureSubscription immutableRef = downstream;
            if (immutableRef != null) {
@@ -386,7 +411,7 @@
        public boolean enableTLS(final SSLEngine sslEngine, final boolean startTls) {
            Reject.ifNull(sslEngine, "sslEngine must not be null");
            synchronized (this) {
                if (filterExists(SSLFilter.class)) {
                if (!connection.isOpen() || filterExists(SSLFilter.class) || filterExists(StartTLSFilter.class)) {
                    return false;
                }
                SSLUtils.setSSLEngine(connection, sslEngine);
@@ -399,13 +424,97 @@
                  System.setProperty(SSLContextConfigurator.TRUST_STORE_FILE, "NONE");
                }

                SSLFilter sslFilter = new SSLFilter();
                sslFilter.setHandshakeTimeout(getLongProperty("org.forgerock.opendj.grizzly.handshakeTimeout", 10000), TimeUnit.MILLISECONDS);
                installFilter(startTls ? new StartTLSFilter(sslFilter) : sslFilter);
                // Grizzly otherwise sees this externally supplied engine as NOT_HANDSHAKING on the first read,
                // starts it implicitly while unwrapping, and enters its blocking re-handshake path. A partial write
                // can then wait for the selector thread that is itself blocked in that handshake.
                final long timeout = getLongProperty("org.forgerock.opendj.grizzly.handshakeTimeout", 10000);
                initialHandshakePending = true;
                final SSLFilter sslFilter = new SSLFilter() {
                    private boolean firstRead = true;
                    private boolean receivedFirstRecord;

                    @Override
                    public NextAction handleRead(final FilterChainContext ctx) throws IOException {
                        if (firstRead) {
                            firstRead = false;
                            if (sslEngine.getHandshakeStatus() == HandshakeStatus.NOT_HANDSHAKING) {
                                sslEngine.beginHandshake();
                            }
                        }
                        if (timeout > 0) {
                            if (!receivedFirstRecord) {
                                final Buffer input = ctx.getMessage();
                                final int packetSize = SSLUtils.getSSLPacketSize(input);
                                receivedFirstRecord = packetSize > 0 && input.remaining() >= packetSize;
                            }
                            if (receivedFirstRecord) {
                                renewHandshakeDeadline(timeout);
                            }
                        }
                        return super.handleRead(ctx);
                    }
                };
                // A negative Grizzly timeout selects non-blocking handshake processing. Preserve its configured
                // read timeout separately, starting when the first complete TLS record arrives and renewing on reads.
                sslFilter.setHandshakeTimeout(-1, TimeUnit.MILLISECONDS);
                sslFilter.addHandshakeListener(new HandshakeListener() {
                    @Override
                    public void onStart(Connection<?> ignored) {
                        // The externally supplied engine is started on the first read.
                    }

                    @Override
                    public void onComplete(Connection<?> ignored) {
                        cancelHandshakeDeadline();
                        // Grizzly also uses this timeout for later re-handshakes. They still take its blocking
                        // path, so restore their read timeout once the initial, asynchronous handshake is done.
                        sslFilter.setHandshakeTimeout(timeout, TimeUnit.MILLISECONDS);
                    }

                    @Override
                    public void onFailure(Connection<?> ignored, Throwable error) {
                        cancelHandshakeDeadline();
                    }
                });
                try {
                    installFilter(startTls ? new StartTLSFilter(sslFilter) : sslFilter);
                } catch (RuntimeException | Error e) {
                    cancelHandshakeDeadline();
                    throw e;
                }
                return true;
            }
        }

        private synchronized void renewHandshakeDeadline(final long timeout) {
            if (!initialHandshakePending) {
                return;
            }
            if (handshakeDeadline != null) {
                handshakeDeadline.cancel(false);
            }
            final long generation = ++handshakeDeadlineGeneration;
            handshakeDeadline = HANDSHAKE_TIMER.schedule(() -> {
                synchronized (ClientConnectionImpl.this) {
                    if (!initialHandshakePending || generation != handshakeDeadlineGeneration) {
                        return;
                    }
                    initialHandshakePending = false;
                    handshakeDeadline = null;
                }
                connection.terminateSilently();
            }, timeout, TimeUnit.MILLISECONDS);
        }

        private synchronized void cancelHandshakeDeadline() {
            initialHandshakePending = false;
            handshakeDeadlineGeneration++;
            if (handshakeDeadline != null) {
                handshakeDeadline.cancel(false);
                handshakeDeadline = null;
            }
        }

        @Override
        public SSLSession getSSLSession() {
            final SSLEngine sslEngine = SSLUtils.getSSLEngine(connection);

 opendj-grizzly/src/test/java/org/forgerock/opendj/grizzly/GrizzlyLDAPListenerTestCase.java

@@ -13,7 +13,7 @@
 *
 * Copyright 2010 Sun Microsystems, Inc.
 * Portions Copyright 2011-2016 ForgeRock AS.
 * Portions Copyright 2017-2026 3A Systems, LLC
 * Portions Copyright 2017-2026 3A Systems, LLC.
 */
package org.forgerock.opendj.grizzly;

@@ -28,18 +28,48 @@
import static org.forgerock.util.Options.defaultOptions;
import static org.mockito.Mockito.mock;

import java.io.DataInputStream;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.lang.management.ThreadInfo;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.math.BigInteger;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.Socket;
import java.net.SocketTimeoutException;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.security.KeyPair;
import java.security.KeyPairGenerator;
import java.security.KeyStore;
import java.security.cert.X509Certificate;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;

import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLEngine;
import javax.net.ssl.SSLEngineResult;
import javax.net.ssl.SSLSocket;
import javax.security.auth.x500.X500Principal;

import org.bouncycastle.asn1.x509.Extension;
import org.bouncycastle.asn1.x509.GeneralName;
import org.bouncycastle.asn1.x509.GeneralNames;
import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder;
import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
import org.forgerock.opendj.ldap.Connection;
import org.forgerock.opendj.ldap.ConnectionException;
import org.forgerock.opendj.ldap.ConnectionFactory;
@@ -55,9 +85,11 @@
import org.forgerock.opendj.ldap.ResultCode;
import org.forgerock.opendj.ldap.SdkTestCase;
import org.forgerock.opendj.ldap.SearchResultHandler;
import org.forgerock.opendj.ldap.SSLContextBuilder;
import org.forgerock.opendj.ldap.ServerConnection;
import org.forgerock.opendj.ldap.ServerConnectionFactory;
import org.forgerock.opendj.ldap.TestCaseUtils;
import org.forgerock.opendj.ldap.TrustManagers;
import org.forgerock.opendj.ldap.requests.AbandonRequest;
import org.forgerock.opendj.ldap.requests.AddRequest;
import org.forgerock.opendj.ldap.requests.BindRequest;
@@ -77,8 +109,10 @@
import org.forgerock.util.promise.PromiseImpl;
import org.glassfish.grizzly.nio.transport.TCPNIOTransport;
import org.glassfish.grizzly.nio.transport.TCPNIOTransportBuilder;
import org.glassfish.grizzly.strategies.SameThreadIOStrategy;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;

import com.forgerock.reactive.ServerConnectionFactoryAdapter;
@@ -374,6 +408,593 @@
        }
    }

    @DataProvider
    public Object[][] tlsProtocols() {
        return new Object[][] { { "TLSv1.2", false, 180 }, { "TLSv1.3", false, 180 },
                { "TLSv1.2", true, 180 }, { "TLSv1.3", true, 180 },
                { "TLSv1.2", false, 600 }, { "TLSv1.3", true, 600 } };
    }

    /** A TLS flight larger than the configured write buffer must reach the client before it waits for a reply. */
    @Test(dataProvider = "tlsProtocols", timeOut = 20000)
    public void testLargeCertificateHandshakeWithSmallWriteBuffer(final String protocol, final boolean startTls,
            final int nameCount) throws Exception {
        final SSLContext serverSSLContext = largeCertificateContext(nameCount);
        final TCPNIOTransport transport = TCPNIOTransportBuilder.newInstance()
                .setIOStrategy(SameThreadIOStrategy.getInstance()).setWriteBufferSize(4096).build();
        transport.start();
        try {
            final MockServerConnection serverConnection = new MockServerConnection() {
                @Override
                public <R extends ExtendedResult> void handleExtendedRequest(final Integer requestContext,
                        final ExtendedRequest<R> request, final IntermediateResponseHandler intermediateResponseHandler,
                        final LdapResultHandler<R> resultHandler) {
                    try {
                        enableServerTLS(context.get(10, TimeUnit.SECONDS), serverSSLContext, protocol, true);
                        resultHandler.handleResult(request.getResultDecoder().newExtendedErrorResult(
                                ResultCode.SUCCESS, "", ""));
                    } catch (Exception e) {
                        resultHandler.handleException(newLdapException(ResultCode.OTHER, e.getMessage(), e));
                    }
                }
            };
            final MockServerConnectionFactory serverFactory = new MockServerConnectionFactory(serverConnection) {
                @Override
                public ServerConnection<Integer> handleAccept(final LDAPClientContext context) throws LdapException {
                    if (!startTls) {
                        enableServerTLS(context, serverSSLContext, protocol, false);
                    }
                    return super.handleAccept(context);
                }
            };
            final Options options = defaultOptions().set(GrizzlyLDAPListener.GRIZZLY_TRANSPORT, transport);
            final LDAPListener listener = new LDAPListener(Collections.singleton(loopbackWithDynamicPort()),
                    new ServerConnectionFactoryAdapter(options.get(LDAP_DECODE_OPTIONS), serverFactory), options);
            try {
                final InetSocketAddress address = listener.firstSocketAddress();
                final SSLContext clientContext = new SSLContextBuilder().setTrustManager(TrustManagers.trustAll())
                        .getSSLContext();
                try (Socket plain = new Socket(address.getAddress(), address.getPort())) {
                    plain.setSoTimeout(7000);
                    if (startTls) {
                        startTLS(plain, (byte) 1);
                    }
                    try (SSLSocket socket = (SSLSocket) clientContext.getSocketFactory().createSocket(
                            plain, "localhost", address.getPort(), false)) {
                        socket.setEnabledProtocols(new String[] { protocol });
                        socket.setSoTimeout(7000);
                        socket.startHandshake();
                        bindAnonymously(socket);
                    }
                }
            } finally {
                listener.close();
            }
        } finally {
            transport.shutdownNow();
        }
    }

    private static boolean enableServerTLS(LDAPClientContext context, SSLContext sslContext, String protocol,
            boolean startTls) {
        final SSLEngine engine = sslContext.createSSLEngine();
        engine.setUseClientMode(false);
        engine.setEnabledProtocols(new String[] { protocol });
        return context.enableTLS(engine, startTls);
    }

    /** The asynchronous handshake must not leave a client which sends only part of its ClientHello connected. */
    @Test(timeOut = 20000)
    public void testIncompleteHandshakeIsClosedAfterTheConfiguredTimeout() throws Exception {
        final SSLContext serverContext = largeCertificateContext();
        final TCPNIOTransport transport = TCPNIOTransportBuilder.newInstance()
                .setIOStrategy(SameThreadIOStrategy.getInstance()).setWriteBufferSize(4096).build();
        transport.start();
        final MockServerConnection serverConnection = new MockServerConnection();
        final LDAPListener listener = secureListener(transport, serverContext, serverConnection);
        final String property = "org.forgerock.opendj.grizzly.handshakeTimeout";
        final String previous = System.getProperty(property);
        System.setProperty(property, "1200");
        try {
            final InetSocketAddress address = listener.firstSocketAddress();
            final long connected = System.nanoTime();
            try (Socket socket = new Socket(address.getAddress(), address.getPort())) {
                serverConnection.context.get(5, TimeUnit.SECONDS);
                // A complete TLS record containing an incomplete ClientHello.
                socket.getOutputStream().write(new byte[] { 0x16, 0x03, 0x03, 0x00, 0x04, 0x01, 0x00, 0x01, 0x00 });
                socket.setSoTimeout(5000);
                org.testng.Assert.assertEquals(socket.getInputStream().read(), -1, "incomplete handshake stayed open");
                org.testng.Assert.assertTrue(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - connected) >= 800,
                        "the handshake was closed before its timeout");
            }
        } finally {
            restoreProperty(property, previous);
            listener.close();
            transport.shutdownNow();
        }
    }

    /** The timeout must not start before the first complete TLS record is received. */
    @Test(timeOut = 20000)
    public void testDelayedClientHelloCanCompleteHandshake() throws Exception {
        final SSLContext serverContext = largeCertificateContext();
        final TCPNIOTransport transport = TCPNIOTransportBuilder.newInstance()
                .setIOStrategy(SameThreadIOStrategy.getInstance()).setWriteBufferSize(4096).build();
        transport.start();
        final MockServerConnection serverConnection = new MockServerConnection();
        final LDAPListener listener = secureListener(transport, serverContext, serverConnection);
        final String property = "org.forgerock.opendj.grizzly.handshakeTimeout";
        final String previous = System.getProperty(property);
        System.setProperty(property, "1200");
        try {
            final InetSocketAddress address = listener.firstSocketAddress();
            try (Socket plain = new Socket(address.getAddress(), address.getPort())) {
                serverConnection.context.get(5, TimeUnit.SECONDS);
                Thread.sleep(2000);
                final SSLContext client = new SSLContextBuilder().setTrustManager(TrustManagers.trustAll())
                        .getSSLContext();
                try (SSLSocket socket = (SSLSocket) client.getSocketFactory().createSocket(
                        plain, address.getHostString(), address.getPort(), true)) {
                    socket.setSoTimeout(5000);
                    socket.startHandshake();
                    bindAnonymously(socket);
                }
            }
        } finally {
            restoreProperty(property, previous);
            listener.close();
            transport.shutdownNow();
        }
    }

    /** A partially received first record does not start Grizzly's handshake read timeout. */
    @Test(timeOut = 20000)
    public void testPartialFirstRecordDoesNotStartHandshakeTimeout() throws Exception {
        final SSLContext serverContext = largeCertificateContext();
        final TCPNIOTransport transport = TCPNIOTransportBuilder.newInstance()
                .setIOStrategy(SameThreadIOStrategy.getInstance()).setWriteBufferSize(4096).build();
        transport.start();
        final MockServerConnection serverConnection = new MockServerConnection();
        final LDAPListener listener = secureListener(transport, serverContext, serverConnection);
        final String property = "org.forgerock.opendj.grizzly.handshakeTimeout";
        final String previous = System.getProperty(property);
        System.setProperty(property, "1200");
        try {
            final InetSocketAddress address = listener.firstSocketAddress();
            try (Socket socket = new Socket(address.getAddress(), address.getPort())) {
                serverConnection.context.get(5, TimeUnit.SECONDS);
                socket.getOutputStream().write(new byte[] { 0x16, 0x03, 0x03, 0x00, 0x04, 0x01 });
                Thread.sleep(2000);
                socket.getOutputStream().write(new byte[] { 0x00, 0x01, 0x00 });
                socket.setSoTimeout(5000);
                final long lastWrite = System.nanoTime();
                org.testng.Assert.assertEquals(socket.getInputStream().read(), -1, "idle handshake stayed open");
                org.testng.Assert.assertTrue(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - lastWrite) >= 800,
                        "the timeout started before the first complete TLS record");
            }
        } finally {
            restoreProperty(property, previous);
            listener.close();
            transport.shutdownNow();
        }
    }

    /** Progress on a fragmented handshake renews the timeout for the next read. */
    @Test(timeOut = 20000)
    public void testHandshakeTimeoutIsRenewedByIncomingRecords() throws Exception {
        final SSLContext serverContext = largeCertificateContext();
        final TCPNIOTransport transport = TCPNIOTransportBuilder.newInstance()
                .setIOStrategy(SameThreadIOStrategy.getInstance()).setWriteBufferSize(4096).build();
        transport.start();
        final MockServerConnection serverConnection = new MockServerConnection();
        final LDAPListener listener = secureListener(transport, serverContext, serverConnection);
        final String property = "org.forgerock.opendj.grizzly.handshakeTimeout";
        final String previous = System.getProperty(property);
        System.setProperty(property, "2000");
        try {
            final InetSocketAddress address = listener.firstSocketAddress();
            try (Socket socket = new Socket(address.getAddress(), address.getPort())) {
                serverConnection.context.get(5, TimeUnit.SECONDS);
                socket.getOutputStream().write(new byte[] { 0x16, 0x03, 0x03, 0x00, 0x04, 0x01, 0x00, 0x01, 0x00 });
                Thread.sleep(750);
                socket.getOutputStream().write(new byte[] { 0x16, 0x03, 0x03, 0x00, 0x01, 0x03 });
                Thread.sleep(750);
                socket.getOutputStream().write(new byte[] { 0x16, 0x03, 0x03, 0x00, 0x01, 0x03 });
                socket.setSoTimeout(1000);
                try {
                    socket.getInputStream().read();
                    fail("the progressing handshake was closed prematurely");
                } catch (SocketTimeoutException expected) {
                    // The total elapsed time exceeds the timeout, but the last read was recent.
                }
                socket.setSoTimeout(5000);
                org.testng.Assert.assertEquals(socket.getInputStream().read(), -1, "idle handshake stayed open");
            }
        } finally {
            restoreProperty(property, previous);
            listener.close();
            transport.shutdownNow();
        }
    }

    /** An expiry which has started before a renewal must not close the renewed handshake. */
    @Test(timeOut = 20000)
    public void testStartedHandshakeExpiryDoesNotCloseRenewedConnection() throws Exception {
        final SSLContext serverContext = largeCertificateContext();
        final TCPNIOTransport transport = TCPNIOTransportBuilder.newInstance().build();
        transport.start();
        final MockServerConnection serverConnection = new MockServerConnection();
        final LDAPListener listener = secureListener(transport, serverContext, serverConnection);
        try {
            final InetSocketAddress address = listener.firstSocketAddress();
            try (Socket socket = new Socket(address.getAddress(), address.getPort())) {
                final LDAPClientContext context = serverConnection.context.get(5, TimeUnit.SECONDS);
                final Class<?> contextClass = context.getClass();
                final Method renew = contextClass.getDeclaredMethod("renewHandshakeDeadline", long.class);
                final Field deadlineField = contextClass.getDeclaredField("handshakeDeadline");
                final Field connectionField = contextClass.getDeclaredField("connection");
                renew.setAccessible(true);
                deadlineField.setAccessible(true);
                connectionField.setAccessible(true);
                final org.glassfish.grizzly.Connection<?> connection =
                        (org.glassfish.grizzly.Connection<?>) connectionField.get(context);
                final Thread expiry;
                synchronized (context) {
                    // Run the actual scheduled task early, then hold it at the context monitor while renewing.
                    renew.invoke(context, 60000L);
                    final ScheduledFuture<?> oldDeadline = (ScheduledFuture<?>) deadlineField.get(context);
                    expiry = new Thread((Runnable) oldDeadline, "test-started-handshake-expiry");
                    expiry.setDaemon(true);
                    expiry.start();
                    final long waitUntil = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
                    while (expiry.getState() != Thread.State.BLOCKED && System.nanoTime() < waitUntil) {
                        Thread.sleep(5);
                    }
                    org.testng.Assert.assertEquals(expiry.getState(), Thread.State.BLOCKED,
                            "the expiry did not start and wait for the context monitor");
                    final ThreadInfo blocked = ManagementFactory.getThreadMXBean().getThreadInfo(expiry.getId());
                    org.testng.Assert.assertNotNull(blocked.getLockInfo(), "the expiry is not waiting on a monitor");
                    org.testng.Assert.assertEquals(blocked.getLockInfo().getIdentityHashCode(),
                            System.identityHashCode(context), "the expiry is blocked on a different monitor");
                    renew.invoke(context, 5000L);
                    org.testng.Assert.assertTrue(oldDeadline.isCancelled(), "the previous deadline was not cancelled");
                }
                expiry.join(5000);
                org.testng.Assert.assertFalse(expiry.isAlive(), "the previous expiry did not finish");
                org.testng.Assert.assertTrue(connection.isOpen(), "the previous expiry closed the renewed handshake");
                socket.setSoTimeout(10000);
                org.testng.Assert.assertEquals(socket.getInputStream().read(), -1, "the renewed deadline did not fire");
            }
        } finally {
            listener.close();
            transport.shutdownNow();
        }
    }

    @DataProvider
    public Object[][] disabledHandshakeTimeouts() {
        return new Object[][] { { "0" }, { "-1" } };
    }

    /** Grizzly's zero and negative timeout values leave a pending handshake without a deadline. */
    @Test(dataProvider = "disabledHandshakeTimeouts", timeOut = 20000)
    public void testDisabledHandshakeTimeoutKeepsAnIncompleteHandshake(final String timeout) throws Exception {
        final SSLContext serverContext = largeCertificateContext();
        final TCPNIOTransport transport = TCPNIOTransportBuilder.newInstance()
                .setIOStrategy(SameThreadIOStrategy.getInstance()).setWriteBufferSize(4096).build();
        transport.start();
        final MockServerConnection serverConnection = new MockServerConnection();
        final LDAPListener listener = secureListener(transport, serverContext, serverConnection);
        final String property = "org.forgerock.opendj.grizzly.handshakeTimeout";
        final String previous = System.getProperty(property);
        System.setProperty(property, timeout);
        try {
            final InetSocketAddress address = listener.firstSocketAddress();
            try (Socket socket = new Socket(address.getAddress(), address.getPort())) {
                serverConnection.context.get(5, TimeUnit.SECONDS);
                // A complete record with an incomplete ClientHello reaches the deadline code.
                socket.getOutputStream().write(new byte[] { 0x16, 0x03, 0x03, 0x00, 0x04, 0x01, 0x00, 0x01, 0x00 });
                socket.setSoTimeout(1500);
                try {
                    socket.getInputStream().read();
                    fail("the incomplete handshake closed with timeout " + timeout);
                } catch (SocketTimeoutException expected) {
                    // The connection is still open.
                }
            }
        } finally {
            restoreProperty(property, previous);
            listener.close();
            transport.shutdownNow();
        }
    }

    @DataProvider
    public Object[][] completedHandshakeProtocols() {
        return new Object[][] { { "TLSv1.2" }, { "TLSv1.3" } };
    }

    /** Completing the handshake cancels its timeout, so it cannot close a subsequently active LDAP connection. */
    @Test(dataProvider = "completedHandshakeProtocols", timeOut = 30000)
    public void testCompletedHandshakeIsNotClosedByItsFormerTimeout(final String protocol) throws Exception {
        final SSLContext serverContext = largeCertificateContext();
        final TCPNIOTransport transport = TCPNIOTransportBuilder.newInstance()
                .setIOStrategy(SameThreadIOStrategy.getInstance()).setWriteBufferSize(4096).build();
        transport.start();
        final MockServerConnection serverConnection = new MockServerConnection();
        final LDAPListener listener = secureListener(transport, serverContext, serverConnection, protocol);
        final String property = "org.forgerock.opendj.grizzly.handshakeTimeout";
        final String previous = System.getProperty(property);
        System.setProperty(property, "5000");
        try {
            final InetSocketAddress address = listener.firstSocketAddress();
            final SSLContext clientContext = new SSLContextBuilder().setTrustManager(TrustManagers.trustAll())
                    .getSSLContext();
            try (SSLSocket socket = (SSLSocket) clientContext.getSocketFactory().createSocket(
                    address.getAddress(), address.getPort())) {
                socket.setSoTimeout(8000);
                socket.startHandshake();
                socket.setSoTimeout(6000);
                try {
                    socket.getInputStream().read();
                    fail("the completed connection was closed by the handshake timeout");
                } catch (SocketTimeoutException expected) {
                    // The connection remained open past the deadline without any LDAP traffic.
                }
                bindAnonymously(socket);
            }
        } finally {
            restoreProperty(property, previous);
            listener.close();
            transport.shutdownNow();
        }
    }

    /** A TLS upgrade must not be accepted after the client connection has closed. */
    @Test(timeOut = 20000)
    public void testEnableTlsRefusesAClosedConnection() throws Exception {
        final SSLContext serverContext = largeCertificateContext();
        final TCPNIOTransport transport = TCPNIOTransportBuilder.newInstance().build();
        transport.start();
        final MockServerConnection serverConnection = new MockServerConnection();
        final Options options = defaultOptions().set(GrizzlyLDAPListener.GRIZZLY_TRANSPORT, transport);
        final LDAPListener listener = new LDAPListener(Collections.singleton(loopbackWithDynamicPort()),
                new ServerConnectionFactoryAdapter(options.get(LDAP_DECODE_OPTIONS),
                        new MockServerConnectionFactory(serverConnection)), options);
        try {
            final InetSocketAddress address = listener.firstSocketAddress();
            final LDAPClientContext clientContext;
            try (Socket socket = new Socket(address.getAddress(), address.getPort())) {
                clientContext = serverConnection.context.get(5, TimeUnit.SECONDS);
            }
            final long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
            while (!clientContext.isClosed() && System.nanoTime() < deadline) {
                Thread.sleep(10);
            }
            org.testng.Assert.assertTrue(clientContext.isClosed(), "the server did not see the client close");
            org.testng.Assert.assertFalse(enableServerTLS(clientContext, serverContext, "TLSv1.2", true),
                    "enableTLS accepted a closed connection");
        } finally {
            listener.close();
            transport.shutdownNow();
        }
    }

    /** A stalled TLS 1.2 renegotiation must retain the configured Grizzly read timeout. */
    @Test(timeOut = 30000)
    public void testIncompleteRenegotiationTimesOut() throws Exception {
        final SSLContext serverContext = largeCertificateContext();
        final TCPNIOTransport transport = TCPNIOTransportBuilder.newInstance()
                .setIOStrategy(SameThreadIOStrategy.getInstance()).setWriteBufferSize(4096).build();
        transport.start();
        final MockServerConnection serverConnection = new MockServerConnection();
        final LDAPListener listener = secureListener(transport, serverContext, serverConnection);
        final String property = "org.forgerock.opendj.grizzly.handshakeTimeout";
        final String previous = System.getProperty(property);
        System.setProperty(property, "1200");
        try {
            final InetSocketAddress address = listener.firstSocketAddress();
            final SSLContext clientContext = new SSLContextBuilder().setTrustManager(TrustManagers.trustAll())
                    .getSSLContext();
            try (Socket socket = new Socket(address.getAddress(), address.getPort())) {
                socket.setSoTimeout(5000);
                final SSLEngine engine = clientContext.createSSLEngine("localhost", address.getPort());
                engine.setUseClientMode(true);
                engine.setEnabledProtocols(new String[] { "TLSv1.2" });
                finishClientHandshake(engine, socket);

                engine.beginHandshake();
                final ByteBuffer hello = ByteBuffer.allocate(engine.getSession().getPacketBufferSize());
                final SSLEngineResult result = engine.wrap(ByteBuffer.allocate(0), hello);
                org.testng.Assert.assertEquals(result.getStatus(), SSLEngineResult.Status.OK);
                org.testng.Assert.assertTrue(hello.position() > 0, "no renegotiation ClientHello was produced");
                final long started = System.nanoTime();
                socket.getOutputStream().write(hello.array(), 0, hello.position());
                // Drain the server's flight, but do not send the rest of the client's handshake.
                int received = 0;
                while (socket.getInputStream().read() != -1) {
                    received++;
                }
                org.testng.Assert.assertTrue(received > 0, "the server did not start renegotiation");
                org.testng.Assert.assertTrue(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - started) >= 800,
                        "the server closed before the renegotiation timeout");
            }
        } finally {
            restoreProperty(property, previous);
            listener.close();
            transport.shutdownNow();
        }
    }

    private static void finishClientHandshake(SSLEngine engine, Socket socket) throws Exception {
        final ByteBuffer input = ByteBuffer.allocate(65536);
        final ByteBuffer output = ByteBuffer.allocate(65536);
        final ByteBuffer application = ByteBuffer.allocate(65536);
        final ByteBuffer empty = ByteBuffer.allocate(0);
        final DataInputStream in = new DataInputStream(socket.getInputStream());
        engine.beginHandshake();
        while (engine.getHandshakeStatus() != SSLEngineResult.HandshakeStatus.NOT_HANDSHAKING) {
            switch (engine.getHandshakeStatus()) {
            case NEED_TASK:
                Runnable task;
                while ((task = engine.getDelegatedTask()) != null) {
                    task.run();
                }
                break;
            case NEED_WRAP:
                output.clear();
                final SSLEngineResult wrapped = engine.wrap(empty, output);
                org.testng.Assert.assertEquals(wrapped.getStatus(), SSLEngineResult.Status.OK);
                socket.getOutputStream().write(output.array(), 0, output.position());
                break;
            case NEED_UNWRAP:
                final byte[] header = new byte[5];
                in.readFully(header);
                final int length = (header[3] & 0xff) * 256 + (header[4] & 0xff);
                input.put(header);
                in.readFully(input.array(), input.position(), length);
                input.position(input.position() + length);
                input.flip();
                final SSLEngineResult unwrapped = engine.unwrap(input, application);
                input.compact();
                org.testng.Assert.assertTrue(unwrapped.getStatus() == SSLEngineResult.Status.OK
                        || unwrapped.getStatus() == SSLEngineResult.Status.BUFFER_UNDERFLOW);
                break;
            default:
                throw new AssertionError("Unexpected handshake status: " + engine.getHandshakeStatus());
            }
        }
    }

    /** StartTLS after an LDAP request still needs a timeout once the TLS handshake has started. */
    @Test(timeOut = 20000)
    public void testStartTlsHandshakeTimesOutAfterAnEarlierLdapRequest() throws Exception {
        final SSLContext serverContext = largeCertificateContext();
        final TCPNIOTransport transport = TCPNIOTransportBuilder.newInstance()
                .setIOStrategy(SameThreadIOStrategy.getInstance()).setWriteBufferSize(4096).build();
        transport.start();
        final MockServerConnection serverConnection = new MockServerConnection() {
            @Override
            public <R extends ExtendedResult> void handleExtendedRequest(final Integer requestContext,
                    final ExtendedRequest<R> request, final IntermediateResponseHandler intermediateResponseHandler,
                    final LdapResultHandler<R> resultHandler) {
                try {
                    final LDAPClientContext clientContext = context.get(5, TimeUnit.SECONDS);
                    org.testng.Assert.assertTrue(enableServerTLS(clientContext, serverContext, "TLSv1.2", true));
                    org.testng.Assert.assertFalse(enableServerTLS(clientContext, serverContext, "TLSv1.2", true),
                            "a second StartTLS was accepted while the first was pending");
                    resultHandler.handleResult(request.getResultDecoder().newExtendedErrorResult(
                            ResultCode.SUCCESS, "", ""));
                } catch (Exception e) {
                    resultHandler.handleException(newLdapException(ResultCode.OTHER, e.getMessage(), e));
                }
            }
        };
        final Options options = defaultOptions().set(GrizzlyLDAPListener.GRIZZLY_TRANSPORT, transport);
        final LDAPListener listener = new LDAPListener(Collections.singleton(loopbackWithDynamicPort()),
                new ServerConnectionFactoryAdapter(options.get(LDAP_DECODE_OPTIONS),
                        new MockServerConnectionFactory(serverConnection)), options);
        final String property = "org.forgerock.opendj.grizzly.handshakeTimeout";
        final String previous = System.getProperty(property);
        System.setProperty(property, "1200");
        try {
            final InetSocketAddress address = listener.firstSocketAddress();
            try (Socket socket = new Socket(address.getAddress(), address.getPort())) {
                socket.setSoTimeout(5000);
                bindAnonymously(socket);
                startTLS(socket, (byte) 2);
                socket.getOutputStream().write(new byte[] { 0x16, 0x03, 0x03, 0x00, 0x04, 0x01, 0x00, 0x01, 0x00 });
                org.testng.Assert.assertEquals(socket.getInputStream().read(), -1, "StartTLS handshake stayed open");
            }
        } finally {
            restoreProperty(property, previous);
            listener.close();
            transport.shutdownNow();
        }
    }

    private static void bindAnonymously(Socket socket) throws IOException {
        socket.getOutputStream().write(new byte[] {
                0x30, 0x0c, 0x02, 0x01, 0x01, 0x60, 0x07, 0x02, 0x01, 0x03, 0x04, 0x00, (byte) 0x80, 0x00 });
        final byte[] response = new byte[14];
        new DataInputStream(socket.getInputStream()).readFully(response);
        org.testng.Assert.assertEquals(response, new byte[] {
                0x30, 0x0c, 0x02, 0x01, 0x01, 0x61, 0x07, 0x0a, 0x01, 0x00, 0x04, 0x00, 0x04, 0x00 });
    }

    private static void startTLS(Socket socket, byte messageId) throws IOException {
        socket.getOutputStream().write(new byte[] {
                0x30, 0x1d, 0x02, 0x01, messageId, 0x77, 0x18, (byte) 0x80, 0x16 });
        socket.getOutputStream().write("1.3.6.1.4.1.1466.20037".getBytes(StandardCharsets.US_ASCII));
        final byte[] response = new byte[14];
        new DataInputStream(socket.getInputStream()).readFully(response);
        org.testng.Assert.assertEquals(response, new byte[] {
                0x30, 0x0c, 0x02, 0x01, messageId, 0x78, 0x07, 0x0a, 0x01, 0x00, 0x04, 0x00, 0x04, 0x00 });
    }

    private static LDAPListener secureListener(TCPNIOTransport transport, SSLContext serverContext,
            MockServerConnection serverConnection) throws IOException {
        return secureListener(transport, serverContext, serverConnection, "TLSv1.2");
    }

    private static LDAPListener secureListener(TCPNIOTransport transport, SSLContext serverContext,
            MockServerConnection serverConnection, String protocol) throws IOException {
        final MockServerConnectionFactory factory = new MockServerConnectionFactory(serverConnection) {
            @Override
            public ServerConnection<Integer> handleAccept(final LDAPClientContext context) throws LdapException {
                enableServerTLS(context, serverContext, protocol, false);
                return super.handleAccept(context);
            }
        };
        final Options options = defaultOptions().set(GrizzlyLDAPListener.GRIZZLY_TRANSPORT, transport);
        return new LDAPListener(Collections.singleton(loopbackWithDynamicPort()),
                new ServerConnectionFactoryAdapter(options.get(LDAP_DECODE_OPTIONS), factory), options);
    }

    private static void restoreProperty(String name, String value) {
        if (value == null) {
            System.clearProperty(name);
        } else {
            System.setProperty(name, value);
        }
    }

    private static SSLContext largeCertificateContext() throws Exception {
        return largeCertificateContext(180);
    }

    private static SSLContext largeCertificateContext(int nameCount) throws Exception {
        final KeyPairGenerator generator = KeyPairGenerator.getInstance("RSA");
        generator.initialize(2048);
        final KeyPair pair = generator.generateKeyPair();
        final X500Principal subject = new X500Principal("CN=localhost");
        final Instant now = Instant.now();
        final JcaX509v3CertificateBuilder builder = new JcaX509v3CertificateBuilder(subject, BigInteger.ONE,
                Date.from(now.minus(1, ChronoUnit.DAYS)), Date.from(now.plus(1, ChronoUnit.DAYS)), subject,
                pair.getPublic());
        final GeneralName[] names = new GeneralName[nameCount];
        for (int i = 0; i < names.length; i++) {
            names[i] = new GeneralName(GeneralName.dNSName, String.format("host-%03d.diagnostic-1287.example.test", i));
        }
        builder.addExtension(Extension.subjectAlternativeName, false, new GeneralNames(names));
        final X509Certificate certificate = new JcaX509CertificateConverter().getCertificate(
                builder.build(new JcaContentSignerBuilder("SHA256withRSA").build(pair.getPrivate())));
        org.testng.Assert.assertTrue(certificate.getEncoded().length > 6144, "certificate is too small for this test");
        if (nameCount >= 600) {
            org.testng.Assert.assertTrue(certificate.getEncoded().length > 16384,
                    "certificate does not exceed one TLS record");
        }
        final KeyStore keyStore = KeyStore.getInstance("JKS");
        keyStore.load(null, null);
        final char[] password = "test-password".toCharArray();
        keyStore.setKeyEntry("server", pair.getPrivate(), password, new X509Certificate[] { certificate });
        final KeyManagerFactory factory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
        factory.init(keyStore, password);
        final SSLContext context = SSLContext.getInstance("TLS");
        context.init(factory.getKeyManagers(), null, null);
        return context;
    }

    /**
     * Tests LDAP listener which attempts to open a connection to a remote
     * offline server at the point when the listener accepts the client

 opendj-server-legacy/src/main/java/org/forgerock/opendj/reactive/LDAPClientConnection2.java

@@ -44,7 +44,6 @@
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;

import javax.net.ssl.SSLEngine;
import javax.net.ssl.SSLException;
import javax.net.ssl.SSLPeerUnverifiedException;
import javax.net.ssl.SSLSession;
@@ -1665,7 +1664,7 @@
        }
        try {
            if (!clientContext.enableTLS(connectionHandler.createSSLEngine(), true)) {
                unavailableReason.append(ERR_LDAP_TLS_EXISTING_SECURITY_PROVIDER.get(SSLEngine.class.getName()));
                unavailableReason.append(ERR_LDAP_TLS_UPGRADE_UNAVAILABLE.get());
                return false;
            }
        } catch (DirectoryException de) {

 opendj-server-legacy/src/messages/org/opends/messages/protocol.properties

@@ -13,7 +13,7 @@
# Copyright 2006-2009 Sun Microsystems, Inc.
# Portions Copyright 2013-2016 ForgeRock AS.
# Portions copyright 2013-2014 Manuel Gaupp
# Portions Copyright 2026 3A Systems LLC.
# Portions Copyright 2026 3A Systems, LLC.


#
@@ -453,6 +453,8 @@
ERR_LDAP_TLS_CANNOT_CREATE_TLS_PROVIDER_273=An error occurred while \
 attempting to create a TLS connection security provider for this client \
 connection for use with StartTLS: %s
ERR_LDAP_TLS_UPGRADE_UNAVAILABLE_1544=StartTLS cannot be enabled because the \
 connection is closed, TLS is already enabled, or a TLS upgrade is pending
NOTE_CONNHANDLER_STARTED_LISTENING_276=Started listening for new \
 connections on %s
NOTE_CONNHANDLER_STOPPED_LISTENING_277=Stopped listening for new \

 opendj-server-legacy/src/test/java/org/opends/server/protocols/ldap/LDAPConnectionLimitsTestCase.java

@@ -48,6 +48,7 @@
import javax.net.ssl.X509TrustManager;

import org.forgerock.i18n.LocalizableMessage;
import org.forgerock.i18n.LocalizableMessageBuilder;
import org.forgerock.opendj.config.DefinedDefaultBehaviorProvider;
import org.forgerock.opendj.config.PropertyDefinition;
import org.forgerock.opendj.ldap.DN;
@@ -512,6 +513,53 @@
    }
  }

  /** A refused TLS upgrade must not claim that TLS is already enabled on a closed connection. */
  @Test
  public void closedConnectionHasAnAccurateStartTlsRefusal() throws Exception
  {
    final int port = TestCaseUtils.findFreePort();
    final LDAPConnectionHandler2 handler = start(configuration(port, true));
    try
    {
      final LDAPClientConnection2 connection;
      try (Socket client = new Socket("127.0.0.1", port))
      {
        awaitClientConnections(handler, 1);
        connection = (LDAPClientConnection2) handler.getClientConnections().iterator().next();
      }
      awaitClientConnections(handler, 0);
      final LocalizableMessageBuilder reason = new LocalizableMessageBuilder();
      assertFalse(connection.prepareTLS(reason));
      assertTrue(reason.toString().contains("closed"), reason.toString());
    }
    finally
    {
      stop(handler);
    }
  }

  /** A second StartTLS attempt while the first upgrade is pending is also reported accurately. */
  @Test
  public void pendingUpgradeHasAnAccurateStartTlsRefusal() throws Exception
  {
    final int port = TestCaseUtils.findFreePort();
    final LDAPConnectionHandler2 handler = start(configuration(port, true));
    try (Socket client = new Socket("127.0.0.1", port))
    {
      awaitClientConnections(handler, 1);
      final LDAPClientConnection2 connection =
          (LDAPClientConnection2) handler.getClientConnections().iterator().next();
      assertTrue(connection.prepareTLS(new LocalizableMessageBuilder()));
      final LocalizableMessageBuilder reason = new LocalizableMessageBuilder();
      assertFalse(connection.prepareTLS(reason));
      assertTrue(reason.toString().contains("pending"), reason.toString());
    }
    finally
    {
      stop(handler);
    }
  }

  /** Connections are counted whatever the limit, so a limit set on a running handler covers those already open. */
  @Test
  public void limitPerAddressSetAtRuntimeCoversTheOpenConnections() throws Exception
@@ -870,6 +918,12 @@

  private static LDAPConnectionHandlerCfg configuration(int port, String... limits) throws Exception
  {
    return configuration(port, false, limits);
  }

  private static LDAPConnectionHandlerCfg configuration(int port, boolean allowStartTls, String... limits)
      throws Exception
  {
    final List<String> lines = new ArrayList<>();
    lines.add("dn: cn=Limits Test Handler,cn=Connection Handlers,cn=config");
    lines.add("objectClass: top");
@@ -882,7 +936,11 @@
    lines.add("ds-cfg-listen-port: " + port);
    lines.add("ds-cfg-keep-stats: false");
    lines.add("ds-cfg-use-ssl: false");
    lines.add("ds-cfg-allow-start-tls: false");
    lines.add("ds-cfg-allow-start-tls: " + allowStartTls);
    if (allowStartTls)
    {
      lines.add("ds-cfg-key-manager-provider: cn=JKS,cn=Key Manager Providers,cn=config");
    }
    lines.add("ds-cfg-allow-ldap-v2: false");
    lines.add("ds-cfg-send-rejection-notice: true");
    for (String limit : limits)