[#1261] Close JMX RMI connector sockets on shutdown (#1264)

Valery Kharseko

12 hours ago cdc884c34f5d7f22624d4e85a8bb61fd50560e5e

[#1261] Close JMX RMI connector sockets on shutdown (#1264)

Fixes #1261.

With `rmi-port: 0`, every JMX handler start created a new RMI connector
listener that remained open after the handler stopped. Unexporting the
connector does not close an anonymous-port listener in the JDK. Repeated
embedded-server restarts therefore accumulated listening ports and
accept threads.

Retain the connector's socket factory and explicitly close its socket
after stopping the connector, for both plain and SSL connections. Both
factories implement `Closeable`, and the SSL factory now remembers its
server socket. Cleanup also runs if connector shutdown throws or a
configuration change keeps the RMI registry running. Repeated
finalization is safe.

The regression tests open real JMX sessions and verify that the old port
refuses connections and can be rebound after shutdown and after a
configuration change. They cover both SSL settings and three successive
handler start/stop cycles.

Validation on macOS:

- All four new regression cases failed on the original code because the
stopped listener still accepted connections.
- `JmxListenAddressTestCase` on JDK 27: 9 passed, 1 skipped.
- Full JMX package on JDK 17, using the Maven reactor and `precommit`
profile: 55 passed, 1 skipped; build successful.
- The existing wildcard-hostname case is skipped because the local
hostname resolves to loopback.
- `git diff --check` passed.

 opendj-server-legacy/src/main/java/org/opends/server/protocols/jmx/DirectoryRMIServerSocketFactory.java

@@ -19,6 +19,7 @@

import org.forgerock.i18n.slf4j.LocalizedLogger;

import java.io.Closeable;
import java.io.IOException;
import java.net.InetAddress;
import java.net.ServerSocket;
@@ -37,7 +38,7 @@
 * Sockets Layer (SSL) or Transport Layer Security (TLS) protocols.
 */
public class DirectoryRMIServerSocketFactory implements
    RMIServerSocketFactory
    RMIServerSocketFactory, Closeable
{
  private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();

@@ -50,6 +51,9 @@
  /** The address to listen on, which could be INADDR_ANY. */
  private final InetAddress listenAddress;

  /** The server socket owned by this factory. */
  private ServerSocket serverSocket;

  /**
   * Constructs a new <code>DirectoryRMIServerSocketFactory</code> with the
   * specified SSL socket configuration.
@@ -105,7 +109,7 @@
  @Override
  public ServerSocket createServerSocket(int port) throws IOException
  {
    return new ServerSocket(port, 0, listenAddress)
    serverSocket = new ServerSocket(port, 0, listenAddress)
    {
      @Override
      public Socket accept() throws IOException
@@ -127,6 +131,22 @@
        return sslSocket;
      }
    };
    return serverSocket;
  }

  /**
   * Closes the underlying server socket, including an anonymous RMI listener.
   *
   * @throws IOException if the socket cannot be closed
   */
  @Override
  public void close() throws IOException
  {
    if (serverSocket != null)
    {
      serverSocket.close();
      serverSocket = null;
    }
  }

  /**

 opendj-server-legacy/src/main/java/org/opends/server/protocols/jmx/OpendsRmiServerSocketFactory.java

@@ -13,9 +13,11 @@
 *
 * Copyright 2006-2008 Sun Microsystems, Inc.
 * Portions Copyright 2014-2015 ForgeRock AS.
 * Portions Copyright 2026 3A Systems, LLC.
 */
package org.opends.server.protocols.jmx;

import java.io.Closeable;
import java.io.IOException;
import java.net.InetAddress;
import java.net.ServerSocket;
@@ -38,7 +40,7 @@
 * <p>
 * Each connector is registered into the JMX MBean server.
 */
public class OpendsRmiServerSocketFactory implements RMIServerSocketFactory
public class OpendsRmiServerSocketFactory implements RMIServerSocketFactory, Closeable
{
  /** The address to listen on, which could be INADDR_ANY. */
  private final InetAddress listenAddress;
@@ -69,7 +71,8 @@
   *
   * @throws IOException If an I/O error occurs when closing the socket.
   */
  void close() throws IOException
  @Override
  public void close() throws IOException
  {
    if (serverSocket != null)
    {

 opendj-server-legacy/src/main/java/org/opends/server/protocols/jmx/RmiConnector.java

@@ -20,6 +20,7 @@
import static org.opends.messages.ProtocolMessages.*;
import static org.opends.server.util.StaticUtils.*;

import java.io.Closeable;
import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
@@ -165,6 +166,9 @@
   */
  private OpendsRmiServerSocketFactory rmiSsf;

  /** The socket factory owned by the RMI connector, separate from the registry's. */
  private Closeable connectorSocketFactory;

  /**
   * The RMI protocol version used by this connector.
   */
@@ -371,8 +375,10 @@
        SSLSocketFactory ssf = ctx.getSocketFactory();

        // set the Server socket factory in the JMX map
        rmiServerSockeyFactory = new DirectoryRMIServerSocketFactory(
        DirectoryRMIServerSocketFactory factory = new DirectoryRMIServerSocketFactory(
            ssf, false, jmxConnectionHandler.getListenAddress());
        connectorSocketFactory = factory;
        rmiServerSockeyFactory = factory;
        env.put(
            "jmx.remote.rmi.server.socket.factory",
            rmiServerSockeyFactory);
@@ -393,8 +399,10 @@
        {
          logger.trace("UNSECURE CONNECTION");
        }
        rmiServerSockeyFactory = new OpendsRmiServerSocketFactory(
        OpendsRmiServerSocketFactory factory = new OpendsRmiServerSocketFactory(
            jmxConnectionHandler.getListenAddress());
        connectorSocketFactory = factory;
        rmiServerSockeyFactory = factory;
      }

      // specify the rmi JMX authenticator to be used
@@ -553,6 +561,11 @@
    jmxRmiConnectorNoClientCertificate = null;
    jmxRmiConnectorClientCertificate = null;

    // RMI keeps anonymous-port listeners open even after the last object is unexported.
    // Close our socket explicitly, also when stop() failed or the registry is being kept.
    close(connectorSocketFactory);
    connectorSocketFactory = null;

    // Unregister connectors and stop them.
    try
    {

 opendj-server-legacy/src/test/java/org/opends/server/protocols/jmx/JmxListenAddressTestCase.java

@@ -21,6 +21,7 @@

import java.io.IOException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.Socket;
import java.rmi.registry.LocateRegistry;
@@ -28,6 +29,8 @@
import java.security.cert.X509Certificate;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.management.remote.JMXConnector;
import javax.net.ssl.SSLContext;
@@ -201,6 +204,83 @@
    }
  }

  /** Each handler restart must release the anonymous RMI port, with and without SSL. */
  @Test(dataProvider = "useSSL")
  public void releasesTheAnonymousRmiPortAfterEachStop(boolean useSSL) throws Exception
  {
    final int listenPort = TestCaseUtils.findFreePort();
    for (int i = 0; i < 3; i++)
    {
      JmxConnectionHandler handler = newHandler(listenAddress, listenPort, 0, useSSL);
      final int rmiPort;
      try
      {
        handler.run();
        rmiPort = anonymousRmiPort(listenAddress, listenPort);
        assertOpensASession(listenAddress, listenPort);
      }
      finally
      {
        handler.finalizeConnectionHandler(STOP_REASON);
        // Shutdown may be called both by processServerShutdown and by finalization.
        handler.finalizeConnectionHandler(STOP_REASON);
      }
      assertPortReleased(listenAddress, rmiPort);
      assertPortReleased(listenAddress, listenPort);
    }
  }

  /** Restarting only the connector must close its old socket while keeping the registry usable. */
  @Test(dataProvider = "useSSL")
  public void releasesTheAnonymousRmiPortAfterAConfigurationChange(boolean useSSL) throws Exception
  {
    final int listenPort = TestCaseUtils.findFreePort();
    JmxConnectionHandler handler = newHandler(listenAddress, listenPort, 0, useSSL);
    try
    {
      handler.run();
      final int oldRmiPort = anonymousRmiPort(listenAddress, listenPort);
      assertOpensASession(listenAddress, listenPort);

      // Select the replacement port while the old connector is listening: it cannot be the old port.
      final int newRmiPort = TestCaseUtils.findFreePort();
      applyConfigurationChange(handler, newConfig(listenAddress, listenPort, newRmiPort, useSSL));

      assertPortReleased(listenAddress, oldRmiPort);
      assertOpensASession(listenAddress, listenPort);
    }
    finally
    {
      handler.finalizeConnectionHandler(STOP_REASON);
    }
  }

  /**
   * The configured port is zero, so read the assigned port from the stub's endpoint description.
   * This avoids opening JDK-internal RMI packages for reflection in the test JVM.
   */
  private static int anonymousRmiPort(InetAddress address, int listenPort) throws Exception
  {
    Registry registry = LocateRegistry.getRegistry(address.getHostAddress(), listenPort);
    String stub = registry.lookup("org.opends.server.protocols.jmx.client-unknown").toString();
    Matcher endpoint = Pattern.compile("endpoint:\\[[^\\]]*?:(\\d+)[,\\]]").matcher(stub);
    assertTrue(endpoint.find(), "Cannot find the RMI endpoint in " + stub);
    int port = Integer.parseInt(endpoint.group(1));
    assertTrue(port > 0, "The RMI stub must advertise the assigned port");
    assertTrue(isAcceptingConnections(address, port), "The RMI connector must listen before it is stopped");
    return port;
  }

  private static void assertPortReleased(InetAddress address, int port) throws IOException
  {
    assertFalse(isAcceptingConnections(address, port), "The stopped RMI listener still accepts connections on " + port);
    try (ServerSocket socket = new ServerSocket())
    {
      socket.setReuseAddress(true);
      socket.bind(new InetSocketAddress(address, port));
    }
  }

  /** A change that restarts the RMI connector keeps the listen address that is reported. */
  @Test
  public void reportsTheListenAddressAfterAChangeOfTheRmiPort() throws Exception