Fix JMX RMI connector startup failure introduced by CVE-2026-46495 hardenin...

Valery Kharseko

4 days ago d4a95c2e37c375756f9829c7ec43a4785b99e2e2

Fix JMX RMI connector startup failure introduced by CVE-2026-46495 hardening (#651)

 opendj-server-legacy/src/main/java/org/opends/server/protocols/jmx/RmiConnector.java

@@ -65,9 +65,6 @@
{
  private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();

  static final String JMX_REMOTE_RMI_SERVER_CREDENTIAL_TYPES =
      "jmx.remote.rmi.server.credential.types";

  /**
   * JDK 10+ JMX environment property scoping a JEP 290 deserialization
   * filter to the credentials object passed during {@code newClient()}.
@@ -75,15 +72,16 @@
   * {@code jmx.remote.rmi.server.serial.filter.pattern}) avoids breaking
   * legitimate JMX traffic such as MBean invocations and notifications,
   * which may legitimately carry non-String types.
   * <p>
   * Note: this property is mutually exclusive with
   * {@code jmx.remote.rmi.server.credential.types}; specifying both makes
   * {@code RMIJRMPServerImpl} throw an {@link IllegalArgumentException} and
   * prevents the connector from starting. The filter pattern is preferred
   * because it additionally constrains array length and nesting depth.
   */
  static final String JMX_REMOTE_RMI_SERVER_CREDENTIALS_FILTER_PATTERN =
      "jmx.remote.rmi.server.credentials.filter.pattern";

  private static final String[] JMX_CREDENTIAL_TYPES =
  {
    String.class.getName(),
    String[].class.getName()
  };

  private static final String JMX_CREDENTIAL_SERIAL_FILTER =
      "maxdepth=3;maxarray=2;java.lang.String;!*";
@@ -392,11 +390,13 @@

  static void configureJmxDeserializationProtection(Map<String, Object> env)
  {
    env.put(JMX_REMOTE_RMI_SERVER_CREDENTIAL_TYPES,
        JMX_CREDENTIAL_TYPES.clone());
    // Scope the JEP 290 deserialization filter to the credentials object
    // only, so legitimate JMX RMI traffic (MBean operations, notifications,
    // etc.) is not affected by the restrictive allowlist.
    //
    // Do NOT also set "jmx.remote.rmi.server.credential.types": the JDK
    // rejects an environment that defines both properties, which would
    // prevent the RMI connector from starting.
    env.put(JMX_REMOTE_RMI_SERVER_CREDENTIALS_FILTER_PATTERN,
        JMX_CREDENTIAL_SERIAL_FILTER);
  }

 opendj-server-legacy/src/test/java/org/opends/server/protocols/jmx/RmiAuthenticatorTest.java

@@ -67,29 +67,15 @@
    Map<String, Object> env = new HashMap<>();
    RmiConnector.configureJmxDeserializationProtection(env);

    assertEquals(env.get(RmiConnector.JMX_REMOTE_RMI_SERVER_CREDENTIAL_TYPES),
        new String[] { String.class.getName(), String[].class.getName() });
    assertEquals(env.get(RmiConnector.JMX_REMOTE_RMI_SERVER_CREDENTIALS_FILTER_PATTERN),
        "maxdepth=3;maxarray=2;java.lang.String;!*");
    // The connector-wide filter must NOT be set, so legitimate JMX traffic
    // (MBean operations, notifications) is not affected by the allowlist.
    assertNull(env.get("jmx.remote.rmi.server.serial.filter.pattern"));
  }

  /** Verifies that each environment receives its own credential type array. */
  @Test
  public void credentialTypesAreDefensivelyCopied()
  {
    Map<String, Object> env = new HashMap<>();
    RmiConnector.configureJmxDeserializationProtection(env);
    String[] credentialTypes =
        (String[]) env.get(RmiConnector.JMX_REMOTE_RMI_SERVER_CREDENTIAL_TYPES);
    credentialTypes[0] = Date.class.getName();

    Map<String, Object> env2 = new HashMap<>();
    RmiConnector.configureJmxDeserializationProtection(env2);
    assertEquals(((String[]) env2.get(RmiConnector.JMX_REMOTE_RMI_SERVER_CREDENTIAL_TYPES))[0],
        String.class.getName());
    // "jmx.remote.rmi.server.credential.types" is mutually exclusive with the
    // credentials filter pattern: setting both prevents the connector from
    // starting, so only the filter pattern must be configured.
    assertNull(env.get("jmx.remote.rmi.server.credential.types"));
  }

  /** Verifies the configured filter allows only the expected credential payload. */