From 7e3a75903159153c877daeb2952a552701e38044 Mon Sep 17 00:00:00 2001
From: Valery Kharseko <vharseko@3a-systems.ru>
Date: Thu, 11 Jun 2026 08:34:19 +0000
Subject: [PATCH] CVE-2026-46495 OpenDJ Unauthenticated RCE via Java Deserialization in JMX RMI
---
opendj-server-legacy/src/main/java/org/opends/server/protocols/jmx/RmiAuthenticator.java | 19 ++++
opendj-server-legacy/src/main/java/org/opends/server/protocols/jmx/RmiConnector.java | 37 +++++++++
opendj-server-legacy/src/test/java/org/opends/server/protocols/jmx/RmiAuthenticatorTest.java | 142 +++++++++++++++++++++++++++++++++++
3 files changed, 195 insertions(+), 3 deletions(-)
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/protocols/jmx/RmiAuthenticator.java b/opendj-server-legacy/src/main/java/org/opends/server/protocols/jmx/RmiAuthenticator.java
index 9377b25..dfc2432 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/protocols/jmx/RmiAuthenticator.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/protocols/jmx/RmiAuthenticator.java
@@ -13,6 +13,7 @@
*
* Copyright 2006-2010 Sun Microsystems, Inc.
* Portions Copyright 2014-2016 ForgeRock AS.
+ * Portions Copyright 2026 3A Systems, LLC.
*/
package org.opends.server.protocols.jmx;
@@ -102,9 +103,21 @@
{
throw new SecurityException();
}
- Object c[] = (Object[]) credentials;
- String authcID = (String) c[0];
- String password = (String) c[1];
+ if (!(credentials instanceof String[]))
+ {
+ logger.trace("Invalid JMX credentials type");
+ throw new SecurityException();
+ }
+
+ String[] c = (String[]) credentials;
+ if (c.length != 2)
+ {
+ logger.trace("Invalid JMX credentials length");
+ throw new SecurityException();
+ }
+
+ String authcID = c[0];
+ String password = c[1];
// The authcID is used at forwarder level to identify the calling client
if (authcID == null)
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/protocols/jmx/RmiConnector.java b/opendj-server-legacy/src/main/java/org/opends/server/protocols/jmx/RmiConnector.java
index 33a9ba6..1a418b9 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/protocols/jmx/RmiConnector.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/protocols/jmx/RmiConnector.java
@@ -13,6 +13,7 @@
*
* Copyright 2006-2009 Sun Microsystems, Inc.
* Portions Copyright 2013-2015 ForgeRock AS.
+ * Portions Copyright 2023-2026 3A Systems, LLC.
*/
package org.opends.server.protocols.jmx;
@@ -22,6 +23,7 @@
import java.rmi.registry.LocateRegistry;
import java.rmi.registry.Registry;
import java.util.HashMap;
+import java.util.Map;
import java.util.SortedSet;
import javax.net.ssl.KeyManager;
@@ -63,6 +65,29 @@
{
private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
+ static final String JMX_REMOTE_RMI_SERVER_CREDENTIAL_TYPES =
+ "jmx.remote.rmi.server.credential.types";
+
+ /**
+ * JDK 10+ JMX environment property scoping a JEP 290 deserialization
+ * filter to the credentials object passed during {@code newClient()}.
+ * Using the credentials-scoped filter (instead of the connector-wide
+ * {@code jmx.remote.rmi.server.serial.filter.pattern}) avoids breaking
+ * legitimate JMX traffic such as MBean invocations and notifications,
+ * which may legitimately carry non-String types.
+ */
+ static final String JMX_REMOTE_RMI_SERVER_CREDENTIALS_FILTER_PATTERN =
+ "jmx.remote.rmi.server.credentials.filter.pattern";
+
+ private static final String[] JMX_CREDENTIAL_TYPES =
+ {
+ String.class.getName(),
+ String[].class.getName()
+ };
+
+ private static final String JMX_CREDENTIAL_SERIAL_FILTER =
+ "maxdepth=3;maxarray=2;java.lang.String;!*";
+
/**
* The MBean server used to handle JMX interaction.
@@ -253,6 +278,7 @@
{
// Environment map
HashMap<String, Object> env = new HashMap<>();
+ configureJmxDeserializationProtection(env);
// ---------------------
// init an ssl context
@@ -364,6 +390,17 @@
}
+ static void configureJmxDeserializationProtection(Map<String, Object> env)
+ {
+ env.put(JMX_REMOTE_RMI_SERVER_CREDENTIAL_TYPES,
+ JMX_CREDENTIAL_TYPES.clone());
+ // Scope the JEP 290 deserialization filter to the credentials object
+ // only, so legitimate JMX RMI traffic (MBean operations, notifications,
+ // etc.) is not affected by the restrictive allowlist.
+ env.put(JMX_REMOTE_RMI_SERVER_CREDENTIALS_FILTER_PATTERN,
+ JMX_CREDENTIAL_SERIAL_FILTER);
+ }
+
/**
* Closes this connection handler so that it will no longer accept new
* client connections. It may or may not disconnect existing client
diff --git a/opendj-server-legacy/src/test/java/org/opends/server/protocols/jmx/RmiAuthenticatorTest.java b/opendj-server-legacy/src/test/java/org/opends/server/protocols/jmx/RmiAuthenticatorTest.java
new file mode 100644
index 0000000..6c7b5eb
--- /dev/null
+++ b/opendj-server-legacy/src/test/java/org/opends/server/protocols/jmx/RmiAuthenticatorTest.java
@@ -0,0 +1,142 @@
+/*
+ * The contents of this file are subject to the terms of the Common Development and
+ * Distribution License (the License). You may not use this file except in compliance with the
+ * License.
+ *
+ * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
+ * specific language governing permission and limitations under the License.
+ *
+ * When distributing Covered Software, include this CDDL Header Notice in each file and include
+ * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
+ * Header, with the fields enclosed by brackets [] replaced by your own identifying
+ * information: "Portions Copyright [year] [name of copyright owner]".
+ *
+ * Copyright 2026 3A Systems, LLC.
+ */
+package org.opends.server.protocols.jmx;
+
+import static org.testng.Assert.*;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.InvalidClassException;
+import java.io.ObjectInputFilter;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+
+
+import org.opends.server.DirectoryServerTestCase;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+/** Tests JMX RMI credential deserialization hardening. */
+@Test(groups = { "precommit", "jmx" }, sequential = true)
+public class RmiAuthenticatorTest extends DirectoryServerTestCase
+{
+ /** Invalid credential shapes rejected before any bind attempt. */
+ @DataProvider(name = "invalidCredentials")
+ public Object[][] invalidCredentials()
+ {
+ return new Object[][]
+ {
+ { null },
+ { new Object[] { "cn=Directory Manager", "password" } },
+ { new Date() },
+ { new String[0] },
+ { new String[] { "cn=Directory Manager" } },
+ { new String[] { "cn=Directory Manager", "password", "extra" } },
+ { new String[] { null, "password" } },
+ { new String[] { "cn=Directory Manager", null } }
+ };
+ }
+
+ /** Verifies that RmiAuthenticator only accepts a two-element String array. */
+ @Test(dataProvider = "invalidCredentials", expectedExceptions = SecurityException.class)
+ public void rejectsInvalidCredentialShapes(Object credentials)
+ {
+ new RmiAuthenticator(null).authenticate(credentials);
+ }
+
+ /** Verifies that RMI connector environment constrains credential unmarshalling. */
+ @Test
+ public void configuresCredentialDeserializationProtection()
+ {
+ Map<String, Object> env = new HashMap<>();
+ RmiConnector.configureJmxDeserializationProtection(env);
+
+ assertEquals(env.get(RmiConnector.JMX_REMOTE_RMI_SERVER_CREDENTIAL_TYPES),
+ new String[] { String.class.getName(), String[].class.getName() });
+ assertEquals(env.get(RmiConnector.JMX_REMOTE_RMI_SERVER_CREDENTIALS_FILTER_PATTERN),
+ "maxdepth=3;maxarray=2;java.lang.String;!*");
+ // The connector-wide filter must NOT be set, so legitimate JMX traffic
+ // (MBean operations, notifications) is not affected by the allowlist.
+ assertNull(env.get("jmx.remote.rmi.server.serial.filter.pattern"));
+ }
+
+ /** Verifies that each environment receives its own credential type array. */
+ @Test
+ public void credentialTypesAreDefensivelyCopied()
+ {
+ Map<String, Object> env = new HashMap<>();
+ RmiConnector.configureJmxDeserializationProtection(env);
+ String[] credentialTypes =
+ (String[]) env.get(RmiConnector.JMX_REMOTE_RMI_SERVER_CREDENTIAL_TYPES);
+ credentialTypes[0] = Date.class.getName();
+
+ Map<String, Object> env2 = new HashMap<>();
+ RmiConnector.configureJmxDeserializationProtection(env2);
+ assertEquals(((String[]) env2.get(RmiConnector.JMX_REMOTE_RMI_SERVER_CREDENTIAL_TYPES))[0],
+ String.class.getName());
+ }
+
+ /** Verifies the configured filter allows only the expected credential payload. */
+ @Test
+ public void serialFilterAllowsOnlyTwoElementStringArray() throws Exception
+ {
+ Map<String, Object> env = new HashMap<>();
+ RmiConnector.configureJmxDeserializationProtection(env);
+ String filterPattern = (String) env.get(RmiConnector.JMX_REMOTE_RMI_SERVER_CREDENTIALS_FILTER_PATTERN);
+
+ assertEquals(readWithFilter(new String[] { "uid", "password" }, filterPattern),
+ new String[] { "uid", "password" });
+ assertRejectedByFilter(new Object[] { "uid", "password" }, filterPattern);
+ assertRejectedByFilter(new String[] { "uid", "password", "extra" }, filterPattern);
+ assertRejectedByFilter(new Date(), filterPattern);
+ }
+
+ private Object readWithFilter(Object object, String filterPattern) throws Exception
+ {
+ byte[] bytes = serialize(object);
+ try (ObjectInputStream in = new ObjectInputStream(new ByteArrayInputStream(bytes)))
+ {
+ in.setObjectInputFilter(ObjectInputFilter.Config.createFilter(filterPattern));
+ return in.readObject();
+ }
+ }
+
+ private byte[] serialize(Object object) throws Exception
+ {
+ ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ try (ObjectOutputStream out = new ObjectOutputStream(bytes))
+ {
+ out.writeObject(object);
+ }
+ return bytes.toByteArray();
+ }
+
+ private void assertRejectedByFilter(Object object, String filterPattern) throws Exception
+ {
+ try
+ {
+ readWithFilter(object, filterPattern);
+ fail("Expected object to be rejected by the JMX credential serial filter");
+ }
+ catch (InvalidClassException expected)
+ {
+ // Expected: ObjectInputFilter rejected the class or array length.
+ }
+ }
+}
--
Gitblit v1.10.0