From 5b2bacde1d7561f28dd20cc965d641fbff558bc2 Mon Sep 17 00:00:00 2001
From: Valera V Harseko <vharseko@3a-systems.ru>
Date: Fri, 17 Jul 2026 11:13:11 +0000
Subject: [PATCH] CVE-2026-62366 Unauthenticated stack exhaustion when decoding an LDAP search filter (DoS)
---
opendj-core/src/test/java/org/forgerock/opendj/io/LDAPFilterDecodeNestingTest.java | 139 +++++++++++++++++
opendj-core/src/main/java/org/forgerock/opendj/io/LDAP.java | 55 +++++-
opendj-server-legacy/src/main/java/org/opends/server/types/RawFilter.java | 50 +++++
opendj-server-legacy/src/test/java/org/opends/server/types/RawFilterDecodeStackOverflowTestCase.java | 212 ++++++++++++++++++++++++++
opendj-server-legacy/src/messages/org/opends/messages/protocol.properties | 3
opendj-core/src/main/resources/com/forgerock/opendj/ldap/core.properties | 3
opendj-server-legacy/src/main/java/org/opends/server/protocols/ldap/LDAPRequestHandler.java | 11 +
7 files changed, 458 insertions(+), 15 deletions(-)
diff --git a/opendj-core/src/main/java/org/forgerock/opendj/io/LDAP.java b/opendj-core/src/main/java/org/forgerock/opendj/io/LDAP.java
index d65d3a1..cedeccf 100644
--- a/opendj-core/src/main/java/org/forgerock/opendj/io/LDAP.java
+++ b/opendj-core/src/main/java/org/forgerock/opendj/io/LDAP.java
@@ -12,9 +12,12 @@
* information: "Portions Copyright [year] [name of copyright owner]".
*
* Copyright 2013-2016 ForgeRock AS.
+ * Portions Copyright 2024-2026 3A Systems, LLC
*/
package org.forgerock.opendj.io;
+import static com.forgerock.opendj.ldap.CoreMessages.ERR_LDAP_FILTER_DECODE_MAX_NESTING_DEPTH;
+
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
@@ -44,6 +47,15 @@
public final class LDAP {
// @Checkstyle:ignore AvoidNestedBlocks
+ /**
+ * The maximum number of nested AND/OR/NOT components allowed in a search
+ * filter when decoding it from its ASN.1 representation. This bounds the
+ * decoder recursion so that a maliciously over-nested filter is rejected
+ * rather than overflowing the JVM stack (GHSA-rv4q-c6mr-wxp7). It matches
+ * the limit applied on the filter evaluation path.
+ */
+ private static final int MAX_NESTED_FILTER_DEPTH = 100;
+
/** The OID for the Kerberos V GSSAPI mechanism. */
public static final String OID_GSSAPI_KERBEROS_V = "1.2.840.113554.1.2.2";
@@ -402,14 +414,39 @@
* If an error occurs while reading from {@code reader}.
*/
public static Filter readFilter(final ASN1Reader reader) throws IOException {
+ return readFilter(reader, 0);
+ }
+
+ /**
+ * Reads the next ASN.1 element from the provided {@code ASN1Reader} as a
+ * {@code Filter}, tracking the current nesting depth so that a maliciously
+ * over-nested filter is rejected rather than overflowing the JVM stack
+ * (GHSA-rv4q-c6mr-wxp7).
+ *
+ * @param reader
+ * The {@code ASN1Reader} from which the ASN.1 encoded
+ * {@code Filter} should be read.
+ * @param depth
+ * The current filter nesting depth (0 for the top-level filter).
+ * @return The decoded {@code Filter}.
+ * @throws IOException
+ * If an error occurs while reading from {@code reader}, or if
+ * the filter is nested more deeply than
+ * {@link #MAX_NESTED_FILTER_DEPTH}.
+ */
+ private static Filter readFilter(final ASN1Reader reader, final int depth) throws IOException {
+ if (depth >= MAX_NESTED_FILTER_DEPTH) {
+ throw DecodeException.fatalError(
+ ERR_LDAP_FILTER_DECODE_MAX_NESTING_DEPTH.get(MAX_NESTED_FILTER_DEPTH));
+ }
final byte type = reader.peekType();
switch (type) {
case LDAP.TYPE_FILTER_AND:
- return readAndFilter(reader);
+ return readAndFilter(reader, depth);
case LDAP.TYPE_FILTER_OR:
- return readOrFilter(reader);
+ return readOrFilter(reader, depth);
case LDAP.TYPE_FILTER_NOT:
- return readNotFilter(reader);
+ return readNotFilter(reader, depth);
case LDAP.TYPE_FILTER_EQUALITY:
return readEqualityMatchFilter(reader);
case LDAP.TYPE_FILTER_GREATER_OR_EQUAL:
@@ -568,13 +605,13 @@
writer.writeEndSequence();
}
- private static Filter readAndFilter(final ASN1Reader reader) throws IOException {
+ private static Filter readAndFilter(final ASN1Reader reader, final int depth) throws IOException {
reader.readStartSequence(LDAP.TYPE_FILTER_AND);
try {
if (reader.hasNextElement()) {
final List<Filter> subFilters = new LinkedList<>();
do {
- subFilters.add(readFilter(reader));
+ subFilters.add(readFilter(reader, depth + 1));
} while (reader.hasNextElement());
return Filter.and(subFilters);
} else {
@@ -654,22 +691,22 @@
}
}
- private static Filter readNotFilter(final ASN1Reader reader) throws IOException {
+ private static Filter readNotFilter(final ASN1Reader reader, final int depth) throws IOException {
reader.readStartSequence(LDAP.TYPE_FILTER_NOT);
try {
- return Filter.not(readFilter(reader));
+ return Filter.not(readFilter(reader, depth + 1));
} finally {
reader.readEndSequence();
}
}
- private static Filter readOrFilter(final ASN1Reader reader) throws IOException {
+ private static Filter readOrFilter(final ASN1Reader reader, final int depth) throws IOException {
reader.readStartSequence(LDAP.TYPE_FILTER_OR);
try {
if (reader.hasNextElement()) {
final List<Filter> subFilters = new LinkedList<>();
do {
- subFilters.add(readFilter(reader));
+ subFilters.add(readFilter(reader, depth + 1));
} while (reader.hasNextElement());
return Filter.or(subFilters);
} else {
diff --git a/opendj-core/src/main/resources/com/forgerock/opendj/ldap/core.properties b/opendj-core/src/main/resources/com/forgerock/opendj/ldap/core.properties
index a73a842..a857887 100644
--- a/opendj-core/src/main/resources/com/forgerock/opendj/ldap/core.properties
+++ b/opendj-core/src/main/resources/com/forgerock/opendj/ldap/core.properties
@@ -112,6 +112,9 @@
ERR_ATTR_SYNTAX_DN_MAX_DEPTH=The provided value "%s" could not be parsed as a \
valid distinguished name because it contains more than %d levels of nested \
distinguished name (DN-syntax) attribute values
+ERR_LDAP_FILTER_DECODE_MAX_NESTING_DEPTH=Cannot decode the provided ASN.1 \
+ element as an LDAP search filter because it is nested more than %d levels \
+ deep, which exceeds the maximum allowed filter nesting depth
WARN_ATTR_SYNTAX_INTEGER_INITIAL_ZERO=The provided value "%s" could \
not be parsed as a valid integer because the first digit may not be zero \
unless it is the only digit
diff --git a/opendj-core/src/test/java/org/forgerock/opendj/io/LDAPFilterDecodeNestingTest.java b/opendj-core/src/test/java/org/forgerock/opendj/io/LDAPFilterDecodeNestingTest.java
new file mode 100644
index 0000000..8d5a3f4
--- /dev/null
+++ b/opendj-core/src/test/java/org/forgerock/opendj/io/LDAPFilterDecodeNestingTest.java
@@ -0,0 +1,139 @@
+/*
+ * The contents of this file are subject to the terms of the Common Development and
+ * Distribution License (the License). You may not use this file except in compliance with the
+ * License.
+ *
+ * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
+ * specific language governing permission and limitations under the License.
+ *
+ * When distributing Covered Software, include this CDDL Header Notice in each file and include
+ * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
+ * Header, with the fields enclosed by brackets [] replaced by your own identifying
+ * information: "Portions Copyright [year] [name of copyright owner]".
+ *
+ * Copyright 2026 3A Systems, LLC.
+ */
+package org.forgerock.opendj.io;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.forgerock.opendj.ldap.ByteStringBuilder;
+import org.forgerock.opendj.ldap.DecodeException;
+import org.forgerock.opendj.ldap.Filter;
+import org.forgerock.opendj.ldap.SdkTestCase;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.*;
+
+/**
+ * Regression test for GHSA-rv4q-c6mr-wxp7 (OPENDJ-001), SDK decoder twin.
+ *
+ * <p>{@link LDAP#readFilter(ASN1Reader)} (AND/OR/NOT) recursed once per nesting
+ * level with no depth bound, so a maliciously over-nested search filter could
+ * overflow the JVM stack with a {@link StackOverflowError} during decode. The
+ * fix threads a depth counter and rejects over-nested filters with a
+ * {@link DecodeException} before the stack is exhausted.
+ *
+ * @see org.opends.server.types.RawFilterDecodeStackOverflowTestCase the matching
+ * test for the legacy server decoder ({@code RawFilter.decode}).
+ */
+@SuppressWarnings("javadoc")
+public class LDAPFilterDecodeNestingTest extends SdkTestCase
+{
+ private static final int OVERFLOW_DEPTH = 100_000;
+ private static final long DECODE_STACK_BYTES = 256L * 1024L;
+
+ @Test
+ public void shallowNestedFilterDecodes() throws Exception
+ {
+ Throwable result = decodeOnBoundedStack(buildNestedAndFilter(50));
+ assertNull(result, "A 50-level filter must decode cleanly, but threw: " + result);
+ }
+
+ @Test
+ public void deeplyNestedFilterIsRejectedWithoutStackOverflow() throws Exception
+ {
+ Throwable result = decodeOnBoundedStack(buildNestedAndFilter(OVERFLOW_DEPTH));
+
+ assertNotNull(result,
+ "Decoding a " + OVERFLOW_DEPTH + "-level filter must be rejected, but it succeeded");
+ assertFalse(result instanceof StackOverflowError,
+ "VULNERABLE: LDAP.readFilter overflowed the stack instead of rejecting: " + result);
+ assertTrue(result instanceof DecodeException,
+ "Expected a controlled DecodeException, but got: " + result);
+ }
+
+ private Throwable decodeOnBoundedStack(final byte[] payload) throws InterruptedException
+ {
+ final Throwable[] escaped = new Throwable[1];
+ Runnable decode = new Runnable()
+ {
+ @Override
+ public void run()
+ {
+ try
+ {
+ LDAP.readFilter(ASN1.getReader(payload));
+ }
+ catch (Throwable t)
+ {
+ escaped[0] = t;
+ }
+ }
+ };
+ Thread worker = new Thread(null, decode, "ghsa-rv4q-sdk-decode", DECODE_STACK_BYTES);
+ worker.start();
+ worker.join();
+ return escaped[0];
+ }
+
+ /** Builds the definite-length BER encoding of an AND filter nested {@code depth} deep. */
+ private static byte[] buildNestedAndFilter(final int depth) throws Exception
+ {
+ ByteStringBuilder leafBuilder = new ByteStringBuilder();
+ ASN1Writer writer = ASN1.getWriter(leafBuilder);
+ LDAP.writeFilter(writer, Filter.present("objectClass"));
+ writer.flush();
+ byte[] leaf = leafBuilder.toByteArray();
+
+ List<byte[]> prefixes = new ArrayList<>(depth);
+ int contentLen = leaf.length;
+ for (int i = 0; i < depth; i++)
+ {
+ byte[] lengthBytes = berLength(contentLen);
+ byte[] prefix = new byte[1 + lengthBytes.length];
+ prefix[0] = LDAP.TYPE_FILTER_AND; // 0xA0
+ System.arraycopy(lengthBytes, 0, prefix, 1, lengthBytes.length);
+ prefixes.add(prefix);
+ contentLen += prefix.length;
+ }
+
+ byte[] out = new byte[contentLen];
+ int pos = 0;
+ for (int i = prefixes.size() - 1; i >= 0; i--)
+ {
+ byte[] prefix = prefixes.get(i);
+ System.arraycopy(prefix, 0, out, pos, prefix.length);
+ pos += prefix.length;
+ }
+ System.arraycopy(leaf, 0, out, pos, leaf.length);
+ return out;
+ }
+
+ private static byte[] berLength(final int len)
+ {
+ if (len < 0x80)
+ {
+ return new byte[] { (byte) len };
+ }
+ int numBytes = len <= 0xFF ? 1 : len <= 0xFFFF ? 2 : len <= 0xFFFFFF ? 3 : 4;
+ byte[] out = new byte[1 + numBytes];
+ out[0] = (byte) (0x80 | numBytes);
+ for (int i = 0; i < numBytes; i++)
+ {
+ out[numBytes - i] = (byte) (len >>> (8 * i));
+ }
+ return out;
+ }
+}
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/protocols/ldap/LDAPRequestHandler.java b/opendj-server-legacy/src/main/java/org/opends/server/protocols/ldap/LDAPRequestHandler.java
index 648be08..b1c394f 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/protocols/ldap/LDAPRequestHandler.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/protocols/ldap/LDAPRequestHandler.java
@@ -13,6 +13,7 @@
*
* Copyright 2006-2010 Sun Microsystems, Inc.
* Portions Copyright 2014-2016 ForgeRock AS.
+ * Portions Copyright 2026 3A Systems, LLC
*/
package org.opends.server.protocols.ldap;
@@ -192,6 +193,16 @@
readyConnection.disconnect(DisconnectReason.PROTOCOL_ERROR, true,
e.getMessageObject());
}
+ catch (StackOverflowError e)
+ {
+ // Defense in depth for over-nested protocol elements (e.g. search
+ // filters - GHSA-rv4q-c6mr-wxp7). A StackOverflowError is an Error,
+ // not an Exception, so without this it would escape and kill the
+ // shared request-handler thread. Drop only the offending connection.
+ logger.traceException(e);
+ readyConnection.disconnect(DisconnectReason.PROTOCOL_ERROR, true,
+ LocalizableMessage.raw(e.toString()));
+ }
catch (Exception e)
{
logger.traceException(e);
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/types/RawFilter.java b/opendj-server-legacy/src/main/java/org/opends/server/types/RawFilter.java
index ddba1d9..a004b1f 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/types/RawFilter.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/types/RawFilter.java
@@ -13,6 +13,7 @@
*
* Copyright 2006-2009 Sun Microsystems, Inc.
* Portions Copyright 2013-2015 ForgeRock AS.
+ * Portions Copyright 2026 3A Systems, LLC
*/
package org.opends.server.types;
@@ -31,6 +32,7 @@
import static org.opends.messages.ProtocolMessages.*;
import static org.opends.server.protocols.ldap.LDAPConstants.*;
import static org.opends.server.protocols.ldap.LDAPResultCode.*;
+import static org.opends.server.util.ServerConstants.MAX_NESTED_FILTER_DEPTH;
import static org.opends.server.util.StaticUtils.*;
@@ -531,6 +533,42 @@
public static LDAPFilter decode(ASN1Reader reader)
throws LDAPException
{
+ return decode(reader, 0);
+ }
+
+ /**
+ * Decodes the elements from the provided ASN.1 reader as a raw
+ * search filter, tracking the current nesting depth.
+ * <p>
+ * The {@code depth} guard bounds the recursion so that a maliciously
+ * over-nested AND/OR/NOT filter is rejected with a
+ * {@code PROTOCOL_ERROR} instead of overflowing the JVM stack with a
+ * {@code StackOverflowError} (GHSA-rv4q-c6mr-wxp7). Without it an
+ * unauthenticated client could kill the request-handler thread, since
+ * a {@code StackOverflowError} is an {@code Error} and escapes the
+ * {@code catch (Exception)} blocks on the decode path.
+ *
+ * @param reader The ASN.1 reader.
+ * @param depth The current filter nesting depth (0 for the top-level
+ * filter).
+ *
+ * @return The decoded search filter.
+ *
+ * @throws LDAPException If the provided ASN.1 element cannot be
+ * decoded as a raw search filter, or if it is
+ * nested more deeply than
+ * {@link org.opends.server.util.ServerConstants#MAX_NESTED_FILTER_DEPTH}.
+ */
+ private static LDAPFilter decode(ASN1Reader reader, int depth)
+ throws LDAPException
+ {
+ if (depth >= MAX_NESTED_FILTER_DEPTH)
+ {
+ LocalizableMessage message =
+ ERR_LDAP_FILTER_DECODE_MAX_NESTING_DEPTH.get(MAX_NESTED_FILTER_DEPTH);
+ throw new LDAPException(PROTOCOL_ERROR, message);
+ }
+
byte type;
try
{
@@ -546,10 +584,10 @@
{
case TYPE_FILTER_AND:
case TYPE_FILTER_OR:
- return decodeCompoundFilter(reader);
+ return decodeCompoundFilter(reader, depth);
case TYPE_FILTER_NOT:
- return decodeNotFilter(reader);
+ return decodeNotFilter(reader, depth);
case TYPE_FILTER_EQUALITY:
case TYPE_FILTER_GREATER_OR_EQUAL:
@@ -586,7 +624,7 @@
* decode the provided ASN.1 element as a
* raw search filter.
*/
- private static LDAPFilter decodeCompoundFilter(ASN1Reader reader)
+ private static LDAPFilter decodeCompoundFilter(ASN1Reader reader, int depth)
throws LDAPException
{
byte type;
@@ -627,7 +665,7 @@
// could also be an absolute true/false filter
while (reader.hasNextElement())
{
- filterComponents.add(LDAPFilter.decode(reader));
+ filterComponents.add(decode(reader, depth + 1));
}
reader.readEndSequence();
}
@@ -659,14 +697,14 @@
* decode the provided ASN.1 element as a
* raw search filter.
*/
- private static LDAPFilter decodeNotFilter(ASN1Reader reader)
+ private static LDAPFilter decodeNotFilter(ASN1Reader reader, int depth)
throws LDAPException
{
RawFilter notComponent;
try
{
reader.readStartSequence();
- notComponent = decode(reader);
+ notComponent = decode(reader, depth + 1);
reader.readEndSequence();
}
catch (LDAPException le)
diff --git a/opendj-server-legacy/src/messages/org/opends/messages/protocol.properties b/opendj-server-legacy/src/messages/org/opends/messages/protocol.properties
index 8d3518e..822087d 100644
--- a/opendj-server-legacy/src/messages/org/opends/messages/protocol.properties
+++ b/opendj-server-legacy/src/messages/org/opends/messages/protocol.properties
@@ -262,6 +262,9 @@
ERR_LDAP_FILTER_DECODE_NOT_COMPONENT_143=Cannot decode the provided \
ASN.1 element as an LDAP search filter because the NOT component element \
could not be decoded as an LDAP filter: %s
+ERR_LDAP_FILTER_DECODE_MAX_NESTING_DEPTH_1538=Cannot decode the provided \
+ ASN.1 element as an LDAP search filter because it is nested more than %d \
+ levels deep, which exceeds the maximum allowed filter nesting depth
ERR_LDAP_FILTER_DECODE_TV_SEQUENCE_144=Cannot decode the provided ASN.1 \
element as an LDAP search filter because the element could not be decoded as \
a type-and-value sequence: %s
diff --git a/opendj-server-legacy/src/test/java/org/opends/server/types/RawFilterDecodeStackOverflowTestCase.java b/opendj-server-legacy/src/test/java/org/opends/server/types/RawFilterDecodeStackOverflowTestCase.java
new file mode 100644
index 0000000..fa8f212
--- /dev/null
+++ b/opendj-server-legacy/src/test/java/org/opends/server/types/RawFilterDecodeStackOverflowTestCase.java
@@ -0,0 +1,212 @@
+/*
+ * The contents of this file are subject to the terms of the Common Development and
+ * Distribution License (the License). You may not use this file except in compliance with the
+ * License.
+ *
+ * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
+ * specific language governing permission and limitations under the License.
+ *
+ * When distributing Covered Software, include this CDDL Header Notice in each file and include
+ * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
+ * Header, with the fields enclosed by brackets [] replaced by your own identifying
+ * information: "Portions Copyright [year] [name of copyright owner]".
+ *
+ * Copyright 2026 3A Systems, LLC.
+ */
+package org.opends.server.types;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.forgerock.opendj.io.ASN1;
+import org.forgerock.opendj.io.ASN1Reader;
+import org.forgerock.opendj.io.ASN1Writer;
+import org.forgerock.opendj.ldap.ByteStringBuilder;
+import org.opends.server.DirectoryServerTestCase;
+import org.opends.server.protocols.ldap.LDAPFilter;
+import org.testng.annotations.Test;
+
+import static org.opends.server.protocols.ldap.LDAPConstants.TYPE_FILTER_AND;
+import static org.opends.server.protocols.ldap.LDAPResultCode.PROTOCOL_ERROR;
+import static org.opends.server.util.ServerConstants.MAX_NESTED_FILTER_DEPTH;
+import static org.testng.Assert.*;
+
+/**
+ * Regression test for GHSA-rv4q-c6mr-wxp7 (OPENDJ-001):
+ * <em>Unauthenticated LDAP search-filter decode stack exhaustion</em>.
+ *
+ * <p>Before the fix, the BER decoder {@link RawFilter#decode(ASN1Reader)} —>
+ * {@code decodeCompoundFilter} (AND/OR) / {@code decodeNotFilter} (NOT) recursed
+ * once per nesting level with <strong>no depth bound</strong>. The
+ * {@code MAX_NESTED_FILTER_DEPTH} guard only protected the filter
+ * <em>evaluation</em> path, which runs strictly <em>after</em> decode. An
+ * anonymous client could therefore send a single small {@code SearchRequest}
+ * whose filter was nested tens of thousands of levels deep; decoding it
+ * overflowed the JVM stack with a {@link StackOverflowError} <strong>before</strong>
+ * the evaluation guard was ever reached. Because {@code StackOverflowError} is a
+ * {@link java.lang.Error} (not an {@link Exception}), it escaped the
+ * {@code catch (Exception)} blocks and killed the shared request-handler thread
+ * → listener-wide DoS.
+ *
+ * <p>After the fix, {@link RawFilter#decode(ASN1Reader)} threads a depth counter
+ * and rejects over-nested filters with a {@link LDAPException}
+ * ({@code PROTOCOL_ERROR}) instead of recursing into a stack overflow.
+ *
+ * <p>This test builds the exact malicious wire encoding (a definite-length BER
+ * tree of nested {@code 0xA0} AND TLVs wrapping an {@code (objectClass=*)}
+ * present leaf) and decodes it on a thread with a small stack, so that — were the
+ * bound ever removed — the overflow would still be deterministic and fast on any
+ * platform.
+ */
+@SuppressWarnings("javadoc")
+public class RawFilterDecodeStackOverflowTestCase extends DirectoryServerTestCase
+{
+ /** Nesting depth that would overflow even an oversized default stack. */
+ private static final int OVERFLOW_DEPTH = 100_000;
+
+ /** A small, deterministic stack so any unbounded recursion overflows quickly. */
+ private static final long DECODE_STACK_BYTES = 256L * 1024L;
+
+ /**
+ * A filter nested well within {@link #MAX_NESTED_FILTER_DEPTH} decodes
+ * normally. This is the control that proves the harness and the wire encoding
+ * are correct, so that the rejection in
+ * {@link #deeplyNestedFilterIsRejectedWithoutStackOverflow()} is attributable
+ * to the depth guard and not to a malformed payload.
+ */
+ @Test
+ public void shallowNestedFilterDecodesWithoutError() throws Exception
+ {
+ byte[] payload = buildNestedAndFilter(MAX_NESTED_FILTER_DEPTH / 2);
+ Throwable result = decodeOnBoundedStack(payload);
+ assertNull(result, "A filter nested " + (MAX_NESTED_FILTER_DEPTH / 2)
+ + " levels deep must decode cleanly, but threw: " + result);
+ }
+
+ /**
+ * Reproduction-turned-regression check: a deeply nested AND filter must be
+ * rejected with a controlled {@link LDAPException} ({@code PROTOCOL_ERROR})
+ * rather than overflowing the stack with a {@link StackOverflowError}.
+ *
+ * <p>On the vulnerable (pre-fix) code this fails because decode recurses until
+ * the stack overflows and the resulting {@code java.lang.Error} escapes. With
+ * the depth guard in place it passes.
+ */
+ @Test
+ public void deeplyNestedFilterIsRejectedWithoutStackOverflow() throws Exception
+ {
+ byte[] payload = buildNestedAndFilter(OVERFLOW_DEPTH);
+
+ // Well under the ~5 MB ds-cfg-max-request-size cap: depth costs only a few
+ // bytes per level, so the request-size limit never bounds the depth.
+ assertTrue(payload.length < 5 * 1024 * 1024,
+ "PoC payload (" + payload.length + " bytes) should be far below the 5 MB request cap");
+
+ Throwable result = decodeOnBoundedStack(payload);
+
+ assertNotNull(result,
+ "Decoding a " + OVERFLOW_DEPTH + "-level filter must be rejected, but it succeeded");
+ assertFalse(result instanceof StackOverflowError,
+ "VULNERABLE: decode overflowed the stack instead of rejecting the over-nested filter: " + result);
+ assertTrue(result instanceof LDAPException,
+ "Expected a controlled LDAPException, but got: " + result);
+ assertEquals(((LDAPException) result).getResultCode(), PROTOCOL_ERROR,
+ "Over-nested filter should be rejected as a protocol error");
+ }
+
+ /**
+ * Decodes {@code payload} via {@link RawFilter#decode(ASN1Reader)} on a worker
+ * thread with a small stack and returns the {@link Throwable} that escaped
+ * decode, or {@code null} if decode completed normally. Captures
+ * {@link Throwable} (including {@link Error}) so a regression to the unbounded
+ * recursion surfaces as a {@link StackOverflowError} rather than killing the
+ * test JVM thread silently.
+ */
+ private Throwable decodeOnBoundedStack(final byte[] payload) throws InterruptedException
+ {
+ final Throwable[] escaped = new Throwable[1];
+ Runnable decode = new Runnable()
+ {
+ @Override
+ public void run()
+ {
+ try
+ {
+ ASN1Reader reader = ASN1.getReader(payload);
+ RawFilter.decode(reader);
+ }
+ catch (Throwable t)
+ {
+ escaped[0] = t;
+ }
+ }
+ };
+
+ Thread worker = new Thread(null, decode, "ghsa-rv4q-decode", DECODE_STACK_BYTES);
+ worker.start();
+ worker.join();
+ return escaped[0];
+ }
+
+ /**
+ * Builds the BER (definite-length) wire encoding of an AND filter nested
+ * {@code depth} levels deep, wrapping an {@code (objectClass=*)} present leaf:
+ * <pre>
+ * A0 L ( A0 L ( ... ( 87 0B "objectClass" ) ... ) )
+ * </pre>
+ * Constructed inside-out so the length fields are exact, which is what
+ * {@link RawFilter#decode} reads. (Indefinite-length BER is not used because
+ * the OpenDJ reader does not support it.)
+ */
+ private static byte[] buildNestedAndFilter(final int depth) throws Exception
+ {
+ // Innermost leaf: presence filter (objectClass=*).
+ ByteStringBuilder leafBuilder = new ByteStringBuilder();
+ ASN1Writer writer = ASN1.getWriter(leafBuilder);
+ LDAPFilter.createPresenceFilter("objectClass").write(writer);
+ writer.flush();
+ byte[] leaf = leafBuilder.toByteString().toByteArray();
+
+ // Build each AND wrapper prefix (0xA0 + definite length) from the inside out.
+ List<byte[]> prefixes = new ArrayList<>(depth);
+ int contentLen = leaf.length;
+ for (int i = 0; i < depth; i++)
+ {
+ byte[] lengthBytes = berLength(contentLen);
+ byte[] prefix = new byte[1 + lengthBytes.length];
+ prefix[0] = TYPE_FILTER_AND; // 0xA0
+ System.arraycopy(lengthBytes, 0, prefix, 1, lengthBytes.length);
+ prefixes.add(prefix);
+ contentLen += prefix.length;
+ }
+
+ // Assemble: outermost prefix first ... innermost prefix ... leaf.
+ byte[] out = new byte[contentLen];
+ int pos = 0;
+ for (int i = prefixes.size() - 1; i >= 0; i--)
+ {
+ byte[] prefix = prefixes.get(i);
+ System.arraycopy(prefix, 0, out, pos, prefix.length);
+ pos += prefix.length;
+ }
+ System.arraycopy(leaf, 0, out, pos, leaf.length);
+ return out;
+ }
+
+ /** Encodes {@code len} as a BER definite-length field (short or long form). */
+ private static byte[] berLength(final int len)
+ {
+ if (len < 0x80)
+ {
+ return new byte[] { (byte) len };
+ }
+ int numBytes = len <= 0xFF ? 1 : len <= 0xFFFF ? 2 : len <= 0xFFFFFF ? 3 : 4;
+ byte[] out = new byte[1 + numBytes];
+ out[0] = (byte) (0x80 | numBytes);
+ for (int i = 0; i < numBytes; i++)
+ {
+ out[numBytes - i] = (byte) (len >>> (8 * i));
+ }
+ return out;
+ }
+}
--
Gitblit v1.10.0