mirror of https://github.com/OpenIdentityPlatform/OpenDJ.git
| .github/benchmark/benchmark.jmx | ●●●●● patch | view | raw | blame | history | |
| .github/benchmark/compare-opendj.sh | ●●●●● patch | view | raw | blame | history | |
| .github/benchmark/people.ldif | ●●●●● patch | view | raw | blame | history | |
| .github/benchmark/summary.sh | ●●●●● patch | view | raw | blame | history | |
| .github/workflows/benchmark.yml | ●●●●● patch | view | raw | blame | history | |
| .github/workflows/build.yml | ●●●●● patch | view | raw | blame | history |
.github/benchmark/benchmark.jmx
New file @@ -0,0 +1,327 @@ <?xml version="1.0" encoding="UTF-8"?> <!-- The contents of this file are subject to the terms of the Common Development and Distribution License (the License). You may not use this file except in compliance with the License. You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the specific language governing permission and limitations under the License. When distributing Covered Software, include this CDDL Header Notice in each file and include the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL Header, with the fields enclosed by brackets [] replaced by your own identifying information: "Portions copyright [year] [name of copyright owner]". Copyright 2026 3A Systems, LLC. --> <jmeterTestPlan version="1.2" properties="5.0" jmeter="5.6.3"> <hashTree> <TestPlan guiclass="TestPlanGui" testclass="TestPlan" testname="OpenDJ vs OpenLDAP - LDAP benchmark" enabled="true"> <stringProp name="TestPlan.comments">Parametrized LDAP benchmark. Admin bind is cached once per thread (Once Only Controller, labelled ADMIN_CONNECT, excluded from metrics). Data ops (ADD/SEARCH/COMPARE/MODIFY/DELETE/READD) reuse the cached admin connection. Entries use mail as the naming/searchable attribute (equality-indexed by default on BOTH OpenDJ and OpenLDAP); only mail + objectClass (both indexed on both servers) are stored, keeping the write cost symmetric. Every created value is unique: ADD uses a per-iteration counter, READD uses a UUID. The measured user authentication is a single bind/unbind (test=sbind, own connection) after MODIFY has set the userPassword.</stringProp> <boolProp name="TestPlan.functional_mode">false</boolProp> <boolProp name="TestPlan.tearDown_on_shutdown">true</boolProp> <boolProp name="TestPlan.serialize_threadgroups">false</boolProp> <elementProp name="TestPlan.user_defined_variables" elementType="Arguments" guiclass="ArgumentsPanel" testclass="Arguments" testname="User Defined Variables" enabled="true"> <collectionProp name="Arguments.arguments"/> </elementProp> <stringProp name="TestPlan.user_define_classpath"></stringProp> </TestPlan> <hashTree> <ThreadGroup guiclass="ThreadGroupGui" testclass="ThreadGroup" testname="LDAP Thread Group" enabled="true"> <stringProp name="ThreadGroup.on_sample_error">continue</stringProp> <elementProp name="ThreadGroup.main_controller" elementType="LoopController" guiclass="LoopControlPanel" testclass="LoopController" testname="Loop Controller" enabled="true"> <boolProp name="LoopController.continue_forever">false</boolProp> <stringProp name="LoopController.loops">-1</stringProp> </elementProp> <stringProp name="ThreadGroup.num_threads">${__P(threads,200)}</stringProp> <stringProp name="ThreadGroup.ramp_time">${__P(rampup,0)}</stringProp> <boolProp name="ThreadGroup.scheduler">true</boolProp> <stringProp name="ThreadGroup.duration">${__P(duration,300)}</stringProp> <stringProp name="ThreadGroup.delay"></stringProp> </ThreadGroup> <hashTree> <!-- Per-thread, per-iteration counter. Combined with ${__threadNum} it yields a value unique across the whole run; it stays constant within a single loop iteration so the same entry is referenced by ADD/SEARCH/COMPARE/MODIFY/BIND/DELETE. --> <CounterConfig guiclass="CounterConfigGui" testclass="CounterConfig" testname="Per-iteration counter" enabled="true"> <stringProp name="CounterConfig.start">1</stringProp> <stringProp name="CounterConfig.end"></stringProp> <stringProp name="CounterConfig.incr">1</stringProp> <stringProp name="CounterConfig.name">iter</stringProp> <stringProp name="CounterConfig.format"></stringProp> <boolProp name="CounterConfig.per_user">true</boolProp> <boolProp name="CounterConfig.reset_on_tg_iteration">false</boolProp> </CounterConfig> <hashTree/> <!-- Admin connection: bound once per thread, cached in ldapContexts[thread], reused by all data operations. Labelled ADMIN_CONNECT and excluded from metrics. --> <OnceOnlyController guiclass="OnceOnlyControllerGui" testclass="OnceOnlyController" testname="Once Only - admin connect" enabled="true"/> <hashTree> <LDAPExtSampler guiclass="LdapExtTestSamplerGui" testclass="LDAPExtSampler" testname="ADMIN_CONNECT" enabled="true"> <stringProp name="servername">${__P(host,localhost)}</stringProp> <stringProp name="port">${__P(port,1389)}</stringProp> <stringProp name="rootdn">ou=People,${__P(basedn,dc=example\,dc=com)}</stringProp> <stringProp name="scope">2</stringProp> <stringProp name="countlimit"></stringProp> <stringProp name="timelimit"></stringProp> <stringProp name="attributes"></stringProp> <stringProp name="return_object">false</stringProp> <stringProp name="deref_aliases">false</stringProp> <stringProp name="connection_timeout">60000</stringProp> <stringProp name="parseflag">false</stringProp> <stringProp name="secure">false</stringProp> <stringProp name="user_dn">${__P(adminbinddn,cn=Directory Manager)}</stringProp> <stringProp name="user_pw">${__P(adminbindpw,password)}</stringProp> <stringProp name="comparedn"></stringProp> <stringProp name="comparefilt"></stringProp> <stringProp name="modddn"></stringProp> <stringProp name="newdn"></stringProp> <stringProp name="test">bind</stringProp> </LDAPExtSampler> <hashTree/> </hashTree> <!-- ADD (admin connection): create the per-iteration user entry, keyed by mail. --> <LDAPExtSampler guiclass="LdapExtTestSamplerGui" testclass="LDAPExtSampler" testname="ADD" enabled="true"> <stringProp name="servername"></stringProp> <stringProp name="port"></stringProp> <stringProp name="rootdn"></stringProp> <stringProp name="scope">2</stringProp> <stringProp name="countlimit"></stringProp> <stringProp name="timelimit"></stringProp> <stringProp name="attributes"></stringProp> <stringProp name="return_object">false</stringProp> <stringProp name="deref_aliases">false</stringProp> <stringProp name="connection_timeout"></stringProp> <stringProp name="parseflag">false</stringProp> <stringProp name="secure">false</stringProp> <stringProp name="user_dn"></stringProp> <stringProp name="user_pw"></stringProp> <stringProp name="comparedn"></stringProp> <stringProp name="comparefilt"></stringProp> <stringProp name="modddn"></stringProp> <stringProp name="newdn"></stringProp> <stringProp name="test">add</stringProp> <stringProp name="base_entry_dn">mail=u_${__threadNum}_${iter}@test.com</stringProp> <elementProp name="arguments" elementType="Arguments" guiclass="ArgumentsPanel" testclass="Arguments" testname="User Defined Variables" enabled="true"> <collectionProp name="Arguments.arguments"> <elementProp name="mail" elementType="Argument"> <stringProp name="Argument.name">mail</stringProp> <stringProp name="Argument.value">u_${__threadNum}_${iter}@test.com</stringProp> <stringProp name="Argument.metadata">=</stringProp> </elementProp> <elementProp name="objectClass" elementType="Argument"> <stringProp name="Argument.name">objectClass</stringProp> <stringProp name="Argument.value">top</stringProp> <stringProp name="Argument.metadata">=</stringProp> </elementProp> <elementProp name="objectClass" elementType="Argument"> <stringProp name="Argument.name">objectClass</stringProp> <stringProp name="Argument.value">locality</stringProp> <stringProp name="Argument.metadata">=</stringProp> </elementProp> <elementProp name="objectClass" elementType="Argument"> <stringProp name="Argument.name">objectClass</stringProp> <stringProp name="Argument.value">extensibleObject</stringProp> <stringProp name="Argument.metadata">=</stringProp> </elementProp> </collectionProp> </elementProp> </LDAPExtSampler> <hashTree/> <!-- SEARCH (admin connection): equality lookup on the indexed mail attribute. --> <LDAPExtSampler guiclass="LdapExtTestSamplerGui" testclass="LDAPExtSampler" testname="SEARCH" enabled="true"> <stringProp name="servername"></stringProp> <stringProp name="port"></stringProp> <stringProp name="rootdn"></stringProp> <stringProp name="scope">2</stringProp> <stringProp name="countlimit">0</stringProp> <stringProp name="timelimit">0</stringProp> <stringProp name="attributes">mail:dn:objectClass</stringProp> <stringProp name="return_object">false</stringProp> <stringProp name="deref_aliases">false</stringProp> <stringProp name="connection_timeout"></stringProp> <stringProp name="parseflag">false</stringProp> <stringProp name="secure">false</stringProp> <stringProp name="user_dn"></stringProp> <stringProp name="user_pw"></stringProp> <stringProp name="comparedn"></stringProp> <stringProp name="comparefilt"></stringProp> <stringProp name="modddn"></stringProp> <stringProp name="newdn"></stringProp> <stringProp name="test">search</stringProp> <stringProp name="search"></stringProp> <stringProp name="searchfilter">(mail=u_${__threadNum}_${iter}@test.com)</stringProp> </LDAPExtSampler> <hashTree/> <!-- COMPARE (admin connection). --> <LDAPExtSampler guiclass="LdapExtTestSamplerGui" testclass="LDAPExtSampler" testname="COMPARE" enabled="true"> <stringProp name="servername"></stringProp> <stringProp name="port"></stringProp> <stringProp name="rootdn"></stringProp> <stringProp name="scope">2</stringProp> <stringProp name="countlimit"></stringProp> <stringProp name="timelimit"></stringProp> <stringProp name="attributes"></stringProp> <stringProp name="return_object">false</stringProp> <stringProp name="deref_aliases">false</stringProp> <stringProp name="connection_timeout"></stringProp> <stringProp name="parseflag">false</stringProp> <stringProp name="secure">false</stringProp> <stringProp name="user_dn"></stringProp> <stringProp name="user_pw"></stringProp> <stringProp name="comparedn">mail=u_${__threadNum}_${iter}@test.com</stringProp> <stringProp name="comparefilt">mail=u_${__threadNum}_${iter}@test.com</stringProp> <stringProp name="modddn"></stringProp> <stringProp name="newdn"></stringProp> <stringProp name="test">compare</stringProp> </LDAPExtSampler> <hashTree/> <!-- MODIFY (admin connection): write a normal attribute (description) and set userPassword (cleartext) so the server hashes it; the user then authenticates with it (sbind). --> <LDAPExtSampler guiclass="LdapExtTestSamplerGui" testclass="LDAPExtSampler" testname="MODIFY" enabled="true"> <stringProp name="servername"></stringProp> <stringProp name="port"></stringProp> <stringProp name="rootdn"></stringProp> <stringProp name="scope">2</stringProp> <stringProp name="countlimit"></stringProp> <stringProp name="timelimit"></stringProp> <stringProp name="attributes"></stringProp> <stringProp name="return_object">false</stringProp> <stringProp name="deref_aliases">false</stringProp> <stringProp name="connection_timeout"></stringProp> <stringProp name="parseflag">false</stringProp> <stringProp name="secure">false</stringProp> <stringProp name="user_dn"></stringProp> <stringProp name="user_pw"></stringProp> <stringProp name="comparedn"></stringProp> <stringProp name="comparefilt"></stringProp> <stringProp name="modddn"></stringProp> <stringProp name="newdn"></stringProp> <stringProp name="test">modify</stringProp> <stringProp name="base_entry_dn">mail=u_${__threadNum}_${iter}@test.com</stringProp> <elementProp name="ldaparguments" elementType="LDAPArguments" guiclass="LDAPArgumentsPanel" testclass="LDAPArguments" testname="LDAP Extended Request Defaults" enabled="true"> <collectionProp name="Arguments.arguments"> <elementProp name="description" elementType="LDAPArgument"> <stringProp name="Argument.name">description</stringProp> <stringProp name="Argument.value">mod_${__threadNum}_${iter}</stringProp> <stringProp name="Argument.opcode">replace</stringProp> <stringProp name="Argument.metadata">=</stringProp> </elementProp> <elementProp name="userPassword" elementType="LDAPArgument"> <stringProp name="Argument.name">userPassword</stringProp> <stringProp name="Argument.value">${__P(benchpw,benchPass1)}</stringProp> <stringProp name="Argument.opcode">replace</stringProp> <stringProp name="Argument.metadata">=</stringProp> </elementProp> </collectionProp> </elementProp> </LDAPExtSampler> <hashTree/> <!-- BIND (measured user authentication): single bind/unbind on its own connection, does NOT disturb the cached admin connection. --> <LDAPExtSampler guiclass="LdapExtTestSamplerGui" testclass="LDAPExtSampler" testname="BIND" enabled="true"> <stringProp name="servername">${__P(host,localhost)}</stringProp> <stringProp name="port">${__P(port,1389)}</stringProp> <stringProp name="rootdn">ou=People,${__P(basedn,dc=example\,dc=com)}</stringProp> <stringProp name="scope">2</stringProp> <stringProp name="countlimit"></stringProp> <stringProp name="timelimit"></stringProp> <stringProp name="attributes"></stringProp> <stringProp name="return_object">false</stringProp> <stringProp name="deref_aliases">false</stringProp> <stringProp name="connection_timeout">60000</stringProp> <stringProp name="parseflag">false</stringProp> <stringProp name="secure">false</stringProp> <stringProp name="user_dn">mail=u_${__threadNum}_${iter}@test.com,ou=People,${__P(basedn,dc=example\,dc=com)}</stringProp> <stringProp name="user_pw">${__P(benchpw,benchPass1)}</stringProp> <stringProp name="comparedn"></stringProp> <stringProp name="comparefilt"></stringProp> <stringProp name="modddn"></stringProp> <stringProp name="newdn"></stringProp> <stringProp name="test">sbind</stringProp> </LDAPExtSampler> <hashTree/> <!-- DELETE (admin connection): remove the per-iteration entry. --> <LDAPExtSampler guiclass="LdapExtTestSamplerGui" testclass="LDAPExtSampler" testname="DELETE" enabled="true"> <stringProp name="servername"></stringProp> <stringProp name="port"></stringProp> <stringProp name="rootdn"></stringProp> <stringProp name="scope">2</stringProp> <stringProp name="countlimit"></stringProp> <stringProp name="timelimit"></stringProp> <stringProp name="attributes"></stringProp> <stringProp name="return_object">false</stringProp> <stringProp name="deref_aliases">false</stringProp> <stringProp name="connection_timeout"></stringProp> <stringProp name="parseflag">false</stringProp> <stringProp name="secure">false</stringProp> <stringProp name="user_dn"></stringProp> <stringProp name="user_pw"></stringProp> <stringProp name="comparedn"></stringProp> <stringProp name="comparefilt"></stringProp> <stringProp name="modddn"></stringProp> <stringProp name="newdn"></stringProp> <stringProp name="test">delete</stringProp> <stringProp name="delete">mail=u_${__threadNum}_${iter}@test.com</stringProp> </LDAPExtSampler> <hashTree/> <!-- READD (admin connection): accumulation; globally unique mail via __UUID, never deleted. --> <LDAPExtSampler guiclass="LdapExtTestSamplerGui" testclass="LDAPExtSampler" testname="READD" enabled="true"> <stringProp name="servername"></stringProp> <stringProp name="port"></stringProp> <stringProp name="rootdn"></stringProp> <stringProp name="scope">2</stringProp> <stringProp name="countlimit"></stringProp> <stringProp name="timelimit"></stringProp> <stringProp name="attributes"></stringProp> <stringProp name="return_object">false</stringProp> <stringProp name="deref_aliases">false</stringProp> <stringProp name="connection_timeout"></stringProp> <stringProp name="parseflag">false</stringProp> <stringProp name="secure">false</stringProp> <stringProp name="user_dn"></stringProp> <stringProp name="user_pw"></stringProp> <stringProp name="comparedn"></stringProp> <stringProp name="comparefilt"></stringProp> <stringProp name="modddn"></stringProp> <stringProp name="newdn"></stringProp> <stringProp name="test">add</stringProp> <stringProp name="base_entry_dn">mail=u_${__UUID}@test.com</stringProp> <elementProp name="arguments" elementType="Arguments" guiclass="ArgumentsPanel" testclass="Arguments" testname="User Defined Variables" enabled="true"> <collectionProp name="Arguments.arguments"> <elementProp name="mail" elementType="Argument"> <stringProp name="Argument.name">mail</stringProp> <stringProp name="Argument.value">u_${__UUID}@test.com</stringProp> <stringProp name="Argument.metadata">=</stringProp> </elementProp> <elementProp name="objectClass" elementType="Argument"> <stringProp name="Argument.name">objectClass</stringProp> <stringProp name="Argument.value">top</stringProp> <stringProp name="Argument.metadata">=</stringProp> </elementProp> <elementProp name="objectClass" elementType="Argument"> <stringProp name="Argument.name">objectClass</stringProp> <stringProp name="Argument.value">locality</stringProp> <stringProp name="Argument.metadata">=</stringProp> </elementProp> <elementProp name="objectClass" elementType="Argument"> <stringProp name="Argument.name">objectClass</stringProp> <stringProp name="Argument.value">extensibleObject</stringProp> <stringProp name="Argument.metadata">=</stringProp> </elementProp> </collectionProp> </elementProp> </LDAPExtSampler> <hashTree/> </hashTree> </hashTree> </hashTree> </jmeterTestPlan> .github/benchmark/compare-opendj.sh
New file @@ -0,0 +1,102 @@ #!/usr/bin/env bash # The contents of this file are subject to the terms of the Common Development and # Distribution License (the License). You may not use this file except in compliance with the # License. # # You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the # specific language governing permission and limitations under the License. # # When distributing Covered Software, include this CDDL Header Notice in each file and include # the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL # Header, with the fields enclosed by brackets [] replaced by your own identifying # information: "Portions copyright [year] [name of copyright owner]". # # Copyright 2026 3A Systems, LLC. # # Compare two OpenDJ Docker images with the LDAP benchmark (.github/benchmark/benchmark.jmx) and # append the comparison report to $GITHUB_STEP_SUMMARY. Both sides are OpenDJ, so no per-server # hashing/index setup is needed (identical product => identical default password scheme). # # Usage: # compare-opendj.sh <A_name> <A_image> <B_name> <B_image> # Env: THREADS (default 200), DURATION (default 150), JMETER_VERSION (default 5.6.3). set -euo pipefail A_NAME="${1:?server A name required}" A_IMAGE="${2:?server A image required}" B_NAME="${3:?server B name required}" B_IMAGE="${4:?server B image required}" THREADS="${THREADS:-200}" DURATION="${DURATION:-150}" JMETER_VERSION="${JMETER_VERSION:-5.6.3}" BASEDN="dc=example,dc=com" BENCHPW="benchPass1" HERE="$(cd "$(dirname "$0")" && pwd)" # .github/benchmark # ---------------------------------------------------------------- dependencies if ! command -v ldapsearch >/dev/null 2>&1 || ! command -v jq >/dev/null 2>&1; then sudo apt-get update -qq sudo apt-get install -y -qq ldap-utils jq fi JM="$HOME/jmeter/apache-jmeter-$JMETER_VERSION/bin/jmeter" if [ ! -x "$JM" ]; then mkdir -p "$HOME/jmeter" curl -fsSL "https://archive.apache.org/dist/jmeter/binaries/apache-jmeter-$JMETER_VERSION.tgz" -o /tmp/jmeter.tgz tar -xzf /tmp/jmeter.tgz -C "$HOME/jmeter" fi wait_dj() { # poll OpenDJ readiness on localhost:1389 for _ in $(seq 1 90); do ldapsearch -x -H ldap://localhost:1389 -D "cn=Directory Manager" -w password \ -b "$BASEDN" -s base dn >/dev/null 2>&1 && return 0 sleep 2 done return 1 } # bench_one <image> <out-slug> -> prints the captured server version (stdout only) bench_one() { local image="$1" out="$2" ver="" docker rm -f opendj-bench >/dev/null 2>&1 || true if docker run -d --name opendj-bench -p 1389:1389 \ -e ROOT_PASSWORD=password -e BASE_DN="$BASEDN" -e ADD_BASE_ENTRY=--addBaseEntry \ "$image" >/dev/null 2>&1; then wait_dj || echo "WARN: $image not ready in time" >&2 ldapadd -x -H ldap://localhost:1389 -D "cn=Directory Manager" -w password \ -f "$HERE/people.ldif" >/dev/null 2>&1 || true ver="$( { ldapsearch -x -LLL -H ldap://localhost:1389 -D 'cn=Directory Manager' -w password \ -b '' -s base fullVendorVersion 2>/dev/null || true; } | sed -n 's/^fullVendorVersion: //p')" rm -rf "$out" "$out.jtl" HEAP="-Xms1g -Xmx2g" "$JM" -n -t "$HERE/benchmark.jmx" \ -Jhost=localhost -Jport=1389 -Jbasedn="$BASEDN" \ -Jadminbinddn="cn=Directory Manager" -Jadminbindpw=password -Jbenchpw="$BENCHPW" \ -Jthreads="$THREADS" -Jduration="$DURATION" -Jrampup=0 \ -Jjmeter.reportgenerator.sample_filter='^(?!ADMIN_CONNECT).*' \ -l "$out.jtl" -e -o "$out" >/dev/null 2>&1 || true docker logs opendj-bench > "$out.docker.log" 2>&1 || true else echo "ERROR: failed to start image $image" >&2 fi docker rm -f opendj-bench >/dev/null 2>&1 || true [ -n "$ver" ] || ver="$image" printf '%s' "$ver" } echo "Benchmarking ${A_NAME} (${A_IMAGE}) @ ${THREADS} threads / ${DURATION}s ..." A_VER="$(bench_one "$A_IMAGE" a)" echo "Benchmarking ${B_NAME} (${B_IMAGE}) @ ${THREADS} threads / ${DURATION}s ..." B_VER="$(bench_one "$B_IMAGE" b)" { bash "$HERE/summary.sh" \ "$A_NAME" a/statistics.json "$A_VER" "$A_IMAGE" \ "$B_NAME" b/statistics.json "$B_VER" "$B_IMAGE" echo "" echo "### Notes" echo "" echo "- **${A_NAME}** = freshly built image; **${B_NAME}** = latest released image. Both are" echo " OpenDJ, so they share the same default password storage scheme (hashing parity is automatic)." echo "- The admin connection bind (\`ADMIN_CONNECT\`) is cached per thread and excluded; \`BIND\` is" echo " the measured user authentication (\`test=sbind\`, single bind/unbind)." } >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}" .github/benchmark/people.ldif
New file @@ -0,0 +1,17 @@ # The contents of this file are subject to the terms of the Common Development and # Distribution License (the License). You may not use this file except in compliance with the # License. # # You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the # specific language governing permission and limitations under the License. # # When distributing Covered Software, include this CDDL Header Notice in each file and include # the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL # Header, with the fields enclosed by brackets [] replaced by your own identifying # information: "Portions copyright [year] [name of copyright owner]". # # Copyright 2026 3A Systems, LLC. dn: ou=People,dc=example,dc=com objectClass: top objectClass: organizationalUnit ou: People .github/benchmark/summary.sh
New file @@ -0,0 +1,129 @@ #!/usr/bin/env bash # The contents of this file are subject to the terms of the Common Development and # Distribution License (the License). You may not use this file except in compliance with the # License. # # You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the # specific language governing permission and limitations under the License. # # When distributing Covered Software, include this CDDL Header Notice in each file and include # the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL # Header, with the fields enclosed by brackets [] replaced by your own identifying # information: "Portions copyright [year] [name of copyright owner]". # # Copyright 2026 3A Systems, LLC. # # Render an LDAP benchmark comparison report (versions + per-operation table + QuickChart charts) # for two servers A and B to stdout — intended to be appended to $GITHUB_STEP_SUMMARY. The report # is generic: server names, versions and images are all parameters, and benchmark-specific notes # are NOT included here (append them separately, e.g. `cat notes.md >> $GITHUB_STEP_SUMMARY`). # # Usage: # summary.sh <A_name> <A_statistics.json> <A_version> <A_image> \ # <B_name> <B_statistics.json> <B_version> <B_image> # # The admin connection bind is labelled ADMIN_CONNECT in the plan and is intentionally # skipped here, so it never pollutes the per-operation comparison. set -euo pipefail A_NAME="${1:?server A name required}" A_JSON="${2:?server A statistics.json required}" A_VER="${3:-unknown}" A_IMG="${4:-}" B_NAME="${5:?server B name required}" B_JSON="${6:?server B statistics.json required}" B_VER="${7:-unknown}" B_IMG="${8:-}" A_COLOR="#4e79a7" # blue = server A B_COLOR="#f28e2b" # orange = server B # Operations to compare, in workflow order. ADMIN_CONNECT and Total are excluded. OPS=("ADD" "SEARCH" "COMPARE" "MODIFY" "BIND" "DELETE" "READD") # m <file> <label> <field> -> numeric value (0 if absent), rounded to 1 decimal. m() { jq -r --arg l "$2" --arg f "$3" '((.[$l][$f]) // 0) | (.*10 | round / 10)' "$1"; } # mi <file> <label> <field> -> integer value (0 if absent). mi() { jq -r --arg l "$2" --arg f "$3" '((.[$l][$f]) // 0) | round' "$1"; } echo "## 🔬 Benchmark: ${A_NAME} vs ${B_NAME}" echo "" # ---------------------------------------------------------------- Versions echo "### Versions" echo "" echo "| Server | Version | Image |" echo "|---|---|---|" echo "| **${A_NAME}** | \`${A_VER}\` | \`${A_IMG:-n/a}\` |" echo "| **${B_NAME}** | \`${B_VER}\` | \`${B_IMG:-n/a}\` |" echo "" # ---------------------------------------------------------------- Totals a_tot_tp=$(m "$A_JSON" Total throughput) b_tot_tp=$(m "$B_JSON" Total throughput) a_tot_n=$(mi "$A_JSON" Total sampleCount) b_tot_n=$(mi "$B_JSON" Total sampleCount) a_tot_e=$(mi "$A_JSON" Total errorCount) b_tot_e=$(mi "$B_JSON" Total errorCount) a_tot_mean=$(m "$A_JSON" Total meanResTime) b_tot_mean=$(m "$B_JSON" Total meanResTime) echo "### Totals (all operations, ADMIN_CONNECT excluded by the plan label)" echo "" echo "| Server | Throughput (ops/s) | Mean (ms) | Samples | Errors |" echo "|---|--:|--:|--:|--:|" echo "| **${A_NAME}** | ${a_tot_tp} | ${a_tot_mean} | ${a_tot_n} | ${a_tot_e} |" echo "| **${B_NAME}** | ${b_tot_tp} | ${b_tot_mean} | ${b_tot_n} | ${b_tot_e} |" echo "" # ---------------------------------------------------------------- Per-op table echo "### Per-operation latency" echo "" echo "| Operation | mean ms ${A_NAME} | mean ms ${B_NAME} | p99 ms ${A_NAME} | p99 ms ${B_NAME} | err ${A_NAME} | err ${B_NAME} |" echo "|---|--:|--:|--:|--:|--:|--:|" for op in "${OPS[@]}"; do printf '| %s | %s | %s | %s | %s | %s | %s |\n' \ "$op" \ "$(m "$A_JSON" "$op" meanResTime)" "$(m "$B_JSON" "$op" meanResTime)" \ "$(mi "$A_JSON" "$op" pct3ResTime)" "$(mi "$B_JSON" "$op" pct3ResTime)" \ "$(mi "$A_JSON" "$op" errorCount)" "$(mi "$B_JSON" "$op" errorCount)" done echo "" # ---------------------------------------------------------------- Chart helpers (QuickChart) # Mermaid xychart-beta can't do grouped bars / a legend and crowds 14 x-labels, so render proper # grouped bar charts via QuickChart (Chart.js) as images: https://quickchart.io/chart?c=<config>. # JSON array of the OPS labels: ["ADD","SEARCH",...]. labels_json() { local out="" op for op in "${OPS[@]}"; do out+="${out:+,}\"${op}\""; done printf '[%s]' "$out" } # Comma-joined values for all OPS from <file> <field> via the <m> helper. vals() { # <fn> <file> <field> local fn="$1" file="$2" field="$3" out="" v for op in "${OPS[@]}"; do v=$("$fn" "$file" "$op" "$field"); out+="${out:+,}${v}"; done printf '%s' "$out" } urienc() { jq -rn --arg s "$1" '$s|@uri'; } # URL-encode the chart config qc() { printf 'https://quickchart.io/chart?w=%s&h=%s&c=%s' "$1" "$2" "$(urienc "$3")"; } # ---------------------------------------------------------------- Total throughput chart echo "### Total throughput (ops/s, higher is better)" echo "" echo "_Per-operation throughput is not charted: every op runs once per loop iteration, so each" echo "op's throughput just equals the loop rate. The meaningful throughput is the aggregate._" echo "" TP_CFG="{\"type\":\"bar\",\"data\":{\"labels\":[\"${A_NAME}\",\"${B_NAME}\"],\"datasets\":[{\"label\":\"ops/s\",\"backgroundColor\":[\"$A_COLOR\",\"$B_COLOR\"],\"data\":[${a_tot_tp},${b_tot_tp}]}]},\"options\":{\"legend\":{\"display\":false},\"title\":{\"display\":true,\"text\":\"Total throughput (ops/s)\"}}}" echo ")" echo "" # ---------------------------------------------------------------- Latency chart (grouped bars) echo "### p99 latency per operation (ms, lower is better)" echo "" echo "_🟦 ${A_NAME} · 🟧 ${B_NAME} — grouped bars per operation._" echo "" LAT_CFG="{\"type\":\"bar\",\"data\":{\"labels\":$(labels_json),\"datasets\":[{\"label\":\"${A_NAME}\",\"backgroundColor\":\"$A_COLOR\",\"data\":[$(vals mi "$A_JSON" pct3ResTime)]},{\"label\":\"${B_NAME}\",\"backgroundColor\":\"$B_COLOR\",\"data\":[$(vals mi "$B_JSON" pct3ResTime)]}]},\"options\":{\"title\":{\"display\":true,\"text\":\"p99 latency per operation (ms)\"}}}" echo ")" echo "" .github/workflows/benchmark.yml
New file @@ -0,0 +1,287 @@ # The contents of this file are subject to the terms of the Common Development and # Distribution License (the License). You may not use this file except in compliance with the # License. # # You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the # specific language governing permission and limitations under the License. # # When distributing Covered Software, include this CDDL Header Notice in each file and include # the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL # Header, with the fields enclosed by brackets [] replaced by your own identifying # information: "Portions copyright [year] [name of copyright owner]". # # Copyright 2026 3A Systems, LLC. name: "Benchmark" # Benchmark: OpenDJ vs OpenLDAP. # Runs manually (workflow_dispatch) and automatically after the "Release" workflow completes. # Starts the latest OpenLDAP and OpenDJ from Docker (image tags overridable via inputs), # benchmarks OpenLDAP first then OpenDJ with Apache JMeter, and writes versions + comparison # table + comparative charts to the job summary. Full JMeter HTML dashboards are uploaded as # the "jmeter-reports" artifact. on: workflow_dispatch: inputs: openldap_image: description: "OpenLDAP Docker image" default: "vegardit/openldap:latest" opendj_image: description: "OpenDJ Docker image" default: "openidentityplatform/opendj:latest" threads: description: "Concurrent JMeter threads" default: "200" duration: description: "Test duration per server (seconds)" default: "300" rampup: description: "Ramp-up period (seconds)" default: "0" jmeter_version: description: "Apache JMeter version" default: "5.6.3" workflow_run: workflows: ["Release"] types: [completed] permissions: contents: read concurrency: group: benchmark-${{ github.ref }} cancel-in-progress: true defaults: run: shell: bash jobs: benchmark: # Manual runs always proceed; chained runs only after a successful Release. if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest env: # `${{ inputs.X || 'default' }}` so workflow_run (which carries no inputs) falls back. OPENLDAP_IMAGE: ${{ inputs.openldap_image || 'vegardit/openldap:latest' }} OPENDJ_IMAGE: ${{ inputs.opendj_image || 'openidentityplatform/opendj:latest' }} THREADS: ${{ inputs.threads || '200' }} DURATION: ${{ inputs.duration || '300' }} RAMPUP: ${{ inputs.rampup || '0' }} JMETER: ${{ inputs.jmeter_version || '5.6.3' }} BENCHPW: benchPass1 BASEDN: dc=example,dc=com # Exclude the cached admin connection bind from the dashboard / statistics.json. SAMPLE_FILTER: '^(?!ADMIN_CONNECT).*' HEAP: -Xms1g -Xmx2g steps: - uses: actions/checkout@v6 - name: Cache JMeter uses: actions/cache@v5 with: path: ~/jmeter key: jmeter-${{ env.JMETER }} - name: Install tooling (ldap-utils + JMeter) run: | sudo apt-get update sudo apt-get install -y ldap-utils if [ ! -x "$HOME/jmeter/apache-jmeter-$JMETER/bin/jmeter" ]; then mkdir -p "$HOME/jmeter" curl -fsSL "https://archive.apache.org/dist/jmeter/binaries/apache-jmeter-$JMETER.tgz" -o /tmp/jmeter.tgz tar -xzf /tmp/jmeter.tgz -C "$HOME/jmeter" fi echo "JMETER_BIN=$HOME/jmeter/apache-jmeter-$JMETER/bin/jmeter" >> "$GITHUB_ENV" - name: Start OpenLDAP run: | docker run -d --name openldap -p 2389:389 \ -e LDAP_INIT_ORG_NAME="Example" \ -e LDAP_INIT_ORG_DN="$BASEDN" \ -e LDAP_INIT_ROOT_USER_DN="cn=admin,$BASEDN" \ -e LDAP_INIT_ROOT_USER_PW="password" \ -e LDAP_TLS_ENABLED=false \ -e LDAP_LDAPS_ENABLED=false \ -e LDAP_INIT_PPOLICY_PW_MIN_LENGTH=1 \ -e LDAP_INIT_PPOLICY_MAX_FAILURES=0 \ -e LDAP_PPOLICY_PQCHECKER_RULE="0|00000000" \ "$OPENLDAP_IMAGE" - name: Configure OpenLDAP (SSHA hash-on-write, seed) run: | wait_ldap() { for i in $(seq 1 90); do ldapsearch -x -H ldap://localhost:2389 -D "cn=admin,$BASEDN" -w password \ -b "$BASEDN" -s base dn >/dev/null 2>&1 && { echo "OpenLDAP is up"; return 0; } sleep 2 done } # cn=config edits via EXTERNAL over ldapi as root (-u 0). le() { docker exec -u 0 -i openldap ldapmodify -Y EXTERNAL -H ldapi:/// >/dev/null 2>&1 || true; } wait_ldap # This image ships no SHA-2 module, so use {SSHA} (Salted SHA-1, OpenLDAP core) — also a # built-in OpenDJ scheme, so both servers hash identically. Set it as the global hash. printf 'dn: olcDatabase={-1}frontend,cn=config\nchangetype: modify\nreplace: olcPasswordHash\nolcPasswordHash: {SSHA}\n' | le # The image already loads the ppolicy overlay; enable hash-cleartext on it so a plain # admin modify of a cleartext userPassword is hashed with {SSHA} on write. PPDN="$(docker exec -u 0 openldap ldapsearch -Y EXTERNAL -H ldapi:/// -LLL -b cn=config '(olcOverlay=*ppolicy)' dn 2>/dev/null | sed -n 's/^dn: //p' | head -1)" [ -z "$PPDN" ] || printf 'dn: %s\nchangetype: modify\nreplace: olcPPolicyHashCleartext\nolcPPolicyHashCleartext: TRUE\n' "$PPDN" | le # Seed ou=People. ldapadd -x -H ldap://localhost:2389 -D "cn=admin,$BASEDN" -w password \ -f .github/benchmark/people.ldif || true - name: Capture OpenLDAP version run: | # No `| head -1`: head closes the pipe early, slapd -VV then gets SIGPIPE and under # `pipefail` fails the step with exit 141. Capture fully, take the first line in bash. ver="$(docker exec openldap sh -c '/usr/sbin/slapd -VV 2>&1 || slapd -VV 2>&1' || true)" ver="${ver%%$'\n'*}" [ -n "$ver" ] || ver="$OPENLDAP_IMAGE" echo "OpenLDAP: $ver" { echo "OPENLDAP_VER<<EOF" echo "$ver" echo "EOF" } >> "$GITHUB_ENV" - name: Benchmark OpenLDAP run: | rm -rf openldap openldap.jtl HEAP="$HEAP" "$JMETER_BIN" -n -t .github/benchmark/benchmark.jmx \ -Jhost=localhost -Jport=2389 \ -Jbasedn="$BASEDN" \ -Jadminbinddn="cn=admin,$BASEDN" -Jadminbindpw=password \ -Jbenchpw="$BENCHPW" \ -Jthreads="$THREADS" -Jduration="$DURATION" -Jrampup="$RAMPUP" \ -Jjmeter.reportgenerator.sample_filter="$SAMPLE_FILTER" \ -l openldap.jtl -e -o openldap - name: Stop OpenLDAP if: always() run: | mkdir -p logs/openldap docker logs openldap > logs/openldap/server.log 2>&1 || true # slapd mostly logs to stdout (captured above); grab the in-container /var/log too. docker cp openldap:/var/log logs/openldap/var-log 2>/dev/null || true tail -n 100 logs/openldap/server.log || true docker rm -f openldap || true - name: Start OpenDJ run: | docker run -d --name opendj -p 1389:1389 \ -e ROOT_PASSWORD=password \ -e BASE_DN="$BASEDN" \ -e ADD_BASE_ENTRY=--addBaseEntry \ "$OPENDJ_IMAGE" - name: Wait for OpenDJ + seed ou=People run: | for i in $(seq 1 90); do if ldapsearch -x -H ldap://localhost:1389 -D "cn=Directory Manager" -w password \ -b "$BASEDN" -s base dn >/dev/null 2>&1; then echo "OpenDJ is up"; break fi sleep 2 done ldapadd -x -H ldap://localhost:1389 -D "cn=Directory Manager" -w password \ -f .github/benchmark/people.ldif || true - name: Configure OpenDJ password policy (SSHA hash-on-write) run: | # Hash cleartext userPassword with Salted SHA-1 on write, matching OpenLDAP {SSHA}. docker exec opendj /opt/opendj/bin/dsconfig set-password-policy-prop \ --policy-name "Default Password Policy" \ --set default-password-storage-scheme:"Salted SHA-1" \ --hostname localhost --port 4444 \ --bindDN "cn=Directory Manager" --bindPassword password \ --trustAll --no-prompt - name: Capture OpenDJ version run: | # `|| true` guards the bind so a transient ldapsearch failure can't trip pipefail; # sed reads all input (no early pipe close), so no SIGPIPE here. ver="$( { ldapsearch -x -LLL -H ldap://localhost:1389 -D 'cn=Directory Manager' -w password \ -b '' -s base fullVendorVersion 2>/dev/null || true; } | sed -n 's/^fullVendorVersion: //p')" [ -n "$ver" ] || ver="$OPENDJ_IMAGE" echo "OpenDJ: $ver" { echo "OPENDJ_VER<<EOF" echo "$ver" echo "EOF" } >> "$GITHUB_ENV" - name: Benchmark OpenDJ run: | rm -rf opendj opendj.jtl HEAP="$HEAP" "$JMETER_BIN" -n -t .github/benchmark/benchmark.jmx \ -Jhost=localhost -Jport=1389 \ -Jbasedn="$BASEDN" \ -Jadminbinddn="cn=Directory Manager" -Jadminbindpw=password \ -Jbenchpw="$BENCHPW" \ -Jthreads="$THREADS" -Jduration="$DURATION" -Jrampup="$RAMPUP" \ -Jjmeter.reportgenerator.sample_filter="$SAMPLE_FILTER" \ -l opendj.jtl -e -o opendj - name: Stop OpenDJ if: always() run: | mkdir -p logs/opendj docker logs opendj > logs/opendj/server.log 2>&1 || true # OpenDJ's internal log directory (errors, access, replication, server.out, ...). docker cp opendj:/opt/opendj/data/logs logs/opendj/internal 2>/dev/null \ || docker cp opendj:/opt/opendj/logs logs/opendj/internal 2>/dev/null || true tail -n 100 logs/opendj/server.log || true docker rm -f opendj || true # ---------------------------------------------------------------- Report - name: Build job summary run: | # summary.sh is generic: <name> <statistics.json> <version> <image> per server. bash .github/benchmark/summary.sh \ "OpenLDAP" openldap/statistics.json "$OPENLDAP_VER" "$OPENLDAP_IMAGE" \ "OpenDJ" opendj/statistics.json "$OPENDJ_VER" "$OPENDJ_IMAGE" \ >> "$GITHUB_STEP_SUMMARY" # benchmark-specific notes (kept out of the reusable script) { echo "" echo "### Notes" echo "" echo '- `BIND` is the measured **user authentication** (`test=sbind`, single bind/unbind on its' echo ' own connection) as `mail=u_<n>,ou=People` with the password set by `MODIFY`. The admin' echo ' connection bind (`ADMIN_CONNECT`) is cached once per thread and excluded from these results.' echo '- `MODIFY` sends the password in cleartext; each server hashes it on write with the **same' echo ' scheme (`{SSHA}`, Salted SHA-1)** — OpenLDAP via the ppolicy hash-cleartext overlay, OpenDJ' echo ' via its Salted SHA-1 default scheme — so `BIND` authentication is compared on equal footing.' echo '- Full interactive JMeter HTML dashboards are attached as the `jmeter-reports` artifact.' } >> "$GITHUB_STEP_SUMMARY" - name: Upload JMeter reports if: always() uses: actions/upload-artifact@v7 with: name: jmeter-reports path: | openldap/ opendj/ *.jtl if-no-files-found: warn retention-days: 90 - name: Upload OpenLDAP logs if: always() uses: actions/upload-artifact@v7 with: name: logs-openldap path: logs/openldap/ if-no-files-found: warn retention-days: 90 - name: Upload OpenDJ logs if: always() uses: actions/upload-artifact@v7 with: name: logs-opendj path: logs/opendj/ if-no-files-found: warn retention-days: 90 .github/workflows/build.yml
@@ -308,6 +308,9 @@ ports: - 5000:5000 steps: - uses: actions/checkout@v6 with: sparse-checkout: .github/benchmark - name: Download artifacts uses: actions/download-artifact@v8 with: @@ -369,6 +372,17 @@ timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_custom | grep -q \"healthy\"; do sleep 10; done' docker exec test_custom 'sh' '-c' '/opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword custom_password --useSsl --trustAll --baseDN "dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1' docker kill test_custom - name: Cache JMeter uses: actions/cache@v5 with: path: ~/jmeter key: jmeter-5.6.3 - name: Benchmark Build vs Release shell: bash run: | THREADS=200 DURATION=150 bash .github/benchmark/compare-opendj.sh \ "Build" localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }} \ "Release" openidentityplatform/opendj:latest build-docker-alpine: needs: build-maven @@ -379,6 +393,9 @@ ports: - 5000:5000 steps: - uses: actions/checkout@v6 with: sparse-checkout: .github/benchmark - name: Download artifacts uses: actions/download-artifact@v8 with: @@ -441,3 +458,14 @@ timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_custom | grep -q \"healthy\"; do sleep 10; done' docker exec test_custom 'sh' '-c' '/opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword custom_password --useSsl --trustAll --baseDN "dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1' docker kill test_custom - name: Cache JMeter uses: actions/cache@v5 with: path: ~/jmeter key: jmeter-5.6.3 - name: Benchmark Build-alpine vs Release-alpine shell: bash run: | THREADS=200 DURATION=150 bash .github/benchmark/compare-opendj.sh \ "Build-alpine" localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}-alpine \ "Release-alpine" openidentityplatform/opendj:alpine
