Benchmark: search on indexed mail, unique entries, QuickChart charts

Valera V Harseko

2 days ago 82016da7b8049db5b02f3f70b00a5c7aedeb4226

Benchmark: search on indexed mail, unique entries, QuickChart charts

benchmark.jmx
- SEARCH now filters on `mail` instead of `sn`. mail is equality-indexed by
  default on BOTH OpenDJ and OpenLDAP/osixia, whereas sn is indexed on OpenDJ
  only, so the search becomes a real indexed lookup on both servers.
- Every created value is unique: ADD keys each entry by a per-iteration JMeter
  counter, READD by a UUID. The search now matches exactly one entry, and the
  accumulated (never-deleted) READD entries no longer inflate the result set.
- Entries are minimal and index-symmetric: RDN is mail, objectClass is
  top/locality/extensibleObject, and no cn/sn/uid/givenName/telephoneNumber/
  member/uniqueMember are stored (those are indexed on OpenDJ but not osixia,
  which would bias the write cost). MODIFY writes description + userPassword.

summary.sh
- Replace the Mermaid xychart-beta charts with QuickChart (Chart.js) grouped bar
  charts rendered as images: proper side-by-side OpenLDAP/OpenDJ bars per
  operation, a legend, readable labels and no overlap. xychart-beta cannot group
  bars or show a legend and crowded the 14 x-axis labels. Throughput is a two-bar
  chart; latency is grouped bars per operation. Config is built with jq and
  URL-encoded.

 .github/benchmark/benchmark.jmx

@@ -17,7 +17,7 @@
<jmeterTestPlan version="1.2" properties="5.0" jmeter="5.6.3">
  <hashTree>
    <TestPlan guiclass="TestPlanGui" testclass="TestPlan" testname="OpenDJ vs OpenLDAP - LDAP benchmark" enabled="true">
      <stringProp name="TestPlan.comments">Parametrized LDAP benchmark. Admin bind is cached once per thread (Once Only Controller, labelled ADMIN_CONNECT, excluded from metrics). Data ops (ADD/SEARCH/COMPARE/MODIFY/DELETE/READD) reuse the cached admin connection. The measured user authentication is a single bind/unbind (test=sbind, own connection) after MODIFY has set the userPassword.</stringProp>
      <stringProp name="TestPlan.comments">Parametrized LDAP benchmark. Admin bind is cached once per thread (Once Only Controller, labelled ADMIN_CONNECT, excluded from metrics). Data ops (ADD/SEARCH/COMPARE/MODIFY/DELETE/READD) reuse the cached admin connection. Entries use mail as the naming/searchable attribute (equality-indexed by default on BOTH OpenDJ and OpenLDAP/osixia); no cn/sn/uid/givenName/telephoneNumber/member/uniqueMember are stored (those are indexed on OpenDJ but not osixia, which would bias the write cost). Every created value is unique: ADD uses a per-iteration counter, READD uses a UUID. The measured user authentication is a single bind/unbind (test=sbind, own connection) after MODIFY has set the userPassword.</stringProp>
      <boolProp name="TestPlan.functional_mode">false</boolProp>
      <boolProp name="TestPlan.tearDown_on_shutdown">true</boolProp>
      <boolProp name="TestPlan.serialize_threadgroups">false</boolProp>
@@ -40,6 +40,20 @@
        <stringProp name="ThreadGroup.delay"></stringProp>
      </ThreadGroup>
      <hashTree>
        <!-- Per-thread, per-iteration counter. Combined with ${__threadNum} it yields a value
             unique across the whole run; it stays constant within a single loop iteration so the
             same entry is referenced by ADD/SEARCH/COMPARE/MODIFY/BIND/DELETE. -->
        <CounterConfig guiclass="CounterConfigGui" testclass="CounterConfig" testname="Per-iteration counter" enabled="true">
          <stringProp name="CounterConfig.start">1</stringProp>
          <stringProp name="CounterConfig.end"></stringProp>
          <stringProp name="CounterConfig.incr">1</stringProp>
          <stringProp name="CounterConfig.name">iter</stringProp>
          <stringProp name="CounterConfig.format"></stringProp>
          <boolProp name="CounterConfig.per_user">true</boolProp>
          <boolProp name="CounterConfig.reset_on_tg_iteration">false</boolProp>
        </CounterConfig>
        <hashTree/>

        <!-- Admin connection: bound once per thread, cached in ldapContexts[thread], reused by all
             data operations. Labelled ADMIN_CONNECT and excluded from metrics. -->
        <OnceOnlyController guiclass="OnceOnlyControllerGui" testclass="OnceOnlyController" testname="Once Only - admin connect" enabled="true"/>
@@ -68,7 +82,7 @@
          <hashTree/>
        </hashTree>

        <!-- ADD (admin connection): create the per-thread user entry. -->
        <!-- ADD (admin connection): create the per-iteration user entry, keyed by mail. -->
        <LDAPExtSampler guiclass="LdapExtTestSamplerGui" testclass="LDAPExtSampler" testname="ADD" enabled="true">
          <stringProp name="servername"></stringProp>
          <stringProp name="port"></stringProp>
@@ -89,17 +103,12 @@
          <stringProp name="modddn"></stringProp>
          <stringProp name="newdn"></stringProp>
          <stringProp name="test">add</stringProp>
          <stringProp name="base_entry_dn">cn=user_${__threadNum}</stringProp>
          <stringProp name="base_entry_dn">mail=u_${__threadNum}_${iter}@test.com</stringProp>
          <elementProp name="arguments" elementType="Arguments" guiclass="ArgumentsPanel" testclass="Arguments" testname="User Defined Variables" enabled="true">
            <collectionProp name="Arguments.arguments">
              <elementProp name="sn" elementType="Argument">
                <stringProp name="Argument.name">sn</stringProp>
                <stringProp name="Argument.value">user_${__threadNum}</stringProp>
                <stringProp name="Argument.metadata">=</stringProp>
              </elementProp>
              <elementProp name="mail" elementType="Argument">
                <stringProp name="Argument.name">mail</stringProp>
                <stringProp name="Argument.value">user_${__threadNum}@test.com</stringProp>
                <stringProp name="Argument.value">u_${__threadNum}_${iter}@test.com</stringProp>
                <stringProp name="Argument.metadata">=</stringProp>
              </elementProp>
              <elementProp name="objectClass" elementType="Argument">
@@ -109,17 +118,12 @@
              </elementProp>
              <elementProp name="objectClass" elementType="Argument">
                <stringProp name="Argument.name">objectClass</stringProp>
                <stringProp name="Argument.value">inetOrgPerson</stringProp>
                <stringProp name="Argument.value">locality</stringProp>
                <stringProp name="Argument.metadata">=</stringProp>
              </elementProp>
              <elementProp name="objectClass" elementType="Argument">
                <stringProp name="Argument.name">objectClass</stringProp>
                <stringProp name="Argument.value">organizationalPerson</stringProp>
                <stringProp name="Argument.metadata">=</stringProp>
              </elementProp>
              <elementProp name="objectClass" elementType="Argument">
                <stringProp name="Argument.name">objectClass</stringProp>
                <stringProp name="Argument.value">person</stringProp>
                <stringProp name="Argument.value">extensibleObject</stringProp>
                <stringProp name="Argument.metadata">=</stringProp>
              </elementProp>
            </collectionProp>
@@ -127,7 +131,7 @@
        </LDAPExtSampler>
        <hashTree/>

        <!-- SEARCH (admin connection). -->
        <!-- SEARCH (admin connection): equality lookup on the indexed mail attribute. -->
        <LDAPExtSampler guiclass="LdapExtTestSamplerGui" testclass="LDAPExtSampler" testname="SEARCH" enabled="true">
          <stringProp name="servername"></stringProp>
          <stringProp name="port"></stringProp>
@@ -135,7 +139,7 @@
          <stringProp name="scope">2</stringProp>
          <stringProp name="countlimit">0</stringProp>
          <stringProp name="timelimit">0</stringProp>
          <stringProp name="attributes">cn:dn:objectClass</stringProp>
          <stringProp name="attributes">mail:dn:objectClass</stringProp>
          <stringProp name="return_object">false</stringProp>
          <stringProp name="deref_aliases">false</stringProp>
          <stringProp name="connection_timeout"></stringProp>
@@ -149,7 +153,7 @@
          <stringProp name="newdn"></stringProp>
          <stringProp name="test">search</stringProp>
          <stringProp name="search"></stringProp>
          <stringProp name="searchfilter">(sn=user_${__threadNum})</stringProp>
          <stringProp name="searchfilter">(mail=u_${__threadNum}_${iter}@test.com)</stringProp>
        </LDAPExtSampler>
        <hashTree/>

@@ -169,15 +173,16 @@
          <stringProp name="secure">false</stringProp>
          <stringProp name="user_dn"></stringProp>
          <stringProp name="user_pw"></stringProp>
          <stringProp name="comparedn">cn=user_${__threadNum}</stringProp>
          <stringProp name="comparefilt">mail=user_${__threadNum}@test.com</stringProp>
          <stringProp name="comparedn">mail=u_${__threadNum}_${iter}@test.com</stringProp>
          <stringProp name="comparefilt">mail=u_${__threadNum}_${iter}@test.com</stringProp>
          <stringProp name="modddn"></stringProp>
          <stringProp name="newdn"></stringProp>
          <stringProp name="test">compare</stringProp>
        </LDAPExtSampler>
        <hashTree/>

        <!-- MODIFY (admin connection): rename sn and set userPassword so the user can authenticate. -->
        <!-- MODIFY (admin connection): write a normal attribute (description) and set userPassword
             (cleartext) so the server hashes it; the user then authenticates with it (sbind). -->
        <LDAPExtSampler guiclass="LdapExtTestSamplerGui" testclass="LDAPExtSampler" testname="MODIFY" enabled="true">
          <stringProp name="servername"></stringProp>
          <stringProp name="port"></stringProp>
@@ -198,12 +203,12 @@
          <stringProp name="modddn"></stringProp>
          <stringProp name="newdn"></stringProp>
          <stringProp name="test">modify</stringProp>
          <stringProp name="base_entry_dn">cn=user_${__threadNum}</stringProp>
          <stringProp name="base_entry_dn">mail=u_${__threadNum}_${iter}@test.com</stringProp>
          <elementProp name="ldaparguments" elementType="LDAPArguments" guiclass="LDAPArgumentsPanel" testclass="LDAPArguments" testname="LDAP Extended Request Defaults" enabled="true">
            <collectionProp name="Arguments.arguments">
              <elementProp name="sn" elementType="LDAPArgument">
                <stringProp name="Argument.name">sn</stringProp>
                <stringProp name="Argument.value">rename_${__threadNum}</stringProp>
              <elementProp name="description" elementType="LDAPArgument">
                <stringProp name="Argument.name">description</stringProp>
                <stringProp name="Argument.value">mod_${__threadNum}_${iter}</stringProp>
                <stringProp name="Argument.opcode">replace</stringProp>
                <stringProp name="Argument.metadata">=</stringProp>
              </elementProp>
@@ -233,7 +238,7 @@
          <stringProp name="connection_timeout">60000</stringProp>
          <stringProp name="parseflag">false</stringProp>
          <stringProp name="secure">false</stringProp>
          <stringProp name="user_dn">cn=user_${__threadNum},ou=People,${__P(basedn,dc=example\,dc=com)}</stringProp>
          <stringProp name="user_dn">mail=u_${__threadNum}_${iter}@test.com,ou=People,${__P(basedn,dc=example\,dc=com)}</stringProp>
          <stringProp name="user_pw">${__P(benchpw,benchPass1)}</stringProp>
          <stringProp name="comparedn"></stringProp>
          <stringProp name="comparefilt"></stringProp>
@@ -243,7 +248,7 @@
        </LDAPExtSampler>
        <hashTree/>

        <!-- DELETE (admin connection): remove the per-thread user entry. -->
        <!-- DELETE (admin connection): remove the per-iteration entry. -->
        <LDAPExtSampler guiclass="LdapExtTestSamplerGui" testclass="LDAPExtSampler" testname="DELETE" enabled="true">
          <stringProp name="servername"></stringProp>
          <stringProp name="port"></stringProp>
@@ -264,11 +269,11 @@
          <stringProp name="modddn"></stringProp>
          <stringProp name="newdn"></stringProp>
          <stringProp name="test">delete</stringProp>
          <stringProp name="delete">cn=user_${__threadNum}</stringProp>
          <stringProp name="delete">mail=u_${__threadNum}_${iter}@test.com</stringProp>
        </LDAPExtSampler>
        <hashTree/>

        <!-- READD (admin connection): re-add without delete (accumulation); unique RDN via __UUID. -->
        <!-- READD (admin connection): accumulation; globally unique mail via __UUID, never deleted. -->
        <LDAPExtSampler guiclass="LdapExtTestSamplerGui" testclass="LDAPExtSampler" testname="READD" enabled="true">
          <stringProp name="servername"></stringProp>
          <stringProp name="port"></stringProp>
@@ -289,17 +294,12 @@
          <stringProp name="modddn"></stringProp>
          <stringProp name="newdn"></stringProp>
          <stringProp name="test">add</stringProp>
          <stringProp name="base_entry_dn">cn=user_${__threadNum}_${__UUID}</stringProp>
          <stringProp name="base_entry_dn">mail=u_${__UUID}@test.com</stringProp>
          <elementProp name="arguments" elementType="Arguments" guiclass="ArgumentsPanel" testclass="Arguments" testname="User Defined Variables" enabled="true">
            <collectionProp name="Arguments.arguments">
              <elementProp name="sn" elementType="Argument">
                <stringProp name="Argument.name">sn</stringProp>
                <stringProp name="Argument.value">user_${__threadNum}</stringProp>
                <stringProp name="Argument.metadata">=</stringProp>
              </elementProp>
              <elementProp name="mail" elementType="Argument">
                <stringProp name="Argument.name">mail</stringProp>
                <stringProp name="Argument.value">user_${__threadNum}@test.com</stringProp>
                <stringProp name="Argument.value">u_${__UUID}@test.com</stringProp>
                <stringProp name="Argument.metadata">=</stringProp>
              </elementProp>
              <elementProp name="objectClass" elementType="Argument">
@@ -309,17 +309,12 @@
              </elementProp>
              <elementProp name="objectClass" elementType="Argument">
                <stringProp name="Argument.name">objectClass</stringProp>
                <stringProp name="Argument.value">inetOrgPerson</stringProp>
                <stringProp name="Argument.value">locality</stringProp>
                <stringProp name="Argument.metadata">=</stringProp>
              </elementProp>
              <elementProp name="objectClass" elementType="Argument">
                <stringProp name="Argument.name">objectClass</stringProp>
                <stringProp name="Argument.value">organizationalPerson</stringProp>
                <stringProp name="Argument.metadata">=</stringProp>
              </elementProp>
              <elementProp name="objectClass" elementType="Argument">
                <stringProp name="Argument.name">objectClass</stringProp>
                <stringProp name="Argument.value">person</stringProp>
                <stringProp name="Argument.value">extensibleObject</stringProp>
                <stringProp name="Argument.metadata">=</stringProp>
              </elementProp>
            </collectionProp>

 .github/benchmark/summary.sh

@@ -83,70 +83,44 @@
done
echo ""

# ---------------------------------------------------------------- Chart helpers
# Mermaid xychart-beta has no grouped bars/legend, so to get two adjacent (non-overlapping)
# columns per operation we repeat each op on the x-axis and zero-pad the two series: OpenLDAP
# bars land on the left tick of each pair, OpenDJ on the right tick.
#
# x-axis: each op as two *distinct* labels -> "ADD OL", "ADD DJ", "SEARCH OL", ...
# The labels MUST be unique: xychart-beta merges duplicate category labels into one slot, which
# would put both series back on the same column (overlapping). Distinct labels keep the columns
# separate so the zero-padded series render side by side.
xaxis_pairs() {
  local out="" op
  for op in "${OPS[@]}"; do out+="${out:+, }\"${op} OL\", \"${op} DJ\""; done
  printf '%s' "$out"
}
# OpenLDAP series: value then 0 per op (bar on the left tick of each pair).
series_ol() { # <fn> <file> <field>
  local fn="$1" file="$2" field="$3" out="" v
  for op in "${OPS[@]}"; do v=$("$fn" "$file" "$op" "$field"); out+="${out:+, }${v}, 0"; done
  printf '%s' "$out"
}
# OpenDJ series: 0 then value per op (bar on the right tick of each pair).
series_dj() { # <fn> <file> <field>
  local fn="$1" file="$2" field="$3" out="" v
  for op in "${OPS[@]}"; do v=$("$fn" "$file" "$op" "$field"); out+="${out:+, }0, ${v}"; done
  printf '%s' "$out"
}
# ---------------------------------------------------------------- Chart helpers (QuickChart)
# Mermaid xychart-beta can't do grouped bars / a legend and crowds 14 x-labels, so render proper
# grouped bar charts via QuickChart (Chart.js) as images: https://quickchart.io/chart?c=<config>.
OL_COLOR="#4e79a7"   # blue   = OpenLDAP
DJ_COLOR="#f28e2b"   # orange = OpenDJ

XAXIS="$(xaxis_pairs)"
# Fix the two series colors (series 1 = OpenLDAP blue, series 2 = OpenDJ orange).
PALETTE='%%{init: {"themeVariables": {"xyChart": {"plotColorPalette": "#4e79a7, #f28e2b"}}}}%%'
CAPTION="_Each operation has two columns: 🟦 OpenLDAP (left) · 🟧 OpenDJ (right)._"
# JSON array of the OPS labels: ["ADD","SEARCH",...].
labels_json() {
  local out="" op
  for op in "${OPS[@]}"; do out+="${out:+,}\"${op}\""; done
  printf '[%s]' "$out"
}
# Comma-joined values for all OPS from <file> <field> via the <m> helper.
vals() { # <fn> <file> <field>
  local fn="$1" file="$2" field="$3" out="" v
  for op in "${OPS[@]}"; do v=$("$fn" "$file" "$op" "$field"); out+="${out:+,}${v}"; done
  printf '%s' "$out"
}
urienc() { jq -rn --arg s "$1" '$s|@uri'; }                       # URL-encode the chart config
qc() { printf 'https://quickchart.io/chart?w=%s&h=%s&c=%s' "$1" "$2" "$(urienc "$3")"; }

# ---------------------------------------------------------------- Total throughput chart
echo "### Total throughput (ops/s, higher is better)"
echo ""
echo "_🟦 OpenLDAP · 🟧 OpenDJ. Per-operation throughput is intentionally not charted: every"
echo "operation runs once per loop iteration, so each op's throughput just equals the loop rate"
echo "(nearly identical across ops). The meaningful throughput is the aggregate shown here._"
echo "_Per-operation throughput is not charted: every op runs once per loop iteration, so each"
echo "op's throughput just equals the loop rate. The meaningful throughput is the aggregate._"
echo ""
echo '```mermaid'
echo "$PALETTE"
echo "xychart-beta"
echo "    title \"Total throughput (ops/s) — OpenLDAP vs OpenDJ\""
echo "    x-axis [\"OpenLDAP\", \"OpenDJ\"]"
echo "    y-axis \"ops/s\""
echo "    bar [${ol_tot_tp}, 0]"
echo "    bar [0, ${dj_tot_tp}]"
echo '```'
TP_CFG="{\"type\":\"bar\",\"data\":{\"labels\":[\"OpenLDAP\",\"OpenDJ\"],\"datasets\":[{\"label\":\"ops/s\",\"backgroundColor\":[\"$OL_COLOR\",\"$DJ_COLOR\"],\"data\":[${ol_tot_tp},${dj_tot_tp}]}]},\"options\":{\"legend\":{\"display\":false},\"title\":{\"display\":true,\"text\":\"Total throughput (ops/s)\"}}}"
echo "![Total throughput (ops/s)]($(qc 500 320 "$TP_CFG"))"
echo ""

# ---------------------------------------------------------------- Latency chart
echo "### Comparative chart — mean latency per operation (ms, lower is better)"
# ---------------------------------------------------------------- Latency chart (grouped bars)
echo "### Mean latency per operation (ms, lower is better)"
echo ""
echo "$CAPTION"
echo "_🟦 OpenLDAP · 🟧 OpenDJ — grouped bars per operation._"
echo ""
echo '```mermaid'
echo "$PALETTE"
echo "xychart-beta"
echo "    title \"Mean latency per operation (ms) — OpenLDAP vs OpenDJ\""
echo "    x-axis [${XAXIS}]"
echo "    y-axis \"ms\""
echo "    bar [$(series_ol m "$OL_JSON" meanResTime)]"
echo "    bar [$(series_dj m "$DJ_JSON" meanResTime)]"
echo '```'
LAT_CFG="{\"type\":\"bar\",\"data\":{\"labels\":$(labels_json),\"datasets\":[{\"label\":\"OpenLDAP\",\"backgroundColor\":\"$OL_COLOR\",\"data\":[$(vals m "$OL_JSON" meanResTime)]},{\"label\":\"OpenDJ\",\"backgroundColor\":\"$DJ_COLOR\",\"data\":[$(vals m "$DJ_JSON" meanResTime)]}]},\"options\":{\"title\":{\"display\":true,\"text\":\"Mean latency per operation (ms)\"}}}"
echo "![Mean latency per operation (ms)]($(qc 900 400 "$LAT_CFG"))"
echo ""

# ---------------------------------------------------------------- Caveats