From 5ec57539279164d5be9797a4975f2e9bdaf0263c Mon Sep 17 00:00:00 2001
From: Valera V Harseko <vharseko@3a-systems.ru>
Date: Tue, 23 Jun 2026 08:44:55 +0000
Subject: [PATCH] Benchmark: make summary.sh reusable, move notes into the workflow

---
 .github/benchmark/summary.sh    |   80 ++++++++++++++++++---------------------
 .github/workflows/benchmark.yml |   20 ++++++++-
 2 files changed, 54 insertions(+), 46 deletions(-)

diff --git a/.github/benchmark/summary.sh b/.github/benchmark/summary.sh
index 7608f4b..992c431 100644
--- a/.github/benchmark/summary.sh
+++ b/.github/benchmark/summary.sh
@@ -13,23 +13,30 @@
 #
 # Copyright 2026 3A Systems, LLC.
 #
-# Render an OpenDJ-vs-OpenLDAP LDAP benchmark report (versions + comparison table +
-# comparative Mermaid charts) to stdout — intended to be appended to $GITHUB_STEP_SUMMARY.
+# Render an LDAP benchmark comparison report (versions + per-operation table + QuickChart charts)
+# for two servers A and B to stdout — intended to be appended to $GITHUB_STEP_SUMMARY. The report
+# is generic: server names, versions and images are all parameters, and benchmark-specific notes
+# are NOT included here (append them separately, e.g. `cat notes.md >> $GITHUB_STEP_SUMMARY`).
 #
 # Usage:
-#   summary.sh <openldap_statistics.json> <opendj_statistics.json> \
-#              <openldap_version> <opendj_version> [openldap_image] [opendj_image]
+#   summary.sh <A_name> <A_statistics.json> <A_version> <A_image> \
+#              <B_name> <B_statistics.json> <B_version> <B_image>
 #
 # The admin connection bind is labelled ADMIN_CONNECT in the plan and is intentionally
 # skipped here, so it never pollutes the per-operation comparison.
 set -euo pipefail
 
-OL_JSON="${1:?openldap statistics.json required}"
-DJ_JSON="${2:?opendj statistics.json required}"
-OL_VER="${3:-unknown}"
-DJ_VER="${4:-unknown}"
-OL_IMG="${5:-}"
-DJ_IMG="${6:-}"
+A_NAME="${1:?server A name required}"
+A_JSON="${2:?server A statistics.json required}"
+A_VER="${3:-unknown}"
+A_IMG="${4:-}"
+B_NAME="${5:?server B name required}"
+B_JSON="${6:?server B statistics.json required}"
+B_VER="${7:-unknown}"
+B_IMG="${8:-}"
+
+A_COLOR="#4e79a7"   # blue   = server A
+B_COLOR="#f28e2b"   # orange = server B
 
 # Operations to compare, in workflow order. ADMIN_CONNECT and Total are excluded.
 OPS=("ADD" "SEARCH" "COMPARE" "MODIFY" "BIND" "DELETE" "READD")
@@ -39,7 +46,7 @@
 # mi <file> <label> <field> -> integer value (0 if absent).
 mi() { jq -r --arg l "$2" --arg f "$3" '((.[$l][$f]) // 0) | round' "$1"; }
 
-echo "## 🔬 Benchmark: OpenDJ vs OpenLDAP"
+echo "## 🔬 Benchmark: ${A_NAME} vs ${B_NAME}"
 echo ""
 
 # ---------------------------------------------------------------- Versions
@@ -47,47 +54,45 @@
 echo ""
 echo "| Server | Version | Image |"
 echo "|---|---|---|"
-echo "| **OpenLDAP** | \`${OL_VER}\` | \`${OL_IMG:-n/a}\` |"
-echo "| **OpenDJ** | \`${DJ_VER}\` | \`${DJ_IMG:-n/a}\` |"
+echo "| **${A_NAME}** | \`${A_VER}\` | \`${A_IMG:-n/a}\` |"
+echo "| **${B_NAME}** | \`${B_VER}\` | \`${B_IMG:-n/a}\` |"
 echo ""
 
 # ---------------------------------------------------------------- Totals
-ol_tot_tp=$(m  "$OL_JSON" Total throughput)
-dj_tot_tp=$(m  "$DJ_JSON" Total throughput)
-ol_tot_n=$(mi  "$OL_JSON" Total sampleCount)
-dj_tot_n=$(mi  "$DJ_JSON" Total sampleCount)
-ol_tot_e=$(mi  "$OL_JSON" Total errorCount)
-dj_tot_e=$(mi  "$DJ_JSON" Total errorCount)
-ol_tot_mean=$(m "$OL_JSON" Total meanResTime)
-dj_tot_mean=$(m "$DJ_JSON" Total meanResTime)
+a_tot_tp=$(m  "$A_JSON" Total throughput)
+b_tot_tp=$(m  "$B_JSON" Total throughput)
+a_tot_n=$(mi  "$A_JSON" Total sampleCount)
+b_tot_n=$(mi  "$B_JSON" Total sampleCount)
+a_tot_e=$(mi  "$A_JSON" Total errorCount)
+b_tot_e=$(mi  "$B_JSON" Total errorCount)
+a_tot_mean=$(m "$A_JSON" Total meanResTime)
+b_tot_mean=$(m "$B_JSON" Total meanResTime)
 
 echo "### Totals (all operations, ADMIN_CONNECT excluded by the plan label)"
 echo ""
 echo "| Server | Throughput (ops/s) | Mean (ms) | Samples | Errors |"
 echo "|---|--:|--:|--:|--:|"
-echo "| **OpenLDAP** | ${ol_tot_tp} | ${ol_tot_mean} | ${ol_tot_n} | ${ol_tot_e} |"
-echo "| **OpenDJ** | ${dj_tot_tp} | ${dj_tot_mean} | ${dj_tot_n} | ${dj_tot_e} |"
+echo "| **${A_NAME}** | ${a_tot_tp} | ${a_tot_mean} | ${a_tot_n} | ${a_tot_e} |"
+echo "| **${B_NAME}** | ${b_tot_tp} | ${b_tot_mean} | ${b_tot_n} | ${b_tot_e} |"
 echo ""
 
 # ---------------------------------------------------------------- Per-op table
 echo "### Per-operation latency"
 echo ""
-echo "| Operation | mean ms OpenLDAP | mean ms OpenDJ | p99 ms OpenLDAP | p99 ms OpenDJ | err OL | err DJ |"
+echo "| Operation | mean ms ${A_NAME} | mean ms ${B_NAME} | p99 ms ${A_NAME} | p99 ms ${B_NAME} | err ${A_NAME} | err ${B_NAME} |"
 echo "|---|--:|--:|--:|--:|--:|--:|"
 for op in "${OPS[@]}"; do
   printf '| %s | %s | %s | %s | %s | %s | %s |\n' \
     "$op" \
-    "$(m  "$OL_JSON" "$op" meanResTime)"  "$(m  "$DJ_JSON" "$op" meanResTime)" \
-    "$(mi "$OL_JSON" "$op" pct3ResTime)"  "$(mi "$DJ_JSON" "$op" pct3ResTime)" \
-    "$(mi "$OL_JSON" "$op" errorCount)"   "$(mi "$DJ_JSON" "$op" errorCount)"
+    "$(m  "$A_JSON" "$op" meanResTime)"  "$(m  "$B_JSON" "$op" meanResTime)" \
+    "$(mi "$A_JSON" "$op" pct3ResTime)"  "$(mi "$B_JSON" "$op" pct3ResTime)" \
+    "$(mi "$A_JSON" "$op" errorCount)"   "$(mi "$B_JSON" "$op" errorCount)"
 done
 echo ""
 
 # ---------------------------------------------------------------- Chart helpers (QuickChart)
 # Mermaid xychart-beta can't do grouped bars / a legend and crowds 14 x-labels, so render proper
 # grouped bar charts via QuickChart (Chart.js) as images: https://quickchart.io/chart?c=<config>.
-OL_COLOR="#4e79a7"   # blue   = OpenLDAP
-DJ_COLOR="#f28e2b"   # orange = OpenDJ
 
 # JSON array of the OPS labels: ["ADD","SEARCH",...].
 labels_json() {
@@ -110,26 +115,15 @@
 echo "_Per-operation throughput is not charted: every op runs once per loop iteration, so each"
 echo "op's throughput just equals the loop rate. The meaningful throughput is the aggregate._"
 echo ""
-TP_CFG="{\"type\":\"bar\",\"data\":{\"labels\":[\"OpenLDAP\",\"OpenDJ\"],\"datasets\":[{\"label\":\"ops/s\",\"backgroundColor\":[\"$OL_COLOR\",\"$DJ_COLOR\"],\"data\":[${ol_tot_tp},${dj_tot_tp}]}]},\"options\":{\"legend\":{\"display\":false},\"title\":{\"display\":true,\"text\":\"Total throughput (ops/s)\"}}}"
+TP_CFG="{\"type\":\"bar\",\"data\":{\"labels\":[\"${A_NAME}\",\"${B_NAME}\"],\"datasets\":[{\"label\":\"ops/s\",\"backgroundColor\":[\"$A_COLOR\",\"$B_COLOR\"],\"data\":[${a_tot_tp},${b_tot_tp}]}]},\"options\":{\"legend\":{\"display\":false},\"title\":{\"display\":true,\"text\":\"Total throughput (ops/s)\"}}}"
 echo "![Total throughput (ops/s)]($(qc 500 320 "$TP_CFG"))"
 echo ""
 
 # ---------------------------------------------------------------- Latency chart (grouped bars)
 echo "### p99 latency per operation (ms, lower is better)"
 echo ""
-echo "_🟦 OpenLDAP · 🟧 OpenDJ — grouped bars per operation._"
+echo "_🟦 ${A_NAME} · 🟧 ${B_NAME} — grouped bars per operation._"
 echo ""
-LAT_CFG="{\"type\":\"bar\",\"data\":{\"labels\":$(labels_json),\"datasets\":[{\"label\":\"OpenLDAP\",\"backgroundColor\":\"$OL_COLOR\",\"data\":[$(vals mi "$OL_JSON" pct3ResTime)]},{\"label\":\"OpenDJ\",\"backgroundColor\":\"$DJ_COLOR\",\"data\":[$(vals mi "$DJ_JSON" pct3ResTime)]}]},\"options\":{\"title\":{\"display\":true,\"text\":\"p99 latency per operation (ms)\"}}}"
+LAT_CFG="{\"type\":\"bar\",\"data\":{\"labels\":$(labels_json),\"datasets\":[{\"label\":\"${A_NAME}\",\"backgroundColor\":\"$A_COLOR\",\"data\":[$(vals mi "$A_JSON" pct3ResTime)]},{\"label\":\"${B_NAME}\",\"backgroundColor\":\"$B_COLOR\",\"data\":[$(vals mi "$B_JSON" pct3ResTime)]}]},\"options\":{\"title\":{\"display\":true,\"text\":\"p99 latency per operation (ms)\"}}}"
 echo "![p99 latency per operation (ms)]($(qc 900 400 "$LAT_CFG"))"
 echo ""
-
-# ---------------------------------------------------------------- Caveats
-echo "### Notes"
-echo ""
-echo "- \`BIND\` is the measured **user authentication** (\`test=sbind\`, single bind/unbind on its"
-echo "  own connection) as \`cn=user_<n>,ou=People\` with the password set by \`MODIFY\`. The admin"
-echo "  connection bind (\`ADMIN_CONNECT\`) is cached once per thread and excluded from these results."
-echo "- MODIFY sends the password in cleartext; each server hashes it on write with the **same"
-echo "  scheme (SSHA, Salted SHA-1)** — OpenLDAP via the ppolicy hash-cleartext overlay, OpenDJ via"
-echo "  its Salted SHA-1 default scheme — so BIND authentication is compared on equal footing."
-echo "- Full interactive JMeter HTML dashboards are attached as the \`jmeter-reports\` artifact."
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index f64cdc7..4a27339 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -237,10 +237,24 @@
       # ---------------------------------------------------------------- Report
       - name: Build job summary
         run: |
+          # summary.sh is generic: <name> <statistics.json> <version> <image> per server.
           bash .github/benchmark/summary.sh \
-            openldap/statistics.json opendj/statistics.json \
-            "$OPENLDAP_VER" "$OPENDJ_VER" \
-            "$OPENLDAP_IMAGE" "$OPENDJ_IMAGE" >> "$GITHUB_STEP_SUMMARY"
+            "OpenLDAP" openldap/statistics.json "$OPENLDAP_VER" "$OPENLDAP_IMAGE" \
+            "OpenDJ"   opendj/statistics.json   "$OPENDJ_VER"   "$OPENDJ_IMAGE" \
+            >> "$GITHUB_STEP_SUMMARY"
+          # benchmark-specific notes (kept out of the reusable script)
+          {
+            echo ""
+            echo "### Notes"
+            echo ""
+            echo '- `BIND` is the measured **user authentication** (`test=sbind`, single bind/unbind on its'
+            echo '  own connection) as `mail=u_<n>,ou=People` with the password set by `MODIFY`. The admin'
+            echo '  connection bind (`ADMIN_CONNECT`) is cached once per thread and excluded from these results.'
+            echo '- `MODIFY` sends the password in cleartext; each server hashes it on write with the **same'
+            echo '  scheme (`{SSHA}`, Salted SHA-1)** — OpenLDAP via the ppolicy hash-cleartext overlay, OpenDJ'
+            echo '  via its Salted SHA-1 default scheme — so `BIND` authentication is compared on equal footing.'
+            echo '- Full interactive JMeter HTML dashboards are attached as the `jmeter-reports` artifact.'
+          } >> "$GITHUB_STEP_SUMMARY"
 
       - name: Upload JMeter reports
         if: always()

--
Gitblit v1.10.0