Benchmark: 200 threads default, p99 latency chart (linear)

Valera V Harseko

2 days ago 736c83dc1c71b19abc0554b7df10ec0dc117cc84

Benchmark: 200 threads default, p99 latency chart (linear)

- Raise the default concurrent thread count to 200 (was 128) in the workflow
  input and env fallback and in the JMX __P default.
- Latency chart now plots p99 instead of mean: tail latency is the more
  meaningful metric for the skewed LDAP latency distributions under load (mean
  hides the tail). The Y axis is linear (logarithmic scale removed).

 .github/benchmark/benchmark.jmx

@@ -33,7 +33,7 @@
          <boolProp name="LoopController.continue_forever">false</boolProp>
          <stringProp name="LoopController.loops">-1</stringProp>
        </elementProp>
        <stringProp name="ThreadGroup.num_threads">${__P(threads,128)}</stringProp>
        <stringProp name="ThreadGroup.num_threads">${__P(threads,200)}</stringProp>
        <stringProp name="ThreadGroup.ramp_time">${__P(rampup,0)}</stringProp>
        <boolProp name="ThreadGroup.scheduler">true</boolProp>
        <stringProp name="ThreadGroup.duration">${__P(duration,300)}</stringProp>

 .github/benchmark/summary.sh

@@ -115,13 +115,12 @@
echo ""

# ---------------------------------------------------------------- Latency chart (grouped bars)
echo "### Mean latency per operation (ms, log scale — lower is better)"
echo "### p99 latency per operation (ms, lower is better)"
echo ""
echo "_🟦 OpenLDAP · 🟧 OpenDJ — grouped bars per operation. Y axis is logarithmic so the small"
echo "values stay visible next to the large ones._"
echo "_🟦 OpenLDAP · 🟧 OpenDJ — grouped bars per operation._"
echo ""
LAT_CFG="{\"type\":\"bar\",\"data\":{\"labels\":$(labels_json),\"datasets\":[{\"label\":\"OpenLDAP\",\"backgroundColor\":\"$OL_COLOR\",\"data\":[$(vals m "$OL_JSON" meanResTime)]},{\"label\":\"OpenDJ\",\"backgroundColor\":\"$DJ_COLOR\",\"data\":[$(vals m "$DJ_JSON" meanResTime)]}]},\"options\":{\"title\":{\"display\":true,\"text\":\"Mean latency per operation (ms, log scale)\"},\"scales\":{\"yAxes\":[{\"type\":\"logarithmic\"}]}}}"
echo "![Mean latency per operation (ms)]($(qc 900 400 "$LAT_CFG"))"
LAT_CFG="{\"type\":\"bar\",\"data\":{\"labels\":$(labels_json),\"datasets\":[{\"label\":\"OpenLDAP\",\"backgroundColor\":\"$OL_COLOR\",\"data\":[$(vals mi "$OL_JSON" pct3ResTime)]},{\"label\":\"OpenDJ\",\"backgroundColor\":\"$DJ_COLOR\",\"data\":[$(vals mi "$DJ_JSON" pct3ResTime)]}]},\"options\":{\"title\":{\"display\":true,\"text\":\"p99 latency per operation (ms)\"}}}"
echo "![p99 latency per operation (ms)]($(qc 900 400 "$LAT_CFG"))"
echo ""

# ---------------------------------------------------------------- Caveats

 .github/workflows/benchmark.yml

@@ -31,7 +31,7 @@
        default: "openidentityplatform/opendj:latest"
      threads:
        description: "Concurrent JMeter threads"
        default: "128"
        default: "200"
      duration:
        description: "Test duration per server (seconds)"
        default: "300"
@@ -65,7 +65,7 @@
      # `${{ inputs.X || 'default' }}` so workflow_run (which carries no inputs) falls back.
      OPENLDAP_IMAGE: ${{ inputs.openldap_image || 'osixia/openldap:latest' }}
      OPENDJ_IMAGE: ${{ inputs.opendj_image || 'openidentityplatform/opendj:latest' }}
      THREADS: ${{ inputs.threads || '128' }}
      THREADS: ${{ inputs.threads || '200' }}
      DURATION: ${{ inputs.duration || '300' }}
      RAMPUP: ${{ inputs.rampup || '0' }}
      JMETER: ${{ inputs.jmeter_version || '5.6.3' }}