Benchmark: lighter default load, log-scale latency, per-server log artifacts

Valera V Harseko

2 days ago af83007d1fd2f906a8aefa3e7c33a1de1f471437

Benchmark: lighter default load, log-scale latency, per-server log artifacts

- Lower the default load to 128 threads / 300s (was 256 / 600), in the workflow
  inputs and env fallbacks and in the JMX __P defaults.
- Latency chart: switch the Y axis to logarithmic so the small per-operation
  values stay readable next to the large ones (e.g. OpenLDAP BIND).
- Capture server logs and upload them as two per-server artifacts. Each Stop step
  (if: always(), so logs are kept on failure too) now saves the container's
  `docker logs` to <server>/server.log and copies the in-container log directory
  (OpenDJ /opt/opendj/data/logs -> opendj/internal; OpenLDAP /var/log ->
  openldap/var-log). Uploaded as artifacts logs-opendj and logs-openldap.
- Set retention-days: 90 on all artifacts (jmeter-reports, logs-openldap,
  logs-opendj).

 .github/benchmark/benchmark.jmx

@@ -33,10 +33,10 @@
          <boolProp name="LoopController.continue_forever">false</boolProp>
          <stringProp name="LoopController.loops">-1</stringProp>
        </elementProp>
        <stringProp name="ThreadGroup.num_threads">${__P(threads,256)}</stringProp>
        <stringProp name="ThreadGroup.num_threads">${__P(threads,128)}</stringProp>
        <stringProp name="ThreadGroup.ramp_time">${__P(rampup,0)}</stringProp>
        <boolProp name="ThreadGroup.scheduler">true</boolProp>
        <stringProp name="ThreadGroup.duration">${__P(duration,600)}</stringProp>
        <stringProp name="ThreadGroup.duration">${__P(duration,300)}</stringProp>
        <stringProp name="ThreadGroup.delay"></stringProp>
      </ThreadGroup>
      <hashTree>

 .github/benchmark/summary.sh

@@ -115,11 +115,12 @@
echo ""

# ---------------------------------------------------------------- Latency chart (grouped bars)
echo "### Mean latency per operation (ms, lower is better)"
echo "### Mean latency per operation (ms, log scale — lower is better)"
echo ""
echo "_🟦 OpenLDAP · 🟧 OpenDJ — grouped bars per operation._"
echo "_🟦 OpenLDAP · 🟧 OpenDJ — grouped bars per operation. Y axis is logarithmic so the small"
echo "values stay visible next to the large ones._"
echo ""
LAT_CFG="{\"type\":\"bar\",\"data\":{\"labels\":$(labels_json),\"datasets\":[{\"label\":\"OpenLDAP\",\"backgroundColor\":\"$OL_COLOR\",\"data\":[$(vals m "$OL_JSON" meanResTime)]},{\"label\":\"OpenDJ\",\"backgroundColor\":\"$DJ_COLOR\",\"data\":[$(vals m "$DJ_JSON" meanResTime)]}]},\"options\":{\"title\":{\"display\":true,\"text\":\"Mean latency per operation (ms)\"}}}"
LAT_CFG="{\"type\":\"bar\",\"data\":{\"labels\":$(labels_json),\"datasets\":[{\"label\":\"OpenLDAP\",\"backgroundColor\":\"$OL_COLOR\",\"data\":[$(vals m "$OL_JSON" meanResTime)]},{\"label\":\"OpenDJ\",\"backgroundColor\":\"$DJ_COLOR\",\"data\":[$(vals m "$DJ_JSON" meanResTime)]}]},\"options\":{\"title\":{\"display\":true,\"text\":\"Mean latency per operation (ms, log scale)\"},\"scales\":{\"yAxes\":[{\"type\":\"logarithmic\"}]}}}"
echo "![Mean latency per operation (ms)]($(qc 900 400 "$LAT_CFG"))"
echo ""


 .github/workflows/benchmark.yml

@@ -31,10 +31,10 @@
        default: "openidentityplatform/opendj:latest"
      threads:
        description: "Concurrent JMeter threads"
        default: "256"
        default: "128"
      duration:
        description: "Test duration per server (seconds)"
        default: "600"
        default: "300"
      rampup:
        description: "Ramp-up period (seconds)"
        default: "0"
@@ -65,8 +65,8 @@
      # `${{ inputs.X || 'default' }}` so workflow_run (which carries no inputs) falls back.
      OPENLDAP_IMAGE: ${{ inputs.openldap_image || 'osixia/openldap:latest' }}
      OPENDJ_IMAGE: ${{ inputs.opendj_image || 'openidentityplatform/opendj:latest' }}
      THREADS: ${{ inputs.threads || '256' }}
      DURATION: ${{ inputs.duration || '600' }}
      THREADS: ${{ inputs.threads || '128' }}
      DURATION: ${{ inputs.duration || '300' }}
      RAMPUP: ${{ inputs.rampup || '0' }}
      JMETER: ${{ inputs.jmeter_version || '5.6.3' }}
      BENCHPW: benchPass1
@@ -159,8 +159,13 @@
            -l openldap.jtl -e -o openldap

      - name: Stop OpenLDAP
        if: always()
        run: |
          docker logs openldap 2>&1 | tail -n 100 || true
          mkdir -p logs/openldap
          docker logs openldap > logs/openldap/server.log 2>&1 || true
          # osixia mostly logs to stdout (captured above); grab the in-container /var/log too.
          docker cp openldap:/var/log logs/openldap/var-log 2>/dev/null || true
          tail -n 100 logs/openldap/server.log || true
          docker rm -f openldap || true

      - name: Start OpenDJ
@@ -220,8 +225,14 @@
            -l opendj.jtl -e -o opendj

      - name: Stop OpenDJ
        if: always()
        run: |
          docker logs opendj 2>&1 | tail -n 100 || true
          mkdir -p logs/opendj
          docker logs opendj > logs/opendj/server.log 2>&1 || true
          # OpenDJ's internal log directory (errors, access, replication, server.out, ...).
          docker cp opendj:/opt/opendj/data/logs logs/opendj/internal 2>/dev/null \
            || docker cp opendj:/opt/opendj/logs logs/opendj/internal 2>/dev/null || true
          tail -n 100 logs/opendj/server.log || true
          docker rm -f opendj || true

      # ---------------------------------------------------------------- Report
@@ -242,10 +253,22 @@
            opendj/
            *.jtl
          if-no-files-found: warn
          retention-days: 90

      - name: Container logs + cleanup
      - name: Upload OpenLDAP logs
        if: always()
        run: |
          docker logs openldap 2>&1 | tail -n 100 || true
          docker logs opendj 2>&1 | tail -n 100 || true
          docker rm -f openldap opendj || true
        uses: actions/upload-artifact@v7
        with:
          name: logs-openldap
          path: logs/openldap/
          if-no-files-found: warn
          retention-days: 90

      - name: Upload OpenDJ logs
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: logs-opendj
          path: logs/opendj/
          if-no-files-found: warn
          retention-days: 90