From 616d5f6c17161905dc9608029826dbf043a68e39 Mon Sep 17 00:00:00 2001
From: Valery Kharseko <vharseko@3a-systems.ru>
Date: Sun, 18 Oct 2026 08:09:20 +0000
Subject: [PATCH] [#1090] Check replication convergence under load next to the Docker tests, and run the Docker tests of both images after their builds (#1192)
---
.github/scripts/replication/assert-converged.sh | 229 +++++++
.github/scripts/replication/collect.sh | 102 +++
.github/scripts/replication/monitor.sh | 111 +++
.github/scripts/replication/topology-up.sh | 56 +
.github/scripts/replication/load.sh | 118 ++++
.github/scripts/replication/run.sh | 100 +++
.github/workflows/build.yml | 700 +++++++----------------
.github/scripts/replication/lib.sh | 158 +++++
.github/scripts/replication/changelog-gaps.py | 164 +++++
9 files changed, 1,253 insertions(+), 485 deletions(-)
diff --git a/.github/scripts/replication/assert-converged.sh b/.github/scripts/replication/assert-converged.sh
new file mode 100644
index 0000000..d3b8d85
--- /dev/null
+++ b/.github/scripts/replication/assert-converged.sh
@@ -0,0 +1,229 @@
+#
+#
+# The contents of this file are subject to the terms of the Common Development and
+# Distribution License (the License). You may not use this file except in compliance with the
+# License.
+#
+# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
+# specific language governing permission and limitations under the License.
+#
+# When distributing Covered Software, include this CDDL Header Notice in each file and include
+# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
+# Header, with the fields enclosed by brackets [] replaced by your own identifying
+# information: "Portions copyright [year] [name of copyright owner]".
+#
+# Copyright 2026 3A Systems, LLC.
+
+# The checks of the replication convergence run (#1090), sourced by run.sh: the replicas hold
+# the same data, compared on the data itself rather than on the replication counters.
+
+# Writes to the second directory server only, which the checks must report: the replication
+# repair control (ReplicationRepairRequestControl) applies an operation without replicating it
+inject_divergence() {
+ local n
+ n=$(echo $DS_NODES | cut -d' ' -f2)
+ printf 'dn: uid=user.0,ou=People,%s\nchangetype: modify\nreplace: description\ndescription: only on %s\n' "$BASE_DN" "$n" \
+ | docker exec -i "$n" /opt/opendj/bin/ldapmodify --noPropertiesFile --hostname localhost --port 1636 \
+ --useSsl --trustAll --bindDN "cn=Directory Manager" --bindPassword "$ROOT_PASSWORD" \
+ --control 1.3.6.1.4.1.26027.1.5.2 >/dev/null
+ # an entry of its own, which the searches of the other replicas do not return
+ printf 'dn: uid=only-on-%s,ou=People,%s\nobjectClass: inetOrgPerson\nuid: only-on-%s\ncn: Diverged\nsn: Diverged\n' "$n" "$BASE_DN" "$n" \
+ | docker exec -i "$n" /opt/opendj/bin/ldapmodify --noPropertiesFile --hostname localhost --port 1636 \
+ --useSsl --trustAll --bindDN "cn=Directory Manager" --bindPassword "$ROOT_PASSWORD" \
+ --control 1.3.6.1.4.1.26027.1.5.2 --defaultAdd >/dev/null
+}
+
+# An offline export and verify-index need the servers stopped; docker stop gives them the time
+# of a normal shutdown, so what they write on the way down is on their volumes
+stop_all() {
+ docker stop -t 120 $ALL_NODES >/dev/null
+}
+
+# Runs a tool of the image on the volume of a stopped server. run.sh of the image points the
+# installation at the volume (instance.loc) when the container starts, which a one-off
+# container has to do itself; $OUT is mounted on /out
+offline() { # <node> <tool> [<argument>...]
+ local node=$1 tool=$2
+ shift 2
+ docker run --rm -v "vol-$node:/opt/opendj/data" -v "$OUT:/out" --entrypoint sh "$IMAGE" -c \
+ 'echo /opt/opendj/data >/opt/opendj/instance.loc && exec /opt/opendj/bin/"$0" "$@"' "$tool" "$@"
+}
+
+assert_offline() {
+ local n first entries failed=
+ # the tools run as the user of the image, not as the one running this script
+ mkdir -p "$OUT/export" "$OUT/diff"
+ chmod 777 "$OUT/export" "$OUT/diff"
+ for n in $DS_NODES; do
+ # ds-sync-hist is the record of the changes each replica applied, ds-sync-state the position
+ # of each replica: both are its own
+ # --offline: a tools.properties with connection settings would turn the export into a task
+ # of a server that is not running
+ if ! offline $n export-ldif --offline --noPropertiesFile --backendID userRoot --ldifFile /out/export/$n.ldif \
+ --excludeAttribute ds-sync-hist --excludeAttribute ds-sync-state >"$OUT/export/$n.out" 2>&1; then
+ echo "::error::export-ldif failed on $n, see export/$n.out"
+ failed=1
+ continue
+ fi
+ # two exports that failed alike, or exported another backend, are as identical as two good ones
+ entries=$(grep -c '^dn: ' "$OUT/export/$n.ldif" || true)
+ if [ "${entries:-0}" -le "$SAMPLE_DATA" ]; then
+ echo "::error::the export of $n holds $entries entries, not more than the $SAMPLE_DATA seeded users"
+ failed=1
+ fi
+ offline $n verify-index --baseDN "$BASE_DN" --countErrors >"$OUT/diff/$n.verify-index.txt" 2>&1 \
+ || { echo "::error::verify-index found errors on $n, see diff/$n.verify-index.txt"; failed=1; }
+ done
+ first=${DS_NODES%% *}
+ for n in ${DS_NODES#* }; do
+ # ldifdiff exits with 1 when the files differ
+ offline $n ldifdiff --outputLDIF /out/diff/$n.ldif /out/export/$first.ldif /out/export/$n.ldif >/dev/null \
+ || { echo "::error::$n holds other data than $first, see diff/$n.ldif"; failed=1; }
+ done
+ assert_changelogs || failed=1
+ [ -z "$failed" ] || fail "the replicas do not hold the same data"
+}
+
+# The changes of every replica in the changelog of every replication server, one after the
+# other: a gap in the sequence numbers of a replica's changes during the load is a change that
+# reached no replication server. The data shows it only while no later change of the same
+# attribute overwrites it, and missing-changes never (#1195). docker cp streams the changelog
+# out of the stopped container; changelog-gaps.py reads it
+assert_changelogs() {
+ local n id code failed=
+ mkdir -p "$OUT/changelog"
+ for n in $RS_NODES; do
+ code=0
+ docker cp "$n:/opt/opendj/data/changelogDb" - \
+ | python3 -I "$HERE/changelog-gaps.py" "$BASE_DN" "$LOAD_START_MS" "$LOAD_END_MS" \
+ | awk 'FILENAME == ARGV[1] { name[$1] = $2; next }
+ $1 == "replica" { id = $2; sub(/:$/, "", id); if (id in name) $2 = id " (" name[id] "):" }
+ { print }' "$OUT/server-ids.txt" - >"$OUT/changelog/$n.txt" || code=$?
+ case $code in
+ 0) ;;
+ 1) echo "::error::changes of the load are missing from the changelog of $n, see changelog/$n.txt"; failed=1 ;;
+ *) echo "::error::the changelog of $n could not be read, see changelog/$n.txt"; failed=1; continue ;;
+ esac
+ # every directory server made changes in the load: a changelog whose records the script
+ # no longer reads - another layout, another message type - would otherwise show no gap
+ for id in $(awk '{ print $1 }' "$OUT/server-ids.txt"); do
+ grep -q "^replica $id (" "$OUT/changelog/$n.txt" \
+ || { echo "::error::the changelog of $n holds no change of the load by $(awk -v id=$id '$1 == id { print $2 }' "$OUT/server-ids.txt")"; failed=1; }
+ done
+ done
+ [ -z "$failed" ]
+}
+
+# Equality filters on an attribute the default setup indexes (sn), each matching fewer entries
+# than the index entry limit (4000 by default): the search goes through the index even for the
+# Directory Manager, so an entry missing from the index of one replica (#1021) is missing from
+# its answer, which an export of the entries does not show. A key that holds more entries than
+# the limit is not indexed, and the search then matches every entry by its filter -
+# (objectClass=inetOrgPerson) over the seeded users did - so unindexed_searches checks that
+# each search went through the index
+INDEXED_FILTERS="(sn=AddRate) (sn=Diverged)"
+
+# The searches of INDEXED_FILTERS a directory server ran without an index, or that its access
+# log does not hold, as errors. Every access log file is read: the load rotates them
+unindexed_searches() { # <node>
+ docker exec $1 sh -c 'grep -h " SEARCH " /opt/opendj/data/logs/access*; [ $? -le 1 ]' \
+ | awk -v node=$1 -v base="$BASE_DN" -v filters="$INDEXED_FILTERS" '
+ BEGIN { n = split(filters, f, " ") }
+ $3 == "SEARCH" && $4 == "REQ" && index($0, " base=\"" base "\" scope=sub ") {
+ for (i = 1; i <= n; i++) if (index($0, " filter=\"" f[i] "\" ")) req[$5 " " $6] = f[i]
+ }
+ $3 == "SEARCH" && $4 == "RES" && / unindexed( |$)/ { unindexed[$5 " " $6] = 1 }
+ END {
+ for (k in req) { seen[req[k]] = 1; if (k in unindexed) printf "::error::%s ran %s without an index\n", node, req[k] }
+ for (i = 1; i <= n; i++) if (!(f[i] in seen)) printf "::error::the access log of %s holds no search of %s\n", node, f[i]
+ }'
+}
+
+# The number of lines each server's errors log held once the topology was up: the joins log
+# replication errors of their own while the servers come up one after the other, and only what
+# follows is the run's
+mark_logs() {
+ local n
+ mkdir -p "$OUT/logs"
+ for n in $ALL_NODES; do
+ docker exec $n sh -c 'cat /opt/opendj/data/logs/errors | wc -l' >"$OUT/logs/$n.mark"
+ done
+}
+
+since_mark() { # <node>
+ docker exec $1 tail -n +$(($(cat "$OUT/logs/$1.mark") + 1)) /opt/opendj/data/logs/errors
+}
+
+# How often each directory server connected to a replication server again after the topology
+# came up, whatever made it: a switch the broker decided on (msgID 188, or 213 to spread the
+# load), a replication server that closed the connection (63), a connection that broke. Each
+# connection ends in msgID 62 (96 where the generation IDs differ), so a switch that went back
+# to the same replication server (#1215) counts as well. Changes made around a reconnection
+# under load were lost in #1195, so the summary shows how many a run went through
+record_reconnections() {
+ local n lines
+ for n in $DS_NODES; do
+ # a log that cannot be read leaves the node out, which the summary shows as -
+ lines=$(since_mark $n) || continue
+ echo "$n $(echo "$lines" | grep -E "msgID=(62|96) " | grep -cF "for domain \"$BASE_DN\"" || true)"
+ done >"$OUT/reconnections.txt"
+}
+
+# A directory server is connected to one replication server at a time, so the replication
+# servers reach each other only through their own connections: each one names every peer it is
+# connected to "Connected replication server RS(<id>) <host:port>" (ReplicationServerHandler)
+assert_rs_mesh() {
+ local rs want got failed=
+ want=$(($(echo $RS_NODES | wc -w) - 1))
+ for rs in $RS_NODES; do
+ got=$(admin_search $rs "$MONITOR_DOMAIN" sub "(cn=Connected replication server*)" 1.1 | grep -c '^dn:' || true)
+ [ "${got:-0}" -eq "$want" ] || { echo "::error::$rs is connected to ${got:-0} replication servers, not $want"; failed=1; }
+ done
+ [ -z "$failed" ]
+}
+
+# run.sh calls this in a condition, where neither errexit nor the ERR trap stops a command
+# that fails: every search and every read of a log checks its own status, since a search that
+# failed alike on every replica gives the same empty answer everywhere
+assert_online() {
+ local f i=0 n first out lines failed=
+ first=${DS_NODES%% *}
+ mkdir -p "$OUT/dns"
+ record_reconnections
+ for f in $INDEXED_FILTERS; do
+ i=$((i + 1))
+ for n in $DS_NODES; do
+ ldaps_search $n "$BASE_DN" sub "$f" 1.1 >"$OUT/dns/$i.$n.ldif" 2>&1 \
+ || { echo "::error::$f failed on $n, see dns/$i.$n.ldif"; failed=1; }
+ awk '/^dn: /{ print tolower($0) }' "$OUT/dns/$i.$n.ldif" | sort >"$OUT/dns/$i.$n.txt"
+ done
+ for n in ${DS_NODES#* }; do
+ cmp -s "$OUT/dns/$i.$first.txt" "$OUT/dns/$i.$n.txt" \
+ || { echo "::error::$f returns other entries on $n than on $first, see dns/$i.*"; failed=1; }
+ done
+ done
+ for n in $DS_NODES; do
+ if ! out=$(unindexed_searches $n); then
+ echo "::error::the access log of $n could not be read"; failed=1
+ elif [ -n "$out" ]; then
+ echo "$out"; failed=1
+ fi
+ done
+ for n in $DS_NODES; do
+ # the load makes no naming conflicts: each server adds under a uid prefix of its own
+ if ! out=$(ldaps_search $n "$BASE_DN" sub "(ds-sync-conflict=*)" 1.1); then
+ echo "::error::the ds-sync-conflict search failed on $n"; failed=1
+ elif grep -q '^dn:' <<<"$out"; then
+ echo "::error::$n holds ds-sync-conflict entries"; failed=1
+ fi
+ done
+ for n in $ALL_NODES; do
+ if ! lines=$(since_mark $n); then
+ echo "::error::the errors log of $n could not be read"; failed=1; continue
+ fi
+ lines=$(grep -E "category=SYNC severity=ERROR|SSLHandshakeException|bad_certificate" <<<"$lines" || true)
+ [ -z "$lines" ] || { printf '%s\n' "$lines" >"$OUT/logs/$n.errors"; echo "::error::$n logged replication errors, see logs/$n.errors"; failed=1; }
+ done
+ assert_rs_mesh || failed=1
+ [ -z "$failed" ] || fail "the replicas do not answer the same"
+}
diff --git a/.github/scripts/replication/changelog-gaps.py b/.github/scripts/replication/changelog-gaps.py
new file mode 100644
index 0000000..2412ead
--- /dev/null
+++ b/.github/scripts/replication/changelog-gaps.py
@@ -0,0 +1,164 @@
+#
+# The contents of this file are subject to the terms of the Common Development and
+# Distribution License (the License). You may not use this file except in compliance with the
+# License.
+#
+# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
+# specific language governing permission and limitations under the License.
+#
+# When distributing Covered Software, include this CDDL Header Notice in each file and include
+# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
+# Header, with the fields enclosed by brackets [] replaced by your own identifying
+# information: "Portions copyright [year] [name of copyright owner]".
+#
+# Copyright 2026 3A Systems, LLC.
+
+"""Finds the changes a replication server's changelog lacks (#1090, #1195).
+
+A directory server numbers the changes it makes one after the other: CSNGenerator.newCSN()
+adds one to the sequence number of the CSN for every change, whatever its time. So the
+changes of one replica in a changelog have no gap in their sequence numbers unless a change
+was never published - an operation that failed after it had its CSN, or one that was lost on
+its way to the replication servers. During the load of the convergence run no operation fails
+(load.sh checks that), so a gap there is a lost change: a change the data of the other
+replicas misses, while their ServerState and missing-changes say nothing, since both only
+know the newest CSN.
+
+Usage: changelog-gaps.py <base DN> <from ms> <to ms> [<changelogDb directory>]
+
+Without a directory it reads the changelogDb as a tar stream on its standard input, as
+"docker cp <container>:/opt/opendj/data/changelogDb -" writes it. It reports, for every
+replica of the domain of <base DN>, the changes whose CSN time is between <from ms> and
+<to ms>, and the sequence numbers missing between them. Exit code 1: a change is missing;
+2: the changelog cannot be read (no such domain, or it is encrypted).
+
+What it cannot tell. A change lost at either end of a replica's changes in the window - its
+last ones, or its first ones when no earlier change of the replica is in the changelog - leaves
+no gap between two changes that are there; the data of the other replicas shows such a change
+unless a later change of another server overwrites it. And not every sequence number that no
+record holds is a lost change: a directory server that goes offline (a restart, its domain
+disabled) numbers its ReplicaOfflineMsg like a change, and the changelog keeps only the newest
+of them, in the offline.state of the replica, not in a log file. The convergence run restarts
+no server after the topology is up; a run that did would read each restart in the window as a
+lost change.
+
+The file format is the one of FileReplicaDB: every log file is cut into blocks of 256 bytes,
+and every block after the first starts with a 4-byte offset (BlockLogWriter); in between, a
+record is its length in 4 bytes and the bytes of its UpdateMsg, whose header is the message
+type, the protocol version and the CSN as 28 hexadecimal characters (LDAPUpdateMsg).
+"""
+
+import os
+import re
+import sys
+import tarfile
+
+BLOCK_SIZE = 256
+SIZE_OF_BLOCK_OFFSET = 4
+# ReplicationMsg: MSG_TYPE_MODIFY, ADD, DELETE, MODIFYDN of the protocol versions after 1
+UPDATE_TYPES = {22: "modify", 23: "add", 24: "delete", 25: "modifydn"}
+DISK_ENCODING = 0xFF
+CSN = re.compile(rb"[0-9a-f]{28}")
+LOG_FILE = re.compile(r"(?:^|/)(\d+)\.dom/(\d+)\.server/[^/.][^/]*\.log$")
+
+
+def records(data):
+ """The UpdateMsg bytes of the records of one log file."""
+ stream = bytearray(data[:BLOCK_SIZE])
+ for start in range(BLOCK_SIZE, len(data), BLOCK_SIZE):
+ stream += data[start + SIZE_OF_BLOCK_OFFSET:start + BLOCK_SIZE]
+ pos = 0
+ while pos + 4 <= len(stream):
+ length = int.from_bytes(stream[pos:pos + 4], "big")
+ if length <= 0 or pos + 4 + length > len(stream):
+ raise ValueError("record of %d bytes at %d of %d" % (length, pos, len(stream)))
+ yield bytes(stream[pos + 4:pos + 4 + length])
+ pos += 4 + length
+
+
+def read_changelog(entries):
+ """(domains, changes): domain id by base DN, and for each (domain id, server id) a
+ list of (sequence number, time in ms, type)."""
+ domains = {}
+ changes = {}
+ for name, data in entries:
+ if os.path.basename(name) == "domains.state":
+ for line in data.decode("utf-8").splitlines():
+ if ":" in line:
+ domain_id, dn = line.split(":", 1)
+ domains[dn.strip().lower()] = domain_id
+ continue
+ match = LOG_FILE.search(name)
+ if not match:
+ continue
+ found = changes.setdefault((match.group(1), int(match.group(2))), [])
+ for message in records(data):
+ if message[0] == DISK_ENCODING:
+ raise EncryptedChangelog(name)
+ if message[0] not in UPDATE_TYPES or not CSN.fullmatch(message[2:30]):
+ continue
+ csn = message[2:30].decode("ascii")
+ found.append((int(csn[20:28], 16), int(csn[0:16], 16), UPDATE_TYPES[message[0]]))
+ return domains, changes
+
+
+class EncryptedChangelog(Exception):
+ pass
+
+
+def from_directory(root):
+ for directory, _, files in os.walk(root):
+ for file in files:
+ path = os.path.join(directory, file)
+ with open(path, "rb") as f:
+ yield os.path.relpath(path, root), f.read()
+
+
+def from_tar(stream):
+ with tarfile.open(fileobj=stream, mode="r|") as tar:
+ for member in tar:
+ if member.isfile():
+ yield member.name, tar.extractfile(member).read()
+
+
+def main(argv):
+ if len(argv) not in (4, 5):
+ sys.stderr.write(__doc__)
+ return 2
+ base_dn, start, end = argv[1].lower(), int(argv[2]), int(argv[3])
+ entries = from_directory(argv[4]) if len(argv) == 5 else from_tar(sys.stdin.buffer)
+ try:
+ domains, changes = read_changelog(entries)
+ except EncryptedChangelog as e:
+ print("the changelog is encrypted (%s): its changes cannot be read" % e)
+ return 2
+ except (tarfile.TarError, ValueError, OSError) as e:
+ print("the changelog cannot be read: %s" % e)
+ return 2
+ if base_dn not in domains:
+ print("no domain %s in the changelog, which holds %s" % (base_dn, sorted(domains)))
+ return 2
+ lost = False
+ for (domain_id, server_id), found in sorted(changes.items()):
+ if domain_id != domains[base_dn]:
+ continue
+ load = sorted((seqnum, time, kind) for seqnum, time, kind in found if start <= time <= end)
+ if not load:
+ continue
+ seqnums = [seqnum for seqnum, _, _ in load]
+ missing = []
+ for previous, following in zip(load, load[1:]):
+ missing += [(seqnum, previous, following) for seqnum in range(previous[0] + 1, following[0])]
+ print("replica %d: %d changes in the load, sequence numbers %d to %d, %d missing"
+ % (server_id, len(load), seqnums[0], seqnums[-1], len(missing)))
+ for seqnum, previous, following in missing[:20]:
+ print(" missing %d, between the %s %d (%d) and the %s %d (%d)"
+ % (seqnum, previous[2], previous[0], previous[1], following[2], following[0], following[1]))
+ if len(missing) > 20:
+ print(" and %d more" % (len(missing) - 20))
+ lost = lost or bool(missing)
+ return 1 if lost else 0
+
+
+if __name__ == "__main__":
+ sys.exit(main(sys.argv))
diff --git a/.github/scripts/replication/collect.sh b/.github/scripts/replication/collect.sh
new file mode 100644
index 0000000..8aee017
--- /dev/null
+++ b/.github/scripts/replication/collect.sh
@@ -0,0 +1,102 @@
+#
+#
+# The contents of this file are subject to the terms of the Common Development and
+# Distribution License (the License). You may not use this file except in compliance with the
+# License.
+#
+# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
+# specific language governing permission and limitations under the License.
+#
+# When distributing Covered Software, include this CDDL Header Notice in each file and include
+# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
+# Header, with the fields enclosed by brackets [] replaced by your own identifying
+# information: "Portions copyright [year] [name of copyright owner]".
+#
+# Copyright 2026 3A Systems, LLC.
+
+# What a replication convergence run (#1090) leaves behind, sourced by run.sh: the summary of
+# every run, and the state of every server of a failed one - what #534 could not get from its
+# reporter.
+
+# The server id each directory server has in the replication domain of BASE_DN, which the
+# samples name the servers by: "Directory server DS(<id>) <host:port>" (ReplicationMonitor)
+record_server_ids() {
+ local n
+ for n in $DS_NODES; do
+ admin_search $n "$MONITOR_DOMAIN" one "(objectClass=*)" 1.1 \
+ | sed -n 's/^dn: cn=Directory server DS(\([0-9]*\)).*/\1/p' | sed "s/\$/ $n/"
+ done >"$OUT/server-ids.txt"
+ # the changelog check looks for the changes of each of them
+ [ "$(wc -l <"$OUT/server-ids.txt")" -eq "$(echo $DS_NODES | wc -w)" ] \
+ || fail "the server id of a directory server is missing, see server-ids.txt"
+}
+
+# Every server's container log, logs/, config/ and changelogDb, packed into state.tgz; docker cp
+# reads a stopped or killed container as well. Nothing here may fail the run a second time
+collect() {
+ local n
+ for n in $ALL_NODES; do
+ docker inspect $n >/dev/null 2>&1 || continue
+ mkdir -p "$OUT/state/$n"
+ docker logs $n >"$OUT/state/$n/container.log" 2>&1 || true
+ docker cp $n:/opt/opendj/data/logs "$OUT/state/$n/" >/dev/null 2>&1 || true
+ docker cp $n:/opt/opendj/data/config "$OUT/state/$n/" >/dev/null 2>&1 || true
+ docker cp $n:/opt/opendj/data/changelogDb "$OUT/state/$n/" >/dev/null 2>&1 || true
+ done
+ [ ! -d "$OUT/state" ] || { tar czf "$OUT/state.tgz" -C "$OUT" state 2>/dev/null && rm -rf "$OUT/state"; } || true
+}
+
+# The summary of the run, into $OUT/summary.md and, in GitHub Actions, the job summary
+summarize() { # <result>
+ {
+ echo "### Replication convergence ($TOPOLOGY): $1"
+ echo
+ echo "| | |"
+ echo "|---|---|"
+ echo "| servers | $ALL_NODES |"
+ echo "| load | $DURATION s on each directory server, at most $LOAD_MOD_RATE modifications and $LOAD_ADD_RATE adds or deletes a second |"
+ echo "| seeded users | $SAMPLE_DATA |"
+ echo "| converged | ${CONVERGED_IN:--} s after the load |"
+ if [ -s "$OUT/samples.txt" ]; then
+ awk '{ t[$1] = 1 } END { for (s in t) { n++; if (!min || s < min) min = s; if (s > max) max = s }
+ printf "| samples of the monitor | %d, one every %d s |\n", n, (n > 1 ? (max - min) / (n - 1) : 0) }' "$OUT/samples.txt"
+ fi
+ if [ -s "$OUT/samples.txt" ] && [ -s "$OUT/server-ids.txt" ]; then
+ echo
+ echo "Peaks during the load, over every replication server (a restart resets the counters, so these are maxima of the samples, not sums), and the connections to a replication server again after the topology came up, whatever made them:"
+ echo
+ echo "| directory server | peak missing-changes | peak approximate-delay (s) | reconnections |"
+ echo "|---|---|---|---|"
+ awk 'FILENAME == ARGV[1] { reconnections[$1] = $2; next }
+ FILENAME == ARGV[2] { name[$1] = $2; next }
+ ($3 in name) { if ($4 > m[$3]) m[$3] = $4; if ($5 > d[$3]) d[$3] = $5; seen[$3] = 1 }
+ END { for (id in seen) printf "| %s | %d | %d | %s |\n", name[id], m[id], d[id], \
+ (name[id] in reconnections) ? reconnections[name[id]] : "-" }' \
+ "$([ -f "$OUT/reconnections.txt" ] && echo "$OUT/reconnections.txt" || echo /dev/null)" \
+ "$OUT/server-ids.txt" "$OUT/samples.txt" | sort
+ fi
+ if ls "$OUT"/changelog/*.txt >/dev/null 2>&1; then
+ echo
+ echo "Changes of the load missing from the changelog of each replication server (gaps in the sequence numbers of a replica; which ones in changelog/):"
+ echo
+ echo '```'
+ for f in "$OUT"/changelog/*.txt; do
+ grep -E ' [1-9][0-9]* missing$|cannot be read|encrypted|no domain' "$f" | sed "s/^/$(basename "$f" .txt): /"
+ done | grep . || echo "none"
+ echo '```'
+ fi
+ if ls "$OUT"/load/*.csv >/dev/null 2>&1; then
+ echo
+ echo "| client | average ops/s | intervals |"
+ echo "|---|---|---|"
+ for f in "$OUT"/load/*.csv; do
+ awk -F, -v c="$(basename "$f" .csv)" '$1 ~ /^[0-9.]+$/ { rows++; avg = $3 } END { printf "| %s | %s | %d |\n", c, avg, rows }' "$f"
+ done
+ fi
+ echo
+ echo '```'
+ sort -rn <<<"${PHASE_TIMES%$'\n'}"
+ echo '```'
+ } >"$OUT/summary.md" 2>/dev/null || true
+ [ -z "${GITHUB_STEP_SUMMARY:-}" ] || cat "$OUT/summary.md" >>"$GITHUB_STEP_SUMMARY" || true
+}
diff --git a/.github/scripts/replication/lib.sh b/.github/scripts/replication/lib.sh
new file mode 100644
index 0000000..5cdbd49
--- /dev/null
+++ b/.github/scripts/replication/lib.sh
@@ -0,0 +1,158 @@
+#
+#
+# The contents of this file are subject to the terms of the Common Development and
+# Distribution License (the License). You may not use this file except in compliance with the
+# License.
+#
+# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
+# specific language governing permission and limitations under the License.
+#
+# When distributing Covered Software, include this CDDL Header Notice in each file and include
+# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
+# Header, with the fields enclosed by brackets [] replaced by your own identifying
+# information: "Portions copyright [year] [name of copyright owner]".
+#
+# Copyright 2026 3A Systems, LLC.
+
+# Settings and helpers of the replication convergence run (#1090), sourced by run.sh. Every
+# setting comes from the environment, so that the same run goes in CI and on a laptop.
+
+TOPOLOGY=${TOPOLOGY:-rs534}
+# seconds of load
+DURATION=${DURATION:-120}
+# the most modifications, and adds or deletes, per second the load sends to each directory
+# server; a runner that cannot keep up sends fewer, which the summary shows
+LOAD_MOD_RATE=${LOAD_MOD_RATE:-423}
+LOAD_ADD_RATE=${LOAD_ADD_RATE:-85}
+# users the first directory server imports; the others get them from its initialize
+SAMPLE_DATA=${SAMPLE_DATA:-2000}
+# the heap of every server and every load client. Without a bound each JVM may take a quarter
+# of the memory of the host, fourteen of them on a runner of 16 GB, and the one run without it
+# lost both its runners (run 38694855431). No container memory limit: one would kill a server
+# in the middle of the run where the heap bound only slows it down
+SERVER_JAVA_ARGS=${SERVER_JAVA_ARGS:--server -Xmx1g}
+CLIENT_JAVA_ARGS=${CLIENT_JAVA_ARGS:--Xmx256m}
+# seconds the whole topology is given to come up: the joins take their turns one at a time
+UP_TIMEOUT=${UP_TIMEOUT:-3600}
+# seconds the topology is given to converge once the load stops
+QUIESCE_TIMEOUT=${QUIESCE_TIMEOUT:-900}
+# 1: write to one replica only, which the run must report (a self-test of its checks)
+INJECT_DIVERGENCE=${INJECT_DIVERGENCE:-0}
+# where the summary, the samples, the exports, the differences and a failed run's state go: a
+# directory that is new or empty
+OUT=${OUT:-$PWD/replication-out}
+# 1: leave the servers of a failed run up, to look at them; the next run removes them
+KEEP=${KEEP:-0}
+
+# awk reads 0.5 as 0 where the locale's decimal separator is a comma, and the checks sum the
+# rates the load tools print with a dot
+export LC_ALL=C
+
+NETWORK=convergence
+ROOT_PASSWORD=password
+BASE_DN="dc=example,dc=com"
+# the monitor of the replication domain of BASE_DN: cn=Replication,cn=monitor holds one per
+# replicated suffix (cn=admin data and cn=schema as well), and a server has another server id
+# in each of them
+MONITOR_DOMAIN="cn=$(echo "$BASE_DN" | tr '=,' '__'),cn=Replication,cn=monitor"
+
+case $TOPOLOGY in
+ # #534: directory servers without a replication server and replication servers without data.
+ # The directory servers come first in REPLICATION_PEERS: the first of them seeds
+ rs534)
+ DS_NODES="cv-d0 cv-d1 cv-d2 cv-d3"
+ RS_ONLY="cv-r0 cv-r1"
+ RS_NODES=$RS_ONLY
+ ;;
+ # every server holds the data and runs a replication server, which most deployments do
+ combined)
+ DS_NODES="cv-c0 cv-c1 cv-c2 cv-c3"
+ RS_ONLY=
+ RS_NODES=$DS_NODES
+ ;;
+ *)
+ echo "unknown TOPOLOGY $TOPOLOGY: rs534 or combined" >&2
+ exit 2
+ ;;
+esac
+ALL_NODES=$(echo $DS_NODES $RS_ONLY)
+PEERS=$(echo $ALL_NODES | tr ' ' ',')
+
+cleanup() {
+ local n
+ # the load clients of a run that failed before it removed them (load.sh)
+ docker rm -f $(docker ps -aq --filter "name=^load-") >/dev/null 2>&1 || true
+ docker rm -f $ALL_NODES >/dev/null 2>&1 || true
+ docker network rm $NETWORK >/dev/null 2>&1 || true
+ for n in $ALL_NODES; do
+ docker volume rm -f "vol-$n" >/dev/null 2>&1 || true
+ done
+}
+
+# Where the time goes: each phase logs when it starts, and the run ends - failed or not - with
+# the phases sorted by their duration
+PHASE=
+PHASE_START=$SECONDS
+PHASE_TIMES=
+phase() { # <name>
+ [ -z "$PHASE" ] || PHASE_TIMES+="$((SECONDS - PHASE_START)) s $PHASE"$'\n'
+ PHASE=$1
+ PHASE_START=$SECONDS
+ echo "=== [${SECONDS} s] $PHASE"
+}
+
+timings() {
+ phase "(end)"
+ echo "::group::time per phase, longest first"
+ sort -rn <<<"${PHASE_TIMES%$'\n'}"
+ echo "total: ${SECONDS} s"
+ echo "::endgroup::"
+}
+
+fail() {
+ echo "::error::$1"
+ false
+}
+
+wait_until() { # <seconds> <what> <command> [<argument>...]
+ local deadline=$((SECONDS + $1)) what=$2 start=$SECONDS
+ shift 2
+ until "$@"; do
+ [ "$SECONDS" -lt "$deadline" ] || fail "timed out waiting until $what"
+ sleep 2
+ done
+ [ $((SECONDS - start)) -lt 20 ] || echo " [$((SECONDS - start)) s] until $what"
+}
+
+is_healthy() { # <container>
+ [ "$(docker inspect --format='{{.State.Health.Status}}' "$1")" = healthy ]
+}
+
+# The background join of the image goes on once the container reports itself healthy: it adds
+# replication servers to the replication server lists while its peers register
+# (repair_replication_servers in join.sh). Ended once no bootstrap script runs any more, as
+# docker-test-replication.sh waits for it
+bootstrap_ended() { # <container>
+ ! docker exec "$1" sh -c "grep -qs -e 'bootstrap/[j]oin\.sh' -e 'bootstrap/[r]eplicate\.sh' /proc/[0-9]*/cmdline"
+}
+
+ldaps_search() { # <container> <base DN> <scope> <filter> [<attribute>...]
+ local container=$1 base=$2 scope=$3 filter=$4
+ shift 4
+ docker exec "$container" /opt/opendj/bin/ldapsearch --noPropertiesFile --hostname localhost --port 1636 \
+ --useSsl --trustAll --bindDN "cn=Directory Manager" --bindPassword "$ROOT_PASSWORD" \
+ --baseDN "$base" --searchScope "$scope" "$filter" "$@"
+}
+
+# cn=config and cn=monitor through the administration connector, which every role has
+admin_search() { # <container> <base DN> <scope> <filter> [<attribute>...]
+ local container=$1 base=$2 scope=$3 filter=$4
+ shift 4
+ docker exec "$container" /opt/opendj/bin/ldapsearch --noPropertiesFile --hostname localhost --port 4444 \
+ --useSsl --trustAll --bindDN "cn=Directory Manager" --bindPassword "$ROOT_PASSWORD" \
+ --baseDN "$base" --searchScope "$scope" "$filter" "$@"
+}
+
+runs_replication_server() { # <container>
+ admin_search "$1" "cn=config" sub "(objectClass=ds-cfg-replication-server)" 1.1 | grep "^dn:" >/dev/null
+}
diff --git a/.github/scripts/replication/load.sh b/.github/scripts/replication/load.sh
new file mode 100644
index 0000000..3e71283
--- /dev/null
+++ b/.github/scripts/replication/load.sh
@@ -0,0 +1,118 @@
+#
+#
+# The contents of this file are subject to the terms of the Common Development and
+# Distribution License (the License). You may not use this file except in compliance with the
+# License.
+#
+# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
+# specific language governing permission and limitations under the License.
+#
+# When distributing Covered Software, include this CDDL Header Notice in each file and include
+# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
+# Header, with the fields enclosed by brackets [] replaced by your own identifying
+# information: "Portions copyright [year] [name of copyright owner]".
+#
+# Copyright 2026 3A Systems, LLC.
+
+# The load of the replication convergence run (#1090), sourced by run.sh: every directory
+# server is written to at once, for DURATION seconds, by rate tools that run in containers of
+# the same image on the network of the topology. Only writes: a read sends nothing to the
+# replication servers.
+
+# <client> <tool> <argument>... : a container that runs one tool of the image against one server
+start_client() {
+ local name=$1 tool=$2
+ shift 2
+ docker run -d --name "$name" --network $NETWORK -e OPENDJ_JAVA_ARGS="$CLIENT_JAVA_ARGS" \
+ -v "$OUT/load:/load:ro" --entrypoint /opt/opendj/bin/$tool "$IMAGE" "$@" >/dev/null
+}
+
+CLIENTS=
+# seconds between two lines of the output of a rate tool
+STAT_INTERVAL=10
+
+# Plain LDAP: the run is after the changes the replication carries, and TLS between a client
+# and its server only takes CPU from them. --noRebind: every client keeps its connections open
+# and binds once, as an application does, rather than connecting and binding for every
+# operation, which would cost more than the operation itself. --connectTimeout: the clients
+# start at once next to the servers, and a runner that is short of CPU for a while accepts
+# their connections late
+run_load() {
+ local n c failed
+ mkdir -p "$OUT/load"
+ for n in $DS_NODES; do
+ # entries of its own on each server, so that the adds and deletes of two servers never meet
+ # on one DN: a naming conflict is not what this load is after
+ cat >"$OUT/load/addrate-$n.template" <<TEMPLATE
+define suffix=$BASE_DN
+
+branch: ou=People,[suffix]
+subordinateTemplate: person
+
+template: person
+rdnAttr: uid
+objectClass: top
+objectClass: person
+objectClass: organizationalPerson
+objectClass: inetOrgPerson
+uid: addrate-$n.<sequential:0>
+cn: AddRate {uid}
+sn: AddRate
+userPassword: password
+TEMPLATE
+ chmod 644 "$OUT/load/addrate-$n.template"
+ # the same entries from every server at once: concurrent modifications of one attribute of
+ # one entry, applied in the order of their change numbers on every replica (#139). The value
+ # starts with the name of the server: rand and randstr are seeded alike in every client, so
+ # all of them write the same values in the same order, and a change lost on its way would
+ # hide behind the same value written later by another server; the name also tells which
+ # server a value in a diff or a changelog comes from (#1206)
+ start_client load-mod-$n modrate --hostname $n --port 1389 --connectTimeout 120000 --noRebind \
+ --bindDN "cn=Directory Manager" --bindPassword "$ROOT_PASSWORD" --maxDuration "$DURATION" \
+ --statInterval $STAT_INTERVAL --numConnections 8 --numThreads 1 --scriptFriendly --targetThroughput "$LOAD_MOD_RATE" \
+ --targetDN "uid=user.%d,ou=People,$BASE_DN" --argument "rand(0,$((SAMPLE_DATA - 1)))" \
+ --argument "randstr(16)" "description:$n-%2\$s"
+ # adds, and deletes of the oldest once 100 of them are there: a slow runner gets to the deletes as well
+ start_client load-add-$n addrate --hostname $n --port 1389 --connectTimeout 120000 --noRebind \
+ --bindDN "cn=Directory Manager" --bindPassword "$ROOT_PASSWORD" --maxDuration "$DURATION" \
+ --statInterval $STAT_INTERVAL --numConnections 8 --numThreads 1 --scriptFriendly --targetThroughput "$LOAD_ADD_RATE" \
+ --deleteMode fifo --deleteSizeThreshold 100 "/load/addrate-$n.template"
+ CLIENTS="$CLIENTS load-mod-$n load-add-$n"
+ done
+ for c in $CLIENTS; do
+ docker wait $c >/dev/null
+ done
+ # every client's output first, so that one which failed does not hide how the others did
+ for c in $CLIENTS; do
+ docker logs $c >"$OUT/load/$c.csv" 2>&1
+ docker inspect -f '{{.State.ExitCode}} {{.State.OOMKilled}}' $c >"$OUT/load/$c.exit"
+ docker rm $c >/dev/null
+ done
+ failed=
+ for c in $CLIENTS; do
+ check_client $c || failed=1
+ done
+ [ -z "$failed" ] || fail "the load did not run as it should, see load/"
+}
+
+# A rate tool exits with 0 whatever its requests got back, and one that could not reach its
+# server at all exits at once. One whose connection breaks stops at once as well, exits with 0
+# too, leaves out the line of its last interval - the one that would count the error - and
+# prints the error to its error output, which goes to the same file. So the client has to have
+# done work, every interval of its CSV output has to report no errors (the "Errors/second"
+# column), the output has to cover the load - a line every STAT_INTERVAL seconds, of which the
+# last may be missing - and hold nothing but the CSV
+check_client() { # <client>
+ awk -F, -v c=$1 -v exit_state="$(cat "$OUT/load/$1.exit")" -v want=$((DURATION / STAT_INTERVAL - 1)) '
+ /^Time \(seconds\)/ { for (i = 1; i <= NF; i++) { if ($i == "Recent throughput (ops/second)") t = i; if ($i == "Errors/second") e = i }; next }
+ t && $1 ~ /^[0-9.]+$/ { rows++; ops += $t; errors += $e; last = $0; next }
+ NF { other++ }
+ END {
+ if (!rows || ops == 0) { printf "::error::%s did no work (exit code and OOM kill: %s)\n", c, exit_state; exit 1 }
+ if (errors > 0) { printf "::error::%s got errors back\n", c; exit 1 }
+ if (exit_state != "0 false") { printf "::error::%s ended with exit code and OOM kill %s\n", c, exit_state; exit 1 }
+ if (other) { printf "::error::%s printed %d lines besides its CSV output, see load/%s.csv\n", c, other, c; exit 1 }
+ if (rows < want) { printf "::error::%s stopped after %d of %d intervals\n", c, rows, want; exit 1 }
+ printf "%s: %d intervals, last: %s\n", c, rows, last
+ }' "$OUT/load/$1.csv"
+}
diff --git a/.github/scripts/replication/monitor.sh b/.github/scripts/replication/monitor.sh
new file mode 100644
index 0000000..907ad3f
--- /dev/null
+++ b/.github/scripts/replication/monitor.sh
@@ -0,0 +1,111 @@
+#
+#
+# The contents of this file are subject to the terms of the Common Development and
+# Distribution License (the License). You may not use this file except in compliance with the
+# License.
+#
+# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
+# specific language governing permission and limitations under the License.
+#
+# When distributing Covered Software, include this CDDL Header Notice in each file and include
+# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
+# Header, with the fields enclosed by brackets [] replaced by your own identifying
+# information: "Portions copyright [year] [name of copyright owner]".
+#
+# Copyright 2026 3A Systems, LLC.
+
+# The replication monitor of the convergence run (#1090), sourced by run.sh: how far behind
+# each directory server is, seen from each replication server, sampled during the load, and
+# the wait for the topology to converge once it stops.
+
+# One line per directory server known to each replication server - connected to it, or to a
+# peer of it - in the replication domain of BASE_DN: <epoch> <replication server> <server id>
+# <missing-changes> <approximate-delay>. Both kinds of entries are named "Connected directory
+# server DS(<id>) ..." (DataServerHandler, LightweightServerHandler)
+# The replication servers are asked at once: an ldapsearch starts a JVM, which takes tens of
+# seconds on a runner the load keeps busy, and asked one after the other they took a sample
+# every minute or so instead of every five seconds
+sample() {
+ local rs now dir pid pids=
+ now=$(date +%s)
+ dir=$(mktemp -d "$OUT/sample.XXXXXX")
+ for rs in $RS_NODES; do
+ admin_search $rs "$MONITOR_DOMAIN" sub "(cn=Connected directory server*)" \
+ server-id missing-changes approximate-delay \
+ | awk -v t=$now -v rs=$rs '
+ tolower($1) == "server-id:" { id = $2 }
+ tolower($1) == "missing-changes:" { m = $2 }
+ tolower($1) == "approximate-delay:" { d = $2 }
+ /^$/ && id != "" { print t, rs, id, m, d; id = "" }
+ END { if (id != "") print t, rs, id, m, d }' >"$dir/$rs" &
+ pids="$pids $!"
+ done
+ # a replication server that does not answer only leaves its lines out, which
+ # converged_once does not take for a zero
+ for pid in $pids; do
+ wait $pid || true
+ done
+ for rs in $RS_NODES; do
+ cat "$dir/$rs"
+ done
+ rm -rf "$dir"
+}
+
+# Converged once every replication server knows every directory server and none of them misses
+# a change. An entry that is not there is not a zero: right after a server connects again its
+# entry is missing until the replication server has heard of it
+converged_once() {
+ local s rs want
+ s=$(sample) || return 1
+ want=$(echo $DS_NODES | wc -w)
+ for rs in $RS_NODES; do
+ [ "$(echo "$s" | awk -v rs=$rs '$2 == rs { print $3 }' | sort -u | wc -l)" -eq "$want" ] || return 1
+ done
+ echo "$s" | awk '$4 != 0 { behind = 1 } END { exit behind }'
+}
+
+# A replication server computes missing-changes once a monitoring-period (MonitoringPublisher),
+# which this run leaves at its default, and a directory server its last computation does not
+# know reads as 0: samples that do not span a period may all read one computation, made while
+# changes were on their way
+MONITORING_PERIOD=60
+
+# Three converged samples in a row, the last one started more than a monitoring period (and
+# the publisher's 5 s latch) after the first ended. Sets CONVERGED_IN to the seconds until the
+# first of them
+wait_converged() {
+ local start=$SECONDS hits=0 first= last=
+ until [ $hits -ge 3 ] && [ $((last - first)) -gt $((MONITORING_PERIOD + 5)) ]; do
+ last=$SECONDS
+ if converged_once; then hits=$((hits + 1)); first=${first:-$SECONDS}; else hits=0; first=; fi
+ if [ $((SECONDS - start)) -ge "$QUIESCE_TIMEOUT" ]; then
+ echo "::group::the monitor when the wait timed out (epoch, replication server, server id, missing changes, delay)"
+ sample || true
+ echo "::endgroup::"
+ fail "the topology did not converge within $QUIESCE_TIMEOUT s"
+ fi
+ sleep 5
+ done
+ CONVERGED_IN=$((first - start))
+}
+
+# Samples every 5 s in the background into $OUT/samples.txt, for the peaks of the run; a
+# sample that fails - a server restarting - is only a gap
+sampler_start() {
+ ( while :; do sample >>"$OUT/samples.txt" 2>/dev/null || true; sleep 5; done ) &
+ SAMPLER=$!
+}
+
+sampler_stop() {
+ if [ -n "${SAMPLER:-}" ]; then
+ # waited for, so that the shell does not report the job it killed
+ kill $SAMPLER 2>/dev/null || true
+ wait $SAMPLER 2>/dev/null || true
+ # a sample takes tens of seconds under the load, so the kill nearly always cuts one short
+ # before it removes its directory, which would then be uploaded with the run. The kill
+ # reaches the sampler only, not the searches it started: one that opens its file in the
+ # directory after rm read it makes rm fail, which must not fail the run
+ rm -rf "$OUT"/sample.* || true
+ fi
+ SAMPLER=
+}
diff --git a/.github/scripts/replication/run.sh b/.github/scripts/replication/run.sh
new file mode 100755
index 0000000..306fed0
--- /dev/null
+++ b/.github/scripts/replication/run.sh
@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+#
+#
+# The contents of this file are subject to the terms of the Common Development and
+# Distribution License (the License). You may not use this file except in compliance with the
+# License.
+#
+# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
+# specific language governing permission and limitations under the License.
+#
+# When distributing Covered Software, include this CDDL Header Notice in each file and include
+# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
+# Header, with the fields enclosed by brackets [] replaced by your own identifying
+# information: "Portions copyright [year] [name of copyright owner]".
+#
+# Copyright 2026 3A Systems, LLC.
+
+# Writes to every directory server of a replication topology at once, then checks that the
+# replicas hold the same data (#1090). The Docker image jobs of build.yml hand their image to
+# the job that runs it; it runs as well on a laptop with Docker alone.
+#
+# Usage: run.sh <image>; the settings and their defaults are in lib.sh
+
+set -eE -o pipefail
+
+IMAGE=${1:?usage: $0 <image>}
+HERE=$(cd "$(dirname "$0")" && pwd)
+# changelog-gaps.py reads the changelogs of the replication servers
+command -v python3 >/dev/null || { echo "run.sh needs python3" >&2; exit 2; }
+. "$HERE/lib.sh"
+. "$HERE/topology-up.sh"
+. "$HERE/monitor.sh"
+. "$HERE/load.sh"
+. "$HERE/assert-converged.sh"
+. "$HERE/collect.sh"
+
+# the files of an earlier run would mix into this one's: the samples are appended to, and the
+# summary of a run that failed early would show what a run that got further left
+if [ -n "$(ls -A "$OUT" 2>/dev/null)" ]; then
+ echo "$OUT holds the files of an earlier run: remove it, or set OUT to another directory" >&2
+ exit 2
+fi
+mkdir -p "$OUT"
+# set -E hands the trap to every $(...) as well: there a failure only ends the subshell, and
+# the main shell decides. A run that fails before the online checks - a load check, a wait
+# that timed out, which a reconnection under load would explain - counts the reconnections
+# while the servers are still up
+trap 'code=$?; [ "$BASH_SUBSHELL" -eq 0 ] || exit $code; sampler_stop; [ -f "$OUT/reconnections.txt" ] || [ ! -f "$OUT/logs/${DS_NODES%% *}.mark" ] || record_reconnections || true; timings; collect; summarize failed; [ "$KEEP" = 1 ] || cleanup; exit $code' ERR
+cleanup
+
+phase "topology up ($TOPOLOGY)"
+topology_up
+for n in $RS_NODES; do
+ runs_replication_server "$n" || fail "$n runs no replication server"
+done
+for n in $DS_NODES; do
+ case " $RS_NODES " in
+ *" $n "*) ;;
+ *) ! runs_replication_server "$n" || fail "$n runs a replication server although its role is directory" ;;
+ esac
+done
+for n in $DS_NODES; do
+ users=$(ldaps_search "$n" "ou=People,$BASE_DN" one "(objectClass=inetOrgPerson)" 1.1 | grep -c '^dn:' || true)
+ [ "$users" -eq "$SAMPLE_DATA" ] || fail "$n holds $users of the $SAMPLE_DATA seeded users"
+done
+
+mark_logs
+record_server_ids
+
+phase "load ($DURATION s)"
+# the changes of the load: assert_changelogs looks for gaps among the changes made in between
+LOAD_START_MS=$(($(date +%s) * 1000))
+sampler_start
+run_load
+sampler_stop
+
+phase "converge"
+wait_converged
+echo "converged in $CONVERGED_IN s"
+LOAD_END_MS=$(($(date +%s) * 1000))
+
+if [ "$INJECT_DIVERGENCE" = 1 ]; then
+ phase "inject a divergence"
+ inject_divergence
+fi
+
+# the offline checks run after online checks that failed as well: they tell which changes are
+# missing where
+phase "online checks"
+checks=
+assert_online || checks="the online checks"
+
+phase "offline checks"
+stop_all
+assert_offline || checks="${checks:+$checks and }the offline checks"
+[ -z "$checks" ] || fail "$checks failed"
+
+timings
+summarize passed
+cleanup
diff --git a/.github/scripts/replication/topology-up.sh b/.github/scripts/replication/topology-up.sh
new file mode 100644
index 0000000..db0a29d
--- /dev/null
+++ b/.github/scripts/replication/topology-up.sh
@@ -0,0 +1,56 @@
+#
+#
+# The contents of this file are subject to the terms of the Common Development and
+# Distribution License (the License). You may not use this file except in compliance with the
+# License.
+#
+# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
+# specific language governing permission and limitations under the License.
+#
+# When distributing Covered Software, include this CDDL Header Notice in each file and include
+# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
+# Header, with the fields enclosed by brackets [] replaced by your own identifying
+# information: "Portions copyright [year] [name of copyright owner]".
+#
+# Copyright 2026 3A Systems, LLC.
+
+# Starts the topology of the replication convergence run (#1090), sourced by run.sh. Every
+# server joins through the background join of the image (OPENDJ_REPLICATION_TYPE=simple) in
+# its REPLICATION_ROLE, as a StatefulSet would start it.
+
+start_server() { # <name> <role> [<docker run option>...]
+ local name=$1 role=$2
+ shift 2
+ docker volume create "vol-$name" >/dev/null
+ docker run -d --network $NETWORK --name "$name" --hostname "$name" \
+ -v "vol-$name:/opt/opendj/data" \
+ -e ROOT_PASSWORD="$ROOT_PASSWORD" -e ADD_BASE_ENTRY="--addBaseEntry" \
+ -e OPENDJ_JAVA_ARGS="$SERVER_JAVA_ARGS" \
+ -e OPENDJ_REPLICATION_TYPE=simple -e REPLICATION_ROLE="$role" -e REPLICATION_PEERS="$PEERS" \
+ -e REPLICATION_RETRY_INTERVAL=3 \
+ "$@" "$IMAGE" >/dev/null
+}
+
+topology_up() {
+ local n deadline role=directory first=${DS_NODES%% *}
+ [ "$TOPOLOGY" != combined ] || role=combined
+ docker network create $NETWORK >/dev/null
+ # only the first server imports the users; the others are initialized from it when they join
+ start_server "$first" $role -e SAMPLE_DATA="$SAMPLE_DATA"
+ for n in ${DS_NODES#* }; do
+ start_server "$n" $role
+ done
+ for n in $RS_ONLY; do
+ start_server "$n" replication
+ done
+ # one deadline for all of them: the joins take their turns, so the last server may wait for
+ # every other one's
+ deadline=$((SECONDS + UP_TIMEOUT))
+ for n in $ALL_NODES; do
+ wait_until $((deadline > SECONDS ? deadline - SECONDS : 1)) "$n is healthy" is_healthy "$n"
+ done
+ # what a join still changes would otherwise fall into the errors and reconnections of the run
+ for n in $ALL_NODES; do
+ wait_until $((deadline > SECONDS ? deadline - SECONDS : 1)) "the join of $n ends" bootstrap_ended "$n"
+ done
+}
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index cb8614f..210950a 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -468,27 +468,25 @@
opendj-rest2ldap-servlet/target/*.war
build-docker:
needs: build-maven
+ # builds the image for every platform and hands its linux/amd64 image to docker-test (and
+ # docker-convergence): the registry service below is gone with this job
+ outputs:
+ release_version: ${{ steps.version.outputs.release_version }}
runs-on: 'ubuntu-latest'
permissions:
contents: read
- security-events: write
services:
registry:
image: registry:3
ports:
- 5000:5000
steps:
- # .github/scripts holds the replication test that both image jobs run
- - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- with:
- sparse-checkout: |
- .github/benchmark
- .github/scripts
- name: Download artifacts
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
name: ubuntu-latest-11
- name: Get latest release version
+ id: version
shell: bash
env:
# An anonymous call shares the 60-an-hour budget of the runner's address with every
@@ -499,6 +497,7 @@
export git_version_last="$(curl -i -o - --silent -H "Authorization: Bearer $GH_TOKEN" https://api.github.com/repos/OpenIdentityPlatform/OpenDJ/releases/latest | grep -m1 "\"name\"" | cut -d\" -f4)" ; echo "last release: $git_version_last"
[ -n "$git_version_last" ] || { echo "::error::The latest release of OpenIdentityPlatform/OpenDJ could not be read, so there is no version to tag the image with"; exit 1; }
echo "release_version=$git_version_last" >> $GITHUB_ENV
+ echo "release_version=$git_version_last" >> $GITHUB_OUTPUT
echo "image_repository=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV
- name: Docker meta
id: meta
@@ -531,11 +530,163 @@
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
+ - name: Save the linux/amd64 image for the test jobs
+ # the layers come from the build above, so this only exports them
+ uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
+ with:
+ context: ./opendj-packages/opendj-docker
+ file: ./opendj-packages/opendj-docker/Dockerfile
+ build-args: |
+ VERSION=${{ env.release_version }}
+ platforms: linux/amd64
+ tags: ${{ steps.meta.outputs.tags }}
+ outputs: type=docker,dest=${{ runner.temp }}/image.tar
+ - name: Upload the linux/amd64 image
+ uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+ with:
+ name: docker-image-default
+ path: ${{ runner.temp }}/image.tar
+ # about 250 MB. "Re-run failed jobs" does not run this job again and downloads the
+ # image it uploaded: within 3 days of the run; later only "Re-run all jobs" works
+ retention-days: 3
+
+ build-docker-alpine:
+ needs: build-maven
+ # builds the image for every platform and hands its linux/amd64 image to docker-test (and
+ # docker-convergence): the registry service below is gone with this job
+ outputs:
+ release_version: ${{ steps.version.outputs.release_version }}
+ runs-on: 'ubuntu-latest'
+ permissions:
+ contents: read
+ services:
+ registry:
+ image: registry:3
+ ports:
+ - 5000:5000
+ steps:
+ - name: Download artifacts
+ uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+ with:
+ name: ubuntu-latest-11
+ - name: Get latest release version
+ id: version
+ shell: bash
+ env:
+ # An anonymous call shares the 60-an-hour budget of the runner's address with every
+ # other job there; once it is spent the API answers with no release, the version comes
+ # out empty and buildx refuses the untagged push (run 34854564649).
+ GH_TOKEN: ${{ github.token }}
+ run: |
+ export git_version_last="$(curl -i -o - --silent -H "Authorization: Bearer $GH_TOKEN" https://api.github.com/repos/OpenIdentityPlatform/OpenDJ/releases/latest | grep -m1 "\"name\"" | cut -d\" -f4)" ; echo "last release: $git_version_last"
+ [ -n "$git_version_last" ] || { echo "::error::The latest release of OpenIdentityPlatform/OpenDJ could not be read, so there is no version to tag the image with"; exit 1; }
+ echo "release_version=$git_version_last" >> $GITHUB_ENV
+ echo "release_version=$git_version_last" >> $GITHUB_OUTPUT
+ echo "image_repository=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV
+ - name: Docker meta
+ id: meta
+ uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
+ with:
+ images: |
+ localhost:5000/${{ github.repository }}
+ tags: |
+ type=raw,value=alpine
+ type=raw,value=${{ env.release_version }}-alpine
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
+ with:
+ driver-opts: network=host
+ - name: Prepare Dockerfile
+ shell: bash
+ run: |
+ unzip -d ./opendj-packages/opendj-docker ./opendj-packages/opendj-docker/target/Dockerfile.zip
+ cp ./opendj-server-legacy/target/package/opendj-*.zip ./opendj-packages/opendj-docker
+ sed -i -E '/^#COPY opendj/s/^#//' ./opendj-packages/opendj-docker/Dockerfile-alpine
+ - name: Build image
+ uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
+ with:
+ context: ./opendj-packages/opendj-docker
+ file: ./opendj-packages/opendj-docker/Dockerfile-alpine
+ build-args: |
+ VERSION=${{ env.release_version }}
+ platforms: linux/amd64, linux/arm64/8, linux/s390x, linux/ppc64le, linux/386, linux/riscv64
+ push: true
+ tags: ${{ steps.meta.outputs.tags }}
+ labels: ${{ steps.meta.outputs.labels }}
+ - name: Save the linux/amd64 image for the test jobs
+ # the layers come from the build above, so this only exports them
+ uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
+ with:
+ context: ./opendj-packages/opendj-docker
+ file: ./opendj-packages/opendj-docker/Dockerfile-alpine
+ build-args: |
+ VERSION=${{ env.release_version }}
+ platforms: linux/amd64
+ tags: ${{ steps.meta.outputs.tags }}
+ outputs: type=docker,dest=${{ runner.temp }}/image.tar
+ - name: Upload the linux/amd64 image
+ uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+ with:
+ name: docker-image-alpine
+ path: ${{ runner.temp }}/image.tar
+ # see the image of build-docker: a re-run of the failed jobs within 3 days
+ retention-days: 3
+
+ docker-test:
+ # the tests of both images, each on the linux/amd64 image its build job saved
+ name: docker-test (${{ matrix.flavor }})
+ needs: [build-docker, build-docker-alpine]
+ # a matrix row cannot need its own build job only, and a job-level if cannot read the
+ # matrix: the tests of one image run when the build of the other failed, the row of the
+ # image that was not built fails at its download, and re-running that build re-runs both rows
+ if: ${{ !cancelled() && (needs.build-docker.result == 'success' || needs.build-docker-alpine.result == 'success') }}
+ runs-on: 'ubuntu-latest'
+ permissions:
+ contents: read
+ security-events: write
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - flavor: default
+ suffix: ""
+ trivy_category: trivy-build-default
+ build_label: Build
+ release_label: Release
+ release_image: openidentityplatform/opendj:latest
+ - flavor: alpine
+ suffix: "-alpine"
+ trivy_category: trivy-build-alpine
+ build_label: Build-alpine
+ release_label: Release-alpine
+ release_image: openidentityplatform/opendj:alpine
+ env:
+ release_version: ${{ matrix.flavor == 'alpine' && needs.build-docker-alpine.outputs.release_version || needs.build-docker.outputs.release_version }}
+ steps:
+ # .github/scripts holds the replication test of both images
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ sparse-checkout: |
+ .github/benchmark
+ .github/scripts
+ - name: Download the image
+ uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+ with:
+ name: docker-image-${{ matrix.flavor }}
+ path: ${{ runner.temp }}/image
+ - name: Load the image
+ # under the tag its build job pushed to its registry, which every step below names
+ shell: bash
+ run: |
+ docker load -i "$RUNNER_TEMP/image/image.tar"
+ echo "image_repository=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV
- name: Docker test
shell: bash
run: |
trap 'code=$?; echo "::group::container logs (test)"; docker logs test 2>&1 || true; echo "::endgroup::"; exit $code' ERR
- docker run -it -d --memory="512m" --name=test localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}
+ docker run -it -d --memory="512m" --name=test localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}${{ matrix.suffix }}
# the bootstrap's server is down before the marker the health check waits for is
# written: the stop has logged its record by the time the marker is there
timeout 3m bash -c 'until docker exec test sh -c "test -f \"\$BOOTSTRAP_COMPLETE\"" 2>/dev/null; do sleep 0.2; done'
@@ -565,7 +716,7 @@
shell: bash
run: |
trap 'code=$?; echo "::group::container logs (test_custom)"; docker logs test_custom 2>&1 || true; echo "::endgroup::"; exit $code' ERR
- docker run --rm -it -d --memory="512m" -e ADD_BASE_ENTRY="--addBaseEntry" -e ROOT_PASSWORD=custom_password --name=test_custom localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}
+ docker run --rm -it -d --memory="512m" -e ADD_BASE_ENTRY="--addBaseEntry" -e ROOT_PASSWORD=custom_password --name=test_custom localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}${{ matrix.suffix }}
timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_custom | grep -q \"healthy\"; do sleep 10; done'
docker exec test_custom 'sh' '-c' '/opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword custom_password --useSsl --trustAll --baseDN "dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1'
docker kill test_custom
@@ -575,7 +726,7 @@
shell: bash
run: |
trap 'code=$?; echo "::group::container logs (test_base_dn)"; docker logs test_base_dn 2>&1 || true; echo "::endgroup::"; exit $code' ERR
- docker run --rm -it -d --memory="512m" -e ADD_BASE_ENTRY="--addBaseEntry" -e BASE_DN="o=My Company,c=US" --name=test_base_dn localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}
+ docker run --rm -it -d --memory="512m" -e ADD_BASE_ENTRY="--addBaseEntry" -e BASE_DN="o=My Company,c=US" --name=test_base_dn localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}${{ matrix.suffix }}
timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_base_dn | grep -q \"healthy\"; do sleep 10; done'
docker exec test_base_dn 'sh' '-c' '/opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword password --useSsl --trustAll --baseDN "o=My Company,c=US" --searchScope base "(objectClass=*)" 1.1 | grep -qx "dn: o=My Company,c=US"'
docker kill test_base_dn
@@ -585,7 +736,7 @@
run: |
trap 'code=$?; echo "::group::container logs (test_uid)"; docker logs test_uid 2>&1 || true; echo "::endgroup::"; exit $code' ERR
uid=$((1000000000 + RANDOM))
- docker run --rm -it -d --memory="512m" --user $uid:0 -e ADD_BASE_ENTRY="--addBaseEntry" --name=test_uid localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}
+ docker run --rm -it -d --memory="512m" --user $uid:0 -e ADD_BASE_ENTRY="--addBaseEntry" --name=test_uid localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}${{ matrix.suffix }}
timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_uid | grep -q \"healthy\"; do sleep 10; done'
docker exec test_uid 'sh' '-c' "test \"\$(stat -c %u /opt/opendj/instance.loc)\" = $uid"
docker exec test_uid 'sh' '-c' '/opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword password --useSsl --trustAll --baseDN "dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1'
@@ -597,7 +748,7 @@
# the background join of OPENDJ_REPLICATION_TYPE=simple and the one-shot sdsr path, the same
# scenarios for both images
shell: bash
- run: .github/scripts/docker-test-replication.sh "localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}"
+ run: .github/scripts/docker-test-replication.sh "localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}${{ matrix.suffix }}"
- name: Docker test secret volume
# a keystore mounted at SECRET_VOLUME is what LDAPS serves from the first start on, a
# renewed one - a new password included - is copied while the server runs and served
@@ -606,7 +757,7 @@
# instance already there, and SECRET_VOLUME_REFRESH=0 runs no watcher
shell: bash
env:
- IMAGE: localhost:5000/${{ env.image_repository }}:${{ env.release_version }}
+ IMAGE: localhost:5000/${{ env.image_repository }}:${{ env.release_version }}${{ matrix.suffix }}
run: |
set -E
trap 'code=$?; echo "::group::container logs (test_secret)"; docker logs test_secret 2>&1 || true; echo "::endgroup::"; exit $code' ERR
@@ -685,7 +836,7 @@
- name: Docker test bootstrap LDIFs
shell: bash
run: |
- IMAGE=localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}
+ IMAGE=localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}${{ matrix.suffix }}
trap 'code=$?; for c in test_bootstrap_lf test_bootstrap_cr test_bootstrap; do echo "::group::container logs ($c)"; docker logs $c 2>&1 || true; echo "::endgroup::"; done; exit $code' ERR
# a root password with a line break is refused: the tools read only the first line of the password file
docker run -d --memory="512m" -e ROOT_PASSWORD=$'ab\ncd' --name=test_bootstrap_lf "$IMAGE"
@@ -747,7 +898,7 @@
set -o errtrace
# the containers are run without --rm, so that one whose bootstrap failed is still there for the trap to print
trap 'code=$?; for c in test_health test_health_bind; do echo "::group::container logs ($c)"; docker logs $c 2>&1 || true; docker inspect --format="{{json .State.Health}}" $c 2>&1 || true; echo "::endgroup::"; done; docker rm -f test_health test_health_bind >/dev/null 2>&1 || true; exit $code' ERR
- IMAGE=localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}
+ IMAGE=localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}${{ matrix.suffix }}
# a failed probe leaves a failing streak until the next probe passes, 5 s later at the
# earliest, so a container found "healthy 0" every 2 s for 45 s passed every probe since
stays_healthy() {
@@ -798,7 +949,7 @@
# configuration must not report healthy either (#1182)
shell: bash
run: |
- IMAGE=localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}
+ IMAGE=localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}${{ matrix.suffix }}
MEMORY=512m
trap 'code=$?; echo "::group::container logs (test_failed)"; docker logs test_failed 2>&1 || true; docker inspect --format="{{json .State.Health}}" test_failed 2>&1 || true; echo "::endgroup::"; docker rm -f test_failed >/dev/null 2>&1 || true; docker volume rm test_failed_data >/dev/null 2>&1 || true; exit $code' ERR
# an unknown backend type fails create-backend once setup has written data/config
@@ -826,8 +977,8 @@
# upgrade that did not complete keeps it unhealthy on later starts too (#1185)
shell: bash
run: |
- IMAGE=localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}
- RELEASED=openidentityplatform/opendj:${{ env.release_version }}
+ IMAGE=localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}${{ matrix.suffix }}
+ RELEASED=openidentityplatform/opendj:${{ env.release_version }}${{ matrix.suffix }}
MEMORY=512m
trap 'code=$?; echo "::group::container logs (test_upgrade)"; docker logs test_upgrade 2>&1 || true; docker inspect --format="{{json .State.Health}}" test_upgrade 2>&1 || true; echo "::endgroup::"; docker rm -f test_upgrade >/dev/null 2>&1 || true; docker volume rm test_upgrade_data >/dev/null 2>&1 || true; exit $code' ERR
search() { docker exec test_upgrade /opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --useSsl --trustAll --bindDN "cn=Directory Manager" --bindPassword password "$@"; }
@@ -896,7 +1047,7 @@
# evicting the m2-repository caches out of the repo's 10GB actions-cache quota
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
with:
- image-ref: localhost:5000/${{ env.image_repository }}:${{ env.release_version }}
+ image-ref: localhost:5000/${{ env.image_repository }}:${{ env.release_version }}${{ matrix.suffix }}
format: sarif
output: trivy-results.sarif
severity: CRITICAL,HIGH
@@ -911,20 +1062,22 @@
with:
sarif_file: trivy-results.sarif
# distinct from the docker-scan.yml categories, which track the published images
- category: trivy-build-default
+ category: ${{ matrix.trivy_category }}
- name: Cache JMeter
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ~/jmeter
key: jmeter-5.6.3
- name: Benchmark PDB vs JE
+ # the default image only, as before
+ if: matrix.flavor == 'default'
shell: bash
run: |
A_BACKEND=je B_BACKEND=pdb THREADS=200 DURATION=150 bash .github/benchmark/compare-opendj.sh \
- "JE" localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }} \
- "PDB" localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}
+ "JE" localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}${{ matrix.suffix }} \
+ "PDB" localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}${{ matrix.suffix }}
- name: Upload benchmark artifacts (PDB vs JE)
- if: always()
+ if: always() && matrix.flavor == 'default'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: benchmark-pdb-vs-je
@@ -935,18 +1088,18 @@
*.docker.log
*.jmeter.out
if-no-files-found: warn
- retention-days: 3
- - name: Benchmark Build vs Release
+ retention-days: 5
+ - name: Benchmark ${{ matrix.build_label }} vs ${{ matrix.release_label }}
shell: bash
run: |
THREADS=200 DURATION=150 bash .github/benchmark/compare-opendj.sh \
- "Build" localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }} \
- "Release" openidentityplatform/opendj:latest
+ "${{ matrix.build_label }}" localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}${{ matrix.suffix }} \
+ "${{ matrix.release_label }}" ${{ matrix.release_image }}
- name: Upload benchmark artifacts
if: always()
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
- name: benchmark-build-vs-release
+ name: benchmark-build-vs-release${{ matrix.suffix }}
path: |
a/
b/
@@ -954,479 +1107,56 @@
*.docker.log
*.jmeter.out
if-no-files-found: warn
- retention-days: 3
+ retention-days: 5
- build-docker-alpine:
- needs: build-maven
+ docker-convergence:
+ # #1090: writes to every directory server of a replication topology at once, then checks
+ # that the replicas hold the same data; next to docker-test, on the image of build-docker
+ needs: build-docker
+ # a red row turns the Build run red, and deploy.yml then skips the snapshot deploy and the
+ # publishing of the docs of a push to master: rs534 fails until #1206 is fixed and can fail
+ # until #1215 is, combined can fail until #1195 is
runs-on: 'ubuntu-latest'
permissions:
contents: read
- security-events: write
- services:
- registry:
- image: registry:3
- ports:
- - 5000:5000
+ strategy:
+ fail-fast: false
+ matrix:
+ topology: [rs534, combined]
steps:
- # .github/scripts holds the replication test that both image jobs run
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
sparse-checkout: |
- .github/benchmark
- .github/scripts
- - name: Download artifacts
+ .github/scripts/replication
+ - name: Download the image
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
- name: ubuntu-latest-11
- - name: Get latest release version
+ name: docker-image-default
+ path: ${{ runner.temp }}/image
+ - name: Load the image
+ shell: bash
+ run: docker load -i "$RUNNER_TEMP/image/image.tar"
+ - name: Convergence run (${{ matrix.topology }})
shell: bash
env:
- # An anonymous call shares the 60-an-hour budget of the runner's address with every
- # other job there; once it is spent the API answers with no release, the version comes
- # out empty and buildx refuses the untagged push (run 34854564649).
- GH_TOKEN: ${{ github.token }}
- run: |
- export git_version_last="$(curl -i -o - --silent -H "Authorization: Bearer $GH_TOKEN" https://api.github.com/repos/OpenIdentityPlatform/OpenDJ/releases/latest | grep -m1 "\"name\"" | cut -d\" -f4)" ; echo "last release: $git_version_last"
- [ -n "$git_version_last" ] || { echo "::error::The latest release of OpenIdentityPlatform/OpenDJ could not be read, so there is no version to tag the image with"; exit 1; }
- echo "release_version=$git_version_last" >> $GITHUB_ENV
- echo "image_repository=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV
- - name: Docker meta
- id: meta
- uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
- with:
- images: |
- localhost:5000/${{ github.repository }}
- tags: |
- type=raw,value=alpine
- type=raw,value=${{ env.release_version }}-alpine
- - name: Set up QEMU
- uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0
- - name: Set up Docker Buildx
- uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
- with:
- driver-opts: network=host
- - name: Prepare Dockerfile
- shell: bash
- run: |
- unzip -d ./opendj-packages/opendj-docker ./opendj-packages/opendj-docker/target/Dockerfile.zip
- cp ./opendj-server-legacy/target/package/opendj-*.zip ./opendj-packages/opendj-docker
- sed -i -E '/^#COPY opendj/s/^#//' ./opendj-packages/opendj-docker/Dockerfile-alpine
- - name: Build image
- uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
- with:
- context: ./opendj-packages/opendj-docker
- file: ./opendj-packages/opendj-docker/Dockerfile-alpine
- build-args: |
- VERSION=${{ env.release_version }}
- platforms: linux/amd64, linux/arm64/8, linux/s390x, linux/ppc64le, linux/386, linux/riscv64
- push: true
- tags: ${{ steps.meta.outputs.tags }}
- labels: ${{ steps.meta.outputs.labels }}
- - name: Docker test
- shell: bash
- run: |
- trap 'code=$?; echo "::group::container logs (test)"; docker logs test 2>&1 || true; echo "::endgroup::"; exit $code' ERR
- docker run -it -d --memory="1g" --name=test localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}-alpine
- # the bootstrap's server is down before the marker the health check waits for is
- # written: the stop has logged its record by the time the marker is there
- timeout 3m bash -c 'until docker exec test sh -c "test -f \"\$BOOTSTRAP_COMPLETE\"" 2>/dev/null; do sleep 0.2; done'
- test "$(docker exec test grep -c "The Directory Server is now stopped" /opt/opendj/data/logs/errors)" -ge 1
- timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test | grep -q \"healthy\"; do sleep 10; done'
- docker exec test 'sh' '-c' '/opt/opendj/bin/dsconfig create-backend --hostname localhost --port 4444 --bindDN "cn=Directory Manager" --bindPassword password --backend-name=example2 --type je --set=base-dn:dc=example2,dc=com --set=enabled:true --no-prompt --trustAll'
- docker exec test 'sh' '-c' '/opt/opendj/bin/makeldif -o /tmp/test.ldif -c suffix=dc=example2,dc=com /opt/opendj/data/config/MakeLDIF/example.template'
- docker exec test 'sh' '-c' '/opt/opendj/bin/import-ldif --hostname localhost --port 4444 --bindDN "cn=Directory Manager" --bindPassword password --ldifFile /tmp/test.ldif --backendID=example2 --trustAll'
- # the server is PID 1 of the container, so stopping it stops the container; the
- # container that bootstrapped the instance has to stop the server on SIGTERM, not
- # sit out the timeout and be killed. The server run in the foreground reports its
- # shutdown only in the error log, which is read once the container is back up
- stopped=$(docker exec test grep -c "The Directory Server is now stopped" /opt/opendj/data/logs/errors || true)
- start=$SECONDS
- docker stop -t 60 test
- echo "stopped in $((SECONDS - start)) s, exit code $(docker inspect --format='{{.State.ExitCode}}' test)"
- test $((SECONDS - start)) -lt 50
- test "$(docker inspect --format='{{.State.ExitCode}}' test)" -ne 137
- # a restart runs the server of the instance already there
- docker start test
- timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test | grep -q \"healthy\"; do sleep 10; done'
- test "$(docker exec test grep -c "The Directory Server is now stopped" /opt/opendj/data/logs/errors)" -gt "$stopped"
- docker exec test 'sh' '-c' '/opt/opendj/bin/rebuild-index --hostname localhost --port 4444 --bindDN "cn=Directory Manager" --bindPassword password --baseDN "dc=example2,dc=com" --rebuildAll --trustAll'
- docker exec test 'sh' '-c' '/opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword password --useSsl --trustAll --baseDN "ou=people,dc=example2,dc=com" --searchScope sub "(uid=user.*)" dn | grep ^dn: | wc -l | grep -q 10000'
- docker rm -f test
- - name: Docker test custom password
- shell: bash
- run: |
- trap 'code=$?; echo "::group::container logs (test_custom)"; docker logs test_custom 2>&1 || true; echo "::endgroup::"; exit $code' ERR
- docker run --rm -it -d --memory="1g" -e ADD_BASE_ENTRY="--addBaseEntry" -e ROOT_PASSWORD=custom_password --name=test_custom localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}-alpine
- timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_custom | grep -q \"healthy\"; do sleep 10; done'
- docker exec test_custom 'sh' '-c' '/opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword custom_password --useSsl --trustAll --baseDN "dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1'
- docker kill test_custom
- - name: Docker test base DN with a space
- # the shell must hand the base DN to dsconfig as one value, or the bootstrap stops at
- # "creating backend" (#1157)
- shell: bash
- run: |
- trap 'code=$?; echo "::group::container logs (test_base_dn)"; docker logs test_base_dn 2>&1 || true; echo "::endgroup::"; exit $code' ERR
- docker run --rm -it -d --memory="1g" -e ADD_BASE_ENTRY="--addBaseEntry" -e BASE_DN="o=My Company,c=US" --name=test_base_dn localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}-alpine
- timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_base_dn | grep -q \"healthy\"; do sleep 10; done'
- docker exec test_base_dn 'sh' '-c' '/opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword password --useSsl --trustAll --baseDN "o=My Company,c=US" --searchScope base "(objectClass=*)" 1.1 | grep -qx "dn: o=My Company,c=US"'
- docker kill test_base_dn
- - name: Docker test arbitrary uid
- # OpenShift runs a container under an arbitrary uid that is only in group 0 (#1088)
- shell: bash
- run: |
- trap 'code=$?; echo "::group::container logs (test_uid)"; docker logs test_uid 2>&1 || true; echo "::endgroup::"; exit $code' ERR
- uid=$((1000000000 + RANDOM))
- docker run --rm -it -d --memory="1g" --user $uid:0 -e ADD_BASE_ENTRY="--addBaseEntry" --name=test_uid localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}-alpine
- timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_uid | grep -q \"healthy\"; do sleep 10; done'
- docker exec test_uid 'sh' '-c' "test \"\$(stat -c %u /opt/opendj/instance.loc)\" = $uid"
- docker exec test_uid 'sh' '-c' '/opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword password --useSsl --trustAll --baseDN "dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1'
- docker restart test_uid
- timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_uid | grep -q \"healthy\"; do sleep 10; done'
- docker exec test_uid 'sh' '-c' '/opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword password --useSsl --trustAll --baseDN "dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1'
- docker kill test_uid
- - name: Docker test replication
- # the background join of OPENDJ_REPLICATION_TYPE=simple and the one-shot sdsr path, the same
- # scenarios for both images
- shell: bash
- run: .github/scripts/docker-test-replication.sh "localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}-alpine"
- - name: Docker test secret volume
- # a keystore mounted at SECRET_VOLUME is what LDAPS serves from the first start on, a
- # renewed one - a new password included - is copied while the server runs and served
- # without a restart, and the server is PID 1 of the container, stopping on SIGTERM
- # (#1087, #1085); a start copies what was renewed while no container ran, over the
- # instance already there, and SECRET_VOLUME_REFRESH=0 runs no watcher
- shell: bash
- env:
- IMAGE: localhost:5000/${{ env.image_repository }}:${{ env.release_version }}-alpine
- run: |
- set -E
- trap 'code=$?; echo "::group::container logs (test_secret)"; docker logs test_secret 2>&1 || true; echo "::endgroup::"; exit $code' ERR
- SECRETS=$(mktemp -d)
- chmod 777 "$SECRETS"
- # a key under an alias other than setup's own, since setup binds no connection handler
- # to an alias, and a truststore holding its certificate; the alias is kept by a
- # renewal the server is to load while it runs, which takes a keystore only when it
- # holds a key under an alias the one before it did
- keystore() {
- local pass=${2:-changeit} alias=${3:-$1}
- docker run --rm --entrypoint keytool -v "$SECRETS":/secrets "$IMAGE" -genkeypair -alias "$alias" \
- -keyalg RSA -keysize 2048 -validity 30 -dname "CN=$1" -storetype PKCS12 \
- -keystore /secrets/keystore.new -storepass "$pass" -keypass "$pass"
- docker run --rm --entrypoint keytool -v "$SECRETS":/secrets "$IMAGE" -exportcert -rfc -alias "$alias" \
- -keystore /secrets/keystore.new -storepass "$pass" -file /secrets/cert.pem
- rm -f "$SECRETS/truststore.new"
- docker run --rm --entrypoint keytool -v "$SECRETS":/secrets "$IMAGE" -importcert -noprompt -alias "$1" \
- -file /secrets/cert.pem -keystore /secrets/truststore.new -storetype JKS -storepass changeit
- rm -f "$SECRETS/cert.pem"
- printf %s "$pass" > "$SECRETS/keystore.pin"
- printf changeit > "$SECRETS/truststore.pin"
- mv -f "$SECRETS/truststore.new" "$SECRETS/truststore"
- mv -f "$SECRETS/keystore.new" "$SECRETS/keystore"
- }
- served() { echo | openssl s_client -connect 127.0.0.1:1637 2>/dev/null | openssl x509 -noout -subject -nameopt RFC2253; }
- healthy() { timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_secret | grep -q \"healthy\"; do sleep 10; done'; }
- copies() { docker logs test_secret 2>&1 | grep -c "^Copied $1 from the secret volume$" || true; }
- keystore secret-v1
- docker run -d --memory="1g" -e SECRET_VOLUME_REFRESH=5 -p 127.0.0.1:1637:1636 --name=test_secret \
- -v "$SECRETS":/var/secrets/opendj:ro -v test_secret_data:/opt/opendj/data "$IMAGE"
- healthy
- grep -q "CN=secret-v1" <<< "$(served)"
- test "$(docker exec test_secret cat /proc/1/comm)" = java
- test "$(docker exec test_secret stat -c %a /opt/opendj/data/config/keystore.pin)" = 600
- docker exec test_secret cmp -s /var/secrets/opendj/truststore /opt/opendj/data/config/truststore
- # the bootstrap's server is stopped before the instance is marked bootstrapped
- test "$(docker logs test_secret 2>&1 | grep -e '^Stopping Server' -e '^The instance is bootstrapped' | paste -sd '|' -)" \
- = "Stopping Server...|The instance is bootstrapped, the health check may probe it"
- keystore secret-v2 changeit secret-v1
- timeout 1m bash -c 'until docker exec test_secret cmp -s /var/secrets/opendj/keystore /opt/opendj/data/config/keystore; do sleep 5; done'
- # the server loads the keystore the watcher copied on the next handshake
- grep -q "CN=secret-v2" <<< "$(served)"
- # the watcher has looked at the volume every 5 s since the start, and copied the
- # keystore only when it changed: on the start and once more for v2
- sleep 6
- test "$(copies keystore)" = 2
- # a new password is copied along with its keystore, and the server loads both
- keystore secret-v3 changeit2 secret-v1
- timeout 1m bash -c 'until docker exec test_secret sh -c "cmp -s /var/secrets/opendj/keystore /opt/opendj/data/config/keystore && cmp -s /var/secrets/opendj/keystore.pin /opt/opendj/data/config/keystore.pin"; do sleep 5; done'
- grep -q "CN=secret-v3" <<< "$(served)"
- test "$(copies keystore.pin)" = 2
- start=$SECONDS
- docker stop -t 60 test_secret
- test $((SECONDS - start)) -lt 30
- test "$(docker inspect --format='{{.State.ExitCode}}' test_secret)" = 143
- docker start test_secret
- healthy
- grep -q "CN=secret-v3" <<< "$(served)"
- docker stop -t 60 test_secret
- docker rm test_secret
- # renewed while no container runs, the keystore, its new password and its new alias
- # are copied by the start, over the instance already there
- keystore secret-v4 changeit3
- docker run -d --memory="1g" -e SECRET_VOLUME_REFRESH=0 -p 127.0.0.1:1637:1636 --name=test_secret \
- -v "$SECRETS":/var/secrets/opendj:ro -v test_secret_data:/opt/opendj/data "$IMAGE"
- healthy
- grep -q "CN=secret-v4" <<< "$(served)"
- # no watcher, whose sleep is nearly always there, and nothing copied while the server runs
- docker exec test_secret sh -c '! grep -sqx sleep /proc/[0-9]*/comm'
- keystore secret-v5 changeit3
- sleep 15
- docker exec test_secret sh -c '! cmp -s /var/secrets/opendj/keystore /opt/opendj/data/config/keystore'
- docker rm -f test_secret
- docker volume rm test_secret_data
- - name: Docker test bootstrap LDIFs
- shell: bash
- run: |
- IMAGE=localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}-alpine
- trap 'code=$?; for c in test_bootstrap_lf test_bootstrap_cr test_bootstrap; do echo "::group::container logs ($c)"; docker logs $c 2>&1 || true; echo "::endgroup::"; done; exit $code' ERR
- # a root password with a line break is refused: the tools read only the first line of the password file
- docker run -d --memory="1g" -e ROOT_PASSWORD=$'ab\ncd' --name=test_bootstrap_lf "$IMAGE"
- docker run -d --memory="1g" -e ROOT_PASSWORD=$'ab\rcd' --name=test_bootstrap_cr "$IMAGE"
- for c in test_bootstrap_lf test_bootstrap_cr; do
- timeout 2m bash -c 'until docker logs $0 2>&1 | grep -qF "ROOT_PASSWORD must not contain a line break"; do sleep 2; done' $c
- done
- docker rm -f test_bootstrap_lf test_bootstrap_cr
- # a password with a space and a pattern that matches files in /opt/opendj: every tool
- # of the bootstrap must get it as one value (#1093); the LDIF names have a space as well
- ROOT_PASSWORD='p@ss b*'
- BOOTSTRAP_DIR=$(mktemp -d)
- mkdir -p "$BOOTSTRAP_DIR/schema" "$BOOTSTRAP_DIR/data" "$BOOTSTRAP_DIR/config/schema"
- printf "dn: cn=schema\nchangetype: modify\nadd: attributeTypes\nattributeTypes: ( 1.3.6.1.4.1.99999.1.1 NAME 'bootstrapProbe' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 )\n" > "$BOOTSTRAP_DIR/schema/10 schema.ldif"
- # a schema file of bootstrap/config/schema is copied into the configuration of the instance before setup runs
- printf "dn: cn=schema\nobjectClass: top\nobjectClass: ldapSubentry\nobjectClass: subschema\ncn: schema\nattributeTypes: ( 1.3.6.1.4.1.99999.1.2 NAME 'bootstrapConfigProbe' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 )\n" > "$BOOTSTRAP_DIR/config/schema/99 config.ldif"
- # a pre-encoded password is accepted only once the bootstrap has allowed it on the admin
- # port the server listens on, which is not the default one here
- USER_PASSWORD=$(python3 -c 'import base64, hashlib, os; s = os.urandom(8); print("{SSHA}" + base64.b64encode(hashlib.sha1(b"userpw" + s).digest() + s).decode())')
- printf 'dn: ou=probe,dc=example,dc=com\nobjectClass: organizationalUnit\nou: probe\n\ndn: uid=pre,ou=probe,dc=example,dc=com\nobjectClass: inetOrgPerson\nuid: pre\ncn: pre\nsn: pre\nuserPassword: %s\n' "$USER_PASSWORD" > "$BOOTSTRAP_DIR/data/10 probe.ldif"
- chmod -R a+rX "$BOOTSTRAP_DIR"
- # the containers of a Kubernetes pod share its /dev/shm, which outlives a restart of one of them: the holder stands in for the pod
- docker run -d --no-healthcheck --ipc=shareable --entrypoint sleep --name=test_bootstrap_shm "$IMAGE" 3600
- docker run -it -d --memory="1g" --ipc=container:test_bootstrap_shm -e ADD_BASE_ENTRY="--addBaseEntry" -e ROOT_PASSWORD="$ROOT_PASSWORD" -e ADMIN_PORT=5444 -v "$BOOTSTRAP_DIR/schema:/opt/opendj/bootstrap/schema:ro" -v "$BOOTSTRAP_DIR/data:/opt/opendj/bootstrap/data:ro" -v "$BOOTSTRAP_DIR/config/schema:/opt/opendj/bootstrap/config/schema:ro" --name=test_bootstrap "$IMAGE"
- # while the bootstrap runs, the password file is on the tmpfs of /dev/shm rather than in the writable
- # layer, and its name carries the admin port of the container
- timeout 2m bash -c 'until docker exec test_bootstrap sh -c "ls /dev/shm/opendj-setup-password.5444.*" >/dev/null 2>&1; do sleep 1; done'
- timeout 5m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_bootstrap | grep -q \"healthy\"; do sleep 10; done'
- # the schema and the data LDIFs were loaded: the user of the data LDIF binds with its pre-encoded password
- docker exec test_bootstrap /opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword "$ROOT_PASSWORD" --useSsl --trustAll --baseDN "cn=schema" --searchScope base "(objectClass=*)" attributeTypes > "$BOOTSTRAP_DIR/schema.out"
- grep -q bootstrapProbe "$BOOTSTRAP_DIR/schema.out"
- grep -q bootstrapConfigProbe "$BOOTSTRAP_DIR/schema.out"
- docker exec test_bootstrap /opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "uid=pre,ou=probe,dc=example,dc=com" --bindPassword userpw --useSsl --trustAll --baseDN "" --searchScope base "(objectClass=*)" 1.1
- if docker logs test_bootstrap 2>&1 | grep -F "$ROOT_PASSWORD"; then echo "::error::The root password is in the container log"; false; fi
- # the password file of setup.sh is gone once the bootstrap is over; the perf data of the
- # JVM of the HEALTHCHECK's ldapsearch holds the password too while it runs (#1092)
- left=$(docker exec test_bootstrap grep -rlsF -- "$ROOT_PASSWORD" /tmp /dev/shm | grep -v '^/tmp/hsperfdata_' || true)
- if [ -n "$left" ]; then echo "::error::The root password is left in $left"; false; fi
- # no tool of the bootstrap gets the password on its command line, which only a sample taken
- # while that tool runs would show; grep exits with 1 only when it has read the file and found nothing
- rc=0; docker exec test_bootstrap grep -nE -- '(^|[[:space:]])(-w|--(bindPassword|rootUserPassword))([[:space:]=]|$)' /opt/opendj/bootstrap/setup.sh || rc=$?
- if [ $rc -ne 1 ]; then echo "::error::setup.sh passes the root password on a command line, or grep could not read it"; false; fi
- # run.sh removes the password file that a bootstrap killed before its EXIT trap leaves behind, in /tmp
- # and in /dev/shm, but not the one of another container of the pod, which listens on another admin port
- left=$(docker exec test_bootstrap mktemp /tmp/opendj-setup-password.5444.XXXXXX)
- shm_left=$(docker exec test_bootstrap mktemp -p /dev/shm opendj-setup-password.5444.XXXXXX)
- shm_other=$(docker exec test_bootstrap mktemp -p /dev/shm opendj-setup-password.4444.XXXXXX)
- docker restart test_bootstrap
- timeout 5m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_bootstrap | grep -q \"healthy\"; do sleep 10; done'
- for f in "$left" "$shm_left"; do
- docker exec test_bootstrap test ! -e "$f" || { echo "::error::$f is still there after a restart"; false; }
- done
- docker exec test_bootstrap test -e "$shm_other" || { echo "::error::run.sh removed $shm_other, the password file of another container"; false; }
- docker kill test_bootstrap test_bootstrap_shm
- - name: Docker test health check
- shell: bash
- run: |
- # the ERR trap below has to fire for a check failing inside stays_healthy too
- set -o errtrace
- # the containers are run without --rm, so that one whose bootstrap failed is still there for the trap to print
- trap 'code=$?; for c in test_health test_health_bind; do echo "::group::container logs ($c)"; docker logs $c 2>&1 || true; docker inspect --format="{{json .State.Health}}" $c 2>&1 || true; echo "::endgroup::"; done; docker rm -f test_health test_health_bind >/dev/null 2>&1 || true; exit $code' ERR
- IMAGE=localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}-alpine
- # a failed probe leaves a failing streak until the next probe passes, 5 s later at the
- # earliest, so a container found "healthy 0" every 2 s for 45 s passed every probe since
- stays_healthy() {
- local end=$((SECONDS + 45))
- while [ $SECONDS -lt $end ]; do
- test "$(docker inspect --format='{{.State.Health.Status}} {{.State.Health.FailingStreak}}' "$1")" = "healthy 0"
- sleep 2
- done
- }
- # ROOT_PASSWORD is only the initial root password: changing it must not turn the container unhealthy
- docker run -it -d --memory="1g" --health-interval=5s -e ROOT_PASSWORD=initial_password --name=test_health $IMAGE
- timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_health | grep -q \"healthy\"; do sleep 10; done'
- docker exec test_health /opt/opendj/bin/ldappasswordmodify --hostname localhost --port 1636 --useSsl --trustAll --bindDN "cn=Directory Manager" --bindPassword initial_password --currentPassword initial_password --newPassword rotated_password
- # bind settings kept for the CLI in the home of the image user must not reach the probe
- docker exec test_health sh -c 'mkdir -p /home/opendj/.opendj && printf "bindDN=cn=Directory Manager\nbindPassword=wrong_password\n" > /home/opendj/.opendj/tools.properties'
- stays_healthy test_health
- docker rm -f test_health
- # an instance rejecting unauthenticated requests is probed with the account it is given, whose password is read from a file
- # a password of its own, so the command lines below can be searched for it
- printf hc_secret_1092 > "$RUNNER_TEMP/healthcheck_password"
- chmod 644 "$RUNNER_TEMP/healthcheck_password"
- docker run -it -d --memory="1g" --health-interval=5s -v "$RUNNER_TEMP/healthcheck_password:/tmp/healthcheck_password:ro" -e ROOT_PASSWORD=hc_secret_1092 -e HEALTHCHECK_BIND_DN="cn=Directory Manager" -e HEALTHCHECK_BIND_PASSWORD_FILE=/tmp/healthcheck_password --name=test_health_bind $IMAGE
- timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_health_bind | grep -q \"healthy\"; do sleep 10; done'
- docker exec test_health_bind /opt/opendj/bin/dsconfig set-global-configuration-prop --hostname localhost --port 4444 --bindDN "cn=Directory Manager" --bindPasswordFile /tmp/healthcheck_password --set reject-unauthenticated-requests:true --no-prompt --trustAll
- # the setting has taken: the anonymous probe would now be refused
- rc=0
- docker exec test_health_bind /opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --useSsl --trustAll --baseDN "" --searchScope base "(objectClass=*)" 1.1 || rc=$?
- test "$rc" = 53
- # and the probe reports that refusal when it has no account to bind with
- rc=0
- docker exec -e HEALTHCHECK_BIND_DN= test_health_bind /opt/opendj/healthcheck.sh || rc=$?
- test "$rc" = 1
- # a password file it cannot read is reported as such, whether it is missing or there but not
- # readable: the image runs as its own user, not root, so mode 000 keeps the probe out
- docker exec test_health_bind sh -c 'touch /tmp/unreadable_password && chmod 000 /tmp/unreadable_password'
- for f in /nonexistent /tmp/unreadable_password; do
- rc=0
- out=$(docker exec -e HEALTHCHECK_BIND_PASSWORD_FILE=$f test_health_bind /opt/opendj/healthcheck.sh) || rc=$?
- test "$rc" = 1
- grep -q 'is not a readable file' <<< "$out"
- done
- stays_healthy test_health_bind
- # the password never shows on a command line: sample every process's for two probe intervals
- docker exec test_health_bind sh -c 'end=$(($(date +%s) + 12)); while [ "$(date +%s)" -lt "$end" ]; do for f in /proc/[0-9]*/cmdline; do tr "\0" " " < "$f" 2>/dev/null; echo; done | grep -q "[h]c_secret_1092" && exit 1; sleep 0.2; done; exit 0'
- docker rm -f test_health_bind
- - name: Docker test failed bootstrap
- # a start over a volume whose bootstrap failed once setup had written its
- # configuration must not report healthy either (#1182)
- shell: bash
- run: |
- IMAGE=localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}-alpine
- MEMORY=1g
- trap 'code=$?; echo "::group::container logs (test_failed)"; docker logs test_failed 2>&1 || true; docker inspect --format="{{json .State.Health}}" test_failed 2>&1 || true; echo "::endgroup::"; docker rm -f test_failed >/dev/null 2>&1 || true; docker volume rm test_failed_data >/dev/null 2>&1 || true; exit $code' ERR
- # an unknown backend type fails create-backend once setup has written data/config
- docker run -d --memory="$MEMORY" --health-interval=5s -e ADD_BASE_ENTRY="--addBaseEntry" -e BACKEND_TYPE=bogus -v test_failed_data:/opt/opendj/data --name=test_failed "$IMAGE"
- timeout 3m bash -c 'until docker logs test_failed 2>&1 | grep -qF "failed, this container will not report itself healthy"; do sleep 2; done'
- docker restart -t 60 test_failed
- timeout 3m bash -c 'until docker logs test_failed 2>&1 | grep -qF "The bootstrap of this volume did not complete"; do sleep 2; done'
- # the server is up to be looked at, and no probe passes over 45 s of probes every 5 s
- timeout 3m bash -c 'until docker exec test_failed /opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --useSsl --trustAll --bindDN "cn=Directory Manager" --bindPassword password --baseDN "" --searchScope base "(objectClass=*)" 1.1 >/dev/null 2>&1; do sleep 5; done'
- end=$((SECONDS + 45))
- while [ $SECONDS -lt $end ]; do
- test "$(docker inspect --format='{{.State.Health.Status}}' test_failed)" != healthy
- sleep 2
- done
- # the setup finished by hand and the mark removed, the next start reports healthy
- docker exec test_failed /opt/opendj/bin/dsconfig create-backend --hostname localhost --port 4444 --bindDN "cn=Directory Manager" --bindPassword password --backend-name=userRoot --type je --set=base-dn:dc=example,dc=com --set=enabled:true --no-prompt --trustAll
- docker exec test_failed rm /opt/opendj/data/.bootstrap-pending
- docker restart -t 60 test_failed
- timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_failed | grep -q \"healthy\"; do sleep 5; done'
- docker rm -f test_failed
- docker volume rm test_failed_data
- - name: Docker test upgrade from a released image
- # the build image started over a volume that the latest release bootstrapped
- # upgrades it on start, the road the Installation Guide documents (#1183), and an
- # upgrade that did not complete keeps it unhealthy on later starts too (#1185)
- shell: bash
- run: |
- IMAGE=localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}-alpine
- RELEASED=openidentityplatform/opendj:${{ env.release_version }}-alpine
- MEMORY=1g
- trap 'code=$?; echo "::group::container logs (test_upgrade)"; docker logs test_upgrade 2>&1 || true; docker inspect --format="{{json .State.Health}}" test_upgrade 2>&1 || true; echo "::endgroup::"; docker rm -f test_upgrade >/dev/null 2>&1 || true; docker volume rm test_upgrade_data >/dev/null 2>&1 || true; exit $code' ERR
- search() { docker exec test_upgrade /opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --useSsl --trustAll --bindDN "cn=Directory Manager" --bindPassword password "$@"; }
- # release.yml publishes the GitHub release before it pushes the images of it: in between,
- # and while a failed push stays unfixed, the latest release has no image to start from.
- # Only that skips the test - a tag the registry does not have, which the classic image
- # store reports as "manifest unknown" and the containerd one as a reference it could not
- # resolve, "not found"; a pull that failed otherwise - a rate limit, the registry or the
- # network down - fails it rather than passing it untested
- if ! out=$(docker pull "$RELEASED" 2>&1); then
- echo "$out"
- case $out in
- *"manifest unknown"*|*"failed to resolve reference"*": not found"*) echo "::warning::$RELEASED is not on Docker Hub yet, the upgrade test is skipped"; exit 0 ;;
- *) exit 1 ;;
- esac
- fi
- docker run -d --memory="$MEMORY" -e ADD_BASE_ENTRY="--addBaseEntry" -e SAMPLE_DATA=10 -v test_upgrade_data:/opt/opendj/data --name=test_upgrade "$RELEASED"
- # the entries, not the health status: the health check of 5.1.2 turned healthy before
- # import-ldif had finished
- export -f search
- timeout 5m bash -c 'until search --baseDN "ou=People,dc=example,dc=com" --searchScope one "(uid=user.9)" 1.1 2>/dev/null | grep -q "^dn:"; do sleep 5; done'
- docker stop -t 60 test_upgrade
- docker rm test_upgrade
- docker run -d --memory="$MEMORY" -v test_upgrade_data:/opt/opendj/data --name=test_upgrade "$IMAGE"
- timeout 10m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_upgrade | grep -q \"healthy\"; do sleep 5; done'
- # the upgrade ran its tasks: on equal versions it would be a no-op
- if docker logs test_upgrade 2>&1 | grep -F "has already been upgraded"; then echo "::error::The build image found nothing to upgrade on the volume of $RELEASED"; false; fi
- test "$(search --baseDN "ou=People,dc=example,dc=com" --searchScope one "(uid=user.*)" 1.1 | grep -c "^dn:")" = 10
- # a successful upgrade leaves no mark behind (#1185)
- docker exec test_upgrade test ! -e /opt/opendj/data/.upgrade-pending
- # an upgrade that recorded the new version but did not complete its post-upgrade tasks:
- # the next run of upgrade finds nothing to do, and the mark keeps the container unhealthy
- # until the operator removes it
- docker stop -t 60 test_upgrade
- docker rm test_upgrade
- docker run --rm --entrypoint sh -v test_upgrade_data:/opt/opendj/data "$IMAGE" -c 'echo "an earlier version to this one" >/opt/opendj/data/.upgrade-pending'
- docker run -d --memory="$MEMORY" --health-interval=5s -v test_upgrade_data:/opt/opendj/data --name=test_upgrade "$IMAGE"
- timeout 3m bash -c 'until docker logs test_upgrade 2>&1 | grep -qF "did not complete on an earlier start"; do sleep 2; done'
- # the server is up to be looked at, and no probe passes over 45 s of probes every 5 s
- timeout 3m bash -c 'until search --baseDN "" --searchScope base "(objectClass=*)" 1.1 >/dev/null 2>&1; do sleep 5; done'
- end=$((SECONDS + 45))
- while [ $SECONDS -lt $end ]; do
- test "$(docker inspect --format='{{.State.Health.Status}}' test_upgrade)" != healthy
- sleep 2
- done
- docker exec test_upgrade rm /opt/opendj/data/.upgrade-pending
- docker restart -t 60 test_upgrade
- timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_upgrade | grep -q \"healthy\"; do sleep 5; done'
- docker rm -f test_upgrade
- docker volume rm test_upgrade_data
- # the mark goes on before the upgrade and stays when it fails: a read-only buildinfo
- # fails it where it records the new version, after its upgrade tasks
- docker run -d --memory="$MEMORY" -e ADD_BASE_ENTRY="--addBaseEntry" -v test_upgrade_data:/opt/opendj/data --name=test_upgrade "$RELEASED"
- timeout 5m bash -c 'until search --baseDN "dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1 2>/dev/null | grep -q "^dn:"; do sleep 5; done'
- docker stop -t 60 test_upgrade
- docker rm test_upgrade
- docker run --rm --user root --entrypoint chmod -v test_upgrade_data:/opt/opendj/data "$IMAGE" 444 /opt/opendj/data/config/buildinfo
- docker run -d --memory="$MEMORY" -v test_upgrade_data:/opt/opendj/data --name=test_upgrade "$IMAGE"
- timeout 5m bash -c 'until docker logs test_upgrade 2>&1 | grep -qF "The upgrade did not complete, this container will not report itself healthy"; do sleep 2; done'
- docker run --rm --entrypoint test -v test_upgrade_data:/opt/opendj/data "$IMAGE" -s /opt/opendj/data/.upgrade-pending
- docker rm -f test_upgrade
- docker volume rm test_upgrade_data
- - name: Scan image for vulnerabilities (Trivy)
- # trivy resolves the image from the local Docker daemon, so only the runner's
- # linux/amd64 manifest is scanned; cache: false keeps the ~1GB trivy DBs from
- # evicting the m2-repository caches out of the repo's 10GB actions-cache quota
- uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
- with:
- image-ref: localhost:5000/${{ env.image_repository }}:${{ env.release_version }}-alpine
- format: sarif
- output: trivy-results.sarif
- severity: CRITICAL,HIGH
- limit-severities-for-sarif: true
- ignore-unfixed: true
- scanners: vuln
- cache: false
- - name: Upload Trivy report to GitHub Security
- uses: github/codeql-action/upload-sarif@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3
- # upload even if a preceding step failed, but not without a report to upload
- if: ${{ always() && hashFiles('trivy-results.sarif') != '' }}
- with:
- sarif_file: trivy-results.sarif
- # distinct from the docker-scan.yml categories, which track the published images
- category: trivy-build-alpine
- - name: Cache JMeter
- uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
- with:
- path: ~/jmeter
- key: jmeter-5.6.3
- - name: Benchmark Build-alpine vs Release-alpine
- shell: bash
- run: |
- THREADS=200 DURATION=150 bash .github/benchmark/compare-opendj.sh \
- "Build-alpine" localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}-alpine \
- "Release-alpine" openidentityplatform/opendj:alpine
- - name: Upload benchmark artifacts
+ TOPOLOGY: ${{ matrix.topology }}
+ DURATION: "1800"
+ # the replicas fall behind during the load and catch up after it: 900 s at 325 and 65
+ # a second converged in 396 s (run 38944365728), 1800 s at 423 and 85 in 2489 s (run
+ # 39043600905), the backlog growing faster than the load above the replay capacity; a
+ # run that does not converge in time skips the checks of the data
+ QUIESCE_TIMEOUT: "4500"
+ SAMPLE_DATA: "10000"
+ OUT: ${{ runner.temp }}/replication-out
+ run: .github/scripts/replication/run.sh "localhost:5000/${GITHUB_REPOSITORY,,}:${{ needs.build-docker.outputs.release_version }}"
+ - name: Upload the convergence run
if: always()
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
- name: benchmark-build-vs-release-alpine
- path: |
- a/
- b/
- *.jtl
- *.docker.log
- *.jmeter.out
+ name: convergence-${{ matrix.topology }}
+ path: ${{ runner.temp }}/replication-out
if-no-files-found: warn
- retention-days: 90
+ retention-days: 5
test-deb:
needs: build-maven
--
Gitblit v1.10.0