#
|
#
|
# The contents of this file are subject to the terms of the Common Development and
|
# Distribution License (the License). You may not use this file except in compliance with the
|
# License.
|
#
|
# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
|
# specific language governing permission and limitations under the License.
|
#
|
# When distributing Covered Software, include this CDDL Header Notice in each file and include
|
# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
|
# Header, with the fields enclosed by brackets [] replaced by your own identifying
|
# information: "Portions copyright [year] [name of copyright owner]".
|
#
|
# Copyright 2026 3A Systems, LLC.
|
|
# The checks of the replication convergence run (#1090), sourced by run.sh: the replicas hold
|
# the same data, compared on the data itself rather than on the replication counters.
|
|
# Writes to the second directory server only, which the checks must report: the replication
|
# repair control (ReplicationRepairRequestControl) applies an operation without replicating it
|
inject_divergence() {
|
local n
|
n=$(echo $DS_NODES | cut -d' ' -f2)
|
printf 'dn: uid=user.0,ou=People,%s\nchangetype: modify\nreplace: description\ndescription: only on %s\n' "$BASE_DN" "$n" \
|
| docker exec -i "$n" /opt/opendj/bin/ldapmodify --noPropertiesFile --hostname localhost --port 1636 \
|
--useSsl --trustAll --bindDN "cn=Directory Manager" --bindPassword "$ROOT_PASSWORD" \
|
--control 1.3.6.1.4.1.26027.1.5.2 >/dev/null
|
# an entry of its own, which the searches of the other replicas do not return
|
printf 'dn: uid=only-on-%s,ou=People,%s\nobjectClass: inetOrgPerson\nuid: only-on-%s\ncn: Diverged\nsn: Diverged\n' "$n" "$BASE_DN" "$n" \
|
| docker exec -i "$n" /opt/opendj/bin/ldapmodify --noPropertiesFile --hostname localhost --port 1636 \
|
--useSsl --trustAll --bindDN "cn=Directory Manager" --bindPassword "$ROOT_PASSWORD" \
|
--control 1.3.6.1.4.1.26027.1.5.2 --defaultAdd >/dev/null
|
}
|
|
# An offline export and verify-index need the servers stopped; docker stop gives them the time
|
# of a normal shutdown, so what they write on the way down is on their volumes
|
stop_all() {
|
docker stop -t 120 $ALL_NODES >/dev/null
|
}
|
|
# Runs a tool of the image on the volume of a stopped server. run.sh of the image points the
|
# installation at the volume (instance.loc) when the container starts, which a one-off
|
# container has to do itself; $OUT is mounted on /out
|
offline() { # <node> <tool> [<argument>...]
|
local node=$1 tool=$2
|
shift 2
|
docker run --rm -v "vol-$node:/opt/opendj/data" -v "$OUT:/out" --entrypoint sh "$IMAGE" -c \
|
'echo /opt/opendj/data >/opt/opendj/instance.loc && exec /opt/opendj/bin/"$0" "$@"' "$tool" "$@"
|
}
|
|
assert_offline() {
|
local n first entries failed=
|
# the tools run as the user of the image, not as the one running this script
|
mkdir -p "$OUT/export" "$OUT/diff"
|
chmod 777 "$OUT/export" "$OUT/diff"
|
for n in $DS_NODES; do
|
# ds-sync-hist is the record of the changes each replica applied, ds-sync-state the position
|
# of each replica: both are its own
|
# --offline: a tools.properties with connection settings would turn the export into a task
|
# of a server that is not running
|
if ! offline $n export-ldif --offline --noPropertiesFile --backendID userRoot --ldifFile /out/export/$n.ldif \
|
--excludeAttribute ds-sync-hist --excludeAttribute ds-sync-state >"$OUT/export/$n.out" 2>&1; then
|
echo "::error::export-ldif failed on $n, see export/$n.out"
|
failed=1
|
continue
|
fi
|
# two exports that failed alike, or exported another backend, are as identical as two good ones
|
entries=$(grep -c '^dn: ' "$OUT/export/$n.ldif" || true)
|
if [ "${entries:-0}" -le "$SAMPLE_DATA" ]; then
|
echo "::error::the export of $n holds $entries entries, not more than the $SAMPLE_DATA seeded users"
|
failed=1
|
fi
|
offline $n verify-index --baseDN "$BASE_DN" --countErrors >"$OUT/diff/$n.verify-index.txt" 2>&1 \
|
|| { echo "::error::verify-index found errors on $n, see diff/$n.verify-index.txt"; failed=1; }
|
done
|
first=${DS_NODES%% *}
|
for n in ${DS_NODES#* }; do
|
# ldifdiff exits with 1 when the files differ
|
offline $n ldifdiff --outputLDIF /out/diff/$n.ldif /out/export/$first.ldif /out/export/$n.ldif >/dev/null \
|
|| { echo "::error::$n holds other data than $first, see diff/$n.ldif"; failed=1; }
|
done
|
assert_changelogs || failed=1
|
[ -z "$failed" ] || fail "the replicas do not hold the same data"
|
}
|
|
# The changes of every replica in the changelog of every replication server, one after the
|
# other: a gap in the sequence numbers of a replica's changes during the load is a change that
|
# reached no replication server. The data shows it only while no later change of the same
|
# attribute overwrites it, and missing-changes never (#1195). docker cp streams the changelog
|
# out of the stopped container; changelog-gaps.py reads it
|
assert_changelogs() {
|
local n id code failed=
|
mkdir -p "$OUT/changelog"
|
for n in $RS_NODES; do
|
code=0
|
docker cp "$n:/opt/opendj/data/changelogDb" - \
|
| python3 -I "$HERE/changelog-gaps.py" "$BASE_DN" "$LOAD_START_MS" "$LOAD_END_MS" \
|
| awk 'FILENAME == ARGV[1] { name[$1] = $2; next }
|
$1 == "replica" { id = $2; sub(/:$/, "", id); if (id in name) $2 = id " (" name[id] "):" }
|
{ print }' "$OUT/server-ids.txt" - >"$OUT/changelog/$n.txt" || code=$?
|
case $code in
|
0) ;;
|
1) echo "::error::changes of the load are missing from the changelog of $n, see changelog/$n.txt"; failed=1 ;;
|
*) echo "::error::the changelog of $n could not be read, see changelog/$n.txt"; failed=1; continue ;;
|
esac
|
# every directory server made changes in the load: a changelog whose records the script
|
# no longer reads - another layout, another message type - would otherwise show no gap
|
for id in $(awk '{ print $1 }' "$OUT/server-ids.txt"); do
|
grep -q "^replica $id (" "$OUT/changelog/$n.txt" \
|
|| { echo "::error::the changelog of $n holds no change of the load by $(awk -v id=$id '$1 == id { print $2 }' "$OUT/server-ids.txt")"; failed=1; }
|
done
|
done
|
[ -z "$failed" ]
|
}
|
|
# Equality filters on an attribute the default setup indexes (sn), each matching fewer entries
|
# than the index entry limit (4000 by default): the search goes through the index even for the
|
# Directory Manager, so an entry missing from the index of one replica (#1021) is missing from
|
# its answer, which an export of the entries does not show. A key that holds more entries than
|
# the limit is not indexed, and the search then matches every entry by its filter -
|
# (objectClass=inetOrgPerson) over the seeded users did - so unindexed_searches checks that
|
# each search went through the index
|
INDEXED_FILTERS="(sn=AddRate) (sn=Diverged)"
|
|
# The searches of INDEXED_FILTERS a directory server ran without an index, or that its access
|
# log does not hold, as errors. Every access log file is read: the load rotates them
|
unindexed_searches() { # <node>
|
docker exec $1 sh -c 'grep -h " SEARCH " /opt/opendj/data/logs/access*; [ $? -le 1 ]' \
|
| awk -v node=$1 -v base="$BASE_DN" -v filters="$INDEXED_FILTERS" '
|
BEGIN { n = split(filters, f, " ") }
|
$3 == "SEARCH" && $4 == "REQ" && index($0, " base=\"" base "\" scope=sub ") {
|
for (i = 1; i <= n; i++) if (index($0, " filter=\"" f[i] "\" ")) req[$5 " " $6] = f[i]
|
}
|
$3 == "SEARCH" && $4 == "RES" && / unindexed( |$)/ { unindexed[$5 " " $6] = 1 }
|
END {
|
for (k in req) { seen[req[k]] = 1; if (k in unindexed) printf "::error::%s ran %s without an index\n", node, req[k] }
|
for (i = 1; i <= n; i++) if (!(f[i] in seen)) printf "::error::the access log of %s holds no search of %s\n", node, f[i]
|
}'
|
}
|
|
# The number of lines each server's errors log held once the topology was up: the joins log
|
# replication errors of their own while the servers come up one after the other, and only what
|
# follows is the run's
|
mark_logs() {
|
local n
|
mkdir -p "$OUT/logs"
|
for n in $ALL_NODES; do
|
docker exec $n sh -c 'cat /opt/opendj/data/logs/errors | wc -l' >"$OUT/logs/$n.mark"
|
done
|
}
|
|
since_mark() { # <node>
|
docker exec $1 tail -n +$(($(cat "$OUT/logs/$1.mark") + 1)) /opt/opendj/data/logs/errors
|
}
|
|
# How often each directory server connected to a replication server again after the topology
|
# came up, whatever made it: a switch the broker decided on (msgID 188, or 213 to spread the
|
# load), a replication server that closed the connection (63), a connection that broke. Each
|
# connection ends in msgID 62 (96 where the generation IDs differ), so a switch that went back
|
# to the same replication server (#1215) counts as well. Changes made around a reconnection
|
# under load were lost in #1195, so the summary shows how many a run went through
|
record_reconnections() {
|
local n lines
|
for n in $DS_NODES; do
|
# a log that cannot be read leaves the node out, which the summary shows as -
|
lines=$(since_mark $n) || continue
|
echo "$n $(echo "$lines" | grep -E "msgID=(62|96) " | grep -cF "for domain \"$BASE_DN\"" || true)"
|
done >"$OUT/reconnections.txt"
|
}
|
|
# A directory server is connected to one replication server at a time, so the replication
|
# servers reach each other only through their own connections: each one names every peer it is
|
# connected to "Connected replication server RS(<id>) <host:port>" (ReplicationServerHandler)
|
assert_rs_mesh() {
|
local rs want got failed=
|
want=$(($(echo $RS_NODES | wc -w) - 1))
|
for rs in $RS_NODES; do
|
got=$(admin_search $rs "$MONITOR_DOMAIN" sub "(cn=Connected replication server*)" 1.1 | grep -c '^dn:' || true)
|
[ "${got:-0}" -eq "$want" ] || { echo "::error::$rs is connected to ${got:-0} replication servers, not $want"; failed=1; }
|
done
|
[ -z "$failed" ]
|
}
|
|
# run.sh calls this in a condition, where neither errexit nor the ERR trap stops a command
|
# that fails: every search and every read of a log checks its own status, since a search that
|
# failed alike on every replica gives the same empty answer everywhere
|
assert_online() {
|
local f i=0 n first out lines failed=
|
first=${DS_NODES%% *}
|
mkdir -p "$OUT/dns"
|
record_reconnections
|
for f in $INDEXED_FILTERS; do
|
i=$((i + 1))
|
for n in $DS_NODES; do
|
ldaps_search $n "$BASE_DN" sub "$f" 1.1 >"$OUT/dns/$i.$n.ldif" 2>&1 \
|
|| { echo "::error::$f failed on $n, see dns/$i.$n.ldif"; failed=1; }
|
awk '/^dn: /{ print tolower($0) }' "$OUT/dns/$i.$n.ldif" | sort >"$OUT/dns/$i.$n.txt"
|
done
|
for n in ${DS_NODES#* }; do
|
cmp -s "$OUT/dns/$i.$first.txt" "$OUT/dns/$i.$n.txt" \
|
|| { echo "::error::$f returns other entries on $n than on $first, see dns/$i.*"; failed=1; }
|
done
|
done
|
for n in $DS_NODES; do
|
if ! out=$(unindexed_searches $n); then
|
echo "::error::the access log of $n could not be read"; failed=1
|
elif [ -n "$out" ]; then
|
echo "$out"; failed=1
|
fi
|
done
|
for n in $DS_NODES; do
|
# the load makes no naming conflicts: each server adds under a uid prefix of its own
|
if ! out=$(ldaps_search $n "$BASE_DN" sub "(ds-sync-conflict=*)" 1.1); then
|
echo "::error::the ds-sync-conflict search failed on $n"; failed=1
|
elif grep -q '^dn:' <<<"$out"; then
|
echo "::error::$n holds ds-sync-conflict entries"; failed=1
|
fi
|
done
|
for n in $ALL_NODES; do
|
if ! lines=$(since_mark $n); then
|
echo "::error::the errors log of $n could not be read"; failed=1; continue
|
fi
|
lines=$(grep -E "category=SYNC severity=ERROR|SSLHandshakeException|bad_certificate" <<<"$lines" || true)
|
[ -z "$lines" ] || { printf '%s\n' "$lines" >"$OUT/logs/$n.errors"; echo "::error::$n logged replication errors, see logs/$n.errors"; failed=1; }
|
done
|
assert_rs_mesh || failed=1
|
[ -z "$failed" ] || fail "the replicas do not answer the same"
|
}
|
