#
|
#
|
# The contents of this file are subject to the terms of the Common Development and
|
# Distribution License (the License). You may not use this file except in compliance with the
|
# License.
|
#
|
# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
|
# specific language governing permission and limitations under the License.
|
#
|
# When distributing Covered Software, include this CDDL Header Notice in each file and include
|
# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
|
# Header, with the fields enclosed by brackets [] replaced by your own identifying
|
# information: "Portions copyright [year] [name of copyright owner]".
|
#
|
# Copyright 2026 3A Systems, LLC.
|
|
# Settings and helpers of the replication convergence run (#1090), sourced by run.sh. Every
|
# setting comes from the environment, so that the same run goes in CI and on a laptop.
|
|
TOPOLOGY=${TOPOLOGY:-rs534}
|
# what disturbs the topology during the load (#1208): none, or the <name> of a disturb-<name>.sh
|
# next to this script, which names the DISTURBED server and defines disturb, run during the load,
|
# and settle, run after it
|
DISTURBANCE=${DISTURBANCE:-none}
|
# seconds of load. A disturbed run takes the load of its CI rows: a restarted server served
|
# again 131 to 225 s after the restart there, and the load has to go on after that
|
if [ "$DISTURBANCE" = none ]; then
|
DURATION=${DURATION:-120}
|
else
|
DURATION=${DURATION:-900}
|
fi
|
# the most modifications, and adds or deletes, per second the load sends to each directory
|
# server; a runner that cannot keep up sends fewer, which the summary shows
|
LOAD_MOD_RATE=${LOAD_MOD_RATE:-423}
|
LOAD_ADD_RATE=${LOAD_ADD_RATE:-85}
|
# users the first directory server imports; the others get them from its initialize
|
SAMPLE_DATA=${SAMPLE_DATA:-2000}
|
# the heap of every server and every load client. Without a bound each JVM may take a quarter
|
# of the memory of the host, fourteen of them on a runner of 16 GB, and the one run without it
|
# lost both its runners (run 38694855431). No container memory limit: one would kill a server
|
# in the middle of the run where the heap bound only slows it down
|
SERVER_JAVA_ARGS=${SERVER_JAVA_ARGS:--server -Xmx1g}
|
CLIENT_JAVA_ARGS=${CLIENT_JAVA_ARGS:--Xmx256m}
|
# seconds the whole topology is given to come up: the joins take their turns one at a time
|
UP_TIMEOUT=${UP_TIMEOUT:-3600}
|
# seconds the topology is given to converge once the load stops
|
QUIESCE_TIMEOUT=${QUIESCE_TIMEOUT:-900}
|
# 1: write to one replica only, which the run must report (a self-test of its checks)
|
INJECT_DIVERGENCE=${INJECT_DIVERGENCE:-0}
|
# seconds into the load at which the disturbance starts
|
DISTURB_AT=${DISTURB_AT:-$((DURATION / 3))}
|
# seconds a disturbed server is given to serve again, and its join to end after the load
|
DISTURB_TIMEOUT=${DISTURB_TIMEOUT:-900}
|
# where the summary, the samples, the exports, the differences and a failed run's state go: a
|
# directory that is new or empty
|
OUT=${OUT:-$PWD/replication-out}
|
# 1: leave the servers of a failed run up, to look at them; the next run removes them
|
KEEP=${KEEP:-0}
|
|
# awk reads 0.5 as 0 where the locale's decimal separator is a comma, and the checks sum the
|
# rates the load tools print with a dot
|
export LC_ALL=C
|
|
NETWORK=convergence
|
ROOT_PASSWORD=password
|
BASE_DN="dc=example,dc=com"
|
# the monitor of the replication domain of BASE_DN: cn=Replication,cn=monitor holds one per
|
# replicated suffix (cn=admin data and cn=schema as well), and a server has another server id
|
# in each of them
|
MONITOR_DOMAIN="cn=$(echo "$BASE_DN" | tr '=,' '__'),cn=Replication,cn=monitor"
|
|
case $TOPOLOGY in
|
# #534: directory servers without a replication server and replication servers without data.
|
# The directory servers come first in REPLICATION_PEERS: the first of them seeds
|
rs534)
|
DS_NODES="cv-d0 cv-d1 cv-d2 cv-d3"
|
RS_ONLY="cv-r0 cv-r1"
|
RS_NODES=$RS_ONLY
|
;;
|
# every server holds the data and runs a replication server, which most deployments do
|
combined)
|
DS_NODES="cv-c0 cv-c1 cv-c2 cv-c3"
|
RS_ONLY=
|
RS_NODES=$DS_NODES
|
;;
|
*)
|
echo "unknown TOPOLOGY $TOPOLOGY: rs534 or combined" >&2
|
exit 2
|
;;
|
esac
|
ALL_NODES=$(echo $DS_NODES $RS_ONLY)
|
PEERS=$(echo $ALL_NODES | tr ' ' ',')
|
|
cleanup() {
|
local n
|
# the load clients of a run that failed before it removed them (load.sh)
|
docker rm -f $(docker ps -aq --filter "name=^load-") >/dev/null 2>&1 || true
|
docker rm -f $ALL_NODES >/dev/null 2>&1 || true
|
docker network rm $NETWORK >/dev/null 2>&1 || true
|
for n in $ALL_NODES; do
|
docker volume rm -f "vol-$n" >/dev/null 2>&1 || true
|
done
|
}
|
|
# Where the time goes: each phase logs when it starts, and the run ends - failed or not - with
|
# the phases sorted by their duration
|
PHASE=
|
PHASE_START=$SECONDS
|
PHASE_TIMES=
|
phase() { # <name>
|
[ -z "$PHASE" ] || PHASE_TIMES+="$((SECONDS - PHASE_START)) s $PHASE"$'\n'
|
PHASE=$1
|
PHASE_START=$SECONDS
|
echo "=== [${SECONDS} s] $PHASE"
|
}
|
|
timings() {
|
phase "(end)"
|
echo "::group::time per phase, longest first"
|
sort -rn <<<"${PHASE_TIMES%$'\n'}"
|
echo "total: ${SECONDS} s"
|
echo "::endgroup::"
|
}
|
|
fail() {
|
echo "::error::$1"
|
false
|
}
|
|
wait_until() { # <seconds> <what> <command> [<argument>...]
|
local deadline=$((SECONDS + $1)) what=$2 start=$SECONDS
|
shift 2
|
until "$@"; do
|
[ "$SECONDS" -lt "$deadline" ] || fail "timed out waiting until $what"
|
sleep 2
|
done
|
[ $((SECONDS - start)) -lt 20 ] || echo " [$((SECONDS - start)) s] until $what"
|
}
|
|
is_healthy() { # <container>
|
[ "$(docker inspect --format='{{.State.Health.Status}}' "$1")" = healthy ]
|
}
|
|
# The server of the container logged that it started (msgID 135, once its connection handlers
|
# listen) after the Unix time <seconds>: it serves. Not the health of the container: its probe,
|
# a JVM of its own, does not get through its 30 s on a runner under load - in run 39834888428 a
|
# restarted server served 7 minutes before its container was healthy again. No grep -q: it
|
# would cut docker logs off, and pipefail would take that for no match
|
started_since() { # <container> <seconds>
|
docker logs --since "$2" "$1" 2>&1 | grep 'category=CORE severity=NOTICE msgID=135 ' >/dev/null
|
}
|
|
# The background join of the image goes on once the container reports itself healthy: it adds
|
# replication servers to the replication server lists while its peers register
|
# (repair_replication_servers in join.sh). Ended once no bootstrap script runs any more, as
|
# docker-test-replication.sh waits for it
|
bootstrap_ended() { # <container>
|
! docker exec "$1" sh -c "grep -qs -e 'bootstrap/[j]oin\.sh' -e 'bootstrap/[r]eplicate\.sh' /proc/[0-9]*/cmdline"
|
}
|
|
ldaps_search() { # <container> <base DN> <scope> <filter> [<attribute>...]
|
local container=$1 base=$2 scope=$3 filter=$4
|
shift 4
|
docker exec "$container" /opt/opendj/bin/ldapsearch --noPropertiesFile --hostname localhost --port 1636 \
|
--useSsl --trustAll --bindDN "cn=Directory Manager" --bindPassword "$ROOT_PASSWORD" \
|
--baseDN "$base" --searchScope "$scope" "$filter" "$@"
|
}
|
|
# cn=config and cn=monitor through the administration connector, which every role has
|
admin_search() { # <container> <base DN> <scope> <filter> [<attribute>...]
|
local container=$1 base=$2 scope=$3 filter=$4
|
shift 4
|
docker exec "$container" /opt/opendj/bin/ldapsearch --noPropertiesFile --hostname localhost --port 4444 \
|
--useSsl --trustAll --bindDN "cn=Directory Manager" --bindPassword "$ROOT_PASSWORD" \
|
--baseDN "$base" --searchScope "$scope" "$filter" "$@"
|
}
|
|
runs_replication_server() { # <container>
|
admin_search "$1" "cn=config" sub "(objectClass=ds-cfg-replication-server)" 1.1 | grep "^dn:" >/dev/null
|
}
|
