From f251af3f68a38e2f83154ed80be92667326d286c Mon Sep 17 00:00:00 2001
From: Valery Kharseko <vharseko@3a-systems.ru>
Date: Sat, 07 Nov 2026 11:48:53 +0000
Subject: [PATCH] [#1208] Test directory and replication server crashes and restarts (#1244)
---
.github/scripts/replication/disturb-restart-ds.sh | 11
.github/scripts/replication/collect.sh | 15
.github/scripts/replication/disturb-restart-rs.sh | 27 +
.github/scripts/replication/load.sh | 30 +
.github/scripts/replication/disturb-kill-ds.sh | 50 +++
.github/scripts/replication/selftest-disturb-kill-ds.sh | 112 ++++++++
.github/scripts/replication/rs-disturbance.sh | 71 +++++
.github/scripts/replication/selftest-disturb-rs.sh | 147 ++++++++++
.github/scripts/replication/selftest-summary.sh | 59 ++++
.github/scripts/replication/assert-converged.sh | 83 ++++-
.github/scripts/replication/disturb-kill-rs.sh | 28 ++
.github/scripts/replication/selftest-disturb-restart-ds.sh | 51 ++-
.github/workflows/build.yml | 24 +
.github/scripts/replication/lib.sh | 3
.github/scripts/replication/selftest-replication-errors.sh | 119 ++++++++
15 files changed, 765 insertions(+), 65 deletions(-)
diff --git a/.github/scripts/replication/assert-converged.sh b/.github/scripts/replication/assert-converged.sh
index 98ea1d6..75c31a0 100644
--- a/.github/scripts/replication/assert-converged.sh
+++ b/.github/scripts/replication/assert-converged.sh
@@ -161,20 +161,25 @@
}
# A disturbance window over the errors log of every server (#1208): window_open before the
-# disturbance, window_close once the disturbed server serves again, a line "<lines before>
-# <lines after>" in logs/<node>.windows
-window_open() {
- local n
+# disturbance, with the msgIDs of the errors it expects besides the failed handshakes,
+# or dns:<hostname> for ERROR 51 naming exactly that temporarily unavailable host,
+# window_close once the disturbed server serves again, a line "<lines before> <lines after>
+# [<msgID>|dns:<hostname>...]" in logs/<node>.windows
+window_open() { # [<msgID>|dns:<hostname>...]
+ local n lines
for n in $ALL_NODES; do
- errors_lines $n >"$OUT/logs/$n.window"
+ # an assignment, so that a log that cannot be read stops the run here as well
+ lines=$(errors_lines $n)
+ echo "$lines${*:+ $*}" >"$OUT/logs/$n.window"
done
}
window_close() {
- local n lines
+ local n lines before ids
for n in $ALL_NODES; do
lines=$(errors_lines $n)
- echo "$(cat "$OUT/logs/$n.window") $lines" >>"$OUT/logs/$n.windows"
+ read -r before ids <"$OUT/logs/$n.window"
+ echo "$before $lines${ids:+ $ids}" >>"$OUT/logs/$n.windows"
rm "$OUT/logs/$n.window"
done
}
@@ -189,24 +194,57 @@
# timed out connecting (212) and its replication server logged 178, and a replication server
# logged 311 with SocketTimeoutException when the restarted server logged 119 with "Remote host
# terminated the handshake"; in run 40463887505 a replication server logged 145 on the
-# connection of the restarted one. Those go to logs/<node>.expected; outside a window, and
-# anything else inside one, they are errors. A window expects them on any server, not only on
-# the connections of the disturbed one: 178 and 179 name the servers by id alone, and a server
-# has one in each of its domains and another for its replication server. A handshake that fails
-# is followed by a connection elsewhere, and a change it lost the checks of the data find
+# connection of the restarted one. A window opened with msgIDs expects the SYNC errors of those
+# msgIDs as well. A dns:<hostname> token expects only ERROR 51's exact resolution-failure
+# message for that hostname, not other DNS errors inside the window. This covers the stopped
+# RS container's name temporarily disappearing during kill-rs, restart-rs or a combined
+# kill-ds or restart-ds. DS-only crashes and restarts pass no DNS token, and other hosts remain
+# errors.
+# The other allowances cover what one disturbance leaves behind besides the handshakes, such
+# as the connections a killed replication server breaks (disturb-kill-rs.sh), which a window of
+# another one does not expect. Those go to logs/<node>.expected; outside a window, and anything
+# else inside one, they are errors. A window expects them on any server, not only on the
+# connections of the disturbed one: 178 and 179 name the servers by id alone, and a server has
+# one in each of its domains and another for its replication server. A handshake that fails is
+# followed by a connection elsewhere, and a change it lost the checks of the data find
replication_errors() { # <node>
local windows=
- [ ! -f "$OUT/logs/$1.windows" ] || windows=$(tr '\n' ' ' <"$OUT/logs/$1.windows")
+ [ ! -f "$OUT/logs/$1.windows" ] || windows=$(tr '\n' ';' <"$OUT/logs/$1.windows")
since_mark $1 | awk -v first=$(($(cat "$OUT/logs/$1.mark") + 1)) -v windows="$windows" \
-v expected="$OUT/logs/$1.expected" '
- BEGIN { n = split(windows, w, " ") }
+ BEGIN {
+ n = split(windows, w, ";")
+ for (i = 1; i <= n; i++) {
+ k = split(w[i], f, " ")
+ from[i] = f[1]; to[i] = f[2]; ids[i] = " "
+ for (j = 3; j <= k; j++) ids[i] = ids[i] f[j] " "
+ }
+ }
!/category=SYNC severity=ERROR|SSLHandshakeException|bad_certificate/ { next }
{
line = first + NR - 1
- if (/category=SYNC severity=ERROR msgID=(145|178|179) / \
- || /category=SYNC severity=WARNING .*SSLHandshakeException\(Remote host terminated the handshake\)/)
- for (i = 1; i < n; i += 2)
- if (line > w[i] + 0 && line <= w[i + 1] + 0) { print >expected; next }
+ handshake = /category=SYNC severity=ERROR msgID=(145|178|179) / \
+ || /category=SYNC severity=WARNING .*SSLHandshakeException\(Remote host terminated the handshake\)/
+ id = ""
+ if (match($0, /category=SYNC severity=ERROR msgID=[0-9]+ /)) {
+ id = substr($0, RSTART, RLENGTH)
+ sub(/.*msgID=/, "", id)
+ sub(/ $/, "", id)
+ }
+ dns = ""
+ if (id == "51") {
+ message = substr($0, index($0, " msg=") + 5)
+ if (message ~ /^The hostname [^ ]+ could not be resolved as an IP address$/) {
+ split(message, parts, " ")
+ dns = " dns:" parts[3] " "
+ }
+ }
+ for (i = 1; i <= n; i++)
+ if (line > from[i] + 0 && line <= to[i] + 0 \
+ && (handshake || index(ids[i], " " id " ") || (dns != "" && index(ids[i], dns)))) {
+ print >expected
+ next
+ }
print
}'
}
@@ -228,12 +266,17 @@
# A directory server is connected to one replication server at a time, so the replication
# servers reach each other only through their own connections: each one names every peer it is
-# connected to "Connected replication server RS(<id>) <host:port>" (ReplicationServerHandler)
+# connected to "Connected replication server RS(<id>) <host:port>" (ReplicationServerHandler).
+# The number of them, 0 where the search fails
+rs_peers() { # <replication server>
+ admin_search $1 "$MONITOR_DOMAIN" sub "(cn=Connected replication server*)" 1.1 | grep -c '^dn:' || true
+}
+
assert_rs_mesh() {
local rs want got failed=
want=$(($(echo $RS_NODES | wc -w) - 1))
for rs in $RS_NODES; do
- got=$(admin_search $rs "$MONITOR_DOMAIN" sub "(cn=Connected replication server*)" 1.1 | grep -c '^dn:' || true)
+ got=$(rs_peers $rs)
[ "${got:-0}" -eq "$want" ] || { echo "::error::$rs is connected to ${got:-0} replication servers, not $want"; failed=1; }
done
[ -z "$failed" ]
diff --git a/.github/scripts/replication/collect.sh b/.github/scripts/replication/collect.sh
index a67b25a..eb766c1 100644
--- a/.github/scripts/replication/collect.sh
+++ b/.github/scripts/replication/collect.sh
@@ -48,6 +48,7 @@
# The summary of the run, into $OUT/summary.md and, in GitHub Actions, the job summary
summarize() { # <result>
+ local completed=0
{
echo "### Replication convergence ($TOPOLOGY): $1"
echo
@@ -57,13 +58,19 @@
echo "| load | $DURATION s on each directory server, at most $LOAD_MOD_RATE modifications and $LOAD_ADD_RATE adds or deletes a second |"
echo "| seeded users | $SAMPLE_DATA |"
if [ -s "$OUT/disturbances.txt" ]; then
+ completed=$(wc -l <"$OUT/disturbances.txt")
awk -v start="${LOAD_START_MS:-0}" '{ printf "| disturbance | %s of %s %d s into the load, serving again %d s later |\n",
$4, $3, ($1 - start) / 1000, ($2 - $1) / 1000 }' "$OUT/disturbances.txt"
- elif [ "$DISTURBANCE" != none ] && [ -z "${LOAD_START_MS:-}" ]; then
+ fi
+ if [ "$DISTURBANCE" != none ] && [ -z "${LOAD_START_MS:-}" ]; then
# run.sh sets LOAD_START_MS as the load starts
echo "| disturbance | $DISTURBANCE at $DISTURB_AT s into the load, not started: the run ended before the load |"
- elif [ "$DISTURBANCE" != none ]; then
- echo "| disturbance | $DISTURBANCE at $DISTURB_AT s into the load, not completed |"
+ elif [ "$DISTURBANCE" != none ] && [ "$completed" -lt "$DISTURB_STEPS" ]; then
+ if [ "$completed" -eq 0 ]; then
+ echo "| disturbance | $DISTURBANCE scheduled at $DISTURB_AT s into the load, not completed: 0 of $DISTURB_STEPS steps completed |"
+ else
+ echo "| disturbance | $DISTURBANCE not completed: $((completed)) of $DISTURB_STEPS steps completed |"
+ fi
fi
echo "| converged | ${CONVERGED_IN:--} s after the load |"
if [ -s "$OUT/samples.txt" ]; then
@@ -96,7 +103,7 @@
fi
if ls "$OUT"/logs/*.expected >/dev/null 2>&1; then
echo
- echo "Handshakes failed while a disturbed server came back, which the disturbance expects (logs/*.expected):"
+ echo "Errors logged while a disturbed server went down and came back, which the disturbance expects - failed handshakes, and what a kill leaves behind (logs/*.expected):"
echo
echo '```'
for f in "$OUT"/logs/*.expected; do
diff --git a/.github/scripts/replication/disturb-kill-ds.sh b/.github/scripts/replication/disturb-kill-ds.sh
new file mode 100644
index 0000000..26e5297
--- /dev/null
+++ b/.github/scripts/replication/disturb-kill-ds.sh
@@ -0,0 +1,50 @@
+#
+#
+# The contents of this file are subject to the terms of the Common Development and
+# Distribution License (the License). You may not use this file except in compliance with the
+# License.
+#
+# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
+# specific language governing permission and limitations under the License.
+#
+# When distributing Covered Software, include this CDDL Header Notice in each file and include
+# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
+# Header, with the fields enclosed by brackets [] replaced by your own identifying
+# information: "Portions copyright [year] [name of copyright owner]".
+#
+# Copyright 2026 3A Systems, LLC.
+
+# Crash and recover the same directory server as restart-ds (#1208). Its clients finish
+# before the kill and resume after recovery; the other directory servers keep writing.
+# In combined the container also runs an RS, so this crashes both roles on the same volume.
+DISTURBED=$(echo $DS_NODES | cut -d' ' -f2)
+DISTURB_STEPS=1
+
+disturb() {
+ local from served
+ echo " [$SECONDS s] killing $DISTURBED"
+ # A killed DS leaves a broken DS connection (211). A combined server also breaks DS-to-RS
+ # (180) and RS-to-RS (181) connections and can recover a torn changelog record (284). Peers
+ # resolving it as an RS can log ERROR 51 while its container's name is unavailable.
+ # Allow these only during the crash/recovery window; 66/108 and data/gap checks stay strict.
+ case " $RS_NODES " in
+ *" $DISTURBED "*) window_open "dns:$DISTURBED" 180 181 211 284 ;;
+ *) window_open 211 ;;
+ esac
+ from=$(date +%s)
+ docker kill $DISTURBED >/dev/null
+ docker wait $DISTURBED >/dev/null
+ docker start $DISTURBED >/dev/null
+ wait_until $DISTURB_TIMEOUT "$DISTURBED serves again" started_since $DISTURBED $from
+ served=$(date +%s)
+ echo " [$SECONDS s] $DISTURBED serves again"
+ # Match restart-ds: pending handshakes can finish after the startup message.
+ sleep 10
+ window_close
+ echo "$((from * 1000)) $((served * 1000)) $DISTURBED kill" >>"$OUT/disturbances.txt"
+}
+
+# As for restart-ds, finish the join before the final convergence wait.
+settle() {
+ wait_until $DISTURB_TIMEOUT "the join of $DISTURBED ends" bootstrap_ended $DISTURBED
+}
diff --git a/.github/scripts/replication/disturb-kill-rs.sh b/.github/scripts/replication/disturb-kill-rs.sh
new file mode 100644
index 0000000..42ad048
--- /dev/null
+++ b/.github/scripts/replication/disturb-kill-rs.sh
@@ -0,0 +1,28 @@
+#
+#
+# The contents of this file are subject to the terms of the Common Development and
+# Distribution License (the License). You may not use this file except in compliance with the
+# License.
+#
+# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
+# specific language governing permission and limitations under the License.
+#
+# When distributing Covered Software, include this CDDL Header Notice in each file and include
+# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
+# Header, with the fields enclosed by brackets [] replaced by your own identifying
+# information: "Portions copyright [year] [name of copyright owner]".
+#
+# Copyright 2026 3A Systems, LLC.
+
+# Kill and recover one replication-only server under load (#1208). The other RS stays up
+# throughout this run, including while the recovered server catches up after the load.
+. "$HERE/rs-disturbance.sh"
+
+# A kill leaves DS disconnects (180), RS disconnects (181), and possibly recovery of an
+# incomplete changelog record (284). The stopped container's name may also stop resolving
+# (51, observed in CI run 42046591833); expect that message only for this RS's exact hostname.
+# These SYNC errors are expected only inside its window; monitor-send errors 66/108 and all
+# other unexpected errors remain failures.
+disturb() {
+ down_and_up kill "dns:$DISTURBED" 180 181 284
+}
diff --git a/.github/scripts/replication/disturb-restart-ds.sh b/.github/scripts/replication/disturb-restart-ds.sh
index 6cb18b5..a589038 100644
--- a/.github/scripts/replication/disturb-restart-ds.sh
+++ b/.github/scripts/replication/disturb-restart-ds.sh
@@ -22,6 +22,7 @@
# runs restarts with it.
DISTURBED=$(echo $DS_NODES | cut -d' ' -f2)
+DISTURB_STEPS=1
# docker restart sends SIGTERM and kills the server once its timeout is over: 10 s by default,
# less than a server under load may need to stop, and a killed server is another disturbance.
@@ -30,11 +31,17 @@
# its start fails on a loaded runner are expected there (replication_errors). The window closes
# 10 s after the server serves again: a handshake its start began can still fail then, and a
# peer logs 145 only once its wait for the domain lock, of up to 8 s, is over
-# (ServerHandler.lockDomainWithTimeout). A handshake that fails later is an error again
+# (ServerHandler.lockDomainWithTimeout). A handshake that fails later is an error again.
+# A combined server also restarts its RS, whose hostname can temporarily stop resolving.
+# Expect ERROR 51 only for that exact hostname inside this window; a DS-only restart receives
+# no DNS allowance, and disconnect, recovery and monitor errors remain failures.
disturb() {
local from served
echo " [$SECONDS s] restarting $DISTURBED"
- window_open
+ case " $RS_NODES " in
+ *" $DISTURBED "*) window_open "dns:$DISTURBED" ;;
+ *) window_open ;;
+ esac
from=$(date +%s)
docker restart -t 120 $DISTURBED >/dev/null
wait_until $DISTURB_TIMEOUT "$DISTURBED serves again" started_since $DISTURBED $from
diff --git a/.github/scripts/replication/disturb-restart-rs.sh b/.github/scripts/replication/disturb-restart-rs.sh
new file mode 100644
index 0000000..43d1ee6
--- /dev/null
+++ b/.github/scripts/replication/disturb-restart-rs.sh
@@ -0,0 +1,27 @@
+#
+#
+# The contents of this file are subject to the terms of the Common Development and
+# Distribution License (the License). You may not use this file except in compliance with the
+# License.
+#
+# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
+# specific language governing permission and limitations under the License.
+#
+# When distributing Covered Software, include this CDDL Header Notice in each file and include
+# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
+# Header, with the fields enclosed by brackets [] replaced by your own identifying
+# information: "Portions copyright [year] [name of copyright owner]".
+#
+# Copyright 2026 3A Systems, LLC.
+
+# Gracefully restart one replication-only server under load (#1208). The other RS stays up
+# throughout this run. A separate kill-rs run exercises crash recovery on a fresh topology.
+. "$HERE/rs-disturbance.sh"
+
+# A graceful RS restart under load can still leave DS receive failures (180), as in run
+# 42190249091. Expect those only inside this restart window; RS disconnects (181) and torn
+# changelog recovery (284) remain errors. Docker can briefly stop resolving the container's
+# name between its exit and start even after an orderly stop; allow ERROR 51 only for that RS.
+disturb() {
+ down_and_up restart "dns:$DISTURBED" 180
+}
diff --git a/.github/scripts/replication/lib.sh b/.github/scripts/replication/lib.sh
index 9b9c289..f207943 100644
--- a/.github/scripts/replication/lib.sh
+++ b/.github/scripts/replication/lib.sh
@@ -19,7 +19,8 @@
TOPOLOGY=${TOPOLOGY:-rs534}
# what disturbs the topology during the load (#1208): none, or the <name> of a disturb-<name>.sh
-# next to this script, which names the DISTURBED server and defines disturb, run during the load,
+# next to this script, which names the DISTURBED servers - one directory server, or replication
+# servers - the number of steps in DISTURB_STEPS, and defines disturb, run during the load,
# and settle, run after it
DISTURBANCE=${DISTURBANCE:-none}
# seconds of load. A disturbed run takes the load of its CI rows: a restarted server served
diff --git a/.github/scripts/replication/load.sh b/.github/scripts/replication/load.sh
index 534ea2c..e555831 100644
--- a/.github/scripts/replication/load.sh
+++ b/.github/scripts/replication/load.sh
@@ -17,8 +17,9 @@
# The load of the replication convergence run (#1090), sourced by run.sh: every directory
# server is written to at once, for DURATION seconds, by rate tools that run in containers of
# the same image on the network of the topology. Only writes: a read sends nothing to the
-# replication servers. The server a disturbance takes down (#1208) is written to before it and
-# after it, by clients of their own.
+# replication servers. A directory server a disturbance takes down (#1208) is written to before
+# it and after it, by clients of their own; replication servers it takes down leave the clients
+# be, and the directory servers write through the replication server still up.
# <client> <tool> <argument>... : a container that runs one tool of the image against one server
start_client() {
@@ -99,14 +100,23 @@
fi
done
if [ "$DISTURBANCE" != none ]; then
- # the clients of the disturbed server end on their own before the disturbance, which then
- # cuts none of their operations off: one that had its CSN would read as a lost change
- docker wait load-mod-$DISTURBED load-add-$DISTURBED >/dev/null
- disturb
- left=$((DURATION - (SECONDS - start)))
- [ "$left" -ge $((3 * STAT_INTERVAL)) ] \
- || fail "$DISTURBED served again $((SECONDS - start)) s into the load of $DURATION s, too late to take load again"
- start_clients $DISTURBED 2 $left
+ case " $DS_NODES " in
+ *" $DISTURBED "*)
+ # the clients of the disturbed server end on their own before the disturbance, which
+ # then cuts none of their operations off: one that had its CSN would read as a lost change
+ docker wait load-mod-$DISTURBED load-add-$DISTURBED >/dev/null
+ disturb
+ left=$((DURATION - (SECONDS - start)))
+ [ "$left" -ge $((3 * STAT_INTERVAL)) ] \
+ || fail "$DISTURBED served again $((SECONDS - start)) s into the load of $DURATION s, too late to take load again"
+ start_clients $DISTURBED 2 $left
+ ;;
+ *)
+ left=$((DISTURB_AT - (SECONDS - start)))
+ [ "$left" -le 0 ] || sleep $left
+ disturb
+ ;;
+ esac
fi
for c in $CLIENTS; do
docker wait $c >/dev/null
diff --git a/.github/scripts/replication/rs-disturbance.sh b/.github/scripts/replication/rs-disturbance.sh
new file mode 100644
index 0000000..ddd211c
--- /dev/null
+++ b/.github/scripts/replication/rs-disturbance.sh
@@ -0,0 +1,71 @@
+#
+#
+# The contents of this file are subject to the terms of the Common Development and
+# Distribution License (the License). You may not use this file except in compliance with the
+# License.
+#
+# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
+# specific language governing permission and limitations under the License.
+#
+# When distributing Covered Software, include this CDDL Header Notice in each file and include
+# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
+# Header, with the fields enclosed by brackets [] replaced by your own identifying
+# information: "Portions copyright [year] [name of copyright owner]".
+#
+# Copyright 2026 3A Systems, LLC.
+
+# Shared lifecycle for the independent kill-rs and restart-rs convergence scenarios (#1208).
+# Each run takes down one RS-only server while the other continues serving every DS. The
+# combined topology also takes down a DS and belongs to a different disturbance.
+[ $(echo $RS_ONLY | wc -w) -ge 2 ] \
+ || { echo "$DISTURBANCE needs at least two replication servers without data: TOPOLOGY=rs534" >&2; exit 2; }
+DISTURBED=$(echo $RS_ONLY | cut -d' ' -f1)
+DISTURB_STEPS=1
+
+# Startup after <seconds> and connections to every other RS establish availability. Changelog
+# catch-up is checked by the final convergence/data checks after the load; no second RS is
+# stopped in this run, so the recovered RS need not catch up before another disturbance.
+rs_serves_since() { # <rs> <seconds>
+ started_since $1 $2 && [ "$(rs_peers $1)" -eq $(($(echo $RS_NODES | wc -w) - 1)) ]
+}
+
+# <kill|restart> [<msgID>|dns:<hostname>...]: open the expected-error window, recover the RS, and retain
+# its actual downtime. A kill needs docker wait before start; a graceful restart gets 120 s
+# to stop. The window includes 10 s after startup/peer connections for pending handshakes.
+# DS load balancing may move connections later or not at all, so it does not end the window.
+down_and_up() {
+ local how=$1 rs=$DISTURBED from served left
+ shift
+ window_open "$@"
+ from=$(date +%s)
+ # Opening the window reads each server's log and can itself be slow. Check the remaining
+ # load immediately before the operation, including when client startup delayed disturb().
+ left=$((LOAD_START_MS / 1000 + DURATION - from))
+ [ "$left" -ge $((3 * STAT_INTERVAL)) ] \
+ || fail "only $left s left in the load of $DURATION s, too late to $how $rs under it"
+ echo " [$SECONDS s] $how $rs"
+ if [ "$how" = kill ]; then
+ docker kill $rs >/dev/null
+ docker wait $rs >/dev/null
+ docker start $rs >/dev/null
+ else
+ docker restart -t 120 $rs >/dev/null
+ fi
+ wait_until $DISTURB_TIMEOUT "$rs serves again" rs_serves_since $rs $from
+ served=$(date +%s)
+ echo " [$SECONDS s] $rs serves again"
+ sleep 10
+ window_close
+ echo "$((from * 1000)) $((served * 1000)) $rs $how" >>"$OUT/disturbances.txt"
+ # Record a completed recovery even if it was too slow, but fail rather than silently
+ # accepting a restart that left no useful load. Match restart-ds's three sample intervals.
+ left=$((LOAD_START_MS / 1000 + DURATION - $(date +%s)))
+ [ "$left" -ge $((3 * STAT_INTERVAL)) ] \
+ || fail "$rs recovered too late: only $left s left in the load of $DURATION s"
+}
+
+# The image runs its join on every start. Leave the disturbed server as topology_up does
+# before the final convergence wait, without requiring catch-up while writes are still active.
+settle() {
+ wait_until $DISTURB_TIMEOUT "the join of $DISTURBED ends" bootstrap_ended $DISTURBED
+}
diff --git a/.github/scripts/replication/selftest-disturb-kill-ds.sh b/.github/scripts/replication/selftest-disturb-kill-ds.sh
new file mode 100755
index 0000000..451aa7a
--- /dev/null
+++ b/.github/scripts/replication/selftest-disturb-kill-ds.sh
@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+#
+#
+# The contents of this file are subject to the terms of the Common Development and
+# Distribution License (the License). You may not use this file except in compliance with the
+# License.
+#
+# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
+# specific language governing permission and limitations under the License.
+#
+# When distributing Covered Software, include this CDDL Header Notice in each file and include
+# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
+# Header, with the fields enclosed by brackets [] replaced by your own identifying
+# information: "Portions copyright [year] [name of copyright owner]".
+#
+# Copyright 2026 3A Systems, LLC.
+
+# Exercise a DS crash in both topologies without Docker. Keep the real disturbance,
+# load segmentation and summary; only container commands, readiness and time are stubs.
+set -eE -o pipefail
+HERE=$(cd "$(dirname "$0")" && pwd)
+OUT=$(mktemp -d)
+trap 'rm -rf "$OUT"' EXIT
+failed=0 checks=0
+check() {
+ checks=$((checks + 1))
+ [ "$2" = "$3" ] || { echo "::error::$1: got '$2', expected '$3'"; failed=$((failed + 1)); }
+}
+
+for TOPOLOGY in rs534 combined; do
+ DISTURBANCE=kill-ds DURATION=900 DISTURB_AT=300 DISTURB_TIMEOUT=900
+ if [ "$TOPOLOGY" = rs534 ]; then
+ DS_NODES="d0 d1 d2 d3" RS_NODES="r0 r1" extras="211" target=d1
+ else
+ DS_NODES="c0 c1 c2 c3" RS_NODES="$DS_NODES" extras="dns:c1 180 181 211 284" target=c1
+ fi
+ [ -f "$HERE/disturb-kill-ds.sh" ] || { echo "::error::kill-ds is not implemented"; exit 1; }
+ . "$HERE/disturb-kill-ds.sh"
+ check "$TOPOLOGY target" "$DISTURBED" "$target"
+ check "$TOPOLOGY steps" "$DISTURB_STEPS" 1
+ unset SECONDS
+ SECONDS=0 now=400 calls=
+ date() { echo "$now"; }
+ sleep() { calls="$calls sleep $1"; now=$((now + $1)); }
+ docker() {
+ calls="$calls docker $*"
+ echo "$*" >>"$OUT/docker.calls"
+ [ "${DOCKER_FAIL:-}" != "$1" ] || return 1
+ now=$((now + 5))
+ }
+ wait_until() {
+ calls="$calls wait_until $1 [$2] ${*:3}"
+ [ "${WAIT_FAIL:-no}" != yes ] || return 1
+ now=$((now + 30))
+ }
+ window_open() { calls="$calls window_open $*"; }
+ window_close() { calls="$calls window_close"; }
+ rm -f "$OUT/disturbances.txt"
+ disturb >/dev/null
+ check "$TOPOLOGY crash lifecycle and error window" "$calls" \
+ " window_open $extras docker kill $DISTURBED docker wait $DISTURBED docker start $DISTURBED wait_until 900 [$DISTURBED serves again] started_since $DISTURBED 400 sleep 10 window_close"
+ check "$TOPOLOGY completed crash" "$(cat "$OUT/disturbances.txt")" "400000 445000 $DISTURBED kill"
+ calls=
+ settle
+ check "$TOPOLOGY bootstrap" "$calls" " wait_until 900 [the join of $DISTURBED ends] bootstrap_ended $DISTURBED"
+
+ # Failures run outside if/|| so errexit has the same effect as in run.sh.
+ for failure in kill wait start readiness; do
+ rm -f "$OUT/disturbances.txt"
+ : >"$OUT/docker.calls"
+ set +e
+ (
+ set -e
+ if [ "$failure" = readiness ]; then WAIT_FAIL=yes; else DOCKER_FAIL=$failure; fi
+ disturb >/dev/null
+ )
+ status=$?
+ set -e
+ check "$TOPOLOGY $failure failure" "$status" 1
+ check "$TOPOLOGY $failure leaves no completed step" "$(test -e "$OUT/disturbances.txt" && echo yes || echo no)" no
+ if [ "$failure" = kill ] || [ "$failure" = wait ]; then
+ check "$TOPOLOGY $failure must not start" "$(grep -c '^start ' "$OUT/docker.calls" || true)" 0
+ fi
+ done
+done
+
+# The killed DS's clients finish before the crash and resume only after recovery.
+# The other DS keeps writing throughout. A slow recovery must still leave useful load.
+. "$HERE/load.sh"
+DS_NODES="d0 d1" DISTURBED=d1
+start_clients() { calls="$calls clients $*"; }
+docker() { calls="$calls docker $*"; }
+disturb() { calls="$calls disturb"; SECONDS=$((SECONDS + RECOVERY)); }
+fail() { echo "::error::$1"; false; }
+for RECOVERY in 870 871; do
+ calls= SECONDS=0
+ set +e
+ (set -e; run_load; echo "$calls" >"$OUT/load.calls") >"$OUT/load.log"
+ status=$?
+ set -e
+ if [ "$RECOVERY" = 870 ]; then
+ check "30 s of resumed load" "$status" 0
+ check "clients around the crash" "$(cat "$OUT/load.calls")" \
+ " clients d0 1 900 clients d1 1 300 docker wait load-mod-d1 load-add-d1 disturb clients d1 2 30"
+ else
+ check "29 s is too late to resume" "$status" 1
+ check "slow recovery diagnostic" "$(grep -c 'too late to take load again' "$OUT/load.log" || true)" 1
+ fi
+done
+
+echo "disturb-kill-ds: $((checks - failed)) checks passed, $failed failed"
+[ "$failed" -eq 0 ]
diff --git a/.github/scripts/replication/selftest-disturb-restart-ds.sh b/.github/scripts/replication/selftest-disturb-restart-ds.sh
index af0340e..1f5921f 100755
--- a/.github/scripts/replication/selftest-disturb-restart-ds.sh
+++ b/.github/scripts/replication/selftest-disturb-restart-ds.sh
@@ -16,7 +16,7 @@
# Copyright 2026 3A Systems, LLC.
# A self-test of the disturbance restart-ds of the replication convergence run (#1208), with no
-# topology: what disturb and settle call, in which order, and what disturb writes to
+# topology: what disturb, settle and run_load call, in which order, and what disturb writes to
# disturbances.txt. The window of the errors logs opens before the restart and closes 10 s after
# the server serves again, and the restart ends when the server served, not after the grace. A
# restart-ds row shows neither, unless a handshake record lands where the order moved. docker,
@@ -28,17 +28,14 @@
set -eE -o pipefail
HERE=$(cd "$(dirname "$0")" && pwd)
-DS_NODES="d0 d1" DISTURB_TIMEOUT=900
+DISTURB_TIMEOUT=900
OUT=$(mktemp -d)
trap 'rm -rf "$OUT"' EXIT
-. "$HERE/disturb-restart-ds.sh"
-
-now=100 calls=
date() { echo $now; }
sleep() { calls="$calls sleep $1"; now=$((now + $1)); }
docker() { calls="$calls docker $*"; now=$((now + 5)); }
-wait_until() { calls="$calls wait_until $*"; now=$((now + 30)); }
-window_open() { calls="$calls window_open"; }
+wait_until() { calls="$calls wait_until $1 [$2] ${*:3}"; now=$((now + 30)); }
+window_open() { calls="$calls window_open${*:+ $*}"; }
window_close() { calls="$calls window_close"; }
failed=0
@@ -46,13 +43,37 @@
[ "$2" = "$3" ] || { echo "::error::$1:$2, not$3"; failed=$((failed + 1)); }
}
-disturb >/dev/null
-check "disturb called" "$calls" \
- " window_open docker restart -t 120 d1 wait_until 900 d1 serves again started_since d1 100 sleep 10 window_close"
-check "disturbances.txt holds" " $(cat "$OUT/disturbances.txt")" " 100000 135000 d1 restart"
-calls=
-settle
-check "settle called" "$calls" " wait_until 900 the join of d1 ends bootstrap_ended d1"
+for TOPOLOGY in rs534 combined similar-host; do
+ case $TOPOLOGY in
+ rs534) DS_NODES="d0 d1" RS_NODES="r0 r1" target=d1 extras= ;;
+ combined) DS_NODES="c0 c1" RS_NODES="$DS_NODES" target=c1 extras=" dns:c1" ;;
+ similar-host) DS_NODES="d0 d1" RS_NODES="d10 d11" target=d1 extras= ;;
+ esac
+ . "$HERE/disturb-restart-ds.sh"
+ now=100 calls=
+ rm -f "$OUT/disturbances.txt"
+ disturb >/dev/null
+ check "$TOPOLOGY disturb called" "$calls" \
+ " window_open$extras docker restart -t 120 $target wait_until 900 [$target serves again] started_since $target 100 sleep 10 window_close"
+ check "$TOPOLOGY disturbances.txt holds" " $(cat "$OUT/disturbances.txt")" " 100000 135000 $target restart"
+ calls=
+ settle
+ check "$TOPOLOGY settle called" "$calls" " wait_until 900 [the join of $target ends] bootstrap_ended $target"
+done
-echo "disturb-restart-ds: $((3 - failed)) checks passed, $failed failed"
+# run_load stops the clients of the disturbed server at DISTURB_AT, waits for them to end before
+# it disturbs, and has clients of its own write to it for the rest of the load. SECONDS without
+# its special properties, so that run_load reads the clock of the stubs: the restart takes 200 s
+DURATION=900 DISTURB_AT=300 DISTURBANCE=restart-ds
+DS_NODES="d0 d1" DISTURBED=d1
+. "$HERE/load.sh"
+unset SECONDS
+start_clients() { calls="$calls clients $*"; }
+disturb() { calls="$calls disturb"; SECONDS=$((SECONDS + 200)); }
+calls= SECONDS=0
+run_load
+check "run_load called" "$calls" \
+ " clients d0 1 900 clients d1 1 300 docker wait load-mod-d1 load-add-d1 disturb clients d1 2 700"
+
+echo "disturb-restart-ds: $((10 - failed)) checks passed, $failed failed"
[ "$failed" -eq 0 ]
diff --git a/.github/scripts/replication/selftest-disturb-rs.sh b/.github/scripts/replication/selftest-disturb-rs.sh
new file mode 100755
index 0000000..ec31bf2
--- /dev/null
+++ b/.github/scripts/replication/selftest-disturb-rs.sh
@@ -0,0 +1,147 @@
+#!/usr/bin/env bash
+#
+#
+# The contents of this file are subject to the terms of the Common Development and
+# Distribution License (the License). You may not use this file except in compliance with the
+# License.
+#
+# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
+# specific language governing permission and limitations under the License.
+#
+# When distributing Covered Software, include this CDDL Header Notice in each file and include
+# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
+# Header, with the fields enclosed by brackets [] replaced by your own identifying
+# information: "Portions copyright [year] [name of copyright owner]".
+#
+# Copyright 2026 3A Systems, LLC.
+
+# Exercise both independent RS disturbances without Docker. External commands and the clock
+# are stubs; the entry points, readiness checks and run_load are the production functions.
+set -eE -o pipefail
+HERE=$(cd "$(dirname "$0")" && pwd)
+OUT=$(mktemp -d)
+trap 'rm -rf "$OUT"' EXIT
+
+variant() (
+ local kind=$1 how=$2 recovery=$3 command=$4 extras=$5
+ local code started peers
+ failed=0 checks=0
+ check() {
+ checks=$((checks + 1))
+ [ "$2" = "$3" ] || { echo "::error::$kind $1: got '$2', expected '$3'"; failed=$((failed + 1)); }
+ }
+ for topology in '' r0; do
+ code=0
+ (RS_ONLY=$topology; . "$HERE/disturb-$kind.sh") >/dev/null 2>&1 || code=$?
+ check "refuses fewer than two RS-only servers" "$code" 2
+ done
+ DS_NODES="d0 d1" RS_ONLY="r0 r1" RS_NODES="r0 r1" DURATION=900 DISTURB_AT=300 DISTURB_TIMEOUT=900
+ DISTURBANCE=$kind LOAD_START_MS=100000
+ . "$HERE/disturb-$kind.sh"
+ . "$HERE/load.sh"
+ check "disturbed server" "$DISTURBED" r0
+ check "expected step count" "$DISTURB_STEPS" 1
+ unset SECONDS
+ SECONDS=0 now=400 SERVE=30 calls=
+ date() { echo "$now"; }
+ sleep() { calls="$calls sleep $1"; now=$((now + $1)); SECONDS=$((SECONDS + $1)); }
+ docker() {
+ calls="$calls docker $*"
+ echo "$*" >>"$OUT/docker.calls"
+ [ "${DOCKER_FAIL:-}" != "$1" ] || return 1
+ now=$((now + 5))
+ }
+ wait_until() {
+ calls="$calls wait_until $1 [$2] ${*:3}"
+ [ "${WAIT_FAIL:-no}" != yes ] || fail "readiness timed out"
+ now=$((now + SERVE))
+ }
+ window_open() { calls="$calls window_open${*:+ $*}"; now=$((now + ${WINDOW_DELAY:-0})); }
+ window_close() { calls="$calls window_close"; }
+ fail() { echo "::error::$1"; false; }
+ rm -f "$OUT/disturbances.txt"
+ disturb >"$OUT/disturb.log"
+ check "single operation and error window" "$calls" \
+ " window_open$extras$command wait_until 900 [r0 serves again] rs_serves_since r0 400 sleep 10 window_close"
+ check "single completed step" "$(cat "$OUT/disturbances.txt")" "400000 $((400000 + recovery * 1000)) r0 $how"
+ calls=
+ settle
+ check "settle only the disturbed server" "$calls" " wait_until 900 [the join of r0 ends] bootstrap_ended r0"
+
+ started_since() { [ "$1 $2" = 'r0 400' ] && [ "$started" = yes ]; }
+ rs_peers() { [ "$1" = r0 ] && echo "$peers"; }
+ serves() { if rs_serves_since r0 400; then echo yes; else echo no; fi; }
+ started=yes peers=1
+ check "startup and peer present" "$(serves)" yes
+ peers=0
+ check "missing peer" "$(serves)" no
+ peers=2
+ check "unexpected peer count" "$(serves)" no
+ started=no peers=1
+ check "no startup message" "$(serves)" no
+ started=yes peers=2 RS_NODES="r0 r1 r2"
+ check "three RS topology, both peers present" "$(serves)" yes
+ peers=1
+ check "three RS topology, one peer absent" "$(serves)" no
+ RS_NODES="r0 r1"
+
+ # Run failures outside if/|| so errexit remains active as it is in run.sh.
+ budget_case() { # <label> <start time> <window delay> <status> <completed> <docker calls>
+ local status lines
+ rm -f "$OUT/disturbances.txt"
+ : >"$OUT/docker.calls"
+ set +e
+ (set -e; now=$2 WINDOW_DELAY=$3; disturb >"$OUT/budget.log")
+ status=$?
+ set -e
+ check "$1 status" "$status" "$4"
+ lines=0
+ [ ! -f "$OUT/disturbances.txt" ] || lines=$(awk 'END {print NR}' "$OUT/disturbances.txt")
+ check "$1 completed steps" "$lines" "$5"
+ check "$1 docker calls" "$(awk 'END {print NR}' "$OUT/docker.calls")" "$6"
+ if [ "$4" -ne 0 ]; then
+ check "$1 diagnostic" "$(grep -c 'too late' "$OUT/budget.log" || true)" 1
+ fi
+ }
+ op_calls=1
+ [ "$how" != kill ] || op_calls=3
+ budget_case '30 seconds after recovery and grace' "$((1000 - recovery - 10 - 30))" 0 0 1 "$op_calls"
+ budget_case '29 seconds after recovery and grace' "$((1000 - recovery - 10 - 29))" 0 1 1 "$op_calls"
+ budget_case '30 seconds before operation' 970 0 1 1 "$op_calls"
+ budget_case '29 seconds before operation' 971 0 1 0 0
+ budget_case 'load already ended' 1001 0 1 0 0
+ budget_case 'window capture consumes remaining budget' 960 11 1 0 0
+
+ for failure in docker readiness; do
+ rm -f "$OUT/disturbances.txt"
+ : >"$OUT/docker.calls"
+ set +e
+ (
+ set -e
+ now=400
+ if [ "$failure" = docker ]; then DOCKER_FAIL=$how; else WAIT_FAIL=yes; fi
+ disturb >"$OUT/failure.log"
+ )
+ code=$?
+ set -e
+ check "$failure failure propagates" "$code" 1
+ check "$failure failure not marked complete" "$([ -f "$OUT/disturbances.txt" ] && echo yes || echo no)" no
+ done
+
+ # No DS client is interrupted or segmented for either RS disturbance.
+ start_clients() { calls="$calls clients $*"; }
+ disturb() { calls="$calls disturb at $SECONDS"; }
+ calls= SECONDS=0 CLIENTS=
+ run_load
+ check "whole-load DS clients" "$calls" ' clients d0 1 900 clients d1 1 900 sleep 300 disturb at 300'
+ start_clients() { calls="$calls clients $*"; SECONDS=$((SECONDS + 160)); }
+ calls= SECONDS=0
+ run_load
+ check "late client startup, no negative sleep" "$calls" ' clients d0 1 900 clients d1 1 900 disturb at 320'
+ echo "$kind: $((checks - failed)) checks passed, $failed failed"
+ [ "$failed" -eq 0 ]
+)
+
+# Separate processes keep sourced entry points and fixture overrides from leaking.
+variant restart-rs restart 35 ' docker restart -t 120 r0' ' dns:r0 180'
+variant kill-rs kill 45 ' docker kill r0 docker wait r0 docker start r0' ' dns:r0 180 181 284'
diff --git a/.github/scripts/replication/selftest-replication-errors.sh b/.github/scripts/replication/selftest-replication-errors.sh
index add3f84..b7c4ae5 100755
--- a/.github/scripts/replication/selftest-replication-errors.sh
+++ b/.github/scripts/replication/selftest-replication-errors.sh
@@ -17,8 +17,9 @@
# A self-test of the errors log check of the replication convergence run (#1208), with no
# topology: which records replication_errors takes for errors and which ones a disturbance
-# window expects. A green row does not show it - nothing a disturbance expects is logged outside
-# its window there, and the base rows have no window - so docker-convergence runs this first.
+# window expects: failed handshakes and the msgIDs or dns:<hostname> tokens it was opened with.
+# A green row does not show it - nothing a disturbance expects is logged outside its window
+# there, and the base rows have no window - so docker-convergence runs this first.
# docker is a stub that reads a local errors log: mark_logs, window_open and window_close count
# its lines, since_mark reads it, as they do in a container.
#
@@ -55,11 +56,51 @@
E106ID='[t] category=SYNC severity=ERROR msgID=106 msg=Replication server 32179 lost its connection'
N62='[t] category=SYNC severity=NOTICE msgID=62 msg=Directory server DS(1) has connected'
CORE178='[t] category=CORE severity=ERROR msgID=178 msg=not a replication error'
+# What an RS disturbance can leave behind, and ids like its allowances.
+E180='[t] category=SYNC severity=ERROR msgID=180 msg=Directory server DS(13411) encountered an error while receiving changes for domain "dc=example,dc=com" from replication server RS(23837) at cv-r0:8989. The connection will be closed, and this directory server will now try to connect to another replication server'
+E181='[t] category=SYNC severity=ERROR msgID=181 msg=The connection from this replication server RS(31531) to replication server RS(23837) at cv-r0:8989 for domain "dc=example,dc=com" has failed'
+E284="[t] category=SYNC severity=ERROR msgID=284 msg=Log file '/opt/opendj/data/changelogDb/2.dom/13411.server/head.log' was successfully recovered by removing a partially written record"
+E211='[t] category=SYNC severity=ERROR msgID=211 msg=The connection from this replication server RS(31531) to directory server DS(24329) at cv-d1:8989 for domain "dc=example,dc=com" has failed'
+E66='[t] category=SYNC severity=ERROR msgID=66 msg=monitor reply failed'
+E108='[t] category=SYNC severity=ERROR msgID=108 msg=monitor request failed'
+E51='[05/Nov/2026:19:18:17 +0000] category=SYNC severity=ERROR msgID=51 msg=The hostname cv-r0 could not be resolved as an IP address'
+E18='[t] category=SYNC severity=ERROR msgID=18 msg=an id that starts the first one a window lists'
+E80='[t] category=SYNC severity=ERROR msgID=80 msg=an id that ends the first one a window lists'
+E106KILL='[t] category=SYNC severity=ERROR msgID=106 msg=an id a window lists, 181, in the message'
+# records the check reads for their certificate alert, under an id a window lists
+WBAD181=${WBAD/msgID=119/msgID=181}
+WBADCORE181=${WBADCORE/msgID=1/msgID=181}
+KILL='[ dns:cv-r0 180 181 284'
+# Read the restart allowances from its entry point so the filter cases also detect a missing
+# or widened production allowance. Only the Docker lifecycle is replaced here.
+RESTART=$(
+ RS_ONLY='cv-r0 cv-r1' RS_NODES='cv-r0 cv-r1'
+ . "$HERE/disturb-restart-rs.sh"
+ down_and_up() { shift; printf '[ %s' "$*"; }
+ disturb
+)
+# Run the DS entry point too: the combined restart receives a DNS token, while a DS-only
+# restart does not. Keep the production window arguments and replace only the lifecycle.
+ds_restart_window() (
+ DS_NODES=$1 RS_NODES=$2 DISTURB_TIMEOUT=900
+ . "$HERE/disturb-restart-ds.sh"
+ window_open() { printf '[%s' "${*:+ $*}" >"$OUT/ds-restart-window"; }
+ window_close() { :; }
+ docker() { :; }
+ wait_until() { :; }
+ sleep() { :; }
+ date() { echo 100; }
+ disturb >/dev/null
+ cat "$OUT/ds-restart-window"
+)
+RESTART_DS=$(ds_restart_window 'cv-d0 cv-d1' 'cv-r0 cv-r1')
+RESTART_COMBINED=$(ds_restart_window 'cv-c0 cv-c1 cv-c2 cv-c3' 'cv-c0 cv-c1 cv-c2 cv-c3')
passed=0 failed=0
-# case <name> <errors> <expected> <record, [ or ]>...: the errors log holds two errors before
-# mark_logs, which are not the run's, then the records given; [ opens a window and ] closes it.
-# The number of records replication_errors prints, and of those it sends to logs/n0.expected
+# case <name> <errors> <expected> <record, [, [ <msgID>|dns:<hostname>... or ]>...: the errors log
+# holds two errors before mark_logs, which are not the run's, then the records given; [ opens a
+# window with the msgIDs or DNS tokens that follow it, and ] closes it. Count replication_errors
+# output records and those it sends to logs/n0.expected.
case_() {
local name=$1 want_errors=$2 want_expected=$3 record errors expected
shift 3
@@ -69,6 +110,7 @@
for record; do
case $record in
'[') window_open ;;
+ '[ '*) window_open ${record#\[ } ;;
']') window_close ;;
*) echo "$record" >>"$LOG" ;;
esac
@@ -103,6 +145,73 @@
case_ two-windows 0 2 '[' "$E178" ']' "$N62" '[' "$E179" ']'
case_ between-two-windows 1 2 '[' "$E178" ']' "$E179" '[' "$E178" ']'
case_ mixed 2 2 '[' "$E178" "$E106" "$W119" "$WBAD" "$N62" ']'
+case_ 180-in-a-kill-window 0 1 "$KILL" "$E180" ']'
+case_ 181-in-a-kill-window 0 1 "$KILL" "$E181" ']'
+case_ 284-in-a-kill-window 0 1 "$KILL" "$E284" ']'
+case_ handshakes-in-kill-window 0 2 "$KILL" "$E178" "$W119" ']'
+case_ 181-in-another-window 1 0 '[' "$E181" ']'
+case_ 181-outside-a-window 1 0 "$E181"
+case_ 181-after-a-kill-window 1 1 "$KILL" "$E284" ']' "$E181"
+case_ ids-like-listed-ones 2 0 "$KILL" "$E18" "$E80" ']'
+case_ listed-id-in-the-message 1 0 "$KILL" "$E106KILL" ']'
+case_ another-in-a-kill-window 1 0 "$KILL" "$E106" ']'
+case_ listed-id-not-sync-error 2 0 "$KILL" "$WBAD181" "$WBADCORE181" ']'
+case_ kill-window-then-another 1 2 "$KILL" "$E180" ']' "$N62" '[' "$E181" "$E145" ']'
+case_ another-then-kill-window 0 2 '[' ']' "$N62" "$KILL" "$E181" "$E145" ']'
+
+# DNS failures of only the named RS are expected, and only in its own window.
+# Removing the host, message, ID or window checks must expose a failure here.
+case_ killed-rs-dns 0 1 "$KILL" "$E51" ']'
+case_ dns-before-after-window 2 1 "$E51" "$KILL" "$E51" ']' "$E51"
+case_ dns-without-window 1 0 "$E51"
+case_ dns-window-without-token 1 0 '[' "$E51" ']'
+case_ dns-ds-crash 1 0 '[ 211' "$E51" ']'
+case_ dns-combined-without-token 1 0 '[ 180 181 211 284' "$E51" ']'
+case_ dns-combined-host 0 1 '[ dns:cv-c1 180 181 211 284' "${E51/cv-r0/cv-c1}" ']'
+case_ dns-restarted-rs 0 1 "$RESTART" "$E51" ']'
+case_ dns-combined-ds-restart 0 1 "$RESTART_COMBINED" "${E51/cv-r0/cv-c1}" ']'
+case_ dns-ds-only-restart 1 0 "$RESTART_DS" "${E51/cv-r0/cv-d1}" ']'
+case_ dns-other-host-ds-restart 1 0 "$RESTART_COMBINED" "${E51/cv-r0/cv-c2}" ']'
+case_ dns-before-after-ds-restart 2 1 "${E51/cv-r0/cv-c1}" "$RESTART_COMBINED" \
+ "${E51/cv-r0/cv-c1}" ']' "${E51/cv-r0/cv-c1}"
+case_ dns-other-host 1 0 "$KILL" "${E51/cv-r0/cv-r1}" ']'
+case_ dns-host-prefix 1 0 "$KILL" "${E51/cv-r0/cv-r}" ']'
+case_ dns-host-suffix 1 0 "$KILL" "${E51/cv-r0/cv-r01}" ']'
+case_ dns-other-id 1 0 "$KILL" "${E51/msgID=51/msgID=151}" ']'
+case_ dns-other-message 1 0 "$KILL" "${E51/msg=The hostname/msg=Unexpected hostname}" ']'
+case_ dns-message-suffix 1 0 "$KILL" "$E51 extra" ']'
+case_ dns-embedded-message 1 0 "$KILL" "${E51/msg=The hostname/msg=Unexpected DNS failure msg=The hostname}" ']'
+case_ dns-next-window 1 1 "$KILL" "$E51" ']' '[' "$E51" ']'
+case_ dns-window-hosts 2 2 "$KILL" "$E51" "${E51/cv-r0/cv-r1}" ']' \
+ '[ dns:cv-r1' "$E51" "${E51/cv-r0/cv-r1}" ']'
+case_ monitors-in-rs-kill 2 0 "$KILL" "$E66" "$E108" ']'
+
+# DS crash allowances do not leak into graceful restarts, later records, or RS-only kills.
+case_ ds-kill-disconnect 0 1 '[ 211' "$E211" ']'
+case_ ds-kill-without-rs 2 0 '[ 211' "$E180" "$E284" ']'
+case_ combined-kill 0 4 '[ dns:cv-c1 180 181 211 284' "$E180" "$E181" "$E211" "$E284" ']'
+case_ ds-disconnect-graceful 1 0 '[' "$E211" ']'
+case_ 180-in-rs-restart 0 1 "$RESTART" "$E180" ']'
+case_ 180-before-after-restart 2 1 "$E180" "$RESTART" "$E180" ']' "$E180"
+case_ 180-in-ds-restart 1 0 "$RESTART_DS" "$E180" ']'
+case_ combined-ds-restart-strict 6 0 "$RESTART_COMBINED" "$E180" "$E181" "$E211" "$E284" "$E66" "$E108" ']'
+case_ rs-restart-strict 5 0 "$RESTART" "$E181" "$E211" "$E284" "$E66" "$E108" ']'
+case_ ds-disconnect-rs-kill 1 0 "$KILL" "$E211" ']'
+case_ ds-disconnect-before-after 2 1 "$E211" '[ 211' "$E211" ']' "$E211"
+case_ monitors-in-combined-kill 2 0 '[ dns:cv-c1 180 181 211 284' "$E66" "$E108" ']'
+
+# A log that cannot be read stops window_open, which would otherwise open a window from its first
+# line. In the background: errexit is off in a subshell on the left of ||, and on in a job
+code=0
+rm -f "$LOG"
+(window_open 180 181 284) 2>/dev/null &
+wait $! || code=$?
+if [ "$code" -ne 0 ]; then
+ passed=$((passed + 1))
+else
+ failed=$((failed + 1))
+ echo "::error::window_open went on over an errors log that cannot be read"
+fi
echo "replication_errors: $passed cases passed, $failed failed"
[ "$failed" -eq 0 ]
diff --git a/.github/scripts/replication/selftest-summary.sh b/.github/scripts/replication/selftest-summary.sh
new file mode 100755
index 0000000..11a53ab
--- /dev/null
+++ b/.github/scripts/replication/selftest-summary.sh
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+#
+#
+# The contents of this file are subject to the terms of the Common Development and
+# Distribution License (the License). You may not use this file except in compliance with the
+# License.
+#
+# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
+# specific language governing permission and limitations under the License.
+#
+# When distributing Covered Software, include this CDDL Header Notice in each file and include
+# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
+# Header, with the fields enclosed by brackets [] replaced by your own identifying
+# information: "Portions copyright [year] [name of copyright owner]".
+#
+# Copyright 2026 3A Systems, LLC.
+
+# Check the real summary for absent, partial and complete disturbances, without Docker.
+set -eE -o pipefail
+# The fixture summaries must not be appended to the real GitHub Actions job summary.
+unset GITHUB_STEP_SUMMARY
+HERE=$(cd "$(dirname "$0")" && pwd)
+OUT=$(mktemp -d)
+trap 'rm -rf "$OUT"' EXIT
+. "$HERE/collect.sh"
+TOPOLOGY=rs534 ALL_NODES="d0 d1 r0 r1" DS_NODES="d0 d1" RS_ONLY="r0 r1"
+DURATION=900 DISTURB_AT=300 LOAD_MOD_RATE=250 LOAD_ADD_RATE=50 SAMPLE_DATA=10000 PHASE_TIMES=
+failed=0 checks=0
+check() { # <case> <expected disturbance rows>
+ summarize failed
+ checks=$((checks + 1))
+ got=$(sed -n '/^| disturbance |/p' "$OUT/summary.md")
+ [ "$got" = "$2" ] || { echo "::error::$1: got '$got', expected '$2'"; failed=$((failed + 1)); }
+}
+DISTURBANCE=none
+check "no disturbance" ""
+for DISTURBANCE in restart-rs kill-rs; do
+ . "$HERE/disturb-$DISTURBANCE.sh"
+ rm -f "$OUT/disturbances.txt"
+ unset LOAD_START_MS
+ check "$DISTURBANCE load not started" "| disturbance | $DISTURBANCE at 300 s into the load, not started: the run ended before the load |"
+ LOAD_START_MS=100000
+ check "$DISTURBANCE not completed" "| disturbance | $DISTURBANCE scheduled at 300 s into the load, not completed: 0 of 1 steps completed |"
+ how=restart
+ [ "$DISTURBANCE" != kill-rs ] || how=kill
+ echo "400000 445000 r0 $how" >"$OUT/disturbances.txt"
+ check "$DISTURBANCE completed" "| disturbance | $how of r0 300 s into the load, serving again 45 s later |"
+done
+for DISTURBANCE in restart-ds kill-ds; do
+ . "$HERE/disturb-$DISTURBANCE.sh"
+ rm "$OUT/disturbances.txt"
+ check "$DISTURBANCE not completed" "| disturbance | $DISTURBANCE scheduled at 300 s into the load, not completed: 0 of 1 steps completed |"
+ how=restart
+ [ "$DISTURBANCE" != kill-ds ] || how=kill
+ echo "400000 435000 d1 $how" >"$OUT/disturbances.txt"
+ check "$DISTURBANCE completed" "| disturbance | $how of d1 300 s into the load, serving again 35 s later |"
+done
+echo "summary: $((checks - failed)) checks passed, $failed failed"
+[ "$failed" -eq 0 ]
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 06fca12..a6843b1 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -1136,8 +1136,15 @@
fail-fast: false
matrix:
topology: [rs534, combined]
- # none: the base run; restart-ds: a directory server restarts during the load
- disturbance: [none, restart-ds]
+ # none: the base run; restart-ds and kill-ds: a directory server restarts or crashes;
+ # kill-rs and restart-rs: one RS is killed or gracefully restarted per fresh topology
+ # while every DS writes. Only rs534 has replication servers apart from directory servers.
+ disturbance: [none, restart-ds, kill-ds, kill-rs, restart-rs]
+ exclude:
+ - topology: combined
+ disturbance: kill-rs
+ - topology: combined
+ disturbance: restart-rs
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
@@ -1145,12 +1152,15 @@
.github/scripts/replication
# what a green row does not show: what a disturbance window of the errors log check
# expects - nothing it expects is logged outside the window there, and a base row has
- # none - and where restart-ds opens and closes the window
- - name: Self-tests of the errors log check and of restart-ds
+ # none - and where each disturbance opens and closes its windows
+ - name: Self-tests of the errors log check, disturbances and summary
shell: bash
run: |
.github/scripts/replication/selftest-replication-errors.sh
.github/scripts/replication/selftest-disturb-restart-ds.sh
+ .github/scripts/replication/selftest-disturb-kill-ds.sh
+ .github/scripts/replication/selftest-disturb-rs.sh
+ .github/scripts/replication/selftest-summary.sh
- name: Download the image
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
@@ -1164,10 +1174,8 @@
env:
TOPOLOGY: ${{ matrix.topology }}
DISTURBANCE: ${{ matrix.disturbance }}
- # a disturbed row runs half the load of the base one: what it is after happens around
- # the disturbance, a third of the way in, and for as long as it runs it holds one of
- # the runners the whole organization shares
- DURATION: ${{ matrix.disturbance == 'none' && '1800' || '900' }}
+ # Every row runs 900 s of load; disturbed rows act a third of the way in.
+ DURATION: "900"
# the replicas fall behind during the load and catch up after it: 900 s at 325 and 65
# a second converged in 396 s (run 38944365728), 1800 s at 423 and 85 in 2489 s (run
# 39043600905), the backlog growing faster than the load above the replay capacity; a
--
Gitblit v1.10.0