From c4d91fab9e005179803a3ab978e8f2896a5a5b16 Mon Sep 17 00:00:00 2001
From: Valery Kharseko <vharseko@3a-systems.ru>
Date: Wed, 28 Oct 2026 16:25:28 +0000
Subject: [PATCH] [#1208] Restart a directory server during the load of the replication convergence run, in rows of its own (#1233)
---
.github/scripts/replication/assert-converged.sh | 64 ++++++++++++++++++++++++++++++--
1 files changed, 60 insertions(+), 4 deletions(-)
diff --git a/.github/scripts/replication/assert-converged.sh b/.github/scripts/replication/assert-converged.sh
index d3b8d85..98ea1d6 100644
--- a/.github/scripts/replication/assert-converged.sh
+++ b/.github/scripts/replication/assert-converged.sh
@@ -141,12 +141,18 @@
# The number of lines each server's errors log held once the topology was up: the joins log
# replication errors of their own while the servers come up one after the other, and only what
-# follows is the run's
+# follows is the run's. A log that cannot be read fails the run: a cat that fails in a pipe
+# would count 0 lines and exit 0, and window_open would take that for a window from the first
+# line, while a redirect that fails ends sh with 2
+errors_lines() { # <node>
+ docker exec $1 sh -c 'wc -l </opt/opendj/data/logs/errors'
+}
+
mark_logs() {
local n
mkdir -p "$OUT/logs"
for n in $ALL_NODES; do
- docker exec $n sh -c 'cat /opt/opendj/data/logs/errors | wc -l' >"$OUT/logs/$n.mark"
+ errors_lines $n >"$OUT/logs/$n.mark"
done
}
@@ -154,6 +160,57 @@
docker exec $1 tail -n +$(($(cat "$OUT/logs/$1.mark") + 1)) /opt/opendj/data/logs/errors
}
+# A disturbance window over the errors log of every server (#1208): window_open before the
+# disturbance, window_close once the disturbed server serves again, a line "<lines before>
+# <lines after>" in logs/<node>.windows
+window_open() {
+ local n
+ for n in $ALL_NODES; do
+ errors_lines $n >"$OUT/logs/$n.window"
+ done
+}
+
+window_close() {
+ local n lines
+ for n in $ALL_NODES; do
+ lines=$(errors_lines $n)
+ echo "$(cat "$OUT/logs/$n.window") $lines" >>"$OUT/logs/$n.windows"
+ rm "$OUT/logs/$n.window"
+ done
+}
+
+# The replication errors <node> logged after mark_logs. A disturbance window expects the
+# handshakes that a server starting on a loaded runner fails, the other end giving up first: a
+# directory server (178) or a replication server (179) that disconnected in the handshake, both
+# at ERROR on the replication server it connected to, a replication server whose connection
+# crossed one the other way and lost the domain lock to it (145, a warning that abortStart logs
+# at ERROR, as it does 178 and 179: #1241), and the SSLHandshakeException of a server whose peer
+# closed the handshake once its read timed out. In run 39909337627 a restarted directory server
+# timed out connecting (212) and its replication server logged 178, and a replication server
+# logged 311 with SocketTimeoutException when the restarted server logged 119 with "Remote host
+# terminated the handshake"; in run 40463887505 a replication server logged 145 on the
+# connection of the restarted one. Those go to logs/<node>.expected; outside a window, and
+# anything else inside one, they are errors. A window expects them on any server, not only on
+# the connections of the disturbed one: 178 and 179 name the servers by id alone, and a server
+# has one in each of its domains and another for its replication server. A handshake that fails
+# is followed by a connection elsewhere, and a change it lost the checks of the data find
+replication_errors() { # <node>
+ local windows=
+ [ ! -f "$OUT/logs/$1.windows" ] || windows=$(tr '\n' ' ' <"$OUT/logs/$1.windows")
+ since_mark $1 | awk -v first=$(($(cat "$OUT/logs/$1.mark") + 1)) -v windows="$windows" \
+ -v expected="$OUT/logs/$1.expected" '
+ BEGIN { n = split(windows, w, " ") }
+ !/category=SYNC severity=ERROR|SSLHandshakeException|bad_certificate/ { next }
+ {
+ line = first + NR - 1
+ if (/category=SYNC severity=ERROR msgID=(145|178|179) / \
+ || /category=SYNC severity=WARNING .*SSLHandshakeException\(Remote host terminated the handshake\)/)
+ for (i = 1; i < n; i += 2)
+ if (line > w[i] + 0 && line <= w[i + 1] + 0) { print >expected; next }
+ print
+ }'
+}
+
# How often each directory server connected to a replication server again after the topology
# came up, whatever made it: a switch the broker decided on (msgID 188, or 213 to spread the
# load), a replication server that closed the connection (63), a connection that broke. Each
@@ -218,10 +275,9 @@
fi
done
for n in $ALL_NODES; do
- if ! lines=$(since_mark $n); then
+ if ! lines=$(replication_errors $n); then
echo "::error::the errors log of $n could not be read"; failed=1; continue
fi
- lines=$(grep -E "category=SYNC severity=ERROR|SSLHandshakeException|bad_certificate" <<<"$lines" || true)
[ -z "$lines" ] || { printf '%s\n' "$lines" >"$OUT/logs/$n.errors"; echo "::error::$n logged replication errors, see logs/$n.errors"; failed=1; }
done
assert_rs_mesh || failed=1
--
Gitblit v1.10.0