10 files modified
7 files added
| | |
| | | } |
| | | |
| | | # A disturbance window over the errors log of every server (#1208): window_open before the |
| | | # disturbance, window_close once the disturbed server serves again, a line "<lines before> |
| | | # <lines after>" in logs/<node>.windows |
| | | window_open() { |
| | | local n |
| | | # disturbance, with the msgIDs of the errors it expects besides the failed handshakes, |
| | | # or dns:<hostname> for ERROR 51 naming exactly that temporarily unavailable host, |
| | | # window_close once the disturbed server serves again, a line "<lines before> <lines after> |
| | | # [<msgID>|dns:<hostname>...]" in logs/<node>.windows |
| | | window_open() { # [<msgID>|dns:<hostname>...] |
| | | local n lines |
| | | for n in $ALL_NODES; do |
| | | errors_lines $n >"$OUT/logs/$n.window" |
| | | # an assignment, so that a log that cannot be read stops the run here as well |
| | | lines=$(errors_lines $n) |
| | | echo "$lines${*:+ $*}" >"$OUT/logs/$n.window" |
| | | done |
| | | } |
| | | |
| | | window_close() { |
| | | local n lines |
| | | local n lines before ids |
| | | for n in $ALL_NODES; do |
| | | lines=$(errors_lines $n) |
| | | echo "$(cat "$OUT/logs/$n.window") $lines" >>"$OUT/logs/$n.windows" |
| | | read -r before ids <"$OUT/logs/$n.window" |
| | | echo "$before $lines${ids:+ $ids}" >>"$OUT/logs/$n.windows" |
| | | rm "$OUT/logs/$n.window" |
| | | done |
| | | } |
| | |
| | | # timed out connecting (212) and its replication server logged 178, and a replication server |
| | | # logged 311 with SocketTimeoutException when the restarted server logged 119 with "Remote host |
| | | # terminated the handshake"; in run 40463887505 a replication server logged 145 on the |
| | | # connection of the restarted one. Those go to logs/<node>.expected; outside a window, and |
| | | # anything else inside one, they are errors. A window expects them on any server, not only on |
| | | # the connections of the disturbed one: 178 and 179 name the servers by id alone, and a server |
| | | # has one in each of its domains and another for its replication server. A handshake that fails |
| | | # is followed by a connection elsewhere, and a change it lost the checks of the data find |
| | | # connection of the restarted one. A window opened with msgIDs expects the SYNC errors of those |
| | | # msgIDs as well. A dns:<hostname> token expects only ERROR 51's exact resolution-failure |
| | | # message for that hostname, not other DNS errors inside the window. This covers the stopped |
| | | # RS container's name temporarily disappearing during kill-rs, restart-rs or a combined |
| | | # kill-ds or restart-ds. DS-only crashes and restarts pass no DNS token, and other hosts remain |
| | | # errors. |
| | | # The other allowances cover what one disturbance leaves behind besides the handshakes, such |
| | | # as the connections a killed replication server breaks (disturb-kill-rs.sh), which a window of |
| | | # another one does not expect. Those go to logs/<node>.expected; outside a window, and anything |
| | | # else inside one, they are errors. A window expects them on any server, not only on the |
| | | # connections of the disturbed one: 178 and 179 name the servers by id alone, and a server has |
| | | # one in each of its domains and another for its replication server. A handshake that fails is |
| | | # followed by a connection elsewhere, and a change it lost the checks of the data find |
| | | replication_errors() { # <node> |
| | | local windows= |
| | | [ ! -f "$OUT/logs/$1.windows" ] || windows=$(tr '\n' ' ' <"$OUT/logs/$1.windows") |
| | | [ ! -f "$OUT/logs/$1.windows" ] || windows=$(tr '\n' ';' <"$OUT/logs/$1.windows") |
| | | since_mark $1 | awk -v first=$(($(cat "$OUT/logs/$1.mark") + 1)) -v windows="$windows" \ |
| | | -v expected="$OUT/logs/$1.expected" ' |
| | | BEGIN { n = split(windows, w, " ") } |
| | | BEGIN { |
| | | n = split(windows, w, ";") |
| | | for (i = 1; i <= n; i++) { |
| | | k = split(w[i], f, " ") |
| | | from[i] = f[1]; to[i] = f[2]; ids[i] = " " |
| | | for (j = 3; j <= k; j++) ids[i] = ids[i] f[j] " " |
| | | } |
| | | } |
| | | !/category=SYNC severity=ERROR|SSLHandshakeException|bad_certificate/ { next } |
| | | { |
| | | line = first + NR - 1 |
| | | if (/category=SYNC severity=ERROR msgID=(145|178|179) / \ |
| | | || /category=SYNC severity=WARNING .*SSLHandshakeException\(Remote host terminated the handshake\)/) |
| | | for (i = 1; i < n; i += 2) |
| | | if (line > w[i] + 0 && line <= w[i + 1] + 0) { print >expected; next } |
| | | handshake = /category=SYNC severity=ERROR msgID=(145|178|179) / \ |
| | | || /category=SYNC severity=WARNING .*SSLHandshakeException\(Remote host terminated the handshake\)/ |
| | | id = "" |
| | | if (match($0, /category=SYNC severity=ERROR msgID=[0-9]+ /)) { |
| | | id = substr($0, RSTART, RLENGTH) |
| | | sub(/.*msgID=/, "", id) |
| | | sub(/ $/, "", id) |
| | | } |
| | | dns = "" |
| | | if (id == "51") { |
| | | message = substr($0, index($0, " msg=") + 5) |
| | | if (message ~ /^The hostname [^ ]+ could not be resolved as an IP address$/) { |
| | | split(message, parts, " ") |
| | | dns = " dns:" parts[3] " " |
| | | } |
| | | } |
| | | for (i = 1; i <= n; i++) |
| | | if (line > from[i] + 0 && line <= to[i] + 0 \ |
| | | && (handshake || index(ids[i], " " id " ") || (dns != "" && index(ids[i], dns)))) { |
| | | print >expected |
| | | next |
| | | } |
| | | print |
| | | }' |
| | | } |
| | |
| | | |
| | | # A directory server is connected to one replication server at a time, so the replication |
| | | # servers reach each other only through their own connections: each one names every peer it is |
| | | # connected to "Connected replication server RS(<id>) <host:port>" (ReplicationServerHandler) |
| | | # connected to "Connected replication server RS(<id>) <host:port>" (ReplicationServerHandler). |
| | | # The number of them, 0 where the search fails |
| | | rs_peers() { # <replication server> |
| | | admin_search $1 "$MONITOR_DOMAIN" sub "(cn=Connected replication server*)" 1.1 | grep -c '^dn:' || true |
| | | } |
| | | |
| | | assert_rs_mesh() { |
| | | local rs want got failed= |
| | | want=$(($(echo $RS_NODES | wc -w) - 1)) |
| | | for rs in $RS_NODES; do |
| | | got=$(admin_search $rs "$MONITOR_DOMAIN" sub "(cn=Connected replication server*)" 1.1 | grep -c '^dn:' || true) |
| | | got=$(rs_peers $rs) |
| | | [ "${got:-0}" -eq "$want" ] || { echo "::error::$rs is connected to ${got:-0} replication servers, not $want"; failed=1; } |
| | | done |
| | | [ -z "$failed" ] |
| | |
| | | |
| | | # The summary of the run, into $OUT/summary.md and, in GitHub Actions, the job summary |
| | | summarize() { # <result> |
| | | local completed=0 |
| | | { |
| | | echo "### Replication convergence ($TOPOLOGY): $1" |
| | | echo |
| | |
| | | echo "| load | $DURATION s on each directory server, at most $LOAD_MOD_RATE modifications and $LOAD_ADD_RATE adds or deletes a second |" |
| | | echo "| seeded users | $SAMPLE_DATA |" |
| | | if [ -s "$OUT/disturbances.txt" ]; then |
| | | completed=$(wc -l <"$OUT/disturbances.txt") |
| | | awk -v start="${LOAD_START_MS:-0}" '{ printf "| disturbance | %s of %s %d s into the load, serving again %d s later |\n", |
| | | $4, $3, ($1 - start) / 1000, ($2 - $1) / 1000 }' "$OUT/disturbances.txt" |
| | | elif [ "$DISTURBANCE" != none ] && [ -z "${LOAD_START_MS:-}" ]; then |
| | | fi |
| | | if [ "$DISTURBANCE" != none ] && [ -z "${LOAD_START_MS:-}" ]; then |
| | | # run.sh sets LOAD_START_MS as the load starts |
| | | echo "| disturbance | $DISTURBANCE at $DISTURB_AT s into the load, not started: the run ended before the load |" |
| | | elif [ "$DISTURBANCE" != none ]; then |
| | | echo "| disturbance | $DISTURBANCE at $DISTURB_AT s into the load, not completed |" |
| | | elif [ "$DISTURBANCE" != none ] && [ "$completed" -lt "$DISTURB_STEPS" ]; then |
| | | if [ "$completed" -eq 0 ]; then |
| | | echo "| disturbance | $DISTURBANCE scheduled at $DISTURB_AT s into the load, not completed: 0 of $DISTURB_STEPS steps completed |" |
| | | else |
| | | echo "| disturbance | $DISTURBANCE not completed: $((completed)) of $DISTURB_STEPS steps completed |" |
| | | fi |
| | | fi |
| | | echo "| converged | ${CONVERGED_IN:--} s after the load |" |
| | | if [ -s "$OUT/samples.txt" ]; then |
| | |
| | | fi |
| | | if ls "$OUT"/logs/*.expected >/dev/null 2>&1; then |
| | | echo |
| | | echo "Handshakes failed while a disturbed server came back, which the disturbance expects (logs/*.expected):" |
| | | echo "Errors logged while a disturbed server went down and came back, which the disturbance expects - failed handshakes, and what a kill leaves behind (logs/*.expected):" |
| | | echo |
| | | echo '```' |
| | | for f in "$OUT"/logs/*.expected; do |
| New file |
| | |
| | | # |
| | | # |
| | | # The contents of this file are subject to the terms of the Common Development and |
| | | # Distribution License (the License). You may not use this file except in compliance with the |
| | | # License. |
| | | # |
| | | # You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the |
| | | # specific language governing permission and limitations under the License. |
| | | # |
| | | # When distributing Covered Software, include this CDDL Header Notice in each file and include |
| | | # the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL |
| | | # Header, with the fields enclosed by brackets [] replaced by your own identifying |
| | | # information: "Portions copyright [year] [name of copyright owner]". |
| | | # |
| | | # Copyright 2026 3A Systems, LLC. |
| | | |
| | | # Crash and recover the same directory server as restart-ds (#1208). Its clients finish |
| | | # before the kill and resume after recovery; the other directory servers keep writing. |
| | | # In combined the container also runs an RS, so this crashes both roles on the same volume. |
| | | DISTURBED=$(echo $DS_NODES | cut -d' ' -f2) |
| | | DISTURB_STEPS=1 |
| | | |
| | | disturb() { |
| | | local from served |
| | | echo " [$SECONDS s] killing $DISTURBED" |
| | | # A killed DS leaves a broken DS connection (211). A combined server also breaks DS-to-RS |
| | | # (180) and RS-to-RS (181) connections and can recover a torn changelog record (284). Peers |
| | | # resolving it as an RS can log ERROR 51 while its container's name is unavailable. |
| | | # Allow these only during the crash/recovery window; 66/108 and data/gap checks stay strict. |
| | | case " $RS_NODES " in |
| | | *" $DISTURBED "*) window_open "dns:$DISTURBED" 180 181 211 284 ;; |
| | | *) window_open 211 ;; |
| | | esac |
| | | from=$(date +%s) |
| | | docker kill $DISTURBED >/dev/null |
| | | docker wait $DISTURBED >/dev/null |
| | | docker start $DISTURBED >/dev/null |
| | | wait_until $DISTURB_TIMEOUT "$DISTURBED serves again" started_since $DISTURBED $from |
| | | served=$(date +%s) |
| | | echo " [$SECONDS s] $DISTURBED serves again" |
| | | # Match restart-ds: pending handshakes can finish after the startup message. |
| | | sleep 10 |
| | | window_close |
| | | echo "$((from * 1000)) $((served * 1000)) $DISTURBED kill" >>"$OUT/disturbances.txt" |
| | | } |
| | | |
| | | # As for restart-ds, finish the join before the final convergence wait. |
| | | settle() { |
| | | wait_until $DISTURB_TIMEOUT "the join of $DISTURBED ends" bootstrap_ended $DISTURBED |
| | | } |
| New file |
| | |
| | | # |
| | | # |
| | | # The contents of this file are subject to the terms of the Common Development and |
| | | # Distribution License (the License). You may not use this file except in compliance with the |
| | | # License. |
| | | # |
| | | # You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the |
| | | # specific language governing permission and limitations under the License. |
| | | # |
| | | # When distributing Covered Software, include this CDDL Header Notice in each file and include |
| | | # the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL |
| | | # Header, with the fields enclosed by brackets [] replaced by your own identifying |
| | | # information: "Portions copyright [year] [name of copyright owner]". |
| | | # |
| | | # Copyright 2026 3A Systems, LLC. |
| | | |
| | | # Kill and recover one replication-only server under load (#1208). The other RS stays up |
| | | # throughout this run, including while the recovered server catches up after the load. |
| | | . "$HERE/rs-disturbance.sh" |
| | | |
| | | # A kill leaves DS disconnects (180), RS disconnects (181), and possibly recovery of an |
| | | # incomplete changelog record (284). The stopped container's name may also stop resolving |
| | | # (51, observed in CI run 42046591833); expect that message only for this RS's exact hostname. |
| | | # These SYNC errors are expected only inside its window; monitor-send errors 66/108 and all |
| | | # other unexpected errors remain failures. |
| | | disturb() { |
| | | down_and_up kill "dns:$DISTURBED" 180 181 284 |
| | | } |
| | |
| | | # runs restarts with it. |
| | | |
| | | DISTURBED=$(echo $DS_NODES | cut -d' ' -f2) |
| | | DISTURB_STEPS=1 |
| | | |
| | | # docker restart sends SIGTERM and kills the server once its timeout is over: 10 s by default, |
| | | # less than a server under load may need to stop, and a killed server is another disturbance. |
| | |
| | | # its start fails on a loaded runner are expected there (replication_errors). The window closes |
| | | # 10 s after the server serves again: a handshake its start began can still fail then, and a |
| | | # peer logs 145 only once its wait for the domain lock, of up to 8 s, is over |
| | | # (ServerHandler.lockDomainWithTimeout). A handshake that fails later is an error again |
| | | # (ServerHandler.lockDomainWithTimeout). A handshake that fails later is an error again. |
| | | # A combined server also restarts its RS, whose hostname can temporarily stop resolving. |
| | | # Expect ERROR 51 only for that exact hostname inside this window; a DS-only restart receives |
| | | # no DNS allowance, and disconnect, recovery and monitor errors remain failures. |
| | | disturb() { |
| | | local from served |
| | | echo " [$SECONDS s] restarting $DISTURBED" |
| | | window_open |
| | | case " $RS_NODES " in |
| | | *" $DISTURBED "*) window_open "dns:$DISTURBED" ;; |
| | | *) window_open ;; |
| | | esac |
| | | from=$(date +%s) |
| | | docker restart -t 120 $DISTURBED >/dev/null |
| | | wait_until $DISTURB_TIMEOUT "$DISTURBED serves again" started_since $DISTURBED $from |
| New file |
| | |
| | | # |
| | | # |
| | | # The contents of this file are subject to the terms of the Common Development and |
| | | # Distribution License (the License). You may not use this file except in compliance with the |
| | | # License. |
| | | # |
| | | # You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the |
| | | # specific language governing permission and limitations under the License. |
| | | # |
| | | # When distributing Covered Software, include this CDDL Header Notice in each file and include |
| | | # the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL |
| | | # Header, with the fields enclosed by brackets [] replaced by your own identifying |
| | | # information: "Portions copyright [year] [name of copyright owner]". |
| | | # |
| | | # Copyright 2026 3A Systems, LLC. |
| | | |
| | | # Gracefully restart one replication-only server under load (#1208). The other RS stays up |
| | | # throughout this run. A separate kill-rs run exercises crash recovery on a fresh topology. |
| | | . "$HERE/rs-disturbance.sh" |
| | | |
| | | # A graceful RS restart under load can still leave DS receive failures (180), as in run |
| | | # 42190249091. Expect those only inside this restart window; RS disconnects (181) and torn |
| | | # changelog recovery (284) remain errors. Docker can briefly stop resolving the container's |
| | | # name between its exit and start even after an orderly stop; allow ERROR 51 only for that RS. |
| | | disturb() { |
| | | down_and_up restart "dns:$DISTURBED" 180 |
| | | } |
| | |
| | | |
| | | TOPOLOGY=${TOPOLOGY:-rs534} |
| | | # what disturbs the topology during the load (#1208): none, or the <name> of a disturb-<name>.sh |
| | | # next to this script, which names the DISTURBED server and defines disturb, run during the load, |
| | | # next to this script, which names the DISTURBED servers - one directory server, or replication |
| | | # servers - the number of steps in DISTURB_STEPS, and defines disturb, run during the load, |
| | | # and settle, run after it |
| | | DISTURBANCE=${DISTURBANCE:-none} |
| | | # seconds of load. A disturbed run takes the load of its CI rows: a restarted server served |
| | |
| | | # The load of the replication convergence run (#1090), sourced by run.sh: every directory |
| | | # server is written to at once, for DURATION seconds, by rate tools that run in containers of |
| | | # the same image on the network of the topology. Only writes: a read sends nothing to the |
| | | # replication servers. The server a disturbance takes down (#1208) is written to before it and |
| | | # after it, by clients of their own. |
| | | # replication servers. A directory server a disturbance takes down (#1208) is written to before |
| | | # it and after it, by clients of their own; replication servers it takes down leave the clients |
| | | # be, and the directory servers write through the replication server still up. |
| | | |
| | | # <client> <tool> <argument>... : a container that runs one tool of the image against one server |
| | | start_client() { |
| | |
| | | fi |
| | | done |
| | | if [ "$DISTURBANCE" != none ]; then |
| | | # the clients of the disturbed server end on their own before the disturbance, which then |
| | | # cuts none of their operations off: one that had its CSN would read as a lost change |
| | | docker wait load-mod-$DISTURBED load-add-$DISTURBED >/dev/null |
| | | disturb |
| | | left=$((DURATION - (SECONDS - start))) |
| | | [ "$left" -ge $((3 * STAT_INTERVAL)) ] \ |
| | | || fail "$DISTURBED served again $((SECONDS - start)) s into the load of $DURATION s, too late to take load again" |
| | | start_clients $DISTURBED 2 $left |
| | | case " $DS_NODES " in |
| | | *" $DISTURBED "*) |
| | | # the clients of the disturbed server end on their own before the disturbance, which |
| | | # then cuts none of their operations off: one that had its CSN would read as a lost change |
| | | docker wait load-mod-$DISTURBED load-add-$DISTURBED >/dev/null |
| | | disturb |
| | | left=$((DURATION - (SECONDS - start))) |
| | | [ "$left" -ge $((3 * STAT_INTERVAL)) ] \ |
| | | || fail "$DISTURBED served again $((SECONDS - start)) s into the load of $DURATION s, too late to take load again" |
| | | start_clients $DISTURBED 2 $left |
| | | ;; |
| | | *) |
| | | left=$((DISTURB_AT - (SECONDS - start))) |
| | | [ "$left" -le 0 ] || sleep $left |
| | | disturb |
| | | ;; |
| | | esac |
| | | fi |
| | | for c in $CLIENTS; do |
| | | docker wait $c >/dev/null |
| New file |
| | |
| | | # |
| | | # |
| | | # The contents of this file are subject to the terms of the Common Development and |
| | | # Distribution License (the License). You may not use this file except in compliance with the |
| | | # License. |
| | | # |
| | | # You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the |
| | | # specific language governing permission and limitations under the License. |
| | | # |
| | | # When distributing Covered Software, include this CDDL Header Notice in each file and include |
| | | # the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL |
| | | # Header, with the fields enclosed by brackets [] replaced by your own identifying |
| | | # information: "Portions copyright [year] [name of copyright owner]". |
| | | # |
| | | # Copyright 2026 3A Systems, LLC. |
| | | |
| | | # Shared lifecycle for the independent kill-rs and restart-rs convergence scenarios (#1208). |
| | | # Each run takes down one RS-only server while the other continues serving every DS. The |
| | | # combined topology also takes down a DS and belongs to a different disturbance. |
| | | [ $(echo $RS_ONLY | wc -w) -ge 2 ] \ |
| | | || { echo "$DISTURBANCE needs at least two replication servers without data: TOPOLOGY=rs534" >&2; exit 2; } |
| | | DISTURBED=$(echo $RS_ONLY | cut -d' ' -f1) |
| | | DISTURB_STEPS=1 |
| | | |
| | | # Startup after <seconds> and connections to every other RS establish availability. Changelog |
| | | # catch-up is checked by the final convergence/data checks after the load; no second RS is |
| | | # stopped in this run, so the recovered RS need not catch up before another disturbance. |
| | | rs_serves_since() { # <rs> <seconds> |
| | | started_since $1 $2 && [ "$(rs_peers $1)" -eq $(($(echo $RS_NODES | wc -w) - 1)) ] |
| | | } |
| | | |
| | | # <kill|restart> [<msgID>|dns:<hostname>...]: open the expected-error window, recover the RS, and retain |
| | | # its actual downtime. A kill needs docker wait before start; a graceful restart gets 120 s |
| | | # to stop. The window includes 10 s after startup/peer connections for pending handshakes. |
| | | # DS load balancing may move connections later or not at all, so it does not end the window. |
| | | down_and_up() { |
| | | local how=$1 rs=$DISTURBED from served left |
| | | shift |
| | | window_open "$@" |
| | | from=$(date +%s) |
| | | # Opening the window reads each server's log and can itself be slow. Check the remaining |
| | | # load immediately before the operation, including when client startup delayed disturb(). |
| | | left=$((LOAD_START_MS / 1000 + DURATION - from)) |
| | | [ "$left" -ge $((3 * STAT_INTERVAL)) ] \ |
| | | || fail "only $left s left in the load of $DURATION s, too late to $how $rs under it" |
| | | echo " [$SECONDS s] $how $rs" |
| | | if [ "$how" = kill ]; then |
| | | docker kill $rs >/dev/null |
| | | docker wait $rs >/dev/null |
| | | docker start $rs >/dev/null |
| | | else |
| | | docker restart -t 120 $rs >/dev/null |
| | | fi |
| | | wait_until $DISTURB_TIMEOUT "$rs serves again" rs_serves_since $rs $from |
| | | served=$(date +%s) |
| | | echo " [$SECONDS s] $rs serves again" |
| | | sleep 10 |
| | | window_close |
| | | echo "$((from * 1000)) $((served * 1000)) $rs $how" >>"$OUT/disturbances.txt" |
| | | # Record a completed recovery even if it was too slow, but fail rather than silently |
| | | # accepting a restart that left no useful load. Match restart-ds's three sample intervals. |
| | | left=$((LOAD_START_MS / 1000 + DURATION - $(date +%s))) |
| | | [ "$left" -ge $((3 * STAT_INTERVAL)) ] \ |
| | | || fail "$rs recovered too late: only $left s left in the load of $DURATION s" |
| | | } |
| | | |
| | | # The image runs its join on every start. Leave the disturbed server as topology_up does |
| | | # before the final convergence wait, without requiring catch-up while writes are still active. |
| | | settle() { |
| | | wait_until $DISTURB_TIMEOUT "the join of $DISTURBED ends" bootstrap_ended $DISTURBED |
| | | } |
| New file |
| | |
| | | #!/usr/bin/env bash |
| | | # |
| | | # |
| | | # The contents of this file are subject to the terms of the Common Development and |
| | | # Distribution License (the License). You may not use this file except in compliance with the |
| | | # License. |
| | | # |
| | | # You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the |
| | | # specific language governing permission and limitations under the License. |
| | | # |
| | | # When distributing Covered Software, include this CDDL Header Notice in each file and include |
| | | # the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL |
| | | # Header, with the fields enclosed by brackets [] replaced by your own identifying |
| | | # information: "Portions copyright [year] [name of copyright owner]". |
| | | # |
| | | # Copyright 2026 3A Systems, LLC. |
| | | |
| | | # Exercise a DS crash in both topologies without Docker. Keep the real disturbance, |
| | | # load segmentation and summary; only container commands, readiness and time are stubs. |
| | | set -eE -o pipefail |
| | | HERE=$(cd "$(dirname "$0")" && pwd) |
| | | OUT=$(mktemp -d) |
| | | trap 'rm -rf "$OUT"' EXIT |
| | | failed=0 checks=0 |
| | | check() { |
| | | checks=$((checks + 1)) |
| | | [ "$2" = "$3" ] || { echo "::error::$1: got '$2', expected '$3'"; failed=$((failed + 1)); } |
| | | } |
| | | |
| | | for TOPOLOGY in rs534 combined; do |
| | | DISTURBANCE=kill-ds DURATION=900 DISTURB_AT=300 DISTURB_TIMEOUT=900 |
| | | if [ "$TOPOLOGY" = rs534 ]; then |
| | | DS_NODES="d0 d1 d2 d3" RS_NODES="r0 r1" extras="211" target=d1 |
| | | else |
| | | DS_NODES="c0 c1 c2 c3" RS_NODES="$DS_NODES" extras="dns:c1 180 181 211 284" target=c1 |
| | | fi |
| | | [ -f "$HERE/disturb-kill-ds.sh" ] || { echo "::error::kill-ds is not implemented"; exit 1; } |
| | | . "$HERE/disturb-kill-ds.sh" |
| | | check "$TOPOLOGY target" "$DISTURBED" "$target" |
| | | check "$TOPOLOGY steps" "$DISTURB_STEPS" 1 |
| | | unset SECONDS |
| | | SECONDS=0 now=400 calls= |
| | | date() { echo "$now"; } |
| | | sleep() { calls="$calls sleep $1"; now=$((now + $1)); } |
| | | docker() { |
| | | calls="$calls docker $*" |
| | | echo "$*" >>"$OUT/docker.calls" |
| | | [ "${DOCKER_FAIL:-}" != "$1" ] || return 1 |
| | | now=$((now + 5)) |
| | | } |
| | | wait_until() { |
| | | calls="$calls wait_until $1 [$2] ${*:3}" |
| | | [ "${WAIT_FAIL:-no}" != yes ] || return 1 |
| | | now=$((now + 30)) |
| | | } |
| | | window_open() { calls="$calls window_open $*"; } |
| | | window_close() { calls="$calls window_close"; } |
| | | rm -f "$OUT/disturbances.txt" |
| | | disturb >/dev/null |
| | | check "$TOPOLOGY crash lifecycle and error window" "$calls" \ |
| | | " window_open $extras docker kill $DISTURBED docker wait $DISTURBED docker start $DISTURBED wait_until 900 [$DISTURBED serves again] started_since $DISTURBED 400 sleep 10 window_close" |
| | | check "$TOPOLOGY completed crash" "$(cat "$OUT/disturbances.txt")" "400000 445000 $DISTURBED kill" |
| | | calls= |
| | | settle |
| | | check "$TOPOLOGY bootstrap" "$calls" " wait_until 900 [the join of $DISTURBED ends] bootstrap_ended $DISTURBED" |
| | | |
| | | # Failures run outside if/|| so errexit has the same effect as in run.sh. |
| | | for failure in kill wait start readiness; do |
| | | rm -f "$OUT/disturbances.txt" |
| | | : >"$OUT/docker.calls" |
| | | set +e |
| | | ( |
| | | set -e |
| | | if [ "$failure" = readiness ]; then WAIT_FAIL=yes; else DOCKER_FAIL=$failure; fi |
| | | disturb >/dev/null |
| | | ) |
| | | status=$? |
| | | set -e |
| | | check "$TOPOLOGY $failure failure" "$status" 1 |
| | | check "$TOPOLOGY $failure leaves no completed step" "$(test -e "$OUT/disturbances.txt" && echo yes || echo no)" no |
| | | if [ "$failure" = kill ] || [ "$failure" = wait ]; then |
| | | check "$TOPOLOGY $failure must not start" "$(grep -c '^start ' "$OUT/docker.calls" || true)" 0 |
| | | fi |
| | | done |
| | | done |
| | | |
| | | # The killed DS's clients finish before the crash and resume only after recovery. |
| | | # The other DS keeps writing throughout. A slow recovery must still leave useful load. |
| | | . "$HERE/load.sh" |
| | | DS_NODES="d0 d1" DISTURBED=d1 |
| | | start_clients() { calls="$calls clients $*"; } |
| | | docker() { calls="$calls docker $*"; } |
| | | disturb() { calls="$calls disturb"; SECONDS=$((SECONDS + RECOVERY)); } |
| | | fail() { echo "::error::$1"; false; } |
| | | for RECOVERY in 870 871; do |
| | | calls= SECONDS=0 |
| | | set +e |
| | | (set -e; run_load; echo "$calls" >"$OUT/load.calls") >"$OUT/load.log" |
| | | status=$? |
| | | set -e |
| | | if [ "$RECOVERY" = 870 ]; then |
| | | check "30 s of resumed load" "$status" 0 |
| | | check "clients around the crash" "$(cat "$OUT/load.calls")" \ |
| | | " clients d0 1 900 clients d1 1 300 docker wait load-mod-d1 load-add-d1 disturb clients d1 2 30" |
| | | else |
| | | check "29 s is too late to resume" "$status" 1 |
| | | check "slow recovery diagnostic" "$(grep -c 'too late to take load again' "$OUT/load.log" || true)" 1 |
| | | fi |
| | | done |
| | | |
| | | echo "disturb-kill-ds: $((checks - failed)) checks passed, $failed failed" |
| | | [ "$failed" -eq 0 ] |
| | |
| | | # Copyright 2026 3A Systems, LLC. |
| | | |
| | | # A self-test of the disturbance restart-ds of the replication convergence run (#1208), with no |
| | | # topology: what disturb and settle call, in which order, and what disturb writes to |
| | | # topology: what disturb, settle and run_load call, in which order, and what disturb writes to |
| | | # disturbances.txt. The window of the errors logs opens before the restart and closes 10 s after |
| | | # the server serves again, and the restart ends when the server served, not after the grace. A |
| | | # restart-ds row shows neither, unless a handshake record lands where the order moved. docker, |
| | |
| | | set -eE -o pipefail |
| | | |
| | | HERE=$(cd "$(dirname "$0")" && pwd) |
| | | DS_NODES="d0 d1" DISTURB_TIMEOUT=900 |
| | | DISTURB_TIMEOUT=900 |
| | | OUT=$(mktemp -d) |
| | | trap 'rm -rf "$OUT"' EXIT |
| | | . "$HERE/disturb-restart-ds.sh" |
| | | |
| | | now=100 calls= |
| | | date() { echo $now; } |
| | | sleep() { calls="$calls sleep $1"; now=$((now + $1)); } |
| | | docker() { calls="$calls docker $*"; now=$((now + 5)); } |
| | | wait_until() { calls="$calls wait_until $*"; now=$((now + 30)); } |
| | | window_open() { calls="$calls window_open"; } |
| | | wait_until() { calls="$calls wait_until $1 [$2] ${*:3}"; now=$((now + 30)); } |
| | | window_open() { calls="$calls window_open${*:+ $*}"; } |
| | | window_close() { calls="$calls window_close"; } |
| | | |
| | | failed=0 |
| | |
| | | [ "$2" = "$3" ] || { echo "::error::$1:$2, not$3"; failed=$((failed + 1)); } |
| | | } |
| | | |
| | | disturb >/dev/null |
| | | check "disturb called" "$calls" \ |
| | | " window_open docker restart -t 120 d1 wait_until 900 d1 serves again started_since d1 100 sleep 10 window_close" |
| | | check "disturbances.txt holds" " $(cat "$OUT/disturbances.txt")" " 100000 135000 d1 restart" |
| | | calls= |
| | | settle |
| | | check "settle called" "$calls" " wait_until 900 the join of d1 ends bootstrap_ended d1" |
| | | for TOPOLOGY in rs534 combined similar-host; do |
| | | case $TOPOLOGY in |
| | | rs534) DS_NODES="d0 d1" RS_NODES="r0 r1" target=d1 extras= ;; |
| | | combined) DS_NODES="c0 c1" RS_NODES="$DS_NODES" target=c1 extras=" dns:c1" ;; |
| | | similar-host) DS_NODES="d0 d1" RS_NODES="d10 d11" target=d1 extras= ;; |
| | | esac |
| | | . "$HERE/disturb-restart-ds.sh" |
| | | now=100 calls= |
| | | rm -f "$OUT/disturbances.txt" |
| | | disturb >/dev/null |
| | | check "$TOPOLOGY disturb called" "$calls" \ |
| | | " window_open$extras docker restart -t 120 $target wait_until 900 [$target serves again] started_since $target 100 sleep 10 window_close" |
| | | check "$TOPOLOGY disturbances.txt holds" " $(cat "$OUT/disturbances.txt")" " 100000 135000 $target restart" |
| | | calls= |
| | | settle |
| | | check "$TOPOLOGY settle called" "$calls" " wait_until 900 [the join of $target ends] bootstrap_ended $target" |
| | | done |
| | | |
| | | echo "disturb-restart-ds: $((3 - failed)) checks passed, $failed failed" |
| | | # run_load stops the clients of the disturbed server at DISTURB_AT, waits for them to end before |
| | | # it disturbs, and has clients of its own write to it for the rest of the load. SECONDS without |
| | | # its special properties, so that run_load reads the clock of the stubs: the restart takes 200 s |
| | | DURATION=900 DISTURB_AT=300 DISTURBANCE=restart-ds |
| | | DS_NODES="d0 d1" DISTURBED=d1 |
| | | . "$HERE/load.sh" |
| | | unset SECONDS |
| | | start_clients() { calls="$calls clients $*"; } |
| | | disturb() { calls="$calls disturb"; SECONDS=$((SECONDS + 200)); } |
| | | calls= SECONDS=0 |
| | | run_load |
| | | check "run_load called" "$calls" \ |
| | | " clients d0 1 900 clients d1 1 300 docker wait load-mod-d1 load-add-d1 disturb clients d1 2 700" |
| | | |
| | | echo "disturb-restart-ds: $((10 - failed)) checks passed, $failed failed" |
| | | [ "$failed" -eq 0 ] |
| New file |
| | |
| | | #!/usr/bin/env bash |
| | | # |
| | | # |
| | | # The contents of this file are subject to the terms of the Common Development and |
| | | # Distribution License (the License). You may not use this file except in compliance with the |
| | | # License. |
| | | # |
| | | # You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the |
| | | # specific language governing permission and limitations under the License. |
| | | # |
| | | # When distributing Covered Software, include this CDDL Header Notice in each file and include |
| | | # the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL |
| | | # Header, with the fields enclosed by brackets [] replaced by your own identifying |
| | | # information: "Portions copyright [year] [name of copyright owner]". |
| | | # |
| | | # Copyright 2026 3A Systems, LLC. |
| | | |
| | | # Exercise both independent RS disturbances without Docker. External commands and the clock |
| | | # are stubs; the entry points, readiness checks and run_load are the production functions. |
| | | set -eE -o pipefail |
| | | HERE=$(cd "$(dirname "$0")" && pwd) |
| | | OUT=$(mktemp -d) |
| | | trap 'rm -rf "$OUT"' EXIT |
| | | |
| | | variant() ( |
| | | local kind=$1 how=$2 recovery=$3 command=$4 extras=$5 |
| | | local code started peers |
| | | failed=0 checks=0 |
| | | check() { |
| | | checks=$((checks + 1)) |
| | | [ "$2" = "$3" ] || { echo "::error::$kind $1: got '$2', expected '$3'"; failed=$((failed + 1)); } |
| | | } |
| | | for topology in '' r0; do |
| | | code=0 |
| | | (RS_ONLY=$topology; . "$HERE/disturb-$kind.sh") >/dev/null 2>&1 || code=$? |
| | | check "refuses fewer than two RS-only servers" "$code" 2 |
| | | done |
| | | DS_NODES="d0 d1" RS_ONLY="r0 r1" RS_NODES="r0 r1" DURATION=900 DISTURB_AT=300 DISTURB_TIMEOUT=900 |
| | | DISTURBANCE=$kind LOAD_START_MS=100000 |
| | | . "$HERE/disturb-$kind.sh" |
| | | . "$HERE/load.sh" |
| | | check "disturbed server" "$DISTURBED" r0 |
| | | check "expected step count" "$DISTURB_STEPS" 1 |
| | | unset SECONDS |
| | | SECONDS=0 now=400 SERVE=30 calls= |
| | | date() { echo "$now"; } |
| | | sleep() { calls="$calls sleep $1"; now=$((now + $1)); SECONDS=$((SECONDS + $1)); } |
| | | docker() { |
| | | calls="$calls docker $*" |
| | | echo "$*" >>"$OUT/docker.calls" |
| | | [ "${DOCKER_FAIL:-}" != "$1" ] || return 1 |
| | | now=$((now + 5)) |
| | | } |
| | | wait_until() { |
| | | calls="$calls wait_until $1 [$2] ${*:3}" |
| | | [ "${WAIT_FAIL:-no}" != yes ] || fail "readiness timed out" |
| | | now=$((now + SERVE)) |
| | | } |
| | | window_open() { calls="$calls window_open${*:+ $*}"; now=$((now + ${WINDOW_DELAY:-0})); } |
| | | window_close() { calls="$calls window_close"; } |
| | | fail() { echo "::error::$1"; false; } |
| | | rm -f "$OUT/disturbances.txt" |
| | | disturb >"$OUT/disturb.log" |
| | | check "single operation and error window" "$calls" \ |
| | | " window_open$extras$command wait_until 900 [r0 serves again] rs_serves_since r0 400 sleep 10 window_close" |
| | | check "single completed step" "$(cat "$OUT/disturbances.txt")" "400000 $((400000 + recovery * 1000)) r0 $how" |
| | | calls= |
| | | settle |
| | | check "settle only the disturbed server" "$calls" " wait_until 900 [the join of r0 ends] bootstrap_ended r0" |
| | | |
| | | started_since() { [ "$1 $2" = 'r0 400' ] && [ "$started" = yes ]; } |
| | | rs_peers() { [ "$1" = r0 ] && echo "$peers"; } |
| | | serves() { if rs_serves_since r0 400; then echo yes; else echo no; fi; } |
| | | started=yes peers=1 |
| | | check "startup and peer present" "$(serves)" yes |
| | | peers=0 |
| | | check "missing peer" "$(serves)" no |
| | | peers=2 |
| | | check "unexpected peer count" "$(serves)" no |
| | | started=no peers=1 |
| | | check "no startup message" "$(serves)" no |
| | | started=yes peers=2 RS_NODES="r0 r1 r2" |
| | | check "three RS topology, both peers present" "$(serves)" yes |
| | | peers=1 |
| | | check "three RS topology, one peer absent" "$(serves)" no |
| | | RS_NODES="r0 r1" |
| | | |
| | | # Run failures outside if/|| so errexit remains active as it is in run.sh. |
| | | budget_case() { # <label> <start time> <window delay> <status> <completed> <docker calls> |
| | | local status lines |
| | | rm -f "$OUT/disturbances.txt" |
| | | : >"$OUT/docker.calls" |
| | | set +e |
| | | (set -e; now=$2 WINDOW_DELAY=$3; disturb >"$OUT/budget.log") |
| | | status=$? |
| | | set -e |
| | | check "$1 status" "$status" "$4" |
| | | lines=0 |
| | | [ ! -f "$OUT/disturbances.txt" ] || lines=$(awk 'END {print NR}' "$OUT/disturbances.txt") |
| | | check "$1 completed steps" "$lines" "$5" |
| | | check "$1 docker calls" "$(awk 'END {print NR}' "$OUT/docker.calls")" "$6" |
| | | if [ "$4" -ne 0 ]; then |
| | | check "$1 diagnostic" "$(grep -c 'too late' "$OUT/budget.log" || true)" 1 |
| | | fi |
| | | } |
| | | op_calls=1 |
| | | [ "$how" != kill ] || op_calls=3 |
| | | budget_case '30 seconds after recovery and grace' "$((1000 - recovery - 10 - 30))" 0 0 1 "$op_calls" |
| | | budget_case '29 seconds after recovery and grace' "$((1000 - recovery - 10 - 29))" 0 1 1 "$op_calls" |
| | | budget_case '30 seconds before operation' 970 0 1 1 "$op_calls" |
| | | budget_case '29 seconds before operation' 971 0 1 0 0 |
| | | budget_case 'load already ended' 1001 0 1 0 0 |
| | | budget_case 'window capture consumes remaining budget' 960 11 1 0 0 |
| | | |
| | | for failure in docker readiness; do |
| | | rm -f "$OUT/disturbances.txt" |
| | | : >"$OUT/docker.calls" |
| | | set +e |
| | | ( |
| | | set -e |
| | | now=400 |
| | | if [ "$failure" = docker ]; then DOCKER_FAIL=$how; else WAIT_FAIL=yes; fi |
| | | disturb >"$OUT/failure.log" |
| | | ) |
| | | code=$? |
| | | set -e |
| | | check "$failure failure propagates" "$code" 1 |
| | | check "$failure failure not marked complete" "$([ -f "$OUT/disturbances.txt" ] && echo yes || echo no)" no |
| | | done |
| | | |
| | | # No DS client is interrupted or segmented for either RS disturbance. |
| | | start_clients() { calls="$calls clients $*"; } |
| | | disturb() { calls="$calls disturb at $SECONDS"; } |
| | | calls= SECONDS=0 CLIENTS= |
| | | run_load |
| | | check "whole-load DS clients" "$calls" ' clients d0 1 900 clients d1 1 900 sleep 300 disturb at 300' |
| | | start_clients() { calls="$calls clients $*"; SECONDS=$((SECONDS + 160)); } |
| | | calls= SECONDS=0 |
| | | run_load |
| | | check "late client startup, no negative sleep" "$calls" ' clients d0 1 900 clients d1 1 900 disturb at 320' |
| | | echo "$kind: $((checks - failed)) checks passed, $failed failed" |
| | | [ "$failed" -eq 0 ] |
| | | ) |
| | | |
| | | # Separate processes keep sourced entry points and fixture overrides from leaking. |
| | | variant restart-rs restart 35 ' docker restart -t 120 r0' ' dns:r0 180' |
| | | variant kill-rs kill 45 ' docker kill r0 docker wait r0 docker start r0' ' dns:r0 180 181 284' |
| | |
| | | |
| | | # A self-test of the errors log check of the replication convergence run (#1208), with no |
| | | # topology: which records replication_errors takes for errors and which ones a disturbance |
| | | # window expects. A green row does not show it - nothing a disturbance expects is logged outside |
| | | # its window there, and the base rows have no window - so docker-convergence runs this first. |
| | | # window expects: failed handshakes and the msgIDs or dns:<hostname> tokens it was opened with. |
| | | # A green row does not show it - nothing a disturbance expects is logged outside its window |
| | | # there, and the base rows have no window - so docker-convergence runs this first. |
| | | # docker is a stub that reads a local errors log: mark_logs, window_open and window_close count |
| | | # its lines, since_mark reads it, as they do in a container. |
| | | # |
| | |
| | | E106ID='[t] category=SYNC severity=ERROR msgID=106 msg=Replication server 32179 lost its connection' |
| | | N62='[t] category=SYNC severity=NOTICE msgID=62 msg=Directory server DS(1) has connected' |
| | | CORE178='[t] category=CORE severity=ERROR msgID=178 msg=not a replication error' |
| | | # What an RS disturbance can leave behind, and ids like its allowances. |
| | | E180='[t] category=SYNC severity=ERROR msgID=180 msg=Directory server DS(13411) encountered an error while receiving changes for domain "dc=example,dc=com" from replication server RS(23837) at cv-r0:8989. The connection will be closed, and this directory server will now try to connect to another replication server' |
| | | E181='[t] category=SYNC severity=ERROR msgID=181 msg=The connection from this replication server RS(31531) to replication server RS(23837) at cv-r0:8989 for domain "dc=example,dc=com" has failed' |
| | | E284="[t] category=SYNC severity=ERROR msgID=284 msg=Log file '/opt/opendj/data/changelogDb/2.dom/13411.server/head.log' was successfully recovered by removing a partially written record" |
| | | E211='[t] category=SYNC severity=ERROR msgID=211 msg=The connection from this replication server RS(31531) to directory server DS(24329) at cv-d1:8989 for domain "dc=example,dc=com" has failed' |
| | | E66='[t] category=SYNC severity=ERROR msgID=66 msg=monitor reply failed' |
| | | E108='[t] category=SYNC severity=ERROR msgID=108 msg=monitor request failed' |
| | | E51='[05/Nov/2026:19:18:17 +0000] category=SYNC severity=ERROR msgID=51 msg=The hostname cv-r0 could not be resolved as an IP address' |
| | | E18='[t] category=SYNC severity=ERROR msgID=18 msg=an id that starts the first one a window lists' |
| | | E80='[t] category=SYNC severity=ERROR msgID=80 msg=an id that ends the first one a window lists' |
| | | E106KILL='[t] category=SYNC severity=ERROR msgID=106 msg=an id a window lists, 181, in the message' |
| | | # records the check reads for their certificate alert, under an id a window lists |
| | | WBAD181=${WBAD/msgID=119/msgID=181} |
| | | WBADCORE181=${WBADCORE/msgID=1/msgID=181} |
| | | KILL='[ dns:cv-r0 180 181 284' |
| | | # Read the restart allowances from its entry point so the filter cases also detect a missing |
| | | # or widened production allowance. Only the Docker lifecycle is replaced here. |
| | | RESTART=$( |
| | | RS_ONLY='cv-r0 cv-r1' RS_NODES='cv-r0 cv-r1' |
| | | . "$HERE/disturb-restart-rs.sh" |
| | | down_and_up() { shift; printf '[ %s' "$*"; } |
| | | disturb |
| | | ) |
| | | # Run the DS entry point too: the combined restart receives a DNS token, while a DS-only |
| | | # restart does not. Keep the production window arguments and replace only the lifecycle. |
| | | ds_restart_window() ( |
| | | DS_NODES=$1 RS_NODES=$2 DISTURB_TIMEOUT=900 |
| | | . "$HERE/disturb-restart-ds.sh" |
| | | window_open() { printf '[%s' "${*:+ $*}" >"$OUT/ds-restart-window"; } |
| | | window_close() { :; } |
| | | docker() { :; } |
| | | wait_until() { :; } |
| | | sleep() { :; } |
| | | date() { echo 100; } |
| | | disturb >/dev/null |
| | | cat "$OUT/ds-restart-window" |
| | | ) |
| | | RESTART_DS=$(ds_restart_window 'cv-d0 cv-d1' 'cv-r0 cv-r1') |
| | | RESTART_COMBINED=$(ds_restart_window 'cv-c0 cv-c1 cv-c2 cv-c3' 'cv-c0 cv-c1 cv-c2 cv-c3') |
| | | |
| | | passed=0 failed=0 |
| | | # case <name> <errors> <expected> <record, [ or ]>...: the errors log holds two errors before |
| | | # mark_logs, which are not the run's, then the records given; [ opens a window and ] closes it. |
| | | # The number of records replication_errors prints, and of those it sends to logs/n0.expected |
| | | # case <name> <errors> <expected> <record, [, [ <msgID>|dns:<hostname>... or ]>...: the errors log |
| | | # holds two errors before mark_logs, which are not the run's, then the records given; [ opens a |
| | | # window with the msgIDs or DNS tokens that follow it, and ] closes it. Count replication_errors |
| | | # output records and those it sends to logs/n0.expected. |
| | | case_() { |
| | | local name=$1 want_errors=$2 want_expected=$3 record errors expected |
| | | shift 3 |
| | |
| | | for record; do |
| | | case $record in |
| | | '[') window_open ;; |
| | | '[ '*) window_open ${record#\[ } ;; |
| | | ']') window_close ;; |
| | | *) echo "$record" >>"$LOG" ;; |
| | | esac |
| | |
| | | case_ two-windows 0 2 '[' "$E178" ']' "$N62" '[' "$E179" ']' |
| | | case_ between-two-windows 1 2 '[' "$E178" ']' "$E179" '[' "$E178" ']' |
| | | case_ mixed 2 2 '[' "$E178" "$E106" "$W119" "$WBAD" "$N62" ']' |
| | | case_ 180-in-a-kill-window 0 1 "$KILL" "$E180" ']' |
| | | case_ 181-in-a-kill-window 0 1 "$KILL" "$E181" ']' |
| | | case_ 284-in-a-kill-window 0 1 "$KILL" "$E284" ']' |
| | | case_ handshakes-in-kill-window 0 2 "$KILL" "$E178" "$W119" ']' |
| | | case_ 181-in-another-window 1 0 '[' "$E181" ']' |
| | | case_ 181-outside-a-window 1 0 "$E181" |
| | | case_ 181-after-a-kill-window 1 1 "$KILL" "$E284" ']' "$E181" |
| | | case_ ids-like-listed-ones 2 0 "$KILL" "$E18" "$E80" ']' |
| | | case_ listed-id-in-the-message 1 0 "$KILL" "$E106KILL" ']' |
| | | case_ another-in-a-kill-window 1 0 "$KILL" "$E106" ']' |
| | | case_ listed-id-not-sync-error 2 0 "$KILL" "$WBAD181" "$WBADCORE181" ']' |
| | | case_ kill-window-then-another 1 2 "$KILL" "$E180" ']' "$N62" '[' "$E181" "$E145" ']' |
| | | case_ another-then-kill-window 0 2 '[' ']' "$N62" "$KILL" "$E181" "$E145" ']' |
| | | |
| | | # DNS failures of only the named RS are expected, and only in its own window. |
| | | # Removing the host, message, ID or window checks must expose a failure here. |
| | | case_ killed-rs-dns 0 1 "$KILL" "$E51" ']' |
| | | case_ dns-before-after-window 2 1 "$E51" "$KILL" "$E51" ']' "$E51" |
| | | case_ dns-without-window 1 0 "$E51" |
| | | case_ dns-window-without-token 1 0 '[' "$E51" ']' |
| | | case_ dns-ds-crash 1 0 '[ 211' "$E51" ']' |
| | | case_ dns-combined-without-token 1 0 '[ 180 181 211 284' "$E51" ']' |
| | | case_ dns-combined-host 0 1 '[ dns:cv-c1 180 181 211 284' "${E51/cv-r0/cv-c1}" ']' |
| | | case_ dns-restarted-rs 0 1 "$RESTART" "$E51" ']' |
| | | case_ dns-combined-ds-restart 0 1 "$RESTART_COMBINED" "${E51/cv-r0/cv-c1}" ']' |
| | | case_ dns-ds-only-restart 1 0 "$RESTART_DS" "${E51/cv-r0/cv-d1}" ']' |
| | | case_ dns-other-host-ds-restart 1 0 "$RESTART_COMBINED" "${E51/cv-r0/cv-c2}" ']' |
| | | case_ dns-before-after-ds-restart 2 1 "${E51/cv-r0/cv-c1}" "$RESTART_COMBINED" \ |
| | | "${E51/cv-r0/cv-c1}" ']' "${E51/cv-r0/cv-c1}" |
| | | case_ dns-other-host 1 0 "$KILL" "${E51/cv-r0/cv-r1}" ']' |
| | | case_ dns-host-prefix 1 0 "$KILL" "${E51/cv-r0/cv-r}" ']' |
| | | case_ dns-host-suffix 1 0 "$KILL" "${E51/cv-r0/cv-r01}" ']' |
| | | case_ dns-other-id 1 0 "$KILL" "${E51/msgID=51/msgID=151}" ']' |
| | | case_ dns-other-message 1 0 "$KILL" "${E51/msg=The hostname/msg=Unexpected hostname}" ']' |
| | | case_ dns-message-suffix 1 0 "$KILL" "$E51 extra" ']' |
| | | case_ dns-embedded-message 1 0 "$KILL" "${E51/msg=The hostname/msg=Unexpected DNS failure msg=The hostname}" ']' |
| | | case_ dns-next-window 1 1 "$KILL" "$E51" ']' '[' "$E51" ']' |
| | | case_ dns-window-hosts 2 2 "$KILL" "$E51" "${E51/cv-r0/cv-r1}" ']' \ |
| | | '[ dns:cv-r1' "$E51" "${E51/cv-r0/cv-r1}" ']' |
| | | case_ monitors-in-rs-kill 2 0 "$KILL" "$E66" "$E108" ']' |
| | | |
| | | # DS crash allowances do not leak into graceful restarts, later records, or RS-only kills. |
| | | case_ ds-kill-disconnect 0 1 '[ 211' "$E211" ']' |
| | | case_ ds-kill-without-rs 2 0 '[ 211' "$E180" "$E284" ']' |
| | | case_ combined-kill 0 4 '[ dns:cv-c1 180 181 211 284' "$E180" "$E181" "$E211" "$E284" ']' |
| | | case_ ds-disconnect-graceful 1 0 '[' "$E211" ']' |
| | | case_ 180-in-rs-restart 0 1 "$RESTART" "$E180" ']' |
| | | case_ 180-before-after-restart 2 1 "$E180" "$RESTART" "$E180" ']' "$E180" |
| | | case_ 180-in-ds-restart 1 0 "$RESTART_DS" "$E180" ']' |
| | | case_ combined-ds-restart-strict 6 0 "$RESTART_COMBINED" "$E180" "$E181" "$E211" "$E284" "$E66" "$E108" ']' |
| | | case_ rs-restart-strict 5 0 "$RESTART" "$E181" "$E211" "$E284" "$E66" "$E108" ']' |
| | | case_ ds-disconnect-rs-kill 1 0 "$KILL" "$E211" ']' |
| | | case_ ds-disconnect-before-after 2 1 "$E211" '[ 211' "$E211" ']' "$E211" |
| | | case_ monitors-in-combined-kill 2 0 '[ dns:cv-c1 180 181 211 284' "$E66" "$E108" ']' |
| | | |
| | | # A log that cannot be read stops window_open, which would otherwise open a window from its first |
| | | # line. In the background: errexit is off in a subshell on the left of ||, and on in a job |
| | | code=0 |
| | | rm -f "$LOG" |
| | | (window_open 180 181 284) 2>/dev/null & |
| | | wait $! || code=$? |
| | | if [ "$code" -ne 0 ]; then |
| | | passed=$((passed + 1)) |
| | | else |
| | | failed=$((failed + 1)) |
| | | echo "::error::window_open went on over an errors log that cannot be read" |
| | | fi |
| | | |
| | | echo "replication_errors: $passed cases passed, $failed failed" |
| | | [ "$failed" -eq 0 ] |
| New file |
| | |
| | | #!/usr/bin/env bash |
| | | # |
| | | # |
| | | # The contents of this file are subject to the terms of the Common Development and |
| | | # Distribution License (the License). You may not use this file except in compliance with the |
| | | # License. |
| | | # |
| | | # You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the |
| | | # specific language governing permission and limitations under the License. |
| | | # |
| | | # When distributing Covered Software, include this CDDL Header Notice in each file and include |
| | | # the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL |
| | | # Header, with the fields enclosed by brackets [] replaced by your own identifying |
| | | # information: "Portions copyright [year] [name of copyright owner]". |
| | | # |
| | | # Copyright 2026 3A Systems, LLC. |
| | | |
| | | # Check the real summary for absent, partial and complete disturbances, without Docker. |
| | | set -eE -o pipefail |
| | | # The fixture summaries must not be appended to the real GitHub Actions job summary. |
| | | unset GITHUB_STEP_SUMMARY |
| | | HERE=$(cd "$(dirname "$0")" && pwd) |
| | | OUT=$(mktemp -d) |
| | | trap 'rm -rf "$OUT"' EXIT |
| | | . "$HERE/collect.sh" |
| | | TOPOLOGY=rs534 ALL_NODES="d0 d1 r0 r1" DS_NODES="d0 d1" RS_ONLY="r0 r1" |
| | | DURATION=900 DISTURB_AT=300 LOAD_MOD_RATE=250 LOAD_ADD_RATE=50 SAMPLE_DATA=10000 PHASE_TIMES= |
| | | failed=0 checks=0 |
| | | check() { # <case> <expected disturbance rows> |
| | | summarize failed |
| | | checks=$((checks + 1)) |
| | | got=$(sed -n '/^| disturbance |/p' "$OUT/summary.md") |
| | | [ "$got" = "$2" ] || { echo "::error::$1: got '$got', expected '$2'"; failed=$((failed + 1)); } |
| | | } |
| | | DISTURBANCE=none |
| | | check "no disturbance" "" |
| | | for DISTURBANCE in restart-rs kill-rs; do |
| | | . "$HERE/disturb-$DISTURBANCE.sh" |
| | | rm -f "$OUT/disturbances.txt" |
| | | unset LOAD_START_MS |
| | | check "$DISTURBANCE load not started" "| disturbance | $DISTURBANCE at 300 s into the load, not started: the run ended before the load |" |
| | | LOAD_START_MS=100000 |
| | | check "$DISTURBANCE not completed" "| disturbance | $DISTURBANCE scheduled at 300 s into the load, not completed: 0 of 1 steps completed |" |
| | | how=restart |
| | | [ "$DISTURBANCE" != kill-rs ] || how=kill |
| | | echo "400000 445000 r0 $how" >"$OUT/disturbances.txt" |
| | | check "$DISTURBANCE completed" "| disturbance | $how of r0 300 s into the load, serving again 45 s later |" |
| | | done |
| | | for DISTURBANCE in restart-ds kill-ds; do |
| | | . "$HERE/disturb-$DISTURBANCE.sh" |
| | | rm "$OUT/disturbances.txt" |
| | | check "$DISTURBANCE not completed" "| disturbance | $DISTURBANCE scheduled at 300 s into the load, not completed: 0 of 1 steps completed |" |
| | | how=restart |
| | | [ "$DISTURBANCE" != kill-ds ] || how=kill |
| | | echo "400000 435000 d1 $how" >"$OUT/disturbances.txt" |
| | | check "$DISTURBANCE completed" "| disturbance | $how of d1 300 s into the load, serving again 35 s later |" |
| | | done |
| | | echo "summary: $((checks - failed)) checks passed, $failed failed" |
| | | [ "$failed" -eq 0 ] |
| | |
| | | scanners: vuln |
| | | cache: false |
| | | - name: Upload Trivy report to GitHub Security |
| | | uses: github/codeql-action/upload-sarif@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 |
| | | uses: github/codeql-action/upload-sarif@1fc58d04a4d4374f921bdac573c3811c240f9c26 # v4.39.2 |
| | | # upload even if a preceding step failed, but not without a report to upload |
| | | if: ${{ always() && hashFiles('trivy-results.sarif') != '' }} |
| | | with: |
| | |
| | | fail-fast: false |
| | | matrix: |
| | | topology: [rs534, combined] |
| | | # none: the base run; restart-ds: a directory server restarts during the load |
| | | disturbance: [none, restart-ds] |
| | | # none: the base run; restart-ds and kill-ds: a directory server restarts or crashes; |
| | | # kill-rs and restart-rs: one RS is killed or gracefully restarted per fresh topology |
| | | # while every DS writes. Only rs534 has replication servers apart from directory servers. |
| | | disturbance: [none, restart-ds, kill-ds, kill-rs, restart-rs] |
| | | exclude: |
| | | - topology: combined |
| | | disturbance: kill-rs |
| | | - topology: combined |
| | | disturbance: restart-rs |
| | | steps: |
| | | - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 |
| | | with: |
| | |
| | | .github/scripts/replication |
| | | # what a green row does not show: what a disturbance window of the errors log check |
| | | # expects - nothing it expects is logged outside the window there, and a base row has |
| | | # none - and where restart-ds opens and closes the window |
| | | - name: Self-tests of the errors log check and of restart-ds |
| | | # none - and where each disturbance opens and closes its windows |
| | | - name: Self-tests of the errors log check, disturbances and summary |
| | | shell: bash |
| | | run: | |
| | | .github/scripts/replication/selftest-replication-errors.sh |
| | | .github/scripts/replication/selftest-disturb-restart-ds.sh |
| | | .github/scripts/replication/selftest-disturb-kill-ds.sh |
| | | .github/scripts/replication/selftest-disturb-rs.sh |
| | | .github/scripts/replication/selftest-summary.sh |
| | | - name: Download the image |
| | | uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 |
| | | with: |
| | |
| | | env: |
| | | TOPOLOGY: ${{ matrix.topology }} |
| | | DISTURBANCE: ${{ matrix.disturbance }} |
| | | # a disturbed row runs half the load of the base one: what it is after happens around |
| | | # the disturbance, a third of the way in, and for as long as it runs it holds one of |
| | | # the runners the whole organization shares |
| | | DURATION: ${{ matrix.disturbance == 'none' && '1800' || '900' }} |
| | | # Every row runs 900 s of load; disturbed rows act a third of the way in. |
| | | DURATION: "900" |
| | | # the replicas fall behind during the load and catch up after it: 900 s at 325 and 65 |
| | | # a second converged in 396 s (run 38944365728), 1800 s at 423 and 85 in 2489 s (run |
| | | # 39043600905), the backlog growing faster than the load above the replay capacity; a |
| | |
| | | distribution: 'temurin' |
| | | |
| | | - name: Initialize CodeQL |
| | | uses: github/codeql-action/init@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 |
| | | uses: github/codeql-action/init@1fc58d04a4d4374f921bdac573c3811c240f9c26 # v4.39.2 |
| | | with: |
| | | languages: ${{ matrix.language }} |
| | | build-mode: ${{ matrix.build-mode }} |
| | |
| | | # --------------------------------------------------------------------- |
| | | |
| | | - name: Perform CodeQL Analysis |
| | | uses: github/codeql-action/analyze@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 |
| | | uses: github/codeql-action/analyze@1fc58d04a4d4374f921bdac573c3811c240f9c26 # v4.39.2 |
| | | with: |
| | | category: "/language:${{ matrix.language }}" |
| | |
| | | scanners: vuln |
| | | cache: false |
| | | - name: Upload report to GitHub Security |
| | | uses: github/codeql-action/upload-sarif@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 |
| | | uses: github/codeql-action/upload-sarif@1fc58d04a4d4374f921bdac573c3811c240f9c26 # v4.39.2 |
| | | # upload even if a preceding step failed, but not without a report to upload |
| | | if: ${{ always() && hashFiles(format('trivy-{0}.sarif', matrix.tag)) != '' }} |
| | | with: |