#
|
#
|
# The contents of this file are subject to the terms of the Common Development and
|
# Distribution License (the License). You may not use this file except in compliance with the
|
# License.
|
#
|
# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
|
# specific language governing permission and limitations under the License.
|
#
|
# When distributing Covered Software, include this CDDL Header Notice in each file and include
|
# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
|
# Header, with the fields enclosed by brackets [] replaced by your own identifying
|
# information: "Portions copyright [year] [name of copyright owner]".
|
#
|
# Copyright 2026 3A Systems, LLC.
|
|
# Shared lifecycle for the independent kill-rs and restart-rs convergence scenarios (#1208).
|
# Each run takes down one RS-only server while the other continues serving every DS. The
|
# combined topology also takes down a DS and belongs to a different disturbance.
|
[ $(echo $RS_ONLY | wc -w) -ge 2 ] \
|
|| { echo "$DISTURBANCE needs at least two replication servers without data: TOPOLOGY=rs534" >&2; exit 2; }
|
DISTURBED=$(echo $RS_ONLY | cut -d' ' -f1)
|
DISTURB_STEPS=1
|
|
# Startup after <seconds> and connections to every other RS establish availability. Changelog
|
# catch-up is checked by the final convergence/data checks after the load; no second RS is
|
# stopped in this run, so the recovered RS need not catch up before another disturbance.
|
rs_serves_since() { # <rs> <seconds>
|
started_since $1 $2 && [ "$(rs_peers $1)" -eq $(($(echo $RS_NODES | wc -w) - 1)) ]
|
}
|
|
# <kill|restart> [<msgID>|dns:<hostname>...]: open the expected-error window, recover the RS, and retain
|
# its actual downtime. A kill needs docker wait before start; a graceful restart gets 120 s
|
# to stop. The window includes 10 s after startup/peer connections for pending handshakes.
|
# DS load balancing may move connections later or not at all, so it does not end the window.
|
down_and_up() {
|
local how=$1 rs=$DISTURBED from served left
|
shift
|
window_open "$@"
|
from=$(date +%s)
|
# Opening the window reads each server's log and can itself be slow. Check the remaining
|
# load immediately before the operation, including when client startup delayed disturb().
|
left=$((LOAD_START_MS / 1000 + DURATION - from))
|
[ "$left" -ge $((3 * STAT_INTERVAL)) ] \
|
|| fail "only $left s left in the load of $DURATION s, too late to $how $rs under it"
|
echo " [$SECONDS s] $how $rs"
|
if [ "$how" = kill ]; then
|
docker kill $rs >/dev/null
|
docker wait $rs >/dev/null
|
docker start $rs >/dev/null
|
else
|
docker restart -t 120 $rs >/dev/null
|
fi
|
wait_until $DISTURB_TIMEOUT "$rs serves again" rs_serves_since $rs $from
|
served=$(date +%s)
|
echo " [$SECONDS s] $rs serves again"
|
sleep 10
|
window_close
|
echo "$((from * 1000)) $((served * 1000)) $rs $how" >>"$OUT/disturbances.txt"
|
# Record a completed recovery even if it was too slow, but fail rather than silently
|
# accepting a restart that left no useful load. Match restart-ds's three sample intervals.
|
left=$((LOAD_START_MS / 1000 + DURATION - $(date +%s)))
|
[ "$left" -ge $((3 * STAT_INTERVAL)) ] \
|
|| fail "$rs recovered too late: only $left s left in the load of $DURATION s"
|
}
|
|
# The image runs its join on every start. Leave the disturbed server as topology_up does
|
# before the final convergence wait, without requiring catch-up while writes are still active.
|
settle() {
|
wait_until $DISTURB_TIMEOUT "the join of $DISTURBED ends" bootstrap_ended $DISTURBED
|
}
|