#!/usr/bin/env bash
#
#
# The contents of this file are subject to the terms of the Common Development and
# Distribution License (the License). You may not use this file except in compliance with the
# License.
#
# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
# specific language governing permission and limitations under the License.
#
# When distributing Covered Software, include this CDDL Header Notice in each file and include
# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
# Header, with the fields enclosed by brackets [] replaced by your own identifying
# information: "Portions copyright [year] [name of copyright owner]".
#
# Copyright 2026 3A Systems, LLC.

# Exercise both independent RS disturbances without Docker. External commands and the clock
# are stubs; the entry points, readiness checks and run_load are the production functions.
set -eE -o pipefail
HERE=$(cd "$(dirname "$0")" && pwd)
OUT=$(mktemp -d)
trap 'rm -rf "$OUT"' EXIT

variant() (
  local kind=$1 how=$2 recovery=$3 command=$4 extras=$5
  local code started peers
  failed=0 checks=0
  check() {
    checks=$((checks + 1))
    [ "$2" = "$3" ] || { echo "::error::$kind $1: got '$2', expected '$3'"; failed=$((failed + 1)); }
  }
  for topology in '' r0; do
    code=0
    (RS_ONLY=$topology; . "$HERE/disturb-$kind.sh") >/dev/null 2>&1 || code=$?
    check "refuses fewer than two RS-only servers" "$code" 2
  done
  DS_NODES="d0 d1" RS_ONLY="r0 r1" RS_NODES="r0 r1" DURATION=900 DISTURB_AT=300 DISTURB_TIMEOUT=900
  DISTURBANCE=$kind LOAD_START_MS=100000
  . "$HERE/disturb-$kind.sh"
  . "$HERE/load.sh"
  check "disturbed server" "$DISTURBED" r0
  check "expected step count" "$DISTURB_STEPS" 1
  unset SECONDS
  SECONDS=0 now=400 SERVE=30 calls=
  date() { echo "$now"; }
  sleep() { calls="$calls sleep $1"; now=$((now + $1)); SECONDS=$((SECONDS + $1)); }
  docker() {
    calls="$calls docker $*"
    echo "$*" >>"$OUT/docker.calls"
    [ "${DOCKER_FAIL:-}" != "$1" ] || return 1
    now=$((now + 5))
  }
  wait_until() {
    calls="$calls wait_until $1 [$2] ${*:3}"
    [ "${WAIT_FAIL:-no}" != yes ] || fail "readiness timed out"
    now=$((now + SERVE))
  }
  window_open() { calls="$calls window_open${*:+ $*}"; now=$((now + ${WINDOW_DELAY:-0})); }
  window_close() { calls="$calls window_close"; }
  fail() { echo "::error::$1"; false; }
  rm -f "$OUT/disturbances.txt"
  disturb >"$OUT/disturb.log"
  check "single operation and error window" "$calls" \
    " window_open$extras$command wait_until 900 [r0 serves again] rs_serves_since r0 400 sleep 10 window_close"
  check "single completed step" "$(cat "$OUT/disturbances.txt")" "400000 $((400000 + recovery * 1000)) r0 $how"
  calls=
  settle
  check "settle only the disturbed server" "$calls" " wait_until 900 [the join of r0 ends] bootstrap_ended r0"

  started_since() { [ "$1 $2" = 'r0 400' ] && [ "$started" = yes ]; }
  rs_peers() { [ "$1" = r0 ] && echo "$peers"; }
  serves() { if rs_serves_since r0 400; then echo yes; else echo no; fi; }
  started=yes peers=1
  check "startup and peer present" "$(serves)" yes
  peers=0
  check "missing peer" "$(serves)" no
  peers=2
  check "unexpected peer count" "$(serves)" no
  started=no peers=1
  check "no startup message" "$(serves)" no
  started=yes peers=2 RS_NODES="r0 r1 r2"
  check "three RS topology, both peers present" "$(serves)" yes
  peers=1
  check "three RS topology, one peer absent" "$(serves)" no
  RS_NODES="r0 r1"

  # Run failures outside if/|| so errexit remains active as it is in run.sh.
  budget_case() { # <label> <start time> <window delay> <status> <completed> <docker calls>
    local status lines
    rm -f "$OUT/disturbances.txt"
    : >"$OUT/docker.calls"
    set +e
    (set -e; now=$2 WINDOW_DELAY=$3; disturb >"$OUT/budget.log")
    status=$?
    set -e
    check "$1 status" "$status" "$4"
    lines=0
    [ ! -f "$OUT/disturbances.txt" ] || lines=$(awk 'END {print NR}' "$OUT/disturbances.txt")
    check "$1 completed steps" "$lines" "$5"
    check "$1 docker calls" "$(awk 'END {print NR}' "$OUT/docker.calls")" "$6"
    if [ "$4" -ne 0 ]; then
      check "$1 diagnostic" "$(grep -c 'too late' "$OUT/budget.log" || true)" 1
    fi
  }
  op_calls=1
  [ "$how" != kill ] || op_calls=3
  budget_case '30 seconds after recovery and grace' "$((1000 - recovery - 10 - 30))" 0 0 1 "$op_calls"
  budget_case '29 seconds after recovery and grace' "$((1000 - recovery - 10 - 29))" 0 1 1 "$op_calls"
  budget_case '30 seconds before operation' 970 0 1 1 "$op_calls"
  budget_case '29 seconds before operation' 971 0 1 0 0
  budget_case 'load already ended' 1001 0 1 0 0
  budget_case 'window capture consumes remaining budget' 960 11 1 0 0

  for failure in docker readiness; do
    rm -f "$OUT/disturbances.txt"
    : >"$OUT/docker.calls"
    set +e
    (
      set -e
      now=400
      if [ "$failure" = docker ]; then DOCKER_FAIL=$how; else WAIT_FAIL=yes; fi
      disturb >"$OUT/failure.log"
    )
    code=$?
    set -e
    check "$failure failure propagates" "$code" 1
    check "$failure failure not marked complete" "$([ -f "$OUT/disturbances.txt" ] && echo yes || echo no)" no
  done

  # No DS client is interrupted or segmented for either RS disturbance.
  start_clients() { calls="$calls clients $*"; }
  disturb() { calls="$calls disturb at $SECONDS"; }
  calls= SECONDS=0 CLIENTS=
  run_load
  check "whole-load DS clients" "$calls" ' clients d0 1 900 clients d1 1 900 sleep 300 disturb at 300'
  start_clients() { calls="$calls clients $*"; SECONDS=$((SECONDS + 160)); }
  calls= SECONDS=0
  run_load
  check "late client startup, no negative sleep" "$calls" ' clients d0 1 900 clients d1 1 900 disturb at 320'
  echo "$kind: $((checks - failed)) checks passed, $failed failed"
  [ "$failed" -eq 0 ]
)

# Separate processes keep sourced entry points and fixture overrides from leaking.
variant restart-rs restart 35 ' docker restart -t 120 r0' ' dns:r0 180'
variant kill-rs kill 45 ' docker kill r0 docker wait r0 docker start r0' ' dns:r0 180 181 284'