#!/usr/bin/env bash
#
#
# The contents of this file are subject to the terms of the Common Development and
# Distribution License (the License). You may not use this file except in compliance with the
# License.
#
# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
# specific language governing permission and limitations under the License.
#
# When distributing Covered Software, include this CDDL Header Notice in each file and include
# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
# Header, with the fields enclosed by brackets [] replaced by your own identifying
# information: "Portions copyright [year] [name of copyright owner]".
#
# Copyright 2026 3A Systems, LLC.

# A self-test of the errors log check of the replication convergence run (#1208), with no
# topology: which records replication_errors takes for errors and which ones a disturbance
# window expects: failed handshakes and the msgIDs or dns:<hostname> tokens it was opened with.
# A green row does not show it - nothing a disturbance expects is logged outside its window
# there, and the base rows have no window - so docker-convergence runs this first.
# docker is a stub that reads a local errors log: mark_logs, window_open and window_close count
# its lines, since_mark reads it, as they do in a container.
#
# Usage: selftest-replication-errors.sh

set -eE -o pipefail

HERE=$(cd "$(dirname "$0")" && pwd)
. "$HERE/assert-converged.sh"
ALL_NODES=n0
OUT=$(mktemp -d)
trap 'rm -rf "$OUT"' EXIT
LOG=$OUT/errors

docker() { # exec <node> sh -c <the line count of the errors log> | exec <node> tail -n +<first> <errors log>
  case "$1 $3" in
    "exec sh") wc -l <"$LOG" | tr -d ' ' ;;
    "exec tail") tail -n "$5" "$LOG" ;;
    *) echo "unexpected docker $*" >&2; return 1 ;;
  esac
}

E178='[t] category=SYNC severity=ERROR msgID=178 msg=Directory server 32536 was attempting to connect to replication server 31531 but has disconnected in handshake phase'
E179='[t] category=SYNC severity=ERROR msgID=179 msg=Replication server 1 was attempting to connect to replication server 2 but has disconnected in handshake phase'
E145='[t] category=SYNC severity=ERROR msgID=145 msg=Timed out while trying to acquire the domain lock for domain "dc=example,dc=com". The connection attempt from replication server RS(29892) at cv-c1.convergence/172.18.0.3:52382 to this replication server RS(27915) will be aborted. This is probably benign and a result of a simultaneous cross connection attempt'
W119='[t] category=SYNC severity=WARNING msgID=119 msg=Directory server DS(24329) encountered an unexpected error while connecting to replication server cv-r0:8989 for domain "cn=schema": SSLHandshakeException(Remote host terminated the handshake)'
E119=${W119/severity=WARNING/severity=ERROR}
W119CORE=${W119/category=SYNC/category=CORE}
WBAD='[t] category=SYNC severity=WARNING msgID=119 msg=Directory server DS(1) encountered an unexpected error while connecting: SSLHandshakeException(Received fatal alert: bad_certificate)'
WBADCORE='[t] category=CORE severity=WARNING msgID=1 msg=SSLException(Received fatal alert: bad_certificate)'
E106='[t] category=SYNC severity=ERROR msgID=106 msg=another replication error'
E1450='[t] category=SYNC severity=ERROR msgID=1450 msg=an id that starts like one a window expects'
E45='[t] category=SYNC severity=ERROR msgID=45 msg=an id that ends like one a window expects'
E106ID='[t] category=SYNC severity=ERROR msgID=106 msg=Replication server 32179 lost its connection'
N62='[t] category=SYNC severity=NOTICE msgID=62 msg=Directory server DS(1) has connected'
CORE178='[t] category=CORE severity=ERROR msgID=178 msg=not a replication error'
# What an RS disturbance can leave behind, and ids like its allowances.
E180='[t] category=SYNC severity=ERROR msgID=180 msg=Directory server DS(13411) encountered an error while receiving changes for domain "dc=example,dc=com" from replication server RS(23837) at cv-r0:8989. The connection will be closed, and this directory server will now try to connect to another replication server'
E181='[t] category=SYNC severity=ERROR msgID=181 msg=The connection from this replication server RS(31531) to replication server RS(23837) at cv-r0:8989 for domain "dc=example,dc=com" has failed'
E284="[t] category=SYNC severity=ERROR msgID=284 msg=Log file '/opt/opendj/data/changelogDb/2.dom/13411.server/head.log' was successfully recovered by removing a partially written record"
E211='[t] category=SYNC severity=ERROR msgID=211 msg=The connection from this replication server RS(31531) to directory server DS(24329) at cv-d1:8989 for domain "dc=example,dc=com" has failed'
E66='[t] category=SYNC severity=ERROR msgID=66 msg=monitor reply failed'
E108='[t] category=SYNC severity=ERROR msgID=108 msg=monitor request failed'
E51='[05/Nov/2026:19:18:17 +0000] category=SYNC severity=ERROR msgID=51 msg=The hostname cv-r0 could not be resolved as an IP address'
E18='[t] category=SYNC severity=ERROR msgID=18 msg=an id that starts the first one a window lists'
E80='[t] category=SYNC severity=ERROR msgID=80 msg=an id that ends the first one a window lists'
E106KILL='[t] category=SYNC severity=ERROR msgID=106 msg=an id a window lists, 181, in the message'
# records the check reads for their certificate alert, under an id a window lists
WBAD181=${WBAD/msgID=119/msgID=181}
WBADCORE181=${WBADCORE/msgID=1/msgID=181}
KILL='[ dns:cv-r0 180 181 284'
# Read the restart allowances from its entry point so the filter cases also detect a missing
# or widened production allowance. Only the Docker lifecycle is replaced here.
RESTART=$(
  RS_ONLY='cv-r0 cv-r1' RS_NODES='cv-r0 cv-r1'
  . "$HERE/disturb-restart-rs.sh"
  down_and_up() { shift; printf '[ %s' "$*"; }
  disturb
)
# Run the DS entry point too: the combined restart receives a DNS token, while a DS-only
# restart does not. Keep the production window arguments and replace only the lifecycle.
ds_restart_window() (
  DS_NODES=$1 RS_NODES=$2 DISTURB_TIMEOUT=900
  . "$HERE/disturb-restart-ds.sh"
  window_open() { printf '[%s' "${*:+ $*}" >"$OUT/ds-restart-window"; }
  window_close() { :; }
  docker() { :; }
  wait_until() { :; }
  sleep() { :; }
  date() { echo 100; }
  disturb >/dev/null
  cat "$OUT/ds-restart-window"
)
RESTART_DS=$(ds_restart_window 'cv-d0 cv-d1' 'cv-r0 cv-r1')
RESTART_COMBINED=$(ds_restart_window 'cv-c0 cv-c1 cv-c2 cv-c3' 'cv-c0 cv-c1 cv-c2 cv-c3')

passed=0 failed=0
# case <name> <errors> <expected> <record, [, [ <msgID>|dns:<hostname>... or ]>...: the errors log
# holds two errors before mark_logs, which are not the run's, then the records given; [ opens a
# window with the msgIDs or DNS tokens that follow it, and ] closes it. Count replication_errors
# output records and those it sends to logs/n0.expected.
case_() {
  local name=$1 want_errors=$2 want_expected=$3 record errors expected
  shift 3
  rm -rf "$OUT/logs"
  printf '%s\n' "$E178" "$E106" >"$LOG"
  mark_logs
  for record; do
    case $record in
      '[') window_open ;;
      '[ '*) window_open ${record#\[ } ;;
      ']') window_close ;;
      *) echo "$record" >>"$LOG" ;;
    esac
  done
  errors=$(replication_errors n0 | grep -c . || true)
  expected=$( [ ! -f "$OUT/logs/n0.expected" ] || grep -c . "$OUT/logs/n0.expected" || true)
  if [ "$errors" = "$want_errors" ] && [ "${expected:-0}" = "$want_expected" ]; then
    passed=$((passed + 1))
  else
    failed=$((failed + 1))
    echo "::error::$name: $errors errors and ${expected:-0} expected, not $want_errors and $want_expected"
  fi
}

case_ 178-outside-a-window       1 0     "$E178"
case_ 145-outside-a-window       1 0     "$E145"
case_ 178-in-a-window            0 1 '[' "$E178" ']'
case_ 179-in-a-window            0 1 '[' "$E179" ']'
case_ 145-in-a-window            0 1 '[' "$E145" ']'
case_ terminated-handshake       0 1 '[' "$W119" ']'
case_ terminated-handshake-error 1 0 '[' "$E119" ']'
case_ terminated-elsewhere       1 0 '[' "$W119CORE" ']'
case_ bad-certificate            1 0 '[' "$WBAD" ']'
case_ bad-certificate-elsewhere  1 0 '[' "$WBADCORE" ']'
case_ another-error-in-a-window  1 0 '[' "$E106" ']'
case_ ids-like-expected-ones     2 0 '[' "$E1450" "$E45" ']'
case_ an-id-in-the-message       1 0 '[' "$E106ID" ']'
case_ another-category           0 0 '[' "$CORE178" ']'
case_ a-notice                   0 0     "$N62"
case_ before-a-window            1 1     "$E178" '[' "$E178" ']'
case_ after-a-window             1 1 '[' "$N62" "$E178" ']' "$E178"
case_ two-windows                0 2 '[' "$E178" ']' "$N62" '[' "$E179" ']'
case_ between-two-windows        1 2 '[' "$E178" ']' "$E179" '[' "$E178" ']'
case_ mixed                      2 2 '[' "$E178" "$E106" "$W119" "$WBAD" "$N62" ']'
case_ 180-in-a-kill-window       0 1 "$KILL" "$E180" ']'
case_ 181-in-a-kill-window       0 1 "$KILL" "$E181" ']'
case_ 284-in-a-kill-window       0 1 "$KILL" "$E284" ']'
case_ handshakes-in-kill-window  0 2 "$KILL" "$E178" "$W119" ']'
case_ 181-in-another-window      1 0 '[' "$E181" ']'
case_ 181-outside-a-window       1 0     "$E181"
case_ 181-after-a-kill-window    1 1 "$KILL" "$E284" ']' "$E181"
case_ ids-like-listed-ones       2 0 "$KILL" "$E18" "$E80" ']'
case_ listed-id-in-the-message   1 0 "$KILL" "$E106KILL" ']'
case_ another-in-a-kill-window   1 0 "$KILL" "$E106" ']'
case_ listed-id-not-sync-error   2 0 "$KILL" "$WBAD181" "$WBADCORE181" ']'
case_ kill-window-then-another   1 2 "$KILL" "$E180" ']' "$N62" '[' "$E181" "$E145" ']'
case_ another-then-kill-window   0 2 '[' ']' "$N62" "$KILL" "$E181" "$E145" ']'

# DNS failures of only the named RS are expected, and only in its own window.
# Removing the host, message, ID or window checks must expose a failure here.
case_ killed-rs-dns              0 1 "$KILL" "$E51" ']'
case_ dns-before-after-window    2 1 "$E51" "$KILL" "$E51" ']' "$E51"
case_ dns-without-window         1 0 "$E51"
case_ dns-window-without-token   1 0 '[' "$E51" ']'
case_ dns-ds-crash               1 0 '[ 211' "$E51" ']'
case_ dns-combined-without-token 1 0 '[ 180 181 211 284' "$E51" ']'
case_ dns-combined-host          0 1 '[ dns:cv-c1 180 181 211 284' "${E51/cv-r0/cv-c1}" ']'
case_ dns-restarted-rs           0 1 "$RESTART" "$E51" ']'
case_ dns-combined-ds-restart    0 1 "$RESTART_COMBINED" "${E51/cv-r0/cv-c1}" ']'
case_ dns-ds-only-restart        1 0 "$RESTART_DS" "${E51/cv-r0/cv-d1}" ']'
case_ dns-other-host-ds-restart  1 0 "$RESTART_COMBINED" "${E51/cv-r0/cv-c2}" ']'
case_ dns-before-after-ds-restart 2 1 "${E51/cv-r0/cv-c1}" "$RESTART_COMBINED" \
                                   "${E51/cv-r0/cv-c1}" ']' "${E51/cv-r0/cv-c1}"
case_ dns-other-host             1 0 "$KILL" "${E51/cv-r0/cv-r1}" ']'
case_ dns-host-prefix            1 0 "$KILL" "${E51/cv-r0/cv-r}" ']'
case_ dns-host-suffix            1 0 "$KILL" "${E51/cv-r0/cv-r01}" ']'
case_ dns-other-id               1 0 "$KILL" "${E51/msgID=51/msgID=151}" ']'
case_ dns-other-message          1 0 "$KILL" "${E51/msg=The hostname/msg=Unexpected hostname}" ']'
case_ dns-message-suffix         1 0 "$KILL" "$E51 extra" ']'
case_ dns-embedded-message       1 0 "$KILL" "${E51/msg=The hostname/msg=Unexpected DNS failure msg=The hostname}" ']'
case_ dns-next-window            1 1 "$KILL" "$E51" ']' '[' "$E51" ']'
case_ dns-window-hosts           2 2 "$KILL" "$E51" "${E51/cv-r0/cv-r1}" ']' \
                                   '[ dns:cv-r1' "$E51" "${E51/cv-r0/cv-r1}" ']'
case_ monitors-in-rs-kill        2 0 "$KILL" "$E66" "$E108" ']'

# DS crash allowances do not leak into graceful restarts, later records, or RS-only kills.
case_ ds-kill-disconnect         0 1 '[ 211' "$E211" ']'
case_ ds-kill-without-rs         2 0 '[ 211' "$E180" "$E284" ']'
case_ combined-kill              0 4 '[ dns:cv-c1 180 181 211 284' "$E180" "$E181" "$E211" "$E284" ']'
case_ ds-disconnect-graceful     1 0 '[' "$E211" ']'
case_ 180-in-rs-restart          0 1 "$RESTART" "$E180" ']'
case_ 180-before-after-restart   2 1 "$E180" "$RESTART" "$E180" ']' "$E180"
case_ 180-in-ds-restart          1 0 "$RESTART_DS" "$E180" ']'
case_ combined-ds-restart-strict 6 0 "$RESTART_COMBINED" "$E180" "$E181" "$E211" "$E284" "$E66" "$E108" ']'
case_ rs-restart-strict          5 0 "$RESTART" "$E181" "$E211" "$E284" "$E66" "$E108" ']'
case_ ds-disconnect-rs-kill      1 0 "$KILL" "$E211" ']'
case_ ds-disconnect-before-after 2 1 "$E211" '[ 211' "$E211" ']' "$E211"
case_ monitors-in-combined-kill  2 0 '[ dns:cv-c1 180 181 211 284' "$E66" "$E108" ']'

# A log that cannot be read stops window_open, which would otherwise open a window from its first
# line. In the background: errexit is off in a subshell on the left of ||, and on in a job
code=0
rm -f "$LOG"
(window_open 180 181 284) 2>/dev/null &
wait $! || code=$?
if [ "$code" -ne 0 ]; then
  passed=$((passed + 1))
else
  failed=$((failed + 1))
  echo "::error::window_open went on over an errors log that cannot be read"
fi

echo "replication_errors: $passed cases passed, $failed failed"
[ "$failed" -eq 0 ]