#
#
# The contents of this file are subject to the terms of the Common Development and
# Distribution License (the License). You may not use this file except in compliance with the
# License.
#
# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
# specific language governing permission and limitations under the License.
#
# When distributing Covered Software, include this CDDL Header Notice in each file and include
# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
# Header, with the fields enclosed by brackets [] replaced by your own identifying
# information: "Portions copyright [year] [name of copyright owner]".
#
# Copyright 2026 3A Systems, LLC.

# The replication monitor of the convergence run (#1090), sourced by run.sh: how far behind
# each directory server is, seen from each replication server, sampled during the load, and
# the wait for the topology to converge once it stops.

# One line per directory server known to each replication server - connected to it, or to a
# peer of it - in the replication domain of BASE_DN: <epoch> <replication server> <server id>
# <missing-changes> <approximate-delay>. Both kinds of entries are named "Connected directory
# server DS(<id>) ..." (DataServerHandler, LightweightServerHandler)
# The replication servers are asked at once: an ldapsearch starts a JVM, which takes tens of
# seconds on a runner the load keeps busy, and asked one after the other they took a sample
# every minute or so instead of every five seconds
sample() {
  local rs now dir pid pids=
  now=$(date +%s)
  dir=$(mktemp -d "$OUT/sample.XXXXXX")
  for rs in $RS_NODES; do
    admin_search $rs "$MONITOR_DOMAIN" sub "(cn=Connected directory server*)" \
        server-id missing-changes approximate-delay \
      | awk -v t=$now -v rs=$rs '
          tolower($1) == "server-id:" { id = $2 }
          tolower($1) == "missing-changes:" { m = $2 }
          tolower($1) == "approximate-delay:" { d = $2 }
          /^$/ && id != "" { print t, rs, id, m, d; id = "" }
          END { if (id != "") print t, rs, id, m, d }' >"$dir/$rs" &
    pids="$pids $!"
  done
  # a replication server that does not answer only leaves its lines out, which
  # converged_once does not take for a zero
  for pid in $pids; do
    wait $pid || true
  done
  for rs in $RS_NODES; do
    cat "$dir/$rs"
  done
  rm -rf "$dir"
}

# Converged once every replication server knows every directory server and none of them misses
# a change. An entry that is not there is not a zero: right after a server connects again its
# entry is missing until the replication server has heard of it
converged_once() {
  local s rs want
  s=$(sample) || return 1
  want=$(echo $DS_NODES | wc -w)
  for rs in $RS_NODES; do
    [ "$(echo "$s" | awk -v rs=$rs '$2 == rs { print $3 }' | sort -u | wc -l)" -eq "$want" ] || return 1
  done
  echo "$s" | awk '$4 != 0 { behind = 1 } END { exit behind }'
}

# A replication server computes missing-changes once a monitoring-period (MonitoringPublisher),
# which this run leaves at its default, and a directory server its last computation does not
# know reads as 0: samples that do not span a period may all read one computation, made while
# changes were on their way
MONITORING_PERIOD=60

# Three converged samples in a row, the last one started more than a monitoring period (and
# the publisher's 5 s latch) after the first ended. Sets CONVERGED_IN to the seconds until the
# first of them
wait_converged() {
  local start=$SECONDS hits=0 first= last=
  until [ $hits -ge 3 ] && [ $((last - first)) -gt $((MONITORING_PERIOD + 5)) ]; do
    last=$SECONDS
    if converged_once; then hits=$((hits + 1)); first=${first:-$SECONDS}; else hits=0; first=; fi
    if [ $((SECONDS - start)) -ge "$QUIESCE_TIMEOUT" ]; then
      echo "::group::the monitor when the wait timed out (epoch, replication server, server id, missing changes, delay)"
      sample || true
      echo "::endgroup::"
      fail "the topology did not converge within $QUIESCE_TIMEOUT s"
    fi
    sleep 5
  done
  CONVERGED_IN=$((first - start))
}

# Samples every 5 s in the background into $OUT/samples.txt, for the peaks of the run; a
# sample that fails - a server restarting - is only a gap
sampler_start() {
  ( while :; do sample >>"$OUT/samples.txt" 2>/dev/null || true; sleep 5; done ) &
  SAMPLER=$!
}

sampler_stop() {
  if [ -n "${SAMPLER:-}" ]; then
    # waited for, so that the shell does not report the job it killed
    kill $SAMPLER 2>/dev/null || true
    wait $SAMPLER 2>/dev/null || true
    # a sample takes tens of seconds under the load, so the kill nearly always cuts one short
    # before it removes its directory, which would then be uploaded with the run. The kill
    # reaches the sampler only, not the searches it started: one that opens its file in the
    # directory after rm read it makes rm fail, which must not fail the run
    rm -rf "$OUT"/sample.* || true
  fi
  SAMPLER=
}