[#1208] Test directory and replication server crashes and restarts (#1244)

Valery Kharseko

5 hours ago f251af3f68a38e2f83154ed80be92667326d286c

 .github/workflows/build.yml

@@ -1136,8 +1136,15 @@
      fail-fast: false
      matrix:
        topology: [rs534, combined]
        # none: the base run; restart-ds: a directory server restarts during the load
        disturbance: [none, restart-ds]
        # none: the base run; restart-ds and kill-ds: a directory server restarts or crashes;
        # kill-rs and restart-rs: one RS is killed or gracefully restarted per fresh topology
        # while every DS writes. Only rs534 has replication servers apart from directory servers.
        disturbance: [none, restart-ds, kill-ds, kill-rs, restart-rs]
        exclude:
          - topology: combined
            disturbance: kill-rs
          - topology: combined
            disturbance: restart-rs
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
@@ -1145,12 +1152,15 @@
            .github/scripts/replication
      # what a green row does not show: what a disturbance window of the errors log check
      # expects - nothing it expects is logged outside the window there, and a base row has
      # none - and where restart-ds opens and closes the window
      - name: Self-tests of the errors log check and of restart-ds
      # none - and where each disturbance opens and closes its windows
      - name: Self-tests of the errors log check, disturbances and summary
        shell: bash
        run: |
          .github/scripts/replication/selftest-replication-errors.sh
          .github/scripts/replication/selftest-disturb-restart-ds.sh
          .github/scripts/replication/selftest-disturb-kill-ds.sh
          .github/scripts/replication/selftest-disturb-rs.sh
          .github/scripts/replication/selftest-summary.sh
      - name: Download the image
        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
        with:
@@ -1164,10 +1174,8 @@
        env:
          TOPOLOGY: ${{ matrix.topology }}
          DISTURBANCE: ${{ matrix.disturbance }}
          # a disturbed row runs half the load of the base one: what it is after happens around
          # the disturbance, a third of the way in, and for as long as it runs it holds one of
          # the runners the whole organization shares
          DURATION: ${{ matrix.disturbance == 'none' && '1800' || '900' }}
          # Every row runs 900 s of load; disturbed rows act a third of the way in.
          DURATION: "900"
          # the replicas fall behind during the load and catch up after it: 900 s at 325 and 65
          # a second converged in 396 s (run 38944365728), 1800 s at 423 and 85 in 2489 s (run
          # 39043600905), the backlog growing faster than the load above the replay capacity; a