From 7925442d32b631630b78e214cb1d2a9d71bb920a Mon Sep 17 00:00:00 2001
From: Valery Kharseko <vharseko@3a-systems.ru>
Date: Tue, 13 Oct 2026 09:06:13 +0000
Subject: [PATCH] [#1178] Give the background join of the Docker image a server role, keep a failed bootstrap or upgrade unhealthy, and test the upgrade from a released image (#1184)
---
opendj-packages/opendj-docker/run.sh | 63 ++
opendj-doc-generated-ref/src/main/asciidoc/install-guide/chap-install.adoc | 2
opendj-packages/opendj-docker/Dockerfile-alpine | 2
opendj-packages/opendj-docker/README.md | 120 +++++
opendj-packages/opendj-docker/bootstrap/join.sh | 401 ++++++++++++++++--
opendj-packages/opendj-docker/bootstrap/setup.sh | 38 +
opendj-doc-generated-ref/src/main/asciidoc/install-guide/chap-upgrade.adoc | 2
.github/workflows/build.yml | 194 +++++++++
.github/scripts/docker-test-replication.sh | 381 +++++++++++++++++
opendj-doc-generated-ref/src/main/asciidoc/admin-guide/chap-replication.adoc | 2
opendj-packages/opendj-docker/Dockerfile | 2
11 files changed, 1,112 insertions(+), 95 deletions(-)
diff --git a/.github/scripts/docker-test-replication.sh b/.github/scripts/docker-test-replication.sh
index f5cdca0..ca838be 100755
--- a/.github/scripts/docker-test-replication.sh
+++ b/.github/scripts/docker-test-replication.sh
@@ -23,8 +23,8 @@
set -eE -o pipefail
IMAGE=${1:?usage: $0 <image>}
-NODES="dj-0 dj-1 dj-2 dj-x dj-solo dj-bf dj-fi dj-f dj-p0 dj-p1 dj-p2 dj-ipm dj-ipr dj-sdsr dj-shm-probe dj-shm"
-VOLUMES="vol-dj-0 vol-dj-1 vol-dj-2 vol-dj-x vol-dj-solo vol-dj-bf vol-dj-fi vol-dj-f vol-dj-ipr vol-dj-p0 vol-dj-p1 vol-dj-p2"
+NODES="dj-0 dj-1 dj-2 dj-x dj-solo dj-bf dj-fi dj-f dj-p0 dj-p1 dj-p2 dj-ipm dj-ipr dj-sdsr dj-shm-probe dj-shm dj-d0 dj-d1 dj-r0 dj-r1 dj-dn0 dj-dn1 dj-rf dj-rfd dj-nm dj-nmr dj-bogus"
+VOLUMES="vol-dj-0 vol-dj-1 vol-dj-2 vol-dj-x vol-dj-solo vol-dj-bf vol-dj-fi vol-dj-f vol-dj-ipr vol-dj-p0 vol-dj-p1 vol-dj-p2 vol-dj-d0 vol-dj-d1 vol-dj-r0 vol-dj-r1 vol-dj-dn0 vol-dj-dn1 vol-dj-rf vol-dj-rfd vol-dj-nm vol-dj-nmr vol-dj-bogus"
# a bootstrap that imports the base entry and then fails, mounted into dj-bf
FAILING_BOOTSTRAP=$(mktemp)
NETWORK=test_replication
@@ -56,7 +56,28 @@
# set -E hands the trap to every $(...) as well, where a failing command would dump the logs
# into the value being captured and remove the containers under the running test: there the
# failure only ends the subshell, and the main shell decides
-trap 'code=$?; [ "$BASH_SUBSHELL" -eq 0 ] || exit $code; dump_logs; cleanup; exit $code' ERR
+trap 'code=$?; [ "$BASH_SUBSHELL" -eq 0 ] || exit $code; dump_logs; timings; cleanup; exit $code' ERR
+
+# Where the time goes: each case logs when it starts, in seconds since the script started, and
+# a wait that takes long logs how long; the run ends - failed or not - with the cases sorted by
+# their duration, the place to start when making the script faster
+PHASE=
+PHASE_START=$SECONDS
+PHASE_TIMES=
+phase() { # <case>
+ [ -z "$PHASE" ] || PHASE_TIMES+="$((SECONDS - PHASE_START)) s $PHASE"$'\n'
+ PHASE=$1
+ PHASE_START=$SECONDS
+ echo "=== [${SECONDS} s] $PHASE"
+}
+
+timings() {
+ phase "(end)"
+ echo "::group::time per case, longest first"
+ sort -rn <<<"${PHASE_TIMES%$'\n'}"
+ echo "total: ${SECONDS} s"
+ echo "::endgroup::"
+}
fail() {
echo "::error::$1"
@@ -81,12 +102,13 @@
}
wait_until() { # <seconds> <what> <command> [<argument>...]
- local deadline=$((SECONDS + $1)) what=$2
+ local deadline=$((SECONDS + $1)) what=$2 start=$SECONDS
shift 2
until "$@"; do
[ "$SECONDS" -lt "$deadline" ] || fail "timed out waiting until $what"
sleep 2
done
+ [ $((SECONDS - start)) -lt 20 ] || echo " [$((SECONDS - start)) s] until $what"
}
health() {
@@ -143,6 +165,16 @@
--bindDN "cn=Directory Manager" --bindPassword "$ROOT_PASSWORD" --trustAll --no-prompt >/dev/null
}
+runs_replication_server() { # <container>
+ admin_search "$1" "cn=config" sub "(objectClass=ds-cfg-replication-server)" 1.1 | grep "^dn:" >/dev/null
+}
+
+# the role setup.sh recorded on the volume of the container, or the join of a volume set up
+# before roles
+recorded_role_of() { # <container>
+ docker exec "$1" cat /opt/opendj/data/.replication-role 2>/dev/null
+}
+
# the writability-mode the backend of dc=example,dc=com is configured with, empty where its
# entry sets none
backend_mode() { # <container>
@@ -234,6 +266,7 @@
done
}
+phase "image checks"
# every tool reads the root password from a file (#1084, #1092); dsreplication run with -n
# prints no command line, so a password put back on one would pass every check below
rc=0
@@ -273,12 +306,14 @@
"$@" "$IMAGE" >/dev/null
}
+phase "seed"
# the first peer of REPLICATION_PEERS, and only it, seeds a topology its retries could not
# find; it imports entries no other server ever gets but from an initialize
start_node dj-0 dj-0,dj-1 -e SAMPLE_DATA=10
wait_healthy dj-0
logs_have dj-0 "seeding it with this server's data" || fail "dj-0 did not seed the topology"
+phase "seed after a reset"
# a seed that follows a reset lets the writes of clients in again, as a rejoin does: dj-0 is
# left the way a reset cut off after its disable leaves a volume - the backend held,
# $REJOIN_PENDING naming it, no replication domain - and started again while no other peer
@@ -302,6 +337,7 @@
add_ou dj-0 seeded-after-reset || fail "dj-0 refuses the writes of clients after it seeded"
[ "$(published_state dj-0)" = ready ] || fail "dj-0 does not publish itself ready after it seeded"
+phase "join retries an unreachable peer"
# a joining server tries again while its peer is unreachable
docker network disconnect $NETWORK dj-0
start_node dj-1 dj-0,dj-1
@@ -320,14 +356,20 @@
add_ou dj-0 replicated
wait_has dj-1 "ou=replicated,dc=example,dc=com"
+phase "restart of a member"
# a member is ready again right after a restart, without waiting for its peers - gating it
# on them would deadlock a whole-cluster restart under OrderedReady - and replication still
-# flows
+# flows. dj-1 comes back the way a volume set up before roles does, without the role setup.sh
+# records: its join tells the role from its replication configuration, and records it
+docker exec dj-1 rm /opt/opendj/data/.replication-role
docker stop dj-1 >/dev/null
docker restart dj-0 >/dev/null
wait_until 120 "dj-0 is healthy while dj-1 is down" is_healthy dj-0
docker start dj-1 >/dev/null
wait_healthy dj-1
+# a member is healthy before its join runs: the role is recorded once the join found it a member
+wait_until 300 "the restarted dj-1 is a member again" logs_have_since dj-1 "Starting OpenDJ" "already a member of the replication topology"
+[ "$(recorded_role_of dj-1)" = combined ] || fail "dj-1 was recorded $(recorded_role_of dj-1) from its combined replication configuration"
add_ou dj-1 replicated2
wait_has dj-0 "ou=replicated2,dc=example,dc=com"
# and neither of them took its replication down to enable it anew: only a server that no
@@ -337,6 +379,7 @@
if logs_have $n "taking its replication configuration down"; then fail "$n reset its replication on a plain restart"; fi
done
+phase "seed without a domain"
# a seed that no peer joined holds the data without a replication domain, and its join binds
# with the ROOT_PASSWORD of the bootstrap: once the root password is changed, a restart is
# still healthy, as it is without replication
@@ -363,6 +406,7 @@
docker rm -f dj-solo >/dev/null
docker volume rm vol-dj-solo >/dev/null
+phase "/dev/shm across restarts"
# a Kubernetes pod keeps its /dev/shm across container restarts, shared by all its
# containers: a starting container removes the password files a killed join or replicate.sh
# left there - only those of its own ADMIN_PORT, the files of the other containers of the pod
@@ -378,6 +422,7 @@
docker exec dj-shm test -e /dev/shm/opendj-join.5444.other || fail "run.sh removed the password file of another container"
docker rm -f dj-shm-probe dj-shm >/dev/null
+phase "failed join stays unhealthy"
# a join that cannot succeed keeps the container from reporting itself healthy - across a
# restart too, where the health marker used to follow the upgrade and a failed join turned
# into a healthy, unreplicated server. dj-x may not seed: it is not the first peer of its list
@@ -392,6 +437,7 @@
docker rm -f dj-x >/dev/null
docker volume rm vol-dj-x >/dev/null
+phase "seed lost its volume"
# the seed lost its volume: behind the same name, a fresh dj-0 finds the topology at dj-1 and
# takes its data from it instead of seeding an empty one next to it
docker rm -f dj-0 >/dev/null
@@ -403,21 +449,30 @@
ldaps_has dj-0 "ou=replicated,dc=example,dc=com" || fail "the reborn dj-0 lacks the data of the topology"
ldaps_has dj-0 "uid=user.0,ou=People,dc=example,dc=com" || fail "the reborn dj-0 lacks the entries of the seed"
+phase "failed bootstrap (dj-bf)"
# a bootstrap that failed after it imported the base entry leaves a volume that never counts
-# as holding the data of the topology: once restarted, dj-bf initializes from dj-0 rather
-# than joining with what its bootstrap left
+# as holding the data of the topology. A restart does not join it either: the initialize
+# would bring the data of the topology, but nothing else the bootstrap was to configure
+# (#1182). Once the setup is finished by hand and the mark removed, dj-bf initializes from
+# dj-0 rather than joining with what its bootstrap left
printf 'sh /opt/opendj/bootstrap/setup.sh\nexit 1\n' >"$FAILING_BOOTSTRAP"
chmod 644 "$FAILING_BOOTSTRAP"
start_node dj-bf dj-0,dj-1,dj-bf -v "$FAILING_BOOTSTRAP:/opt/opendj/failing-bootstrap.sh:ro" \
-e BOOTSTRAP=/opt/opendj/failing-bootstrap.sh
wait_until 300 "the bootstrap of dj-bf fails" logs_have dj-bf "failing-bootstrap.sh failed"
docker restart dj-bf >/dev/null
+wait_until 300 "the restarted dj-bf reports its pending bootstrap" logs_have_since dj-bf "failing-bootstrap.sh failed" "The bootstrap of this volume did not complete"
+stays_unhealthy dj-bf "its bootstrap did not complete"
+if logs_have dj-bf "join: enabling replication"; then fail "dj-bf joined over a volume whose bootstrap did not complete"; fi
+docker exec dj-bf rm /opt/opendj/data/.bootstrap-pending
+docker restart dj-bf >/dev/null
wait_healthy dj-bf
logs_have dj-bf "initializing from dj-0" || fail "dj-bf joined with the data of its failed bootstrap"
ldaps_has dj-bf "uid=user.0,ou=People,dc=example,dc=com" || fail "dj-bf lacks the entries only an initialize from dj-0 brings"
docker rm -f dj-bf >/dev/null
docker volume rm vol-dj-bf >/dev/null
+phase "failed initialize"
# an initialize that fails keeps the volume waiting for the data of the topology: a
# one-second bound cuts the dsreplication JVM off before it connects
start_node dj-fi dj-0,dj-fi -e REPLICATION_INITIALIZE_TIMEOUT=1 -e REPLICATION_RETRY_COUNT=2
@@ -428,6 +483,7 @@
docker rm -f dj-fi >/dev/null
docker volume rm vol-dj-fi >/dev/null
+phase "scale up to three"
# scale up to three - the peer list is configuration, so the running servers are replaced
# with the longer list before the third one starts, as a rolling update would
docker rm -f dj-0 dj-1 >/dev/null
@@ -439,6 +495,7 @@
wait_healthy dj-2
ldaps_has dj-2 "ou=replicated,dc=example,dc=com" || fail "dj-2 lacks the data of the topology"
+phase "scale down to two"
# scale down to two: the survivors, restarted with the shorter list, remove dj-2 from
# cn=admin data and from every replication server list they hold - that of BASE_DN, and
# those of cn=schema and cn=admin data that dsreplication enable configures next to it.
@@ -463,6 +520,7 @@
add_ou dj-0 replicated3
wait_has dj-1 "ou=replicated3,dc=example,dc=com"
+phase "scale up on a kept volume, locks held"
# scaled up again on the volume it kept: dj-2 still replicates, but the survivors no longer
# register it, and an enable between two servers whose cn=admin data is replicated registers
# nobody - so dj-2 takes its replication configuration down, enables again and registers.
@@ -520,6 +578,7 @@
add_ou dj-2 rejoined
wait_has dj-0 "ou=rejoined,dc=example,dc=com"
+phase "two servers back together"
# two servers taken out together and scaled up again together on their volumes: both take
# their replication down, and with it every registration in their cn=admin data. While the
# lock of the survivor is held, neither may enable through the other - an enable between two
@@ -575,6 +634,7 @@
wait_has dj-0 "ou=rejoined-together,dc=example,dc=com"
wait_has dj-2 "ou=rejoined-together,dc=example,dc=com"
+phase "half-enabled rejoins"
# an enable that stopped half-way - its initialize of cn=admin data failed - leaves BASE_DN
# replicated and the server registered at its peers, but not in its own cn=admin data, and
# every enable after that exits 5, "already replicated", without registering it there. So a
@@ -606,6 +666,55 @@
wait_has dj-0 "ou=rejoined-half-enabled,dc=example,dc=com"
wait_has dj-2 "ou=rejoined-half-enabled,dc=example,dc=com"
+phase "domain-less pre-roles volume"
+# an enable that stopped before it configured the domain of BASE_DN, on a volume that records
+# no role, as an image before roles leaves it: a replication server and a backend, no domain.
+# Its join takes it for the combined server it is - the replication server of the one-shot
+# srs is told from it only by REPLICATION_ROLE=replication - and enables it again
+dsconfig_on dj-1 delete-replication-domain --provider-name "Multimaster Synchronization" --domain-name "dc=example,dc=com"
+replicates_example dj-1 && fail "dj-1 still replicates dc=example,dc=com"
+docker exec dj-1 rm /opt/opendj/data/.replication-role
+# counted, not looked for after the last start: right after docker restart, that is still the
+# start before, whose first join logged the same line
+joins=$(logs_count dj-1 "joined the replication topology through dj-0")
+docker restart dj-1 >/dev/null
+joined_again() { [ "$(logs_count dj-1 "joined the replication topology through dj-0")" -gt "$joins" ]; }
+wait_until 300 "the domain-less dj-1 joins again" joined_again
+wait_healthy dj-1
+[ "$(recorded_role_of dj-1)" = combined ] || fail "dj-1 was recorded $(recorded_role_of dj-1) from a replication server and a backend under REPLICATION_ROLE=combined"
+replicates_example dj-1 || fail "dj-1 does not replicate dc=example,dc=com after it joined again"
+add_ou dj-1 rejoined-domain-less
+wait_has dj-2 "ou=rejoined-domain-less,dc=example,dc=com"
+
+phase "repair defers an unanswering peer"
+# a peer that a replication server list lacks and that does not answer is left out of it,
+# whatever the roles around - here all combined - and added by the repair of a later start:
+# dj-2 leaves the list of the replication server of dj-1 and goes down while dj-1 restarts
+dsconfig_on dj-1 set-replication-server-prop --provider-name "Multimaster Synchronization" --remove replication-server:dj-2:8989
+docker stop dj-2 >/dev/null
+# counted, as above: the repair of an earlier start may have logged either line already -
+# an add of dj-2 to a domain list as well, so the adds back below are counted against the
+# server list lines alone
+adds=$(logs_count dj-1 "adding replication server dj-2:")
+server_adds=$(logs_count dj-1 "adding replication server dj-2:8989 to the server list")
+deferrals=$(logs_count dj-1 "could not ask dj-2 whether it runs a replication server")
+docker restart dj-1 >/dev/null
+deferred() { [ "$(logs_count dj-1 "could not ask dj-2 whether it runs a replication server")" -gt "$deferrals" ]; }
+wait_until 300 "the restarted dj-1 leaves dj-2 for a later start" deferred
+if server_lists dj-1 dj-2; then fail "dj-1 added dj-2, which did not answer, to the list of its replication server"; fi
+[ "$(logs_count dj-1 "adding replication server dj-2:")" -eq "$adds" ] || fail "dj-1 added dj-2, which did not answer, to a replication server list"
+docker start dj-2 >/dev/null
+wait_healthy dj-2
+docker restart dj-1 >/dev/null
+added_back() { [ "$(logs_count dj-1 "adding replication server dj-2:8989 to the server list")" -gt "$server_adds" ]; }
+wait_until 300 "the restarted dj-1 adds dj-2 back" added_back
+wait_until 120 "the replication server of dj-1 lists dj-2 again" server_lists dj-1 dj-2
+wait_healthy dj-1
+# dj-2 keeps the read-only backend the operator gave it above
+add_ou dj-1 repaired-list
+wait_has dj-2 "ou=repaired-list,dc=example,dc=com"
+
+phase "sdsr"
# the deprecated one-shot sdsr path of replicate.sh still bootstraps a replica. On a first
# start a server stops the server its bootstrap started and starts it again, and a replica
# can reach it in between: replicate.sh tries a dsreplication enable that could not connect
@@ -641,6 +750,24 @@
fail "replicate.sh for a base DN nobody holds exited with $rc, not with the 5 of its dsreplication enable, or went on after it"
fi
+phase "pre-roles srs volume"
+# the volume the replication server of the one-shot srs has from an image before roles: a
+# replication server and a backend - setup.sh created one for every server then - but no
+# domain and no role recorded. A combined server whose enable stopped half-way has the same,
+# as dj-1 above, and only REPLICATION_ROLE=replication tells the two apart: its join records
+# a replication server here. Last for dj-1, whose volume stays one from then on
+dsconfig_on dj-1 delete-replication-domain --provider-name "Multimaster Synchronization" --domain-name "dc=example,dc=com"
+replicates_example dj-1 && fail "dj-1 still replicates dc=example,dc=com"
+docker exec dj-1 rm /opt/opendj/data/.replication-role
+docker rm -f dj-1 >/dev/null
+start_node dj-1 dj-0,dj-1,dj-2 -e REPLICATION_ROLE=replication -e REPLICATION_RETRY_COUNT=30
+role_recorded() { [ -n "$(recorded_role_of dj-1)" ]; }
+wait_until 300 "the pre-roles srs dj-1 records a role" role_recorded
+[ "$(recorded_role_of dj-1)" = replication ] || fail "dj-1 was recorded $(recorded_role_of dj-1) from a replication server and a backend under REPLICATION_ROLE=replication"
+srs_member() { logs_have dj-1 "already a member of the replication topology" || logs_have dj-1 "joined the replication topology through"; }
+wait_until 300 "the pre-roles srs dj-1 is a member again" srs_member
+wait_healthy dj-1
+
# the root password shows in no container log, and the files the tools read it from are gone
for c in dj-0 dj-1 dj-2 dj-sdsr; do
if docker logs $c 2>&1 | grep -F -- "$ROOT_PASSWORD"; then fail "the root password is in the log of $c"; fi
@@ -650,6 +777,7 @@
docker rm -f dj-0 dj-1 dj-2 dj-sdsr >/dev/null
docker volume rm vol-dj-0 vol-dj-1 vol-dj-2 >/dev/null
+phase "MASTER_SERVER by address"
# MASTER_SERVER may name the master by its address, as the grep of /etc/hosts in the base
# replicate.sh allowed: a master given its own address recognises itself and seeds at once,
# and a replica joins and initializes through that address
@@ -679,6 +807,7 @@
docker rm -f dj-ipm dj-ipr >/dev/null
docker volume rm vol-dj-ipr >/dev/null
+phase "parallel start"
# three fresh servers started together, as Compose or a StatefulSet with
# podManagementPolicy: Parallel start them: none of them takes the bootstrap data of another
# fresh one for the topology's. The first peer seeds, the others initialize from it - its
@@ -713,6 +842,244 @@
for c in dj-p0 dj-p1 dj-p2; do
if docker logs $c 2>&1 | grep -F -- "$ROOT_PASSWORD"; then fail "the root password is in the log of $c"; fi
done
+docker rm -f dj-p0 dj-p1 dj-p2 >/dev/null
+docker volume rm vol-dj-p0 vol-dj-p1 vol-dj-p2 >/dev/null
+phase "unknown role"
+# REPLICATION_ROLE (#1178). A role the join does not know ends the join before it waits for
+# the server
+rc=0
+out=$(docker run --rm --hostname dj-bogus --entrypoint /opt/opendj/bootstrap/join.sh \
+ -e REPLICATION_PEERS=dj-0,dj-bogus -e REPLICATION_ROLE=bogus "$IMAGE" 2>&1) || rc=$?
+[ "$rc" -eq 1 ] && grep -F "REPLICATION_ROLE=bogus is not one of combined, directory or replication" <<<"$out" >/dev/null \
+ || { echo "$out"; fail "the join went on with an unknown REPLICATION_ROLE (exit $rc)"; }
+# and so does setup.sh, before it sets anything up: the volume records no role, and holds no
+# instance set up for a role nobody asked for. The volume is looked at once run.sh says that
+# setup.sh ended, not when the refusal is logged: a setup.sh that logged it first and set the
+# instance up all the same would still be running
+start_node dj-bogus dj-0,dj-bogus -e REPLICATION_ROLE=bogus
+wait_until 300 "setup.sh ends" logs_have dj-bogus "/opt/opendj/bootstrap/setup.sh failed, this container will not report itself healthy"
+logs_have dj-bogus "REPLICATION_ROLE=bogus is not one of combined, directory or replication" || fail "setup.sh did not say why it refused the role"
+if docker run --rm --entrypoint test -v vol-dj-bogus:/opt/opendj/data "$IMAGE" -e /opt/opendj/data/.replication-role; then
+ fail "setup.sh recorded an unknown role"
+fi
+if docker run --rm --entrypoint test -v vol-dj-bogus:/opt/opendj/data "$IMAGE" -e /opt/opendj/data/config/buildinfo; then
+ fail "setup.sh set the instance up before it refused REPLICATION_ROLE=bogus"
+fi
+docker rm -f dj-bogus >/dev/null
+docker volume rm vol-dj-bogus >/dev/null
+phase "replication server listed first"
+# A replication server listed first is passed over: the first peer that holds data seeds
+# the topology, here a directory server, and the replication server joins through it
+start_node dj-rf dj-rf,dj-rfd -e REPLICATION_ROLE=replication -e REPLICATION_RETRY_COUNT=30
+start_node dj-rfd dj-rf,dj-rfd -e REPLICATION_ROLE=directory -e SAMPLE_DATA=10 -e REPLICATION_RETRY_COUNT=30
+wait_healthy dj-rfd
+wait_healthy dj-rf
+logs_have dj-rfd "seeding it with this server's data" || fail "dj-rfd, the first peer that holds data, did not seed"
+if logs_have dj-rf "seeding it with this server's data"; then fail "dj-rf seeded the topology as a replication server"; fi
+logs_have dj-rf "joined the replication topology through dj-rfd" || fail "dj-rf did not join through dj-rfd"
+admin_search dj-rfd "cn=config" sub "(&(objectClass=ds-cfg-replication-domain)(ds-cfg-base-dn=dc=example,dc=com))" ds-cfg-replication-server | grep -F "ds-cfg-replication-server: dj-rf:" >/dev/null || fail "the domain of dj-rfd does not connect to dj-rf"
+docker rm -f dj-rf dj-rfd >/dev/null
+docker volume rm vol-dj-rf vol-dj-rfd >/dev/null
+
+phase "master without join"
+# A master without MASTER_SERVER runs no join, and publishes neither a state nor a role: a
+# replica that names it in MASTER_SERVER takes it for a combined server, as before roles, and
+# joins and initializes through it
+start_node dj-nm "" -e SAMPLE_DATA=10
+wait_healthy dj-nm
+start_node dj-nmr "" -e MASTER_SERVER=dj-nm
+wait_healthy dj-nmr
+logs_have dj-nmr "initializing from dj-nm" || fail "dj-nmr did not initialize from dj-nm"
+ldaps_has dj-nmr "uid=user.0,ou=People,dc=example,dc=com" || fail "dj-nmr lacks the entries of dj-nm"
+runs_replication_server dj-nm || fail "dj-nm, which publishes no role, was not given a replication server"
+docker rm -f dj-nm dj-nmr >/dev/null
+docker volume rm vol-dj-nm vol-dj-nmr >/dev/null
+
+phase "roles: #534 topology"
+# The topology of #534: directory servers without a replication server, and replication
+# servers without data. The first directory server seeds, the replication servers join
+# through it, and the second directory server joins through whichever peer runs a
+# replication server and initializes from the seed
+ROLE_PEERS=dj-d0,dj-r0,dj-r1,dj-d1
+start_node dj-d0 $ROLE_PEERS -e REPLICATION_ROLE=directory -e SAMPLE_DATA=10 -e REPLICATION_RETRY_COUNT=30
+for n in dj-r0 dj-r1; do
+ start_node $n $ROLE_PEERS -e REPLICATION_ROLE=replication -e REPLICATION_RETRY_COUNT=30
+done
+start_node dj-d1 $ROLE_PEERS -e REPLICATION_ROLE=directory -e REPLICATION_RETRY_COUNT=30
+for n in dj-d0 dj-r0 dj-r1 dj-d1; do
+ wait_healthy $n
+done
+logs_have dj-d0 "seeding it with this server's data" || fail "dj-d0 did not seed the topology"
+logs_have dj-d1 "initializing from dj-d0" || fail "dj-d1 did not initialize from dj-d0"
+ldaps_has dj-d1 "uid=user.0,ou=People,dc=example,dc=com" || fail "dj-d1 lacks the entries of the seed"
+
+holds_example_backend() { # <container>
+ admin_search "$1" "cn=Backends,cn=config" one "(ds-cfg-base-dn=dc=example,dc=com)" 1.1 | grep "^dn:" >/dev/null
+}
+# the replication servers the domain of dc=example,dc=com of the container connects to
+example_domain_lists() { # <container>
+ admin_search "$1" "cn=config" sub "(&(objectClass=ds-cfg-replication-domain)(ds-cfg-base-dn=dc=example,dc=com))" ds-cfg-replication-server \
+ | awk 'tolower($1) == "ds-cfg-replication-server:" { print $2 }'
+}
+# the role the container publishes next to its state, which its peers read while it holds no
+# replication configuration, and itself after a reset
+published_role_of() { # <container>
+ admin_search "$1" "cn=Docker Join,cn=config" base "(objectClass=*)" description \
+ | awk 'tolower($1) == "description:" { print $3 }'
+}
+roles_hold() {
+ local n
+ # dj-r0 joined through dj-d0: an enable that took dj-d0 for a combined server would have
+ # given it a replication server
+ for n in dj-d0 dj-d1; do
+ runs_replication_server $n && fail "the directory server $n runs a replication server"
+ replicates_example $n || fail "the directory server $n does not replicate dc=example,dc=com"
+ [ "$(published_role_of $n)" = directory ] || fail "the directory server $n publishes the role $(published_role_of $n)"
+ [ "$(recorded_role_of $n)" = directory ] || fail "the volume of the directory server $n records the role $(recorded_role_of $n)"
+ done
+ for n in dj-r0 dj-r1; do
+ runs_replication_server $n || fail "the replication server $n runs none"
+ replicates_example $n && fail "the replication server $n replicates dc=example,dc=com"
+ holds_example_backend $n && fail "the replication server $n holds a backend for dc=example,dc=com"
+ [ "$(published_role_of $n)" = replication ] || fail "the replication server $n publishes the role $(published_role_of $n)"
+ [ "$(recorded_role_of $n)" = replication ] || fail "the volume of the replication server $n records the role $(recorded_role_of $n)"
+ done
+ return 0
+}
+roles_hold
+lists_rs_only() { # <container>
+ local lists
+ lists=$(example_domain_lists "$1")
+ grep -x "dj-r0:8989" <<<"$lists" >/dev/null && grep -x "dj-r1:8989" <<<"$lists" >/dev/null \
+ && ! grep -E "^dj-d[01]:" <<<"$lists" >/dev/null
+}
+for n in dj-d0 dj-d1; do
+ wait_until 120 "the domain of $n lists dj-r0 and dj-r1 and no directory server" lists_rs_only $n
+done
+wait_until 120 "the replication server of dj-r0 knows dj-r1" server_lists dj-r0 dj-r1
+wait_until 120 "the replication server of dj-r1 knows dj-r0" server_lists dj-r1 dj-r0
+add_ou dj-d0 roles-from-d0
+wait_has dj-d1 "ou=roles-from-d0,dc=example,dc=com"
+add_ou dj-d1 roles-from-d1
+wait_has dj-d0 "ou=roles-from-d1,dc=example,dc=com"
+
+phase "roles: restarts"
+# a restart keeps each server in its role, without taking its replication down
+for n in dj-r0 dj-d1; do
+ docker restart $n >/dev/null
+ wait_until 300 "the restarted $n is a member again" logs_have_since $n "Starting OpenDJ" "already a member of the replication topology"
+ wait_healthy $n
+ if logs_have $n "taking its replication configuration down"; then fail "$n reset its replication on a plain restart"; fi
+done
+roles_hold
+phase "roles: role-less volumes"
+# a volume that records no role, as one set up before roles, has its join tell the role from
+# the replication configuration and record it, which roles_hold reads - not from
+# REPLICATION_ROLE, which is combined here for both: a domain without a replication server is
+# a directory server, and a replication server without a domain or a backend for BASE_DN is a
+# replication server
+for n in dj-r0 dj-d1; do
+ docker exec $n rm /opt/opendj/data/.replication-role
+done
+docker rm -f dj-r0 dj-d1 >/dev/null
+for n in dj-r0 dj-d1; do
+ start_node $n $ROLE_PEERS -e REPLICATION_ROLE=combined -e REPLICATION_RETRY_COUNT=30
+done
+for n in dj-r0 dj-d1; do
+ wait_until 300 "the role-less $n is a member again" logs_have $n "already a member of the replication topology"
+ wait_healthy $n
+ logs_have $n "keeps that role, REPLICATION_ROLE=combined does not change it" || fail "$n did not say that it keeps its role"
+done
+roles_hold
+phase "roles: started in another role"
+# and so does a new container on the volume with another REPLICATION_ROLE, which is logged
+docker rm -f dj-r0 dj-d1 >/dev/null
+start_node dj-r0 $ROLE_PEERS -e REPLICATION_ROLE=directory -e REPLICATION_RETRY_COUNT=30
+start_node dj-d1 $ROLE_PEERS -e REPLICATION_ROLE=replication -e REPLICATION_RETRY_COUNT=30
+for n in dj-r0 dj-d1; do
+ wait_until 300 "$n is a member again in its former role" logs_have $n "already a member of the replication topology"
+ wait_healthy $n
+ if logs_have $n "taking its replication configuration down"; then fail "$n reset its replication when started in another role"; fi
+done
+logs_have dj-r0 "keeps that role, REPLICATION_ROLE=directory does not change it" || fail "dj-r0 did not say that it keeps its role"
+logs_have dj-d1 "keeps that role, REPLICATION_ROLE=replication does not change it" || fail "dj-d1 did not say that it keeps its role"
+roles_hold
+
+phase "roles: fail-over"
+# one replication server is enough for the directory servers to go on replicating, either
+# one: which of them a directory server connects to is the broker's choice. A change is made
+# once both directory servers are connected to the one left: right after the other stops, a
+# directory server that has not connected again refuses writes, or takes one that its broker
+# still sends to the replication server going down. This runs before the joins on empty
+# volumes below, while every server trusts the certificate of every other: dj-d0, down while
+# dj-r1 comes back on an empty volume, may replay the new instance key of dj-r1 without its
+# certificate and never connect to it (#1188)
+connected_to() { # <directory server> <replication server>
+ local address
+ address=$(docker inspect -f "{{(index .NetworkSettings.Networks \"$NETWORK\").IPAddress}}" "$2") || return 1
+ # the monitor names the replication server by its address
+ admin_search "$1" "cn=dc_example_dc_com,cn=Replication,cn=monitor" one "(connected-to=*)" connected-to \
+ | grep -Fx "connected-to: $address:8989" >/dev/null
+}
+docker stop dj-r1 >/dev/null
+for n in dj-d0 dj-d1; do
+ wait_until 120 "$n is connected to dj-r0" connected_to $n dj-r0
+done
+add_ou dj-d1 roles-one-rs
+wait_has dj-d0 "ou=roles-one-rs,dc=example,dc=com"
+docker start dj-r1 >/dev/null
+wait_healthy dj-r1
+docker stop dj-r0 >/dev/null
+for n in dj-d0 dj-d1; do
+ wait_until 120 "$n is connected to dj-r1" connected_to $n dj-r1
+done
+add_ou dj-d0 roles-other-rs
+wait_has dj-d1 "ou=roles-other-rs,dc=example,dc=com"
+docker start dj-r0 >/dev/null
+wait_healthy dj-r0
+
+phase "roles: joins on empty volumes"
+# A joiner goes through a replication server while the directory server ahead of it in
+# REPLICATION_PEERS is down: dj-r1 and then dj-d1 come back on empty volumes with dj-d0
+# stopped. dj-r1 enables with dj-r0, both sides replication servers only - dsreplication
+# finds BASE_DN through the topology then, on a directory server it reaches, so dj-d1 is
+# still up; dj-d1 enables through a replication server and initializes from dj-d0 once that
+# is back
+docker stop dj-d0 >/dev/null
+docker rm -f dj-r1 >/dev/null
+docker volume rm vol-dj-r1 >/dev/null
+start_node dj-r1 $ROLE_PEERS -e REPLICATION_ROLE=replication -e REPLICATION_RETRY_COUNT=30
+wait_healthy dj-r1
+logs_have dj-r1 "joined the replication topology through dj-r0" || fail "dj-r1 did not join through dj-r0"
+docker rm -f dj-d1 >/dev/null
+docker volume rm vol-dj-d1 >/dev/null
+start_node dj-d1 $ROLE_PEERS -e REPLICATION_ROLE=directory -e REPLICATION_RETRY_COUNT=30
+wait_until 300 "dj-d1 joined through a replication server" logs_have dj-d1 "joined the replication topology through dj-r"
+docker start dj-d0 >/dev/null
+wait_healthy dj-d0
+wait_healthy dj-d1
+logs_have dj-d1 "initializing from dj-d0" || fail "dj-d1 did not initialize from dj-d0"
+ldaps_has dj-d1 "ou=roles-from-d1,dc=example,dc=com" || fail "dj-d1 lacks the entries of the topology"
+roles_hold
+docker rm -f dj-d0 dj-d1 dj-r0 dj-r1 >/dev/null
+docker volume rm vol-dj-d0 vol-dj-d1 vol-dj-r0 vol-dj-r1 >/dev/null
+
+phase "directory servers only"
+# directory servers with no replication server among their peers: the first seeds, the other
+# cannot join anything - dsreplication enable refuses a topology without a replication
+# server - says why, and does not turn healthy. dj-dn0 seeds once its retries find nobody,
+# so that the few rounds of dj-dn1 all run against a seed
+start_node dj-dn0 dj-dn0,dj-dn1 -e REPLICATION_ROLE=directory
+wait_healthy dj-dn0
+start_node dj-dn1 dj-dn0,dj-dn1 -e REPLICATION_ROLE=directory -e REPLICATION_RETRY_COUNT=4
+wait_until 300 "dj-dn1 says that dj-dn0 has no replication server" logs_have dj-dn1 "dj-dn0 is a directory server that no replication server joined yet"
+wait_until 300 "dj-dn1 gives up" logs_have dj-dn1 "could not join the replication topology after 4 attempts"
+stays_unhealthy dj-dn1 "dj-dn0 has no replication server"
+runs_replication_server dj-dn0 && fail "dj-dn0 was given a replication server by the join of dj-dn1"
+docker rm -f dj-dn0 dj-dn1 >/dev/null
+docker volume rm vol-dj-dn0 vol-dj-dn1 >/dev/null
+
+timings
cleanup
echo "Docker replication test passed"
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 879a50b..0319761 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -793,6 +793,103 @@
# the password never shows on a command line: sample every process's for two probe intervals
docker exec test_health_bind sh -c 'end=$(($(date +%s) + 12)); while [ "$(date +%s)" -lt "$end" ]; do for f in /proc/[0-9]*/cmdline; do tr "\0" " " < "$f" 2>/dev/null; echo; done | grep -q "[h]c_secret_1092" && exit 1; sleep 0.2; done; exit 0'
docker rm -f test_health_bind
+ - name: Docker test failed bootstrap
+ # a start over a volume whose bootstrap failed once setup had written its
+ # configuration must not report healthy either (#1182)
+ shell: bash
+ run: |
+ IMAGE=localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}
+ MEMORY=512m
+ trap 'code=$?; echo "::group::container logs (test_failed)"; docker logs test_failed 2>&1 || true; docker inspect --format="{{json .State.Health}}" test_failed 2>&1 || true; echo "::endgroup::"; docker rm -f test_failed >/dev/null 2>&1 || true; docker volume rm test_failed_data >/dev/null 2>&1 || true; exit $code' ERR
+ # an unknown backend type fails create-backend once setup has written data/config
+ docker run -d --memory="$MEMORY" --health-interval=5s -e ADD_BASE_ENTRY="--addBaseEntry" -e BACKEND_TYPE=bogus -v test_failed_data:/opt/opendj/data --name=test_failed "$IMAGE"
+ timeout 3m bash -c 'until docker logs test_failed 2>&1 | grep -qF "failed, this container will not report itself healthy"; do sleep 2; done'
+ docker restart -t 60 test_failed
+ timeout 3m bash -c 'until docker logs test_failed 2>&1 | grep -qF "The bootstrap of this volume did not complete"; do sleep 2; done'
+ # the server is up to be looked at, and no probe passes over 45 s of probes every 5 s
+ timeout 3m bash -c 'until docker exec test_failed /opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --useSsl --trustAll --bindDN "cn=Directory Manager" --bindPassword password --baseDN "" --searchScope base "(objectClass=*)" 1.1 >/dev/null 2>&1; do sleep 5; done'
+ end=$((SECONDS + 45))
+ while [ $SECONDS -lt $end ]; do
+ test "$(docker inspect --format='{{.State.Health.Status}}' test_failed)" != healthy
+ sleep 2
+ done
+ # the setup finished by hand and the mark removed, the next start reports healthy
+ docker exec test_failed /opt/opendj/bin/dsconfig create-backend --hostname localhost --port 4444 --bindDN "cn=Directory Manager" --bindPassword password --backend-name=userRoot --type je --set=base-dn:dc=example,dc=com --set=enabled:true --no-prompt --trustAll
+ docker exec test_failed rm /opt/opendj/data/.bootstrap-pending
+ docker restart -t 60 test_failed
+ timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_failed | grep -q \"healthy\"; do sleep 5; done'
+ docker rm -f test_failed
+ docker volume rm test_failed_data
+ - name: Docker test upgrade from a released image
+ # the build image started over a volume that the latest release bootstrapped
+ # upgrades it on start, the road the Installation Guide documents (#1183), and an
+ # upgrade that did not complete keeps it unhealthy on later starts too (#1185)
+ shell: bash
+ run: |
+ IMAGE=localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}
+ RELEASED=openidentityplatform/opendj:${{ env.release_version }}
+ MEMORY=512m
+ trap 'code=$?; echo "::group::container logs (test_upgrade)"; docker logs test_upgrade 2>&1 || true; docker inspect --format="{{json .State.Health}}" test_upgrade 2>&1 || true; echo "::endgroup::"; docker rm -f test_upgrade >/dev/null 2>&1 || true; docker volume rm test_upgrade_data >/dev/null 2>&1 || true; exit $code' ERR
+ search() { docker exec test_upgrade /opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --useSsl --trustAll --bindDN "cn=Directory Manager" --bindPassword password "$@"; }
+ # release.yml publishes the GitHub release before it pushes the images of it: in between,
+ # and while a failed push stays unfixed, the latest release has no image to start from.
+ # Only that skips the test - a tag the registry does not have, which the classic image
+ # store reports as "manifest unknown" and the containerd one as a reference it could not
+ # resolve, "not found"; a pull that failed otherwise - a rate limit, the registry or the
+ # network down - fails it rather than passing it untested
+ if ! out=$(docker pull "$RELEASED" 2>&1); then
+ echo "$out"
+ case $out in
+ *"manifest unknown"*|*"failed to resolve reference"*": not found"*) echo "::warning::$RELEASED is not on Docker Hub yet, the upgrade test is skipped"; exit 0 ;;
+ *) exit 1 ;;
+ esac
+ fi
+ docker run -d --memory="$MEMORY" -e ADD_BASE_ENTRY="--addBaseEntry" -e SAMPLE_DATA=10 -v test_upgrade_data:/opt/opendj/data --name=test_upgrade "$RELEASED"
+ # the entries, not the health status: the health check of 5.1.2 turned healthy before
+ # import-ldif had finished
+ export -f search
+ timeout 5m bash -c 'until search --baseDN "ou=People,dc=example,dc=com" --searchScope one "(uid=user.9)" 1.1 2>/dev/null | grep -q "^dn:"; do sleep 5; done'
+ docker stop -t 60 test_upgrade
+ docker rm test_upgrade
+ docker run -d --memory="$MEMORY" -v test_upgrade_data:/opt/opendj/data --name=test_upgrade "$IMAGE"
+ timeout 10m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_upgrade | grep -q \"healthy\"; do sleep 5; done'
+ # the upgrade ran its tasks: on equal versions it would be a no-op
+ if docker logs test_upgrade 2>&1 | grep -F "has already been upgraded"; then echo "::error::The build image found nothing to upgrade on the volume of $RELEASED"; false; fi
+ test "$(search --baseDN "ou=People,dc=example,dc=com" --searchScope one "(uid=user.*)" 1.1 | grep -c "^dn:")" = 10
+ # a successful upgrade leaves no mark behind (#1185)
+ docker exec test_upgrade test ! -e /opt/opendj/data/.upgrade-pending
+ # an upgrade that recorded the new version but did not complete its post-upgrade tasks:
+ # the next run of upgrade finds nothing to do, and the mark keeps the container unhealthy
+ # until the operator removes it
+ docker stop -t 60 test_upgrade
+ docker rm test_upgrade
+ docker run --rm --entrypoint sh -v test_upgrade_data:/opt/opendj/data "$IMAGE" -c 'echo "an earlier version to this one" >/opt/opendj/data/.upgrade-pending'
+ docker run -d --memory="$MEMORY" --health-interval=5s -v test_upgrade_data:/opt/opendj/data --name=test_upgrade "$IMAGE"
+ timeout 3m bash -c 'until docker logs test_upgrade 2>&1 | grep -qF "did not complete on an earlier start"; do sleep 2; done'
+ # the server is up to be looked at, and no probe passes over 45 s of probes every 5 s
+ timeout 3m bash -c 'until search --baseDN "" --searchScope base "(objectClass=*)" 1.1 >/dev/null 2>&1; do sleep 5; done'
+ end=$((SECONDS + 45))
+ while [ $SECONDS -lt $end ]; do
+ test "$(docker inspect --format='{{.State.Health.Status}}' test_upgrade)" != healthy
+ sleep 2
+ done
+ docker exec test_upgrade rm /opt/opendj/data/.upgrade-pending
+ docker restart -t 60 test_upgrade
+ timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_upgrade | grep -q \"healthy\"; do sleep 5; done'
+ docker rm -f test_upgrade
+ docker volume rm test_upgrade_data
+ # the mark goes on before the upgrade and stays when it fails: a read-only buildinfo
+ # fails it where it records the new version, after its upgrade tasks
+ docker run -d --memory="$MEMORY" -e ADD_BASE_ENTRY="--addBaseEntry" -v test_upgrade_data:/opt/opendj/data --name=test_upgrade "$RELEASED"
+ timeout 5m bash -c 'until search --baseDN "dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1 2>/dev/null | grep -q "^dn:"; do sleep 5; done'
+ docker stop -t 60 test_upgrade
+ docker rm test_upgrade
+ docker run --rm --user root --entrypoint chmod -v test_upgrade_data:/opt/opendj/data "$IMAGE" 444 /opt/opendj/data/config/buildinfo
+ docker run -d --memory="$MEMORY" -v test_upgrade_data:/opt/opendj/data --name=test_upgrade "$IMAGE"
+ timeout 5m bash -c 'until docker logs test_upgrade 2>&1 | grep -qF "The upgrade did not complete, this container will not report itself healthy"; do sleep 2; done'
+ docker run --rm --entrypoint test -v test_upgrade_data:/opt/opendj/data "$IMAGE" -s /opt/opendj/data/.upgrade-pending
+ docker rm -f test_upgrade
+ docker volume rm test_upgrade_data
- name: Scan image for vulnerabilities (Trivy)
# trivy resolves the image from the local Docker daemon, so only the runner's
# linux/amd64 manifest is scanned; cache: false keeps the ~1GB trivy DBs from
@@ -1187,6 +1284,103 @@
# the password never shows on a command line: sample every process's for two probe intervals
docker exec test_health_bind sh -c 'end=$(($(date +%s) + 12)); while [ "$(date +%s)" -lt "$end" ]; do for f in /proc/[0-9]*/cmdline; do tr "\0" " " < "$f" 2>/dev/null; echo; done | grep -q "[h]c_secret_1092" && exit 1; sleep 0.2; done; exit 0'
docker rm -f test_health_bind
+ - name: Docker test failed bootstrap
+ # a start over a volume whose bootstrap failed once setup had written its
+ # configuration must not report healthy either (#1182)
+ shell: bash
+ run: |
+ IMAGE=localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}-alpine
+ MEMORY=1g
+ trap 'code=$?; echo "::group::container logs (test_failed)"; docker logs test_failed 2>&1 || true; docker inspect --format="{{json .State.Health}}" test_failed 2>&1 || true; echo "::endgroup::"; docker rm -f test_failed >/dev/null 2>&1 || true; docker volume rm test_failed_data >/dev/null 2>&1 || true; exit $code' ERR
+ # an unknown backend type fails create-backend once setup has written data/config
+ docker run -d --memory="$MEMORY" --health-interval=5s -e ADD_BASE_ENTRY="--addBaseEntry" -e BACKEND_TYPE=bogus -v test_failed_data:/opt/opendj/data --name=test_failed "$IMAGE"
+ timeout 3m bash -c 'until docker logs test_failed 2>&1 | grep -qF "failed, this container will not report itself healthy"; do sleep 2; done'
+ docker restart -t 60 test_failed
+ timeout 3m bash -c 'until docker logs test_failed 2>&1 | grep -qF "The bootstrap of this volume did not complete"; do sleep 2; done'
+ # the server is up to be looked at, and no probe passes over 45 s of probes every 5 s
+ timeout 3m bash -c 'until docker exec test_failed /opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --useSsl --trustAll --bindDN "cn=Directory Manager" --bindPassword password --baseDN "" --searchScope base "(objectClass=*)" 1.1 >/dev/null 2>&1; do sleep 5; done'
+ end=$((SECONDS + 45))
+ while [ $SECONDS -lt $end ]; do
+ test "$(docker inspect --format='{{.State.Health.Status}}' test_failed)" != healthy
+ sleep 2
+ done
+ # the setup finished by hand and the mark removed, the next start reports healthy
+ docker exec test_failed /opt/opendj/bin/dsconfig create-backend --hostname localhost --port 4444 --bindDN "cn=Directory Manager" --bindPassword password --backend-name=userRoot --type je --set=base-dn:dc=example,dc=com --set=enabled:true --no-prompt --trustAll
+ docker exec test_failed rm /opt/opendj/data/.bootstrap-pending
+ docker restart -t 60 test_failed
+ timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_failed | grep -q \"healthy\"; do sleep 5; done'
+ docker rm -f test_failed
+ docker volume rm test_failed_data
+ - name: Docker test upgrade from a released image
+ # the build image started over a volume that the latest release bootstrapped
+ # upgrades it on start, the road the Installation Guide documents (#1183), and an
+ # upgrade that did not complete keeps it unhealthy on later starts too (#1185)
+ shell: bash
+ run: |
+ IMAGE=localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}-alpine
+ RELEASED=openidentityplatform/opendj:${{ env.release_version }}-alpine
+ MEMORY=1g
+ trap 'code=$?; echo "::group::container logs (test_upgrade)"; docker logs test_upgrade 2>&1 || true; docker inspect --format="{{json .State.Health}}" test_upgrade 2>&1 || true; echo "::endgroup::"; docker rm -f test_upgrade >/dev/null 2>&1 || true; docker volume rm test_upgrade_data >/dev/null 2>&1 || true; exit $code' ERR
+ search() { docker exec test_upgrade /opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --useSsl --trustAll --bindDN "cn=Directory Manager" --bindPassword password "$@"; }
+ # release.yml publishes the GitHub release before it pushes the images of it: in between,
+ # and while a failed push stays unfixed, the latest release has no image to start from.
+ # Only that skips the test - a tag the registry does not have, which the classic image
+ # store reports as "manifest unknown" and the containerd one as a reference it could not
+ # resolve, "not found"; a pull that failed otherwise - a rate limit, the registry or the
+ # network down - fails it rather than passing it untested
+ if ! out=$(docker pull "$RELEASED" 2>&1); then
+ echo "$out"
+ case $out in
+ *"manifest unknown"*|*"failed to resolve reference"*": not found"*) echo "::warning::$RELEASED is not on Docker Hub yet, the upgrade test is skipped"; exit 0 ;;
+ *) exit 1 ;;
+ esac
+ fi
+ docker run -d --memory="$MEMORY" -e ADD_BASE_ENTRY="--addBaseEntry" -e SAMPLE_DATA=10 -v test_upgrade_data:/opt/opendj/data --name=test_upgrade "$RELEASED"
+ # the entries, not the health status: the health check of 5.1.2 turned healthy before
+ # import-ldif had finished
+ export -f search
+ timeout 5m bash -c 'until search --baseDN "ou=People,dc=example,dc=com" --searchScope one "(uid=user.9)" 1.1 2>/dev/null | grep -q "^dn:"; do sleep 5; done'
+ docker stop -t 60 test_upgrade
+ docker rm test_upgrade
+ docker run -d --memory="$MEMORY" -v test_upgrade_data:/opt/opendj/data --name=test_upgrade "$IMAGE"
+ timeout 10m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_upgrade | grep -q \"healthy\"; do sleep 5; done'
+ # the upgrade ran its tasks: on equal versions it would be a no-op
+ if docker logs test_upgrade 2>&1 | grep -F "has already been upgraded"; then echo "::error::The build image found nothing to upgrade on the volume of $RELEASED"; false; fi
+ test "$(search --baseDN "ou=People,dc=example,dc=com" --searchScope one "(uid=user.*)" 1.1 | grep -c "^dn:")" = 10
+ # a successful upgrade leaves no mark behind (#1185)
+ docker exec test_upgrade test ! -e /opt/opendj/data/.upgrade-pending
+ # an upgrade that recorded the new version but did not complete its post-upgrade tasks:
+ # the next run of upgrade finds nothing to do, and the mark keeps the container unhealthy
+ # until the operator removes it
+ docker stop -t 60 test_upgrade
+ docker rm test_upgrade
+ docker run --rm --entrypoint sh -v test_upgrade_data:/opt/opendj/data "$IMAGE" -c 'echo "an earlier version to this one" >/opt/opendj/data/.upgrade-pending'
+ docker run -d --memory="$MEMORY" --health-interval=5s -v test_upgrade_data:/opt/opendj/data --name=test_upgrade "$IMAGE"
+ timeout 3m bash -c 'until docker logs test_upgrade 2>&1 | grep -qF "did not complete on an earlier start"; do sleep 2; done'
+ # the server is up to be looked at, and no probe passes over 45 s of probes every 5 s
+ timeout 3m bash -c 'until search --baseDN "" --searchScope base "(objectClass=*)" 1.1 >/dev/null 2>&1; do sleep 5; done'
+ end=$((SECONDS + 45))
+ while [ $SECONDS -lt $end ]; do
+ test "$(docker inspect --format='{{.State.Health.Status}}' test_upgrade)" != healthy
+ sleep 2
+ done
+ docker exec test_upgrade rm /opt/opendj/data/.upgrade-pending
+ docker restart -t 60 test_upgrade
+ timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_upgrade | grep -q \"healthy\"; do sleep 5; done'
+ docker rm -f test_upgrade
+ docker volume rm test_upgrade_data
+ # the mark goes on before the upgrade and stays when it fails: a read-only buildinfo
+ # fails it where it records the new version, after its upgrade tasks
+ docker run -d --memory="$MEMORY" -e ADD_BASE_ENTRY="--addBaseEntry" -v test_upgrade_data:/opt/opendj/data --name=test_upgrade "$RELEASED"
+ timeout 5m bash -c 'until search --baseDN "dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1 2>/dev/null | grep -q "^dn:"; do sleep 5; done'
+ docker stop -t 60 test_upgrade
+ docker rm test_upgrade
+ docker run --rm --user root --entrypoint chmod -v test_upgrade_data:/opt/opendj/data "$IMAGE" 444 /opt/opendj/data/config/buildinfo
+ docker run -d --memory="$MEMORY" -v test_upgrade_data:/opt/opendj/data --name=test_upgrade "$IMAGE"
+ timeout 5m bash -c 'until docker logs test_upgrade 2>&1 | grep -qF "The upgrade did not complete, this container will not report itself healthy"; do sleep 2; done'
+ docker run --rm --entrypoint test -v test_upgrade_data:/opt/opendj/data "$IMAGE" -s /opt/opendj/data/.upgrade-pending
+ docker rm -f test_upgrade
+ docker volume rm test_upgrade_data
- name: Scan image for vulnerabilities (Trivy)
# trivy resolves the image from the local Docker daemon, so only the runner's
# linux/amd64 manifest is scanned; cache: false keeps the ~1GB trivy DBs from
diff --git a/opendj-doc-generated-ref/src/main/asciidoc/admin-guide/chap-replication.adoc b/opendj-doc-generated-ref/src/main/asciidoc/admin-guide/chap-replication.adoc
index 8543fb6..eee6daf 100644
--- a/opendj-doc-generated-ref/src/main/asciidoc/admin-guide/chap-replication.adoc
+++ b/opendj-doc-generated-ref/src/main/asciidoc/admin-guide/chap-replication.adoc
@@ -41,7 +41,7 @@
[NOTE]
======
-The Docker image can join its replication topology itself, on every start of a container, when it is started with `OPENDJ_REPLICATION_TYPE=simple` and the servers of the topology listed in `REPLICATION_PEERS`. With `REPLICATION_PEERS` set, a container also removes from the topology the servers that are registered in it by name but not listed. To add a server to such a topology, first replace every container with one that lists the new server in its `REPLICATION_PEERS`, and only then start the new server: a container that still runs with the old list removes the new server from the topology when it starts again. To remove a server, remove its container first, then replace every other container with one that leaves it out of the list. The environment of a container cannot be changed, so restarting a container does not change its list; a server registered by an address, such as a master that `MASTER_SERVER` named by its address, is never removed this way. See the link:https://github.com/OpenIdentityPlatform/OpenDJ/blob/master/opendj-packages/opendj-docker/README.md#replication[Replication section of the README of the image, window=\_blank].
+The Docker image can join its replication topology itself, on every start of a container, when it is started with `OPENDJ_REPLICATION_TYPE=simple` and the servers of the topology listed in `REPLICATION_PEERS`. `REPLICATION_ROLE` sets the part each container takes: a directory server that also runs a replication server (`combined`, the default), a directory server without a replication server (`directory`), or a standalone replication server that holds no data (`replication`); see xref:#repl-dedicated-replica["Standalone Directory Server Replicas"] and xref:#repl-dedicated-servers["Standalone Replication Servers"]. The role is recorded on the volume on the first start, and a later start with another value keeps it. With `REPLICATION_PEERS` set, a container also removes from the topology the servers that are registered in it by name but not listed. To add a server to such a topology, first replace every container with one that lists the new server in its `REPLICATION_PEERS`, and only then start the new server: a container that still runs with the old list removes the new server from the topology when it starts again. To remove a server, remove its container first, then replace every other container with one that leaves it out of the list. The environment of a container cannot be changed, so restarting a container does not change its list; a server registered by an address, such as a master that `MASTER_SERVER` named by its address, is never removed this way. See the link:https://github.com/OpenIdentityPlatform/OpenDJ/blob/master/opendj-packages/opendj-docker/README.md#replication[Replication section of the README of the image, window=\_blank].
======
diff --git a/opendj-doc-generated-ref/src/main/asciidoc/install-guide/chap-install.adoc b/opendj-doc-generated-ref/src/main/asciidoc/install-guide/chap-install.adoc
index 7a7ab55..565a143 100644
--- a/opendj-doc-generated-ref/src/main/asciidoc/install-guide/chap-install.adoc
+++ b/opendj-doc-generated-ref/src/main/asciidoc/install-guide/chap-install.adoc
@@ -686,7 +686,7 @@
$ timeout 5m bash -c 'until [ "$(docker inspect -f "{{.State.Health.Status}}" opendj)" = healthy ]; do sleep 5; done'
----
+
-On its first start the container starts the server, creates the backend, imports the data that `ADD_BASE_ENTRY` asked for, then stops the server and starts it again, so a client that only waits for the port may see its first requests fail. The container reports itself healthy once all of this has succeeded. A first start that fails never reports healthy: what failed is in `docker logs opendj`. To try again, remove the container and its volume, with `docker rm -f opendj` and `docker volume rm opendj-data`, then start over: a container started over the volume of a failed first start can take it for an installed instance and report itself healthy without the backend or the base entry.
+On its first start the container starts the server, creates the backend, imports the data that `ADD_BASE_ENTRY` asked for, then stops the server and starts it again, so a client that only waits for the port may see its first requests fail. The container reports itself healthy once all of this has succeeded. A first start that fails never reports healthy: what failed is in `docker logs opendj`. Nor does a later start over the same volume: the volume carries the file `/opt/opendj/data/.bootstrap-pending` from before the first start sets the instance up until it has succeeded, and a container started over a volume that still carries it starts the server so that it can be looked at, logs that the bootstrap did not complete, and does not report itself healthy. To try again, remove the container and its volume, with `docker rm -f opendj` and `docker volume rm opendj-data`, then start over. Alternatively, finish the setup by hand, then remove the file with `docker exec opendj rm /opt/opendj/data/.bootstrap-pending` and restart the container with `docker restart opendj`.
. (Optional) Check that the server serves the base entry:
+
diff --git a/opendj-doc-generated-ref/src/main/asciidoc/install-guide/chap-upgrade.adoc b/opendj-doc-generated-ref/src/main/asciidoc/install-guide/chap-upgrade.adoc
index 9ee5239..27de861 100644
--- a/opendj-doc-generated-ref/src/main/asciidoc/install-guide/chap-upgrade.adoc
+++ b/opendj-doc-generated-ref/src/main/asciidoc/install-guide/chap-upgrade.adoc
@@ -369,7 +369,7 @@
$ timeout 5m bash -c 'until [ "$(docker inspect -f "{{.State.Health.Status}}" opendj)" = healthy ]; do sleep 5; done'
----
+
-The upgrade tasks that take long, such as verifying or rebuilding indexes, are performed before the server starts, since nobody is there to run them afterwards. On a large instance they can take longer than the start period of the health check, so the container may report `unhealthy` until the upgrade is over, and the command above may give up before: `docker logs opendj` shows which task is running, and the command can be run again. A container whose upgrade fails does not report itself healthy: `docker logs opendj` shows why, and the upgrade log `/opt/opendj/data/logs/upgrade.log` has the details. Do not start it again to get past the failure: when a task that runs after the upgrade fails, such as an index rebuild, the new version is already recorded, so the next start skips the upgrade and reports healthy with the indexes as the failed rebuild left them, and without the tasks that were to follow it. Revert as follows, or rebuild the indexes that `docker logs opendj` names before you rely on the container.
+The upgrade tasks that take long, such as verifying or rebuilding indexes, are performed before the server starts, since nobody is there to run them afterwards. On a large instance they can take longer than the start period of the health check, so the container may report `unhealthy` until the upgrade is over, and the command above may give up before: `docker logs opendj` shows which task is running, and the command can be run again. A container whose upgrade fails does not report itself healthy: `docker logs opendj` shows why, and the upgrade log `/opt/opendj/data/logs/upgrade.log` has the details. Starting it again does not get past the failure either: the volume carries the file `/opt/opendj/data/.upgrade-pending` from before the upgrade until the upgrade, the tasks that run after it included, has succeeded. An upgrade that failed before it recorded the new version is run again on the next start. When a task that runs after the upgrade fails, such as an index rebuild, the new version is already recorded, and the upgrade would find nothing left to do: a container started over the volume then starts the server, logs that the upgrade did not complete, and does not report itself healthy. Revert as follows, or rebuild the indexes that `upgrade.log` names, as described in xref:../admin-guide/chap-indexing.adoc#rebuild-index["Rebuilding Indexes"] in the __Administration Guide__, then remove the file with `docker exec opendj rm /opt/opendj/data/.upgrade-pending` and restart the container with `docker restart opendj`.
To revert, remove the new container, replace the volume with one restored from the backup, and start a container from the older image over it, with the options of the old container. Restore into an empty volume: files that the upgrade added would be left behind in a volume that the archive is extracted over.
diff --git a/opendj-packages/opendj-docker/Dockerfile b/opendj-packages/opendj-docker/Dockerfile
index 0ebaf0a..7fc6145 100644
--- a/opendj-packages/opendj-docker/Dockerfile
+++ b/opendj-packages/opendj-docker/Dockerfile
@@ -32,6 +32,8 @@
# every server of the replication topology, comma separated; the first entry may seed a new
# topology. MASTER_SERVER keeps working as a one-element list. See bootstrap/join.sh
#ENV REPLICATION_PEERS
+# the part this server takes in the topology: combined, directory or replication (#1178)
+#ENV REPLICATION_ROLE=combined
#ENV REPLICATION_PORT=8989
#ENV REPLICATION_RETRY_COUNT=30
#ENV REPLICATION_RETRY_INTERVAL=10
diff --git a/opendj-packages/opendj-docker/Dockerfile-alpine b/opendj-packages/opendj-docker/Dockerfile-alpine
index fba79d4..44957c4 100644
--- a/opendj-packages/opendj-docker/Dockerfile-alpine
+++ b/opendj-packages/opendj-docker/Dockerfile-alpine
@@ -32,6 +32,8 @@
# every server of the replication topology, comma separated; the first entry may seed a new
# topology. MASTER_SERVER keeps working as a one-element list. See bootstrap/join.sh
#ENV REPLICATION_PEERS
+# the part this server takes in the topology: combined, directory or replication (#1178)
+#ENV REPLICATION_ROLE=combined
#ENV REPLICATION_PORT=8989
#ENV REPLICATION_RETRY_COUNT=30
#ENV REPLICATION_RETRY_INTERVAL=10
diff --git a/opendj-packages/opendj-docker/README.md b/opendj-packages/opendj-docker/README.md
index 40415c9..b1134a4 100644
--- a/opendj-packages/opendj-docker/README.md
+++ b/opendj-packages/opendj-docker/README.md
@@ -55,11 +55,24 @@
once it is done that server is stopped and started again in the foreground, so a client
that only waits for the port can have its first requests fail in between.
-A bootstrap that imports `SAMPLE_DATA` can take minutes on a small container, which is what
-the start period allows for. A bootstrap that fails - or an upgrade that fails when starting
-over an instance that is already there - never reports healthy: what failed is in `docker
-logs`, and where the server is up at all the container is left running to be looked at,
-turning `unhealthy` once the start period is over.
+A bootstrap that imports `SAMPLE_DATA` can take minutes on a small container, which is what the
+start period allows for. A bootstrap that fails - or an upgrade that fails when starting over
+an instance that is already there - never reports healthy: what failed is in `docker logs`, and
+where the server is up at all the container is left running to be looked at, turning
+`unhealthy` once the start period is over. Nor does any later start over the same volume: a
+volume carries `.bootstrap-pending` in its data directory from before its bootstrap until the
+bootstrap succeeded, and a container started over a volume that still carries it upgrades the
+instance when it is of another version, starts the server, logs that the bootstrap did not
+complete, and stays `unhealthy`. This holds for a volume of the background join as well: it
+runs no join, since an initialize from the topology would bring the data of the topology but
+nothing else the bootstrap was to configure. Remove the volume and start over, or finish the
+setup by hand and remove the file; a volume of the background join then joins on the restart
+and takes the data of the topology, as on its first start:
+
+```bash
+docker exec opendj rm /opt/opendj/data/.bootstrap-pending
+docker restart opendj
+```
Started over an instance of an older version, the container runs `upgrade --no-prompt --force`
before the server: the upgrade tasks that take long, such as verifying or rebuilding indexes,
@@ -67,6 +80,24 @@
take longer than the start period, so the container may report `unhealthy` until the upgrade
is over; `docker logs` shows which task is running.
+An upgrade that fails never reports healthy, and neither does any later start over the same
+volume, even though the upgrade records the new version before its post-upgrade tasks - the
+index rebuilds and verifications - and a second run would find nothing left to do: the volume
+carries `.upgrade-pending` in its data directory from before the upgrade until the upgrade,
+post-upgrade tasks included, succeeded. An upgrade that failed before it recorded the new
+version is run again on the next start. Once it has recorded it, a container started over the
+marked volume starts the server, logs that the upgrade did not complete, and stays `unhealthy`:
+`/opt/opendj/data/logs/upgrade.log` names the task that failed. Rebuild the indexes it names
+and remove the file, or restore the backup taken before the upgrade:
+
+```bash
+docker exec opendj /opt/opendj/bin/rebuild-index --hostname localhost --port 4444 \
+ --bindDN "cn=Directory Manager" --bindPassword password --trustAll \
+ --baseDN dc=example,dc=com --index <index>
+docker exec opendj rm /opt/opendj/data/.upgrade-pending
+docker restart opendj
+```
+
The server runs as PID 1 of the container, and a JVM does not reap the processes left
behind to it - those of a health check that ran past its timeout, say. Run the container
with `docker run --init` (`init: true` in Compose) to put a PID 1 in front of the server
@@ -102,7 +133,8 @@
removed by hand.
The join decides from what is there, not from an exit code: this server is a member once
-its configuration holds the replication domain for `BASE_DN` and it is registered in
+its configuration holds the replication domain for `BASE_DN` (a `replication` server: its
+replication server, see [Server roles](#server-roles)) and it is registered in
`cn=admin data`. Until then it runs `dsreplication enable` through the listed peers,
`REPLICATION_RETRY_COUNT` times every `REPLICATION_RETRY_INTERVAL` seconds, each enable
bounded by `REPLICATION_ATTEMPT_TIMEOUT` seconds, and the container reports itself healthy
@@ -145,14 +177,17 @@
started by hand - which is where its data has to come from anyway.
With `REPLICATION_PEERS` set explicitly, a joined server also removes every server that is
-registered in the topology but no longer listed: its entries in `cn=admin data` and its
-values in every local replication server list - those of `BASE_DN`, and of `cn=schema` and
-`cn=admin data`, which `dsreplication enable` replicates next to it. A scale-down therefore
-needs no `preStop` hook - `dsreplication disable` on termination would take the server out
-on every rolling restart, and cannot clean up a server that is already gone. It also adds
-every listed peer registered in the topology to the local lists that lack it, so joins that
-ran at the same time cannot leave two replication servers that know nothing of each other.
-With only `MASTER_SERVER` set nothing is removed or added, so servers joined by hand stay.
+registered in the topology but no longer listed: its entries in `cn=admin data` and its values
+in every local replication server list - those of `BASE_DN`, and of `cn=schema` and `cn=admin
+data`, which `dsreplication enable` replicates next to it. A scale-down therefore needs no
+`preStop` hook - `dsreplication disable` on termination would take the server out on every
+rolling restart, and cannot clean up a server that is already gone. It also adds every listed
+peer registered in the topology that runs a replication server to the local lists that lack it,
+so joins that ran at the same time cannot leave two replication servers that know nothing of
+each other. A peer that does not answer whether it runs one is left for a later start, as a
+`directory` server put into a list would stay there for good; a replication server left out
+adds this server to its own lists when it starts. With only `MASTER_SERVER` set nothing is
+removed or added, so servers joined by hand stay.
A server that was removed this way and is scaled up again on the volume it kept still holds
the data and its replication configuration, but no peer registers it any more, and
@@ -179,8 +214,62 @@
failed, say): `BASE_DN` is replicated, and every later enable only reports it as replicated
already, without registering the server where it is missing.
+### Server roles
+
+`REPLICATION_ROLE` sets the part a server of `simple` takes in its topology:
+
+| Value | Server |
+|---|---|
+| `combined` (default) | a directory server with a replication server of its own |
+| `directory` | a directory server without a replication server, which connects to those of its peers |
+| `replication` | a replication server only: it holds no data, and its bootstrap creates no backend for `BASE_DN` and loads no entries |
+
+Every role is listed in the same `REPLICATION_PEERS`, so that each server can reach every other
+one. Only the first entry that holds data - `combined` or `directory` - may seed the topology:
+`replication` servers listed ahead of it are passed over, by the role they publish, and until
+each of them has published one the server after them does not seed. A `replication` server
+publishes `pending` until it is a member, then `ready`, and on later starts `rejoining` until
+it is found a member again; it is never initialized and never seeds. Joining through another
+`replication` server, it needs a server that holds the data to be up as well: `dsreplication`
+finds `BASE_DN` on the servers of the topology it reaches, and until one answers the join tries
+again. A `directory` server joins only a topology that has a replication server: it does not
+enable with a `directory` peer that no replication server joined yet, logs that, and tries the
+next peer or the next round. Each side of an enable is given its own role, the peer's read from
+the peer, so a server that joins through a `directory` server never hands it a replication
+server - unless that server runs an image from before roles, whose join gives every peer one:
+move every server to an image with roles before any of them takes another role than `combined`.
+Roles may be mixed - `combined` servers with additional `directory` ones, say. Separate
+replication servers on Kubernetes take two StatefulSets, one per role, with one
+`REPLICATION_PEERS` that lists the pods of both, the directory servers first:
+
+```
+REPLICATION_PEERS=ds-0.ds,ds-1.ds,ds-2.ds,ds-3.ds,rs-0.rs,rs-1.rs
+```
+
+The role is set when the volume is set up, and recorded on it (`.replication-role` in the
+data directory); a `replication` server is set up without a backend for `BASE_DN`. A later
+start with another `REPLICATION_ROLE` keeps the role the volume has and logs that it does: a
+`replication` server takes an empty volume. A volume set up by an image before roles is read
+on its first start under this one - a replication server without a domain is the replication
+server of `srs` when `REPLICATION_ROLE=replication` says so - and its role recorded then.
+Without `REPLICATION_PEERS` (or `MASTER_SERVER`) there is no join, and `REPLICATION_ROLE` is
+ignored; a value that is not one of the three fails the bootstrap of a new volume, and ends
+the join of a volume set up by an image before roles; on a volume that records its role it
+is logged, and the role stands. The replication servers keep the changes of the topology,
+not its data: should every server that holds data lose its volume, replace the volumes of
+the replication servers as well, as the first peer that holds data seeds a new topology only
+once no other peer is ready.
+
The one-shot types `srs`, `sdsr` and `rg` keep their previous behaviour - they run once,
-during the first bootstrap only - and are deprecated in favour of `simple`. Like `simple`,
+during the first bootstrap only - and are deprecated in favour of `simple`: what `srs` builds
+is `simple` with `REPLICATION_ROLE=replication` for the replication server and `directory` for
+the others, and `sdsr` is `directory` servers next to `combined` ones. Their `MASTER_SERVER`
+does not carry over as it is: under `simple` it becomes the only peer, and the replication
+server of `srs` holds no data to seed the topology with - list every server in
+`REPLICATION_PEERS` instead. The replication server of `srs` keeps
+its role under `simple` when it is started with `REPLICATION_ROLE=replication`, together with
+the backend that the bootstrap of an image before this one created on every server: it holds
+what that bootstrap loaded, and is not replicated. Like `simple`,
they need one `ADMIN_PORT` and one `REPLICATION_PORT` on every server: the tools reach the
master on the ports of the container they run in (images before this one used 4444 and
8989 on both sides, whatever the container was set up with). With
@@ -268,6 +357,7 @@
| SECRET_VOLUME_REFRESH | 60 | While the server runs, `SECRET_VOLUME` is checked again every that many seconds and changed files are copied again; `0` copies them on start only |
| MASTER_SERVER | - | Replication master server; with `simple` it works as a one-element `REPLICATION_PEERS` |
| REPLICATION_PEERS | value of MASTER_SERVER | every server of the replication topology by DNS name, comma separated; the first entry may seed a new topology, and servers no longer listed are removed from it, see [Replication](#replication) |
+| REPLICATION_ROLE | combined | with `OPENDJ_REPLICATION_TYPE=simple`, the part this server takes: `combined`, `directory` (no replication server of its own) or `replication` (a replication server without data), see [Server roles](#server-roles) |
| REPLICATION_PORT | 8989 | replication port, the same on every server of the topology |
| REPLICATION_RETRY_COUNT | 30 | rounds of join attempts through every peer before the join gives up: the first peer of `REPLICATION_PEERS` then seeds the topology, any other server stays `unhealthy`; a value that is not a whole number above 0 falls back to 30 |
| REPLICATION_RETRY_INTERVAL | 10 | seconds between rounds of join attempts, plus a random part of as much again; a value that is not a whole number falls back to 10 |
diff --git a/opendj-packages/opendj-docker/bootstrap/join.sh b/opendj-packages/opendj-docker/bootstrap/join.sh
index 8afa04d..7ecf173 100755
--- a/opendj-packages/opendj-docker/bootstrap/join.sh
+++ b/opendj-packages/opendj-docker/bootstrap/join.sh
@@ -48,8 +48,9 @@
# in $STATE_DN, a local entry of cn=config: "pending" while the marker is there, "ready" once
# the volume holds the data of the topology - it was never bootstrapped by run.sh, it was
# initialized from a ready peer, or it seeded the topology - and "rejoining" while a volume
-# that holds it has its replication taken down to enable it anew. A server joins only through
-# a ready peer, and a pending one initializes only from one, so two fresh servers that start
+# that holds it has its replication taken down to enable it anew, or while a replication
+# server is not found a member yet on this start. A server joins only through a ready peer,
+# and a pending one initializes only from one, so two fresh servers that start
# together never take each other's bootstrap data for the topology's, and two servers whose
# replication is down never form a topology of their own; where the list is only
# MASTER_SERVER, a peer that publishes no state (an image before this one) counts as ready, as
@@ -70,6 +71,21 @@
# run.sh) - unless the join takes its replication down to enable it anew: from then until the
# enable registered it again it is not, across restarts too ($REJOIN_PENDING), and the
# backend of BASE_DN refuses the writes of clients, which no longer replicate meanwhile.
+#
+# REPLICATION_ROLE (#1178) is the part this server takes: combined, a directory server with a
+# replication server of its own (the default); directory, one that connects to the
+# replication servers of other peers; replication, a replication server that holds no data.
+# The role of each side of an enable is passed to dsreplication: unless it is told otherwise,
+# the tool gives both servers a replication server and a domain for BASE_DN, so an enable
+# through a directory server would hand it a replication server. A peer's role is read from
+# the peer - what its configuration holds, or else what it publishes next to its state. A
+# replication server goes the way of a volume that waits for the data of the topology,
+# publishing itself pending until it is a member, but nothing initializes it; it never seeds,
+# and one listed first is passed over. A directory server joins only a topology that has a
+# replication server: dsreplication enable refuses one without, before it changes anything.
+# The role is set when setup.sh sets the volume up - a replication server gets no backend for
+# BASE_DN - and recorded on it: a later start in another role only says so, and the join goes
+# on in the role the server has (ROLE below).
cd /opt/opendj || exit 1
export PATH=/opt/opendj/bin:$PATH
@@ -83,9 +99,15 @@
BOOTSTRAP_COMPLETE=${BOOTSTRAP_COMPLETE:-/opt/opendj/.bootstrap-complete}
INITIALIZE_PENDING=${INITIALIZE_PENDING:-/opt/opendj/data/.replication-initialize-pending}
REJOIN_PENDING=${REJOIN_PENDING:-/opt/opendj/data/.replication-rejoin-pending}
+# the role the volume was set up with (setup.sh), or the one a first start of this image read
+# from a volume set up before roles
+REPLICATION_ROLE_FILE=${REPLICATION_ROLE_FILE:-/opt/opendj/data/.replication-role}
# not replicated: cn=config is this server's alone, and it lives on the volume next to the
# marker, so the two cannot tell different stories
STATE_DN="cn=Docker Join,cn=config"
+# BASE_DN as a value in a search filter (RFC 4515): a parenthesis, an asterisk or a backslash
+# in it would break the filter, or make it match more than BASE_DN
+BASE_DN_VALUE=$(printf '%s' "$BASE_DN" | sed -e 's/\\/\\5c/g' -e 's/\*/\\2a/g' -e 's/(/\\28/g' -e 's/)/\\29/g')
# Sets the variable to its value read as a decimal whole number, or to the default when it
# is not one or is below the least value. $(( )) reads a leading zero as octal and fails on
@@ -118,6 +140,22 @@
REPLICATION_INITIALIZE_TIMEOUT=${REPLICATION_INITIALIZE_TIMEOUT:-0}
whole_number REPLICATION_INITIALIZE_TIMEOUT 0 0
+# a role it does not know is not taken for the default: a server started as a replication
+# server by a misspelt value would otherwise join holding data. A volume that records its
+# role (see ROLE below) joins in that role whatever the value, as it does with another role
+REPLICATION_ROLE=${REPLICATION_ROLE:-combined}
+case $REPLICATION_ROLE in
+ combined | directory | replication) ;;
+ *)
+ if [ -s "$REPLICATION_ROLE_FILE" ]; then
+ echo "join: REPLICATION_ROLE=$REPLICATION_ROLE is not one of combined, directory or replication, the role recorded on the volume stands"
+ else
+ echo "join: REPLICATION_ROLE=$REPLICATION_ROLE is not one of combined, directory or replication, this server does not join"
+ exit 1
+ fi
+ ;;
+esac
+
# Removing servers that left the topology needs the operator's word on who belongs to it:
# with only MASTER_SERVER set, a third server joined by hand would be "not in the list" and
# thrown out. So the cleanup, and the repair of the replication server lists, run only when
@@ -252,10 +290,12 @@
search localhost --baseDN "" --searchScope base "(objectClass=*)" 1.1 >/dev/null
}
+# the state goes with the role of this server, after it: a peer reads the first word as the
+# state, which a join of an image before roles reads alone
publish_state() {
- printf 'dn: %s\nchangetype: modify\nreplace: description\ndescription: %s\n' "$STATE_DN" "$1" \
+ printf 'dn: %s\nchangetype: modify\nreplace: description\ndescription: %s %s\n' "$STATE_DN" "$1" "$ROLE" \
| ldapmodify_local >/dev/null && return 0
- printf 'dn: %s\nobjectClass: top\nobjectClass: ds-cfg-branch\nobjectClass: extensibleObject\ncn: Docker Join\ndescription: %s\n' "$STATE_DN" "$1" \
+ printf 'dn: %s\nobjectClass: top\nobjectClass: ds-cfg-branch\nobjectClass: extensibleObject\ncn: Docker Join\ndescription: %s %s\n' "$STATE_DN" "$1" "$ROLE" \
| ldapmodify_local --defaultAdd >/dev/null && return 0
echo "join: could not publish the state $1 of this server to its peers"
return 1
@@ -299,19 +339,98 @@
printf '%s\n' "$out" | awk 'tolower($1) == "hostname:" { print $2 }'
}
-# a member holds the replication domain for BASE_DN in its configuration and is registered
+# a member holds the replication domain for BASE_DN in its configuration - a replication
+# server its replication server instead (see holds_own_part) - and is registered
# in cn=admin data - under the name the enable that registered it connected with, which for
# a server that never ran an enable of its own is the name a peer listed it by; both are made
# by one dsreplication enable, so requiring both keeps a half-done enable from being declared
# a success, and the next round takes it down (see reset_if_unregistered)
replicates_base_dn() { # <host>
- search "$1" --baseDN "cn=config" --searchScope sub \
- "(&(objectClass=ds-cfg-replication-domain)(ds-cfg-base-dn=$BASE_DN))" 1.1 | grep -q "^dn:"
+ grep -qx domain <<<"$(replication_config "$1")"
+}
+
+# What of the replication the server holds, one line each: "server" for a replication server,
+# "domain" for the replication domain of BASE_DN, "backend" for a backend of BASE_DN, and
+# "published <role>" for the state it publishes - the role is left out where the state comes
+# without one, from an image before roles - all from one search, as every ldapsearch starts a
+# JVM. Nothing for a server that has none of
+# them, and a failure when it cannot be asked.
+replication_config() { # <host>
+ local out
+ out=$(search "$1" --baseDN "cn=config" --searchScope sub \
+ "(|(objectClass=ds-cfg-replication-server)(&(objectClass=ds-cfg-replication-domain)(ds-cfg-base-dn=$BASE_DN_VALUE))(&(objectClass=ds-cfg-backend)(ds-cfg-base-dn=$BASE_DN_VALUE))(cn=Docker Join))" \
+ objectClass description) \
+ || return 1
+ printf '%s\n' "$out" | awk -v state="$(lower "$STATE_DN")" '
+ /^dn: / { is_state = (tolower(substr($0, 5)) == state) }
+ tolower($0) == "objectclass: ds-cfg-replication-server" { print "server" }
+ tolower($0) == "objectclass: ds-cfg-replication-domain" { print "domain" }
+ tolower($0) == "objectclass: ds-cfg-backend" { print "backend" }
+ is_state && tolower($1) == "description:" { print "published", $3 }'
+}
+
+# What a replication configuration holds: "combined" for a replication server and the domain
+# of BASE_DN, "directory" for the domain alone, "server" for a replication server alone,
+# nothing for neither. Only the first two tell a role by themselves.
+config_shape() { # <replication_config output>
+ local server=no domain=no
+ grep -qx server <<<"$1" && server=yes
+ grep -qx domain <<<"$1" && domain=yes
+ case $server$domain in
+ yesyes) echo combined ;;
+ noyes) echo directory ;;
+ yesno) echo server ;;
+ esac
+}
+
+# The role of a peer: the one its configuration tells, or else the one it publishes next to
+# its state, which tells a replication server from a combined server whose enable stopped
+# half-way, before it configured the domain - and combined for a peer that publishes none, an
+# image before roles or a master that runs no join, as every server was before roles.
+peer_role() { # <replication_config output>
+ case $(config_shape "$1") in
+ combined) echo combined ;;
+ directory) echo directory ;;
+ *) published_role "$1" ;;
+ esac
+}
+
+# the role a server publishes next to its state; combined for one that publishes none, as a
+# server of an image before roles is one - or a server that runs no join, whose state entry a
+# search does not find
+published_role() { # <replication_config output>
+ local role
+ role=$(awk '$1 == "published" { print $2; exit }' <<<"$1")
+ case $role in
+ directory | replication) echo "$role" ;;
+ *) echo combined ;;
+ esac
+}
+
+# this server holds what its role gives it: the replication domain of BASE_DN for a combined
+# or a directory server, a replication server for a replication server
+holds_own_part() {
+ local config
+ config=$(replication_config localhost) || return 1
+ if [ "$ROLE" = replication ]; then
+ grep -qx server <<<"$config"
+ else
+ grep -qx domain <<<"$config"
+ fi
+}
+
+# the dsreplication enable arguments that give the server of that side its role, one a line
+role_arguments() { # <1|2> <role>
+ case $2 in
+ directory) printf '%s\n' "--noReplicationServer$1" ;;
+ replication) printf '%s\n' "--onlyReplicationServer$1" "--replicationPort$1" "$REPLICATION_PORT" ;;
+ *) printf '%s\n' "--replicationPort$1" "$REPLICATION_PORT" ;;
+ esac
}
is_member() {
local host
- replicates_base_dn localhost || return 1
+ holds_own_part || return 1
for host in $(registered_hosts); do
is_self "$host" && return 0
done
@@ -321,16 +440,25 @@
# cn=admin data is replicated, but a server that was away while the survivors removed it
# from the topology still holds its own entry until that delete reaches it, and it may not
# have yet when the join looks: the other peers that answer and replicate BASE_DN have the
-# say. Being unregistered takes this server's replication down (reset_replication), so it is
+# say - a replication server as well, which replicates cn=admin data without BASE_DN. Being
+# unregistered takes this server's replication down (reset_replication), so it is
# concluded only when at least one of them listed its registrations and none of them lists
# this server: a search that failed, or a peer whose copy of cn=admin data still lags behind
# the others, decides nothing while another peer lists it. With no such peer the local view
# stands - nothing else can be asked.
registered_with_peers() {
- local peer host hosts asked=no
+ local peer host hosts config asked=no
for peer in "${PEERS[@]}"; do
is_self "$peer" && continue
- replicates_base_dn "$peer" || continue
+ # a replication server alone is a member only as a replication server; on a combined
+ # server it is an enable that stopped before the domain and the initialize of cn=admin
+ # data, whose registrations are its own
+ config=$(replication_config "$peer") || continue
+ case $(config_shape "$config") in
+ combined | directory) ;;
+ server) [ "$(peer_role "$config")" = replication ] || continue ;;
+ *) continue ;;
+ esac
hosts=$(registered_hosts "$peer") || continue
asked=yes
for host in $hosts; do
@@ -430,8 +558,13 @@
# exits 75 without running the enable while another server enables through the peer or
# through this one. Neither lock is waited for, so two servers that each hold their own and
# ask for the other's cannot deadlock; they both give up and try again after pause().
+# It exits 75 as well when the peer's role cannot be read - guessed wrong, the enable would
+# give a directory server a replication server - and when a directory server would enable
+# with a directory server that no replication server joined yet: that topology has none, and
+# dsreplication enable refuses it. The peer is read under its lock, which an enable through
+# it by another server holds while it changes what the peer runs.
enable_through() {
- local rc own peer_lock
+ local rc own peer_lock config role peer_args own_args
take_join_lock localhost || return 75
own=$JOIN_LOCK_VALUE
if ! take_join_lock "$1"; then
@@ -439,14 +572,25 @@
return 75
fi
peer_lock=$JOIN_LOCK_VALUE
- bounded "$REPLICATION_ATTEMPT_TIMEOUT" dsreplication enable \
- --host1 "$1" --port1 "$ADMIN_PORT" --bindDN1 "$ROOT_USER_DN" \
- --bindPasswordFile1 "$PASSWORD_FILE" --replicationPort1 "$REPLICATION_PORT" \
- --host2 "$MYHOSTNAME" --port2 "$ADMIN_PORT" --bindDN2 "$ROOT_USER_DN" \
- --bindPasswordFile2 "$PASSWORD_FILE" --replicationPort2 "$REPLICATION_PORT" \
- --adminUID admin --adminPasswordFile "$PASSWORD_FILE" \
- --baseDN "$BASE_DN" -X -n
- rc=$?
+ if ! config=$(replication_config "$1"); then
+ echo "join: could not read how $1 replicates"
+ rc=75
+ elif role=$(peer_role "$config") && [ "$ROLE" = directory ] && [ "$role" = directory ] \
+ && [ -z "$(config_shape "$config")" ]; then
+ echo "join: $1 is a directory server that no replication server joined yet, and a directory server joins only through a topology that has one"
+ rc=75
+ else
+ mapfile -t peer_args < <(role_arguments 1 "$role")
+ mapfile -t own_args < <(role_arguments 2 "$ROLE")
+ bounded "$REPLICATION_ATTEMPT_TIMEOUT" dsreplication enable \
+ --host1 "$1" --port1 "$ADMIN_PORT" --bindDN1 "$ROOT_USER_DN" \
+ --bindPasswordFile1 "$PASSWORD_FILE" "${peer_args[@]}" \
+ --host2 "$MYHOSTNAME" --port2 "$ADMIN_PORT" --bindDN2 "$ROOT_USER_DN" \
+ --bindPasswordFile2 "$PASSWORD_FILE" "${own_args[@]}" \
+ --adminUID admin --adminPasswordFile "$PASSWORD_FILE" \
+ --baseDN "$BASE_DN" -X -n
+ rc=$?
+ fi
release_join_lock "$1" "$peer_lock"
release_join_lock localhost "$own"
return $rc
@@ -472,7 +616,7 @@
# every directory server connected to it, but only the domain publishes replayed-updates
generation_id() {
search "$1" --baseDN "cn=monitor" --searchScope sub \
- "(&(domain-name=$BASE_DN)(replayed-updates=*))" generation-id \
+ "(&(domain-name=$BASE_DN_VALUE)(replayed-updates=*))" generation-id \
| awk 'tolower($1) == "generation-id:" { print $2; exit }'
}
@@ -606,7 +750,8 @@
return 1
fi
own=$JOIN_LOCK_VALUE
- if ! hold_writes; then
+ # a replication server has no backend of BASE_DN, and no client writes to hold
+ if [ "$ROLE" != replication ] && ! hold_writes; then
echo "join: could not stop the writes of clients, its replication configuration stays as it is"
release_join_lock localhost "$own"
return 1
@@ -623,11 +768,14 @@
}
# "<backend-id> <writability-mode>" of the backend that holds BASE_DN; the mode is left out
-# where the entry does not set one, which means enabled
+# where the entry does not set one, which means enabled. Nothing where no backend holds it,
+# and a failure when the search failed
data_backend() {
- search localhost --baseDN "cn=Backends,cn=config" --searchScope one \
- "(&(objectClass=ds-cfg-backend)(ds-cfg-base-dn=$BASE_DN))" ds-cfg-backend-id ds-cfg-writability-mode \
- | awk 'tolower($1) == "ds-cfg-backend-id:" { id = $2 } tolower($1) == "ds-cfg-writability-mode:" { mode = $2 }
+ local out
+ out=$(search localhost --baseDN "cn=Backends,cn=config" --searchScope one \
+ "(&(objectClass=ds-cfg-backend)(ds-cfg-base-dn=$BASE_DN_VALUE))" ds-cfg-backend-id ds-cfg-writability-mode) \
+ || return 1
+ printf '%s\n' "$out" | awk 'tolower($1) == "ds-cfg-backend-id:" { id = $2 } tolower($1) == "ds-cfg-writability-mode:" { mode = $2 }
END { if (id != "") print id, mode }'
}
@@ -700,9 +848,12 @@
# servers that know nothing of each other, cutting the updates of one off from the other; a
# replication server connects to a server added to its list at once. A peer that is not
# registered yet is waited for, as long as the retries last, after this server already
-# reports itself healthy.
+# reports itself healthy. Only a peer that runs a replication server goes into a list - a
+# directory server has none to connect to - and cn=admin data does not say which do, so a
+# peer that a list lacks is asked; one that does not answer is left for the repair of a later
+# start.
repair_replication_servers() {
- local i peer waiting hosts host lists kind cn value found
+ local i peer waiting hosts host lists kind cn value found config servers=
[ "$PEERS_ARE_EXPLICIT" = yes ] || return 0
for i in $(seq 1 "$REPLICATION_RETRY_COUNT"); do
waiting=
@@ -725,17 +876,27 @@
lists=$(replication_server_lists)
# a here-string of nothing still reads as one empty line
[ -n "$lists" ] || return 0
+ # every ldapsearch starts a JVM: a peer that every list holds already is not asked. One that
+ # does not answer waits for the repair of a later start, whatever the roles around it: a
+ # replication server left out of a list is added then - and its own repair adds this server
+ # to its lists when it starts, which connects the two already - while a directory server put
+ # into one would stay there for good, a replication server that nothing listens on, which
+ # cleanup_departed keeps as long as the host is listed; and a peer that every list holds
+ # is never asked again, so nothing would take it out.
+ for host in $hosts; do
+ is_self "$host" && continue
+ in_peers "$host" || continue
+ lacked_by_a_list "$host" "$lists" || continue
+ if ! config=$(replication_config "$host"); then
+ echo "join: could not ask $host whether it runs a replication server, its place in the replication server lists is checked on a later start"
+ elif grep -qx server <<<"$config"; then
+ servers="$servers $host"
+ fi
+ done
while IFS=$'\t' read -r kind cn value; do
[ -n "$value" ] && continue
- for host in $hosts; do
- is_self "$host" && continue
- in_peers "$host" || continue
- found=no
- while IFS=$'\t' read -r k c v; do
- [ "$k" = "$kind" ] && [ "$c" = "$cn" ] && [ -n "$v" ] || continue
- names_match "$(lower "${v%:*}")" "$(lower "$host")" && { found=yes; break; }
- done <<<"$lists"
- if [ "$found" = no ]; then
+ for host in $servers; do
+ if ! list_holds "$host" "$kind" "$cn" "$lists"; then
echo "join: adding replication server $host:$REPLICATION_PORT to the $kind list of $cn"
change_replication_servers "$kind" "$cn" --add "$host:$REPLICATION_PORT" || true
fi
@@ -743,6 +904,25 @@
done <<<"$lists"
}
+# whether the list of that kind and cn holds the host, in replication_server_lists output
+list_holds() { # <host> <kind> <cn> <lists>
+ local k c v
+ while IFS=$'\t' read -r k c v; do
+ [ "$k" = "$2" ] && [ "$c" = "$3" ] && [ -n "$v" ] || continue
+ names_match "$(lower "${v%:*}")" "$(lower "$1")" && return 0
+ done <<<"$4"
+ return 1
+}
+
+lacked_by_a_list() { # <host> <lists>
+ local kind cn value
+ while IFS=$'\t' read -r kind cn value; do
+ [ -n "$value" ] && continue
+ list_holds "$1" "$kind" "$cn" "$2" || return 0
+ done <<<"$2"
+ return 1
+}
+
# the backend of BASE_DN refuses the writes of clients because this join stopped them: an
# operator's read-only backend leaves $REJOIN_PENDING empty, and a hold whose dsconfig failed
# named the backend without changing its mode
@@ -777,6 +957,11 @@
joined() {
cleanup_departed
leave_rejoin || exit 1
+ # a replication server published itself rejoining until now, on the ready road too
+ if [ "$ROLE" = replication ] && ! publish_state ready; then
+ echo "join: could not publish this server ready, this container will not report itself healthy until a later start does"
+ exit 1
+ fi
touch "$BOOTSTRAP_COMPLETE"
echo "join: this server is a member of the replication topology, the health check may probe it"
repair_replication_servers
@@ -796,7 +981,9 @@
# volume that holds the data of the topology does so from the start, until a reset of its
# replication took that back - and the writes of its clients, where the reset stopped them
give_up() {
- if [ -f "$BOOTSTRAP_COMPLETE" ]; then
+ if [ -f "$BOOTSTRAP_COMPLETE" ] && [ "$ROLE" = replication ]; then
+ echo "join: could not join the replication topology after $REPLICATION_RETRY_COUNT attempts, this replication server stays apart from it until a later start joins it"
+ elif [ -f "$BOOTSTRAP_COMPLETE" ]; then
echo "join: could not join the replication topology after $REPLICATION_RETRY_COUNT attempts, this server keeps serving its data unreplicated until a later start joins it"
elif writes_held; then
echo "join: could not join the replication topology after $REPLICATION_RETRY_COUNT attempts, this container will not report itself healthy and serves its data read-only until a later start joins it"
@@ -808,7 +995,7 @@
echo "join: waiting for the server on the administration connector"
for i in $(seq 1 150); do
- server_up && break
+ server_up && own_config=$(replication_config localhost) && break
if [ "$i" -eq 150 ]; then
echo "join: the server did not come up, or does not take ROOT_PASSWORD any more, giving up"
exit 1
@@ -816,8 +1003,87 @@
sleep 2
done
-FIRST=no
-is_self "${PEERS[0]}" && FIRST=yes
+# The role this server has, which every step below goes by: the one setup.sh set the volume
+# up with, whatever REPLICATION_ROLE says now - a replication server has no backend for
+# BASE_DN, a server with one holds data, and a later start in another role is told rather than
+# left to believe it took effect. A volume set up by an image before roles has no role
+# recorded: its replication configuration tells it, and REPLICATION_ROLE where that does not -
+# a replication server without a domain is the replication server of the one-shot srs, when
+# REPLICATION_ROLE says so, and a combined server whose enable stopped half-way otherwise. One
+# without a backend for BASE_DN as well holds no data whatever REPLICATION_ROLE says: only
+# this image sets a volume up so, for a replication server. The role found is recorded, so
+# that it stays what it is from then on.
+legacy_role() { # <replication_config output>
+ case $(config_shape "$1") in
+ combined) echo combined ;;
+ directory) echo directory ;;
+ server)
+ if [ "$REPLICATION_ROLE" = replication ] || ! grep -qx backend <<<"$1"; then
+ echo replication
+ else
+ echo combined
+ fi
+ ;;
+ *)
+ if [ -f "$REJOIN_PENDING" ]; then
+ # only the join of an image before roles took a volume down, and it ran combined
+ # servers only
+ echo combined
+ elif grep -qx backend <<<"$1" && [ "$REPLICATION_ROLE" = replication ]; then
+ echo "join: this server holds a backend for $BASE_DN, which a replication server does not, and joins as a combined server; a replication server starts from an empty volume" >&2
+ echo combined
+ else
+ echo "$REPLICATION_ROLE"
+ fi
+ ;;
+ esac
+}
+if [ -s "$REPLICATION_ROLE_FILE" ]; then
+ ROLE=$(cat "$REPLICATION_ROLE_FILE")
+ case $ROLE in
+ combined | directory | replication) ;;
+ *)
+ echo "join: $REPLICATION_ROLE_FILE holds $ROLE, which is not one of combined, directory or replication, this server does not join"
+ exit 1 ;;
+ esac
+else
+ ROLE=$(legacy_role "$own_config")
+ printf '%s\n' "$ROLE" >"$REPLICATION_ROLE_FILE" \
+ || echo "join: could not record the role $ROLE in $REPLICATION_ROLE_FILE, the next start reads it again"
+fi
+if [ "$ROLE" != "$REPLICATION_ROLE" ]; then
+ echo "join: this server is a $ROLE server and keeps that role, REPLICATION_ROLE=$REPLICATION_ROLE does not change it"
+fi
+if [ "$ROLE" != replication ] && ! grep -qx backend <<<"$own_config"; then
+ echo "join: this server holds no backend for $BASE_DN, which a $ROLE server needs: its bootstrap did not create one"
+fi
+
+# Only the first peer of REPLICATION_PEERS that holds data may seed a topology: a replication
+# server has none to seed it with, and one listed ahead of the others is passed over. Asked
+# only where a seed is weighed, and only of the peers listed ahead of this one, each by the
+# role it publishes; one that does not answer, or does not publish one yet, counts as holding
+# data, so that two servers never both take themselves for the first. A role stays with its
+# volume, so each answer is kept: a peer ahead that publishes another role holds data for
+# good, and one that publishes replication is not asked again.
+AHEAD_HOLDS_DATA=no
+AHEAD_REPLICATION=
+may_seed() {
+ local peer config
+ [ "$ROLE" = replication ] && return 1
+ [ "$AHEAD_HOLDS_DATA" = yes ] && return 1
+ for peer in "${PEERS[@]}"; do
+ is_self "$peer" && return 0
+ [[ " $AHEAD_REPLICATION " == *" $peer "* ]] && continue
+ config=$(replication_config "$peer") || return 1
+ grep -q '^published' <<<"$config" || return 1
+ if [ "$(peer_role "$config")" != replication ]; then
+ AHEAD_HOLDS_DATA=yes
+ return 1
+ fi
+ AHEAD_REPLICATION="$AHEAD_REPLICATION $peer"
+ done
+ return 1
+}
# Every round on both roads starts here: a replication configuration that the peers no longer
# register is taken down (see reset_replication). So is one that the peers register but the
@@ -828,7 +1094,7 @@
# an enable registers nobody before it.
reset_if_unregistered() { # <the state to publish meanwhile>
local rc=0
- replicates_base_dn localhost || return 0
+ holds_own_part || return 0
if ! registered_with_peers; then
reset_replication "$1" "the peers no longer register this server"
return
@@ -842,8 +1108,12 @@
if [ ! -f "$INITIALIZE_PENDING" ]; then
# This volume holds the data of the topology, or is the one that seeds it: it publishes
# that - unless a reset took its replication down and it has not rejoined yet, also on an
- # earlier start
- if [ -f "$REJOIN_PENDING" ]; then
+ # earlier start. A replication server holds no data: what a peer may join through is its
+ # membership, and a peer that enabled through it before a reset took it out would be
+ # registered in a registry of the two of them. So it publishes itself rejoining until a
+ # round below finds it a member - not pending, which would count it among the peers that
+ # wait for data and let a fresh seed go ahead next to the topology it serves
+ if [ -f "$REJOIN_PENDING" ] || [ "$ROLE" = replication ]; then
publish_state rejoining
else
publish_state ready
@@ -873,7 +1143,12 @@
rc=$?
if member_of_topology; then
echo "join: joined the replication topology through $peer"
- cross_check "$peer" "; replication will not flow until this server or that one is initialized by hand"
+ # a replication server holds no generation ID of BASE_DN to compare with, neither
+ # this one nor the peer joined through
+ if [ "$ROLE" != replication ]; then
+ replicates_base_dn "$peer" || peer=$(trusted_source) || peer=
+ [ -z "$peer" ] || cross_check "$peer" "; replication will not flow until this server or that one is initialized by hand"
+ fi
joined
exit
fi
@@ -882,7 +1157,7 @@
fi
# a list without a single other server leaves nothing to retry against, and peers that
# all wait for data join through this one rather than the other way round
- if [ "$FIRST" = yes ] && all_others_pending; then
+ if may_seed && all_others_pending; then
seed "no other peer holds the data of the topology" || exit 1
exit 0
fi
@@ -891,7 +1166,7 @@
fi
done
# the same rule as at the end of the pending road below
- if [ "$FIRST" = yes ] && ! any_other_may_hold_data; then
+ if may_seed && ! any_other_may_hold_data; then
seed "no topology found after $REPLICATION_RETRY_COUNT attempts" || exit 1
exit 0
fi
@@ -899,11 +1174,12 @@
fi
# This volume was bootstrapped and never received the data of the topology: it joins and
-# initializes only through a peer that holds that data, and only the first peer may decide
-# that nobody does. Its membership is asked of the peers as on the road above: a volume that
-# enabled once but never completed its initialize may have been removed by the survivors
-# while it was away, and its own cn=admin data still registers it until their delete reaches
-# it - after which every enable registers nobody, as reset_replication describes.
+# initializes only through a peer that holds that data, and only the first peer that holds
+# data (see may_seed) may decide that nobody does. Its membership is asked of the peers as on
+# the road above: a volume that enabled once but never completed its initialize may have been
+# removed by the survivors while it was away, and its own cn=admin data still registers it
+# until their delete reaches it - after which every enable registers nobody, as
+# reset_replication describes.
publish_state pending
member=no
for i in $(seq 1 "$REPLICATION_RETRY_COUNT"); do
@@ -933,7 +1209,14 @@
[ "$rc" -eq 75 ] || echo "join: dsreplication enable with $peer exited with $rc and membership is not there"
done
fi
- if [ "$member" = yes ]; then
+ if [ "$member" = yes ] && [ "$ROLE" = replication ]; then
+ # a replication server holds no data that an initialize would bring: membership is all
+ # it waits for. Published before the marker goes, as after an initialize
+ publish_state ready || exit 1
+ rm -f "$INITIALIZE_PENDING"
+ joined
+ exit
+ elif [ "$member" = yes ]; then
source=$(trusted_source)
if [ -n "$source" ]; then
echo "join: initializing from $source"
@@ -949,14 +1232,14 @@
exit
fi
echo "join: initialize from $source exited with $rc"
- elif [ "$FIRST" = yes ] && all_others_pending; then
+ elif may_seed && all_others_pending; then
seed "every other peer waits for the data of the topology as well" || exit 1
joined
exit 0
else
echo "join: no peer holds the data of the topology yet"
fi
- elif [ "$FIRST" = yes ] && all_others_pending; then
+ elif may_seed && all_others_pending; then
seed "every other peer waits for the data of the topology as well" || exit 1
exit 0
fi
@@ -965,10 +1248,10 @@
fi
done
-# Only the first peer may decide that there is no topology to join and that its own data
-# seeds one, and only while no peer that could hold the topology's data answers; anyone
-# else staying unhealthy is what surfaces a lost topology instead of forking it
-if [ "$FIRST" = yes ] && [ "$member" = no ] && ! any_other_may_hold_data; then
+# Only the first peer that holds data may decide that there is no topology to join and that
+# its own data seeds one, and only while no peer that could hold the topology's data answers;
+# anyone else staying unhealthy is what surfaces a lost topology instead of forking it
+if [ "$member" = no ] && may_seed && ! any_other_may_hold_data; then
seed "no topology found after $REPLICATION_RETRY_COUNT attempts" || exit 1
exit 0
fi
diff --git a/opendj-packages/opendj-docker/bootstrap/setup.sh b/opendj-packages/opendj-docker/bootstrap/setup.sh
index ec8aaca..0341cdb 100755
--- a/opendj-packages/opendj-docker/bootstrap/setup.sh
+++ b/opendj-packages/opendj-docker/bootstrap/setup.sh
@@ -22,6 +22,30 @@
echo "Setting up default OpenDJ instance"
+# A replication server of the background join holds no data (#1178): it gets no backend for
+# BASE_DN, and none of the entries below - entries there would replicate nowhere, and be
+# served to any client that reaches it. run.sh marks a volume $INITIALIZE_PENDING before its
+# bootstrap exactly when the background join runs; without the join the server holds its data
+# as any other - one with neither would be healthy and serve nothing - and REPLICATION_ROLE
+# is not read. The role is checked here as join.sh checks it, before anything is set up: a
+# value the join refuses would otherwise leave a volume set up for another role than the one
+# asked for. It is recorded on the volume, where join.sh reads it on every start: the volume
+# is what makes the role, whatever REPLICATION_ROLE says later.
+HOLDS_DATA=true
+if [ -n "$INITIALIZE_PENDING" ] && [ -f "$INITIALIZE_PENDING" ]; then
+ role=${REPLICATION_ROLE:-combined}
+ case $role in
+ combined | directory) ;;
+ replication)
+ HOLDS_DATA=false
+ echo "REPLICATION_ROLE=replication: no backend is created for $BASE_DN, and no entries are loaded" ;;
+ *)
+ echo "REPLICATION_ROLE=$REPLICATION_ROLE is not one of combined, directory or replication" >&2
+ exit 1 ;;
+ esac
+ printf '%s\n' "$role" >"${REPLICATION_ROLE_FILE:-/opt/opendj/data/.replication-role}" || exit 1
+fi
+
# The tools read only the first line of a password file, and a CR ends it as an LF does, so a
# password with a line break would be cut there - silently, as the root password set up would
# not be the one the container was given
@@ -81,13 +105,15 @@
BACKEND_TYPE=${BACKEND_TYPE:-je}
BACKEND_DB_DIRECTORY=${BACKEND_DB_DIRECTORY:-db}
-echo "creating backend: $BACKEND_TYPE db-directory: ${BACKEND_DB_DIRECTORY}"
+if [ "$HOLDS_DATA" = true ]; then
+ echo "creating backend: $BACKEND_TYPE db-directory: ${BACKEND_DB_DIRECTORY}"
-/opt/opendj/bin/dsconfig create-backend -h localhost -p $ADMIN_PORT --bindDN "$ROOT_USER_DN" --bindPasswordFile "$PASSWORD_FILE" \
- --backend-name=userRoot --type $BACKEND_TYPE --set "base-dn:$BASE_DN" --set "db-directory:$BACKEND_DB_DIRECTORY" \
- --set enabled:true --no-prompt --trustAll || exit 1
+ /opt/opendj/bin/dsconfig create-backend -h localhost -p $ADMIN_PORT --bindDN "$ROOT_USER_DN" --bindPasswordFile "$PASSWORD_FILE" \
+ --backend-name=userRoot --type $BACKEND_TYPE --set "base-dn:$BASE_DN" --set "db-directory:$BACKEND_DB_DIRECTORY" \
+ --set enabled:true --no-prompt --trustAll || exit 1
+fi
-if [ "$ADD_BASE_ENTRY" = "--addBaseEntry" ]; then
+if [ "$HOLDS_DATA" = true ] && [ "$ADD_BASE_ENTRY" = "--addBaseEntry" ]; then
BASE_TEMPLATE=$(mktemp)
if [ ! -z ${SAMPLE_DATA} ]; then
echo "generating sample data..."
@@ -116,7 +142,7 @@
done
fi
-if [ -d /opt/opendj/bootstrap/data/ ]; then
+if [ "$HOLDS_DATA" = true ] && [ -d /opt/opendj/bootstrap/data/ ]; then
# allow pre encoded passwords; the port is named, as the tool would otherwise go to 4444
# whatever ADMIN_PORT the server listens on, and the entries carrying them would be refused
/opt/opendj/bin/dsconfig \
diff --git a/opendj-packages/opendj-docker/run.sh b/opendj-packages/opendj-docker/run.sh
index b66314a..35ee12d 100755
--- a/opendj-packages/opendj-docker/run.sh
+++ b/opendj-packages/opendj-docker/run.sh
@@ -54,6 +54,47 @@
# the enable registered it again: it holds the data of the topology, but takes writes that
# replicate nowhere meanwhile
export REJOIN_PENDING=${REJOIN_PENDING:-/opt/opendj/data/.replication-rejoin-pending}
+# A volume this script bootstraps carries this one from before its bootstrap until the whole
+# of it - the bootstrap script, and the replicate.sh of a one-shot replication type - has
+# succeeded (#1182). A bootstrap that failed or was killed after setup wrote ./data/config
+# leaves the volume looking installed, and the next start would otherwise report a server
+# healthy that has no backend or no base entry. Volumes bootstrapped by images before this one
+# carry no mark, and start as they did.
+export BOOTSTRAP_PENDING=${BOOTSTRAP_PENDING:-/opt/opendj/data/.bootstrap-pending}
+# And a volume this script upgrades to another version carries this one from before the
+# upgrade until the upgrade succeeded, post-upgrade tasks included (#1185). The upgrade records
+# the new version in config/buildinfo before it runs those tasks - the index rebuilds, and the
+# verification of the DN equality indexes - so after one of them failed, or the container was
+# stopped in the middle of them, the next run of upgrade finds nothing to do and succeeds.
+UPGRADE_PENDING=${UPGRADE_PENDING:-/opt/opendj/data/.upgrade-pending}
+
+# major.minor.point of a buildinfo file: the part of the version the upgrade compares
+build_version() { # <buildinfo>
+ head -n 1 "$1" 2>/dev/null | cut -d. -f1-3
+}
+
+# Upgrades the instance to the version of this image, and fails where the upgrade fails or
+# where an earlier start left it incomplete: the mark of a volume whose version is already
+# the one of this image can only come from an upgrade that recorded it and did not complete.
+# One that did not get that far is run again, as before. An instance without a version to read
+# is not marked: the upgrade refuses it, and setup never got far enough to call it installed.
+# --force performs the tasks that -n alone answers with their default no, such as rebuilding
+# indexes: nobody is there to run them by hand afterwards, as the native packages do too
+upgrade_instance() {
+ local from to
+ from=$(build_version ./data/config/buildinfo)
+ to=$(build_version ./template/config/buildinfo)
+ if [ -n "$from" ] && [ "$from" != "$to" ] && ! printf '%s\n' "$from to $to" >"$UPGRADE_PENDING"; then
+ echo "Could not write $UPGRADE_PENDING, the upgrade is not started"
+ return 1
+ fi
+ sh ./upgrade -n --force || return 1
+ if [ "$from" = "$to" ] && [ -f "$UPGRADE_PENDING" ]; then
+ echo "The upgrade of this volume from $(cat "$UPGRADE_PENDING") did not complete on an earlier start: its post-upgrade tasks failed or were cut off, which /opt/opendj/data/logs/upgrade.log tells. Rebuild the indexes it names with rebuild-index and remove $UPGRADE_PENDING, or restore the backup taken before the upgrade"
+ return 1
+ fi
+ rm -f "$UPGRADE_PENDING"
+}
# The background join (bootstrap/join.sh) serves OPENDJ_REPLICATION_TYPE=simple on every
# start; srs, sdsr and rg keep the one-shot replicate.sh of the first bootstrap
@@ -136,10 +177,17 @@
# Instance dir does exist? We start opendj without detach
if [ -d ./data/config ]; then
# nothing is bootstrapped here, the instance is already there - but a half-migrated one
- # is not ready to serve either, so the marker follows the upgrade.
- # --force performs the tasks that -n alone answers with their default no, such as rebuilding indexes:
- # nobody is there to run them by hand afterwards, as the native packages do too
- if sh ./upgrade -n --force; then
+ # is not ready to serve either, so the marker follows the upgrade
+ if [ -f "$BOOTSTRAP_PENDING" ]; then
+ # neither the marker nor the join: the join would enable a volume that lacks what the
+ # bootstrap was to set up - a volume of the background join too, whose initialize would
+ # bring the data of the topology but nothing else the bootstrap was to configure - and
+ # the server is only there to be looked at. The upgrade still runs, whatever it says:
+ # start-ds refuses an instance of another version, and the setup could then not be
+ # finished by hand under this image
+ upgrade_instance || echo "The upgrade did not complete"
+ echo "The bootstrap of this volume did not complete, this container will not report itself healthy: remove the volume and start over, or finish the setup by hand and remove $BOOTSTRAP_PENDING"
+ elif upgrade_instance; then
# A server whose volume holds the data of the topology is ready as soon as it serves:
# gating it on its peers would deadlock a whole-cluster restart under OrderedReady,
# where -0 would wait for peers the StatefulSet starts only once -0 is ready - and so
@@ -169,7 +217,7 @@
/opt/opendj/bootstrap/join.sh &
fi
else
- echo "Upgrade failed, this container will not report itself healthy"
+ echo "The upgrade did not complete, this container will not report itself healthy"
fi
start_server
fi
@@ -189,6 +237,7 @@
# volume that counts as holding the data of the topology; a bootstrapped volume has entries
# whether or not it was asked for any, so it is initialized from the topology (see
# bootstrap/join.sh)
+touch "$BOOTSTRAP_PENDING"
if join_requested; then
touch "$INITIALIZE_PENDING"
fi
@@ -226,6 +275,10 @@
# replication - is in place from here on, so the health check may start probing the server.
# With the background join, replication is the one thing still outstanding: the join writes
# the health marker once this server is a member of its topology, and not before.
+if [ "$BOOTSTRAPPED" = true ] && ! rm -f "$BOOTSTRAP_PENDING"; then
+ BOOTSTRAPPED=false
+ echo "Could not remove $BOOTSTRAP_PENDING, this container will not report itself healthy"
+fi
if [ "$BOOTSTRAPPED" = true ]; then
if join_requested; then
/opt/opendj/bootstrap/join.sh &
--
Gitblit v1.10.0