#!/usr/bin/env bash
|
# The contents of this file are subject to the terms of the Common Development and
|
# Distribution License (the License). You may not use this file except in compliance with the
|
# License.
|
#
|
# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
|
# specific language governing permission and limitations under the License.
|
#
|
# When distributing Covered Software, include this CDDL Header Notice in each file and include
|
# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
|
# Header, with the fields enclosed by brackets [] replaced by your own identifying
|
# information: "Portions copyright [year] [name of copyright owner]".
|
#
|
# Copyright 2026 3A Systems, LLC.
|
|
# Joins this server to the replication topology (#1086). run.sh starts it in the background
|
# next to the server on every start, not as a step of the first bootstrap: a join that could
|
# not complete is tried again on the next start, and membership that changed while no
|
# container ran is repaired. It covers OPENDJ_REPLICATION_TYPE=simple; the one-shot srs, sdsr
|
# and rg paths stay in replicate.sh.
|
#
|
# The contract with the operator: REPLICATION_PEERS lists every server of the topology by DNS
|
# name, and all of them share BASE_DN, ROOT_USER_DN, ROOT_PASSWORD, ADMIN_PORT and
|
# REPLICATION_PORT. A StatefulSet chart derives the list from its ordinals
|
# (<sts>-0.<svc> ... <sts>-N-1.<svc>), plain docker run passes the container names of
|
# containers whose --hostname is their --name. MASTER_SERVER keeps working as a one-element
|
# list. This server recognises itself in the list by a name that equals hostname -f, or is
|
# hostname -f cut at a dot (a StatefulSet pod whose FQDN is <sts>-N.<svc>.<ns>.svc.<domain>
|
# is listed as <sts>-N.<svc>), or the other way round; and, as the grep of /etc/hosts it
|
# replaces did, by one of its own addresses or a name /etc/hosts gives one of them. Names
|
# are compared whole - that grep took opendj-1 for opendj-10, a replica for its master when
|
# an --add-host named the master, and a master that lost its volume for a replica of itself -
|
# and cut only at a dot, so opendj-0.opendj.east is not opendj-0.opendj.west.
|
#
|
# Membership is decided from what is there, never from an exit code: exit 5 of dsreplication
|
# enable (REPLICATION_CANNOT_BE_ENABLED_ON_BASEDN) means "no suffix was left to enable",
|
# which covers a base DN that is already replicated and a base DN that one of the two
|
# servers does not hold - the race with a peer whose bootstrap has not created the backend
|
# yet. The step is a member once the replication domain for BASE_DN exists in its cn=config
|
# and the server is registered in cn=admin data; anything else is tried again, whatever the
|
# exit code, because a repeated enable is safe exactly when it is decided this way.
|
#
|
# Whose data the topology carries follows what each volume went through, not whether BASE_DN
|
# has entries: every fresh volume has entries (setup.sh imports the base entry or
|
# SAMPLE_DATA), and two freshly bootstrapped volumes even share a generation ID, so
|
# replication silently carries nothing that reached one of them by import-ldif. run.sh marks
|
# a volume it bootstrapped with $INITIALIZE_PENDING, and the join publishes that to its peers
|
# in $STATE_DN, a local entry of cn=config: "pending" while the marker is there, "ready" once
|
# the volume holds the data of the topology - it was never bootstrapped by run.sh, it was
|
# initialized from a ready peer, or it seeded the topology - and "rejoining" while a volume
|
# that holds it has its replication taken down to enable it anew, or while a replication
|
# server is not found a member yet on this start. A server joins only through a ready peer,
|
# and a pending one initializes only from one, so two fresh servers that start
|
# together never take each other's bootstrap data for the topology's, and two servers whose
|
# replication is down never form a topology of their own; where the list is only
|
# MASTER_SERVER, a peer that publishes no state (an image before this one) counts as ready, as
|
# the old replicate.sh trusted its master. The seed is only the first entry of
|
# REPLICATION_PEERS: at once when every other peer answers and is pending (all of them start
|
# from scratch, together or not), or once the retries are exhausted while no other peer
|
# answers that may hold the data - a ready one, one of an image before this one that
|
# publishes no state but replicates BASE_DN, or one that refuses the bind with ROOT_PASSWORD
|
# (only a server past its bootstrap can have another root password). The residual risk is
|
# documented in the README: every other server down *and* the first peer's volume lost means
|
# the first peer seeds empty.
|
#
|
# On a volume that waits for the data of the topology, the health marker is written only
|
# once the join succeeded or the seed rule fired, so a container that never joined stays
|
# unhealthy - across restarts too, which the bootstrap-only replicate.sh could not say (a
|
# restart wrote the marker right after upgrade and a failed join turned into a healthy,
|
# unreplicated server). A volume that holds the data is healthy as soon as it serves (see
|
# run.sh) - unless the join takes its replication down to enable it anew: from then until the
|
# enable registered it again it is not, across restarts too ($REJOIN_PENDING), and the
|
# backend of BASE_DN refuses the writes of clients, which no longer replicate meanwhile.
|
#
|
# REPLICATION_ROLE (#1178) is the part this server takes: combined, a directory server with a
|
# replication server of its own (the default); directory, one that connects to the
|
# replication servers of other peers; replication, a replication server that holds no data.
|
# The role of each side of an enable is passed to dsreplication: unless it is told otherwise,
|
# the tool gives both servers a replication server and a domain for BASE_DN, so an enable
|
# through a directory server would hand it a replication server. A peer's role is read from
|
# the peer - what its configuration holds, or else what it publishes next to its state. A
|
# replication server goes the way of a volume that waits for the data of the topology,
|
# publishing itself pending until it is a member, but nothing initializes it; it never seeds,
|
# and one listed first is passed over. A directory server joins only a topology that has a
|
# replication server: dsreplication enable refuses one without, before it changes anything.
|
# The role is set when setup.sh sets the volume up - a replication server gets no backend for
|
# BASE_DN - and recorded on it: a later start in another role only says so, and the join goes
|
# on in the role the server has (ROLE below).
|
|
cd /opt/opendj || exit 1
|
export PATH=/opt/opendj/bin:$PATH
|
|
BASE_DN=${BASE_DN:-"dc=example,dc=com"}
|
ROOT_USER_DN=${ROOT_USER_DN:-"cn=Directory Manager"}
|
ROOT_PASSWORD=${ROOT_PASSWORD:-password}
|
ADMIN_PORT=${ADMIN_PORT:-4444}
|
REPLICATION_PORT=${REPLICATION_PORT:-8989}
|
MYHOSTNAME=${MYHOSTNAME:-$(hostname -f)}
|
BOOTSTRAP_COMPLETE=${BOOTSTRAP_COMPLETE:-/opt/opendj/.bootstrap-complete}
|
INITIALIZE_PENDING=${INITIALIZE_PENDING:-/opt/opendj/data/.replication-initialize-pending}
|
REJOIN_PENDING=${REJOIN_PENDING:-/opt/opendj/data/.replication-rejoin-pending}
|
# the role the volume was set up with (setup.sh), or the one a first start of this image read
|
# from a volume set up before roles
|
REPLICATION_ROLE_FILE=${REPLICATION_ROLE_FILE:-/opt/opendj/data/.replication-role}
|
# not replicated: cn=config is this server's alone, and it lives on the volume next to the
|
# marker, so the two cannot tell different stories
|
STATE_DN="cn=Docker Join,cn=config"
|
# BASE_DN as a value in a search filter (RFC 4515): a parenthesis, an asterisk or a backslash
|
# in it would break the filter, or make it match more than BASE_DN
|
BASE_DN_VALUE=$(printf '%s' "$BASE_DN" | sed -e 's/\\/\\5c/g' -e 's/\*/\\2a/g' -e 's/(/\\28/g' -e 's/)/\\29/g')
|
|
# Sets the variable to its value read as a decimal whole number, or to the default when it
|
# is not one or is below the least value. $(( )) reads a leading zero as octal and fails on
|
# 08 - abandoning the whole command it runs in, however far up that is - while test reads 08
|
# as 8, so a check by test alone lets the value through to the arithmetic.
|
whole_number() { # <variable> <default> <least>
|
local value=${!1}
|
case $value in
|
'' | *[!0-9]*) value= ;;
|
*) value=$((10#$value)) ;;
|
esac
|
if [ -z "$value" ] || [ "$value" -lt "$3" ]; then
|
echo "join: $1 is not a whole number of at least $3, using $2"
|
value=$2
|
fi
|
printf -v "$1" '%s' "$value"
|
}
|
|
REPLICATION_RETRY_COUNT=${REPLICATION_RETRY_COUNT:-30}
|
whole_number REPLICATION_RETRY_COUNT 30 1
|
REPLICATION_RETRY_INTERVAL=${REPLICATION_RETRY_INTERVAL:-10}
|
whole_number REPLICATION_RETRY_INTERVAL 10 0
|
# dsreplication enable can hang rather than fail (a peer that stops mid-operation), so no
|
# enable runs without a bound
|
REPLICATION_ATTEMPT_TIMEOUT=${REPLICATION_ATTEMPT_TIMEOUT:-120}
|
whole_number REPLICATION_ATTEMPT_TIMEOUT 120 1
|
# a total update is a full import of BASE_DN, which takes as long as the data needs; a killed
|
# dsreplication initialize leaves its task running on the server, and the next attempt asks
|
# for another full import, so by default it runs without a bound
|
REPLICATION_INITIALIZE_TIMEOUT=${REPLICATION_INITIALIZE_TIMEOUT:-0}
|
whole_number REPLICATION_INITIALIZE_TIMEOUT 0 0
|
|
# a role it does not know is not taken for the default: a server started as a replication
|
# server by a misspelt value would otherwise join holding data. A volume that records its
|
# role (see ROLE below) joins in that role whatever the value, as it does with another role
|
REPLICATION_ROLE=${REPLICATION_ROLE:-combined}
|
case $REPLICATION_ROLE in
|
combined | directory | replication) ;;
|
*)
|
if [ -s "$REPLICATION_ROLE_FILE" ]; then
|
echo "join: REPLICATION_ROLE=$REPLICATION_ROLE is not one of combined, directory or replication, the role recorded on the volume stands"
|
else
|
echo "join: REPLICATION_ROLE=$REPLICATION_ROLE is not one of combined, directory or replication, this server does not join"
|
exit 1
|
fi
|
;;
|
esac
|
|
# Removing servers that left the topology needs the operator's word on who belongs to it:
|
# with only MASTER_SERVER set, a third server joined by hand would be "not in the list" and
|
# thrown out. So the cleanup, and the repair of the replication server lists, run only when
|
# REPLICATION_PEERS itself is set.
|
PEERS_ARE_EXPLICIT=${REPLICATION_PEERS:+yes}
|
REPLICATION_PEERS=${REPLICATION_PEERS:-$MASTER_SERVER}
|
|
if [ -z "$REPLICATION_PEERS" ]; then
|
echo "join: neither REPLICATION_PEERS nor MASTER_SERVER is set, nothing to join"
|
exit 1
|
fi
|
|
IFS=',' read -r -a RAW_PEERS <<<"$REPLICATION_PEERS"
|
PEERS=()
|
for peer in "${RAW_PEERS[@]}"; do
|
peer=$(echo "$peer" | tr -d '[:space:]')
|
[ -n "$peer" ] && PEERS+=("$peer")
|
done
|
if [ "${#PEERS[@]}" -eq 0 ]; then
|
echo "join: REPLICATION_PEERS holds no peer"
|
exit 1
|
fi
|
|
# The tools read the root password from a file (#1084); the file lives on the tmpfs of
|
# /dev/shm where there is one, and run.sh removes what a killed join left behind, by the
|
# name, on every start - in /dev/shm those of its ADMIN_PORT, in /tmp all of them
|
PASSWORD_FILE=$(mktemp -p /dev/shm "opendj-join.$ADMIN_PORT.XXXXXX" 2>/dev/null \
|
|| mktemp "/tmp/opendj-join.$ADMIN_PORT.XXXXXX") || exit 1
|
trap 'rm -f "$PASSWORD_FILE"' EXIT
|
printf '%s\n' "$ROOT_PASSWORD" >"$PASSWORD_FILE" || exit 1
|
|
lower() {
|
printf '%s' "$1" | tr '[:upper:]' '[:lower:]'
|
}
|
|
# an IPv4 address is digits and dots, an IPv6 one has a colon
|
is_address() {
|
case $1 in
|
*:*) return 0 ;;
|
*[!0-9.]* | '') return 1 ;;
|
*) return 0 ;;
|
esac
|
}
|
|
# Two names (lower case) are one server when they are equal, or when one of them is the
|
# other cut at a dot; addresses only when they are equal
|
names_match() {
|
[ "$1" = "$2" ] && return 0
|
if is_address "$1" || is_address "$2"; then
|
return 1
|
fi
|
case $1 in "$2".*) return 0 ;; esac
|
case $2 in "$1".*) return 0 ;; esac
|
return 1
|
}
|
|
SELF_NAME=$(lower "$MYHOSTNAME")
|
# the addresses of this container (loopback aside) and the names /etc/hosts gives them - an
|
# --add-host or a hostAliases entry for itself - as the base replicate.sh recognised them
|
SELF_ADDRESSES=()
|
SELF_ALIASES=()
|
own_names=" $(lower "$(hostname)") $(lower "$(hostname -f 2>/dev/null)") $SELF_NAME "
|
for address in $(hostname -i 2>/dev/null) \
|
$(awk -v names="$own_names" '!/^[[:space:]]*#/ { for (i = 2; i <= NF; i++) if (index(names, " " tolower($i) " ")) { print $1; break } }' /etc/hosts 2>/dev/null); do
|
case $address in 127.* | ::1 | 0.0.0.0) continue ;; esac
|
SELF_ADDRESSES+=("$(lower "$address")")
|
done
|
if [ "${#SELF_ADDRESSES[@]}" -gt 0 ]; then
|
for name in $(awk -v addresses=" ${SELF_ADDRESSES[*]} " '!/^[[:space:]]*#/ && index(addresses, " " tolower($1) " ") { for (i = 2; i <= NF; i++) print tolower($i) }' /etc/hosts 2>/dev/null); do
|
SELF_ALIASES+=("$name")
|
done
|
fi
|
|
is_self() {
|
local peer candidate
|
peer=$(lower "$1")
|
names_match "$peer" "$SELF_NAME" && return 0
|
for candidate in "${SELF_ADDRESSES[@]}" "${SELF_ALIASES[@]}"; do
|
[ "$peer" = "$candidate" ] && return 0
|
done
|
return 1
|
}
|
|
in_peers() {
|
local host peer
|
host=$(lower "$1")
|
for peer in "${PEERS[@]}"; do
|
names_match "$(lower "$peer")" "$host" && return 0
|
done
|
return 1
|
}
|
|
# every LDAP operation goes through the administration connector, which always serves TLS;
|
# the root user may change its password after the bootstrap, so unlike the health check
|
# these tools run only while the join still has the bootstrap's ROOT_PASSWORD to work with
|
search() {
|
local host=$1
|
shift
|
ldapsearch --noPropertiesFile --hostname "$host" --port "$ADMIN_PORT" --useSsl --trustAll \
|
--bindDN "$ROOT_USER_DN" --bindPasswordFile "$PASSWORD_FILE" "$@" 2>/dev/null
|
}
|
|
ldapmodify_on() { # <host> [<option>...]
|
local host=$1
|
shift
|
ldapmodify --noPropertiesFile --hostname "$host" --port "$ADMIN_PORT" --useSsl --trustAll \
|
--bindDN "$ROOT_USER_DN" --bindPasswordFile "$PASSWORD_FILE" "$@" 2>/dev/null
|
}
|
|
ldapmodify_local() {
|
ldapmodify_on localhost "$@"
|
}
|
|
dsconfig_local() {
|
dsconfig "$@" --hostname localhost --port "$ADMIN_PORT" --bindDN "$ROOT_USER_DN" \
|
--bindPasswordFile "$PASSWORD_FILE" --trustAll --no-prompt
|
}
|
|
# runs a command within that many seconds, or without a bound for 0; timeout executes the
|
# command, so it has to be a program, not a function of this script
|
bounded() {
|
local limit=$1
|
shift
|
if [ "$limit" -gt 0 ] 2>/dev/null; then
|
timeout "$limit" "$@"
|
else
|
"$@"
|
fi
|
}
|
|
server_up() {
|
search localhost --baseDN "" --searchScope base "(objectClass=*)" 1.1 >/dev/null
|
}
|
|
# the state goes with the role of this server, after it: a peer reads the first word as the
|
# state, which a join of an image before roles reads alone
|
publish_state() {
|
printf 'dn: %s\nchangetype: modify\nreplace: description\ndescription: %s %s\n' "$STATE_DN" "$1" "$ROLE" \
|
| ldapmodify_local >/dev/null && return 0
|
printf 'dn: %s\nobjectClass: top\nobjectClass: ds-cfg-branch\nobjectClass: extensibleObject\ncn: Docker Join\ndescription: %s %s\n' "$STATE_DN" "$1" "$ROLE" \
|
| ldapmodify_local --defaultAdd >/dev/null && return 0
|
echo "join: could not publish the state $1 of this server to its peers"
|
return 1
|
}
|
|
# ready, pending or rejoining as the peer publishes it, absent when it answers without
|
# publishing any, refused when it answers but no longer takes ROOT_PASSWORD, down when it does
|
# not answer
|
peer_state() {
|
local out state
|
out=$(search "$1" --baseDN "$STATE_DN" --searchScope base "(objectClass=*)" description)
|
case $? in
|
0)
|
state=$(printf '%s\n' "$out" | awk 'tolower($1) == "description:" { print $2; exit }')
|
case $state in
|
ready | pending | rejoining) echo "$state" ;;
|
*) echo absent ;;
|
esac
|
;;
|
32) echo absent ;;
|
# 49: invalidCredentials - the root password of a server is changed in its cn=config,
|
# which is not replicated, and only a server past its bootstrap has another one
|
49) echo refused ;;
|
*) echo down ;;
|
esac
|
}
|
|
trusted() {
|
case $1 in
|
ready) return 0 ;;
|
absent) [ "$PEERS_ARE_EXPLICIT" != yes ] ;;
|
*) return 1 ;;
|
esac
|
}
|
|
# fails when the search does, rather than answering with an empty list
|
registered_hosts() { # [<host>, localhost by default]
|
local out
|
out=$(search "${1:-localhost}" --baseDN "cn=Servers,cn=admin data" --searchScope one "(objectClass=*)" hostname) \
|
|| return 1
|
printf '%s\n' "$out" | awk 'tolower($1) == "hostname:" { print $2 }'
|
}
|
|
# a member holds the replication domain for BASE_DN in its configuration - a replication
|
# server its replication server instead (see holds_own_part) - and is registered
|
# in cn=admin data - under the name the enable that registered it connected with, which for
|
# a server that never ran an enable of its own is the name a peer listed it by; both are made
|
# by one dsreplication enable, so requiring both keeps a half-done enable from being declared
|
# a success, and the next round takes it down (see reset_if_unregistered)
|
replicates_base_dn() { # <host>
|
grep -qx domain <<<"$(replication_config "$1")"
|
}
|
|
# What of the replication the server holds, one line each: "server" for a replication server,
|
# "domain" for the replication domain of BASE_DN, "backend" for a backend of BASE_DN, and
|
# "published <role>" for the state it publishes - the role is left out where the state comes
|
# without one, from an image before roles - all from one search, as every ldapsearch starts a
|
# JVM. Nothing for a server that has none of
|
# them, and a failure when it cannot be asked.
|
replication_config() { # <host>
|
local out
|
out=$(search "$1" --baseDN "cn=config" --searchScope sub \
|
"(|(objectClass=ds-cfg-replication-server)(&(objectClass=ds-cfg-replication-domain)(ds-cfg-base-dn=$BASE_DN_VALUE))(&(objectClass=ds-cfg-backend)(ds-cfg-base-dn=$BASE_DN_VALUE))(cn=Docker Join))" \
|
objectClass description) \
|
|| return 1
|
printf '%s\n' "$out" | awk -v state="$(lower "$STATE_DN")" '
|
/^dn: / { is_state = (tolower(substr($0, 5)) == state) }
|
tolower($0) == "objectclass: ds-cfg-replication-server" { print "server" }
|
tolower($0) == "objectclass: ds-cfg-replication-domain" { print "domain" }
|
tolower($0) == "objectclass: ds-cfg-backend" { print "backend" }
|
is_state && tolower($1) == "description:" { print "published", $3 }'
|
}
|
|
# What a replication configuration holds: "combined" for a replication server and the domain
|
# of BASE_DN, "directory" for the domain alone, "server" for a replication server alone,
|
# nothing for neither. Only the first two tell a role by themselves.
|
config_shape() { # <replication_config output>
|
local server=no domain=no
|
grep -qx server <<<"$1" && server=yes
|
grep -qx domain <<<"$1" && domain=yes
|
case $server$domain in
|
yesyes) echo combined ;;
|
noyes) echo directory ;;
|
yesno) echo server ;;
|
esac
|
}
|
|
# The role of a peer: the one its configuration tells, or else the one it publishes next to
|
# its state, which tells a replication server from a combined server whose enable stopped
|
# half-way, before it configured the domain - and combined for a peer that publishes none, an
|
# image before roles or a master that runs no join, as every server was before roles.
|
peer_role() { # <replication_config output>
|
case $(config_shape "$1") in
|
combined) echo combined ;;
|
directory) echo directory ;;
|
*) published_role "$1" ;;
|
esac
|
}
|
|
# the role a server publishes next to its state; combined for one that publishes none, as a
|
# server of an image before roles is one - or a server that runs no join, whose state entry a
|
# search does not find
|
published_role() { # <replication_config output>
|
local role
|
role=$(awk '$1 == "published" { print $2; exit }' <<<"$1")
|
case $role in
|
directory | replication) echo "$role" ;;
|
*) echo combined ;;
|
esac
|
}
|
|
# this server holds what its role gives it: the replication domain of BASE_DN for a combined
|
# or a directory server, a replication server for a replication server
|
holds_own_part() {
|
local config
|
config=$(replication_config localhost) || return 1
|
if [ "$ROLE" = replication ]; then
|
grep -qx server <<<"$config"
|
else
|
grep -qx domain <<<"$config"
|
fi
|
}
|
|
# the dsreplication enable arguments that give the server of that side its role, one a line
|
role_arguments() { # <1|2> <role>
|
case $2 in
|
directory) printf '%s\n' "--noReplicationServer$1" ;;
|
replication) printf '%s\n' "--onlyReplicationServer$1" "--replicationPort$1" "$REPLICATION_PORT" ;;
|
*) printf '%s\n' "--replicationPort$1" "$REPLICATION_PORT" ;;
|
esac
|
}
|
|
is_member() {
|
local host
|
holds_own_part || return 1
|
for host in $(registered_hosts); do
|
is_self "$host" && return 0
|
done
|
return 1
|
}
|
|
# cn=admin data is replicated, but a server that was away while the survivors removed it
|
# from the topology still holds its own entry until that delete reaches it, and it may not
|
# have yet when the join looks: the other peers that answer and replicate BASE_DN have the
|
# say - a replication server as well, which replicates cn=admin data without BASE_DN. Being
|
# unregistered takes this server's replication down (reset_replication), so it is
|
# concluded only when at least one of them listed its registrations and none of them lists
|
# this server: a search that failed, or a peer whose copy of cn=admin data still lags behind
|
# the others, decides nothing while another peer lists it. With no such peer the local view
|
# stands - nothing else can be asked.
|
registered_with_peers() {
|
local peer host hosts config asked=no
|
for peer in "${PEERS[@]}"; do
|
is_self "$peer" && continue
|
# a replication server alone is a member only as a replication server; on a combined
|
# server it is an enable that stopped before the domain and the initialize of cn=admin
|
# data, whose registrations are its own
|
config=$(replication_config "$peer") || continue
|
case $(config_shape "$config") in
|
combined | directory) ;;
|
server) [ "$(peer_role "$config")" = replication ] || continue ;;
|
*) continue ;;
|
esac
|
hosts=$(registered_hosts "$peer") || continue
|
asked=yes
|
for host in $hosts; do
|
is_self "$host" && return 0
|
done
|
done
|
[ "$asked" = no ]
|
}
|
|
member_of_topology() {
|
is_member && registered_with_peers
|
}
|
|
# whether the cn=admin data of this server registers it: 0 when it does, 1 when a search
|
# answers that it does not - cn=Servers missing altogether included, which an initialize of
|
# cn=admin data that failed half-way leaves - and 2 when the search failed otherwise, which
|
# decides nothing
|
registered_here() {
|
local out host
|
out=$(search localhost --baseDN "cn=Servers,cn=admin data" --searchScope one "(objectClass=*)" hostname)
|
case $? in
|
0) ;;
|
32) return 1 ;;
|
*) return 2 ;;
|
esac
|
for host in $(printf '%s\n' "$out" | awk 'tolower($1) == "hostname:" { print $2 }'); do
|
is_self "$host" && return 0
|
done
|
return 1
|
}
|
|
# Two dsreplication enable runs through one peer at the same time break each other: both
|
# create the replication server that a seed does not have yet, the loser fails half-way
|
# (ManagedObjectAlreadyExistsException, exit 17), and every enable after that exits 5,
|
# "already replicated", without ever completing its membership. So an enable holds a lock on
|
# the peer it runs through - an entry of the peer's cn=config, which only one add creates -
|
# and one on this server: an enable configures both of its servers, and a server that enables
|
# through a peer may at the same time be the peer another server enables through. A server
|
# that takes its own replication down holds its own lock for as long.
|
# A lock whose holder was killed before it removed it is broken once it is older than an
|
# enable may take, by a delete that asserts the value it read: two joins that break it at
|
# once cannot remove the lock one of them took right after. One that holds this server's own
|
# name is broken at once: it was left by a join of an earlier start, and a start runs one.
|
JOIN_LOCK_DN="cn=Docker Join Lock,cn=config"
|
JOIN_LOCK_TTL=$((REPLICATION_ATTEMPT_TIMEOUT + 60))
|
JOIN_LOCK_VALUE=
|
|
add_join_lock() { # <host> <value>
|
printf 'dn: %s\nobjectClass: top\nobjectClass: ds-cfg-branch\nobjectClass: extensibleObject\ncn: Docker Join Lock\ndescription: %s\n' "$JOIN_LOCK_DN" "$2" \
|
| ldapmodify_on "$1" --defaultAdd >/dev/null
|
}
|
|
# Succeeds when this server may go on: it took the lock and left its value in
|
# JOIN_LOCK_VALUE, or the server could not be asked for one - a peer that does not answer
|
# fails the enable anyway, and one that refuses the entry for another reason than holding it
|
# must not keep the join out for good. A lock taken this way is released with
|
# release_join_lock <host> "$JOIN_LOCK_VALUE".
|
take_join_lock() { # <host>
|
local value rc held holder since age where
|
JOIN_LOCK_VALUE=
|
value="$SELF_NAME $(date +%s)"
|
add_join_lock "$1" "$value"
|
rc=$?
|
if [ "$rc" -eq 0 ]; then
|
JOIN_LOCK_VALUE=$value
|
return 0
|
fi
|
# 68: entryAlreadyExists
|
[ "$rc" -eq 68 ] || return 0
|
held=$(search "$1" --baseDN "$JOIN_LOCK_DN" --searchScope base "(objectClass=*)" description \
|
| awk 'tolower($1) == "description:" { print $2, $3; exit }')
|
holder=${held% *}
|
since=${held##* }
|
age=$(($(date +%s) - ${since:-0}))
|
if [ -n "$held" ] && { [ "$holder" = "$SELF_NAME" ] || [ "$age" -gt "$JOIN_LOCK_TTL" ]; }; then
|
echo "join: breaking the lock $holder took on $1 $age s ago"
|
printf 'dn: %s\nchangetype: delete\n' "$JOIN_LOCK_DN" \
|
| ldapmodify_on "$1" --assertionFilter "(description=$held)" >/dev/null
|
if add_join_lock "$1" "$value"; then
|
JOIN_LOCK_VALUE=$value
|
return 0
|
fi
|
fi
|
if [ "$1" = localhost ]; then where="this server"; else where=$1; fi
|
# a lock released between the add and the search has no holder left to name
|
echo "join: ${holder:-another server} is enabling replication through $where, waiting for it"
|
return 1
|
}
|
|
release_join_lock() { # <host> <value>
|
[ -n "$2" ] || return 0
|
printf 'dn: %s\nchangetype: delete\n' "$JOIN_LOCK_DN" \
|
| ldapmodify_on "$1" --assertionFilter "(description=$2)" >/dev/null \
|
|| echo "join: could not remove the lock on $1, it expires in $JOIN_LOCK_TTL s"
|
}
|
|
# exits 75 without running the enable while another server enables through the peer or
|
# through this one. Neither lock is waited for, so two servers that each hold their own and
|
# ask for the other's cannot deadlock; they both give up and try again after pause().
|
# It exits 75 as well when the peer's role cannot be read - guessed wrong, the enable would
|
# give a directory server a replication server - and when a directory server would enable
|
# with a directory server that no replication server joined yet: that topology has none, and
|
# dsreplication enable refuses it. The peer is read under its lock, which an enable through
|
# it by another server holds while it changes what the peer runs.
|
enable_through() {
|
local rc own peer_lock config role peer_args own_args
|
take_join_lock localhost || return 75
|
own=$JOIN_LOCK_VALUE
|
if ! take_join_lock "$1"; then
|
release_join_lock localhost "$own"
|
return 75
|
fi
|
peer_lock=$JOIN_LOCK_VALUE
|
if ! config=$(replication_config "$1"); then
|
echo "join: could not read how $1 replicates"
|
rc=75
|
elif role=$(peer_role "$config") && [ "$ROLE" = directory ] && [ "$role" = directory ] \
|
&& [ -z "$(config_shape "$config")" ]; then
|
echo "join: $1 is a directory server that no replication server joined yet, and a directory server joins only through a topology that has one"
|
rc=75
|
else
|
mapfile -t peer_args < <(role_arguments 1 "$role")
|
mapfile -t own_args < <(role_arguments 2 "$ROLE")
|
bounded "$REPLICATION_ATTEMPT_TIMEOUT" dsreplication enable \
|
--host1 "$1" --port1 "$ADMIN_PORT" --bindDN1 "$ROOT_USER_DN" \
|
--bindPasswordFile1 "$PASSWORD_FILE" "${peer_args[@]}" \
|
--host2 "$MYHOSTNAME" --port2 "$ADMIN_PORT" --bindDN2 "$ROOT_USER_DN" \
|
--bindPasswordFile2 "$PASSWORD_FILE" "${own_args[@]}" \
|
--adminUID admin --adminPasswordFile "$PASSWORD_FILE" \
|
--baseDN "$BASE_DN" -X -n
|
rc=$?
|
fi
|
release_join_lock "$1" "$peer_lock"
|
release_join_lock localhost "$own"
|
return $rc
|
}
|
|
# the retry interval and up to as much again, at random: two joins that keep missing each
|
# other's lock would otherwise try again in step, round after round
|
pause() {
|
local delay=$((REPLICATION_RETRY_INTERVAL + RANDOM % (REPLICATION_RETRY_INTERVAL + 1)))
|
echo "$1, trying again in $delay s"
|
sleep "$delay"
|
}
|
|
initialize_from() {
|
bounded "$REPLICATION_INITIALIZE_TIMEOUT" dsreplication initialize --baseDN "$BASE_DN" \
|
--adminUID admin --adminPasswordFile "$PASSWORD_FILE" \
|
--hostSource "$1" --portSource "$ADMIN_PORT" \
|
--hostDestination "$MYHOSTNAME" --portDestination "$ADMIN_PORT" -X -n
|
}
|
|
# the generation ID of the replication domain, from the domain's own monitor entry: a
|
# replication server also puts domain-name, connected-to and generation-id on the entry of
|
# every directory server connected to it, but only the domain publishes replayed-updates
|
generation_id() {
|
search "$1" --baseDN "cn=monitor" --searchScope sub \
|
"(&(domain-name=$BASE_DN_VALUE)(replayed-updates=*))" generation-id \
|
| awk 'tolower($1) == "generation-id:" { print $2; exit }'
|
}
|
|
cross_check() { # <peer> <what the mismatch means>
|
local local_id peer_id
|
local_id=$(generation_id localhost)
|
peer_id=$(generation_id "$1")
|
if [ -n "$local_id" ] && [ -n "$peer_id" ] && [ "$local_id" != "$peer_id" ]; then
|
echo "join: generation ID $local_id does not match $peer_id of $1$2"
|
fi
|
}
|
|
# the first other peer, in the order of the list, that holds the data of the topology and
|
# replicates BASE_DN - asked of the peer itself, as cn=admin data registers a server under
|
# whatever name the enable that registered it used, which need not be the listed one
|
trusted_source() {
|
local peer
|
for peer in "${PEERS[@]}"; do
|
is_self "$peer" && continue
|
trusted "$(peer_state "$peer")" && replicates_base_dn "$peer" && { echo "$peer"; return 0; }
|
done
|
return 1
|
}
|
|
# every other peer answers and waits for the data of the topology itself; true as well when
|
# the list names no other server
|
all_others_pending() {
|
local peer
|
for peer in "${PEERS[@]}"; do
|
is_self "$peer" && continue
|
[ "$(peer_state "$peer")" = pending ] || return 1
|
done
|
return 0
|
}
|
|
# some other peer answers and may hold the data of the topology: it is ready, it is rejoining
|
# (it holds the data while its replication is down), it refuses ROOT_PASSWORD, or it publishes
|
# no state and replicates BASE_DN. Unlike an initialize source,
|
# the last counts here even where the peers are explicit - a server of an image before this
|
# one, still serving the topology, and seeding next to it would fork it. A server of this
|
# image publishes no state only while it bootstraps, and it replicates nothing then, so it
|
# does not hold the seed off. A refusing peer cannot be asked what it replicates, but only a
|
# server past its bootstrap has a root password other than ROOT_PASSWORD.
|
any_other_may_hold_data() {
|
local peer
|
for peer in "${PEERS[@]}"; do
|
is_self "$peer" && continue
|
case $(peer_state "$peer") in
|
ready | rejoining | refused) return 0 ;;
|
absent) replicates_base_dn "$peer" && return 0 ;;
|
esac
|
done
|
return 1
|
}
|
|
# The values of the replication server lists this server holds: one "<kind>\t<cn>" line for
|
# every list, its replication server entry (kind server) and each of its replication domains
|
# (kind domain) - BASE_DN, and cn=schema and cn=admin data, which dsreplication enable
|
# configures next to it - followed by a "<kind>\t<cn>\t<host:port>" line for every value
|
replication_server_lists() {
|
search localhost --baseDN "cn=config" --searchScope sub \
|
"(|(objectClass=ds-cfg-replication-server)(objectClass=ds-cfg-replication-domain))" \
|
objectClass cn ds-cfg-replication-server \
|
| awk -v OFS='\t' '
|
function flush( i) {
|
if (kind != "") { print kind, cn; for (i = 1; i <= n; i++) print kind, cn, values[i] }
|
kind = ""; cn = ""; n = 0
|
}
|
/^dn: / { flush() }
|
tolower($0) == "objectclass: ds-cfg-replication-domain" { kind = "domain" }
|
tolower($0) == "objectclass: ds-cfg-replication-server" { kind = "server" }
|
tolower($1) == "cn:" { cn = substr($0, 5) }
|
tolower($1) == "ds-cfg-replication-server:" { values[++n] = $2 }
|
END { flush() }'
|
}
|
|
# it runs inside loops that read their lines from stdin, which dsconfig is kept away from
|
change_replication_servers() { # <kind> <cn> --add|--remove <host:port>
|
if [ "$1" = server ]; then
|
dsconfig_local set-replication-server-prop --provider-name "Multimaster Synchronization" \
|
"$3" "replication-server:$4" </dev/null
|
else
|
dsconfig_local set-replication-domain-prop --provider-name "Multimaster Synchronization" \
|
--domain-name "$2" "$3" "replication-server:$4" </dev/null
|
fi
|
}
|
|
# "<dn>\t<hostname>" for every server this server finds in its cn=admin data
|
registrations() {
|
search localhost --baseDN "cn=Servers,cn=admin data" --searchScope one "(objectClass=*)" hostname \
|
| awk '/^dn: /{dn=substr($0,5)} tolower($1) == "hostname:" {print dn "\t" $2}'
|
}
|
|
# removes a server's entry and its group membership from cn=admin data
|
unregister() { # <dn>
|
printf 'dn: %s\nchangetype: delete\n' "$1" | ldapmodify_local >/dev/null || return 1
|
printf 'dn: cn=all-servers,cn=Server Groups,cn=admin data\nchangetype: modify\ndelete: uniqueMember\nuniqueMember: %s\n' "${1%%,cn=Servers,cn=admin data}" \
|
| ldapmodify_local >/dev/null || true
|
}
|
|
# A server that the survivors removed from the topology while it was away, and that comes
|
# back on the volume it kept, still replicates BASE_DN and cn=admin data. dsreplication enable
|
# registers a server only where the administration data of one of the two is not replicated
|
# yet: with both replicated and their registries alike - this server's copy lacks it once the
|
# delete of the survivors reached it - the enable updates the replication server lists and
|
# leaves cn=admin data as it is, and this server stays unregistered for good. So its own
|
# replication configuration is taken down first, and the enable after that registers it
|
# anew. A copy of its entry that the delete has not reached yet is dropped as well: the
|
# registries would differ, the enable would merge them, and the delete arriving later would
|
# take this server out again.
|
# From the first change on, writes this server takes get no change number and reach no other
|
# server. Removing the health marker is not enough to stop them: new probes fail, but a
|
# container turns unhealthy only after its probe's retries (a readiness probe after its
|
# failure threshold), by when the enable has usually run, and connections that are open stay
|
# open. So before anything changes the backend of BASE_DN refuses the writes of clients (see
|
# hold_writes), and joined() or seed() lets them in again. $REJOIN_PENDING keeps run.sh from
|
# reporting a restart healthy until then. A volume that holds the data of the topology
|
# publishes the state rejoining meanwhile, which keeps the peers from enabling or
|
# initializing through it while it has no replication of its own, and keeps a first peer
|
# from seeding next to it; one that waits for that data stays pending. It holds its own join
|
# lock while its configuration goes, so that no server enables through it then. One attempt:
|
# a lock another join holds leaves everything as it was, and the next round tries again.
|
reset_replication() { # <rejoining|pending> <why>
|
local dn host own
|
echo "join: $2, taking its replication configuration down to enable it anew"
|
publish_state "$1" || return 1
|
touch "$REJOIN_PENDING"
|
rm -f "$BOOTSTRAP_COMPLETE"
|
if ! take_join_lock localhost; then
|
echo "join: its replication configuration stays as it is until the next round"
|
return 1
|
fi
|
own=$JOIN_LOCK_VALUE
|
# a replication server has no backend of BASE_DN, and no client writes to hold
|
if [ "$ROLE" != replication ] && ! hold_writes; then
|
echo "join: could not stop the writes of clients, its replication configuration stays as it is"
|
release_join_lock localhost "$own"
|
return 1
|
fi
|
registrations | while IFS=$'\t' read -r dn host; do
|
[ -n "$host" ] && is_self "$host" || continue
|
echo "join: dropping the stale entry $dn"
|
unregister "$dn" || echo "join: could not remove $dn"
|
done
|
bounded "$REPLICATION_ATTEMPT_TIMEOUT" dsreplication disable --hostname "$MYHOSTNAME" --port "$ADMIN_PORT" \
|
--adminUID admin --adminPasswordFile "$PASSWORD_FILE" --disableAll -X -n \
|
|| echo "join: dsreplication disable exited with $?"
|
release_join_lock localhost "$own"
|
}
|
|
# "<backend-id> <writability-mode>" of the backend that holds BASE_DN; the mode is left out
|
# where the entry does not set one, which means enabled. Nothing where no backend holds it,
|
# and a failure when the search failed
|
data_backend() {
|
local out
|
out=$(search localhost --baseDN "cn=Backends,cn=config" --searchScope one \
|
"(&(objectClass=ds-cfg-backend)(ds-cfg-base-dn=$BASE_DN_VALUE))" ds-cfg-backend-id ds-cfg-writability-mode) \
|
|| return 1
|
printf '%s\n' "$out" | awk 'tolower($1) == "ds-cfg-backend-id:" { id = $2 } tolower($1) == "ds-cfg-writability-mode:" { mode = $2 }
|
END { if (id != "") print id, mode }'
|
}
|
|
# The backend of BASE_DN takes internal and replication writes only - dsreplication enable
|
# and initialize go through those, a client's write is refused (Unwilling to Perform). Only
|
# that backend: internal-only on the whole server would refuse the writes dsreplication
|
# enable makes to cn=config and cn=admin data. The mode is configuration, so it outlasts a
|
# restart; $REJOIN_PENDING names the backend, so that release_writes lets the writes in
|
# again on the start that rejoins, and only where this join stopped them: a backend that an
|
# operator made read-only already stays as it is.
|
hold_writes() {
|
local backend mode
|
read -r backend mode <<<"$(data_backend)"
|
if [ -z "$backend" ]; then
|
echo "join: found no backend that holds $BASE_DN"
|
return 1
|
fi
|
[ "${mode:-enabled}" = enabled ] || return 0
|
echo "join: backend $backend refuses the writes of clients until this server rejoins"
|
# named before the mode changes: a container killed in between lets the writes in again
|
# once it rejoined, rather than leaving them refused for good
|
printf '%s\n' "$backend" >"$REJOIN_PENDING" || return 1
|
dsconfig_local set-backend-prop --backend-name "$backend" --set writability-mode:internal-only </dev/null >/dev/null
|
}
|
|
release_writes() {
|
local backend
|
backend=$(cat "$REJOIN_PENDING" 2>/dev/null)
|
[ -n "$backend" ] || return 0
|
dsconfig_local set-backend-prop --backend-name "$backend" --set writability-mode:enabled </dev/null >/dev/null || return 1
|
echo "join: backend $backend takes the writes of clients again"
|
}
|
|
# Removes every server that is registered in the topology but no longer listed in
|
# REPLICATION_PEERS: its entry and group membership in cn=admin data (which is replicated,
|
# so one survivor's delete reaches the others) and its values in every replication server
|
# list of this server (which are configuration of this server alone, so every survivor
|
# prunes its own on its next start). dsreplication disable cannot do this for a dead server:
|
# it changes only the servers it can reach, and OpenDJ 4 has no cleanup subcommand. Every
|
# step tolerates losing the race to another survivor. A server registered by an address - a
|
# master that MASTER_SERVER named by one, whose replicas enabled through that address - is
|
# left alone: nothing tells it from a server listed by name, and the list moving from
|
# MASTER_SERVER to REPLICATION_PEERS would otherwise throw the live master out.
|
cleanup_departed() {
|
local dn host kind cn value
|
[ "$PEERS_ARE_EXPLICIT" = yes ] || return 0
|
registrations | while IFS=$'\t' read -r dn host; do
|
[ -z "$host" ] && continue
|
is_address "$host" && continue
|
if ! in_peers "$host" && ! is_self "$host"; then
|
echo "join: removing departed server $host from cn=admin data"
|
unregister "$dn" || echo "join: could not remove $dn (already removed by another survivor?)"
|
fi
|
done
|
replication_server_lists | while IFS=$'\t' read -r kind cn value; do
|
[ -z "$value" ] && continue
|
host=${value%:*}
|
is_address "$host" && continue
|
if ! in_peers "$host" && ! is_self "$host"; then
|
echo "join: removing departed replication server $value from the $kind list of $cn"
|
change_replication_servers "$kind" "$cn" --remove "$value" || true
|
fi
|
done
|
}
|
|
# Adds every listed peer that is registered in the topology to each replication server list
|
# of this server that lacks it, under the name it is registered by - the name dsreplication
|
# enable writes as well, so no server is listed twice. Joins that run at the same time each
|
# configure the topology as they read it, and two of them can leave a pair of replication
|
# servers that know nothing of each other, cutting the updates of one off from the other; a
|
# replication server connects to a server added to its list at once. A peer that is not
|
# registered yet is waited for, as long as the retries last, after this server already
|
# reports itself healthy. Only a peer that runs a replication server goes into a list - a
|
# directory server has none to connect to - and cn=admin data does not say which do, so a
|
# peer that a list lacks is asked; one that does not answer is left for the repair of a later
|
# start.
|
repair_replication_servers() {
|
local i peer waiting hosts host lists kind cn value found config servers=
|
[ "$PEERS_ARE_EXPLICIT" = yes ] || return 0
|
for i in $(seq 1 "$REPLICATION_RETRY_COUNT"); do
|
waiting=
|
# one search a round: every ldapsearch starts a JVM
|
hosts=$(registered_hosts)
|
for peer in "${PEERS[@]}"; do
|
is_self "$peer" && continue
|
found=no
|
for host in $hosts; do
|
names_match "$(lower "$peer")" "$(lower "$host")" && { found=yes; break; }
|
done
|
[ "$found" = yes ] || waiting="$waiting $peer"
|
done
|
[ -z "$waiting" ] && break
|
[ "$i" -eq "$REPLICATION_RETRY_COUNT" ] && break
|
echo "join: waiting for$waiting to register in the topology before the replication server lists are checked"
|
sleep "$REPLICATION_RETRY_INTERVAL"
|
done
|
hosts=$(registered_hosts)
|
lists=$(replication_server_lists)
|
# a here-string of nothing still reads as one empty line
|
[ -n "$lists" ] || return 0
|
# every ldapsearch starts a JVM: a peer that every list holds already is not asked. One that
|
# does not answer waits for the repair of a later start, whatever the roles around it: a
|
# replication server left out of a list is added then - and its own repair adds this server
|
# to its lists when it starts, which connects the two already - while a directory server put
|
# into one would stay there for good, a replication server that nothing listens on, which
|
# cleanup_departed keeps as long as the host is listed; and a peer that every list holds
|
# is never asked again, so nothing would take it out.
|
for host in $hosts; do
|
is_self "$host" && continue
|
in_peers "$host" || continue
|
lacked_by_a_list "$host" "$lists" || continue
|
if ! config=$(replication_config "$host"); then
|
echo "join: could not ask $host whether it runs a replication server, its place in the replication server lists is checked on a later start"
|
elif grep -qx server <<<"$config"; then
|
servers="$servers $host"
|
fi
|
done
|
while IFS=$'\t' read -r kind cn value; do
|
[ -n "$value" ] && continue
|
for host in $servers; do
|
if ! list_holds "$host" "$kind" "$cn" "$lists"; then
|
echo "join: adding replication server $host:$REPLICATION_PORT to the $kind list of $cn"
|
change_replication_servers "$kind" "$cn" --add "$host:$REPLICATION_PORT" || true
|
fi
|
done
|
done <<<"$lists"
|
}
|
|
# whether the list of that kind and cn holds the host, in replication_server_lists output
|
list_holds() { # <host> <kind> <cn> <lists>
|
local k c v
|
while IFS=$'\t' read -r k c v; do
|
[ "$k" = "$2" ] && [ "$c" = "$3" ] && [ -n "$v" ] || continue
|
names_match "$(lower "${v%:*}")" "$(lower "$1")" && return 0
|
done <<<"$4"
|
return 1
|
}
|
|
lacked_by_a_list() { # <host> <lists>
|
local kind cn value
|
while IFS=$'\t' read -r kind cn value; do
|
[ -n "$value" ] && continue
|
list_holds "$1" "$kind" "$cn" "$2" || return 0
|
done <<<"$2"
|
return 1
|
}
|
|
# the backend of BASE_DN refuses the writes of clients because this join stopped them: an
|
# operator's read-only backend leaves $REJOIN_PENDING empty, and a hold whose dsconfig failed
|
# named the backend without changing its mode
|
writes_held() {
|
[ -s "$REJOIN_PENDING" ] && [ "$(data_backend | awk '{ print $2 }')" = internal-only ]
|
}
|
|
# a server that rejoins keeps $REJOIN_PENDING, and so stays unready on its next start too,
|
# until its clients may write again and its peers see it ready. A dsconfig or a publish that
|
# fails is tried again here, as long as the retries last: the join that got this far would
|
# otherwise end, and only a later start would enable, or initialize, all over again
|
leave_rejoin() {
|
local i
|
[ -f "$REJOIN_PENDING" ] || return 0
|
for i in $(seq 1 "$REPLICATION_RETRY_COUNT"); do
|
if release_writes && publish_state ready; then
|
rm -f "$REJOIN_PENDING"
|
return 0
|
fi
|
if [ "$i" -lt "$REPLICATION_RETRY_COUNT" ]; then
|
pause "join: this server is not out of its rejoin yet ($i of $REPLICATION_RETRY_COUNT)"
|
fi
|
done
|
if writes_held; then
|
echo "join: could not let the writes of clients in again after $REPLICATION_RETRY_COUNT attempts, this container will not report itself healthy and serves its data read-only until a later start does"
|
else
|
echo "join: could not publish this server ready after $REPLICATION_RETRY_COUNT attempts, this container will not report itself healthy until a later start does"
|
fi
|
return 1
|
}
|
|
joined() {
|
cleanup_departed
|
leave_rejoin || exit 1
|
# a replication server published itself rejoining until now, on the ready road too
|
if [ "$ROLE" = replication ] && ! publish_state ready; then
|
echo "join: could not publish this server ready, this container will not report itself healthy until a later start does"
|
exit 1
|
fi
|
touch "$BOOTSTRAP_COMPLETE"
|
echo "join: this server is a member of the replication topology, the health check may probe it"
|
repair_replication_servers
|
}
|
|
# the peers learn that this server holds the data before anything else changes: a seed whose
|
# state never reached them would be healthy while every pending peer skips it
|
seed() { # <why>
|
echo "join: $1, seeding it with this server's data"
|
publish_state ready || return 1
|
leave_rejoin || return 1
|
rm -f "$INITIALIZE_PENDING"
|
touch "$BOOTSTRAP_COMPLETE"
|
}
|
|
# what giving up means depends on whether the container already reports itself healthy: a
|
# volume that holds the data of the topology does so from the start, until a reset of its
|
# replication took that back - and the writes of its clients, where the reset stopped them
|
give_up() {
|
if [ -f "$BOOTSTRAP_COMPLETE" ] && [ "$ROLE" = replication ]; then
|
echo "join: could not join the replication topology after $REPLICATION_RETRY_COUNT attempts, this replication server stays apart from it until a later start joins it"
|
elif [ -f "$BOOTSTRAP_COMPLETE" ]; then
|
echo "join: could not join the replication topology after $REPLICATION_RETRY_COUNT attempts, this server keeps serving its data unreplicated until a later start joins it"
|
elif writes_held; then
|
echo "join: could not join the replication topology after $REPLICATION_RETRY_COUNT attempts, this container will not report itself healthy and serves its data read-only until a later start joins it"
|
else
|
echo "join: could not join the replication topology after $REPLICATION_RETRY_COUNT attempts, this container will not report itself healthy"
|
fi
|
exit 1
|
}
|
|
echo "join: waiting for the server on the administration connector"
|
for i in $(seq 1 150); do
|
server_up && own_config=$(replication_config localhost) && break
|
if [ "$i" -eq 150 ]; then
|
echo "join: the server did not come up, or does not take ROOT_PASSWORD any more, giving up"
|
exit 1
|
fi
|
sleep 2
|
done
|
|
# The role this server has, which every step below goes by: the one setup.sh set the volume
|
# up with, whatever REPLICATION_ROLE says now - a replication server has no backend for
|
# BASE_DN, a server with one holds data, and a later start in another role is told rather than
|
# left to believe it took effect. A volume set up by an image before roles has no role
|
# recorded: its replication configuration tells it, and REPLICATION_ROLE where that does not -
|
# a replication server without a domain is the replication server of the one-shot srs, when
|
# REPLICATION_ROLE says so, and a combined server whose enable stopped half-way otherwise. One
|
# without a backend for BASE_DN as well holds no data whatever REPLICATION_ROLE says: only
|
# this image sets a volume up so, for a replication server. The role found is recorded, so
|
# that it stays what it is from then on.
|
legacy_role() { # <replication_config output>
|
case $(config_shape "$1") in
|
combined) echo combined ;;
|
directory) echo directory ;;
|
server)
|
if [ "$REPLICATION_ROLE" = replication ] || ! grep -qx backend <<<"$1"; then
|
echo replication
|
else
|
echo combined
|
fi
|
;;
|
*)
|
if [ -f "$REJOIN_PENDING" ]; then
|
# only the join of an image before roles took a volume down, and it ran combined
|
# servers only
|
echo combined
|
elif grep -qx backend <<<"$1" && [ "$REPLICATION_ROLE" = replication ]; then
|
echo "join: this server holds a backend for $BASE_DN, which a replication server does not, and joins as a combined server; a replication server starts from an empty volume" >&2
|
echo combined
|
else
|
echo "$REPLICATION_ROLE"
|
fi
|
;;
|
esac
|
}
|
if [ -s "$REPLICATION_ROLE_FILE" ]; then
|
ROLE=$(cat "$REPLICATION_ROLE_FILE")
|
case $ROLE in
|
combined | directory | replication) ;;
|
*)
|
echo "join: $REPLICATION_ROLE_FILE holds $ROLE, which is not one of combined, directory or replication, this server does not join"
|
exit 1 ;;
|
esac
|
else
|
ROLE=$(legacy_role "$own_config")
|
printf '%s\n' "$ROLE" >"$REPLICATION_ROLE_FILE" \
|
|| echo "join: could not record the role $ROLE in $REPLICATION_ROLE_FILE, the next start reads it again"
|
fi
|
if [ "$ROLE" != "$REPLICATION_ROLE" ]; then
|
echo "join: this server is a $ROLE server and keeps that role, REPLICATION_ROLE=$REPLICATION_ROLE does not change it"
|
fi
|
if [ "$ROLE" != replication ] && ! grep -qx backend <<<"$own_config"; then
|
echo "join: this server holds no backend for $BASE_DN, which a $ROLE server needs: its bootstrap did not create one"
|
fi
|
|
# Only the first peer of REPLICATION_PEERS that holds data may seed a topology: a replication
|
# server has none to seed it with, and one listed ahead of the others is passed over. Asked
|
# only where a seed is weighed, and only of the peers listed ahead of this one, each by the
|
# role it publishes; one that does not answer, or does not publish one yet, counts as holding
|
# data, so that two servers never both take themselves for the first. A role stays with its
|
# volume, so each answer is kept: a peer ahead that publishes another role holds data for
|
# good, and one that publishes replication is not asked again.
|
AHEAD_HOLDS_DATA=no
|
AHEAD_REPLICATION=
|
may_seed() {
|
local peer config
|
[ "$ROLE" = replication ] && return 1
|
[ "$AHEAD_HOLDS_DATA" = yes ] && return 1
|
for peer in "${PEERS[@]}"; do
|
is_self "$peer" && return 0
|
[[ " $AHEAD_REPLICATION " == *" $peer "* ]] && continue
|
config=$(replication_config "$peer") || return 1
|
grep -q '^published' <<<"$config" || return 1
|
if [ "$(peer_role "$config")" != replication ]; then
|
AHEAD_HOLDS_DATA=yes
|
return 1
|
fi
|
AHEAD_REPLICATION="$AHEAD_REPLICATION $peer"
|
done
|
return 1
|
}
|
|
# Every round on both roads starts here: a replication configuration that the peers no longer
|
# register is taken down (see reset_replication). So is one that the peers register but the
|
# cn=admin data of this server does not: an enable that stopped half-way - its initialize of
|
# cn=admin data failed, say - leaves BASE_DN replicated and this server registered at the
|
# peer, and every enable after it exits 5, "already replicated", without registering it here.
|
# Fails when a reset is due and could not be done in this round, which then enables nothing -
|
# an enable registers nobody before it.
|
reset_if_unregistered() { # <the state to publish meanwhile>
|
local rc=0
|
holds_own_part || return 0
|
if ! registered_with_peers; then
|
reset_replication "$1" "the peers no longer register this server"
|
return
|
fi
|
registered_here || rc=$?
|
if [ "$rc" -eq 1 ]; then
|
reset_replication "$1" "the peers register this server but its own cn=admin data does not"
|
fi
|
}
|
|
if [ ! -f "$INITIALIZE_PENDING" ]; then
|
# This volume holds the data of the topology, or is the one that seeds it: it publishes
|
# that - unless a reset took its replication down and it has not rejoined yet, also on an
|
# earlier start. A replication server holds no data: what a peer may join through is its
|
# membership, and a peer that enabled through it before a reset took it out would be
|
# registered in a registry of the two of them. So it publishes itself rejoining until a
|
# round below finds it a member - not pending, which would count it among the peers that
|
# wait for data and let a fresh seed go ahead next to the topology it serves
|
if [ -f "$REJOIN_PENDING" ] || [ "$ROLE" = replication ]; then
|
publish_state rejoining
|
else
|
publish_state ready
|
fi
|
for i in $(seq 1 "$REPLICATION_RETRY_COUNT"); do
|
if ! reset_if_unregistered rejoining; then
|
: # the next round tries the reset again
|
elif is_member; then
|
echo "join: already a member of the replication topology"
|
joined
|
exit
|
else
|
# only through a peer that holds the topology, as on the pending road: a rejoining peer
|
# has no replication of its own, a pending one or one that still bootstraps none yet, and
|
# an enable with any of them registers the two in a registry of their own - which counts
|
# as membership, a peer of the pair lists this server - beside the survivors, whom
|
# nothing ever adds to it
|
for peer in "${PEERS[@]}"; do
|
is_self "$peer" && continue
|
state=$(peer_state "$peer")
|
if ! trusted "$state"; then
|
echo "join: $peer is $state, not a peer to join through"
|
continue
|
fi
|
echo "join: enabling replication with $peer"
|
enable_through "$peer"
|
rc=$?
|
if member_of_topology; then
|
echo "join: joined the replication topology through $peer"
|
# a replication server holds no generation ID of BASE_DN to compare with, neither
|
# this one nor the peer joined through
|
if [ "$ROLE" != replication ]; then
|
replicates_base_dn "$peer" || peer=$(trusted_source) || peer=
|
[ -z "$peer" ] || cross_check "$peer" "; replication will not flow until this server or that one is initialized by hand"
|
fi
|
joined
|
exit
|
fi
|
[ "$rc" -eq 75 ] || echo "join: dsreplication enable with $peer exited with $rc and membership is not there"
|
done
|
fi
|
# a list without a single other server leaves nothing to retry against, and peers that
|
# all wait for data join through this one rather than the other way round
|
if may_seed && all_others_pending; then
|
seed "no other peer holds the data of the topology" || exit 1
|
exit 0
|
fi
|
if [ "$i" -lt "$REPLICATION_RETRY_COUNT" ]; then
|
pause "join: no peer joined this server yet ($i of $REPLICATION_RETRY_COUNT)"
|
fi
|
done
|
# the same rule as at the end of the pending road below
|
if may_seed && ! any_other_may_hold_data; then
|
seed "no topology found after $REPLICATION_RETRY_COUNT attempts" || exit 1
|
exit 0
|
fi
|
give_up
|
fi
|
|
# This volume was bootstrapped and never received the data of the topology: it joins and
|
# initializes only through a peer that holds that data, and only the first peer that holds
|
# data (see may_seed) may decide that nobody does. Its membership is asked of the peers as on
|
# the road above: a volume that enabled once but never completed its initialize may have been
|
# removed by the survivors while it was away, and its own cn=admin data still registers it
|
# until their delete reaches it - after which every enable registers nobody, as
|
# reset_replication describes.
|
publish_state pending
|
member=no
|
for i in $(seq 1 "$REPLICATION_RETRY_COUNT"); do
|
member=no
|
reset_due=no
|
if ! reset_if_unregistered pending; then
|
reset_due=yes
|
elif is_member; then
|
member=yes
|
fi
|
if [ "$member" = no ] && [ "$reset_due" = no ]; then
|
for peer in "${PEERS[@]}"; do
|
is_self "$peer" && continue
|
state=$(peer_state "$peer")
|
if ! trusted "$state"; then
|
echo "join: $peer is $state, not a peer to join through"
|
continue
|
fi
|
echo "join: enabling replication with $peer"
|
enable_through "$peer"
|
rc=$?
|
if member_of_topology; then
|
member=yes
|
echo "join: joined the replication topology through $peer"
|
break
|
fi
|
[ "$rc" -eq 75 ] || echo "join: dsreplication enable with $peer exited with $rc and membership is not there"
|
done
|
fi
|
if [ "$member" = yes ] && [ "$ROLE" = replication ]; then
|
# a replication server holds no data that an initialize would bring: membership is all
|
# it waits for. Published before the marker goes, as after an initialize
|
publish_state ready || exit 1
|
rm -f "$INITIALIZE_PENDING"
|
joined
|
exit
|
elif [ "$member" = yes ]; then
|
source=$(trusted_source)
|
if [ -n "$source" ]; then
|
echo "join: initializing from $source"
|
initialize_from "$source"
|
rc=$?
|
if [ "$rc" -eq 0 ]; then
|
# published before the marker goes, so that the marker is never gone while the peers
|
# still see this server pending; the next start initializes it again
|
publish_state ready || exit 1
|
rm -f "$INITIALIZE_PENDING"
|
cross_check "$source" " after initialize"
|
joined
|
exit
|
fi
|
echo "join: initialize from $source exited with $rc"
|
elif may_seed && all_others_pending; then
|
seed "every other peer waits for the data of the topology as well" || exit 1
|
joined
|
exit 0
|
else
|
echo "join: no peer holds the data of the topology yet"
|
fi
|
elif may_seed && all_others_pending; then
|
seed "every other peer waits for the data of the topology as well" || exit 1
|
exit 0
|
fi
|
if [ "$i" -lt "$REPLICATION_RETRY_COUNT" ]; then
|
pause "join: not joined and initialized yet ($i of $REPLICATION_RETRY_COUNT)"
|
fi
|
done
|
|
# Only the first peer that holds data may decide that there is no topology to join and that
|
# its own data seeds one, and only while no peer that could hold the topology's data answers;
|
# anyone else staying unhealthy is what surfaces a lost topology instead of forking it
|
if [ "$member" = no ] && may_seed && ! any_other_may_hold_data; then
|
seed "no topology found after $REPLICATION_RETRY_COUNT attempts" || exit 1
|
exit 0
|
fi
|
|
give_up
|
