[#1289] Make temporary directory checks portable to ksh93 (#1291)

Valery Kharseko

6 hours ago 3fa40c285fec0e2a8318f017426ec325c11be9b8

[#1289] Make temporary directory checks portable to ksh93 (#1291)

Fixes #1289.

`setup`, `upgrade`, and other server scripts source `_script-util.sh`
through `/bin/sh`. On ksh93, its `local` commands fail, leaving the
executable-directory probe without its arguments and preventing the
Bouncy Castle fallback from working.

Run the probe in a subshell to keep its variables private in POSIX
shells, and branch on its result directly when choosing the Bouncy
Castle installation directory. The existing explicit directory and
noexec fallback behavior are preserved.

The regression test exercises the real probe on executable and
non-executable files, checks cleanup of a created directory, and
verifies fallback warnings and explicit-directory handling. CI runs it
under ksh93 on macOS and dash on Linux.

Validation:
- The regression test passed under OpenIndiana Hipster 2026.10
(`/bin/sh`, ksh93u+); the original utility failed there with `local: not
found`.
- It also passed locally with ksh, dash, bash, and `/bin/sh`. The added
checks detected the reviewed probe, cleanup, and warning mutations; dash
rejected the non-POSIX conditional mutation.
- A Linux container test with a real noexec `/dev/shm` selected an
executable `$HOME`.
- Shell syntax checks, workflow YAML parsing, and `git diff --check`
passed. Oracle Solaris was not tested.

This change targets POSIX shells, including ksh93. The scripts already
use POSIX constructs that are incompatible with the traditional SVR4
Bourne shell; Solaris 10 compatibility and its support status remain
unverified.

 .github/scripts/test-script-util.sh

New file
@@ -0,0 +1,95 @@
#!/bin/sh
#
# The contents of this file are subject to the terms of the Common Development and
# Distribution License (the License). You may not use this file except in compliance with the
# License.
#
# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
# specific language governing permission and limitations under the License.
#
# When distributing Covered Software, include this CDDL Header Notice in each file and include
# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
# Header, with the fields enclosed by brackets [] replaced by your own identifying
# information: "Portions copyright [year] [name of copyright owner]".
#
# Copyright 2026 3A Systems, LLC.

set -eu
root=$(CDPATH= cd -- "$(dirname "$0")/../.." && pwd)
test_dir=$(mktemp -d "${TMPDIR:-/tmp}/opendj-script-util.XXXXXX")
trap 'rm -rf "$test_dir"' 0

INSTALL_ROOT=$test_dir
INSTANCE_ROOT=$test_dir
SCRIPT_UTIL_CMD=
SCRIPT_NAME=setup
. "$root/opendj-server-legacy/resource/bin/_script-util.sh"

mkdir "$test_dir/tmp" "$test_dir/home"
OPENDJ_TMP_DIR=$test_dir/tmp
HOME=$test_dir/home
OPENDJ_JAVA_ARGS=

# On an executable filesystem, the real probe must succeed without shell errors
# or leaking its scratch variables into the caller.
res=caller
DIR_TO_TEST=$test_dir/caller
TMP_FILE=$test_dir/caller-file
remove_dir=caller
if ! set_bc_dir >"$test_dir/output" 2>&1; then
  cat "$test_dir/output" >&2
  exit 1
fi
if [ -s "$test_dir/output" ]; then
  cat "$test_dir/output" >&2
  exit 1
fi
test -z "$OPENDJ_JAVA_ARGS"
test "$res:$DIR_TO_TEST:$TMP_FILE:$remove_dir" = "caller:$test_dir/caller:$test_dir/caller-file:caller"
test -z "$(ls -A "$OPENDJ_TMP_DIR")"

# Prevent the probe file in the temporary directory from becoming executable,
# while allowing the real probe to make and execute its file in HOME.
chmod() {
  case $2 in
    "$OPENDJ_TMP_DIR"/*) ;;
    *) command chmod "$@" ;;
  esac
}
set_bc_dir >"$test_dir/output" 2>&1
unset -f chmod
test "$OPENDJ_JAVA_ARGS" = " -Dorg.bouncycastle.native.loader.install_dir=$HOME"
test -z "$(ls -A "$OPENDJ_TMP_DIR")"
test -z "$(ls -A "$HOME")"
OPENDJ_JAVA_ARGS=

# A directory created by the real probe must be removed afterwards.
check_noexec "$test_dir/missing"
test ! -e "$test_dir/missing"

# Simulate a noexec mount: the production selection logic still runs, while
# the probe returns the result of attempting to execute from each directory.
check_noexec() { [ "$1" = "$HOME" ]; }
set_bc_dir >"$test_dir/output" 2>&1
test "$OPENDJ_JAVA_ARGS" = " -Dorg.bouncycastle.native.loader.install_dir=$HOME"
grep -F "WARNING: $OPENDJ_TMP_DIR is mounted as noexec" "$test_dir/output" >/dev/null
if grep -F 'the OpenDJ installation could cause errors' "$test_dir/output" >/dev/null; then
  cat "$test_dir/output" >&2
  exit 1
fi

# An explicit BC installation directory must be preserved without probing.
OPENDJ_JAVA_ARGS='-Dorg.bouncycastle.native.loader.install_dir=/chosen'
check_noexec() { echo 'unexpected probe'; return 1; }
set_bc_dir >"$test_dir/output" 2>&1
test ! -s "$test_dir/output"
test "$OPENDJ_JAVA_ARGS" = '-Dorg.bouncycastle.native.loader.install_dir=/chosen'

# If neither directory permits execution, report it and do not set a fallback.
OPENDJ_JAVA_ARGS=
check_noexec() { return 1; }
set_bc_dir >"$test_dir/output" 2>&1
test -z "$OPENDJ_JAVA_ARGS"
grep -F 'the OpenDJ installation could cause errors' "$test_dir/output" >/dev/null

echo 'script-util checks passed'

 .github/workflows/build.yml

@@ -62,6 +62,14 @@
      if: runner.os == 'Linux' && matrix.java == '11'
      shell: bash
      run: bash .github/scripts/test-refresh-launchers.sh
    - name: Check shell utility with ksh93
      if: runner.os == 'macOS' && matrix.java == '11'
      shell: bash
      run: /bin/ksh .github/scripts/test-script-util.sh
    - name: Check shell utility with dash
      if: runner.os == 'Linux' && matrix.java == '11'
      shell: bash
      run: dash .github/scripts/test-script-util.sh
    - name: Java ${{ matrix.Java }} (${{ matrix.os }})
      uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0
      with:

 opendj-server-legacy/resource/bin/_script-util.sh

@@ -83,17 +83,18 @@
  export OPENDJ_JAVA_BIN
}

check_noexec() { #returns 0 if can execute files in the directory, otherwise 1
  local DIR_TO_TEST=$1
  local res=0
  local remove_dir=0
# Use a subshell to keep the probe's variables private in POSIX shells.
check_noexec() ( #returns 0 if can execute files in the directory, otherwise 1
  DIR_TO_TEST=$1
  res=0
  remove_dir=0

  if [ ! -d "${DIR_TO_TEST}" ]; then
    remove_dir=1
    mkdir ${DIR_TO_TEST}
  fi

  local TMP_FILE=`mktemp ${DIR_TO_TEST}/temp.XXXXXX`
  TMP_FILE=`mktemp ${DIR_TO_TEST}/temp.XXXXXX`

  chmod +x ${TMP_FILE}
  if ! ${TMP_FILE} 2>/dev/null; then
@@ -107,23 +108,19 @@
  fi

  return $res
}
)

set_bc_dir() {
  if echo "$OPENDJ_JAVA_ARGS" | grep -q -o "\-Dorg.bouncycastle.native.loader.install_dir=[^ ]*"; then
    return # bc install dir already set
  fi

  check_noexec "${OPENDJ_TMP_DIR}"
  local res=$?
  if [ $res = 1 ]; then
  if ! check_noexec "${OPENDJ_TMP_DIR}"; then
    if [ "$SCRIPT_NAME" = "setup" ]; then
      echo "WARNING: $OPENDJ_TMP_DIR is mounted as noexec, try switching to $HOME for org.bouncycastle.native.loader.install_dir"
    fi
    check_noexec "${HOME}"
    res=$?
    if [ $res = 0 ]; then
      OPENDJ_JAVA_ARGS="${OPENDJ_JAVA_ARGS} -Dorg.bouncycastle.native.loader.install_dir=${HOME}"
    if check_noexec "${HOME}"; then
      OPENDJ_JAVA_ARGS="${OPENDJ_JAVA_ARGS} -Dorg.bouncycastle.native.loader.install_dir=${HOME}"
    elif [ "$SCRIPT_NAME" = "setup" ]; then
      echo "WARNING: $HOME is mounted as noexec, the OpenDJ installation could cause errors"
    fi