From 91fcddb2d3bc7be859d60c062089ed947f480dd1 Mon Sep 17 00:00:00 2001
From: Valery Kharseko <vharseko@3a-systems.ru>
Date: Sun, 27 Sep 2026 08:55:51 +0000
Subject: [PATCH] Refresh the Windows launchers only when their code changes, not their toolchain stamp (#1103)
---
.github/scripts/test-refresh-launchers.sh | 121 ++++++++++++++++++++
.github/scripts/refresh-launchers.sh | 35 +++++
.github/scripts/same-pe-code.py | 163 +++++++++++++++++++++++++++
opendj-server-legacy/src/build-tools/windows/Makefile | 5
.github/workflows/deploy.yml | 23 +++
.github/workflows/build.yml | 13 +
6 files changed, 350 insertions(+), 10 deletions(-)
diff --git a/.github/scripts/refresh-launchers.sh b/.github/scripts/refresh-launchers.sh
new file mode 100755
index 0000000..96d49f1
--- /dev/null
+++ b/.github/scripts/refresh-launchers.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+#
+# The contents of this file are subject to the terms of the Common Development and
+# Distribution License (the License). You may not use this file except in compliance with the
+# License.
+#
+# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
+# specific language governing permission and limitations under the License.
+#
+# When distributing Covered Software, include this CDDL Header Notice in each file and include
+# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
+# Header, with the fields enclosed by brackets [] replaced by your own identifying
+# information: "Portions copyright [year] [name of copyright owner]".
+#
+# Copyright 2026 3A Systems, LLC.
+
+# Usage: refresh-launchers.sh BUILT LIB
+#
+# Copies each BUILT/*.exe over its namesake in LIB, unless the two differ only in the
+# toolchain stamp (same-pe-code.py). The Package/Deploy workflow runs it before it
+# commits LIB; the Build workflow runs it on known launcher pairs before merge.
+set -e
+here=$(dirname "$0")
+for built in "$1"/*.exe; do
+ committed=$2/$(basename "$built")
+ # Exit 1 (the code differs) and 2 (not readable as a PE image) both take the rebuilt
+ # file: when in doubt, refresh.
+ if [ -f "$committed" ] && python3 "$here/same-pe-code.py" "$committed" "$built"; then
+ if ! cmp -s "$committed" "$built"; then
+ echo "$committed differs from the rebuilt one only in the toolchain stamp - keeping it."
+ fi
+ continue
+ fi
+ cp "$built" "$committed"
+done
diff --git a/.github/scripts/same-pe-code.py b/.github/scripts/same-pe-code.py
new file mode 100755
index 0000000..d497a2a
--- /dev/null
+++ b/.github/scripts/same-pe-code.py
@@ -0,0 +1,163 @@
+#!/usr/bin/env python3
+#
+# The contents of this file are subject to the terms of the Common Development and
+# Distribution License (the License). You may not use this file except in compliance with the
+# License.
+#
+# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
+# specific language governing permission and limitations under the License.
+#
+# When distributing Covered Software, include this CDDL Header Notice in each file and include
+# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
+# Header, with the fields enclosed by brackets [] replaced by your own identifying
+# information: "Portions copyright [year] [name of copyright owner]".
+#
+# Copyright 2026 3A Systems, LLC.
+
+"""Tell whether two PE images differ in anything but the stamp of the toolchain.
+
+Usage: same-pe-code.py COMMITTED REBUILT
+
+Exits 0 when the two files are identical once the build stamp and the header padding
+are left out of both, 1 when they differ anywhere else, and 2 when either one cannot be
+read as a PE image.
+
+The Windows launchers are linked with /Brepro, so their bytes are a function of the
+inputs - and the inputs include the build numbers of cl, link and cvtres. Two runner
+images a patch release of Visual Studio apart (14.51.36256 and 14.51.36257, say) turn
+out byte-for-byte the same code, data and resources, yet different files: the Rich
+header lists those build numbers, the REPRO debug entry holds a hash over them, and
+the COFF and debug directory timestamps and the PE checksum are derived from that
+hash. GitHub rolls a new image out over days, so the Windows job lands on either one
+and a byte comparison refreshes the committed launchers back and forth on every push.
+
+What is left out of the comparison:
+- the Rich header;
+- the COFF and debug directory timestamps and the PE checksum;
+- the REPRO hash, and the PDB GUID and age of an RSDS CodeView entry;
+- the zero padding the linker puts after the DOS stub, up to the PE header, and after
+ the section table, up to SizeOfHeaders, and e_lfanew, which only says where the first
+ of them ends. A patch release may pad differently:
+ 14.51.36252 put the PE header at 0x100, 14.51.36256 at 0xf0, around the same code.
+The DOS stub, the PE headers, the section table and every byte from SizeOfHeaders on
+are still compared, so a change of source or of code generation still shows. A Rich
+header that gains or loses an entry no longer does on its own: it comes with a change
+of the objects linked in, which moves the code as well.
+"""
+
+import struct
+import sys
+
+DEBUG_TYPE_CODEVIEW = 2
+DEBUG_TYPE_REPRO = 16
+DEBUG_ENTRY_SIZE = 28
+
+
+class NotPE(Exception):
+ pass
+
+
+def u16(b, off):
+ return struct.unpack_from("<H", b, off)[0]
+
+
+def u32(b, off):
+ return struct.unpack_from("<I", b, off)[0]
+
+
+def blank(b, off, length):
+ if off < 0 or off + length > len(b):
+ raise NotPE("field at 0x%x runs past the end of the file" % off)
+ b[off:off + length] = bytes(length)
+
+
+def normalize(data):
+ b = bytearray(data)
+ try:
+ if b[:2] != b"MZ":
+ raise NotPE("no MZ signature")
+ pe = u32(b, 0x3C)
+ if b[pe:pe + 4] != b"PE\0\0":
+ raise NotPE("no PE signature")
+
+ # The Rich header sits between the DOS stub and the PE header: "DanS" XOR key,
+ # the (prodId, build, count) records XOR key, then "Rich" and the key itself.
+ rich = b.find(b"Rich", 0x40, pe)
+ if rich >= 0:
+ key = b[rich + 4:rich + 8]
+ dans = bytes(x ^ y for x, y in zip(b"DanS", key))
+ start = b.rfind(dans, 0x40, rich)
+ if start < 0:
+ raise NotPE("Rich header without its DanS marker")
+ blank(b, start, rich + 8 - start)
+
+ coff = pe + 4
+ sections = u16(b, coff + 2)
+ opt_size = u16(b, coff + 16)
+ blank(b, coff + 4, 4) # TimeDateStamp
+
+ opt = coff + 20
+ magic = u16(b, opt)
+ if magic == 0x10B:
+ data_dirs = opt + 96
+ elif magic == 0x20B:
+ data_dirs = opt + 112
+ else:
+ raise NotPE("unknown optional header magic 0x%x" % magic)
+ blank(b, opt + 64, 4) # CheckSum
+ headers_end = u32(b, opt + 60) # SizeOfHeaders
+ if not pe < headers_end <= len(b):
+ raise NotPE("SizeOfHeaders 0x%x out of range" % headers_end)
+
+ table = opt + opt_size
+ spans = []
+ for i in range(sections):
+ s = table + 40 * i
+ spans.append((u32(b, s + 12), u32(b, s + 8), u32(b, s + 20))) # va, vsize, raw
+
+ def file_offset(rva):
+ for va, vsize, raw in spans:
+ if va <= rva < va + vsize:
+ return raw + rva - va
+ raise NotPE("RVA 0x%x lies in no section" % rva)
+
+ debug_rva = u32(b, data_dirs + 6 * 8)
+ debug_size = u32(b, data_dirs + 6 * 8 + 4)
+ if debug_rva and debug_size:
+ base = file_offset(debug_rva)
+ for i in range(debug_size // DEBUG_ENTRY_SIZE):
+ entry = base + DEBUG_ENTRY_SIZE * i
+ blank(b, entry + 4, 4) # TimeDateStamp
+ kind = u32(b, entry + 12)
+ size = u32(b, entry + 16)
+ raw = u32(b, entry + 24)
+ if kind == DEBUG_TYPE_REPRO:
+ blank(b, raw, size) # the hash itself
+ elif kind == DEBUG_TYPE_CODEVIEW and b[raw:raw + 4] == b"RSDS":
+ blank(b, raw + 4, 20) # PDB GUID and age
+ except struct.error as e:
+ raise NotPE(str(e))
+ # Compare what the padding surrounds, wherever it ends (see above). The Rich header
+ # is blanked by now, so it goes with the padding after the DOS stub; e_lfanew goes
+ # too, since it only says where that padding ends.
+ return (bytes(b[:0x3C]) + bytes(b[0x40:pe]).rstrip(b"\0")
+ + bytes(b[pe:headers_end]).rstrip(b"\0") + bytes(b[headers_end:]))
+
+
+def main(argv):
+ if len(argv) != 3:
+ print(__doc__.strip().splitlines()[2], file=sys.stderr)
+ return 2
+ try:
+ images = []
+ for path in argv[1:]:
+ with open(path, "rb") as f:
+ images.append(normalize(f.read()))
+ except (OSError, NotPE) as e:
+ print("same-pe-code: %s" % e, file=sys.stderr)
+ return 2
+ return 0 if images[0] == images[1] else 1
+
+
+if __name__ == "__main__":
+ sys.exit(main(sys.argv))
diff --git a/.github/scripts/test-refresh-launchers.sh b/.github/scripts/test-refresh-launchers.sh
new file mode 100755
index 0000000..50c1e42
--- /dev/null
+++ b/.github/scripts/test-refresh-launchers.sh
@@ -0,0 +1,121 @@
+#!/bin/bash
+#
+# The contents of this file are subject to the terms of the Common Development and
+# Distribution License (the License). You may not use this file except in compliance with the
+# License.
+#
+# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
+# specific language governing permission and limitations under the License.
+#
+# When distributing Covered Software, include this CDDL Header Notice in each file and include
+# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
+# Header, with the fields enclosed by brackets [] replaced by your own identifying
+# information: "Portions copyright [year] [name of copyright owner]".
+#
+# Copyright 2026 3A Systems, LLC.
+
+# Checks same-pe-code.py and refresh-launchers.sh against launchers committed in the
+# history of master, so that neither can start keeping a changed launcher, or refreshing
+# one that differs only in the toolchain stamp, while CI stays green. Needs the full
+# history (fetch-depth: 0). Run from the root of the repository.
+set -e
+here=$(dirname "$0")
+t=$(mktemp -d)
+trap 'rm -rf "$t"' EXIT
+
+g() { git show "$1:opendj-server-legacy/lib/$2.exe" > "$t/$2-$3.exe"; }
+expect() {
+ local rc=0
+ python3 "$here/same-pe-code.py" "$t/$2" "$t/$3" || rc=$?
+ if [ "$rc" != "$1" ]; then
+ echo "::error::same-pe-code.py $2 $3 exited $rc, expected $1 ($4)"
+ exit 1
+ fi
+ echo "ok: $2 $3 -> $rc ($4)"
+}
+
+for f in launcher_administrator winlauncher opendj_service; do
+ g a611c10c46^ $f 36252
+ g a611c10c46 $f 36256
+ g a8a18f000d $f 36257
+ g 74cb4f5a84 $f old-code
+ expect 0 $f-36256.exe $f-36257.exe "the toolchain stamp only"
+ expect 0 $f-36257.exe $f-36256.exe "the toolchain stamp only"
+ expect 1 $f-old-code.exe $f-36257.exe "before and after a change of the launcher sources"
+done
+# winlauncher.exe has CheckSum 0 in both builds, launcher_administrator.exe does not.
+for f in launcher_administrator winlauncher; do
+ expect 0 $f-36252.exe $f-36256.exe "the same code, the PE header at 0x100 and at 0xf0"
+done
+expect 1 opendj_service-36252.exe opendj_service-36256.exe "a code change of the 05.09 refresh"
+expect 1 winlauncher-36257.exe launcher_administrator-36257.exe "two different launchers"
+# A real change moves the headers too; this one does not, so the sections must be compared.
+python3 - "$t/winlauncher-36257.exe" "$t/winlauncher-text-byte.exe" <<'EOF'
+import struct, sys
+b = bytearray(open(sys.argv[1], "rb").read())
+pe = struct.unpack_from("<I", b, 0x3C)[0]
+table = pe + 24 + struct.unpack_from("<H", b, pe + 20)[0]
+b[struct.unpack_from("<I", b, table + 20)[0] + 16] ^= 1 # the first section: .text
+open(sys.argv[2], "wb").write(b)
+EOF
+expect 1 winlauncher-36257.exe winlauncher-text-byte.exe "one byte of .text flipped"
+# A linker flag, the DOS stub or a section flag changes the headers only, so they must be
+# compared as well.
+for field in nx stub section; do
+ python3 - "$t/winlauncher-36257.exe" "$t/winlauncher-$field.exe" $field <<'EOF'
+import struct, sys
+b = bytearray(open(sys.argv[1], "rb").read())
+pe = struct.unpack_from("<I", b, 0x3C)[0]
+table = pe + 24 + struct.unpack_from("<H", b, pe + 20)[0]
+off = {"nx": pe + 24 + 71, # DllCharacteristics, high byte: NX_COMPAT
+ "stub": 0x4E, # "This program cannot be run in DOS mode."
+ "section": table + 39}[sys.argv[3]] # the first section's Characteristics, top byte
+b[off] ^= 1
+open(sys.argv[2], "wb").write(b)
+EOF
+ expect 1 winlauncher-36257.exe winlauncher-$field.exe "$field: a change of the headers only"
+done
+# No committed launcher has a PDB. Retype the POGO debug entry as CodeView in two copies
+# whose PDB GUID and age differ, and nothing else.
+for n in 1 2; do
+ python3 - "$t/winlauncher-36257.exe" "$t/winlauncher-rsds$n.exe" $n <<'EOF'
+import struct, sys
+b = bytearray(open(sys.argv[1], "rb").read())
+pe = struct.unpack_from("<I", b, 0x3C)[0]
+opt = pe + 24
+dirs = opt + (96 if struct.unpack_from("<H", b, opt)[0] == 0x10B else 112)
+table = opt + struct.unpack_from("<H", b, pe + 20)[0]
+rva = struct.unpack_from("<I", b, dirs + 6 * 8)[0] # the debug directory
+for i in range(struct.unpack_from("<H", b, pe + 6)[0]):
+ va, vsize, raw = (struct.unpack_from("<I", b, table + 40 * i + o)[0] for o in (12, 8, 20))
+ if va <= rva < va + vsize:
+ entry = raw + rva - va
+while struct.unpack_from("<I", b, entry + 12)[0] != 13: # the POGO entry
+ entry += 28
+struct.pack_into("<I", b, entry + 12, 2) # retyped as CodeView
+data = struct.unpack_from("<I", b, entry + 24)[0]
+b[data:data + 24] = b"RSDS" + bytes([int(sys.argv[3])]) * 20 # GUID and age differ
+open(sys.argv[2], "wb").write(b)
+EOF
+done
+expect 0 winlauncher-rsds1.exe winlauncher-rsds2.exe "the PDB GUID and age of a CodeView entry"
+head -c 64 "$t/winlauncher-36257.exe" > "$t/truncated.exe"
+expect 2 truncated.exe winlauncher-36257.exe "a file that is not a PE image"
+
+# The loop: keeps a launcher that differs only in the stamp, refreshes one whose code
+# changed, adds one that is not committed yet, and refreshes one whose committed file
+# is not a PE image.
+mkdir "$t/built" "$t/lib"
+cp "$t/winlauncher-36257.exe" "$t/built/winlauncher.exe"
+cp "$t/winlauncher-36256.exe" "$t/lib/winlauncher.exe"
+cp "$t/launcher_administrator-36257.exe" "$t/built/launcher_administrator.exe"
+cp "$t/launcher_administrator-old-code.exe" "$t/lib/launcher_administrator.exe"
+cp "$t/opendj_service-36257.exe" "$t/built/opendj_service.exe"
+cp "$t/winlauncher-36257.exe" "$t/built/unreadable.exe"
+cp "$t/truncated.exe" "$t/lib/unreadable.exe"
+bash "$here/refresh-launchers.sh" "$t/built" "$t/lib"
+cmp "$t/lib/winlauncher.exe" "$t/winlauncher-36256.exe"
+cmp "$t/lib/launcher_administrator.exe" "$t/built/launcher_administrator.exe"
+cmp "$t/lib/opendj_service.exe" "$t/built/opendj_service.exe"
+cmp "$t/lib/unreadable.exe" "$t/built/unreadable.exe"
+echo "ok: refresh-launchers.sh keeps, refreshes and adds as expected"
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 11e941e..4c54e32 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -56,6 +56,12 @@
with:
fetch-depth: 0
submodules: recursive
+ # deploy.yml runs these two scripts only after a push build, so this is the check they
+ # get before merge. It needs the history above: it compares launchers committed on master.
+ - name: Check same-pe-code.py and refresh-launchers.sh against committed launchers
+ if: runner.os == 'Linux' && matrix.java == '11'
+ shell: bash
+ run: bash .github/scripts/test-refresh-launchers.sh
- name: Java ${{ matrix.Java }} (${{ matrix.os }})
uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1
with:
@@ -86,9 +92,10 @@
git status
# Also the source of truth for the committed opendj-server-legacy/lib/*.exe: on a
# successful push build, deploy.yml downloads windows-exe-11 from this very run and
- # commits its contents back to the branch. Nothing here compares them with what is
- # committed - an MSVC toolchain bump on the runner image changes the bytes on its own,
- # so a byte-for-byte gate would fire without a source change.
+ # commits each launcher that differs from the committed one in more than the toolchain
+ # stamp (.github/scripts/refresh-launchers.sh). Nothing here compares them with what
+ # is committed - an MSVC toolchain bump on the runner image changes the bytes on its
+ # own, so a byte-for-byte gate would fire without a source change.
- name: Upload Windows exe artifacts
if: runner.os == 'Windows'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index f1baf9d..6d4b5bb 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -76,10 +76,17 @@
#
# This only works because the Makefile passes /Brepro to both cl and link: the output
# is a function of the sources, not of the build time. Without it every run would
- # produce different bytes and this would commit on every push. An MSVC toolchain bump
- # on the runner image does change them, and that refresh commit is correct - the
- # committed binary then matches what CI verifies. Pushes made with GITHUB_TOKEN do
- # not start new workflow runs, so this cannot loop; a PAT would break that.
+ # produce different bytes and this would commit on every push. The inputs it hashes
+ # include the build numbers of the tools, though, so a Visual Studio patch release
+ # on the runner image changes the stamp of the files - Rich header, REPRO hash,
+ # timestamps, checksum, header padding - while the code comes out the same. GitHub
+ # rolls a new image out over days, and while old and new both serve windows-latest
+ # the launchers were refreshed back and forth on every push (a8a18f000d, then
+ # c6919eaaf4, which undid it). refresh-launchers.sh compares the files with that
+ # stamp left out (same-pe-code.py), so only a change of the code, data or resources
+ # is committed - from a source change or from a toolchain that really generates
+ # different code. Pushes made with GITHUB_TOKEN do not start new workflow runs, so
+ # this cannot loop; a PAT would break that.
- name: Download the launchers built by the triggering Build run
continue-on-error: true
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
@@ -103,7 +110,13 @@
echo "::warning title=No launcher binaries from the Build run::windows-exe-11 could not be downloaded, leaving opendj-server-legacy/lib/*.exe as committed."
exit 0
fi
- cp "$BUILT"/*.exe opendj-server-legacy/lib/
+ # A checked-out branch that predates the script (sustaining/4.10.x, say) takes
+ # the rebuilt files as they are, which is the old behaviour.
+ if [ -f .github/scripts/refresh-launchers.sh ]; then
+ bash .github/scripts/refresh-launchers.sh "$BUILT" opendj-server-legacy/lib
+ else
+ cp "$BUILT"/*.exe opendj-server-legacy/lib/
+ fi
# status --porcelain, not diff: it reports a brand-new launcher that was never
# git-added just as well as a modified one.
if [ -z "$(git status --porcelain -- opendj-server-legacy/lib)" ]; then
diff --git a/opendj-server-legacy/src/build-tools/windows/Makefile b/opendj-server-legacy/src/build-tools/windows/Makefile
index 2f73f59..6399ebf 100644
--- a/opendj-server-legacy/src/build-tools/windows/Makefile
+++ b/opendj-server-legacy/src/build-tools/windows/Makefile
@@ -39,8 +39,9 @@
WINLAUNCHER_PROGNAME=winlauncher.exe
# /Brepro makes the outputs reproducible (content-hash PE timestamps instead of the
# build time). The Package/Deploy workflow commits these binaries back to the branch
-# whenever their bytes differ from the committed ones; without /Brepro every build
-# would differ and it would commit on every push.
+# whenever they differ from the committed ones in more than the toolchain stamp
+# (.github/scripts/same-pe-code.py); without /Brepro every build would differ and it
+# would commit on every push.
LINKER=link -nologo /machine:x86 /Brepro
LIBS=advapi32.lib
--
Gitblit v1.10.0