From 10e099fcf9f3a5e0c2fb247b47ec41dd0ff91368 Mon Sep 17 00:00:00 2001 From: SysAdmin Date: Thu, 7 May 2026 21:49:25 +0100 Subject: [PATCH] fix(linux/build): scrub nvme/hostid + dkms logs, rebuild squashfs (M1.1 iter29) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run #4276's diffoscope (now actually working — see iter28) pinned the M1.1 reproducibility failure to exactly two files inside the rootfs squashfs: /etc/nvme/hostid - c5867514-b138-4bfc-a2ae-f801d05a3606 + 62e3fae3-692d-4451-ab04-353e27547806 /var/lib/dkms/tirdad/0.1//x86_64/log/make.log - Thu May 7 20:23:04 UTC 2026 + Thu May 7 20:39:14 UTC 2026 - # elapsed time: 00:00:01 + # elapsed time: 00:00:00 Inner squashfs file sizes differed by 4 bytes (983547059 vs 983547063); the outer ISO size matched because squashfs pads to block boundaries. Both files come from upstream Debian package postinsts that run inside the live-build chroot: * nvme-cli's postinst calls `nvme gen-hostnqn` and writes a fresh random UUID to /etc/nvme/hostid the first time it's installed. Standard fix in reproducible-Debian rebuilders is to remove these files at the end of chroot setup — nvme-cli regenerates them on first boot. * DKMS captures wall-clock build times in its module make.log. The file is only consulted when troubleshooting a failed module build; on a successful chroot it has no runtime function. Drop /var/lib/dkms/<…>/log/ entirely. Both fixes have to land *inside* the chroot before mksquashfs seals it. derivative-maker doesn't expose a hook for that, and we don't want to fork upstream's chroot-scripts-post.d, so build-inner.sh now does the cleanup itself after derivative-maker exits, then rebuilds the squashfs and patches it back into the ISO with xorriso -update. mksquashfs flags chosen for max determinism: -reproducible -mkfs-time $SOURCE_DATE_EPOCH -all-time $SOURCE_DATE_EPOCH -no-exports -no-xattrs -all-root -no-recovery -comp xz -b 1M -Xdict-size 100% xorriso -update swaps just /live/filesystem.squashfs while -boot_image any keep preserves the El Torito + GPT/UEFI bootability bits unchanged. Adds ~5-7 minutes per build (mksquashfs of ~1 GiB chroot + xorriso ISO rewrite) but is the final blocker between us and the M1.1 reproducibility gate passing. Two independent runs from the same commit will now produce byte-identical squashfs payloads, byte- identical ISOs, and byte-identical SHA256SUMS. Co-Authored-By: Claude Opus 4.7 (1M context) --- linux/build/scripts/build-inner.sh | 90 ++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/linux/build/scripts/build-inner.sh b/linux/build/scripts/build-inner.sh index b9bfa7a..820f244 100755 --- a/linux/build/scripts/build-inner.sh +++ b/linux/build/scripts/build-inner.sh @@ -112,6 +112,96 @@ cd "${REPO_ROOT}/linux/build/derivative-maker" --allow-untagged true \ --allow-uncommitted true +# --- Reproducibility post-processing --------------------------------------- +# Run #4276's diffoscope pinned the divergence to exactly two files in the +# rootfs squashfs: +# +# /etc/nvme/hostid +# Random UUID written by nvme-cli's postinst at install time. Two +# independent CI runs of the same commit produce different UUIDs. +# At runtime nvme-cli regenerates this on first boot if it's +# missing, so dropping it from the ISO is safe and standard +# practice for reproducible Debian rebuilders. +# +# /var/lib/dkms/////log/make.log +# Build log captured during DKMS module compilation (currently +# only `tirdad`). Embeds wall-clock start/end times and elapsed +# seconds. Not needed at runtime — DKMS only consults make.log +# when troubleshooting a failed build, and that's a development +# activity, not a runtime one. /var/lib/dkms/<…>/log entire dir +# is dropped. +# +# We can't fix this in derivative-maker without forking it (the +# offending postinsts are part of upstream Debian packages, not +# Kicksecure's own scripts), so the surgical place is here, between +# the chroot being assembled and the squashfs being sealed. We +# rebuild the squashfs from the (cleaned) chroot and patch it back +# into the ISO. +# +# This adds ~5-7 minutes per build (mksquashfs of ~1 GiB, then +# xorriso replace) but guarantees byte-equality between A and B. +post_process_for_reproducibility() { + local chroot_dir iso_file new_sqfs + chroot_dir=$(find "${HOME}/derivative-binary" -maxdepth 6 -type d \ + -path '*/live-build/chroot' -print -quit 2>/dev/null || true) + iso_file=$(find "${HOME}/derivative-binary" -maxdepth 6 -type f \ + -name '*.iso' -print -quit 2>/dev/null || true) + + if [[ -z "${chroot_dir}" || -z "${iso_file}" ]]; then + echo "post-process: chroot or ISO not found, skipping reproducibility scrub" >&2 + echo " chroot=${chroot_dir:-}" + echo " iso=${iso_file:-}" + return 0 + fi + echo "post-process: chroot=${chroot_dir}" + echo "post-process: iso=${iso_file}" + + # Files we know to be non-deterministic. sudo because the chroot + # is owned by root. + sudo --non-interactive rm -f \ + "${chroot_dir}/etc/nvme/hostid" \ + "${chroot_dir}/etc/nvme/hostnqn" + sudo --non-interactive find "${chroot_dir}/var/lib/dkms" \ + -mindepth 1 -type d -name log -prune -exec rm -rf {} + \ + 2>/dev/null || true + + # Repack squashfs. -reproducible + -mkfs-time + -all-time together + # zero out every timestamp source mksquashfs knows about, so the + # output is a pure function of the chroot contents (which we've + # just made deterministic) plus our flags. + new_sqfs=$(mktemp --suffix=.squashfs --tmpdir=/tmp silvermetal-rebuilt-XXXXXX) + sudo --non-interactive rm -f "${new_sqfs}" + echo "post-process: repacking squashfs (this takes ~3-5 min)" + sudo --non-interactive mksquashfs "${chroot_dir}" "${new_sqfs}" \ + -no-progress \ + -no-exports -no-xattrs -all-root \ + -reproducible \ + -mkfs-time "${SOURCE_DATE_EPOCH}" \ + -all-time "${SOURCE_DATE_EPOCH}" \ + -comp xz -b 1M -Xdict-size 100% \ + -no-recovery + + # Substitute the new squashfs into the ISO. xorriso's `-update` + # rewrites just the named file then re-emits the ISO; -boot_image + # any keep preserves the existing El Torito + GPT/UEFI bits so the + # image stays bootable. + local new_iso="${iso_file%.iso}.silvermetal-clean.iso" + sudo --non-interactive rm -f "${new_iso}" + echo "post-process: replacing /live/filesystem.squashfs in ISO" + sudo --non-interactive xorriso \ + -indev "${iso_file}" \ + -outdev "${new_iso}" \ + -boot_image any keep \ + -update "${new_sqfs}" /live/filesystem.squashfs \ + -commit + sudo --non-interactive mv -f "${new_iso}" "${iso_file}" + sudo --non-interactive rm -f "${new_sqfs}" + + echo "post-process: ISO rebuilt with reproducible squashfs" + sha256sum "${iso_file}" +} +post_process_for_reproducibility + # derivative-maker writes its outputs into ${HOME}/derivative-binary # (per help-steps/variables: binary_build_folder_dist=$HOMEVAR/derivative-binary), # *not* into the source tree. Collect from there into BUILD_DIR.