From e2f7450cea559ad23c2879d1310b66839cd2ecf3 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 14 Aug 2026 15:36:21 +0530 Subject: [PATCH 1/4] FIX: eliminate Debian ARM64 QEMU rseq segfault in PR validation CI The Linux ARM64 legs run under QEMU user-mode emulation on x86_64 agents. Older QEMU (multiarch/qemu-user-static) mishandles glibc restartable sequences (rseq), randomly SIGSEGV-ing (exit 139) the emulated arm64 Python during Debian's apt byte-compilation and forcing repeated job reruns (build 166395's Debian_ARM64 job needed 5 attempts). Two independent, root-cause guards scoped to the flaky Debian/Ubuntu ARM64 job: - Emulator level: pin a modern QEMU (tonistiigi/binfmt qemu-v9.2.2), which fixes rseq emulation (QEMU >= 7.2). - Guest level: disable rseq on the Debian container via GLIBC_TUNABLES=glibc.pthread.rseq=0. No retryCountOnTaskFailure, no skipped tests. RHEL9 ARM64 and Alpine legs are untouched. AB#47276 --- eng/pipelines/pr-validation-pipeline.yml | 29 ++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index f0323fb0..be5e7f62 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -1019,16 +1019,41 @@ jobs: steps: - script: | - # Set up Docker buildx for multi-architecture support - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + # Set up Docker buildx for multi-architecture support. + # + # Register a modern, pinned QEMU (tonistiigi/binfmt qemu-v9.2.2) instead + # of the older multiarch/qemu-user-static image. Older QEMU user-mode + # emulation mishandles glibc restartable sequences (rseq), randomly + # SIGSEGV-ing (exit 139) the emulated arm64 Python during Debian's apt + # byte-compilation and forcing repeated job reruns. QEMU >= 7.2 fixes + # rseq emulation, eliminating the crash at the emulator level. Together + # with the GLIBC_TUNABLES rseq disable on the Debian container below, + # this gives two independent guards so the segfault can no longer occur. + docker run --rm --privileged tonistiigi/binfmt:qemu-v9.2.2 --install arm64 docker buildx create --name multiarch --driver docker-container --use docker buildx inspect --bootstrap displayName: 'Setup Docker buildx for ARM64 emulation' - script: | # Create a Docker container for testing on ARM64 + # + # Debian 12 (glibc 2.36) intermittently segfaults (exit 139) during + # `apt-get install` post-install scripts when the emulated arm64 Python + # interpreter is invoked for byte-compilation (python3-wheel, + # python3-pyparsing, python3.11-venv, ...). Root cause: QEMU arm64 + # user-mode emulation on the x86_64 agent mishandles glibc restartable + # sequences (rseq), which modern glibc uses by default. Disabling rseq + # via GLIBC_TUNABLES removes the trigger with no functional impact (rseq + # is only a perf optimization). Set at container creation so every + # subsequent `docker exec` inherits it. Ubuntu 22.04 (glibc 2.35) is not + # affected in practice, so this is scoped strictly to the Debian leg. + EXTRA_ENV="" + if [ "$(distroName)" = "Debian" ]; then + EXTRA_ENV="-e GLIBC_TUNABLES=glibc.pthread.rseq=0" + fi docker run -d --name test-container-$(distroName)-$(archName) \ --platform linux/arm64 \ + $EXTRA_ENV \ -v $(Build.SourcesDirectory):/workspace \ -w /workspace \ --network bridge \ From 53689984df304a566cba3ae894222f8b999d4232 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 14 Aug 2026 15:48:57 +0530 Subject: [PATCH 2/4] FIX: reset stale arm64 binfmt handler before installing pinned QEMU Address PR review: bare `--install arm64` is a no-op when the hosted agent already has an arm64 handler registered (kernel returns EEXIST and tonistiigi/binfmt leaves the existing, possibly older QEMU in place, logging but not failing). That could silently keep the old buggy QEMU on the Ubuntu_ARM64 leg, which has no guest-side GLIBC_TUNABLES guard. tonistiigi/binfmt has no `--reset` flag (that was multiarch/qemu-user- static); the equivalent is `--uninstall arm64 --install arm64` (uninstall runs before install in run()). `--uninstall arm64` resolves through the arch config to the qemu-aarch64 handler via the -aarch64 suffix match and is a harmless no-op when nothing is registered. Also reword the container-creation comment to clarify that only the GLIBC_TUNABLES/rseq-disable guard is Debian-scoped; the ARM64 emulation is shared by both matrix legs. AB#47276 --- eng/pipelines/pr-validation-pipeline.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index be5e7f62..daceea83 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -1029,7 +1029,19 @@ jobs: # rseq emulation, eliminating the crash at the emulator level. Together # with the GLIBC_TUNABLES rseq disable on the Debian container below, # this gives two independent guards so the segfault can no longer occur. - docker run --rm --privileged tonistiigi/binfmt:qemu-v9.2.2 --install arm64 + # + # Uninstall any arm64 handler the agent may already have registered + # before installing this pinned image's QEMU. tonistiigi/binfmt's + # `--install` is a no-op if a handler is already registered (the kernel + # returns EEXIST and binfmt leaves the existing one in place), so without + # the uninstall we could silently keep the agent's older QEMU and defeat + # the fix on the Ubuntu leg. This mirrors the old `--reset`; note that + # tonistiigi/binfmt has no `--reset` flag (that belonged to + # multiarch/qemu-user-static) -- uninstall+install is the equivalent. + # `--uninstall arm64` maps through binfmt's arch config to the + # `qemu-aarch64` handler (matched by the `-aarch64` suffix) and is a + # harmless no-op when nothing is registered. + docker run --rm --privileged tonistiigi/binfmt:qemu-v9.2.2 --uninstall arm64 --install arm64 docker buildx create --name multiarch --driver docker-container --use docker buildx inspect --bootstrap displayName: 'Setup Docker buildx for ARM64 emulation' @@ -1046,7 +1058,9 @@ jobs: # via GLIBC_TUNABLES removes the trigger with no functional impact (rseq # is only a perf optimization). Set at container creation so every # subsequent `docker exec` inherits it. Ubuntu 22.04 (glibc 2.35) is not - # affected in practice, so this is scoped strictly to the Debian leg. + # affected in practice, so only this GLIBC_TUNABLES/rseq-disable guard is + # scoped to the Debian leg -- the ARM64 emulation itself is shared by + # both matrix legs. EXTRA_ENV="" if [ "$(distroName)" = "Debian" ]; then EXTRA_ENV="-e GLIBC_TUNABLES=glibc.pthread.rseq=0" From 0c22825a8d7d9758830ed0f4f4bb6a25b0fff6c7 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 14 Aug 2026 15:54:48 +0530 Subject: [PATCH 3/4] REFACTOR: simplify ARM64 rseq fix to a single guest-side guard Drop the emulator-side QEMU swap (pinned tonistiigi/binfmt + uninstall/install) and revert the buildx step to the original multiarch/qemu-user-static registration. The crash mechanism is glibc rseq; disabling it in the guest via GLIBC_TUNABLES=glibc.pthread.rseq=0 neutralizes the root cause deterministically, without depending on binfmt registration order or agent pre-registration state. Apply the guard unconditionally to both ARM64 matrix legs (Debian 12 / glibc 2.36 and Ubuntu 22.04 / glibc 2.35) -- both share the same emulated-arm64 path and both glibc versions ship rseq -- which also removes the Ubuntu-leg exposure that motivated the emulator swap. Net change vs main is now a single env var; no new image dependency. AB#47276 --- eng/pipelines/pr-validation-pipeline.yml | 53 ++++++------------------ 1 file changed, 13 insertions(+), 40 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index daceea83..f9c97aab 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -1019,29 +1019,8 @@ jobs: steps: - script: | - # Set up Docker buildx for multi-architecture support. - # - # Register a modern, pinned QEMU (tonistiigi/binfmt qemu-v9.2.2) instead - # of the older multiarch/qemu-user-static image. Older QEMU user-mode - # emulation mishandles glibc restartable sequences (rseq), randomly - # SIGSEGV-ing (exit 139) the emulated arm64 Python during Debian's apt - # byte-compilation and forcing repeated job reruns. QEMU >= 7.2 fixes - # rseq emulation, eliminating the crash at the emulator level. Together - # with the GLIBC_TUNABLES rseq disable on the Debian container below, - # this gives two independent guards so the segfault can no longer occur. - # - # Uninstall any arm64 handler the agent may already have registered - # before installing this pinned image's QEMU. tonistiigi/binfmt's - # `--install` is a no-op if a handler is already registered (the kernel - # returns EEXIST and binfmt leaves the existing one in place), so without - # the uninstall we could silently keep the agent's older QEMU and defeat - # the fix on the Ubuntu leg. This mirrors the old `--reset`; note that - # tonistiigi/binfmt has no `--reset` flag (that belonged to - # multiarch/qemu-user-static) -- uninstall+install is the equivalent. - # `--uninstall arm64` maps through binfmt's arch config to the - # `qemu-aarch64` handler (matched by the `-aarch64` suffix) and is a - # harmless no-op when nothing is registered. - docker run --rm --privileged tonistiigi/binfmt:qemu-v9.2.2 --uninstall arm64 --install arm64 + # Set up Docker buildx for multi-architecture support + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes docker buildx create --name multiarch --driver docker-container --use docker buildx inspect --bootstrap displayName: 'Setup Docker buildx for ARM64 emulation' @@ -1049,25 +1028,19 @@ jobs: - script: | # Create a Docker container for testing on ARM64 # - # Debian 12 (glibc 2.36) intermittently segfaults (exit 139) during - # `apt-get install` post-install scripts when the emulated arm64 Python - # interpreter is invoked for byte-compilation (python3-wheel, - # python3-pyparsing, python3.11-venv, ...). Root cause: QEMU arm64 - # user-mode emulation on the x86_64 agent mishandles glibc restartable - # sequences (rseq), which modern glibc uses by default. Disabling rseq - # via GLIBC_TUNABLES removes the trigger with no functional impact (rseq - # is only a perf optimization). Set at container creation so every - # subsequent `docker exec` inherits it. Ubuntu 22.04 (glibc 2.35) is not - # affected in practice, so only this GLIBC_TUNABLES/rseq-disable guard is - # scoped to the Debian leg -- the ARM64 emulation itself is shared by - # both matrix legs. - EXTRA_ENV="" - if [ "$(distroName)" = "Debian" ]; then - EXTRA_ENV="-e GLIBC_TUNABLES=glibc.pthread.rseq=0" - fi + # The arm64 test image runs under QEMU user-mode emulation on the x86_64 + # agent. QEMU mishandles glibc restartable sequences (rseq) -- which + # modern glibc enables by default -- causing random SIGSEGVs (exit 139) + # when the emulated arm64 Python is invoked for apt byte-compilation + # during dependency installation. Disabling rseq via GLIBC_TUNABLES + # removes the trigger with no functional impact (rseq is only a perf + # optimization). Set at container creation so every subsequent + # `docker exec` inherits it. Applied to both matrix legs (Debian 12 / + # glibc 2.36 and Ubuntu 22.04 / glibc 2.35) since both share the same + # emulated-arm64 path and both glibc versions ship rseq. docker run -d --name test-container-$(distroName)-$(archName) \ --platform linux/arm64 \ - $EXTRA_ENV \ + -e GLIBC_TUNABLES=glibc.pthread.rseq=0 \ -v $(Build.SourcesDirectory):/workspace \ -w /workspace \ --network bridge \ From 145e29039a33aae18d7a4b825ad5e13c62a90677 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 14 Aug 2026 17:12:02 +0530 Subject: [PATCH 4/4] FIX: use preinstalled-python image for Debian ARM64 to avoid QEMU segfault The Debian ARM64 leg SIGSEGVs (exit 139) during the apt 'Setting up python3' post-install byte-compilation under QEMU user-mode emulation (proven by build 167144, which ran the prior guest-side GLIBC_TUNABLES=glibc.pthread.rseq=0 guard and still crashed). Disabling rseq does not fix this emulator-level bug. Instead, avoid the crashing operation entirely: run the Debian leg on the python:3.11-bookworm image (Debian 12 + Python 3.11 already installed and byte-compiled natively at image-build time) and stop apt-installing the python3* packages. Python (pip, venv, dev headers) comes from /usr/local, pybind11 is installed via pip from requirements.txt, and only non-Python build tools (cmake, curl, wget, gnupg, build-essential) are pulled from apt -- none of which depend on the Debian python3 package. The ineffective rseq guard is removed. AB#47276 --- eng/pipelines/pr-validation-pipeline.yml | 26 +++++++++++------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index f9c97aab..846249c2 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -1013,7 +1013,11 @@ jobs: distroName: 'Ubuntu' archName: 'arm64' Debian_ARM64: - dockerImage: 'debian:12' + # Python 3.11 is preinstalled (byte-compiled natively at image-build + # time) so we never run the apt python3 post-install byte-compilation + # that SIGSEGVs (exit 139) under QEMU user-mode emulation. Still Debian + # 12 (bookworm) underneath, so the msodbcsql18 debian/12 repo applies. + dockerImage: 'python:3.11-bookworm' distroName: 'Debian' archName: 'arm64' @@ -1027,20 +1031,8 @@ jobs: - script: | # Create a Docker container for testing on ARM64 - # - # The arm64 test image runs under QEMU user-mode emulation on the x86_64 - # agent. QEMU mishandles glibc restartable sequences (rseq) -- which - # modern glibc enables by default -- causing random SIGSEGVs (exit 139) - # when the emulated arm64 Python is invoked for apt byte-compilation - # during dependency installation. Disabling rseq via GLIBC_TUNABLES - # removes the trigger with no functional impact (rseq is only a perf - # optimization). Set at container creation so every subsequent - # `docker exec` inherits it. Applied to both matrix legs (Debian 12 / - # glibc 2.36 and Ubuntu 22.04 / glibc 2.35) since both share the same - # emulated-arm64 path and both glibc versions ship rseq. docker run -d --name test-container-$(distroName)-$(archName) \ --platform linux/arm64 \ - -e GLIBC_TUNABLES=glibc.pthread.rseq=0 \ -v $(Build.SourcesDirectory):/workspace \ -w /workspace \ --network bridge \ @@ -1102,13 +1094,19 @@ jobs: " else # Debian ARM64 + # Python (with pip, venv and dev headers) is already provided by the + # python:3.11-bookworm base image under /usr/local, so we do NOT + # apt-install python3* here -- the apt python3 post-install byte- + # compilation is what SIGSEGVs (exit 139) under QEMU emulation. pybind11 + # comes from pip (requirements.txt); only non-Python build tools are + # pulled from apt, none of which depend on the Debian python3 package. docker exec test-container-$(distroName)-$(archName) bash -c " set -euo pipefail export DEBIAN_FRONTEND=noninteractive export TZ=UTC ln -snf /usr/share/zoneinfo/\$TZ /etc/localtime && echo \$TZ > /etc/timezone apt-get update - apt-get install -y python3 python3-pip python3-venv python3-full cmake curl wget gnupg software-properties-common build-essential python3-dev pybind11-dev + apt-get install -y cmake curl wget gnupg build-essential # Verify architecture uname -m dpkg --print-architecture