From 4907dbecf23a3f3e68579a8138530d81d5a9f62c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Loipf=C3=BChrer?= Date: Fri, 2 Oct 2026 10:54:49 +0200 Subject: [PATCH 01/48] feat(cli): rework build / test environment handling Adds an explicit environment (env) subcommand which allows users to list and interact with existing build / test environments. Additionally contains extensive internal refactoring to make the environment tracking and cleanup more robust. --- .github/workflows/ci.yaml | 22 +- Cargo.lock | 529 ++++- Cargo.toml | 6 +- README.md | 4 +- debian/control | 7 + debian/rules.py | 24 +- docs/index.md | 3 +- docs/usage/build.md | 31 +- docs/usage/config.md | 11 +- docs/usage/env.md | 68 + docs/usage/test.md | 30 +- packages/debmagic/CHANGELOG.md | 1 + packages/debmagic/Cargo.toml | 1 + packages/debmagic/README.md | 10 +- packages/debmagic/src/build/attach.rs | 93 - packages/debmagic/src/build/mod.rs | 494 ++--- packages/debmagic/src/build/source.rs | 11 +- packages/debmagic/src/build_intent.rs | 71 +- packages/debmagic/src/cli.rs | 102 +- packages/debmagic/src/config.rs | 54 +- packages/debmagic/src/data_dir.rs | 86 + packages/debmagic/src/driver/config.rs | 44 +- packages/debmagic/src/driver/driver_bare.rs | 32 +- packages/debmagic/src/driver/driver_docker.rs | 68 +- packages/debmagic/src/driver/driver_lxd.rs | 97 +- packages/debmagic/src/driver/mod.rs | 196 +- packages/debmagic/src/env.rs | 177 ++ packages/debmagic/src/environment/mod.rs | 1713 +++++++++++++++++ packages/debmagic/src/environment/process.rs | 81 + packages/debmagic/src/environment/registry.rs | 1114 +++++++++++ packages/debmagic/src/main.rs | 56 +- .../debmagic/src/migrations/0001_registry.sql | 55 + packages/debmagic/src/package.rs | 4 +- packages/debmagic/src/test/intent.rs | 42 +- packages/debmagic/src/test/run.rs | 555 +++--- packages/debmagic/tests/assets/config1.toml | 2 +- 36 files changed, 4865 insertions(+), 1029 deletions(-) create mode 100644 docs/usage/env.md delete mode 100644 packages/debmagic/src/build/attach.rs create mode 100644 packages/debmagic/src/data_dir.rs create mode 100644 packages/debmagic/src/env.rs create mode 100644 packages/debmagic/src/environment/mod.rs create mode 100644 packages/debmagic/src/environment/process.rs create mode 100644 packages/debmagic/src/environment/registry.rs create mode 100644 packages/debmagic/src/migrations/0001_registry.sql diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4a5189d0..2c196eb8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -141,6 +141,10 @@ jobs: contents: read # Push of the pre-built CI image is additionally gated to main below. packages: write + env: + # Registry and Environment Host roots together, so the cached build + # tree stays registered under the same Environment id. + DEBMAGIC_DATA_DIR: /tmp/debmagic-data steps: - uses: actions/checkout@v5 - uses: Swatinem/rust-cache@v2 @@ -149,16 +153,18 @@ jobs: - name: Cache the debmagic build tree uses: actions/cache@v4 with: - path: /tmp/debmagic - key: debmagic-build-${{ hashFiles('Cargo.lock', 'debian/changelog', 'debian/control', 'debian/rules') }} - restore-keys: debmagic-build- - - name: Prune stale build trees from the restored cache + path: /tmp/debmagic-data + key: debmagic-data-${{ hashFiles('Cargo.lock', 'debian/changelog', 'debian/control', 'debian/rules') }} + restore-keys: debmagic-data- + - name: Prune Environments of other package versions from the restored cache run: | - mkdir -p /tmp/debmagic current="$(dpkg-parsechangelog -SSource)-$(dpkg-parsechangelog -SVersion)" - find /tmp/debmagic -mindepth 1 -maxdepth 1 -type d \ - ! -name "$current" ! -name "${current}-test" \ - -exec rm -rf {} + + cargo run --locked -q -p debmagic -- env list \ + | awk -v current="$current" 'NR > 1 && $4 != current { print $1 }' \ + | while read -r id; do + cargo run --locked -q -p debmagic -- env clean "$id" \ + || echo "::warning::could not prune Environment $id" + done - name: Resolve CI image tag id: image run: | diff --git a/Cargo.lock b/Cargo.lock index 07ccdad5..b16b7ec5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,6 +11,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -93,6 +99,15 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -183,6 +198,12 @@ version = "3.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.12.1" @@ -286,6 +307,12 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "colorchoice" version = "1.0.4" @@ -397,6 +424,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -416,6 +458,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -447,6 +498,15 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "ctutils" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03bb0e1cc970d482d121d9a1744999169b69a07470b3d644a7894e53fcaf4574" +dependencies = [ + "cmov", +] + [[package]] name = "deb822-derive" version = "0.2.0" @@ -482,7 +542,7 @@ dependencies = [ "log", "rowan", "textwrap", - "whoami", + "whoami 1.6.1", ] [[package]] @@ -523,6 +583,7 @@ dependencies = [ "serde_json", "sha1", "sha2 0.11.0", + "sqlx", "test-case", "tokio", "toml", @@ -577,6 +638,7 @@ dependencies = [ "block-buffer 0.12.1", "const-oid", "crypto-common 0.2.2", + "ctutils", ] [[package]] @@ -620,12 +682,27 @@ dependencies = [ "const-random", ] +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "dunce" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" +dependencies = [ + "serde", +] + [[package]] name = "encoding_rs" version = "0.8.35" @@ -652,18 +729,55 @@ dependencies = [ "typeid", ] +[[package]] +name = "etcetera" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de48cc4d1c1d97a20fd819def54b890cadde72ed3ad0c614822a0a433361be96" +dependencies = [ + "cfg-if", + "windows-sys 0.61.2", +] + +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + [[package]] name = "find-msvc-tools" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "645cbb3a84e60b7531617d5ae4e57f7e27308f6445f5abf653209ea76dec8dff" +[[package]] +name = "flume" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + [[package]] name = "foldhash" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -686,6 +800,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -694,6 +809,28 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + [[package]] name = "futures-io" version = "0.3.34" @@ -702,13 +839,13 @@ checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] name = "futures-macro" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 3.0.3", ] [[package]] @@ -725,9 +862,9 @@ checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-util" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-core", "futures-io", @@ -736,7 +873,6 @@ dependencies = [ "futures-task", "memchr", "pin-project-lite", - "pin-utils", "slab", ] @@ -820,7 +956,18 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "foldhash", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", ] [[package]] @@ -838,12 +985,45 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "hashlink" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824e001ac4f3012dd16a264bec811403a67ca9deb6c102fc5049b32c4574b35f" +dependencies = [ + "hashbrown 0.16.1", +] + [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa26c720c68b866f2c96ef5c1264b3e6f473fe5d4ce61cd44bbe913e553018" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "http" version = "1.5.0" @@ -1239,7 +1419,18 @@ dependencies = [ "bitflags", "libc", "plain", - "redox_syscall", + "redox_syscall 0.9.4", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f111c8c41e7c61a49cd34e44c7619462967221a6443b0ec299e0ac30cfb9b1" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", ] [[package]] @@ -1248,6 +1439,15 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.29" @@ -1349,6 +1549,35 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.18", + "smallvec", + "windows-link", +] + [[package]] name = "pathdiff" version = "0.2.3" @@ -1410,12 +1639,6 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - [[package]] name = "pkg-config" version = "0.3.34" @@ -1550,6 +1773,15 @@ dependencies = [ "rand_core", ] +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + [[package]] name = "redox_syscall" version = "0.9.4" @@ -1807,6 +2039,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "security-framework" version = "3.7.0" @@ -1972,6 +2210,9 @@ name = "smallvec" version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +dependencies = [ + "serde", +] [[package]] name = "smawk" @@ -1989,12 +2230,201 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" +dependencies = [ + "lock_api", +] + +[[package]] +name = "sqlx" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "378620ccc25c62c89d8be1c819e76a88d59bdcc3304733330788948e619bfd71" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b44e85bf579a8eeb4ceaa77a3a523baf2bf0e9bac7e40f405d537b5d2d5ccb" +dependencies = [ + "base64 0.22.1", + "bytes", + "cfg-if", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.16.1", + "hashlink 0.11.1", + "indexmap", + "log", + "memchr", + "percent-encoding", + "serde", + "serde_json", + "sha2 0.10.9", + "smallvec", + "thiserror", + "tokio", + "tokio-stream", + "tracing", + "url", +] + +[[package]] +name = "sqlx-macros" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd2b84f2bc39a5705ef27ec785a11c934a41bbd4a24941e257927cddc26b60bf" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 2.0.111", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb8d96de5fdc85a5c4ec813432b523ec637e80ba98f046555f75f7908ddac7c3" +dependencies = [ + "cfg-if", + "dotenvy", + "either", + "heck", + "hex", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2 0.10.9", + "sqlx-core", + "sqlx-sqlite", + "syn 2.0.111", + "thiserror", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90b8020fe17c5f2c245bfa2505d7ef59c5604839527c740266ad2214acebea27" +dependencies = [ + "bitflags", + "byteorder", + "bytes", + "crc", + "digest 0.11.3", + "dotenvy", + "either", + "futures-core", + "futures-util", + "generic-array", + "log", + "percent-encoding", + "serde", + "sha1", + "sha2 0.11.0", + "sqlx-core", + "thiserror", + "tracing", +] + +[[package]] +name = "sqlx-postgres" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87a2bdd6e83f6b3ea525ca9fee568030508b58355a43d0b2c1674d5f79dcd65e" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags", + "byteorder", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "rand", + "serde", + "serde_json", + "sha2 0.11.0", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "whoami 2.1.3", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488e99c397a62007e4229aec669a179816339afc6d2620ca6fa420dbee2e982c" +dependencies = [ + "atoi", + "flume", + "form_urlencoded", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "sqlx-core", + "thiserror", + "tracing", + "url", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.11.1" @@ -2101,22 +2531,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.17" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.17" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 3.0.3", ] [[package]] @@ -2180,6 +2610,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.19" @@ -2296,10 +2737,23 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "tracing-core" version = "0.1.36" @@ -2333,6 +2787,12 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.22" @@ -2345,6 +2805,21 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "unicode-segmentation" version = "1.12.0" @@ -2399,6 +2874,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" @@ -2552,6 +3033,12 @@ dependencies = [ "wasite", ] +[[package]] +name = "whoami" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" + [[package]] name = "winapi-util" version = "0.1.11" @@ -2731,7 +3218,7 @@ checksum = "2462ea039c445496d8793d052e13787f2b90e750b833afee748e601c17621ed9" dependencies = [ "arraydeque", "encoding_rs", - "hashlink", + "hashlink 0.10.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index fcec6a2d..80522b4e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ members = ["packages/debmagic", "packages/debmagic-common"] [workspace.package] edition = "2024" -rust-version = "1.89" +rust-version = "1.94" [workspace.dependencies] # internal @@ -27,7 +27,7 @@ chrono = { version = ">=0.4.42" } regex = { version = ">=1.12.2" } reqwest = { version = ">=0.13", default-features = false, features = ["rustls", "stream"] } futures-util = "0.3" -tokio = { version = ">=1.43", features = ["macros", "rt-multi-thread"] } +tokio = { version = ">=1.43", features = ["macros", "rt-multi-thread", "sync", "time"] } debian-changelog = { version = ">=0.2.14" } debian-control = { version = ">=0.1.39" } deb822-lossless = { version = ">=0.2" } @@ -35,6 +35,8 @@ ignore = { version = ">=0.4.25" } md-5 = { version = ">=0.11" } sha1 = { version = ">=0.11" } sha2 = { version = ">=0.11" } +# `sqlite` bundles libsqlite3; Debian's sqlx maps it to the system library +sqlx = { version = ">=0.9", default-features = false, features = ["runtime-tokio", "sqlite", "macros", "migrate"] } # dev dependencies test-case = { version = ">=3.3.1" } diff --git a/README.md b/README.md index 2ecba0ae..1c889883 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Modern, robust & easy [Debian](https://debian.org)/[Ubuntu](https://ubuntu.com) Debmagic unifies the the packaging experience as a single, streamlined tool. Additionally, `debmagic-pkg` allows you to define package build recipes in Python. - **Isolated builds without the setup**: `debmagic build binary` builds a `debian/`-packaged source tree in a temporary container (using LXD, Incus, Docker) -- **Fast iteration**: `--persistent`/`--incremental` reuse the environment and sync only source changes; `--shell-on-failure` and `debmagic shell` drop you right where the build broke +- **Fast iteration**: `--persistent`/`--incremental` reuse the environment and sync only source changes; a failed build prints `debmagic env shell ` - **Test & sign integrated**: `debmagic test` runs the package's autopkgtest tests in a fresh environment, `debmagic sign` GPG-signs `.changes`/`.dsc`/`.buildinfo` on the host - **Python packaging API**: replace complicated `debian/rules` Makefiles with typed Python `debian/rules.py`, with optional `dh` compatibility @@ -30,7 +30,7 @@ uvx debmagic build binary --driver docker | `debmagic build binary` | Build a binary package (`.deb`) in an isolated environment | | `debmagic build source` | Create a source package (`.dsc`) for upload (incl signing) | | `debmagic test` | Run the package's autopkgtest tests (`debian/tests/`) against a prior build | -| `debmagic shell` | Attach an interactive shell to the build environment | +| `debmagic env` | List, clean, or open a shell in a build or test environment | | `debmagic sign` | GPG-sign a `.changes` file (and its `.dsc`/`.buildinfo`) on the host | | `debmagic config` | Inspect and edit the effective `debmagic.toml` configuration | | `debmagic check` | Lint the package *(in progress)* | diff --git a/debian/control b/debian/control index 4216a09a..ec31dd9e 100644 --- a/debian/control +++ b/debian/control @@ -38,6 +38,13 @@ Build-Depends: librust-md-5-dev (>= 0.11), librust-sha1-dev (>= 0.11), librust-sha2-dev (>= 0.11), + librust-sqlx-dev (>= 0.9), + librust-sqlx+runtime-tokio-dev (>= 0.9), + librust-sqlx+sqlite-dev (>= 0.9), + librust-sqlx+macros-dev (>= 0.9), + librust-sqlx+migrate-dev (>= 0.9), + libsqlite3-dev, + pkg-config, Rules-Requires-Root: no X-Style: black Standards-Version: 4.7.2 diff --git a/debian/rules.py b/debian/rules.py index 9c3bb05d..68f00037 100755 --- a/debian/rules.py +++ b/debian/rules.py @@ -17,13 +17,15 @@ ) # TODO: add this section to our rust debmagic module to make setting this up less annoying -os.environ.update({ - "PATH": f"/usr/share/cargo/bin:{os.environ['PATH']}", - "CARGO": "/usr/share/cargo/bin/cargo", - "CARGO_HOME": f"{pkg.base_dir}/debian/cargo_home", - "CARGO_REGISTRY": f"{pkg.base_dir}/debian/cargo_registry", - "DEB_CARGO_CRATE": f"{pkg.build_env.DEB_SOURCE}_{pkg.build_env.DEB_VERSION_UPSTREAM}" -}) +os.environ.update( + { + "PATH": f"/usr/share/cargo/bin:{os.environ['PATH']}", + "CARGO": "/usr/share/cargo/bin/cargo", + "CARGO_HOME": f"{pkg.base_dir}/debian/cargo_home", + "CARGO_REGISTRY": f"{pkg.base_dir}/debian/cargo_registry", + "DEB_CARGO_CRATE": f"{pkg.build_env.DEB_SOURCE}_{pkg.build_env.DEB_VERSION_UPSTREAM}", + } +) packages = { "debmagic": ("packages/debmagic", "debmagic"), @@ -33,10 +35,14 @@ cargo_lock = Path("Cargo.lock") cargo_lock_saved = Path("Cargo.lock.saved") + def dh_auto(build: Build, stage: str, use_destdir: bool = False): for pkg_name, (path, python_pkg_name) in packages.items(): destdir = f" --destdir debian/{pkg_name} " if use_destdir else "" - build.cmd(f"{stage} -p {pkg_name} --sourcedirectory {path} --buildsystem=pybuild {destdir} -- --name {python_pkg_name}") + build.cmd( + f"{stage} -p {pkg_name} --sourcedirectory {path} --buildsystem=pybuild {destdir} -- --name {python_pkg_name}" + ) + @dhp.override def dh_auto_configure(build: Build): @@ -68,4 +74,4 @@ def dh_auto_clean(build: Build): dh_auto(build, "dh_auto_clean") -pkg.pack() \ No newline at end of file +pkg.pack() diff --git a/docs/index.md b/docs/index.md index bff6dd8e..15128be9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -9,7 +9,7 @@ Working with Debian packages without `debmagic` means juggling with several inde Debmagic unifies this into one CLI with useful defaults: - **Isolated builds without the setup**: `debmagic build binary` builds a `debian/`-packaged source tree in a temporary container (using LXD, Incus, Docker) -- **Fast iteration**: `--persistent`/`--incremental` reuse the environment and sync only source changes; `--shell-on-failure` and `debmagic shell` drop you right where the build broke +- **Fast iteration**: `--persistent`/`--incremental` reuse the environment and sync only source changes; analyze failures with `debmagic env shell` - **Test & sign integrated**: `debmagic test` runs the package's autopkgtest tests in a fresh environment, `debmagic sign` GPG-signs `.changes`/`.dsc`/`.buildinfo` on the host - **Package uploading**: `debmagic upload` (to `debian`, `ubuntu`, `ppa:user/repo`) pushes signed builds to archives and PPAs via scp/sftp, with configurable pre-upload checks - **Upstream tracking**: `debmagic upstream list`/`switch` replace uscan/uupdate, reading the package's existing `debian/watch` and `debian/copyright` @@ -37,6 +37,7 @@ Then dive into the pages below — start with [Getting started](usage/getting-st usage/getting-started.md usage/build.md usage/test.md +usage/env.md usage/source.md usage/upload.md usage/upstream.md diff --git a/docs/usage/build.md b/docs/usage/build.md index 412b9ca5..9601d2b1 100644 --- a/docs/usage/build.md +++ b/docs/usage/build.md @@ -22,8 +22,8 @@ debmagic build binary --driver lxd \ |---|---| | `--driver <...>` | [Build environment driver to use](#picking-a-driver) | | `--source-sync ` | [Which source files to include](#source-file-staging) | -| `--persistent` | Retain the build environment. This does not do incremental builds. | -| `--incremental` | Do incremental builds by syncing changed sources only; implies `persistent` | +| `--persistent ` | How long the build environment outlives this build: `on-failure` (default) keeps it when the build command fails, `always` keeps it after every build, `no` tears it down. A bare `--persistent` means `always` | +| `--incremental` | Do incremental builds by syncing changed sources only; forces `--persistent=always` | | `--distro ` | [Select the target distro/release](#selecting-a-distrorelease) (e.g. `trixie`, `resolute`) | | `--proposed` | Use [build dependencies from `proposed`](#proposed-dependencies) pocket | | `--sign` | [GPG-sign the resulting `.changes`/`.dsc`/`.buildinfo`](#signing) | @@ -34,9 +34,8 @@ debmagic build binary --driver lxd \ | `--apt-update-age ` | [When a persistent environment re-runs `apt-get update`](#apt-update-age) | | `--source-dir ` | Directory containing the `debian/` package directory | | `--output-dir ` | Directory to put the resulting build artifacts | -| `--shell-on-failure` | On build failure, drop into an interactive shell in the build environment when stdout is a TTY | -[`debmagic shell`](#inspecting-a-failed-build) — attach an interactive shell to a build environment +[`debmagic env shell`](#inspecting-a-failed-build) — attach an interactive shell to an Environment ## Picking a driver @@ -52,18 +51,13 @@ There's no auto-detection; pick one and pass it explicitly every time (or config ## Inspecting a failed build -On failure the build environment is torn down by default. Pass `--shell-on-failure` to drop into an interactive shell inside the build environment when stdout is a TTY (destroyed on shell exit unless `--persistent` was used). - -To inspect after the run finishes, pass `--persistent` up front, then: +When the build command itself fails and this run's Persistence is `on-failure` (the default) or `always`, the Environment stays and debmagic prints `debmagic env shell `. Setup failures, and failures after the build command succeeded (export, signing), tear it down. `no` always tears it down and prints no hint. ```shell -# if you're in the package still -debmagic shell -# from the outside: -debmagic shell --source-dir /path/to/parent/of/debian/dir +debmagic env shell ``` -This attaches an interactive shell inside the still-running (or restartable) build environment, at the package's build directory. +Each `debmagic env shell` is its own shell, so you can open more than one. A bare `debmagic env shell` attaches only when this checkout has exactly one Environment. ## Mirror selection @@ -80,7 +74,7 @@ You can persistently set this flag in `.config/debmagic/config.toml`. ## Apt update age Fresh environments always run `apt-get update` once on creation. -When a persistent environment is reused, `--apt-update-age` decides whether the apt index is refreshed again: +When an Environment is reused, `--apt-update-age` decides whether the apt index is refreshed again: | Value | Behavior | |---|---| @@ -113,23 +107,24 @@ You can iterate on the same build for faster compile times. ### Persistent container -Every `debmagic build binary` invocation creates a new container by default and tears it down afterwards. -For repeated attempts against the same package and distro, add `--persistent` to retain and reuse the running environment while restaging the source tree for each build: +Every `debmagic build binary` keeps the Environment when the build command fails (`--persistent=on-failure`, the default) and tears it down when the build succeeds. Pass `--persistent` (that is, `--persistent=always`) to keep it after success too, and reuse it on the next build while restaging the source tree: ```shell debmagic build binary --driver lxd --persistent \ --source-dir . --output-dir /tmp/out ``` +`--persistent=no` discards a kept Environment and starts fresh. A later `on-failure` success also tears down an Environment a previous `always` left behind. + ### Incremental builds Use `--incremental` to retain the environment and synchronize only source changes while preserving generated files and unchanged source inodes. -This flag implies `--persistent`, and cannot be combined with `--clean yes`. +This flag forces `--persistent=always`, including over an explicit `no` or `on-failure`, and cannot be combined with `--clean yes`. The preserved build tree is kept even when the environment itself is *not* reused (e.g. a fresh CI runner where the tree was restored from a cache). Quilt patches that a previous build or shell session left applied in the build tree are unapplied with `dpkg-source --after-build` before sources are synced, so the worktree remains the source of truth and stale `.pc` state cannot break later builds. -The work dir (`temp_build_dir`, default `/tmp/debmagic`) keeps the artifacts of builds, including their `.changes` files. +The Environment's Host root lives under `environments_dir` (default `$XDG_DATA_HOME/debmagic/environments`). Build artifacts, including `.changes` files, are exported to `output_dir`. ## Selecting a distro/release @@ -234,4 +229,4 @@ Instead of repeating CLI flags on every invocation, drop a [config file](config. ## Internals -- Container/device names are derived and sanitized internally (alphanumeric + hyphen, ≤63 chars for LXD/Incus) — don't try to predict or construct them yourself; use `debmagic shell` instead of `lxc`/`docker` commands directly. +- Container/device names are derived and sanitized internally (alphanumeric + hyphen, ≤63 chars for LXD/Incus) — don't try to predict or construct them yourself; use `debmagic env shell` instead of `lxc`/`docker` commands directly. diff --git a/docs/usage/config.md b/docs/usage/config.md index 781393e6..68a347b9 100644 --- a/docs/usage/config.md +++ b/docs/usage/config.md @@ -14,6 +14,8 @@ Config files are merged in order of increasing precedence, so later files overri Only files that exist are read; missing ones are skipped. Command-line flags override whatever the merged config resolves to. +`DEBMAGIC_DATA_DIR` relocates the Environment registry (`db.sqlite`) and the default `environments_dir` together. A relative value is resolved to an absolute path against the current working directory. It is not a config file key. + ## Options All keys are optional. @@ -21,16 +23,16 @@ All keys are optional. | Key | Type | Default | CLI flag | Description | |---|---|---|---|---| | `driver.default` | enum | — | `--driver` | Build driver (`docker`, `bare`, `lxd`, `incus`) | -| `driver.persistent` | bool | `false` | `--persistent` | Keep and reuse the build environment across runs instead of tearing it down. | +| `driver.persistent` | enum | `"on-failure"` | `--persistent` | How long the Environment outlives the claim: `"no"`, `"on-failure"`, or `"always"`. A bare `--persistent` means `always`. | | `driver.apt_mirror` | string | — | `--apt-mirror` | Mirror used for build-dependency resolution. Not used by the `bare` driver. | | `driver.proposed` | bool | `false` | `--proposed` | Also enable the `-proposed` pocket. Not used by the `bare` driver. | | `driver.apt_update_age` | string | `"1d"` | `--apt-update-age` | When a persistent environment re-runs `apt-get update`: `"now"` (every build), `"never"` (only on creation), or a maximum index age like `"1d"`, `"12h"`, `"30m"`. Fresh environments always update once. Not used by the `bare` driver. | | `driver.docker.base_images` | map | — | — | Base image per distro, keyed by `":"` (e.g. `"debian:trixie"`). Falls back to `docker.io/:`. For non-Debian/Ubuntu suites (e.g. `"yocto:kirkstone"`), the map entry is what makes the suite a known DistroVersion for Docker builds. | | `driver.lxd.project` | string | — | — | LXD/Incus project to use. | | `driver.lxd.base_images` | map | — | — | Base image per distro, keyed by `":"`. Falls back to the driver's default remote image. Same custom-suite registry role as Docker's map for LXD/Incus. | -| `temp_build_dir` | path | `/tmp/debmagic` | — | Where build trees are staged. | +| `environments_dir` | path | `$XDG_DATA_HOME/debmagic/environments` | — | Parent directory of Environment Host roots. Must be absolute or start with `~/`. The Environment registry is always `$XDG_DATA_HOME/debmagic/db.sqlite` (override both with `DEBMAGIC_DATA_DIR`). | | `output_dir` | path | `build` | `-o`/`--output-dir` | Where build artifacts are exported; relative paths resolve against the package root. | -| `incremental` | bool | `false` | `--incremental` | Retain the environment and sync only source changes, preserving generated files. Implies `persistent`; incompatible with `clean`. | +| `incremental` | bool | `false` | `--incremental` | Retain the environment and sync only source changes, preserving generated files. Forces Persistence `always`; incompatible with `clean`. | | `source_sync_mode` | enum | `tracked` | `--source-sync` | Which source files are staged (see below). | | `build_debug_symbols` | bool | `false` | `--debug-symbols` | Build the automatic `-dbgsym` debug symbol package. | | `run_test` | bool | `true` | `--test` | Run the package's test suite during the build; `false` exports `DEB_BUILD_OPTIONS=nocheck` so tests are skipped. | @@ -40,7 +42,6 @@ All keys are optional. | `sign.sign_command` | string | — | `--sign-command` | Custom signing command for `sign.tool = "custom"`, run without a shell with `{file}`/`{key}`/`{email}` placeholders; writes the clearsigned result to stdout. | | `sign.notify` | bool | `false` | `--sign-notify` | Send a desktop notification via `notify-send` just before signing, so a hardware-key touch prompt isn't missed. | | `clean` | bool | `false` | `--clean` | Run `debian/rules clean` before building. Disabled by default; incompatible with `incremental`. | -| `shell_on_failure` | bool | `false` | `--shell-on-failure` | On build or test failure, drop into an interactive shell in the environment when stdout is a TTY. | | `host_arch_variant` | string | — | `--host-arch-variant` | Build for a dpkg architecture variant (e.g. `"amd64v3"` on Ubuntu) -> `DEB_HOST_ARCH_VARIANT`. | | `upstream.verify_signatures` | bool | `true` | `--no-signature-check` | Verify the upstream tarball's PGP signature against `debian/upstream/signing-key.asc` during `upstream switch` (see [Upstream](upstream.md#signature-verification)). | | `upload.targets` | map | — | `debmagic upload ` | Named upload targets for [uploading](upload.md), merged field-by-field over the builtins (`ppa`, `ubuntu`, `debian`). | @@ -106,7 +107,7 @@ notify = true [driver] default = "lxd" -persistent = true +persistent = "always" apt_mirror = "http:///ubuntu" [driver.docker] diff --git a/docs/usage/env.md b/docs/usage/env.md new file mode 100644 index 00000000..aeb88c61 --- /dev/null +++ b/docs/usage/env.md @@ -0,0 +1,68 @@ +# Environments + +Every build and test runs in an Environment: a container, VM or chroot that a Driver creates for one package version and distro. +debmagic remembers all Environments on this machine, so you can inspect, reuse and clean them up later. + +```shell +debmagic env list # what exists, and in what state +debmagic env shell [] # open a shell in an Environment +debmagic env clean # destroy everything Stale +debmagic env rm ... # destroy specific Environments +``` + +`env` is short for `debmagic environment`. + +## list + +Prints every Environment, after asking each Driver whether it still exists: + +| Column | Meaning | +|---|---| +| ID | Environment id, used by `shell` and `rm` | +| PURPOSE | `build` or `test` | +| DRIVER | Isolation backend | +| PACKAGE | Package name and version | +| DISTRO | Codename | +| PERSIST | Persistence of the run that created it: `no`, `on-failure`, or `always` | +| STATUS | `live`, `stale`, or `unreachable` | +| ATTACH | Number of open `debmagic env shell` sessions | +| SOURCE | Source tree path | + +An Environment is **Stale** when its Driver resource or Host root is gone, or when it has Persistence `no` and neither its creating command nor a shell is still running. +`on-failure` and `always` Environments stay `live` after their command exits. +**Unreachable** means the Driver could not be queried, so debmagic cannot tell whether the Environment still exists. + +## shell + +```shell +debmagic env shell # inside a Source tree: its only Environment +debmagic env shell # anywhere +``` + +Without an id, debmagic picks the Environment of the current Source tree. +Environments of the version in `debian/changelog` take precedence over those of earlier versions. +If there are none or several (for example a build and a test Environment), it lists them and asks for an id. + +Each call opens a new shell, so you can open several. +While a shell is open, the Environment is kept, even with Persistence `no`, and a build or test that would reuse it refuses to start instead of resetting it under you. + +## clean and rm + +Each package version gets its own Environment, so kept ones accumulate. + +`debmagic env clean` destroys all **Stale** Environments on the machine and leaves healthy ones alone. +Unreachable Environments are skipped and reported. +An Environment that a concurrent run has just recreated is skipped too. + +`debmagic env rm ...` (alias `remove`) destroys the given Environments, even healthy ones. +It refuses while a shell or the creating command is still running, and refuses Unreachable Environments. +The remaining ids are still processed when one fails. + +If the Driver fails to destroy its resource (for example `docker rm` fails), the Environment stays listed so the resource is not lost; retry once the Driver works. +`--force` removes the Environment anyway, also when it is Unreachable, and only warns if the Driver resource may have leaked. + +## Data directory + +The registry and the Host roots live under `$XDG_DATA_HOME/debmagic` (see [`environments_dir`](config.md)). +Keep this on a local filesystem. +With a network-mounted home directory, set `DEBMAGIC_DATA_DIR` to a local path. diff --git a/docs/usage/test.md b/docs/usage/test.md index 96393f17..268fd5dd 100644 --- a/docs/usage/test.md +++ b/docs/usage/test.md @@ -5,7 +5,7 @@ Quick reference for running a package's declared Debian autopkgtest tests with ` ## TL;DR - Entry point: `debmagic test` — runs tests from `debian/tests/control`; needs a prior `debmagic build` -- Requires a completed build in the same build root (or pass `--changes` to point at exported artifacts) +- Requires a completed binary build (looked up from the Environment registry's Invocation history, or pass `--changes`) ```shell cd your-package @@ -17,7 +17,7 @@ debmagic test --driver docker `debmagic test` installs the binary packages from a prior build and runs the package's declared autopkgtest tests (`debian/tests/control`) inside a **fresh, separate** driver-managed environment. -The test environment is never the build environment — even when `--persistent` reuses a container across runs, +The test environment is never the build environment — even when Persistence `on-failure` or `always` reuses a container across runs, the test tree is reset and the `.debs` are reinstalled each time. The driver *is* the testbed: `autopkgtest` runs with the `null` backend inside the container (or on the host for the bare driver). No `autopkgtest-virt-*` backends are used. @@ -26,23 +26,22 @@ The driver *is* the testbed: `autopkgtest` runs with the `null` backend inside t | Option | Description | |---|---| -| `--driver <...>` | Test environment driver (defaults to the driver recorded in the prior build's `environment.json`) | -| `--persistent` | Retain the test environment after the run for reattach/debug | +| `--driver <...>` | Test environment driver (defaults to the driver recorded on the prior binary-build Invocation) | +| `--persistent ` | How long the test environment outlives this run: `on-failure` (default) keeps it when the tests fail, `always` keeps it after every run, `no` tears it down. A bare `--persistent` means `always` | | `--strict` | Treat skipped tests and "no tests declared" as failures (exit code 2) | -| `--changes ` | Path to a `.changes` file whose directory supplies the built `.debs` (for pipeline use) | -| `--distro ` | Override the target distro for the test environment (defaults to the prior build's distro from `environment.json`, not the changelog) | +| `--changes ` | Path to a `.changes` file whose directory supplies the built `.debs` (for pipeline use). Overrides the artifact path only; driver and distro still default to the prior binary-build Invocation | +| `--distro ` | Override the target distro for the test environment (defaults to the prior binary-build Invocation's distro, not the changelog) | | `--proposed` | Enable the `-proposed` pocket in the test environment | | `--apt-mirror ` | Mirror URL (same as [`debmagic build`](build.md)) | | `--apt-update-age ` | Configure when to run `apt-get update` (same as [`debmagic build`](build.md#apt-update-age)) | | `--source-dir ` | Directory containing the `debian/` package directory | | `--allow-host-test` | Allow the bare driver, which runs autopkgtest as root on the host | -| `--shell-on-failure` | On test failure, drop into an interactive shell in the test environment when stdout is a TTY | Driver-specific flags (`--driver-docker-base-image`, `--driver-lxd-*`) mirror `debmagic build`. ## Picking a driver -Use the same drivers as for builds. Pass `--driver` explicitly (or rely on the driver recorded in the prior build's `environment.json`): +Use the same drivers as for builds. Pass `--driver` explicitly (or rely on the driver recorded on the prior binary-build Invocation): | Driver | Isolation the Environment provides | |---|---| @@ -66,16 +65,21 @@ If no `debian/tests/control` exists (or it declares no tests), the run exits 0 w ## Inspecting a failed test run -On failure the test environment is torn down by default. Pass `--shell-on-failure` to drop into an interactive shell inside the test environment when stdout is a TTY (destroyed on shell exit unless `--persistent` was used). +When the tests themselves fail and this run's Persistence is `on-failure` (the default) or `always`, the Environment stays and debmagic prints `debmagic env shell `. A strict skip tears the Environment down. `no` always tears it down and prints no hint. -Test output and logs are exported to a `test/` subdirectory of the build root; the path is printed at the end of the run. +Test output and logs are exported to a `.test/` directory next to the `.changes` file (for `pkg_1.0-1_amd64.changes`, that is `pkg_1.0-1_amd64.test/`); the path is printed at the end of the run. A later run replaces that directory only if debmagic wrote it; otherwise the run fails rather than deleting it. + +```shell +debmagic env shell +``` ## Prior build required -By default `debmagic test` resolves the prior build from the build root (same layout as `debmagic shell`). If no build artifacts are found: -run `debmagic build` first +By default `debmagic test` looks up the latest successful binary-build Invocation for this Source tree whose exported `.changes` still exists. If more than one DistroVersion matches, it errors and lists them — pass `--distro`. If none are found: + +run `debmagic build binary` first -Use `--changes` to supply a `.changes` file from an exported output directory instead. +Use `--changes` to supply a `.changes` file from an exported output directory. That overrides the artifact path only. Driver and distro still come from the prior binary-build Invocation when one exists; pass `--driver` and `--distro` when it does not. Several DistroVersions still require `--distro`. ## Bare driver diff --git a/packages/debmagic/CHANGELOG.md b/packages/debmagic/CHANGELOG.md index 059f7682..038e312f 100644 --- a/packages/debmagic/CHANGELOG.md +++ b/packages/debmagic/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `debmagic switch ` upstream source fetching and orig tarballs handling with `debmagic upstream` (and caching them) - dynamic interval for running `apt update` configured by `--apt-update-age` / `driver.apt_update_age` +- `debmagic env list`, `debmagic env clean`, and `debmagic env shell` for a machine-wide view of environments ## [0.0.1-alpha.8] - 2026-09-20 diff --git a/packages/debmagic/Cargo.toml b/packages/debmagic/Cargo.toml index f7569c23..439f4e97 100644 --- a/packages/debmagic/Cargo.toml +++ b/packages/debmagic/Cargo.toml @@ -35,6 +35,7 @@ ignore = { workspace = true } md-5 = { workspace = true } sha1 = { workspace = true } sha2 = { workspace = true } +sqlx = { workspace = true } [dev-dependencies] test-case = { workspace = true } diff --git a/packages/debmagic/README.md b/packages/debmagic/README.md index 0ef9552c..7a76bc72 100644 --- a/packages/debmagic/README.md +++ b/packages/debmagic/README.md @@ -5,7 +5,7 @@ Modern, robust & easy tooling for building and packaging [Debian](https://debian - **Build any package** in an isolated container environment with `debmagic build` - **Run Debian autopkgtest tests** against built packages with `debmagic test` - **Lint** with `debmagic check` -- **Debug** build environments interactively with `debmagic shell` +- **Debug** environments interactively with `debmagic env shell` ## Installation @@ -59,8 +59,8 @@ Use `--strict` to fail on skipped or undeclared tests (exit code 2). The bare dr ### Useful options - `--distro ` — select the target distro/release (e.g. `trixie`, `noble`) if the changelog is ambiguous -- `--persistent` — retain the build environment for repeated attempts -- `--incremental` — sync only changed sources for faster rebuilds; implies `--persistent` +- `--persistent` — keep the build environment after every build (`always`); the default `on-failure` keeps it only when the build command fails +- `--incremental` — sync only changed sources for faster rebuilds; forces `--persistent=always` - `--sign` — GPG-sign the resulting `.changes`/`.dsc`/`.buildinfo` - `--apt-mirror ` — use a faster mirror for build-dependency resolution @@ -68,10 +68,10 @@ Any of these can be persisted in a `debmagic.toml` config file instead of repeat ### Inspecting a failed build -Failed builds tear down their environment by default. Pass `--shell-on-failure` to drop into a shell when stdout is a TTY, or build with `--persistent` and attach afterwards: +A failed build or test keeps the Environment when Persistence is `on-failure` (the default) or `always`, and prints how to attach: ```shell -debmagic shell +debmagic env shell ``` ## Documentation diff --git a/packages/debmagic/src/build/attach.rs b/packages/debmagic/src/build/attach.rs deleted file mode 100644 index 15110d4b..00000000 --- a/packages/debmagic/src/build/attach.rs +++ /dev/null @@ -1,93 +0,0 @@ -//! Unix-socket attach/detach tracking for build environments. -//! -//! While a build keeps its environment alive after a failure, a small -//! socket server in the build root lets concurrent `debmagic shell` -//! sessions register themselves, so the environment is only torn down -//! once the last attached shell detaches. - -use core::time; -use std::net::Shutdown; -use std::os::unix::net::{UnixListener, UnixStream}; -use std::sync::{Arc, Mutex}; -use std::{ - fs, io, - io::{Read, Write}, - path::{Path, PathBuf}, - thread, -}; - -use anyhow::Context; - -fn socket_path_for_build(build_root: &Path) -> PathBuf { - build_root.join("build.sock") -} - -pub fn start_socket_server( - build_root: &Path, - should_exit: Arc>, -) -> anyhow::Result> { - let sock = socket_path_for_build(build_root); - if sock.exists() { - // try to remove stale socket file - let _ = fs::remove_file(&sock); - } - - let listener = UnixListener::bind(&sock) - .with_context(|| format!("failed to bind unix socket {}", sock.display()))?; - - // Set non-blocking mode so we can check the exit flag - listener - .set_nonblocking(true) - .context("failed to set socket non-blocking")?; - - let handle = thread::spawn(move || { - let mut num_attached = 0u64; - loop { - // Check if we should exit - let exit_requested = *should_exit.lock().unwrap(); - if exit_requested && num_attached == 0 { - break; - } - - match listener.accept() { - Ok((mut s, _)) => { - let mut buf = String::new(); - if s.read_to_string(&mut buf).is_err() { - let _ = s.shutdown(Shutdown::Both); - continue; - } - let cmd = buf.trim(); - match cmd { - "attach" => { - num_attached += 1; - } - "detach" => { - num_attached = num_attached.saturating_sub(1); - } - _ => {} - } - let _ = s.shutdown(Shutdown::Both); - } - Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => { - // No connection available, sleep briefly to avoid busy-waiting - thread::sleep(time::Duration::from_millis(10)); - } - Err(_) => break, - } - } - let _ = fs::remove_file(&sock); - }); - - Ok(handle) -} - -pub fn send_socket_command(build_root: &Path, cmd: &str) -> anyhow::Result<()> { - let sock = socket_path_for_build(build_root); - let mut stream = UnixStream::connect(&sock) - .with_context(|| format!("failed to connect to socket {}", sock.display()))?; - stream - .write_all(cmd.as_bytes()) - .context("failed to send socket command")?; - stream.shutdown(Shutdown::Write).ok(); - Ok(()) -} diff --git a/packages/debmagic/src/build/mod.rs b/packages/debmagic/src/build/mod.rs index f791c4d0..3b3008ad 100644 --- a/packages/debmagic/src/build/mod.rs +++ b/packages/debmagic/src/build/mod.rs @@ -1,200 +1,64 @@ -use std::sync::{Arc, Mutex}; use std::{ fs, io, - io::{BufReader, IsTerminal, stdout}, - path::{Path, PathBuf}, + path::Path, process::{Command, Stdio}, }; -use crate::build::attach::{send_socket_command, start_socket_server}; use crate::build::source::{source_manifest_path, stage_dir, stage_source_tree}; use crate::build_intent::{BuildIntent, BuildIntentInput, BuildKind, resolve_build_intent}; -use crate::driver::{ - Driver, DriverType, Environment, EnvironmentDriver, EnvironmentMetadata, EnvironmentPurpose, - config::DriverConfig, create_driver, create_driver_from_metadata, remove_environment_root, +use crate::driver::{Driver, DriverType, Environment, EnvironmentDriver, EnvironmentPurpose}; +use crate::environment::{ + ClaimedEnvironment, CommandCompletion, HostRootPolicy, InvocationKind, Registry, claim_command, }; -use crate::{config::Config, package::PackageTarget}; +use crate::package::PackageTarget; use anyhow::{Context, anyhow}; use debmagic_common::debian::source::SourceFormat; use debmagic_common::package::SourcePackage; pub mod artifacts; -pub mod attach; pub mod source; pub use source::SourceSyncMode; -struct Build { +struct PlannedBuild { environment: Environment, - driver: Driver, - attached: bool, - output_dir: PathBuf, - clean: bool, - build_debug_symbols: bool, - run_test: bool, - host_arch_variant: Option, + root_policy: HostRootPolicy, + incremental: bool, } -impl Build { - pub fn create(environment: Environment, intent: &BuildIntent) -> anyhow::Result { - let driver = create_driver( - &environment, - &intent.config.driver, - &intent.driver_overrides, - ) - .context(format!("failed to create {:?} driver", environment.driver))?; - Ok(Self { - environment, - driver, - attached: false, - output_dir: intent.output_dir.clone(), - clean: intent.config.clean, - build_debug_symbols: intent.config.build_debug_symbols, - run_test: intent.config.run_test, - host_arch_variant: intent.config.host_arch_variant.clone(), - }) - } - - pub fn from_build_root( - build_root: &Path, - driver_config: &DriverConfig, - ) -> anyhow::Result { - let metadata_path = build_root.join("environment.json"); - if !metadata_path.is_file() { - return Err(anyhow!("No environment.json found")); - } - let file = fs::OpenOptions::new().read(true).open(&metadata_path)?; - let metadata = || -> anyhow::Result { - let reader = BufReader::new(&file); - let metadata: EnvironmentMetadata = - serde_json::from_reader(reader).with_context(|| { - format!( - "Failed to read environment metadata from {} - invalid json", - metadata_path.display() - ) - })?; - Ok(metadata) - }(); - - let metadata = metadata?; - - let driver = create_driver_from_metadata(driver_config, &metadata)?; - - let attached = send_socket_command(build_root, "attach").is_ok(); - - Ok(Self { - environment: metadata.environment.clone(), - driver, - attached, - output_dir: PathBuf::new(), - clean: false, - build_debug_symbols: false, - run_test: true, - host_arch_variant: None, - }) - } - - pub fn detach(&self) -> anyhow::Result<()> { - let build_root = &self.environment.root_dir; - if self.attached { - send_socket_command(build_root, "detach")?; +fn plan_build(intent: &BuildIntent, target: &PackageTarget) -> PlannedBuild { + let package = &target.package; + let environment = Environment::new( + intent.driver, + package.name(), + &format!("{}-{}", package.name(), package.version()), + &intent.source_dir, + target.distro.clone(), + intent.config.driver.persistent, + EnvironmentPurpose::Build, + &intent.config.environments_dir, + ); + + // An incremental sync needs a manifest from a previous build of the same + // kind; without one the tree is reset so no stale files leak into the + // build. A source build shares the Environment with the binary tree, so + // only its own stage dir is restaged. + let manifest = source_manifest_path(&environment, intent.kind); + let incremental = intent.config.incremental && manifest.is_file(); + let root_policy = if incremental { + HostRootPolicy::Keep + } else if intent.kind.is_source() { + HostRootPolicy::ResetStage { + stage_dir: stage_dir(&environment, intent.kind), } - Ok(()) - } - - pub fn write_metadata(&self) -> anyhow::Result<()> { - let metadata = EnvironmentMetadata { - environment: self.environment.clone(), - driver_metadata: self.driver.driver_metadata(), - }; - let path = self.environment.root_dir.join("environment.json"); - let json = serde_json::to_string_pretty(&metadata) - .context("Failed to serialize environment metadata")?; - fs::write(path, json)?; - Ok(()) - } -} - -fn get_build_root_and_identifier( - temp_build_dir: &Path, - package: &SourcePackage, -) -> (String, PathBuf) { - let package_identifier = format!("{}-{}", package.name(), package.version()); - let build_root = temp_build_dir.join(&package_identifier); - (package_identifier, build_root) -} - -/// Create the output dir and stage the source tree into the environment. -fn stage_sources( - environment: &Environment, - intent: &BuildIntent, - target: &PackageTarget, -) -> anyhow::Result<()> { - fs::create_dir_all(&intent.output_dir).context("failed to create output directory")?; - environment - .create_dirs() - .context("failed to create build directories")?; - stage_source_tree( - environment, - intent.kind, - &target.package, - intent.config.source_sync_mode, - intent.config.incremental, - ) -} - -fn prepare_build_env(intent: &BuildIntent, target: &PackageTarget) -> anyhow::Result { - let (package_identifier, build_root) = - get_build_root_and_identifier(&intent.config.temp_build_dir, &target.package); - - let environment = Environment { - driver: intent.driver, - package_name: target.package.name().to_string(), - package_identifier, - root_dir: build_root.clone(), - distro: target.distro.clone(), - persistent: intent.config.driver.persistent, - purpose: EnvironmentPurpose::Build, + } else { + HostRootPolicy::Reset }; - - if intent.config.driver.persistent && build_root.exists() { - // For persistent containers, starting first lets root inside delete - // container-owned files the host user can't remove. - let build = Build::create(environment.clone(), intent) - .context(format!("failed to create {:?} driver", intent.driver))?; - // An incremental sync needs a manifest from a previous build of the - // same kind; without one the tree is reset so no stale files leak - // into the build. - let sync_incrementally = - intent.config.incremental && source_manifest_path(&environment, intent.kind).is_file(); - if sync_incrementally { - if !build.driver.reused_environment() { - // e.g. a new CI runner with a restored build tree; cargo's own - // fingerprinting discards whatever the new toolchain/archive - // state invalidates - println!("Keeping incremental build tree in a fresh build environment"); - } - unapply_quilt_patches(&build, intent.kind, &target.package)?; - } else if intent.kind.is_source() { - // A source build shares the environment with the binary tree: - // only its own stage dir is restaged, never the whole root. - reset_stage_dir(&build, &environment, intent.kind)?; - } else { - build - .driver - .reset_root() - .context("failed to reset persistent build directory")?; - } - stage_sources(&environment, intent, target)?; - return Ok(build); + PlannedBuild { + environment, + root_policy, + incremental, } - - remove_environment_root(&build_root, &intent.config.driver)?; - - crate::output::step("Staging source tree"); - stage_sources(&environment, intent, target)?; - - Build::create(environment, intent) } /// Whether an incremental sync must first unapply quilt patches in the @@ -212,16 +76,16 @@ fn needs_quilt_unapply(source_format: SourceFormat, staged_source_dir: &Path) -> /// `dpkg-source --after-build --unapply-patches` restores the pre-patch /// state from the `.pc` backups and removes the quilt db entirely. fn unapply_quilt_patches( - build: &Build, + driver: &Driver, + environment: &Environment, kind: BuildKind, package: &SourcePackage, ) -> anyhow::Result<()> { - let staged_source_dir = stage_dir(&build.environment, kind); + let staged_source_dir = stage_dir(environment, kind); if !needs_quilt_unapply(package.source_format(), &staged_source_dir) { return Ok(()); } - build - .driver + driver .run_command_checked( &["dpkg-source", "--after-build", "--unapply-patches", "."], &staged_source_dir, @@ -231,46 +95,6 @@ fn unapply_quilt_patches( .context("failed to unapply quilt patches left in the build tree") } -/// Remove every previous leftover from a stage dir (binary residue like -/// `debian//` install trees makes `dpkg-source -b` abort on "unwanted -/// binary file"). Only this dir is wiped — the other kind's tree sharing -/// the environment keeps its incremental outputs. -fn reset_stage_dir( - build: &Build, - environment: &Environment, - kind: BuildKind, -) -> anyhow::Result<()> { - let dir = stage_dir(environment, kind); - if !dir.exists() { - return Ok(()); - } - build - .driver - .run_command_checked(&["find", ".", "-mindepth", "1", "-delete"], &dir, true, &[]) - .context("failed to reset the source stage directory") -} - -pub fn get_shell_in_build(config: &Config, package: &SourcePackage) -> anyhow::Result<()> { - let (_package_identifier, build_root) = - get_build_root_and_identifier(&config.temp_build_dir, package); - let build = Build::from_build_root(&build_root, &config.driver)?; - // the binary tree is the iteration workflow; fall back to the source - // tree when only source builds ever ran - let kind = if stage_dir(&build.environment, BuildKind::Binary).exists() { - BuildKind::Binary - } else { - BuildKind::Source - }; - let result = build - .driver - .interactive_shell(&stage_dir(&build.environment, kind)); - - build.detach()?; - - result?; - Ok(()) -} - fn deb_build_options(existing: Option<&str>, build_debug_symbols: bool, run_test: bool) -> String { let mut options = existing .unwrap_or_default() @@ -292,16 +116,15 @@ struct BuildRequest<'a> { target: &'a PackageTarget, } -/// Shared build orchestration: prepare the environment, run `build_commands` -/// in it, export the artifacts to the output dir, sign them if requested, and -/// clean up (dropping into a shell first when `--shell-on-failure` is set). -/// While the run is in progress, a socket server lets concurrent -/// `debmagic shell` sessions attach to the environment. -fn run_build( - request: &BuildRequest, - build_commands: impl FnOnce(&Build) -> anyhow::Result<()>, -) -> anyhow::Result { +/// Shared build orchestration: claim the Environment, run `build_commands` +/// in it, export the artifacts to the output dir, and sign them if requested. +/// Recording, the failure shell, and finish belong to [`claim_command`]. +async fn run_build( + request: &BuildRequest<'_>, + build_commands: impl FnOnce(&ClaimedEnvironment) -> anyhow::Result<()>, +) -> anyhow::Result { let sign = &request.intent.config.sign; + let registry = Registry::open_default_or_ephemeral().await?; let package = &request.target.package; crate::output::stage(&format!( @@ -309,96 +132,105 @@ fn run_build( package.name(), package.version() )); - let build = prepare_build_env(request.intent, request.target) - .context("failed to prepare build environment")?; - build - .write_metadata() - .context("failed to write environment metadata")?; - - let should_exit = Arc::new(Mutex::new(false)); - let socket_server_handle = - start_socket_server(&build.environment.root_dir, should_exit.clone())?; - - let stop_socket_server = || { - *should_exit.lock().unwrap() = true; - if !socket_server_handle.is_finished() { - println!("Waiting for all attached shells to exit..."); - } - socket_server_handle.join().ok(); - }; - - let result = build_commands(&build).and_then(|()| { - crate::output::stage("Exporting artifacts"); - let changes_file = artifacts::export_build_artifacts( - &build.environment.work_dir(), - &build.output_dir, - request.intent.kind, - )?; - if sign.source.is_enabled() { - crate::output::stage(&format!("Signing {}", build.environment.package_identifier)); - crate::sign::sign_file( - &changes_file, - sign, - sign.source, - sign.notify, - &build.environment.package_identifier, - )?; - } - Ok(changes_file) - }); - - if let Err(error) = &result { - if request.intent.shell_on_failure && stdout().is_terminal() { - eprintln!("Build failed: {error}. Dropping into shell..."); - if let Err(shell_error) = build - .driver - .interactive_shell(&stage_dir(&build.environment, request.intent.kind)) + let planned = plan_build(request.intent, request.target); + let build_kind = request.intent.kind; + let version = package.version().to_string(); + + claim_command( + ®istry, + planned.environment, + &request.intent.config.driver, + &request.intent.driver_overrides, + planned.root_policy, + |environment, driver| { + fs::create_dir_all(&request.intent.output_dir) + .context("failed to create output directory")?; + if planned.incremental + && let Some(driver) = driver { - eprintln!("Dropping into shell failed: {shell_error}"); + unapply_quilt_patches(driver, environment, build_kind, package)?; } - } else if request.intent.shell_on_failure { - eprintln!("Build failed: {error}"); - eprintln!( - "--shell-on-failure is set but stdout is not a TTY; skipping interactive shell" - ); - } else { - eprintln!("Build failed: {error}"); - eprintln!("Re-run with --shell-on-failure to inspect the build environment"); - } - if let Err(cleanup_error) = build.driver.cleanup() { - eprintln!("Failed to clean up build environment: {cleanup_error}"); - } - stop_socket_server(); - return result; - } - - stop_socket_server(); - build - .driver - .cleanup() - .context("failed to clean up build environment")?; - result + crate::output::step("Staging source tree"); + stage_source_tree( + environment, + build_kind, + package, + request.intent.config.source_sync_mode, + request.intent.config.incremental, + ) + }, + InvocationKind::Build(build_kind), + &version, + |claimed| { + let built = build_commands(claimed); + let shell_worthy = built.is_err(); + let result = built.and_then(|()| { + let environment = claimed.environment(); + crate::output::stage(&format!( + "Exporting artifacts to {}", + request.intent.output_dir.display() + )); + let changes_file = artifacts::export_build_artifacts( + &environment.work_dir(), + &request.intent.output_dir, + build_kind, + )?; + if sign.source.is_enabled() { + crate::output::stage(&format!("Signing {}", environment.package_identifier)); + crate::sign::sign_file( + &changes_file, + sign, + sign.source, + sign.notify, + &environment.package_identifier, + )?; + } + Ok(changes_file) + }); + let success = result.is_ok(); + let changes_path = match (&result, build_kind) { + (Ok(path), BuildKind::Binary) => Some(path.clone()), + _ => None, + }; + let (value, failure_lines) = match result { + Ok(path) => (Ok(path), Vec::new()), + Err(error) => { + let failure_lines = vec![format!("Build failed: {error}")]; + (Err(error), failure_lines) + } + }; + Ok(CommandCompletion { + value, + success, + shell_worthy, + finish_error_fails_command: success, + changes_path, + failure_lines, + }) + }, + ) + .await } -pub fn build_package( +pub async fn build_package( intent: &BuildIntent, target: &PackageTarget, changes_options: &[String], -) -> anyhow::Result { +) -> anyhow::Result { let request = BuildRequest { intent, target }; - run_build(&request, |build| { + run_build(&request, |claimed| { crate::output::stage("Building binary packages"); - let staged_source_dir = stage_dir(&build.environment, BuildKind::Binary); + let staged_source_dir = stage_dir(claimed.environment(), BuildKind::Binary); // build-essential is an implicit dependency that `apt-get build-dep` // won't resolve, so install it explicitly. No-op when the environment // already has it (idempotent, and the bare driver runs on the host). - build.driver.run_command_checked( + claimed.driver().run_command_checked( &["apt-get", "-y", "install", "build-essential"], &staged_source_dir, true, &[], )?; - build.driver.run_command_checked( + claimed.driver().run_command_checked( &["apt-get", "-y", "build-dep", "."], &staged_source_dir, true, @@ -407,11 +239,11 @@ pub fn build_package( let inherited_options = std::env::var("DEB_BUILD_OPTIONS").ok(); let options = deb_build_options( inherited_options.as_deref(), - build.build_debug_symbols, - build.run_test, + intent.config.build_debug_symbols, + intent.config.run_test, ); let mut env_add = vec![("DEB_BUILD_OPTIONS", options.as_str())]; - if let Some(variant) = build.host_arch_variant.as_deref() { + if let Some(variant) = intent.config.host_arch_variant.as_deref() { env_add.push(("DEB_HOST_ARCH_VARIANT", variant)); } let mut dpkg_buildpackage_args: Vec = vec![ @@ -420,7 +252,7 @@ pub fn build_package( "-uc".into(), "-ui".into(), ]; - if !build.clean { + if !intent.config.clean { // Non-incremental builds already stage a clean source tree, while // incremental builds preserve their outputs intentionally. dpkg_buildpackage_args.push("-nc".into()); @@ -431,7 +263,7 @@ pub fn build_package( dpkg_buildpackage_args.push("-b".into()); let dpkg_buildpackage_args: Vec<&str> = dpkg_buildpackage_args.iter().map(String::as_str).collect(); - build.driver.run_command_checked( + claimed.driver().run_command_checked( &dpkg_buildpackage_args, &staged_source_dir, false, @@ -439,6 +271,7 @@ pub fn build_package( )?; Ok(()) }) + .await } /// Confirm `cmd` is on `PATH`, failing with an actionable message (rather @@ -500,16 +333,16 @@ pub async fn build_source_package( target: &PackageTarget, changes_options: &[String], include_orig: bool, -) -> anyhow::Result { +) -> anyhow::Result { if intent.driver == DriverType::Bare { check_dpkg_buildpackage_available()?; } // dpkg-source looks for the orig tarball in the parent of the source - // dir it builds, which inside the environment is the work dir. - let orig_fetch_dir = intent - .config - .temp_build_dir + // dir it builds, which inside the environment is the work dir. Fetch + // into the data directory first; `temp_build_dir` no longer exists. + let orig_fetch_dir = crate::data_dir::data_dir() + .context("locating the debmagic data directory")? .join("orig") .join(target.package.name()); let orig_tarball = crate::upstream::orig::fetch_orig_tarball( @@ -527,20 +360,21 @@ pub async fn build_source_package( })?; let request = BuildRequest { intent, target }; - run_build(&request, |build| { + run_build(&request, |claimed| { crate::output::stage("Building source package"); - let staged_source_dir = stage_dir(&build.environment, BuildKind::Source); + let staged_source_dir = stage_dir(claimed.environment(), BuildKind::Source); if let Some(tarball) = &orig_tarball { // the work dir is the staged source dir's parent, which is where // dpkg-source looks for the tarball - let destination = build - .environment - .work_dir() - .join(tarball.file_name().expect("orig tarball has a file name")); + let destination = claimed.environment().work_dir().join( + tarball + .file_name() + .expect("orig tarball has a file name"), + ); stage_file(tarball, &destination)?; } - if build.clean { - build.driver.run_command_checked( + if intent.config.clean { + claimed.driver().run_command_checked( &["apt-get", "-y", "build-dep", "."], &staged_source_dir, true, @@ -555,7 +389,7 @@ pub async fn build_source_package( "-uc".into(), "-ui".into(), ]; - if !build.clean { + if !intent.config.clean { args.push("-nc".into()); } // -sa/-sd decide whether the .changes references the orig tarball @@ -568,9 +402,10 @@ pub async fn build_source_package( args.push(format!("--changes-option={option}")); } let args: Vec<&str> = args.iter().map(String::as_str).collect(); - let result = build - .driver - .run_command_checked(&args, &staged_source_dir, false, &[]); + let result = + claimed + .driver() + .run_command_checked(&args, &staged_source_dir, false, &[]); if result.is_err() && needs_quilt_unapply(target.package.source_format(), &staged_source_dir) { @@ -581,6 +416,7 @@ pub async fn build_source_package( result?; Ok(()) }) + .await .context("failed to build source package") } @@ -638,6 +474,7 @@ pub async fn build(command: BuildCommand) -> anyhow::Result<()> { .context("host's /etc/os-release does not match the build target distro")?; } build_package(&intent, &target, &command.changes_options) + .await .context("Building the package failed")? }; @@ -662,19 +499,6 @@ pub async fn build(command: BuildCommand) -> anyhow::Result<()> { Ok(()) } -/// Run the `debmagic shell` command: attach to the currently active -/// build environment and open an interactive shell in it. -pub fn shell( - fallback_dir: &Path, - source_dir: Option<&Path>, - config_file: Option<&Path>, -) -> anyhow::Result<()> { - let source_dir = crate::package::resolve_source_dir(fallback_dir, source_dir)?; - let config = Config::load(Some(&source_dir), config_file)?; - let identity = crate::package::load_package(&source_dir)?; - get_shell_in_build(&config, &identity) -} - #[cfg(test)] mod tests { use super::*; diff --git a/packages/debmagic/src/build/source.rs b/packages/debmagic/src/build/source.rs index 08540426..1c898c04 100644 --- a/packages/debmagic/src/build/source.rs +++ b/packages/debmagic/src/build/source.rs @@ -524,7 +524,7 @@ pub fn stage_source_tree( mod tests { use std::os::unix::fs::MetadataExt; - use crate::driver::{DriverType, Environment, EnvironmentPurpose}; + use crate::driver::{DriverType, Environment, EnvironmentPurpose, Persistence}; use super::*; @@ -547,9 +547,10 @@ mod tests { driver: DriverType::Bare, package_name: "example".to_string(), package_identifier: "example-1.0".to_string(), + source_dir: source_dir.clone(), root_dir: build_root_dir.clone(), distro: debmagic_common::distro::get_distro_version("trixie").unwrap(), - persistent: true, + persistence: Persistence::Always, purpose: EnvironmentPurpose::Build, }; environment.create_dirs()?; @@ -615,9 +616,10 @@ mod tests { driver: DriverType::Bare, package_name: "example".to_string(), package_identifier: "example-1.0".to_string(), + source_dir: PathBuf::from("/src"), root_dir: PathBuf::from("/tmp/debmagic/example-1.0"), distro: debmagic_common::distro::get_distro_version("trixie").unwrap(), - persistent: true, + persistence: Persistence::Always, purpose: EnvironmentPurpose::Build, }; @@ -665,9 +667,10 @@ mod tests { driver: DriverType::Bare, package_name: "example".to_string(), package_identifier: "example-1.0".to_string(), + source_dir: source_dir.clone(), root_dir: test_root.join("build"), distro: debmagic_common::distro::get_distro_version("trixie").unwrap(), - persistent: true, + persistence: Persistence::Always, purpose: EnvironmentPurpose::Build, }; environment.create_dirs()?; diff --git a/packages/debmagic/src/build_intent.rs b/packages/debmagic/src/build_intent.rs index 891fc9fd..52cf1f9c 100644 --- a/packages/debmagic/src/build_intent.rs +++ b/packages/debmagic/src/build_intent.rs @@ -5,8 +5,7 @@ use anyhow::Context; use crate::{ build::source::SourceSyncMode, config::Config, - driver::{DriverType, config::DriverOverrides}, - package::resolve_source_dir, + driver::{DriverType, Persistence, config::DriverOverrides}, sign::{SignMode, SignTool}, }; @@ -35,7 +34,7 @@ pub struct BuildIntentInput { pub config_file: Option, pub driver: Option, pub kind: BuildKind, - pub persistent: Option, + pub persistent: Option, pub incremental: Option, pub debug_symbols: Option, pub test: Option, @@ -47,7 +46,6 @@ pub struct BuildIntentInput { pub clean: Option, pub source_sync: Option, pub host_arch_variant: Option, - pub shell_on_failure: Option, pub driver_overrides: DriverOverrides, } @@ -60,13 +58,15 @@ pub struct BuildIntent { pub output_dir: PathBuf, pub driver: DriverType, pub kind: BuildKind, - pub shell_on_failure: bool, pub config: Config, pub driver_overrides: DriverOverrides, } pub fn resolve_build_intent(input: BuildIntentInput) -> anyhow::Result { - let source_dir = resolve_source_dir(&input.fallback_dir, input.source_dir.as_deref())?; + // Canonicalized: Environment ids and registry lookups key on the + // source_dir string, so all entry points must agree on one spelling. + let source_dir = std::fs::canonicalize(input.source_dir.unwrap_or(input.fallback_dir)) + .context("resolving source dir failed")?; let mut config = Config::load(Some(&source_dir), input.config_file.as_deref())?; @@ -121,17 +121,14 @@ pub fn resolve_build_intent(input: BuildIntentInput) -> anyhow::Result anyhow::Result<()> { let cfg = Config::load(None, Some(&asset_config()))?; - assert!(cfg.driver.persistent); + assert_eq!(cfg.driver.persistent, Persistence::Always); assert_eq!( cfg.driver.docker.base_images.get("debian:trixie"), Some(&"some-debian-trixie-image:latest".to_string()) @@ -268,12 +264,12 @@ mod tests { fn resolve_applies_incremental_implies_persistent() -> anyhow::Result<()> { let dir = std::env::temp_dir(); let mut input = base_input(dir); - input.persistent = Some(false); + input.persistent = Some(Persistence::No); input.incremental = Some(true); let intent = resolve_build_intent(input)?; assert!(intent.config.incremental); - assert!(intent.config.driver.persistent); + assert_eq!(intent.config.driver.persistent, Persistence::Always); Ok(()) } @@ -281,34 +277,20 @@ mod tests { fn resolve_honours_persistent_without_incremental() -> anyhow::Result<()> { let dir = std::env::temp_dir(); let mut input = base_input(dir); - // config1.toml has persistent = true; CLI can turn it off - input.persistent = Some(false); + // config1.toml has persistent = "always"; CLI can turn it off + input.persistent = Some(Persistence::No); input.incremental = Some(false); let intent = resolve_build_intent(input)?; assert!(!intent.config.incremental); - assert!(!intent.config.driver.persistent); + assert_eq!(intent.config.driver.persistent, Persistence::No); Ok(()) } #[test] - fn resolve_passes_through_shell_on_failure() -> anyhow::Result<()> { + fn leftover_shell_on_failure_key_is_ignored() -> anyhow::Result<()> { let dir = std::env::temp_dir(); let mut input = base_input(dir); - input.shell_on_failure = Some(true); - - let intent = resolve_build_intent(input)?; - assert!(intent.shell_on_failure); - Ok(()) - } - - #[test] - fn resolve_shell_on_failure_from_config() -> anyhow::Result<()> { - let dir = std::env::temp_dir(); - let mut input = base_input(dir); - input.config_file = None; - input.shell_on_failure = None; - let config_path = std::env::temp_dir().join(format!( "debmagic-shell-on-failure-{}.toml", std::process::id() @@ -317,34 +299,23 @@ mod tests { input.config_file = Some(config_path.clone()); let intent = resolve_build_intent(input)?; - assert!(intent.shell_on_failure); + assert_eq!(intent.config.driver.persistent, Persistence::OnFailure); std::fs::remove_file(config_path)?; Ok(()) } - #[test] - fn resolve_keeps_docker_base_image_override() -> anyhow::Result<()> { - let dir = std::env::temp_dir(); - let mut input = base_input(dir); - input.driver_overrides.docker.base_image = Some("custom:image".to_string()); - - let intent = resolve_build_intent(input)?; - assert_eq!( - intent.driver_overrides.docker.base_image.as_deref(), - Some("custom:image") - ); - Ok(()) - } - #[test] fn resolve_absolutizes_paths() -> anyhow::Result<()> { let dir = std::env::temp_dir(); let intent = resolve_build_intent(base_input(dir.clone()))?; assert!(intent.source_dir.is_absolute()); assert!(intent.output_dir.is_absolute()); - assert_eq!(intent.source_dir, std::path::absolute(&dir)?); + assert_eq!(intent.source_dir, std::fs::canonicalize(&dir)?); // default output dir is build/, relative to the package root - assert_eq!(intent.output_dir, std::path::absolute(dir.join("build"))?); + assert_eq!( + intent.output_dir, + std::path::absolute(std::fs::canonicalize(&dir)?.join("build"))? + ); Ok(()) } diff --git a/packages/debmagic/src/cli.rs b/packages/debmagic/src/cli.rs index 977f4231..37a70202 100644 --- a/packages/debmagic/src/cli.rs +++ b/packages/debmagic/src/cli.rs @@ -48,8 +48,12 @@ pub struct Cli { pub enum Commands { #[command(about = "Build a debian package: 'binary' (.deb) or 'source' (.dsc) packages")] Build(Box), - #[command(about = "Open an interactive shell to the currently active build environment")] - Shell(ShellSubcommandArgs), + #[command( + name = "environment", + alias = "env", + about = "List, clean, or open a shell in a Driver Environment" + )] + Environment(EnvSubcommandArgs), #[command(about = "Run the package's declared Debian autopkgtest tests against a prior build")] Test(TestSubcommandArgs), #[command(about = "Check the project")] @@ -229,7 +233,7 @@ pub struct CommonBuildArgs { num_args = 0..=1, default_missing_value = "true", action = clap::ArgAction::Set, - help = "Synchronize changed source inputs to preserve build outputs. Implies --persistent" + help = "Synchronize changed source inputs to preserve build outputs. Forces --persistent=always" )] pub incremental: Option, @@ -243,11 +247,11 @@ pub struct CommonBuildArgs { #[arg( long, num_args = 0..=1, - default_missing_value = "true", + default_missing_value = "always", action = clap::ArgAction::Set, - help = "Keep the build environment for reuse after the build finishes" + help = "How long the build environment outlives this build: 'on-failure' (the default) keeps it when the build command fails, 'always' keeps it after every build, 'no' tears it down. A bare --persistent means 'always'. A failed build that keeps the environment prints `debmagic env shell `." )] - pub persistent: Option, + pub persistent: Option, #[arg( long = "source-sync", @@ -346,15 +350,6 @@ pub struct CommonBuildArgs { )] pub clean: Option, - #[arg( - long = "shell-on-failure", - num_args = 0..=1, - default_missing_value = "true", - action = clap::ArgAction::Set, - help = "On build failure, drop into an interactive shell in the build environment when stdout is a TTY. Defaults to the 'shell_on_failure' setting in the config file (false if unset)." - )] - pub shell_on_failure: Option, - #[command(flatten)] pub common: CommonCli, @@ -429,9 +424,39 @@ pub struct SourceTargetArgs { } #[derive(Args, Debug)] -pub struct ShellSubcommandArgs { - #[command(flatten)] - pub common: CommonCli, +pub struct EnvSubcommandArgs { + #[command(subcommand)] + pub command: EnvCommands, +} + +#[derive(Subcommand, Debug)] +pub enum EnvCommands { + #[command(about = "List Environments in the machine-local registry")] + List, + #[command(about = "Destroy all Stale Environments")] + Clean, + #[command( + about = "Destroy Environments, even healthy ones", + visible_alias = "remove" + )] + Rm { + #[arg(required = true, help = "Environment ids (see `debmagic env list`)")] + ids: Vec, + #[arg( + long, + help = "Destroy even when the Environment is Unreachable, without Driver cooperation" + )] + force: bool, + }, + #[command(about = "Open an interactive shell in an Environment")] + Shell { + #[arg( + help = "Environment id; when omitted, unique Environment for the current Source tree" + )] + id: Option, + #[command(flatten)] + common: CommonCli, + }, } #[derive(Args, Debug)] @@ -439,18 +464,18 @@ pub struct TestSubcommandArgs { #[arg( short, long, - help = "Driver type for the test environment. Defaults to the driver recorded in the prior build's environment.json." + help = "Driver type for the test environment. Defaults to the driver recorded on the prior binary-build Invocation." )] pub driver: Option, #[arg( long, num_args = 0..=1, - default_missing_value = "true", + default_missing_value = "always", action = clap::ArgAction::Set, - help = "Keep the test environment for reuse after the test run finishes" + help = "How long the test environment outlives this run: 'on-failure' (the default) keeps it when the tests fail, 'always' keeps it after every run, 'no' tears it down. A bare --persistent means 'always'. A failed run that keeps the environment prints `debmagic env shell `." )] - pub persistent: Option, + pub persistent: Option, #[command(flatten)] pub docker: DockerArgs, @@ -481,7 +506,7 @@ pub struct TestSubcommandArgs { #[arg( long, - help = "Override the target distribution for the test environment. Defaults to the distro recorded in the prior build's environment.json, not the changelog." + help = "Override the target distribution for the test environment. Defaults to the distro of the prior binary-build Invocation, not the changelog." )] pub distro: Option, @@ -505,15 +530,6 @@ pub struct TestSubcommandArgs { )] pub allow_host_test: bool, - #[arg( - long = "shell-on-failure", - num_args = 0..=1, - default_missing_value = "true", - action = clap::ArgAction::Set, - help = "On test failure, drop into an interactive shell in the test environment when stdout is a TTY. Defaults to the 'shell_on_failure' setting in the config file (false if unset)." - )] - pub shell_on_failure: Option, - #[command(flatten)] pub common: CommonCli, } @@ -690,6 +706,7 @@ impl TestSubcommandArgs { #[cfg(test)] mod tests { use super::*; + use crate::driver::Persistence; use test_case::test_case; fn parse_test(args: &[&str]) -> TestSubcommandArgs { @@ -700,16 +717,19 @@ mod tests { } } - // an absent flag must stay None so the config value applies #[test_case(&[], None; "absent")] - #[test_case(&["--shell-on-failure"], Some(true); "bare flag")] - #[test_case(&["--shell-on-failure=false"], Some(false); "explicit false")] - fn tri_state_flag(args: &[&str], expected: Option) { + #[test_case(&["--persistent"], Some(Persistence::Always); "bare flag")] + #[test_case(&["--persistent=always"], Some(Persistence::Always); "always")] + #[test_case(&["--persistent=on-failure"], Some(Persistence::OnFailure); "on failure")] + #[test_case(&["--persistent=no"], Some(Persistence::No); "no")] + fn persistent_flag(args: &[&str], expected: Option) { let args = parse_test(args); - assert_eq!(args.shell_on_failure, expected); - if expected.is_none() { - assert_eq!(args.persistent, None); - assert_eq!(args.proposed, None); - } + assert_eq!(args.persistent, expected); + } + + #[test] + fn persistent_rejects_a_boolean() { + let error = Cli::try_parse_from(["debmagic", "test", "--persistent=true"]).unwrap_err(); + assert!(error.to_string().contains("always"), "{error}"); } } diff --git a/packages/debmagic/src/config.rs b/packages/debmagic/src/config.rs index 8f4991ad..8b34247e 100644 --- a/packages/debmagic/src/config.rs +++ b/packages/debmagic/src/config.rs @@ -6,7 +6,7 @@ use crate::driver::config::DriverConfig; use crate::sign::{SignMode, SignTool}; use crate::upload::UploadConfig; use crate::upstream::orig::OrigTarballConfig; -use anyhow::{Context, anyhow}; +use anyhow::{Context, anyhow, bail}; use config::{Config as ConfigBuilder, File}; use serde::{Deserialize, Serialize}; @@ -224,7 +224,8 @@ pub fn run_config(command: ConfigCommand) -> anyhow::Result<()> { #[serde(default)] pub struct Config { pub driver: DriverConfig, - pub temp_build_dir: PathBuf, + /// Parent directory of Environment Host roots. + pub environments_dir: PathBuf, /// Where build artifacts are exported; relative paths resolve against /// the package root. pub output_dir: PathBuf, @@ -249,9 +250,6 @@ pub struct Config { /// builds already stage a clean source tree and incremental builds preserve /// outputs intentionally. pub clean: bool, - /// On build or test failure, drop into an interactive shell in the - /// environment when stdout is a TTY. - pub shell_on_failure: bool, /// Build for a dpkg architecture variant (e.g. `amd64v3` on Ubuntu), /// exported as `DEB_HOST_ARCH_VARIANT` for the build. pub host_arch_variant: Option, @@ -305,7 +303,7 @@ impl Default for Config { fn default() -> Self { Self { driver: DriverConfig::default(), - temp_build_dir: PathBuf::from("/tmp/debmagic"), + environments_dir: crate::data_dir::default_environments_dir(), output_dir: PathBuf::from("build"), incremental: false, source_sync_mode: SourceSyncMode::default(), @@ -316,7 +314,6 @@ impl Default for Config { orig_tarball: OrigTarballConfig::default(), upstream: UpstreamConfig::default(), clean: false, - shell_on_failure: false, host_arch_variant: None, } } @@ -381,11 +378,11 @@ impl Config { let build = builder .build() .context("Failed to initialize config reader")?; - let config: anyhow::Result = build + let mut config: Self = build .try_deserialize() - .map_err(|e| anyhow!("Failed to read config: {e}")); - - config + .map_err(|e| anyhow!("Failed to read config: {e}"))?; + config.environments_dir = resolve_environments_dir(&config.environments_dir)?; + Ok(config) } /// Look up a dotted key (e.g. `sign.key`) in the serialized config, @@ -413,11 +410,44 @@ impl Config { } } +/// Host roots are stored in the Environment registry and used from any +/// working directory later, so `environments_dir` must not depend on the cwd. +/// Config layers are merged before this point, so there is no single file to +/// resolve a relative value against: only `~/` is expanded. +fn resolve_environments_dir(dir: &Path) -> anyhow::Result { + if let Ok(rest) = dir.strip_prefix("~") { + let home = dirs::home_dir().context("cannot expand ~ in environments_dir")?; + return Ok(home.join(rest)); + } + if !dir.is_absolute() { + bail!( + "environments_dir must be an absolute path (or start with ~/), got {}", + dir.display() + ); + } + Ok(dir.to_path_buf()) +} + #[cfg(test)] mod tests { use super::*; use crate::driver::DriverType; + #[test] + fn environments_dir_must_not_depend_on_the_cwd() -> anyhow::Result<()> { + assert_eq!( + resolve_environments_dir(Path::new("/var/debmagic"))?, + PathBuf::from("/var/debmagic") + ); + let home = dirs::home_dir().unwrap(); + assert_eq!( + resolve_environments_dir(Path::new("~/envs"))?, + home.join("envs") + ); + assert!(resolve_environments_dir(Path::new("envs")).is_err()); + Ok(()) + } + #[test] fn it_loads_a_simple_config() -> Result<(), anyhow::Error> { let test_asset_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -429,7 +459,7 @@ mod tests { ConfigPathStatus::Used, )])?; assert_eq!(cfg.driver.default, Some(DriverType::Docker)); - assert!(cfg.driver.persistent); + assert_eq!(cfg.driver.persistent, crate::driver::Persistence::Always); assert!( cfg.driver.docker.base_images.get("debian:trixie") diff --git a/packages/debmagic/src/data_dir.rs b/packages/debmagic/src/data_dir.rs new file mode 100644 index 00000000..3424cd1f --- /dev/null +++ b/packages/debmagic/src/data_dir.rs @@ -0,0 +1,86 @@ +use std::{ffi::OsStr, path::PathBuf}; + +use anyhow::{Context, bail}; + +/// Machine-local data directory: `$DEBMAGIC_DATA_DIR`, or +/// `$XDG_DATA_HOME/debmagic` (`~/.local/share/debmagic`). +/// +/// A relative `DEBMAGIC_DATA_DIR` is resolved to an absolute path against the +/// current directory, so later use does not depend on a different working +/// directory. +pub fn data_dir() -> anyhow::Result { + let override_dir = std::env::var_os("DEBMAGIC_DATA_DIR"); + resolve_data_dir(override_dir.as_deref(), dirs::data_dir().as_deref()) +} + +pub fn registry_path() -> anyhow::Result { + Ok(data_dir()?.join("db.sqlite")) +} + +pub fn default_environments_dir() -> PathBuf { + environments_dir_for(data_dir()) +} + +fn resolve_data_dir( + override_dir: Option<&OsStr>, + xdg_data_home: Option<&std::path::Path>, +) -> anyhow::Result { + if let Some(path) = override_dir { + if path.is_empty() { + bail!("DEBMAGIC_DATA_DIR is set but empty"); + } + return std::path::absolute(path).context("resolving DEBMAGIC_DATA_DIR failed"); + } + + xdg_data_home + .map(|path| path.join("debmagic")) + .context("cannot determine the user data directory") +} + +fn environments_dir_for(data: anyhow::Result) -> PathBuf { + data.unwrap_or_else(|_| PathBuf::from("/tmp/debmagic")) + .join("environments") +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::Path; + + #[test] + fn debmagic_data_dir_overrides_xdg() { + let dir = resolve_data_dir(Some(OsStr::new("/tmp/debmagic-data-test")), None).unwrap(); + assert_eq!(dir, PathBuf::from("/tmp/debmagic-data-test")); + assert_eq!( + environments_dir_for(Ok(dir)), + PathBuf::from("/tmp/debmagic-data-test/environments") + ); + } + + #[test] + fn relative_debmagic_data_dir_is_absolute() { + let dir = resolve_data_dir(Some(OsStr::new("debmagic-data")), None).unwrap(); + assert!(dir.is_absolute()); + assert!(dir.ends_with("debmagic-data")); + } + + #[test] + fn empty_debmagic_data_dir_is_rejected() { + let error = resolve_data_dir(Some(OsStr::new("")), None) + .unwrap_err() + .to_string(); + assert!(error.contains("empty")); + } + + #[test] + fn xdg_data_home_is_used_without_override() { + let dir = resolve_data_dir(None, Some(Path::new("/xdg"))).unwrap(); + assert_eq!(dir, PathBuf::from("/xdg/debmagic")); + } + + #[test] + fn missing_data_dir_falls_back_for_environments() { + let environments = environments_dir_for(Err(anyhow::anyhow!("no home"))); + assert_eq!(environments, PathBuf::from("/tmp/debmagic/environments")); + } +} diff --git a/packages/debmagic/src/driver/config.rs b/packages/debmagic/src/driver/config.rs index a8261a30..93bf145f 100644 --- a/packages/debmagic/src/driver/config.rs +++ b/packages/debmagic/src/driver/config.rs @@ -1,3 +1,4 @@ +use clap::ValueEnum; use serde::{Deserialize, Serialize}; use crate::driver::DriverType; @@ -6,6 +7,46 @@ use crate::driver::driver_docker::{DriverDockerConfig, DriverDockerConfigOverrid use crate::driver::driver_lxd::{DriverLxdConfig, DriverLxdConfigOverrides}; use crate::time::RefreshPolicy; +/// How long the current claim lets an Environment outlive that claim. +/// +/// The claim replaces whatever the registry recorded. `on-failure` and +/// `always` reuse a kept Environment; `no` discards it. +#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, ValueEnum)] +#[serde(rename_all = "kebab-case")] +pub enum Persistence { + /// Torn down when the claim finishes, discarding one already kept. + No, + /// Kept only when the build or TestRun command itself fails. + #[default] + OnFailure, + /// Kept when the claim finishes. + Always, +} + +impl Persistence { + /// Whether this claim reuses an Environment a previous claim kept. + pub fn reuses(self) -> bool { + matches!(self, Self::OnFailure | Self::Always) + } + + pub fn as_str(self) -> &'static str { + match self { + Self::No => "no", + Self::OnFailure => "on-failure", + Self::Always => "always", + } + } + + pub fn from_name(name: &str) -> anyhow::Result { + match name { + "no" => Ok(Self::No), + "on-failure" => Ok(Self::OnFailure), + "always" => Ok(Self::Always), + other => anyhow::bail!("unknown Persistence '{other}'"), + } + } +} + #[derive(Serialize, Deserialize, Debug, Clone, Default)] #[serde(default)] pub struct DriverConfig { @@ -13,7 +54,8 @@ pub struct DriverConfig { /// require a driver, from the CLI or here; source builds fall back to /// `bare`. pub default: Option, - pub persistent: bool, + /// Lifetime of the Environment after this claim. Unset is `on-failure`. + pub persistent: Persistence, /// Not used by the bare driver, which builds on the host's own sources. pub apt_mirror: Option, /// Also enable the `-proposed` pocket. Not used by the bare diff --git a/packages/debmagic/src/driver/driver_bare.rs b/packages/debmagic/src/driver/driver_bare.rs index 86058c3d..9c3cb073 100644 --- a/packages/debmagic/src/driver/driver_bare.rs +++ b/packages/debmagic/src/driver/driver_bare.rs @@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize}; use crate::driver::{ DriverType, Environment, EnvironmentDriver, EnvironmentMetadata, IsolationCapability, - config::DriverConfig, + ResourceStatus, config::DriverConfig, }; use crate::subprocess::{self, Capture, CommandResult}; @@ -75,15 +75,15 @@ impl EnvironmentDriver for DriverBare { subprocess::command(command).capture(capture).run() } - fn cleanup(&self) -> anyhow::Result<()> { - Ok(()) - } - - fn interactive_shell(&self, _cwd: &Path) -> std::io::Result<()> { - println!( - "source directory of current package build in {}", - _cwd.display() - ); + fn interactive_shell(&self, cwd: &Path) -> std::io::Result<()> { + let shell = + std::env::var_os("SHELL").unwrap_or_else(|| std::ffi::OsString::from("/bin/sh")); + let mut command = Command::new(shell); + command.current_dir(cwd); + let status = command.status()?; + if !status.success() { + return Err(std::io::Error::other(format!("shell exited with {status}"))); + } Ok(()) } @@ -101,4 +101,16 @@ impl EnvironmentDriver for DriverBare { } Ok(()) } + + fn probe_resource(&self) -> ResourceStatus { + if self.environment.root_dir.exists() { + ResourceStatus::Present + } else { + ResourceStatus::Absent + } + } + + fn destroy_resource(&self) -> anyhow::Result<()> { + Ok(()) + } } diff --git a/packages/debmagic/src/driver/driver_docker.rs b/packages/debmagic/src/driver/driver_docker.rs index 6daaa1dc..41e9017a 100644 --- a/packages/debmagic/src/driver/driver_docker.rs +++ b/packages/debmagic/src/driver/driver_docker.rs @@ -11,9 +11,9 @@ use serde::{Deserialize, Serialize}; use crate::driver::{ APT_MIRROR_SCRIPT, DriverType, ENVIRONMENT_DIR_IN_CONTAINER, Environment, EnvironmentDriver, - EnvironmentMetadata, IsolationCapability, config::DriverConfig, container_name_from_metadata, - container_name_metadata, environment_fingerprint, refresh_apt_index, resource_name, - run_checked, translate_path_in_container, + EnvironmentMetadata, IsolationCapability, ResourceStatus, config::DriverConfig, + container_name_from_metadata, container_name_metadata, environment_fingerprint, + refresh_apt_index, resource_name, run_checked, translate_path_in_container, }; use crate::subprocess::{self, Capture, CommandResult}; @@ -240,6 +240,19 @@ impl DriverDocker { Ok(output?.exit_code == 0) } + fn container_name_for(environment: &Environment) -> String { + resource_name( + "debmagic", + &environment.package_name, + &sanitize_docker_reference(&environment.id()), + ) + } + + /// Driver metadata known before the container exists. + pub fn planned_metadata(environment: &Environment) -> HashMap { + container_name_metadata(&Self::container_name_for(environment)) + } + pub fn create( environment: &Environment, driver_config: &DriverConfig, @@ -273,11 +286,7 @@ impl DriverDocker { container_fingerprint_parts.push(purpose); } let desired_fingerprint = environment_fingerprint(&container_fingerprint_parts); - let container_name = resource_name( - "debmagic", - &environment.package_name, - &sanitize_docker_reference(&environment.identifier()), - ); + let container_name = Self::container_name_for(environment); let mut driver = Self { environment: environment.clone(), container_name, @@ -287,7 +296,7 @@ impl DriverDocker { .as_deref() == Some(&desired_fingerprint); - let mut reuse = environment.persistent && environment_matches; + let mut reuse = environment.persistence.reuses() && environment_matches; if reuse { if !driver.container_is_running()? { driver.container_start()?; @@ -366,7 +375,7 @@ impl DriverDocker { ) -> anyhow::Result { Ok(Self { environment: environment.clone(), - container_name: container_name_from_metadata(metadata)?, + container_name: container_name_from_metadata(metadata), reused_environment: true, }) } @@ -416,14 +425,6 @@ impl EnvironmentDriver for DriverDocker { subprocess::command(exec_cmd).capture(capture).run() } - fn cleanup(&self) -> anyhow::Result<()> { - if self.environment.persistent { - Ok(()) - } else { - self.container_remove_force() - } - } - fn reset_root(&self) -> std::io::Result<()> { let find_cmd = [ "find", @@ -480,4 +481,35 @@ impl EnvironmentDriver for DriverDocker { fn isolation_capability(&self) -> IsolationCapability { IsolationCapability::Container } + + fn probe_resource(&self) -> ResourceStatus { + match Command::new("docker") + .args(["inspect", &self.container_name]) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .output() + { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + ResourceStatus::Unreachable + } + Err(_) => ResourceStatus::Unreachable, + Ok(output) if output.status.success() => ResourceStatus::Present, + Ok(output) => { + // Docker says "No such object", Podman's docker shim "no such object". + let stderr = String::from_utf8_lossy(&output.stderr).to_lowercase(); + if stderr.contains("no such object") || stderr.contains("no such container") { + ResourceStatus::Absent + } else { + ResourceStatus::Unreachable + } + } + } + } + + fn destroy_resource(&self) -> anyhow::Result<()> { + match self.probe_resource() { + ResourceStatus::Absent => Ok(()), + ResourceStatus::Present | ResourceStatus::Unreachable => self.container_remove_force(), + } + } } diff --git a/packages/debmagic/src/driver/driver_lxd.rs b/packages/debmagic/src/driver/driver_lxd.rs index ea69dabb..9ec62e27 100644 --- a/packages/debmagic/src/driver/driver_lxd.rs +++ b/packages/debmagic/src/driver/driver_lxd.rs @@ -1,4 +1,5 @@ use std::{ + collections::HashMap, fs, path::{Path, PathBuf}, process::{Command, Stdio}, @@ -9,9 +10,9 @@ use serde::{Deserialize, Serialize}; use crate::driver::{ APT_MIRROR_SCRIPT, DriverType, ENVIRONMENT_DIR_IN_CONTAINER, Environment, EnvironmentDriver, - EnvironmentMetadata, IsolationCapability, config::DriverConfig, container_name_from_metadata, - container_name_metadata, environment_fingerprint, refresh_apt_index, resource_name, - run_checked, translate_path_in_container, + EnvironmentMetadata, IsolationCapability, ResourceStatus, config::DriverConfig, + container_name_from_metadata, container_name_metadata, environment_fingerprint, + refresh_apt_index, resource_name, run_checked, translate_path_in_container, }; use crate::subprocess::{self, Capture, CommandResult}; @@ -36,6 +37,7 @@ impl LxdVariant { const BUILD_USER_UID: u32 = 1000; const BUILD_USER_GID: u32 = 1000; const ENVIRONMENT_CONFIG_KEY: &str = "user.debmagic.environment"; +const PROJECT_KEY: &str = "project"; const ENVIRONMENT_SETUP_VERSION: &str = "dpkg-dev-norec python3; build-user-v1; raw.idmap-v1"; // ── Config ──────────────────────────────────────────────────────────────────── @@ -181,7 +183,7 @@ impl DriverLxd { resource_name( "debmagic-src", &self.environment.package_name, - &self.environment.identifier(), + &self.environment.id(), ) } @@ -245,6 +247,42 @@ impl DriverLxd { Ok((!fingerprint.is_empty()).then_some(fingerprint)) } + fn container_name_for(environment: &Environment) -> String { + resource_name("debmagic", &environment.package_name, &environment.id()) + } + + /// Project: CLI override > config file value. + fn resolve_project( + driver_config: &DriverConfig, + overrides: &DriverLxdConfigOverrides, + ) -> Option { + overrides + .project + .clone() + .or_else(|| driver_config.lxd.project.clone()) + } + + fn metadata_for(container_name: &str, project: Option<&str>) -> HashMap { + let mut meta = container_name_metadata(container_name); + if let Some(project) = project { + meta.insert(PROJECT_KEY.to_string(), project.to_string()); + } + meta + } + + /// Driver metadata known before the container exists. The project must be + /// recorded up front: reattachment cannot recover it from the config. + pub fn planned_metadata( + environment: &Environment, + driver_config: &DriverConfig, + overrides: &DriverLxdConfigOverrides, + ) -> HashMap { + Self::metadata_for( + &Self::container_name_for(environment), + Self::resolve_project(driver_config, overrides).as_deref(), + ) + } + pub fn create( variant: LxdVariant, environment: &Environment, @@ -253,17 +291,8 @@ impl DriverLxd { apt_mirror: Option<&str>, proposed: bool, ) -> anyhow::Result { - let container_name = resource_name( - "debmagic", - &environment.package_name, - &environment.identifier(), - ); - - // Project: CLI override > config file value - let project = overrides - .project - .clone() - .or_else(|| driver_config.lxd.project.clone()); + let container_name = Self::container_name_for(environment); + let project = Self::resolve_project(driver_config, overrides); let base_image = overrides.base_image.clone().unwrap_or_else(|| { driver_config .lxd @@ -300,7 +329,7 @@ impl DriverLxd { let container_entry = base.container_list_entry()?; let environment_matches = container_entry.is_some() && base.container_environment_fingerprint()?.as_deref() == Some(&desired_fingerprint); - let reusing_container = environment.persistent && environment_matches; + let reusing_container = environment.persistence.reuses() && environment_matches; base.reused_environment = reusing_container; let mut initialized_container = false; @@ -329,7 +358,7 @@ impl DriverLxd { } let mut init = base.lxd_cmd("init"); - if !environment.persistent { + if environment.persistence == crate::driver::Persistence::No { init.arg("--ephemeral"); } init.args([&base_image, &container_name]); @@ -487,12 +516,12 @@ impl DriverLxd { _driver_config: &DriverConfig, metadata: &EnvironmentMetadata, ) -> anyhow::Result { - let project = metadata.driver_metadata.get("project").cloned(); + let project = metadata.driver_metadata.get(PROJECT_KEY).cloned(); Ok(Self { variant, environment: environment.clone(), - container_name: container_name_from_metadata(metadata)?, + container_name: container_name_from_metadata(metadata), project, reused_environment: true, }) @@ -593,11 +622,7 @@ impl DriverLxd { impl EnvironmentDriver for DriverLxd { fn driver_metadata(&self) -> std::collections::HashMap { - let mut meta = container_name_metadata(&self.container_name); - if let Some(ref p) = self.project { - meta.insert("project".to_string(), p.clone()); - } - meta + Self::metadata_for(&self.container_name, self.project.as_deref()) } fn run_command( @@ -615,15 +640,6 @@ impl EnvironmentDriver for DriverLxd { self.exec_in_container(cmd, Some(&container_path), requires_root, env_add, capture) } - fn cleanup(&self) -> anyhow::Result<()> { - if self.environment.persistent { - Ok(()) - } else { - // Ephemeral containers are auto-deleted after stopping. - self.container_stop() - } - } - fn reset_root(&self) -> std::io::Result<()> { self.with_running_container(|driver| { driver.exec_in_container_checked( @@ -673,4 +689,19 @@ impl EnvironmentDriver for DriverLxd { fn isolation_capability(&self) -> IsolationCapability { IsolationCapability::Container } + + fn probe_resource(&self) -> ResourceStatus { + match self.container_list_entry() { + Ok(Some(_)) => ResourceStatus::Present, + Ok(None) => ResourceStatus::Absent, + Err(_) => ResourceStatus::Unreachable, + } + } + + fn destroy_resource(&self) -> anyhow::Result<()> { + match self.container_list_entry() { + Ok(None) => Ok(()), + Ok(Some(_)) | Err(_) => self.container_delete_force(), + } + } } diff --git a/packages/debmagic/src/driver/mod.rs b/packages/debmagic/src/driver/mod.rs index b22490d0..ff759251 100644 --- a/packages/debmagic/src/driver/mod.rs +++ b/packages/debmagic/src/driver/mod.rs @@ -25,6 +25,8 @@ pub mod driver_bare; pub mod driver_docker; pub mod driver_lxd; +pub use config::Persistence; + /// Path at which the environment root is bind-mounted inside container-based /// drivers (Docker, LXD, Incus). pub const ENVIRONMENT_DIR_IN_CONTAINER: &str = "/debmagic"; @@ -169,12 +171,22 @@ pub fn container_name_metadata(name: &str) -> HashMap { HashMap::from([(CONTAINER_NAME_KEY.to_string(), name.to_string())]) } -pub fn container_name_from_metadata(metadata: &EnvironmentMetadata) -> anyhow::Result { +/// The container name for reattachment. Falls back to the deterministic name +/// derived from the Environment id when the stored metadata is missing it +/// (e.g. the creating run crashed before `driver_metadata` was recorded) — +/// the id is hex, so the Docker and LXD/Incus name schemes coincide. +pub fn container_name_from_metadata(metadata: &EnvironmentMetadata) -> String { metadata .driver_metadata .get(CONTAINER_NAME_KEY) .cloned() - .context("environment metadata has no container_name") + .unwrap_or_else(|| { + resource_name( + "debmagic", + &metadata.environment.package_name, + &metadata.environment.id(), + ) + }) } #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum, Serialize, Deserialize)] @@ -188,11 +200,28 @@ pub enum DriverType { impl std::fmt::Display for DriverType { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str( - self.to_possible_value() - .expect("no skipped variants") - .get_name(), - ) + f.write_str(self.as_str()) + } +} + +impl DriverType { + pub fn as_str(self) -> &'static str { + match self { + Self::Docker => "docker", + Self::Bare => "bare", + Self::Lxd => "lxd", + Self::Incus => "incus", + } + } + + pub fn from_name(name: &str) -> anyhow::Result { + match name { + "docker" => Ok(Self::Docker), + "bare" => Ok(Self::Bare), + "lxd" => Ok(Self::Lxd), + "incus" => Ok(Self::Incus), + other => anyhow::bail!("unknown Driver '{other}'"), + } } } @@ -218,6 +247,21 @@ pub enum EnvironmentPurpose { } impl EnvironmentPurpose { + pub fn as_str(self) -> &'static str { + match self { + Self::Build => "build", + Self::Test => "test", + } + } + + pub fn from_name(name: &str) -> anyhow::Result { + match name { + "build" => Ok(Self::Build), + "test" => Ok(Self::Test), + other => anyhow::bail!("unknown EnvironmentPurpose '{other}'"), + } + } + /// Extra part for environment fingerprints when purpose is not [`Self::Build`]. pub fn fingerprint_part(self) -> Option<&'static str> { match self { @@ -233,15 +277,57 @@ pub struct Environment { #[serde(default)] pub package_name: String, pub package_identifier: String, + pub source_dir: PathBuf, pub root_dir: PathBuf, pub distro: DistroVersion, #[serde(default)] - pub persistent: bool, + pub persistence: Persistence, #[serde(default)] pub purpose: EnvironmentPurpose, } impl Environment { + #[allow(clippy::too_many_arguments)] + pub fn new( + driver: DriverType, + package_name: &str, + package_identifier: &str, + source_dir: &Path, + distro: DistroVersion, + persistence: Persistence, + purpose: EnvironmentPurpose, + environments_dir: &Path, + ) -> Self { + let mut environment = Self { + driver, + package_name: package_name.to_string(), + package_identifier: package_identifier.to_string(), + source_dir: source_dir.to_path_buf(), + root_dir: PathBuf::new(), + distro, + persistence, + purpose, + }; + environment.root_dir = environments_dir.join(environment.id()); + environment + } + + /// Short stable Environment id derived from the identity tuple. + pub fn id(&self) -> String { + let source_dir = self.source_dir.to_string_lossy(); + let mut id = environment_fingerprint(&[ + source_dir.as_ref(), + &self.package_name, + &self.package_identifier, + self.distro.distro.as_str(), + &self.distro.codename, + self.purpose.as_str(), + self.driver.as_str(), + ]); + id.truncate(16); + id + } + pub fn identifier(&self) -> String { let base = format!( "{}-{}-{}", @@ -261,13 +347,27 @@ impl Environment { self.root_dir.join("temp") } + /// Directory the Environment module's own staging uses. Build and test + /// trees live in kind-specific directories under the work dir instead. + pub fn staged_source_dir(&self) -> PathBuf { + self.work_dir().join(&self.package_identifier) + } + pub fn create_dirs(&self) -> io::Result<()> { fs::create_dir_all(self.work_dir())?; fs::create_dir_all(self.temp_dir())?; + fs::create_dir_all(self.staged_source_dir())?; Ok(()) } } +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum ResourceStatus { + Present, + Absent, + Unreachable, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct EnvironmentMetadata { pub environment: Environment, @@ -313,8 +413,6 @@ pub trait EnvironmentDriver { Ok(()) } - fn cleanup(&self) -> anyhow::Result<()>; - fn interactive_shell(&self, cwd: &Path) -> io::Result<()>; fn driver_type(&self) -> DriverType; @@ -327,6 +425,13 @@ pub trait EnvironmentDriver { fn reused_environment(&self) -> bool { true } + + /// Whether the Driver resource for this Environment still exists. + fn probe_resource(&self) -> ResourceStatus; + + /// Destroy the Driver resource even if this is a Persistent driver. + /// Succeeds when the resource is already gone. + fn destroy_resource(&self) -> anyhow::Result<()>; } /// A live environment driver, created for one build/test run. @@ -360,14 +465,6 @@ impl EnvironmentDriver for Driver { } } - fn cleanup(&self) -> anyhow::Result<()> { - match self { - Self::Docker(d) => d.cleanup(), - Self::Bare(d) => d.cleanup(), - Self::Lxd(d) => d.cleanup(), - } - } - fn interactive_shell(&self, cwd: &Path) -> io::Result<()> { match self { Self::Docker(d) => d.interactive_shell(cwd), @@ -407,6 +504,22 @@ impl EnvironmentDriver for Driver { Self::Lxd(d) => d.reused_environment(), } } + + fn probe_resource(&self) -> ResourceStatus { + match self { + Self::Docker(d) => d.probe_resource(), + Self::Bare(d) => d.probe_resource(), + Self::Lxd(d) => d.probe_resource(), + } + } + + fn destroy_resource(&self) -> anyhow::Result<()> { + match self { + Self::Docker(d) => d.destroy_resource(), + Self::Bare(d) => d.destroy_resource(), + Self::Lxd(d) => d.destroy_resource(), + } + } } pub fn create_driver( @@ -454,6 +567,23 @@ pub fn create_driver( } } +/// The Driver metadata [`create_driver`] will produce, computed without +/// touching the Driver. Recorded before creation so a run that dies mid-setup +/// still leaves enough in the registry to find and destroy the resource. +pub fn planned_driver_metadata( + environment: &Environment, + driver_config: &DriverConfig, + overrides: &DriverOverrides, +) -> HashMap { + match environment.driver { + DriverType::Docker => DriverDocker::planned_metadata(environment), + DriverType::Bare => HashMap::new(), + DriverType::Lxd | DriverType::Incus => { + DriverLxd::planned_metadata(environment, driver_config, &overrides.lxd) + } + } +} + pub fn create_driver_from_metadata( driver_config: &DriverConfig, metadata: &EnvironmentMetadata, @@ -487,31 +617,27 @@ pub fn create_driver_from_metadata( /// Remove `root` from the host. If files are owned by a container user the host /// cannot delete, delete them from inside that environment first. Never requires /// host root. -pub fn remove_environment_root(root: &Path, driver_config: &DriverConfig) -> anyhow::Result<()> { +pub fn remove_environment_root( + root: &Path, + driver_config: &DriverConfig, + metadata: Option<&EnvironmentMetadata>, +) -> anyhow::Result<()> { if !root.exists() { return Ok(()); } match fs::remove_dir_all(root) { Ok(()) => Ok(()), Err(e) if e.kind() == io::ErrorKind::PermissionDenied => { - let metadata_path = root.join("environment.json"); - if !metadata_path.is_file() { + let Some(metadata) = metadata else { return Err(e).with_context(|| { format!( - "failed to remove {} (permission denied) and no environment.json is present to delete files from inside the environment", + "failed to remove {} (permission denied) and no Driver metadata is available to delete files from inside the environment", root.display() ) }); - } - let file = fs::OpenOptions::new() - .read(true) - .open(&metadata_path) - .with_context(|| format!("failed to open {}", metadata_path.display()))?; - let metadata: EnvironmentMetadata = - serde_json::from_reader(std::io::BufReader::new(&file)) - .with_context(|| format!("failed to parse {}", metadata_path.display()))?; + }; let driver = - create_driver_from_metadata(driver_config, &metadata).with_context(|| { + create_driver_from_metadata(driver_config, metadata).with_context(|| { format!( "failed to reattach to the environment at {} to delete privileged files", root.display() @@ -566,6 +692,7 @@ mod tests { Environment { driver: DriverType::Docker, package_identifier: package_identifier.to_string(), + source_dir: PathBuf::from("/src"), root_dir: PathBuf::from("/tmp"), distro: DistroVersion { distro: Distro::Debian, @@ -573,7 +700,7 @@ mod tests { version: "15".to_string(), is_devel: false, }, - persistent: false, + persistence: Persistence::No, package_name: "debmagic".to_string(), purpose: EnvironmentPurpose::Build, } @@ -600,6 +727,10 @@ mod tests { environment.identifier(), sample_environment("debmagic-0.0.1~alpha2").identifier() ); + assert_ne!( + environment.id(), + sample_environment("debmagic-0.0.1~alpha2").id() + ); } #[test] @@ -607,6 +738,7 @@ mod tests { let json = r#"{ "driver": "docker", "package_identifier": "pkg-1.0", + "source_dir": "/src", "root_dir": "/tmp/build", "distro": { "distro": "Debian", "codename": "forky", "version": "15" } }"#; diff --git a/packages/debmagic/src/env.rs b/packages/debmagic/src/env.rs new file mode 100644 index 00000000..ed3617d8 --- /dev/null +++ b/packages/debmagic/src/env.rs @@ -0,0 +1,177 @@ +use anyhow::{Context, bail}; + +use crate::config::Config; +use crate::environment::{ + DestroyWhen, EnvironmentStatus, Registry, assess_environment, claim_and_destroy, + select_unique_environment, +}; +use crate::package::load_package; + +pub async fn list_environments(config: &Config) -> anyhow::Result<()> { + let Some(registry) = Registry::open_default_if_exists().await? else { + println!("No Environments in the registry."); + return Ok(()); + }; + let environments = registry.list().await?; + if environments.is_empty() { + println!("No Environments in the registry."); + return Ok(()); + } + + println!( + "{:<16} {:<6} {:<8} {:<18} {:<10} {:<10} {:<6} {:<11} SOURCE", + "ID", "PURPOSE", "DRIVER", "PACKAGE", "DISTRO", "PERSIST", "STATUS", "ATTACH" + ); + for registered in environments { + let assessed = assess_environment(®istry, ®istered, &config.driver).await?; + let env = ®istered.environment; + println!( + "{:<16} {:<6} {:<8} {:<18} {:<10} {:<10} {:<6} {:<11} {}", + env.id(), + env.purpose.as_str(), + env.driver.as_str(), + env.package_identifier, + env.distro.codename, + env.persistence.as_str(), + assessed.status.as_str(), + assessed.attachments, + env.source_dir.display(), + ); + } + Ok(()) +} + +/// Destroy every Stale Environment on the machine. +pub async fn clean_environments(config: &Config) -> anyhow::Result<()> { + let Some(registry) = Registry::open_default_if_exists().await? else { + println!("Cleaned 0 Stale Environment(s); skipped 0."); + return Ok(()); + }; + let environments = registry.list().await?; + let mut cleaned = 0; + let mut skipped = 0; + for registered in environments { + let id = registered.environment.id(); + let status = match assess_environment(®istry, ®istered, &config.driver).await { + Ok(assessed) => assessed.status, + Err(error) => { + // Fail closed: never destroy what we cannot assess. + eprintln!("skipping Environment {id}: {error}"); + skipped += 1; + continue; + } + }; + match status { + EnvironmentStatus::Unreachable => { + eprintln!("skipping unreachable Environment {id}"); + skipped += 1; + } + EnvironmentStatus::Stale => { + match claim_and_destroy(®istry, &config.driver, &id, DestroyWhen::StillStale) + .await + { + Ok(true) => cleaned += 1, + Ok(false) => { + eprintln!("skipping Environment {id}: no longer Stale"); + skipped += 1; + } + Err(error) => { + eprintln!("skipping Environment {id}: {error}"); + skipped += 1; + } + } + } + EnvironmentStatus::Live => {} + } + } + println!("Cleaned {cleaned} Stale Environment(s); skipped {skipped}."); + Ok(()) +} + +/// Destroy the given Environments, even healthy ones. Like `docker rm`, a +/// failure does not stop the others. +pub async fn remove_environments( + config: &Config, + ids: &[String], + force: bool, +) -> anyhow::Result<()> { + let Some(registry) = Registry::open_default_if_exists().await? else { + bail!("no Environment with id {}", ids.join(", ")); + }; + let mut failed = 0; + for id in ids { + match remove_one(®istry, config, id, force).await { + Ok(()) => println!("Destroyed Environment {id}"), + Err(error) => { + eprintln!("Error: {error:#}"); + failed += 1; + } + } + } + if failed > 0 { + bail!("{failed} of {} Environment(s) not removed", ids.len()); + } + Ok(()) +} + +async fn remove_one( + registry: &Registry, + config: &Config, + id: &str, + force: bool, +) -> anyhow::Result<()> { + if registry.get(id).await?.is_none() { + bail!("no Environment with id {id}"); + } + let when = if force { + DestroyWhen::Always + } else { + DestroyWhen::Reachable + }; + if !claim_and_destroy(registry, &config.driver, id, when).await? { + bail!( + "Environment {id} is Unreachable; make the Driver queryable and retry, \ + or pass --force to destroy it without Driver cooperation" + ); + } + Ok(()) +} + +pub async fn shell_environment( + config: &Config, + source_dir: &std::path::Path, + id: Option<&str>, +) -> anyhow::Result<()> { + let Some(registry) = Registry::open_default_if_exists().await? else { + match id { + Some(id) => bail!("no Environment with id {id}"), + None => bail!("no Environment found for this Source tree"), + } + }; + let registered = match id { + Some(id) => registry + .get(id) + .await? + .with_context(|| format!("no Environment with id {id}"))?, + None => { + let package = load_package(source_dir)?; + let candidates = registry.list_for_source_tree(source_dir).await?; + let package_identifier = format!("{}-{}", package.name(), package.version()); + select_unique_environment(&candidates, &package_identifier)?.clone() + } + }; + + crate::environment::shell_environment(®istry, &config.driver, ®istered).await +} + +/// When no explicit id is given, resolve the Source tree from cwd / `--source-dir`. +/// +/// Canonicalized: Environment ids and registry lookups key on the `source_dir` +/// string, so all entry points must agree on one spelling per tree. +pub fn resolve_shell_source_dir( + source_dir: Option<&std::path::Path>, + cwd: &std::path::Path, +) -> anyhow::Result { + let dir = std::fs::canonicalize(source_dir.unwrap_or(cwd))?; + load_package(&dir).map(|_| dir) +} diff --git a/packages/debmagic/src/environment/mod.rs b/packages/debmagic/src/environment/mod.rs new file mode 100644 index 00000000..4fa5368f --- /dev/null +++ b/packages/debmagic/src/environment/mod.rs @@ -0,0 +1,1713 @@ +mod process; +mod registry; + +pub use registry::{InvocationKind, RecordedInvocation, RegisteredEnvironment, Registry}; + +use std::collections::HashMap; +use std::path::PathBuf; +use std::time::Duration; + +use anyhow::{Context, bail}; +use process::Process; +use sqlx::sqlite::SqliteConnection; + +use crate::driver::{ + Driver, Environment, EnvironmentDriver, EnvironmentMetadata, Persistence, ResourceStatus, + config::{DriverConfig, DriverOverrides}, + create_driver_from_metadata, planned_driver_metadata, remove_environment_root, +}; + +/// How [`ClaimedEnvironment::acquire`] treats the Host root of an Environment +/// it reuses (Persistence `on-failure` or `always`). A Host root that is not +/// reused always starts empty. +pub enum HostRootPolicy { + Reset, + /// Keep the tree when `manifest` exists in it, so an Incremental build + /// continues from the previous run's state. + KeepIncremental { + manifest: std::path::PathBuf, + }, + /// Keep the whole Host root. The caller has already decided an incremental + /// manifest for this run exists. + Keep, + /// Delete only this stage directory, so a sibling build kind sharing the + /// Environment keeps its tree. + ResetStage { + stage_dir: std::path::PathBuf, + }, +} + +/// An Environment this process has claimed in the Environment registry, with +/// its Host root prepared and its Driver created. Only [`claim_command`] +/// hands one out, and it always finishes the claim. +pub struct ClaimedEnvironment<'a> { + registry: &'a Registry, + driver_config: &'a DriverConfig, + environment: Environment, + driver: Driver, + /// `on-failure` keeps the Environment only when the build or TestRun + /// command itself failed. Set before [`Self::finish`]. + keep_after_failure: bool, +} + +impl<'a> ClaimedEnvironment<'a> { + /// Claim `environment`, prepare its Host root, run `stage` to fill it + /// (after the Environment directories exist), and create its Driver. + /// + /// A fresh Host root is staged before the Driver is created, because + /// container Drivers bind-mount it. A reused one gets its Driver first, + /// so the Driver can reset the root from inside the Environment. + async fn acquire( + registry: &'a Registry, + environment: Environment, + driver_config: &'a DriverConfig, + overrides: &DriverOverrides, + root_policy: HostRootPolicy, + stage: impl FnOnce(&Environment, Option<&Driver>) -> anyhow::Result<()>, + ) -> anyhow::Result { + let planned = planned_driver_metadata(&environment, driver_config, overrides); + let previous = begin_environment(registry, &environment, Process::current(), &planned) + .await? + .map(|row| row.metadata()); + + if environment.persistence.reuses() && environment.root_dir.exists() { + let claimed = + Self::create_driver(registry, environment, driver_config, overrides).await?; + if let Err(error) = claimed.prepare_reused_root(root_policy, stage) { + return claimed.abort(error).await; + } + return Ok(claimed); + } + + let prepared = + remove_environment_root(&environment.root_dir, driver_config, previous.as_ref()) + .and_then(|()| { + environment + .create_dirs() + .context("failed to create Environment directories") + }) + .and_then(|()| stage(&environment, None)); + if let Err(error) = prepared { + abandon_claim(registry, &environment, driver_config, false).await; + return Err(error); + } + Self::create_driver(registry, environment, driver_config, overrides).await + } + + pub fn environment(&self) -> &Environment { + &self.environment + } + + pub fn driver(&self) -> &Driver { + &self.driver + } + + async fn finish(self) -> anyhow::Result<()> { + finish_environment( + self.registry, + &self.environment, + &self.driver, + self.driver_config, + self.keep_after_failure, + ) + .await + } + + /// Finish after `error` stopped the run, keeping `error` as the result. + async fn abort(self, error: anyhow::Error) -> anyhow::Result { + let id = self.environment.id(); + if let Err(teardown) = self.finish().await { + eprintln!("Warning: failed to tear down Environment {id}: {teardown:#}"); + } + Err(error) + } + + async fn create_driver( + registry: &'a Registry, + environment: Environment, + driver_config: &'a DriverConfig, + overrides: &DriverOverrides, + ) -> anyhow::Result { + let driver = match crate::driver::create_driver(&environment, driver_config, overrides) { + Ok(driver) => driver, + Err(error) => { + abandon_claim(registry, &environment, driver_config, true).await; + return Err( + error.context(format!("failed to create {:?} driver", environment.driver)) + ); + } + }; + // Best effort: the claim already recorded the planned Driver metadata, + // which is enough to find the resource. + if let Err(error) = record_driver_metadata(registry, &environment, &driver).await { + eprintln!("Warning: failed to record Driver metadata: {error:#}"); + } + Ok(Self { + registry, + driver_config, + environment, + driver, + keep_after_failure: false, + }) + } + + fn prepare_reused_root( + &self, + root_policy: HostRootPolicy, + stage: impl FnOnce(&Environment, Option<&Driver>) -> anyhow::Result<()>, + ) -> anyhow::Result<()> { + enum Action { + Reset, + Keep, + ResetStage(std::path::PathBuf), + } + let action = match root_policy { + HostRootPolicy::Reset => Action::Reset, + HostRootPolicy::KeepIncremental { manifest } => { + if manifest.is_file() { + Action::Keep + } else { + Action::Reset + } + } + HostRootPolicy::Keep => Action::Keep, + HostRootPolicy::ResetStage { stage_dir } => Action::ResetStage(stage_dir), + }; + match action { + Action::Reset => { + self.driver + .reset_root() + .context("failed to reset the Host root of the reused Environment")?; + } + Action::ResetStage(stage_dir) => { + if stage_dir.exists() { + self.driver + .run_command_checked( + &["find", ".", "-mindepth", "1", "-delete"], + &stage_dir, + true, + &[], + ) + .context("failed to reset the source stage directory")?; + } + } + Action::Keep => { + if !self.driver.reused_environment() { + println!("Keeping incremental build tree in a fresh build environment"); + } + } + } + self.environment + .create_dirs() + .context("failed to create Environment directories")?; + stage(&self.environment, Some(&self.driver)) + } +} + +async fn record_driver_metadata( + registry: &Registry, + environment: &Environment, + driver: &Driver, +) -> anyhow::Result<()> { + let mut conn = registry.connection().await?; + registry::set_driver_metadata(&mut conn, &environment.id(), &driver.driver_metadata()).await +} + +/// What a command finished with, once an Environment is claimed. +/// +/// `Err` from the command callback is not a completion: nothing is recorded, +/// and the Environment is finished with a warning on failure. +pub struct CommandCompletion { + /// Returned after the Invocation is recorded and the claim is finished. + /// A failed build is a completion whose value is `Err`. + pub value: anyhow::Result, + /// Whether the Invocation succeeded. + pub success: bool, + /// Whether the build or TestRun command itself failed. Persistence + /// `on-failure` keeps the Environment, and the inspection hint is printed. + /// Export, signing, and a strict skip are not this. + pub shell_worthy: bool, + /// Whether a finish error fails the command. Otherwise it is a warning + /// and `value` is still returned. + pub finish_error_fails_command: bool, + /// Exported `.changes` for a successful binary build. Absent for a source + /// build, a TestRun, and a failed binary build. + pub changes_path: Option, + /// Lines printed before the inspection hint when `shell_worthy`. + pub failure_lines: Vec, +} + +/// Hint printed when a failed build or TestRun kept its Environment. +pub fn inspection_hint(environment_id: &str) -> String { + format!("Get a shell with `debmagic env shell {environment_id}`") +} + +/// Claim `environment`, stage it, run `command`, and on a completion record +/// the Invocation, maybe print the inspection hint, and finish. +/// +/// The caller builds the Environment and opens the registry. Distro, Driver, +/// Environment id, package name, and source directory on the Invocation come +/// from the Environment. +#[allow(clippy::too_many_arguments)] +pub async fn claim_command( + registry: &Registry, + environment: Environment, + driver_config: &DriverConfig, + overrides: &DriverOverrides, + root_policy: HostRootPolicy, + stage: impl FnOnce(&Environment, Option<&Driver>) -> anyhow::Result<()>, + kind: InvocationKind, + package_version: &str, + command: impl FnOnce(&ClaimedEnvironment<'_>) -> anyhow::Result>, +) -> anyhow::Result { + let mut claimed = ClaimedEnvironment::acquire( + registry, + environment, + driver_config, + overrides, + root_policy, + stage, + ) + .await?; + let completion = match command(&claimed) { + Ok(completion) => completion, + Err(error) => return claimed.abort(error).await, + }; + + let environment = claimed.environment(); + let environment_id = environment.id(); + let purpose = environment.purpose.as_str(); + let invocation = RecordedInvocation { + kind, + source_dir: environment.source_dir.clone(), + package_name: environment.package_name.clone(), + package_version: package_version.to_string(), + distro: environment.distro.clone(), + driver: Some(environment.driver), + success: completion.success, + changes_path: completion.changes_path.clone(), + environment_id: Some(environment_id.clone()), + }; + if let Err(error) = registry.record_invocation(&invocation).await { + eprintln!("Warning: failed to record Invocation: {error:#}"); + } + + if completion.shell_worthy { + for line in &completion.failure_lines { + eprintln!("{line}"); + } + if environment.persistence.reuses() { + eprintln!("{}", inspection_hint(&environment_id)); + claimed.keep_after_failure = true; + } + } + + let finish_error_fails = completion.finish_error_fails_command; + let value = completion.value; + match claimed.finish().await { + Ok(()) => value, + Err(error) if finish_error_fails => { + Err(error.context(format!("failed to clean up {purpose} environment"))) + } + Err(error) => { + eprintln!("Failed to clean up {purpose} environment: {error:#}"); + value + } + } +} + +/// Register `environment` as claimed by `owner` with the Driver metadata +/// planned for it, refusing when a live different process already owns it +/// or a live Attachment is open on it (the run would reset the Host root, or +/// replace the container, under the shell). +/// +/// Returns the row as it was before, so the caller can still reach a Driver +/// resource recorded under the previous metadata. +async fn begin_environment( + registry: &Registry, + environment: &Environment, + owner: Process, + planned_driver_metadata: &HashMap, +) -> anyhow::Result> { + let id = environment.id(); + let mut tx = registry.write().await?; + let previous = registry::get_environment(&mut tx, &id).await?; + if let Some(previous) = &previous { + if let Some(holder) = previous.owner + && holder.pid != owner.pid + && holder.is_alive() + { + bail!( + "Environment {id} is already in use by process {}", + holder.pid + ); + } + let attachments = live_attachment_count(&mut tx, &id).await?; + if attachments > 0 { + bail!( + "Environment {id} has {attachments} live Attachment(s) \ + (`debmagic env shell`); exit them first" + ); + } + } + registry::upsert_environment(&mut tx, environment, Some(owner)).await?; + registry::set_driver_metadata(&mut tx, &id, planned_driver_metadata).await?; + tx.commit().await?; + Ok(previous) +} + +/// Verify that an Environment has no live Attachments and no live owner, +/// then mark it as being destroyed by this process, returning its row. +/// Until released or deleted, this blocks new `env shell` Attachments +/// (refused while `destroying`) and `begin_environment` (live owner). +async fn claim_for_destroy(registry: &Registry, id: &str) -> anyhow::Result { + let me = Process::current(); + let mut tx = registry.write().await?; + let row = registry::get_environment(&mut tx, id) + .await? + .with_context(|| format!("no Environment with id {id}"))?; + if live_attachment_count(&mut tx, id).await? > 0 { + bail!("Environment {id} has live Attachments; stop them first"); + } + if let Some(holder) = row.owner + && holder.pid != me.pid + && holder.is_alive() + { + if row.destroying { + bail!( + "Environment {id} is already being destroyed by process {}", + holder.pid + ); + } + bail!("Environment {id} still has a creating command running"); + } + registry::set_claim(&mut tx, id, Some(me), true).await?; + tx.commit().await?; + Ok(row) +} + +async fn owned_by_me(conn: &mut SqliteConnection, id: &str) -> anyhow::Result { + let me = Process::current(); + Ok(registry::get_environment(conn, id) + .await? + .and_then(|row| row.owner) + .is_some_and(|owner| owner.pid == me.pid)) +} + +/// Drop this process's hold on an Environment, if it still has it. +async fn release(registry: &Registry, id: &str) -> anyhow::Result<()> { + let mut tx = registry.write().await?; + if owned_by_me(&mut tx, id).await? { + registry::set_claim(&mut tx, id, None, false).await?; + } + tx.commit().await?; + Ok(()) +} + +/// Delete the registry row, if this process still holds it. +async fn delete_if_owned(registry: &Registry, id: &str) -> anyhow::Result<()> { + let mut tx = registry.write().await?; + if owned_by_me(&mut tx, id).await? { + registry::delete_environment(&mut tx, id).await?; + } + tx.commit().await?; + Ok(()) +} + +/// Live Attachments of an Environment, deleting those whose shell is gone. +async fn live_attachment_count(conn: &mut SqliteConnection, id: &str) -> anyhow::Result { + let mut live = 0; + for attachment in registry::attachments(&mut *conn, id).await? { + if attachment.process.is_alive() { + live += 1; + } else { + registry::delete_attachment(&mut *conn, attachment.id).await?; + } + } + Ok(live) +} + +async fn reap_attachments(registry: &Registry, id: &str) -> anyhow::Result { + let mut tx = registry.write().await?; + let live = live_attachment_count(&mut tx, id).await?; + tx.commit().await?; + Ok(live) +} + +/// Wait until no live Attachments remain. With `mark_destroying`, mark the +/// Environment as being destroyed in that same transaction, so no new +/// Attachment slips in before the teardown. +async fn wait_for_detach( + registry: &Registry, + id: &str, + mark_destroying: bool, +) -> anyhow::Result<()> { + let mut announced = false; + loop { + let mut tx = registry.write().await?; + let live = live_attachment_count(&mut tx, id).await?; + if live == 0 && mark_destroying { + if !owned_by_me(&mut tx, id).await? { + bail!("Environment {id} is no longer owned by this process"); + } + registry::set_claim(&mut tx, id, Some(Process::current()), true).await?; + } + tx.commit().await?; + if live == 0 { + return Ok(()); + } + if !announced { + println!("Waiting for all attached shells to exit..."); + announced = true; + } + tokio::time::sleep(Duration::from_millis(200)).await; + } +} + +/// Attach `process` to an Environment, refusing one that is gone or being +/// destroyed. +async fn attach(registry: &Registry, id: &str, process: Process) -> anyhow::Result { + let mut tx = registry.write().await?; + match registry::get_environment(&mut tx, id).await? { + Some(row) if !row.destroying => {} + _ => bail!("Environment {id} is gone or being destroyed"), + } + let attachment = registry::insert_attachment(&mut tx, id, process).await?; + tx.commit().await?; + Ok(attachment) +} + +/// Undo a claim when acquiring failed before a Driver existed. A failed +/// Driver creation may have left a resource under the planned metadata, so +/// then the row stays (unowned, hence Stale) for `debmagic env clean`. +async fn abandon_claim( + registry: &Registry, + environment: &Environment, + driver_config: &DriverConfig, + driver_attempted: bool, +) { + let id = environment.id(); + let destroy = environment.persistence != Persistence::Always + && !driver_attempted + && match remove_environment_root(&environment.root_dir, driver_config, None) { + Ok(()) => true, + Err(error) => { + eprintln!("Warning: failed to remove the Host root of {id}: {error:#}"); + false + } + }; + let released = if destroy { + delete_if_owned(registry, &id).await + } else { + release(registry, &id).await + }; + if let Err(error) = released { + eprintln!("Warning: failed to release Environment {id} in the registry: {error:#}"); + } +} + +async fn finish_environment( + registry: &Registry, + environment: &Environment, + driver: &Driver, + driver_config: &DriverConfig, + keep_after_failure: bool, +) -> anyhow::Result<()> { + let id = environment.id(); + let keep = match environment.persistence { + Persistence::Always => true, + Persistence::OnFailure => keep_after_failure, + Persistence::No => false, + }; + if keep { + wait_for_detach(registry, &id, false).await?; + return release(registry, &id).await; + } + + // Keep ownership while destroying: concurrent `begin_environment` calls + // see a live owner and refuse, and marking `destroying` blocks new + // Attachments, so nothing can start using the Environment mid-teardown. + wait_for_detach(registry, &id, true).await?; + let metadata = EnvironmentMetadata { + environment: environment.clone(), + driver_metadata: driver.driver_metadata(), + }; + destroy_claimed(registry, &metadata, Some(driver), driver_config, false).await +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EnvironmentStatus { + Live, + Stale, + Unreachable, +} + +impl EnvironmentStatus { + pub fn as_str(self) -> &'static str { + match self { + Self::Live => "live", + Self::Stale => "stale", + Self::Unreachable => "unreachable", + } + } +} + +/// Live, Stale, or Unreachable, plus the live Attachment count. Assessing +/// reaps dead Attachments. +pub struct AssessedEnvironment { + pub status: EnvironmentStatus, + pub attachments: usize, +} + +/// Derive whether a registered Environment is Live, Stale, or Unreachable, +/// and how many live Attachments it has. +pub async fn assess_environment( + registry: &Registry, + registered: &RegisteredEnvironment, + driver_config: &DriverConfig, +) -> anyhow::Result { + // Fail closed: a registry error must never look like "no Attachments", + // since this status drives destruction. + let attachments = reap_attachments(registry, ®istered.environment.id()).await?; + let leftover = registered.environment.persistence == Persistence::No + && registered.owner.is_none_or(|owner| !owner.is_alive()) + && attachments == 0; + let root_gone = !registered.environment.root_dir.exists(); + + let status = match create_driver_from_metadata(driver_config, ®istered.metadata()) { + Err(_) if leftover || root_gone => EnvironmentStatus::Stale, + Err(_) => EnvironmentStatus::Unreachable, + Ok(driver) => match driver.probe_resource() { + ResourceStatus::Unreachable => EnvironmentStatus::Unreachable, + ResourceStatus::Absent => EnvironmentStatus::Stale, + ResourceStatus::Present if leftover || root_gone => EnvironmentStatus::Stale, + ResourceStatus::Present => EnvironmentStatus::Live, + }, + }; + Ok(AssessedEnvironment { + status, + attachments, + }) +} + +/// Open a shell on an Environment that is already chosen and Live. Holds an +/// Attachment for the duration of the shell and releases it afterwards. +pub async fn shell_environment( + registry: &Registry, + driver_config: &DriverConfig, + registered: &RegisteredEnvironment, +) -> anyhow::Result<()> { + shell_while_attached(registry, driver_config, registered, open_interactive_shell).await +} + +/// Assess, hold an Attachment around `during`, then release it. `during` is +/// the interactive shell in production. Tests pass their own check so they +/// exercise this same path without spawning `$SHELL` on the terminal. +async fn shell_while_attached( + registry: &Registry, + driver_config: &DriverConfig, + registered: &RegisteredEnvironment, + during: impl FnOnce(&DriverConfig, &RegisteredEnvironment) -> anyhow::Result<()>, +) -> anyhow::Result<()> { + let id = registered.environment.id(); + match assess_environment(registry, registered, driver_config) + .await? + .status + { + EnvironmentStatus::Unreachable => bail!("Environment {id} is Unreachable"), + EnvironmentStatus::Stale => { + bail!("Environment {id} is Stale; run `debmagic env rm {id}`") + } + EnvironmentStatus::Live => {} + } + + let attachment = attach(registry, &id, Process::current()).await?; + let result = during(driver_config, registered); + let detached = + async { registry::delete_attachment(&mut *registry.connection().await?, attachment).await }; + result.and(detached.await) +} + +fn open_interactive_shell( + driver_config: &DriverConfig, + registered: &RegisteredEnvironment, +) -> anyhow::Result<()> { + let driver = create_driver_from_metadata(driver_config, ®istered.metadata())?; + driver + .interactive_shell(&shell_cwd(®istered.environment)) + .map_err(|error| anyhow::anyhow!("interactive shell failed: {error}"))?; + Ok(()) +} + +/// Where an interactive shell starts. Binary and source builds stage into +/// sibling trees; the binary tree is the iteration workflow, with the source +/// tree as fallback. Environments staged only by the Environment module +/// itself use [`Environment::staged_source_dir`]. +fn shell_cwd(environment: &Environment) -> std::path::PathBuf { + let binary = environment + .work_dir() + .join(format!("{}-binary", environment.package_identifier)); + let source = environment + .work_dir() + .join(format!("{}-source", environment.package_identifier)); + if binary.is_dir() { + binary + } else if source.is_dir() { + source + } else { + environment.staged_source_dir() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DestroyWhen { + /// `env clean`: only what is still Stale once claimed. + StillStale, + /// `env rm `: anything but Unreachable. + Reachable, + /// `env rm --force`: also without Driver cooperation; Driver + /// failures only warn. + Always, +} + +/// Claim the Environment for destruction (refusing live Attachments and a +/// live owner), then assess the row read by the claim: between the caller's +/// assessment and the claim, a run may have recreated it. Destroys the +/// Driver resource, Host root, and registry row when `when` still holds; +/// otherwise releases the claim and returns `false`. +/// +/// The Driver commands run outside the claim's transaction, which would +/// otherwise lock every other debmagic out of the registry meanwhile; the +/// `destroying` mark keeps the Environment reserved instead. +pub async fn claim_and_destroy( + registry: &Registry, + driver_config: &DriverConfig, + id: &str, + when: DestroyWhen, +) -> anyhow::Result { + let mut claimed = claim_for_destroy(registry, id).await?; + // The claim verified the previous owner was gone; we are only holding + // the row, which must not make it look in use. + claimed.owner = None; + let status = match assess_environment(registry, &claimed, driver_config).await { + Ok(assessed) => assessed.status, + Err(error) => { + let _ = release(registry, id).await; + return Err(error); + } + }; + let proceed = match when { + DestroyWhen::StillStale => status == EnvironmentStatus::Stale, + DestroyWhen::Reachable => status != EnvironmentStatus::Unreachable, + DestroyWhen::Always => true, + }; + if !proceed { + release(registry, id).await?; + return Ok(false); + } + + let force = when == DestroyWhen::Always; + let metadata = claimed.metadata(); + let driver = match create_driver_from_metadata(driver_config, &metadata) { + Ok(driver) => Some(driver), + Err(error) if force => { + eprintln!( + "Warning: cannot reattach to the Driver of {id} ({error}); its resource may leak" + ); + None + } + Err(error) => { + return Err(error.context(format!( + "cannot reattach to the Driver of Environment {id}; \ + pass --force to destroy it without Driver cooperation" + ))); + } + }; + destroy_claimed(registry, &metadata, driver.as_ref(), driver_config, force).await?; + Ok(true) +} + +/// Destroy an Environment this process has marked `destroying`: its Driver +/// resource, Host root, and registry row, in that order. Any failure keeps +/// the claim, so the row stays `destroying` (no new Attachments) and still +/// records the resource for a later `env rm`. With `force`, a failed +/// resource destroy only warns. +async fn destroy_claimed( + registry: &Registry, + metadata: &EnvironmentMetadata, + driver: Option<&Driver>, + driver_config: &DriverConfig, + force: bool, +) -> anyhow::Result<()> { + let id = metadata.environment.id(); + if let Some(driver) = driver { + if let Err(error) = driver.reset_root() { + eprintln!("Warning: failed to reset the Host root of {id} before destroy: {error}"); + } + if let Err(error) = driver.destroy_resource() { + if !force { + return Err(error.context(format!( + "failed to destroy the Driver resource of Environment {id}" + ))); + } + eprintln!("Warning: failed to destroy the Driver resource of {id}: {error}"); + } + } + remove_environment_root( + &metadata.environment.root_dir, + driver_config, + Some(metadata), + )?; + delete_if_owned(registry, &id).await +} + +/// 0 / 1 / 2+ selection for `debmagic env shell` with no explicit id. +/// +/// Environments of the Source tree's current `package_identifier` win over +/// those left behind by earlier versions (each version is its own +/// Environment), so a changelog bump does not make the choice ambiguous. +pub fn select_unique_environment<'a>( + environments: &'a [RegisteredEnvironment], + current_package_identifier: &str, +) -> anyhow::Result<&'a RegisteredEnvironment> { + let current: Vec<&RegisteredEnvironment> = environments + .iter() + .filter(|registered| { + registered.environment.package_identifier == current_package_identifier + }) + .collect(); + let candidates: Vec<&RegisteredEnvironment> = if current.is_empty() { + environments.iter().collect() + } else { + current + }; + match candidates.as_slice() { + [] => bail!("no Environment found for this Source tree"), + [only] => Ok(*only), + many => { + let mut message = String::from( + "more than one Environment exists for this Source tree; pass an Environment id:\n", + ); + for registered in many { + message.push_str(&format!( + " {} {} {} {} {}\n", + registered.environment.id(), + registered.environment.purpose.as_str(), + registered.environment.driver.as_str(), + registered.environment.distro.codename, + registered.environment.package_identifier, + )); + } + bail!("{}", message.trim_end()); + } + } +} + +/// Latest Invocation per DistroVersion. `invocations` must be newest-first. +fn matching_binary_changes<'a>( + invocations: &'a [RecordedInvocation], + distro_filter: Option<&str>, +) -> Vec<&'a RecordedInvocation> { + let mut seen = std::collections::HashSet::new(); + invocations + .iter() + .filter(|invocation| { + distro_filter.is_none_or(|codename| invocation.distro.codename == codename) + }) + .filter(|invocation| { + seen.insert(( + invocation.distro.distro.as_str().to_string(), + invocation.distro.codename.clone(), + )) + }) + .collect() +} + +/// 0 / 1 / 2+ DistroVersions among binary-build Invocations with existing `.changes`. +fn select_unique_binary_changes( + invocations: &[RecordedInvocation], + distro_filter: Option<&str>, +) -> anyhow::Result { + match matching_binary_changes(invocations, distro_filter).as_slice() { + [] => { + if let Some(codename) = distro_filter { + bail!( + "no binary-build Invocation with a .changes file for distro {codename}; run `debmagic build binary` first" + ); + } + bail!( + "no prior binary build found; run `debmagic build binary` first or pass --changes" + ) + } + [only] => Ok((*only).clone()), + many => { + let mut message = String::from( + "more than one DistroVersion has a binary-build Invocation; pass --distro:\n", + ); + for invocation in many { + message.push_str(&format!(" {}\n", invocation.distro.codename)); + } + bail!("{}", message.trim_end()); + } + } +} + +/// Prior binary-build Invocation used to default driver and distro. +/// +/// `--changes` overrides only the artifact path, so history is still consulted. +/// An explicit `.changes` with no matching Invocation is not an error (the +/// caller then requires `--driver` and `--distro`). Several DistroVersions +/// still are. +pub fn prior_binary_invocation( + invocations: &[RecordedInvocation], + distro_filter: Option<&str>, + changes_explicit: bool, +) -> anyhow::Result> { + let matching = matching_binary_changes(invocations, distro_filter); + if changes_explicit && matching.is_empty() { + return Ok(None); + } + select_unique_binary_changes(invocations, distro_filter).map(Some) +} + +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; + + use debmagic_common::distro::{Distro, DistroVersion}; + + use super::registry::tests::{sample_environment, temp_registry, upsert}; + use super::*; + use crate::build_intent::BuildKind; + use crate::driver::{DriverType, EnvironmentPurpose}; + + fn dead_pid() -> u32 { + let mut child = std::process::Command::new("true").spawn().unwrap(); + let pid = child.id(); + child.wait().unwrap(); + pid + } + + fn registered(environment: Environment) -> RegisteredEnvironment { + RegisteredEnvironment { + environment, + driver_metadata: HashMap::new(), + owner: None, + destroying: false, + } + } + + #[tokio::test] + async fn begin_environment_refuses_live_owner_and_allows_takeover() -> anyhow::Result<()> { + let (_dir, registry) = temp_registry().await; + let environment = sample_environment(Path::new("/src/a"), EnvironmentPurpose::Build); + let none = HashMap::new(); + + // pid 1 is always alive; a different process must not take over. + begin_environment(®istry, &environment, Process::of(1), &none).await?; + let error = begin_environment(®istry, &environment, Process::current(), &none) + .await + .unwrap_err() + .to_string(); + assert!(error.contains("already in use by process 1"), "{error}"); + + // A dead owner can be taken over. + upsert(®istry, &environment, Some(Process::of(dead_pid()))).await; + begin_environment(®istry, &environment, Process::current(), &none).await?; + + // The same process re-beginning is idempotent. + begin_environment(®istry, &environment, Process::current(), &none).await?; + Ok(()) + } + + #[tokio::test] + async fn begin_environment_refuses_live_attachments() -> anyhow::Result<()> { + let (_dir, registry) = temp_registry().await; + let environment = sample_environment(Path::new("/src/a"), EnvironmentPurpose::Build); + upsert(®istry, &environment, None).await; + attach(®istry, &environment.id(), Process::current()).await?; + + let error = begin_environment(®istry, &environment, Process::current(), &HashMap::new()) + .await + .unwrap_err() + .to_string(); + assert!(error.contains("live Attachment"), "{error}"); + Ok(()) + } + + #[tokio::test] + async fn begin_environment_records_planned_metadata_and_returns_previous() -> anyhow::Result<()> + { + let (_dir, registry) = temp_registry().await; + let environment = sample_environment(Path::new("/src/a"), EnvironmentPurpose::Build); + let first = HashMap::from([("project".to_string(), "old".to_string())]); + let second = HashMap::from([("project".to_string(), "new".to_string())]); + + let me = Process::current(); + assert!( + begin_environment(®istry, &environment, me, &first) + .await? + .is_none() + ); + let previous = begin_environment(®istry, &environment, me, &second) + .await? + .expect("previous row"); + assert_eq!(previous.driver_metadata, first); + assert_eq!( + registry + .get(&environment.id()) + .await? + .unwrap() + .driver_metadata, + second + ); + Ok(()) + } + + #[tokio::test] + async fn released_destroy_claim_allows_attachments_again() -> anyhow::Result<()> { + let (_dir, registry) = temp_registry().await; + let environment = sample_environment(Path::new("/src/a"), EnvironmentPurpose::Build); + let id = environment.id(); + upsert(®istry, &environment, None).await; + claim_for_destroy(®istry, &id).await?; + release(®istry, &id).await?; + + attach(®istry, &id, Process::current()).await?; + assert_eq!(registry.get(&id).await?.unwrap().owner, None); + Ok(()) + } + + #[tokio::test] + async fn claim_for_destroy_blocks_attachments_and_delete_removes_row() -> anyhow::Result<()> { + let (_dir, registry) = temp_registry().await; + let environment = sample_environment(Path::new("/src/a"), EnvironmentPurpose::Build); + let id = environment.id(); + upsert(®istry, &environment, None).await; + + claim_for_destroy(®istry, &id).await?; + let error = attach(®istry, &id, Process::current()) + .await + .unwrap_err() + .to_string(); + assert!(error.contains("gone or being destroyed"), "{error}"); + + delete_if_owned(®istry, &id).await?; + assert!(registry.get(&id).await?.is_none()); + Ok(()) + } + + #[tokio::test] + async fn claim_for_destroy_refuses_live_attachments() -> anyhow::Result<()> { + let (_dir, registry) = temp_registry().await; + let environment = sample_environment(Path::new("/src/a"), EnvironmentPurpose::Build); + upsert(®istry, &environment, None).await; + attach(®istry, &environment.id(), Process::current()).await?; + + let error = claim_for_destroy(®istry, &environment.id()) + .await + .unwrap_err() + .to_string(); + assert!(error.contains("live Attachments"), "{error}"); + Ok(()) + } + + #[test] + fn unique_shell_selection_is_zero_one_many() { + let build = sample_environment(Path::new("/src/a"), EnvironmentPurpose::Build); + let test = sample_environment(Path::new("/src/a"), EnvironmentPurpose::Test); + let one = vec![registered(build.clone())]; + assert!(select_unique_environment(&one, "pkg-1.0").is_ok()); + assert!(select_unique_environment(&[], "pkg-1.0").is_err()); + let two = vec![registered(build), registered(test)]; + let error = select_unique_environment(&two, "pkg-1.0") + .unwrap_err() + .to_string(); + assert!(error.contains("more than one Environment")); + } + + #[test] + fn shell_selection_prefers_the_current_package_version() { + let version = |identifier: &str| { + registered(Environment::new( + DriverType::Docker, + "pkg", + identifier, + Path::new("/src/a"), + DistroVersion::new(Distro::Debian, "trixie", "13"), + Persistence::Always, + EnvironmentPurpose::Build, + Path::new("/envs"), + )) + }; + let both = vec![version("pkg-1.0"), version("pkg-1.1")]; + let picked = select_unique_environment(&both, "pkg-1.1").unwrap(); + assert_eq!(picked.environment.package_identifier, "pkg-1.1"); + + // No Environment of the current version: fall back to all of them. + let error = select_unique_environment(&both, "pkg-2.0") + .unwrap_err() + .to_string(); + assert!(error.contains("more than one Environment")); + } + + fn recorded_binary(codename: &str, changes: &str) -> RecordedInvocation { + RecordedInvocation { + kind: InvocationKind::Build(BuildKind::Binary), + source_dir: PathBuf::from("/src"), + package_name: "pkg".to_string(), + package_version: "1.0".to_string(), + distro: DistroVersion::new(Distro::Debian, codename, "13"), + driver: Some(DriverType::Docker), + success: true, + changes_path: Some(PathBuf::from(changes)), + environment_id: None, + } + } + + #[test] + fn binary_changes_selection_splits_on_distro() { + let trixie = recorded_binary("trixie", "/out/trixie.changes"); + let older_trixie = recorded_binary("trixie", "/out/old.changes"); + let sid = recorded_binary("sid", "/out/sid.changes"); + let found = [trixie, older_trixie, sid]; + + let error = select_unique_binary_changes(&found, None) + .unwrap_err() + .to_string(); + assert!(error.contains("more than one DistroVersion")); + let picked = select_unique_binary_changes(&found, Some("trixie")).unwrap(); + assert_eq!( + picked.changes_path.as_deref(), + Some(Path::new("/out/trixie.changes")) + ); + } + + #[test] + fn explicit_changes_still_reads_a_prior_invocation() { + let trixie = recorded_binary("trixie", "/out/trixie.changes"); + let sid = recorded_binary("sid", "/out/sid.changes"); + + let prior = prior_binary_invocation(std::slice::from_ref(&trixie), None, true) + .unwrap() + .unwrap(); + assert_eq!(prior.driver, Some(DriverType::Docker)); + assert_eq!(prior.distro.codename, "trixie"); + + assert!(prior_binary_invocation(&[], None, true).unwrap().is_none()); + + let error = prior_binary_invocation(&[trixie, sid], None, true) + .unwrap_err() + .to_string(); + assert!(error.contains("more than one DistroVersion")); + + let missing = prior_binary_invocation(&[], None, false).unwrap_err(); + assert!(missing.to_string().contains("no prior binary build")); + } +} + +#[cfg(test)] +mod teardown_tests { + use std::path::PathBuf; + + use debmagic_common::distro::{Distro, DistroVersion}; + + use super::registry::tests::upsert; + use super::*; + use crate::driver::{DriverType, EnvironmentPurpose}; + + struct Fixture { + dir: PathBuf, + registry: Registry, + } + + impl Fixture { + async fn new() -> Self { + let dir = + std::env::temp_dir().join(format!("debmagic-lifecycle-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&dir).unwrap(); + let registry = Registry::open(&dir.join("db.sqlite")).await.unwrap(); + Self { dir, registry } + } + + /// A registered Bare-driver Environment whose Host root exists. + async fn environment(&self, persistence: Persistence, owner: Option) -> Environment { + let environment = Environment::new( + DriverType::Bare, + "pkg", + "pkg-1.0", + &self.dir.join("src"), + DistroVersion::new(Distro::Debian, "trixie", "13"), + persistence, + EnvironmentPurpose::Build, + &self.dir.join("envs"), + ); + environment.create_dirs().unwrap(); + upsert(&self.registry, &environment, owner.map(Process::of)).await; + environment + } + + async fn row(&self, environment: &Environment) -> Option { + self.registry.get(&environment.id()).await.unwrap() + } + + async fn assess(&self, environment: &Environment) -> AssessedEnvironment { + let registered = self.row(environment).await.unwrap(); + assess_environment(&self.registry, ®istered, &DriverConfig::default()) + .await + .unwrap() + } + + async fn claim_and_destroy( + &self, + environment: &Environment, + when: DestroyWhen, + ) -> anyhow::Result { + claim_and_destroy( + &self.registry, + &DriverConfig::default(), + &environment.id(), + when, + ) + .await + } + } + + impl Drop for Fixture { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.dir); + } + } + + fn dead_pid() -> u32 { + let mut child = std::process::Command::new("true").spawn().unwrap(); + let pid = child.id(); + child.wait().unwrap(); + pid + } + + #[tokio::test] + async fn non_persistent_environment_with_dead_owner_is_stale() { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::No, Some(dead_pid())).await; + assert_eq!( + fixture.assess(&environment).await.status, + EnvironmentStatus::Stale + ); + } + + #[tokio::test] + async fn non_persistent_environment_with_live_owner_is_live() { + let fixture = Fixture::new().await; + let environment = fixture + .environment(Persistence::No, Some(std::process::id())) + .await; + assert_eq!( + fixture.assess(&environment).await.status, + EnvironmentStatus::Live + ); + } + + #[tokio::test] + async fn persistent_environment_with_root_and_resource_is_live() { + let fixture = Fixture::new().await; + let environment = fixture + .environment(Persistence::Always, Some(dead_pid())) + .await; + assert_eq!( + fixture.assess(&environment).await.status, + EnvironmentStatus::Live + ); + } + + #[tokio::test] + async fn environment_without_host_root_is_stale() { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::Always, None).await; + std::fs::remove_dir_all(&environment.root_dir).unwrap(); + assert_eq!( + fixture.assess(&environment).await.status, + EnvironmentStatus::Stale + ); + } + + #[tokio::test] + async fn still_stale_releases_the_claim_when_no_longer_stale() -> anyhow::Result<()> { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::Always, None).await; + + assert!( + !fixture + .claim_and_destroy(&environment, DestroyWhen::StillStale) + .await? + ); + + let row = fixture.row(&environment).await.unwrap(); + assert_eq!(row.owner, None); + assert!(environment.root_dir.exists()); + // Released means no longer `destroying`, so Attachments are allowed. + attach(&fixture.registry, &environment.id(), Process::current()).await?; + Ok(()) + } + + #[tokio::test] + async fn still_stale_destroys_a_stale_environment() -> anyhow::Result<()> { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::No, Some(dead_pid())).await; + + assert!( + fixture + .claim_and_destroy(&environment, DestroyWhen::StillStale) + .await? + ); + + assert!(fixture.row(&environment).await.is_none()); + assert!(!environment.root_dir.exists()); + Ok(()) + } + + #[tokio::test] + async fn reachable_destroys_a_live_environment() -> anyhow::Result<()> { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::Always, None).await; + + assert!( + fixture + .claim_and_destroy(&environment, DestroyWhen::Reachable) + .await? + ); + + assert!(fixture.row(&environment).await.is_none()); + assert!(!environment.root_dir.exists()); + Ok(()) + } + + #[tokio::test] + async fn claim_is_refused_while_an_attachment_is_live() -> anyhow::Result<()> { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::Always, None).await; + attach(&fixture.registry, &environment.id(), Process::current()).await?; + + let result = fixture + .claim_and_destroy(&environment, DestroyWhen::Always) + .await; + + assert!(result.is_err()); + assert!(fixture.row(&environment).await.is_some()); + assert!(environment.root_dir.exists()); + Ok(()) + } + + #[tokio::test] + async fn assess_reports_live_attachment_count() -> anyhow::Result<()> { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::Always, None).await; + let assessed = fixture.assess(&environment).await; + assert_eq!(assessed.status, EnvironmentStatus::Live); + assert_eq!(assessed.attachments, 0); + + attach(&fixture.registry, &environment.id(), Process::current()).await?; + assert_eq!(fixture.assess(&environment).await.attachments, 1); + + attach( + &fixture.registry, + &environment.id(), + Process::of(dead_pid()), + ) + .await?; + assert_eq!( + fixture.assess(&environment).await.attachments, + 1, + "dead shells are reaped" + ); + Ok(()) + } + + #[tokio::test] + async fn shell_holds_and_releases_attachment() -> anyhow::Result<()> { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::Always, None).await; + let registered = fixture.row(&environment).await.unwrap(); + + let mut attached = None; + shell_while_attached( + &fixture.registry, + &DriverConfig::default(), + ®istered, + |_driver_config, registered| { + attached = Some(registered.environment.id()); + Ok(()) + }, + ) + .await?; + assert_eq!(attached, Some(environment.id())); + + let assessed = fixture.assess(&environment).await; + assert_eq!(assessed.status, EnvironmentStatus::Live); + assert_eq!(assessed.attachments, 0); + Ok(()) + } + + #[tokio::test] + async fn shell_on_stale_environment_leaves_no_attachment() { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::Always, None).await; + std::fs::remove_dir_all(&environment.root_dir).unwrap(); + let registered = fixture.row(&environment).await.unwrap(); + + let error = shell_environment(&fixture.registry, &DriverConfig::default(), ®istered) + .await + .unwrap_err(); + assert!(error.to_string().contains("Stale")); + + let assessed = fixture.assess(&environment).await; + assert_eq!(assessed.status, EnvironmentStatus::Stale); + assert_eq!(assessed.attachments, 0); + } +} + +#[cfg(test)] +mod claimed_environment_tests { + use super::*; + use crate::build_intent::BuildKind; + use crate::driver::{DriverType, EnvironmentPurpose}; + use debmagic_common::distro::{Distro, DistroVersion}; + use std::path::{Path, PathBuf}; + + struct Fixture { + dir: PathBuf, + registry: Registry, + driver_config: DriverConfig, + overrides: DriverOverrides, + } + + impl Fixture { + async fn new() -> Self { + let dir = + std::env::temp_dir().join(format!("debmagic-lifecycle-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&dir).unwrap(); + let registry = Registry::open(&dir.join("db.sqlite")).await.unwrap(); + Self { + dir, + registry, + driver_config: DriverConfig::default(), + overrides: DriverOverrides::default(), + } + } + + fn environment(&self, persistence: Persistence) -> Environment { + Environment::new( + DriverType::Bare, + "pkg", + "pkg-1.0", + &self.dir.join("src"), + DistroVersion::new(Distro::Debian, "trixie", "13"), + persistence, + EnvironmentPurpose::Build, + &self.dir.join("environments"), + ) + } + + async fn acquire( + &self, + environment: Environment, + root_policy: HostRootPolicy, + stage: impl FnOnce(&Environment, Option<&Driver>) -> anyhow::Result<()>, + ) -> anyhow::Result> { + ClaimedEnvironment::acquire( + &self.registry, + environment, + &self.driver_config, + &self.overrides, + root_policy, + stage, + ) + .await + } + + async fn owner(&self, environment: &Environment) -> Option> { + self.registry + .get(&environment.id()) + .await + .unwrap() + .map(|row| row.owner.map(|owner| owner.pid)) + } + } + + impl Drop for Fixture { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.dir); + } + } + + fn write_staged_file(environment: &Environment) -> anyhow::Result<()> { + std::fs::write(environment.staged_source_dir().join("staged"), "")?; + Ok(()) + } + + fn staged_file(environment: &Environment) -> PathBuf { + environment.staged_source_dir().join("staged") + } + + #[tokio::test] + async fn finish_tears_down_non_persistent_environment() -> anyhow::Result<()> { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::No); + let claimed = fixture + .acquire( + environment.clone(), + HostRootPolicy::Reset, + |environment, _| write_staged_file(environment), + ) + .await?; + assert!(staged_file(claimed.environment()).is_file()); + assert_eq!( + fixture.owner(&environment).await, + Some(Some(std::process::id())) + ); + + claimed.finish().await?; + assert!(!environment.root_dir.exists()); + assert_eq!(fixture.owner(&environment).await, None); + Ok(()) + } + + #[tokio::test] + async fn staging_failure_releases_claim_and_host_root() { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::No); + let result = fixture + .acquire( + environment.clone(), + HostRootPolicy::Reset, + |environment, _| { + write_staged_file(environment)?; + anyhow::bail!("staging failed") + }, + ) + .await; + assert!(result.is_err()); + assert!(!environment.root_dir.exists()); + assert_eq!(fixture.owner(&environment).await, None); + } + + #[tokio::test] + async fn persistent_environment_stays_registered_without_owner() -> anyhow::Result<()> { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::Always); + fixture + .acquire( + environment.clone(), + HostRootPolicy::Reset, + |environment, _| write_staged_file(environment), + ) + .await? + .finish() + .await?; + assert!(staged_file(&environment).is_file()); + assert_eq!(fixture.owner(&environment).await, Some(None)); + Ok(()) + } + + #[tokio::test] + async fn reused_host_root_is_kept_only_for_incremental_with_manifest() -> anyhow::Result<()> { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::Always); + let manifest = environment.root_dir.join("manifest"); + let keep = || HostRootPolicy::KeepIncremental { + manifest: manifest.clone(), + }; + let no_stage = |_: &Environment, _: Option<&Driver>| Ok(()); + + fixture + .acquire(environment.clone(), keep(), |environment, _| { + write_staged_file(environment) + }) + .await? + .finish() + .await?; + fixture + .acquire(environment.clone(), keep(), no_stage) + .await? + .finish() + .await?; + assert!(!staged_file(&environment).exists(), "no manifest: reset"); + + fixture + .acquire(environment.clone(), keep(), |environment, _| { + write_staged_file(environment) + }) + .await? + .finish() + .await?; + std::fs::write(&manifest, "")?; + fixture + .acquire(environment.clone(), keep(), no_stage) + .await? + .finish() + .await?; + assert!(staged_file(&environment).is_file(), "manifest: kept"); + + fixture + .acquire(environment.clone(), HostRootPolicy::Reset, no_stage) + .await? + .finish() + .await?; + assert!( + !staged_file(&environment).exists(), + "Reset ignores manifest" + ); + assert!(Path::new(&environment.staged_source_dir()).is_dir()); + Ok(()) + } + + #[tokio::test] + async fn staging_failure_on_reused_root_releases_claim() -> anyhow::Result<()> { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::Always); + fixture + .acquire( + environment.clone(), + HostRootPolicy::Reset, + |environment, _| write_staged_file(environment), + ) + .await? + .finish() + .await?; + let result = fixture + .acquire(environment.clone(), HostRootPolicy::Reset, |_, _| { + anyhow::bail!("staging failed") + }) + .await; + assert!(result.is_err()); + assert_eq!(fixture.owner(&environment).await, Some(None)); + Ok(()) + } + + const SOURCE_BUILD: InvocationKind = InvocationKind::Build(BuildKind::Source); + + async fn claim( + fixture: &Fixture, + environment: Environment, + command: impl FnOnce(&ClaimedEnvironment<'_>) -> anyhow::Result>, + ) -> anyhow::Result<()> { + claim_command( + &fixture.registry, + environment, + &fixture.driver_config, + &fixture.overrides, + HostRootPolicy::Reset, + |_, _| Ok(()), + SOURCE_BUILD, + "9.9", + command, + ) + .await + } + + fn completion(value: anyhow::Result<()>) -> CommandCompletion<()> { + let success = value.is_ok(); + CommandCompletion { + value, + success, + shell_worthy: !success, + finish_error_fails_command: success, + changes_path: None, + failure_lines: Vec::new(), + } + } + + #[tokio::test] + async fn claim_command_records_a_successful_completion() -> anyhow::Result<()> { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::No); + claim(&fixture, environment.clone(), |_| Ok(completion(Ok(())))).await?; + let rows = fixture + .registry + .invocations_for_source(&environment.source_dir) + .await?; + assert_eq!(rows.len(), 1); + assert!(rows[0].success); + assert_eq!(rows[0].kind, SOURCE_BUILD); + assert_eq!(rows[0].package_version, "9.9"); + assert_eq!(rows[0].driver, Some(DriverType::Bare)); + assert_eq!(rows[0].source_dir, environment.source_dir); + // the Environment row is gone, the Invocation stays + assert_eq!(rows[0].environment_id, None); + assert!(!environment.root_dir.exists()); + Ok(()) + } + + #[tokio::test] + async fn claim_command_records_a_failed_completion() -> anyhow::Result<()> { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::No); + let error = claim(&fixture, environment.clone(), |_| { + Ok(completion(Err(anyhow::anyhow!("build failed")))) + }) + .await + .unwrap_err(); + assert!(error.to_string().contains("build failed")); + let rows = fixture + .registry + .invocations_for_source(&environment.source_dir) + .await?; + assert_eq!(rows.len(), 1); + assert!(!rows[0].success); + assert_eq!(rows[0].package_version, "9.9"); + assert!(!environment.root_dir.exists()); + Ok(()) + } + + #[tokio::test] + async fn claim_command_tears_down_and_records_nothing_when_the_command_stops() + -> anyhow::Result<()> { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::No); + let error = claim(&fixture, environment.clone(), |_| anyhow::bail!("stopped")) + .await + .unwrap_err(); + assert!(error.to_string().contains("stopped")); + let rows = fixture + .registry + .invocations_for_source(&environment.source_dir) + .await?; + assert!(rows.is_empty()); + assert!(!environment.root_dir.exists()); + assert_eq!(fixture.owner(&environment).await, None); + Ok(()) + } + + #[tokio::test] + async fn on_failure_success_tears_the_environment_down() -> anyhow::Result<()> { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::OnFailure); + claim(&fixture, environment.clone(), |_| Ok(completion(Ok(())))).await?; + assert!(!environment.root_dir.exists()); + assert_eq!(fixture.owner(&environment).await, None); + Ok(()) + } + + #[tokio::test] + async fn on_failure_command_failure_keeps_the_environment() -> anyhow::Result<()> { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::OnFailure); + let error = claim(&fixture, environment.clone(), |_| { + Ok(completion(Err(anyhow::anyhow!("build failed")))) + }) + .await + .unwrap_err(); + assert!(error.to_string().contains("build failed")); + assert!(environment.root_dir.exists()); + assert_eq!(fixture.owner(&environment).await, Some(None)); + Ok(()) + } + + #[test] + fn inspection_hint_names_the_environment() { + assert_eq!( + inspection_hint("bld-abc"), + "Get a shell with `debmagic env shell bld-abc`" + ); + } +} diff --git a/packages/debmagic/src/environment/process.rs b/packages/debmagic/src/environment/process.rs new file mode 100644 index 00000000..07539411 --- /dev/null +++ b/packages/debmagic/src/environment/process.rs @@ -0,0 +1,81 @@ +/// A process as the Environment registry records it: the pid plus its start +/// time, so a reused pid does not pass for the recorded process. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Process { + pub pid: u32, + /// `None` when unreadable (non-Linux, or the process is already gone). + pub start: Option, +} + +impl Process { + pub fn current() -> Self { + Self::of(std::process::id()) + } + + pub fn of(pid: u32) -> Self { + Self { + pid, + start: start_time(pid), + } + } + + /// Whether the recorded process still runs. Falls back to plain liveness + /// when the start time cannot be compared. + pub fn is_alive(self) -> bool { + if !pid_is_alive(self.pid) { + return false; + } + match (self.start, start_time(self.pid)) { + (Some(recorded), Some(current)) => recorded == current, + _ => true, + } + } +} + +fn pid_is_alive(pid: u32) -> bool { + if pid == 0 { + return false; + } + let result = unsafe { libc::kill(pid as i32, 0) }; + if result == 0 { + return true; + } + std::io::Error::last_os_error() + .raw_os_error() + .is_none_or(|code| code != libc::ESRCH) +} + +/// Field 22 of `/proc//stat`. +fn start_time(pid: u32) -> Option { + let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?; + // comm (field 2) is parenthesized and may itself contain spaces or + // parens; the remaining fields start after the last ')'. + let after_comm = stat.rsplit_once(')')?.1; + // starttime is field 22 overall, i.e. index 19 after comm. + after_comm.split_whitespace().nth(19)?.parse().ok() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn current_process_is_alive() { + assert!(Process::current().is_alive()); + } + + #[test] + fn exited_process_is_not_alive() { + let mut child = std::process::Command::new("true").spawn().unwrap(); + let process = Process::of(child.id()); + child.wait().unwrap(); + assert!(!process.is_alive()); + } + + #[test] + fn reused_pid_is_not_the_recorded_process() { + let mut current = Process::current(); + current.start = current.start.map(|start| start + 1); + assert!(!current.is_alive()); + } +} diff --git a/packages/debmagic/src/environment/registry.rs b/packages/debmagic/src/environment/registry.rs new file mode 100644 index 00000000..4e7f8dd3 --- /dev/null +++ b/packages/debmagic/src/environment/registry.rs @@ -0,0 +1,1114 @@ +//! Storage of the Environment registry: rows of Environments, their +//! Attachments, and Invocations. Row functions take a connection, so the +//! parent module composes them into transactions around its lifecycle rules. + +use std::{ + collections::{HashMap, HashSet}, + path::{Path, PathBuf}, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; + +use anyhow::{Context, bail}; +use sqlx::{ + migrate::{MigrateError, Migrator}, + pool::PoolConnection, + sqlite::{Sqlite, SqliteConnectOptions, SqliteConnection, SqlitePool, SqlitePoolOptions}, +}; + +use super::process::Process; +use crate::build_intent::BuildKind; +use crate::driver::{ + DriverType, Environment, EnvironmentMetadata, EnvironmentPurpose, Persistence, +}; +use debmagic_common::distro::{Distro, DistroVersion}; + +macro_rules! select_environments { + ($rest:literal) => { + concat!( + "select driver, package_name, package_identifier, source_dir, root_dir, + distro_family, distro_codename, distro_version, distro_is_devel, + persistence, purpose, owner_pid, owner_pid_start, destroying, driver_metadata + from environments ", + $rest + ) + }; +} + +macro_rules! select_invocations { + ($rest:literal) => { + concat!( + "select id, kind, source_dir, package_name, package_version, + distro_family, distro_codename, distro_version, distro_is_devel, + driver, success, changes_path, environment_id + from invocations ", + $rest + ) + }; +} + +static MIGRATOR: Migrator = sqlx::migrate!("src/migrations"); + +const BUSY_TIMEOUT: Duration = Duration::from_secs(5); + +pub type Transaction = sqlx::Transaction<'static, Sqlite>; + +#[derive(Debug)] +struct IncompatibleSchema { + version: i64, +} + +impl std::fmt::Display for IncompatibleSchema { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "the Environment registry has schema version {}, which this debmagic does not know; upgrade debmagic", + self.version + ) + } +} + +impl std::error::Error for IncompatibleSchema {} + +fn is_incompatible_schema(error: &anyhow::Error) -> bool { + error + .chain() + .any(|cause| cause.downcast_ref::().is_some()) +} + +fn is_busy(error: &anyhow::Error) -> bool { + error.chain().any(|cause| { + cause + .downcast_ref::() + .is_some_and(is_busy_error) + }) +} + +/// SQLITE_BUSY or SQLITE_LOCKED; extended result codes add the high bits. +fn is_busy_error(error: &sqlx::Error) -> bool { + let sqlx::Error::Database(error) = error else { + return false; + }; + error + .code() + .and_then(|code| code.parse::().ok()) + .is_some_and(|code| matches!(code & 0xff, 5 | 6)) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum InvocationKind { + Build(BuildKind), + Test, +} + +impl InvocationKind { + fn as_str(self) -> &'static str { + match self { + Self::Build(BuildKind::Binary) => "binary_build", + Self::Build(BuildKind::Source) => "source_build", + Self::Test => "test_run", + } + } + + fn from_name(name: &str) -> anyhow::Result { + Ok(match name { + "binary_build" => Self::Build(BuildKind::Binary), + "source_build" => Self::Build(BuildKind::Source), + "test_run" => Self::Test, + other => bail!("unknown Invocation kind '{other}'"), + }) + } +} + +#[derive(Debug, Clone)] +pub struct RecordedInvocation { + pub kind: InvocationKind, + pub source_dir: PathBuf, + pub package_name: String, + pub package_version: String, + pub distro: DistroVersion, + pub driver: Option, + pub success: bool, + pub changes_path: Option, + pub environment_id: Option, +} + +#[derive(Debug, Clone)] +pub struct RegisteredEnvironment { + pub environment: Environment, + pub(super) driver_metadata: HashMap, + /// The command holding the Environment. + pub(super) owner: Option, + pub(super) destroying: bool, +} + +impl RegisteredEnvironment { + pub(super) fn metadata(&self) -> EnvironmentMetadata { + EnvironmentMetadata { + environment: self.environment.clone(), + driver_metadata: self.driver_metadata.clone(), + } + } +} + +pub(super) struct Attachment { + pub id: i64, + pub process: Process, +} + +pub struct Registry { + pool: SqlitePool, +} + +impl Registry { + async fn connect(options: SqliteConnectOptions) -> anyhow::Result { + // One connection: no caller holds two at once, and SQLite serializes + // writers anyway. Never closed while idle, so an in-memory registry + // lives as long as the process. + let pool = SqlitePoolOptions::new() + .max_connections(1) + .idle_timeout(None) + .max_lifetime(None) + .connect_with(options.busy_timeout(BUSY_TIMEOUT).foreign_keys(true)) + .await?; + Ok(Self { pool }) + } + + pub async fn open(path: &Path) -> anyhow::Result { + Self::open_file(path) + .await + .with_context(|| format!("opening the Environment registry {} failed", path.display())) + } + + async fn open_file(path: &Path) -> anyhow::Result { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("creating {} failed", parent.display()))?; + } + let options = SqliteConnectOptions::new() + .filename(path) + .create_if_missing(true); + let registry = Self::connect(options).await?; + registry.enable_wal().await?; + registry.migrate().await?; + Ok(registry) + } + + /// Open `path` when the file already exists. Does not create the file or + /// its parent directory. + pub async fn open_if_exists(path: &Path) -> anyhow::Result> { + if !path.is_file() { + return Ok(None); + } + Self::open(path).await.map(Some) + } + + /// Open the on-disk registry when it already exists. `debmagic env` reads + /// use this so listing or cleaning does not create an empty database. + pub async fn open_default_if_exists() -> anyhow::Result> { + Self::open_if_exists(&crate::data_dir::registry_path()?).await + } + + async fn open_memory() -> anyhow::Result { + let registry = Self::connect(SqliteConnectOptions::new().in_memory(true)).await?; + registry.migrate().await?; + Ok(registry) + } + + async fn warn_and_open_memory(error: &anyhow::Error) -> Self { + eprintln!( + "Warning: cannot open the Environment registry ({error:#}); \ + continuing without persistent bookkeeping" + ); + Self::open_memory() + .await + .expect("an in-memory registry cannot fail to open") + } + + /// Open `path`, or a process-local in-memory registry when the file cannot + /// be used. Still an error: + /// - a schema newer than this binary — falling back would hide + /// Environments the newer debmagic already recorded; + /// - a registry that is busy or locked — it works, another debmagic holds + /// it, and bypassing it would skip the "already in use" check that keeps + /// two runs off the same Environment. + async fn open_or_ephemeral(path: &Path) -> anyhow::Result { + match Self::open(path).await { + Ok(registry) => Ok(registry), + Err(error) if is_incompatible_schema(&error) || is_busy(&error) => Err(error), + Err(error) => Ok(Self::warn_and_open_memory(&error).await), + } + } + + /// For build/test bookkeeping, which must not fail the actual work: see + /// [`Self::open_or_ephemeral`]. `debmagic env` commands *are* registry + /// operations and use [`Self::open_default_if_exists`]. + pub async fn open_default_or_ephemeral() -> anyhow::Result { + match crate::data_dir::registry_path() { + Ok(path) => Self::open_or_ephemeral(&path).await, + Err(error) => Ok(Self::warn_and_open_memory(&error).await), + } + } + + /// Switch to WAL, retrying on SQLITE_BUSY. Changing the journal mode + /// upgrades a shared lock, and SQLite reports that conflict immediately + /// instead of invoking the busy handler, so concurrent first opens must + /// retry by hand. + async fn enable_wal(&self) -> anyhow::Result<()> { + let deadline = Instant::now() + BUSY_TIMEOUT; + loop { + match sqlx::query("pragma journal_mode = wal") + .execute(&self.pool) + .await + { + Ok(_) => return Ok(()), + Err(error) if is_busy_error(&error) && Instant::now() < deadline => { + tokio::time::sleep(Duration::from_millis(10)).await; + } + Err(error) => return Err(error).context("enabling WAL failed"), + } + } + } + + async fn migrate(&self) -> anyhow::Result<()> { + // The SQLite migrator does not lock, so concurrent first opens are + // serialized by running it inside our own write transaction. + let mut tx = self.write().await?; + match MIGRATOR.run(&mut *tx).await { + Ok(()) => {} + Err(MigrateError::VersionMissing(version)) => { + return Err(IncompatibleSchema { version }.into()); + } + Err(MigrateError::VersionMismatch(version)) => bail!( + "schema version {version} of the Environment registry differs from this debmagic's; \ + delete the registry to recreate it" + ), + Err(error) => return Err(error).context("migrating the schema failed"), + } + let violations = sqlx::query("pragma foreign_key_check") + .fetch_all(&mut *tx) + .await + .context("checking foreign keys failed")?; + if !violations.is_empty() { + bail!( + "the Environment registry has {} foreign key violations", + violations.len() + ); + } + tx.commit().await?; + Ok(()) + } + + /// A write transaction. IMMEDIATE takes the write lock up front, so a + /// read-check-write sequence cannot interleave with another debmagic. + pub(super) async fn write(&self) -> anyhow::Result { + self.pool + .begin_with("begin immediate") + .await + .context("starting an Environment registry transaction failed") + } + + /// The connection for a single statement outside a transaction. Release + /// it before calling another registry function: there is only one. + pub(super) async fn connection(&self) -> anyhow::Result> { + self.pool + .acquire() + .await + .context("connecting to the Environment registry failed") + } + + pub async fn get(&self, id: &str) -> anyhow::Result> { + get_environment(&mut *self.connection().await?, id).await + } + + pub async fn list(&self) -> anyhow::Result> { + let rows = sqlx::query_as::<_, EnvironmentRow>(select_environments!( + "order by package_name, source_dir, purpose, driver" + )) + .fetch_all(&mut *self.connection().await?) + .await + .context("listing the Environments failed")?; + rows.into_iter().map(TryInto::try_into).collect() + } + + pub async fn list_for_source_tree( + &self, + source_dir: &Path, + ) -> anyhow::Result> { + let rows = sqlx::query_as::<_, EnvironmentRow>(select_environments!( + "where source_dir = ? order by purpose, driver, distro_codename" + )) + .bind(path_to_string(source_dir)) + .fetch_all(&mut *self.connection().await?) + .await + .context("listing the Environments of the Source tree failed")?; + rows.into_iter().map(TryInto::try_into).collect() + } + + pub async fn record_invocation(&self, invocation: &RecordedInvocation) -> anyhow::Result<()> { + let mut tx = self.write().await?; + sqlx::query( + "insert into invocations ( + kind, source_dir, package_name, package_version, + distro_family, distro_codename, distro_version, distro_is_devel, + driver, success, changes_path, environment_id, created_at + ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ) + .bind(invocation.kind.as_str()) + .bind(path_to_string(&invocation.source_dir)) + .bind(&invocation.package_name) + .bind(&invocation.package_version) + .bind(invocation.distro.distro.as_str()) + .bind(&invocation.distro.codename) + .bind(&invocation.distro.version) + .bind(invocation.distro.is_devel) + .bind(invocation.driver.map(DriverType::as_str)) + .bind(invocation.success) + .bind(invocation.changes_path.as_deref().map(path_to_string)) + .bind(&invocation.environment_id) + .bind(unix_time(SystemTime::now())) + .execute(&mut *tx) + .await + .context("inserting the Invocation failed")?; + prune_invocations(&mut tx).await?; + tx.commit().await?; + Ok(()) + } + + /// Successful binary-build Invocations for this Source tree whose + /// `.changes` still exists, newest first. + pub async fn binary_build_changes_for_tree( + &self, + source_dir: &Path, + ) -> anyhow::Result> { + // Housekeeping only: the filter below skips missing files anyway. + if let Err(error) = self.prune_invocations().await { + eprintln!("Warning: failed to prune Invocation history: {error:#}"); + } + let rows = sqlx::query_as::<_, InvocationRow>(select_invocations!( + "where source_dir = ? and kind = ? and success = 1 order by created_at desc, id desc" + )) + .bind(path_to_string(source_dir)) + .bind(InvocationKind::Build(BuildKind::Binary).as_str()) + .fetch_all(&mut *self.connection().await?) + .await + .context("reading binary-build Invocations failed")?; + let mut found = Vec::new(); + for row in rows { + let invocation = RecordedInvocation::try_from(row)?; + if invocation + .changes_path + .as_ref() + .is_some_and(|path| path.is_file()) + { + found.push(invocation); + } + } + Ok(found) + } + + async fn prune_invocations(&self) -> anyhow::Result<()> { + let mut tx = self.write().await?; + prune_invocations(&mut tx).await?; + tx.commit().await?; + Ok(()) + } + + /// Every Invocation recorded for this Source tree, newest first. + #[cfg(test)] + pub(super) async fn invocations_for_source( + &self, + source_dir: &Path, + ) -> anyhow::Result> { + let rows = sqlx::query_as::<_, InvocationRow>(select_invocations!( + "where source_dir = ? order by created_at desc, id desc" + )) + .bind(path_to_string(source_dir)) + .fetch_all(&mut *self.connection().await?) + .await?; + rows.into_iter().map(TryInto::try_into).collect() + } +} + +/// Drop Invocation rows that can no longer affect a lookup. +/// +/// Keeps the newest row per kind, Source tree, and DistroVersion. A +/// binary-build row is kept only when it succeeded and its `.changes` +/// file still exists, so a later failure does not erase the last usable +/// artifact and a deleted artifact does not linger. +async fn prune_invocations(conn: &mut SqliteConnection) -> anyhow::Result<()> { + let rows = sqlx::query_as::<_, InvocationRow>(select_invocations!( + "order by created_at desc, id desc" + )) + .fetch_all(&mut *conn) + .await + .context("reading the Invocation history failed")?; + let binary_build = InvocationKind::Build(BuildKind::Binary).as_str(); + let mut seen = HashSet::new(); + for row in rows { + let usable = row.kind != binary_build + || (row.success + && row + .changes_path + .as_deref() + .is_some_and(|path| Path::new(path).is_file())); + let key = ( + row.kind, + row.source_dir, + row.distro_family, + row.distro_codename, + ); + if !(usable && seen.insert(key)) { + sqlx::query("delete from invocations where id = ?") + .bind(row.id) + .execute(&mut *conn) + .await + .context("pruning the Invocation history failed")?; + } + } + Ok(()) +} + +pub(super) async fn get_environment( + conn: &mut SqliteConnection, + id: &str, +) -> anyhow::Result> { + sqlx::query_as::<_, EnvironmentRow>(select_environments!("where id = ?")) + .bind(id) + .fetch_optional(conn) + .await + .context("reading the Environment from the registry failed")? + .map(TryInto::try_into) + .transpose() +} + +/// Insert or replace `environment`, keeping its Driver metadata, apt +/// upgrade time, Attachments, and Invocations. +pub(super) async fn upsert_environment( + conn: &mut SqliteConnection, + environment: &Environment, + owner: Option, +) -> anyhow::Result<()> { + sqlx::query( + "insert into environments ( + id, driver, package_name, package_identifier, source_dir, root_dir, + distro_family, distro_codename, distro_version, distro_is_devel, + persistence, purpose, owner_pid, owner_pid_start + ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + on conflict(id) do update set + driver = excluded.driver, + package_name = excluded.package_name, + package_identifier = excluded.package_identifier, + source_dir = excluded.source_dir, + root_dir = excluded.root_dir, + distro_family = excluded.distro_family, + distro_codename = excluded.distro_codename, + distro_version = excluded.distro_version, + distro_is_devel = excluded.distro_is_devel, + persistence = excluded.persistence, + purpose = excluded.purpose, + owner_pid = excluded.owner_pid, + owner_pid_start = excluded.owner_pid_start, + destroying = 0", + ) + .bind(environment.id()) + .bind(environment.driver.as_str()) + .bind(&environment.package_name) + .bind(&environment.package_identifier) + .bind(path_to_string(&environment.source_dir)) + .bind(path_to_string(&environment.root_dir)) + .bind(environment.distro.distro.as_str()) + .bind(&environment.distro.codename) + .bind(&environment.distro.version) + .bind(environment.distro.is_devel) + .bind(environment.persistence.as_str()) + .bind(environment.purpose.as_str()) + .bind(owner.map(|owner| i64::from(owner.pid))) + .bind(owner.and_then(|owner| owner.start)) + .execute(conn) + .await + .context("writing the Environment to the registry failed")?; + Ok(()) +} + +pub(super) async fn set_driver_metadata( + conn: &mut SqliteConnection, + id: &str, + metadata: &HashMap, +) -> anyhow::Result<()> { + let updated = sqlx::query("update environments set driver_metadata = ? where id = ?") + .bind(serde_json::to_string(metadata)?) + .bind(id) + .execute(conn) + .await + .context("writing the Driver metadata failed")?; + if updated.rows_affected() == 0 { + bail!("Environment {id} is not in the registry"); + } + Ok(()) +} + +pub(super) async fn set_claim( + conn: &mut SqliteConnection, + id: &str, + owner: Option, + destroying: bool, +) -> anyhow::Result<()> { + sqlx::query( + "update environments set owner_pid = ?, owner_pid_start = ?, destroying = ? where id = ?", + ) + .bind(owner.map(|owner| i64::from(owner.pid))) + .bind(owner.and_then(|owner| owner.start)) + .bind(destroying) + .bind(id) + .execute(conn) + .await + .context("writing the claim of the Environment failed")?; + Ok(()) +} + +pub(super) async fn delete_environment( + conn: &mut SqliteConnection, + id: &str, +) -> anyhow::Result<()> { + sqlx::query("delete from environments where id = ?") + .bind(id) + .execute(conn) + .await + .context("deleting the Environment from the registry failed")?; + Ok(()) +} + +pub(super) async fn attachments( + conn: &mut SqliteConnection, + environment_id: &str, +) -> anyhow::Result> { + let rows: Vec<(i64, i64, Option)> = + sqlx::query_as("select id, pid, pid_start from attachments where environment_id = ?") + .bind(environment_id) + .fetch_all(conn) + .await + .context("reading the Attachments failed")?; + Ok(rows + .into_iter() + .map(|(id, pid, start)| Attachment { + id, + process: Process { + pid: pid as u32, + start, + }, + }) + .collect()) +} + +pub(super) async fn insert_attachment( + conn: &mut SqliteConnection, + environment_id: &str, + process: Process, +) -> anyhow::Result { + let inserted = + sqlx::query("insert into attachments (environment_id, pid, pid_start) values (?, ?, ?)") + .bind(environment_id) + .bind(i64::from(process.pid)) + .bind(process.start) + .execute(conn) + .await + .context("recording the Attachment failed")?; + Ok(inserted.last_insert_rowid()) +} + +pub(super) async fn delete_attachment(conn: &mut SqliteConnection, id: i64) -> anyhow::Result<()> { + sqlx::query("delete from attachments where id = ?") + .bind(id) + .execute(conn) + .await + .context("removing the Attachment failed")?; + Ok(()) +} + +fn path_to_string(path: &Path) -> String { + path.to_string_lossy().into_owned() +} + +fn unix_time(time: SystemTime) -> i64 { + time.duration_since(UNIX_EPOCH) + .map(|duration| duration.as_secs() as i64) + .unwrap_or(0) +} + +#[derive(sqlx::FromRow)] +struct EnvironmentRow { + driver: String, + package_name: String, + package_identifier: String, + source_dir: String, + root_dir: String, + distro_family: String, + distro_codename: String, + distro_version: String, + distro_is_devel: bool, + persistence: String, + purpose: String, + owner_pid: Option, + owner_pid_start: Option, + destroying: bool, + driver_metadata: String, +} + +impl TryFrom for RegisteredEnvironment { + type Error = anyhow::Error; + + fn try_from(row: EnvironmentRow) -> anyhow::Result { + Ok(Self { + environment: Environment { + driver: DriverType::from_name(&row.driver)?, + package_name: row.package_name, + package_identifier: row.package_identifier, + source_dir: PathBuf::from(row.source_dir), + root_dir: PathBuf::from(row.root_dir), + distro: DistroVersion { + distro: Distro::parse(&row.distro_family), + codename: row.distro_codename, + version: row.distro_version, + is_devel: row.distro_is_devel, + }, + persistence: Persistence::from_name(&row.persistence)?, + purpose: EnvironmentPurpose::from_name(&row.purpose)?, + }, + driver_metadata: serde_json::from_str(&row.driver_metadata) + .context("parsing the Driver metadata failed")?, + owner: row.owner_pid.map(|pid| Process { + pid: pid as u32, + start: row.owner_pid_start, + }), + destroying: row.destroying, + }) + } +} + +#[derive(sqlx::FromRow)] +struct InvocationRow { + id: i64, + kind: String, + source_dir: String, + package_name: String, + package_version: String, + distro_family: String, + distro_codename: String, + distro_version: String, + distro_is_devel: bool, + driver: Option, + success: bool, + changes_path: Option, + environment_id: Option, +} + +impl TryFrom for RecordedInvocation { + type Error = anyhow::Error; + + fn try_from(row: InvocationRow) -> anyhow::Result { + Ok(Self { + kind: InvocationKind::from_name(&row.kind)?, + source_dir: PathBuf::from(row.source_dir), + package_name: row.package_name, + package_version: row.package_version, + distro: DistroVersion { + distro: Distro::parse(&row.distro_family), + codename: row.distro_codename, + version: row.distro_version, + is_devel: row.distro_is_devel, + }, + driver: row + .driver + .as_deref() + .map(DriverType::from_name) + .transpose()?, + success: row.success, + changes_path: row.changes_path.map(PathBuf::from), + environment_id: row.environment_id, + }) + } +} + +#[cfg(test)] +pub(super) mod tests { + use super::*; + + pub(in crate::environment) async fn temp_registry() -> (PathBuf, Registry) { + let dir = std::env::temp_dir().join(format!("debmagic-registry-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&dir).unwrap(); + let registry = Registry::open(&dir.join("db.sqlite")).await.unwrap(); + (dir, registry) + } + + pub(in crate::environment) fn sample_environment( + source_dir: &Path, + purpose: EnvironmentPurpose, + ) -> Environment { + Environment::new( + DriverType::Docker, + "pkg", + "pkg-1.0", + source_dir, + DistroVersion::new(Distro::Debian, "trixie", "13"), + Persistence::Always, + purpose, + &source_dir.join("envs"), + ) + } + + pub(in crate::environment) async fn upsert( + registry: &Registry, + environment: &Environment, + owner: Option, + ) { + let mut conn = registry.connection().await.unwrap(); + upsert_environment(&mut conn, environment, owner) + .await + .unwrap(); + } + + fn invocation( + kind: InvocationKind, + source_dir: &Path, + version: &str, + distro: &DistroVersion, + success: bool, + changes_path: Option<&Path>, + ) -> RecordedInvocation { + RecordedInvocation { + kind, + source_dir: source_dir.to_path_buf(), + package_name: "pkg".to_string(), + package_version: version.to_string(), + distro: distro.clone(), + driver: Some(DriverType::Docker), + success, + changes_path: changes_path.map(Path::to_path_buf), + environment_id: None, + } + } + + const BINARY: InvocationKind = InvocationKind::Build(BuildKind::Binary); + + async fn invocation_count(registry: &Registry) -> i64 { + sqlx::query_scalar("select count(*) from invocations") + .fetch_one(&mut *registry.connection().await.unwrap()) + .await + .unwrap() + } + + #[test] + fn environment_id_differs_across_source_trees_and_purposes() { + let first = sample_environment(Path::new("/src/a"), EnvironmentPurpose::Build); + let second = sample_environment(Path::new("/src/b"), EnvironmentPurpose::Build); + let test = sample_environment(Path::new("/src/a"), EnvironmentPurpose::Test); + assert_ne!(first.id(), second.id()); + assert_ne!(first.id(), test.id()); + assert_eq!( + first.id(), + sample_environment(Path::new("/src/a"), EnvironmentPurpose::Build).id() + ); + assert_eq!(first.id().len(), 16); + } + + #[tokio::test] + async fn upsert_and_list_round_trip() -> anyhow::Result<()> { + let (_dir, registry) = temp_registry().await; + let environment = sample_environment(Path::new("/src/a"), EnvironmentPurpose::Build); + upsert(®istry, &environment, Some(Process::of(1))).await; + let listed = registry.list().await?; + assert_eq!(listed.len(), 1); + assert_eq!(listed[0].environment.id(), environment.id()); + assert_eq!(listed[0].environment.source_dir, PathBuf::from("/src/a")); + assert_eq!(listed[0].environment.persistence, Persistence::Always); + assert_eq!(listed[0].owner.map(|owner| owner.pid), Some(1)); + Ok(()) + } + + #[tokio::test] + async fn unknown_enum_values_are_rejected() { + let (_dir, registry) = temp_registry().await; + let environment = sample_environment(Path::new("/src/a"), EnvironmentPurpose::Build); + upsert(®istry, &environment, None).await; + let error = sqlx::query("update environments set purpose = 'deploy'") + .execute(&mut *registry.connection().await.unwrap()) + .await + .unwrap_err(); + assert!(error.to_string().contains("CHECK"), "{error}"); + } + + #[tokio::test] + async fn deleting_an_environment_drops_its_attachments_and_keeps_invocations() + -> anyhow::Result<()> { + let (_dir, registry) = temp_registry().await; + let environment = sample_environment(Path::new("/src/a"), EnvironmentPurpose::Build); + let id = environment.id(); + upsert(®istry, &environment, None).await; + let mut conn = registry.connection().await?; + insert_attachment(&mut conn, &id, Process::current()).await?; + drop(conn); + let distro = DistroVersion::new(Distro::Debian, "trixie", "13"); + let mut recorded = invocation( + InvocationKind::Test, + Path::new("/src/a"), + "1.0", + &distro, + true, + None, + ); + recorded.environment_id = Some(id.clone()); + registry.record_invocation(&recorded).await?; + + let mut conn = registry.connection().await?; + delete_environment(&mut conn, &id).await?; + assert!(attachments(&mut conn, &id).await?.is_empty()); + drop(conn); + let invocations = registry.invocations_for_source(Path::new("/src/a")).await?; + assert_eq!(invocations.len(), 1); + assert_eq!(invocations[0].environment_id, None); + Ok(()) + } + + #[tokio::test] + async fn invocation_records_full_distro_version() -> anyhow::Result<()> { + let (dir, registry) = temp_registry().await; + let source = dir.join("src"); + let changes = dir.join("pkg.changes"); + std::fs::write(&changes, "changes")?; + let distro = DistroVersion { + distro: Distro::Custom("yocto".to_string()), + codename: "kirkstone".to_string(), + version: "4.0".to_string(), + is_devel: true, + }; + registry + .record_invocation(&invocation( + BINARY, + &source, + "1.0", + &distro, + true, + Some(&changes), + )) + .await?; + + let found = registry.binary_build_changes_for_tree(&source).await?; + assert_eq!(found.len(), 1); + assert_eq!(found[0].distro, distro); + assert_eq!(found[0].kind, BINARY); + Ok(()) + } + + #[tokio::test] + async fn binary_changes_lookup_skips_missing_files() -> anyhow::Result<()> { + let (dir, registry) = temp_registry().await; + let source = dir.join("src"); + let trixie = dir.join("trixie.changes"); + std::fs::write(&trixie, "changes")?; + let missing = dir.join("gone.changes"); + let distro_trixie = DistroVersion::new(Distro::Debian, "trixie", "13"); + let distro_sid = DistroVersion::new(Distro::Debian, "sid", ""); + for (distro, changes) in [ + (&distro_trixie, &missing), + (&distro_trixie, &trixie), + (&distro_sid, &trixie), + ] { + registry + .record_invocation(&invocation( + BINARY, + &source, + "1.0", + distro, + true, + Some(changes), + )) + .await?; + } + + assert_eq!(invocation_count(®istry).await, 2); + let found = registry.binary_build_changes_for_tree(&source).await?; + let codenames: Vec<_> = found.iter().map(|i| i.distro.codename.as_str()).collect(); + assert_eq!(codenames, ["sid", "trixie"]); + Ok(()) + } + + #[tokio::test] + async fn invocation_history_keeps_the_newest_usable_binary_build() -> anyhow::Result<()> { + let (dir, registry) = temp_registry().await; + let source = dir.join("src"); + let older = dir.join("older.changes"); + let newer = dir.join("newer.changes"); + std::fs::write(&older, "old")?; + std::fs::write(&newer, "new")?; + let distro = DistroVersion::new(Distro::Debian, "trixie", "13"); + + registry + .record_invocation(&invocation( + BINARY, + &source, + "1.0", + &distro, + true, + Some(&older), + )) + .await?; + let mut lxd = invocation(BINARY, &source, "1.1", &distro, true, Some(&newer)); + lxd.driver = Some(DriverType::Lxd); + registry.record_invocation(&lxd).await?; + registry + .record_invocation(&invocation(BINARY, &source, "1.2", &distro, false, None)) + .await?; + + assert_eq!(invocation_count(®istry).await, 1); + let found = registry.binary_build_changes_for_tree(&source).await?; + assert_eq!(found.len(), 1); + assert_eq!(found[0].changes_path.as_deref(), Some(newer.as_path())); + assert_eq!(found[0].driver, Some(DriverType::Lxd)); + assert_eq!(found[0].package_version, "1.1"); + Ok(()) + } + + #[tokio::test] + async fn deleting_a_changes_file_prunes_that_invocation() -> anyhow::Result<()> { + let (dir, registry) = temp_registry().await; + let source = dir.join("src"); + let changes = dir.join("pkg.changes"); + std::fs::write(&changes, "changes")?; + let distro = DistroVersion::new(Distro::Debian, "trixie", "13"); + registry + .record_invocation(&invocation( + BINARY, + &source, + "1.0", + &distro, + true, + Some(&changes), + )) + .await?; + std::fs::remove_file(&changes)?; + + assert!( + registry + .binary_build_changes_for_tree(&source) + .await? + .is_empty() + ); + assert_eq!(invocation_count(®istry).await, 0); + Ok(()) + } + + #[tokio::test] + async fn non_binary_invocations_collapse_to_the_newest() -> anyhow::Result<()> { + let (dir, registry) = temp_registry().await; + let source = dir.join("src"); + let distro = DistroVersion::new(Distro::Debian, "trixie", "13"); + for version in ["1.0", "1.1"] { + registry + .record_invocation(&invocation( + InvocationKind::Test, + &source, + version, + &distro, + true, + None, + )) + .await?; + } + let rows = registry.invocations_for_source(&source).await?; + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].package_version, "1.1"); + Ok(()) + } + + #[tokio::test] + async fn open_if_exists_does_not_create_the_registry() -> anyhow::Result<()> { + let path = std::env::temp_dir().join(format!( + "debmagic-missing-{}/db.sqlite", + uuid::Uuid::new_v4() + )); + assert!(Registry::open_if_exists(&path).await?.is_none()); + assert!(!path.parent().unwrap().exists()); + Ok(()) + } + + #[tokio::test] + async fn reopening_keeps_the_rows() -> anyhow::Result<()> { + let (dir, registry) = temp_registry().await; + let environment = sample_environment(Path::new("/src/a"), EnvironmentPurpose::Build); + upsert(®istry, &environment, None).await; + registry.pool.close().await; + + let registry = Registry::open(&dir.join("db.sqlite")).await?; + let listed = registry.list().await?; + assert_eq!(listed.len(), 1); + assert_eq!(listed[0].environment.id(), environment.id()); + Ok(()) + } + + async fn add_future_migration(path: &Path) { + let registry = Registry::open(path).await.unwrap(); + sqlx::query( + "insert into _sqlx_migrations (version, description, success, checksum, execution_time) + values (99, 'future', 1, x'00', 0)", + ) + .execute(®istry.pool) + .await + .unwrap(); + registry.pool.close().await; + } + + #[tokio::test] + async fn newer_schema_is_refused() { + let (dir, registry) = temp_registry().await; + registry.pool.close().await; + let path = dir.join("db.sqlite"); + add_future_migration(&path).await; + + let Err(error) = Registry::open(&path).await else { + panic!("expected a newer schema to be refused"); + }; + let rendered = format!("{error:#}"); + assert!(rendered.contains("upgrade debmagic"), "{rendered}"); + assert!(rendered.contains("99"), "{rendered}"); + + let Err(error) = Registry::open_or_ephemeral(&path).await else { + panic!("expected the fallback not to hide a newer schema"); + }; + assert!(is_incompatible_schema(&error), "{error:#}"); + } + + #[tokio::test] + async fn open_or_ephemeral_falls_back_when_the_file_is_unusable() -> anyhow::Result<()> { + let (dir, _registry) = temp_registry().await; + // a directory is not a database + let registry = Registry::open_or_ephemeral(&dir).await?; + let environment = sample_environment(Path::new("/src/a"), EnvironmentPurpose::Build); + upsert(®istry, &environment, None).await; + assert_eq!(registry.list().await?.len(), 1); + Ok(()) + } + + /// Each opener gets its own runtime, like separate debmagic processes. + #[test] + fn concurrent_first_opens_all_succeed() -> anyhow::Result<()> { + let dir = std::env::temp_dir().join(format!("debmagic-registry-{}", uuid::Uuid::new_v4())); + let path = dir.join("db.sqlite"); + let barrier = std::sync::Barrier::new(8); + std::thread::scope(|scope| { + let openers: Vec<_> = (0..8) + .map(|_| { + scope.spawn(|| { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + barrier.wait(); + runtime.block_on(Registry::open(&path)).map(|_| ()) + }) + }) + .collect(); + openers + .into_iter() + .try_for_each(|opener| opener.join().expect("opener thread panicked")) + }) + } +} diff --git a/packages/debmagic/src/main.rs b/packages/debmagic/src/main.rs index 9224fc8e..1cbd5191 100644 --- a/packages/debmagic/src/main.rs +++ b/packages/debmagic/src/main.rs @@ -1,10 +1,9 @@ -use std::env; use std::process::ExitCode; use anyhow::Context; use clap::{CommandFactory, Parser}; -use crate::cli::{BuildTarget, Cli, Commands, ConfigCommands, UpstreamCommands}; +use crate::cli::{BuildTarget, Cli, Commands, ConfigCommands, EnvCommands, UpstreamCommands}; pub mod build; pub mod build_intent; @@ -13,7 +12,10 @@ pub mod changes; pub mod cli; pub mod config; pub mod control; +pub mod data_dir; pub mod driver; +pub mod env; +pub mod environment; pub mod output; pub mod package; pub mod requests; @@ -42,7 +44,7 @@ async fn run() -> anyhow::Result { let cli = Cli::parse(); output::init_color(cli.color); - let current_dir = env::current_dir()?; + let current_dir = std::env::current_dir()?; match &cli.command { Commands::Build(args) => { let (build_args, debug_symbols, kind, include_orig, run_test) = match &args.target { @@ -81,7 +83,6 @@ async fn run() -> anyhow::Result { clean: build_args.clean, source_sync: build_args.source_sync, host_arch_variant: build_args.host_arch_variant.clone(), - shell_on_failure: build_args.shell_on_failure, driver_overrides: build_args.driver_overrides(), }, distro: build_args.distro.clone(), @@ -92,13 +93,41 @@ async fn run() -> anyhow::Result { }) .await?; } - Commands::Shell(args) => { - build::shell( - ¤t_dir, - args.common.source_dir.as_deref(), - cli.config.as_deref(), - )?; - } + Commands::Environment(args) => match &args.command { + EnvCommands::List => { + let config = config::Config::load(None, cli.config.as_deref())?; + env::list_environments(&config).await?; + } + EnvCommands::Clean => { + let config = config::Config::load(None, cli.config.as_deref())?; + env::clean_environments(&config).await?; + } + EnvCommands::Rm { ids, force } => { + let config = config::Config::load(None, cli.config.as_deref())?; + env::remove_environments(&config, ids, *force).await?; + } + EnvCommands::Shell { id, common } => { + if let Some(id) = id { + let config = config::Config::load(None, cli.config.as_deref())?; + env::shell_environment(&config, ¤t_dir, Some(id)).await?; + } else { + let source_dir = common.source_dir.as_deref().unwrap_or(¤t_dir); + let source_dir = match env::resolve_shell_source_dir( + Some(source_dir), + ¤t_dir, + ) { + Ok(dir) => dir, + Err(_) => { + anyhow::bail!( + "not a Source tree; pass an Environment id (see `debmagic env list`)" + ); + } + }; + let config = config::Config::load(Some(&source_dir), cli.config.as_deref())?; + env::shell_environment(&config, &source_dir, None).await?; + } + } + }, Commands::Test(args) => { let intent = test::resolve_test_intent(test::TestIntentInput { fallback_dir: current_dir.clone(), @@ -109,12 +138,13 @@ async fn run() -> anyhow::Result { strict: args.strict, changes: args.changes.clone(), allow_host_test: args.allow_host_test, - shell_on_failure: args.shell_on_failure, distro: args.distro.clone(), driver_overrides: args.driver_overrides(), })?; - let outcome = test::run_test(&intent).context("running tests failed")?; + let outcome = test::run_test(&intent) + .await + .context("running tests failed")?; return Ok(match outcome { test::TestOutcome::Passed => ExitCode::SUCCESS, test::TestOutcome::Failed => ExitCode::from(1), diff --git a/packages/debmagic/src/migrations/0001_registry.sql b/packages/debmagic/src/migrations/0001_registry.sql new file mode 100644 index 00000000..527cefcd --- /dev/null +++ b/packages/debmagic/src/migrations/0001_registry.sql @@ -0,0 +1,55 @@ +-- SQLite has no enums: each enum column checks the values of the Rust +-- conversion named above it. + +create table environments ( + -- Environment::id + id text primary key, + -- DriverType::as_str / DriverType::from_name + driver text not null check (driver in ('docker', 'bare', 'lxd', 'incus')), + package_name text not null, + package_identifier text not null, + source_dir text not null, + root_dir text not null, + -- Distro::as_str / Distro::parse, free-form for custom distros + distro_family text not null, + distro_codename text not null, + distro_version text not null, + distro_is_devel integer not null check (distro_is_devel in (0, 1)), + -- Persistence::as_str / Persistence::from_name + persistence text not null check (persistence in ('no', 'on-failure', 'always')), + -- EnvironmentPurpose::as_str / EnvironmentPurpose::from_name + purpose text not null check (purpose in ('build', 'test')), + -- the command holding the Environment, see Process + owner_pid integer, + owner_pid_start integer, + destroying integer not null default 0 check (destroying in (0, 1)), + -- JSON object from Driver::driver_metadata + driver_metadata text not null default '{}' +) strict; + +-- `debmagic env shell` sessions, see Process +create table attachments ( + id integer primary key autoincrement, + environment_id text not null references environments(id) on delete cascade, + pid integer not null, + pid_start integer +) strict; + +create table invocations ( + id integer primary key autoincrement, + -- InvocationKind::as_str / InvocationKind::from_name + kind text not null check (kind in ('binary_build', 'source_build', 'test_run')), + source_dir text not null, + package_name text not null, + package_version text not null, + distro_family text not null, + distro_codename text not null, + distro_version text not null, + distro_is_devel integer not null check (distro_is_devel in (0, 1)), + driver text check (driver in ('docker', 'bare', 'lxd', 'incus')), + success integer not null check (success in (0, 1)), + changes_path text, + environment_id text references environments(id) on delete set null, + -- unix time + created_at integer not null +) strict; diff --git a/packages/debmagic/src/package.rs b/packages/debmagic/src/package.rs index 9635d682..267247ee 100644 --- a/packages/debmagic/src/package.rs +++ b/packages/debmagic/src/package.rs @@ -157,7 +157,9 @@ fn custom_distro_from_base_images( } } -fn lookup_distro(name: &str, mode: DistroResolveMode<'_>) -> anyhow::Result { +/// Resolve a distro codename (built-in or a custom suite from the Driver's +/// `base_images` map) to a full [`DistroVersion`]. +pub fn lookup_distro(name: &str, mode: DistroResolveMode<'_>) -> anyhow::Result { if let DistroResolveMode::Container { base_images, .. } = mode { validate_base_images_keys(base_images.keys().map(|k| k.as_str()))?; } diff --git a/packages/debmagic/src/test/intent.rs b/packages/debmagic/src/test/intent.rs index 1221a967..cbb5c139 100644 --- a/packages/debmagic/src/test/intent.rs +++ b/packages/debmagic/src/test/intent.rs @@ -4,7 +4,7 @@ use anyhow::Context; use crate::{ config::Config, - driver::{DriverType, config::DriverOverrides}, + driver::{DriverType, Persistence, config::DriverOverrides}, }; /// Inputs for resolving a [`TestIntent`]. @@ -15,11 +15,10 @@ pub struct TestIntentInput { pub source_dir: Option, pub config_file: Option, pub driver: Option, - pub persistent: Option, + pub persistent: Option, pub strict: bool, pub changes: Option, pub allow_host_test: bool, - pub shell_on_failure: Option, pub distro: Option, pub driver_overrides: DriverOverrides, } @@ -34,14 +33,15 @@ pub struct TestIntent { pub strict: bool, pub changes: Option, pub allow_host_test: bool, - pub shell_on_failure: bool, pub distro: Option, pub config: Config, pub driver_overrides: DriverOverrides, } pub fn resolve_test_intent(input: TestIntentInput) -> anyhow::Result { - let source_dir = std::path::absolute(input.source_dir.unwrap_or(input.fallback_dir)) + // Canonicalized: Environment ids and registry lookups key on the + // source_dir string, so all entry points must agree on one spelling. + let source_dir = std::fs::canonicalize(input.source_dir.unwrap_or(input.fallback_dir)) .context("resolving source dir failed")?; let mut config = Config::load(Some(&source_dir), input.config_file.as_deref())?; @@ -56,15 +56,12 @@ pub fn resolve_test_intent(input: TestIntentInput) -> anyhow::Result None }; - let shell_on_failure = input.shell_on_failure.unwrap_or(config.shell_on_failure); - Ok(TestIntent { source_dir, driver: input.driver, strict: input.strict, changes, allow_host_test: input.allow_host_test, - shell_on_failure, distro: input.distro, config, driver_overrides: input.driver_overrides, @@ -96,7 +93,6 @@ mod tests { strict: false, changes: None, allow_host_test: false, - shell_on_failure: None, distro: None, driver_overrides: DriverOverrides { apt_mirror: None, @@ -116,32 +112,10 @@ mod tests { fn resolve_applies_persistent_override() -> anyhow::Result<()> { let dir = std::env::temp_dir(); let mut input = base_input(dir); - input.persistent = Some(false); - - let intent = resolve_test_intent(input)?; - assert!(!intent.config.driver.persistent); - Ok(()) - } - - #[test] - fn resolve_passes_through_strict() -> anyhow::Result<()> { - let dir = std::env::temp_dir(); - let mut input = base_input(dir); - input.strict = true; - - let intent = resolve_test_intent(input)?; - assert!(intent.strict); - Ok(()) - } - - #[test] - fn resolve_passes_through_shell_on_failure() -> anyhow::Result<()> { - let dir = std::env::temp_dir(); - let mut input = base_input(dir); - input.shell_on_failure = Some(true); + input.persistent = Some(Persistence::No); let intent = resolve_test_intent(input)?; - assert!(intent.shell_on_failure); + assert_eq!(intent.config.driver.persistent, Persistence::No); Ok(()) } @@ -150,7 +124,7 @@ mod tests { let dir = std::env::temp_dir(); let intent = resolve_test_intent(base_input(dir.clone()))?; assert!(intent.source_dir.is_absolute()); - assert_eq!(intent.source_dir, std::path::absolute(&dir)?); + assert_eq!(intent.source_dir, std::fs::canonicalize(&dir)?); Ok(()) } } diff --git a/packages/debmagic/src/test/run.rs b/packages/debmagic/src/test/run.rs index 9a33a019..c2ca0f8b 100644 --- a/packages/debmagic/src/test/run.rs +++ b/packages/debmagic/src/test/run.rs @@ -1,23 +1,23 @@ use std::{ fs, io, - io::{BufReader, IsTerminal, stdout}, path::{Path, PathBuf}, }; use super::intent::TestIntent; -use crate::build::artifacts::{self, copy_changes_artifacts, copy_dir_all}; -use crate::build::source::stage_dir; -use crate::build::source::stage_source_tree; +use crate::build::artifacts::{copy_changes_artifacts, copy_dir_all}; +use crate::build::source::{stage_dir, stage_source_tree}; use crate::build_intent::BuildKind; -use crate::driver::{ - Driver, DriverType, Environment, EnvironmentDriver, EnvironmentMetadata, EnvironmentPurpose, - IsolationCapability, - config::{DriverConfig, DriverOverrides}, - create_driver, remove_environment_root, +use crate::driver::{DriverType, Environment, EnvironmentDriver, EnvironmentPurpose}; +use crate::driver::{IsolationCapability, config::DriverConfig}; +use crate::environment::{ + CommandCompletion, HostRootPolicy, InvocationKind, RecordedInvocation, Registry, claim_command, + prior_binary_invocation, }; +use crate::package::distro_resolve_mode_for_driver; use crate::package::load_package; use crate::subprocess::Capture; use anyhow::{Context, anyhow, bail}; +use debian_control::lossless::changes::Changes; use debmagic_common::distro::DistroVersion; use debmagic_common::package::SourcePackage; @@ -38,28 +38,6 @@ pub enum TestOutcome { StrictFailure, } -struct TestRun { - environment: Environment, - driver: Driver, -} - -fn get_build_root_and_identifier( - temp_build_dir: &Path, - package: &SourcePackage, -) -> (String, PathBuf) { - let package_identifier = format!("{}-{}", package.name(), package.version()); - let build_root = temp_build_dir.join(&package_identifier); - (package_identifier, build_root) -} - -fn test_build_root(build_root: &Path) -> PathBuf { - let package_identifier = build_root - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or_default(); - build_root.with_file_name(format!("{package_identifier}-test")) -} - fn map_autopkgtest_exit(exit_code: i32, strict: bool) -> TestOutcome { if exit_code == AUTOPKGTEST_EXIT_PASS { return TestOutcome::Passed; @@ -85,99 +63,95 @@ fn map_autopkgtest_exit(exit_code: i32, strict: bool) -> TestOutcome { TestOutcome::Passed } -fn lookup_distro(name: &str) -> anyhow::Result { - debmagic_common::distro::get_distro_version(name) - .ok_or_else(|| anyhow!("unknown distro codename '{name}'")) +/// Resolve an explicit `--distro` override like a build would: built-in +/// suites, plus custom suites declared in the chosen Driver's `base_images`. +fn lookup_distro_override( + name: &str, + driver: DriverType, + driver_config: &DriverConfig, +) -> anyhow::Result { + let mode = distro_resolve_mode_for_driver( + driver, + &driver_config.docker.base_images, + &driver_config.lxd.base_images, + ); + crate::package::lookup_distro(name, mode) } -fn load_environment_metadata(root: &Path) -> anyhow::Result { - let metadata_path = root.join("environment.json"); - if !metadata_path.is_file() { - bail!("No environment.json found"); - } - let file = fs::OpenOptions::new().read(true).open(&metadata_path)?; - let reader = BufReader::new(&file); - serde_json::from_reader(reader).with_context(|| { - format!( - "Failed to read environment metadata from {} - invalid json", - metadata_path.display() - ) +/// Whether any suite in a `.changes` `Distribution` field (space-separated, +/// codenames or aliases like `unstable`) names `distro`. +fn distribution_matches(distribution: &str, distro: &DistroVersion) -> bool { + distribution.split_whitespace().any(|suite| { + suite == distro.codename + || debmagic_common::distro::get_distro_version(suite) + .is_some_and(|resolved| resolved.codename == distro.codename) }) } -impl TestRun { - fn create( - environment: &Environment, - driver_config: &DriverConfig, - driver_overrides: &DriverOverrides, - ) -> anyhow::Result { - let driver = create_driver(environment, driver_config, driver_overrides) - .context(format!("failed to create {:?} driver", environment.driver))?; - Ok(Self { - environment: environment.clone(), - driver, - }) - } - - fn write_metadata(&self) -> anyhow::Result<()> { - let metadata = EnvironmentMetadata { - environment: self.environment.clone(), - driver_metadata: self.driver.driver_metadata(), - }; - let path = self.environment.root_dir.join("environment.json"); - let json = serde_json::to_string_pretty(&metadata) - .context("Failed to serialize environment metadata")?; - fs::write(path, json)?; - Ok(()) - } -} - -fn prepare_test_env( +/// Point out where the prior binary-build Invocation that supplies the +/// defaults disagrees with what is about to be tested. +fn warn_on_prior_build_mismatch( intent: &TestIntent, - environment: &Environment, package: &SourcePackage, + invocation: &RecordedInvocation, changes_path: &Path, -) -> anyhow::Result { - let test_root = &environment.root_dir; - - if intent.config.driver.persistent && test_root.exists() { - let test_run = - TestRun::create(environment, &intent.config.driver, &intent.driver_overrides) - .context(format!("failed to create {:?} driver", environment.driver))?; - test_run - .driver - .reset_root() - .context("failed to reset persistent test directory")?; - environment - .create_dirs() - .context("failed to create test directories")?; - stage_source_tree( - environment, - BuildKind::Binary, - package, - intent.config.source_sync_mode, - false, - )?; - copy_changes_artifacts(changes_path, &environment.work_dir())?; - return Ok(test_run); +) { + let current_version = package.version().to_string(); + if intent.changes.is_none() && invocation.package_version != current_version { + eprintln!( + "Warning: testing {} {} from the last successful binary build; \ + debian/changelog is now at {current_version}", + package.name(), + invocation.package_version + ); } + if intent.changes.is_some() + && intent.distro.is_none() + && let Ok(changes) = Changes::from_file(changes_path) + && let Some(distribution) = changes.distribution() + && !distribution_matches(&distribution, &invocation.distro) + { + eprintln!( + "Warning: {} targets {distribution}, but the test defaults to {} from the \ + prior binary build; pass --distro to override", + changes_path.display(), + invocation.distro.codename + ); + } +} - remove_environment_root(test_root, &intent.config.driver)?; +/// Written into every exported test directory; only directories carrying it +/// are ever replaced, so an unrelated folder next to the `.changes` survives. +const TEST_OUTPUT_MARKER: &str = ".debmagic-test-output"; + +/// `/.test` — named after the `.changes` so it cannot +/// clash with a generic folder (e.g. a package's own `test/`) when the +/// output directory is the source tree or its parent. +fn exported_test_dir(changes_path: &Path) -> anyhow::Result { + let stem = changes_path + .file_stem() + .and_then(|stem| stem.to_str()) + .ok_or_else(|| anyhow!("invalid .changes path: {}", changes_path.display()))?; + let parent = changes_path.parent().unwrap_or_else(|| Path::new(".")); + Ok(parent.join(format!("{stem}.test"))) +} - environment - .create_dirs() - .context("failed to create test directories")?; - stage_source_tree( - environment, - BuildKind::Binary, - package, - intent.config.source_sync_mode, - false, - )?; - copy_changes_artifacts(changes_path, &environment.work_dir())?; +fn ensure_exported_test_dir_replaceable(dir: &Path) -> anyhow::Result<()> { + if dir.exists() && !dir.join(TEST_OUTPUT_MARKER).is_file() { + bail!( + "{} exists but was not written by debmagic; move it away and retry", + dir.display() + ); + } + Ok(()) +} - let test_run = TestRun::create(environment, &intent.config.driver, &intent.driver_overrides)?; - Ok(test_run) +fn replace_exported_test_dir(dir: &Path) -> anyhow::Result<()> { + ensure_exported_test_dir_replaceable(dir)?; + if !dir.exists() { + return Ok(()); + } + fs::remove_dir_all(dir).with_context(|| format!("failed to remove {}", dir.display())) } fn print_autopkgtest_notices(exit_code: i32, summary_path: &Path) { @@ -228,10 +202,24 @@ fn autopkgtest_isolation_args(isolation: IsolationCapability) -> Vec<&'static st } } -pub fn run_test(intent: &TestIntent) -> anyhow::Result { +pub async fn run_test(intent: &TestIntent) -> anyhow::Result { let package = load_package(&intent.source_dir)?; - let (package_identifier, build_root) = - get_build_root_and_identifier(&intent.config.temp_build_dir, &package); + let registry = Registry::open_default_or_ephemeral().await?; + + // `--changes` overrides only the artifact path. Driver and distro still + // default to the prior binary-build Invocation when one exists. + let invocations = registry + .binary_build_changes_for_tree(&intent.source_dir) + .await + .unwrap_or_else(|error| { + eprintln!("Warning: cannot read prior binary builds from the registry: {error}"); + Vec::new() + }); + let prior_invocation = prior_binary_invocation( + &invocations, + intent.distro.as_deref(), + intent.changes.is_some(), + )?; let changes_path = if let Some(ref explicit) = intent.changes { if !explicit.is_file() { @@ -239,27 +227,26 @@ pub fn run_test(intent: &TestIntent) -> anyhow::Result { } explicit.clone() } else { - let metadata_path = build_root.join("environment.json"); - if !metadata_path.is_file() { - bail!( - "no prior build found at {}; run `debmagic build binary` first", - build_root.display() - ); - } - // tests install built binaries, so a binary build's .changes is what - // to export — not a leftover source one from the same persistent dir - artifacts::find_changes_file(&build_root.join("work"), BuildKind::Binary)? - }; - - let prior_build = if build_root.join("environment.json").is_file() { - Some(load_environment_metadata(&build_root)?) - } else { - None + prior_invocation + .as_ref() + .and_then(|invocation| invocation.changes_path.clone()) + .context( + "no prior binary build found; run `debmagic build binary` first or pass --changes", + )? }; + if let Some(invocation) = prior_invocation.as_ref() { + warn_on_prior_build_mismatch(intent, &package, invocation, &changes_path); + } + let exported_test_dir = exported_test_dir(&changes_path)?; + ensure_exported_test_dir_replaceable(&exported_test_dir)?; let driver = intent .driver - .or_else(|| prior_build.as_ref().map(|metadata| metadata.environment.driver)) + .or_else(|| { + prior_invocation + .as_ref() + .and_then(|invocation| invocation.driver) + }) .ok_or_else(|| { anyhow!( "no driver specified and no prior build found; pass --driver or run `debmagic build binary` first" @@ -274,159 +261,193 @@ pub fn run_test(intent: &TestIntent) -> anyhow::Result { } let distro = if let Some(ref override_distro) = intent.distro { - lookup_distro(override_distro)? - } else if let Some(ref metadata) = prior_build { - metadata.environment.distro.clone() + lookup_distro_override(override_distro, driver, &intent.config.driver)? + } else if let Some(ref invocation) = prior_invocation { + // The Invocation recorded the full DistroVersion, so custom suites + // from a Driver's base_images work without re-resolving. + invocation.distro.clone() } else { bail!( - "no prior build metadata found; pass --distro when using --changes without a build root" + "no prior build metadata found; pass --distro when using --changes without a prior Invocation" ); }; - let test_root = test_build_root(&build_root); - - let environment = Environment { + let environment = Environment::new( driver, - package_name: package.name().to_string(), - package_identifier, - root_dir: test_root.clone(), + package.name(), + &format!("{}-{}", package.name(), package.version()), + &intent.source_dir, distro, - persistent: intent.config.driver.persistent, - purpose: EnvironmentPurpose::Test, - }; - - let test_run = prepare_test_env(intent, &environment, &package, &changes_path) - .context("failed to prepare test environment")?; - test_run - .write_metadata() - .context("failed to write test metadata")?; - - crate::output::stage("Installing autopkgtest"); - let staged_source_dir = stage_dir(&environment, BuildKind::Binary); - test_run.driver.run_command_checked( - &["apt-get", "install", "-y", "autopkgtest"], - &staged_source_dir, - true, - &[("DEBIAN_FRONTEND", "noninteractive")], - )?; - - let work_dir = environment.work_dir(); - let changes_filename = changes_path - .file_name() - .and_then(|name| name.to_str()) - .ok_or_else(|| anyhow!("invalid .changes path: {}", changes_path.display()))?; - let source_tree_name = staged_source_dir - .file_name() - .and_then(|name| name.to_str()) - .context("stage dir has no file name")?; - let autopkgtest_out_host = test_root.join("autopkgtest-out"); - if autopkgtest_out_host.exists() { - fs::remove_dir_all(&autopkgtest_out_host)?; - } - let output_dir_arg = "../autopkgtest-out"; - let summary_arg = "../autopkgtest-out/summary"; - - // Binary-only builds have no .dsc in the .changes; pass the staged source - // tree alongside the .changes so debian/tests/ is found without rebuilding - // (-B). See autopkgtest(1) "TESTING A DEBIAN PACKAGE" (.changes + tree). - // IsolationCapabilities become `--ignore-restrictions` (autopkgtest args, - // not virt-null `--fake-capability`; see autopkgtest_isolation_args). - let output_dir_flag = format!("--output-dir={output_dir_arg}"); - let summary_flag = format!("--summary={summary_arg}"); - let source_tree_arg = format!("{source_tree_name}/"); - let isolation_args = autopkgtest_isolation_args(test_run.driver.isolation_capability()); - let mut autopkgtest_cmd = vec!["autopkgtest", "-B", "--no-auto-control"]; - autopkgtest_cmd.extend(isolation_args); - autopkgtest_cmd.extend([ - output_dir_flag.as_str(), - summary_flag.as_str(), - changes_filename, - source_tree_arg.as_str(), - "--", - "null", - ]); - - crate::output::stage(&format!("Running autopkgtest for {}", package.name())); - let exit_code = test_run - .driver - .run_command(&autopkgtest_cmd, &work_dir, true, &[], Capture::NONE) - .map(|result| result.exit_code) - .unwrap_or(-1); - - let summary_path = autopkgtest_out_host.join("summary"); - print_autopkgtest_notices(exit_code, &summary_path); - - let export_root = prior_build - .as_ref() - .map(|_| build_root.as_path()) - .unwrap_or_else(|| changes_path.parent().unwrap()); - let exported_test_dir = export_root.join("test"); - if exported_test_dir.exists() { - fs::remove_dir_all(&exported_test_dir)?; - } - copy_dir_all(&autopkgtest_out_host, &exported_test_dir).with_context(|| { - format!( - "failed to copy test output to {}", - exported_test_dir.display() - ) - })?; - println!("Test output written to {}", exported_test_dir.display()); - - let outcome = map_autopkgtest_exit(exit_code, intent.strict); - - if outcome == TestOutcome::Failed { - eprintln!("Tests failed (autopkgtest exit code {exit_code})."); - eprintln!("Test logs: {}", exported_test_dir.display()); - if intent.shell_on_failure && stdout().is_terminal() { - eprintln!("Dropping into shell..."); - if let Err(shell_error) = test_run.driver.interactive_shell(&staged_source_dir) { - eprintln!("Dropping into shell failed: {shell_error}"); + intent.config.driver.persistent, + EnvironmentPurpose::Test, + &intent.config.environments_dir, + ); + let version = package.version().to_string(); + + claim_command( + ®istry, + environment, + &intent.config.driver, + &intent.driver_overrides, + HostRootPolicy::Reset, + |environment, _| { + stage_source_tree( + environment, + BuildKind::Binary, + &package, + intent.config.source_sync_mode, + false, + )?; + copy_changes_artifacts(&changes_path, &environment.work_dir()) + }, + InvocationKind::Test, + &version, + |claimed| { + crate::output::stage("Installing autopkgtest"); + let staged_source_dir = stage_dir(claimed.environment(), BuildKind::Binary); + claimed.driver().run_command_checked( + &["apt-get", "install", "-y", "autopkgtest"], + &staged_source_dir, + true, + &[("DEBIAN_FRONTEND", "noninteractive")], + )?; + + let work_dir = claimed.environment().work_dir(); + let changes_filename = changes_path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| anyhow!("invalid .changes path: {}", changes_path.display()))?; + let source_tree_name = staged_source_dir + .file_name() + .and_then(|name| name.to_str()) + .context("stage dir has no file name")?; + let autopkgtest_out_host = claimed.environment().root_dir.join("autopkgtest-out"); + if autopkgtest_out_host.exists() { + fs::remove_dir_all(&autopkgtest_out_host)?; } - } else if intent.shell_on_failure { - eprintln!( - "--shell-on-failure is set but stdout is not a TTY; skipping interactive shell" - ); - } else { - eprintln!("Re-run with --shell-on-failure to inspect the test environment"); - } - } - - if !environment.persistent { - // Clear container-owned files from the bind mount before destroying - // the container; otherwise the host user cannot remove them later. - if let Err(e) = test_run.driver.reset_root() { - eprintln!("Warning: failed to reset test root before cleanup: {e}"); - } - } - - if let Err(cleanup_error) = test_run.driver.cleanup() { - eprintln!("Failed to clean up test environment: {cleanup_error}"); - } - - Ok(outcome) + let output_dir_arg = "../autopkgtest-out"; + let summary_arg = "../autopkgtest-out/summary"; + + // Binary-only builds have no .dsc in the .changes; pass the staged source + // tree alongside the .changes so debian/tests/ is found without rebuilding + // (-B). See autopkgtest(1) "TESTING A DEBIAN PACKAGE" (.changes + tree). + // IsolationCapabilities become `--ignore-restrictions` (autopkgtest args, + // not virt-null `--fake-capability`; see autopkgtest_isolation_args). + let output_dir_flag = format!("--output-dir={output_dir_arg}"); + let summary_flag = format!("--summary={summary_arg}"); + let source_tree_arg = format!("{source_tree_name}/"); + let isolation_args = + autopkgtest_isolation_args(claimed.driver().isolation_capability()); + let mut autopkgtest_cmd = vec!["autopkgtest", "-B", "--no-auto-control"]; + autopkgtest_cmd.extend(isolation_args); + autopkgtest_cmd.extend([ + output_dir_flag.as_str(), + summary_flag.as_str(), + changes_filename, + source_tree_arg.as_str(), + "--", + "null", + ]); + + crate::output::stage(&format!("Running autopkgtest for {}", package.name())); + let exit_code = claimed + .driver() + .run_command(&autopkgtest_cmd, &work_dir, true, &[], Capture::NONE) + .map(|result| result.exit_code) + .unwrap_or(-1); + + let summary_path = autopkgtest_out_host.join("summary"); + print_autopkgtest_notices(exit_code, &summary_path); + + replace_exported_test_dir(&exported_test_dir)?; + copy_dir_all(&autopkgtest_out_host, &exported_test_dir).with_context(|| { + format!( + "failed to copy test output to {}", + exported_test_dir.display() + ) + })?; + fs::write(exported_test_dir.join(TEST_OUTPUT_MARKER), "")?; + println!("Test output written to {}", exported_test_dir.display()); + + let outcome = map_autopkgtest_exit(exit_code, intent.strict); + let failure_lines = if outcome == TestOutcome::Failed { + vec![ + format!("Tests failed (autopkgtest exit code {exit_code})."), + format!("Test logs: {}", exported_test_dir.display()), + ] + } else { + Vec::new() + }; + Ok(CommandCompletion { + value: Ok(outcome), + success: outcome == TestOutcome::Passed, + shell_worthy: outcome == TestOutcome::Failed, + finish_error_fails_command: false, + changes_path: None, + failure_lines, + }) + }, + ) + .await } #[cfg(test)] mod tests { use super::*; + use crate::driver::Persistence; - fn sample_package() -> SourcePackage { - SourcePackage::from_files([( - "changelog", - "pkg (1.0-1) unstable; urgency=medium\n\n * Change.\n\n -- A Mon, 01 Jan 2024 00:00:00 +0000\n", - )]) - .unwrap() + #[test] + fn exported_test_dir_is_named_after_the_changes_file() -> anyhow::Result<()> { + assert_eq!( + exported_test_dir(Path::new("/out/pkg_1.0-1_amd64.changes"))?, + PathBuf::from("/out/pkg_1.0-1_amd64.test") + ); + Ok(()) } #[test] - fn test_build_root_appends_test_suffix() { - let (_, build_root) = - get_build_root_and_identifier(Path::new("/tmp/debmagic"), &sample_package()); - assert_eq!(build_root, PathBuf::from("/tmp/debmagic/pkg-1.0-1")); - assert_eq!( - test_build_root(&build_root), - PathBuf::from("/tmp/debmagic/pkg-1.0-1-test") + fn foreign_test_dir_is_never_removed() -> anyhow::Result<()> { + let dir = std::env::temp_dir().join(format!("debmagic-test-{}", uuid::Uuid::new_v4())); + let foreign = dir.join("pkg.test"); + fs::create_dir_all(&foreign)?; + fs::write(foreign.join("keep"), "user data")?; + assert!(replace_exported_test_dir(&foreign).is_err()); + assert!(foreign.join("keep").is_file()); + + fs::write(foreign.join(TEST_OUTPUT_MARKER), "")?; + replace_exported_test_dir(&foreign)?; + assert!(!foreign.exists()); + fs::remove_dir_all(&dir)?; + Ok(()) + } + + #[test] + fn test_environment_id_differs_from_build() { + let distro = DistroVersion::new(debmagic_common::distro::Distro::Debian, "trixie", "13"); + let envs = PathBuf::from("/envs"); + let source_dir = PathBuf::from("/src"); + let build = Environment::new( + DriverType::Docker, + "pkg", + "pkg-1.0-1", + &source_dir, + distro.clone(), + Persistence::Always, + EnvironmentPurpose::Build, + &envs, + ); + let test = Environment::new( + DriverType::Docker, + "pkg", + "pkg-1.0-1", + &source_dir, + distro, + Persistence::Always, + EnvironmentPurpose::Test, + &envs, ); + assert_ne!(build.id(), test.id()); + assert_ne!(build.root_dir, test.root_dir); } #[test] diff --git a/packages/debmagic/tests/assets/config1.toml b/packages/debmagic/tests/assets/config1.toml index 542e4aca..e4af8717 100644 --- a/packages/debmagic/tests/assets/config1.toml +++ b/packages/debmagic/tests/assets/config1.toml @@ -1,6 +1,6 @@ [driver] default = "docker" -persistent = true +persistent = "always" [driver.docker] base_images = { "debian:trixie" = "some-debian-trixie-image:latest" } From e71ce99e0046b0f217dc46d139f0b131ce2c8ede Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Tue, 6 Oct 2026 16:55:39 +0200 Subject: [PATCH 02/48] feat(changelog): add new changelog entries --- docs/index.md | 1 + docs/usage/changelog.md | 29 ++++ docs/usage/upstream.md | 2 +- .../debmagic-common/src/debian/changelog.rs | 82 +++++++++++ packages/debmagic/src/changelog.rs | 139 +++++++++++++++++- packages/debmagic/src/cli.rs | 22 +++ packages/debmagic/src/main.rs | 14 +- 7 files changed, 285 insertions(+), 4 deletions(-) create mode 100644 docs/usage/changelog.md diff --git a/docs/index.md b/docs/index.md index 15128be9..8b4b1055 100644 --- a/docs/index.md +++ b/docs/index.md @@ -41,6 +41,7 @@ usage/env.md usage/source.md usage/upload.md usage/upstream.md +usage/changelog.md usage/config.md usage/packaging.md usage/modules/index.md diff --git a/docs/usage/changelog.md b/docs/usage/changelog.md new file mode 100644 index 00000000..1e4f4bac --- /dev/null +++ b/docs/usage/changelog.md @@ -0,0 +1,29 @@ +# Changelog + +`debmagic changelog` edits `debian/changelog`. + +```shell +debmagic changelog new # add an UNRELEASED entry on top, keeping the current version +``` + +## `changelog new` + +Prepends an entry for the current package and version, with distribution `UNRELEASED`, urgency `medium`, an empty `* ` bullet and a trailer signed by you with the current local time: + +```text +foo (1.2.3-1ubuntu2) UNRELEASED; urgency=medium + + * + + -- Larry Cow Mue, 34 Oct 2026 11:67:56 +0200 +``` + +Fill in the bullet and adjust the version in your editor. +The rest of the file is kept byte for byte. + +If the top entry already is `UNRELEASED`, `new` refuses: keep editing that entry instead. + +Your identity is taken from `DEBFULLNAME` and `DEBEMAIL`, then `NAME` and `EMAIL` (an email variable may also carry the name as `Elsa Cow `), then git's `user.name` and `user.email`. +Without one, `new` fails rather than guessing from your login and hostname. + +`--source-dir` points at a package other than the one in the current directory. diff --git a/docs/usage/upstream.md b/docs/usage/upstream.md index 914cca64..10ed6208 100644 --- a/docs/usage/upstream.md +++ b/docs/usage/upstream.md @@ -21,7 +21,7 @@ The changelog's version is normalized with the watch file's `Dversion-Mangle` ru ## `upstream switch` Fetches the chosen version's tarball, applies the package's repack configuration, writes the `orig` tarball into the package's output dir (`build/` by default) and replaces the source tree's contents — everything except `debian/`, which is kept as-is. -The changelog is not touched; add the new version entry yourself (automated changelog handling is planned). +The changelog is not touched; add the new version entry with [`debmagic changelog new`](changelog.md). `--dry-run` reports what would happen — tarball URL, repack excludes, version suffix — without downloading anything. diff --git a/packages/debmagic-common/src/debian/changelog.rs b/packages/debmagic-common/src/debian/changelog.rs index 0f772db3..f80f51b1 100644 --- a/packages/debmagic-common/src/debian/changelog.rs +++ b/packages/debmagic-common/src/debian/changelog.rs @@ -25,6 +25,48 @@ impl ChangelogHead { } } +/// Who signs off a changelog entry, as `(name, email)`. +pub type Maintainer = (String, String); + +/// Prepend an `UNRELEASED` entry for the head's package at `version`, with an +/// empty bullet to fill in. Identity and time come from the caller. +pub fn add_entry( + changelog: &mut debian_changelog::ChangeLog, + version: &PackageVersion, + maintainer: Maintainer, + timestamp: chrono::DateTime, +) -> anyhow::Result<()> { + let head = changelog + .iter() + .next() + .ok_or_else(|| anyhow::anyhow!("changelog has no entry to base the new one on"))?; + if head.is_unreleased() == Some(true) { + anyhow::bail!( + "the top entry ({}) is already UNRELEASED; edit it instead", + head.version() + .map_or_else(|| "no version".into(), |v| v.to_string()) + ); + } + let package = head + .package() + .ok_or_else(|| anyhow::anyhow!("the top entry has no package name"))?; + let version = version + .version() + .parse() + .map_err(|error| anyhow::anyhow!("invalid version {version}: {error}"))?; + changelog + .new_empty_entry() + .package(package) + .version(version) + .distribution(debian_changelog::UNRELEASED.into()) + .urgency(debian_changelog::Urgency::Medium) + .maintainer(maintainer) + .datetime(timestamp) + .change_line("* ".into()) + .finish(); + Ok(()) +} + /// Whether a changelog's top entry looks like an Ubuntu deltarebase /// onto a Debian version Ubuntu has never seen: the current revision /// carries an `ubuntu` component while the previous entry's does not. @@ -152,6 +194,46 @@ mod tests { assert_eq!(head.distributions, ["unstable"]); } + fn add(changelog: &str) -> anyhow::Result { + let mut changelog = parse(changelog); + let version = ChangelogHead::from_changelog(&changelog).unwrap().version; + let timestamp = chrono::DateTime::parse_from_rfc2822("Tue, 06 Oct 2026 12:34:56 +0200")?; + add_entry( + &mut changelog, + &version, + ("Jane Doe".into(), "jane@example.com".into()), + timestamp, + )?; + Ok(changelog.to_string()) + } + + #[test] + fn test_add_entry_copies_version() { + let old = format!("{}\n{}", entry("1:1.2.3-1ubuntu2"), entry("1:1.2.3-1")); + assert_eq!( + add(&old).unwrap(), + format!( + "foo (1:1.2.3-1ubuntu2) UNRELEASED; urgency=medium\n\n * \n\n -- Jane Doe Tue, 06 Oct 2026 12:34:56 +0200\n\n{old}" + ) + ); + } + + #[test] + fn test_add_entry_result_parses() { + let new = add(&entry("1.2.3-1")).unwrap(); + let head = ChangelogHead::from_changelog(&parse(&new)).unwrap(); + assert_eq!(head.version.version(), "1.2.3-1"); + assert_eq!(head.distributions, ["UNRELEASED"]); + assert_eq!(parse(&new).iter().count(), 2); + } + + #[test] + fn test_add_entry_refuses_on_unreleased_head() { + let old = entry("1.2.3-1").replace("unstable", "UNRELEASED"); + let error = add(&old).unwrap_err().to_string(); + assert!(error.contains("already UNRELEASED"), "{error}"); + } + #[test] fn test_head_from_empty_changelog() { let changelog = parse(""); diff --git a/packages/debmagic/src/changelog.rs b/packages/debmagic/src/changelog.rs index 4204bf43..8930a9c2 100644 --- a/packages/debmagic/src/changelog.rs +++ b/packages/debmagic/src/changelog.rs @@ -1,9 +1,89 @@ -use std::path::Path; +use std::path::{Path, PathBuf}; +use std::process::Command; -use anyhow::Context; +use anyhow::{Context, bail}; +use debmagic_common::debian::changelog::Maintainer; use debmagic_common::debian::source::SourceFormat; +/// Inputs for the `debmagic changelog new` command. +#[derive(Debug, Clone)] +pub struct ChangelogNewCommand { + pub source_dir: PathBuf, +} + +/// Run `debmagic changelog new`: prepend an `UNRELEASED` entry that keeps +/// the current version, signed by the user. +pub fn new_entry(command: ChangelogNewCommand) -> anyhow::Result<()> { + let source_dir = command.source_dir; + let path = source_dir.join("debian").join("changelog"); + let mut changelog = load_changelog(&source_dir)?; + let head = changelog_head(&changelog) + .with_context(|| format!("failed to read the head entry of {}", path.display()))?; + let maintainer = maintainer(&source_dir, |name| std::env::var(name).ok())?; + debmagic_common::debian::changelog::add_entry( + &mut changelog, + &head.version, + maintainer, + chrono::Local::now().fixed_offset(), + ) + .with_context(|| format!("adding an entry to {} failed", path.display()))?; + + let partial = path.with_extension("dch"); + std::fs::write(&partial, changelog.to_string()) + .with_context(|| format!("failed to write {}", partial.display()))?; + std::fs::rename(&partial, &path) + .with_context(|| format!("failed to replace {}", path.display()))?; + println!( + "debmagic: added {} ({}) UNRELEASED to {}", + head.package, + head.version, + path.display() + ); + Ok(()) +} + +/// The user signing changelog entries, found like dch does (`DEBFULLNAME`, +/// `DEBEMAIL`, `NAME`, `EMAIL`, where an email var may be `Name `), +/// then from git config. Never guessed from the login or hostname. +fn maintainer( + source_dir: &Path, + get_env: impl Fn(&str) -> Option, +) -> anyhow::Result { + let env = |key| get_env(key).filter(|value: &String| !value.trim().is_empty()); + let split = |value: String| -> (Option, String) { + let (name, email) = debian_changelog::parseaddr(&value); + (name.map(str::to_string), email.to_string()) + }; + let (debemail_name, debemail) = env("DEBEMAIL").map(split).unzip(); + let (email_name, email) = env("EMAIL").map(split).unzip(); + let name = env("DEBFULLNAME") + .or(debemail_name.flatten()) + .or_else(|| env("NAME")) + .or(email_name.flatten()) + .or_else(|| git_config(source_dir, "user.name")); + let email = debemail + .or(email) + .or_else(|| git_config(source_dir, "user.email")); + match (name, email) { + (Some(name), Some(email)) => Ok((name.trim().to_string(), email.trim().to_string())), + _ => bail!( + "unknown changelog maintainer: set DEBFULLNAME and DEBEMAIL (or git's user.name and user.email)" + ), + } +} + +fn git_config(dir: &Path, key: &str) -> Option { + let output = Command::new("git") + .arg("-C") + .arg(dir) + .args(["config", "--get", key]) + .output() + .ok()?; + let value = String::from_utf8(output.stdout).ok()?.trim().to_string(); + (output.status.success() && !value.is_empty()).then_some(value) +} + /// Read `debian/source/format`; a missing file is `1.0`, like dpkg-source. pub fn load_source_format(dir: &Path) -> SourceFormat { std::fs::read_to_string(dir.join("debian").join("source").join("format")) @@ -47,3 +127,58 @@ pub fn changelog_head( anyhow::anyhow!("changelog head entry has no package, version or distribution") }) } + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::*; + + fn maintainer_from(vars: &[(&str, &str)]) -> Maintainer { + let vars: HashMap<_, _> = vars.iter().copied().collect(); + // a dir outside any git worktree with no config, so only the env counts + maintainer(Path::new("/nonexistent"), |key| { + vars.get(key).map(|value| value.to_string()) + }) + .unwrap() + } + + #[test] + fn test_maintainer_deb_vars() { + assert_eq!( + maintainer_from(&[ + ("DEBFULLNAME", "Jane Doe"), + ("DEBEMAIL", "jane@example.com"), + ("EMAIL", "other@example.com"), + ]), + ("Jane Doe".into(), "jane@example.com".into()) + ); + } + + #[test] + fn test_maintainer_name_from_email_var() { + assert_eq!( + maintainer_from(&[("DEBEMAIL", "Jane Doe ")]), + ("Jane Doe".into(), "jane@example.com".into()) + ); + } + + #[test] + fn test_maintainer_debfullname_wins_over_debemail_name() { + assert_eq!( + maintainer_from(&[ + ("DEBFULLNAME", "Jane Doe"), + ("DEBEMAIL", "J. D. "), + ]), + ("Jane Doe".into(), "jane@example.com".into()) + ); + } + + #[test] + fn test_maintainer_generic_vars() { + assert_eq!( + maintainer_from(&[("NAME", "Jane Doe"), ("EMAIL", "jane@example.com")]), + ("Jane Doe".into(), "jane@example.com".into()) + ); + } +} diff --git a/packages/debmagic/src/cli.rs b/packages/debmagic/src/cli.rs index 37a70202..df21e9f5 100644 --- a/packages/debmagic/src/cli.rs +++ b/packages/debmagic/src/cli.rs @@ -68,10 +68,32 @@ pub enum Commands { Config(ConfigSubcommandArgs), #[command(about = "Query and switch upstream versions")] Upstream(UpstreamSubcommandArgs), + #[command(about = "Add entries to debian/changelog")] + Changelog(ChangelogSubcommandArgs), #[command(about = "Show version information")] Version {}, } +#[derive(Args, Debug)] +pub struct ChangelogSubcommandArgs { + #[command(subcommand)] + pub command: ChangelogCommands, +} + +#[derive(Subcommand, Debug)] +pub enum ChangelogCommands { + #[command( + about = "Add an UNRELEASED entry on top of debian/changelog, keeping the current version" + )] + New(ChangelogNewArgs), +} + +#[derive(Args, Debug)] +pub struct ChangelogNewArgs { + #[command(flatten)] + pub common: CommonCli, +} + #[derive(Args, Debug)] pub struct UpstreamSubcommandArgs { #[command(subcommand)] diff --git a/packages/debmagic/src/main.rs b/packages/debmagic/src/main.rs index 1cbd5191..9be6cf59 100644 --- a/packages/debmagic/src/main.rs +++ b/packages/debmagic/src/main.rs @@ -3,7 +3,9 @@ use std::process::ExitCode; use anyhow::Context; use clap::{CommandFactory, Parser}; -use crate::cli::{BuildTarget, Cli, Commands, ConfigCommands, EnvCommands, UpstreamCommands}; +use crate::cli::{ + BuildTarget, ChangelogCommands, Cli, Commands, ConfigCommands, EnvCommands, UpstreamCommands, +}; pub mod build; pub mod build_intent; @@ -241,6 +243,16 @@ async fn run() -> anyhow::Result { .await?; } }, + Commands::Changelog(args) => match &args.command { + ChangelogCommands::New(new_args) => { + changelog::new_entry(changelog::ChangelogNewCommand { + source_dir: package::resolve_source_dir( + ¤t_dir, + new_args.common.source_dir.as_deref(), + )?, + })?; + } + }, Commands::Version {} => { let cmd = Cli::command(); println!("{}", cmd.render_version()); From e8f9e909fd916118e87258309f12093e90ba34c8 Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Tue, 6 Oct 2026 17:06:12 +0200 Subject: [PATCH 03/48] refactor(cli): resolve the source dir in main for every command --- packages/debmagic/src/build_intent.rs | 13 +++++------- packages/debmagic/src/config.rs | 7 ++----- packages/debmagic/src/main.rs | 30 ++++++++++++++++++--------- packages/debmagic/src/test/intent.rs | 13 +++++------- packages/debmagic/src/upstream.rs | 14 +++++-------- 5 files changed, 37 insertions(+), 40 deletions(-) diff --git a/packages/debmagic/src/build_intent.rs b/packages/debmagic/src/build_intent.rs index 52cf1f9c..f0a1fa5b 100644 --- a/packages/debmagic/src/build_intent.rs +++ b/packages/debmagic/src/build_intent.rs @@ -27,9 +27,7 @@ impl BuildKind { /// Inputs for resolving a [`BuildIntent`]. #[derive(Debug, Clone)] pub struct BuildIntentInput { - /// Directory used when `source_dir` / `output_dir` are unset (typically cwd). - pub fallback_dir: PathBuf, - pub source_dir: Option, + pub source_dir: PathBuf, pub output_dir: Option, pub config_file: Option, pub driver: Option, @@ -65,8 +63,8 @@ pub struct BuildIntent { pub fn resolve_build_intent(input: BuildIntentInput) -> anyhow::Result { // Canonicalized: Environment ids and registry lookups key on the // source_dir string, so all entry points must agree on one spelling. - let source_dir = std::fs::canonicalize(input.source_dir.unwrap_or(input.fallback_dir)) - .context("resolving source dir failed")?; + let source_dir = + std::fs::canonicalize(input.source_dir).context("resolving source dir failed")?; let mut config = Config::load(Some(&source_dir), input.config_file.as_deref())?; @@ -165,10 +163,9 @@ mod tests { .join("config1.toml") } - fn base_input(fallback: PathBuf) -> BuildIntentInput { + fn base_input(source_dir: PathBuf) -> BuildIntentInput { BuildIntentInput { - fallback_dir: fallback, - source_dir: None, + source_dir, output_dir: None, config_file: Some(asset_config()), driver: Some(DriverType::Docker), diff --git a/packages/debmagic/src/config.rs b/packages/debmagic/src/config.rs index 8b34247e..587e3cd8 100644 --- a/packages/debmagic/src/config.rs +++ b/packages/debmagic/src/config.rs @@ -148,9 +148,7 @@ pub fn resolve_set_target( /// Inputs for the `debmagic config` command. #[derive(Debug, Clone)] pub struct ConfigCommand { - /// Directory used when `source_dir` is unset (typically cwd). - pub fallback_dir: PathBuf, - pub source_dir: Option, + pub source_dir: PathBuf, pub config_file: Option, pub command: ConfigCommandKind, } @@ -174,8 +172,7 @@ pub enum ConfigCommandKind { /// Run the `debmagic config` command: show, get or set config values. pub fn run_config(command: ConfigCommand) -> anyhow::Result<()> { - let source_dir = - crate::package::resolve_source_dir(&command.fallback_dir, command.source_dir.as_deref())?; + let source_dir = command.source_dir; match &command.command { ConfigCommandKind::Show => { let paths = Config::resolve_paths(Some(&source_dir), command.config_file.as_deref())?; diff --git a/packages/debmagic/src/main.rs b/packages/debmagic/src/main.rs index 9be6cf59..3219b3c3 100644 --- a/packages/debmagic/src/main.rs +++ b/packages/debmagic/src/main.rs @@ -67,8 +67,10 @@ async fn run() -> anyhow::Result { }; build::build(build::BuildCommand { intent: build_intent::BuildIntentInput { - fallback_dir: current_dir.clone(), - source_dir: build_args.common.source_dir.clone(), + source_dir: package::resolve_source_dir( + ¤t_dir, + build_args.common.source_dir.as_deref(), + )?, output_dir: build_args.output_dir.clone(), config_file: cli.config.clone(), driver: build_args.driver, @@ -132,8 +134,10 @@ async fn run() -> anyhow::Result { }, Commands::Test(args) => { let intent = test::resolve_test_intent(test::TestIntentInput { - fallback_dir: current_dir.clone(), - source_dir: args.common.source_dir.clone(), + source_dir: package::resolve_source_dir( + ¤t_dir, + args.common.source_dir.as_deref(), + )?, config_file: cli.config.clone(), driver: args.driver, persistent: args.persistent, @@ -213,8 +217,10 @@ async fn run() -> anyhow::Result { ), }; config::run_config(config::ConfigCommand { - fallback_dir: current_dir.clone(), - source_dir: common.source_dir.clone(), + source_dir: package::resolve_source_dir( + ¤t_dir, + common.source_dir.as_deref(), + )?, config_file: cli.config.clone(), command, })?; @@ -222,8 +228,10 @@ async fn run() -> anyhow::Result { Commands::Upstream(args) => match &args.command { UpstreamCommands::List(list_args) => { upstream::list(upstream::UpstreamListCommand { - fallback_dir: current_dir.clone(), - source_dir: list_args.common.source_dir.clone(), + source_dir: package::resolve_source_dir( + ¤t_dir, + list_args.common.source_dir.as_deref(), + )?, config_file: cli.config.clone(), all: list_args.all, previous: list_args.previous, @@ -232,8 +240,10 @@ async fn run() -> anyhow::Result { } UpstreamCommands::Switch(switch_args) => { upstream::switch(upstream::UpstreamSwitchCommand { - fallback_dir: current_dir.clone(), - source_dir: switch_args.common.source_dir.clone(), + source_dir: package::resolve_source_dir( + ¤t_dir, + switch_args.common.source_dir.as_deref(), + )?, config_file: cli.config.clone(), version: switch_args.version.clone(), dry_run: switch_args.dry_run, diff --git a/packages/debmagic/src/test/intent.rs b/packages/debmagic/src/test/intent.rs index cbb5c139..57023ffd 100644 --- a/packages/debmagic/src/test/intent.rs +++ b/packages/debmagic/src/test/intent.rs @@ -10,9 +10,7 @@ use crate::{ /// Inputs for resolving a [`TestIntent`]. #[derive(Debug, Clone)] pub struct TestIntentInput { - /// Directory used when `source_dir` is unset (typically cwd). - pub fallback_dir: PathBuf, - pub source_dir: Option, + pub source_dir: PathBuf, pub config_file: Option, pub driver: Option, pub persistent: Option, @@ -41,8 +39,8 @@ pub struct TestIntent { pub fn resolve_test_intent(input: TestIntentInput) -> anyhow::Result { // Canonicalized: Environment ids and registry lookups key on the // source_dir string, so all entry points must agree on one spelling. - let source_dir = std::fs::canonicalize(input.source_dir.unwrap_or(input.fallback_dir)) - .context("resolving source dir failed")?; + let source_dir = + std::fs::canonicalize(input.source_dir).context("resolving source dir failed")?; let mut config = Config::load(Some(&source_dir), input.config_file.as_deref())?; @@ -83,10 +81,9 @@ mod tests { .join("config1.toml") } - fn base_input(fallback: PathBuf) -> TestIntentInput { + fn base_input(source_dir: PathBuf) -> TestIntentInput { TestIntentInput { - fallback_dir: fallback, - source_dir: None, + source_dir, config_file: Some(asset_config()), driver: None, persistent: None, diff --git a/packages/debmagic/src/upstream.rs b/packages/debmagic/src/upstream.rs index aa108284..07a8087f 100644 --- a/packages/debmagic/src/upstream.rs +++ b/packages/debmagic/src/upstream.rs @@ -4,7 +4,7 @@ use std::str::FromStr; use anyhow::{Context, bail}; use crate::config::Config; -use crate::package::{load_package, resolve_source_dir}; +use crate::package::load_package; pub mod mangle; pub mod orig; @@ -16,9 +16,7 @@ pub mod watch; /// Inputs for the `debmagic upstream list` command. #[derive(Debug, Clone)] pub struct UpstreamListCommand { - /// Directory used when `source_dir` is unset (typically cwd). - pub fallback_dir: PathBuf, - pub source_dir: Option, + pub source_dir: PathBuf, pub config_file: Option, /// Show all candidate versions, not just the newest newer ones. pub all: bool, @@ -29,9 +27,7 @@ pub struct UpstreamListCommand { /// Inputs for the `debmagic upstream switch` command. #[derive(Debug, Clone)] pub struct UpstreamSwitchCommand { - /// Directory used when `source_dir` is unset (typically cwd). - pub fallback_dir: PathBuf, - pub source_dir: Option, + pub source_dir: PathBuf, pub config_file: Option, /// The upstream version to switch to, or "latest" for the newest eligible. pub version: String, @@ -46,7 +42,7 @@ pub struct UpstreamSwitchCommand { /// Run the `debmagic upstream list` command: query each watch source and /// print its current and newer upstream versions. pub async fn list(command: UpstreamListCommand) -> anyhow::Result<()> { - let source_dir = resolve_source_dir(&command.fallback_dir, command.source_dir.as_deref())?; + let source_dir = command.source_dir; let identity = load_package(&source_dir)?; let sources = query::load_watch(&source_dir, identity.name())?; @@ -89,7 +85,7 @@ pub async fn list(command: UpstreamListCommand) -> anyhow::Result<()> { /// Run the `debmagic upstream switch` command: resolve the candidate /// version, then switch the main source and every component source to it. pub async fn switch(command: UpstreamSwitchCommand) -> anyhow::Result<()> { - let source_dir = resolve_source_dir(&command.fallback_dir, command.source_dir.as_deref())?; + let source_dir = command.source_dir.clone(); let identity = load_package(&source_dir)?; let sources = query::load_watch(&source_dir, identity.name())?; let mut config = Config::load(Some(&source_dir), command.config_file.as_deref())?; From 8803a521530cc7052e09e63ff985e08a390ff71a Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Wed, 7 Oct 2026 13:52:36 +0200 Subject: [PATCH 04/48] refactor(common): model debian/control and debian/copyright SourcePackage parses each lazily once and caches it. The .dsc/.changes checksum helpers move from debian::control to checksums. --- packages/debmagic-common/src/changes.rs | 2 +- packages/debmagic-common/src/checksums.rs | 232 ++++++++++++++++ .../debmagic-common/src/debian/control.rs | 247 +++--------------- .../debmagic-common/src/debian/copyright.rs | 16 +- packages/debmagic-common/src/lib.rs | 1 + packages/debmagic-common/src/package.rs | 68 ++++- packages/debmagic/src/control.rs | 2 +- packages/debmagic/src/upload/orig.rs | 2 +- packages/debmagic/src/upstream/orig.rs | 3 +- packages/debmagic/src/upstream/repack.rs | 2 +- 10 files changed, 342 insertions(+), 233 deletions(-) create mode 100644 packages/debmagic-common/src/checksums.rs diff --git a/packages/debmagic-common/src/changes.rs b/packages/debmagic-common/src/changes.rs index af07b618..20ad3a39 100644 --- a/packages/debmagic-common/src/changes.rs +++ b/packages/debmagic-common/src/changes.rs @@ -1,6 +1,6 @@ use deb822_lossless::Paragraph; -use crate::debian::control::Hash; +use crate::checksums::Hash; /// A parsed `.changes` file listing: which files it references and /// whether the `orig` tarball(s) are among them. diff --git a/packages/debmagic-common/src/checksums.rs b/packages/debmagic-common/src/checksums.rs new file mode 100644 index 00000000..033142b7 --- /dev/null +++ b/packages/debmagic-common/src/checksums.rs @@ -0,0 +1,232 @@ +//! The `Files:`/`Checksums-*:` fields of `.dsc`, `.changes` and +//! `.buildinfo` files: child-file lookup, verification and rewriting. + +use deb822_lossless::Paragraph; + +/// Filenames ending in `.` listed under `Files:` or any +/// `Checksums-*:` field. +pub fn child_filename(paragraph: &Paragraph, ext: &str) -> Option { + let suffix = format!(".{ext}"); + for key in paragraph.keys() { + if key != "Files" && !key.starts_with("Checksums-") { + continue; + } + for line in paragraph + .get(&key) + .into_iter() + .flat_map(|v| v.lines().map(str::to_string).collect::>()) + { + if let Some(name) = line.split_whitespace().next_back() + && name.ends_with(&suffix) + { + return Some(name.to_string()); + } + } + } + None +} + +/// The hash algorithms used by the checksum fields of a control file. +#[derive(Clone, Copy)] +pub enum Hash { + Md5, + Sha1, + Sha256, +} + +impl Hash { + pub fn hex(self, data: &[u8]) -> String { + fn hex(mut hasher: D, data: &[u8]) -> String { + hasher.update(data); + hasher + .finalize() + .iter() + .map(|b| format!("{b:02x}")) + .collect() + } + match self { + Hash::Md5 => hex(md5::Md5::default(), data), + Hash::Sha1 => hex(sha1::Sha1::default(), data), + Hash::Sha256 => hex(sha2::Sha256::default(), data), + } + } +} + +/// The checksum fields debmagic understands, mapped to their hash. +pub const CHECKSUM_FIELDS: &[(&str, Hash)] = &[ + ("Files", Hash::Md5), + ("Checksums-Sha1", Hash::Sha1), + ("Checksums-Sha256", Hash::Sha256), +]; + +/// Rewrite one `Files:`/`Checksums-*:` line: the first token is the +/// checksum, the second the size, the last the filename; entries for +/// other files pass through unchanged. +fn rewrite_checksum_line(line: &str, filename: &str, checksum: &str, size: usize) -> String { + let tokens: Vec<&str> = line.split_whitespace().collect(); + match tokens.as_slice() { + [old_checksum, old_size, middle @ .., name] if *name == filename => { + let middle = if middle.is_empty() { + String::new() + } else { + format!(" {}", middle.join(" ")) + }; + format!("{checksum} {size}{middle} {name}") + } + _ => line.to_string(), + } +} + +/// Rewrite the size and checksum entries for the file listings in a control file +pub fn fixup_checksums( + paragraph: &mut Paragraph, + filename: &str, + data: &[u8], +) -> anyhow::Result<()> { + let size = data.len(); + + for key in paragraph.keys() { + if key.starts_with("Checksums-") && !CHECKSUM_FIELDS.iter().any(|(field, ..)| *field == key) + { + // An unknown checksum format would keep a stale checksum for a + // re-signed file, producing an upload that fails verification + // far away from here. + anyhow::bail!("unknown checksum field '{key}:' in control file"); + } + } + + for (key, hash) in CHECKSUM_FIELDS { + let Some(value) = paragraph.get(key) else { + continue; + }; + let checksum = hash.hex(data); + let updated = value + .lines() + .filter(|line| !line.is_empty()) + .map(|line| rewrite_checksum_line(line, filename, &checksum, size)) + .collect::>() + .join("\n"); + paragraph.set(key, &updated); + } + Ok(()) +} + +/// The digests of one file, for every algorithm a control file's +/// checksum fields may use. +#[derive(Debug, Clone, Default)] +pub struct Digests { + pub md5: String, + pub sha1: String, + pub sha256: String, +} + +impl Digests { + /// Digest `data` with every checksum-field algorithm at once. + pub fn of(data: &[u8]) -> Self { + Self { + md5: Hash::Md5.hex(data), + sha1: Hash::Sha1.hex(data), + sha256: Hash::Sha256.hex(data), + } + } + + fn for_hash(&self, hash: Hash) -> &str { + match hash { + Hash::Md5 => &self.md5, + Hash::Sha1 => &self.sha1, + Hash::Sha256 => &self.sha256, + } + } +} + +/// Verify that the checksum entries a paragraph records for `filename` +/// match `digests`. Returns whether any entry was checked; a paragraph +/// listing no entry for the file is not an error. +pub fn verify_checksums( + paragraph: &Paragraph, + filename: &str, + digests: &Digests, +) -> anyhow::Result { + let mut checked = false; + for (field, hash) in CHECKSUM_FIELDS { + let Some(value) = paragraph.get(field) else { + continue; + }; + let actual = digests.for_hash(*hash); + for line in value.lines().filter(|l| !l.is_empty()) { + let tokens: Vec<&str> = line.split_whitespace().collect(); + if tokens.last() != Some(&filename) { + continue; + } + let Some(&expected) = tokens.first() else { + continue; + }; + checked = true; + if actual != expected { + anyhow::bail!("{filename} does not match the checksum {field} recorded for it"); + } + } + } + Ok(checked) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn verify_checksums_matches_and_mismatches() { + let data = b"tarball content"; + let digests = Digests::of(data); + let control: Paragraph = format!( + "Format: 3.0 (quilt)\nFiles:\n {} 15 debiandev optional pkg_1.0.orig.tar.xz\nChecksums-Sha256:\n {} 15 pkg_1.0.orig.tar.xz\n", + digests.md5, digests.sha256 + ) + .parse() + .unwrap(); + + assert!(verify_checksums(&control, "pkg_1.0.orig.tar.xz", &digests).unwrap()); + + // a stale tarball fails + let stale = Digests::of(b"different content"); + assert!(verify_checksums(&control, "pkg_1.0.orig.tar.xz", &stale).is_err()); + + // a file the paragraph does not list is not checked + assert!(!verify_checksums(&control, "other.tar.xz", &digests).unwrap()); + } + + fn parse_control(content: &str) -> Paragraph { + content.parse().unwrap() + } + + #[test] + fn child_filename_finds_dsc_and_buildinfo() { + let control = parse_control( + "Format: 1.8\nSource: pkg\nFiles:\n abc 123 pkg_1.0.dsc\n def 456 pkg_1.0.buildinfo\n ghi 789 other.txt\nChecksums-Sha256:\n xyz 123 pkg_1.0.dsc\n", + ); + assert_eq!( + child_filename(&control, "dsc").as_deref(), + Some("pkg_1.0.dsc") + ); + assert_eq!( + child_filename(&control, "buildinfo").as_deref(), + Some("pkg_1.0.buildinfo") + ); + assert_eq!(child_filename(&control, "deb"), None); + } + + #[test] + fn fixup_rewrites_all_checksum_sections() { + let mut control = parse_control( + "Format: 1.8\nFiles:\n oldmd5 3 hash optional pkg_1.0.dsc\nChecksums-Sha1:\n oldsha1 3 pkg_1.0.dsc\nChecksums-Sha256:\n oldsha256 3 pkg_1.0.dsc\n", + ); + let data = b"abc"; + fixup_checksums(&mut control, "pkg_1.0.dsc", data).unwrap(); + + let md5 = Hash::Md5.hex(data); + let joined = control.to_string(); + assert!(joined.contains(&format!("{md5} 3 hash optional pkg_1.0.dsc"))); + assert!(joined.contains(&format!(" {} 3 pkg_1.0.dsc", Hash::Sha1.hex(data)))); + assert!(joined.contains(&format!(" {} 3 pkg_1.0.dsc", Hash::Sha256.hex(data)))); + } +} diff --git a/packages/debmagic-common/src/debian/control.rs b/packages/debmagic-common/src/debian/control.rs index b193bb17..c241009a 100644 --- a/packages/debmagic-common/src/debian/control.rs +++ b/packages/debmagic-common/src/debian/control.rs @@ -1,229 +1,46 @@ -use deb822_lossless::Paragraph; +use anyhow::Context; -/// Filenames ending in `.` listed under `Files:` or any -/// `Checksums-*:` field. -pub fn child_filename(paragraph: &Paragraph, ext: &str) -> Option { - let suffix = format!(".{ext}"); - for key in paragraph.keys() { - if key != "Files" && !key.starts_with("Checksums-") { - continue; - } - for line in paragraph - .get(&key) - .into_iter() - .flat_map(|v| v.lines().map(str::to_string).collect::>()) - { - if let Some(name) = line.split_whitespace().next_back() - && name.ends_with(&suffix) - { - return Some(name.to_string()); - } - } - } - None -} - -/// The hash algorithms used by the checksum fields of a control file. -#[derive(Clone, Copy)] -pub enum Hash { - Md5, - Sha1, - Sha256, -} - -impl Hash { - pub fn hex(self, data: &[u8]) -> String { - fn hex(mut hasher: D, data: &[u8]) -> String { - hasher.update(data); - hasher - .finalize() - .iter() - .map(|b| format!("{b:02x}")) - .collect() - } - match self { - Hash::Md5 => hex(md5::Md5::default(), data), - Hash::Sha1 => hex(sha1::Sha1::default(), data), - Hash::Sha256 => hex(sha2::Sha256::default(), data), - } - } -} - -/// The checksum fields debmagic understands, mapped to their hash. -pub const CHECKSUM_FIELDS: &[(&str, Hash)] = &[ - ("Files", Hash::Md5), - ("Checksums-Sha1", Hash::Sha1), - ("Checksums-Sha256", Hash::Sha256), -]; - -/// Rewrite one `Files:`/`Checksums-*:` line: the first token is the -/// checksum, the second the size, the last the filename; entries for -/// other files pass through unchanged. -fn rewrite_checksum_line(line: &str, filename: &str, checksum: &str, size: usize) -> String { - let tokens: Vec<&str> = line.split_whitespace().collect(); - match tokens.as_slice() { - [old_checksum, old_size, middle @ .., name] if *name == filename => { - let middle = if middle.is_empty() { - String::new() - } else { - format!(" {}", middle.join(" ")) - }; - format!("{checksum} {size}{middle} {name}") - } - _ => line.to_string(), - } -} - -/// Rewrite the size and checksum entries for the file listings in a control file -pub fn fixup_checksums( - paragraph: &mut Paragraph, - filename: &str, - data: &[u8], -) -> anyhow::Result<()> { - let size = data.len(); - - for key in paragraph.keys() { - if key.starts_with("Checksums-") && !CHECKSUM_FIELDS.iter().any(|(field, ..)| *field == key) - { - // An unknown checksum format would keep a stale checksum for a - // re-signed file, producing an upload that fails verification - // far away from here. - anyhow::bail!("unknown checksum field '{key}:' in control file"); - } - } - - for (key, hash) in CHECKSUM_FIELDS { - let Some(value) = paragraph.get(key) else { - continue; - }; - let checksum = hash.hex(data); - let updated = value - .lines() - .filter(|line| !line.is_empty()) - .map(|line| rewrite_checksum_line(line, filename, &checksum, size)) - .collect::>() - .join("\n"); - paragraph.set(key, &updated); - } - Ok(()) -} - -/// The digests of one file, for every algorithm a control file's -/// checksum fields may use. +/// The model of a package's `debian/control`. #[derive(Debug, Clone, Default)] -pub struct Digests { - pub md5: String, - pub sha1: String, - pub sha256: String, +pub struct Control { + /// The source paragraph's `Rules-Requires-Root` keywords, `None` if absent. + pub rules_requires_root: Option>, } -impl Digests { - /// Digest `data` with every checksum-field algorithm at once. - pub fn of(data: &[u8]) -> Self { - Self { - md5: Hash::Md5.hex(data), - sha1: Hash::Sha1.hex(data), - sha256: Hash::Sha256.hex(data), - } - } - - fn for_hash(&self, hash: Hash) -> &str { - match hash { - Hash::Md5 => &self.md5, - Hash::Sha1 => &self.sha1, - Hash::Sha256 => &self.sha256, - } +impl Control { + pub fn parse(content: &str) -> anyhow::Result { + let deb822 = content + .parse::() + .context("invalid deb822")?; + let source = deb822.paragraphs().next().context("no source paragraph")?; + Ok(Self { + rules_requires_root: source + .get("Rules-Requires-Root") + .map(|value| value.split_whitespace().map(str::to_string).collect()), + }) } -} -/// Verify that the checksum entries a paragraph records for `filename` -/// match `digests`. Returns whether any entry was checked; a paragraph -/// listing no entry for the file is not an error. -pub fn verify_checksums( - paragraph: &Paragraph, - filename: &str, - digests: &Digests, -) -> anyhow::Result { - let mut checked = false; - for (field, hash) in CHECKSUM_FIELDS { - let Some(value) = paragraph.get(field) else { - continue; - }; - let actual = digests.for_hash(*hash); - for line in value.lines().filter(|l| !l.is_empty()) { - let tokens: Vec<&str> = line.split_whitespace().collect(); - if tokens.last() != Some(&filename) { - continue; - } - let Some(&expected) = tokens.first() else { - continue; - }; - checked = true; - if actual != expected { - anyhow::bail!("{filename} does not match the checksum {field} recorded for it"); - } - } + /// Whether `dpkg-buildpackage` demands a gain-root command (`fakeroot`) + /// at startup, whatever targets it runs: unless `Rules-Requires-Root` + /// contains `no`. An absent field counts as needing one, since dpkg + /// before 1.22.13 defaults it to `binary-targets`. + pub fn needs_gain_root(&self) -> bool { + self.rules_requires_root + .as_ref() + .is_none_or(|keywords| !keywords.iter().any(|keyword| keyword == "no")) } - Ok(checked) } #[cfg(test)] mod tests { use super::*; - - #[test] - fn verify_checksums_matches_and_mismatches() { - let data = b"tarball content"; - let digests = Digests::of(data); - let control: Paragraph = format!( - "Format: 3.0 (quilt)\nFiles:\n {} 15 debiandev optional pkg_1.0.orig.tar.xz\nChecksums-Sha256:\n {} 15 pkg_1.0.orig.tar.xz\n", - digests.md5, digests.sha256 - ) - .parse() - .unwrap(); - - assert!(verify_checksums(&control, "pkg_1.0.orig.tar.xz", &digests).unwrap()); - - // a stale tarball fails - let stale = Digests::of(b"different content"); - assert!(verify_checksums(&control, "pkg_1.0.orig.tar.xz", &stale).is_err()); - - // a file the paragraph does not list is not checked - assert!(!verify_checksums(&control, "other.tar.xz", &digests).unwrap()); - } - - fn parse_control(content: &str) -> Paragraph { - content.parse().unwrap() - } - - #[test] - fn child_filename_finds_dsc_and_buildinfo() { - let control = parse_control( - "Format: 1.8\nSource: pkg\nFiles:\n abc 123 pkg_1.0.dsc\n def 456 pkg_1.0.buildinfo\n ghi 789 other.txt\nChecksums-Sha256:\n xyz 123 pkg_1.0.dsc\n", - ); - assert_eq!( - child_filename(&control, "dsc").as_deref(), - Some("pkg_1.0.dsc") - ); - assert_eq!( - child_filename(&control, "buildinfo").as_deref(), - Some("pkg_1.0.buildinfo") - ); - assert_eq!(child_filename(&control, "deb"), None); - } - - #[test] - fn fixup_rewrites_all_checksum_sections() { - let mut control = parse_control( - "Format: 1.8\nFiles:\n oldmd5 3 hash optional pkg_1.0.dsc\nChecksums-Sha1:\n oldsha1 3 pkg_1.0.dsc\nChecksums-Sha256:\n oldsha256 3 pkg_1.0.dsc\n", - ); - let data = b"abc"; - fixup_checksums(&mut control, "pkg_1.0.dsc", data).unwrap(); - - let md5 = Hash::Md5.hex(data); - let joined = control.to_string(); - assert!(joined.contains(&format!("{md5} 3 hash optional pkg_1.0.dsc"))); - assert!(joined.contains(&format!(" {} 3 pkg_1.0.dsc", Hash::Sha1.hex(data)))); - assert!(joined.contains(&format!(" {} 3 pkg_1.0.dsc", Hash::Sha256.hex(data)))); + use test_case::test_case; + + #[test_case("Source: foo\nRules-Requires-Root: no\n", false ; "no")] + #[test_case("Source: foo\nRules-Requires-Root: binary-targets\n", true ; "binary targets")] + #[test_case("Source: foo\nRules-Requires-Root: dpkg/target-subcommand\n", true ; "implementation keyword")] + #[test_case("Source: foo\n\nPackage: foo\nRules-Requires-Root: no\n", true ; "absent from source paragraph")] + fn test_needs_gain_root(control: &str, expected: bool) { + assert_eq!(Control::parse(control).unwrap().needs_gain_root(), expected); } } diff --git a/packages/debmagic-common/src/debian/copyright.rs b/packages/debmagic-common/src/debian/copyright.rs index 3b853dbf..85d35577 100644 --- a/packages/debmagic-common/src/debian/copyright.rs +++ b/packages/debmagic-common/src/debian/copyright.rs @@ -1,3 +1,17 @@ +/// The model of a package's `debian/copyright`. +#[derive(Debug, Clone, Default)] +pub struct Copyright { + pub files_excluded: FilesExcluded, +} + +impl Copyright { + pub fn parse(content: &str) -> Self { + Self { + files_excluded: files_excluded(content), + } + } +} + /// The `Files-Excluded` patterns from a `debian/copyright` file. /// /// The field is a machine-readable header (not part of the deb822 @@ -14,7 +28,7 @@ pub struct FilesExcluded { /// Parse the `Files-Excluded` fields from a `debian/copyright` /// file's content. -pub fn files_excluded(content: &str) -> FilesExcluded { +fn files_excluded(content: &str) -> FilesExcluded { let mut result = FilesExcluded::default(); // the component whose patterns we are collecting; `None` = main let mut current: Option> = None; diff --git a/packages/debmagic-common/src/lib.rs b/packages/debmagic-common/src/lib.rs index 44ece3ad..5ebd9eb9 100644 --- a/packages/debmagic-common/src/lib.rs +++ b/packages/debmagic-common/src/lib.rs @@ -1,4 +1,5 @@ pub mod changes; +pub mod checksums; pub mod debian; pub mod distro; pub mod package; diff --git a/packages/debmagic-common/src/package.rs b/packages/debmagic-common/src/package.rs index ba1b0dc3..838713fc 100644 --- a/packages/debmagic-common/src/package.rs +++ b/packages/debmagic-common/src/package.rs @@ -1,8 +1,12 @@ +use std::cell::OnceCell; use std::path::{Path, PathBuf}; use std::rc::Rc; +use anyhow::Context; + use crate::debian::changelog::ChangelogHead; -use crate::debian::copyright::FilesExcluded; +use crate::debian::control::Control; +use crate::debian::copyright::Copyright; use crate::debian::source::SourceFormat; use crate::debian::version::PackageVersion; @@ -35,6 +39,8 @@ pub struct SourcePackage { location: Location, changelog: ChangelogHead, source_format: SourceFormat, + control: OnceCell, + copyright: OnceCell, } impl std::fmt::Debug for SourcePackage { @@ -68,6 +74,8 @@ impl SourcePackage { location, changelog, source_format, + control: OnceCell::new(), + copyright: OnceCell::new(), }) } @@ -136,13 +144,32 @@ impl SourcePackage { } } - /// The `Files-Excluded` patterns from `debian/copyright`. - /// Read lazily; a missing file yields no excludes. - pub fn copyright_excludes(&self) -> anyhow::Result { - Ok((self.reader)("copyright") - .map(|content| crate::debian::copyright::files_excluded(&content)) - .unwrap_or_default()) + /// The `debian/control` model. + pub fn control(&self) -> anyhow::Result<&Control> { + lazy(&self.control, || { + let content = (self.reader)("control")?; + Control::parse(&content).context("failed to parse debian/control") + }) + } + + /// The `debian/copyright` model; a missing file is an empty one. + pub fn copyright(&self) -> anyhow::Result<&Copyright> { + lazy(&self.copyright, || { + Ok((self.reader)("copyright") + .map(|content| Copyright::parse(&content)) + .unwrap_or_default()) + }) + } +} + +/// A substructure read and parsed on first access, then cached; a failed +/// load is not cached, so the next access retries. +fn lazy(cell: &OnceCell, load: impl FnOnce() -> anyhow::Result) -> anyhow::Result<&T> { + if let Some(value) = cell.get() { + return Ok(value); } + let value = load()?; + Ok(cell.get_or_init(|| value)) } #[cfg(test)] @@ -207,17 +234,33 @@ mod tests { ]); let package = SourcePackage::from_reader(reader, Location::InMemory).unwrap(); assert_eq!(*reads.borrow(), ["changelog", "source/format"]); - let excludes = package.copyright_excludes().unwrap(); + let excludes = &package.copyright().unwrap().files_excluded; + // cached: a second access doesn't read again + package.copyright().unwrap(); assert_eq!(excludes.main, ["win32/*"]); assert_eq!(*reads.borrow(), ["changelog", "source/format", "copyright"]); } + #[test] + fn test_control_lazy_and_cached() { + let (reader, reads) = reader(&[ + ("changelog", CHANGELOG), + ("source/format", "3.0 (quilt)\n"), + ("control", "Source: postfix\nRules-Requires-Root: no\n"), + ]); + let package = SourcePackage::from_reader(reader, Location::InMemory).unwrap(); + assert_eq!(*reads.borrow(), ["changelog", "source/format"]); + assert!(!package.control().unwrap().needs_gain_root()); + assert!(!package.control().unwrap().needs_gain_root()); + assert_eq!(*reads.borrow(), ["changelog", "source/format", "control"]); + } + #[test] fn test_copyright_excludes_missing_file() { let (reader, _) = reader(&[("changelog", CHANGELOG), ("source/format", "3.0 (quilt)\n")]); let package = SourcePackage::from_reader(reader, Location::InMemory).unwrap(); // a missing copyright is no error, just no excludes - assert!(package.copyright_excludes().unwrap().main.is_empty()); + assert!(package.copyright().unwrap().files_excluded.main.is_empty()); } #[test] @@ -230,7 +273,10 @@ mod tests { .unwrap(); assert_eq!(package.name(), "postfix"); assert!(!package.is_native()); - assert_eq!(package.copyright_excludes().unwrap().main, ["win32/*"]); + assert_eq!( + package.copyright().unwrap().files_excluded.main, + ["win32/*"] + ); } #[test] @@ -241,7 +287,7 @@ mod tests { ("source/format", "3.0 (quilt)\n"), ]) .unwrap(); - assert!(package.copyright_excludes().unwrap().main.is_empty()); + assert!(package.copyright().unwrap().files_excluded.main.is_empty()); } #[test] diff --git a/packages/debmagic/src/control.rs b/packages/debmagic/src/control.rs index 43aec63e..b3078bb6 100644 --- a/packages/debmagic/src/control.rs +++ b/packages/debmagic/src/control.rs @@ -6,7 +6,7 @@ use debian_control::lossless::changes::Changes; use debian_control::pgp; use sha2::Digest; -pub use debmagic_common::debian::control::{ +pub use debmagic_common::checksums::{ CHECKSUM_FIELDS, Digests, Hash, child_filename, fixup_checksums, verify_checksums, }; diff --git a/packages/debmagic/src/upload/orig.rs b/packages/debmagic/src/upload/orig.rs index 92542d66..07ff9bb2 100644 --- a/packages/debmagic/src/upload/orig.rs +++ b/packages/debmagic/src/upload/orig.rs @@ -140,7 +140,7 @@ fn verify_against_dsc(tarball: &Path, changes_file: &Path) -> anyhow::Result<()> let digests = crate::control::digest_file(tarball)?; - debmagic_common::debian::control::verify_checksums(&dsc, &tarball_name, &digests).map_err( + debmagic_common::checksums::verify_checksums(&dsc, &tarball_name, &digests).map_err( |error| { anyhow::anyhow!( "{error}; the tarball next to the .changes is stale (recorded by {dsc_name})" diff --git a/packages/debmagic/src/upstream/orig.rs b/packages/debmagic/src/upstream/orig.rs index 2fbf0130..de669198 100644 --- a/packages/debmagic/src/upstream/orig.rs +++ b/packages/debmagic/src/upstream/orig.rs @@ -400,8 +400,7 @@ async fn fetch_from_pool( let dsc = crate::control::parse_control(&dsc) .with_context(|| format!("parsing {dsc_name} from {pool_url} failed"))?; let digests = crate::control::digest_file(&orig_path)?; - let verified = - debmagic_common::debian::control::verify_checksums(&dsc, &orig_name, &digests); + let verified = debmagic_common::checksums::verify_checksums(&dsc, &orig_name, &digests); if let Err(error) = verified { // never leave a tarball behind that the archive would reject std::fs::remove_file(&orig_path).ok(); diff --git a/packages/debmagic/src/upstream/repack.rs b/packages/debmagic/src/upstream/repack.rs index 8719de18..c5ed7929 100644 --- a/packages/debmagic/src/upstream/repack.rs +++ b/packages/debmagic/src/upstream/repack.rs @@ -21,7 +21,7 @@ pub fn load_repack_config( package: &debmagic_common::package::SourcePackage, watch: &crate::upstream::watch::WatchSource, ) -> anyhow::Result { - let mut excluded = package.copyright_excludes()?; + let mut excluded = package.copyright()?.files_excluded.clone(); let (excludes, field) = match &watch.component { Some(component) => ( excluded.components.remove(component).unwrap_or_default(), From 63a833b615a27d507e6bc569dc7b95213d013688 Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Tue, 6 Oct 2026 23:43:06 +0200 Subject: [PATCH 05/48] fix(build): install fakeroot when Rules-Requires-Root needs it Source builds fail with "fakeroot not found" on dpkg before 1.22.13, which checks for the gain-root command at startup even for -S. Both source and binary builds now install fakeroot unless Rules-Requires-Root is no. --- packages/debmagic/src/build/mod.rs | 45 ++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/packages/debmagic/src/build/mod.rs b/packages/debmagic/src/build/mod.rs index 3b3008ad..c9bc5aed 100644 --- a/packages/debmagic/src/build/mod.rs +++ b/packages/debmagic/src/build/mod.rs @@ -212,24 +212,21 @@ async fn run_build( .await } -pub async fn build_package( +pub async fn build_binary_package( intent: &BuildIntent, target: &PackageTarget, changes_options: &[String], ) -> anyhow::Result { let request = BuildRequest { intent, target }; run_build(&request, |claimed| { - crate::output::stage("Building binary packages"); + crate::output::stage("Building binary package(s)"); let staged_source_dir = stage_dir(claimed.environment(), BuildKind::Binary); // build-essential is an implicit dependency that `apt-get build-dep` - // won't resolve, so install it explicitly. No-op when the environment - // already has it (idempotent, and the bare driver runs on the host). - claimed.driver().run_command_checked( - &["apt-get", "-y", "install", "build-essential"], - &staged_source_dir, - true, - &[], - )?; + // won't pull in. No-op when the environment already has it + // (idempotent, and the bare driver runs on the host). + let mut packages = vec!["build-essential"]; + packages.extend(gain_root_package(target)?); + install_packages(claimed, &staged_source_dir, &packages)?; claimed.driver().run_command_checked( &["apt-get", "-y", "build-dep", "."], &staged_source_dir, @@ -274,6 +271,28 @@ pub async fn build_package( .await } +/// `fakeroot` if `dpkg-buildpackage` will demand it for this package. It is +/// only recommended by dpkg-dev, which `apt-get build-dep` doesn't pull in. +fn gain_root_package(target: &PackageTarget) -> anyhow::Result> { + Ok(target + .package + .control() + .context("checking Rules-Requires-Root failed")? + .needs_gain_root() + .then_some("fakeroot")) +} + +fn install_packages( + claimed: &ClaimedEnvironment, + dir: &Path, + packages: &[&str], +) -> anyhow::Result<()> { + let mut cmd = vec!["apt-get", "-y", "install"]; + cmd.extend(packages); + claimed.driver().run_command_checked(&cmd, dir, true, &[])?; + Ok(()) +} + /// Confirm `cmd` is on `PATH`, failing with an actionable message (rather /// than a raw "command not found") if it isn't. fn check_command_available(cmd: &str, install_hint: &str) -> anyhow::Result<()> { @@ -373,6 +392,10 @@ pub async fn build_source_package( ); stage_file(tarball, &destination)?; } + // dpkg-buildpackage looks for the gain-root command even for -S + if let Some(package) = gain_root_package(target)? { + install_packages(claimed, &staged_source_dir, &[package])?; + } if intent.config.clean { claimed.driver().run_command_checked( &["apt-get", "-y", "build-dep", "."], @@ -473,7 +496,7 @@ pub async fn build(command: BuildCommand) -> anyhow::Result<()> { ) .context("host's /etc/os-release does not match the build target distro")?; } - build_package(&intent, &target, &command.changes_options) + build_binary_package(&intent, &target, &command.changes_options) .await .context("Building the package failed")? }; From 4c7af37095dee77aaa4821265c10794db68adc3e Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Wed, 7 Oct 2026 00:15:50 +0200 Subject: [PATCH 06/48] fix(env): adjust table column widths --- packages/debmagic/src/env.rs | 34 +++++++++++++++++--------------- packages/debmagic/src/output.rs | 35 +++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 16 deletions(-) diff --git a/packages/debmagic/src/env.rs b/packages/debmagic/src/env.rs index ed3617d8..25cc7bc9 100644 --- a/packages/debmagic/src/env.rs +++ b/packages/debmagic/src/env.rs @@ -18,26 +18,28 @@ pub async fn list_environments(config: &Config) -> anyhow::Result<()> { return Ok(()); } - println!( - "{:<16} {:<6} {:<8} {:<18} {:<10} {:<10} {:<6} {:<11} SOURCE", - "ID", "PURPOSE", "DRIVER", "PACKAGE", "DISTRO", "PERSIST", "STATUS", "ATTACH" - ); + let mut rows = vec![ + [ + "ID", "PURPOSE", "DRIVER", "PACKAGE", "DISTRO", "PERSIST", "STATUS", "ATTACH", "SOURCE", + ] + .map(String::from), + ]; for registered in environments { let assessed = assess_environment(®istry, ®istered, &config.driver).await?; let env = ®istered.environment; - println!( - "{:<16} {:<6} {:<8} {:<18} {:<10} {:<10} {:<6} {:<11} {}", - env.id(), - env.purpose.as_str(), - env.driver.as_str(), - env.package_identifier, - env.distro.codename, - env.persistence.as_str(), - assessed.status.as_str(), - assessed.attachments, - env.source_dir.display(), - ); + rows.push([ + env.id().to_string(), + env.purpose.as_str().to_string(), + env.driver.as_str().to_string(), + env.package_identifier.to_string(), + env.distro.codename.to_string(), + env.persistence.as_str().to_string(), + assessed.status.as_str().to_string(), + assessed.attachments.to_string(), + env.source_dir.display().to_string(), + ]); } + print!("{}", crate::output::table(&rows)); Ok(()) } diff --git a/packages/debmagic/src/output.rs b/packages/debmagic/src/output.rs index b08b3198..a2521159 100644 --- a/packages/debmagic/src/output.rs +++ b/packages/debmagic/src/output.rs @@ -160,10 +160,45 @@ pub fn step(text: &str) { eprintln!("{}", styled(Style::Bold, &format!(" -> {text}"))); } +/// Lay out `rows` (header first) in columns as wide as their widest cell. +/// The last column is not padded, so a long path does not widen the table. +pub fn table(rows: &[[String; N]]) -> String { + let widths: [usize; N] = std::array::from_fn(|column| { + rows.iter() + .map(|row| row[column].chars().count()) + .max() + .unwrap_or(0) + }); + let mut out = String::new(); + for row in rows { + let line: Vec = row + .iter() + .zip(widths) + .map(|(cell, width)| format!("{cell: = Some("xterm-256color"); #[test] From ce430875ed60d3f05107f484eebf9d339afc9d74 Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Wed, 7 Oct 2026 00:16:13 +0200 Subject: [PATCH 07/48] fix(upload): display counter before filename --- packages/debmagic/src/upload/sftp.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/debmagic/src/upload/sftp.rs b/packages/debmagic/src/upload/sftp.rs index 0b74347a..af36c058 100644 --- a/packages/debmagic/src/upload/sftp.rs +++ b/packages/debmagic/src/upload/sftp.rs @@ -79,7 +79,7 @@ impl Uploader for SftpUploader { .file_name() .map(|n| n.to_string_lossy().into_owned()) .unwrap_or_else(|| path.to_string()); - println!("debmagic: uploading {name} ({index}/{})", files.len()); + println!("debmagic: uploading {index}/{}: {name}", files.len()); } if line.contains("Remote working directory:") { done = true; From a6ccf2d2b93781e0e09c5d49f7e14a35ed04c0cd Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Wed, 7 Oct 2026 14:51:54 +0200 Subject: [PATCH 08/48] fix(build): find the orig tarball in the output dir `debmagic build source` fails with "no upstream tarball found" after `debmagic upstream switch`: switch writes the orig tarball to the output dir (`build/`), but the source build only looked in the data dir and `../`. The source build now uses the output dir too. --- packages/debmagic/src/build/artifacts.rs | 11 +++++++++ packages/debmagic/src/build/mod.rs | 30 ++++++++++++------------ 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/packages/debmagic/src/build/artifacts.rs b/packages/debmagic/src/build/artifacts.rs index dc352aa9..4fb43f2b 100644 --- a/packages/debmagic/src/build/artifacts.rs +++ b/packages/debmagic/src/build/artifacts.rs @@ -50,6 +50,12 @@ fn artifact_filename(filename: &str) -> anyhow::Result<&OsStr> { } } +fn is_same_file(source: &fs::Metadata, destination: &Path) -> bool { + use std::os::unix::fs::MetadataExt; + fs::metadata(destination) + .is_ok_and(|dest| dest.dev() == source.dev() && dest.ino() == source.ino()) +} + fn reject_destination_symlink(path: &Path) -> anyhow::Result<()> { match fs::symlink_metadata(path) { Ok(metadata) if metadata.file_type().is_symlink() => { @@ -106,6 +112,11 @@ pub fn export_build_artifacts( } let destination = output_dir.join(filename); reject_destination_symlink(&destination)?; + // an orig tarball staged into the build dir as a hardlink of the + // output dir's copy: copying it onto itself would truncate it + if is_same_file(&metadata, &destination) { + continue; + } fs::copy(&source, destination) .with_context(|| format!("failed to copy build artifact {}", source.display()))?; } diff --git a/packages/debmagic/src/build/mod.rs b/packages/debmagic/src/build/mod.rs index c9bc5aed..7605e897 100644 --- a/packages/debmagic/src/build/mod.rs +++ b/packages/debmagic/src/build/mod.rs @@ -287,7 +287,7 @@ fn install_packages( dir: &Path, packages: &[&str], ) -> anyhow::Result<()> { - let mut cmd = vec!["apt-get", "-y", "install"]; + let mut cmd = vec!["apt-get", "-y", "-q", "--no-install-recommends", "install"]; cmd.extend(packages); claimed.driver().run_command_checked(&cmd, dir, true, &[])?; Ok(()) @@ -358,25 +358,25 @@ pub async fn build_source_package( } // dpkg-source looks for the orig tarball in the parent of the source - // dir it builds, which inside the environment is the work dir. Fetch - // into the data directory first; `temp_build_dir` no longer exists. - let orig_fetch_dir = crate::data_dir::data_dir() - .context("locating the debmagic data directory")? - .join("orig") - .join(target.package.name()); + // dir it builds, which inside the environment is the work dir. The + // output dir is where `upstream switch` puts it and where the source + // artifacts land anyway, so fetch into it and stage from there. + let name = target.package.name(); + let upstream_version = target.package.version().upstream_version(); let orig_tarball = crate::upstream::orig::fetch_orig_tarball( &intent.config.orig_tarball, &target.package, - &orig_fetch_dir, + &intent.output_dir, ) .await - .with_context(|| { - format!( - "fetching the orig tarball for {} {} failed", - target.package.name(), - target.package.version().upstream_version() - ) - })?; + .with_context(|| format!("fetching the orig tarball for {name} {upstream_version} failed"))?; + if orig_tarball.is_none() && target.package.source_format() == SourceFormat::Quilt { + anyhow::bail!( + "no orig tarball for {name} {upstream_version} in {}, the source tree's parent dir or the distro archives; \ + run 'debmagic upstream switch {upstream_version}' or place it in one of those dirs", + intent.output_dir.display() + ); + } let request = BuildRequest { intent, target }; run_build(&request, |claimed| { From 35a8504af42a5b3c78a717f371bbbaafef368085 Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Wed, 7 Oct 2026 15:10:45 +0200 Subject: [PATCH 09/48] feat(cli): accept yes/no for boolean flags --test, --clean, --incremental, --proposed, --debug-symbols and --sign-notify now take yes/no and on/off besides true/false. --- docs/usage/build.md | 8 +++++--- packages/debmagic/src/cli.rs | 33 ++++++++++++++++++++++++--------- 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/docs/usage/build.md b/docs/usage/build.md index 9601d2b1..d3ecd98e 100644 --- a/docs/usage/build.md +++ b/docs/usage/build.md @@ -159,12 +159,14 @@ Or set `build_debug_symbols = true` in the [`debmagic.toml`](config.md). ## Running tests By default the build runs the package's test suite (the `test` stage of `debian/rules.py`, or `dh_auto_test` via the dh preset). -Pass `--test=false` to skip it for one invocation, or set `run_test = false` in the [`debmagic.toml`](config.md): +Pass `--test=no` to skip it for one invocation, or set `run_test = false` in the [`debmagic.toml`](config.md): ```shell -debmagic build binary --test=false +debmagic build binary --test=no ``` +Boolean flags accept `yes`/`no`, `true`/`false` and `on`/`off`; a bare flag means `yes`. + This exports `DEB_BUILD_OPTIONS=nocheck`, the standard dpkg mechanism: dpkg-buildpackage propagates it into the build, debmagic's `test` stage is skipped, and classic debhelper packages skip `dh_auto_test` as usual. ## Signing @@ -219,7 +221,7 @@ Without a file argument, it locates the `.changes` via `debian/changelog` and `- ## Cleaning -`--clean` runs `debian/rules clean` before building, like plain `dpkg-buildpackage` does unless passed `-nc`; `--clean=false` skips it even if the config file defaults to cleaning. +`--clean` runs `debian/rules clean` before building, like plain `dpkg-buildpackage` does unless passed `-nc`; `--clean=no` skips it even if the config file defaults to cleaning. Non-incremental builds already stage a clean source tree, while incremental builds preserve outputs intentionally. Enable cleaning only for packages whose `clean` target performs required setup or code generation. diff --git a/packages/debmagic/src/cli.rs b/packages/debmagic/src/cli.rs index df21e9f5..2bc2c23f 100644 --- a/packages/debmagic/src/cli.rs +++ b/packages/debmagic/src/cli.rs @@ -10,18 +10,19 @@ use crate::sign::{SignMode, SignTool}; use crate::time::RefreshPolicy; use crate::upload::UploadMethod; use crate::upload::orig::IncludeOrig; +use clap::builder::BoolishValueParser; use clap::{Args, Parser, Subcommand}; /// When to use colored output. Mirrors common CLI conventions; `auto` is the /// default and respects the `NO_COLOR` environment variable. #[derive(Debug, Default, Copy, Clone, PartialEq, Eq, clap::ValueEnum)] pub enum ColorChoice { - /// Color when stderr is a terminal and `NO_COLOR` is unset. + /// enable when stderr is a terminal and NO_COLOR is unset #[default] Auto, - /// Always color, even when piped or `NO_COLOR` is set. + /// force enable colors Always, - /// Never color. + /// disable colors Never, } @@ -36,7 +37,7 @@ pub struct Cli { global = true, value_enum, default_value_t = ColorChoice::Auto, - help = "When to colorize output: 'auto' (default) colors on a terminal and respects NO_COLOR, 'always' forces color, 'never' disables it" + help = "configure output coloring" )] pub color: ColorChoice, @@ -255,6 +256,7 @@ pub struct CommonBuildArgs { num_args = 0..=1, default_missing_value = "true", action = clap::ArgAction::Set, + value_parser = BoolishValueParser::new(), help = "Synchronize changed source inputs to preserve build outputs. Forces --persistent=always" )] pub incremental: Option, @@ -298,6 +300,7 @@ pub struct CommonBuildArgs { num_args = 0..=1, default_missing_value = "true", action = clap::ArgAction::Set, + value_parser = BoolishValueParser::new(), help = "Also enable the '-proposed' pocket in the build environment. Ignored by the bare driver." )] pub proposed: Option, @@ -358,7 +361,7 @@ pub struct CommonBuildArgs { long = "sign-notify", num_args = 0..=1, default_missing_value = "true", - value_parser = clap::value_parser!(bool), + value_parser = BoolishValueParser::new(), help = "Send a desktop notification via notify-send just before signing, so a hardware-key touch prompt isn't missed. Defaults to the 'sign.notify' setting in the config file (false if unset)." )] pub sign_notify: Option, @@ -367,7 +370,7 @@ pub struct CommonBuildArgs { long, num_args = 0..=1, default_missing_value = "true", - value_parser = clap::value_parser!(bool), + value_parser = BoolishValueParser::new(), help = "Run 'debian/rules clean' before building, like plain dpkg-buildpackage does unless passed -nc. Defaults to the 'clean' setting in the config file (false if unset); non-incremental builds already stage a clean source tree, while incremental builds preserve outputs by design. For source builds this also installs build-dependencies first, since a clean target usually needs its own tooling." )] pub clean: Option, @@ -417,6 +420,7 @@ pub struct BinaryTargetArgs { num_args = 0..=1, default_missing_value = "true", action = clap::ArgAction::Set, + value_parser = BoolishValueParser::new(), help = "Also build the automatic '-dbgsym' debug symbol package" )] pub debug_symbols: Option, @@ -425,8 +429,8 @@ pub struct BinaryTargetArgs { long = "test", num_args = 0..=1, default_missing_value = "true", - value_parser = clap::value_parser!(bool), - help = "Run the package's test suite during the build. Defaults to the 'run_test' setting in the config file (true if unset); --test=false exports DEB_BUILD_OPTIONS=nocheck so dpkg-buildpackage skips tests." + value_parser = BoolishValueParser::new(), + help = "Run the package's test suite during the build. Defaults to the 'run_test' setting in the config file (true if unset); --test=no exports DEB_BUILD_OPTIONS=nocheck so dpkg-buildpackage skips tests." )] pub test: Option, } @@ -516,6 +520,7 @@ pub struct TestSubcommandArgs { num_args = 0..=1, default_missing_value = "true", action = clap::ArgAction::Set, + value_parser = BoolishValueParser::new(), help = "Also enable the '-proposed' pocket in the test environment. Ignored by the bare driver." )] pub proposed: Option, @@ -595,7 +600,7 @@ pub struct SignSubcommandArgs { long = "sign-notify", num_args = 0..=1, default_missing_value = "true", - value_parser = clap::value_parser!(bool), + value_parser = BoolishValueParser::new(), help = "Send a desktop notification via notify-send just before signing, so a hardware-key touch prompt isn't missed. Defaults to the 'sign.notify' setting in the config file (false if unset)." )] pub sign_notify: Option, @@ -749,6 +754,16 @@ mod tests { assert_eq!(args.persistent, expected); } + #[test_case(&[], None; "absent")] + #[test_case(&["--proposed"], Some(true); "bare flag")] + #[test_case(&["--proposed=yes"], Some(true); "yes")] + #[test_case(&["--proposed=no"], Some(false); "no")] + #[test_case(&["--proposed=off"], Some(false); "off")] + #[test_case(&["--proposed=true"], Some(true); "literal true")] + fn boolish_flag(args: &[&str], expected: Option) { + assert_eq!(parse_test(args).proposed, expected); + } + #[test] fn persistent_rejects_a_boolean() { let error = Cli::try_parse_from(["debmagic", "test", "--persistent=true"]).unwrap_err(); From 339929ba70ad4f336c10fa4684ab8e1d659eb6fa Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Wed, 7 Oct 2026 16:14:16 +0200 Subject: [PATCH 10/48] feat(upstream): add `upstream fetch` for orig tarballs `debmagic upstream fetch [VERSION]` puts the orig tarball into the output dir, keeping an existing one unless `--force`. `upstream switch` fetches the same way, then replaces the tree. - `--from auto|upstream|launchpad|ubuntu|debian|custom` picks the source - `--repack[=no]` (`upstream.repack`) strips `Files-Excluded` from upstream and custom tarballs; without it the tarball stays pristine and every excluded file it contains is listed as a warning - `--signature-check[=no]` replaces `--no-signature-check`, and the `upstream.verify_signatures` config key is now `upstream.signature_check` - upstream tarballs without excluded files are no longer re-created --- docs/usage/config.md | 3 +- docs/usage/upstream.md | 43 +- packages/debmagic/src/cli.rs | 65 +- packages/debmagic/src/config.rs | 7 +- packages/debmagic/src/main.rs | 46 +- packages/debmagic/src/upstream.rs | 193 +++--- packages/debmagic/src/upstream/fetch.rs | 821 +++++++++++++++++++++++ packages/debmagic/src/upstream/orig.rs | 56 +- packages/debmagic/src/upstream/query.rs | 4 +- packages/debmagic/src/upstream/repack.rs | 2 +- packages/debmagic/src/upstream/switch.rs | 481 ++----------- 11 files changed, 1140 insertions(+), 581 deletions(-) create mode 100644 packages/debmagic/src/upstream/fetch.rs diff --git a/docs/usage/config.md b/docs/usage/config.md index 68a347b9..5a6c2b46 100644 --- a/docs/usage/config.md +++ b/docs/usage/config.md @@ -43,7 +43,8 @@ All keys are optional. | `sign.notify` | bool | `false` | `--sign-notify` | Send a desktop notification via `notify-send` just before signing, so a hardware-key touch prompt isn't missed. | | `clean` | bool | `false` | `--clean` | Run `debian/rules clean` before building. Disabled by default; incompatible with `incremental`. | | `host_arch_variant` | string | — | `--host-arch-variant` | Build for a dpkg architecture variant (e.g. `"amd64v3"` on Ubuntu) -> `DEB_HOST_ARCH_VARIANT`. | -| `upstream.verify_signatures` | bool | `true` | `--no-signature-check` | Verify the upstream tarball's PGP signature against `debian/upstream/signing-key.asc` during `upstream switch` (see [Upstream](upstream.md#signature-verification)). | +| `upstream.signature_check` | bool | `true` | `--signature-check` | Verify the upstream tarball's PGP signature against `debian/upstream/signing-key.asc` during `upstream fetch`/`switch` (see [Upstream](upstream.md#signature-verification)). | +| `upstream.repack` | bool | `true` | `--repack` | Strip the `Files-Excluded` paths from upstream and custom tarballs during `upstream fetch`/`switch` (see [Upstream](upstream.md#repacking)). | | `upload.targets` | map | — | `debmagic upload ` | Named upload targets for [uploading](upload.md), merged field-by-field over the builtins (`ppa`, `ubuntu`, `debian`). | ### `source_sync_mode` diff --git a/docs/usage/upstream.md b/docs/usage/upstream.md index 10ed6208..eec6af51 100644 --- a/docs/usage/upstream.md +++ b/docs/usage/upstream.md @@ -1,11 +1,13 @@ # Upstream versions -`debmagic upstream` replaces `uscan`/`uupdate`: it reads the package's existing `debian/watch` and `debian/copyright` and offers two operations — checking for new upstream versions and switching the package tree to one. +`debmagic upstream` replaces `uscan`/`uupdate`: it reads the package's existing `debian/watch` and `debian/copyright` and offers three operations — checking for new upstream versions, fetching an orig tarball, and switching the package tree to one. ```shell debmagic upstream list # newest upstream version newer than the changelog's debmagic upstream list --all # every candidate, newest first debmagic upstream list --previous 3 # the 3 newest versions above the current one +debmagic upstream fetch # the orig tarball of the changelog's version +debmagic upstream fetch --from upstream --repack=no # the unmodified upstream release debmagic upstream switch latest # switch the tree to the newest version debmagic upstream switch 3.12.0 # switch to a specific version debmagic upstream switch latest --dry-run @@ -18,16 +20,35 @@ By default only versions newer than the changelog's current upstream version are The changelog's version is normalized with the watch file's `Dversion-Mangle` rules before comparing, so `+dfsg` suffixes and similar don't hide available updates. +## `upstream fetch` + +Puts the `orig` tarball of an upstream version into the package's output dir (`build/` by default, `-o` overrides it), where `debmagic build source` finds it. +Without a version it fetches the changelog's upstream version; MUT component tarballs are fetched along with the main one. + +An orig tarball already in the output dir is kept, and one next to the source tree (`../`) is linked into it: re-creating a tarball changes its checksums, and the archive rejects an upload whose orig differs from the one it has. +`--force` replaces it. + +`--from` picks where the tarball comes from: + +- `auto` (default): the configured [`orig_tarball.method`](#orig-tarballs-in-builds), then upstream — a version Debian or Ubuntu published must be reused as-is +- `upstream`: the release `debian/watch` points to +- `launchpad`, `ubuntu`, `debian`: the distro archives +- `custom`: the `orig_tarball.command` + +Upstream and custom tarballs are [repacked](#repacking) unless `--repack=no` (or `upstream.repack = false`) is given. +Then the tarball is kept byte for byte, and every file in it that `Files-Excluded` says must go is listed as a warning. +Distro tarballs are never repacked: the archive already holds them. + +`--dry-run` reports what would happen — method, tarball URL, repack excludes — without downloading anything. + ## `upstream switch` -Fetches the chosen version's tarball, applies the package's repack configuration, writes the `orig` tarball into the package's output dir (`build/` by default) and replaces the source tree's contents — everything except `debian/`, which is kept as-is. +Fetches the chosen version's orig tarball exactly like `upstream fetch` — with the same options — and replaces the source tree's contents with it, everything except `debian/`, which is kept as-is. The changelog is not touched; add the new version entry with [`debmagic changelog new`](changelog.md). -`--dry-run` reports what would happen — tarball URL, repack excludes, version suffix — without downloading anything. - For a concrete version, the tarball URL is resolved without scraping the download listing: -1. the distro archives (the `launchpad` orig method) — a version Debian or Ubuntu published needs no upstream download at all +1. the distro archives (with `--from auto`) — a version Debian or Ubuntu published needs no upstream download at all 2. the watch pattern, read as a URL template: `@ANY_VERSION@` becomes the version, `@ARCHIVE_EXT@` each archive extension in turn (the extension of the project's existing orig tarball is probed first), and the constructed URLs are probed — this also finds versions that have already fallen out of the listing 3. the listing, fetched as a *structure teacher*: any listed sibling reveals the prefix/extension shape, and the requested version's URL is constructed from it — this covers patterns pure inversion rejects (multiple capture groups, regex constructs) and compression changes 4. the listing, filtered for the requested version — the last resort @@ -40,11 +61,11 @@ After downloading, the tarball's PGP signature is verified against the keyring i Verification is on by default and uses the same OpenPGP backend as signing — `gpg` by default, `sq` (Sequoia) when `sign.tool = "sequoia"`, or your custom command when `sign.tool = "custom"`: `sign.verify_command` with `{file}`, `{signature}` and `{keyring}` placeholders (without `{signature}` the signature path is appended as the last argument), falling back to `sign.sign_command`. The signature is looked up via the watch file's `Pgp-Sig-Url-Mangle` rules; without them the common suffixes (`.asc`, `.sig`, `.sign`, `.pgp`, `.gpg`) are tried next to the tarball. -If no signature exists upstream, the switch fails — disable the check per-run with `--no-signature-check` or persistently with `verify_signatures = false`: +If no signature exists upstream, the fetch fails — disable the check per-run with `--signature-check=no` or persistently with `signature_check = false`: ```toml [upstream] -verify_signatures = false +signature_check = false ``` Export the upstream key with `gpg --export --armor > debian/upstream/signing-key.asc`. @@ -56,7 +77,13 @@ Two sources are merged: - `Files-Excluded` in `debian/copyright` — the standard way to strip non-DFSG or useless files; `Files-Excluded-` works for multiple-upstream-tarball packages - the watch file's `Repack`/`Repacksuffix` options, e.g. `Repacksuffix: +dfsg` appends the suffix to the upstream version of the repacked tarball -The repacked `orig` tarball uses the canonical Debian layout: a single top-level `_/` directory, xz-compressed. +A tarball containing excluded files is re-created in the canonical Debian layout: a single top-level `-/` directory, xz-compressed. +One without excluded files is used unchanged, unless dpkg-source can't read its compression (e.g. zip or zstd). + +```toml +[upstream] +repack = false # keep upstream tarballs as-is, warning about excluded files +``` ## Watch file support diff --git a/packages/debmagic/src/cli.rs b/packages/debmagic/src/cli.rs index 2bc2c23f..8781d9d6 100644 --- a/packages/debmagic/src/cli.rs +++ b/packages/debmagic/src/cli.rs @@ -10,6 +10,7 @@ use crate::sign::{SignMode, SignTool}; use crate::time::RefreshPolicy; use crate::upload::UploadMethod; use crate::upload::orig::IncludeOrig; +use crate::upstream::fetch::FetchFrom; use clap::builder::BoolishValueParser; use clap::{Args, Parser, Subcommand}; @@ -107,8 +108,10 @@ pub enum UpstreamCommands { about = "List available upstream versions from debian/watch, newest eligible by default" )] List(UpstreamListArgs), + #[command(about = "Fetch the orig tarball of an upstream version into the output dir")] + Fetch(UpstreamFetchCliArgs), #[command( - about = "Switch the package tree to an upstream version: fetch, repack, replace the tree (keeping debian/)" + about = "Switch the package tree to an upstream version: fetch its orig tarball, replace the tree (keeping debian/)" )] Switch(UpstreamSwitchArgs), } @@ -131,22 +134,57 @@ pub struct UpstreamListArgs { pub common: CommonCli, } +#[derive(Args, Debug)] +pub struct UpstreamFetchCliArgs { + #[arg( + help = "The upstream version to fetch, or 'latest' for the newest. Defaults to the changelog's upstream version" + )] + pub version: Option, + + #[command(flatten)] + pub fetch: UpstreamFetchArgs, +} + #[derive(Args, Debug)] pub struct UpstreamSwitchArgs { - #[arg(help = "The upstream version to switch to, or 'latest' for the newest eligible")] + #[arg(help = "The upstream version to switch to, or 'latest' for the newest")] pub version: String, + #[command(flatten)] + pub fetch: UpstreamFetchArgs, +} + +/// Flags shared between `debmagic upstream fetch` and `debmagic upstream switch`. +#[derive(Args, Debug)] +pub struct UpstreamFetchArgs { #[arg( + short, long, - help = "Only report what would happen, without touching anything" + value_enum, + default_value_t = FetchFrom::Auto, + help = "Where to get the orig tarball: 'auto' (default) tries the 'orig_tarball.method' setting, then upstream; 'upstream' downloads the release from debian/watch; 'launchpad', 'ubuntu' and 'debian' fetch it from the distro archives; 'custom' runs 'orig_tarball.command'" )] - pub dry_run: bool, + pub from: FetchFrom, #[arg( long, - help = "Skip verifying the upstream tarball signature against debian/upstream/signing-key.asc" + num_args = 0..=1, + default_missing_value = "true", + action = clap::ArgAction::Set, + value_parser = BoolishValueParser::new(), + help = "Strip the Files-Excluded paths of debian/copyright from upstream and custom tarballs, re-creating them as xz with the watch file's repack suffix. --repack=no keeps the tarball as-is and lists the excluded files it still contains. Distro archive tarballs are never repacked. Defaults to the 'upstream.repack' setting in the config file (yes if unset)" )] - pub no_signature_check: bool, + pub repack: Option, + + #[arg( + long = "signature-check", + num_args = 0..=1, + default_missing_value = "true", + action = clap::ArgAction::Set, + value_parser = BoolishValueParser::new(), + help = "Verify the upstream tarball signature against debian/upstream/signing-key.asc. Defaults to the 'upstream.signature_check' setting in the config file (yes if unset)" + )] + pub signature_check: Option, #[arg( long = "verify-command", @@ -154,6 +192,21 @@ pub struct UpstreamSwitchArgs { )] pub verify_command: Option, + #[arg( + long, + help = "Replace an existing orig tarball for the version instead of keeping it" + )] + pub force: bool, + + #[arg( + long, + help = "Only report what would happen, without touching anything" + )] + pub dry_run: bool, + + #[arg(short, long, help = "Output directory for the orig tarball")] + pub output_dir: Option, + #[command(flatten)] pub common: CommonCli, } diff --git a/packages/debmagic/src/config.rs b/packages/debmagic/src/config.rs index 587e3cd8..8c7b8e34 100644 --- a/packages/debmagic/src/config.rs +++ b/packages/debmagic/src/config.rs @@ -258,13 +258,16 @@ pub struct Config { pub struct UpstreamConfig { /// Verify upstream tarball signatures against /// `debian/upstream/signing-key.asc` when it exists. - pub verify_signatures: bool, + pub signature_check: bool, + /// Strip the `Files-Excluded` paths from upstream tarballs. + pub repack: bool, } impl Default for UpstreamConfig { fn default() -> Self { Self { - verify_signatures: true, + signature_check: true, + repack: true, } } } diff --git a/packages/debmagic/src/main.rs b/packages/debmagic/src/main.rs index 3219b3c3..054d2f17 100644 --- a/packages/debmagic/src/main.rs +++ b/packages/debmagic/src/main.rs @@ -42,6 +42,26 @@ fn main() -> ExitCode { } } +fn upstream_fetch_command( + current_dir: &std::path::Path, + cli: &Cli, + version: Option, + args: &cli::UpstreamFetchArgs, +) -> anyhow::Result { + Ok(upstream::UpstreamFetchCommand { + source_dir: package::resolve_source_dir(current_dir, args.common.source_dir.as_deref())?, + config_file: cli.config.clone(), + version, + from: args.from, + repack: args.repack, + signature_check: args.signature_check, + verify_command: args.verify_command.clone(), + force: args.force, + dry_run: args.dry_run, + output_dir: args.output_dir.clone(), + }) +} + async fn run() -> anyhow::Result { let cli = Cli::parse(); output::init_color(cli.color); @@ -238,18 +258,22 @@ async fn run() -> anyhow::Result { }) .await?; } + UpstreamCommands::Fetch(fetch_args) => { + upstream::fetch(upstream_fetch_command( + ¤t_dir, + &cli, + fetch_args.version.clone(), + &fetch_args.fetch, + )?) + .await?; + } UpstreamCommands::Switch(switch_args) => { - upstream::switch(upstream::UpstreamSwitchCommand { - source_dir: package::resolve_source_dir( - ¤t_dir, - switch_args.common.source_dir.as_deref(), - )?, - config_file: cli.config.clone(), - version: switch_args.version.clone(), - dry_run: switch_args.dry_run, - no_signature_check: switch_args.no_signature_check, - verify_command: switch_args.verify_command.clone(), - }) + upstream::switch(upstream_fetch_command( + ¤t_dir, + &cli, + Some(switch_args.version.clone()), + &switch_args.fetch, + )?) .await?; } }, diff --git a/packages/debmagic/src/upstream.rs b/packages/debmagic/src/upstream.rs index 07a8087f..9a9f6e86 100644 --- a/packages/debmagic/src/upstream.rs +++ b/packages/debmagic/src/upstream.rs @@ -1,11 +1,13 @@ use std::path::PathBuf; use std::str::FromStr; -use anyhow::{Context, bail}; +use anyhow::Context; use crate::config::Config; use crate::package::load_package; +use fetch::FetchFrom; +pub mod fetch; pub mod mangle; pub mod orig; pub mod query; @@ -24,21 +26,6 @@ pub struct UpstreamListCommand { pub previous: Option, } -/// Inputs for the `debmagic upstream switch` command. -#[derive(Debug, Clone)] -pub struct UpstreamSwitchCommand { - pub source_dir: PathBuf, - pub config_file: Option, - /// The upstream version to switch to, or "latest" for the newest eligible. - pub version: String, - /// Only report what would happen, without touching anything. - pub dry_run: bool, - /// Skip verifying the upstream tarball signature. - pub no_signature_check: bool, - /// Custom verification command overriding `sign.verify_command`. - pub verify_command: Option, -} - /// Run the `debmagic upstream list` command: query each watch source and /// print its current and newer upstream versions. pub async fn list(command: UpstreamListCommand) -> anyhow::Result<()> { @@ -82,14 +69,66 @@ pub async fn list(command: UpstreamListCommand) -> anyhow::Result<()> { Ok(()) } -/// Run the `debmagic upstream switch` command: resolve the candidate -/// version, then switch the main source and every component source to it. -pub async fn switch(command: UpstreamSwitchCommand) -> anyhow::Result<()> { - let source_dir = command.source_dir.clone(); - let identity = load_package(&source_dir)?; - let sources = query::load_watch(&source_dir, identity.name())?; +/// Inputs shared by the `debmagic upstream fetch` and `switch` commands. +#[derive(Debug, Clone)] +pub struct UpstreamFetchCommand { + pub source_dir: PathBuf, + pub config_file: Option, + /// An upstream or Debian version, or `latest`; `None` means the + /// changelog's upstream version. + pub version: Option, + pub from: FetchFrom, + /// Overrides `upstream.repack`. + pub repack: Option, + /// Overrides `upstream.signature_check`. + pub signature_check: Option, + /// Custom verification command overriding `sign.verify_command`. + pub verify_command: Option, + pub force: bool, + pub dry_run: bool, + /// Overrides the configured output dir. + pub output_dir: Option, +} + +/// Run `debmagic upstream fetch`: put the orig tarballs (main and MUT +/// components) of an upstream version into the output dir. +pub async fn fetch(command: UpstreamFetchCommand) -> anyhow::Result<()> { + run_fetch(command, false).await +} + +/// Run `debmagic upstream switch`: fetch like `upstream fetch`, then +/// replace the tree (and each component's subdir) with the tarballs' +/// contents. +pub async fn switch(command: UpstreamFetchCommand) -> anyhow::Result<()> { + run_fetch(command, true).await +} + +async fn run_fetch(command: UpstreamFetchCommand, switch_tree: bool) -> anyhow::Result<()> { + let source_dir = command.source_dir; + let package = load_package(&source_dir)?; + let sources = query::load_watch(&source_dir, package.name())?; let mut config = Config::load(Some(&source_dir), command.config_file.as_deref())?; - let output_dir = source_dir.join(&config.output_dir); + if let Some(verify_command) = command.verify_command { + config.sign.verify_command = Some(verify_command); + } + let output_dir = std::path::absolute( + command + .output_dir + .unwrap_or_else(|| source_dir.join(&config.output_dir)), + ) + .context("resolving the output dir failed")?; + let options = fetch::FetchOptions { + from: command.from, + orig_tarball: &config.orig_tarball, + repack: command.repack.unwrap_or(config.upstream.repack), + signature_check: command + .signature_check + .unwrap_or(config.upstream.signature_check), + sign: &config.sign, + output_dir: &output_dir, + force: command.force, + dry_run: command.dry_run, + }; // the main source (no Component field) drives the version; // component sources contribute their own tarballs @@ -97,106 +136,34 @@ pub async fn switch(command: UpstreamSwitchCommand) -> anyhow::Result<()> { .iter() .find(|s| s.untrackable.is_none() && s.component.is_none()) .context("no usable main watch source found")?; - let candidate = if command.version == "latest" { - // discovery needs the listing - let candidates = query::query_source(main_source, identity.name()).await?; - candidates.first().cloned() - } else { - // a concrete version: construct the URL from the watch - // pattern directly; only fall back to scraping when the - // pattern is not invertible or the URL does not exist. - // an existing orig tarball hints at the extension first. - let existing_orig = crate::changes::find_orig_in_dir( - &output_dir, - identity.name(), - identity.version().upstream_version(), - None, - ) - .or_else(|| { - source_dir.parent().and_then(|parent| { - crate::changes::find_orig_in_dir( - parent, - identity.name(), - identity.version().upstream_version(), - None, - ) - }) - }); - match query::resolve_concrete( - main_source, - identity.name(), - &command.version, - existing_orig.as_deref(), - ) - .await - { - Ok(Some(candidate)) => Some(candidate), - Ok(None) | Err(_) => { - let candidates = query::query_source(main_source, identity.name()).await?; - query::find_candidate(&candidates, &command.version).cloned() - } - } - }; - let Some(candidate) = candidate else { - if command.version == "latest" { - bail!("no upstream candidates found for {}", identity.name()); - } - bail!( - "upstream version {} not found for {}; run 'debmagic upstream list' to see the available versions", - command.version, - identity.name() - ); - }; - - let package = load_package(&source_dir)?; - let repack = repack::load_repack_config(&package, main_source)?; - let verify = config.upstream.verify_signatures && !command.no_signature_check; - if let Some(verify_command) = &command.verify_command { - config.sign.verify_command = Some(verify_command.clone()); - } let components: Vec = sources .iter() .filter_map(|source| source.component.clone()) .collect(); - let options = switch::SwitchOptions { - repack: &repack, - output_dir: &output_dir, - orig_tarball_config: Some(&config.orig_tarball), - verify_signatures: verify, - sign_options: &config.sign, - dry_run: command.dry_run, - components: &components, + let requested = command + .version + .unwrap_or_else(|| package.version().upstream_version().to_string()); + + let run = async |source: &watch::WatchSource, version: &str| { + let what = match &source.component { + Some(component) => format!("component {component} of {}", package.name()), + None => package.name().to_string(), + }; + let result = if switch_tree { + switch::switch(&package, source, version, &options, &components).await + } else { + fetch::fetch(&package, source, version, &options).await + }; + result.with_context(|| format!("fetching {what} {version} failed")) }; - switch::switch(&source_dir, main_source, &candidate, &options).await?; - // MUT: switch each component source to the same version + let main = run(main_source, &requested).await?; + // MUT: the components follow the main tarball's version for source in &sources { if source.untrackable.is_some() || source.component.is_none() { continue; } - let candidates = query::query_source(source, identity.name()).await?; - let Some(component_candidate) = candidates - .iter() - .find(|c| c.version == candidate.version) - .or_else(|| candidates.first()) - else { - bail!( - "no candidates for component {} at version {}", - source.component.as_deref().unwrap_or_default(), - candidate.version - ); - }; - let repack = repack::load_repack_config(&package, source)?; - let options = switch::SwitchOptions { - repack: &repack, - output_dir: &output_dir, - orig_tarball_config: Some(&config.orig_tarball), - verify_signatures: verify, - sign_options: &config.sign, - dry_run: command.dry_run, - components: &components, - }; - switch::switch(&source_dir, source, component_candidate, &options).await?; + run(source, &main.upstream_version).await?; } Ok(()) } diff --git a/packages/debmagic/src/upstream/fetch.rs b/packages/debmagic/src/upstream/fetch.rs new file mode 100644 index 00000000..8a8b7b7e --- /dev/null +++ b/packages/debmagic/src/upstream/fetch.rs @@ -0,0 +1,821 @@ +use std::path::{Path, PathBuf}; + +use anyhow::{Context, bail}; +use clap::ValueEnum; +use debmagic_common::changes::{component_orig_prefix, orig_filenames, orig_prefix}; +use debmagic_common::package::SourcePackage; + +use crate::changes::find_orig_in_dir; +use crate::config::SignConfig; +use crate::upstream::mangle::mangle; +use crate::upstream::orig::{ + OrigTarballConfig, OrigTarballMethod, fetch_with_method, link_or_copy, +}; +use crate::upstream::query::{self, Candidate}; +use crate::upstream::repack::{RepackConfig, apply_excludes, is_excluded, load_repack_config}; +use crate::upstream::watch::WatchSource; + +/// Where `upstream fetch` and `upstream switch` get the orig tarball from. +#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, ValueEnum)] +pub enum FetchFrom { + /// The configured `orig_tarball.method`, then upstream: a version the + /// distros already have must be reused, since a re-created tarball + /// would have different checksums and the upload would be rejected. + #[default] + Auto, + /// The upstream release from `debian/watch`. + Upstream, + /// The distro archives via Launchpad. + Launchpad, + /// Ubuntu's archive pool. + Ubuntu, + /// Debian's archive pool. + Debian, + /// The `orig_tarball.command`. + Custom, +} + +impl FetchFrom { + /// The orig tarball method to try before upstream, if any. + fn method(self, config: &OrigTarballConfig) -> Option { + match self { + Self::Auto => Some(config.method()), + Self::Upstream => None, + Self::Launchpad => Some(OrigTarballMethod::Launchpad), + Self::Ubuntu => Some(OrigTarballMethod::Ubuntu), + Self::Debian => Some(OrigTarballMethod::Debian), + Self::Custom => Some(OrigTarballMethod::Custom), + } + } +} + +/// How [`fetch`] gets and places one source's orig tarball. +pub struct FetchOptions<'a> { + pub from: FetchFrom, + pub orig_tarball: &'a OrigTarballConfig, + /// Strip the `Files-Excluded` paths from upstream and custom + /// tarballs; distro tarballs are immutable. + pub repack: bool, + pub signature_check: bool, + pub sign: &'a SignConfig, + pub output_dir: &'a Path, + /// Replace an existing orig tarball instead of keeping it. + pub force: bool, + /// Only report what would happen. + pub dry_run: bool, +} + +/// A fetched orig tarball. +#[derive(Debug)] +pub struct Fetched { + /// The tarball in the output dir; in a dry run, where it would go. + pub tarball: PathBuf, + /// The upstream version, without a repack suffix. + pub upstream_version: String, +} + +/// A scratch dir, removed when dropped on success and error alike. +struct WorkDir(PathBuf); + +impl WorkDir { + fn new(name: &str) -> anyhow::Result { + let path = + std::env::temp_dir().join(format!("debmagic-fetch-{name}-{}", std::process::id())); + std::fs::create_dir_all(&path) + .with_context(|| format!("failed to create {}", path.display()))?; + Ok(Self(path)) + } +} + +impl Drop for WorkDir { + fn drop(&mut self) { + std::fs::remove_dir_all(&self.0).ok(); + } +} + +/// Fetch the orig tarball of one watch source (the main tarball, or a +/// MUT component's) for `requested` into the output dir. `requested` +/// is an upstream or full Debian version, or `latest`. +/// +/// An existing orig tarball for the version is kept unless forced: +/// re-creating it would change its checksums and break uploads. +pub async fn fetch( + package: &SourcePackage, + source: &WatchSource, + requested: &str, + options: &FetchOptions<'_>, +) -> anyhow::Result { + let name = package.name(); + let component = source.component.as_deref(); + let output_dir = options.output_dir; + let repack = load_repack_config(package, source)?; + let suffix = repack + .suffix + .as_deref() + .filter(|_| options.repack) + .unwrap_or_default(); + + // a concrete version is known up front, so the distro archives can + // be asked without contacting upstream; `latest` needs the listing + let mut candidate = None; + let upstream_version = if requested == "latest" { + let latest = resolve_candidate(package, source, requested, output_dir).await?; + let version = latest.version.clone(); + candidate = Some(latest); + version + } else { + let version = query::current_upstream_version(source, requested)?; + match repack + .suffix + .as_deref() + .and_then(|s| version.strip_suffix(s)) + { + Some(base) => base.to_string(), + None => version, + } + }; + let orig_version = format!("{upstream_version}{suffix}"); + let fetched = |tarball| Fetched { + tarball, + upstream_version: upstream_version.clone(), + }; + + let changelog_upstream = package.version().upstream_version(); + if !options.repack + && let Some(repack_suffix) = &repack.suffix + && changelog_upstream.ends_with(repack_suffix.as_str()) + { + println!( + "debmagic: warning: the changelog's upstream version {changelog_upstream} carries the repack suffix {repack_suffix}, but without repacking the orig tarball is for {orig_version}" + ); + } + + if options.force { + if !options.dry_run { + remove_existing(output_dir, name, &orig_version, component)?; + } + } else if let Some(existing) = use_existing(package, &orig_version, component, options)? { + return Ok(fetched(existing)); + } + + if let Some(method) = options.from.method(options.orig_tarball) + && method != OrigTarballMethod::Disabled + { + let method_name = method + .to_possible_value() + .expect("no orig tarball method is skipped") + .get_name() + .to_string(); + // a custom command's output is pristine like an upstream + // download and repacked like one, so it is for the bare version + let custom = method == OrigTarballMethod::Custom; + let version = if custom { + &upstream_version + } else { + &orig_version + }; + if options.dry_run { + println!("debmagic: would fetch {name} {version} via the {method_name} method"); + if options.from != FetchFrom::Auto { + let prefix = orig_name_prefix(name, &orig_version, component); + return Ok(fetched(output_dir.join(format!("{prefix}xz")))); + } + } else { + match fetch_with_method( + options.orig_tarball, + method, + package, + output_dir, + version, + component, + ) + .await + { + Ok(Some(tarball)) if custom => { + let work_dir = WorkDir::new(name)?; + return place_orig( + &tarball, + &work_dir.0, + name, + &orig_version, + component, + &repack, + options, + ) + .map(fetched); + } + Ok(Some(tarball)) => { + println!("debmagic: fetched {}", tarball.display()); + return Ok(fetched(tarball)); + } + Ok(None) if options.from == FetchFrom::Auto => {} + Ok(None) => { + bail!("the {method_name} method has no orig tarball for {name} {version}") + } + Err(error) if options.from == FetchFrom::Auto => { + println!( + "debmagic: no orig tarball via the {method_name} method ({error:#}); fetching from upstream" + ); + } + Err(error) => return Err(error), + } + } + } + + let candidate = match candidate { + Some(candidate) => candidate, + None => resolve_candidate(package, source, &upstream_version, output_dir).await?, + }; + let url = download_url(source, &candidate)?; + if options.dry_run { + println!("debmagic: would download {url}"); + let repacking = options.repack && !repack.excludes.is_empty(); + if repacking { + println!("debmagic: would repack, excluding:"); + for pattern in &repack.excludes { + println!(" {pattern}"); + } + } + let compression = if repacking { + Some("xz") + } else { + orig_compression(&download_filename(source, &candidate)?) + }; + let prefix = orig_name_prefix(name, &orig_version, component); + return Ok(fetched( + output_dir.join(format!("{prefix}{}", compression.unwrap_or("xz"))), + )); + } + + let work_dir = WorkDir::new(name)?; + let download_dir = work_dir.0.join("download"); + std::fs::create_dir_all(&download_dir) + .with_context(|| format!("failed to create {}", download_dir.display()))?; + let tarball = download_tarball(source, &candidate, &download_dir).await?; + if options.signature_check { + verify_tarball_signature( + source, + package.source_dir()?, + &candidate, + &tarball, + options.sign, + ) + .await?; + } + place_orig( + &tarball, + &work_dir.0, + name, + &orig_version, + component, + &repack, + options, + ) + .map(fetched) +} + +/// The orig tarball filename up to the compression extension. +fn orig_name_prefix(name: &str, version: &str, component: Option<&str>) -> String { + match component { + None => orig_prefix(name, version), + Some(component) => component_orig_prefix(name, version, component), + } +} + +/// An existing orig tarball for `orig_version`: in the output dir, or +/// next to the source tree (`../`), where developers conventionally keep +/// them; that one is linked into the output dir. +fn use_existing( + package: &SourcePackage, + orig_version: &str, + component: Option<&str>, + options: &FetchOptions<'_>, +) -> anyhow::Result> { + let name = package.name(); + let output_dir = options.output_dir; + if let Some(existing) = find_orig_in_dir(output_dir, name, orig_version, component) { + println!( + "debmagic: keeping existing orig tarball {} (--force replaces it)", + existing.display() + ); + return Ok(Some(existing)); + } + let Some(existing) = package + .source_dir()? + .parent() + .and_then(|parent| find_orig_in_dir(parent, name, orig_version, component)) + else { + return Ok(None); + }; + let destination = output_dir.join(existing.file_name().expect("orig tarball has a file name")); + println!( + "debmagic: using existing orig tarball {}", + existing.display() + ); + if !options.dry_run { + std::fs::create_dir_all(output_dir) + .with_context(|| format!("failed to create {}", output_dir.display()))?; + link_or_copy(&existing, &destination)?; + } + Ok(Some(destination)) +} + +/// Remove the output dir's orig tarballs for `orig_version`, in any +/// compression. +fn remove_existing( + output_dir: &Path, + name: &str, + orig_version: &str, + component: Option<&str>, +) -> anyhow::Result<()> { + for filename in orig_filenames(name, orig_version, component) { + let path = output_dir.join(filename); + if path.is_file() { + std::fs::remove_file(&path) + .with_context(|| format!("failed to remove {}", path.display()))?; + } + } + Ok(()) +} + +/// Resolve `requested` (a version or `latest`) to a downloadable +/// candidate of the watch source. +async fn resolve_candidate( + package: &SourcePackage, + source: &WatchSource, + requested: &str, + output_dir: &Path, +) -> anyhow::Result { + let name = package.name(); + if requested == "latest" { + return query::query_source(source, name) + .await? + .into_iter() + .next() + .with_context(|| format!("no upstream candidates found for {name}")); + } + + // a concrete version: construct the URL from the watch pattern + // directly, the extension of an existing orig tarball first; only + // fall back to scraping when that fails + let component = source.component.as_deref(); + let current = package.version().upstream_version(); + let existing_orig = find_orig_in_dir(output_dir, name, current, component).or_else(|| { + package + .source_dir() + .ok()? + .parent() + .and_then(|parent| find_orig_in_dir(parent, name, current, component)) + }); + if let Ok(Some(candidate)) = + query::resolve_concrete(source, name, requested, existing_orig.as_deref()).await + { + return Ok(candidate); + } + let candidates = query::query_source(source, name).await?; + query::find_candidate(&candidates, requested) + .cloned() + .with_context(|| { + format!( + "upstream version {requested} not found for {name}; run 'debmagic upstream list' to see the available versions" + ) + }) +} + +/// Turn a pristine upstream tarball into the orig tarball in the output +/// dir: repacked with `options.repack` when excluded files are in it (or +/// dpkg-source can't use its compression), else moved as-is, with a +/// warning listing every file `debian/copyright` excludes. +fn place_orig( + pristine: &Path, + work_dir: &Path, + name: &str, + orig_version: &str, + component: Option<&str>, + repack: &RepackConfig, + options: &FetchOptions<'_>, +) -> anyhow::Result { + let output_dir = options.output_dir; + std::fs::create_dir_all(output_dir) + .with_context(|| format!("failed to create {}", output_dir.display()))?; + let prefix = orig_name_prefix(name, orig_version, component); + let pristine_name = pristine + .file_name() + .unwrap_or_default() + .to_string_lossy() + .into_owned(); + let compression = orig_compression(&pristine_name); + + let excluded: Vec = match compression { + Some(_) if !repack.excludes.is_empty() => list_tarball(pristine)? + .into_iter() + .filter(|path| is_excluded(path, &repack.excludes)) + .collect(), + _ => Vec::new(), + }; + + if options.repack && (compression.is_none() || !excluded.is_empty()) { + let tree = work_dir.join("tree"); + extract_tarball(pristine, &tree)?; + let removed = apply_excludes(&tree, &repack.excludes)?; + let repacked = work_dir.join(format!("{prefix}xz")); + create_tarball(&tree, &format!("{name}-{orig_version}"), &repacked)?; + let destination = output_dir.join(format!("{prefix}xz")); + move_file(&repacked, &destination)?; + // a custom command leaves its pristine tarball in the output dir + if pristine != destination && pristine.starts_with(output_dir) { + std::fs::remove_file(pristine) + .with_context(|| format!("failed to remove {}", pristine.display()))?; + } + println!( + "debmagic: repacked {pristine_name} as {} ({removed} excluded paths removed)", + destination.display() + ); + return Ok(destination); + } + + let Some(compression) = compression else { + bail!( + "dpkg-source only accepts .tar.{{gz,xz,bz2,lzma}} orig tarballs, not {pristine_name}; pass --repack=yes to re-create it" + ); + }; + if !excluded.is_empty() { + let field = match component { + Some(component) => format!("Files-Excluded-{component}"), + None => "Files-Excluded".to_string(), + }; + println!( + "debmagic: warning: {} file(s) matching {field} in debian/copyright kept without repacking:", + excluded.len() + ); + for path in &excluded { + println!(" {path}"); + } + } + let destination = output_dir.join(format!("{prefix}{compression}")); + if pristine != destination { + move_file(pristine, &destination)?; + } + println!("debmagic: fetched {}", destination.display()); + Ok(destination) +} + +/// The orig tarball compression for an upstream tarball's filename, if +/// dpkg-source accepts it; maps the short `.tgz`-style forms. +fn orig_compression(filename: &str) -> Option<&'static str> { + [ + (".tar.gz", "gz"), + (".tgz", "gz"), + (".tar.xz", "xz"), + (".txz", "xz"), + (".tar.bz2", "bz2"), + (".tbz2", "bz2"), + (".tbz", "bz2"), + (".tar.lzma", "lzma"), + ] + .into_iter() + .find(|(suffix, _)| filename.ends_with(suffix)) + .map(|(_, compression)| compression) +} + +/// The file paths in a tarball, without the leading `-/` +/// component, like [`extract_tarball`] unpacks them. +fn list_tarball(tarball: &Path) -> anyhow::Result> { + let output = std::process::Command::new("tar") + .arg("--list") + .arg("--file") + .arg(tarball) + .output() + .with_context(|| format!("failed to run tar for {}", tarball.display()))?; + if !output.status.success() { + bail!( + "listing {} failed: {}", + tarball.display(), + String::from_utf8_lossy(&output.stderr).trim() + ); + } + let mut paths: Vec = String::from_utf8_lossy(&output.stdout) + .lines() + .filter_map(|line| line.split_once('/').map(|(_, path)| path)) + .filter(|path| !path.is_empty() && !path.ends_with('/')) + .map(str::to_string) + .collect(); + paths.sort(); + Ok(paths) +} + +/// Create an xz tarball of `tree` with the canonical Debian layout: a +/// single top-level `top_level/` dir. Moves `tree` into place. +fn create_tarball(tree: &Path, top_level: &str, destination: &Path) -> anyhow::Result<()> { + let stage = tree.with_file_name("stage"); + std::fs::create_dir_all(&stage) + .with_context(|| format!("failed to create {}", stage.display()))?; + std::fs::rename(tree, stage.join(top_level)) + .with_context(|| format!("failed to stage {}", tree.display()))?; + let output = std::process::Command::new("tar") + .current_dir(&stage) + .args(["--create", "--xz", "--file"]) + .arg(destination) + .arg(top_level) + .output() + .with_context(|| format!("failed to run tar for {}", destination.display()))?; + if !output.status.success() { + bail!( + "creating {} failed: {}", + destination.display(), + String::from_utf8_lossy(&output.stderr).trim() + ); + } + Ok(()) +} + +/// Move a file, copying when the rename crosses filesystems. +fn move_file(from: &Path, to: &Path) -> anyhow::Result<()> { + if std::fs::rename(from, to).is_ok() { + return Ok(()); + } + std::fs::copy(from, to) + .and_then(|_| std::fs::remove_file(from)) + .with_context(|| format!("failed to move {} to {}", from.display(), to.display())) +} + +/// Resolve the absolute download URL for a candidate: hrefs resolve +/// against the watch source's URL like a browser (RFC 3986, as uscan), +/// then `Download-Url-Mangle` applies. +pub fn download_url(source: &WatchSource, candidate: &Candidate) -> anyhow::Result { + let base = reqwest::Url::parse(&source.source) + .with_context(|| format!("invalid watch source URL {}", source.source))?; + let mut url = base + .join(&candidate.href) + .with_context(|| format!("resolving {} against {base} failed", candidate.href))? + .to_string(); + if let Some(rules) = &source.download_url_mangle { + url = mangle(rules, &url)?; + } + Ok(url) +} + +/// The local filename for a downloaded tarball: `Filename-Mangle` if +/// set, else the last URL component without any query fragment. +fn download_filename(source: &WatchSource, candidate: &Candidate) -> anyhow::Result { + if let Some(rules) = &source.filename_mangle { + return mangle(rules, &candidate.href); + } + let url = download_url(source, candidate)?; + let last = url.rsplit('/').next().unwrap_or_default(); + Ok(last + .split(['?', '#']) + .next() + .unwrap_or_default() + .to_string()) +} + +/// Download a candidate's tarball into `download_dir`, returning its path. +/// Uses curl for ftp, the requests module for http(s). +async fn download_tarball( + source: &WatchSource, + candidate: &Candidate, + download_dir: &Path, +) -> anyhow::Result { + let url = download_url(source, candidate)?; + let filename = download_filename(source, candidate)?; + if filename.is_empty() { + bail!("could not determine a filename for {url}"); + } + let destination = download_dir.join(filename); + println!("debmagic: downloading {url}"); + if url.starts_with("ftp://") || url.starts_with("ftps://") { + crate::requests::ftp_download(&url, &destination).await?; + } else { + crate::requests::http_download(&url, &destination).await?; + } + Ok(destination) +} + +/// Download and verify the signature of a downloaded tarball. +/// The signature URL comes from `Pgp-Sig-Url-Mangle` or the common +/// suffixes (`.asc`, `.sig`, ...); verification runs against +/// `debian/upstream/signing-key.asc` when that file exists. +/// Returns Ok(()) when no keyring or no signature is available. +async fn verify_tarball_signature( + source: &WatchSource, + source_dir: &Path, + candidate: &Candidate, + tarball: &Path, + sign_options: &SignConfig, +) -> anyhow::Result<()> { + let keyring = source_dir + .join("debian") + .join("upstream") + .join("signing-key.asc"); + if !keyring.is_file() { + return Ok(()); + } + + let url = download_url(source, candidate)?; + let sig_url = match &source.pgp_sig_url_mangle { + Some(rules) => mangle(rules, &url)?, + None => { + let mut found = None; + for suffix in ["asc", "sig", "sign", "pgp", "gpg"] { + let candidate_url = format!("{url}.{suffix}"); + if crate::requests::http_exists(&candidate_url).await { + found = Some(candidate_url); + break; + } + } + match found { + Some(url) => url, + None => return Ok(()), + } + } + }; + + let sig_name = sig_url.rsplit('/').next().unwrap_or_default(); + let sig_path = tarball + .parent() + .unwrap_or_else(|| Path::new(".")) + .join(sig_name); + println!("debmagic: downloading signature {sig_url}"); + crate::requests::http_download(&sig_url, &sig_path).await?; + + crate::sign::verify_signature(sign_options, tarball, &sig_path, &keyring)?; + println!( + "debmagic: verified upstream signature of {}", + tarball.display() + ); + Ok(()) +} + +/// Extract a tarball into `extract_dir`, stripping the leading +/// `-/` component so the tree sits directly in +/// `extract_dir`. +pub fn extract_tarball(tarball: &Path, extract_dir: &Path) -> anyhow::Result<()> { + std::fs::create_dir_all(extract_dir) + .with_context(|| format!("failed to create {}", extract_dir.display()))?; + let output = std::process::Command::new("tar") + .arg("--extract") + .arg("--file") + .arg(tarball) + .arg("--directory") + .arg(extract_dir) + .arg("--strip-components=1") + .output() + .with_context(|| format!("failed to run tar for {}", tarball.display()))?; + if !output.status.success() { + bail!( + "extracting {} failed: {}", + tarball.display(), + String::from_utf8_lossy(&output.stderr).trim() + ); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn source() -> WatchSource { + WatchSource { + source: "https://example.com/releases/".to_string(), + matching_pattern: "foo-@ANY_VERSION@@ARCHIVE_EXT@".to_string(), + ..Default::default() + } + } + + #[test_case::test_case("foo-1.0.tar.gz", "https://example.com/releases/foo-1.0.tar.gz"; "relative")] + #[test_case::test_case("/dl/foo-1.0.tar.gz", "https://example.com/dl/foo-1.0.tar.gz"; "host absolute")] + #[test_case::test_case("../old/foo-1.0.tar.gz", "https://example.com/old/foo-1.0.tar.gz"; "parent")] + #[test_case::test_case("//cdn.example.org/foo-1.0.tar.gz", "https://cdn.example.org/foo-1.0.tar.gz"; "scheme relative")] + fn test_download_url_relative(href: &str, expected: &str) { + let source = source(); + let candidate = Candidate { + version: "1.0".to_string(), + href: href.to_string(), + }; + assert_eq!(download_url(&source, &candidate).unwrap(), expected); + } + + #[test] + fn test_download_url_absolute() { + let source = source(); + let candidate = Candidate { + version: "1.0".to_string(), + href: "https://mirror.example.com/foo-1.0.tar.gz".to_string(), + }; + assert_eq!( + download_url(&source, &candidate).unwrap(), + "https://mirror.example.com/foo-1.0.tar.gz" + ); + } + + #[test] + fn test_download_url_mangle() { + let mut source = source(); + source.download_url_mangle = Some("s/prdownload/download/".to_string()); + let candidate = Candidate { + version: "1.0".to_string(), + href: "https://prdownload.example.com/foo-1.0.tar.gz".to_string(), + }; + assert_eq!( + download_url(&source, &candidate).unwrap(), + "https://download.example.com/foo-1.0.tar.gz" + ); + } + + #[test] + fn test_download_filename() { + let source = source(); + let candidate = Candidate { + version: "1.0".to_string(), + href: "foo-1.0.tar.gz".to_string(), + }; + assert_eq!( + download_filename(&source, &candidate).unwrap(), + "foo-1.0.tar.gz" + ); + } + + #[test_case::test_case("foo-1.0.tar.gz", Some("gz"); "tar gz")] + #[test_case::test_case("foo-1.0.tgz", Some("gz"); "tgz")] + #[test_case::test_case("foo-1.0.tar.xz", Some("xz"); "tar xz")] + #[test_case::test_case("foo-1.0.tbz2", Some("bz2"); "tbz2")] + #[test_case::test_case("foo-1.0.tar.lzma", Some("lzma"); "lzma")] + #[test_case::test_case("foo-1.0.tar.zst", None; "zstd")] + #[test_case::test_case("foo-1.0.zip", None; "zip")] + fn test_orig_compression(filename: &str, expected: Option<&str>) { + assert_eq!(orig_compression(filename), expected); + } + + /// A `pkg-1.0/` upstream tarball with a `win32/` dir to exclude. + fn pristine_tarball(base: &Path) -> PathBuf { + let tree = base.join("upstream").join("pkg-1.0"); + std::fs::create_dir_all(tree.join("win32")).unwrap(); + std::fs::create_dir_all(tree.join("src")).unwrap(); + std::fs::write(tree.join("win32").join("a.c"), "x").unwrap(); + std::fs::write(tree.join("src").join("main.c"), "x").unwrap(); + let tarball = base.join("pkg-1.0.tgz"); + let status = std::process::Command::new("tar") + .current_dir(base.join("upstream")) + .args(["--create", "--gzip", "--file"]) + .arg(&tarball) + .arg("pkg-1.0") + .status() + .unwrap(); + assert!(status.success()); + tarball + } + + fn place(base: &Path, pristine: &Path, repack: bool, suffix: Option<&str>) -> PathBuf { + let work_dir = base.join("work"); + std::fs::create_dir_all(&work_dir).unwrap(); + let config = RepackConfig { + excludes: vec!["win32/*".to_string()], + suffix: suffix.map(str::to_string), + }; + let orig_tarball = OrigTarballConfig::default(); + let sign = SignConfig::default(); + let output_dir = base.join("out"); + let options = FetchOptions { + from: FetchFrom::Upstream, + orig_tarball: &orig_tarball, + repack, + signature_check: false, + sign: &sign, + output_dir: &output_dir, + force: false, + dry_run: false, + }; + let version = format!("1.0{}", suffix.unwrap_or_default()); + place_orig( + pristine, &work_dir, "pkg", &version, None, &config, &options, + ) + .unwrap() + } + + #[test] + fn test_place_orig_repacks_excluded_files() { + let base = std::env::temp_dir().join(format!("debmagic-repack-{}", std::process::id())); + let orig = place(&base, &pristine_tarball(&base), true, Some("+dfsg")); + assert_eq!(orig, base.join("out").join("pkg_1.0+dfsg.orig.tar.xz")); + assert_eq!(list_tarball(&orig).unwrap(), vec!["src/main.c"]); + std::fs::remove_dir_all(&base).unwrap(); + } + + #[test] + fn test_place_orig_without_repack_keeps_tarball() { + let base = std::env::temp_dir().join(format!("debmagic-pristine-{}", std::process::id())); + let tarball = pristine_tarball(&base); + let pristine = std::fs::read(&tarball).unwrap(); + let orig = place(&base, &tarball, false, None); + // the upstream tarball, byte for byte, under the orig name + assert_eq!(orig, base.join("out").join("pkg_1.0.orig.tar.gz")); + assert_eq!(std::fs::read(&orig).unwrap(), pristine); + assert_eq!( + list_tarball(&orig).unwrap(), + vec!["src/main.c", "win32/a.c"] + ); + std::fs::remove_dir_all(&base).unwrap(); + } +} diff --git a/packages/debmagic/src/upstream/orig.rs b/packages/debmagic/src/upstream/orig.rs index de669198..bfd6f157 100644 --- a/packages/debmagic/src/upstream/orig.rs +++ b/packages/debmagic/src/upstream/orig.rs @@ -57,7 +57,7 @@ pub struct OrigTarballConfig { impl OrigTarballConfig { /// The resolved method: the configured one, or the `launchpad` /// default when unset. - fn method(&self) -> OrigTarballMethod { + pub fn method(&self) -> OrigTarballMethod { self.method.unwrap_or_default() } @@ -73,7 +73,12 @@ impl OrigTarballConfig { /// Substitute `{name}`, `{version}`, `{upstream_version}`, `{source_dir}` /// and `{output_dir}` placeholders. -fn substitute_placeholders(s: &str, package: &SourcePackage, output_dir: &Path) -> String { +fn substitute_placeholders( + s: &str, + package: &SourcePackage, + upstream_version: &str, + output_dir: &Path, +) -> String { let version = package.version().to_string(); let source_dir = package.source_dir().unwrap().to_string_lossy(); let out = output_dir.to_string_lossy(); @@ -82,7 +87,7 @@ fn substitute_placeholders(s: &str, package: &SourcePackage, output_dir: &Path) &[ ("name", package.name()), ("version", version.as_str()), - ("upstream_version", package.version().upstream_version()), + ("upstream_version", upstream_version), ("source_dir", &source_dir), ("output_dir", &out), ], @@ -161,10 +166,33 @@ pub async fn fetch_orig_tarball_for_version( return Ok(Some(existing)); } - let method = config.method(); + fetch_with_method( + config, + config.method(), + package, + output_dir, + upstream_version, + component, + ) + .await +} + +/// Fetch the `orig` tarball for `upstream_version` with an explicit +/// `method` into `output_dir`, without looking for local copies first. +/// `None` when the method does not have it. +pub async fn fetch_with_method( + config: &OrigTarballConfig, + method: OrigTarballMethod, + package: &SourcePackage, + output_dir: &Path, + upstream_version: &str, + component: Option<&str>, +) -> anyhow::Result> { if method == OrigTarballMethod::Disabled { return Ok(None); } + let source_dir = package.source_dir()?; + let name = package.name(); std::fs::create_dir_all(output_dir) .with_context(|| format!("failed to create {}", output_dir.display()))?; @@ -213,7 +241,7 @@ pub async fn fetch_orig_tarball_for_version( let command = config.command.as_deref().context( "orig_tarball.method = \"custom\" requires orig_tarball.command to be set", )?; - let command = substitute_placeholders(command, package, output_dir); + let command = substitute_placeholders(command, package, upstream_version, output_dir); println!("debmagic: running orig tarball command: {command}"); let status = std::process::Command::new("sh") .arg("-c") @@ -318,17 +346,24 @@ async fn fetch_from_distro_archive( /// filesystem, a copy otherwise. fn stage_from_cache(cached: &Path, output_dir: &Path) -> anyhow::Result { let destination = output_dir.join(cached.file_name().unwrap_or_default()); - if std::fs::hard_link(cached, &destination).is_ok() { - return Ok(destination); + link_or_copy(cached, &destination)?; + Ok(destination) +} + +/// Hardlink `source` to `destination` when both are on the same +/// filesystem, copy otherwise. +pub fn link_or_copy(source: &Path, destination: &Path) -> anyhow::Result<()> { + if std::fs::hard_link(source, destination).is_ok() { + return Ok(()); } - std::fs::copy(cached, &destination).with_context(|| { + std::fs::copy(source, destination).with_context(|| { format!( "failed to copy {} to {}", - cached.display(), + source.display(), destination.display() ) })?; - Ok(destination) + Ok(()) } /// The `debian`/`ubuntu` methods: a distro archive pool directly, @@ -594,6 +629,7 @@ mod tests { substitute_placeholders( "fetch {name} {version} {upstream_version} from {source_dir} to {output_dir}", &pkg, + "3.11.7", &out ), format!( diff --git a/packages/debmagic/src/upstream/query.rs b/packages/debmagic/src/upstream/query.rs index 6cf1b220..ed26ff8a 100644 --- a/packages/debmagic/src/upstream/query.rs +++ b/packages/debmagic/src/upstream/query.rs @@ -269,7 +269,7 @@ pub async fn resolve_concrete( version: version.clone(), href: href.clone(), }; - let url = crate::upstream::switch::download_url(source, &candidate)?; + let url = crate::upstream::fetch::download_url(source, &candidate)?; if crate::requests::http_exists(&url).await { return Ok(Some(candidate)); } @@ -286,7 +286,7 @@ pub async fn resolve_concrete( version: version.clone(), href, }; - let url = crate::upstream::switch::download_url(source, &candidate)?; + let url = crate::upstream::fetch::download_url(source, &candidate)?; if crate::requests::http_exists(&url).await { return Ok(Some(candidate)); } diff --git a/packages/debmagic/src/upstream/repack.rs b/packages/debmagic/src/upstream/repack.rs index c5ed7929..9ac84004 100644 --- a/packages/debmagic/src/upstream/repack.rs +++ b/packages/debmagic/src/upstream/repack.rs @@ -41,7 +41,7 @@ pub fn load_repack_config( /// Whether any exclude pattern matches `path` (relative, forward slashes). /// A `dir/*` pattern excludes everything under `dir`, at any depth. -fn is_excluded(path: &str, excludes: &[String]) -> bool { +pub fn is_excluded(path: &str, excludes: &[String]) -> bool { excludes.iter().any(|pattern| { let Ok(glob) = glob::Pattern::new(pattern) else { return false; diff --git a/packages/debmagic/src/upstream/switch.rs b/packages/debmagic/src/upstream/switch.rs index 9f16d62e..386c0fcc 100644 --- a/packages/debmagic/src/upstream/switch.rs +++ b/packages/debmagic/src/upstream/switch.rs @@ -1,386 +1,74 @@ use std::path::{Path, PathBuf}; -use anyhow::{Context, bail}; +use anyhow::Context; +use debmagic_common::package::SourcePackage; -use crate::upstream::mangle::mangle; -use crate::upstream::orig::{OrigTarballConfig, fetch_orig_tarball_for_version}; -use crate::upstream::query::Candidate; -use crate::upstream::repack::{RepackConfig, apply_excludes}; +use crate::upstream::fetch::{FetchOptions, Fetched, extract_tarball, fetch}; use crate::upstream::watch::WatchSource; -/// Resolve the absolute download URL for a candidate: hrefs resolve -/// against the watch source's URL like a browser (RFC 3986, as uscan), -/// then `Download-Url-Mangle` applies. -pub fn download_url(source: &WatchSource, candidate: &Candidate) -> anyhow::Result { - let base = reqwest::Url::parse(&source.source) - .with_context(|| format!("invalid watch source URL {}", source.source))?; - let mut url = base - .join(&candidate.href) - .with_context(|| format!("resolving {} against {base} failed", candidate.href))? - .to_string(); - if let Some(rules) = &source.download_url_mangle { - url = mangle(rules, &url)?; - } - Ok(url) -} - -/// The local filename for a downloaded tarball: `Filename-Mangle` if -/// set, else the last URL component without any query fragment. -pub fn download_filename(source: &WatchSource, candidate: &Candidate) -> anyhow::Result { - if let Some(rules) = &source.filename_mangle { - return mangle(rules, &candidate.href); - } - let url = download_url(source, candidate)?; - let last = url.rsplit('/').next().unwrap_or_default(); - Ok(last - .split(['?', '#']) - .next() - .unwrap_or_default() - .to_string()) -} - -/// Download a candidate's tarball into `download_dir`, returning its path. -/// Uses curl for ftp, the requests module for http(s). -pub async fn download_tarball( - source: &WatchSource, - candidate: &Candidate, - download_dir: &Path, -) -> anyhow::Result { - let url = download_url(source, candidate)?; - let filename = download_filename(source, candidate)?; - if filename.is_empty() { - bail!("could not determine a filename for {url}"); - } - let destination = download_dir.join(filename); - if destination.exists() { - println!( - "debmagic: using already-downloaded {}", - destination.display() - ); - return Ok(destination); - } - println!("debmagic: downloading {url}"); - if url.starts_with("ftp://") || url.starts_with("ftps://") { - crate::requests::ftp_download(&url, &destination).await?; - } else { - crate::requests::http_download(&url, &destination).await?; - } - Ok(destination) -} - -/// Download and verify the signature of a downloaded tarball. -/// The signature URL comes from `Pgp-Sig-Url-Mangle` or the common -/// suffixes (`.asc`, `.sig`, ...); verification runs against -/// `debian/upstream/signing-key.asc` when that file exists. -/// Returns Ok(()) when no keyring or no signature is available. -pub async fn verify_tarball_signature( +/// Switch the package tree to an upstream version: fetch its orig +/// tarball like `upstream fetch`, then replace the tree's contents with +/// the tarball's (keeping `debian/`). A component source swaps its +/// `/` subdir instead; `components` are the package's MUT +/// component names, whose subdirs the main tree swap keeps. +pub async fn switch( + package: &SourcePackage, source: &WatchSource, - source_dir: &Path, - candidate: &Candidate, - tarball: &Path, - sign_options: &crate::config::SignConfig, -) -> anyhow::Result<()> { - let keyring = source_dir - .join("debian") - .join("upstream") - .join("signing-key.asc"); - if !keyring.is_file() { - return Ok(()); - } - - let url = download_url(source, candidate)?; - let sig_url = match &source.pgp_sig_url_mangle { - Some(rules) => mangle(rules, &url)?, - None => { - let mut found = None; - for suffix in ["asc", "sig", "sign", "pgp", "gpg"] { - let candidate_url = format!("{url}.{suffix}"); - if crate::requests::http_exists(&candidate_url).await { - found = Some(candidate_url); - break; - } - } - match found { - Some(url) => url, - None => return Ok(()), - } - } + requested: &str, + options: &FetchOptions<'_>, + components: &[String], +) -> anyhow::Result { + let source_dir = package.source_dir()?; + let fetched = fetch(package, source, requested, options).await?; + let target_dir = match &source.component { + Some(component) => source_dir.join(component), + None => source_dir.to_path_buf(), }; - - let sig_name = sig_url.rsplit('/').next().unwrap_or_default(); - let sig_path = tarball - .parent() - .unwrap_or_else(|| Path::new(".")) - .join(sig_name); - println!("debmagic: downloading signature {sig_url}"); - crate::requests::http_download(&sig_url, &sig_path).await?; - - crate::sign::verify_signature(sign_options, tarball, &sig_path, &keyring)?; - println!( - "debmagic: verified upstream signature of {}", - tarball.display() - ); - Ok(()) -} - -/// Extract a tarball into `extract_dir`, stripping the leading -/// `-/` component so the tree sits directly in -/// `extract_dir`. -pub fn extract_tarball(tarball: &Path, extract_dir: &Path) -> anyhow::Result<()> { - std::fs::create_dir_all(extract_dir) - .with_context(|| format!("failed to create {}", extract_dir.display()))?; - let output = std::process::Command::new("tar") - .arg("--extract") - .arg("--file") - .arg(tarball) - .arg("--directory") - .arg(extract_dir) - .arg("--strip-components=1") - .output() - .with_context(|| format!("failed to run tar for {}", tarball.display()))?; - if !output.status.success() { - bail!( - "extracting {} failed: {}", - tarball.display(), - String::from_utf8_lossy(&output.stderr).trim() + if options.dry_run { + println!( + "debmagic: would replace {} with the contents of {}", + target_dir.display(), + fetched.tarball.display() ); + return Ok(fetched); } - Ok(()) -} - -/// The result of a switch: what was done and to which version. -#[derive(Debug)] -pub struct SwitchResult { - pub version: String, - pub orig_tarball: PathBuf, - pub removed_files: usize, -} -/// How [`switch`] fetches, repacks and places one source's tarball. -pub struct SwitchOptions<'a> { - pub repack: &'a RepackConfig, - pub output_dir: &'a Path, - /// How to fetch an orig tarball the distros may already have; - /// `None` skips the distro lookup. - pub orig_tarball_config: Option<&'a OrigTarballConfig>, - pub verify_signatures: bool, - pub sign_options: &'a crate::config::SignConfig, - pub dry_run: bool, - /// The package's MUT component names, whose subdirs the main tree - /// swap keeps. - pub components: &'a [String], -} - -/// Switch the package tree to `candidate`'s upstream version: download, -/// extract, apply the repack excludes, write the orig tarball to the -/// output dir, and swap the tree contents (keeping `debian/`). A -/// component source swaps its `/` subdir instead. With -/// `dry_run`, only report what would happen. -pub async fn switch( - source_dir: &Path, - source: &WatchSource, - candidate: &Candidate, - options: &SwitchOptions<'_>, -) -> anyhow::Result { - let package = crate::package::load_package(source_dir)?; - let name = package.name().to_string(); - let repack = options.repack; - let output_dir = options.output_dir; - let dry_run = options.dry_run; - let version = &candidate.version; - let oversion = match &repack.suffix { - Some(suffix) => format!("{version}{suffix}"), - None => version.clone(), - }; - - let work_dir = - std::env::temp_dir().join(format!("debmagic-switch-{}-{}", name, std::process::id())); - let download_dir = work_dir.join("download"); + let work_dir = std::env::temp_dir().join(format!( + "debmagic-switch-{}-{}", + package.name(), + std::process::id() + )); let extract_dir = work_dir.join("tree"); - - if dry_run { - let url = download_url(source, candidate)?; - println!("debmagic: would switch {} to upstream {version}", name); - println!("debmagic: tarball: {url}"); - if !repack.excludes.is_empty() { - println!("debmagic: would repack, excluding:"); - for pattern in &repack.excludes { - println!(" {pattern}"); - } - } - if let Some(suffix) = &repack.suffix { - println!("debmagic: version suffix: {suffix}"); - } - match &source.component { - Some(component) => println!("debmagic: would replace the {component}/ subdir"), - None => println!("debmagic: would replace the source tree, keeping debian/"), - } - return Ok(SwitchResult { - version: oversion, - orig_tarball: download_dir.join(download_filename(source, candidate)?), - removed_files: 0, - }); - } - - std::fs::create_dir_all(&download_dir) - .with_context(|| format!("failed to create {}", download_dir.display()))?; - - // the distros may already have this version (a re-switch, or a - // version Debian/Ubuntu published): their orig tarball must be - // reused, since a re-created one would have different checksums - // and the upload would be rejected. Only when they don't have it - // is the tarball fetched from upstream and repacked. - let mut distro_orig = None; - if let Some(orig_config) = options.orig_tarball_config { - match fetch_orig_tarball_for_version( - orig_config, - &package, - options.output_dir, - &oversion, - source.component.as_deref(), - ) - .await - { - Ok(Some(tarball)) => { - println!( - "debmagic: using orig tarball {} (already available; \ - skipping the upstream download and repack)", - tarball.display() - ); - distro_orig = Some(tarball); - } - Ok(None) => {} - Err(error) => { - println!( - "debmagic: no orig tarball from the configured method ({error}); \ - fetching from upstream" - ); - } - } - } - - let tarball = match &distro_orig { - Some(tarball) => tarball.clone(), - None => { - let tarball = download_tarball(source, candidate, &download_dir).await?; - - if options.verify_signatures { - verify_tarball_signature( - source, - source_dir, - candidate, - &tarball, - options.sign_options, - ) - .await?; - } - tarball - } - }; - - extract_tarball(&tarball, &extract_dir)?; - let removed = apply_excludes(&extract_dir, &repack.excludes)?; - - // canonical Debian layout: one top-level `-/` dir - let top_level = format!("{}-{}", name, oversion); - let stage_dir = work_dir.join("stage").join(&top_level); - std::fs::create_dir_all(&stage_dir) - .with_context(|| format!("failed to create {}", stage_dir.display()))?; - move_tree_contents(&extract_dir, &stage_dir)?; - - // an orig tarball for this version may already exist — from a prior - // switch or a developer-provided one. Re-creating it would produce - // different checksums than the archive already holds, and the upload - // would be rejected, so the existing one is kept as-is. - std::fs::create_dir_all(output_dir) - .with_context(|| format!("failed to create {}", output_dir.display()))?; - let orig_path = match crate::changes::find_orig_in_dir( - output_dir, - &name, - &oversion, - source.component.as_deref(), - ) { - Some(existing) => { - println!( - "debmagic: keeping existing orig tarball {} (re-creating it would \ - change its checksums and break uploads)", - existing.display() - ); - existing - } - None => { - let orig_name = match &source.component { - Some(component) => format!( - "{}xz", - debmagic_common::changes::component_orig_prefix(&name, &oversion, component) - ), - None => format!( - "{}xz", - debmagic_common::changes::orig_prefix(&name, &oversion) - ), - }; - let orig_path = output_dir.join(&orig_name); - let output = std::process::Command::new("tar") - .current_dir(stage_dir.parent().expect("stage dir has a parent")) - .args(["--create", "--xz", "--file"]) - .arg(&orig_path) - .arg(&top_level) - .output() - .with_context(|| format!("failed to run tar for {}", orig_path.display()))?; - if !output.status.success() { - bail!( - "creating {} failed: {}", - orig_path.display(), - String::from_utf8_lossy(&output.stderr).trim() - ); - } - orig_path - } - }; - - // after the tarball: the swap moves the staged tree away. A component - // lives in its own subdir, like dpkg-source unpacks it; the main tree - // keeps the component subdirs and the output dir. - let (target_dir, mut keep) = match &source.component { - Some(component) => (source_dir.join(component), Vec::new()), - None => ( - source_dir.to_path_buf(), - options - .components + let swapped = extract_tarball(&fetched.tarball, &extract_dir).and_then(|()| { + // a component lives in its own subdir, like dpkg-source unpacks + // it; the main tree keeps the component subdirs and the output dir + let mut keep: Vec = match &source.component { + Some(_) => Vec::new(), + None => components .iter() .map(|component| source_dir.join(component)) .collect(), - ), - }; - keep.push(output_dir.to_path_buf()); - std::fs::create_dir_all(&target_dir) - .with_context(|| format!("failed to create {}", target_dir.display()))?; - swap_tree(&target_dir, &stage_dir, &keep)?; - + }; + keep.push(options.output_dir.to_path_buf()); + std::fs::create_dir_all(&target_dir) + .with_context(|| format!("failed to create {}", target_dir.display()))?; + swap_tree(&target_dir, &extract_dir, &keep) + }); std::fs::remove_dir_all(&work_dir).ok(); + swapped.with_context(|| { + format!( + "replacing {} with {} failed", + target_dir.display(), + fetched.tarball.display() + ) + })?; println!( - "debmagic: switched {} to upstream {oversion} (orig: {}, {} files excluded)", - name, - orig_path.display(), - removed + "debmagic: switched {} to upstream {} (orig: {})", + target_dir.display(), + fetched.upstream_version, + fetched.tarball.display() ); - Ok(SwitchResult { - version: oversion, - orig_tarball: orig_path, - removed_files: removed, - }) -} - -/// Move every entry of `from_dir` into `to_dir`. -fn move_tree_contents(from_dir: &Path, to_dir: &Path) -> anyhow::Result<()> { - for entry in std::fs::read_dir(from_dir) - .with_context(|| format!("failed to read {}", from_dir.display()))? - .flatten() - { - let destination = to_dir.join(entry.file_name()); - move_entry(&entry.path(), &destination)?; - } - Ok(()) + Ok(fetched) } /// Move one entry, falling back to copy+remove when a rename crosses @@ -497,67 +185,6 @@ fn is_kept(name: &str) -> bool { mod tests { use super::*; - fn source() -> WatchSource { - WatchSource { - source: "https://example.com/releases/".to_string(), - matching_pattern: "foo-@ANY_VERSION@@ARCHIVE_EXT@".to_string(), - ..Default::default() - } - } - - #[test_case::test_case("foo-1.0.tar.gz", "https://example.com/releases/foo-1.0.tar.gz"; "relative")] - #[test_case::test_case("/dl/foo-1.0.tar.gz", "https://example.com/dl/foo-1.0.tar.gz"; "host absolute")] - #[test_case::test_case("../old/foo-1.0.tar.gz", "https://example.com/old/foo-1.0.tar.gz"; "parent")] - #[test_case::test_case("//cdn.example.org/foo-1.0.tar.gz", "https://cdn.example.org/foo-1.0.tar.gz"; "scheme relative")] - fn test_download_url_relative(href: &str, expected: &str) { - let source = source(); - let candidate = Candidate { - version: "1.0".to_string(), - href: href.to_string(), - }; - assert_eq!(download_url(&source, &candidate).unwrap(), expected); - } - - #[test] - fn test_download_url_absolute() { - let source = source(); - let candidate = Candidate { - version: "1.0".to_string(), - href: "https://mirror.example.com/foo-1.0.tar.gz".to_string(), - }; - assert_eq!( - download_url(&source, &candidate).unwrap(), - "https://mirror.example.com/foo-1.0.tar.gz" - ); - } - - #[test] - fn test_download_url_mangle() { - let mut source = source(); - source.download_url_mangle = Some("s/prdownload/download/".to_string()); - let candidate = Candidate { - version: "1.0".to_string(), - href: "https://prdownload.example.com/foo-1.0.tar.gz".to_string(), - }; - assert_eq!( - download_url(&source, &candidate).unwrap(), - "https://download.example.com/foo-1.0.tar.gz" - ); - } - - #[test] - fn test_download_filename() { - let source = source(); - let candidate = Candidate { - version: "1.0".to_string(), - href: "foo-1.0.tar.gz".to_string(), - }; - assert_eq!( - download_filename(&source, &candidate).unwrap(), - "foo-1.0.tar.gz" - ); - } - #[test] fn test_extract_and_swap() { let base = std::env::temp_dir().join(format!("debmagic-switch-{}", std::process::id())); From 94c7bc12c12195103fee8e9fe714fe175406a548 Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Wed, 7 Oct 2026 16:35:53 +0200 Subject: [PATCH 11/48] fix(upstream): fetch GitHub tag tarballs `upstream fetch --from upstream` fails for watch files like valkey's: - the `.zip` GitHub lists first won the version dedup; a tarball dpkg-source can use as-is is now preferred - `@ARCHIVE_EXT@` and the version macros are now expanded in mangle rules too, like uscan does, so `filenamemangle` matches - a mangled filename is always reduced to its last path component --- packages/debmagic/src/upstream/fetch.rs | 15 ++++--- packages/debmagic/src/upstream/query.rs | 59 ++++++++++++++++++++++--- 2 files changed, 61 insertions(+), 13 deletions(-) diff --git a/packages/debmagic/src/upstream/fetch.rs b/packages/debmagic/src/upstream/fetch.rs index 8a8b7b7e..4e4e4d0f 100644 --- a/packages/debmagic/src/upstream/fetch.rs +++ b/packages/debmagic/src/upstream/fetch.rs @@ -462,7 +462,7 @@ fn place_orig( /// The orig tarball compression for an upstream tarball's filename, if /// dpkg-source accepts it; maps the short `.tgz`-style forms. -fn orig_compression(filename: &str) -> Option<&'static str> { +pub fn orig_compression(filename: &str) -> Option<&'static str> { [ (".tar.gz", "gz"), (".tgz", "gz"), @@ -556,13 +556,14 @@ pub fn download_url(source: &WatchSource, candidate: &Candidate) -> anyhow::Resu } /// The local filename for a downloaded tarball: `Filename-Mangle` if -/// set, else the last URL component without any query fragment. +/// set, else the last URL component without any query fragment. Always +/// a bare filename: a mangle rule that doesn't match leaves a path. fn download_filename(source: &WatchSource, candidate: &Candidate) -> anyhow::Result { - if let Some(rules) = &source.filename_mangle { - return mangle(rules, &candidate.href); - } - let url = download_url(source, candidate)?; - let last = url.rsplit('/').next().unwrap_or_default(); + let name = match &source.filename_mangle { + Some(rules) => mangle(rules, &candidate.href)?, + None => download_url(source, candidate)?, + }; + let last = name.rsplit('/').next().unwrap_or_default(); Ok(last .split(['?', '#']) .next() diff --git a/packages/debmagic/src/upstream/query.rs b/packages/debmagic/src/upstream/query.rs index ed26ff8a..deaf91bf 100644 --- a/packages/debmagic/src/upstream/query.rs +++ b/packages/debmagic/src/upstream/query.rs @@ -5,12 +5,18 @@ use anyhow::{Context, bail}; use debmagic_common::debian::version::PackageVersion; use regex::Regex; +use crate::upstream::fetch::orig_compression; use crate::upstream::mangle::mangle; use crate::upstream::watch::{SearchMode, WatchSource}; fn substitute(pattern: &str, package: &str) -> String { + substitute_macros(&pattern.replace("@PACKAGE@", ®ex::escape(package))) +} + +/// Expand the version and extension macros to their regexes; like uscan +/// does on the whole watch line, so mangle rules can use them too. +pub fn substitute_macros(pattern: &str) -> String { pattern - .replace("@PACKAGE@", ®ex::escape(package)) .replace( "@ANY_VERSION@", r"[-_]?[Vv]?(\d[\-+\.:\~\da-zA-Z]*)", @@ -363,17 +369,21 @@ fn parse_candidates( }); } - // stable sort: among equal versions the first listed href wins the dedup + // stable sort: among equal versions a tarball dpkg-source can use + // as-is wins the dedup (GitHub lists the .zip before the .tar.gz), + // then the first listed href candidates.sort_by(|a, b| { - let a = PackageVersion::from_str(&a.version); - let b = PackageVersion::from_str(&b.version); - match (a, b) { + let version_a = PackageVersion::from_str(&a.version); + let version_b = PackageVersion::from_str(&b.version); + let by_version = match (version_a, version_b) { (Ok(a), Ok(b)) => b.cmp(&a), // unparseable versions sink to the bottom (Ok(_), Err(_)) => std::cmp::Ordering::Less, (Err(_), Ok(_)) => std::cmp::Ordering::Greater, (Err(_), Err(_)) => std::cmp::Ordering::Equal, - } + }; + let unusable = |c: &Candidate| orig_compression(&c.href).is_none(); + by_version.then_with(|| unusable(a).cmp(&unusable(b))) }); candidates.dedup_by(|a, b| a.version == b.version); Ok(candidates) @@ -427,6 +437,18 @@ pub fn load_watch(source_dir: &Path, package: &str) -> anyhow::Result + "#; + let candidates = parse_candidates(&source, "foo", page).unwrap(); + assert_eq!(candidates.len(), 1); + assert_eq!(candidates[0].href, "/foo/foo/archive/refs/tags/1.0.tar.gz"); + } + + #[test] + fn test_substitute_macros_in_mangle_rules() { + // valkey's filenamemangle uses @ARCHIVE_EXT@ + let rule = substitute_macros(r"s%(?:.*?)?v?(\d[\d.]*@ARCHIVE_EXT@)%valkey-$1%"); + assert_eq!( + mangle(&rule, "/valkey-io/valkey/archive/refs/tags/8.1.3.tar.gz").unwrap(), + "valkey-8.1.3.tar.gz" + ); + } + #[test] fn test_find_candidate_accepts_debian_version() { let candidates = vec![ From 81075922513471b5e0d6b96ab77f9b55cb7350ba Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Wed, 7 Oct 2026 16:35:53 +0200 Subject: [PATCH 12/48] feat(upload): replace --no-hooks with --hooks[=no] --- docs/usage/upload.md | 2 +- packages/debmagic/src/cli.rs | 10 +++++++--- packages/debmagic/src/main.rs | 2 +- packages/debmagic/src/upload.rs | 5 +++-- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/docs/usage/upload.md b/docs/usage/upload.md index 94fe9ced..5cc40df2 100644 --- a/docs/usage/upload.md +++ b/docs/usage/upload.md @@ -72,7 +72,7 @@ A target's own `pre_upload_commands` run after the global ones. With dput-ng installed, it runs the pre-upload hooks the dput-ng profile of the same name enables (e.g. `ppa`), exactly as `dput` would; pass `--profile ` to use another profile. Without dput-ng, it runs every `"pre": true` hook with an emulation of the python API dput-ng provides, so unmodified dput-ng hook modules work when placed in `/usr/share/dput-ng`, `/etc/dput.d` and `~/.dput.d`. -Pass `--no-hooks` to skip the checks. +Pass `--hooks=no` to skip the checks. ## Signing before the upload diff --git a/packages/debmagic/src/cli.rs b/packages/debmagic/src/cli.rs index 8781d9d6..21550db4 100644 --- a/packages/debmagic/src/cli.rs +++ b/packages/debmagic/src/cli.rs @@ -708,10 +708,14 @@ pub struct UploadSubcommandArgs { #[arg( long, - action = clap::ArgAction::SetTrue, - help = "Skip the target's pre_upload_commands" + num_args = 0..=1, + default_missing_value = "true", + default_value_t = true, + action = clap::ArgAction::Set, + value_parser = BoolishValueParser::new(), + help = "Run the target's pre_upload_commands; --hooks=no skips them" )] - pub no_hooks: bool, + pub hooks: bool, #[arg( long, diff --git a/packages/debmagic/src/main.rs b/packages/debmagic/src/main.rs index 054d2f17..5332d13e 100644 --- a/packages/debmagic/src/main.rs +++ b/packages/debmagic/src/main.rs @@ -193,7 +193,7 @@ async fn run() -> anyhow::Result { incoming: args.incoming.clone(), login: args.login.clone(), port: args.port, - no_hooks: args.no_hooks, + hooks: args.hooks, force: args.force, sign: args.sign, include_orig: args.include_orig, diff --git a/packages/debmagic/src/upload.rs b/packages/debmagic/src/upload.rs index 88002fbd..ff064823 100644 --- a/packages/debmagic/src/upload.rs +++ b/packages/debmagic/src/upload.rs @@ -70,7 +70,8 @@ pub struct UploadIntent { pub incoming: Option, pub login: Option, pub port: Option, - pub no_hooks: bool, + /// Run the target's `pre_upload_commands`. + pub hooks: bool, pub force: bool, pub sign: Option, pub include_orig: Option, @@ -147,7 +148,7 @@ pub fn upload(input: UploadIntent) -> anyhow::Result<()> { &upload_target, &input.target, &changes_file, - input.no_hooks, + !input.hooks, input.force, Some(identity.version().upstream_version()), ) From 2d98f57aff45b70807221ec5bad26f0af879da89 Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Wed, 7 Oct 2026 16:43:49 +0200 Subject: [PATCH 13/48] feat(upstream): support the oversionmangle watch option The orig tarball version is the upstream version with `oversionmangle` (`Oversion-Mangle` in v5) applied, e.g. dovecot's `s/(.*)/$1+dfsg1/`. --- docs/usage/upstream.md | 2 +- packages/debmagic/src/upstream/fetch.rs | 6 ++++- packages/debmagic/src/upstream/query.rs | 1 + packages/debmagic/src/upstream/watch.rs | 32 ++++++++++++++++++++++++- 4 files changed, 38 insertions(+), 3 deletions(-) diff --git a/docs/usage/upstream.md b/docs/usage/upstream.md index eec6af51..51ee5dce 100644 --- a/docs/usage/upstream.md +++ b/docs/usage/upstream.md @@ -88,7 +88,7 @@ repack = false # keep upstream tarballs as-is, warning about excluded files ## Watch file support `debian/watch` versions 2–5 are parsed (version 1 is rejected). -The common options work: `Source`, `Matching-Pattern` (including the `@ANY_VERSION@`/`@ARCHIVE_EXT@`/`@PACKAGE@` substitutions), `Search-Mode`, `Uversion-Mangle`, `Dversion-Mangle`, `Filename-Mangle`, `Download-Url-Mangle`, `Pgp-Mode`/`Pgp-Sig-Url-Mangle`, `Repack`, `Repacksuffix`, `Component` (for MUT packages), `Untrackable`. +The common options work: `Source`, `Matching-Pattern` (including the `@ANY_VERSION@`/`@ARCHIVE_EXT@`/`@PACKAGE@` substitutions), `Search-Mode`, `Uversion-Mangle`, `Dversion-Mangle`, `Oversion-Mangle`, `Filename-Mangle`, `Download-Url-Mangle`, `Pgp-Mode`/`Pgp-Sig-Url-Mangle`, `Repack`, `Repacksuffix`, `Component` (for MUT packages), `Untrackable`. http(s) sources are fetched directly; ftp sources fall back to `curl` (install it when a watch file needs it). diff --git a/packages/debmagic/src/upstream/fetch.rs b/packages/debmagic/src/upstream/fetch.rs index 4e4e4d0f..31f4e45e 100644 --- a/packages/debmagic/src/upstream/fetch.rs +++ b/packages/debmagic/src/upstream/fetch.rs @@ -134,7 +134,11 @@ pub async fn fetch( None => version, } }; - let orig_version = format!("{upstream_version}{suffix}"); + let orig_base = match &source.oversion_mangle { + Some(rules) => mangle(rules, &upstream_version)?, + None => upstream_version.clone(), + }; + let orig_version = format!("{orig_base}{suffix}"); let fetched = |tarball| Fetched { tarball, upstream_version: upstream_version.clone(), diff --git a/packages/debmagic/src/upstream/query.rs b/packages/debmagic/src/upstream/query.rs index deaf91bf..ae20b071 100644 --- a/packages/debmagic/src/upstream/query.rs +++ b/packages/debmagic/src/upstream/query.rs @@ -440,6 +440,7 @@ pub fn load_watch(source_dir: &Path, package: &str) -> anyhow::Result, pub dversion_mangle: Option, + /// Turns the upstream version into the orig tarball's version, e.g. + /// appending `+dfsg` to a repacked MUT package. + pub oversion_mangle: Option, pub filename_mangle: Option, pub download_url_mangle: Option, pub pgp_sig_url_mangle: Option, @@ -46,6 +49,7 @@ impl WatchSource { [ &mut self.uversion_mangle, &mut self.dversion_mangle, + &mut self.oversion_mangle, &mut self.filename_mangle, &mut self.download_url_mangle, &mut self.pgp_sig_url_mangle, @@ -257,6 +261,9 @@ fn parse_watch_v5(content: &str) -> anyhow::Result> { "dversion-mangle" | "dversionmangle" => { source.dversion_mangle = Some(value.clone()) } + "oversion-mangle" | "oversionmangle" => { + source.oversion_mangle = Some(value.clone()) + } "filename-mangle" | "filenamemangle" => { source.filename_mangle = Some(value.clone()) } @@ -282,7 +289,7 @@ fn parse_watch_v5(content: &str) -> anyhow::Result> { | "page-mangle" | "mode" | "compression" | "unzip-opt" | "git-pretty" | "git-date" | "git-export" | "git-mode" | "git-modules" | "bare" | "maturity" | "hrefdecode" | "decompress" | "user-agent" | "template" | "dist" | "owner" - | "project" | "oversion-mangle" => { + | "project" => { bail!( "watch file option '{key}' is not supported yet; \ declare the upstream source in debmagic.toml instead" @@ -328,6 +335,9 @@ fn merge_source(defaults: &WatchSource, source: WatchSource) -> WatchSource { merged.dversion_mangle = merged .dversion_mangle .or_else(|| defaults.dversion_mangle.clone()); + merged.oversion_mangle = merged + .oversion_mangle + .or_else(|| defaults.oversion_mangle.clone()); merged.filename_mangle = merged .filename_mangle .or_else(|| defaults.filename_mangle.clone()); @@ -354,6 +364,8 @@ fn apply_opts(source: &mut WatchSource, opts: &str) -> anyhow::Result<()> { source.uversion_mangle = Some(value.to_string()); } else if let Some(value) = opt.strip_prefix("dversionmangle=") { source.dversion_mangle = Some(value.to_string()); + } else if let Some(value) = opt.strip_prefix("oversionmangle=") { + source.oversion_mangle = Some(value.to_string()); } else if let Some(value) = opt.strip_prefix("filenamemangle=") { source.filename_mangle = Some(value.to_string()); } else if let Some(value) = opt.strip_prefix("downloadurlmangle=") { @@ -458,6 +470,24 @@ mod tests { assert_eq!(sources[0].matching_pattern, "foo-(.*)\\.tar\\.gz"); } + #[test] + fn test_parse_oversion_mangle() { + let sources = parse_watch( + "version=4\n\ + opts=\"oversionmangle=s/(.*)/$1+dfsg1/\" https://example.com/ foo-(.*)\\.tar\\.gz\n", + ) + .unwrap(); + assert_eq!( + sources[0].oversion_mangle.as_deref(), + Some("s/(.*)/$1+dfsg1/") + ); + let sources = parse_watch( + "Version: 5\n\nSource: https://example.com/\nMatching-Pattern: foo-(.*)\\.tar\\.gz\nOversion-Mangle: s/$/+ds/\n", + ) + .unwrap(); + assert_eq!(sources[0].oversion_mangle.as_deref(), Some("s/$/+ds/")); + } + #[test] fn test_parse_v4_embedded_pattern() { let sources = parse_watch( From 0bbb488522ea349a0d881d9956531c4d565e46cb Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Wed, 7 Oct 2026 17:33:29 +0200 Subject: [PATCH 14/48] fix(upstream): exclude bare directory patterns from listed files A `Files-Excluded` entry naming a directory (dovecot's `doc/rfc`) never matched the files under it: `--repack=no` printed no warning, and repacking was skipped as if nothing were excluded. --- packages/debmagic/src/upstream/repack.rs | 30 +++++++++++++++--------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/packages/debmagic/src/upstream/repack.rs b/packages/debmagic/src/upstream/repack.rs index 9ac84004..ba232e79 100644 --- a/packages/debmagic/src/upstream/repack.rs +++ b/packages/debmagic/src/upstream/repack.rs @@ -40,22 +40,20 @@ pub fn load_repack_config( } /// Whether any exclude pattern matches `path` (relative, forward slashes). -/// A `dir/*` pattern excludes everything under `dir`, at any depth. +/// A pattern matching a directory (`doc/rfc`, `win32/*`) excludes +/// everything under it, at any depth. pub fn is_excluded(path: &str, excludes: &[String]) -> bool { excludes.iter().any(|pattern| { let Ok(glob) = glob::Pattern::new(pattern) else { return false; }; - if glob.matches(path) { - return true; - } - // the pattern may match an ancestor: `win32/*` covers win32/nested/b.c - let mut ancestor = String::new(); - path.split('/').any(|part| { - ancestor.push_str(part); - ancestor.push('/'); - glob.matches(&ancestor) - }) + path.match_indices('/') + .map(|(end, _)| end) + .chain([path.len()]) + .any(|end| { + let dir = &path[..end]; + glob.matches(dir) || glob.matches(&format!("{dir}/")) + }) }) } @@ -176,5 +174,15 @@ mod tests { assert!(is_excluded("docs/manual.pdf", &excludes)); assert!(!is_excluded("src/main.c", &excludes)); assert!(!is_excluded("win32.c", &excludes)); + // the directory itself, so it is removed as a whole + assert!(is_excluded("win32", &excludes)); + } + + #[test] + fn test_is_excluded_bare_dir() { + let excludes = vec!["doc/rfc".to_string()]; + assert!(is_excluded("doc/rfc", &excludes)); + assert!(is_excluded("doc/rfc/rfc5228.txt", &excludes)); + assert!(!is_excluded("doc/rfc2822.c", &excludes)); } } From 8099d86d57cb5424f12fdaf2dabe695223e4b637 Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Wed, 7 Oct 2026 17:33:29 +0200 Subject: [PATCH 15/48] fix(sign): verify upstream signatures against armored keys gpg's `--keyring` only reads binary keyrings, so every `debian/upstream/signing-key.asc` failed with "invalid packet (ctb=2d)". The key is now dearmored first, like uscan does. --- packages/debmagic/src/sign.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/packages/debmagic/src/sign.rs b/packages/debmagic/src/sign.rs index afde93e7..ccac1194 100644 --- a/packages/debmagic/src/sign.rs +++ b/packages/debmagic/src/sign.rs @@ -147,6 +147,34 @@ impl SignBackend for Gpg { signature: &Path, keyring: &Path, ) -> anyhow::Result { + // gpg's --keyring only reads binary keyrings, while + // debian/upstream/signing-key.asc is armored + let armored = std::fs::read(keyring) + .with_context(|| format!("failed to read {}", keyring.display()))? + .starts_with(b"-----BEGIN PGP"); + let keyring = if armored { + let mut dearmored = signature.as_os_str().to_owned(); + dearmored.push(".keyring.gpg"); + let dearmored = PathBuf::from(dearmored); + let mut cmd = Command::new("gpg"); + cmd.args(["--yes", "--dearmor", "--output"]) + .arg(&dearmored) + .arg(keyring); + let result = crate::subprocess::command(cmd) + .capture(Capture::ALL) + .run() + .context("failed to run gpg --dearmor")?; + if result.exit_code != 0 { + bail!( + "dearmoring {} failed:\n{}", + keyring.display(), + result.stderr.as_deref().unwrap_or_default().trim() + ); + } + dearmored + } else { + keyring.to_path_buf() + }; let mut cmd = Command::new("gpg"); cmd.args(["--no-default-keyring", "--keyring"]) .arg(keyring) From 16ef46fdc307f708168d3a0be2e4fb3957d37273 Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Wed, 7 Oct 2026 17:33:29 +0200 Subject: [PATCH 16/48] fix: send a User-Agent with HTTP requests The GitHub API answers requests without one with 403 Forbidden, breaking watch files on api.github.com (e.g. mdevctl). --- packages/debmagic/src/requests.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/debmagic/src/requests.rs b/packages/debmagic/src/requests.rs index 4d19e6f0..760ad39b 100644 --- a/packages/debmagic/src/requests.rs +++ b/packages/debmagic/src/requests.rs @@ -5,9 +5,11 @@ use futures_util::StreamExt; /// The HTTP client for all external requests: no overall timeout /// (tarballs can be huge and slow), but a connect timeout so dead -/// hosts fail fast. +/// hosts fail fast. The GitHub API rejects requests without a +/// User-Agent. fn http_client() -> reqwest::Client { reqwest::Client::builder() + .user_agent(concat!("debmagic/", env!("CARGO_PKG_VERSION"))) .connect_timeout(std::time::Duration::from_secs(30)) .build() .expect("reqwest client with a custom timeout") From ad4a94474eeaf8bc98cccc4aaab2b356e2ce99d1 Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Wed, 7 Oct 2026 17:33:29 +0200 Subject: [PATCH 17/48] feat(upstream): support v4 MUT watch lines - `component=` in `opts` - version field `same`, and `ignore`: a component without a release for the main version falls back to its newest one (also v5 `Version-Schema`) - update script `uupdate` is accepted, as `upstream switch` does its job (also v5 `Update-Script`) --- docs/usage/upstream.md | 5 ++- packages/debmagic/src/upstream/fetch.rs | 14 +++++- packages/debmagic/src/upstream/watch.rs | 57 ++++++++++++++++++++----- 3 files changed, 63 insertions(+), 13 deletions(-) diff --git a/docs/usage/upstream.md b/docs/usage/upstream.md index 51ee5dce..c15c3edc 100644 --- a/docs/usage/upstream.md +++ b/docs/usage/upstream.md @@ -90,9 +90,12 @@ repack = false # keep upstream tarballs as-is, warning about excluded files `debian/watch` versions 2–5 are parsed (version 1 is rejected). The common options work: `Source`, `Matching-Pattern` (including the `@ANY_VERSION@`/`@ARCHIVE_EXT@`/`@PACKAGE@` substitutions), `Search-Mode`, `Uversion-Mangle`, `Dversion-Mangle`, `Oversion-Mangle`, `Filename-Mangle`, `Download-Url-Mangle`, `Pgp-Mode`/`Pgp-Sig-Url-Mangle`, `Repack`, `Repacksuffix`, `Component` (for MUT packages), `Untrackable`. +In version 2–4 lines, the version field may be `debian`, or `same`/`ignore` for MUT components: components are fetched for the main tarball's version, and an `ignore` component that has no such release falls back to its newest one. +The `uupdate` update script is accepted, since `debmagic upstream switch` does its job. + http(s) sources are fetched directly; ftp sources fall back to `curl` (install it when a watch file needs it). -Rare options with no debmagic equivalent yet (`Mode: git`/`svn`, `Version-Schema: group/checksum`, `Page-Mangle`, `Update-Script`, ...) produce a clear error suggesting an explicit upstream declaration instead of silently misbehaving. +Rare options with no debmagic equivalent yet (`Mode: git`/`svn`, `Version-Schema: group/checksum`, `Page-Mangle`, custom update scripts, ...) produce a clear error suggesting an explicit upstream declaration instead of silently misbehaving. ## Orig tarballs in builds diff --git a/packages/debmagic/src/upstream/fetch.rs b/packages/debmagic/src/upstream/fetch.rs index 31f4e45e..7cef4ae8 100644 --- a/packages/debmagic/src/upstream/fetch.rs +++ b/packages/debmagic/src/upstream/fetch.rs @@ -228,7 +228,19 @@ pub async fn fetch( let candidate = match candidate { Some(candidate) => candidate, - None => resolve_candidate(package, source, &upstream_version, output_dir).await?, + None => match resolve_candidate(package, source, &upstream_version, output_dir).await { + // its orig tarball is still named after the main version + Err(error) + if let Some(component) = component + && source.version_unrestricted => + { + println!( + "debmagic: {error:#}; the watch file doesn't tie component {component} to the main version, using its newest release" + ); + resolve_candidate(package, source, "latest", output_dir).await? + } + result => result?, + }, }; let url = download_url(source, &candidate)?; if options.dry_run { diff --git a/packages/debmagic/src/upstream/watch.rs b/packages/debmagic/src/upstream/watch.rs index 1d0f504d..d90573a0 100644 --- a/packages/debmagic/src/upstream/watch.rs +++ b/packages/debmagic/src/upstream/watch.rs @@ -22,6 +22,9 @@ pub struct WatchSource { /// MUT component name: this source describes one component tarball /// of a multiple-upstream-tarballs package. pub component: Option, + /// A MUT component's version is not tied to the main tarball's + /// (v4 version field `ignore`). + pub version_unrestricted: bool, pub untrackable: Option, } @@ -177,18 +180,21 @@ fn parse_watch_line(line: &str) -> anyhow::Result { source.source = url; source.matching_pattern = pattern.to_string(); - // uscan's optional version (`debian`, `prev`, a literal version or a - // version schema) and update script; `debian` is what debmagic does - let version = tokens.next(); - if let Some(version) = version - && version != "debian" - { - bail!( + // uscan's optional version field: debmagic fetches the requested + // version (`debian`), MUT components follow the main tarball's + // version (`same`) or are unrestricted (`ignore`) + match tokens.next() { + None | Some("debian" | "same") => {} + Some("ignore") => source.version_unrestricted = true, + Some(version) => bail!( "watch line version field '{version}' is not supported yet; \ declare the upstream source in debmagic.toml instead" - ); + ), } - if let Some(script) = tokens.next() { + // `upstream switch` does uupdate's job in place + if let Some(script) = tokens.next() + && script != "uupdate" + { bail!( "watch line update script '{script}' is not supported yet; \ declare the upstream source in debmagic.toml instead" @@ -285,6 +291,12 @@ fn parse_watch_v5(content: &str) -> anyhow::Result> { "repack" => source.repack = value.eq_ignore_ascii_case("yes"), "repacksuffix" => source.repack_suffix = Some(value.clone()), "component" => source.component = Some(value.clone()), + "version-schema" | "versionschema" + if matches!(value.as_str(), "debian" | "same" | "ignore") => + { + source.version_unrestricted = value == "ignore" + } + "update-script" | "updatescript" if value == "uupdate" => {} "ctype" | "version-schema" | "versionseparator" | "update-script" | "page-mangle" | "mode" | "compression" | "unzip-opt" | "git-pretty" | "git-date" | "git-export" | "git-mode" | "git-modules" | "bare" | "maturity" @@ -353,6 +365,7 @@ fn merge_source(defaults: &WatchSource, source: WatchSource) -> WatchSource { .or_else(|| defaults.repack_suffix.clone()); merged.repack = merged.repack || defaults.repack; merged.component = merged.component.or_else(|| defaults.component.clone()); + merged.version_unrestricted = merged.version_unrestricted || defaults.version_unrestricted; merged } @@ -382,6 +395,8 @@ fn apply_opts(source: &mut WatchSource, opts: &str) -> anyhow::Result<()> { }); } else if let Some(value) = opt.strip_prefix("repacksuffix=") { source.repack_suffix = Some(value.to_string()); + } else if let Some(value) = opt.strip_prefix("component=") { + source.component = Some(value.to_string()); } else if opt == "repack" { source.repack = true; } else if opt == "pasv" || opt == "passive" { @@ -448,12 +463,29 @@ mod tests { assert!(sources[0].uversion_mangle.is_some()); } - #[test_case("https://example.com/ foo-(.*)\\.tar\\.gz debian uupdate"; "update script")] + #[test_case("https://example.com/ foo-(.*)\\.tar\\.gz debian debian/update.sh"; "custom update script")] #[test_case("https://example.com/ foo-(.*)\\.tar\\.gz 1.2"; "fixed version")] + #[test_case("https://example.com/ foo-(.*)\\.tar\\.gz group"; "grouped version")] fn test_parse_v4_unsupported_trailing_fields(line: &str) { assert!(parse_watch(&format!("version=4\n{line}\n")).is_err()); } + #[test] + fn test_parse_v4_mut() { + let sources = parse_watch( + "version=4\n\ + https://example.com/ foo-(.*)\\.tar\\.gz debian uupdate\n\ + opts=\"pgpmode=none, component=bar\" https://example.com/ bar-(.*)\\.tar\\.gz same\n\ + opts=component=baz https://example.com/ baz-(.*)\\.tar\\.gz ignore uupdate\n", + ) + .unwrap(); + let components: Vec<_> = sources.iter().map(|s| s.component.as_deref()).collect(); + assert_eq!(components, [None, Some("bar"), Some("baz")]); + assert_eq!(sources[1].pgp_mode, Some(PgpMode::None)); + let unrestricted: Vec<_> = sources.iter().map(|s| s.version_unrestricted).collect(); + assert_eq!(unrestricted, [false, false, true]); + } + #[test] fn test_parse_v4_trailing_debian_and_quoted_opts() { let sources = parse_watch( @@ -574,11 +606,14 @@ mod tests { "Version: 5\n\ Source: https://example.com/a/\nMatching-Pattern: a-@ANY_VERSION@\n\ \n\ - Source: https://example.com/b/\nMatching-Pattern: b-@ANY_VERSION@\nComponent: bar\n", + Source: https://example.com/b/\nMatching-Pattern: b-@ANY_VERSION@\nComponent: bar\n\ + Version-Schema: ignore\nUpdate-Script: uupdate\n", ) .unwrap(); assert_eq!(sources.len(), 2); assert_eq!(sources[0].component, None); assert_eq!(sources[1].component.as_deref(), Some("bar")); + assert!(!sources[0].version_unrestricted); + assert!(sources[1].version_unrestricted); } } From aba5bc2a552b927af56d898f01154fb12f88cc0c Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Wed, 7 Oct 2026 17:59:00 +0200 Subject: [PATCH 18/48] fix(upstream): replace whole dirs on switch `upstream switch` left stale content behind: - a dir present in both the old and the new tree was merged, keeping files the new version dropped - a component's `debian/` was kept from the old tree, while dpkg-source only replaces the main orig's `debian/` The old tree is now removed entirely before the new one moves in, except `.git`, kept paths and the main tree's `debian/`. --- packages/debmagic/src/upstream/switch.rs | 54 +++++++++++++++++------- 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/packages/debmagic/src/upstream/switch.rs b/packages/debmagic/src/upstream/switch.rs index 386c0fcc..f52714a6 100644 --- a/packages/debmagic/src/upstream/switch.rs +++ b/packages/debmagic/src/upstream/switch.rs @@ -52,7 +52,7 @@ pub async fn switch( keep.push(options.output_dir.to_path_buf()); std::fs::create_dir_all(&target_dir) .with_context(|| format!("failed to create {}", target_dir.display()))?; - swap_tree(&target_dir, &extract_dir, &keep) + swap_tree(&target_dir, &extract_dir, &keep, source.component.is_none()) }); std::fs::remove_dir_all(&work_dir).ok(); swapped.with_context(|| { @@ -134,26 +134,32 @@ fn copy_dir(from: &Path, to: &Path) -> anyhow::Result<()> { } /// Replace the source tree's contents with the extracted tree's, -/// keeping `debian/`, any VCS metadata dir (`.git`, ...) and every -/// `keep` path (or path containing one) intact. -fn swap_tree(source_dir: &Path, extract_dir: &Path, keep: &[PathBuf]) -> anyhow::Result<()> { +/// keeping any VCS metadata dir (`.git`, ...), `debian/` when +/// `main_tree` and every `keep` path (or path containing one) intact. +/// dpkg-source only replaces the main orig's `debian/`, a component's +/// is regular upstream content. +fn swap_tree( + source_dir: &Path, + extract_dir: &Path, + keep: &[PathBuf], + main_tree: bool, +) -> anyhow::Result<()> { + let is_kept = |name: &str| name == ".git" || (main_tree && name == "debian"); let new_names: Vec = std::fs::read_dir(extract_dir) .with_context(|| format!("failed to read {}", extract_dir.display()))? .flatten() .map(|entry| entry.file_name().to_string_lossy().into_owned()) .collect(); - // stale entries from the old tree that the new one doesn't have + // the whole old tree, so a dir in both trees doesn't keep files the + // new one dropped for entry in std::fs::read_dir(source_dir) .with_context(|| format!("failed to read {}", source_dir.display()))? .flatten() { let name = entry.file_name().to_string_lossy().into_owned(); let path = entry.path(); - if is_kept(&name) - || new_names.contains(&name) - || keep.iter().any(|kept| kept.starts_with(&path)) - { + if is_kept(&name) || keep.iter().any(|kept| kept.starts_with(&path)) { continue; } if path.is_dir() { @@ -175,12 +181,6 @@ fn swap_tree(source_dir: &Path, extract_dir: &Path, keep: &[PathBuf]) -> anyhow: Ok(()) } -/// Entries that always survive a tree swap: the packaging dir and -/// VCS storage (`.git`) -fn is_kept(name: &str) -> bool { - name == "debian" || name == ".git" -} - #[cfg(test)] mod tests { use super::*; @@ -192,8 +192,10 @@ mod tests { let extract_dir = base.join("tree"); std::fs::create_dir_all(source_dir.join("debian")).unwrap(); std::fs::create_dir_all(&extract_dir).unwrap(); - // old tree: a file that the new tree lacks, plus debian/ + // old tree: files that the new tree lacks, plus debian/ std::fs::write(source_dir.join("old.txt"), "old").unwrap(); + std::fs::create_dir_all(source_dir.join("src")).unwrap(); + std::fs::write(source_dir.join("src").join("gone.c"), "x").unwrap(); std::fs::write(source_dir.join("debian").join("changelog"), "keep").unwrap(); let output_dir = source_dir.join("build"); std::fs::create_dir_all(&output_dir).unwrap(); @@ -211,15 +213,35 @@ mod tests { &source_dir, &extract_dir, &[output_dir.clone(), component_dir.clone()], + true, ) .unwrap(); assert!(source_dir.join("new.txt").exists()); assert!(source_dir.join("src").join("main.c").exists()); assert!(!source_dir.join("old.txt").exists()); + assert!(!source_dir.join("src").join("gone.c").exists()); assert!(source_dir.join("debian").join("changelog").exists()); assert!(output_dir.join("pkg_2.0.orig.tar.xz").exists()); assert!(component_dir.join("plugin.c").exists()); std::fs::remove_dir_all(&base).unwrap(); } + + #[test] + fn test_swap_component_replaces_debian() { + let base = + std::env::temp_dir().join(format!("debmagic-switch-comp-{}", std::process::id())); + let component_dir = base.join("plugin"); + let extract_dir = base.join("tree"); + std::fs::create_dir_all(component_dir.join("debian")).unwrap(); + std::fs::create_dir_all(extract_dir.join("debian")).unwrap(); + std::fs::write(component_dir.join("debian").join("old"), "old").unwrap(); + std::fs::write(extract_dir.join("debian").join("new"), "new").unwrap(); + + swap_tree(&component_dir, &extract_dir, &[], false).unwrap(); + + assert!(component_dir.join("debian").join("new").exists()); + assert!(!component_dir.join("debian").join("old").exists()); + std::fs::remove_dir_all(&base).unwrap(); + } } From f970abdf0f39e1a236e489f99b1c8d78a8c19bd6 Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Wed, 7 Oct 2026 18:03:52 +0200 Subject: [PATCH 19/48] fix(distro): resolve pocket suites like resolute-security A changelog targeting `-security`, `-updates`, `-proposed` or `-backports` fails the build with an unknown distro codename. Such suites now resolve to their release. --- packages/debmagic-common/src/distro.rs | 32 +++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/packages/debmagic-common/src/distro.rs b/packages/debmagic-common/src/distro.rs index 78200b65..6496a22c 100644 --- a/packages/debmagic-common/src/distro.rs +++ b/packages/debmagic-common/src/distro.rs @@ -161,8 +161,23 @@ static DISTRO_INFO_MAP: LazyLock> = LazyLoc /// Alias targets are maintained manually when Debian/Ubuntu roll. /// Non-built-in suites are not returned here; callers resolve those via Driver /// `base_images` or Bare `/etc/os-release` checks. +/// +/// A pocket suffix (`resolute-security`, `bookworm-backports`, ...) resolves to +/// its release: the build environment is the same. pub fn get_distro_version(name: &str) -> Option { - DISTRO_INFO_MAP.get(name).cloned() + DISTRO_INFO_MAP + .get(name) + .or_else(|| DISTRO_INFO_MAP.get(strip_pocket(name)?)) + .cloned() +} + +/// Pocket suffixes of Debian and Ubuntu suites +const POCKETS: &[&str] = &["security", "updates", "proposed", "backports"]; + +/// The release part of a `-` suite name +fn strip_pocket(name: &str) -> Option<&str> { + let (release, pocket) = name.rsplit_once('-')?; + POCKETS.contains(&pocket).then_some(release) } #[cfg(test)] @@ -176,6 +191,21 @@ mod tests { assert_eq!(serde_json::to_string(&d).unwrap(), "\"debian\""); } + #[test] + fn pocket_suffix_resolves_to_release() { + let resolute = get_distro_version("resolute"); + assert!(resolute.is_some()); + for suite in ["resolute-security", "resolute-updates", "resolute-proposed"] { + assert_eq!(get_distro_version(suite), resolute, "{suite}"); + } + assert_eq!( + get_distro_version("bookworm-backports"), + get_distro_version("bookworm") + ); + assert_eq!(get_distro_version("resolute-nope"), None); + assert_eq!(get_distro_version("yocto-security"), None); + } + #[test] fn distro_serde_custom_is_plain_string() { let d = Distro::Custom("yocto".into()); From d3d5e10c31b38d80a10d9b14a66d929275e20e5f Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Wed, 7 Oct 2026 18:12:58 +0200 Subject: [PATCH 20/48] feat(build): notify when a build finishes `--notify[=no]` (config `notify`) sends a desktop notification via notify-send and rings the terminal bell when `debmagic build` succeeds or fails, so a long build in a background window isn't forgotten. --- docs/usage/build.md | 1 + docs/usage/config.md | 3 ++- packages/debmagic/src/build/mod.rs | 24 +++++++++++++++++++++--- packages/debmagic/src/build_intent.rs | 5 +++++ packages/debmagic/src/cli.rs | 9 +++++++++ packages/debmagic/src/config.rs | 4 ++++ packages/debmagic/src/main.rs | 1 + packages/debmagic/src/output.rs | 2 +- 8 files changed, 44 insertions(+), 5 deletions(-) diff --git a/docs/usage/build.md b/docs/usage/build.md index d3ecd98e..1b5e626c 100644 --- a/docs/usage/build.md +++ b/docs/usage/build.md @@ -30,6 +30,7 @@ debmagic build binary --driver lxd \ | `--clean` | Run [`debian/rules clean` before building](#cleaning) | | `--debug-symbols` | [Build the automatic `-dbgsym` debug symbol packages](#building-debug-symbol-packages) | | `--test` | [Run the package's test suite](#running-tests) | +| `--notify` | Desktop notification (`notify-send`) and terminal bell when the build finishes; config `notify` | | `--apt-mirror ` | [Mirror URL](#mirror-selection) | | `--apt-update-age ` | [When a persistent environment re-runs `apt-get update`](#apt-update-age) | | `--source-dir ` | Directory containing the `debian/` package directory | diff --git a/docs/usage/config.md b/docs/usage/config.md index 5a6c2b46..33a5da96 100644 --- a/docs/usage/config.md +++ b/docs/usage/config.md @@ -26,7 +26,7 @@ All keys are optional. | `driver.persistent` | enum | `"on-failure"` | `--persistent` | How long the Environment outlives the claim: `"no"`, `"on-failure"`, or `"always"`. A bare `--persistent` means `always`. | | `driver.apt_mirror` | string | — | `--apt-mirror` | Mirror used for build-dependency resolution. Not used by the `bare` driver. | | `driver.proposed` | bool | `false` | `--proposed` | Also enable the `-proposed` pocket. Not used by the `bare` driver. | -| `driver.apt_update_age` | string | `"1d"` | `--apt-update-age` | When a persistent environment re-runs `apt-get update`: `"now"` (every build), `"never"` (only on creation), or a maximum index age like `"1d"`, `"12h"`, `"30m"`. Fresh environments always update once. Not used by the `bare` driver. | +| `driver.apt_update_age` | string | `"1d"` | `--apt-update-age` | When a persistent environment re-runs `apt-get update && upgrade`: `"now"` (every build), `"never"` (only on creation), or a maximum age like `"1d"`, `"12h"`, `"30m"`. Not used by the `bare` driver. | | `driver.docker.base_images` | map | — | — | Base image per distro, keyed by `":"` (e.g. `"debian:trixie"`). Falls back to `docker.io/:`. For non-Debian/Ubuntu suites (e.g. `"yocto:kirkstone"`), the map entry is what makes the suite a known DistroVersion for Docker builds. | | `driver.lxd.project` | string | — | — | LXD/Incus project to use. | | `driver.lxd.base_images` | map | — | — | Base image per distro, keyed by `":"`. Falls back to the driver's default remote image. Same custom-suite registry role as Docker's map for LXD/Incus. | @@ -42,6 +42,7 @@ All keys are optional. | `sign.sign_command` | string | — | `--sign-command` | Custom signing command for `sign.tool = "custom"`, run without a shell with `{file}`/`{key}`/`{email}` placeholders; writes the clearsigned result to stdout. | | `sign.notify` | bool | `false` | `--sign-notify` | Send a desktop notification via `notify-send` just before signing, so a hardware-key touch prompt isn't missed. | | `clean` | bool | `false` | `--clean` | Run `debian/rules clean` before building. Disabled by default; incompatible with `incremental`. | +| `notify` | bool | `false` | `--notify` | Send a desktop notification via `notify-send` and ring the terminal bell when a build finishes. | | `host_arch_variant` | string | — | `--host-arch-variant` | Build for a dpkg architecture variant (e.g. `"amd64v3"` on Ubuntu) -> `DEB_HOST_ARCH_VARIANT`. | | `upstream.signature_check` | bool | `true` | `--signature-check` | Verify the upstream tarball's PGP signature against `debian/upstream/signing-key.asc` during `upstream fetch`/`switch` (see [Upstream](upstream.md#signature-verification)). | | `upstream.repack` | bool | `true` | `--repack` | Strip the `Files-Excluded` paths from upstream and custom tarballs during `upstream fetch`/`switch` (see [Upstream](upstream.md#repacking)). | diff --git a/packages/debmagic/src/build/mod.rs b/packages/debmagic/src/build/mod.rs index 7605e897..f666bff3 100644 --- a/packages/debmagic/src/build/mod.rs +++ b/packages/debmagic/src/build/mod.rs @@ -466,7 +466,7 @@ pub struct BuildCommand { /// build the binary or source package, and upload the result when /// `--upload` was passed. pub async fn build(command: BuildCommand) -> anyhow::Result<()> { - let intent = resolve_build_intent(command.intent)?; + let intent = resolve_build_intent(command.intent.clone())?; let target = crate::package::resolve_package_target( &intent.source_dir, command.distro.as_deref(), @@ -478,13 +478,31 @@ pub async fn build(command: BuildCommand) -> anyhow::Result<()> { ) .context("failed to determine package target")?; + let package = format!("{} {}", target.package.name(), target.package.version()); + let notify = intent.config.notify; + let result = build_and_upload(&intent, target, &command).await; + if notify { + let summary = match &result { + Ok(()) => "debmagic: build finished", + Err(_) => "debmagic: build failed", + }; + crate::output::notify_send_bell(summary, &package); + } + result +} + +async fn build_and_upload( + intent: &BuildIntent, + target: PackageTarget, + command: &BuildCommand, +) -> anyhow::Result<()> { let upstream_version = target.package.version().upstream_version().to_string(); let changes_file = if intent.kind.is_source() { let include_orig = match command.include_orig { Some(mode) => crate::upload::orig::decide_orig_upload(mode, &intent.source_dir)?, None => true, }; - build_source_package(&intent, &target, &command.changes_options, include_orig) + build_source_package(intent, &target, &command.changes_options, include_orig) .await .context("Building the source package failed")? } else { @@ -496,7 +514,7 @@ pub async fn build(command: BuildCommand) -> anyhow::Result<()> { ) .context("host's /etc/os-release does not match the build target distro")?; } - build_binary_package(&intent, &target, &command.changes_options) + build_binary_package(intent, &target, &command.changes_options) .await .context("Building the package failed")? }; diff --git a/packages/debmagic/src/build_intent.rs b/packages/debmagic/src/build_intent.rs index f0a1fa5b..6bec5933 100644 --- a/packages/debmagic/src/build_intent.rs +++ b/packages/debmagic/src/build_intent.rs @@ -41,6 +41,7 @@ pub struct BuildIntentInput { pub sign_tool: Option, pub sign_command: Option, pub sign_notify: Option, + pub notify: Option, pub clean: Option, pub source_sync: Option, pub host_arch_variant: Option, @@ -106,6 +107,9 @@ pub fn resolve_build_intent(input: BuildIntentInput) -> anyhow::Result, + #[arg( + long, + num_args = 0..=1, + default_missing_value = "true", + value_parser = BoolishValueParser::new(), + help = "Send a desktop notification via notify-send and ring the terminal bell when the build finishes, successfully or not. Defaults to the 'notify' setting in the config file (false if unset)." + )] + pub notify: Option, + #[arg( long, num_args = 0..=1, diff --git a/packages/debmagic/src/config.rs b/packages/debmagic/src/config.rs index 8c7b8e34..fc3ee3c4 100644 --- a/packages/debmagic/src/config.rs +++ b/packages/debmagic/src/config.rs @@ -247,6 +247,9 @@ pub struct Config { /// builds already stage a clean source tree and incremental builds preserve /// outputs intentionally. pub clean: bool, + /// Send a desktop notification via `notify-send` and ring the terminal + /// bell when a build finishes. + pub notify: bool, /// Build for a dpkg architecture variant (e.g. `amd64v3` on Ubuntu), /// exported as `DEB_HOST_ARCH_VARIANT` for the build. pub host_arch_variant: Option, @@ -314,6 +317,7 @@ impl Default for Config { orig_tarball: OrigTarballConfig::default(), upstream: UpstreamConfig::default(), clean: false, + notify: false, host_arch_variant: None, } } diff --git a/packages/debmagic/src/main.rs b/packages/debmagic/src/main.rs index 5332d13e..591bae19 100644 --- a/packages/debmagic/src/main.rs +++ b/packages/debmagic/src/main.rs @@ -104,6 +104,7 @@ async fn run() -> anyhow::Result { sign_tool: build_args.sign_tool, sign_command: build_args.sign_command.clone(), sign_notify: build_args.sign_notify, + notify: build_args.notify, clean: build_args.clean, source_sync: build_args.source_sync, host_arch_variant: build_args.host_arch_variant.clone(), diff --git a/packages/debmagic/src/output.rs b/packages/debmagic/src/output.rs index a2521159..d385030e 100644 --- a/packages/debmagic/src/output.rs +++ b/packages/debmagic/src/output.rs @@ -70,7 +70,7 @@ pub fn notify_send(summary: &str, body: &str) { { Ok(_) => {} Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - eprintln!("notify-send not found on PATH; cannot send signing notification"); + eprintln!("notify-send not found on PATH; cannot send notification"); } Err(e) => eprintln!("failed to run notify-send: {e}"), } From a9948dde539075c145e41e50b7dfcff2e0d7b5f9 Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Wed, 7 Oct 2026 20:41:10 +0200 Subject: [PATCH 21/48] feat(env): upgrade packages in build environments Fresh container environments only ran `apt-get update`, so a cached Docker image or a long-lived Environment built against outdated packages. They now also run `apt-get dist-upgrade`, and `--apt-update-age` (config `driver.apt_update_age`) decides when a reused Environment upgrades again. The time of the last upgrade is kept in the Environment registry instead of a stamp file inside the container. --- docs/usage/build.md | 13 ++-- packages/debmagic/src/cli.rs | 4 +- packages/debmagic/src/driver/config.rs | 4 +- packages/debmagic/src/driver/driver_docker.rs | 24 +------ packages/debmagic/src/driver/driver_lxd.rs | 22 ++---- packages/debmagic/src/driver/mod.rs | 70 ++++++------------- packages/debmagic/src/environment/mod.rs | 41 ++++++++++- packages/debmagic/src/environment/registry.rs | 47 +++++++++++++ .../debmagic/src/migrations/0001_registry.sql | 4 +- packages/debmagic/src/time.rs | 28 +------- 10 files changed, 133 insertions(+), 124 deletions(-) diff --git a/docs/usage/build.md b/docs/usage/build.md index 1b5e626c..0c0e8a33 100644 --- a/docs/usage/build.md +++ b/docs/usage/build.md @@ -32,7 +32,7 @@ debmagic build binary --driver lxd \ | `--test` | [Run the package's test suite](#running-tests) | | `--notify` | Desktop notification (`notify-send`) and terminal bell when the build finishes; config `notify` | | `--apt-mirror ` | [Mirror URL](#mirror-selection) | -| `--apt-update-age ` | [When a persistent environment re-runs `apt-get update`](#apt-update-age) | +| `--apt-update-age ` | [When a persistent environment re-runs `apt-get update` and `upgrade`](#apt-update-age) | | `--source-dir ` | Directory containing the `debian/` package directory | | `--output-dir ` | Directory to put the resulting build artifacts | @@ -74,14 +74,15 @@ You can persistently set this flag in `.config/debmagic/config.toml`. ## Apt update age -Fresh environments always run `apt-get update` once on creation. -When an Environment is reused, `--apt-update-age` decides whether the apt index is refreshed again: +When an Environment is reused, `--apt-update-age` decides when `apt-get update/upgrade` are run again: | Value | Behavior | |---|---| -| `now` | Update before every build | -| `never` | Only the initial update on creation | -| `1d` (default), `12h`, `30m`, … | Update again once the last one is older than this | +| `now` | Upgrade before every build | +| `never` | Only the initial upgrade on creation | +| `1d` (default), `12h`, `30m`, … | Upgrade again once the last one is older than this | + +The bare driver builds on the host and never upgrades it. The default `1d` matches a typical developer machine's daily apt refresh: repeated builds stay fast, while the index can't go arbitrarily stale. Persist the setting as `apt_update_age = "1d"` in [`debmagic.toml`](config.md). diff --git a/packages/debmagic/src/cli.rs b/packages/debmagic/src/cli.rs index bdfb87d2..61813a56 100644 --- a/packages/debmagic/src/cli.rs +++ b/packages/debmagic/src/cli.rs @@ -360,7 +360,7 @@ pub struct CommonBuildArgs { #[arg( long = "apt-update-age", - help = "When a persistent build environment runs 'apt-get update' again: 'now' (every build), 'never' (only on first creation), or a maximum age of the apt index like '1d' (the default), '12h', '30m'. Fresh environments always update once. Defaults to the 'apt_update_age' setting in the config file. Ignored by the bare driver." + help = "When a persistent build environment runs 'apt-get update/upgrade' again: 'now' (every build), 'never' (only on first creation), or a maximum age of the last upgrade like '1d' (the default), '12h', '30m'. Defaults to the 'apt_update_age' setting in the config file. Ignored by the bare driver." )] pub apt_update_age: Option, @@ -589,7 +589,7 @@ pub struct TestSubcommandArgs { #[arg( long = "apt-update-age", - help = "When a persistent test environment runs 'apt-get update' again: 'now' (every run), 'never' (only on first creation), or a maximum age of the apt index like '1d' (the default), '12h', '30m'. Fresh environments always update once. Defaults to the 'apt_update_age' setting in the config file. Ignored by the bare driver." + help = "When a persistent build environment runs 'apt-get update/upgrade' again: 'now' (every build), 'never' (only on first creation), or a maximum age of the last upgrade like '1d' (the default), '12h', '30m'. Defaults to the 'apt_update_age' setting in the config file. Ignored by the bare driver." )] pub apt_update_age: Option, diff --git a/packages/debmagic/src/driver/config.rs b/packages/debmagic/src/driver/config.rs index 93bf145f..b08a4f8b 100644 --- a/packages/debmagic/src/driver/config.rs +++ b/packages/debmagic/src/driver/config.rs @@ -61,9 +61,7 @@ pub struct DriverConfig { /// Also enable the `-proposed` pocket. Not used by the bare /// driver, which builds on the host's own sources. pub proposed: bool, - /// How old the apt index in a persistent environment may get before - /// `apt-get update` runs again; fresh environments always update once. - /// Not used by the bare driver, which builds on the host's own sources. + /// When to run apt-get update/upgrade again. pub apt_update_age: RefreshPolicy, pub docker: DriverDockerConfig, pub bare: DriverBareConfig, diff --git a/packages/debmagic/src/driver/driver_docker.rs b/packages/debmagic/src/driver/driver_docker.rs index 41e9017a..766481ac 100644 --- a/packages/debmagic/src/driver/driver_docker.rs +++ b/packages/debmagic/src/driver/driver_docker.rs @@ -12,8 +12,8 @@ use serde::{Deserialize, Serialize}; use crate::driver::{ APT_MIRROR_SCRIPT, DriverType, ENVIRONMENT_DIR_IN_CONTAINER, Environment, EnvironmentDriver, EnvironmentMetadata, IsolationCapability, ResourceStatus, config::DriverConfig, - container_name_from_metadata, container_name_metadata, environment_fingerprint, - refresh_apt_index, resource_name, run_checked, translate_path_in_container, + container_name_from_metadata, container_name_metadata, environment_fingerprint, resource_name, + run_checked, translate_path_in_container, }; use crate::subprocess::{self, Capture, CommandResult}; @@ -307,9 +307,8 @@ impl DriverDocker { } } - let created_container = if reuse { + if reuse { driver.reused_environment = true; - false } else { // The container may not exist; removal errors don't matter here. let _ = Command::new("docker") @@ -346,23 +345,6 @@ impl DriverDocker { .arg(&docker_image_name), "starting docker container", )?; - true - }; - - // cwd is the build root (the bind mount itself), not the source dir: - // create() must not assume the source tree has been staged yet. - // Fresh containers always update once; reused ones only when the - // configured apt update age says their index has gone stale. - let update_result = - refresh_apt_index(&driver, environment, driver_config.apt_update_age, reuse) - .map_err(|error| anyhow!("Error running apt-get update in container: {error}")); - if let Err(error) = update_result { - if created_container && let Err(cleanup_error) = driver.container_remove_force() { - return Err(error.context(format!( - "also failed to remove Docker container: {cleanup_error}" - ))); - } - return Err(error); } Ok(driver) diff --git a/packages/debmagic/src/driver/driver_lxd.rs b/packages/debmagic/src/driver/driver_lxd.rs index 9ec62e27..5bee551e 100644 --- a/packages/debmagic/src/driver/driver_lxd.rs +++ b/packages/debmagic/src/driver/driver_lxd.rs @@ -11,8 +11,8 @@ use serde::{Deserialize, Serialize}; use crate::driver::{ APT_MIRROR_SCRIPT, DriverType, ENVIRONMENT_DIR_IN_CONTAINER, Environment, EnvironmentDriver, EnvironmentMetadata, IsolationCapability, ResourceStatus, config::DriverConfig, - container_name_from_metadata, container_name_metadata, environment_fingerprint, - refresh_apt_index, resource_name, run_checked, translate_path_in_container, + container_name_from_metadata, container_name_metadata, environment_fingerprint, resource_name, + run_checked, translate_path_in_container, }; use crate::subprocess::{self, Capture, CommandResult}; @@ -417,20 +417,12 @@ impl DriverLxd { } } - // Fresh containers always update once; reused ones only when the - // configured apt update age says their index has gone stale — a - // previous invocation that crashed before finishing this setup (or - // a long-lived incremental container with an aging package cache) - // must not leave `apt-get build-dep` unable to resolve anything. - refresh_apt_index( - &base, - environment, - driver_config.apt_update_age, - reusing_container, - ) - .map_err(|e| anyhow::anyhow!("Error running apt-get update in container: {e}"))?; - if !reusing_container { + // stock images ship without an apt index + base.exec_in_container_checked(&["apt-get", "update"], None, true, &[]) + .map_err(|e| { + anyhow::anyhow!("Error running apt-get update in container: {e}") + })?; // Install the base tooling that stock images don't include. // build-essential is intentionally omitted: binary builds pull // it in explicitly, so source-only environments stay lean. diff --git a/packages/debmagic/src/driver/mod.rs b/packages/debmagic/src/driver/mod.rs index ff759251..03962bd2 100644 --- a/packages/debmagic/src/driver/mod.rs +++ b/packages/debmagic/src/driver/mod.rs @@ -18,7 +18,7 @@ use crate::driver::{ driver_lxd::{DriverLxd, LxdVariant}, }; use crate::subprocess::{Capture, CommandResult}; -use crate::time::{RefreshPolicy, iso_timestamp, parse_iso_timestamp}; +use crate::time::RefreshPolicy; pub mod config; pub mod driver_bare; @@ -104,49 +104,31 @@ pub fn environment_fingerprint(parts: &[&str]) -> String { .to_string() } -/// Stamp file in a container environment's own filesystem (not the -/// bind-mounted build root, which `reset_root` wipes) holding the ISO -/// timestamp of the last successful `apt-get update` there. -pub const APT_UPDATE_STAMP: &str = "/var/lib/debmagic/apt-updated"; - -/// Bring a reused environment's apt index up to date under `policy`: check -/// the stamp file for when `apt-get update` last ran, and when the policy -/// says it has gone stale (or the stamp is missing), run the update and -/// record a fresh timestamp. Fresh environments (`reused = false`) always -/// update once. -pub fn refresh_apt_index( - driver: &dyn EnvironmentDriver, - environment: &Environment, - policy: RefreshPolicy, - reused: bool, -) -> io::Result<()> { - let last_update = if reused { - driver - .run_command( - &["cat", APT_UPDATE_STAMP], - &environment.root_dir, - true, - &[], - Capture::STDOUT, - ) - .ok() - .filter(|result| result.exit_code == 0) - .and_then(|result| result.stdout) - .and_then(|stdout| parse_iso_timestamp(&stdout)) - } else { - None - }; - if reused && !apt_update_needed_on_reuse(policy, last_update) { - return Ok(()); - } - driver.run_command_checked(&["apt-get", "update"], &environment.root_dir, true, &[])?; - let stamp = apt_update_stamp_script(&iso_timestamp()); - driver.run_command_checked(&["sh", "-ec", &stamp], &environment.root_dir, true, &[]) +/// Refresh the apt index and upgrade the installed packages, so a cached +/// image or a long-lived Environment builds against current packages, like +/// sbuild does. +pub fn upgrade_packages(driver: &dyn EnvironmentDriver, cwd: &Path) -> io::Result<()> { + let env = [("DEBIAN_FRONTEND", "noninteractive")]; + driver.run_command_checked(&["apt-get", "update"], cwd, true, &env)?; + driver.run_command_checked( + &[ + "apt-get", + "-y", + "-o", + "Dpkg::Options::=--force-confdef", + "-o", + "Dpkg::Options::=--force-confold", + "upgrade", + ], + cwd, + true, + &env, + ) } -/// Whether a reused environment needs `apt-get update` again under `policy`. +/// Whether a reused environment needs an apt refresh again under `policy`. /// `last_update` is when its apt index was last refreshed; `None` means -/// never, which includes a missing stamp file. +/// never. pub fn apt_update_needed_on_reuse(policy: RefreshPolicy, last_update: Option) -> bool { match policy { RefreshPolicy::Now => true, @@ -157,12 +139,6 @@ pub fn apt_update_needed_on_reuse(policy: RefreshPolicy, last_update: Option String { - format!("mkdir -p /var/lib/debmagic && printf %s '{timestamp}' > {APT_UPDATE_STAMP}") -} - /// Metadata key under which container-based drivers store their container's /// name for later reattachment via `create_driver_from_metadata`. const CONTAINER_NAME_KEY: &str = "container_name"; diff --git a/packages/debmagic/src/environment/mod.rs b/packages/debmagic/src/environment/mod.rs index 4fa5368f..9a915198 100644 --- a/packages/debmagic/src/environment/mod.rs +++ b/packages/debmagic/src/environment/mod.rs @@ -142,13 +142,50 @@ impl<'a> ClaimedEnvironment<'a> { if let Err(error) = record_driver_metadata(registry, &environment, &driver).await { eprintln!("Warning: failed to record Driver metadata: {error:#}"); } - Ok(Self { + let claimed = Self { registry, driver_config, environment, driver, keep_after_failure: false, - }) + }; + let policy = overrides + .apt_update_age + .unwrap_or(driver_config.apt_update_age); + if let Err(error) = claimed.upgrade_packages(policy).await { + return claimed + .abort(error.context("upgrading the packages in the Environment failed")) + .await; + } + Ok(claimed) + } + + /// Upgrade the packages of a container Environment: always when it is + /// fresh, when reused only once `policy` says the last upgrade (recorded + /// in the registry) has gone stale. The Bare driver is the host, which + /// the user maintains. + async fn upgrade_packages(&self, policy: crate::time::RefreshPolicy) -> anyhow::Result<()> { + if self.environment.driver == crate::driver::DriverType::Bare { + return Ok(()); + } + let id = self.environment.id(); + if self.driver.reused_environment() { + let last_update = + registry::apt_updated_at(&mut *self.registry.connection().await?, &id).await?; + if !crate::driver::apt_update_needed_on_reuse(policy, last_update) { + return Ok(()); + } + } + crate::output::step("Upgrading the packages in the Environment"); + crate::driver::upgrade_packages(&self.driver, &self.environment.root_dir)?; + let recorded = async { + let mut conn = self.registry.connection().await?; + registry::set_apt_updated_at(&mut conn, &id, std::time::SystemTime::now()).await + }; + if let Err(error) = recorded.await { + eprintln!("Warning: failed to record the apt upgrade time: {error:#}"); + } + Ok(()) } fn prepare_reused_root( diff --git a/packages/debmagic/src/environment/registry.rs b/packages/debmagic/src/environment/registry.rs index 4e7f8dd3..9b2bf1dd 100644 --- a/packages/debmagic/src/environment/registry.rs +++ b/packages/debmagic/src/environment/registry.rs @@ -578,6 +578,35 @@ pub(super) async fn delete_environment( Ok(()) } +pub(super) async fn apt_updated_at( + conn: &mut SqliteConnection, + id: &str, +) -> anyhow::Result> { + let seconds: Option> = + sqlx::query_scalar("select apt_updated_at from environments where id = ?") + .bind(id) + .fetch_optional(conn) + .await + .context("reading the apt upgrade time failed")?; + Ok(seconds + .flatten() + .map(|seconds| UNIX_EPOCH + Duration::from_secs(seconds as u64))) +} + +pub(super) async fn set_apt_updated_at( + conn: &mut SqliteConnection, + id: &str, + time: SystemTime, +) -> anyhow::Result<()> { + sqlx::query("update environments set apt_updated_at = ? where id = ?") + .bind(unix_time(time)) + .bind(id) + .execute(conn) + .await + .context("writing the apt upgrade time failed")?; + Ok(()) +} + pub(super) async fn attachments( conn: &mut SqliteConnection, environment_id: &str, @@ -1020,6 +1049,24 @@ pub(super) mod tests { Ok(()) } + #[tokio::test] + async fn apt_updated_at_roundtrips() -> anyhow::Result<()> { + let (_dir, registry) = temp_registry().await; + let environment = sample_environment(Path::new("/src/a"), EnvironmentPurpose::Build); + let id = environment.id(); + upsert(®istry, &environment, None).await; + let mut conn = registry.connection().await?; + assert_eq!(apt_updated_at(&mut conn, &id).await?, None); + + let time = UNIX_EPOCH + Duration::from_secs(1_800_000_000); + set_apt_updated_at(&mut conn, &id, time).await?; + assert_eq!(apt_updated_at(&mut conn, &id).await?, Some(time)); + // a re-claim keeps the record + upsert_environment(&mut conn, &environment, None).await?; + assert_eq!(apt_updated_at(&mut conn, &id).await?, Some(time)); + Ok(()) + } + #[tokio::test] async fn open_if_exists_does_not_create_the_registry() -> anyhow::Result<()> { let path = std::env::temp_dir().join(format!( diff --git a/packages/debmagic/src/migrations/0001_registry.sql b/packages/debmagic/src/migrations/0001_registry.sql index 527cefcd..cc2ce326 100644 --- a/packages/debmagic/src/migrations/0001_registry.sql +++ b/packages/debmagic/src/migrations/0001_registry.sql @@ -24,7 +24,9 @@ create table environments ( owner_pid_start integer, destroying integer not null default 0 check (destroying in (0, 1)), -- JSON object from Driver::driver_metadata - driver_metadata text not null default '{}' + driver_metadata text not null default '{}', + -- unix time of the last apt upgrade, null for never + apt_updated_at integer ) strict; -- `debmagic env shell` sessions, see Process diff --git a/packages/debmagic/src/time.rs b/packages/debmagic/src/time.rs index 8e04d15c..2cc9cf1f 100644 --- a/packages/debmagic/src/time.rs +++ b/packages/debmagic/src/time.rs @@ -1,7 +1,6 @@ -use std::time::{Duration, SystemTime}; +use std::time::Duration; use anyhow::anyhow; -use chrono::{DateTime, SecondsFormat, Utc}; use serde::{Deserialize, Serialize}; /// When something cached is refreshed again: before every use, only the first @@ -97,18 +96,6 @@ impl std::str::FromStr for RefreshPolicy { } } -/// UTC timestamp in RFC 3339 with second precision -pub fn iso_timestamp() -> String { - Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true) -} - -/// Parse a timestamp written by [`iso_timestamp`] -pub fn parse_iso_timestamp(value: &str) -> Option { - DateTime::parse_from_rfc3339(value.trim()) - .ok() - .map(|time| time.into()) -} - #[cfg(test)] mod tests { use super::*; @@ -163,17 +150,4 @@ mod tests { "37s" ); } - - #[test] - fn iso_timestamps_roundtrip() { - let parsed = parse_iso_timestamp(&iso_timestamp()).unwrap(); - let age = SystemTime::now().duration_since(parsed).unwrap(); - assert!(age < Duration::from_secs(120)); - } - - #[test] - fn iso_timestamp_parsing_rejects_garbage() { - assert_eq!(parse_iso_timestamp("not a timestamp"), None); - assert_eq!(parse_iso_timestamp(""), None); - } } From d6b06380c4fdcf1623272ad7d6553b2742358816 Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Wed, 7 Oct 2026 20:51:52 +0200 Subject: [PATCH 22/48] fix(build): copy the orig tarball when it can't be hardlinked A source build in a container environment fails with `cannot fstat file ./.orig.tar.*` when the output dir is on another filesystem than the environments dir: the symlink fallback pointed at a host path the container doesn't have. The tarball is now copied instead. --- packages/debmagic/src/build/mod.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/debmagic/src/build/mod.rs b/packages/debmagic/src/build/mod.rs index f666bff3..d4eafe6a 100644 --- a/packages/debmagic/src/build/mod.rs +++ b/packages/debmagic/src/build/mod.rs @@ -317,9 +317,9 @@ fn check_dpkg_buildpackage_available() -> anyhow::Result<()> { ) } -/// Make `source` available at `destination` without copying bytes when -/// avoidable: hardlink (same filesystem), then symlink, then copy as the -/// last resort. dpkg-source only reads the file, so a link is fine. +/// Make `source` available at `destination`: hardlink on the same +/// filesystem, else copy. A symlink would point at a host path that +/// doesn't exist inside a container environment. fn stage_file(source: &Path, destination: &Path) -> anyhow::Result<()> { if destination.exists() { std::fs::remove_file(destination) @@ -328,9 +328,6 @@ fn stage_file(source: &Path, destination: &Path) -> anyhow::Result<()> { if std::fs::hard_link(source, destination).is_ok() { return Ok(()); } - if std::os::unix::fs::symlink(source, destination).is_ok() { - return Ok(()); - } fs::copy(source, destination).map(|_| ()).with_context(|| { format!( "failed to stage {} into the build environment", From e9aad8a9656e9d263262f730f9742d4a474e9e47 Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Thu, 8 Oct 2026 13:24:57 +0200 Subject: [PATCH 23/48] fix(build): stage MUT component origs for source builds A source build of a package with debian/watch components (like mdevctl's vendor/) fails with dpkg-source 'unrepresentable changes to source', since only the main orig tarball was staged. Component origs are now fetched and staged next to it. --- packages/debmagic/src/build/mod.rs | 35 ++++++++++++++++++++++++-- packages/debmagic/src/upstream/orig.rs | 31 +++++++++++++++++++++++ 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/packages/debmagic/src/build/mod.rs b/packages/debmagic/src/build/mod.rs index d4eafe6a..b15705a2 100644 --- a/packages/debmagic/src/build/mod.rs +++ b/packages/debmagic/src/build/mod.rs @@ -374,14 +374,45 @@ pub async fn build_source_package( intent.output_dir.display() ); } + let mut orig_tarballs: Vec = orig_tarball.into_iter().collect(); + if !target.package.is_native() { + // dpkg-source picks up component origs next to the main one; + // without them their subdirs become unrepresentable changes + let components = + crate::upstream::orig::mut_components(&target.package).unwrap_or_else(|error| { + eprintln!("debmagic: warning: ignoring MUT components: {error:#}"); + Vec::new() + }); + for component in components { + let tarball = crate::upstream::orig::fetch_orig_tarball_for_version( + &intent.config.orig_tarball, + &target.package, + &intent.output_dir, + upstream_version, + Some(&component), + ) + .await + .with_context(|| { + format!("fetching the orig tarball of component {component} for {name} {upstream_version} failed") + })? + .with_context(|| { + format!( + "no orig tarball of component {component} for {name} {upstream_version} in {}, the source tree's parent dir or the distro archives; \ + run 'debmagic upstream fetch' or place it in one of those dirs", + intent.output_dir.display() + ) + })?; + orig_tarballs.push(tarball); + } + } let request = BuildRequest { intent, target }; run_build(&request, |claimed| { crate::output::stage("Building source package"); let staged_source_dir = stage_dir(claimed.environment(), BuildKind::Source); - if let Some(tarball) = &orig_tarball { + for tarball in &orig_tarballs { // the work dir is the staged source dir's parent, which is where - // dpkg-source looks for the tarball + // dpkg-source looks for the tarballs let destination = claimed.environment().work_dir().join( tarball .file_name() diff --git a/packages/debmagic/src/upstream/orig.rs b/packages/debmagic/src/upstream/orig.rs index bfd6f157..90c5c3fd 100644 --- a/packages/debmagic/src/upstream/orig.rs +++ b/packages/debmagic/src/upstream/orig.rs @@ -177,6 +177,20 @@ pub async fn fetch_orig_tarball_for_version( .await } +/// The MUT component names `debian/watch` declares; none without a +/// watch file. +pub fn mut_components(package: &SourcePackage) -> anyhow::Result> { + let source_dir = package.source_dir()?; + if !source_dir.join("debian").join("watch").is_file() { + return Ok(Vec::new()); + } + let sources = crate::upstream::query::load_watch(source_dir, package.name())?; + Ok(sources + .into_iter() + .filter_map(|source| source.component) + .collect()) +} + /// Fetch the `orig` tarball for `upstream_version` with an explicit /// `method` into `output_dir`, without looking for local copies first. /// `None` when the method does not have it. @@ -582,6 +596,23 @@ mod tests { std::fs::remove_dir_all(&base).unwrap(); } + #[test] + fn test_mut_components_from_watch() { + let base = std::env::temp_dir().join(format!("debmagic-mut-{}", std::process::id())); + let source = base.join("postfix"); + let pkg = package(&source, None); + assert!(mut_components(&pkg).unwrap().is_empty()); + + std::fs::write( + source.join("debian").join("watch"), + "version=4\nhttps://example.com/ postfix-(.*)\\.tar\\.gz\n\ + opts=component=vendor https://example.com/ vendor-(.*)\\.tar\\.gz same\n", + ) + .unwrap(); + assert_eq!(mut_components(&pkg).unwrap(), ["vendor"]); + std::fs::remove_dir_all(&base).unwrap(); + } + #[tokio::test] async fn test_fetch_component_uses_component_tarball() { let base = std::env::temp_dir().join(format!("debmagic-orig-comp-{}", std::process::id())); From c4a6c2a0d8601062cbaceb83565d7f5d39c43a57 Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Thu, 8 Oct 2026 14:19:38 +0200 Subject: [PATCH 24/48] fix(upload): let dput-ng compat shim see ppa uploads not as main archive uploads --- packages/debmagic/src/build/mod.rs | 2 +- packages/debmagic/src/upload.rs | 4 ++-- scripts/dput_ng_hooks.py | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/debmagic/src/build/mod.rs b/packages/debmagic/src/build/mod.rs index b15705a2..279bc0c1 100644 --- a/packages/debmagic/src/build/mod.rs +++ b/packages/debmagic/src/build/mod.rs @@ -549,7 +549,7 @@ async fn build_and_upload( if let Some(spec) = &command.upload { let upload_target = crate::upload::resolve_target(spec, Some(&intent.config.upload))?; - crate::output::stage(&format!("Uploading to {}", upload_target.name)); + crate::output::stage(&format!("Uploading to {spec}")); crate::upload::upload_changes( &upload_target, spec, diff --git a/packages/debmagic/src/upload.rs b/packages/debmagic/src/upload.rs index ff064823..4dc97ebb 100644 --- a/packages/debmagic/src/upload.rs +++ b/packages/debmagic/src/upload.rs @@ -142,7 +142,7 @@ pub fn upload(input: UploadIntent) -> anyhow::Result<()> { crate::output::stage(&format!( "Uploading {} to {}", changes_file.display(), - upload_target.name + input.target )); upload_changes( &upload_target, @@ -249,7 +249,7 @@ pub fn upload_changes( format!( "transferring {} files to {} failed", files.len(), - target.name + target_spec ) })?; log.record(changes_file, target_spec, upstream_version)?; diff --git a/scripts/dput_ng_hooks.py b/scripts/dput_ng_hooks.py index e13360b6..875a0e02 100644 --- a/scripts/dput_ng_hooks.py +++ b/scripts/dput_ng_hooks.py @@ -272,7 +272,8 @@ def run_shim(changes_path: Path, target: str, config_dirs: list[Path]) -> int: changes = Changes(changes_path) profile = { - "name": target, + # dput-ng names the profile after the block ("ppa"), not the full "ppa:user/repo" spec + "name": target.split(":", 1)[0], "incoming": os.environ.get("DEBMAGIC_UPLOAD_TARGET_INCOMING", ""), "fqdn": os.environ.get("DEBMAGIC_UPLOAD_TARGET_SERVER", ""), } From c4c2c8d178ee7b20a41fe328e829b286ed15ded4 Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Thu, 8 Oct 2026 16:14:54 +0200 Subject: [PATCH 25/48] refactor(cli): share identical flags between commands --- packages/debmagic/src/build_intent.rs | 25 +-- packages/debmagic/src/cli.rs | 230 ++++++++++---------------- packages/debmagic/src/main.rs | 18 +- packages/debmagic/src/sign.rs | 45 +++-- 4 files changed, 127 insertions(+), 191 deletions(-) diff --git a/packages/debmagic/src/build_intent.rs b/packages/debmagic/src/build_intent.rs index 6bec5933..ba07d1dc 100644 --- a/packages/debmagic/src/build_intent.rs +++ b/packages/debmagic/src/build_intent.rs @@ -6,7 +6,7 @@ use crate::{ build::source::SourceSyncMode, config::Config, driver::{DriverType, Persistence, config::DriverOverrides}, - sign::{SignMode, SignTool}, + sign::{SignMode, SignOverrides}, }; /// Which kind of package a build produces: a source-only upload or @@ -37,10 +37,7 @@ pub struct BuildIntentInput { pub debug_symbols: Option, pub test: Option, pub sign: Option, - pub sign_key: Option, - pub sign_tool: Option, - pub sign_command: Option, - pub sign_notify: Option, + pub sign_overrides: SignOverrides, pub notify: Option, pub clean: Option, pub source_sync: Option, @@ -95,18 +92,7 @@ pub fn resolve_build_intent(input: BuildIntentInput) -> anyhow::Result, } -/// Flags shared between `debmagic build binary` and `debmagic build source`. +/// Flags shared by the commands that run in a driver environment +/// (`debmagic build`, `debmagic test`). #[derive(Args, Debug)] -pub struct CommonBuildArgs { +pub struct EnvironmentArgs { #[arg( - short, long, num_args = 0..=1, - default_missing_value = "true", + default_missing_value = "always", action = clap::ArgAction::Set, - value_parser = BoolishValueParser::new(), - help = "Synchronize changed source inputs to preserve build outputs. Forces --persistent=always" + help = "How long the environment outlives this run: 'on-failure' (the default) keeps it when the command fails, 'always' keeps it after every run, 'no' tears it down. A bare --persistent means 'always'. A failed run that keeps the environment prints `debmagic env shell `." )] - pub incremental: Option, + pub persistent: Option, + + #[command(flatten)] + pub docker: DockerArgs, + + #[command(flatten)] + pub lxd: LxdArgs, #[arg( - short, - long, - help = "Build driver type. Defaults to the 'driver' key in debmagic.toml; without either, source-only builds use 'bare', since those need no build-deps or compilation." + long = "apt-mirror", + help = "Apt mirror URL to use inside the environment instead of the default archive.ubuntu.com/security.ubuntu.com/deb.debian.org, e.g. http://my-mirror.example/ubuntu. Ignored by the bare driver." )] - pub driver: Option, + pub apt_mirror: Option, #[arg( long, num_args = 0..=1, - default_missing_value = "always", + default_missing_value = "true", action = clap::ArgAction::Set, - help = "How long the build environment outlives this build: 'on-failure' (the default) keeps it when the build command fails, 'always' keeps it after every build, 'no' tears it down. A bare --persistent means 'always'. A failed build that keeps the environment prints `debmagic env shell `." + value_parser = BoolishValueParser::new(), + help = "Also enable the '-proposed' pocket in the environment. Ignored by the bare driver." )] - pub persistent: Option, + pub proposed: Option, #[arg( - long = "source-sync", - help = "Which source files are staged into the build tree: 'tracked' stages git-tracked files including uncommitted changes and warns about untracked files (default), 'committed' additionally fails if the worktree is dirty, 'worktree' stages everything that is not git-ignored. Defaults to the 'source_sync_mode' setting in the config file." + long = "apt-update-age", + help = "When a persistent environment runs 'apt-get update/upgrade' again: 'now' (every run), 'never' (only on first creation), or a maximum age of the last upgrade like '1d' (the default), '12h', '30m'. Defaults to the 'apt_update_age' setting in the config file. Ignored by the bare driver." )] - pub source_sync: Option, + pub apt_update_age: Option, +} - #[command(flatten)] - pub docker: DockerArgs, +/// Signing flags shared by `debmagic build` and `debmagic sign`. +#[derive(Args, Debug)] +pub struct SignArgs { + #[arg( + long = "sign-key", + help = "GPG key ID/email to sign with. Defaults to the 'sign.key' setting in the config file, or the Changed-By/Maintainer address of the file being signed if unset." + )] + pub sign_key: Option, - #[command(flatten)] - pub lxd: LxdArgs, + #[arg( + long = "sign-tool", + value_enum, + help = "OpenPGP implementation to sign with: 'gpg' (default), 'sequoia' (sq), or 'custom' (uses --sign-command). Defaults to the 'sign.tool' setting in the config file." + )] + pub sign_tool: Option, #[arg( - long = "apt-mirror", - help = "Apt mirror URL to use inside the build environment instead of the default archive.ubuntu.com/security.ubuntu.com/deb.debian.org, e.g. http://my-mirror.example/ubuntu. Ignored by the bare driver." + long = "sign-command", + help = "Custom signing command for --sign-tool custom, run without a shell. Supports {file}, {key} and {email} placeholders; writes the clearsigned result to stdout. Defaults to the 'sign.sign_command' setting in the config file." )] - pub apt_mirror: Option, + pub sign_command: Option, + + #[arg( + long = "sign-notify", + num_args = 0..=1, + default_missing_value = "true", + value_parser = BoolishValueParser::new(), + help = "Send a desktop notification via notify-send just before signing, so a hardware-key touch prompt isn't missed. Defaults to the 'sign.notify' setting in the config file (false if unset)." + )] + pub sign_notify: Option, +} + +impl SignArgs { + pub fn overrides(&self) -> SignOverrides { + SignOverrides { + key: self.sign_key.clone(), + tool: self.sign_tool, + command: self.sign_command.clone(), + notify: self.sign_notify, + } + } +} +/// Flags shared between `debmagic build binary` and `debmagic build source`. +#[derive(Args, Debug)] +pub struct CommonBuildArgs { #[arg( + short, long, num_args = 0..=1, default_missing_value = "true", action = clap::ArgAction::Set, value_parser = BoolishValueParser::new(), - help = "Also enable the '-proposed' pocket in the build environment. Ignored by the bare driver." + help = "Synchronize changed source inputs to preserve build outputs. Forces --persistent=always" )] - pub proposed: Option, + pub incremental: Option, #[arg( - long = "apt-update-age", - help = "When a persistent build environment runs 'apt-get update/upgrade' again: 'now' (every build), 'never' (only on first creation), or a maximum age of the last upgrade like '1d' (the default), '12h', '30m'. Defaults to the 'apt_update_age' setting in the config file. Ignored by the bare driver." + short, + long, + help = "Build driver type. Defaults to the 'driver' key in debmagic.toml; without either, source-only builds use 'bare', since those need no build-deps or compilation." )] - pub apt_update_age: Option, + pub driver: Option, + + #[arg( + long = "source-sync", + help = "Which source files are staged into the build tree: 'tracked' stages git-tracked files including uncommitted changes and warns about untracked files (default), 'committed' additionally fails if the worktree is dirty, 'worktree' stages everything that is not git-ignored. Defaults to the 'source_sync_mode' setting in the config file." + )] + pub source_sync: Option, + + #[command(flatten)] + pub environment: EnvironmentArgs, #[arg( long, @@ -391,33 +442,8 @@ pub struct CommonBuildArgs { )] pub sign: Option, - #[arg( - long = "sign-key", - help = "GPG key ID/email to sign with. Defaults to the 'sign.key' setting in the config file, or the Changed-By/Maintainer address of the file being signed if unset." - )] - pub sign_key: Option, - - #[arg( - long = "sign-tool", - value_enum, - help = "OpenPGP implementation to sign with: 'gpg' (default), 'sequoia' (sq), or 'custom' (uses --sign-command). Defaults to the 'sign.tool' setting in the config file." - )] - pub sign_tool: Option, - - #[arg( - long = "sign-command", - help = "Custom signing command for --sign-tool custom, run without a shell. Supports {file}, {key} and {email} placeholders; writes the clearsigned result to stdout. Defaults to the 'sign.sign_command' setting in the config file." - )] - pub sign_command: Option, - - #[arg( - long = "sign-notify", - num_args = 0..=1, - default_missing_value = "true", - value_parser = BoolishValueParser::new(), - help = "Send a desktop notification via notify-send just before signing, so a hardware-key touch prompt isn't missed. Defaults to the 'sign.notify' setting in the config file (false if unset)." - )] - pub sign_notify: Option, + #[command(flatten)] + pub signing: SignArgs, #[arg( long, @@ -556,42 +582,8 @@ pub struct TestSubcommandArgs { )] pub driver: Option, - #[arg( - long, - num_args = 0..=1, - default_missing_value = "always", - action = clap::ArgAction::Set, - help = "How long the test environment outlives this run: 'on-failure' (the default) keeps it when the tests fail, 'always' keeps it after every run, 'no' tears it down. A bare --persistent means 'always'. A failed run that keeps the environment prints `debmagic env shell `." - )] - pub persistent: Option, - - #[command(flatten)] - pub docker: DockerArgs, - #[command(flatten)] - pub lxd: LxdArgs, - - #[arg( - long = "apt-mirror", - help = "Apt mirror URL to use inside the test environment instead of the default archive mirrors. Ignored by the bare driver." - )] - pub apt_mirror: Option, - - #[arg( - long, - num_args = 0..=1, - default_missing_value = "true", - action = clap::ArgAction::Set, - value_parser = BoolishValueParser::new(), - help = "Also enable the '-proposed' pocket in the test environment. Ignored by the bare driver." - )] - pub proposed: Option, - - #[arg( - long = "apt-update-age", - help = "When a persistent build environment runs 'apt-get update/upgrade' again: 'now' (every build), 'never' (only on first creation), or a maximum age of the last upgrade like '1d' (the default), '12h', '30m'. Defaults to the 'apt_update_age' setting in the config file. Ignored by the bare driver." - )] - pub apt_update_age: Option, + pub environment: EnvironmentArgs, #[arg( long, @@ -639,33 +631,8 @@ pub struct SignSubcommandArgs { )] pub mode: SignMode, - #[arg( - long = "sign-key", - help = "GPG key ID/email to sign with. Defaults to the 'sign.key' setting in the config file, or the Changed-By/Maintainer address of the file being signed if unset." - )] - pub sign_key: Option, - - #[arg( - long = "sign-tool", - value_enum, - help = "OpenPGP implementation to sign with: 'gpg' (default), 'sequoia' (sq), or 'custom' (uses --sign-command). Defaults to the 'sign.tool' setting in the config file." - )] - pub sign_tool: Option, - - #[arg( - long = "sign-command", - help = "Custom signing command for --sign-tool custom, run without a shell. Supports {file}, {key} and {email} placeholders; writes the clearsigned result to stdout. Defaults to the 'sign.sign_command' setting in the config file." - )] - pub sign_command: Option, - - #[arg( - long = "sign-notify", - num_args = 0..=1, - default_missing_value = "true", - value_parser = BoolishValueParser::new(), - help = "Send a desktop notification via notify-send just before signing, so a hardware-key touch prompt isn't missed. Defaults to the 'sign.notify' setting in the config file (false if unset)." - )] - pub sign_notify: Option, + #[command(flatten)] + pub signing: SignArgs, #[arg( short, @@ -758,26 +725,7 @@ pub struct UploadSubcommandArgs { pub changes: Option, } -impl CommonBuildArgs { - /// Collect the driver-specific overrides from the CLI flags. - pub fn driver_overrides(&self) -> DriverOverrides { - DriverOverrides { - apt_mirror: self.apt_mirror.clone(), - proposed: self.proposed, - apt_update_age: self.apt_update_age, - docker: DriverDockerConfigOverrides { - base_image: self.docker.base_image.clone(), - }, - bare: DriverBareConfigOverrides {}, - lxd: DriverLxdConfigOverrides { - base_image: self.lxd.base_image.clone(), - project: self.lxd.project.clone(), - }, - } - } -} - -impl TestSubcommandArgs { +impl EnvironmentArgs { /// Collect the driver-specific overrides from the CLI flags. pub fn driver_overrides(&self) -> DriverOverrides { DriverOverrides { @@ -817,7 +765,7 @@ mod tests { #[test_case(&["--persistent=no"], Some(Persistence::No); "no")] fn persistent_flag(args: &[&str], expected: Option) { let args = parse_test(args); - assert_eq!(args.persistent, expected); + assert_eq!(args.environment.persistent, expected); } #[test_case(&[], None; "absent")] @@ -827,7 +775,7 @@ mod tests { #[test_case(&["--proposed=off"], Some(false); "off")] #[test_case(&["--proposed=true"], Some(true); "literal true")] fn boolish_flag(args: &[&str], expected: Option) { - assert_eq!(parse_test(args).proposed, expected); + assert_eq!(parse_test(args).environment.proposed, expected); } #[test] diff --git a/packages/debmagic/src/main.rs b/packages/debmagic/src/main.rs index 591bae19..0f07f0ee 100644 --- a/packages/debmagic/src/main.rs +++ b/packages/debmagic/src/main.rs @@ -95,20 +95,17 @@ async fn run() -> anyhow::Result { config_file: cli.config.clone(), driver: build_args.driver, kind, - persistent: build_args.persistent, + persistent: build_args.environment.persistent, incremental: build_args.incremental, debug_symbols, test: run_test, sign: build_args.sign, - sign_key: build_args.sign_key.clone(), - sign_tool: build_args.sign_tool, - sign_command: build_args.sign_command.clone(), - sign_notify: build_args.sign_notify, + sign_overrides: build_args.signing.overrides(), notify: build_args.notify, clean: build_args.clean, source_sync: build_args.source_sync, host_arch_variant: build_args.host_arch_variant.clone(), - driver_overrides: build_args.driver_overrides(), + driver_overrides: build_args.environment.driver_overrides(), }, distro: build_args.distro.clone(), include_orig, @@ -161,12 +158,12 @@ async fn run() -> anyhow::Result { )?, config_file: cli.config.clone(), driver: args.driver, - persistent: args.persistent, + persistent: args.environment.persistent, strict: args.strict, changes: args.changes.clone(), allow_host_test: args.allow_host_test, distro: args.distro.clone(), - driver_overrides: args.driver_overrides(), + driver_overrides: args.environment.driver_overrides(), })?; let outcome = test::run_test(&intent) @@ -209,10 +206,7 @@ async fn run() -> anyhow::Result { )?, config_file: cli.config.clone(), mode: args.mode, - sign_key: args.sign_key.clone(), - sign_tool: args.sign_tool, - sign_command: args.sign_command.clone(), - sign_notify: args.sign_notify, + overrides: args.signing.overrides(), output_dir: args.output_dir.clone(), file: args.file.clone(), })?; diff --git a/packages/debmagic/src/sign.rs b/packages/debmagic/src/sign.rs index ccac1194..3007ebd4 100644 --- a/packages/debmagic/src/sign.rs +++ b/packages/debmagic/src/sign.rs @@ -433,6 +433,32 @@ fn substitute_verify_placeholders( Ok(out) } +/// CLI overrides of the `[sign]` config section. +#[derive(Debug, Clone, Default)] +pub struct SignOverrides { + pub key: Option, + pub tool: Option, + pub command: Option, + pub notify: Option, +} + +impl SignOverrides { + pub fn apply(self, config: &mut crate::config::SignConfig) { + if let Some(key) = self.key { + config.key = Some(key); + } + if let Some(tool) = self.tool { + config.tool = tool; + } + if let Some(command) = self.command { + config.sign_command = Some(command); + } + if let Some(notify) = self.notify { + config.notify = notify; + } + } +} + /// Sign the `.changes` file (and its `.dsc`/`.buildinfo` children) on the /// host, as resolved from the build's `sign` config. #[derive(Debug, Clone)] @@ -440,10 +466,7 @@ pub struct SignIntent { pub source_dir: PathBuf, pub config_file: Option, pub mode: SignMode, - pub sign_key: Option, - pub sign_tool: Option, - pub sign_command: Option, - pub sign_notify: Option, + pub overrides: SignOverrides, pub output_dir: Option, /// The explicit file to sign; located via changelog + output dir when unset. pub file: Option, @@ -454,19 +477,7 @@ pub struct SignIntent { pub fn sign(input: SignIntent) -> anyhow::Result<()> { let source_dir = input.source_dir; let mut config = crate::config::Config::load(Some(&source_dir), input.config_file.as_deref())?; - - if let Some(key) = &input.sign_key { - config.sign.key = Some(key.clone()); - } - if let Some(tool) = input.sign_tool { - config.sign.tool = tool; - } - if let Some(command) = &input.sign_command { - config.sign.sign_command = Some(command.clone()); - } - if let Some(notify) = input.sign_notify { - config.sign.notify = notify; - } + input.overrides.apply(&mut config.sign); let options = SignConfig { key: config.sign.key.clone(), From 7effced0decde0403018e1f18bd3b80897de5ea4 Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Thu, 8 Oct 2026 16:31:50 +0200 Subject: [PATCH 26/48] feat(notify): global notify default, --notify on test, sign and upload The top-level `notify` now defaults the new `build.notify`, `test.notify` and the now optional `sign.notify`; one `--notify[=no]` overrides all steps of a command and replaces `--sign-notify`. - `debmagic test` notifies when the run finishes - a build that signs notifies once, when signing starts --- docs/usage/build.md | 20 ++++++- docs/usage/config.md | 8 +-- docs/usage/test.md | 1 + packages/debmagic/src/build/mod.rs | 30 +++++++---- packages/debmagic/src/build_intent.rs | 4 +- packages/debmagic/src/cli.rs | 42 ++++++++------- packages/debmagic/src/config.rs | 77 +++++++++++++++++++++++++-- packages/debmagic/src/main.rs | 5 +- packages/debmagic/src/output.rs | 21 ++++++++ packages/debmagic/src/sign.rs | 33 +++++++----- packages/debmagic/src/test/intent.rs | 4 ++ packages/debmagic/src/test/run.rs | 13 ++++- packages/debmagic/src/upload.rs | 16 +++--- 13 files changed, 208 insertions(+), 66 deletions(-) diff --git a/docs/usage/build.md b/docs/usage/build.md index 0c0e8a33..90370c31 100644 --- a/docs/usage/build.md +++ b/docs/usage/build.md @@ -30,7 +30,7 @@ debmagic build binary --driver lxd \ | `--clean` | Run [`debian/rules clean` before building](#cleaning) | | `--debug-symbols` | [Build the automatic `-dbgsym` debug symbol packages](#building-debug-symbol-packages) | | `--test` | [Run the package's test suite](#running-tests) | -| `--notify` | Desktop notification (`notify-send`) and terminal bell when the build finishes; config `notify` | +| `--notify` | [Desktop notification and terminal bell](#notifications) when the build finishes or signing needs a key touch | | `--apt-mirror ` | [Mirror URL](#mirror-selection) | | `--apt-update-age ` | [When a persistent environment re-runs `apt-get update` and `upgrade`](#apt-update-age) | | `--source-dir ` | Directory containing the `debian/` package directory | @@ -196,7 +196,7 @@ The `--sign` mode decides what happens to an existing signature: | `--sign-key ` | `sign.key` | Key ID/fingerprint/email; defaults to the `Changed-By:`/`Maintainer:` address of the file being signed | | `--sign-tool ` | `sign.tool` | OpenPGP implementation: `gpg` (default), `sequoia` (sq), or `custom` | | `--sign-command ` | `sign.sign_command` | Custom signing command for `--sign-tool custom` (see below) | -| `--sign-notify` | `sign.notify` | Desktop notification + terminal bell just before signing, so a hardware-key touch prompt isn't missed after a long build | +| `--notify` | `sign.notify` | Desktop notification + terminal bell just before signing, so a hardware-key touch prompt isn't missed after a long build (see [Notifications](#notifications)) | A custom signing command runs without a shell and must write the clearsigned result to stdout. The file to sign is passed via the `{file}` placeholder (or, if no placeholder is used, as the last argument). @@ -211,6 +211,22 @@ Unknown placeholders are an error. Defaults can be set in [`debmagic.toml`](config.md). +## Notifications + +A desktop notification (`notify-send`) plus a terminal bell tells you when a long build needs you. +The top-level `notify` setting is the default for each step; `build.notify` and `sign.notify` override it per step: + +```toml +notify = true + +[sign] +notify = false # no touch prompt notification, but "build finished" +``` + +A build that signs notifies once, when signing starts: "build finished, signing" if `build.notify` is on, else just the touch prompt if `sign.notify` is. +An unsigned build notifies when it finishes, a failed one in any case. +`--notify[=no]` overrides all steps of one run. + ## Signing an existing build `debmagic sign` signs a `.changes` file (and its `.dsc`/`.buildinfo` children) that already exists — the same code path `--sign` uses after a build: diff --git a/docs/usage/config.md b/docs/usage/config.md index 33a5da96..7b9ec6b7 100644 --- a/docs/usage/config.md +++ b/docs/usage/config.md @@ -40,9 +40,11 @@ All keys are optional. | `sign.key` | string | — | `--sign-key` | GPG key ID/email to sign with; falls back to the Changed-By/Maintainer address. | | `sign.tool` | enum | `gpg` | `--sign-tool` | OpenPGP implementation: `gpg`, `sequoia` (sq) or `custom` (uses `sign.sign_command`). | | `sign.sign_command` | string | — | `--sign-command` | Custom signing command for `sign.tool = "custom"`, run without a shell with `{file}`/`{key}`/`{email}` placeholders; writes the clearsigned result to stdout. | -| `sign.notify` | bool | `false` | `--sign-notify` | Send a desktop notification via `notify-send` just before signing, so a hardware-key touch prompt isn't missed. | +| `sign.notify` | bool | `notify` | `--notify` | Send a desktop notification via `notify-send` just before signing, so a hardware-key touch prompt isn't missed. | | `clean` | bool | `false` | `--clean` | Run `debian/rules clean` before building. Disabled by default; incompatible with `incremental`. | -| `notify` | bool | `false` | `--notify` | Send a desktop notification via `notify-send` and ring the terminal bell when a build finishes. | +| `notify` | bool | `false` | `--notify` | Default for `build.notify`, `test.notify` and `sign.notify`. | +| `build.notify` | bool | `notify` | `--notify` | Desktop notification via `notify-send` and terminal bell when a build finishes; a build that signs notifies once, when signing starts (see [Notifications](build.md#notifications)). | +| `test.notify` | bool | `notify` | `--notify` | Desktop notification via `notify-send` and terminal bell when `debmagic test` finishes. | | `host_arch_variant` | string | — | `--host-arch-variant` | Build for a dpkg architecture variant (e.g. `"amd64v3"` on Ubuntu) -> `DEB_HOST_ARCH_VARIANT`. | | `upstream.signature_check` | bool | `true` | `--signature-check` | Verify the upstream tarball's PGP signature against `debian/upstream/signing-key.asc` during `upstream fetch`/`switch` (see [Upstream](upstream.md#signature-verification)). | | `upstream.repack` | bool | `true` | `--repack` | Strip the `Files-Excluded` paths from upstream and custom tarballs during `upstream fetch`/`switch` (see [Upstream](upstream.md#repacking)). | @@ -63,7 +65,7 @@ All keys are optional. | `source` | enum | `no` | `--sign` | Sign the source package (`.changes`/`.dsc`) after building: `no` never, `keep` keeps any existing signature, `auto` skips what our key already signed, `force` always re-signs. | | `key` | string | — | `--sign-key` | GPG key ID/email to sign with; falls back to the Changed-By/Maintainer address. | | `tool` | enum | `gpg` | `--sign-tool` | OpenPGP implementation: `gpg` (sequoia's `sq`) or `custom` (uses `sign_command`). | -| `notify` | bool | `false` | `--sign-notify` | Desktop notification via `notify-send` before signing. | +| `notify` | bool | top-level `notify` | `--notify` | Desktop notification via `notify-send` before signing. | | `sign_command` | string | — | `--sign-command` | Custom signing command for `tool = "custom"`, like `debsign`'s `-p`. | | `verify_command` | string | — | `--verify-command` (upstream switch) | Custom verification command for `sign.tool = "custom"`, run without a shell with `{file}`/`{signature}`/`{keyring}` placeholders; without `{signature}` the signature path is appended. Falls back to `sign_command`. | | `signed_by_command` | string | — | — | Custom same-key check for `tool = "custom"` under `source = "auto"`: exit code 0 means the file's signature is ours (skip), anything else re-signs. Without it, `auto` always re-signs. | diff --git a/docs/usage/test.md b/docs/usage/test.md index 268fd5dd..c063b93c 100644 --- a/docs/usage/test.md +++ b/docs/usage/test.md @@ -36,6 +36,7 @@ The driver *is* the testbed: `autopkgtest` runs with the `null` backend inside t | `--apt-update-age ` | Configure when to run `apt-get update` (same as [`debmagic build`](build.md#apt-update-age)) | | `--source-dir ` | Directory containing the `debian/` package directory | | `--allow-host-test` | Allow the bare driver, which runs autopkgtest as root on the host | +| `--notify` | Desktop notification and terminal bell when the run finishes | Driver-specific flags (`--driver-docker-base-image`, `--driver-lxd-*`) mirror `debmagic build`. diff --git a/packages/debmagic/src/build/mod.rs b/packages/debmagic/src/build/mod.rs index 279bc0c1..677d9c78 100644 --- a/packages/debmagic/src/build/mod.rs +++ b/packages/debmagic/src/build/mod.rs @@ -176,14 +176,22 @@ async fn run_build( build_kind, )?; if sign.source.is_enabled() { - crate::output::stage(&format!("Signing {}", environment.package_identifier)); - crate::sign::sign_file( - &changes_file, - sign, - sign.source, - sign.notify, - &environment.package_identifier, - )?; + let package = &environment.package_identifier; + crate::output::stage(&format!("Signing {package}")); + // one notification for both: the build is done when + // the key wants a touch + let config = &request.intent.config; + let notice = if config.build_notify() { + Some(crate::output::Notice::new( + "debmagic: build finished, signing", + &format!("touch your key to sign {package}"), + )) + } else { + config + .sign_notify() + .then(|| crate::sign::sign_notice(package)) + }; + crate::sign::sign_file(&changes_file, sign, sign.source, notice.as_ref())?; } Ok(changes_file) }); @@ -507,9 +515,11 @@ pub async fn build(command: BuildCommand) -> anyhow::Result<()> { .context("failed to determine package target")?; let package = format!("{} {}", target.package.name(), target.package.version()); - let notify = intent.config.notify; + let notify = intent.config.build_notify(); let result = build_and_upload(&intent, target, &command).await; - if notify { + // a signed build already notified when signing started + let notified = result.is_ok() && intent.config.sign.source.is_enabled(); + if notify && !notified { let summary = match &result { Ok(()) => "debmagic: build finished", Err(_) => "debmagic: build failed", diff --git a/packages/debmagic/src/build_intent.rs b/packages/debmagic/src/build_intent.rs index ba07d1dc..f2d3327a 100644 --- a/packages/debmagic/src/build_intent.rs +++ b/packages/debmagic/src/build_intent.rs @@ -93,9 +93,7 @@ pub fn resolve_build_intent(input: BuildIntentInput) -> anyhow::Result, - - #[arg( - long = "sign-notify", - num_args = 0..=1, - default_missing_value = "true", - value_parser = BoolishValueParser::new(), - help = "Send a desktop notification via notify-send just before signing, so a hardware-key touch prompt isn't missed. Defaults to the 'sign.notify' setting in the config file (false if unset)." - )] - pub sign_notify: Option, } impl SignArgs { @@ -380,11 +371,23 @@ impl SignArgs { key: self.sign_key.clone(), tool: self.sign_tool, command: self.sign_command.clone(), - notify: self.sign_notify, } } } +/// `--notify`, shared by the commands that take long or need attention. +#[derive(Args, Debug)] +pub struct NotifyArgs { + #[arg( + long, + num_args = 0..=1, + default_missing_value = "true", + value_parser = BoolishValueParser::new(), + help = "Send a desktop notification via notify-send and ring the terminal bell when the command finishes or signing needs a key touch, overriding the 'notify', 'build.notify', 'test.notify' and 'sign.notify' settings. A finished build that signs notifies once, when signing starts." + )] + pub notify: Option, +} + /// Flags shared between `debmagic build binary` and `debmagic build source`. #[derive(Args, Debug)] pub struct CommonBuildArgs { @@ -445,14 +448,8 @@ pub struct CommonBuildArgs { #[command(flatten)] pub signing: SignArgs, - #[arg( - long, - num_args = 0..=1, - default_missing_value = "true", - value_parser = BoolishValueParser::new(), - help = "Send a desktop notification via notify-send and ring the terminal bell when the build finishes, successfully or not. Defaults to the 'notify' setting in the config file (false if unset)." - )] - pub notify: Option, + #[command(flatten)] + pub notify: NotifyArgs, #[arg( long, @@ -611,6 +608,9 @@ pub struct TestSubcommandArgs { )] pub allow_host_test: bool, + #[command(flatten)] + pub notify: NotifyArgs, + #[command(flatten)] pub common: CommonCli, } @@ -634,6 +634,9 @@ pub struct SignSubcommandArgs { #[command(flatten)] pub signing: SignArgs, + #[command(flatten)] + pub notify: NotifyArgs, + #[arg( short, long, @@ -716,6 +719,9 @@ pub struct UploadSubcommandArgs { )] pub include_orig: Option, + #[command(flatten)] + pub notify: NotifyArgs, + #[command(flatten)] pub common: CommonCli, diff --git a/packages/debmagic/src/config.rs b/packages/debmagic/src/config.rs index fc3ee3c4..76d7cfcb 100644 --- a/packages/debmagic/src/config.rs +++ b/packages/debmagic/src/config.rs @@ -247,14 +247,59 @@ pub struct Config { /// builds already stage a clean source tree and incremental builds preserve /// outputs intentionally. pub clean: bool, - /// Send a desktop notification via `notify-send` and ring the terminal - /// bell when a build finishes. + /// Default for the `build`, `test` and `sign` notifications. pub notify: bool, + /// `[build]` section. + pub build: BuildConfig, + /// `[test]` section. + pub test: TestConfig, /// Build for a dpkg architecture variant (e.g. `amd64v3` on Ubuntu), /// exported as `DEB_HOST_ARCH_VARIANT` for the build. pub host_arch_variant: Option, } +/// `[build]` section: `debmagic build` behavior. +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +#[serde(default)] +pub struct BuildConfig { + /// Desktop notification and terminal bell when the build finishes; + /// the top-level `notify` when unset. + pub notify: Option, +} + +/// `[test]` section: `debmagic test` behavior. +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +#[serde(default)] +pub struct TestConfig { + /// Desktop notification and terminal bell when the tests finish; + /// the top-level `notify` when unset. + pub notify: Option, +} + +impl Config { + pub fn build_notify(&self) -> bool { + self.build.notify.unwrap_or(self.notify) + } + + pub fn test_notify(&self) -> bool { + self.test.notify.unwrap_or(self.notify) + } + + pub fn sign_notify(&self) -> bool { + self.sign.notify.unwrap_or(self.notify) + } + + /// Apply `--notify`: it covers every step of the command. + pub fn override_notify(&mut self, notify: Option) { + if let Some(notify) = notify { + self.notify = notify; + self.build.notify = Some(notify); + self.test.notify = Some(notify); + self.sign.notify = Some(notify); + } + } +} + /// `[upstream]` section: `upstream` command behavior. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] @@ -298,8 +343,9 @@ pub struct SignConfig { /// is skipped, anything else means re-sign. pub signed_by_command: Option, /// Send a desktop notification via `notify-send` just before signing, - /// so a hardware-key touch prompt isn't missed. - pub notify: bool, + /// so a hardware-key touch prompt isn't missed; the top-level `notify` + /// when unset. + pub notify: Option, } impl Default for Config { @@ -318,6 +364,8 @@ impl Default for Config { upstream: UpstreamConfig::default(), clean: false, notify: false, + build: BuildConfig::default(), + test: TestConfig::default(), host_arch_variant: None, } } @@ -491,7 +539,26 @@ mod tests { assert_eq!(cfg.sign.source, crate::sign::SignMode::Auto); assert_eq!(cfg.sign.key.as_deref(), Some("you@example.com")); assert_eq!(cfg.sign.sign_command.as_deref(), Some("gpg --foo")); - assert!(cfg.sign.notify); + assert_eq!(cfg.sign.notify, Some(true)); Ok(()) } + + #[test] + fn notify_defaults_each_step() { + let mut cfg = Config { + notify: true, + ..Config::default() + }; + cfg.sign.notify = Some(false); + assert!(cfg.build_notify()); + assert!(cfg.test_notify()); + assert!(!cfg.sign_notify()); + + cfg.override_notify(None); + assert!(!cfg.sign_notify()); + cfg.override_notify(Some(false)); + assert!(!cfg.build_notify() && !cfg.test_notify() && !cfg.sign_notify()); + cfg.override_notify(Some(true)); + assert!(cfg.build_notify() && cfg.test_notify() && cfg.sign_notify()); + } } diff --git a/packages/debmagic/src/main.rs b/packages/debmagic/src/main.rs index 0f07f0ee..110928bc 100644 --- a/packages/debmagic/src/main.rs +++ b/packages/debmagic/src/main.rs @@ -101,7 +101,7 @@ async fn run() -> anyhow::Result { test: run_test, sign: build_args.sign, sign_overrides: build_args.signing.overrides(), - notify: build_args.notify, + notify: build_args.notify.notify, clean: build_args.clean, source_sync: build_args.source_sync, host_arch_variant: build_args.host_arch_variant.clone(), @@ -159,6 +159,7 @@ async fn run() -> anyhow::Result { config_file: cli.config.clone(), driver: args.driver, persistent: args.environment.persistent, + notify: args.notify.notify, strict: args.strict, changes: args.changes.clone(), allow_host_test: args.allow_host_test, @@ -194,6 +195,7 @@ async fn run() -> anyhow::Result { hooks: args.hooks, force: args.force, sign: args.sign, + notify: args.notify.notify, include_orig: args.include_orig, changes: args.changes.clone(), })?; @@ -207,6 +209,7 @@ async fn run() -> anyhow::Result { config_file: cli.config.clone(), mode: args.mode, overrides: args.signing.overrides(), + notify: args.notify.notify, output_dir: args.output_dir.clone(), file: args.file.clone(), })?; diff --git a/packages/debmagic/src/output.rs b/packages/debmagic/src/output.rs index d385030e..cd58f2ed 100644 --- a/packages/debmagic/src/output.rs +++ b/packages/debmagic/src/output.rs @@ -93,6 +93,27 @@ pub fn notify_send_bell(summary: &str, body: &str) { bell(); } +/// A [`notify_send_bell`] notification, decided ahead of the step that +/// sends it. +#[derive(Debug, Clone)] +pub struct Notice { + pub summary: String, + pub body: String, +} + +impl Notice { + pub fn new(summary: &str, body: &str) -> Self { + Self { + summary: summary.to_string(), + body: body.to_string(), + } + } + + pub fn send(&self) { + notify_send_bell(&self.summary, &self.body); + } +} + /// Whether color is on, after [`init_color`] has run. pub fn color_enabled() -> bool { COLOR.load(Ordering::Relaxed) diff --git a/packages/debmagic/src/sign.rs b/packages/debmagic/src/sign.rs index 3007ebd4..c13cb01a 100644 --- a/packages/debmagic/src/sign.rs +++ b/packages/debmagic/src/sign.rs @@ -18,7 +18,7 @@ use serde::{Deserialize, Serialize}; use crate::config::SignConfig; use crate::control::{child_filename, fixup_checksums, read_control, write_control}; -use crate::output::notify_send_bell; +use crate::output::Notice; use crate::subprocess::Capture; /// Which OpenPGP implementation performs the signing. @@ -439,7 +439,6 @@ pub struct SignOverrides { pub key: Option, pub tool: Option, pub command: Option, - pub notify: Option, } impl SignOverrides { @@ -453,12 +452,17 @@ impl SignOverrides { if let Some(command) = self.command { config.sign_command = Some(command); } - if let Some(notify) = self.notify { - config.notify = notify; - } } } +/// The notification right before the key-touch prompt of signing `package`. +pub fn sign_notice(package: &str) -> Notice { + Notice::new( + "debmagic: signing requested", + &format!("touch your key to sign {package}"), + ) +} + /// Sign the `.changes` file (and its `.dsc`/`.buildinfo` children) on the /// host, as resolved from the build's `sign` config. #[derive(Debug, Clone)] @@ -467,6 +471,8 @@ pub struct SignIntent { pub config_file: Option, pub mode: SignMode, pub overrides: SignOverrides, + /// `--notify`, overriding the configured notifications. + pub notify: Option, pub output_dir: Option, /// The explicit file to sign; located via changelog + output dir when unset. pub file: Option, @@ -478,6 +484,7 @@ pub fn sign(input: SignIntent) -> anyhow::Result<()> { let source_dir = input.source_dir; let mut config = crate::config::Config::load(Some(&source_dir), input.config_file.as_deref())?; input.overrides.apply(&mut config.sign); + config.override_notify(input.notify); let options = SignConfig { key: config.sign.key.clone(), @@ -511,7 +518,8 @@ pub fn sign(input: SignIntent) -> anyhow::Result<()> { .file_stem() .map(|stem| stem.to_string_lossy().into_owned()) .unwrap_or_default(); - sign_file(&file, &options, input.mode, config.sign.notify, &package) + let notice = config.sign_notify().then(|| sign_notice(&package)); + sign_file(&file, &options, input.mode, notice.as_ref()) } /// Is the file already clearsigned? @@ -645,12 +653,12 @@ fn sign_one(path: &Path, signas: &str, options: &SignConfig) -> anyhow::Result<( /// files our own key already signed, `force` always re-signs. When any /// child is (re)signed the parent is force-signed regardless, since its /// checksums change and the old signature would no longer verify. +/// `notice` is sent right before signing, so a key-touch prompt isn't missed. pub fn sign_file( path: &Path, options: &SignConfig, mode: SignMode, - notify: bool, - package: &str, + notice: Option<&Notice>, ) -> anyhow::Result<()> { let dir = path .parent() @@ -664,11 +672,8 @@ pub fn sign_file( path.display() ); } - if notify { - notify_send_bell( - "debmagic: signing requested", - &format!("touch your key to sign {package}"), - ); + if let Some(notice) = notice { + notice.send(); } // Children first: .dsc, then .buildinfo. @@ -816,7 +821,7 @@ mod tests { verify_command: None, signed_by_command: None, source: SignMode::No, - notify: false, + notify: None, }; let control = parse_control("Maintainer: A \nChanged-By: B \n"); diff --git a/packages/debmagic/src/test/intent.rs b/packages/debmagic/src/test/intent.rs index 57023ffd..8e6e7f52 100644 --- a/packages/debmagic/src/test/intent.rs +++ b/packages/debmagic/src/test/intent.rs @@ -14,6 +14,8 @@ pub struct TestIntentInput { pub config_file: Option, pub driver: Option, pub persistent: Option, + /// `--notify`, overriding the configured notifications. + pub notify: Option, pub strict: bool, pub changes: Option, pub allow_host_test: bool, @@ -47,6 +49,7 @@ pub fn resolve_test_intent(input: TestIntentInput) -> anyhow::Result if let Some(persistent) = input.persistent { config.driver.persistent = persistent; } + config.override_notify(input.notify); let changes = if let Some(changes) = input.changes { Some(std::path::absolute(changes).context("resolving --changes path failed")?) @@ -87,6 +90,7 @@ mod tests { config_file: Some(asset_config()), driver: None, persistent: None, + notify: None, strict: false, changes: None, allow_host_test: false, diff --git a/packages/debmagic/src/test/run.rs b/packages/debmagic/src/test/run.rs index c2ca0f8b..f95ff46f 100644 --- a/packages/debmagic/src/test/run.rs +++ b/packages/debmagic/src/test/run.rs @@ -284,7 +284,7 @@ pub async fn run_test(intent: &TestIntent) -> anyhow::Result { ); let version = package.version().to_string(); - claim_command( + let result = claim_command( ®istry, environment, &intent.config.driver, @@ -388,7 +388,16 @@ pub async fn run_test(intent: &TestIntent) -> anyhow::Result { }) }, ) - .await + .await; + if intent.config.test_notify() { + let summary = match &result { + Ok(TestOutcome::Passed) => "debmagic: tests passed", + Ok(TestOutcome::Failed | TestOutcome::StrictFailure) => "debmagic: tests failed", + Err(_) => "debmagic: test run failed", + }; + crate::output::notify_send_bell(summary, &format!("{} {version}", package.name())); + } + result } #[cfg(test)] diff --git a/packages/debmagic/src/upload.rs b/packages/debmagic/src/upload.rs index 4dc97ebb..5c0a6823 100644 --- a/packages/debmagic/src/upload.rs +++ b/packages/debmagic/src/upload.rs @@ -74,6 +74,8 @@ pub struct UploadIntent { pub hooks: bool, pub force: bool, pub sign: Option, + /// `--notify`, overriding the configured notifications. + pub notify: Option, pub include_orig: Option, /// The explicit `.changes` to upload; located via changelog + output dir when unset. pub changes: Option, @@ -83,7 +85,8 @@ pub struct UploadIntent { /// the `.changes` and adjust its orig entries, then upload it. pub fn upload(input: UploadIntent) -> anyhow::Result<()> { let source_dir = input.source_dir; - let config = crate::config::Config::load(Some(&source_dir), input.config_file.as_deref())?; + let mut config = crate::config::Config::load(Some(&source_dir), input.config_file.as_deref())?; + config.override_notify(input.notify); let mut upload_target = resolve_target(&input.target, Some(&config.upload))?; upload_target.apply_overrides(&UploadOverrides { @@ -130,13 +133,10 @@ pub fn upload(input: UploadIntent) -> anyhow::Result<()> { .file_stem() .map(|stem| stem.to_string_lossy().into_owned()) .unwrap_or_default(); - crate::sign::sign_file( - &changes_file, - &config.sign, - sign_mode, - config.sign.notify, - &package, - )?; + let notice = config + .sign_notify() + .then(|| crate::sign::sign_notice(&package)); + crate::sign::sign_file(&changes_file, &config.sign, sign_mode, notice.as_ref())?; } crate::output::stage(&format!( From 008d08ffa5758bb80e4671f6fd66bd6971771d8d Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Thu, 8 Oct 2026 16:35:14 +0200 Subject: [PATCH 27/48] feat(upload): name the upload target in the signing notification --- packages/debmagic/src/upload.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/debmagic/src/upload.rs b/packages/debmagic/src/upload.rs index 5c0a6823..85963af1 100644 --- a/packages/debmagic/src/upload.rs +++ b/packages/debmagic/src/upload.rs @@ -133,9 +133,12 @@ pub fn upload(input: UploadIntent) -> anyhow::Result<()> { .file_stem() .map(|stem| stem.to_string_lossy().into_owned()) .unwrap_or_default(); - let notice = config - .sign_notify() - .then(|| crate::sign::sign_notice(&package)); + let notice = config.sign_notify().then(|| { + crate::output::Notice::new( + "debmagic: signing for upload", + &format!("touch your key to sign {package} for {}", input.target), + ) + }); crate::sign::sign_file(&changes_file, &config.sign, sign_mode, notice.as_ref())?; } From e1dcd2a891464e19eec2c0b6069c65830e71d778 Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Thu, 8 Oct 2026 18:33:03 +0200 Subject: [PATCH 28/48] fix(build): don't sign binary builds by default You can still sign them with `--sign` or `build.sign_binary`. --- docs/usage/build.md | 12 ++++- docs/usage/config.md | 10 ++-- packages/debmagic/src/build_intent.rs | 70 +++++++++++++++++++++++++-- packages/debmagic/src/cli.rs | 2 +- packages/debmagic/src/config.rs | 63 +++++++++++++++++++++++- 5 files changed, 148 insertions(+), 9 deletions(-) diff --git a/docs/usage/build.md b/docs/usage/build.md index 90370c31..bc0e912d 100644 --- a/docs/usage/build.md +++ b/docs/usage/build.md @@ -174,6 +174,16 @@ This exports `DEB_BUILD_OPTIONS=nocheck`, the standard dpkg mechanism: dpkg-buil ## Signing `--sign` GPG-signs the resulting `.changes`/`.dsc`/`.buildinfo` after building — mainly useful for [source builds destined for Launchpad](source.md#uploading-to-launchpad), but works for binary builds too. + +Without `--sign`, source builds are signed and binary builds are not (if `sign.key`, or `sign.tool = "custom"` are configured). +`build.sign_source` and `build.sign_binary` change that per kind; `--sign` signs regardless, and `--sign=no` never does. + +```toml +[build] +sign_binary = true # also sign binary builds +sign_source = false # but not source builds +``` + Signing is debmagic's own reimplementation of `debsign` and always runs on the host with your gpg keyring: the artifacts are exported to the host output dir first, so no container or agent forwarding is involved. Children are signed first (`.dsc`, then `.buildinfo`) and the `.changes` checksums are rewritten after each, exactly like `debsign`. @@ -192,7 +202,7 @@ The `--sign` mode decides what happens to an existing signature: | Option | Config | Description | |---|---|---| -| `--sign ` | `sign.source` | Sign after building: `no`, `keep`, `auto` or `force` | +| `--sign ` | `build.sign_source`, `build.sign_binary`, `sign.source` | Sign after building: `no`, `keep`, `auto` (default) or `force`; the config decides what happens without the flag | | `--sign-key ` | `sign.key` | Key ID/fingerprint/email; defaults to the `Changed-By:`/`Maintainer:` address of the file being signed | | `--sign-tool ` | `sign.tool` | OpenPGP implementation: `gpg` (default), `sequoia` (sq), or `custom` | | `--sign-command ` | `sign.sign_command` | Custom signing command for `--sign-tool custom` (see below) | diff --git a/docs/usage/config.md b/docs/usage/config.md index 7b9ec6b7..371c748e 100644 --- a/docs/usage/config.md +++ b/docs/usage/config.md @@ -36,7 +36,7 @@ All keys are optional. | `source_sync_mode` | enum | `tracked` | `--source-sync` | Which source files are staged (see below). | | `build_debug_symbols` | bool | `false` | `--debug-symbols` | Build the automatic `-dbgsym` debug symbol package. | | `run_test` | bool | `true` | `--test` | Run the package's test suite during the build; `false` exports `DEB_BUILD_OPTIONS=nocheck` so tests are skipped. | -| `sign.source` | enum | `no` | `--sign` | Sign the resulting `.changes`/`.dsc`: `no` never, `keep` keeps any existing signature, `auto` skips what our key already signed, `force` always re-signs (see below). | +| `sign.source` | enum | `auto` | `--sign` | How to sign the resulting `.changes`/`.dsc`/`.buildinfo`: `no` never, `keep` keeps any existing signature, `auto` skips what our key already signed, `force` always re-signs (see below). Whether a build signs at all is up to `build.sign_source`/`build.sign_binary`. | | `sign.key` | string | — | `--sign-key` | GPG key ID/email to sign with; falls back to the Changed-By/Maintainer address. | | `sign.tool` | enum | `gpg` | `--sign-tool` | OpenPGP implementation: `gpg`, `sequoia` (sq) or `custom` (uses `sign.sign_command`). | | `sign.sign_command` | string | — | `--sign-command` | Custom signing command for `sign.tool = "custom"`, run without a shell with `{file}`/`{key}`/`{email}` placeholders; writes the clearsigned result to stdout. | @@ -44,6 +44,8 @@ All keys are optional. | `clean` | bool | `false` | `--clean` | Run `debian/rules clean` before building. Disabled by default; incompatible with `incremental`. | | `notify` | bool | `false` | `--notify` | Default for `build.notify`, `test.notify` and `sign.notify`. | | `build.notify` | bool | `notify` | `--notify` | Desktop notification via `notify-send` and terminal bell when a build finishes; a build that signs notifies once, when signing starts (see [Notifications](build.md#notifications)). | +| `build.sign_source` | bool | `true` | `--sign` | Sign `build source` results without `--sign`, if a signing key is configured (see [Signing](build.md#signing)). | +| `build.sign_binary` | bool | `false` | `--sign` | Sign `build binary` results without `--sign`, if a signing key is configured. | | `test.notify` | bool | `notify` | `--notify` | Desktop notification via `notify-send` and terminal bell when `debmagic test` finishes. | | `host_arch_variant` | string | — | `--host-arch-variant` | Build for a dpkg architecture variant (e.g. `"amd64v3"` on Ubuntu) -> `DEB_HOST_ARCH_VARIANT`. | | `upstream.signature_check` | bool | `true` | `--signature-check` | Verify the upstream tarball's PGP signature against `debian/upstream/signing-key.asc` during `upstream fetch`/`switch` (see [Upstream](upstream.md#signature-verification)). | @@ -62,7 +64,7 @@ All keys are optional. | Key | Type | Default | CLI flag | Description | |---|---|---|---|---| -| `source` | enum | `no` | `--sign` | Sign the source package (`.changes`/`.dsc`) after building: `no` never, `keep` keeps any existing signature, `auto` skips what our key already signed, `force` always re-signs. | +| `source` | enum | `auto` | `--sign` | How to sign after building: `no` never, `keep` keeps any existing signature, `auto` skips what our key already signed, `force` always re-signs. Whether a build signs at all is up to `build.sign_source`/`build.sign_binary`. | | `key` | string | — | `--sign-key` | GPG key ID/email to sign with; falls back to the Changed-By/Maintainer address. | | `tool` | enum | `gpg` | `--sign-tool` | OpenPGP implementation: `gpg` (sequoia's `sq`) or `custom` (uses `sign_command`). | | `notify` | bool | top-level `notify` | `--notify` | Desktop notification via `notify-send` before signing. | @@ -104,8 +106,10 @@ See [Uploading](upload.md) for target resolution, builtins and pre-upload checks build_debug_symbols = true clean = false +[build] +sign_binary = true # source builds sign by default, binary builds don't + [sign] -source = true key = "you@example.com or gpg key id" notify = true diff --git a/packages/debmagic/src/build_intent.rs b/packages/debmagic/src/build_intent.rs index f2d3327a..9140a76b 100644 --- a/packages/debmagic/src/build_intent.rs +++ b/packages/debmagic/src/build_intent.rs @@ -89,10 +89,8 @@ pub fn resolve_build_intent(input: BuildIntentInput) -> anyhow::Result anyhow::Result, config: &Config, kind: BuildKind) -> SignMode { + if let Some(mode) = cli { + return mode; + } + let by_default = match kind { + BuildKind::Source => config.build.sign_source, + BuildKind::Binary => config.build.sign_binary, + }; + if by_default && config.sign.is_configured() { + config.sign.source + } else { + SignMode::No + } +} + /// CLI `--driver` wins; else the config's `driver.default`; source-only /// builds fall back to bare, since they need no build-deps or compilation. fn resolve_driver( @@ -143,6 +161,7 @@ mod tests { DriverType, Persistence, config::DriverOverrides, driver_bare::DriverBareConfigOverrides, driver_docker::DriverDockerConfigOverrides, driver_lxd::DriverLxdConfigOverrides, }; + use test_case::test_case; fn asset_config() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -302,6 +321,51 @@ mod tests { Ok(()) } + fn sign_test_config(key: bool, sign_source: bool, sign_binary: bool) -> Config { + let mut config = Config::default(); + config.sign.key = key.then(|| "you@example.com".to_string()); + config.build.sign_source = sign_source; + config.build.sign_binary = sign_binary; + config + } + + #[test_case(BuildKind::Source, None, true, true, false, SignMode::Auto; "source builds sign by default")] + #[test_case(BuildKind::Binary, None, true, true, false, SignMode::No; "binary builds do not")] + #[test_case(BuildKind::Binary, Some(SignMode::Auto), true, true, false, SignMode::Auto; "--sign signs a binary build")] + #[test_case(BuildKind::Binary, Some(SignMode::Auto), false, true, false, SignMode::Auto; "--sign needs no configured key")] + #[test_case(BuildKind::Binary, Some(SignMode::Force), true, true, false, SignMode::Force; "--sign=force")] + #[test_case(BuildKind::Source, Some(SignMode::No), true, true, false, SignMode::No; "--sign=no")] + #[test_case(BuildKind::Source, None, false, true, false, SignMode::No; "no key, no default signing")] + #[test_case(BuildKind::Binary, None, true, true, true, SignMode::Auto; "build.sign_binary")] + #[test_case(BuildKind::Source, None, true, false, false, SignMode::No; "build.sign_source = false")] + fn resolve_sign_mode_by_kind( + kind: BuildKind, + cli: Option, + key: bool, + sign_source: bool, + sign_binary: bool, + expected: SignMode, + ) { + let config = sign_test_config(key, sign_source, sign_binary); + assert_eq!(resolve_sign_mode(cli, &config, kind), expected); + } + + #[test] + fn resolve_sign_mode_uses_the_configured_mode() { + let mut config = sign_test_config(true, true, true); + config.sign.source = SignMode::Force; + assert_eq!( + resolve_sign_mode(None, &config, BuildKind::Binary), + SignMode::Force + ); + + config.sign.source = SignMode::No; + assert_eq!( + resolve_sign_mode(None, &config, BuildKind::Source), + SignMode::No + ); + } + #[test] fn resolve_rejects_incremental_with_clean() { let dir = std::env::temp_dir(); diff --git a/packages/debmagic/src/cli.rs b/packages/debmagic/src/cli.rs index d1fac702..1472c872 100644 --- a/packages/debmagic/src/cli.rs +++ b/packages/debmagic/src/cli.rs @@ -441,7 +441,7 @@ pub struct CommonBuildArgs { value_enum, num_args = 0..=1, default_missing_value = "auto", - help = "Sign the resulting .changes/.dsc after building: 'auto' (the default when passed without a value) signs what is unsigned and skips what our key already signed, 'force' always re-signs, 'no' never signs. Defaults to the 'sign.source' setting in the config file (no if unset)." + help = "Sign the resulting .changes/.dsc/.buildinfo after building: 'auto' (the default when passed without a value) signs what is unsigned and skips what our key already signed, 'keep' keeps any existing signature, 'force' always re-signs, 'no' never signs. Without --sign, source builds sign if 'build.sign_source' (default yes) and binary builds if 'build.sign_binary' (default no) is set, with the 'sign.source' mode (auto if unset), but only when a signing key is configured." )] pub sign: Option, diff --git a/packages/debmagic/src/config.rs b/packages/debmagic/src/config.rs index 76d7cfcb..b6bd6c5b 100644 --- a/packages/debmagic/src/config.rs +++ b/packages/debmagic/src/config.rs @@ -259,12 +259,26 @@ pub struct Config { } /// `[build]` section: `debmagic build` behavior. -#[derive(Serialize, Deserialize, Debug, Clone, Default)] +#[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct BuildConfig { /// Desktop notification and terminal bell when the build finishes; /// the top-level `notify` when unset. pub notify: Option, + /// Sign source builds without `--sign`, when signing is configured. + pub sign_source: bool, + /// Sign binary builds without `--sign`, when signing is configured. + pub sign_binary: bool, +} + +impl Default for BuildConfig { + fn default() -> Self { + Self { + notify: None, + sign_source: true, + sign_binary: false, + } + } } /// `[test]` section: `debmagic test` behavior. @@ -348,6 +362,16 @@ pub struct SignConfig { pub notify: Option, } +impl SignConfig { + /// Whether signing is set up explicitly: a key, or a custom command + /// that brings its own. Without either the key is guessed from the + /// `Changed-By:`/`Maintainer:` address, which is fine when asked for + /// with `--sign`, but not for signing by default. + pub fn is_configured(&self) -> bool { + self.key.is_some() || (self.tool == SignTool::Custom && self.sign_command.is_some()) + } +} + impl Default for Config { fn default() -> Self { Self { @@ -543,6 +567,43 @@ mod tests { Ok(()) } + #[test] + fn build_sign_defaults_survive_a_partial_section() -> Result<(), anyhow::Error> { + let defaults = Config::default(); + assert!(defaults.build.sign_source && !defaults.build.sign_binary); + + let dir = + std::env::temp_dir().join(format!("debmagic-test-build-sign-{}", std::process::id())); + std::fs::create_dir_all(&dir)?; + let file = dir.join("build.toml"); + std::fs::write(&file, "[build]\nsign_binary = true\n")?; + let cfg = Config::new(&[ConfigPath::new( + ConfigLayer::Explicit, + file, + ConfigPathStatus::Used, + )])?; + std::fs::remove_dir_all(&dir).ok(); + assert!(cfg.build.sign_source && cfg.build.sign_binary); + Ok(()) + } + + #[test] + fn sign_is_configured_by_a_key_or_a_custom_command() { + let mut sign = SignConfig::default(); + assert!(!sign.is_configured()); + + sign.sign_command = Some("my-signer {file}".into()); + assert!(!sign.is_configured(), "a command is ignored by gpg"); + sign.tool = SignTool::Custom; + assert!(sign.is_configured()); + + let with_key = SignConfig { + key: Some("you@example.com".into()), + ..SignConfig::default() + }; + assert!(with_key.is_configured()); + } + #[test] fn notify_defaults_each_step() { let mut cfg = Config { From 4d63ea37b1526c646c7655f5479c427e77eda72d Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Thu, 8 Oct 2026 18:36:15 +0200 Subject: [PATCH 29/48] fix(upstream): stage an orig tarball kept as a symlink in `..` `upstream switch` and `build source` fail with "No such file or directory" when the orig tarball in `..` is a relative symlink (e.g. into the git-ubuntu cache). Fix by resolving the symlink and replacing whatever is at the destination, dangling links included. `--force` also removes dangling links. --- packages/debmagic/src/build/mod.rs | 49 ++----------- packages/debmagic/src/upstream/fetch.rs | 3 +- packages/debmagic/src/upstream/orig.rs | 94 +++++++++++++++++++++++-- 3 files changed, 99 insertions(+), 47 deletions(-) diff --git a/packages/debmagic/src/build/mod.rs b/packages/debmagic/src/build/mod.rs index 677d9c78..6ed61bfc 100644 --- a/packages/debmagic/src/build/mod.rs +++ b/packages/debmagic/src/build/mod.rs @@ -11,6 +11,7 @@ use crate::environment::{ ClaimedEnvironment, CommandCompletion, HostRootPolicy, InvocationKind, Registry, claim_command, }; use crate::package::PackageTarget; +use crate::upstream::orig::link_or_copy; use anyhow::{Context, anyhow}; use debmagic_common::debian::source::SourceFormat; use debmagic_common::package::SourcePackage; @@ -325,25 +326,6 @@ fn check_dpkg_buildpackage_available() -> anyhow::Result<()> { ) } -/// Make `source` available at `destination`: hardlink on the same -/// filesystem, else copy. A symlink would point at a host path that -/// doesn't exist inside a container environment. -fn stage_file(source: &Path, destination: &Path) -> anyhow::Result<()> { - if destination.exists() { - std::fs::remove_file(destination) - .with_context(|| format!("failed to remove {}", destination.display()))?; - } - if std::fs::hard_link(source, destination).is_ok() { - return Ok(()); - } - fs::copy(source, destination).map(|_| ()).with_context(|| { - format!( - "failed to stage {} into the build environment", - source.display() - ) - }) -} - /// Build a `.dsc` + tarball + `.buildinfo` + `.changes` source package. /// /// If `config.clean` is set, build-dependencies are installed before @@ -426,7 +408,12 @@ pub async fn build_source_package( .file_name() .expect("orig tarball has a file name"), ); - stage_file(tarball, &destination)?; + link_or_copy(tarball, &destination).with_context(|| { + format!( + "staging {} into the build environment failed", + tarball.display() + ) + })?; } // dpkg-buildpackage looks for the gain-root command even for -S if let Some(package) = gain_root_package(target)? { @@ -594,28 +581,6 @@ mod tests { std::fs::remove_dir_all(&dir).unwrap(); } - #[test] - fn stage_file_prefers_hardlink() { - let dir = std::env::temp_dir().join(format!("debmagic-stage-{}", std::process::id())); - std::fs::create_dir_all(&dir).unwrap(); - let source = dir.join("src.tar"); - std::fs::write(&source, "data").unwrap(); - let destination = dir.join("dest.tar"); - stage_file(&source, &destination).unwrap(); - // same filesystem: a hardlink shares the inode - assert_eq!( - std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(&source).unwrap()), - std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(&destination).unwrap()) - ); - // restaging replaces the destination - stage_file(&source, &destination).unwrap(); - assert_eq!( - std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(&source).unwrap()), - std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(&destination).unwrap()) - ); - std::fs::remove_dir_all(&dir).unwrap(); - } - #[test] fn debug_symbol_option_preserves_other_build_options() { assert_eq!( diff --git a/packages/debmagic/src/upstream/fetch.rs b/packages/debmagic/src/upstream/fetch.rs index 7cef4ae8..8a2b267d 100644 --- a/packages/debmagic/src/upstream/fetch.rs +++ b/packages/debmagic/src/upstream/fetch.rs @@ -346,7 +346,8 @@ fn remove_existing( ) -> anyhow::Result<()> { for filename in orig_filenames(name, orig_version, component) { let path = output_dir.join(filename); - if path.is_file() { + // a dangling symlink is leftover too + if path.symlink_metadata().is_ok() { std::fs::remove_file(&path) .with_context(|| format!("failed to remove {}", path.display()))?; } diff --git a/packages/debmagic/src/upstream/orig.rs b/packages/debmagic/src/upstream/orig.rs index 90c5c3fd..3efb49c0 100644 --- a/packages/debmagic/src/upstream/orig.rs +++ b/packages/debmagic/src/upstream/orig.rs @@ -364,13 +364,41 @@ fn stage_from_cache(cached: &Path, output_dir: &Path) -> anyhow::Result Ok(destination) } -/// Hardlink `source` to `destination` when both are on the same -/// filesystem, copy otherwise. +/// Make `source` available at `destination`: a hardlink when both are on +/// the same filesystem, a copy otherwise (a symlink would point at a +/// host path that may not exist where the tarball is used). +/// +/// A symlinked `source` is resolved first, as `link(2)` would hardlink +/// the symlink itself, whose relative target dangles from any other +/// directory. An existing `destination`, a dangling symlink included, is +/// replaced instead of written through. pub fn link_or_copy(source: &Path, destination: &Path) -> anyhow::Result<()> { - if std::fs::hard_link(source, destination).is_ok() { + let source = source + .canonicalize() + .with_context(|| format!("failed to resolve {}", source.display()))?; + // nothing to do, and removing it would delete the source + let is_symlink = destination + .symlink_metadata() + .is_ok_and(|metadata| metadata.file_type().is_symlink()); + if !is_symlink + && destination + .canonicalize() + .is_ok_and(|existing| existing == source) + { + return Ok(()); + } + match std::fs::remove_file(destination) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error) + .with_context(|| format!("failed to remove {}", destination.display())); + } + } + if std::fs::hard_link(&source, destination).is_ok() { return Ok(()); } - std::fs::copy(source, destination).with_context(|| { + std::fs::copy(&source, destination).with_context(|| { format!( "failed to copy {} to {}", source.display(), @@ -650,6 +678,64 @@ mod tests { std::fs::remove_dir_all(&base).unwrap(); } + #[test] + fn test_link_or_copy_resolves_a_relative_symlink_source() { + use std::os::unix::fs::MetadataExt; + let base = + std::env::temp_dir().join(format!("debmagic-orig-symlink-{}", std::process::id())); + let cache = base.join("pkg").join("cache"); + let build = base.join("pkg").join("build"); + std::fs::create_dir_all(&cache).unwrap(); + std::fs::create_dir_all(&build).unwrap(); + let real = cache.join("foo_1.0.orig.tar.xz"); + std::fs::write(&real, "x").unwrap(); + // like a tarball kept in `..` as a link into the git cache + let link = base.join("foo_1.0.orig.tar.xz"); + std::os::unix::fs::symlink("pkg/cache/foo_1.0.orig.tar.xz", &link).unwrap(); + + let staged = build.join("foo_1.0.orig.tar.xz"); + link_or_copy(&link, &staged).unwrap(); + assert!( + !std::fs::symlink_metadata(&staged) + .unwrap() + .file_type() + .is_symlink() + ); + assert_eq!( + std::fs::metadata(&real).unwrap().ino(), + std::fs::metadata(&staged).unwrap().ino() + ); + std::fs::remove_dir_all(&base).unwrap(); + } + + #[test] + fn test_link_or_copy_replaces_the_destination() { + let base = + std::env::temp_dir().join(format!("debmagic-orig-replace-{}", std::process::id())); + let out = base.join("out"); + std::fs::create_dir_all(&out).unwrap(); + let source = base.join("foo_1.0.orig.tar.xz"); + std::fs::write(&source, "new").unwrap(); + let destination = out.join("foo_1.0.orig.tar.xz"); + + // a stale file + std::fs::write(&destination, "old").unwrap(); + link_or_copy(&source, &destination).unwrap(); + assert_eq!(std::fs::read_to_string(&destination).unwrap(), "new"); + + // a dangling link must not be written through + std::fs::remove_file(&destination).unwrap(); + std::os::unix::fs::symlink("missing/dir/file", &destination).unwrap(); + link_or_copy(&source, &destination).unwrap(); + assert_eq!(std::fs::read_to_string(&destination).unwrap(), "new"); + assert!(!out.join("missing").exists()); + + // the source itself stays intact when it already is the destination + link_or_copy(&source, &source).unwrap(); + assert_eq!(std::fs::read_to_string(&source).unwrap(), "new"); + std::fs::remove_dir_all(&base).unwrap(); + } + #[test] fn test_substitute_placeholders() { let base = std::env::temp_dir().join(format!("debmagic-orig-subst-{}", std::process::id())); From 9bebfa9bb1faffb8b2cda32108fb261dc751a6f0 Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Fri, 9 Oct 2026 13:55:00 +0200 Subject: [PATCH 30/48] feat(build): use build-debmagic/ when upstream owns build/ debmagic tags its output dir and the orig download cache with a `CACHEDIR.TAG`, so backup tools skip them. An untagged `build/` holding more than artifacts belongs to upstream: every command then uses `output_dir_fallback` (`build-debmagic` by default), or stops with `output_dir_clash = "fail"`. `upstream switch` moves the output dir to the fallback when the new version starts shipping its name, instead of merging upstream's files into it. Builds no longer treat the output dir's contents as source: - it is never staged, in any `--source-sync` mode - the untracked-files warning of `--source-sync tracked` skips it --- docs/usage/build.md | 9 + docs/usage/config.md | 4 +- docs/usage/upstream.md | 3 + packages/debmagic-common/src/changes.rs | 39 ++++ packages/debmagic/src/build/mod.rs | 6 +- packages/debmagic/src/build/source.rs | 77 +++++++- packages/debmagic/src/build_intent.rs | 7 +- packages/debmagic/src/config.rs | 7 + packages/debmagic/src/main.rs | 1 + packages/debmagic/src/output_dir.rs | 231 +++++++++++++++++++++++ packages/debmagic/src/sign.rs | 8 +- packages/debmagic/src/test/run.rs | 1 + packages/debmagic/src/upload.rs | 3 +- packages/debmagic/src/upstream.rs | 33 ++-- packages/debmagic/src/upstream/fetch.rs | 3 + packages/debmagic/src/upstream/orig.rs | 48 ++--- packages/debmagic/src/upstream/switch.rs | 124 +++++++++++- 17 files changed, 545 insertions(+), 59 deletions(-) create mode 100644 packages/debmagic/src/output_dir.rs diff --git a/docs/usage/build.md b/docs/usage/build.md index bc0e912d..46b771d3 100644 --- a/docs/usage/build.md +++ b/docs/usage/build.md @@ -99,10 +99,19 @@ Before building, `debmagic` stages the source tree into the build environment. | `worktree` | everything that isn't git-ignored, tracked or not | | If the source directory is not a git worktree, `tracked` and `committed` fall back to `worktree` with a warning. +The [output dir](#output-dir) is never staged. Git submodules are skipped with a note, since their contents aren't tracked by the parent repository. To persist a mode, set `source_sync_mode = "committed"` in [`debmagic.toml`](config.md). +## Output dir + +Build artifacts land in `build/` under the package root (`output_dir`, `-o`). +debmagic marks the output dirs it creates with a [`CACHEDIR.TAG`](https://bford.info/cachedir/), so backup tools skip them. +The tag also tells debmagic's `build/` apart from one an upstream project ships: an existing `build/` without the tag that holds more than build artifacts belongs to upstream. +debmagic then uses `build-debmagic/` instead (`output_dir_fallback`), or stops with `output_dir_clash = "fail"`. +`-o` is always used as given. + ## Repeating a build You can iterate on the same build for faster compile times. diff --git a/docs/usage/config.md b/docs/usage/config.md index 371c748e..9db7a2b3 100644 --- a/docs/usage/config.md +++ b/docs/usage/config.md @@ -31,7 +31,9 @@ All keys are optional. | `driver.lxd.project` | string | — | — | LXD/Incus project to use. | | `driver.lxd.base_images` | map | — | — | Base image per distro, keyed by `":"`. Falls back to the driver's default remote image. Same custom-suite registry role as Docker's map for LXD/Incus. | | `environments_dir` | path | `$XDG_DATA_HOME/debmagic/environments` | — | Parent directory of Environment Host roots. Must be absolute or start with `~/`. The Environment registry is always `$XDG_DATA_HOME/debmagic/db.sqlite` (override both with `DEBMAGIC_DATA_DIR`). | -| `output_dir` | path | `build` | `-o`/`--output-dir` | Where build artifacts are exported; relative paths resolve against the package root. | +| `output_dir` | path | `build` | `-o`/`--output-dir` | Where build artifacts are exported; relative paths resolve against the package root. See [Output dir](build.md#output-dir). | +| `output_dir_fallback` | path | `build-debmagic` | — | Used instead of `output_dir` when upstream owns that directory. | +| `output_dir_clash` | enum | `fallback` | — | When upstream owns `output_dir`: `fallback` uses `output_dir_fallback`, `fail` stops. | | `incremental` | bool | `false` | `--incremental` | Retain the environment and sync only source changes, preserving generated files. Forces Persistence `always`; incompatible with `clean`. | | `source_sync_mode` | enum | `tracked` | `--source-sync` | Which source files are staged (see below). | | `build_debug_symbols` | bool | `false` | `--debug-symbols` | Build the automatic `-dbgsym` debug symbol package. | diff --git a/docs/usage/upstream.md b/docs/usage/upstream.md index c15c3edc..b089a23f 100644 --- a/docs/usage/upstream.md +++ b/docs/usage/upstream.md @@ -46,6 +46,9 @@ Distro tarballs are never repacked: the archive already holds them. Fetches the chosen version's orig tarball exactly like `upstream fetch` — with the same options — and replaces the source tree's contents with it, everything except `debian/`, which is kept as-is. The changelog is not touched; add the new version entry with [`debmagic changelog new`](changelog.md). +The output dir (`build/` by default) is kept across the switch. +If the new version starts shipping a directory of that name — autotools projects sometimes do — the output dir moves to `build-debmagic/` (see [Output dir](build.md#output-dir)); with `output_dir_clash = "fail"` or `-o`, the switch stops before changing anything. + For a concrete version, the tarball URL is resolved without scraping the download listing: 1. the distro archives (with `--from auto`) — a version Debian or Ubuntu published needs no upstream download at all diff --git a/packages/debmagic-common/src/changes.rs b/packages/debmagic-common/src/changes.rs index 20ad3a39..9a8dbaa6 100644 --- a/packages/debmagic-common/src/changes.rs +++ b/packages/debmagic-common/src/changes.rs @@ -48,6 +48,27 @@ pub fn is_orig_tarball(name: &str) -> bool { } } +/// Whether a filename looks like a build or upload artifact: +/// `_...` with a package, source, `.changes`, `.buildinfo`, +/// dput `.upload` or debmagic `.upload.json` suffix, or a tarball, possibly +/// with an `.asc` signature. +pub fn is_artifact_filename(name: &str) -> bool { + const SUFFIXES: [&str; 9] = [ + ".changes", + ".buildinfo", + ".dsc", + ".deb", + ".udeb", + ".ddeb", + ".upload", + ".upload.json", + ".diff.gz", + ]; + let name = name.strip_suffix(".asc").unwrap_or(name); + name.contains('_') + && (name.contains(".tar.") || SUFFIXES.iter().any(|suffix| name.ends_with(suffix))) +} + /// The `orig` tarball filename prefix for a source package version: /// `_.orig.tar.` — the compression extension varies. /// For a component, `_.orig-.tar.`. @@ -165,6 +186,24 @@ mod tests { assert_eq!(is_source_changes(name), expected); } + #[test_case("pkg_1.0.orig.tar.xz", true; "orig")] + #[test_case("pkg_1.0.orig.tar.xz.asc", true; "orig signature")] + #[test_case("pkg_1.0-1.debian.tar.xz", true; "debian tarball")] + #[test_case("pkg_1.0.tar.xz", true; "native tarball")] + #[test_case("pkg_1.0-1.dsc", true; "dsc")] + #[test_case("pkg_1.0-1.diff.gz", true; "1.0 diff")] + #[test_case("libpkg1_1.0-1_amd64.deb", true; "deb")] + #[test_case("pkg-dbgsym_1.0-1_amd64.ddeb", true; "ddeb")] + #[test_case("pkg_1.0-1_amd64.buildinfo", true; "buildinfo")] + #[test_case("pkg_1.0-1_source.ppa.upload", true; "dput log")] + #[test_case("pkg_1.0-1_source.upload.json", true; "debmagic upload record")] + #[test_case("config.guess", false; "upstream script")] + #[test_case("lib_shared.mk", false; "upstream makefile with underscore")] + #[test_case("pkg.tar.gz", false; "tarball without version")] + fn test_is_artifact_filename(name: &str, expected: bool) { + assert_eq!(is_artifact_filename(name), expected); + } + #[test] fn test_changes_files_orig_tarballs() { let changes = ChangesFiles { diff --git a/packages/debmagic/src/build/mod.rs b/packages/debmagic/src/build/mod.rs index 6ed61bfc..dc99df18 100644 --- a/packages/debmagic/src/build/mod.rs +++ b/packages/debmagic/src/build/mod.rs @@ -1,5 +1,5 @@ use std::{ - fs, io, + io, path::Path, process::{Command, Stdio}, }; @@ -144,7 +144,7 @@ async fn run_build( &request.intent.driver_overrides, planned.root_policy, |environment, driver| { - fs::create_dir_all(&request.intent.output_dir) + crate::output_dir::create(&request.intent.output_dir) .context("failed to create output directory")?; if planned.incremental && let Some(driver) = driver @@ -158,6 +158,7 @@ async fn run_build( package, request.intent.config.source_sync_mode, request.intent.config.incremental, + &request.intent.output_dir, ) }, InvocationKind::Build(build_kind), @@ -350,6 +351,7 @@ pub async fn build_source_package( // artifacts land anyway, so fetch into it and stage from there. let name = target.package.name(); let upstream_version = target.package.version().upstream_version(); + crate::output_dir::create(&intent.output_dir)?; let orig_tarball = crate::upstream::orig::fetch_orig_tarball( &intent.config.orig_tarball, &target.package, diff --git a/packages/debmagic/src/build/source.rs b/packages/debmagic/src/build/source.rs index 1c898c04..dc0af797 100644 --- a/packages/debmagic/src/build/source.rs +++ b/packages/debmagic/src/build/source.rs @@ -196,6 +196,34 @@ fn source_tree_entries(src: &Path, mode: SourceSyncMode) -> anyhow::Result Option { + output_dir + .strip_prefix(src) + .ok() + .filter(|relative| { + !relative.as_os_str().is_empty() && crate::output_dir::is_tagged(output_dir) + }) + .map(Path::to_path_buf) +} + +/// The entries to stage: the sync mode's selection, minus debmagic's +/// output dir. +fn staged_entries( + src: &Path, + mode: SourceSyncMode, + output_dir: &Path, +) -> anyhow::Result> { + let entries = source_tree_entries(src, mode)?; + let Some(output) = owned_output_dir(src, output_dir) else { + return Ok(entries); + }; + Ok(entries + .into_iter() + .filter(|entry| !entry.path.starts_with(&output)) + .collect()) +} + fn entry_kind(path: &Path) -> anyhow::Result { let metadata = fs::symlink_metadata(path) .with_context(|| format!("failed to stat source path {}", path.display()))?; @@ -420,6 +448,7 @@ fn sync_source_tree( kind: BuildKind, source_dir: &Path, source_sync_mode: SourceSyncMode, + output_dir: &Path, ) -> anyhow::Result<()> { let manifest_path = source_manifest_path(environment, kind); let previous: Vec = serde_json::from_reader(BufReader::new( @@ -430,7 +459,7 @@ fn sync_source_tree( for entry in &previous { validate_source_path(&entry.path)?; } - let current = source_tree_entries(source_dir, source_sync_mode)?; + let current = staged_entries(source_dir, source_sync_mode, output_dir)?; let current_kinds = current .iter() @@ -470,10 +499,20 @@ pub fn stage_source_tree( package: &SourcePackage, source_sync_mode: SourceSyncMode, incremental: bool, + output_dir: &Path, ) -> anyhow::Result<()> { let source_dir = package.source_dir()?; if source_sync_mode == SourceSyncMode::Tracked { - let untracked = git_untracked_paths(source_dir); + // the artifacts of earlier builds are not the user's to `git add` + let output = owned_output_dir(source_dir, output_dir); + let untracked: Vec = git_untracked_paths(source_dir) + .into_iter() + .filter(|path| { + output + .as_ref() + .is_none_or(|output| !path.starts_with(output)) + }) + .collect(); if !untracked.is_empty() { eprintln!( "debmagic: warning: {} untracked file(s) not staged into the build tree:", @@ -489,10 +528,10 @@ pub fn stage_source_tree( } } if incremental && source_manifest_path(environment, kind).is_file() { - sync_source_tree(environment, kind, source_dir, source_sync_mode) + sync_source_tree(environment, kind, source_dir, source_sync_mode, output_dir) .context("failed to synchronize source tree")?; } else { - let entries = source_tree_entries(source_dir, source_sync_mode)?; + let entries = staged_entries(source_dir, source_sync_mode, output_dir)?; copy_source_entries(source_dir, &stage_dir(environment, kind), &entries) .context("failed to copy source tree to build directory")?; write_source_manifest(environment, kind, &entries)?; @@ -581,6 +620,7 @@ mod tests { BuildKind::Binary, &source_dir, SourceSyncMode::Worktree, + &source_dir.join("build"), )?; let staged = stage_dir(&environment, BuildKind::Binary); @@ -683,6 +723,7 @@ mod tests { &package, SourceSyncMode::Worktree, false, + &source_dir.join("build"), )?; fs::write( stage_dir(&environment, BuildKind::Binary).join("compiled.o"), @@ -698,6 +739,7 @@ mod tests { &package, SourceSyncMode::Worktree, false, + &source_dir.join("build"), )?; assert!( stage_dir(&environment, BuildKind::Source) @@ -718,6 +760,7 @@ mod tests { &package, SourceSyncMode::Worktree, true, + &source_dir.join("build"), )?; assert_eq!( fs::read_to_string(stage_dir(&environment, BuildKind::Source).join("README"))?, @@ -801,6 +844,32 @@ mod tests { Ok(()) } + #[test] + fn staging_skips_the_tagged_output_dir_only() -> anyhow::Result<()> { + let dir = std::env::temp_dir().join(format!("debmagic-output-{}", uuid::Uuid::new_v4())); + // upstream's build/ next to debmagic's output dir + fs::create_dir_all(dir.join("build"))?; + fs::write(dir.join("build/config.sub"), "upstream")?; + crate::output_dir::create(&dir.join("build-debmagic"))?; + fs::write(dir.join("build-debmagic/pkg_1.0_amd64.deb"), "artifact")?; + + let entries = staged_entries(&dir, SourceSyncMode::Worktree, &dir.join("build-debmagic"))?; + let paths: Vec<&Path> = entries.iter().map(|e| e.path.as_path()).collect(); + assert!(paths.contains(&Path::new("build/config.sub"))); + assert!(!paths.iter().any(|path| path.starts_with("build-debmagic"))); + + // -o pointing at upstream's untagged dir stages it + let entries = staged_entries(&dir, SourceSyncMode::Worktree, &dir.join("build"))?; + assert!( + entries + .iter() + .any(|e| e.path == Path::new("build/config.sub")) + ); + + fs::remove_dir_all(dir)?; + Ok(()) + } + #[test] fn committed_sync_requires_clean_worktree() -> anyhow::Result<()> { let repo = git_test_repo()?; diff --git a/packages/debmagic/src/build_intent.rs b/packages/debmagic/src/build_intent.rs index 9140a76b..209728ab 100644 --- a/packages/debmagic/src/build_intent.rs +++ b/packages/debmagic/src/build_intent.rs @@ -68,12 +68,7 @@ pub fn resolve_build_intent(input: BuildIntentInput) -> anyhow::Result std::path::absolute(dir).context("resolving output dir failed")?, - None => std::path::absolute(source_dir.join(&config.output_dir)) - .context("resolving output dir failed")?, - }; + let output_dir = crate::output_dir::resolve(&source_dir, &config, input.output_dir.as_deref())?; if let Some(persistent) = input.persistent { config.driver.persistent = persistent; diff --git a/packages/debmagic/src/config.rs b/packages/debmagic/src/config.rs index b6bd6c5b..556b4867 100644 --- a/packages/debmagic/src/config.rs +++ b/packages/debmagic/src/config.rs @@ -3,6 +3,7 @@ use std::path::{Path, PathBuf}; use crate::build::source::SourceSyncMode; use crate::driver::DriverType; use crate::driver::config::DriverConfig; +use crate::output_dir::OutputDirClash; use crate::sign::{SignMode, SignTool}; use crate::upload::UploadConfig; use crate::upstream::orig::OrigTarballConfig; @@ -226,6 +227,10 @@ pub struct Config { /// Where build artifacts are exported; relative paths resolve against /// the package root. pub output_dir: PathBuf, + /// Used instead of `output_dir` when that holds upstream's files. + pub output_dir_fallback: PathBuf, + /// How to handle an output dir clash with upstream's files. + pub output_dir_clash: OutputDirClash, pub incremental: bool, /// Which source files are staged into the build tree. pub source_sync_mode: SourceSyncMode, @@ -378,6 +383,8 @@ impl Default for Config { driver: DriverConfig::default(), environments_dir: crate::data_dir::default_environments_dir(), output_dir: PathBuf::from("build"), + output_dir_fallback: PathBuf::from("build-debmagic"), + output_dir_clash: OutputDirClash::default(), incremental: false, source_sync_mode: SourceSyncMode::default(), build_debug_symbols: false, diff --git a/packages/debmagic/src/main.rs b/packages/debmagic/src/main.rs index 110928bc..5db7a8d9 100644 --- a/packages/debmagic/src/main.rs +++ b/packages/debmagic/src/main.rs @@ -19,6 +19,7 @@ pub mod driver; pub mod env; pub mod environment; pub mod output; +pub mod output_dir; pub mod package; pub mod requests; pub mod sign; diff --git a/packages/debmagic/src/output_dir.rs b/packages/debmagic/src/output_dir.rs new file mode 100644 index 00000000..8d039c7b --- /dev/null +++ b/packages/debmagic/src/output_dir.rs @@ -0,0 +1,231 @@ +//! The output dir, where builds export artifacts and `upstream` puts orig +//! tarballs. A `CACHEDIR.TAG` marks it as debmagic's: that tells it apart +//! from a `build/` dir upstream ships, and backup tools skip it. + +use std::path::{Component, Path, PathBuf}; + +use anyhow::{Context, bail}; +use serde::{Deserialize, Serialize}; + +use crate::config::Config; + +const TAG_NAME: &str = "CACHEDIR.TAG"; +/// The fixed start of a tag, see . +const TAG_SIGNATURE: &str = "Signature: 8a477f597d28d172789f06886806bc55"; +const OUTPUT_DIR_HINT: &str = + "debmagic output dir: build artifacts and orig tarballs, not part of the package source."; + +/// What to do when the configured output dir holds files debmagic didn't +/// put there, e.g. a `build/` dir upstream ships. +#[derive(Serialize, Deserialize, Debug, Clone, Copy, Default, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum OutputDirClash { + /// Use `output_dir_fallback` instead. + #[default] + Fallback, + /// Fail, asking for another output dir. + Fail, +} + +/// Resolve the output dir: `-o` as given, else the configured one relative +/// to the source dir, or its fallback when upstream owns it. +pub fn resolve(source_dir: &Path, config: &Config, cli: Option<&Path>) -> anyhow::Result { + if let Some(dir) = cli { + return std::path::absolute(dir).context("resolving the output dir failed"); + } + let configured = std::path::absolute(source_dir.join(&config.output_dir)) + .context("resolving the output dir failed")?; + // only a dir in the tree can be upstream's; `..` is never debmagic's alone + if !is_in_tree(&config.output_dir) || !is_foreign(&configured)? { + return Ok(configured); + } + if config.output_dir_clash == OutputDirClash::Fail { + bail!( + "the output dir {} holds files debmagic didn't put there, likely upstream's; \ + set `output_dir` or pass `-o` to use another directory", + configured.display() + ); + } + let fallback = std::path::absolute(source_dir.join(&config.output_dir_fallback)) + .context("resolving the fallback output dir failed")?; + if is_foreign(&fallback)? { + bail!( + "both the output dir {} and its fallback {} hold files debmagic didn't put there; \ + set `output_dir` or pass `-o` to use another directory", + configured.display(), + fallback.display() + ); + } + println!( + "debmagic: {} belongs to upstream, using {}", + configured.display(), + fallback.display() + ); + Ok(fallback) +} + +/// Where `upstream switch` moves the output dir when the new upstream +/// version starts shipping a directory of its name; `None` when the +/// output dir is explicit (`-o`) or clashes are configured to fail. +pub fn switch_fallback( + source_dir: &Path, + config: &Config, + cli: Option<&Path>, +) -> anyhow::Result> { + if cli.is_some() + || config.output_dir_clash == OutputDirClash::Fail + || !is_in_tree(&config.output_dir) + { + return Ok(None); + } + std::path::absolute(source_dir.join(&config.output_dir_fallback)) + .map(Some) + .context("resolving the fallback output dir failed") +} + +/// Create the output dir and tag it as debmagic's, unless it holds other +/// files (`-o ..`, upstream's `build/`). +pub fn create(dir: &Path) -> anyhow::Result<()> { + std::fs::create_dir_all(dir).with_context(|| format!("failed to create {}", dir.display()))?; + if !is_foreign(dir)? { + tag(dir, OUTPUT_DIR_HINT) + .with_context(|| format!("failed to tag {} as debmagic's", dir.display()))?; + } + Ok(()) +} + +/// Write a `CACHEDIR.TAG` into `dir`, with `hint` telling humans what it is. +pub fn tag(dir: &Path, hint: &str) -> std::io::Result<()> { + if is_tagged(dir) { + return Ok(()); + } + std::fs::write( + dir.join(TAG_NAME), + format!("{TAG_SIGNATURE}\n# {hint}\n# More info: https://bford.info/cachedir/\n"), + ) +} + +pub fn is_tagged(dir: &Path) -> bool { + std::fs::read(dir.join(TAG_NAME)) + .is_ok_and(|content| content.starts_with(TAG_SIGNATURE.as_bytes())) +} + +fn is_in_tree(dir: &Path) -> bool { + dir.components() + .all(|component| matches!(component, Component::Normal(_) | Component::CurDir)) +} + +/// Whether `dir` holds files debmagic didn't put there: it is untagged and +/// holds more than artifacts (an output dir from before the tag holds only those). +fn is_foreign(dir: &Path) -> anyhow::Result { + if is_tagged(dir) { + return Ok(false); + } + let entries = match std::fs::read_dir(dir) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(error) => { + return Err(error).with_context(|| format!("failed to read {}", dir.display())); + } + }; + for entry in entries { + let entry = entry.with_context(|| format!("failed to read {}", dir.display()))?; + let is_dir = entry.file_type().is_ok_and(|kind| kind.is_dir()); + let name = entry.file_name(); + if is_dir || !debmagic_common::changes::is_artifact_filename(&name.to_string_lossy()) { + return Ok(true); + } + } + Ok(false) +} + +#[cfg(test)] +mod tests { + use super::*; + use test_case::test_case; + + fn test_dir() -> PathBuf { + let dir = std::env::temp_dir().join(format!("debmagic-output-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&dir).unwrap(); + dir + } + + fn config(clash: OutputDirClash) -> Config { + Config { + output_dir_clash: clash, + ..Config::default() + } + } + + #[test_case(&[], "build"; "no build dir")] + #[test_case(&["pkg_1.0-1_amd64.changes", "pkg_1.0.orig.tar.xz"], "build"; "untagged artifacts")] + #[test_case(&["config.guess"], "build-debmagic"; "upstream build dir")] + fn test_resolve(files: &[&str], expected: &str) { + let source_dir = test_dir(); + if !files.is_empty() { + std::fs::create_dir_all(source_dir.join("build")).unwrap(); + } + for file in files { + std::fs::write(source_dir.join("build").join(file), "").unwrap(); + } + + let resolved = resolve(&source_dir, &config(OutputDirClash::Fallback), None).unwrap(); + + assert_eq!(resolved, source_dir.join(expected)); + std::fs::remove_dir_all(&source_dir).unwrap(); + } + + #[test] + fn test_resolve_tagged_upstream_content_is_ours() { + let source_dir = test_dir(); + create(&source_dir.join("build")).unwrap(); + std::fs::write(source_dir.join("build/config.guess"), "").unwrap(); + + let resolved = resolve(&source_dir, &config(OutputDirClash::Fallback), None).unwrap(); + + assert_eq!(resolved, source_dir.join("build")); + std::fs::remove_dir_all(&source_dir).unwrap(); + } + + #[test] + fn test_resolve_clash_fails_when_configured() { + let source_dir = test_dir(); + std::fs::create_dir_all(source_dir.join("build/aux")).unwrap(); + + let error = resolve(&source_dir, &config(OutputDirClash::Fail), None).unwrap_err(); + + assert!(error.to_string().contains("didn't put there")); + std::fs::remove_dir_all(&source_dir).unwrap(); + } + + #[test] + fn test_resolve_outside_the_tree_is_taken_as_is() { + let source_dir = test_dir(); + let config = Config { + output_dir: PathBuf::from(".."), + ..Config::default() + }; + + let resolved = resolve(&source_dir, &config, None).unwrap(); + + assert_eq!(resolved, source_dir.join("..")); + std::fs::remove_dir_all(&source_dir).unwrap(); + } + + #[test] + fn test_create_tags_only_debmagic_dirs() { + let base = test_dir(); + create(&base.join("new")).unwrap(); + std::fs::create_dir_all(base.join("upstream/aux")).unwrap(); + create(&base.join("upstream")).unwrap(); + + assert!(is_tagged(&base.join("new"))); + assert!(!is_tagged(&base.join("upstream"))); + assert!( + std::fs::read_to_string(base.join("new").join(TAG_NAME)) + .unwrap() + .contains("debmagic") + ); + std::fs::remove_dir_all(&base).unwrap(); + } +} diff --git a/packages/debmagic/src/sign.rs b/packages/debmagic/src/sign.rs index c13cb01a..6c773418 100644 --- a/packages/debmagic/src/sign.rs +++ b/packages/debmagic/src/sign.rs @@ -500,12 +500,8 @@ pub fn sign(input: SignIntent) -> anyhow::Result<()> { Some(file) => std::path::absolute(file).context("resolving the file to sign failed")?, None => { let identity = crate::package::load_package(&source_dir)?; - // -o wins; else the config value, relative to the package root. - let output_dir = match &input.output_dir { - Some(dir) => std::path::absolute(dir).context("resolving output dir failed")?, - None => std::path::absolute(source_dir.join(&config.output_dir)) - .context("resolving output dir failed")?, - }; + let output_dir = + crate::output_dir::resolve(&source_dir, &config, input.output_dir.as_deref())?; find_changes_file( identity.name(), &identity.version().to_string(), diff --git a/packages/debmagic/src/test/run.rs b/packages/debmagic/src/test/run.rs index f95ff46f..d1c82e18 100644 --- a/packages/debmagic/src/test/run.rs +++ b/packages/debmagic/src/test/run.rs @@ -297,6 +297,7 @@ pub async fn run_test(intent: &TestIntent) -> anyhow::Result { &package, intent.config.source_sync_mode, false, + &crate::output_dir::resolve(&intent.source_dir, &intent.config, None)?, )?; copy_changes_artifacts(&changes_path, &environment.work_dir()) }, diff --git a/packages/debmagic/src/upload.rs b/packages/debmagic/src/upload.rs index 85963af1..d4d590e1 100644 --- a/packages/debmagic/src/upload.rs +++ b/packages/debmagic/src/upload.rs @@ -101,8 +101,7 @@ pub fn upload(input: UploadIntent) -> anyhow::Result<()> { let changes_file = match &input.changes { Some(file) => std::path::absolute(file).context("resolving the changes file failed")?, None => { - let output_dir = std::path::absolute(source_dir.join(&config.output_dir)) - .context("resolving output dir failed")?; + let output_dir = crate::output_dir::resolve(&source_dir, &config, None)?; crate::sign::find_changes_file( identity.name(), &identity.version().to_string(), diff --git a/packages/debmagic/src/upstream.rs b/packages/debmagic/src/upstream.rs index 9a9f6e86..4191f935 100644 --- a/packages/debmagic/src/upstream.rs +++ b/packages/debmagic/src/upstream.rs @@ -111,12 +111,13 @@ async fn run_fetch(command: UpstreamFetchCommand, switch_tree: bool) -> anyhow:: if let Some(verify_command) = command.verify_command { config.sign.verify_command = Some(verify_command); } - let output_dir = std::path::absolute( - command - .output_dir - .unwrap_or_else(|| source_dir.join(&config.output_dir)), - ) - .context("resolving the output dir failed")?; + let output_dir = + crate::output_dir::resolve(&source_dir, &config, command.output_dir.as_deref())?; + let output_dir_fallback = + crate::output_dir::switch_fallback(&source_dir, &config, command.output_dir.as_deref())?; + if !command.dry_run { + crate::output_dir::create(&output_dir)?; + } let options = fetch::FetchOptions { from: command.from, orig_tarball: &config.orig_tarball, @@ -126,6 +127,7 @@ async fn run_fetch(command: UpstreamFetchCommand, switch_tree: bool) -> anyhow:: .unwrap_or(config.upstream.signature_check), sign: &config.sign, output_dir: &output_dir, + output_dir_fallback: output_dir_fallback.as_deref(), force: command.force, dry_run: command.dry_run, }; @@ -144,26 +146,35 @@ async fn run_fetch(command: UpstreamFetchCommand, switch_tree: bool) -> anyhow:: .version .unwrap_or_else(|| package.version().upstream_version().to_string()); - let run = async |source: &watch::WatchSource, version: &str| { + let run = async |source: &watch::WatchSource, version: &str, options: &fetch::FetchOptions| { let what = match &source.component { Some(component) => format!("component {component} of {}", package.name()), None => package.name().to_string(), }; let result = if switch_tree { - switch::switch(&package, source, version, &options, &components).await + switch::switch(&package, source, version, options, &components).await } else { - fetch::fetch(&package, source, version, &options).await + fetch::fetch(&package, source, version, options).await }; result.with_context(|| format!("fetching {what} {version} failed")) }; - let main = run(main_source, &requested).await?; + let main = run(main_source, &requested, &options).await?; + // the switch may have moved the output dir aside for upstream's + let output_dir = match main.tarball.parent() { + Some(dir) if !command.dry_run => dir.to_path_buf(), + _ => output_dir.clone(), + }; + let options = fetch::FetchOptions { + output_dir: &output_dir, + ..options + }; // MUT: the components follow the main tarball's version for source in &sources { if source.untrackable.is_some() || source.component.is_none() { continue; } - run(source, &main.upstream_version).await?; + run(source, &main.upstream_version, &options).await?; } Ok(()) } diff --git a/packages/debmagic/src/upstream/fetch.rs b/packages/debmagic/src/upstream/fetch.rs index 8a2b267d..830f5530 100644 --- a/packages/debmagic/src/upstream/fetch.rs +++ b/packages/debmagic/src/upstream/fetch.rs @@ -59,6 +59,8 @@ pub struct FetchOptions<'a> { pub signature_check: bool, pub sign: &'a SignConfig, pub output_dir: &'a Path, + /// If upstream ships the output dir, use this instead (`None` fails) + pub output_dir_fallback: Option<&'a Path>, /// Replace an existing orig tarball instead of keeping it. pub force: bool, /// Only report what would happen. @@ -802,6 +804,7 @@ mod tests { signature_check: false, sign: &sign, output_dir: &output_dir, + output_dir_fallback: None, force: false, dry_run: false, }; diff --git a/packages/debmagic/src/upstream/orig.rs b/packages/debmagic/src/upstream/orig.rs index 3efb49c0..ca5c587f 100644 --- a/packages/debmagic/src/upstream/orig.rs +++ b/packages/debmagic/src/upstream/orig.rs @@ -94,16 +94,34 @@ fn substitute_placeholders( ) } +/// The download cache: `/debmagic/orig/`. +fn cache_root() -> Option { + dirs::cache_dir().map(|cache| cache.join("debmagic").join("orig")) +} + /// The download cache dir for one package version: /// `/debmagic/orig///`. fn cache_dir(name: &str, upstream_version: &str) -> Option { - dirs::cache_dir().map(|cache| { - cache - .join("debmagic") - .join("orig") - .join(name) - .join(upstream_version) - }) + cache_root().map(|root| root.join(name).join(upstream_version)) +} + +/// Copy a downloaded tarball into its cache dir. A failing cache must +/// never fail the build, so errors are only reported. +fn populate_cache(cache_dir: &Path, orig_path: &Path) { + let result = std::fs::create_dir_all(cache_dir) + .and_then(|()| match cache_root() { + Some(root) => crate::output_dir::tag(&root, "debmagic orig tarball download cache."), + None => Ok(()), + }) + .and_then(|()| { + std::fs::copy( + orig_path, + cache_dir.join(orig_path.file_name().unwrap_or_default()), + ) + }); + if let Err(error) = result { + println!("debmagic: populating the orig cache failed: {error}"); + } } /// Fetch the `orig` tarball for `package` into `output_dir`, unless a @@ -343,13 +361,7 @@ async fn fetch_from_distro_archive( .with_context(|| format!("downloading {orig_name} failed"))?; if let Some(cache_dir) = &cache { - let cached = cache_dir.join(orig_name); - if let Err(error) = - std::fs::create_dir_all(cache_dir).and_then(|()| std::fs::copy(&orig_path, &cached)) - { - // a failing cache must never fail the build - println!("debmagic: populating the orig cache failed: {error}"); - } + populate_cache(cache_dir, &orig_path); } Ok(Some(orig_path)) @@ -488,13 +500,7 @@ async fn fetch_from_pool( } if let Some(cache_dir) = &cache { - let cached = cache_dir.join(&orig_name); - if let Err(error) = - std::fs::create_dir_all(cache_dir).and_then(|()| std::fs::copy(&orig_path, &cached)) - { - // a failing cache must never fail the build - println!("debmagic: populating the orig cache failed: {error}"); - } + populate_cache(cache_dir, &orig_path); } return Ok(Some(orig_path)); diff --git a/packages/debmagic/src/upstream/switch.rs b/packages/debmagic/src/upstream/switch.rs index f52714a6..2fa37858 100644 --- a/packages/debmagic/src/upstream/switch.rs +++ b/packages/debmagic/src/upstream/switch.rs @@ -1,6 +1,6 @@ -use std::path::{Path, PathBuf}; +use std::path::{Component, Path, PathBuf}; -use anyhow::Context; +use anyhow::{Context, bail}; use debmagic_common::package::SourcePackage; use crate::upstream::fetch::{FetchOptions, Fetched, extract_tarball, fetch}; @@ -19,7 +19,7 @@ pub async fn switch( components: &[String], ) -> anyhow::Result { let source_dir = package.source_dir()?; - let fetched = fetch(package, source, requested, options).await?; + let mut fetched = fetch(package, source, requested, options).await?; let target_dir = match &source.component { Some(component) => source_dir.join(component), None => source_dir.to_path_buf(), @@ -49,19 +49,24 @@ pub async fn switch( .map(|component| source_dir.join(component)) .collect(), }; - keep.push(options.output_dir.to_path_buf()); + let output_dir = make_room_for_upstream(&target_dir, &extract_dir, options)?; + keep.push(output_dir.clone()); std::fs::create_dir_all(&target_dir) .with_context(|| format!("failed to create {}", target_dir.display()))?; - swap_tree(&target_dir, &extract_dir, &keep, source.component.is_none()) + swap_tree(&target_dir, &extract_dir, &keep, source.component.is_none())?; + Ok(output_dir) }); std::fs::remove_dir_all(&work_dir).ok(); - swapped.with_context(|| { + let output_dir = swapped.with_context(|| { format!( "replacing {} with {} failed", target_dir.display(), fetched.tarball.display() ) })?; + if let Ok(relative) = fetched.tarball.strip_prefix(options.output_dir) { + fetched.tarball = output_dir.join(relative); + } println!( "debmagic: switched {} to upstream {} (orig: {})", target_dir.display(), @@ -71,6 +76,56 @@ pub async fn switch( Ok(fetched) } +/// The output dir the swap must keep. When the tarball ships the top-level +/// entry holding it, the swap would merge upstream's files into the +/// artifacts: a debmagic-owned output dir moves to the fallback, anything +/// else fails. +fn make_room_for_upstream( + target_dir: &Path, + extract_dir: &Path, + options: &FetchOptions, +) -> anyhow::Result { + let output_dir = options.output_dir; + let Ok(relative) = output_dir.strip_prefix(target_dir) else { + return Ok(output_dir.to_path_buf()); + }; + let Some(Component::Normal(top)) = relative.components().next() else { + return Ok(output_dir.to_path_buf()); + }; + if std::fs::symlink_metadata(extract_dir.join(top)).is_err() { + return Ok(output_dir.to_path_buf()); + } + if let Some(fallback) = options.output_dir_fallback + && target_dir.join(top) == output_dir + && crate::output_dir::is_tagged(output_dir) + && std::fs::symlink_metadata(fallback).is_err() + && extract_dir + .join(fallback.strip_prefix(target_dir).unwrap_or(fallback)) + .symlink_metadata() + .is_err() + { + std::fs::rename(output_dir, fallback).with_context(|| { + format!( + "failed to move {} to {}", + output_dir.display(), + fallback.display() + ) + })?; + println!( + "debmagic: upstream ships its own {}, moved the output dir to {}", + top.to_string_lossy(), + fallback.display() + ); + return Ok(fallback.to_path_buf()); + } + bail!( + "the upstream tarball ships its own {}, which holds the output dir {}; \ + set `output_dir` or pass `-o` to use another directory", + top.to_string_lossy(), + output_dir.display() + ); +} + /// Move one entry, falling back to copy+remove when a rename crosses /// filesystems (the work dir is in /tmp, the source tree may not be). fn move_entry(from: &Path, to: &Path) -> anyhow::Result<()> { @@ -184,6 +239,63 @@ fn swap_tree( #[cfg(test)] mod tests { use super::*; + use test_case::test_case; + + #[test_case("build", &["build"], true, true, Some("build-debmagic"); "own output dir moves aside")] + #[test_case("build", &["build"], false, true, None; "untagged output dir")] + #[test_case("build", &["build"], true, false, None; "clash configured to fail")] + #[test_case("build", &["build", "build-debmagic"], true, true, None; "fallback shipped too")] + #[test_case("build/out", &["build"], true, true, None; "output dir below a shipped dir")] + #[test_case("build", &["src"], true, true, Some("build"); "output dir not shipped")] + #[test_case("/elsewhere/out", &["build"], false, false, Some("/elsewhere/out"); "output dir outside the tree")] + #[test_case("..", &["build"], false, false, Some(".."); "output dir is the parent")] + fn test_make_room_for_upstream( + output_dir: &str, + shipped: &[&str], + tagged: bool, + fallback: bool, + expected: Option<&str>, + ) { + let base = std::env::temp_dir().join(format!("debmagic-clash-{}", uuid::Uuid::new_v4())); + let target_dir = base.join("pkg"); + let extract_dir = base.join("tree"); + let output_dir = target_dir.join(output_dir); + std::fs::create_dir_all(&target_dir).unwrap(); + if tagged { + crate::output_dir::create(&output_dir).unwrap(); + } + for name in shipped { + std::fs::create_dir_all(extract_dir.join(name)).unwrap(); + } + let orig_tarball = crate::upstream::orig::OrigTarballConfig::default(); + let sign = crate::config::SignConfig::default(); + let fallback_dir = target_dir.join("build-debmagic"); + let options = FetchOptions { + from: crate::upstream::fetch::FetchFrom::Upstream, + orig_tarball: &orig_tarball, + repack: false, + signature_check: false, + sign: &sign, + output_dir: &output_dir, + output_dir_fallback: fallback.then_some(fallback_dir.as_path()), + force: false, + dry_run: false, + }; + + let result = make_room_for_upstream(&target_dir, &extract_dir, &options); + + match expected { + Some(expected) => { + let kept = result.unwrap(); + assert_eq!(kept, target_dir.join(expected)); + if tagged { + assert!(crate::output_dir::is_tagged(&kept)); + } + } + None => assert!(result.is_err()), + } + std::fs::remove_dir_all(&base).unwrap(); + } #[test] fn test_extract_and_swap() { From ea39ad65325da8e80be1c24af675d11bf1b0973e Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Wed, 14 Oct 2026 17:19:40 +0200 Subject: [PATCH 31/48] fix(build): parse debian/control with comments inside a field `build source` fails with "invalid deb822: expected key" when a continuation field contains a `#` comment line (e.g. openldap's Build-Depends). deb822-lossless >= 0.5 parses these like dpkg does; raise it and debian-control >= 0.3, which depends on it. --- Cargo.lock | 23 +++++++++++++------ Cargo.toml | 4 ++-- debian/control | 4 ++-- .../debmagic-common/src/debian/control.rs | 1 + 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b16b7ec5..9a2fbab1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -509,9 +509,9 @@ dependencies = [ [[package]] name = "deb822-derive" -version = "0.2.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b6e5cafe61e77421a090e2a33b8a2e4e2ff1b106fd906ebade111307064d981" +checksum = "e83ef29a094bcb2b7dd0f609ace7f5a34ef9a62e0731ebd350637640320a3b15" dependencies = [ "proc-macro2", "quote", @@ -519,12 +519,20 @@ dependencies = [ ] [[package]] -name = "deb822-lossless" -version = "0.2.4" +name = "deb822-fast" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812bb5c8052a89edc6d45d1bc3b3400e8186dd166e9b0a9520bfa5a2bd8477ee" +checksum = "114c474fa4cd5d6d24bb5e68b36fa4ef70f5b830e3cc14a9b66a12e71a15aeb9" dependencies = [ "deb822-derive", +] + +[[package]] +name = "deb822-lossless" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45bfeac719b1f3441c6f8c29652c60376e11eabbdbd99c53b3f14e0e8585d053" +dependencies = [ "regex", "rowan", "serde", @@ -547,11 +555,12 @@ dependencies = [ [[package]] name = "debian-control" -version = "0.1.39" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3367f1dc94222cb9cf16e07c782e442b1928a26dd8dcee3aa4d2dbd32ca11d21" +checksum = "99ffab14a76cf08121d4bf7fb37b40bff0985e49cf050a7cc00b402e0d31e00d" dependencies = [ "chrono", + "deb822-fast", "deb822-lossless", "debversion", "regex", diff --git a/Cargo.toml b/Cargo.toml index 80522b4e..12ecbbb4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,8 +29,8 @@ reqwest = { version = ">=0.13", default-features = false, features = ["rustls", futures-util = "0.3" tokio = { version = ">=1.43", features = ["macros", "rt-multi-thread", "sync", "time"] } debian-changelog = { version = ">=0.2.14" } -debian-control = { version = ">=0.1.39" } -deb822-lossless = { version = ">=0.2" } +debian-control = { version = ">=0.3" } +deb822-lossless = { version = ">=0.5" } ignore = { version = ">=0.4.25" } md-5 = { version = ">=0.11" } sha1 = { version = ">=0.11" } diff --git a/debian/control b/debian/control index ec31dd9e..c2ff096d 100644 --- a/debian/control +++ b/debian/control @@ -31,9 +31,9 @@ Build-Depends: librust-tokio-dev (>= 1.43), librust-test-case-dev (>=3.3.1), librust-pyo3-dev (>=0.27.2), - librust-deb822-lossless-dev (>=0.5.18), + librust-deb822-lossless-dev (>=0.5.19), librust-debian-changelog-dev (>=0.2.14), - librust-debian-control-dev (>= 0.1.39), + librust-debian-control-dev (>= 0.3.17), librust-ignore-dev (>=0.4.25), librust-md-5-dev (>= 0.11), librust-sha1-dev (>= 0.11), diff --git a/packages/debmagic-common/src/debian/control.rs b/packages/debmagic-common/src/debian/control.rs index c241009a..795b9983 100644 --- a/packages/debmagic-common/src/debian/control.rs +++ b/packages/debmagic-common/src/debian/control.rs @@ -40,6 +40,7 @@ mod tests { #[test_case("Source: foo\nRules-Requires-Root: binary-targets\n", true ; "binary targets")] #[test_case("Source: foo\nRules-Requires-Root: dpkg/target-subcommand\n", true ; "implementation keyword")] #[test_case("Source: foo\n\nPackage: foo\nRules-Requires-Root: no\n", true ; "absent from source paragraph")] + #[test_case("Source: foo\nBuild-Depends: a,\n# comment\n b\nRules-Requires-Root: no\n", false ; "comment in continuation")] fn test_needs_gain_root(control: &str, expected: bool) { assert_eq!(Control::parse(control).unwrap().needs_gain_root(), expected); } From 3f40b73ef4c9e1f5c383752d8f28ce3564d7a02b Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Wed, 14 Oct 2026 17:25:41 +0200 Subject: [PATCH 32/48] fix(build): don't truncate an orig tarball staged twice A second `build source` empties the orig tarball (and the file it is hardlinked to, e.g. in the git-ubuntu cache): the work dir already holds a hardlink of it, and copying `..`'s symlink onto it truncates the shared inode. Stage the tarballs from `..` and the test artifacts by replacing the destination instead. --- packages/debmagic/src/build/artifacts.rs | 5 ++-- packages/debmagic/src/build/source.rs | 29 +++++++++++++++++++++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/packages/debmagic/src/build/artifacts.rs b/packages/debmagic/src/build/artifacts.rs index 4fb43f2b..99dae1b7 100644 --- a/packages/debmagic/src/build/artifacts.rs +++ b/packages/debmagic/src/build/artifacts.rs @@ -5,6 +5,7 @@ use std::{ }; use crate::build_intent::BuildKind; +use crate::upstream::orig::link_or_copy; use anyhow::{Context, anyhow, bail}; use debmagic_common::changes::is_source_changes; @@ -169,8 +170,8 @@ pub fn copy_changes_artifacts(changes_path: &Path, dest_dir: &Path) -> anyhow::R } let destination = dest_dir.join(filename); reject_destination_symlink(&destination)?; - fs::copy(&source, destination) - .with_context(|| format!("failed to copy build artifact {}", source.display()))?; + link_or_copy(&source, &destination) + .with_context(|| format!("failed to stage build artifact {}", source.display()))?; } let changes_filename = changes_path diff --git a/packages/debmagic/src/build/source.rs b/packages/debmagic/src/build/source.rs index dc0af797..dc090f35 100644 --- a/packages/debmagic/src/build/source.rs +++ b/packages/debmagic/src/build/source.rs @@ -20,6 +20,7 @@ use clap::ValueEnum; use crate::build_intent::BuildKind; use crate::driver::Environment; +use crate::upstream::orig::link_or_copy; use debmagic_common::package::SourcePackage; /// The staged tree a build kind uses inside the work dir. The `-binary` tree @@ -425,7 +426,7 @@ fn copy_glob(src_dir: &Path, pattern: &str, dest_dir: &Path) -> anyhow::Result<( "Could not retrieve filename from {}", path.display() ))?; - fs::copy(&path, dest_dir.join(filename))?; + link_or_copy(&path, &dest_dir.join(filename))?; } } Ok(()) @@ -642,6 +643,32 @@ mod tests { Ok(()) } + /// The orig tarball in `..` symlinks to a file already hardlinked into + /// the work dir: re-staging it must not truncate the shared inode. + #[test] + fn copy_glob_keeps_hardlinked_destination() -> anyhow::Result<()> { + let test_root = + std::env::temp_dir().join(format!("debmagic-copy-glob-test-{}", uuid::Uuid::new_v4())); + let parent = test_root.join("parent"); + let work = test_root.join("work"); + fs::create_dir_all(&parent)?; + fs::create_dir_all(&work)?; + let cached = test_root.join("cached.orig.tar.xz"); + fs::write(&cached, "tarball")?; + symlink(&cached, parent.join("pkg_1.0.orig.tar.xz"))?; + fs::hard_link(&cached, work.join("pkg_1.0.orig.tar.xz"))?; + + copy_glob(&parent, "pkg_1.0.orig.tar*", &work)?; + + assert_eq!(fs::read_to_string(&cached)?, "tarball"); + assert_eq!( + fs::read_to_string(work.join("pkg_1.0.orig.tar.xz"))?, + "tarball" + ); + fs::remove_dir_all(test_root)?; + Ok(()) + } + #[test] fn source_manifest_paths_must_be_relative_and_normal() { assert!(validate_source_path(Path::new("debian/control")).is_ok()); From e0fd7c7d0808bae141ea282369abc408cffb0cf9 Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Wed, 14 Oct 2026 17:28:02 +0200 Subject: [PATCH 33/48] test(config): ignore the user-global config in unit tests --- packages/debmagic/src/config.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/debmagic/src/config.rs b/packages/debmagic/src/config.rs index 556b4867..3b9a7dfb 100644 --- a/packages/debmagic/src/config.rs +++ b/packages/debmagic/src/config.rs @@ -24,6 +24,10 @@ pub enum ConfigPathStatus { /// default XDG location (like git's `GIT_CONFIG_GLOBAL`); `/dev/null` /// disables it. fn global_config_path() -> anyhow::Result { + // unit tests must not depend on the developer's config + if cfg!(test) { + return Ok(PathBuf::from("/dev/null")); + } if let Ok(path) = std::env::var("DEBMAGIC_CONFIG_GLOBAL") { return Ok(PathBuf::from(path)); } From 0eb3f95778e0620bbe42a278a0fc326fd829cb5c Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Wed, 14 Oct 2026 17:38:20 +0200 Subject: [PATCH 34/48] fix(config): let `config set` create debian/debmagic.toml by default Before, `config set` changed the user-global config when debian/debmagic.toml was missing --- packages/debmagic/src/config.rs | 38 ++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/packages/debmagic/src/config.rs b/packages/debmagic/src/config.rs index 3b9a7dfb..844321ce 100644 --- a/packages/debmagic/src/config.rs +++ b/packages/debmagic/src/config.rs @@ -118,8 +118,8 @@ impl ConfigPath { /// Config file that `set` should target, selected from the candidate /// locations: the explicit `-c` file, else with `global` the user-global -/// config file (created on demand), else the project `debian/debmagic.toml` -/// if it exists, else the user-global config file. +/// config file, else the project `debian/debmagic.toml`. The global and +/// project files are created on demand. pub fn resolve_set_target( source_dir: Option<&Path>, config_file: Option<&Path>, @@ -131,21 +131,18 @@ pub fn resolve_set_target( return Ok(explicit.clone()); } - if !global - && let Some(project) = paths - .iter() - .find(|p| p.layer == ConfigLayer::Project && p.is_used()) - { - return Ok(project.clone()); - } - - let global_config = paths + let layer = if global { + ConfigLayer::Global + } else { + ConfigLayer::Project + }; + let target = paths .iter() - .find(|p| p.layer == ConfigLayer::Global) - .context("the user-global config location is unknown")?; + .find(|p| p.layer == layer) + .with_context(|| format!("the {layer:?} config location is unknown"))?; Ok(ConfigPath::new( - ConfigLayer::Global, - global_config.path.clone(), + layer, + target.path.clone(), ConfigPathStatus::Used, )) } @@ -535,6 +532,17 @@ mod tests { Ok(()) } + #[test] + fn set_targets_the_project_config_unless_global() -> anyhow::Result<()> { + let source_dir = Path::new("/src/pkg"); + let target = resolve_set_target(Some(source_dir), None, false)?; + assert_eq!(target.layer, ConfigLayer::Project); + assert_eq!(target.path, source_dir.join("debian/debmagic.toml")); + let target = resolve_set_target(Some(source_dir), None, true)?; + assert_eq!(target.layer, ConfigLayer::Global); + Ok(()) + } + #[test] fn it_loads_a_simple_config() -> Result<(), anyhow::Error> { let test_asset_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) From 8fd41bef9abe022f6fa5d7a78ce452369b9430f1 Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Wed, 14 Oct 2026 18:43:08 +0200 Subject: [PATCH 35/48] refactor(common): move artifact filename knowledge out of changes Orig tarball naming, `is_source_changes` and `is_artifact_filename` need no `.changes` file, so they move to `debmagic_common::artifact`; `changes` keeps the `.changes` entry handling. --- packages/debmagic-common/src/artifact.rs | 138 +++++++++++++++++++++++ packages/debmagic-common/src/changes.rs | 132 +--------------------- packages/debmagic-common/src/lib.rs | 1 + packages/debmagic/src/build/artifacts.rs | 2 +- packages/debmagic/src/build/source.rs | 7 +- packages/debmagic/src/changes.rs | 2 +- packages/debmagic/src/output_dir.rs | 2 +- packages/debmagic/src/upload/orig.rs | 2 +- packages/debmagic/src/upstream/fetch.rs | 2 +- packages/debmagic/src/upstream/orig.rs | 8 +- 10 files changed, 154 insertions(+), 142 deletions(-) create mode 100644 packages/debmagic-common/src/artifact.rs diff --git a/packages/debmagic-common/src/artifact.rs b/packages/debmagic-common/src/artifact.rs new file mode 100644 index 00000000..c75b7f78 --- /dev/null +++ b/packages/debmagic-common/src/artifact.rs @@ -0,0 +1,138 @@ +//! Naming conventions of build and upload artifacts: orig tarballs, +//! `.changes` files and everything else a build or upload leaves behind. + +/// Whether a `.changes` filename names a source-only upload: +/// `dpkg-buildpackage -S` writes `__source.changes`, while +/// binary builds name the file after the build architecture instead. +pub fn is_source_changes(name: &str) -> bool { + let Some(stem) = name.strip_suffix(".changes") else { + return false; + }; + stem.ends_with("_source") +} + +/// Whether a filename is an `orig` tarball (main or component). +pub fn is_orig_tarball(name: &str) -> bool { + // main: foo_1.0.orig.tar.gz; component: foo_1.0.orig-bar.tar.gz + let Some((_, rest)) = name.split_once(".orig") else { + return false; + }; + if rest.starts_with(".tar.") { + return true; + } + // component: the part between `.orig-` and `.tar.` is the component name + match rest.strip_prefix('-').and_then(|r| r.split_once(".tar.")) { + Some((component, _)) => !component.is_empty() && !component.contains('/'), + None => false, + } +} + +/// Whether a filename looks like a build or upload artifact: +/// `_...` with a package, source, `.changes`, `.buildinfo`, +/// dput `.upload` or debmagic `.upload.json` suffix, or a tarball, possibly +/// with an `.asc` signature. +pub fn is_artifact_filename(name: &str) -> bool { + const SUFFIXES: [&str; 9] = [ + ".changes", + ".buildinfo", + ".dsc", + ".deb", + ".udeb", + ".ddeb", + ".upload", + ".upload.json", + ".diff.gz", + ]; + let name = name.strip_suffix(".asc").unwrap_or(name); + name.contains('_') + && (name.contains(".tar.") || SUFFIXES.iter().any(|suffix| name.ends_with(suffix))) +} + +/// The `orig` tarball filename prefix for a source package version: +/// `_.orig.tar.` — the compression extension varies. +/// For a component, `_.orig-.tar.`. +pub fn orig_prefix(name: &str, upstream_version: &str) -> String { + format!("{name}_{upstream_version}.orig.tar.") +} + +/// The `orig` tarball filename prefix for a component of a source +/// package version: `_.orig-.tar.`. +pub fn component_orig_prefix(name: &str, upstream_version: &str, component: &str) -> String { + format!("{name}_{upstream_version}.orig-{component}.tar.") +} + +/// The possible `orig` tarball filenames of a source package version, +/// one per compression dpkg-source accepts (its `%COMP` table): the main +/// tarball, or a component's when `component` is set. +pub fn orig_filenames(name: &str, upstream_version: &str, component: Option<&str>) -> Vec { + let prefix = match component { + None => orig_prefix(name, upstream_version), + Some(component) => component_orig_prefix(name, upstream_version, component), + }; + ["gz", "xz", "bz2", "lzma"] + .iter() + .map(|ext| format!("{prefix}{ext}")) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use test_case::test_case; + + #[test_case("pkg_1.0.orig.tar.gz", true; "main orig")] + #[test_case("pkg_1.0.orig.tar.xz", true; "main orig xz")] + #[test_case("pkg_1.0.orig-bar.tar.xz", true; "component orig")] + #[test_case("pkg_1.0-1.debian.tar.xz", false; "debian tarball")] + #[test_case("pkg_1.0-1.dsc", false; "dsc")] + #[test_case("pkg_1.0.orig.tar", false; "no compression ext")] + fn test_is_orig_tarball(name: &str, expected: bool) { + assert_eq!(is_orig_tarball(name), expected); + } + + #[test_case("pkg_1.0-1_source.changes", true; "source upload")] + #[test_case("pkg_1.0-1_amd64.changes", false; "binary upload")] + #[test_case("pkg_1.0-1_all.changes", false; "arch-indep upload")] + #[test_case("pkg_1.0-1_multi.changes", false; "multiarch upload")] + #[test_case("pkg_1.0-1_amd64+all.changes", false; "combined upload")] + #[test_case("pkg_1.0-1_amd64v3.changes", false; "arch variant upload")] + #[test_case("pkg_1.0-1_source.buildinfo", false; "not a changes file")] + fn test_is_source_changes(name: &str, expected: bool) { + assert_eq!(is_source_changes(name), expected); + } + + #[test_case("pkg_1.0.orig.tar.xz", true; "orig")] + #[test_case("pkg_1.0.orig.tar.xz.asc", true; "orig signature")] + #[test_case("pkg_1.0-1.debian.tar.xz", true; "debian tarball")] + #[test_case("pkg_1.0.tar.xz", true; "native tarball")] + #[test_case("pkg_1.0-1.dsc", true; "dsc")] + #[test_case("pkg_1.0-1.diff.gz", true; "1.0 diff")] + #[test_case("libpkg1_1.0-1_amd64.deb", true; "deb")] + #[test_case("pkg-dbgsym_1.0-1_amd64.ddeb", true; "ddeb")] + #[test_case("pkg_1.0-1_amd64.buildinfo", true; "buildinfo")] + #[test_case("pkg_1.0-1_source.ppa.upload", true; "dput log")] + #[test_case("pkg_1.0-1_source.upload.json", true; "debmagic upload record")] + #[test_case("config.guess", false; "upstream script")] + #[test_case("lib_shared.mk", false; "upstream makefile with underscore")] + #[test_case("pkg.tar.gz", false; "tarball without version")] + fn test_is_artifact_filename(name: &str, expected: bool) { + assert_eq!(is_artifact_filename(name), expected); + } + + #[test] + fn test_orig_filenames() { + assert_eq!( + orig_filenames("pkg", "1.0", None), + [ + "pkg_1.0.orig.tar.gz", + "pkg_1.0.orig.tar.xz", + "pkg_1.0.orig.tar.bz2", + "pkg_1.0.orig.tar.lzma" + ] + ); + assert_eq!( + orig_filenames("pkg", "1.0", Some("bar"))[0], + "pkg_1.0.orig-bar.tar.gz" + ); + } +} diff --git a/packages/debmagic-common/src/changes.rs b/packages/debmagic-common/src/changes.rs index 9a8dbaa6..0e8264ed 100644 --- a/packages/debmagic-common/src/changes.rs +++ b/packages/debmagic-common/src/changes.rs @@ -1,5 +1,6 @@ use deb822_lossless::Paragraph; +use crate::artifact::is_orig_tarball; use crate::checksums::Hash; /// A parsed `.changes` file listing: which files it references and @@ -22,80 +23,6 @@ impl ChangesFiles { } } -/// Whether a `.changes` filename names a source-only upload: -/// `dpkg-buildpackage -S` writes `__source.changes`, while -/// binary builds name the file after the build architecture instead. -pub fn is_source_changes(name: &str) -> bool { - let Some(stem) = name.strip_suffix(".changes") else { - return false; - }; - stem.ends_with("_source") -} - -/// Whether a filename is an `orig` tarball (main or component). -pub fn is_orig_tarball(name: &str) -> bool { - // main: foo_1.0.orig.tar.gz; component: foo_1.0.orig-bar.tar.gz - let Some((_, rest)) = name.split_once(".orig") else { - return false; - }; - if rest.starts_with(".tar.") { - return true; - } - // component: the part between `.orig-` and `.tar.` is the component name - match rest.strip_prefix('-').and_then(|r| r.split_once(".tar.")) { - Some((component, _)) => !component.is_empty() && !component.contains('/'), - None => false, - } -} - -/// Whether a filename looks like a build or upload artifact: -/// `_...` with a package, source, `.changes`, `.buildinfo`, -/// dput `.upload` or debmagic `.upload.json` suffix, or a tarball, possibly -/// with an `.asc` signature. -pub fn is_artifact_filename(name: &str) -> bool { - const SUFFIXES: [&str; 9] = [ - ".changes", - ".buildinfo", - ".dsc", - ".deb", - ".udeb", - ".ddeb", - ".upload", - ".upload.json", - ".diff.gz", - ]; - let name = name.strip_suffix(".asc").unwrap_or(name); - name.contains('_') - && (name.contains(".tar.") || SUFFIXES.iter().any(|suffix| name.ends_with(suffix))) -} - -/// The `orig` tarball filename prefix for a source package version: -/// `_.orig.tar.` — the compression extension varies. -/// For a component, `_.orig-.tar.`. -pub fn orig_prefix(name: &str, upstream_version: &str) -> String { - format!("{name}_{upstream_version}.orig.tar.") -} - -/// The `orig` tarball filename prefix for a component of a source -/// package version: `_.orig-.tar.`. -pub fn component_orig_prefix(name: &str, upstream_version: &str, component: &str) -> String { - format!("{name}_{upstream_version}.orig-{component}.tar.") -} - -/// The possible `orig` tarball filenames of a source package version, -/// one per compression dpkg-source accepts (its `%COMP` table): the main -/// tarball, or a component's when `component` is set. -pub fn orig_filenames(name: &str, upstream_version: &str, component: Option<&str>) -> Vec { - let prefix = match component { - None => orig_prefix(name, upstream_version), - Some(component) => component_orig_prefix(name, upstream_version, component), - }; - ["gz", "xz", "bz2", "lzma"] - .iter() - .map(|ext| format!("{prefix}{ext}")) - .collect() -} - /// Add one `orig` tarball entry (or, with `None`, remove all orig /// entries) from a `.changes` paragraph's `Files:`/`Checksums-*:` /// listings. `tarball_data` is the content of the tarball named by @@ -163,46 +90,6 @@ pub fn set_changes_orig_entries( #[cfg(test)] mod tests { use super::*; - use test_case::test_case; - - #[test_case("pkg_1.0.orig.tar.gz", true; "main orig")] - #[test_case("pkg_1.0.orig.tar.xz", true; "main orig xz")] - #[test_case("pkg_1.0.orig-bar.tar.xz", true; "component orig")] - #[test_case("pkg_1.0-1.debian.tar.xz", false; "debian tarball")] - #[test_case("pkg_1.0-1.dsc", false; "dsc")] - #[test_case("pkg_1.0.orig.tar", false; "no compression ext")] - fn test_is_orig_tarball(name: &str, expected: bool) { - assert_eq!(is_orig_tarball(name), expected); - } - - #[test_case("pkg_1.0-1_source.changes", true; "source upload")] - #[test_case("pkg_1.0-1_amd64.changes", false; "binary upload")] - #[test_case("pkg_1.0-1_all.changes", false; "arch-indep upload")] - #[test_case("pkg_1.0-1_multi.changes", false; "multiarch upload")] - #[test_case("pkg_1.0-1_amd64+all.changes", false; "combined upload")] - #[test_case("pkg_1.0-1_amd64v3.changes", false; "arch variant upload")] - #[test_case("pkg_1.0-1_source.buildinfo", false; "not a changes file")] - fn test_is_source_changes(name: &str, expected: bool) { - assert_eq!(is_source_changes(name), expected); - } - - #[test_case("pkg_1.0.orig.tar.xz", true; "orig")] - #[test_case("pkg_1.0.orig.tar.xz.asc", true; "orig signature")] - #[test_case("pkg_1.0-1.debian.tar.xz", true; "debian tarball")] - #[test_case("pkg_1.0.tar.xz", true; "native tarball")] - #[test_case("pkg_1.0-1.dsc", true; "dsc")] - #[test_case("pkg_1.0-1.diff.gz", true; "1.0 diff")] - #[test_case("libpkg1_1.0-1_amd64.deb", true; "deb")] - #[test_case("pkg-dbgsym_1.0-1_amd64.ddeb", true; "ddeb")] - #[test_case("pkg_1.0-1_amd64.buildinfo", true; "buildinfo")] - #[test_case("pkg_1.0-1_source.ppa.upload", true; "dput log")] - #[test_case("pkg_1.0-1_source.upload.json", true; "debmagic upload record")] - #[test_case("config.guess", false; "upstream script")] - #[test_case("lib_shared.mk", false; "upstream makefile with underscore")] - #[test_case("pkg.tar.gz", false; "tarball without version")] - fn test_is_artifact_filename(name: &str, expected: bool) { - assert_eq!(is_artifact_filename(name), expected); - } #[test] fn test_changes_files_orig_tarballs() { @@ -220,23 +107,6 @@ mod tests { ); } - #[test] - fn test_orig_filenames() { - assert_eq!( - orig_filenames("pkg", "1.0", None), - [ - "pkg_1.0.orig.tar.gz", - "pkg_1.0.orig.tar.xz", - "pkg_1.0.orig.tar.bz2", - "pkg_1.0.orig.tar.lzma" - ] - ); - assert_eq!( - orig_filenames("pkg", "1.0", Some("bar"))[0], - "pkg_1.0.orig-bar.tar.gz" - ); - } - #[test] fn set_changes_orig_entries_adds_and_removes() { let mut control: Paragraph = "Format: 1.8\nSource: pkg\nFiles:\n abc 3 devel optional pkg_1.0-1.debian.tar.xz\n def 4 devel optional pkg_1.0-1.dsc\nChecksums-Sha256:\n xyz 3 pkg_1.0-1.debian.tar.xz\n zzz 4 pkg_1.0-1.dsc\n" diff --git a/packages/debmagic-common/src/lib.rs b/packages/debmagic-common/src/lib.rs index 5ebd9eb9..ecf01801 100644 --- a/packages/debmagic-common/src/lib.rs +++ b/packages/debmagic-common/src/lib.rs @@ -1,3 +1,4 @@ +pub mod artifact; pub mod changes; pub mod checksums; pub mod debian; diff --git a/packages/debmagic/src/build/artifacts.rs b/packages/debmagic/src/build/artifacts.rs index 99dae1b7..4d3cf543 100644 --- a/packages/debmagic/src/build/artifacts.rs +++ b/packages/debmagic/src/build/artifacts.rs @@ -7,7 +7,7 @@ use std::{ use crate::build_intent::BuildKind; use crate::upstream::orig::link_or_copy; use anyhow::{Context, anyhow, bail}; -use debmagic_common::changes::is_source_changes; +use debmagic_common::artifact::is_source_changes; /// Locate the `.changes` file a build of `kind` produced in a build work /// directory. dpkg names source-only uploads `__source.changes` diff --git a/packages/debmagic/src/build/source.rs b/packages/debmagic/src/build/source.rs index dc090f35..af539792 100644 --- a/packages/debmagic/src/build/source.rs +++ b/packages/debmagic/src/build/source.rs @@ -545,11 +545,14 @@ pub fn stage_source_tree( // wildcard for "any compression" is added here let orig_pattern = format!( "{}*", - debmagic_common::changes::orig_prefix(package.name(), package.version().upstream_version()) + debmagic_common::artifact::orig_prefix( + package.name(), + package.version().upstream_version() + ) ); let component_pattern = format!( "{}*", - debmagic_common::changes::component_orig_prefix( + debmagic_common::artifact::component_orig_prefix( package.name(), package.version().upstream_version(), "*" diff --git a/packages/debmagic/src/changes.rs b/packages/debmagic/src/changes.rs index 7f1be532..e0a70c95 100644 --- a/packages/debmagic/src/changes.rs +++ b/packages/debmagic/src/changes.rs @@ -12,7 +12,7 @@ pub fn find_orig_in_dir( upstream_version: &str, component: Option<&str>, ) -> Option { - debmagic_common::changes::orig_filenames(name, upstream_version, component) + debmagic_common::artifact::orig_filenames(name, upstream_version, component) .into_iter() .map(|filename| dir.join(filename)) .find(|path| path.is_file()) diff --git a/packages/debmagic/src/output_dir.rs b/packages/debmagic/src/output_dir.rs index 8d039c7b..052bcdb7 100644 --- a/packages/debmagic/src/output_dir.rs +++ b/packages/debmagic/src/output_dir.rs @@ -132,7 +132,7 @@ fn is_foreign(dir: &Path) -> anyhow::Result { let entry = entry.with_context(|| format!("failed to read {}", dir.display()))?; let is_dir = entry.file_type().is_ok_and(|kind| kind.is_dir()); let name = entry.file_name(); - if is_dir || !debmagic_common::changes::is_artifact_filename(&name.to_string_lossy()) { + if is_dir || !debmagic_common::artifact::is_artifact_filename(&name.to_string_lossy()) { return Ok(true); } } diff --git a/packages/debmagic/src/upload/orig.rs b/packages/debmagic/src/upload/orig.rs index 07ff9bb2..2fc6907d 100644 --- a/packages/debmagic/src/upload/orig.rs +++ b/packages/debmagic/src/upload/orig.rs @@ -69,7 +69,7 @@ pub fn changes_include_orig( .flatten() { let name = entry.file_name().to_string_lossy().into_owned(); - if debmagic_common::changes::is_orig_tarball(&name) + if debmagic_common::artifact::is_orig_tarball(&name) && name.contains(".orig-") && name.starts_with(&format!( "{}_{}.", diff --git a/packages/debmagic/src/upstream/fetch.rs b/packages/debmagic/src/upstream/fetch.rs index 830f5530..93cd0d4e 100644 --- a/packages/debmagic/src/upstream/fetch.rs +++ b/packages/debmagic/src/upstream/fetch.rs @@ -2,7 +2,7 @@ use std::path::{Path, PathBuf}; use anyhow::{Context, bail}; use clap::ValueEnum; -use debmagic_common::changes::{component_orig_prefix, orig_filenames, orig_prefix}; +use debmagic_common::artifact::{component_orig_prefix, orig_filenames, orig_prefix}; use debmagic_common::package::SourcePackage; use crate::changes::find_orig_in_dir; diff --git a/packages/debmagic/src/upstream/orig.rs b/packages/debmagic/src/upstream/orig.rs index ca5c587f..2abb9f02 100644 --- a/packages/debmagic/src/upstream/orig.rs +++ b/packages/debmagic/src/upstream/orig.rs @@ -317,7 +317,7 @@ async fn launchpad_orig_url( component: Option<&str>, ) -> Option { let base = format!("https://launchpad.net/{distro}/+archive/primary/+files/"); - for filename in debmagic_common::changes::orig_filenames(name, upstream_version, component) { + for filename in debmagic_common::artifact::orig_filenames(name, upstream_version, component) { let url = format!("{base}{filename}"); if crate::requests::http_exists(&url).await { return Some(url); @@ -452,13 +452,13 @@ async fn fetch_from_pool( // the pool holds origs of many versions; the filename must carry // the wanted one let orig_prefix = match component { - None => debmagic_common::changes::orig_prefix(name, upstream_version), + None => debmagic_common::artifact::orig_prefix(name, upstream_version), Some(component) => { - debmagic_common::changes::component_orig_prefix(name, upstream_version, component) + debmagic_common::artifact::component_orig_prefix(name, upstream_version, component) } }; let wanted = |filename: &str| -> bool { - filename.starts_with(&orig_prefix) && debmagic_common::changes::is_orig_tarball(filename) + filename.starts_with(&orig_prefix) && debmagic_common::artifact::is_orig_tarball(filename) }; for section in sections { From af9a2ec39c59e60f082ecf13978786f28bd93f0f Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Fri, 23 Oct 2026 15:29:33 +0200 Subject: [PATCH 36/48] feat(build): install what a build needs in one apt transaction A fresh container ran apt-get update up to three times and four separate dpkg transactions before building. - container creation runs no apt (LXD) or bakes the refresh into the cached image (Docker); `--apt-update-age` then also applies to fresh Docker containers, measured from the image creation - tooling and build dependencies install in one `apt-get satisfy` (`install` + `build-dep` before Debian 11 and Ubuntu 20.04) - `--test=no` also sets the `nocheck` build profile, skipping test-only build dependencies - build containers skip recommends, translations, index diffs, fsync, man-db rebuilds and service starts - debmagic writes the `--apt-mirror`/`--proposed` sources itself, so containers no longer need python3 --- docs/usage/build.md | 5 +- docs/usage/config.md | 4 +- packages/debmagic-common/src/apt.rs | 178 ++++++++++ .../debmagic-common/src/debian/control.rs | 97 +++++ packages/debmagic-common/src/lib.rs | 1 + packages/debmagic/src/apt.rs | 336 ++++++++++++++++++ packages/debmagic/src/build/mod.rs | 102 ++++-- packages/debmagic/src/cli.rs | 2 +- packages/debmagic/src/driver/driver_docker.rs | 132 ++++--- packages/debmagic/src/driver/driver_lxd.rs | 75 +--- packages/debmagic/src/driver/mod.rs | 43 +-- .../debmagic/src/driver/scripts/mirror.py | 196 ---------- packages/debmagic/src/environment/mod.rs | 38 +- packages/debmagic/src/main.rs | 1 + packages/debmagic/src/test/run.rs | 17 +- 15 files changed, 827 insertions(+), 400 deletions(-) create mode 100644 packages/debmagic-common/src/apt.rs create mode 100644 packages/debmagic/src/apt.rs delete mode 100644 packages/debmagic/src/driver/scripts/mirror.py diff --git a/docs/usage/build.md b/docs/usage/build.md index 46b771d3..6f0f6b51 100644 --- a/docs/usage/build.md +++ b/docs/usage/build.md @@ -64,7 +64,7 @@ Each `debmagic env shell` is its own shell, so you can open more than one. A bar ## Mirror selection Fresh containers install their base tooling plus every `Build-Depends`, so slow mirrors directly translate into slow builds. -Pass `--apt-mirror` to use a faster mirror for build-dependency resolution after the base tooling is bootstrapped from the image's configured archives: +Pass `--apt-mirror` to replace the image's apt sources with a different mirror: ```shell debmagic build binary --driver lxd --apt-mirror http:///ubuntu ... @@ -74,7 +74,7 @@ You can persistently set this flag in `.config/debmagic/config.toml`. ## Apt update age -When an Environment is reused, `--apt-update-age` decides when `apt-get update/upgrade` are run again: +`--apt-update-age` decides when `apt-get update/upgrade` run again, measured from the last upgrade: | Value | Behavior | |---|---| @@ -179,6 +179,7 @@ debmagic build binary --test=no Boolean flags accept `yes`/`no`, `true`/`false` and `on`/`off`; a bare flag means `yes`. This exports `DEB_BUILD_OPTIONS=nocheck`, the standard dpkg mechanism: dpkg-buildpackage propagates it into the build, debmagic's `test` stage is skipped, and classic debhelper packages skip `dh_auto_test` as usual. +It also adds the `nocheck` build profile to `DEB_BUILD_PROFILES`, so test-only build dependencies (``) aren't installed. ## Signing diff --git a/docs/usage/config.md b/docs/usage/config.md index 9db7a2b3..f364c72a 100644 --- a/docs/usage/config.md +++ b/docs/usage/config.md @@ -24,7 +24,7 @@ All keys are optional. |---|---|---|---|---| | `driver.default` | enum | — | `--driver` | Build driver (`docker`, `bare`, `lxd`, `incus`) | | `driver.persistent` | enum | `"on-failure"` | `--persistent` | How long the Environment outlives the claim: `"no"`, `"on-failure"`, or `"always"`. A bare `--persistent` means `always`. | -| `driver.apt_mirror` | string | — | `--apt-mirror` | Mirror used for build-dependency resolution. Not used by the `bare` driver. | +| `driver.apt_mirror` | string | — | `--apt-mirror` | Mirror replacing the image's apt sources. Not used by the `bare` driver. | | `driver.proposed` | bool | `false` | `--proposed` | Also enable the `-proposed` pocket. Not used by the `bare` driver. | | `driver.apt_update_age` | string | `"1d"` | `--apt-update-age` | When a persistent environment re-runs `apt-get update && upgrade`: `"now"` (every build), `"never"` (only on creation), or a maximum age like `"1d"`, `"12h"`, `"30m"`. Not used by the `bare` driver. | | `driver.docker.base_images` | map | — | — | Base image per distro, keyed by `":"` (e.g. `"debian:trixie"`). Falls back to `docker.io/:`. For non-Debian/Ubuntu suites (e.g. `"yocto:kirkstone"`), the map entry is what makes the suite a known DistroVersion for Docker builds. | @@ -37,7 +37,7 @@ All keys are optional. | `incremental` | bool | `false` | `--incremental` | Retain the environment and sync only source changes, preserving generated files. Forces Persistence `always`; incompatible with `clean`. | | `source_sync_mode` | enum | `tracked` | `--source-sync` | Which source files are staged (see below). | | `build_debug_symbols` | bool | `false` | `--debug-symbols` | Build the automatic `-dbgsym` debug symbol package. | -| `run_test` | bool | `true` | `--test` | Run the package's test suite during the build; `false` exports `DEB_BUILD_OPTIONS=nocheck` so tests are skipped. | +| `run_test` | bool | `true` | `--test` | Run the package's test suite during the build; `false` exports `DEB_BUILD_OPTIONS=nocheck` and the `nocheck` build profile, so tests and their build dependencies are skipped. | | `sign.source` | enum | `auto` | `--sign` | How to sign the resulting `.changes`/`.dsc`/`.buildinfo`: `no` never, `keep` keeps any existing signature, `auto` skips what our key already signed, `force` always re-signs (see below). Whether a build signs at all is up to `build.sign_source`/`build.sign_binary`. | | `sign.key` | string | — | `--sign-key` | GPG key ID/email to sign with; falls back to the Changed-By/Maintainer address. | | `sign.tool` | enum | `gpg` | `--sign-tool` | OpenPGP implementation: `gpg`, `sequoia` (sq) or `custom` (uses `sign.sign_command`). | diff --git a/packages/debmagic-common/src/apt.rs b/packages/debmagic-common/src/apt.rs new file mode 100644 index 00000000..8f3edd18 --- /dev/null +++ b/packages/debmagic-common/src/apt.rs @@ -0,0 +1,178 @@ +//! What apt needs to know about a target distro: the archive layout a build +//! environment pulls from, and the features of the release's apt. + +use anyhow::bail; + +use crate::distro::{Distro, DistroVersion}; + +/// Debian suites without `-updates`/`-security` pockets. +const DEBIAN_UNSTABLE: &[&str] = &["unstable", "sid", "experimental"]; +const DEBIAN_ARCHIVE: &str = "http://deb.debian.org/debian"; +/// Debian mirrors don't carry the security pocket. +const DEBIAN_SECURITY_ARCHIVE: &str = "http://deb.debian.org/debian-security"; +const UBUNTU_ARCHIVE: &str = "http://archive.ubuntu.com/ubuntu"; +const UBUNTU_PORTS_ARCHIVE: &str = "http://ports.ubuntu.com/ubuntu-ports"; + +/// A deb822 `.sources` file pointing apt at `mirror` (the distro's default +/// archive if `None`) for the release, its `-updates` and `-security` +/// pockets, plus its proposed pocket if `proposed`. It +/// replaces the base image's own sources. `dpkg_arch` is the environment's +/// architecture: Ubuntu serves most of them from a separate ports archive. +pub fn sources_file( + distro: &DistroVersion, + mirror: Option<&str>, + proposed: bool, + dpkg_arch: &str, +) -> anyhow::Result { + let codename = distro.codename.as_str(); + let (archive, components) = match &distro.distro { + Distro::Debian => (DEBIAN_ARCHIVE, debian_components(&distro.version)), + Distro::Ubuntu => ( + match dpkg_arch { + "amd64" | "i386" => UBUNTU_ARCHIVE, + _ => UBUNTU_PORTS_ARCHIVE, + }, + &["main", "restricted", "universe", "multiverse"][..], + ), + Distro::Custom(family) => { + bail!("no known apt archive for {family}; configure the base image's sources instead") + } + }; + let debian = distro.distro == Distro::Debian; + let rolling = debian && DEBIAN_UNSTABLE.contains(&codename); + let security = format!("{codename}-security"); + let mut suites = vec![codename.to_string()]; + if !rolling { + suites.push(format!("{codename}-updates")); + if !debian { + suites.push(security.clone()); + } + } + if proposed { + suites.extend(proposed_suite(distro)); + } + + let mut groups = vec![(mirror.unwrap_or(archive), suites)]; + if debian && !rolling { + groups.push((DEBIAN_SECURITY_ARCHIVE, vec![security])); + } + let paragraphs: Vec = groups + .into_iter() + .map(|(uri, suites)| { + format!( + "Types: deb deb-src\nURIs: {uri}\nSuites: {}\nComponents: {}\n", + suites.join(" "), + components.join(" ") + ) + }) + .collect(); + Ok(paragraphs.join("\n")) +} + +fn debian_components(version: &str) -> &'static [&'static str] { + // rolling suites have no version and are past 12 + match version.parse::() { + Ok(major) if major < 12 => &["main", "contrib", "non-free"], + _ => &["main", "contrib", "non-free", "non-free-firmware"], + } +} + +/// The release's proposed pocket, `None` for unstable, which has none. +fn proposed_suite(distro: &DistroVersion) -> Option { + let codename = &distro.codename; + match distro.distro { + Distro::Debian if DEBIAN_UNSTABLE.contains(&codename.as_str()) => None, + Distro::Debian => Some(format!("{codename}-proposed-updates")), + _ => Some(format!("{codename}-proposed")), + } +} + +/// Whether the release's apt has `apt-get satisfy` (apt 1.9: Debian 11, +/// Ubuntu 20.04). Unknown versions count as recent. +pub fn has_satisfy(distro: &DistroVersion) -> bool { + let major = distro + .version + .split('.') + .next() + .and_then(|major| major.parse::().ok()); + match (&distro.distro, major) { + (Distro::Debian, Some(major)) => major >= 11, + (Distro::Ubuntu, Some(major)) => major >= 20, + _ => true, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::distro::get_distro_version; + use test_case::test_case; + + fn distro(name: &str) -> DistroVersion { + get_distro_version(name).unwrap() + } + + #[test] + fn debian_stable_keeps_security_on_the_debian_archive() { + let sources = sources_file(&distro("trixie"), Some("http://m/debian"), true, "amd64"); + assert_eq!( + sources.unwrap(), + "Types: deb deb-src\nURIs: http://m/debian\n\ + Suites: trixie trixie-updates trixie-proposed-updates\n\ + Components: main contrib non-free non-free-firmware\n\ + \n\ + Types: deb deb-src\nURIs: http://deb.debian.org/debian-security\n\ + Suites: trixie-security\n\ + Components: main contrib non-free non-free-firmware\n" + ); + } + + #[test_case(false ; "release")] + #[test_case(true ; "no proposed pocket")] + fn debian_unstable_is_one_suite(proposed: bool) { + let sources = sources_file(&distro("sid"), None, proposed, "arm64").unwrap(); + assert_eq!( + sources, + "Types: deb deb-src\nURIs: http://deb.debian.org/debian\nSuites: unstable\n\ + Components: main contrib non-free non-free-firmware\n" + ); + } + + #[test] + fn old_debian_has_no_firmware_component() { + let sources = sources_file(&distro("bullseye"), None, false, "amd64").unwrap(); + assert!( + sources.contains("Components: main contrib non-free\n"), + "{sources}" + ); + } + + #[test_case("amd64", UBUNTU_ARCHIVE ; "archive")] + #[test_case("arm64", UBUNTU_PORTS_ARCHIVE ; "ports")] + fn ubuntu_proposed_uses_the_default_archive(arch: &str, archive: &str) { + let sources = sources_file(&distro("noble"), None, true, arch).unwrap(); + assert_eq!( + sources, + format!( + "Types: deb deb-src\nURIs: {archive}\n\ + Suites: noble noble-updates noble-security noble-proposed\n\ + Components: main restricted universe multiverse\n" + ) + ); + } + + #[test] + fn custom_distro_has_no_known_archive() { + let custom = DistroVersion::custom(Distro::Custom("yocto".into()), "scarthgap"); + assert!(sources_file(&custom, Some("http://m"), false, "amd64").is_err()); + } + + #[test_case("buster", false)] + #[test_case("bullseye", true)] + #[test_case("unstable", true)] + #[test_case("bionic", false)] + #[test_case("focal", true)] + fn test_has_satisfy(name: &str, expected: bool) { + assert_eq!(has_satisfy(&distro(name)), expected); + } +} diff --git a/packages/debmagic-common/src/debian/control.rs b/packages/debmagic-common/src/debian/control.rs index 795b9983..fe85fde1 100644 --- a/packages/debmagic-common/src/debian/control.rs +++ b/packages/debmagic-common/src/debian/control.rs @@ -1,10 +1,43 @@ use anyhow::Context; +/// The source paragraph's build relationship fields, `Build-Depends` first: +/// a [`BuildScope::Clean`] needs only that one. +const BUILD_DEPENDS_FIELDS: [&str; 3] = + ["Build-Depends", "Build-Depends-Arch", "Build-Depends-Indep"]; +const BUILD_CONFLICTS_FIELDS: [&str; 3] = [ + "Build-Conflicts", + "Build-Conflicts-Arch", + "Build-Conflicts-Indep", +]; + +/// Which build relationship fields a build has to satisfy. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BuildScope { + /// `debian/rules clean` (and so a source build): `Build-Depends` and + /// `Build-Conflicts` only. + Clean, + /// A full binary build: also the `-Arch` and `-Indep` fields. + Full, +} + +/// The relations a build has to satisfy, one comma-separated list per field, +/// with arch qualifiers and build profiles still in it. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct BuildRelations { + pub depends: Vec, + pub conflicts: Vec, +} + /// The model of a package's `debian/control`. #[derive(Debug, Clone, Default)] pub struct Control { /// The source paragraph's `Rules-Requires-Root` keywords, `None` if absent. pub rules_requires_root: Option>, + /// The source paragraph's build relationship fields in the order of + /// [`BUILD_DEPENDS_FIELDS`] and [`BUILD_CONFLICTS_FIELDS`], normalized + /// to one line without empty entries; `None` if absent or empty. + pub build_depends: [Option; 3], + pub build_conflicts: [Option; 3], } impl Control { @@ -13,13 +46,37 @@ impl Control { .parse::() .context("invalid deb822")?; let source = deb822.paragraphs().next().context("no source paragraph")?; + let relations = |fields: [&str; 3]| { + fields.map(|field| { + source + .get(field) + .and_then(|value| normalize_relations(&value)) + }) + }; Ok(Self { rules_requires_root: source .get("Rules-Requires-Root") .map(|value| value.split_whitespace().map(str::to_string).collect()), + build_depends: relations(BUILD_DEPENDS_FIELDS), + build_conflicts: relations(BUILD_CONFLICTS_FIELDS), }) } + /// The build relations `scope` has to satisfy. + pub fn build_relations(&self, scope: BuildScope) -> BuildRelations { + let fields = match scope { + BuildScope::Clean => 1, + BuildScope::Full => 3, + }; + let collect = |relations: &[Option; 3]| { + relations[..fields].iter().flatten().cloned().collect() + }; + BuildRelations { + depends: collect(&self.build_depends), + conflicts: collect(&self.build_conflicts), + } + } + /// Whether `dpkg-buildpackage` demands a gain-root command (`fakeroot`) /// at startup, whatever targets it runs: unless `Rules-Requires-Root` /// contains `no`. An absent field counts as needing one, since dpkg @@ -31,6 +88,17 @@ impl Control { } } +/// A relations field on one line, without the empty entries a trailing +/// comma leaves (old apt rejects them); `None` if nothing remains. +fn normalize_relations(value: &str) -> Option { + let relations: Vec = value + .split(',') + .map(|relation| relation.split_whitespace().collect::>().join(" ")) + .filter(|relation| !relation.is_empty()) + .collect(); + (!relations.is_empty()).then(|| relations.join(", ")) +} + #[cfg(test)] mod tests { use super::*; @@ -44,4 +112,33 @@ mod tests { fn test_needs_gain_root(control: &str, expected: bool) { assert_eq!(Control::parse(control).unwrap().needs_gain_root(), expected); } + + const BUILD_RELATIONS: &str = "Source: foo\n\ + Build-Depends: debhelper-compat (= 13),\n\ + # comment\n\ + \x20libfoo-dev [amd64] ,\n\ + Build-Depends-Indep: python3-sphinx\n\ + Build-Conflicts: libbar-dev\n\ + Build-Conflicts-Arch:\n\ + \n\ + Package: foo\n\ + Build-Depends: ignored\n"; + + #[test_case(BuildScope::Clean, &["debhelper-compat (= 13), libfoo-dev [amd64] "] ; "clean")] + #[test_case(BuildScope::Full, &["debhelper-compat (= 13), libfoo-dev [amd64] ", "python3-sphinx"] ; "full")] + fn test_build_relations(scope: BuildScope, depends: &[&str]) { + let relations = Control::parse(BUILD_RELATIONS) + .unwrap() + .build_relations(scope); + assert_eq!( + relations, + BuildRelations { + depends: depends + .iter() + .map(|relation| relation.to_string()) + .collect(), + conflicts: vec!["libbar-dev".to_string()], + } + ); + } } diff --git a/packages/debmagic-common/src/lib.rs b/packages/debmagic-common/src/lib.rs index ecf01801..26079ea6 100644 --- a/packages/debmagic-common/src/lib.rs +++ b/packages/debmagic-common/src/lib.rs @@ -1,3 +1,4 @@ +pub mod apt; pub mod artifact; pub mod changes; pub mod checksums; diff --git a/packages/debmagic/src/apt.rs b/packages/debmagic/src/apt.rs new file mode 100644 index 00000000..78136791 --- /dev/null +++ b/packages/debmagic/src/apt.rs @@ -0,0 +1,336 @@ +//! Every apt call debmagic makes in an Environment: the one-time setup of a +//! container, refreshing its packages, and installing what a run needs in +//! one transaction. + +use std::{fs, io, os::unix::fs::PermissionsExt, path::Path}; + +use anyhow::Context; +use debmagic_common::{apt, debian::control::BuildRelations, distro::DistroVersion}; + +use crate::driver::EnvironmentDriver; + +const ENV: [(&str, &str); 1] = [("DEBIAN_FRONTEND", "noninteractive")]; + +/// Installed with every [`Install`], so a container needs no apt run at +/// creation. +const BASE_PACKAGES: &[&str] = &["dpkg-dev"]; + +/// Name of the setup tree in an Environment's temp dir. +pub const SETUP_DIR: &str = "apt-setup"; + +/// Improve performance in build containers +/// no recommends, no translations or index diffs to fetch, +/// no fsync, no man-db rebuilds, keep changed conffiles. +const SETUP_FILES: &[(&str, &str, u32)] = &[ + ( + "etc/apt/apt.conf.d/90debmagic", + "APT::Install-Recommends \"false\";\n\ + Acquire::Languages \"none\";\n\ + Acquire::PDiffs \"false\";\n\ + Dpkg::Options { \"--force-confdef\"; \"--force-confold\"; };\n", + 0o644, + ), + ("etc/dpkg/dpkg.cfg.d/debmagic", "force-unsafe-io\n", 0o644), +]; + +/// Keeps maintainer scripts from starting services, which builds never use. +const POLICY_RC_D: (&str, &str, u32) = ("usr/sbin/policy-rc.d", "#!/bin/sh\nexit 101\n", 0o755); + +const SOURCES_FILE: &str = "etc/apt/sources.list.d/debmagic.sources"; + +const MAN_DB_PRESEED: &str = "'man-db man-db/auto-update boolean false'"; + +/// The base images' own sources, replaced by [`SOURCES_FILE`]. +const DEFAULT_SOURCES: &[&str] = &[ + "/etc/apt/sources.list", + "/etc/apt/sources.list.d/ubuntu.sources", + "/etc/apt/sources.list.d/debian.sources", +]; + +/// The apt and dpkg configuration of a fresh container: a file tree to copy +/// over its root, written on the host so the container needs nothing to +/// create it. Apt sources replace the base image's only for a `mirror` or +/// the `proposed` pocket. +pub struct Setup<'a> { + pub distro: &'a DistroVersion, + pub mirror: Option<&'a str>, + pub proposed: bool, + /// Let installed packages start their services, which tests may expect. + pub services: bool, +} + +impl Setup<'_> { + fn replaces_sources(&self) -> bool { + self.mirror.is_some() || self.proposed + } + + /// The setup files apart from the apt sources. + fn files(&self) -> impl Iterator { + SETUP_FILES + .iter() + .copied() + .chain((!self.services).then_some(POLICY_RC_D)) + } + + /// Changes whenever the container would be set up differently, for + /// Driver fingerprints. + pub fn fingerprint(&self) -> String { + self.files() + .map(|(path, content, mode)| format!("{path}:{mode:o}:{content}")) + .chain([ + self.script(""), + self.mirror.unwrap_or_default().to_string(), + self.proposed.to_string(), + ]) + .collect::>() + .join("\0") + } + + /// Write the setup tree into `dir`, replacing an old one. + pub fn write(&self, dir: &Path) -> anyhow::Result<()> { + let sources = self + .replaces_sources() + .then(|| apt::sources_file(self.distro, self.mirror, self.proposed, host_dpkg_arch())) + .transpose() + .context("generating the apt sources failed")?; + if dir.exists() { + fs::remove_dir_all(dir) + .with_context(|| format!("removing the old {} failed", dir.display()))?; + } + let mut files: Vec<(&str, &str, u32)> = self.files().collect(); + files.extend( + sources + .as_deref() + .map(|content| (SOURCES_FILE, content, 0o644)), + ); + for (path, content, mode) in files { + let path = dir.join(path); + fs::create_dir_all(path.parent().expect("setup files are in a dir")) + .and_then(|()| fs::write(&path, content)) + .and_then(|()| fs::set_permissions(&path, fs::Permissions::from_mode(mode))) + .with_context(|| format!("writing {} failed", path.display()))?; + } + Ok(()) + } + + /// A `sh -c` command line applying the setup tree found at + /// `dir_in_container`. + pub fn script(&self, dir_in_container: &str) -> String { + let mut steps = Vec::new(); + if self.replaces_sources() { + steps.push(format!("rm -f {}", DEFAULT_SOURCES.join(" "))); + } + steps.push(format!("cp -R {dir_in_container}/. /")); + // man-db's postinst recreates the flag file from debconf + steps.push(format!("echo {MAN_DB_PRESEED} | debconf-set-selections")); + steps.push("rm -f /var/lib/man-db/auto-update".to_string()); + steps.join(" && ") + } +} + +/// The dpkg architecture of containers, which run on the host's. +fn host_dpkg_arch() -> &'static str { + match std::env::consts::ARCH { + "x86_64" => "amd64", + "x86" => "i386", + "aarch64" => "arm64", + "powerpc64" => "ppc64el", + "loongarch64" => "loong64", + other => other, + } +} + +/// `apt-get update` and `upgrade`, so a cached image or a long-lived +/// Environment builds against current packages, like sbuild does. +pub fn refresh_commands() -> Vec> { + [&["update"][..], &["-y", "upgrade"]] + .iter() + .map(|args| apt_get(args.iter().copied())) + .collect() +} + +pub fn refresh(driver: &dyn EnvironmentDriver, cwd: &Path) -> io::Result<()> { + run(driver, cwd, &refresh_commands()) +} + +/// What a run needs installed, in one apt transaction where the release's +/// apt allows it. [`BASE_PACKAGES`] are always part of it. +#[derive(Debug, Default)] +pub struct Install<'a> { + pub packages: Vec<&'a str>, + /// The build relations of the source tree in the install's working + /// dir. Before `apt-get satisfy`, `apt-get build-dep .` installs all of + /// them from its `debian/control` instead. + pub build_relations: Option, + /// Build profiles the build relations are evaluated with. + pub profiles: Vec, +} + +pub fn install_commands(satisfy: bool, install: &Install) -> Vec> { + let options = ["-y", "-q", "--no-install-recommends"]; + let mut profiled = apt_get(options); + if !install.profiles.is_empty() { + profiled.extend(["-P".to_string(), install.profiles.join(",")]); + } + let packages = BASE_PACKAGES.iter().chain(&install.packages).copied(); + + if satisfy { + let mut command = profiled; + command.push("satisfy".to_string()); + command.extend(packages.map(str::to_string)); + if let Some(relations) = &install.build_relations { + command.extend(relations.depends.iter().cloned()); + let conflicts = relations.conflicts.iter(); + command.extend(conflicts.map(|conflicts| format!("Conflicts: {conflicts}"))); + } + return vec![command]; + } + + let mut commands = vec![apt_get( + options.into_iter().chain(["install"]).chain(packages), + )]; + if install.build_relations.is_some() { + profiled.extend(["build-dep".to_string(), ".".to_string()]); + commands.push(profiled); + } + commands +} + +pub fn install( + driver: &dyn EnvironmentDriver, + cwd: &Path, + distro: &DistroVersion, + install: &Install, +) -> io::Result<()> { + run( + driver, + cwd, + &install_commands(apt::has_satisfy(distro), install), + ) +} + +/// `commands` as one `sh -c` command line. +pub fn shell_line(commands: &[Vec]) -> String { + commands + .iter() + .map(|command| { + command + .iter() + .map(|arg| shell_quote(arg)) + .collect::>() + .join(" ") + }) + .collect::>() + .join(" && ") +} + +/// `s` as a single POSIX shell word, quoted only if it needs to be. +pub fn shell_quote(s: &str) -> String { + let plain = !s.is_empty() + && s.bytes() + .all(|b| b.is_ascii_alphanumeric() || b"-_./=:,+@%".contains(&b)); + if plain { + s.to_string() + } else { + format!("'{}'", s.replace('\'', "'\\''")) + } +} + +fn apt_get>(args: impl IntoIterator) -> Vec { + std::iter::once("apt-get".to_string()) + .chain(args.into_iter().map(Into::into)) + .collect() +} + +fn run(driver: &dyn EnvironmentDriver, cwd: &Path, commands: &[Vec]) -> io::Result<()> { + for command in commands { + let command: Vec<&str> = command.iter().map(String::as_str).collect(); + driver.run_command_checked(&command, cwd, true, &ENV)?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn build_install() -> Install<'static> { + Install { + packages: vec!["build-essential"], + build_relations: Some(BuildRelations { + depends: vec!["debhelper-compat (= 13), libfoo-dev ".to_string()], + conflicts: vec!["libbar-dev".to_string()], + }), + profiles: vec!["nocheck".to_string(), "nodoc".to_string()], + } + } + + #[test] + fn satisfy_installs_everything_in_one_transaction() { + assert_eq!( + shell_line(&install_commands(true, &build_install())), + "apt-get -y -q --no-install-recommends -P nocheck,nodoc satisfy dpkg-dev build-essential \ + 'debhelper-compat (= 13), libfoo-dev ' 'Conflicts: libbar-dev'" + ); + } + + #[test] + fn without_satisfy_build_dep_reads_the_source_tree() { + assert_eq!( + shell_line(&install_commands(false, &build_install())), + "apt-get -y -q --no-install-recommends install dpkg-dev build-essential && \ + apt-get -y -q --no-install-recommends -P nocheck,nodoc build-dep ." + ); + } + + #[test] + fn plain_install_has_no_build_dep() { + let install = Install { + packages: vec!["autopkgtest"], + ..Install::default() + }; + assert_eq!( + shell_line(&install_commands(false, &install)), + "apt-get -y -q --no-install-recommends install dpkg-dev autopkgtest" + ); + } + + #[test] + fn setup_replaces_the_default_sources_only_with_its_own() -> anyhow::Result<()> { + let dir = std::env::temp_dir().join(format!("debmagic-apt-setup-{}", std::process::id())); + let trixie = debmagic_common::distro::get_distro_version("trixie").unwrap(); + + let mut setup = Setup { + distro: &trixie, + mirror: None, + proposed: false, + services: false, + }; + setup.write(&dir)?; + assert!(!dir.join(SOURCES_FILE).exists()); + assert!(!setup.script("/s").contains("sources.list")); + let policy = fs::metadata(dir.join("usr/sbin/policy-rc.d"))?; + assert_eq!(policy.permissions().mode() & 0o777, 0o755); + let plain = setup.fingerprint(); + + setup.services = true; + setup.write(&dir)?; + assert!(!dir.join("usr/sbin/policy-rc.d").exists()); + assert_ne!(setup.fingerprint(), plain); + setup.services = false; + + setup.mirror = Some("http://m/debian"); + setup.write(&dir)?; + assert_ne!(setup.fingerprint(), plain); + assert!(fs::read_to_string(dir.join(SOURCES_FILE))?.contains("URIs: http://m/debian\n")); + assert_eq!( + setup.script("/s"), + "rm -f /etc/apt/sources.list /etc/apt/sources.list.d/ubuntu.sources \ + /etc/apt/sources.list.d/debian.sources && cp -R /s/. / && \ + echo 'man-db man-db/auto-update boolean false' | debconf-set-selections && \ + rm -f /var/lib/man-db/auto-update" + ); + fs::remove_dir_all(&dir)?; + Ok(()) + } +} diff --git a/packages/debmagic/src/build/mod.rs b/packages/debmagic/src/build/mod.rs index dc99df18..7567c72f 100644 --- a/packages/debmagic/src/build/mod.rs +++ b/packages/debmagic/src/build/mod.rs @@ -4,6 +4,7 @@ use std::{ process::{Command, Stdio}, }; +use crate::apt; use crate::build::source::{source_manifest_path, stage_dir, stage_source_tree}; use crate::build_intent::{BuildIntent, BuildIntentInput, BuildKind, resolve_build_intent}; use crate::driver::{Driver, DriverType, Environment, EnvironmentDriver, EnvironmentPurpose}; @@ -13,6 +14,7 @@ use crate::environment::{ use crate::package::PackageTarget; use crate::upstream::orig::link_or_copy; use anyhow::{Context, anyhow}; +use debmagic_common::debian::control::BuildScope; use debmagic_common::debian::source::SourceFormat; use debmagic_common::package::SourcePackage; @@ -231,17 +233,20 @@ pub async fn build_binary_package( run_build(&request, |claimed| { crate::output::stage("Building binary package(s)"); let staged_source_dir = stage_dir(claimed.environment(), BuildKind::Binary); - // build-essential is an implicit dependency that `apt-get build-dep` - // won't pull in. No-op when the environment already has it - // (idempotent, and the bare driver runs on the host). + let profiles = build_profiles( + std::env::var("DEB_BUILD_PROFILES").ok().as_deref(), + intent.config.run_test, + ); + // build-essential is an implicit build dependency let mut packages = vec!["build-essential"]; packages.extend(gain_root_package(target)?); - install_packages(claimed, &staged_source_dir, &packages)?; - claimed.driver().run_command_checked( - &["apt-get", "-y", "build-dep", "."], - &staged_source_dir, - true, - &[], + install_packages( + claimed, + target, + BuildKind::Binary, + packages, + Some(BuildScope::Full), + &profiles, )?; let inherited_options = std::env::var("DEB_BUILD_OPTIONS").ok(); let options = deb_build_options( @@ -249,7 +254,11 @@ pub async fn build_binary_package( intent.config.build_debug_symbols, intent.config.run_test, ); + let profiles = profiles.join(" "); let mut env_add = vec![("DEB_BUILD_OPTIONS", options.as_str())]; + if !profiles.is_empty() { + env_add.push(("DEB_BUILD_PROFILES", &profiles)); + } if let Some(variant) = intent.config.host_arch_variant.as_deref() { env_add.push(("DEB_HOST_ARCH_VARIANT", variant)); } @@ -292,15 +301,57 @@ fn gain_root_package(target: &PackageTarget) -> anyhow::Result, + scope: Option, + profiles: &[String], ) -> anyhow::Result<()> { - let mut cmd = vec!["apt-get", "-y", "-q", "--no-install-recommends", "install"]; - cmd.extend(packages); - claimed.driver().run_command_checked(&cmd, dir, true, &[])?; - Ok(()) + let build_relations = scope + .map(|scope| { + anyhow::Ok( + target + .package + .control() + .context("reading the build dependencies failed")? + .build_relations(scope), + ) + }) + .transpose()?; + let install = apt::Install { + packages, + build_relations, + profiles: profiles.to_vec(), + }; + let environment = claimed.environment(); + let staged_source_dir = stage_dir(environment, kind); + apt::install( + claimed.driver(), + &staged_source_dir, + &environment.distro, + &install, + ) + .context("installing the build dependencies failed") +} + +/// The build profiles from `inherited` (`DEB_BUILD_PROFILES`), plus +/// `nocheck` when tests don't run, so test-only build dependencies are +/// skipped. +fn build_profiles(inherited: Option<&str>, run_test: bool) -> Vec { + let mut profiles: Vec = inherited + .unwrap_or_default() + .split_whitespace() + .filter(|profile| *profile != "nocheck") + .map(str::to_string) + .collect(); + if !run_test { + profiles.push("nocheck".to_string()); + } + profiles } /// Confirm `cmd` is on `PATH`, failing with an actionable message (rather @@ -418,17 +469,9 @@ pub async fn build_source_package( })?; } // dpkg-buildpackage looks for the gain-root command even for -S - if let Some(package) = gain_root_package(target)? { - install_packages(claimed, &staged_source_dir, &[package])?; - } - if intent.config.clean { - claimed.driver().run_command_checked( - &["apt-get", "-y", "build-dep", "."], - &staged_source_dir, - true, - &[], - )?; - } + let packages = gain_root_package(target)?.into_iter().collect(); + let scope = intent.config.clean.then_some(BuildScope::Clean); + install_packages(claimed, target, BuildKind::Source, packages, scope, &[])?; let mut args: Vec = vec![ "dpkg-buildpackage".into(), "-S".into(), @@ -610,4 +653,11 @@ mod tests { "parallel=8" ); } + + #[test] + fn run_test_profile_adds_nocheck() { + assert_eq!(build_profiles(None, true), Vec::::new()); + assert_eq!(build_profiles(Some("nodoc"), false), ["nodoc", "nocheck"]); + assert_eq!(build_profiles(Some("nocheck nodoc"), true), ["nodoc"]); + } } diff --git a/packages/debmagic/src/cli.rs b/packages/debmagic/src/cli.rs index 1472c872..3a49680b 100644 --- a/packages/debmagic/src/cli.rs +++ b/packages/debmagic/src/cli.rs @@ -515,7 +515,7 @@ pub struct BinaryTargetArgs { num_args = 0..=1, default_missing_value = "true", value_parser = BoolishValueParser::new(), - help = "Run the package's test suite during the build. Defaults to the 'run_test' setting in the config file (true if unset); --test=no exports DEB_BUILD_OPTIONS=nocheck so dpkg-buildpackage skips tests." + help = "Run the package's test suite during the build. Defaults to the 'run_test' setting in the config file (true if unset); --test=no exports DEB_BUILD_OPTIONS=nocheck and the nocheck build profile, so tests and test-only build dependencies are skipped." )] pub test: Option, } diff --git a/packages/debmagic/src/driver/driver_docker.rs b/packages/debmagic/src/driver/driver_docker.rs index 766481ac..4471d92b 100644 --- a/packages/debmagic/src/driver/driver_docker.rs +++ b/packages/debmagic/src/driver/driver_docker.rs @@ -3,15 +3,17 @@ use std::{ fs, path::{Path, PathBuf}, process::{Command, Stdio}, + time::SystemTime, }; -use anyhow::anyhow; +use anyhow::{Context, anyhow}; use debmagic_common::distro::DistroVersion; use serde::{Deserialize, Serialize}; +use crate::apt; use crate::driver::{ - APT_MIRROR_SCRIPT, DriverType, ENVIRONMENT_DIR_IN_CONTAINER, Environment, EnvironmentDriver, - EnvironmentMetadata, IsolationCapability, ResourceStatus, config::DriverConfig, + DriverType, ENVIRONMENT_DIR_IN_CONTAINER, Environment, EnvironmentDriver, EnvironmentMetadata, + EnvironmentPurpose, IsolationCapability, ResourceStatus, config::DriverConfig, container_name_from_metadata, container_name_metadata, environment_fingerprint, resource_name, run_checked, translate_path_in_container, }; @@ -47,8 +49,9 @@ const DOCKERFILE_TEMPLATE: &str = r#" FROM {base_image} ARG USER_UID=1000 ARG USER_GID=$USER_UID -RUN apt-get update && apt-get install -y --no-install-recommends dpkg-dev python3 -{apt_mirror_setup} +ARG DEBIAN_FRONTEND=noninteractive +COPY {setup_dir} /tmp/{setup_dir} +RUN {setup} && rm -rf /tmp/{setup_dir} && {apt} RUN set -e; \ getent group "$USER_GID" >/dev/null || groupadd --gid "$USER_GID" debmagic; \ getent passwd "$USER_UID" >/dev/null || useradd --uid "$USER_UID" --gid "$USER_GID" -m debmagic @@ -57,20 +60,29 @@ USER $USER_UID:$USER_GID ENTRYPOINT ["sleep", "infinity"] "#; -// Ubuntu images ship python3 already; installed explicitly above for the -// others (e.g. Debian's slim images) before the mirror script needs it. -const APT_MIRROR_SCRIPT_FILENAME: &str = "mirror.py"; - -/// Quote `s` as a single POSIX shell word for embedding in a Dockerfile -/// `RUN` instruction (which is interpreted by `/bin/sh -c`). -fn shell_quote(s: &str) -> String { - format!("'{}'", s.replace('\'', "'\\''")) +/// The image applies the apt setup and refreshes its packages, so a fresh +/// container from a cached image is as current as the image. +fn dockerfile(environment: &Environment, base_image: &str, setup: &apt::Setup) -> String { + let setup_dir = apt::SETUP_DIR; + let mut apt_commands = apt::refresh_commands(); + apt_commands.extend(apt::install_commands( + debmagic_common::apt::has_satisfy(&environment.distro), + &apt::Install::default(), + )); + DOCKERFILE_TEMPLATE + .replace("{base_image}", base_image) + .replace("{setup}", &setup.script(&format!("/tmp/{setup_dir}"))) + .replace("{apt}", &apt::shell_line(&apt_commands)) + .replace("{setup_dir}", setup_dir) + .replace("{build_dir}", ENVIRONMENT_DIR_IN_CONTAINER) } pub struct DriverDocker { environment: Environment, container_name: String, reused_environment: bool, + /// Creation time of the image a fresh container was started from. + image_created: Option, } /// Replace characters that Docker image/container names disallow. @@ -85,46 +97,15 @@ fn sanitize_docker_reference(name: &str) -> String { fn build_build_image( environment: &Environment, - base_image: &str, - apt_mirror: Option<&str>, - proposed: bool, + dockerfile: &str, + setup: &apt::Setup, image_name: &str, ) -> anyhow::Result<()> { - let apt_mirror_setup = if apt_mirror.is_some() || proposed { - fs::write( - environment.temp_dir().join(APT_MIRROR_SCRIPT_FILENAME), - APT_MIRROR_SCRIPT, - ) - .map_err(|e| anyhow!("Failed to write apt mirror script: {e}"))?; - - let mut args = vec![ - "--codename".to_string(), - shell_quote(&environment.distro.codename), - ]; - if let Some(mirror) = apt_mirror { - args.extend(["--mirror".to_string(), shell_quote(mirror)]); - } - if proposed { - args.push("--proposed".to_string()); - } - - let setup = format!( - "COPY {script} /tmp/{script}\nRUN python3 /tmp/{script} {args} && rm /tmp/{script} && apt-get update\n", - script = APT_MIRROR_SCRIPT_FILENAME, - args = args.join(" "), - ); - setup - } else { - String::new() - }; - - let formatted_dockerfile = DOCKERFILE_TEMPLATE - .replace("{base_image}", base_image) - .replace("{apt_mirror_setup}", &apt_mirror_setup) - .replace("{build_dir}", ENVIRONMENT_DIR_IN_CONTAINER); - + setup + .write(&environment.temp_dir().join(apt::SETUP_DIR)) + .context("writing the apt setup failed")?; let dockerfile_path = environment.temp_dir().join("Dockerfile"); - fs::write(&dockerfile_path, formatted_dockerfile) + fs::write(&dockerfile_path, dockerfile) .map_err(|e| anyhow!("Failed to write Dockerfile, {e}"))?; let mut build_args = Vec::new(); @@ -151,14 +132,20 @@ fn build_build_image( Ok(()) } -fn does_image_exist(image_name: &str) -> anyhow::Result { - let status = Command::new("docker") - .args(["image", "inspect", image_name]) - .stdout(Stdio::null()) +/// The image's creation time, `None` if it doesn't exist. +fn image_created(image_name: &str) -> anyhow::Result> { + let output = Command::new("docker") + .args(["image", "inspect", "--format", "{{.Created}}", image_name]) .stderr(Stdio::null()) - .status() + .output() .map_err(|error| anyhow!("Failed to inspect Docker image: {error}"))?; - Ok(status.success()) + if !output.status.success() { + return Ok(None); + } + let created = String::from_utf8_lossy(&output.stdout); + let created = chrono::DateTime::parse_from_rfc3339(created.trim()) + .with_context(|| format!("parsing the creation time {created:?} of {image_name} failed"))?; + Ok(Some(created.into())) } /// The environment fingerprint stored in the container's labels, or `None` @@ -268,15 +255,18 @@ impl DriverDocker { let uid = unsafe { libc::geteuid() }.to_string(); let gid = unsafe { libc::getegid() }.to_string(); let build_root = environment.root_dir.to_string_lossy(); - let proposed_fingerprint = proposed.to_string(); + let setup = apt::Setup { + distro: &environment.distro, + mirror: apt_mirror, + proposed, + services: environment.purpose == EnvironmentPurpose::Test, + }; + let dockerfile = dockerfile(environment, &base_image, &setup); let image_fingerprint = environment_fingerprint(&[ "docker", - DOCKERFILE_TEMPLATE, - APT_MIRROR_SCRIPT, - &base_image, + &dockerfile, + &setup.fingerprint(), &environment.distro.codename, - apt_mirror.unwrap_or(""), - &proposed_fingerprint, &uid, &gid, ]); @@ -291,6 +281,7 @@ impl DriverDocker { environment: environment.clone(), container_name, reused_environment: false, + image_created: None, }; let environment_matches = container_environment_fingerprint(&driver.container_name)? .as_deref() @@ -318,14 +309,10 @@ impl DriverDocker { .status(); let docker_image_name = format!("debmagic-env-{}", &image_fingerprint[..16]); - if !does_image_exist(&docker_image_name)? { - build_build_image( - environment, - &base_image, - apt_mirror, - proposed, - &docker_image_name, - )?; + driver.image_created = image_created(&docker_image_name)?; + if driver.image_created.is_none() { + build_build_image(environment, &dockerfile, &setup, &docker_image_name)?; + driver.image_created = image_created(&docker_image_name)?; } run_checked( Command::new("docker") @@ -359,6 +346,7 @@ impl DriverDocker { environment: environment.clone(), container_name: container_name_from_metadata(metadata), reused_environment: true, + image_created: None, }) } @@ -432,6 +420,10 @@ impl EnvironmentDriver for DriverDocker { self.reused_environment } + fn packages_refreshed_at(&self) -> Option { + self.image_created + } + fn interactive_shell(&self, cwd: &Path) -> std::io::Result<()> { let workdir = self.translate_path_in_container(cwd)?; let was_running = self.container_is_running().map_err(std::io::Error::other)?; diff --git a/packages/debmagic/src/driver/driver_lxd.rs b/packages/debmagic/src/driver/driver_lxd.rs index 5bee551e..3d560c75 100644 --- a/packages/debmagic/src/driver/driver_lxd.rs +++ b/packages/debmagic/src/driver/driver_lxd.rs @@ -1,6 +1,5 @@ use std::{ collections::HashMap, - fs, path::{Path, PathBuf}, process::{Command, Stdio}, }; @@ -8,9 +7,10 @@ use std::{ use debmagic_common::distro::{Distro, DistroVersion}; use serde::{Deserialize, Serialize}; +use crate::apt; use crate::driver::{ - APT_MIRROR_SCRIPT, DriverType, ENVIRONMENT_DIR_IN_CONTAINER, Environment, EnvironmentDriver, - EnvironmentMetadata, IsolationCapability, ResourceStatus, config::DriverConfig, + DriverType, ENVIRONMENT_DIR_IN_CONTAINER, Environment, EnvironmentDriver, EnvironmentMetadata, + EnvironmentPurpose, IsolationCapability, ResourceStatus, config::DriverConfig, container_name_from_metadata, container_name_metadata, environment_fingerprint, resource_name, run_checked, translate_path_in_container, }; @@ -38,7 +38,7 @@ const BUILD_USER_UID: u32 = 1000; const BUILD_USER_GID: u32 = 1000; const ENVIRONMENT_CONFIG_KEY: &str = "user.debmagic.environment"; const PROJECT_KEY: &str = "project"; -const ENVIRONMENT_SETUP_VERSION: &str = "dpkg-dev-norec python3; build-user-v1; raw.idmap-v1"; +const ENVIRONMENT_SETUP_VERSION: &str = "build-user-v1; raw.idmap-v1"; // ── Config ──────────────────────────────────────────────────────────────────── @@ -301,14 +301,19 @@ impl DriverLxd { let host_uid = unsafe { libc::geteuid() }.to_string(); let host_gid = unsafe { libc::getegid() }.to_string(); let build_root = environment.root_dir.to_string_lossy(); - let proposed_fingerprint = proposed.to_string(); + let setup = apt::Setup { + distro: &environment.distro, + mirror: apt_mirror, + proposed, + services: environment.purpose == EnvironmentPurpose::Test, + }; + let setup_fingerprint = setup.fingerprint(); let mut fingerprint_parts = vec![ variant.binary(), ENVIRONMENT_SETUP_VERSION, + &setup_fingerprint, &base_image, &environment.distro.codename, - apt_mirror.unwrap_or(""), - &proposed_fingerprint, &host_uid, &host_gid, build_root.as_ref(), @@ -418,30 +423,13 @@ impl DriverLxd { } if !reusing_container { - // stock images ship without an apt index - base.exec_in_container_checked(&["apt-get", "update"], None, true, &[]) - .map_err(|e| { - anyhow::anyhow!("Error running apt-get update in container: {e}") - })?; - // Install the base tooling that stock images don't include. - // build-essential is intentionally omitted: binary builds pull - // it in explicitly, so source-only environments stay lean. - // build-dep is omitted too: build.rs runs it for every driver - // against the real mounted source tree. - base.exec_in_container_checked( - &[ - "apt-get", - "install", - "-y", - "--no-install-recommends", - "dpkg-dev", - "python3", - ], - None, - true, - &[], - ) - .map_err(|e| anyhow::anyhow!("Error installing base packages in container: {e}"))?; + // apt runs once the run knows what to install + let setup_dir = environment.temp_dir().join(apt::SETUP_DIR); + setup.write(&setup_dir)?; + let setup_dir = base.translate_path_in_container(&setup_dir)?; + let script = setup.script(&setup_dir.to_string_lossy()); + base.exec_in_container_checked(&["sh", "-ec", &script], None, true, &[]) + .map_err(|e| anyhow::anyhow!("Error setting up apt in container: {e}"))?; let ensure_build_user = format!( "getent group {gid} >/dev/null || groupadd --gid {gid} debmagic; \ @@ -452,31 +440,6 @@ impl DriverLxd { base.exec_in_container_checked(&["sh", "-ec", &ensure_build_user], None, true, &[]) .map_err(|e| anyhow::anyhow!("Error creating build user in container: {e}"))?; - if apt_mirror.is_some() || proposed { - let script_path = environment.temp_dir().join("mirror.py"); - fs::write(&script_path, APT_MIRROR_SCRIPT)?; - let container_script_path = base.translate_path_in_container(&script_path)?; - let mut args = vec![ - "python3".to_string(), - container_script_path.to_string_lossy().into_owned(), - "--codename".to_string(), - environment.distro.codename.clone(), - ]; - if let Some(mirror) = apt_mirror { - args.extend(["--mirror".to_string(), mirror.to_string()]); - } - if proposed { - args.push("--proposed".to_string()); - } - let args = args.iter().map(String::as_str).collect::>(); - base.exec_in_container_checked(&args, None, true, &[]) - .map_err(|e| anyhow::anyhow!("Error configuring apt sources: {e}"))?; - base.exec_in_container_checked(&["apt-get", "update"], None, true, &[]) - .map_err(|e| { - anyhow::anyhow!("Error updating configured apt sources: {e}") - })?; - } - run_checked( base.lxd_cmd("config") .arg("set") diff --git a/packages/debmagic/src/driver/mod.rs b/packages/debmagic/src/driver/mod.rs index 03962bd2..87f137ab 100644 --- a/packages/debmagic/src/driver/mod.rs +++ b/packages/debmagic/src/driver/mod.rs @@ -104,28 +104,6 @@ pub fn environment_fingerprint(parts: &[&str]) -> String { .to_string() } -/// Refresh the apt index and upgrade the installed packages, so a cached -/// image or a long-lived Environment builds against current packages, like -/// sbuild does. -pub fn upgrade_packages(driver: &dyn EnvironmentDriver, cwd: &Path) -> io::Result<()> { - let env = [("DEBIAN_FRONTEND", "noninteractive")]; - driver.run_command_checked(&["apt-get", "update"], cwd, true, &env)?; - driver.run_command_checked( - &[ - "apt-get", - "-y", - "-o", - "Dpkg::Options::=--force-confdef", - "-o", - "Dpkg::Options::=--force-confold", - "upgrade", - ], - cwd, - true, - &env, - ) -} - /// Whether a reused environment needs an apt refresh again under `policy`. /// `last_update` is when its apt index was last refreshed; `None` means /// never. @@ -350,13 +328,6 @@ pub struct EnvironmentMetadata { pub driver_metadata: HashMap, } -/// Source of a Python script that rewrites the default Debian/Ubuntu apt -/// sources to point at a mirror. Replaces the base image's default sources -/// file(s) outright with one debmagic owns, rather than parsing and -/// patching them in place, defaulting to the release/updates/security -/// pockets it detects from `/etc/os-release`. -pub const APT_MIRROR_SCRIPT: &str = include_str!("scripts/mirror.py"); - pub trait EnvironmentDriver { fn driver_metadata(&self) -> HashMap; @@ -402,6 +373,12 @@ pub trait EnvironmentDriver { true } + /// When the packages of a fresh Environment were last refreshed, if it + /// comes with refreshed ones (a container from a cached image). + fn packages_refreshed_at(&self) -> Option { + None + } + /// Whether the Driver resource for this Environment still exists. fn probe_resource(&self) -> ResourceStatus; @@ -481,6 +458,14 @@ impl EnvironmentDriver for Driver { } } + fn packages_refreshed_at(&self) -> Option { + match self { + Self::Docker(d) => d.packages_refreshed_at(), + Self::Bare(d) => d.packages_refreshed_at(), + Self::Lxd(d) => d.packages_refreshed_at(), + } + } + fn probe_resource(&self) -> ResourceStatus { match self { Self::Docker(d) => d.probe_resource(), diff --git a/packages/debmagic/src/driver/scripts/mirror.py b/packages/debmagic/src/driver/scripts/mirror.py deleted file mode 100644 index 85239c0d..00000000 --- a/packages/debmagic/src/driver/scripts/mirror.py +++ /dev/null @@ -1,196 +0,0 @@ -#!/usr/bin/env python3 -"""Point apt at a mirror by replacing the base image's default sources. - -debmagic picks the base image itself, so the default sources file it ships -(classic `sources.list` or deb822 `*.sources`) is a known quantity: rather -than parsing and patching it in place, it is simply removed and replaced -with a single deb822 file that debmagic owns and fully controls. Unless -overridden, the suites/components enabled default to the release, its -`-updates` and its `-security` pockets - the same ones any base image -ships out of the box. -""" - -from __future__ import annotations - -import argparse -import glob -import os - -OS_RELEASE_FILE = "/etc/os-release" - -# Default apt configuration shipped by the base images debmagic uses; -# removed unconditionally in favour of MANAGED_SOURCES_FILE below. -DEFAULT_SOURCE_FILES = [ - "/etc/apt/sources.list", - "/etc/apt/sources.list.d/ubuntu.sources", - "/etc/apt/sources.list.d/debian.sources", -] - -MANAGED_SOURCES_FILE = "/etc/apt/sources.list.d/debmagic.sources" - -# Debian codenames with no separate updates/security/backports pockets. -DEBIAN_ROLLING_CODENAMES = {"unstable", "sid", "testing", "experimental"} -DEBIAN_WITHOUT_PROPOSED = {"unstable", "sid", "experimental"} - - -def read_os_release(path: str = OS_RELEASE_FILE) -> dict[str, str]: - values: dict[str, str] = {} - with open(path, encoding="utf-8") as f: - for line in f: - stripped_line = line.strip() - if not stripped_line or stripped_line.startswith("#") or "=" not in stripped_line: - continue - key, _, value = stripped_line.partition("=") - values[key] = value.strip().strip('"') - return values - - -def default_suites(os_release: dict[str, str], codename: str) -> list[str]: - if os_release.get("ID") == "debian" and codename in DEBIAN_ROLLING_CODENAMES: - return [codename] - return [codename, f"{codename}-updates", f"{codename}-security"] - - -def default_components(os_release: dict[str, str]) -> list[str]: - if os_release.get("ID") == "ubuntu": - return ["main", "restricted", "universe", "multiverse"] - - major = int((os_release.get("VERSION_ID") or "0").split(".")[0] or 0) - if major >= 12: - return ["main", "contrib", "non-free", "non-free-firmware"] - return ["main", "contrib", "non-free"] - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--mirror", - help="apt mirror base URI, e.g. http://archive.ubuntu.com/ubuntu/", - ) - parser.add_argument( - "--codename", - help="target distribution codename; defaults to VERSION_CODENAME from the base image", - ) - parser.add_argument( - "--suite", - dest="suites", - action="append", - help="suite/pocket to enable, e.g. noble or noble-security (repeatable); " - "defaults to the release plus its '-updates' and '-security' pockets", - ) - parser.add_argument( - "--component", - dest="components", - action="append", - help="component to enable, e.g. main or universe (repeatable); defaults to those enabled by the base image", - ) - parser.add_argument( - "--proposed", - action="store_true", - help="also enable the '-proposed' pocket", - ) - return parser.parse_args() - - -def iter_sources() -> list[tuple[str, list[str]]]: - """Yield (uri, suites) from deb822 *.sources files and legacy sources.list.""" - sources: list[tuple[str, list[str]]] = [] - for path in glob.glob("/etc/apt/sources.list.d/*.sources"): - uri: str | None = None - suites: list[str] = [] - enabled = True - with open(path, encoding="utf-8") as f: - for line in [*f, "\n"]: - stripped_line = line.strip() - if not stripped_line: - if uri and suites and enabled: - sources.append((uri, suites)) - uri, suites, enabled = None, [], True - continue - key, _, value = stripped_line.partition(":") - key, value = key.strip(), value.strip() - if key == "URIs": - uri = value.split()[0] - elif key == "Suites": - suites = value.split() - elif key == "Enabled": - enabled = value.lower() != "no" - for path in ["/etc/apt/sources.list", *glob.glob("/etc/apt/sources.list.d/*.list")]: - try: - f = open(path, encoding="utf-8") - except FileNotFoundError: - continue - with f: - for line in f: - fields = line.split() - if len(fields) >= 3 and fields[0] in {"deb", "deb-src"}: - fields = fields[1:] - if fields[0].startswith("["): - fields = fields[fields.index(next(f for f in fields if f.endswith("]"))) + 1 :] - if len(fields) >= 2: - sources.append((fields[0], fields[1:])) - return sources - - -def configured_mirror(codename: str) -> str: - for uri, suites in iter_sources(): - if codename in suites: - return uri - raise RuntimeError(f"could not determine the configured mirror for {codename}") - - -def proposed_suite(os_release: dict[str, str], codename: str) -> str: - if os_release.get("ID") == "debian": - if codename in DEBIAN_WITHOUT_PROPOSED: - raise ValueError(f"Debian {codename} has no proposed-updates pocket") - return f"{codename}-proposed-updates" - return f"{codename}-proposed" - - -def main() -> None: - args = parse_args() - os_release = read_os_release() - codename = args.codename or os_release["VERSION_CODENAME"] - - suites = args.suites or default_suites(os_release, codename) - components = args.components or default_components(os_release) - if args.proposed: - suites = [*suites, proposed_suite(os_release, codename)] - - if not args.mirror: - if not args.proposed: - raise ValueError("--mirror or --proposed is required") - suites = [proposed_suite(os_release, codename)] - args.mirror = configured_mirror(codename) - source_files = [] - source_groups = [(args.mirror, suites)] - else: - source_files = DEFAULT_SOURCE_FILES - if os_release.get("ID") == "debian" and codename not in DEBIAN_ROLLING_CODENAMES: - security_suite = f"{codename}-security" - source_groups = [ - (args.mirror, [suite for suite in suites if suite != security_suite]), - (configured_mirror(security_suite), [security_suite]), - ] - else: - source_groups = [(args.mirror, suites)] - - for path in source_files: - try: - os.remove(path) - except FileNotFoundError: - pass - - os.makedirs(os.path.dirname(MANAGED_SOURCES_FILE), exist_ok=True) - with open(MANAGED_SOURCES_FILE, "w", encoding="utf-8") as f: - for index, (mirror, group_suites) in enumerate(source_groups): - if index: - f.write("\n") - f.write("Types: deb deb-src\n") - f.write(f"URIs: {mirror}\n") - f.write(f"Suites: {' '.join(group_suites)}\n") - f.write(f"Components: {' '.join(components)}\n") - - -if __name__ == "__main__": - main() diff --git a/packages/debmagic/src/environment/mod.rs b/packages/debmagic/src/environment/mod.rs index 9a915198..3807241b 100644 --- a/packages/debmagic/src/environment/mod.rs +++ b/packages/debmagic/src/environment/mod.rs @@ -160,27 +160,39 @@ impl<'a> ClaimedEnvironment<'a> { Ok(claimed) } - /// Upgrade the packages of a container Environment: always when it is - /// fresh, when reused only once `policy` says the last upgrade (recorded - /// in the registry) has gone stale. The Bare driver is the host, which - /// the user maintains. + /// Upgrade the packages of a container Environment unless `policy` says + /// the last upgrade is recent enough: the one recorded in the registry + /// for a reused Environment, the image's for a fresh one. A fresh one + /// without a refreshed image always upgrades. The Bare driver is the + /// host, which the user maintains. async fn upgrade_packages(&self, policy: crate::time::RefreshPolicy) -> anyhow::Result<()> { if self.environment.driver == crate::driver::DriverType::Bare { return Ok(()); } let id = self.environment.id(); - if self.driver.reused_environment() { - let last_update = - registry::apt_updated_at(&mut *self.registry.connection().await?, &id).await?; - if !crate::driver::apt_update_needed_on_reuse(policy, last_update) { - return Ok(()); - } + let reused = self.driver.reused_environment(); + let last_update = if reused { + registry::apt_updated_at(&mut *self.registry.connection().await?, &id).await? + } else { + self.driver.packages_refreshed_at() + }; + let current = (reused || last_update.is_some()) + && !crate::driver::apt_update_needed_on_reuse(policy, last_update); + if current && reused { + return Ok(()); } - crate::output::step("Upgrading the packages in the Environment"); - crate::driver::upgrade_packages(&self.driver, &self.environment.root_dir)?; + // record a fresh Environment's image time for its reuse + let updated_at = match last_update.filter(|_| current) { + Some(image_refreshed_at) => image_refreshed_at, + None => { + crate::output::step("Upgrading the packages in the Environment"); + crate::apt::refresh(&self.driver, &self.environment.root_dir)?; + std::time::SystemTime::now() + } + }; let recorded = async { let mut conn = self.registry.connection().await?; - registry::set_apt_updated_at(&mut conn, &id, std::time::SystemTime::now()).await + registry::set_apt_updated_at(&mut conn, &id, updated_at).await }; if let Err(error) = recorded.await { eprintln!("Warning: failed to record the apt upgrade time: {error:#}"); diff --git a/packages/debmagic/src/main.rs b/packages/debmagic/src/main.rs index 5db7a8d9..694954cb 100644 --- a/packages/debmagic/src/main.rs +++ b/packages/debmagic/src/main.rs @@ -7,6 +7,7 @@ use crate::cli::{ BuildTarget, ChangelogCommands, Cli, Commands, ConfigCommands, EnvCommands, UpstreamCommands, }; +pub mod apt; pub mod build; pub mod build_intent; pub mod changelog; diff --git a/packages/debmagic/src/test/run.rs b/packages/debmagic/src/test/run.rs index d1c82e18..bddfc425 100644 --- a/packages/debmagic/src/test/run.rs +++ b/packages/debmagic/src/test/run.rs @@ -306,12 +306,19 @@ pub async fn run_test(intent: &TestIntent) -> anyhow::Result { |claimed| { crate::output::stage("Installing autopkgtest"); let staged_source_dir = stage_dir(claimed.environment(), BuildKind::Binary); - claimed.driver().run_command_checked( - &["apt-get", "install", "-y", "autopkgtest"], + // the former recommends autopkgtest needs for generated tests + // and build-needed ones + let install = crate::apt::Install { + packages: vec!["autopkgtest", "autodep8", "fakeroot"], + ..Default::default() + }; + crate::apt::install( + claimed.driver(), &staged_source_dir, - true, - &[("DEBIAN_FRONTEND", "noninteractive")], - )?; + &claimed.environment().distro, + &install, + ) + .context("installing autopkgtest failed")?; let work_dir = claimed.environment().work_dir(); let changes_filename = changes_path From 72f67e00605337cbfcfce9fc367063338055c286 Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Fri, 23 Oct 2026 15:35:23 +0200 Subject: [PATCH 37/48] feat(build): read the Debian and Ubuntu releases from distro-info-data New releases and moving aliases needed a debmagic update. debmagic now reads /usr/share/distro-info, or fetches the data into ~/.cache/debmagic/distro-info/ when that is missing or predates the latest release, and derives `stable`, `oldstable`, `testing` and `devel` from the release dates. - `testing` is accepted as a suite alias - the Debian package depends on distro-info-data --- debian/control | 1 + docs/usage/build.md | 6 +- packages/debmagic-common/src/apt.rs | 15 +- packages/debmagic-common/src/distro.rs | 290 +++++++++++++++++++------ packages/debmagic/src/apt.rs | 2 +- packages/debmagic/src/build/mod.rs | 4 + packages/debmagic/src/build/source.rs | 18 +- packages/debmagic/src/distro_info.rs | 117 ++++++++++ packages/debmagic/src/main.rs | 1 + packages/debmagic/src/package.rs | 134 +++++++++--- packages/debmagic/src/test/run.rs | 20 +- 11 files changed, 485 insertions(+), 123 deletions(-) create mode 100644 packages/debmagic/src/distro_info.rs diff --git a/debian/control b/debian/control index c2ff096d..d4101c31 100644 --- a/debian/control +++ b/debian/control @@ -80,6 +80,7 @@ Description: dpkg build driver for debmagic rules.py packages. Package: debmagic Architecture: any Depends: + distro-info-data, ${misc:Depends}, ${shlibs:Depends}, Suggests: diff --git a/docs/usage/build.md b/docs/usage/build.md index 6f0f6b51..1f789560 100644 --- a/docs/usage/build.md +++ b/docs/usage/build.md @@ -146,8 +146,10 @@ If the changelog has a single unambiguous entry, omit it. `--distro` also overrides the changelog's distribution: you can use it to rebuild a package released for an older release on a newer one, or to attempt a backport. On the Bare driver the target must still match the host's os-release; pass `--bare-ignore-release` to build for a different suite anyway, with the host providing the build dependencies itself. -Suite aliases in the changelog (or via `--distro`) resolve to a concrete release: Debian `stable` / `oldstable` / `sid` (→ `unstable`), and Ubuntu `devel`. -Alias targets are updated manually when Debian/Ubuntu roll. +Suite aliases in the changelog (or via `--distro`) resolve to a concrete release: Debian `stable` / `oldstable` / `testing` / `sid` (→ `unstable`), and Ubuntu `devel`. + +The releases and their dates come from [distro-info-data](https://salsa.debian.org/debian/distro-info-data) in `/usr/share/distro-info`. +Without it, or when it predates the latest release, debmagic fetches the data into `~/.cache/debmagic/distro-info/` and refreshes that copy every 30 days. Non-Debian/Ubuntu suites (still apt/dpkg-based) are supported when declared for the active container Driver via `base_images`, e.g. `driver.docker.base_images = { "yocto:kirkstone" = "my-registry/yocto-kirkstone:latest" }`. The changelog/`--distro` value stays the bare codename (`kirkstone`). On the Bare driver, binary builds require the host `/etc/os-release` to match: built-in Debian/Ubuntu need matching `ID` and codename, other suites a matching `VERSION_CODENAME`. diff --git a/packages/debmagic-common/src/apt.rs b/packages/debmagic-common/src/apt.rs index 8f3edd18..a3374c9c 100644 --- a/packages/debmagic-common/src/apt.rs +++ b/packages/debmagic-common/src/apt.rs @@ -105,11 +105,20 @@ pub fn has_satisfy(distro: &DistroVersion) -> bool { #[cfg(test)] mod tests { use super::*; - use crate::distro::get_distro_version; use test_case::test_case; fn distro(name: &str) -> DistroVersion { - get_distro_version(name).unwrap() + let (distro, version) = match name { + "buster" => (Distro::Debian, "10"), + "bullseye" => (Distro::Debian, "11"), + "trixie" => (Distro::Debian, "13"), + "unstable" => (Distro::Debian, ""), + "bionic" => (Distro::Ubuntu, "18.04"), + "focal" => (Distro::Ubuntu, "20.04"), + "noble" => (Distro::Ubuntu, "24.04"), + other => panic!("no test release {other}"), + }; + DistroVersion::new(distro, name, version) } #[test] @@ -130,7 +139,7 @@ mod tests { #[test_case(false ; "release")] #[test_case(true ; "no proposed pocket")] fn debian_unstable_is_one_suite(proposed: bool) { - let sources = sources_file(&distro("sid"), None, proposed, "arm64").unwrap(); + let sources = sources_file(&distro("unstable"), None, proposed, "arm64").unwrap(); assert_eq!( sources, "Types: deb deb-src\nURIs: http://deb.debian.org/debian\nSuites: unstable\n\ diff --git a/packages/debmagic-common/src/distro.rs b/packages/debmagic-common/src/distro.rs index 6496a22c..adc06e46 100644 --- a/packages/debmagic-common/src/distro.rs +++ b/packages/debmagic-common/src/distro.rs @@ -1,7 +1,8 @@ +use anyhow::Context; +use chrono::NaiveDate; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::fmt; -use std::sync::LazyLock; /// Distribution family. Known Debian/Ubuntu variants plus an open [`Custom`] name /// for other apt/dpkg targets (`yocto`, …). @@ -98,77 +99,153 @@ impl DistroVersion { Self::new(distro, codename, "") } - fn devel(mut self) -> Self { - self.is_devel = true; - self - } - pub fn key(&self) -> String { format!("{}:{}", self.distro, self.codename) } } -static DISTRO_INFO_MAP: LazyLock> = LazyLock::new(|| { - use Distro::{Debian, Ubuntu}; - HashMap::from([ - // debian - ( - "experimental", - DistroVersion::new(Debian, "experimental", ""), - ), - ("unstable", DistroVersion::new(Debian, "unstable", "")), - // Suite alias: sid → unstable (concrete release identity). - ("sid", DistroVersion::new(Debian, "unstable", "")), - ("testing", DistroVersion::new(Debian, "testing", "")), - ("duke", DistroVersion::new(Debian, "duke", "15")), - ("forky", DistroVersion::new(Debian, "forky", "14")), - ("trixie", DistroVersion::new(Debian, "trixie", "13")), - // Suite alias: stable → current stable release (update when Debian rolls). - ("stable", DistroVersion::new(Debian, "trixie", "13")), - ("bookworm", DistroVersion::new(Debian, "bookworm", "12")), - // Suite alias: oldstable → current oldstable release. - ("oldstable", DistroVersion::new(Debian, "bookworm", "12")), - ("bullseye", DistroVersion::new(Debian, "bullseye", "11")), - ("buster", DistroVersion::new(Debian, "buster", "10")), - ("stretch", DistroVersion::new(Debian, "stretch", "9")), - // ubuntu - ( - "stonking", - DistroVersion::new(Ubuntu, "stonking", "26.10").devel(), - ), - // Suite alias: devel → current Ubuntu development release. - ( - "devel", - DistroVersion::new(Ubuntu, "stonking", "26.10").devel(), - ), - ("resolute", DistroVersion::new(Ubuntu, "resolute", "26.04")), - ("noble", DistroVersion::new(Ubuntu, "noble", "24.04")), - ("jammy", DistroVersion::new(Ubuntu, "jammy", "22.04")), - ("focal", DistroVersion::new(Ubuntu, "focal", "20.04")), - ("bionic", DistroVersion::new(Ubuntu, "bionic", "18.04")), - ("xenial", DistroVersion::new(Ubuntu, "xenial", "16.04")), - ("trusty", DistroVersion::new(Ubuntu, "trusty", "14.04")), - ]) -}); - -/// Look up a built-in distribution by codename or suite alias. -/// -/// Suite aliases are map keys that resolve to a concrete release [`DistroVersion`]: -/// - Debian: `stable` → current stable release, `oldstable` → current oldstable, -/// `sid` → `unstable` -/// - Ubuntu: `devel` → current development release +/// The known Debian and Ubuntu releases and suite aliases, from +/// distro-info-data's `debian.csv` and `ubuntu.csv`. /// -/// Alias targets are maintained manually when Debian/Ubuntu roll. -/// Non-built-in suites are not returned here; callers resolve those via Driver -/// `base_images` or Bare `/etc/os-release` checks. -/// -/// A pocket suffix (`resolute-security`, `bookworm-backports`, ...) resolves to -/// its release: the build environment is the same. -pub fn get_distro_version(name: &str) -> Option { - DISTRO_INFO_MAP - .get(name) - .or_else(|| DISTRO_INFO_MAP.get(strip_pocket(name)?)) - .cloned() +/// A release's [`DistroVersion::codename`] is the CSV's `series` (`trixie`); +/// its `codename` column (`Trixie`) is only a display name. +#[derive(Debug, Clone, Default)] +pub struct DistroTable { + by_name: HashMap, + /// Families without a release in development: their data predates the + /// latest release, like python-distro-info's `DistroDataOutdated`. + outdated: Vec, +} + +impl DistroTable { + /// The table as of `today`, which decides the suite aliases: Debian's + /// `stable` and `oldstable` are the last two releases, `testing` the + /// one in development (created, not released yet), as is Ubuntu's + /// `devel`. `sid` is `unstable`. + pub fn from_distro_info( + debian_csv: &str, + ubuntu_csv: &str, + today: NaiveDate, + ) -> anyhow::Result { + let mut table = Self::default(); + let debian = parse_distro_info(debian_csv).context("parsing debian.csv failed")?; + table.add(Distro::Debian, &debian, today); + let ubuntu = parse_distro_info(ubuntu_csv).context("parsing ubuntu.csv failed")?; + table.add(Distro::Ubuntu, &ubuntu, today); + Ok(table) + } + + fn add(&mut self, distro: Distro, releases: &[Release], today: NaiveDate) { + let (devel_alias, released_aliases) = match distro { + Distro::Debian => ("testing", &["stable", "oldstable"][..]), + _ => ("devel", &[][..]), + }; + let mut released = Vec::new(); + let mut has_devel = false; + for release in releases { + // unstable and experimental have no version + let numbered = !release.version.is_empty(); + let is_released = release.release.is_some_and(|date| date <= today); + let version = DistroVersion { + distro: distro.clone(), + codename: match (&distro, release.series.as_str()) { + (Distro::Debian, "sid") => "unstable".to_string(), + (_, series) => series.to_string(), + }, + version: release.version.clone(), + is_devel: numbered && release.created <= today && !is_released, + }; + if version.is_devel { + has_devel = true; + self.insert(devel_alias, &version); + } + if numbered && is_released { + released.push(version.clone()); + } + self.insert(&release.series, &version); + self.insert(&version.codename, &version); + } + for (alias, version) in released_aliases.iter().zip(released.iter().rev()) { + self.insert(alias, version); + } + if !has_devel { + self.outdated.push(distro); + } + } + + fn insert(&mut self, name: &str, version: &DistroVersion) { + self.by_name.insert(name.to_string(), version.clone()); + } + + /// The families whose data is older than their latest release, so it + /// lacks the release now in development. + pub fn outdated(&self) -> &[Distro] { + &self.outdated + } + + /// Look up a release by codename or suite alias. + /// + /// A pocket suffix (`resolute-security`, `bookworm-backports`, ...) + /// resolves to its release: the build environment is the same. + /// Other suites are not returned here; callers resolve those via Driver + /// `base_images` or Bare `/etc/os-release` checks. + pub fn get(&self, name: &str) -> Option { + self.by_name + .get(name) + .or_else(|| self.by_name.get(strip_pocket(name)?)) + .cloned() + } +} + +/// One row of a distro-info-data CSV. +struct Release { + /// `26.04` without the ` LTS` suffix, empty for Debian's unstable and + /// experimental. + version: String, + series: String, + created: NaiveDate, + release: Option, +} + +fn parse_distro_info(csv: &str) -> anyhow::Result> { + let mut lines = csv.lines().filter(|line| !line.trim().is_empty()); + let header: Vec<&str> = lines.next().context("no header")?.split(',').collect(); + let column = |name: &str| { + header + .iter() + .position(|column| *column == name) + .with_context(|| format!("no {name} column")) + }; + let [version, series, created, release] = [ + column("version")?, + column("series")?, + column("created")?, + column("release")?, + ]; + let date = |field: &str| NaiveDate::parse_from_str(field, "%Y-%m-%d"); + lines + .enumerate() + .map(|(index, line)| { + let fields: Vec<&str> = line.split(',').collect(); + let field = |column: usize| fields.get(column).copied().unwrap_or_default(); + let release = (|| { + Ok::<_, anyhow::Error>(Release { + version: field(version) + .split_whitespace() + .next() + .unwrap_or_default() + .to_string(), + series: field(series).to_string(), + created: date(field(created)).context("invalid created date")?, + release: match field(release) { + "" => None, + release => Some(date(release).context("invalid release date")?), + }, + }) + })(); + release.with_context(|| format!("line {} is invalid: {line}", index + 2)) + }) + .collect() } /// Pocket suffixes of Debian and Ubuntu suites @@ -191,19 +268,88 @@ mod tests { assert_eq!(serde_json::to_string(&d).unwrap(), "\"debian\""); } + const DEBIAN: &str = "version,codename,series,created,release,eol\n\ + 12,Bookworm,bookworm,2021-08-14,2023-06-10,2026-07-11\n\ + 13,Trixie,trixie,2023-06-10,2025-08-09\n\ + 14,Forky,forky,2025-08-09\n\ + 15,Duke,duke,2027-08-01\n\ + ,Sid,sid,1993-08-16\n\ + ,Experimental,experimental,1993-08-16\n"; + const UBUNTU: &str = "version,codename,series,created,release,eol\n\ + 26.04 LTS,Resolute Raccoon,resolute,2025-10-09,2026-04-23,2031-05-29\n\ + 26.10,Stonking Stingray,stonking,2026-04-24,2026-10-15,2027-07-15\n\ + 27.04,Tenacious T-Rex,tenacious,2026-10-16,2027-04-22\n"; + + fn table(year: i32, month: u32, day: u32) -> DistroTable { + let today = NaiveDate::from_ymd_opt(year, month, day).unwrap(); + DistroTable::from_distro_info(DEBIAN, UBUNTU, today).unwrap() + } + #[test] fn pocket_suffix_resolves_to_release() { - let resolute = get_distro_version("resolute"); + let table = table(2026, 10, 23); + let resolute = table.get("resolute"); assert!(resolute.is_some()); for suite in ["resolute-security", "resolute-updates", "resolute-proposed"] { - assert_eq!(get_distro_version(suite), resolute, "{suite}"); + assert_eq!(table.get(suite), resolute, "{suite}"); } + assert_eq!(table.get("bookworm-backports"), table.get("bookworm")); + assert_eq!(table.get("resolute-nope"), None); + assert_eq!(table.get("yocto-security"), None); + } + + #[test_case::test_case("stable", "trixie", "13", false)] + #[test_case::test_case("oldstable", "bookworm", "12", false)] + #[test_case::test_case("testing", "forky", "14", true)] + #[test_case::test_case("forky", "forky", "14", true)] + #[test_case::test_case("duke", "duke", "15", false ; "not created yet")] + #[test_case::test_case("sid", "unstable", "", false)] + #[test_case::test_case("unstable", "unstable", "", false)] + #[test_case::test_case("experimental", "experimental", "", false)] + #[test_case::test_case("devel", "tenacious", "27.04", true)] + #[test_case::test_case("stonking", "stonking", "26.10", false ; "just released")] + #[test_case::test_case("resolute", "resolute", "26.04", false ; "lts suffix")] + fn table_resolves(name: &str, codename: &str, version: &str, is_devel: bool) { + let resolved = table(2026, 10, 23).get(name).unwrap(); assert_eq!( - get_distro_version("bookworm-backports"), - get_distro_version("bookworm") + ( + resolved.codename.as_str(), + resolved.version.as_str(), + resolved.is_devel + ), + (codename, version, is_devel) + ); + } + + #[test] + fn aliases_follow_the_date() { + let before_release = table(2026, 10, 14); + assert_eq!(before_release.get("devel").unwrap().codename, "stonking"); + assert!(before_release.get("stonking").unwrap().is_devel); + let old = table(2024, 1, 1); + assert_eq!(old.get("stable").unwrap().codename, "bookworm"); + assert_eq!(old.get("testing").unwrap().codename, "trixie"); + } + + #[test] + fn data_without_a_release_in_development_is_outdated() { + assert_eq!(table(2026, 10, 23).outdated(), &[]); + // after tenacious and before duke was created + assert_eq!(table(2027, 5, 1).outdated(), &[Distro::Ubuntu]); + } + + #[test] + fn invalid_distro_info_names_the_line() { + let error = DistroTable::from_distro_info( + "version,codename,series,created,release\n13,Trixie,trixie,soon,\n", + UBUNTU, + NaiveDate::from_ymd_opt(2026, 10, 23).unwrap(), + ) + .unwrap_err(); + assert!( + format!("{error:#}").contains("line 2 is invalid"), + "{error:#}" ); - assert_eq!(get_distro_version("resolute-nope"), None); - assert_eq!(get_distro_version("yocto-security"), None); } #[test] diff --git a/packages/debmagic/src/apt.rs b/packages/debmagic/src/apt.rs index 78136791..cfaaa658 100644 --- a/packages/debmagic/src/apt.rs +++ b/packages/debmagic/src/apt.rs @@ -298,7 +298,7 @@ mod tests { #[test] fn setup_replaces_the_default_sources_only_with_its_own() -> anyhow::Result<()> { let dir = std::env::temp_dir().join(format!("debmagic-apt-setup-{}", std::process::id())); - let trixie = debmagic_common::distro::get_distro_version("trixie").unwrap(); + let trixie = DistroVersion::new(debmagic_common::distro::Distro::Debian, "trixie", "13"); let mut setup = Setup { distro: &trixie, diff --git a/packages/debmagic/src/build/mod.rs b/packages/debmagic/src/build/mod.rs index 7567c72f..49e05cdf 100644 --- a/packages/debmagic/src/build/mod.rs +++ b/packages/debmagic/src/build/mod.rs @@ -535,9 +535,13 @@ pub struct BuildCommand { /// `--upload` was passed. pub async fn build(command: BuildCommand) -> anyhow::Result<()> { let intent = resolve_build_intent(command.intent.clone())?; + let distros = crate::distro_info::load() + .await + .context("loading the Debian and Ubuntu releases failed")?; let target = crate::package::resolve_package_target( &intent.source_dir, command.distro.as_deref(), + &distros, crate::package::distro_resolve_mode_for_driver( intent.driver, &intent.config.driver.docker.base_images, diff --git a/packages/debmagic/src/build/source.rs b/packages/debmagic/src/build/source.rs index af539792..d959699e 100644 --- a/packages/debmagic/src/build/source.rs +++ b/packages/debmagic/src/build/source.rs @@ -592,7 +592,11 @@ mod tests { package_identifier: "example-1.0".to_string(), source_dir: source_dir.clone(), root_dir: build_root_dir.clone(), - distro: debmagic_common::distro::get_distro_version("trixie").unwrap(), + distro: debmagic_common::distro::DistroVersion::new( + debmagic_common::distro::Distro::Debian, + "trixie", + "13", + ), persistence: Persistence::Always, purpose: EnvironmentPurpose::Build, }; @@ -688,7 +692,11 @@ mod tests { package_identifier: "example-1.0".to_string(), source_dir: PathBuf::from("/src"), root_dir: PathBuf::from("/tmp/debmagic/example-1.0"), - distro: debmagic_common::distro::get_distro_version("trixie").unwrap(), + distro: debmagic_common::distro::DistroVersion::new( + debmagic_common::distro::Distro::Debian, + "trixie", + "13", + ), persistence: Persistence::Always, purpose: EnvironmentPurpose::Build, }; @@ -739,7 +747,11 @@ mod tests { package_identifier: "example-1.0".to_string(), source_dir: source_dir.clone(), root_dir: test_root.join("build"), - distro: debmagic_common::distro::get_distro_version("trixie").unwrap(), + distro: debmagic_common::distro::DistroVersion::new( + debmagic_common::distro::Distro::Debian, + "trixie", + "13", + ), persistence: Persistence::Always, purpose: EnvironmentPurpose::Build, }; diff --git a/packages/debmagic/src/distro_info.rs b/packages/debmagic/src/distro_info.rs new file mode 100644 index 00000000..263b460b --- /dev/null +++ b/packages/debmagic/src/distro_info.rs @@ -0,0 +1,117 @@ +//! The Debian and Ubuntu release table from distro-info-data: the host's +//! copy if it's current, else one fetched from upstream into the cache. + +use std::path::{Path, PathBuf}; +use std::time::{Duration, SystemTime}; + +use anyhow::Context; +use debmagic_common::distro::DistroTable; + +const HOST_DIR: &str = "/usr/share/distro-info"; +const UPSTREAM: &str = "https://salsa.debian.org/debian/distro-info-data/-/raw/main"; +const DEBIAN_CSV: &str = "debian.csv"; +const UBUNTU_CSV: &str = "ubuntu.csv"; + +/// Picks up upstream corrections and announced release dates, which the +/// outdated check can't see. +const CACHE_MAX_AGE: Duration = Duration::from_secs(30 * 24 * 60 * 60); +/// Upstream lags behind a release for a while: don't refetch it on every run. +const OUTDATED_RETRY_AGE: Duration = Duration::from_secs(24 * 60 * 60); + +/// The release table as of today, from the first current copy of +/// distro-info-data: the host's, the cached one, a freshly fetched one. +/// When fetching fails, an outdated copy is used with a warning. +pub async fn load() -> anyhow::Result { + let mut host = read(Path::new(HOST_DIR)).transpose()?; + if let Some(table) = host.take_if(|table| table.outdated().is_empty()) { + return Ok(table); + } + + let cache_dir = dirs::cache_dir() + .map(|cache| cache.join("debmagic").join("distro-info")) + .context("no cache dir to keep distro-info-data in")?; + let mut cached = read(&cache_dir).transpose()?; + let age = cache_age(&cache_dir); + if let Some(table) = cached.take_if(|table| { + let max_age = match table.outdated() { + [] => CACHE_MAX_AGE, + _ => OUTDATED_RETRY_AGE, + }; + age.is_some_and(|age| age < max_age) + }) { + warn_if_outdated(&table, &cache_dir.display().to_string()); + return Ok(table); + } + + match (fetch(&cache_dir).await, cached.or(host)) { + (Ok(table), _) => { + warn_if_outdated(&table, UPSTREAM); + Ok(table) + } + (Err(error), Some(stale)) => { + eprintln!( + "debmagic: warning: using outdated distro-info-data, updating it failed: {error:#}" + ); + Ok(stale) + } + (Err(error), None) => Err(error.context(format!( + "no distro-info-data in {HOST_DIR} or the cache; install distro-info-data" + ))), + } +} + +fn today() -> chrono::NaiveDate { + chrono::Local::now().date_naive() +} + +/// The table from the CSVs in `dir`, `None` if it has none. +fn read(dir: &Path) -> Option> { + let debian = std::fs::read_to_string(dir.join(DEBIAN_CSV)).ok()?; + let table = std::fs::read_to_string(dir.join(UBUNTU_CSV)) + .with_context(|| format!("reading {UBUNTU_CSV} failed")) + .and_then(|ubuntu| DistroTable::from_distro_info(&debian, &ubuntu, today())) + .with_context(|| format!("loading the distro-info-data in {} failed", dir.display())); + Some(table) +} + +fn cache_age(dir: &Path) -> Option { + let modified = std::fs::metadata(dir.join(DEBIAN_CSV)) + .ok()? + .modified() + .ok()?; + SystemTime::now().duration_since(modified).ok() +} + +/// Fetch the CSVs from upstream into `dir`, only once both parse. +async fn fetch(dir: &Path) -> anyhow::Result { + let debian = crate::requests::http_get(&format!("{UPSTREAM}/{DEBIAN_CSV}")).await?; + let ubuntu = crate::requests::http_get(&format!("{UPSTREAM}/{UBUNTU_CSV}")).await?; + let table = DistroTable::from_distro_info(&debian, &ubuntu, today()) + .context("loading the fetched distro-info-data failed")?; + std::fs::create_dir_all(dir).with_context(|| format!("creating {} failed", dir.display()))?; + for (name, content) in [(UBUNTU_CSV, &ubuntu), (DEBIAN_CSV, &debian)] { + write_atomic(&dir.join(name), content)?; + } + Ok(table) +} + +fn write_atomic(path: &Path, content: &str) -> anyhow::Result<()> { + let partial = PathBuf::from(format!("{}.part", path.display())); + std::fs::write(&partial, content) + .and_then(|()| std::fs::rename(&partial, path)) + .with_context(|| format!("writing {} failed", path.display())) +} + +fn warn_if_outdated(table: &DistroTable, source: &str) { + let outdated: Vec<&str> = table + .outdated() + .iter() + .map(|distro| distro.as_str()) + .collect(); + if !outdated.is_empty() { + eprintln!( + "debmagic: warning: the distro-info-data from {source} predates the latest {} release", + outdated.join(" and ") + ); + } +} diff --git a/packages/debmagic/src/main.rs b/packages/debmagic/src/main.rs index 694954cb..40c427be 100644 --- a/packages/debmagic/src/main.rs +++ b/packages/debmagic/src/main.rs @@ -16,6 +16,7 @@ pub mod cli; pub mod config; pub mod control; pub mod data_dir; +pub mod distro_info; pub mod driver; pub mod env; pub mod environment; diff --git a/packages/debmagic/src/package.rs b/packages/debmagic/src/package.rs index 267247ee..f87d9bdc 100644 --- a/packages/debmagic/src/package.rs +++ b/packages/debmagic/src/package.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; use anyhow::{Context, anyhow, bail}; -use debmagic_common::distro::{Distro, DistroVersion, get_distro_version}; +use debmagic_common::distro::{Distro, DistroTable, DistroVersion}; use debmagic_common::package::{FileReader, Location, SourcePackage}; use crate::driver::DriverType; @@ -58,10 +58,11 @@ pub fn load_package(dir: &Path) -> anyhow::Result { pub fn resolve_package_target( dir: &Path, explicit_distro: Option<&str>, + distros: &DistroTable, mode: DistroResolveMode<'_>, ) -> anyhow::Result { let package = load_package(dir)?; - let distro = select_distro_version(package.distributions(), explicit_distro, mode)?; + let distro = select_distro_version(package.distributions(), explicit_distro, distros, mode)?; Ok(PackageTarget { package, distro }) } @@ -95,7 +96,7 @@ fn parse_base_image_key(key: &str) -> Option<(&str, &str)> { /// Reject custom (non-debian/ubuntu family) map keys whose codename collides /// with a built-in release or suite alias. -fn validate_base_images_keys<'a, I>(keys: I) -> anyhow::Result<()> +fn validate_base_images_keys<'a, I>(keys: I, distros: &DistroTable) -> anyhow::Result<()> where I: IntoIterator, { @@ -107,7 +108,7 @@ where }; match Distro::parse(family) { Distro::Debian | Distro::Ubuntu => {} - Distro::Custom(_) if get_distro_version(codename).is_some() => { + Distro::Custom(_) if distros.get(codename).is_some() => { return Err(anyhow!( "base_images key '{key}' uses codename '{codename}' which is a built-in \ Debian/Ubuntu suite; use a different codename or a debian:/ubuntu: image override" @@ -159,12 +160,16 @@ fn custom_distro_from_base_images( /// Resolve a distro codename (built-in or a custom suite from the Driver's /// `base_images` map) to a full [`DistroVersion`]. -pub fn lookup_distro(name: &str, mode: DistroResolveMode<'_>) -> anyhow::Result { +pub fn lookup_distro( + name: &str, + distros: &DistroTable, + mode: DistroResolveMode<'_>, +) -> anyhow::Result { if let DistroResolveMode::Container { base_images, .. } = mode { - validate_base_images_keys(base_images.keys().map(|k| k.as_str()))?; + validate_base_images_keys(base_images.keys().map(|k| k.as_str()), distros)?; } - if let Some(builtin) = get_distro_version(name) { + if let Some(builtin) = distros.get(name) { return Ok(builtin); } @@ -281,12 +286,13 @@ fn parse_os_release(contents: &str) -> OsRelease { fn select_distro_version( changelog_distros: &[String], explicit_distro: Option<&str>, + distros: &DistroTable, mode: DistroResolveMode<'_>, ) -> anyhow::Result { let Some(explicit) = explicit_distro else { return match changelog_distros { [] => bail!("changelog contains no distributions"), - [single] => lookup_distro(single, mode), + [single] => lookup_distro(single, distros, mode), many => bail!( "changelog contains multiple distributions ({}), please specify which one to build for with --distro", many.join(", ") @@ -294,10 +300,10 @@ fn select_distro_version( }; }; - let resolved = lookup_distro(explicit, mode)?; - let in_changelog = changelog_distros - .iter() - .any(|name| name == explicit || lookup_distro(name, mode).is_ok_and(|d| d == resolved)); + let resolved = lookup_distro(explicit, distros, mode)?; + let in_changelog = changelog_distros.iter().any(|name| { + name == explicit || lookup_distro(name, distros, mode).is_ok_and(|d| d == resolved) + }); if !in_changelog { println!( "debmagic: building for '{explicit}', changelog targets {}", @@ -326,6 +332,18 @@ mod tests { .join("test_package_multi_distro") } + fn table() -> DistroTable { + let debian = "version,codename,series,created,release\n\ + 13,Trixie,trixie,2023-06-10,2025-08-09\n\ + 14,Forky,forky,2025-08-09\n\ + 15,Duke,duke,2027-08-01\n\ + ,Sid,sid,1993-08-16\n"; + let ubuntu = "version,codename,series,created,release\n\ + 27.04,Tenacious T-Rex,tenacious,2026-10-16,2027-04-22\n"; + let today = chrono::NaiveDate::from_ymd_opt(2026, 10, 23).unwrap(); + DistroTable::from_distro_info(debian, ubuntu, today).unwrap() + } + fn docker_mode(map: &HashMap) -> DistroResolveMode<'_> { DistroResolveMode::Container { base_images: map, @@ -347,7 +365,8 @@ mod tests { #[test] fn resolve_package_target_stable_aliases_to_trixie() -> anyhow::Result<()> { let empty = HashMap::new(); - let target = resolve_package_target(&test_package_dir(), None, docker_mode(&empty))?; + let target = + resolve_package_target(&test_package_dir(), None, &table(), docker_mode(&empty))?; assert_eq!(target.distro.codename, "trixie"); assert_eq!(target.distro.distro, Distro::Debian); Ok(()) @@ -356,8 +375,12 @@ mod tests { #[test] fn select_distro_version_alias_matches_canonical_explicit() -> anyhow::Result<()> { let empty = HashMap::new(); - let distro = - select_distro_version(&["stable".to_string()], Some("trixie"), docker_mode(&empty))?; + let distro = select_distro_version( + &["stable".to_string()], + Some("trixie"), + &table(), + docker_mode(&empty), + )?; assert_eq!(distro.codename, "trixie"); Ok(()) } @@ -365,8 +388,12 @@ mod tests { #[test] fn select_distro_version_sid_matches_unstable_explicit() -> anyhow::Result<()> { let empty = HashMap::new(); - let distro = - select_distro_version(&["sid".to_string()], Some("unstable"), docker_mode(&empty))?; + let distro = select_distro_version( + &["sid".to_string()], + Some("unstable"), + &table(), + docker_mode(&empty), + )?; assert_eq!(distro.codename, "unstable"); Ok(()) } @@ -374,8 +401,12 @@ mod tests { #[test] fn resolve_package_target_multiple_distros_requires_explicit() { let empty = HashMap::new(); - let result = - resolve_package_target(&test_package_multi_distro_dir(), None, docker_mode(&empty)); + let result = resolve_package_target( + &test_package_multi_distro_dir(), + None, + &table(), + docker_mode(&empty), + ); assert!(result.is_err()); assert!( result @@ -391,6 +422,7 @@ mod tests { let target = resolve_package_target( &test_package_multi_distro_dir(), Some("unstable"), + &table(), docker_mode(&empty), )?; assert_eq!(target.package.name(), "test-package"); @@ -402,7 +434,8 @@ mod tests { #[test] fn select_distro_version_single_no_explicit() -> anyhow::Result<()> { let empty = HashMap::new(); - let distro = select_distro_version(&["forky".to_string()], None, docker_mode(&empty))?; + let distro = + select_distro_version(&["forky".to_string()], None, &table(), docker_mode(&empty))?; assert_eq!(distro.codename, "forky"); assert_eq!(distro.distro, Distro::Debian); Ok(()) @@ -411,8 +444,12 @@ mod tests { #[test] fn select_distro_version_single_matching_explicit() -> anyhow::Result<()> { let empty = HashMap::new(); - let distro = - select_distro_version(&["forky".to_string()], Some("forky"), docker_mode(&empty))?; + let distro = select_distro_version( + &["forky".to_string()], + Some("forky"), + &table(), + docker_mode(&empty), + )?; assert_eq!(distro.codename, "forky"); Ok(()) } @@ -420,8 +457,12 @@ mod tests { #[test] fn select_distro_version_single_conflicting_explicit_overrides() -> anyhow::Result<()> { let empty = HashMap::new(); - let distro = - select_distro_version(&["forky".to_string()], Some("duke"), docker_mode(&empty))?; + let distro = select_distro_version( + &["forky".to_string()], + Some("duke"), + &table(), + docker_mode(&empty), + )?; assert_eq!(distro.codename, "duke"); Ok(()) } @@ -432,6 +473,7 @@ mod tests { let result = select_distro_version( &["forky".to_string(), "duke".to_string()], None, + &table(), docker_mode(&empty), ); assert!(result.is_err()); @@ -449,6 +491,7 @@ mod tests { let distro = select_distro_version( &["forky".to_string(), "duke".to_string()], Some("duke"), + &table(), docker_mode(&empty), )?; assert_eq!(distro.codename, "duke"); @@ -461,6 +504,7 @@ mod tests { let distro = select_distro_version( &["forky".to_string(), "duke".to_string()], Some("trixie"), + &table(), docker_mode(&empty), )?; assert_eq!(distro.codename, "trixie"); @@ -470,7 +514,7 @@ mod tests { #[test] fn select_distro_version_empty_distros() { let empty = HashMap::new(); - let result = select_distro_version(&[], None, docker_mode(&empty)); + let result = select_distro_version(&[], None, &table(), docker_mode(&empty)); assert!(result.is_err()); assert!( result @@ -487,7 +531,12 @@ mod tests { "yocto:kirkstone".to_string(), "registry.example/yocto-kirkstone:latest".to_string(), ); - let distro = select_distro_version(&["kirkstone".to_string()], None, docker_mode(&map))?; + let distro = select_distro_version( + &["kirkstone".to_string()], + None, + &table(), + docker_mode(&map), + )?; assert_eq!(distro.distro, Distro::Custom("yocto".into())); assert_eq!(distro.codename, "kirkstone"); assert_eq!(distro.version, ""); @@ -497,9 +546,14 @@ mod tests { #[test] fn select_unknown_without_base_images_errors_with_hint() { let empty = HashMap::new(); - let err = select_distro_version(&["kirkstone".to_string()], None, docker_mode(&empty)) - .unwrap_err() - .to_string(); + let err = select_distro_version( + &["kirkstone".to_string()], + None, + &table(), + docker_mode(&empty), + ) + .unwrap_err() + .to_string(); assert!(err.contains("unknown distro codename 'kirkstone'")); assert!(err.contains("driver.docker.base_images")); assert!(err.contains("yocto:kirkstone")); @@ -509,8 +563,8 @@ mod tests { fn select_rejects_colliding_custom_base_images_key() { let mut map = HashMap::new(); map.insert("yocto:trixie".to_string(), "img".to_string()); - let err = - select_distro_version(&["trixie".to_string()], None, docker_mode(&map)).unwrap_err(); + let err = select_distro_version(&["trixie".to_string()], None, &table(), docker_mode(&map)) + .unwrap_err(); assert!(err.to_string().contains("built-in")); } @@ -518,7 +572,8 @@ mod tests { fn select_allows_debian_base_image_override() -> anyhow::Result<()> { let mut map = HashMap::new(); map.insert("debian:trixie".to_string(), "my-trixie:latest".to_string()); - let distro = select_distro_version(&["trixie".to_string()], None, docker_mode(&map))?; + let distro = + select_distro_version(&["trixie".to_string()], None, &table(), docker_mode(&map))?; assert_eq!(distro.distro, Distro::Debian); assert_eq!(distro.codename, "trixie"); Ok(()) @@ -539,12 +594,21 @@ UBUNTU_CODENAME=ignored #[test] fn bare_mode_resolves_any_suite_without_host_check() -> anyhow::Result<()> { - let distro = select_distro_version(&["trixie".to_string()], None, DistroResolveMode::Bare)?; + let distro = select_distro_version( + &["trixie".to_string()], + None, + &table(), + DistroResolveMode::Bare, + )?; assert_eq!(distro.distro, Distro::Debian); assert_eq!(distro.codename, "trixie"); - let distro = - select_distro_version(&["kirkstone".to_string()], None, DistroResolveMode::Bare)?; + let distro = select_distro_version( + &["kirkstone".to_string()], + None, + &table(), + DistroResolveMode::Bare, + )?; assert_eq!(distro.distro, Distro::Custom("kirkstone".into())); assert_eq!(distro.codename, "kirkstone"); Ok(()) diff --git a/packages/debmagic/src/test/run.rs b/packages/debmagic/src/test/run.rs index bddfc425..ff1ea1fe 100644 --- a/packages/debmagic/src/test/run.rs +++ b/packages/debmagic/src/test/run.rs @@ -18,7 +18,7 @@ use crate::package::load_package; use crate::subprocess::Capture; use anyhow::{Context, anyhow, bail}; use debian_control::lossless::changes::Changes; -use debmagic_common::distro::DistroVersion; +use debmagic_common::distro::{DistroTable, DistroVersion}; use debmagic_common::package::SourcePackage; /// autopkgtest(1) exit status values (Debian autopkgtest 6.x). @@ -67,6 +67,7 @@ fn map_autopkgtest_exit(exit_code: i32, strict: bool) -> TestOutcome { /// suites, plus custom suites declared in the chosen Driver's `base_images`. fn lookup_distro_override( name: &str, + distros: &DistroTable, driver: DriverType, driver_config: &DriverConfig, ) -> anyhow::Result { @@ -75,15 +76,16 @@ fn lookup_distro_override( &driver_config.docker.base_images, &driver_config.lxd.base_images, ); - crate::package::lookup_distro(name, mode) + crate::package::lookup_distro(name, distros, mode) } /// Whether any suite in a `.changes` `Distribution` field (space-separated, /// codenames or aliases like `unstable`) names `distro`. -fn distribution_matches(distribution: &str, distro: &DistroVersion) -> bool { +fn distribution_matches(distribution: &str, distro: &DistroVersion, distros: &DistroTable) -> bool { distribution.split_whitespace().any(|suite| { suite == distro.codename - || debmagic_common::distro::get_distro_version(suite) + || distros + .get(suite) .is_some_and(|resolved| resolved.codename == distro.codename) }) } @@ -95,6 +97,7 @@ fn warn_on_prior_build_mismatch( package: &SourcePackage, invocation: &RecordedInvocation, changes_path: &Path, + distros: &DistroTable, ) { let current_version = package.version().to_string(); if intent.changes.is_none() && invocation.package_version != current_version { @@ -109,7 +112,7 @@ fn warn_on_prior_build_mismatch( && intent.distro.is_none() && let Ok(changes) = Changes::from_file(changes_path) && let Some(distribution) = changes.distribution() - && !distribution_matches(&distribution, &invocation.distro) + && !distribution_matches(&distribution, &invocation.distro, distros) { eprintln!( "Warning: {} targets {distribution}, but the test defaults to {} from the \ @@ -204,6 +207,9 @@ fn autopkgtest_isolation_args(isolation: IsolationCapability) -> Vec<&'static st pub async fn run_test(intent: &TestIntent) -> anyhow::Result { let package = load_package(&intent.source_dir)?; + let distros = crate::distro_info::load() + .await + .context("loading the Debian and Ubuntu releases failed")?; let registry = Registry::open_default_or_ephemeral().await?; // `--changes` overrides only the artifact path. Driver and distro still @@ -235,7 +241,7 @@ pub async fn run_test(intent: &TestIntent) -> anyhow::Result { )? }; if let Some(invocation) = prior_invocation.as_ref() { - warn_on_prior_build_mismatch(intent, &package, invocation, &changes_path); + warn_on_prior_build_mismatch(intent, &package, invocation, &changes_path, &distros); } let exported_test_dir = exported_test_dir(&changes_path)?; ensure_exported_test_dir_replaceable(&exported_test_dir)?; @@ -261,7 +267,7 @@ pub async fn run_test(intent: &TestIntent) -> anyhow::Result { } let distro = if let Some(ref override_distro) = intent.distro { - lookup_distro_override(override_distro, driver, &intent.config.driver)? + lookup_distro_override(override_distro, &distros, driver, &intent.config.driver)? } else if let Some(ref invocation) = prior_invocation { // The Invocation recorded the full DistroVersion, so custom suites // from a Driver's base_images work without re-resolving. From cdd591586c6e05bd697a2a93cc4f1d1670246b9c Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Fri, 23 Oct 2026 15:37:21 +0200 Subject: [PATCH 38/48] feat(build): build against what a release migrates from with `--proposed` `--proposed` now uses the suite the release's updates come from and takes everything newer there, like Launchpad builds for -proposed: - Ubuntu: `-proposed`, pinned to the release's priority, since apt otherwise only upgrades installed packages from it - Debian testing: unstable, instead of its proposed-updates, which only carries freeze exceptions - Debian unstable: nothing, instead of an error A build for experimental always gets unstable and experimental sources; its install is retried preferring experimental when unstable alone can't satisfy the build dependencies. --- docs/usage/build.md | 15 +++- packages/debmagic-common/src/apt.rs | 82 +++++++++++++++++-- packages/debmagic/src/apt.rs | 68 +++++++++++---- packages/debmagic/src/driver/driver_docker.rs | 1 + 4 files changed, 141 insertions(+), 25 deletions(-) diff --git a/docs/usage/build.md b/docs/usage/build.md index 1f789560..7cc32b9a 100644 --- a/docs/usage/build.md +++ b/docs/usage/build.md @@ -155,8 +155,19 @@ Non-Debian/Ubuntu suites (still apt/dpkg-based) are supported when declared for ## Proposed dependencies -If needed, build dependencies can be used from `-proposed`. -Pass `--proposed` to enable the proposed pocket in the build environment. +Pass `--proposed` to build against the suite the release's updates come from, like Launchpad builds for `-proposed` do: everything newer there is installed and upgraded. + +| Target | Proposed suite | +|---|---| +| Ubuntu release | `-proposed` | +| Debian testing (devel release) | `unstable` | +| Debian stable release | `-proposed-updates` | +| Debian `unstable`, `experimental` | none | + +## Experimental + +A build for Debian `experimental` installs its build dependencies from unstable. +Only when unstable can't satisfy them, the install is retried preferring the packages from experimental. ## Building debug symbol packages diff --git a/packages/debmagic-common/src/apt.rs b/packages/debmagic-common/src/apt.rs index a3374c9c..12b94fa2 100644 --- a/packages/debmagic-common/src/apt.rs +++ b/packages/debmagic-common/src/apt.rs @@ -5,7 +5,8 @@ use anyhow::bail; use crate::distro::{Distro, DistroVersion}; -/// Debian suites without `-updates`/`-security` pockets. +/// Debian suites without `-updates`/`-security` pockets: unstable, and +/// experimental, an add-on to it. const DEBIAN_UNSTABLE: &[&str] = &["unstable", "sid", "experimental"]; const DEBIAN_ARCHIVE: &str = "http://deb.debian.org/debian"; /// Debian mirrors don't carry the security pocket. @@ -15,7 +16,7 @@ const UBUNTU_PORTS_ARCHIVE: &str = "http://ports.ubuntu.com/ubuntu-ports"; /// A deb822 `.sources` file pointing apt at `mirror` (the distro's default /// archive if `None`) for the release, its `-updates` and `-security` -/// pockets, plus its proposed pocket if `proposed`. It +/// pockets, plus the suite its updates migrate from if `proposed`. It /// replaces the base image's own sources. `dpkg_arch` is the environment's /// architecture: Ubuntu serves most of them from a separate ports archive. pub fn sources_file( @@ -41,7 +42,11 @@ pub fn sources_file( let debian = distro.distro == Distro::Debian; let rolling = debian && DEBIAN_UNSTABLE.contains(&codename); let security = format!("{codename}-security"); - let mut suites = vec![codename.to_string()]; + let mut suites = Vec::new(); + if not_automatic_suite(distro).is_some() { + suites.push("unstable".to_string()); + } + suites.push(codename.to_string()); if !rolling { suites.push(format!("{codename}-updates")); if !debian { @@ -77,16 +82,45 @@ fn debian_components(version: &str) -> &'static [&'static str] { } } -/// The release's proposed pocket, `None` for unstable, which has none. +/// The suite updates of the release migrate from, `None` for unstable +/// itself: unstable migrates into Debian's devel release (testing). fn proposed_suite(distro: &DistroVersion) -> Option { let codename = &distro.codename; match distro.distro { Distro::Debian if DEBIAN_UNSTABLE.contains(&codename.as_str()) => None, + Distro::Debian if distro.is_devel => Some("unstable".to_string()), Distro::Debian => Some(format!("{codename}-proposed-updates")), _ => Some(format!("{codename}-proposed")), } } +/// The NotAutomatic suite a build for the release takes packages from only +/// when nothing else satisfies its build dependencies: experimental, which +/// only adds to unstable. +pub fn not_automatic_suite(distro: &DistroVersion) -> Option<&'static str> { + (distro.distro == Distro::Debian && distro.codename == "experimental").then_some("experimental") +} + +/// Whether a build for the release needs [`sources_file`] whatever the +/// mirror: the base images of experimental lack its suite. +pub fn needs_own_sources(distro: &DistroVersion) -> bool { + not_automatic_suite(distro).is_some() +} + +/// apt preferences for the sources of [`sources_file`]. Ubuntu marks +/// `-proposed` NotAutomatic, so apt would only upgrade installed packages +/// from it; pinned to the release's priority like on Launchpad's builders, +/// the build takes everything newer from it. Debian's proposed suites +/// aren't NotAutomatic. +pub fn preferences_file(distro: &DistroVersion, proposed: bool) -> Option { + (proposed && distro.distro == Distro::Ubuntu).then(|| { + format!( + "Package: *\nPin: release a={}-proposed\nPin-Priority: 500\n", + distro.codename + ) + }) +} + /// Whether the release's apt has `apt-get satisfy` (apt 1.9: Debian 11, /// Ubuntu 20.04). Unknown versions count as recent. pub fn has_satisfy(distro: &DistroVersion) -> bool { @@ -112,7 +146,8 @@ mod tests { "buster" => (Distro::Debian, "10"), "bullseye" => (Distro::Debian, "11"), "trixie" => (Distro::Debian, "13"), - "unstable" => (Distro::Debian, ""), + "forky" => (Distro::Debian, "14"), + "unstable" | "experimental" => (Distro::Debian, ""), "bionic" => (Distro::Ubuntu, "18.04"), "focal" => (Distro::Ubuntu, "20.04"), "noble" => (Distro::Ubuntu, "24.04"), @@ -137,7 +172,7 @@ mod tests { } #[test_case(false ; "release")] - #[test_case(true ; "no proposed pocket")] + #[test_case(true ; "proposed is unstable itself")] fn debian_unstable_is_one_suite(proposed: bool) { let sources = sources_file(&distro("unstable"), None, proposed, "arm64").unwrap(); assert_eq!( @@ -147,6 +182,41 @@ mod tests { ); } + #[test] + fn debian_devel_proposed_is_unstable() { + let mut forky = distro("forky"); + forky.is_devel = true; + let sources = sources_file(&forky, None, true, "amd64").unwrap(); + assert!( + sources.contains("Suites: forky forky-updates unstable\n"), + "{sources}" + ); + assert!(sources.contains("Suites: forky-security\n"), "{sources}"); + assert_eq!(preferences_file(&forky, true), None); + } + + #[test] + fn debian_experimental_adds_to_unstable() { + let experimental = distro("experimental"); + assert!(needs_own_sources(&experimental)); + assert!(!needs_own_sources(&distro("unstable"))); + assert_eq!(not_automatic_suite(&experimental), Some("experimental")); + let sources = sources_file(&experimental, None, true, "amd64").unwrap(); + assert!( + sources.contains("Suites: unstable experimental\n"), + "{sources}" + ); + } + + #[test] + fn ubuntu_proposed_is_pinned_like_the_release() { + assert_eq!( + preferences_file(&distro("noble"), true).unwrap(), + "Package: *\nPin: release a=noble-proposed\nPin-Priority: 500\n" + ); + assert_eq!(preferences_file(&distro("noble"), false), None); + } + #[test] fn old_debian_has_no_firmware_component() { let sources = sources_file(&distro("bullseye"), None, false, "amd64").unwrap(); diff --git a/packages/debmagic/src/apt.rs b/packages/debmagic/src/apt.rs index cfaaa658..c84b2798 100644 --- a/packages/debmagic/src/apt.rs +++ b/packages/debmagic/src/apt.rs @@ -37,6 +37,7 @@ const SETUP_FILES: &[(&str, &str, u32)] = &[ const POLICY_RC_D: (&str, &str, u32) = ("usr/sbin/policy-rc.d", "#!/bin/sh\nexit 101\n", 0o755); const SOURCES_FILE: &str = "etc/apt/sources.list.d/debmagic.sources"; +const PREFERENCES_FILE: &str = "etc/apt/preferences.d/debmagic.pref"; const MAN_DB_PRESEED: &str = "'man-db man-db/auto-update boolean false'"; @@ -61,7 +62,7 @@ pub struct Setup<'a> { impl Setup<'_> { fn replaces_sources(&self) -> bool { - self.mirror.is_some() || self.proposed + self.mirror.is_some() || self.proposed || apt::needs_own_sources(self.distro) } /// The setup files apart from the apt sources. @@ -97,12 +98,11 @@ impl Setup<'_> { fs::remove_dir_all(dir) .with_context(|| format!("removing the old {} failed", dir.display()))?; } + let preferences = apt::preferences_file(self.distro, self.proposed); let mut files: Vec<(&str, &str, u32)> = self.files().collect(); - files.extend( - sources - .as_deref() - .map(|content| (SOURCES_FILE, content, 0o644)), - ); + for (path, content) in [(SOURCES_FILE, &sources), (PREFERENCES_FILE, &preferences)] { + files.extend(content.as_deref().map(|content| (path, content, 0o644))); + } for (path, content, mode) in files { let path = dir.join(path); fs::create_dir_all(path.parent().expect("setup files are in a dir")) @@ -166,9 +166,18 @@ pub struct Install<'a> { pub profiles: Vec, } -pub fn install_commands(satisfy: bool, install: &Install) -> Vec> { - let options = ["-y", "-q", "--no-install-recommends"]; - let mut profiled = apt_get(options); +/// The commands for `install`, preferring the packages of `target_release` +/// if given. +pub fn install_commands( + satisfy: bool, + install: &Install, + target_release: Option<&str>, +) -> Vec> { + let mut options = vec!["-y", "-q", "--no-install-recommends"]; + if let Some(release) = target_release { + options.extend(["-t", release]); + } + let mut profiled = apt_get(options.iter().copied()); if !install.profiles.is_empty() { profiled.extend(["-P".to_string(), install.profiles.join(",")]); } @@ -196,17 +205,29 @@ pub fn install_commands(satisfy: bool, install: &Install) -> Vec> { commands } +/// Install `install`, for a release with a NotAutomatic suite like +/// experimental retried preferring that suite: apt doesn't take a version +/// only that suite has otherwise, while preferring it upfront would take +/// everything it has. pub fn install( driver: &dyn EnvironmentDriver, cwd: &Path, distro: &DistroVersion, install: &Install, ) -> io::Result<()> { - run( - driver, - cwd, - &install_commands(apt::has_satisfy(distro), install), - ) + let satisfy = apt::has_satisfy(distro); + let result = run(driver, cwd, &install_commands(satisfy, install, None)); + match apt::not_automatic_suite(distro) { + Some(suite) if result.is_err() => { + println!("debmagic: retrying preferring the packages from {suite}"); + run( + driver, + cwd, + &install_commands(satisfy, install, Some(suite)), + ) + } + _ => result, + } } /// `commands` as one `sh -c` command line. @@ -268,16 +289,29 @@ mod tests { #[test] fn satisfy_installs_everything_in_one_transaction() { assert_eq!( - shell_line(&install_commands(true, &build_install())), + shell_line(&install_commands(true, &build_install(), None)), "apt-get -y -q --no-install-recommends -P nocheck,nodoc satisfy dpkg-dev build-essential \ 'debhelper-compat (= 13), libfoo-dev ' 'Conflicts: libbar-dev'" ); } + #[test] + fn target_release_applies_to_every_command() { + assert_eq!( + shell_line(&install_commands( + false, + &build_install(), + Some("experimental") + )), + "apt-get -y -q --no-install-recommends -t experimental install dpkg-dev build-essential && \ + apt-get -y -q --no-install-recommends -t experimental -P nocheck,nodoc build-dep ." + ); + } + #[test] fn without_satisfy_build_dep_reads_the_source_tree() { assert_eq!( - shell_line(&install_commands(false, &build_install())), + shell_line(&install_commands(false, &build_install(), None)), "apt-get -y -q --no-install-recommends install dpkg-dev build-essential && \ apt-get -y -q --no-install-recommends -P nocheck,nodoc build-dep ." ); @@ -290,7 +324,7 @@ mod tests { ..Install::default() }; assert_eq!( - shell_line(&install_commands(false, &install)), + shell_line(&install_commands(false, &install, None)), "apt-get -y -q --no-install-recommends install dpkg-dev autopkgtest" ); } diff --git a/packages/debmagic/src/driver/driver_docker.rs b/packages/debmagic/src/driver/driver_docker.rs index 4471d92b..cd03a184 100644 --- a/packages/debmagic/src/driver/driver_docker.rs +++ b/packages/debmagic/src/driver/driver_docker.rs @@ -68,6 +68,7 @@ fn dockerfile(environment: &Environment, base_image: &str, setup: &apt::Setup) - apt_commands.extend(apt::install_commands( debmagic_common::apt::has_satisfy(&environment.distro), &apt::Install::default(), + None, )); DOCKERFILE_TEMPLATE .replace("{base_image}", base_image) From 44ccc2b829f0c01c7433daa4429c949d26973aa7 Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Fri, 23 Oct 2026 16:07:53 +0200 Subject: [PATCH 39/48] feat(build): take only some packages from proposed or experimental `--proposed=libfoo1,src:bar` takes only those packages from the proposed suite, everything else stays with the release. `--experimental[=]` adds experimental to unstable and testing builds; other releases reject it. Both work for `debmagic test` too, to run a test suite against them. In the config, `driver.proposed` and the new `driver.experimental` take a bool or a list of packages. --- docs/usage/build.md | 10 +- docs/usage/config.md | 3 +- docs/usage/test.md | 3 +- packages/debmagic-common/src/apt.rs | 334 +++++++++++++++--- packages/debmagic/src/apt.rs | 27 +- packages/debmagic/src/build/mod.rs | 27 +- packages/debmagic/src/build_intent.rs | 1 + packages/debmagic/src/cli.rs | 37 +- packages/debmagic/src/driver/config.rs | 13 +- packages/debmagic/src/driver/driver_docker.rs | 6 +- packages/debmagic/src/driver/driver_lxd.rs | 9 +- packages/debmagic/src/driver/mod.rs | 28 +- packages/debmagic/src/test/intent.rs | 1 + packages/debmagic/src/test/run.rs | 7 +- 14 files changed, 420 insertions(+), 86 deletions(-) diff --git a/docs/usage/build.md b/docs/usage/build.md index 7cc32b9a..7ede741f 100644 --- a/docs/usage/build.md +++ b/docs/usage/build.md @@ -25,7 +25,8 @@ debmagic build binary --driver lxd \ | `--persistent ` | How long the build environment outlives this build: `on-failure` (default) keeps it when the build command fails, `always` keeps it after every build, `no` tears it down. A bare `--persistent` means `always` | | `--incremental` | Do incremental builds by syncing changed sources only; forces `--persistent=always` | | `--distro ` | [Select the target distro/release](#selecting-a-distrorelease) (e.g. `trixie`, `resolute`) | -| `--proposed` | Use [build dependencies from `proposed`](#proposed-dependencies) pocket | +| `--proposed[=]` | Use [build dependencies from `proposed`](#proposed-dependencies) | +| `--experimental[=]` | Use build dependencies from [experimental](#experimental) | | `--sign` | [GPG-sign the resulting `.changes`/`.dsc`/`.buildinfo`](#signing) | | `--clean` | Run [`debian/rules clean` before building](#cleaning) | | `--debug-symbols` | [Build the automatic `-dbgsym` debug symbol packages](#building-debug-symbol-packages) | @@ -164,11 +165,18 @@ Pass `--proposed` to build against the suite the release's updates come from, li | Debian stable release | `-proposed-updates` | | Debian `unstable`, `experimental` | none | +Pass a comma-separated list to take only those packages from it, e.g. `--proposed=libfoo1,src:bar`. +The names are [apt preferences](https://manpages.debian.org/apt_preferences) patterns: binary packages, `src:` for all binaries of a source, or globs. +Everything else stays with the release. + ## Experimental A build for Debian `experimental` installs its build dependencies from unstable. Only when unstable can't satisfy them, the install is retried preferring the packages from experimental. +Builds for unstable or testing do the same with `--experimental`, or take only the listed packages from it with `--experimental=` like `--proposed`. +Other releases reject it. + ## Building debug symbol packages By default `debmagic build binary` passes `DEB_BUILD_OPTIONS=noautodbgsym` to `dpkg-buildpackage`, which suppresses debhelper's automatic `-dbgsym` package (the detached debug info package debhelper otherwise builds by default from compat 9 onward). diff --git a/docs/usage/config.md b/docs/usage/config.md index f364c72a..b133e2c3 100644 --- a/docs/usage/config.md +++ b/docs/usage/config.md @@ -25,7 +25,8 @@ All keys are optional. | `driver.default` | enum | — | `--driver` | Build driver (`docker`, `bare`, `lxd`, `incus`) | | `driver.persistent` | enum | `"on-failure"` | `--persistent` | How long the Environment outlives the claim: `"no"`, `"on-failure"`, or `"always"`. A bare `--persistent` means `always`. | | `driver.apt_mirror` | string | — | `--apt-mirror` | Mirror replacing the image's apt sources. Not used by the `bare` driver. | -| `driver.proposed` | bool | `false` | `--proposed` | Also enable the `-proposed` pocket. Not used by the `bare` driver. | +| `driver.proposed` | bool or list | `false` | `--proposed` | Packages from the [proposed suite](build.md#proposed-dependencies): `true` for all newer ones, or a list like `["libfoo1", "src:bar"]`. Not used by the `bare` driver. | +| `driver.experimental` | bool or list | `false` | `--experimental` | Packages from [experimental](build.md#experimental) for unstable and testing. Not used by the `bare` driver. | | `driver.apt_update_age` | string | `"1d"` | `--apt-update-age` | When a persistent environment re-runs `apt-get update && upgrade`: `"now"` (every build), `"never"` (only on creation), or a maximum age like `"1d"`, `"12h"`, `"30m"`. Not used by the `bare` driver. | | `driver.docker.base_images` | map | — | — | Base image per distro, keyed by `":"` (e.g. `"debian:trixie"`). Falls back to `docker.io/:`. For non-Debian/Ubuntu suites (e.g. `"yocto:kirkstone"`), the map entry is what makes the suite a known DistroVersion for Docker builds. | | `driver.lxd.project` | string | — | — | LXD/Incus project to use. | diff --git a/docs/usage/test.md b/docs/usage/test.md index c063b93c..500a161d 100644 --- a/docs/usage/test.md +++ b/docs/usage/test.md @@ -31,7 +31,8 @@ The driver *is* the testbed: `autopkgtest` runs with the `null` backend inside t | `--strict` | Treat skipped tests and "no tests declared" as failures (exit code 2) | | `--changes ` | Path to a `.changes` file whose directory supplies the built `.debs` (for pipeline use). Overrides the artifact path only; driver and distro still default to the prior binary-build Invocation | | `--distro ` | Override the target distro for the test environment (defaults to the prior binary-build Invocation's distro, not the changelog) | -| `--proposed` | Enable the `-proposed` pocket in the test environment | +| `--proposed[=]` | Test against packages from the [proposed suite](build.md#proposed-dependencies): all newer ones, or only the listed ones | +| `--experimental[=]` | Test against the listed packages from [experimental](build.md#experimental); without a list, only what nothing else provides | | `--apt-mirror ` | Mirror URL (same as [`debmagic build`](build.md)) | | `--apt-update-age ` | Configure when to run `apt-get update` (same as [`debmagic build`](build.md#apt-update-age)) | | `--source-dir ` | Directory containing the `debian/` package directory | diff --git a/packages/debmagic-common/src/apt.rs b/packages/debmagic-common/src/apt.rs index 12b94fa2..f50b701f 100644 --- a/packages/debmagic-common/src/apt.rs +++ b/packages/debmagic-common/src/apt.rs @@ -1,7 +1,10 @@ //! What apt needs to know about a target distro: the archive layout a build //! environment pulls from, and the features of the release's apt. +use std::str::FromStr; + use anyhow::bail; +use serde::{Deserialize, Serialize}; use crate::distro::{Distro, DistroVersion}; @@ -13,16 +16,136 @@ const DEBIAN_ARCHIVE: &str = "http://deb.debian.org/debian"; const DEBIAN_SECURITY_ARCHIVE: &str = "http://deb.debian.org/debian-security"; const UBUNTU_ARCHIVE: &str = "http://archive.ubuntu.com/ubuntu"; const UBUNTU_PORTS_ARCHIVE: &str = "http://ports.ubuntu.com/ubuntu-ports"; +const EXPERIMENTAL: &str = "experimental"; + +/// Which packages a build takes from a suite beyond its release. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(try_from = "SelectionValue", into = "SelectionValue")] +pub enum Selection { + #[default] + Off, + /// Whatever apt prefers with the suite enabled. + All, + /// Only these packages: apt preferences patterns like binary package + /// names, `src:` or globs. Everything else stays with the + /// release. + Only(Vec), +} + +/// [`Selection`] in config files: a bool or a list of packages. +#[derive(Serialize, Deserialize)] +#[serde(untagged)] +enum SelectionValue { + Enabled(bool), + Packages(Vec), +} + +impl Selection { + fn packages(packages: Vec) -> Result { + if let Some(invalid) = packages + .iter() + .find(|package| package.is_empty() || package.contains(char::is_whitespace)) + { + return Err(format!("invalid package pattern '{invalid}'")); + } + Ok(if packages.is_empty() { + Self::Off + } else { + Self::Only(packages) + }) + } + + pub fn is_on(&self) -> bool { + *self != Self::Off + } +} + +/// A bool, or a comma-separated list of packages. +impl FromStr for Selection { + type Err = String; + + fn from_str(value: &str) -> Result { + match value.to_ascii_lowercase().as_str() { + "y" | "yes" | "t" | "true" | "on" | "1" => Ok(Self::All), + "n" | "no" | "f" | "false" | "off" | "0" => Ok(Self::Off), + _ => Self::packages(value.split(',').map(|p| p.trim().to_string()).collect()), + } + } +} + +impl TryFrom for Selection { + type Error = String; + + fn try_from(value: SelectionValue) -> Result { + match value { + SelectionValue::Enabled(true) => Ok(Self::All), + SelectionValue::Enabled(false) => Ok(Self::Off), + SelectionValue::Packages(packages) => Self::packages(packages), + } + } +} + +impl From for SelectionValue { + fn from(selection: Selection) -> Self { + match selection { + Selection::Off => Self::Enabled(false), + Selection::All => Self::Enabled(true), + Selection::Only(packages) => Self::Packages(packages), + } + } +} + +/// The suites beyond its release a build environment takes packages from. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Suites { + /// From the suite the release's updates migrate from. + pub proposed: Selection, + /// From Debian experimental. + pub experimental: Selection, +} + +impl Suites { + /// The suites for a build for `distro`. Experimental only adds to + /// Debian unstable and testing, and is all of the experimental release. + pub fn for_distro( + distro: &DistroVersion, + proposed: Selection, + experimental: Selection, + ) -> anyhow::Result { + let experimental = match experimental { + Selection::Off if is_experimental(distro) => Selection::All, + Selection::Off => Selection::Off, + _ if distro.distro == Distro::Debian + && (DEBIAN_UNSTABLE.contains(&distro.codename.as_str()) || distro.is_devel) => + { + experimental + } + _ => bail!( + "experimental only adds to Debian unstable and testing, not to {} {}", + distro.distro, + distro.codename + ), + }; + Ok(Self { + proposed, + experimental, + }) + } +} + +fn is_experimental(distro: &DistroVersion) -> bool { + distro.distro == Distro::Debian && distro.codename == EXPERIMENTAL +} /// A deb822 `.sources` file pointing apt at `mirror` (the distro's default /// archive if `None`) for the release, its `-updates` and `-security` -/// pockets, plus the suite its updates migrate from if `proposed`. It -/// replaces the base image's own sources. `dpkg_arch` is the environment's -/// architecture: Ubuntu serves most of them from a separate ports archive. +/// pockets, plus the opted-in `suites`. It replaces the base image's own +/// sources. `dpkg_arch` is the environment's architecture: Ubuntu serves +/// most of them from a separate ports archive. pub fn sources_file( distro: &DistroVersion, mirror: Option<&str>, - proposed: bool, + extra: &Suites, dpkg_arch: &str, ) -> anyhow::Result { let codename = distro.codename.as_str(); @@ -42,20 +165,23 @@ pub fn sources_file( let debian = distro.distro == Distro::Debian; let rolling = debian && DEBIAN_UNSTABLE.contains(&codename); let security = format!("{codename}-security"); - let mut suites = Vec::new(); - if not_automatic_suite(distro).is_some() { - suites.push("unstable".to_string()); - } - suites.push(codename.to_string()); + let mut suites = vec![if is_experimental(distro) { + "unstable".to_string() + } else { + codename.to_string() + }]; if !rolling { suites.push(format!("{codename}-updates")); if !debian { suites.push(security.clone()); } } - if proposed { + if extra.proposed.is_on() { suites.extend(proposed_suite(distro)); } + if extra.experimental.is_on() { + suites.push(EXPERIMENTAL.to_string()); + } let mut groups = vec![(mirror.unwrap_or(archive), suites)]; if debian && !rolling { @@ -94,31 +220,47 @@ fn proposed_suite(distro: &DistroVersion) -> Option { } } -/// The NotAutomatic suite a build for the release takes packages from only -/// when nothing else satisfies its build dependencies: experimental, which -/// only adds to unstable. -pub fn not_automatic_suite(distro: &DistroVersion) -> Option<&'static str> { - (distro.distro == Distro::Debian && distro.codename == "experimental").then_some("experimental") +/// The NotAutomatic suite a build takes all packages from only when nothing +/// else satisfies its build dependencies: experimental. +pub fn retry_suite(extra: &Suites) -> Option<&'static str> { + (extra.experimental == Selection::All).then_some(EXPERIMENTAL) } -/// Whether a build for the release needs [`sources_file`] whatever the -/// mirror: the base images of experimental lack its suite. -pub fn needs_own_sources(distro: &DistroVersion) -> bool { - not_automatic_suite(distro).is_some() +/// apt preferences for the `extra` suites of [`sources_file`]. Ubuntu +/// marks `-proposed` NotAutomatic, so apt would only upgrade installed +/// packages from it; pinned to the release's priority like on Launchpad's +/// builders, the build takes everything newer from it. Debian's proposed +/// suites aren't NotAutomatic, so with only some packages from them the +/// rest is pinned down like Ubuntu's. +pub fn preferences_file(distro: &DistroVersion, extra: &Suites) -> Option { + let mut pins = Vec::new(); + if let Some(suite) = proposed_suite(distro) { + // Debian's proposed-updates have a Suite like `proposed-updates` + let release = if suite.ends_with("-proposed-updates") { + format!("n={suite}") + } else { + format!("a={suite}") + }; + match &extra.proposed { + Selection::Off => {} + Selection::All if distro.distro == Distro::Ubuntu => { + pins.push(pin("*", &release, 500)); + } + Selection::All => {} + Selection::Only(packages) => { + pins.push(pin(&packages.join(" "), &release, 990)); + pins.push(pin("*", &release, 100)); + } + } + } + if let Selection::Only(packages) = &extra.experimental { + pins.push(pin(&packages.join(" "), &format!("a={EXPERIMENTAL}"), 990)); + } + (!pins.is_empty()).then(|| pins.join("\n")) } -/// apt preferences for the sources of [`sources_file`]. Ubuntu marks -/// `-proposed` NotAutomatic, so apt would only upgrade installed packages -/// from it; pinned to the release's priority like on Launchpad's builders, -/// the build takes everything newer from it. Debian's proposed suites -/// aren't NotAutomatic. -pub fn preferences_file(distro: &DistroVersion, proposed: bool) -> Option { - (proposed && distro.distro == Distro::Ubuntu).then(|| { - format!( - "Package: *\nPin: release a={}-proposed\nPin-Priority: 500\n", - distro.codename - ) - }) +fn pin(packages: &str, release: &str, priority: u32) -> String { + format!("Package: {packages}\nPin: release {release}\nPin-Priority: {priority}\n") } /// Whether the release's apt has `apt-get satisfy` (apt 1.9: Debian 11, @@ -156,9 +298,66 @@ mod tests { DistroVersion::new(distro, name, version) } + fn suites(distro: &DistroVersion, proposed: &str, experimental: &str) -> Suites { + let proposed = proposed.parse().unwrap(); + Suites::for_distro(distro, proposed, experimental.parse().unwrap()).unwrap() + } + + fn proposed() -> Suites { + Suites { + proposed: Selection::All, + ..Suites::default() + } + } + + #[test_case("yes", Selection::All)] + #[test_case("off", Selection::Off)] + #[test_case("libfoo, src:bar", Selection::Only(vec!["libfoo".into(), "src:bar".into()]))] + fn selection_from_str(value: &str, expected: Selection) { + assert_eq!(value.parse::(), Ok(expected)); + } + + #[test_case("")] + #[test_case("libfoo,")] + #[test_case("lib foo")] + fn selection_rejects_invalid_packages(value: &str) { + assert!(value.parse::().is_err()); + } + + #[test] + fn selection_in_config_is_bool_or_packages() { + let parse = |json| serde_json::from_str::(json).unwrap(); + assert_eq!(parse("true"), Selection::All); + assert_eq!(parse("[]"), Selection::Off); + assert_eq!( + parse(r#"["libfoo"]"#), + Selection::Only(vec!["libfoo".into()]) + ); + assert!(serde_json::from_str::(r#"["a b"]"#).is_err()); + } + + #[test_case("noble")] + #[test_case("trixie")] + fn experimental_only_adds_to_debian_unstable_and_testing(name: &str) { + let error = Suites::for_distro(&distro(name), Selection::Off, Selection::All); + assert!(error.is_err()); + } + + #[test] + fn experimental_release_is_all_of_experimental() { + let experimental = suites(&distro("experimental"), "no", "no"); + assert_eq!(experimental.experimental, Selection::All); + assert_eq!(retry_suite(&experimental), Some("experimental")); + assert_eq!( + preferences_file(&distro("experimental"), &experimental), + None + ); + } + #[test] fn debian_stable_keeps_security_on_the_debian_archive() { - let sources = sources_file(&distro("trixie"), Some("http://m/debian"), true, "amd64"); + let trixie = distro("trixie"); + let sources = sources_file(&trixie, Some("http://m/debian"), &proposed(), "amd64"); assert_eq!( sources.unwrap(), "Types: deb deb-src\nURIs: http://m/debian\n\ @@ -173,8 +372,13 @@ mod tests { #[test_case(false ; "release")] #[test_case(true ; "proposed is unstable itself")] - fn debian_unstable_is_one_suite(proposed: bool) { - let sources = sources_file(&distro("unstable"), None, proposed, "arm64").unwrap(); + fn debian_unstable_is_one_suite(with_proposed: bool) { + let extra = if with_proposed { + proposed() + } else { + Suites::default() + }; + let sources = sources_file(&distro("unstable"), None, &extra, "arm64").unwrap(); assert_eq!( sources, "Types: deb deb-src\nURIs: http://deb.debian.org/debian\nSuites: unstable\n\ @@ -186,40 +390,80 @@ mod tests { fn debian_devel_proposed_is_unstable() { let mut forky = distro("forky"); forky.is_devel = true; - let sources = sources_file(&forky, None, true, "amd64").unwrap(); + let sources = sources_file(&forky, None, &proposed(), "amd64").unwrap(); assert!( sources.contains("Suites: forky forky-updates unstable\n"), "{sources}" ); assert!(sources.contains("Suites: forky-security\n"), "{sources}"); - assert_eq!(preferences_file(&forky, true), None); + assert_eq!(preferences_file(&forky, &proposed()), None); + + let extra = suites(&forky, "libfoo", "src:bar"); + let sources = sources_file(&forky, None, &extra, "amd64").unwrap(); + assert!( + sources.contains("Suites: forky forky-updates unstable experimental\n"), + "{sources}" + ); + assert_eq!( + preferences_file(&forky, &extra).unwrap(), + "Package: libfoo\nPin: release a=unstable\nPin-Priority: 990\n\n\ + Package: *\nPin: release a=unstable\nPin-Priority: 100\n\n\ + Package: src:bar\nPin: release a=experimental\nPin-Priority: 990\n" + ); + assert_eq!(retry_suite(&extra), None); + } + + #[test] + fn debian_proposed_updates_is_pinned_by_codename() { + let trixie = distro("trixie"); + assert_eq!( + preferences_file(&trixie, &suites(&trixie, "libfoo", "no")).unwrap(), + "Package: libfoo\nPin: release n=trixie-proposed-updates\nPin-Priority: 990\n\n\ + Package: *\nPin: release n=trixie-proposed-updates\nPin-Priority: 100\n" + ); + assert_eq!(preferences_file(&trixie, &proposed()), None); } #[test] fn debian_experimental_adds_to_unstable() { let experimental = distro("experimental"); - assert!(needs_own_sources(&experimental)); - assert!(!needs_own_sources(&distro("unstable"))); - assert_eq!(not_automatic_suite(&experimental), Some("experimental")); - let sources = sources_file(&experimental, None, true, "amd64").unwrap(); + let sources = sources_file( + &experimental, + None, + &suites(&experimental, "yes", "no"), + "amd64", + ); + assert!( + sources + .as_ref() + .unwrap() + .contains("Suites: unstable experimental\n"), + "{sources:?}" + ); + let unstable = distro("unstable"); + let extra = suites(&unstable, "no", "yes"); + let sources = sources_file(&unstable, None, &extra, "amd64").unwrap(); assert!( sources.contains("Suites: unstable experimental\n"), "{sources}" ); + assert_eq!(retry_suite(&extra), Some("experimental")); } #[test] fn ubuntu_proposed_is_pinned_like_the_release() { + let noble = distro("noble"); assert_eq!( - preferences_file(&distro("noble"), true).unwrap(), + preferences_file(&noble, &proposed()).unwrap(), "Package: *\nPin: release a=noble-proposed\nPin-Priority: 500\n" ); - assert_eq!(preferences_file(&distro("noble"), false), None); + assert_eq!(preferences_file(&noble, &Suites::default()), None); } #[test] fn old_debian_has_no_firmware_component() { - let sources = sources_file(&distro("bullseye"), None, false, "amd64").unwrap(); + let extra = Suites::default(); + let sources = sources_file(&distro("bullseye"), None, &extra, "amd64").unwrap(); assert!( sources.contains("Components: main contrib non-free\n"), "{sources}" @@ -229,7 +473,7 @@ mod tests { #[test_case("amd64", UBUNTU_ARCHIVE ; "archive")] #[test_case("arm64", UBUNTU_PORTS_ARCHIVE ; "ports")] fn ubuntu_proposed_uses_the_default_archive(arch: &str, archive: &str) { - let sources = sources_file(&distro("noble"), None, true, arch).unwrap(); + let sources = sources_file(&distro("noble"), None, &proposed(), arch).unwrap(); assert_eq!( sources, format!( @@ -243,7 +487,7 @@ mod tests { #[test] fn custom_distro_has_no_known_archive() { let custom = DistroVersion::custom(Distro::Custom("yocto".into()), "scarthgap"); - assert!(sources_file(&custom, Some("http://m"), false, "amd64").is_err()); + assert!(sources_file(&custom, Some("http://m"), &Suites::default(), "amd64").is_err()); } #[test_case("buster", false)] diff --git a/packages/debmagic/src/apt.rs b/packages/debmagic/src/apt.rs index c84b2798..8b659935 100644 --- a/packages/debmagic/src/apt.rs +++ b/packages/debmagic/src/apt.rs @@ -5,7 +5,11 @@ use std::{fs, io, os::unix::fs::PermissionsExt, path::Path}; use anyhow::Context; -use debmagic_common::{apt, debian::control::BuildRelations, distro::DistroVersion}; +use debmagic_common::{ + apt::{self, Suites}, + debian::control::BuildRelations, + distro::DistroVersion, +}; use crate::driver::EnvironmentDriver; @@ -51,18 +55,18 @@ const DEFAULT_SOURCES: &[&str] = &[ /// The apt and dpkg configuration of a fresh container: a file tree to copy /// over its root, written on the host so the container needs nothing to /// create it. Apt sources replace the base image's only for a `mirror` or -/// the `proposed` pocket. +/// extra suites. pub struct Setup<'a> { pub distro: &'a DistroVersion, pub mirror: Option<&'a str>, - pub proposed: bool, + pub suites: &'a Suites, /// Let installed packages start their services, which tests may expect. pub services: bool, } impl Setup<'_> { fn replaces_sources(&self) -> bool { - self.mirror.is_some() || self.proposed || apt::needs_own_sources(self.distro) + self.mirror.is_some() || *self.suites != Suites::default() } /// The setup files apart from the apt sources. @@ -81,7 +85,7 @@ impl Setup<'_> { .chain([ self.script(""), self.mirror.unwrap_or_default().to_string(), - self.proposed.to_string(), + format!("{:?}", self.suites), ]) .collect::>() .join("\0") @@ -91,14 +95,14 @@ impl Setup<'_> { pub fn write(&self, dir: &Path) -> anyhow::Result<()> { let sources = self .replaces_sources() - .then(|| apt::sources_file(self.distro, self.mirror, self.proposed, host_dpkg_arch())) + .then(|| apt::sources_file(self.distro, self.mirror, self.suites, host_dpkg_arch())) .transpose() .context("generating the apt sources failed")?; if dir.exists() { fs::remove_dir_all(dir) .with_context(|| format!("removing the old {} failed", dir.display()))?; } - let preferences = apt::preferences_file(self.distro, self.proposed); + let preferences = apt::preferences_file(self.distro, self.suites); let mut files: Vec<(&str, &str, u32)> = self.files().collect(); for (path, content) in [(SOURCES_FILE, &sources), (PREFERENCES_FILE, &preferences)] { files.extend(content.as_deref().map(|content| (path, content, 0o644))); @@ -205,19 +209,20 @@ pub fn install_commands( commands } -/// Install `install`, for a release with a NotAutomatic suite like -/// experimental retried preferring that suite: apt doesn't take a version +/// Install `install`, with all of a NotAutomatic suite like experimental +/// in `suites` retried preferring that suite: apt doesn't take a version /// only that suite has otherwise, while preferring it upfront would take /// everything it has. pub fn install( driver: &dyn EnvironmentDriver, cwd: &Path, distro: &DistroVersion, + suites: &Suites, install: &Install, ) -> io::Result<()> { let satisfy = apt::has_satisfy(distro); let result = run(driver, cwd, &install_commands(satisfy, install, None)); - match apt::not_automatic_suite(distro) { + match apt::retry_suite(suites) { Some(suite) if result.is_err() => { println!("debmagic: retrying preferring the packages from {suite}"); run( @@ -337,7 +342,7 @@ mod tests { let mut setup = Setup { distro: &trixie, mirror: None, - proposed: false, + suites: &Suites::default(), services: false, }; setup.write(&dir)?; diff --git a/packages/debmagic/src/build/mod.rs b/packages/debmagic/src/build/mod.rs index 49e05cdf..707a4726 100644 --- a/packages/debmagic/src/build/mod.rs +++ b/packages/debmagic/src/build/mod.rs @@ -7,7 +7,9 @@ use std::{ use crate::apt; use crate::build::source::{source_manifest_path, stage_dir, stage_source_tree}; use crate::build_intent::{BuildIntent, BuildIntentInput, BuildKind, resolve_build_intent}; -use crate::driver::{Driver, DriverType, Environment, EnvironmentDriver, EnvironmentPurpose}; +use crate::driver::{ + Driver, DriverType, Environment, EnvironmentDriver, EnvironmentPurpose, apt_suites, +}; use crate::environment::{ ClaimedEnvironment, CommandCompletion, HostRootPolicy, InvocationKind, Registry, claim_command, }; @@ -242,6 +244,7 @@ pub async fn build_binary_package( packages.extend(gain_root_package(target)?); install_packages( claimed, + intent, target, BuildKind::Binary, packages, @@ -305,6 +308,7 @@ fn gain_root_package(target: &PackageTarget) -> anyhow::Result, @@ -328,11 +332,17 @@ fn install_packages( profiles: profiles.to_vec(), }; let environment = claimed.environment(); + let suites = apt_suites( + &environment.distro, + &intent.config.driver, + &intent.driver_overrides, + )?; let staged_source_dir = stage_dir(environment, kind); apt::install( claimed.driver(), &staged_source_dir, &environment.distro, + &suites, &install, ) .context("installing the build dependencies failed") @@ -471,7 +481,15 @@ pub async fn build_source_package( // dpkg-buildpackage looks for the gain-root command even for -S let packages = gain_root_package(target)?.into_iter().collect(); let scope = intent.config.clean.then_some(BuildScope::Clean); - install_packages(claimed, target, BuildKind::Source, packages, scope, &[])?; + install_packages( + claimed, + intent, + target, + BuildKind::Source, + packages, + scope, + &[], + )?; let mut args: Vec = vec![ "dpkg-buildpackage".into(), "-S".into(), @@ -549,6 +567,11 @@ pub async fn build(command: BuildCommand) -> anyhow::Result<()> { ), ) .context("failed to determine package target")?; + apt_suites( + &target.distro, + &intent.config.driver, + &intent.driver_overrides, + )?; let package = format!("{} {}", target.package.name(), target.package.version()); let notify = intent.config.build_notify(); diff --git a/packages/debmagic/src/build_intent.rs b/packages/debmagic/src/build_intent.rs index 209728ab..31d63046 100644 --- a/packages/debmagic/src/build_intent.rs +++ b/packages/debmagic/src/build_intent.rs @@ -185,6 +185,7 @@ mod tests { driver_overrides: DriverOverrides { apt_mirror: None, proposed: None, + experimental: None, apt_update_age: None, docker: DriverDockerConfigOverrides { base_image: None }, bare: DriverBareConfigOverrides {}, diff --git a/packages/debmagic/src/cli.rs b/packages/debmagic/src/cli.rs index 3a49680b..4b5b6355 100644 --- a/packages/debmagic/src/cli.rs +++ b/packages/debmagic/src/cli.rs @@ -13,6 +13,7 @@ use crate::upload::orig::IncludeOrig; use crate::upstream::fetch::FetchFrom; use clap::builder::BoolishValueParser; use clap::{Args, Parser, Subcommand}; +use debmagic_common::apt::Selection; /// When to use colored output. Mirrors common CLI conventions; `auto` is the /// default and respects the `NO_COLOR` environment variable. @@ -330,10 +331,20 @@ pub struct EnvironmentArgs { num_args = 0..=1, default_missing_value = "true", action = clap::ArgAction::Set, - value_parser = BoolishValueParser::new(), - help = "Also enable the '-proposed' pocket in the environment. Ignored by the bare driver." + value_name = "PACKAGES", + help = "Take packages from the suite the release's updates migrate from (e.g. '-proposed'): all newer ones, or only the comma-separated PACKAGES (binary names, 'src:' or globs). 'no' disables it. Ignored by the bare driver." )] - pub proposed: Option, + pub proposed: Option, + + #[arg( + long, + num_args = 0..=1, + default_missing_value = "true", + action = clap::ArgAction::Set, + value_name = "PACKAGES", + help = "Take packages from Debian experimental for an unstable or testing environment: those nothing else provides, or only the comma-separated PACKAGES. 'no' disables it. Ignored by the bare driver." + )] + pub experimental: Option, #[arg( long = "apt-update-age", @@ -736,7 +747,8 @@ impl EnvironmentArgs { pub fn driver_overrides(&self) -> DriverOverrides { DriverOverrides { apt_mirror: self.apt_mirror.clone(), - proposed: self.proposed, + proposed: self.proposed.clone(), + experimental: self.experimental.clone(), apt_update_age: self.apt_update_age, docker: DriverDockerConfigOverrides { base_image: self.docker.base_image.clone(), @@ -775,15 +787,20 @@ mod tests { } #[test_case(&[], None; "absent")] - #[test_case(&["--proposed"], Some(true); "bare flag")] - #[test_case(&["--proposed=yes"], Some(true); "yes")] - #[test_case(&["--proposed=no"], Some(false); "no")] - #[test_case(&["--proposed=off"], Some(false); "off")] - #[test_case(&["--proposed=true"], Some(true); "literal true")] - fn boolish_flag(args: &[&str], expected: Option) { + #[test_case(&["--proposed"], Some(Selection::All); "bare flag")] + #[test_case(&["--proposed=no"], Some(Selection::Off); "no")] + #[test_case(&["--proposed=libfoo,src:bar"], Some(Selection::Only(vec!["libfoo".into(), "src:bar".into()])); "packages")] + fn proposed_flag(args: &[&str], expected: Option) { assert_eq!(parse_test(args).environment.proposed, expected); } + #[test] + fn experimental_flag_is_a_selection() { + let args = parse_test(&["--experimental"]).environment; + assert_eq!(args.experimental, Some(Selection::All)); + assert!(Cli::try_parse_from(["debmagic", "test", "--experimental=a b"]).is_err()); + } + #[test] fn persistent_rejects_a_boolean() { let error = Cli::try_parse_from(["debmagic", "test", "--persistent=true"]).unwrap_err(); diff --git a/packages/debmagic/src/driver/config.rs b/packages/debmagic/src/driver/config.rs index b08a4f8b..fca66a7b 100644 --- a/packages/debmagic/src/driver/config.rs +++ b/packages/debmagic/src/driver/config.rs @@ -1,4 +1,5 @@ use clap::ValueEnum; +use debmagic_common::apt::Selection; use serde::{Deserialize, Serialize}; use crate::driver::DriverType; @@ -58,9 +59,12 @@ pub struct DriverConfig { pub persistent: Persistence, /// Not used by the bare driver, which builds on the host's own sources. pub apt_mirror: Option, - /// Also enable the `-proposed` pocket. Not used by the bare - /// driver, which builds on the host's own sources. - pub proposed: bool, + /// Packages from the suite the release's updates migrate from: `true` + /// for all, or a list. Not used by the bare driver, which builds on the + /// host's own sources, like `experimental`. + pub proposed: Selection, + /// Packages from Debian experimental, for unstable and testing builds. + pub experimental: Selection, /// When to run apt-get update/upgrade again. pub apt_update_age: RefreshPolicy, pub docker: DriverDockerConfig, @@ -71,7 +75,8 @@ pub struct DriverConfig { #[derive(Deserialize, Debug, Clone, Default)] pub struct DriverOverrides { pub apt_mirror: Option, - pub proposed: Option, + pub proposed: Option, + pub experimental: Option, pub apt_update_age: Option, pub docker: DriverDockerConfigOverrides, pub bare: DriverBareConfigOverrides, diff --git a/packages/debmagic/src/driver/driver_docker.rs b/packages/debmagic/src/driver/driver_docker.rs index cd03a184..83faf8a5 100644 --- a/packages/debmagic/src/driver/driver_docker.rs +++ b/packages/debmagic/src/driver/driver_docker.rs @@ -7,7 +7,7 @@ use std::{ }; use anyhow::{Context, anyhow}; -use debmagic_common::distro::DistroVersion; +use debmagic_common::{apt::Suites, distro::DistroVersion}; use serde::{Deserialize, Serialize}; use crate::apt; @@ -246,7 +246,7 @@ impl DriverDocker { driver_config: &DriverConfig, overrides: &DriverDockerConfigOverrides, apt_mirror: Option<&str>, - proposed: bool, + suites: &Suites, ) -> anyhow::Result { let base_image = overrides.base_image.clone().unwrap_or_else(|| { driver_config @@ -259,7 +259,7 @@ impl DriverDocker { let setup = apt::Setup { distro: &environment.distro, mirror: apt_mirror, - proposed, + suites, services: environment.purpose == EnvironmentPurpose::Test, }; let dockerfile = dockerfile(environment, &base_image, &setup); diff --git a/packages/debmagic/src/driver/driver_lxd.rs b/packages/debmagic/src/driver/driver_lxd.rs index 3d560c75..a89e7cef 100644 --- a/packages/debmagic/src/driver/driver_lxd.rs +++ b/packages/debmagic/src/driver/driver_lxd.rs @@ -4,7 +4,10 @@ use std::{ process::{Command, Stdio}, }; -use debmagic_common::distro::{Distro, DistroVersion}; +use debmagic_common::{ + apt::Suites, + distro::{Distro, DistroVersion}, +}; use serde::{Deserialize, Serialize}; use crate::apt; @@ -289,7 +292,7 @@ impl DriverLxd { driver_config: &DriverConfig, overrides: &DriverLxdConfigOverrides, apt_mirror: Option<&str>, - proposed: bool, + suites: &Suites, ) -> anyhow::Result { let container_name = Self::container_name_for(environment); let project = Self::resolve_project(driver_config, overrides); @@ -304,7 +307,7 @@ impl DriverLxd { let setup = apt::Setup { distro: &environment.distro, mirror: apt_mirror, - proposed, + suites, services: environment.purpose == EnvironmentPurpose::Test, }; let setup_fingerprint = setup.fingerprint(); diff --git a/packages/debmagic/src/driver/mod.rs b/packages/debmagic/src/driver/mod.rs index 87f137ab..0be436d6 100644 --- a/packages/debmagic/src/driver/mod.rs +++ b/packages/debmagic/src/driver/mod.rs @@ -1,6 +1,9 @@ use anyhow::Context; use clap::ValueEnum; -use debmagic_common::distro::DistroVersion; +use debmagic_common::{ + apt::{Selection, Suites}, + distro::DistroVersion, +}; use serde::{Deserialize, Serialize}; use std::{ collections::HashMap, @@ -492,7 +495,7 @@ pub fn create_driver( .apt_mirror .as_deref() .or(driver_config.apt_mirror.as_deref()); - let proposed = overrides.proposed.unwrap_or(driver_config.proposed); + let suites = apt_suites(&environment.distro, driver_config, overrides)?; let mut driver_config = driver_config.clone(); if let Some(apt_update_age) = overrides.apt_update_age { driver_config.apt_update_age = apt_update_age; @@ -504,7 +507,7 @@ pub fn create_driver( &driver_config, &overrides.docker, apt_mirror, - proposed, + &suites, )?)), DriverType::Bare => Ok(Driver::Bare(DriverBare::create( environment, @@ -522,12 +525,29 @@ pub fn create_driver( &driver_config, &overrides.lxd, apt_mirror, - proposed, + &suites, )?)) } } } +/// The suites beyond its release an Environment for `distro` takes +/// packages from. +pub fn apt_suites( + distro: &DistroVersion, + driver_config: &DriverConfig, + overrides: &DriverOverrides, +) -> anyhow::Result { + let pick = |chosen: &Option, configured: &Selection| { + chosen.clone().unwrap_or_else(|| configured.clone()) + }; + Suites::for_distro( + distro, + pick(&overrides.proposed, &driver_config.proposed), + pick(&overrides.experimental, &driver_config.experimental), + ) +} + /// The Driver metadata [`create_driver`] will produce, computed without /// touching the Driver. Recorded before creation so a run that dies mid-setup /// still leaves enough in the registry to find and destroy the resource. diff --git a/packages/debmagic/src/test/intent.rs b/packages/debmagic/src/test/intent.rs index 8e6e7f52..46eb39ba 100644 --- a/packages/debmagic/src/test/intent.rs +++ b/packages/debmagic/src/test/intent.rs @@ -98,6 +98,7 @@ mod tests { driver_overrides: DriverOverrides { apt_mirror: None, proposed: None, + experimental: None, apt_update_age: None, docker: DriverDockerConfigOverrides { base_image: None }, bare: DriverBareConfigOverrides {}, diff --git a/packages/debmagic/src/test/run.rs b/packages/debmagic/src/test/run.rs index ff1ea1fe..daf2b8a3 100644 --- a/packages/debmagic/src/test/run.rs +++ b/packages/debmagic/src/test/run.rs @@ -277,6 +277,7 @@ pub async fn run_test(intent: &TestIntent) -> anyhow::Result { "no prior build metadata found; pass --distro when using --changes without a prior Invocation" ); }; + crate::driver::apt_suites(&distro, &intent.config.driver, &intent.driver_overrides)?; let environment = Environment::new( driver, @@ -318,10 +319,14 @@ pub async fn run_test(intent: &TestIntent) -> anyhow::Result { packages: vec!["autopkgtest", "autodep8", "fakeroot"], ..Default::default() }; + let distro = &claimed.environment().distro; + let suites = + crate::driver::apt_suites(distro, &intent.config.driver, &intent.driver_overrides)?; crate::apt::install( claimed.driver(), &staged_source_dir, - &claimed.environment().distro, + distro, + &suites, &install, ) .context("installing autopkgtest failed")?; From 470c94c7d6c3a13a21084a0461d63270d471e4fd Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Fri, 23 Oct 2026 16:28:30 +0200 Subject: [PATCH 40/48] feat(snap): ship distro-info-data and allow network and home access The snap reads the release table from the distro-info-data it ships, or from its base snap, and can fetch a newer one. --- packages/debmagic/src/distro_info.rs | 76 ++++++++++++++++++++++++---- snap/snapcraft.yaml | 2 + 2 files changed, 69 insertions(+), 9 deletions(-) diff --git a/packages/debmagic/src/distro_info.rs b/packages/debmagic/src/distro_info.rs index 263b460b..1192c4d4 100644 --- a/packages/debmagic/src/distro_info.rs +++ b/packages/debmagic/src/distro_info.rs @@ -1,4 +1,4 @@ -//! The Debian and Ubuntu release table from distro-info-data: the host's +//! The Debian and Ubuntu release table from distro-info-data: the system's //! copy if it's current, else one fetched from upstream into the cache. use std::path::{Path, PathBuf}; @@ -7,7 +7,7 @@ use std::time::{Duration, SystemTime}; use anyhow::Context; use debmagic_common::distro::DistroTable; -const HOST_DIR: &str = "/usr/share/distro-info"; +const DATA_DIR: &str = "usr/share/distro-info"; const UPSTREAM: &str = "https://salsa.debian.org/debian/distro-info-data/-/raw/main"; const DEBIAN_CSV: &str = "debian.csv"; const UBUNTU_CSV: &str = "ubuntu.csv"; @@ -19,12 +19,16 @@ const CACHE_MAX_AGE: Duration = Duration::from_secs(30 * 24 * 60 * 60); const OUTDATED_RETRY_AGE: Duration = Duration::from_secs(24 * 60 * 60); /// The release table as of today, from the first current copy of -/// distro-info-data: the host's, the cached one, a freshly fetched one. +/// distro-info-data: the system's, the cached one, a freshly fetched one. /// When fetching fails, an outdated copy is used with a warning. pub async fn load() -> anyhow::Result { - let mut host = read(Path::new(HOST_DIR)).transpose()?; - if let Some(table) = host.take_if(|table| table.outdated().is_empty()) { - return Ok(table); + let system_dirs = system_dirs(std::env::var_os("SNAP").map(PathBuf::from).as_deref()); + let mut host = None; + for dir in &system_dirs { + match read(dir).transpose()? { + Some(table) if table.outdated().is_empty() => return Ok(table), + table => host = host.or(table), + } } let cache_dir = dirs::cache_dir() @@ -54,10 +58,42 @@ pub async fn load() -> anyhow::Result { ); Ok(stale) } - (Err(error), None) => Err(error.context(format!( - "no distro-info-data in {HOST_DIR} or the cache; install distro-info-data" - ))), + (Err(error), None) => { + let searched: Vec = system_dirs + .iter() + .chain([&cache_dir]) + .map(|dir| dir.display().to_string()) + .collect(); + Err(error.context(format!( + "no distro-info-data in {}; install distro-info-data", + searched.join(", ") + ))) + } + } +} + +/// Where distro-info-data may be installed: the system, and in a snap +/// (whose `/usr` is its base's under strict confinement) the snap itself and +/// its base snap. +fn system_dirs(snap: Option<&Path>) -> Vec { + let mut dirs = vec![Path::new("/").join(DATA_DIR)]; + if let Some(snap) = snap { + dirs.push(snap.join(DATA_DIR)); + let meta = std::fs::read_to_string(snap.join("meta/snap.yaml")).unwrap_or_default(); + dirs.extend( + snap_base(&meta) + .map(|base| Path::new("/snap").join(base).join("current").join(DATA_DIR)), + ); } + dirs +} + +/// The `base` of a snap from its `meta/snap.yaml`. +fn snap_base(meta: &str) -> Option<&str> { + meta.lines() + .find_map(|line| line.strip_prefix("base:")) + .map(|base| base.trim().trim_matches(['"', '\''])) + .filter(|base| !base.is_empty()) } fn today() -> chrono::NaiveDate { @@ -115,3 +151,25 @@ fn warn_if_outdated(table: &DistroTable, source: &str) { ); } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn snap_also_searches_itself_and_its_base() { + let dir = std::env::temp_dir().join(format!("debmagic-snap-{}", std::process::id())); + std::fs::create_dir_all(dir.join("meta")).unwrap(); + std::fs::write(dir.join("meta/snap.yaml"), "name: debmagic\nbase: core26\n").unwrap(); + assert_eq!( + system_dirs(Some(&dir)), + [ + PathBuf::from("/usr/share/distro-info"), + dir.join("usr/share/distro-info"), + PathBuf::from("/snap/core26/current/usr/share/distro-info"), + ] + ); + std::fs::remove_dir_all(&dir).unwrap(); + assert_eq!(system_dirs(None), [PathBuf::from("/usr/share/distro-info")]); + } +} diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index ec68826b..428d3259 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -20,6 +20,7 @@ confinement: strict apps: debmagic: command: bin/debmagic + plugs: [home, network, network-bind] parts: debmagic: @@ -28,6 +29,7 @@ parts: rust-path: [packages/debmagic] rust-channel: "none" build-packages: [git, rustc, cargo] + stage-packages: [distro-info-data] override-pull: | craftctl default # store has 32 char limit (-> use +d as dirty and format manually). From be9c743464aa16279998b945129c4893b7c0d985 Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Fri, 23 Oct 2026 16:34:26 +0200 Subject: [PATCH 41/48] feat(test): test the archive's build of a package named in --proposed `debmagic test --proposed=libfoo1` (or `src:foo`, or `--experimental=...`) runs the source tree's tests against that suite's build of the package instead of a local one, so no prior `debmagic build` is needed. All its binaries come from that suite, so their versions match. --- docs/usage/test.md | 15 +- packages/debmagic-common/src/apt.rs | 54 ++++++ .../debmagic-common/src/debian/control.rs | 15 +- packages/debmagic/src/package.rs | 2 +- packages/debmagic/src/test/run.rs | 168 ++++++++++++++---- 5 files changed, 220 insertions(+), 34 deletions(-) diff --git a/docs/usage/test.md b/docs/usage/test.md index 500a161d..e7258b99 100644 --- a/docs/usage/test.md +++ b/docs/usage/test.md @@ -22,6 +22,17 @@ the test tree is reset and the `.debs` are reinstalled each time. The driver *is* the testbed: `autopkgtest` runs with the `null` backend inside the container (or on the host for the bare driver). No `autopkgtest-virt-*` backends are used. +## Testing the archive's build + +Naming the package (`src:`) or one of its binaries in `--proposed` or `--experimental` runs the source tree's tests against that suite's build instead of a local one, so no prior build is needed: + +```shell +debmagic test --driver lxd --proposed=libfoo1 +``` + +All binaries of the source come from that suite, so their versions match. +The distro comes from `--distro` or the changelog, and the test output goes to `/_.test`. + ## Available options | Option | Description | @@ -31,8 +42,8 @@ The driver *is* the testbed: `autopkgtest` runs with the `null` backend inside t | `--strict` | Treat skipped tests and "no tests declared" as failures (exit code 2) | | `--changes ` | Path to a `.changes` file whose directory supplies the built `.debs` (for pipeline use). Overrides the artifact path only; driver and distro still default to the prior binary-build Invocation | | `--distro ` | Override the target distro for the test environment (defaults to the prior binary-build Invocation's distro, not the changelog) | -| `--proposed[=]` | Test against packages from the [proposed suite](build.md#proposed-dependencies): all newer ones, or only the listed ones | -| `--experimental[=]` | Test against the listed packages from [experimental](build.md#experimental); without a list, only what nothing else provides | +| `--proposed[=]` | Test against packages from the [proposed suite](build.md#proposed-dependencies): all newer ones, or only the listed ones. Naming this package [tests the archive's build](#testing-the-archives-build) | +| `--experimental[=]` | Test against the listed packages from [experimental](build.md#experimental); without a list, only what nothing else provides. Naming this package [tests the archive's build](#testing-the-archives-build) | | `--apt-mirror ` | Mirror URL (same as [`debmagic build`](build.md)) | | `--apt-update-age ` | Configure when to run `apt-get update` (same as [`debmagic build`](build.md#apt-update-age)) | | `--source-dir ` | Directory containing the `debian/` package directory | diff --git a/packages/debmagic-common/src/apt.rs b/packages/debmagic-common/src/apt.rs index f50b701f..6eb9e050 100644 --- a/packages/debmagic-common/src/apt.rs +++ b/packages/debmagic-common/src/apt.rs @@ -58,6 +58,37 @@ impl Selection { pub fn is_on(&self) -> bool { *self != Self::Off } + + /// Whether it names the source package `source` or one of its + /// `binaries`, globs aside. + pub fn names_package(&self, source: &str, binaries: &[String]) -> bool { + let Self::Only(packages) = self else { + return false; + }; + packages + .iter() + .any(|package| match package.strip_prefix("src:") { + Some(name) => without_arch(name) == source, + None => binaries + .iter() + .any(|binary| binary == without_arch(package)), + }) + } + + /// It with all binaries of `source`, if it names packages. + pub fn with_source(mut self, source: &str) -> Self { + if let Self::Only(packages) = &mut self { + let pattern = format!("src:{source}"); + if !packages.contains(&pattern) { + packages.push(pattern); + } + } + self + } +} + +fn without_arch(package: &str) -> &str { + package.split_once(':').map_or(package, |(name, _)| name) } /// A bool, or a comma-separated list of packages. @@ -317,6 +348,29 @@ mod tests { assert_eq!(value.parse::(), Ok(expected)); } + #[test_case("libfoo1:amd64", true ; "binary")] + #[test_case("src:foo:any", true ; "source")] + #[test_case("libfoo-dev", false ; "other binary")] + #[test_case("src:food", false ; "other source")] + #[test_case("yes", false ; "all")] + fn selection_names_package(value: &str, expected: bool) { + let selection = value.parse::().unwrap(); + assert_eq!( + selection.names_package("foo", &["libfoo1".into()]), + expected + ); + } + + #[test] + fn with_source_adds_all_binaries_once() { + let selection = Selection::Only(vec!["libfoo1".into()]).with_source("foo"); + assert_eq!( + selection.clone().with_source("foo"), + Selection::Only(vec!["libfoo1".into(), "src:foo".into()]) + ); + assert_eq!(Selection::All.with_source("foo"), Selection::All); + } + #[test_case("")] #[test_case("libfoo,")] #[test_case("lib foo")] diff --git a/packages/debmagic-common/src/debian/control.rs b/packages/debmagic-common/src/debian/control.rs index fe85fde1..94c11275 100644 --- a/packages/debmagic-common/src/debian/control.rs +++ b/packages/debmagic-common/src/debian/control.rs @@ -38,6 +38,8 @@ pub struct Control { /// to one line without empty entries; `None` if absent or empty. pub build_depends: [Option; 3], pub build_conflicts: [Option; 3], + /// The binary packages it builds. + pub binaries: Vec, } impl Control { @@ -45,7 +47,8 @@ impl Control { let deb822 = content .parse::() .context("invalid deb822")?; - let source = deb822.paragraphs().next().context("no source paragraph")?; + let mut paragraphs = deb822.paragraphs(); + let source = paragraphs.next().context("no source paragraph")?; let relations = |fields: [&str; 3]| { fields.map(|field| { source @@ -59,6 +62,10 @@ impl Control { .map(|value| value.split_whitespace().map(str::to_string).collect()), build_depends: relations(BUILD_DEPENDS_FIELDS), build_conflicts: relations(BUILD_CONFLICTS_FIELDS), + binaries: paragraphs + .filter_map(|binary| binary.get("Package")) + .map(|name| name.trim().to_string()) + .collect(), }) } @@ -141,4 +148,10 @@ mod tests { } ); } + + #[test] + fn test_binaries() { + let control = Control::parse(&format!("{BUILD_RELATIONS}\nPackage: libfoo1\n")).unwrap(); + assert_eq!(control.binaries, ["foo", "libfoo1"]); + } } diff --git a/packages/debmagic/src/package.rs b/packages/debmagic/src/package.rs index f87d9bdc..39292cbc 100644 --- a/packages/debmagic/src/package.rs +++ b/packages/debmagic/src/package.rs @@ -283,7 +283,7 @@ fn parse_os_release(contents: &str) -> OsRelease { } } -fn select_distro_version( +pub fn select_distro_version( changelog_distros: &[String], explicit_distro: Option<&str>, distros: &DistroTable, diff --git a/packages/debmagic/src/test/run.rs b/packages/debmagic/src/test/run.rs index daf2b8a3..d3b7c345 100644 --- a/packages/debmagic/src/test/run.rs +++ b/packages/debmagic/src/test/run.rs @@ -8,7 +8,10 @@ use crate::build::artifacts::{copy_changes_artifacts, copy_dir_all}; use crate::build::source::{stage_dir, stage_source_tree}; use crate::build_intent::BuildKind; use crate::driver::{DriverType, Environment, EnvironmentDriver, EnvironmentPurpose}; -use crate::driver::{IsolationCapability, config::DriverConfig}; +use crate::driver::{ + IsolationCapability, + config::{DriverConfig, DriverOverrides}, +}; use crate::environment::{ CommandCompletion, HostRootPolicy, InvocationKind, RecordedInvocation, Registry, claim_command, prior_binary_invocation, @@ -205,12 +208,54 @@ fn autopkgtest_isolation_args(isolation: IsolationCapability) -> Vec<&'static st } } +/// The overrides to test the archive's build of `package` instead of a +/// local one, `None` to test a local build: when the proposed or +/// experimental selection names it or one of its binaries, with all its +/// binaries from that suite so their versions match. +fn archive_build_overrides( + package: &SourcePackage, + config: &DriverConfig, + overrides: &DriverOverrides, +) -> anyhow::Result> { + let binaries = &package + .control() + .context("reading the binary packages failed")? + .binaries; + let name = package.name(); + let mut archive = overrides.clone(); + let mut named = false; + for (chosen, configured) in [ + (&mut archive.proposed, &config.proposed), + (&mut archive.experimental, &config.experimental), + ] { + let selection = chosen.clone().unwrap_or_else(|| configured.clone()); + if selection.names_package(name, binaries) { + *chosen = Some(selection.with_source(name)); + named = true; + } + } + Ok(named.then_some(archive)) +} + pub async fn run_test(intent: &TestIntent) -> anyhow::Result { let package = load_package(&intent.source_dir)?; let distros = crate::distro_info::load() .await .context("loading the Debian and Ubuntu releases failed")?; let registry = Registry::open_default_or_ephemeral().await?; + let archive_overrides = + archive_build_overrides(&package, &intent.config.driver, &intent.driver_overrides)?; + let from_archive = archive_overrides.is_some(); + let overrides = archive_overrides + .as_ref() + .unwrap_or(&intent.driver_overrides); + if from_archive && intent.changes.is_some() { + bail!( + "--changes tests a local build, but the proposed or experimental packages name {} \ + to test the archive's", + package.name() + ); + } // `--changes` overrides only the artifact path. Driver and distro still // default to the prior binary-build Invocation when one exists. @@ -225,25 +270,44 @@ pub async fn run_test(intent: &TestIntent) -> anyhow::Result { &invocations, intent.distro.as_deref(), intent.changes.is_some(), - )?; + ); + // the archive's build needs a prior one at most for its driver + let prior_invocation = match from_archive { + true => prior_invocation.ok().flatten(), + false => prior_invocation?, + }; - let changes_path = if let Some(ref explicit) = intent.changes { + let changes_path = if from_archive { + None + } else if let Some(ref explicit) = intent.changes { if !explicit.is_file() { bail!("--changes file {} does not exist", explicit.display()); } - explicit.clone() + Some(explicit.clone()) } else { - prior_invocation - .as_ref() - .and_then(|invocation| invocation.changes_path.clone()) - .context( - "no prior binary build found; run `debmagic build binary` first or pass --changes", - )? + Some( + prior_invocation + .as_ref() + .and_then(|invocation| invocation.changes_path.clone()) + .context( + "no prior binary build found; run `debmagic build binary` first, pass \ + --changes, or name the package in --proposed/--experimental to test the \ + archive's build", + )?, + ) }; - if let Some(invocation) = prior_invocation.as_ref() { - warn_on_prior_build_mismatch(intent, &package, invocation, &changes_path, &distros); + if let (Some(invocation), Some(changes_path)) = (&prior_invocation, &changes_path) { + warn_on_prior_build_mismatch(intent, &package, invocation, changes_path, &distros); } - let exported_test_dir = exported_test_dir(&changes_path)?; + let exported_test_dir = match &changes_path { + Some(changes_path) => exported_test_dir(changes_path)?, + None => { + let output_dir = crate::output_dir::resolve(&intent.source_dir, &intent.config, None)?; + crate::output_dir::create(&output_dir)?; + let version = package.version().upstream_revision(); + output_dir.join(format!("{}_{version}.test", package.name())) + } + }; ensure_exported_test_dir_replaceable(&exported_test_dir)?; let driver = intent @@ -266,7 +330,15 @@ pub async fn run_test(intent: &TestIntent) -> anyhow::Result { ); } - let distro = if let Some(ref override_distro) = intent.distro { + let distro = if from_archive { + let mode = distro_resolve_mode_for_driver( + driver, + &intent.config.driver.docker.base_images, + &intent.config.driver.lxd.base_images, + ); + let changelog = package.distributions(); + crate::package::select_distro_version(changelog, intent.distro.as_deref(), &distros, mode)? + } else if let Some(ref override_distro) = intent.distro { lookup_distro_override(override_distro, &distros, driver, &intent.config.driver)? } else if let Some(ref invocation) = prior_invocation { // The Invocation recorded the full DistroVersion, so custom suites @@ -277,7 +349,7 @@ pub async fn run_test(intent: &TestIntent) -> anyhow::Result { "no prior build metadata found; pass --distro when using --changes without a prior Invocation" ); }; - crate::driver::apt_suites(&distro, &intent.config.driver, &intent.driver_overrides)?; + crate::driver::apt_suites(&distro, &intent.config.driver, overrides)?; let environment = Environment::new( driver, @@ -295,7 +367,7 @@ pub async fn run_test(intent: &TestIntent) -> anyhow::Result { ®istry, environment, &intent.config.driver, - &intent.driver_overrides, + overrides, HostRootPolicy::Reset, |environment, _| { stage_source_tree( @@ -306,7 +378,10 @@ pub async fn run_test(intent: &TestIntent) -> anyhow::Result { false, &crate::output_dir::resolve(&intent.source_dir, &intent.config, None)?, )?; - copy_changes_artifacts(&changes_path, &environment.work_dir()) + match &changes_path { + Some(changes_path) => copy_changes_artifacts(changes_path, &environment.work_dir()), + None => Ok(()), + } }, InvocationKind::Test, &version, @@ -320,8 +395,7 @@ pub async fn run_test(intent: &TestIntent) -> anyhow::Result { ..Default::default() }; let distro = &claimed.environment().distro; - let suites = - crate::driver::apt_suites(distro, &intent.config.driver, &intent.driver_overrides)?; + let suites = crate::driver::apt_suites(distro, &intent.config.driver, overrides)?; crate::apt::install( claimed.driver(), &staged_source_dir, @@ -333,9 +407,13 @@ pub async fn run_test(intent: &TestIntent) -> anyhow::Result { let work_dir = claimed.environment().work_dir(); let changes_filename = changes_path - .file_name() - .and_then(|name| name.to_str()) - .ok_or_else(|| anyhow!("invalid .changes path: {}", changes_path.display()))?; + .as_ref() + .map(|path| { + path.file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| anyhow!("invalid .changes path: {}", path.display())) + }) + .transpose()?; let source_tree_name = staged_source_dir .file_name() .and_then(|name| name.to_str()) @@ -350,6 +428,7 @@ pub async fn run_test(intent: &TestIntent) -> anyhow::Result { // Binary-only builds have no .dsc in the .changes; pass the staged source // tree alongside the .changes so debian/tests/ is found without rebuilding // (-B). See autopkgtest(1) "TESTING A DEBIAN PACKAGE" (.changes + tree). + // Without a .changes, -B takes the source's binaries from the archive. // IsolationCapabilities become `--ignore-restrictions` (autopkgtest args, // not virt-null `--fake-capability`; see autopkgtest_isolation_args). let output_dir_flag = format!("--output-dir={output_dir_arg}"); @@ -359,14 +438,9 @@ pub async fn run_test(intent: &TestIntent) -> anyhow::Result { autopkgtest_isolation_args(claimed.driver().isolation_capability()); let mut autopkgtest_cmd = vec!["autopkgtest", "-B", "--no-auto-control"]; autopkgtest_cmd.extend(isolation_args); - autopkgtest_cmd.extend([ - output_dir_flag.as_str(), - summary_flag.as_str(), - changes_filename, - source_tree_arg.as_str(), - "--", - "null", - ]); + autopkgtest_cmd.extend([output_dir_flag.as_str(), summary_flag.as_str()]); + autopkgtest_cmd.extend(changes_filename); + autopkgtest_cmd.extend([source_tree_arg.as_str(), "--", "null"]); crate::output::stage(&format!("Running autopkgtest for {}", package.name())); let exit_code = claimed @@ -424,6 +498,40 @@ mod tests { use super::*; use crate::driver::Persistence; + #[test] + fn naming_the_package_tests_the_archive_build_of_all_its_binaries() -> anyhow::Result<()> { + use debmagic_common::apt::Selection; + + let package = SourcePackage::from_files([ + ( + "changelog", + "foo (1.0-1) unstable; urgency=medium\n\n * Change.\n\n -- A Mon, 01 Jan 2024 00:00:00 +0000\n", + ), + ( + "control", + "Source: foo\n\nPackage: libfoo1\n\nPackage: foo-bin\n", + ), + ])?; + let config = DriverConfig::default(); + let only = |packages: &[&str]| { + Selection::Only(packages.iter().map(|package| package.to_string()).collect()) + }; + let mut overrides = DriverOverrides { + proposed: Some(only(&["libfoo1", "libbar1"])), + ..DriverOverrides::default() + }; + let archive = archive_build_overrides(&package, &config, &overrides)?.unwrap(); + assert_eq!( + archive.proposed, + Some(only(&["libfoo1", "libbar1", "src:foo"])) + ); + assert_eq!(archive.experimental, None); + + overrides.proposed = Some(Selection::All); + assert!(archive_build_overrides(&package, &config, &overrides)?.is_none()); + Ok(()) + } + #[test] fn exported_test_dir_is_named_after_the_changes_file() -> anyhow::Result<()> { assert_eq!( From 2a2d87cd22b823eb7f59c2d9b3b5401ba6a17c19 Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Fri, 23 Oct 2026 17:20:57 +0200 Subject: [PATCH 42/48] feat(config)!: move the apt settings into an [apt] section `driver.apt_mirror`, `driver.apt_update_age`, `driver.proposed` and `driver.experimental` are now `apt.mirror`, `apt.update_age`, `apt.proposed` and `apt.experimental`. The old keys are rejected with their new name. The CLI flags stay the same. --- docs/usage/build.md | 2 +- docs/usage/config.md | 12 +++-- packages/debmagic/CHANGELOG.md | 3 +- packages/debmagic/src/apt.rs | 54 ++++++++++++++++++++- packages/debmagic/src/build/mod.rs | 17 ++----- packages/debmagic/src/build_intent.rs | 8 ++-- packages/debmagic/src/cli.rs | 16 +++++-- packages/debmagic/src/config.rs | 41 ++++++++++++++++ packages/debmagic/src/driver/config.rs | 16 ------- packages/debmagic/src/driver/mod.rs | 41 ++++------------ packages/debmagic/src/environment/mod.rs | 19 +++++--- packages/debmagic/src/main.rs | 2 + packages/debmagic/src/test/intent.rs | 8 ++-- packages/debmagic/src/test/run.rs | 61 ++++++++++-------------- 14 files changed, 174 insertions(+), 126 deletions(-) diff --git a/docs/usage/build.md b/docs/usage/build.md index 7ede741f..12a3a9f9 100644 --- a/docs/usage/build.md +++ b/docs/usage/build.md @@ -86,7 +86,7 @@ You can persistently set this flag in `.config/debmagic/config.toml`. The bare driver builds on the host and never upgrades it. The default `1d` matches a typical developer machine's daily apt refresh: repeated builds stay fast, while the index can't go arbitrarily stale. -Persist the setting as `apt_update_age = "1d"` in [`debmagic.toml`](config.md). +Persist the setting as `update_age = "1d"` in the `[apt]` section of [`debmagic.toml`](config.md). ## Source file staging diff --git a/docs/usage/config.md b/docs/usage/config.md index b133e2c3..86dcc25b 100644 --- a/docs/usage/config.md +++ b/docs/usage/config.md @@ -24,13 +24,13 @@ All keys are optional. |---|---|---|---|---| | `driver.default` | enum | — | `--driver` | Build driver (`docker`, `bare`, `lxd`, `incus`) | | `driver.persistent` | enum | `"on-failure"` | `--persistent` | How long the Environment outlives the claim: `"no"`, `"on-failure"`, or `"always"`. A bare `--persistent` means `always`. | -| `driver.apt_mirror` | string | — | `--apt-mirror` | Mirror replacing the image's apt sources. Not used by the `bare` driver. | -| `driver.proposed` | bool or list | `false` | `--proposed` | Packages from the [proposed suite](build.md#proposed-dependencies): `true` for all newer ones, or a list like `["libfoo1", "src:bar"]`. Not used by the `bare` driver. | -| `driver.experimental` | bool or list | `false` | `--experimental` | Packages from [experimental](build.md#experimental) for unstable and testing. Not used by the `bare` driver. | -| `driver.apt_update_age` | string | `"1d"` | `--apt-update-age` | When a persistent environment re-runs `apt-get update && upgrade`: `"now"` (every build), `"never"` (only on creation), or a maximum age like `"1d"`, `"12h"`, `"30m"`. Not used by the `bare` driver. | | `driver.docker.base_images` | map | — | — | Base image per distro, keyed by `":"` (e.g. `"debian:trixie"`). Falls back to `docker.io/:`. For non-Debian/Ubuntu suites (e.g. `"yocto:kirkstone"`), the map entry is what makes the suite a known DistroVersion for Docker builds. | | `driver.lxd.project` | string | — | — | LXD/Incus project to use. | | `driver.lxd.base_images` | map | — | — | Base image per distro, keyed by `":"`. Falls back to the driver's default remote image. Same custom-suite registry role as Docker's map for LXD/Incus. | +| `apt.mirror` | string | — | `--apt-mirror` | Mirror replacing the image's apt sources. | +| `apt.update_age` | string | `"1d"` | `--apt-update-age` | When a persistent environment re-runs `apt-get update && upgrade`: `"now"` (every build), `"never"` (only on creation), or a maximum age like `"1d"`, `"12h"`, `"30m"`. | +| `apt.proposed` | bool or list | `false` | `--proposed` | Packages from the [proposed suite](build.md#proposed-dependencies): `true` for all newer ones, or a list like `["libfoo1", "src:bar"]`. | +| `apt.experimental` | bool or list | `false` | `--experimental` | Packages from [experimental](build.md#experimental) for unstable and testing. | | `environments_dir` | path | `$XDG_DATA_HOME/debmagic/environments` | — | Parent directory of Environment Host roots. Must be absolute or start with `~/`. The Environment registry is always `$XDG_DATA_HOME/debmagic/db.sqlite` (override both with `DEBMAGIC_DATA_DIR`). | | `output_dir` | path | `build` | `-o`/`--output-dir` | Where build artifacts are exported; relative paths resolve against the package root. See [Output dir](build.md#output-dir). | | `output_dir_fallback` | path | `build-debmagic` | — | Used instead of `output_dir` when upstream owns that directory. | @@ -119,7 +119,9 @@ notify = true [driver] default = "lxd" persistent = "always" -apt_mirror = "http:///ubuntu" + +[apt] +mirror = "http:///ubuntu" [driver.docker] # Optional image overrides for known Debian/Ubuntu releases, and the registry diff --git a/packages/debmagic/CHANGELOG.md b/packages/debmagic/CHANGELOG.md index 038e312f..7e111dfc 100644 --- a/packages/debmagic/CHANGELOG.md +++ b/packages/debmagic/CHANGELOG.md @@ -8,7 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - `debmagic switch ` upstream source fetching and orig tarballs handling with `debmagic upstream` (and caching them) -- dynamic interval for running `apt update` configured by `--apt-update-age` / `driver.apt_update_age` +- dynamic interval for running `apt update` configured by `--apt-update-age` / `apt.update_age` +- apt settings move from `[driver]` to `[apt]`: `driver.apt_mirror` is now `apt.mirror`, `driver.proposed` is `apt.proposed` - `debmagic env list`, `debmagic env clean`, and `debmagic env shell` for a machine-wide view of environments ## [0.0.1-alpha.8] - 2026-09-20 diff --git a/packages/debmagic/src/apt.rs b/packages/debmagic/src/apt.rs index 8b659935..0463b91e 100644 --- a/packages/debmagic/src/apt.rs +++ b/packages/debmagic/src/apt.rs @@ -6,12 +6,64 @@ use std::{fs, io, os::unix::fs::PermissionsExt, path::Path}; use anyhow::Context; use debmagic_common::{ - apt::{self, Suites}, + apt::{self, Selection, Suites}, debian::control::BuildRelations, distro::DistroVersion, }; +use serde::{Deserialize, Serialize}; use crate::driver::EnvironmentDriver; +use crate::time::RefreshPolicy; + +/// The `[apt]` section: where an Environment's packages come from. Not +/// used by the bare driver, which builds on the host's own sources. +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +#[serde(default)] +pub struct AptConfig { + /// Mirror replacing the distro's default archive. + pub mirror: Option, + /// When a reused Environment runs apt-get update/upgrade again. + pub update_age: RefreshPolicy, + /// Packages from the suite the release's updates migrate from: `true` + /// for all, or a list. + pub proposed: Selection, + /// Packages from Debian experimental, for unstable and testing builds. + pub experimental: Selection, +} + +impl AptConfig { + /// The suites beyond its release an Environment for `distro` takes + /// packages from. + pub fn suites(&self, distro: &DistroVersion) -> anyhow::Result { + Suites::for_distro(distro, self.proposed.clone(), self.experimental.clone()) + } +} + +/// The `[apt]` settings given on the command line. +#[derive(Debug, Clone, Default)] +pub struct AptOverrides { + pub mirror: Option, + pub update_age: Option, + pub proposed: Option, + pub experimental: Option, +} + +impl AptOverrides { + pub fn apply(self, config: &mut AptConfig) { + if let Some(mirror) = self.mirror { + config.mirror = Some(mirror); + } + if let Some(update_age) = self.update_age { + config.update_age = update_age; + } + if let Some(proposed) = self.proposed { + config.proposed = proposed; + } + if let Some(experimental) = self.experimental { + config.experimental = experimental; + } + } +} const ENV: [(&str, &str); 1] = [("DEBIAN_FRONTEND", "noninteractive")]; diff --git a/packages/debmagic/src/build/mod.rs b/packages/debmagic/src/build/mod.rs index 707a4726..181fe94e 100644 --- a/packages/debmagic/src/build/mod.rs +++ b/packages/debmagic/src/build/mod.rs @@ -7,9 +7,7 @@ use std::{ use crate::apt; use crate::build::source::{source_manifest_path, stage_dir, stage_source_tree}; use crate::build_intent::{BuildIntent, BuildIntentInput, BuildKind, resolve_build_intent}; -use crate::driver::{ - Driver, DriverType, Environment, EnvironmentDriver, EnvironmentPurpose, apt_suites, -}; +use crate::driver::{Driver, DriverType, Environment, EnvironmentDriver, EnvironmentPurpose}; use crate::environment::{ ClaimedEnvironment, CommandCompletion, HostRootPolicy, InvocationKind, Registry, claim_command, }; @@ -145,6 +143,7 @@ async fn run_build( ®istry, planned.environment, &request.intent.config.driver, + &request.intent.config.apt, &request.intent.driver_overrides, planned.root_policy, |environment, driver| { @@ -332,11 +331,7 @@ fn install_packages( profiles: profiles.to_vec(), }; let environment = claimed.environment(); - let suites = apt_suites( - &environment.distro, - &intent.config.driver, - &intent.driver_overrides, - )?; + let suites = intent.config.apt.suites(&environment.distro)?; let staged_source_dir = stage_dir(environment, kind); apt::install( claimed.driver(), @@ -567,11 +562,7 @@ pub async fn build(command: BuildCommand) -> anyhow::Result<()> { ), ) .context("failed to determine package target")?; - apt_suites( - &target.distro, - &intent.config.driver, - &intent.driver_overrides, - )?; + intent.config.apt.suites(&target.distro)?; let package = format!("{} {}", target.package.name(), target.package.version()); let notify = intent.config.build_notify(); diff --git a/packages/debmagic/src/build_intent.rs b/packages/debmagic/src/build_intent.rs index 31d63046..3cd19a11 100644 --- a/packages/debmagic/src/build_intent.rs +++ b/packages/debmagic/src/build_intent.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; use anyhow::Context; use crate::{ + apt::AptOverrides, build::source::SourceSyncMode, config::Config, driver::{DriverType, Persistence, config::DriverOverrides}, @@ -42,6 +43,7 @@ pub struct BuildIntentInput { pub clean: Option, pub source_sync: Option, pub host_arch_variant: Option, + pub apt_overrides: AptOverrides, pub driver_overrides: DriverOverrides, } @@ -85,6 +87,7 @@ pub fn resolve_build_intent(input: BuildIntentInput) -> anyhow::Result DriverOverrides { - DriverOverrides { - apt_mirror: self.apt_mirror.clone(), + /// Collect the `[apt]` overrides from the CLI flags. + pub fn apt_overrides(&self) -> AptOverrides { + AptOverrides { + mirror: self.apt_mirror.clone(), + update_age: self.apt_update_age, proposed: self.proposed.clone(), experimental: self.experimental.clone(), - apt_update_age: self.apt_update_age, + } + } + + /// Collect the driver-specific overrides from the CLI flags. + pub fn driver_overrides(&self) -> DriverOverrides { + DriverOverrides { docker: DriverDockerConfigOverrides { base_image: self.docker.base_image.clone(), }, diff --git a/packages/debmagic/src/config.rs b/packages/debmagic/src/config.rs index 844321ce..df330e72 100644 --- a/packages/debmagic/src/config.rs +++ b/packages/debmagic/src/config.rs @@ -1,5 +1,6 @@ use std::path::{Path, PathBuf}; +use crate::apt::AptConfig; use crate::build::source::SourceSyncMode; use crate::driver::DriverType; use crate::driver::config::DriverConfig; @@ -223,6 +224,8 @@ pub fn run_config(command: ConfigCommand) -> anyhow::Result<()> { #[serde(default)] pub struct Config { pub driver: DriverConfig, + /// `[apt]` section. + pub apt: AptConfig, /// Parent directory of Environment Host roots. pub environments_dir: PathBuf, /// Where build artifacts are exported; relative paths resolve against @@ -378,10 +381,17 @@ impl SignConfig { } } +const MOVED_KEYS: &[(&str, &str)] = &[ + ("driver.apt_mirror", "apt.mirror"), + ("driver.apt_update_age", "apt.update_age"), + ("driver.proposed", "apt.proposed"), +]; + impl Default for Config { fn default() -> Self { Self { driver: DriverConfig::default(), + apt: AptConfig::default(), environments_dir: crate::data_dir::default_environments_dir(), output_dir: PathBuf::from("build"), output_dir_fallback: PathBuf::from("build-debmagic"), @@ -462,6 +472,12 @@ impl Config { let build = builder .build() .context("Failed to initialize config reader")?; + // unknown keys are ignored, which would silently drop these + for (old, new) in MOVED_KEYS { + if build.get::(old).is_ok() { + bail!("config key '{old}' is now '{new}'"); + } + } let mut config: Self = build .try_deserialize() .map_err(|e| anyhow!("Failed to read config: {e}"))?; @@ -586,6 +602,31 @@ mod tests { Ok(()) } + #[test] + fn it_loads_the_apt_section_and_rejects_moved_keys() -> Result<(), anyhow::Error> { + let dir = std::env::temp_dir().join(format!("debmagic-apt-{}", std::process::id())); + std::fs::create_dir_all(&dir)?; + let load = |content: &str| { + let file = dir.join("apt.toml"); + std::fs::write(&file, content)?; + Config::new(&[ConfigPath::new( + ConfigLayer::Explicit, + file, + ConfigPathStatus::Used, + )]) + }; + let cfg = load("[apt]\nmirror = \"http://m/ubuntu\"\nproposed = [\"libfoo1\"]\n")?; + assert_eq!(cfg.apt.mirror.as_deref(), Some("http://m/ubuntu")); + assert_eq!( + cfg.apt.proposed, + debmagic_common::apt::Selection::Only(vec!["libfoo1".into()]) + ); + let error = load("[driver]\nproposed = true\n").unwrap_err(); + std::fs::remove_dir_all(&dir).ok(); + assert!(error.to_string().contains("'apt.proposed'"), "{error}"); + Ok(()) + } + #[test] fn build_sign_defaults_survive_a_partial_section() -> Result<(), anyhow::Error> { let defaults = Config::default(); diff --git a/packages/debmagic/src/driver/config.rs b/packages/debmagic/src/driver/config.rs index fca66a7b..4553de42 100644 --- a/packages/debmagic/src/driver/config.rs +++ b/packages/debmagic/src/driver/config.rs @@ -1,12 +1,10 @@ use clap::ValueEnum; -use debmagic_common::apt::Selection; use serde::{Deserialize, Serialize}; use crate::driver::DriverType; use crate::driver::driver_bare::{DriverBareConfig, DriverBareConfigOverrides}; use crate::driver::driver_docker::{DriverDockerConfig, DriverDockerConfigOverrides}; use crate::driver::driver_lxd::{DriverLxdConfig, DriverLxdConfigOverrides}; -use crate::time::RefreshPolicy; /// How long the current claim lets an Environment outlive that claim. /// @@ -57,16 +55,6 @@ pub struct DriverConfig { pub default: Option, /// Lifetime of the Environment after this claim. Unset is `on-failure`. pub persistent: Persistence, - /// Not used by the bare driver, which builds on the host's own sources. - pub apt_mirror: Option, - /// Packages from the suite the release's updates migrate from: `true` - /// for all, or a list. Not used by the bare driver, which builds on the - /// host's own sources, like `experimental`. - pub proposed: Selection, - /// Packages from Debian experimental, for unstable and testing builds. - pub experimental: Selection, - /// When to run apt-get update/upgrade again. - pub apt_update_age: RefreshPolicy, pub docker: DriverDockerConfig, pub bare: DriverBareConfig, pub lxd: DriverLxdConfig, @@ -74,10 +62,6 @@ pub struct DriverConfig { #[derive(Deserialize, Debug, Clone, Default)] pub struct DriverOverrides { - pub apt_mirror: Option, - pub proposed: Option, - pub experimental: Option, - pub apt_update_age: Option, pub docker: DriverDockerConfigOverrides, pub bare: DriverBareConfigOverrides, pub lxd: DriverLxdConfigOverrides, diff --git a/packages/debmagic/src/driver/mod.rs b/packages/debmagic/src/driver/mod.rs index 0be436d6..1aadeb62 100644 --- a/packages/debmagic/src/driver/mod.rs +++ b/packages/debmagic/src/driver/mod.rs @@ -1,9 +1,6 @@ use anyhow::Context; use clap::ValueEnum; -use debmagic_common::{ - apt::{Selection, Suites}, - distro::DistroVersion, -}; +use debmagic_common::distro::DistroVersion; use serde::{Deserialize, Serialize}; use std::{ collections::HashMap, @@ -14,6 +11,7 @@ use std::{ time::SystemTime, }; +use crate::apt::AptConfig; use crate::driver::{ config::{DriverConfig, DriverOverrides}, driver_bare::DriverBare, @@ -489,29 +487,23 @@ impl EnvironmentDriver for Driver { pub fn create_driver( environment: &Environment, driver_config: &DriverConfig, + apt: &AptConfig, overrides: &DriverOverrides, ) -> anyhow::Result { - let apt_mirror = overrides - .apt_mirror - .as_deref() - .or(driver_config.apt_mirror.as_deref()); - let suites = apt_suites(&environment.distro, driver_config, overrides)?; - let mut driver_config = driver_config.clone(); - if let Some(apt_update_age) = overrides.apt_update_age { - driver_config.apt_update_age = apt_update_age; - } + let apt_mirror = apt.mirror.as_deref(); + let suites = apt.suites(&environment.distro)?; match environment.driver { DriverType::Docker => Ok(Driver::Docker(DriverDocker::create( environment, - &driver_config, + driver_config, &overrides.docker, apt_mirror, &suites, )?)), DriverType::Bare => Ok(Driver::Bare(DriverBare::create( environment, - &driver_config, + driver_config, &overrides.bare, ))), DriverType::Lxd | DriverType::Incus => { @@ -522,7 +514,7 @@ pub fn create_driver( Ok(Driver::Lxd(DriverLxd::create( variant, environment, - &driver_config, + driver_config, &overrides.lxd, apt_mirror, &suites, @@ -531,23 +523,6 @@ pub fn create_driver( } } -/// The suites beyond its release an Environment for `distro` takes -/// packages from. -pub fn apt_suites( - distro: &DistroVersion, - driver_config: &DriverConfig, - overrides: &DriverOverrides, -) -> anyhow::Result { - let pick = |chosen: &Option, configured: &Selection| { - chosen.clone().unwrap_or_else(|| configured.clone()) - }; - Suites::for_distro( - distro, - pick(&overrides.proposed, &driver_config.proposed), - pick(&overrides.experimental, &driver_config.experimental), - ) -} - /// The Driver metadata [`create_driver`] will produce, computed without /// touching the Driver. Recorded before creation so a run that dies mid-setup /// still leaves enough in the registry to find and destroy the resource. diff --git a/packages/debmagic/src/environment/mod.rs b/packages/debmagic/src/environment/mod.rs index 3807241b..e09afdf1 100644 --- a/packages/debmagic/src/environment/mod.rs +++ b/packages/debmagic/src/environment/mod.rs @@ -11,6 +11,7 @@ use anyhow::{Context, bail}; use process::Process; use sqlx::sqlite::SqliteConnection; +use crate::apt::AptConfig; use crate::driver::{ Driver, Environment, EnvironmentDriver, EnvironmentMetadata, Persistence, ResourceStatus, config::{DriverConfig, DriverOverrides}, @@ -61,6 +62,7 @@ impl<'a> ClaimedEnvironment<'a> { registry: &'a Registry, environment: Environment, driver_config: &'a DriverConfig, + apt: &AptConfig, overrides: &DriverOverrides, root_policy: HostRootPolicy, stage: impl FnOnce(&Environment, Option<&Driver>) -> anyhow::Result<()>, @@ -72,7 +74,7 @@ impl<'a> ClaimedEnvironment<'a> { if environment.persistence.reuses() && environment.root_dir.exists() { let claimed = - Self::create_driver(registry, environment, driver_config, overrides).await?; + Self::create_driver(registry, environment, driver_config, apt, overrides).await?; if let Err(error) = claimed.prepare_reused_root(root_policy, stage) { return claimed.abort(error).await; } @@ -91,7 +93,7 @@ impl<'a> ClaimedEnvironment<'a> { abandon_claim(registry, &environment, driver_config, false).await; return Err(error); } - Self::create_driver(registry, environment, driver_config, overrides).await + Self::create_driver(registry, environment, driver_config, apt, overrides).await } pub fn environment(&self) -> &Environment { @@ -126,9 +128,11 @@ impl<'a> ClaimedEnvironment<'a> { registry: &'a Registry, environment: Environment, driver_config: &'a DriverConfig, + apt: &AptConfig, overrides: &DriverOverrides, ) -> anyhow::Result { - let driver = match crate::driver::create_driver(&environment, driver_config, overrides) { + let driver = match crate::driver::create_driver(&environment, driver_config, apt, overrides) + { Ok(driver) => driver, Err(error) => { abandon_claim(registry, &environment, driver_config, true).await; @@ -149,10 +153,7 @@ impl<'a> ClaimedEnvironment<'a> { driver, keep_after_failure: false, }; - let policy = overrides - .apt_update_age - .unwrap_or(driver_config.apt_update_age); - if let Err(error) = claimed.upgrade_packages(policy).await { + if let Err(error) = claimed.upgrade_packages(apt.update_age).await { return claimed .abort(error.context("upgrading the packages in the Environment failed")) .await; @@ -302,6 +303,7 @@ pub async fn claim_command( registry: &Registry, environment: Environment, driver_config: &DriverConfig, + apt: &AptConfig, overrides: &DriverOverrides, root_policy: HostRootPolicy, stage: impl FnOnce(&Environment, Option<&Driver>) -> anyhow::Result<()>, @@ -313,6 +315,7 @@ pub async fn claim_command( registry, environment, driver_config, + apt, overrides, root_policy, stage, @@ -1467,6 +1470,7 @@ mod claimed_environment_tests { &self.registry, environment, &self.driver_config, + &AptConfig::default(), &self.overrides, root_policy, stage, @@ -1644,6 +1648,7 @@ mod claimed_environment_tests { &fixture.registry, environment, &fixture.driver_config, + &AptConfig::default(), &fixture.overrides, HostRootPolicy::Reset, |_, _| Ok(()), diff --git a/packages/debmagic/src/main.rs b/packages/debmagic/src/main.rs index 40c427be..73197f4f 100644 --- a/packages/debmagic/src/main.rs +++ b/packages/debmagic/src/main.rs @@ -108,6 +108,7 @@ async fn run() -> anyhow::Result { clean: build_args.clean, source_sync: build_args.source_sync, host_arch_variant: build_args.host_arch_variant.clone(), + apt_overrides: build_args.environment.apt_overrides(), driver_overrides: build_args.environment.driver_overrides(), }, distro: build_args.distro.clone(), @@ -167,6 +168,7 @@ async fn run() -> anyhow::Result { changes: args.changes.clone(), allow_host_test: args.allow_host_test, distro: args.distro.clone(), + apt_overrides: args.environment.apt_overrides(), driver_overrides: args.environment.driver_overrides(), })?; diff --git a/packages/debmagic/src/test/intent.rs b/packages/debmagic/src/test/intent.rs index 46eb39ba..c6c54899 100644 --- a/packages/debmagic/src/test/intent.rs +++ b/packages/debmagic/src/test/intent.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; use anyhow::Context; use crate::{ + apt::AptOverrides, config::Config, driver::{DriverType, Persistence, config::DriverOverrides}, }; @@ -20,6 +21,7 @@ pub struct TestIntentInput { pub changes: Option, pub allow_host_test: bool, pub distro: Option, + pub apt_overrides: AptOverrides, pub driver_overrides: DriverOverrides, } @@ -50,6 +52,7 @@ pub fn resolve_test_intent(input: TestIntentInput) -> anyhow::Result config.driver.persistent = persistent; } config.override_notify(input.notify); + input.apt_overrides.apply(&mut config.apt); let changes = if let Some(changes) = input.changes { Some(std::path::absolute(changes).context("resolving --changes path failed")?) @@ -95,11 +98,8 @@ mod tests { changes: None, allow_host_test: false, distro: None, + apt_overrides: AptOverrides::default(), driver_overrides: DriverOverrides { - apt_mirror: None, - proposed: None, - experimental: None, - apt_update_age: None, docker: DriverDockerConfigOverrides { base_image: None }, bare: DriverBareConfigOverrides {}, lxd: DriverLxdConfigOverrides { diff --git a/packages/debmagic/src/test/run.rs b/packages/debmagic/src/test/run.rs index d3b7c345..c1542250 100644 --- a/packages/debmagic/src/test/run.rs +++ b/packages/debmagic/src/test/run.rs @@ -4,14 +4,12 @@ use std::{ }; use super::intent::TestIntent; +use crate::apt::AptConfig; use crate::build::artifacts::{copy_changes_artifacts, copy_dir_all}; use crate::build::source::{stage_dir, stage_source_tree}; use crate::build_intent::BuildKind; use crate::driver::{DriverType, Environment, EnvironmentDriver, EnvironmentPurpose}; -use crate::driver::{ - IsolationCapability, - config::{DriverConfig, DriverOverrides}, -}; +use crate::driver::{IsolationCapability, config::DriverConfig}; use crate::environment::{ CommandCompletion, HostRootPolicy, InvocationKind, RecordedInvocation, Registry, claim_command, prior_binary_invocation, @@ -208,29 +206,24 @@ fn autopkgtest_isolation_args(isolation: IsolationCapability) -> Vec<&'static st } } -/// The overrides to test the archive's build of `package` instead of a +/// The apt config to test the archive's build of `package` instead of a /// local one, `None` to test a local build: when the proposed or /// experimental selection names it or one of its binaries, with all its /// binaries from that suite so their versions match. -fn archive_build_overrides( +fn archive_build_apt( package: &SourcePackage, - config: &DriverConfig, - overrides: &DriverOverrides, -) -> anyhow::Result> { + apt: &AptConfig, +) -> anyhow::Result> { let binaries = &package .control() .context("reading the binary packages failed")? .binaries; let name = package.name(); - let mut archive = overrides.clone(); + let mut archive = apt.clone(); let mut named = false; - for (chosen, configured) in [ - (&mut archive.proposed, &config.proposed), - (&mut archive.experimental, &config.experimental), - ] { - let selection = chosen.clone().unwrap_or_else(|| configured.clone()); + for selection in [&mut archive.proposed, &mut archive.experimental] { if selection.names_package(name, binaries) { - *chosen = Some(selection.with_source(name)); + *selection = std::mem::take(selection).with_source(name); named = true; } } @@ -243,12 +236,9 @@ pub async fn run_test(intent: &TestIntent) -> anyhow::Result { .await .context("loading the Debian and Ubuntu releases failed")?; let registry = Registry::open_default_or_ephemeral().await?; - let archive_overrides = - archive_build_overrides(&package, &intent.config.driver, &intent.driver_overrides)?; - let from_archive = archive_overrides.is_some(); - let overrides = archive_overrides - .as_ref() - .unwrap_or(&intent.driver_overrides); + let archive_apt = archive_build_apt(&package, &intent.config.apt)?; + let from_archive = archive_apt.is_some(); + let apt = archive_apt.as_ref().unwrap_or(&intent.config.apt); if from_archive && intent.changes.is_some() { bail!( "--changes tests a local build, but the proposed or experimental packages name {} \ @@ -349,7 +339,7 @@ pub async fn run_test(intent: &TestIntent) -> anyhow::Result { "no prior build metadata found; pass --distro when using --changes without a prior Invocation" ); }; - crate::driver::apt_suites(&distro, &intent.config.driver, overrides)?; + apt.suites(&distro)?; let environment = Environment::new( driver, @@ -367,7 +357,8 @@ pub async fn run_test(intent: &TestIntent) -> anyhow::Result { ®istry, environment, &intent.config.driver, - overrides, + apt, + &intent.driver_overrides, HostRootPolicy::Reset, |environment, _| { stage_source_tree( @@ -395,7 +386,7 @@ pub async fn run_test(intent: &TestIntent) -> anyhow::Result { ..Default::default() }; let distro = &claimed.environment().distro; - let suites = crate::driver::apt_suites(distro, &intent.config.driver, overrides)?; + let suites = apt.suites(distro)?; crate::apt::install( claimed.driver(), &staged_source_dir, @@ -512,23 +503,19 @@ mod tests { "Source: foo\n\nPackage: libfoo1\n\nPackage: foo-bin\n", ), ])?; - let config = DriverConfig::default(); let only = |packages: &[&str]| { Selection::Only(packages.iter().map(|package| package.to_string()).collect()) }; - let mut overrides = DriverOverrides { - proposed: Some(only(&["libfoo1", "libbar1"])), - ..DriverOverrides::default() + let mut apt = AptConfig { + proposed: only(&["libfoo1", "libbar1"]), + ..AptConfig::default() }; - let archive = archive_build_overrides(&package, &config, &overrides)?.unwrap(); - assert_eq!( - archive.proposed, - Some(only(&["libfoo1", "libbar1", "src:foo"])) - ); - assert_eq!(archive.experimental, None); + let archive = archive_build_apt(&package, &apt)?.unwrap(); + assert_eq!(archive.proposed, only(&["libfoo1", "libbar1", "src:foo"])); + assert_eq!(archive.experimental, Selection::Off); - overrides.proposed = Some(Selection::All); - assert!(archive_build_overrides(&package, &config, &overrides)?.is_none()); + apt.proposed = Selection::All; + assert!(archive_build_apt(&package, &apt)?.is_none()); Ok(()) } From 318ce871b92df4f97fe8cf27ce288781ba3d8941 Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Fri, 23 Oct 2026 17:34:20 +0200 Subject: [PATCH 43/48] feat(build): install a package in a version or from an archive with `--package` `--package [=][/]` uses apt-get's syntax; the archive is `release`, `proposed` or `experimental`. A version is installed even if older. Naming the tested package makes `debmagic test` test that build, e.g. `--package src:foo/release` for what the release ships. The config takes the same as `[apt.packages]`, e.g. `"src:foo" = { from = "proposed" }`. `apt.proposed` and `apt.experimental` are now bools; `--proposed=` stays a shorthand for `--package /proposed`. --- docs/usage/build.md | 26 +- docs/usage/config.md | 5 +- docs/usage/test.md | 9 +- packages/debmagic-common/src/apt.rs | 492 ++++++++++-------- packages/debmagic/src/apt.rs | 49 +- packages/debmagic/src/build/mod.rs | 6 +- packages/debmagic/src/cli.rs | 64 ++- packages/debmagic/src/config.rs | 12 +- packages/debmagic/src/driver/driver_docker.rs | 6 +- packages/debmagic/src/driver/driver_lxd.rs | 6 +- packages/debmagic/src/driver/mod.rs | 6 +- packages/debmagic/src/test/run.rs | 62 ++- 12 files changed, 460 insertions(+), 283 deletions(-) diff --git a/docs/usage/build.md b/docs/usage/build.md index 12a3a9f9..0a56375f 100644 --- a/docs/usage/build.md +++ b/docs/usage/build.md @@ -27,6 +27,7 @@ debmagic build binary --driver lxd \ | `--distro ` | [Select the target distro/release](#selecting-a-distrorelease) (e.g. `trixie`, `resolute`) | | `--proposed[=]` | Use [build dependencies from `proposed`](#proposed-dependencies) | | `--experimental[=]` | Use build dependencies from [experimental](#experimental) | +| `--package [=][/]` | Install a [package in a version or from an archive](#package-versions-and-archives) | | `--sign` | [GPG-sign the resulting `.changes`/`.dsc`/`.buildinfo`](#signing) | | `--clean` | Run [`debian/rules clean` before building](#cleaning) | | `--debug-symbols` | [Build the automatic `-dbgsym` debug symbol packages](#building-debug-symbol-packages) | @@ -165,8 +166,7 @@ Pass `--proposed` to build against the suite the release's updates come from, li | Debian stable release | `-proposed-updates` | | Debian `unstable`, `experimental` | none | -Pass a comma-separated list to take only those packages from it, e.g. `--proposed=libfoo1,src:bar`. -The names are [apt preferences](https://manpages.debian.org/apt_preferences) patterns: binary packages, `src:` for all binaries of a source, or globs. +Pass a comma-separated list to take only those packages from it, e.g. `--proposed=libfoo1,src:bar`, short for [`--package libfoo1/proposed --package src:bar/proposed`](#package-versions-and-archives). Everything else stays with the release. ## Experimental @@ -177,6 +177,28 @@ Only when unstable can't satisfy them, the install is retried preferring the pac Builds for unstable or testing do the same with `--experimental`, or take only the listed packages from it with `--experimental=` like `--proposed`. Other releases reject it. +## Package versions and archives + +`--package` sets how one package is installed, in apt-get's syntax: + +```shell +debmagic build --package libfoo1/proposed # from proposed +debmagic build --package src:bar=2.0-1 # all binaries of bar in version 2.0-1 +debmagic build --package libbaz1=1.2-3/proposed +``` + +The package is an [apt preferences](https://manpages.debian.org/apt_preferences) pattern: a binary package, `src:` for all binaries of a source, or a glob, each with an optional `:`. +The archive is `release` (with its updates), `proposed` or `experimental`; an archive packages come from is added to the environment. +A version is installed even if older than the installed one, from whichever archive has it. + +The [config](config.md) takes the same as a table: + +```toml +[apt.packages] +libfoo1 = { from = "proposed" } +"src:bar" = { version = "2.0-1" } +``` + ## Building debug symbol packages By default `debmagic build binary` passes `DEB_BUILD_OPTIONS=noautodbgsym` to `dpkg-buildpackage`, which suppresses debhelper's automatic `-dbgsym` package (the detached debug info package debhelper otherwise builds by default from compat 9 onward). diff --git a/docs/usage/config.md b/docs/usage/config.md index 86dcc25b..e0e01007 100644 --- a/docs/usage/config.md +++ b/docs/usage/config.md @@ -29,8 +29,9 @@ All keys are optional. | `driver.lxd.base_images` | map | — | — | Base image per distro, keyed by `":"`. Falls back to the driver's default remote image. Same custom-suite registry role as Docker's map for LXD/Incus. | | `apt.mirror` | string | — | `--apt-mirror` | Mirror replacing the image's apt sources. | | `apt.update_age` | string | `"1d"` | `--apt-update-age` | When a persistent environment re-runs `apt-get update && upgrade`: `"now"` (every build), `"never"` (only on creation), or a maximum age like `"1d"`, `"12h"`, `"30m"`. | -| `apt.proposed` | bool or list | `false` | `--proposed` | Packages from the [proposed suite](build.md#proposed-dependencies): `true` for all newer ones, or a list like `["libfoo1", "src:bar"]`. | -| `apt.experimental` | bool or list | `false` | `--experimental` | Packages from [experimental](build.md#experimental) for unstable and testing. | +| `apt.proposed` | bool | `false` | `--proposed` | All newer packages from the [proposed suite](build.md#proposed-dependencies). | +| `apt.experimental` | bool | `false` | `--experimental` | What nothing else provides from [experimental](build.md#experimental), for unstable and testing. | +| `apt.packages` | map | — | `--package` | [Package settings](build.md#package-versions-and-archives) by pattern: `from` (an archive) and `version`, e.g. `{ "src:bar" = { from = "proposed" } }`. | | `environments_dir` | path | `$XDG_DATA_HOME/debmagic/environments` | — | Parent directory of Environment Host roots. Must be absolute or start with `~/`. The Environment registry is always `$XDG_DATA_HOME/debmagic/db.sqlite` (override both with `DEBMAGIC_DATA_DIR`). | | `output_dir` | path | `build` | `-o`/`--output-dir` | Where build artifacts are exported; relative paths resolve against the package root. See [Output dir](build.md#output-dir). | | `output_dir_fallback` | path | `build-debmagic` | — | Used instead of `output_dir` when upstream owns that directory. | diff --git a/docs/usage/test.md b/docs/usage/test.md index e7258b99..49c49f4c 100644 --- a/docs/usage/test.md +++ b/docs/usage/test.md @@ -24,13 +24,15 @@ The driver *is* the testbed: `autopkgtest` runs with the `null` backend inside t ## Testing the archive's build -Naming the package (`src:`) or one of its binaries in `--proposed` or `--experimental` runs the source tree's tests against that suite's build instead of a local one, so no prior build is needed: +Naming the package (`src:`) or one of its binaries in a [`--package`](build.md#package-versions-and-archives), `--proposed` or `--experimental` runs the source tree's tests against that build instead of a local one, so no prior build is needed: ```shell -debmagic test --driver lxd --proposed=libfoo1 +debmagic test --driver lxd --proposed=libfoo1 # proposed's build +debmagic test --driver lxd --package src:foo/release # what the release ships +debmagic test --driver lxd --package src:foo=1.2-3 # a version ``` -All binaries of the source come from that suite, so their versions match. +All binaries of the source get the same setting, so their versions match. The distro comes from `--distro` or the changelog, and the test output goes to `/_.test`. ## Available options @@ -44,6 +46,7 @@ The distro comes from `--distro` or the changelog, and the test output goes to ` | `--distro ` | Override the target distro for the test environment (defaults to the prior binary-build Invocation's distro, not the changelog) | | `--proposed[=]` | Test against packages from the [proposed suite](build.md#proposed-dependencies): all newer ones, or only the listed ones. Naming this package [tests the archive's build](#testing-the-archives-build) | | `--experimental[=]` | Test against the listed packages from [experimental](build.md#experimental); without a list, only what nothing else provides. Naming this package [tests the archive's build](#testing-the-archives-build) | +| `--package [=][/]` | Test against a [package in a version or from an archive](build.md#package-versions-and-archives). Naming this package [tests the archive's build](#testing-the-archives-build) | | `--apt-mirror ` | Mirror URL (same as [`debmagic build`](build.md)) | | `--apt-update-age ` | Configure when to run `apt-get update` (same as [`debmagic build`](build.md#apt-update-age)) | | `--source-dir ` | Directory containing the `debian/` package directory | diff --git a/packages/debmagic-common/src/apt.rs b/packages/debmagic-common/src/apt.rs index 6eb9e050..2eb4e9b3 100644 --- a/packages/debmagic-common/src/apt.rs +++ b/packages/debmagic-common/src/apt.rs @@ -1,7 +1,7 @@ //! What apt needs to know about a target distro: the archive layout a build //! environment pulls from, and the features of the release's apt. -use std::str::FromStr; +use std::{collections::BTreeMap, str::FromStr}; use anyhow::bail; use serde::{Deserialize, Serialize}; @@ -16,74 +16,71 @@ const DEBIAN_ARCHIVE: &str = "http://deb.debian.org/debian"; const DEBIAN_SECURITY_ARCHIVE: &str = "http://deb.debian.org/debian-security"; const UBUNTU_ARCHIVE: &str = "http://archive.ubuntu.com/ubuntu"; const UBUNTU_PORTS_ARCHIVE: &str = "http://ports.ubuntu.com/ubuntu-ports"; +const RELEASE: &str = "release"; +const PROPOSED: &str = "proposed"; const EXPERIMENTAL: &str = "experimental"; +/// The archives a package can be taken from. +const ARCHIVES: &[&str] = &[RELEASE, PROPOSED, EXPERIMENTAL]; -/// Which packages a build takes from a suite beyond its release. +/// How a build environment installs the packages matching an apt +/// preferences pattern: a binary package name, `src:` for all its +/// binaries, or a glob. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(try_from = "SelectionValue", into = "SelectionValue")] -pub enum Selection { - #[default] - Off, - /// Whatever apt prefers with the suite enabled. - All, - /// Only these packages: apt preferences patterns like binary package - /// names, `src:` or globs. Everything else stays with the - /// release. - Only(Vec), +#[serde(default, deny_unknown_fields)] +pub struct PackageSettings { + /// The archive to take it from: `release` (with its updates), + /// `proposed` or `experimental`. + pub from: Option, + /// The version to install, even if older, from whichever archive in use + /// has it. + pub version: Option, } -/// [`Selection`] in config files: a bool or a list of packages. -#[derive(Serialize, Deserialize)] -#[serde(untagged)] -enum SelectionValue { - Enabled(bool), - Packages(Vec), -} +/// [`PackageSettings`] by pattern. +pub type Packages = BTreeMap; -impl Selection { - fn packages(packages: Vec) -> Result { - if let Some(invalid) = packages - .iter() - .find(|package| package.is_empty() || package.contains(char::is_whitespace)) - { - return Err(format!("invalid package pattern '{invalid}'")); - } - Ok(if packages.is_empty() { - Self::Off - } else { - Self::Only(packages) - }) +/// A `[=][/]` package spec, apt-get's syntax. +pub fn parse_package(spec: &str) -> Result<(String, PackageSettings), String> { + let (package, from) = match spec.split_once('/') { + Some((package, from)) => (package, Some(from.to_string())), + None => (spec, None), + }; + let (pattern, version) = match package.split_once('=') { + Some((pattern, version)) => (pattern, Some(version.to_string())), + None => (package, None), + }; + validate_pattern(pattern)?; + let settings = PackageSettings { from, version }; + if settings == PackageSettings::default() { + return Err(format!( + "'{spec}' needs a version (=) or an archive (/)" + )); + } + if let Some(invalid) = [&settings.from, &settings.version] + .into_iter() + .flatten() + .find(|value| value.is_empty() || value.contains(char::is_whitespace)) + { + return Err(format!("invalid '{invalid}' in '{spec}'")); } + Ok((pattern.to_string(), settings)) +} - pub fn is_on(&self) -> bool { - *self != Self::Off +fn validate_pattern(pattern: &str) -> Result<(), String> { + match pattern.is_empty() || pattern.contains(char::is_whitespace) { + true => Err(format!("invalid package pattern '{pattern}'")), + false => Ok(()), } +} - /// Whether it names the source package `source` or one of its - /// `binaries`, globs aside. - pub fn names_package(&self, source: &str, binaries: &[String]) -> bool { - let Self::Only(packages) = self else { - return false; - }; - packages +/// Whether `pattern` names the source package `source` or one of its +/// `binaries`, globs aside. +pub fn names_package(pattern: &str, source: &str, binaries: &[String]) -> bool { + match pattern.strip_prefix("src:") { + Some(name) => without_arch(name) == source, + None => binaries .iter() - .any(|package| match package.strip_prefix("src:") { - Some(name) => without_arch(name) == source, - None => binaries - .iter() - .any(|binary| binary == without_arch(package)), - }) - } - - /// It with all binaries of `source`, if it names packages. - pub fn with_source(mut self, source: &str) -> Self { - if let Self::Only(packages) = &mut self { - let pattern = format!("src:{source}"); - if !packages.contains(&pattern) { - packages.push(pattern); - } - } - self + .any(|binary| binary == without_arch(pattern)), } } @@ -91,7 +88,16 @@ fn without_arch(package: &str) -> &str { package.split_once(':').map_or(package, |(name, _)| name) } -/// A bool, or a comma-separated list of packages. +/// `--proposed`/`--experimental`: a bool, or a comma-separated list of +/// packages to take from that suite. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Selection { + Off, + /// Whatever apt prefers with the suite enabled. + All, + Only(Vec), +} + impl FromStr for Selection { type Err = String; @@ -99,68 +105,80 @@ impl FromStr for Selection { match value.to_ascii_lowercase().as_str() { "y" | "yes" | "t" | "true" | "on" | "1" => Ok(Self::All), "n" | "no" | "f" | "false" | "off" | "0" => Ok(Self::Off), - _ => Self::packages(value.split(',').map(|p| p.trim().to_string()).collect()), - } - } -} - -impl TryFrom for Selection { - type Error = String; - - fn try_from(value: SelectionValue) -> Result { - match value { - SelectionValue::Enabled(true) => Ok(Self::All), - SelectionValue::Enabled(false) => Ok(Self::Off), - SelectionValue::Packages(packages) => Self::packages(packages), - } - } -} - -impl From for SelectionValue { - fn from(selection: Selection) -> Self { - match selection { - Selection::Off => Self::Enabled(false), - Selection::All => Self::Enabled(true), - Selection::Only(packages) => Self::Packages(packages), + _ => { + let packages: Vec = + value.split(',').map(|p| p.trim().to_string()).collect(); + for package in &packages { + validate_pattern(package)?; + } + Ok(Self::Only(packages)) + } } } } -/// The suites beyond its release a build environment takes packages from. +/// The archives beyond its release a build environment takes packages +/// from, and how it installs some packages. #[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct Suites { - /// From the suite the release's updates migrate from. - pub proposed: Selection, - /// From Debian experimental. - pub experimental: Selection, +pub struct Archives { + /// All of the proposed suite, as apt prefers it. + pub proposed: bool, + /// All of experimental, for what nothing else provides. + pub experimental: bool, + pub packages: Packages, } -impl Suites { - /// The suites for a build for `distro`. Experimental only adds to - /// Debian unstable and testing, and is all of the experimental release. +impl Archives { + /// The archives for a build for `distro`: all of experimental for the + /// experimental release, which only adds to unstable. pub fn for_distro( distro: &DistroVersion, - proposed: Selection, - experimental: Selection, + proposed: bool, + experimental: bool, + packages: Packages, ) -> anyhow::Result { - let experimental = match experimental { - Selection::Off if is_experimental(distro) => Selection::All, - Selection::Off => Selection::Off, - _ if distro.distro == Distro::Debian - && (DEBIAN_UNSTABLE.contains(&distro.codename.as_str()) || distro.is_devel) => - { - experimental + let archives = Self { + proposed, + experimental: experimental || is_experimental(distro), + packages, + }; + let release = format!("{} {}", distro.distro, distro.codename); + for (pattern, settings) in &archives.packages { + match settings.from.as_deref() { + Some(from) if !ARCHIVES.contains(&from) => bail!( + "unknown archive '{from}' for {pattern}, known are {}", + ARCHIVES.join(", ") + ), + Some(PROPOSED) if proposed_suite(distro).is_none() => { + bail!("{release} has no proposed suite to take {pattern} from") + } + _ => {} } - _ => bail!( - "experimental only adds to Debian unstable and testing, not to {} {}", - distro.distro, - distro.codename - ), + } + let adds_to_experimental = distro.distro == Distro::Debian + && (DEBIAN_UNSTABLE.contains(&distro.codename.as_str()) || distro.is_devel); + if archives.uses(EXPERIMENTAL) && !adds_to_experimental { + bail!("experimental only adds to Debian unstable and testing, not to {release}"); + } + Ok(archives) + } + + /// Whether it takes packages from `archive`. + fn uses(&self, archive: &str) -> bool { + let all = match archive { + PROPOSED => self.proposed, + EXPERIMENTAL => self.experimental, + _ => false, }; - Ok(Self { - proposed, - experimental, - }) + all || self + .packages + .values() + .any(|settings| settings.from.as_deref() == Some(archive)) + } + + /// Whether it adds suites to the release's. + pub fn adds_suites(&self) -> bool { + self.uses(PROPOSED) || self.uses(EXPERIMENTAL) } } @@ -170,13 +188,13 @@ fn is_experimental(distro: &DistroVersion) -> bool { /// A deb822 `.sources` file pointing apt at `mirror` (the distro's default /// archive if `None`) for the release, its `-updates` and `-security` -/// pockets, plus the opted-in `suites`. It replaces the base image's own +/// pockets, plus the suites of `archives`. It replaces the base image's own /// sources. `dpkg_arch` is the environment's architecture: Ubuntu serves /// most of them from a separate ports archive. pub fn sources_file( distro: &DistroVersion, mirror: Option<&str>, - extra: &Suites, + archives: &Archives, dpkg_arch: &str, ) -> anyhow::Result { let codename = distro.codename.as_str(); @@ -194,28 +212,22 @@ pub fn sources_file( } }; let debian = distro.distro == Distro::Debian; - let rolling = debian && DEBIAN_UNSTABLE.contains(&codename); + let release = release_suites(distro); let security = format!("{codename}-security"); - let mut suites = vec![if is_experimental(distro) { - "unstable".to_string() - } else { - codename.to_string() - }]; - if !rolling { - suites.push(format!("{codename}-updates")); - if !debian { - suites.push(security.clone()); - } - } - if extra.proposed.is_on() { + let mut suites: Vec = release + .iter() + .filter(|suite| !debian || **suite != security) + .cloned() + .collect(); + if archives.uses(PROPOSED) { suites.extend(proposed_suite(distro)); } - if extra.experimental.is_on() { + if archives.uses(EXPERIMENTAL) { suites.push(EXPERIMENTAL.to_string()); } let mut groups = vec![(mirror.unwrap_or(archive), suites)]; - if debian && !rolling { + if debian && release.contains(&security) { groups.push((DEBIAN_SECURITY_ARCHIVE, vec![security])); } let paragraphs: Vec = groups @@ -239,6 +251,23 @@ fn debian_components(version: &str) -> &'static [&'static str] { } } +/// The suites of the release: its pockets, or unstable for Debian's +/// rolling ones. +fn release_suites(distro: &DistroVersion) -> Vec { + let codename = distro.codename.as_str(); + if distro.distro == Distro::Debian && DEBIAN_UNSTABLE.contains(&codename) { + let base = if is_experimental(distro) { + "unstable" + } else { + codename + }; + return vec![base.to_string()]; + } + ["", "-updates", "-security"] + .map(|pocket| format!("{codename}{pocket}")) + .to_vec() +} + /// The suite updates of the release migrate from, `None` for unstable /// itself: unstable migrates into Debian's devel release (testing). fn proposed_suite(distro: &DistroVersion) -> Option { @@ -253,45 +282,69 @@ fn proposed_suite(distro: &DistroVersion) -> Option { /// The NotAutomatic suite a build takes all packages from only when nothing /// else satisfies its build dependencies: experimental. -pub fn retry_suite(extra: &Suites) -> Option<&'static str> { - (extra.experimental == Selection::All).then_some(EXPERIMENTAL) +pub fn retry_suite(archives: &Archives) -> Option<&'static str> { + archives.experimental.then_some(EXPERIMENTAL) } -/// apt preferences for the `extra` suites of [`sources_file`]. Ubuntu -/// marks `-proposed` NotAutomatic, so apt would only upgrade installed -/// packages from it; pinned to the release's priority like on Launchpad's -/// builders, the build takes everything newer from it. Debian's proposed -/// suites aren't NotAutomatic, so with only some packages from them the -/// rest is pinned down like Ubuntu's. -pub fn preferences_file(distro: &DistroVersion, extra: &Suites) -> Option { +/// apt preferences for the `archives` of [`sources_file`]: +/// - a package with a version gets it, even if older; +/// - one from an archive gets that archive's; +/// - Ubuntu marks `-proposed` NotAutomatic, so apt would only upgrade +/// installed packages from it: with all of it, it's pinned to the +/// release's priority like on Launchpad's builders, so the build takes +/// everything newer from it; +/// - with only some packages from it, the rest of a proposed suite only +/// upgrades installed packages, also on Debian, which doesn't mark them +/// NotAutomatic. +pub fn preferences_file(distro: &DistroVersion, archives: &Archives) -> Option { let mut pins = Vec::new(); + for (pattern, settings) in &archives.packages { + match (&settings.version, &settings.from) { + (Some(version), _) => pins.push(pin(pattern, &format!("version {version}"), 1001)), + (None, Some(from)) => { + for suite in archive_suites(distro, from) { + pins.push(pin(pattern, &release_pin(distro, &suite), 990)); + } + } + (None, None) => {} + } + } if let Some(suite) = proposed_suite(distro) { - // Debian's proposed-updates have a Suite like `proposed-updates` - let release = if suite.ends_with("-proposed-updates") { - format!("n={suite}") - } else { - format!("a={suite}") - }; - match &extra.proposed { - Selection::Off => {} - Selection::All if distro.distro == Distro::Ubuntu => { + let release = release_pin(distro, &suite); + if archives.proposed { + if distro.distro == Distro::Ubuntu { pins.push(pin("*", &release, 500)); } - Selection::All => {} - Selection::Only(packages) => { - pins.push(pin(&packages.join(" "), &release, 990)); - pins.push(pin("*", &release, 100)); - } + } else if archives.uses(PROPOSED) { + pins.push(pin("*", &release, 100)); } } - if let Selection::Only(packages) = &extra.experimental { - pins.push(pin(&packages.join(" "), &format!("a={EXPERIMENTAL}"), 990)); - } (!pins.is_empty()).then(|| pins.join("\n")) } -fn pin(packages: &str, release: &str, priority: u32) -> String { - format!("Package: {packages}\nPin: release {release}\nPin-Priority: {priority}\n") +fn archive_suites(distro: &DistroVersion, archive: &str) -> Vec { + match archive { + PROPOSED => proposed_suite(distro).into_iter().collect(), + EXPERIMENTAL => vec![EXPERIMENTAL.to_string()], + _ => release_suites(distro), + } +} + +/// The apt preferences pin of `suite`. Ubuntu's Release files name the +/// pocket in their Suite, while their Codename is the release's. Debian's +/// name it in their Codename, while their Suite is an alias like +/// `stable-updates`, except for unstable (`sid`) and experimental +/// (`rc-buggy`). +fn release_pin(distro: &DistroVersion, suite: &str) -> String { + if distro.distro == Distro::Ubuntu || DEBIAN_UNSTABLE.contains(&suite) { + format!("release a={suite}") + } else { + format!("release n={suite}") + } +} + +fn pin(packages: &str, pin: &str, priority: u32) -> String { + format!("Package: {packages}\nPin: {pin}\nPin-Priority: {priority}\n") } /// Whether the release's apt has `apt-get satisfy` (apt 1.9: Debian 11, @@ -329,18 +382,39 @@ mod tests { DistroVersion::new(distro, name, version) } - fn suites(distro: &DistroVersion, proposed: &str, experimental: &str) -> Suites { - let proposed = proposed.parse().unwrap(); - Suites::for_distro(distro, proposed, experimental.parse().unwrap()).unwrap() + /// The archives for `distro` with all of proposed if `proposed`, and + /// the package `specs`. + fn archives(distro: &DistroVersion, proposed: bool, specs: &[&str]) -> Archives { + let packages = specs.iter().map(|spec| parse_package(spec).unwrap()); + Archives::for_distro(distro, proposed, false, packages.collect()).unwrap() } - fn proposed() -> Suites { - Suites { - proposed: Selection::All, - ..Suites::default() + fn proposed() -> Archives { + Archives { + proposed: true, + ..Archives::default() } } + #[test_case("libfoo1/proposed", "libfoo1", None, Some("proposed") ; "archive")] + #[test_case("src:foo=1:2.0-1", "src:foo", Some("1:2.0-1"), None ; "version with epoch")] + #[test_case("libfoo1:amd64=2.0/release", "libfoo1:amd64", Some("2.0"), Some("release") ; "both")] + fn package_spec(spec: &str, pattern: &str, version: Option<&str>, from: Option<&str>) { + let settings = PackageSettings { + from: from.map(str::to_string), + version: version.map(str::to_string), + }; + assert_eq!(parse_package(spec), Ok((pattern.to_string(), settings))); + } + + #[test_case("libfoo1" ; "no settings")] + #[test_case("=1.0" ; "no pattern")] + #[test_case("libfoo1/" ; "empty archive")] + #[test_case("lib foo/proposed" ; "whitespace")] + fn package_spec_rejects(spec: &str) { + assert!(parse_package(spec).is_err()); + } + #[test_case("yes", Selection::All)] #[test_case("off", Selection::Off)] #[test_case("libfoo, src:bar", Selection::Only(vec!["libfoo".into(), "src:bar".into()]))] @@ -348,59 +422,49 @@ mod tests { assert_eq!(value.parse::(), Ok(expected)); } + #[test_case("")] + #[test_case("libfoo,")] + #[test_case("lib foo")] + fn selection_rejects_invalid_packages(value: &str) { + assert!(value.parse::().is_err()); + } + #[test_case("libfoo1:amd64", true ; "binary")] #[test_case("src:foo:any", true ; "source")] #[test_case("libfoo-dev", false ; "other binary")] #[test_case("src:food", false ; "other source")] - #[test_case("yes", false ; "all")] - fn selection_names_package(value: &str, expected: bool) { - let selection = value.parse::().unwrap(); - assert_eq!( - selection.names_package("foo", &["libfoo1".into()]), - expected - ); + fn test_names_package(pattern: &str, expected: bool) { + assert_eq!(names_package(pattern, "foo", &["libfoo1".into()]), expected); } #[test] - fn with_source_adds_all_binaries_once() { - let selection = Selection::Only(vec!["libfoo1".into()]).with_source("foo"); + fn settings_in_config_reject_unknown_keys() { + let parse = |json| serde_json::from_str::(json); assert_eq!( - selection.clone().with_source("foo"), - Selection::Only(vec!["libfoo1".into(), "src:foo".into()]) + parse(r#"{"from": "proposed"}"#).unwrap().from.as_deref(), + Some("proposed") ); - assert_eq!(Selection::All.with_source("foo"), Selection::All); + assert!(parse(r#"{"form": "proposed"}"#).is_err()); } - #[test_case("")] - #[test_case("libfoo,")] - #[test_case("lib foo")] - fn selection_rejects_invalid_packages(value: &str) { - assert!(value.parse::().is_err()); + #[test_case("noble", "src:foo/experimental" ; "ubuntu")] + #[test_case("trixie", "src:foo/experimental" ; "debian stable")] + #[test_case("unstable", "src:foo/proposed" ; "proposed of unstable")] + #[test_case("noble", "src:foo/backports" ; "unknown archive")] + fn archives_reject(name: &str, spec: &str) { + let packages = [parse_package(spec).unwrap()].into(); + assert!(Archives::for_distro(&distro(name), false, false, packages).is_err()); } #[test] - fn selection_in_config_is_bool_or_packages() { - let parse = |json| serde_json::from_str::(json).unwrap(); - assert_eq!(parse("true"), Selection::All); - assert_eq!(parse("[]"), Selection::Off); - assert_eq!( - parse(r#"["libfoo"]"#), - Selection::Only(vec!["libfoo".into()]) - ); - assert!(serde_json::from_str::(r#"["a b"]"#).is_err()); - } - - #[test_case("noble")] - #[test_case("trixie")] - fn experimental_only_adds_to_debian_unstable_and_testing(name: &str) { - let error = Suites::for_distro(&distro(name), Selection::Off, Selection::All); - assert!(error.is_err()); + fn experimental_only_adds_to_debian_unstable_and_testing() { + assert!(Archives::for_distro(&distro("noble"), false, true, Packages::new()).is_err()); } #[test] fn experimental_release_is_all_of_experimental() { - let experimental = suites(&distro("experimental"), "no", "no"); - assert_eq!(experimental.experimental, Selection::All); + let experimental = archives(&distro("experimental"), false, &[]); + assert!(experimental.experimental); assert_eq!(retry_suite(&experimental), Some("experimental")); assert_eq!( preferences_file(&distro("experimental"), &experimental), @@ -408,6 +472,22 @@ mod tests { ); } + #[test] + fn ubuntu_package_from_the_release_is_pinned_to_all_its_pockets() { + let noble = distro("noble"); + let extra = archives(&noble, true, &["src:foo/release", "libbar1=2.0"]); + assert_eq!( + preferences_file(&noble, &extra).unwrap(), + "Package: libbar1\nPin: version 2.0\nPin-Priority: 1001\n\n\ + Package: src:foo\nPin: release a=noble\nPin-Priority: 990\n\n\ + Package: src:foo\nPin: release a=noble-updates\nPin-Priority: 990\n\n\ + Package: src:foo\nPin: release a=noble-security\nPin-Priority: 990\n\n\ + Package: *\nPin: release a=noble-proposed\nPin-Priority: 500\n" + ); + let release_only = archives(&noble, false, &["src:foo/release"]); + assert!(!release_only.adds_suites()); + } + #[test] fn debian_stable_keeps_security_on_the_debian_archive() { let trixie = distro("trixie"); @@ -430,7 +510,7 @@ mod tests { let extra = if with_proposed { proposed() } else { - Suites::default() + Archives::default() }; let sources = sources_file(&distro("unstable"), None, &extra, "arm64").unwrap(); assert_eq!( @@ -452,7 +532,7 @@ mod tests { assert!(sources.contains("Suites: forky-security\n"), "{sources}"); assert_eq!(preferences_file(&forky, &proposed()), None); - let extra = suites(&forky, "libfoo", "src:bar"); + let extra = archives(&forky, false, &["libfoo/proposed", "src:bar/experimental"]); let sources = sources_file(&forky, None, &extra, "amd64").unwrap(); assert!( sources.contains("Suites: forky forky-updates unstable experimental\n"), @@ -461,8 +541,8 @@ mod tests { assert_eq!( preferences_file(&forky, &extra).unwrap(), "Package: libfoo\nPin: release a=unstable\nPin-Priority: 990\n\n\ - Package: *\nPin: release a=unstable\nPin-Priority: 100\n\n\ - Package: src:bar\nPin: release a=experimental\nPin-Priority: 990\n" + Package: src:bar\nPin: release a=experimental\nPin-Priority: 990\n\n\ + Package: *\nPin: release a=unstable\nPin-Priority: 100\n" ); assert_eq!(retry_suite(&extra), None); } @@ -471,7 +551,7 @@ mod tests { fn debian_proposed_updates_is_pinned_by_codename() { let trixie = distro("trixie"); assert_eq!( - preferences_file(&trixie, &suites(&trixie, "libfoo", "no")).unwrap(), + preferences_file(&trixie, &archives(&trixie, false, &["libfoo/proposed"])).unwrap(), "Package: libfoo\nPin: release n=trixie-proposed-updates\nPin-Priority: 990\n\n\ Package: *\nPin: release n=trixie-proposed-updates\nPin-Priority: 100\n" ); @@ -484,7 +564,7 @@ mod tests { let sources = sources_file( &experimental, None, - &suites(&experimental, "yes", "no"), + &archives(&experimental, true, &[]), "amd64", ); assert!( @@ -495,7 +575,7 @@ mod tests { "{sources:?}" ); let unstable = distro("unstable"); - let extra = suites(&unstable, "no", "yes"); + let extra = Archives::for_distro(&unstable, false, true, Packages::new()).unwrap(); let sources = sources_file(&unstable, None, &extra, "amd64").unwrap(); assert!( sources.contains("Suites: unstable experimental\n"), @@ -511,12 +591,12 @@ mod tests { preferences_file(&noble, &proposed()).unwrap(), "Package: *\nPin: release a=noble-proposed\nPin-Priority: 500\n" ); - assert_eq!(preferences_file(&noble, &Suites::default()), None); + assert_eq!(preferences_file(&noble, &Archives::default()), None); } #[test] fn old_debian_has_no_firmware_component() { - let extra = Suites::default(); + let extra = Archives::default(); let sources = sources_file(&distro("bullseye"), None, &extra, "amd64").unwrap(); assert!( sources.contains("Components: main contrib non-free\n"), @@ -541,7 +621,7 @@ mod tests { #[test] fn custom_distro_has_no_known_archive() { let custom = DistroVersion::custom(Distro::Custom("yocto".into()), "scarthgap"); - assert!(sources_file(&custom, Some("http://m"), &Suites::default(), "amd64").is_err()); + assert!(sources_file(&custom, Some("http://m"), &Archives::default(), "amd64").is_err()); } #[test_case("buster", false)] diff --git a/packages/debmagic/src/apt.rs b/packages/debmagic/src/apt.rs index 0463b91e..f3f010ba 100644 --- a/packages/debmagic/src/apt.rs +++ b/packages/debmagic/src/apt.rs @@ -6,7 +6,7 @@ use std::{fs, io, os::unix::fs::PermissionsExt, path::Path}; use anyhow::Context; use debmagic_common::{ - apt::{self, Selection, Suites}, + apt::{self, Archives, PackageSettings, Packages}, debian::control::BuildRelations, distro::DistroVersion, }; @@ -24,18 +24,24 @@ pub struct AptConfig { pub mirror: Option, /// When a reused Environment runs apt-get update/upgrade again. pub update_age: RefreshPolicy, - /// Packages from the suite the release's updates migrate from: `true` - /// for all, or a list. - pub proposed: Selection, - /// Packages from Debian experimental, for unstable and testing builds. - pub experimental: Selection, + /// All newer packages from the suite the release's updates migrate + /// from. + pub proposed: bool, + /// What nothing else provides from Debian experimental, for unstable + /// and testing builds. + pub experimental: bool, + /// Where to take packages from, or which version, by apt preferences + /// pattern. + pub packages: Packages, } impl AptConfig { - /// The suites beyond its release an Environment for `distro` takes + /// The archives beyond its release an Environment for `distro` takes /// packages from. - pub fn suites(&self, distro: &DistroVersion) -> anyhow::Result { - Suites::for_distro(distro, self.proposed.clone(), self.experimental.clone()) + pub fn archives(&self, distro: &DistroVersion) -> anyhow::Result { + let packages = self.packages.clone(); + Archives::for_distro(distro, self.proposed, self.experimental, packages) + .context("invalid apt package settings") } } @@ -44,8 +50,10 @@ impl AptConfig { pub struct AptOverrides { pub mirror: Option, pub update_age: Option, - pub proposed: Option, - pub experimental: Option, + pub proposed: Option, + pub experimental: Option, + /// Replacing the configured settings of the same pattern. + pub packages: Vec<(String, PackageSettings)>, } impl AptOverrides { @@ -62,6 +70,7 @@ impl AptOverrides { if let Some(experimental) = self.experimental { config.experimental = experimental; } + config.packages.extend(self.packages); } } @@ -111,14 +120,14 @@ const DEFAULT_SOURCES: &[&str] = &[ pub struct Setup<'a> { pub distro: &'a DistroVersion, pub mirror: Option<&'a str>, - pub suites: &'a Suites, + pub archives: &'a Archives, /// Let installed packages start their services, which tests may expect. pub services: bool, } impl Setup<'_> { fn replaces_sources(&self) -> bool { - self.mirror.is_some() || *self.suites != Suites::default() + self.mirror.is_some() || self.archives.adds_suites() } /// The setup files apart from the apt sources. @@ -137,7 +146,7 @@ impl Setup<'_> { .chain([ self.script(""), self.mirror.unwrap_or_default().to_string(), - format!("{:?}", self.suites), + format!("{:?}", self.archives), ]) .collect::>() .join("\0") @@ -147,14 +156,14 @@ impl Setup<'_> { pub fn write(&self, dir: &Path) -> anyhow::Result<()> { let sources = self .replaces_sources() - .then(|| apt::sources_file(self.distro, self.mirror, self.suites, host_dpkg_arch())) + .then(|| apt::sources_file(self.distro, self.mirror, self.archives, host_dpkg_arch())) .transpose() .context("generating the apt sources failed")?; if dir.exists() { fs::remove_dir_all(dir) .with_context(|| format!("removing the old {} failed", dir.display()))?; } - let preferences = apt::preferences_file(self.distro, self.suites); + let preferences = apt::preferences_file(self.distro, self.archives); let mut files: Vec<(&str, &str, u32)> = self.files().collect(); for (path, content) in [(SOURCES_FILE, &sources), (PREFERENCES_FILE, &preferences)] { files.extend(content.as_deref().map(|content| (path, content, 0o644))); @@ -262,19 +271,19 @@ pub fn install_commands( } /// Install `install`, with all of a NotAutomatic suite like experimental -/// in `suites` retried preferring that suite: apt doesn't take a version +/// in `archives` retried preferring that suite: apt doesn't take a version /// only that suite has otherwise, while preferring it upfront would take /// everything it has. pub fn install( driver: &dyn EnvironmentDriver, cwd: &Path, distro: &DistroVersion, - suites: &Suites, + archives: &Archives, install: &Install, ) -> io::Result<()> { let satisfy = apt::has_satisfy(distro); let result = run(driver, cwd, &install_commands(satisfy, install, None)); - match apt::retry_suite(suites) { + match apt::retry_suite(archives) { Some(suite) if result.is_err() => { println!("debmagic: retrying preferring the packages from {suite}"); run( @@ -394,7 +403,7 @@ mod tests { let mut setup = Setup { distro: &trixie, mirror: None, - suites: &Suites::default(), + archives: &Archives::default(), services: false, }; setup.write(&dir)?; diff --git a/packages/debmagic/src/build/mod.rs b/packages/debmagic/src/build/mod.rs index 181fe94e..09f62562 100644 --- a/packages/debmagic/src/build/mod.rs +++ b/packages/debmagic/src/build/mod.rs @@ -331,13 +331,13 @@ fn install_packages( profiles: profiles.to_vec(), }; let environment = claimed.environment(); - let suites = intent.config.apt.suites(&environment.distro)?; + let archives = intent.config.apt.archives(&environment.distro)?; let staged_source_dir = stage_dir(environment, kind); apt::install( claimed.driver(), &staged_source_dir, &environment.distro, - &suites, + &archives, &install, ) .context("installing the build dependencies failed") @@ -562,7 +562,7 @@ pub async fn build(command: BuildCommand) -> anyhow::Result<()> { ), ) .context("failed to determine package target")?; - intent.config.apt.suites(&target.distro)?; + intent.config.apt.archives(&target.distro)?; let package = format!("{} {}", target.package.name(), target.package.version()); let notify = intent.config.build_notify(); diff --git a/packages/debmagic/src/cli.rs b/packages/debmagic/src/cli.rs index 43b47c92..92061299 100644 --- a/packages/debmagic/src/cli.rs +++ b/packages/debmagic/src/cli.rs @@ -14,7 +14,7 @@ use crate::upload::orig::IncludeOrig; use crate::upstream::fetch::FetchFrom; use clap::builder::BoolishValueParser; use clap::{Args, Parser, Subcommand}; -use debmagic_common::apt::Selection; +use debmagic_common::apt::{PackageSettings, Selection, parse_package}; /// When to use colored output. Mirrors common CLI conventions; `auto` is the /// default and respects the `NO_COLOR` environment variable. @@ -333,7 +333,7 @@ pub struct EnvironmentArgs { default_missing_value = "true", action = clap::ArgAction::Set, value_name = "PACKAGES", - help = "Take packages from the suite the release's updates migrate from (e.g. '-proposed'): all newer ones, or only the comma-separated PACKAGES (binary names, 'src:' or globs). 'no' disables it. Ignored by the bare driver." + help = "Take packages from the suite the release's updates migrate from (e.g. '-proposed'): all newer ones, or only the comma-separated PACKAGES (binary names, 'src:' or globs), like --package /proposed. 'no' disables it. Ignored by the bare driver." )] pub proposed: Option, @@ -343,10 +343,18 @@ pub struct EnvironmentArgs { default_missing_value = "true", action = clap::ArgAction::Set, value_name = "PACKAGES", - help = "Take packages from Debian experimental for an unstable or testing environment: those nothing else provides, or only the comma-separated PACKAGES. 'no' disables it. Ignored by the bare driver." + help = "Take packages from Debian experimental for an unstable or testing environment: those nothing else provides, or only the comma-separated PACKAGES, like --package /experimental. 'no' disables it. Ignored by the bare driver." )] pub experimental: Option, + #[arg( + long = "package", + value_name = "PACKAGE[=VERSION][/ARCHIVE]", + value_parser = parse_package, + help = "Install PACKAGE (a binary name, 'src:' for all its binaries, or a glob) in VERSION, even if older, or from ARCHIVE: 'release' (with its updates), 'proposed' or 'experimental'. Naming the tested package makes `debmagic test` test that build. Repeatable. Ignored by the bare driver." + )] + pub packages: Vec<(String, PackageSettings)>, + #[arg( long = "apt-update-age", help = "When a persistent environment runs 'apt-get update/upgrade' again: 'now' (every run), 'never' (only on first creation), or a maximum age of the last upgrade like '1d' (the default), '12h', '30m'. Defaults to the 'apt_update_age' setting in the config file. Ignored by the bare driver." @@ -747,11 +755,32 @@ impl EnvironmentArgs { /// Collect the driver-specific overrides from the CLI flags. /// Collect the `[apt]` overrides from the CLI flags. pub fn apt_overrides(&self) -> AptOverrides { + let mut packages = Vec::new(); + let mut whole = |selection: &Option, archive: &str| match selection { + None => None, + Some(Selection::All) => Some(true), + Some(Selection::Off) => Some(false), + Some(Selection::Only(patterns)) => { + packages.extend(patterns.iter().map(|pattern| { + let from = Some(archive.to_string()); + let settings = PackageSettings { + from, + version: None, + }; + (pattern.clone(), settings) + })); + None + } + }; + let proposed = whole(&self.proposed, "proposed"); + let experimental = whole(&self.experimental, "experimental"); + packages.extend(self.packages.iter().cloned()); AptOverrides { mirror: self.apt_mirror.clone(), update_age: self.apt_update_age, - proposed: self.proposed.clone(), - experimental: self.experimental.clone(), + proposed, + experimental, + packages, } } @@ -802,6 +831,31 @@ mod tests { assert_eq!(parse_test(args).environment.proposed, expected); } + #[test] + fn package_lists_become_package_settings() { + let args = parse_test(&[ + "--proposed=libfoo1", + "--experimental", + "--package", + "src:bar=1.0/release", + ]); + let overrides = args.environment.apt_overrides(); + assert_eq!(overrides.proposed, None); + assert_eq!(overrides.experimental, Some(true)); + let packages: Vec<_> = overrides + .packages + .iter() + .map(|(p, s)| (p.as_str(), s)) + .collect(); + assert_eq!( + packages, + [ + ("libfoo1", &parse_package("x/proposed").unwrap().1), + ("src:bar", &parse_package("x=1.0/release").unwrap().1), + ] + ); + } + #[test] fn experimental_flag_is_a_selection() { let args = parse_test(&["--experimental"]).environment; diff --git a/packages/debmagic/src/config.rs b/packages/debmagic/src/config.rs index df330e72..5fbe0fe8 100644 --- a/packages/debmagic/src/config.rs +++ b/packages/debmagic/src/config.rs @@ -615,12 +615,14 @@ mod tests { ConfigPathStatus::Used, )]) }; - let cfg = load("[apt]\nmirror = \"http://m/ubuntu\"\nproposed = [\"libfoo1\"]\n")?; + let cfg = load( + "[apt]\nmirror = \"http://m/ubuntu\"\n\ + [apt.packages]\n\"src:foo\" = { from = \"proposed\", version = \"1.0\" }\n", + )?; assert_eq!(cfg.apt.mirror.as_deref(), Some("http://m/ubuntu")); - assert_eq!( - cfg.apt.proposed, - debmagic_common::apt::Selection::Only(vec!["libfoo1".into()]) - ); + let settings = &cfg.apt.packages["src:foo"]; + assert_eq!(settings.from.as_deref(), Some("proposed")); + assert_eq!(settings.version.as_deref(), Some("1.0")); let error = load("[driver]\nproposed = true\n").unwrap_err(); std::fs::remove_dir_all(&dir).ok(); assert!(error.to_string().contains("'apt.proposed'"), "{error}"); diff --git a/packages/debmagic/src/driver/driver_docker.rs b/packages/debmagic/src/driver/driver_docker.rs index 83faf8a5..1504e119 100644 --- a/packages/debmagic/src/driver/driver_docker.rs +++ b/packages/debmagic/src/driver/driver_docker.rs @@ -7,7 +7,7 @@ use std::{ }; use anyhow::{Context, anyhow}; -use debmagic_common::{apt::Suites, distro::DistroVersion}; +use debmagic_common::{apt::Archives, distro::DistroVersion}; use serde::{Deserialize, Serialize}; use crate::apt; @@ -246,7 +246,7 @@ impl DriverDocker { driver_config: &DriverConfig, overrides: &DriverDockerConfigOverrides, apt_mirror: Option<&str>, - suites: &Suites, + archives: &Archives, ) -> anyhow::Result { let base_image = overrides.base_image.clone().unwrap_or_else(|| { driver_config @@ -259,7 +259,7 @@ impl DriverDocker { let setup = apt::Setup { distro: &environment.distro, mirror: apt_mirror, - suites, + archives, services: environment.purpose == EnvironmentPurpose::Test, }; let dockerfile = dockerfile(environment, &base_image, &setup); diff --git a/packages/debmagic/src/driver/driver_lxd.rs b/packages/debmagic/src/driver/driver_lxd.rs index a89e7cef..602bb1e1 100644 --- a/packages/debmagic/src/driver/driver_lxd.rs +++ b/packages/debmagic/src/driver/driver_lxd.rs @@ -5,7 +5,7 @@ use std::{ }; use debmagic_common::{ - apt::Suites, + apt::Archives, distro::{Distro, DistroVersion}, }; use serde::{Deserialize, Serialize}; @@ -292,7 +292,7 @@ impl DriverLxd { driver_config: &DriverConfig, overrides: &DriverLxdConfigOverrides, apt_mirror: Option<&str>, - suites: &Suites, + archives: &Archives, ) -> anyhow::Result { let container_name = Self::container_name_for(environment); let project = Self::resolve_project(driver_config, overrides); @@ -307,7 +307,7 @@ impl DriverLxd { let setup = apt::Setup { distro: &environment.distro, mirror: apt_mirror, - suites, + archives, services: environment.purpose == EnvironmentPurpose::Test, }; let setup_fingerprint = setup.fingerprint(); diff --git a/packages/debmagic/src/driver/mod.rs b/packages/debmagic/src/driver/mod.rs index 1aadeb62..f58d41dd 100644 --- a/packages/debmagic/src/driver/mod.rs +++ b/packages/debmagic/src/driver/mod.rs @@ -491,7 +491,7 @@ pub fn create_driver( overrides: &DriverOverrides, ) -> anyhow::Result { let apt_mirror = apt.mirror.as_deref(); - let suites = apt.suites(&environment.distro)?; + let archives = apt.archives(&environment.distro)?; match environment.driver { DriverType::Docker => Ok(Driver::Docker(DriverDocker::create( @@ -499,7 +499,7 @@ pub fn create_driver( driver_config, &overrides.docker, apt_mirror, - &suites, + &archives, )?)), DriverType::Bare => Ok(Driver::Bare(DriverBare::create( environment, @@ -517,7 +517,7 @@ pub fn create_driver( driver_config, &overrides.lxd, apt_mirror, - &suites, + &archives, )?)) } } diff --git a/packages/debmagic/src/test/run.rs b/packages/debmagic/src/test/run.rs index c1542250..90303cf5 100644 --- a/packages/debmagic/src/test/run.rs +++ b/packages/debmagic/src/test/run.rs @@ -19,6 +19,7 @@ use crate::package::load_package; use crate::subprocess::Capture; use anyhow::{Context, anyhow, bail}; use debian_control::lossless::changes::Changes; +use debmagic_common::apt::names_package; use debmagic_common::distro::{DistroTable, DistroVersion}; use debmagic_common::package::SourcePackage; @@ -207,9 +208,9 @@ fn autopkgtest_isolation_args(isolation: IsolationCapability) -> Vec<&'static st } /// The apt config to test the archive's build of `package` instead of a -/// local one, `None` to test a local build: when the proposed or -/// experimental selection names it or one of its binaries, with all its -/// binaries from that suite so their versions match. +/// local one, `None` to test a local build: when the package settings name +/// it or one of its binaries, with the same settings for all its binaries +/// so their versions match. fn archive_build_apt( package: &SourcePackage, apt: &AptConfig, @@ -219,15 +220,20 @@ fn archive_build_apt( .context("reading the binary packages failed")? .binaries; let name = package.name(); + let Some(settings) = apt + .packages + .iter() + .find(|(pattern, _)| names_package(pattern, name, binaries)) + .map(|(_, settings)| settings.clone()) + else { + return Ok(None); + }; let mut archive = apt.clone(); - let mut named = false; - for selection in [&mut archive.proposed, &mut archive.experimental] { - if selection.names_package(name, binaries) { - *selection = std::mem::take(selection).with_source(name); - named = true; - } - } - Ok(named.then_some(archive)) + archive + .packages + .entry(format!("src:{name}")) + .or_insert(settings); + Ok(Some(archive)) } pub async fn run_test(intent: &TestIntent) -> anyhow::Result { @@ -241,8 +247,8 @@ pub async fn run_test(intent: &TestIntent) -> anyhow::Result { let apt = archive_apt.as_ref().unwrap_or(&intent.config.apt); if from_archive && intent.changes.is_some() { bail!( - "--changes tests a local build, but the proposed or experimental packages name {} \ - to test the archive's", + "--changes tests a local build, but the package settings name {} to test the \ + archive's", package.name() ); } @@ -339,7 +345,7 @@ pub async fn run_test(intent: &TestIntent) -> anyhow::Result { "no prior build metadata found; pass --distro when using --changes without a prior Invocation" ); }; - apt.suites(&distro)?; + apt.archives(&distro)?; let environment = Environment::new( driver, @@ -386,12 +392,12 @@ pub async fn run_test(intent: &TestIntent) -> anyhow::Result { ..Default::default() }; let distro = &claimed.environment().distro; - let suites = apt.suites(distro)?; + let archives = apt.archives(distro)?; crate::apt::install( claimed.driver(), &staged_source_dir, distro, - &suites, + &archives, &install, ) .context("installing autopkgtest failed")?; @@ -491,7 +497,7 @@ mod tests { #[test] fn naming_the_package_tests_the_archive_build_of_all_its_binaries() -> anyhow::Result<()> { - use debmagic_common::apt::Selection; + use debmagic_common::apt::parse_package; let package = SourcePackage::from_files([ ( @@ -503,19 +509,19 @@ mod tests { "Source: foo\n\nPackage: libfoo1\n\nPackage: foo-bin\n", ), ])?; - let only = |packages: &[&str]| { - Selection::Only(packages.iter().map(|package| package.to_string()).collect()) - }; - let mut apt = AptConfig { - proposed: only(&["libfoo1", "libbar1"]), + let apt = |specs: &[&str]| AptConfig { + packages: specs + .iter() + .map(|spec| parse_package(spec).unwrap()) + .collect(), ..AptConfig::default() }; - let archive = archive_build_apt(&package, &apt)?.unwrap(); - assert_eq!(archive.proposed, only(&["libfoo1", "libbar1", "src:foo"])); - assert_eq!(archive.experimental, Selection::Off); - - apt.proposed = Selection::All; - assert!(archive_build_apt(&package, &apt)?.is_none()); + let archive = archive_build_apt(&package, &apt(&["libbar1/proposed", "libfoo1=2.0"]))?; + assert_eq!( + archive.unwrap().packages, + apt(&["libbar1/proposed", "libfoo1=2.0", "src:foo=2.0"]).packages + ); + assert!(archive_build_apt(&package, &apt(&["libbar1/proposed"]))?.is_none()); Ok(()) } From 43aa859312e87ec1c24a3f037555187e6d74f20b Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Fri, 23 Oct 2026 18:01:36 +0200 Subject: [PATCH 44/48] feat(build): add PPAs with `--archive` `--archive [=]ppa:/[@]` adds a Launchpad PPA to the environment, also as `[apt.archives]` in the config. Without packages picked from it with `--package /`, apt takes all newer ones from it. `--package libfoo1/ppa:me/stuff` adds the PPA itself. Its signing key is looked up on Launchpad, fetched from keyserver.ubuntu.com and checked against the fingerprint before anything is built. Debian targets need the `@`. --- docs/usage/build.md | 27 ++- docs/usage/config.md | 2 + docs/usage/test.md | 1 + packages/debmagic-common/src/apt.rs | 291 +++++++++++++++++++++-- packages/debmagic/src/apt.rs | 153 +++++++++++- packages/debmagic/src/build/mod.rs | 40 +++- packages/debmagic/src/cli.rs | 9 + packages/debmagic/src/config.rs | 3 + packages/debmagic/src/driver/mod.rs | 10 +- packages/debmagic/src/environment/mod.rs | 21 +- packages/debmagic/src/sign.rs | 153 ++++++++++-- packages/debmagic/src/test/run.rs | 11 +- 12 files changed, 635 insertions(+), 86 deletions(-) diff --git a/docs/usage/build.md b/docs/usage/build.md index 0a56375f..d29c33e6 100644 --- a/docs/usage/build.md +++ b/docs/usage/build.md @@ -28,6 +28,7 @@ debmagic build binary --driver lxd \ | `--proposed[=]` | Use [build dependencies from `proposed`](#proposed-dependencies) | | `--experimental[=]` | Use build dependencies from [experimental](#experimental) | | `--package [=][/]` | Install a [package in a version or from an archive](#package-versions-and-archives) | +| `--archive [=]` | Add an [archive like a PPA](#extra-archives) | | `--sign` | [GPG-sign the resulting `.changes`/`.dsc`/`.buildinfo`](#signing) | | `--clean` | Run [`debian/rules clean` before building](#cleaning) | | `--debug-symbols` | [Build the automatic `-dbgsym` debug symbol packages](#building-debug-symbol-packages) | @@ -188,7 +189,7 @@ debmagic build --package libbaz1=1.2-3/proposed ``` The package is an [apt preferences](https://manpages.debian.org/apt_preferences) pattern: a binary package, `src:` for all binaries of a source, or a glob, each with an optional `:`. -The archive is `release` (with its updates), `proposed` or `experimental`; an archive packages come from is added to the environment. +The archive is `release` (with its updates), `proposed`, `experimental` or an [extra archive](#extra-archives); an archive packages come from is added to the environment. A version is installed even if older than the installed one, from whichever archive has it. The [config](config.md) takes the same as a table: @@ -199,6 +200,30 @@ libfoo1 = { from = "proposed" } "src:bar" = { version = "2.0-1" } ``` +## Extra archives + +`--archive` adds an archive beyond the distro's, for now a Launchpad PPA: + +```shell +debmagic build --archive ppa:me/stuff # all newer packages from it +debmagic build --package libfoo1/ppa:me/stuff # only libfoo1 from it +debmagic build --archive mine=ppa:me/stuff --package libfoo1/mine +``` + +An archive is named after its spec unless `=` names it; `--package` takes packages from it by that name. +Once packages are picked from an archive, its other packages only upgrade installed ones. + +A PPA is taken for the target Ubuntu release; `ppa:/@` picks another, which Debian targets need. +Its signing key is looked up on Launchpad and fetched from keyserver.ubuntu.com before the build starts; if either fails, nothing is built. +The key is then kept in `~/.cache/debmagic/ppa-keys//.asc` and not fetched again; delete it when the PPA's key changes. + +In the [config](config.md): + +```toml +[apt.archives] +mine = "ppa:me/stuff" +``` + ## Building debug symbol packages By default `debmagic build binary` passes `DEB_BUILD_OPTIONS=noautodbgsym` to `dpkg-buildpackage`, which suppresses debhelper's automatic `-dbgsym` package (the detached debug info package debhelper otherwise builds by default from compat 9 onward). diff --git a/docs/usage/config.md b/docs/usage/config.md index e0e01007..086e7072 100644 --- a/docs/usage/config.md +++ b/docs/usage/config.md @@ -31,6 +31,7 @@ All keys are optional. | `apt.update_age` | string | `"1d"` | `--apt-update-age` | When a persistent environment re-runs `apt-get update && upgrade`: `"now"` (every build), `"never"` (only on creation), or a maximum age like `"1d"`, `"12h"`, `"30m"`. | | `apt.proposed` | bool | `false` | `--proposed` | All newer packages from the [proposed suite](build.md#proposed-dependencies). | | `apt.experimental` | bool | `false` | `--experimental` | What nothing else provides from [experimental](build.md#experimental), for unstable and testing. | +| `apt.archives` | map | — | `--archive` | [Extra archives](build.md#extra-archives) by name, e.g. `{ mine = "ppa:me/stuff" }`. | | `apt.packages` | map | — | `--package` | [Package settings](build.md#package-versions-and-archives) by pattern: `from` (an archive) and `version`, e.g. `{ "src:bar" = { from = "proposed" } }`. | | `environments_dir` | path | `$XDG_DATA_HOME/debmagic/environments` | — | Parent directory of Environment Host roots. Must be absolute or start with `~/`. The Environment registry is always `$XDG_DATA_HOME/debmagic/db.sqlite` (override both with `DEBMAGIC_DATA_DIR`). | | `output_dir` | path | `build` | `-o`/`--output-dir` | Where build artifacts are exported; relative paths resolve against the package root. See [Output dir](build.md#output-dir). | @@ -75,6 +76,7 @@ All keys are optional. | `sign_command` | string | — | `--sign-command` | Custom signing command for `tool = "custom"`, like `debsign`'s `-p`. | | `verify_command` | string | — | `--verify-command` (upstream switch) | Custom verification command for `sign.tool = "custom"`, run without a shell with `{file}`/`{signature}`/`{keyring}` placeholders; without `{signature}` the signature path is appended. Falls back to `sign_command`. | | `signed_by_command` | string | — | — | Custom same-key check for `tool = "custom"` under `source = "auto"`: exit code 0 means the file's signature is ours (skip), anything else re-signs. Without it, `auto` always re-signs. | +| `fingerprint_command` | string | — | — | Custom key fingerprint reader for `tool = "custom"`, run without a shell with the `{file}` placeholder (or the key path appended): prints the primary key's fingerprint. Checks the signing keys of [PPAs](build.md#extra-archives). | Signing always runs on the host with your gpg keyring — see [Signing](build.md#signing). ### `orig_tarball` diff --git a/docs/usage/test.md b/docs/usage/test.md index 49c49f4c..7d2346dc 100644 --- a/docs/usage/test.md +++ b/docs/usage/test.md @@ -47,6 +47,7 @@ The distro comes from `--distro` or the changelog, and the test output goes to ` | `--proposed[=]` | Test against packages from the [proposed suite](build.md#proposed-dependencies): all newer ones, or only the listed ones. Naming this package [tests the archive's build](#testing-the-archives-build) | | `--experimental[=]` | Test against the listed packages from [experimental](build.md#experimental); without a list, only what nothing else provides. Naming this package [tests the archive's build](#testing-the-archives-build) | | `--package [=][/]` | Test against a [package in a version or from an archive](build.md#package-versions-and-archives). Naming this package [tests the archive's build](#testing-the-archives-build) | +| `--archive [=]` | Add an [archive like a PPA](build.md#extra-archives), e.g. to test a PPA's build with `--package src:/` | | `--apt-mirror ` | Mirror URL (same as [`debmagic build`](build.md)) | | `--apt-update-age ` | Configure when to run `apt-get update` (same as [`debmagic build`](build.md#apt-update-age)) | | `--source-dir ` | Directory containing the `debian/` package directory | diff --git a/packages/debmagic-common/src/apt.rs b/packages/debmagic-common/src/apt.rs index 2eb4e9b3..c657ef9e 100644 --- a/packages/debmagic-common/src/apt.rs +++ b/packages/debmagic-common/src/apt.rs @@ -117,6 +117,113 @@ impl FromStr for Selection { } } +/// A Launchpad PPA, `ppa:/[@]`: the suite is the +/// Ubuntu release to take its packages for, by default the target's. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Ppa { + pub owner: String, + pub name: String, + pub suite: Option, +} + +impl FromStr for Ppa { + type Err = String; + + fn from_str(spec: &str) -> Result { + let invalid = || format!("'{spec}' is no ppa:/[@]"); + let rest = spec.strip_prefix("ppa:").ok_or_else(invalid)?; + let (path, suite) = match rest.split_once('@') { + Some((path, suite)) => (path, Some(suite)), + None => (rest, None), + }; + let valid = |part: &str| { + !part.is_empty() + && part + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"+-.".contains(&b)) + }; + match path.split_once('/') { + Some((owner, name)) if valid(owner) && valid(name) && suite.is_none_or(valid) => { + Ok(Self { + owner: owner.to_string(), + name: name.to_string(), + suite: suite.map(str::to_string), + }) + } + _ => Err(invalid()), + } + } +} + +impl Ppa { + /// Its Launchpad API resource, which names its signing key. + pub fn api_url(&self) -> String { + format!( + "https://api.launchpad.net/devel/~{}/+archive/ubuntu/{}", + self.owner, self.name + ) + } + + /// It as the archive `name` of a build for `distro`, signed with the + /// ASCII-armored `key`. + pub fn archive( + &self, + name: &str, + distro: &DistroVersion, + key: String, + ) -> anyhow::Result { + let (owner, ppa) = (&self.owner, &self.name); + let suite = match (&self.suite, &distro.distro) { + (Some(suite), _) => suite.clone(), + (None, Distro::Ubuntu) => distro.codename.clone(), + (None, other) => bail!( + "PPAs are built for Ubuntu releases: pick one for {other} {} with \ + ppa:{owner}/{ppa}@", + distro.codename + ), + }; + // Launchpad leaves the name out of the Origin of a PPA named `ppa` + let origin = match ppa.as_str() { + "ppa" => format!("LP-PPA-{owner}"), + _ => format!("LP-PPA-{owner}-{ppa}"), + }; + Ok(ExtraArchive { + name: name.to_string(), + uri: format!("https://ppa.launchpadcontent.net/{owner}/{ppa}/ubuntu"), + suite, + components: vec!["main".to_string()], + key, + pin: format!("release o={origin}"), + }) + } +} + +/// An archive beyond the distro's, with what apt needs to use it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ExtraArchive { + /// What `--package /` calls it. + pub name: String, + pub uri: String, + pub suite: String, + pub components: Vec, + /// The ASCII-armored key its Release file is signed with. + pub key: String, + /// The apt preferences `Pin` of its packages. + pub pin: String, +} + +impl ExtraArchive { + /// Where its key goes, relative to the environment's root. + pub fn key_path(&self) -> String { + let name: String = self + .name + .chars() + .map(|c| if c.is_ascii_alphanumeric() { c } else { '-' }) + .collect(); + format!("etc/apt/keyrings/debmagic-{name}.asc") + } +} + /// The archives beyond its release a build environment takes packages /// from, and how it installs some packages. #[derive(Clone, Debug, Default, PartialEq, Eq)] @@ -126,6 +233,8 @@ pub struct Archives { /// All of experimental, for what nothing else provides. pub experimental: bool, pub packages: Packages, + /// Taken all newer packages from, unless some are picked from them. + pub extra: Vec, } impl Archives { @@ -136,18 +245,39 @@ impl Archives { proposed: bool, experimental: bool, packages: Packages, + extra: Vec, ) -> anyhow::Result { let archives = Self { proposed, experimental: experimental || is_experimental(distro), packages, + extra, }; + let mut key_paths = Vec::new(); + for archive in &archives.extra { + if ARCHIVES.contains(&archive.name.as_str()) { + bail!("'{}' names a built-in archive", archive.name); + } + let key_path = archive.key_path(); + if key_paths.contains(&key_path) { + bail!( + "archive names clash in {key_path}, rename '{}'", + archive.name + ); + } + key_paths.push(key_path); + } + let known: Vec<&str> = ARCHIVES + .iter() + .copied() + .chain(archives.extra.iter().map(|archive| archive.name.as_str())) + .collect(); let release = format!("{} {}", distro.distro, distro.codename); for (pattern, settings) in &archives.packages { match settings.from.as_deref() { - Some(from) if !ARCHIVES.contains(&from) => bail!( + Some(from) if !known.contains(&from) => bail!( "unknown archive '{from}' for {pattern}, known are {}", - ARCHIVES.join(", ") + known.join(", ") ), Some(PROPOSED) if proposed_suite(distro).is_none() => { bail!("{release} has no proposed suite to take {pattern} from") @@ -178,7 +308,7 @@ impl Archives { /// Whether it adds suites to the release's. pub fn adds_suites(&self) -> bool { - self.uses(PROPOSED) || self.uses(EXPERIMENTAL) + self.uses(PROPOSED) || self.uses(EXPERIMENTAL) || !self.extra.is_empty() } } @@ -226,20 +356,30 @@ pub fn sources_file( suites.push(EXPERIMENTAL.to_string()); } - let mut groups = vec![(mirror.unwrap_or(archive), suites)]; + let components = components.join(" "); + let mut groups = vec![( + mirror.unwrap_or(archive), + suites.join(" "), + components.clone(), + )]; if debian && release.contains(&security) { - groups.push((DEBIAN_SECURITY_ARCHIVE, vec![security])); + groups.push((DEBIAN_SECURITY_ARCHIVE, security, components)); } - let paragraphs: Vec = groups + let mut paragraphs: Vec = groups .into_iter() - .map(|(uri, suites)| { - format!( - "Types: deb deb-src\nURIs: {uri}\nSuites: {}\nComponents: {}\n", - suites.join(" "), - components.join(" ") - ) + .map(|(uri, suites, components)| { + format!("Types: deb deb-src\nURIs: {uri}\nSuites: {suites}\nComponents: {components}\n") }) .collect(); + paragraphs.extend(archives.extra.iter().map(|extra| { + format!( + "Types: deb deb-src\nURIs: {}\nSuites: {}\nComponents: {}\nSigned-By: /{}\n", + extra.uri, + extra.suite, + extra.components.join(" "), + extra.key_path() + ) + })); Ok(paragraphs.join("\n")) } @@ -293,22 +433,29 @@ pub fn retry_suite(archives: &Archives) -> Option<&'static str> { /// installed packages from it: with all of it, it's pinned to the /// release's priority like on Launchpad's builders, so the build takes /// everything newer from it; -/// - with only some packages from it, the rest of a proposed suite only -/// upgrades installed packages, also on Debian, which doesn't mark them -/// NotAutomatic. +/// - with only some packages from it, the rest of a proposed suite or an +/// extra archive only upgrades installed packages, also on Debian, which +/// doesn't mark them NotAutomatic. pub fn preferences_file(distro: &DistroVersion, archives: &Archives) -> Option { let mut pins = Vec::new(); for (pattern, settings) in &archives.packages { match (&settings.version, &settings.from) { (Some(version), _) => pins.push(pin(pattern, &format!("version {version}"), 1001)), (None, Some(from)) => { - for suite in archive_suites(distro, from) { - pins.push(pin(pattern, &release_pin(distro, &suite), 990)); + for release in archive_pins(distro, archives, from) { + pins.push(pin(pattern, &release, 990)); } } (None, None) => {} } } + for extra in archives + .extra + .iter() + .filter(|extra| archives.uses(&extra.name)) + { + pins.push(pin("*", &extra.pin, 100)); + } if let Some(suite) = proposed_suite(distro) { let release = release_pin(distro, &suite); if archives.proposed { @@ -322,12 +469,21 @@ pub fn preferences_file(distro: &DistroVersion, archives: &Archives) -> Option Vec { - match archive { +/// The apt preferences pins of `archive`'s packages. +fn archive_pins(distro: &DistroVersion, archives: &Archives, archive: &str) -> Vec { + let suites = match archive { PROPOSED => proposed_suite(distro).into_iter().collect(), EXPERIMENTAL => vec![EXPERIMENTAL.to_string()], - _ => release_suites(distro), - } + RELEASE => release_suites(distro), + _ => { + let extra = archives.extra.iter().filter(|extra| extra.name == archive); + return extra.map(|extra| extra.pin.clone()).collect(); + } + }; + suites + .iter() + .map(|suite| release_pin(distro, suite)) + .collect() } /// The apt preferences pin of `suite`. Ubuntu's Release files name the @@ -386,7 +542,7 @@ mod tests { /// the package `specs`. fn archives(distro: &DistroVersion, proposed: bool, specs: &[&str]) -> Archives { let packages = specs.iter().map(|spec| parse_package(spec).unwrap()); - Archives::for_distro(distro, proposed, false, packages.collect()).unwrap() + Archives::for_distro(distro, proposed, false, packages.collect(), Vec::new()).unwrap() } fn proposed() -> Archives { @@ -453,12 +609,15 @@ mod tests { #[test_case("noble", "src:foo/backports" ; "unknown archive")] fn archives_reject(name: &str, spec: &str) { let packages = [parse_package(spec).unwrap()].into(); - assert!(Archives::for_distro(&distro(name), false, false, packages).is_err()); + assert!(Archives::for_distro(&distro(name), false, false, packages, Vec::new()).is_err()); } #[test] fn experimental_only_adds_to_debian_unstable_and_testing() { - assert!(Archives::for_distro(&distro("noble"), false, true, Packages::new()).is_err()); + assert!( + Archives::for_distro(&distro("noble"), false, true, Packages::new(), Vec::new()) + .is_err() + ); } #[test] @@ -575,7 +734,8 @@ mod tests { "{sources:?}" ); let unstable = distro("unstable"); - let extra = Archives::for_distro(&unstable, false, true, Packages::new()).unwrap(); + let extra = + Archives::for_distro(&unstable, false, true, Packages::new(), Vec::new()).unwrap(); let sources = sources_file(&unstable, None, &extra, "amd64").unwrap(); assert!( sources.contains("Suites: unstable experimental\n"), @@ -618,6 +778,87 @@ mod tests { ); } + #[test_case("ppa:me/stuff", "me", "stuff", None ; "plain")] + #[test_case("ppa:me/stuff@noble", "me", "stuff", Some("noble") ; "suite")] + fn ppa_spec(spec: &str, owner: &str, name: &str, suite: Option<&str>) { + let ppa = spec.parse::().unwrap(); + assert_eq!((ppa.owner.as_str(), ppa.name.as_str()), (owner, name)); + assert_eq!(ppa.suite.as_deref(), suite); + } + + #[test_case("me/stuff" ; "no prefix")] + #[test_case("ppa:me" ; "no name")] + #[test_case("ppa:me/stuff@" ; "empty suite")] + #[test_case("ppa:me/st uff" ; "whitespace")] + fn ppa_spec_rejects(spec: &str) { + assert!(spec.parse::().is_err()); + } + + fn ppa(spec: &str, distro: &DistroVersion) -> anyhow::Result { + spec.parse::() + .unwrap() + .archive(spec, distro, "KEY".to_string()) + } + + #[test] + fn ppa_named_ppa_has_the_owner_as_origin() { + let noble = distro("noble"); + assert_eq!( + ppa("ppa:me/ppa", &noble).unwrap().pin, + "release o=LP-PPA-me" + ); + assert_eq!( + ppa("ppa:me/stuff", &noble).unwrap().pin, + "release o=LP-PPA-me-stuff" + ); + } + + #[test] + fn ppa_for_debian_needs_a_suite() { + let trixie = distro("trixie"); + assert!(ppa("ppa:me/stuff", &trixie).is_err()); + assert_eq!(ppa("ppa:me/stuff@noble", &trixie).unwrap().suite, "noble"); + } + + #[test] + fn ppa_is_signed_by_its_key_and_pinned_when_picked_from() { + let noble = distro("noble"); + let extra = vec![ppa("ppa:me/stuff", &noble).unwrap()]; + let packages = [parse_package("libfoo1/ppa:me/stuff").unwrap()].into(); + let archives = Archives::for_distro(&noble, false, false, packages, extra).unwrap(); + let sources = sources_file(&noble, None, &archives, "amd64").unwrap(); + assert!( + sources.ends_with( + "\n\nTypes: deb deb-src\nURIs: https://ppa.launchpadcontent.net/me/stuff/ubuntu\n\ + Suites: noble\nComponents: main\n\ + Signed-By: /etc/apt/keyrings/debmagic-ppa-me-stuff.asc\n" + ), + "{sources}" + ); + assert_eq!( + preferences_file(&noble, &archives).unwrap(), + "Package: libfoo1\nPin: release o=LP-PPA-me-stuff\nPin-Priority: 990\n\n\ + Package: *\nPin: release o=LP-PPA-me-stuff\nPin-Priority: 100\n" + ); + + let all = Archives::for_distro(&noble, false, false, Packages::new(), archives.extra); + let all = all.unwrap(); + assert!(all.adds_suites()); + assert_eq!(preferences_file(&noble, &all), None); + } + + #[test] + fn extra_archives_must_not_shadow_or_clash() { + let noble = distro("noble"); + let mut proposed = ppa("ppa:me/stuff", &noble).unwrap(); + proposed.name = "proposed".to_string(); + let mut clash = ppa("ppa:me/stuff", &noble).unwrap(); + clash.name = "ppa:me-stuff".to_string(); + let extra = |extra| Archives::for_distro(&noble, false, false, Packages::new(), extra); + assert!(extra(vec![proposed]).is_err()); + assert!(extra(vec![ppa("ppa:me/stuff", &noble).unwrap(), clash]).is_err()); + } + #[test] fn custom_distro_has_no_known_archive() { let custom = DistroVersion::custom(Distro::Custom("yocto".into()), "scarthgap"); diff --git a/packages/debmagic/src/apt.rs b/packages/debmagic/src/apt.rs index f3f010ba..b9dab72e 100644 --- a/packages/debmagic/src/apt.rs +++ b/packages/debmagic/src/apt.rs @@ -2,16 +2,17 @@ //! container, refreshing its packages, and installing what a run needs in //! one transaction. -use std::{fs, io, os::unix::fs::PermissionsExt, path::Path}; +use std::{collections::BTreeMap, fs, io, os::unix::fs::PermissionsExt, path::Path}; -use anyhow::Context; +use anyhow::{Context, bail}; use debmagic_common::{ - apt::{self, Archives, PackageSettings, Packages}, + apt::{self, Archives, ExtraArchive, PackageSettings, Packages, Ppa}, debian::control::BuildRelations, distro::DistroVersion, }; use serde::{Deserialize, Serialize}; +use crate::config::SignConfig; use crate::driver::EnvironmentDriver; use crate::time::RefreshPolicy; @@ -33,18 +34,125 @@ pub struct AptConfig { /// Where to take packages from, or which version, by apt preferences /// pattern. pub packages: Packages, + /// Archives beyond the distro's by name, like `ppa:/`. + pub archives: BTreeMap, } impl AptConfig { - /// The archives beyond its release an Environment for `distro` takes - /// packages from. - pub fn archives(&self, distro: &DistroVersion) -> anyhow::Result { + /// What an Environment for `distro` takes its packages from, with the + /// keys of its extra archives fetched and checked with `sign`'s tool. + pub async fn plan(&self, distro: &DistroVersion, sign: &SignConfig) -> anyhow::Result { + let mut specs = self.archives.clone(); + // an archive spec in `from` adds the archive under that name + for settings in self.packages.values() { + if let Some(from) = &settings.from + && from.starts_with("ppa:") + { + specs.entry(from.clone()).or_insert_with(|| from.clone()); + } + } + let mut extra = Vec::new(); + for (name, spec) in &specs { + let archive = resolve_archive(name, spec, distro, sign) + .await + .with_context(|| format!("adding the archive '{name}' failed"))?; + extra.push(archive); + } let packages = self.packages.clone(); - Archives::for_distro(distro, self.proposed, self.experimental, packages) - .context("invalid apt package settings") + let archives = + Archives::for_distro(distro, self.proposed, self.experimental, packages, extra) + .context("invalid apt package settings")?; + Ok(AptPlan { + mirror: self.mirror.clone(), + update_age: self.update_age, + archives, + }) } } +/// The `[apt]` settings resolved for one target distro. +#[derive(Debug, Clone, Default)] +pub struct AptPlan { + pub mirror: Option, + pub update_age: RefreshPolicy, + pub archives: Archives, +} + +const KEYSERVER: &str = "https://keyserver.ubuntu.com"; + +async fn resolve_archive( + name: &str, + spec: &str, + distro: &DistroVersion, + sign: &SignConfig, +) -> anyhow::Result { + let ppa: Ppa = spec.parse().map_err(anyhow::Error::msg)?; + let key = ppa_key(&ppa, sign).await?; + ppa.archive(name, distro, key) +} + +/// The ASCII-armored signing key of `ppa`, fetched once and then kept in +/// the cache, like add-apt-repository does: its fingerprint from +/// Launchpad, the key from the keyserver, checked against the fingerprint. +async fn ppa_key(ppa: &Ppa, sign: &SignConfig) -> anyhow::Result { + let path = dirs::cache_dir() + .context("no cache dir to keep PPA keys in")? + .join("debmagic/ppa-keys") + .join(&ppa.owner) + .join(format!("{}.asc", ppa.name)); + match fs::read_to_string(&path) { + Ok(key) => return Ok(key), + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error).with_context(|| format!("reading {} failed", path.display())); + } + } + + #[derive(Deserialize)] + struct Archive { + signing_key_fingerprint: Option, + } + + let url = ppa.api_url(); + let archive: Archive = serde_json::from_str(&crate::requests::http_get(&url).await?) + .with_context(|| format!("reading {url} failed"))?; + let fingerprint = archive + .signing_key_fingerprint + .context("Launchpad has no signing key for it yet")?; + let url = format!("{KEYSERVER}/pks/lookup?op=get&options=mr&exact=on&search=0x{fingerprint}"); + let key = crate::requests::http_get(&url).await?; + + let partial = path.with_extension("asc.partial"); + fs::create_dir_all(path.parent().unwrap_or(&path)) + .and_then(|()| fs::write(&partial, &key)) + .with_context(|| format!("writing {} failed", partial.display()))?; + let fetched = crate::sign::key_fingerprint(sign, &partial) + .with_context(|| format!("checking the key from {url} failed"))?; + if !fetched.eq_ignore_ascii_case(&fingerprint) { + bail!("{url} returned the key {fetched} instead of {fingerprint}"); + } + fs::rename(&partial, &path) + .with_context(|| format!("caching the key in {} failed", path.display()))?; + Ok(key) +} + +/// `--archive [=]`: the name defaults to the spec. +pub fn parse_archive_arg(arg: &str) -> Result<(String, String), String> { + let (name, spec) = match arg.split_once('=') { + Some((name, spec)) + if !name.is_empty() + && name + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"-_.".contains(&b)) => + { + (name, spec) + } + _ => (arg, arg), + }; + spec.parse::()?; + Ok((name.to_string(), spec.to_string())) +} + /// The `[apt]` settings given on the command line. #[derive(Debug, Clone, Default)] pub struct AptOverrides { @@ -54,6 +162,8 @@ pub struct AptOverrides { pub experimental: Option, /// Replacing the configured settings of the same pattern. pub packages: Vec<(String, PackageSettings)>, + /// Replacing the configured archives of the same name. + pub archives: Vec<(String, String)>, } impl AptOverrides { @@ -71,6 +181,7 @@ impl AptOverrides { config.experimental = experimental; } config.packages.extend(self.packages); + config.archives.extend(self.archives); } } @@ -164,10 +275,14 @@ impl Setup<'_> { .with_context(|| format!("removing the old {} failed", dir.display()))?; } let preferences = apt::preferences_file(self.distro, self.archives); + let keys: Vec<(String, &str)> = (self.archives.extra.iter()) + .map(|archive| (archive.key_path(), archive.key.as_str())) + .collect(); let mut files: Vec<(&str, &str, u32)> = self.files().collect(); for (path, content) in [(SOURCES_FILE, &sources), (PREFERENCES_FILE, &preferences)] { files.extend(content.as_deref().map(|content| (path, content, 0o644))); } + files.extend(keys.iter().map(|(path, key)| (path.as_str(), *key, 0o644))); for (path, content, mode) in files { let path = dir.join(path); fs::create_dir_all(path.parent().expect("setup files are in a dir")) @@ -430,7 +545,29 @@ mod tests { echo 'man-db man-db/auto-update boolean false' | debconf-set-selections && \ rm -f /var/lib/man-db/auto-update" ); + + let ppa: Ppa = "ppa:me/stuff@noble".parse().unwrap(); + let extra = vec![ppa.archive("stuff", &trixie, "KEY".to_string())?]; + let archives = Archives::for_distro(&trixie, false, false, Packages::new(), extra)?; + setup.archives = &archives; + setup.write(&dir)?; + let key = dir.join("etc/apt/keyrings/debmagic-stuff.asc"); + assert_eq!(fs::read_to_string(key)?, "KEY"); fs::remove_dir_all(&dir)?; Ok(()) } + + #[test_case::test_case("ppa:me/stuff", "ppa:me/stuff" ; "unnamed")] + #[test_case::test_case("mine=ppa:me/stuff@noble", "mine" ; "named")] + fn archive_arg(arg: &str, name: &str) { + let (parsed, spec) = parse_archive_arg(arg).unwrap(); + assert_eq!(parsed, name); + assert!(spec.starts_with("ppa:me/stuff")); + } + + #[test_case::test_case("mine=" ; "empty spec")] + #[test_case::test_case("deb http://x trixie main" ; "no ppa")] + fn archive_arg_rejects(arg: &str) { + assert!(parse_archive_arg(arg).is_err()); + } } diff --git a/packages/debmagic/src/build/mod.rs b/packages/debmagic/src/build/mod.rs index 09f62562..0b1d8475 100644 --- a/packages/debmagic/src/build/mod.rs +++ b/packages/debmagic/src/build/mod.rs @@ -4,7 +4,7 @@ use std::{ process::{Command, Stdio}, }; -use crate::apt; +use crate::apt::{self, AptPlan}; use crate::build::source::{source_manifest_path, stage_dir, stage_source_tree}; use crate::build_intent::{BuildIntent, BuildIntentInput, BuildKind, resolve_build_intent}; use crate::driver::{Driver, DriverType, Environment, EnvironmentDriver, EnvironmentPurpose}; @@ -117,6 +117,7 @@ fn deb_build_options(existing: Option<&str>, build_debug_symbols: bool, run_test struct BuildRequest<'a> { intent: &'a BuildIntent, target: &'a PackageTarget, + apt: &'a AptPlan, } /// Shared build orchestration: claim the Environment, run `build_commands` @@ -143,7 +144,7 @@ async fn run_build( ®istry, planned.environment, &request.intent.config.driver, - &request.intent.config.apt, + request.apt, &request.intent.driver_overrides, planned.root_policy, |environment, driver| { @@ -228,9 +229,14 @@ async fn run_build( pub async fn build_binary_package( intent: &BuildIntent, target: &PackageTarget, + apt: &AptPlan, changes_options: &[String], ) -> anyhow::Result { - let request = BuildRequest { intent, target }; + let request = BuildRequest { + intent, + target, + apt, + }; run_build(&request, |claimed| { crate::output::stage("Building binary package(s)"); let staged_source_dir = stage_dir(claimed.environment(), BuildKind::Binary); @@ -243,7 +249,6 @@ pub async fn build_binary_package( packages.extend(gain_root_package(target)?); install_packages( claimed, - intent, target, BuildKind::Binary, packages, @@ -307,7 +312,6 @@ fn gain_root_package(target: &PackageTarget) -> anyhow::Result, @@ -331,13 +335,13 @@ fn install_packages( profiles: profiles.to_vec(), }; let environment = claimed.environment(); - let archives = intent.config.apt.archives(&environment.distro)?; + let archives = &claimed.apt().archives; let staged_source_dir = stage_dir(environment, kind); apt::install( claimed.driver(), &staged_source_dir, &environment.distro, - &archives, + archives, &install, ) .context("installing the build dependencies failed") @@ -394,6 +398,7 @@ fn check_dpkg_buildpackage_available() -> anyhow::Result<()> { pub async fn build_source_package( intent: &BuildIntent, target: &PackageTarget, + apt: &AptPlan, changes_options: &[String], include_orig: bool, ) -> anyhow::Result { @@ -454,7 +459,11 @@ pub async fn build_source_package( } } - let request = BuildRequest { intent, target }; + let request = BuildRequest { + intent, + target, + apt, + }; run_build(&request, |claimed| { crate::output::stage("Building source package"); let staged_source_dir = stage_dir(claimed.environment(), BuildKind::Source); @@ -478,7 +487,6 @@ pub async fn build_source_package( let scope = intent.config.clean.then_some(BuildScope::Clean); install_packages( claimed, - intent, target, BuildKind::Source, packages, @@ -562,11 +570,16 @@ pub async fn build(command: BuildCommand) -> anyhow::Result<()> { ), ) .context("failed to determine package target")?; - intent.config.apt.archives(&target.distro)?; + let apt = intent + .config + .apt + .plan(&target.distro, &intent.config.sign) + .await + .context("resolving the apt archives failed")?; let package = format!("{} {}", target.package.name(), target.package.version()); let notify = intent.config.build_notify(); - let result = build_and_upload(&intent, target, &command).await; + let result = build_and_upload(&intent, target, &apt, &command).await; // a signed build already notified when signing started let notified = result.is_ok() && intent.config.sign.source.is_enabled(); if notify && !notified { @@ -582,6 +595,7 @@ pub async fn build(command: BuildCommand) -> anyhow::Result<()> { async fn build_and_upload( intent: &BuildIntent, target: PackageTarget, + apt: &AptPlan, command: &BuildCommand, ) -> anyhow::Result<()> { let upstream_version = target.package.version().upstream_version().to_string(); @@ -590,7 +604,7 @@ async fn build_and_upload( Some(mode) => crate::upload::orig::decide_orig_upload(mode, &intent.source_dir)?, None => true, }; - build_source_package(intent, &target, &command.changes_options, include_orig) + build_source_package(intent, &target, apt, &command.changes_options, include_orig) .await .context("Building the source package failed")? } else { @@ -602,7 +616,7 @@ async fn build_and_upload( ) .context("host's /etc/os-release does not match the build target distro")?; } - build_binary_package(intent, &target, &command.changes_options) + build_binary_package(intent, &target, apt, &command.changes_options) .await .context("Building the package failed")? }; diff --git a/packages/debmagic/src/cli.rs b/packages/debmagic/src/cli.rs index 92061299..48bf2fcf 100644 --- a/packages/debmagic/src/cli.rs +++ b/packages/debmagic/src/cli.rs @@ -355,6 +355,14 @@ pub struct EnvironmentArgs { )] pub packages: Vec<(String, PackageSettings)>, + #[arg( + long = "archive", + value_name = "[NAME=]SPEC", + value_parser = crate::apt::parse_archive_arg, + help = "Add an archive: a PPA as 'ppa:/[@]', the suite defaulting to the target Ubuntu release. NAME (by default SPEC) is the ARCHIVE of --package; without packages picked from it, apt takes all newer ones from it. Repeatable. Ignored by the bare driver." + )] + pub archives: Vec<(String, String)>, + #[arg( long = "apt-update-age", help = "When a persistent environment runs 'apt-get update/upgrade' again: 'now' (every run), 'never' (only on first creation), or a maximum age of the last upgrade like '1d' (the default), '12h', '30m'. Defaults to the 'apt_update_age' setting in the config file. Ignored by the bare driver." @@ -781,6 +789,7 @@ impl EnvironmentArgs { proposed, experimental, packages, + archives: self.archives.clone(), } } diff --git a/packages/debmagic/src/config.rs b/packages/debmagic/src/config.rs index 5fbe0fe8..b1ba37dd 100644 --- a/packages/debmagic/src/config.rs +++ b/packages/debmagic/src/config.rs @@ -365,6 +365,9 @@ pub struct SignConfig { /// "auto"`: exit code 0 means the file's signature is ours and it /// is skipped, anything else means re-sign. pub signed_by_command: Option, + /// Custom key fingerprint reader when `tool` is `custom`, for PPA + /// signing keys: prints the primary key's fingerprint of `{file}`. + pub fingerprint_command: Option, /// Send a desktop notification via `notify-send` just before signing, /// so a hardware-key touch prompt isn't missed; the top-level `notify` /// when unset. diff --git a/packages/debmagic/src/driver/mod.rs b/packages/debmagic/src/driver/mod.rs index f58d41dd..bc5074bb 100644 --- a/packages/debmagic/src/driver/mod.rs +++ b/packages/debmagic/src/driver/mod.rs @@ -11,7 +11,7 @@ use std::{ time::SystemTime, }; -use crate::apt::AptConfig; +use crate::apt::AptPlan; use crate::driver::{ config::{DriverConfig, DriverOverrides}, driver_bare::DriverBare, @@ -487,11 +487,11 @@ impl EnvironmentDriver for Driver { pub fn create_driver( environment: &Environment, driver_config: &DriverConfig, - apt: &AptConfig, + apt: &AptPlan, overrides: &DriverOverrides, ) -> anyhow::Result { let apt_mirror = apt.mirror.as_deref(); - let archives = apt.archives(&environment.distro)?; + let archives = &apt.archives; match environment.driver { DriverType::Docker => Ok(Driver::Docker(DriverDocker::create( @@ -499,7 +499,7 @@ pub fn create_driver( driver_config, &overrides.docker, apt_mirror, - &archives, + archives, )?)), DriverType::Bare => Ok(Driver::Bare(DriverBare::create( environment, @@ -517,7 +517,7 @@ pub fn create_driver( driver_config, &overrides.lxd, apt_mirror, - &archives, + archives, )?)) } } diff --git a/packages/debmagic/src/environment/mod.rs b/packages/debmagic/src/environment/mod.rs index e09afdf1..6ac3a521 100644 --- a/packages/debmagic/src/environment/mod.rs +++ b/packages/debmagic/src/environment/mod.rs @@ -11,7 +11,7 @@ use anyhow::{Context, bail}; use process::Process; use sqlx::sqlite::SqliteConnection; -use crate::apt::AptConfig; +use crate::apt::AptPlan; use crate::driver::{ Driver, Environment, EnvironmentDriver, EnvironmentMetadata, Persistence, ResourceStatus, config::{DriverConfig, DriverOverrides}, @@ -44,6 +44,7 @@ pub enum HostRootPolicy { pub struct ClaimedEnvironment<'a> { registry: &'a Registry, driver_config: &'a DriverConfig, + apt: &'a AptPlan, environment: Environment, driver: Driver, /// `on-failure` keeps the Environment only when the build or TestRun @@ -62,7 +63,7 @@ impl<'a> ClaimedEnvironment<'a> { registry: &'a Registry, environment: Environment, driver_config: &'a DriverConfig, - apt: &AptConfig, + apt: &'a AptPlan, overrides: &DriverOverrides, root_policy: HostRootPolicy, stage: impl FnOnce(&Environment, Option<&Driver>) -> anyhow::Result<()>, @@ -104,6 +105,11 @@ impl<'a> ClaimedEnvironment<'a> { &self.driver } + /// Where the Environment's packages come from. + pub fn apt(&self) -> &AptPlan { + self.apt + } + async fn finish(self) -> anyhow::Result<()> { finish_environment( self.registry, @@ -128,7 +134,7 @@ impl<'a> ClaimedEnvironment<'a> { registry: &'a Registry, environment: Environment, driver_config: &'a DriverConfig, - apt: &AptConfig, + apt: &'a AptPlan, overrides: &DriverOverrides, ) -> anyhow::Result { let driver = match crate::driver::create_driver(&environment, driver_config, apt, overrides) @@ -149,6 +155,7 @@ impl<'a> ClaimedEnvironment<'a> { let claimed = Self { registry, driver_config, + apt, environment, driver, keep_after_failure: false, @@ -303,7 +310,7 @@ pub async fn claim_command( registry: &Registry, environment: Environment, driver_config: &DriverConfig, - apt: &AptConfig, + apt: &AptPlan, overrides: &DriverOverrides, root_policy: HostRootPolicy, stage: impl FnOnce(&Environment, Option<&Driver>) -> anyhow::Result<()>, @@ -1430,6 +1437,7 @@ mod claimed_environment_tests { dir: PathBuf, registry: Registry, driver_config: DriverConfig, + apt: AptPlan, overrides: DriverOverrides, } @@ -1443,6 +1451,7 @@ mod claimed_environment_tests { dir, registry, driver_config: DriverConfig::default(), + apt: AptPlan::default(), overrides: DriverOverrides::default(), } } @@ -1470,7 +1479,7 @@ mod claimed_environment_tests { &self.registry, environment, &self.driver_config, - &AptConfig::default(), + &self.apt, &self.overrides, root_policy, stage, @@ -1648,7 +1657,7 @@ mod claimed_environment_tests { &fixture.registry, environment, &fixture.driver_config, - &AptConfig::default(), + &fixture.apt, &fixture.overrides, HostRootPolicy::Reset, |_, _| Ok(()), diff --git a/packages/debmagic/src/sign.rs b/packages/debmagic/src/sign.rs index 6c773418..02327abd 100644 --- a/packages/debmagic/src/sign.rs +++ b/packages/debmagic/src/sign.rs @@ -69,9 +69,10 @@ enum SignInput { File, } -/// One OpenPGP implementation's way of performing the three operations -/// debmagic needs: clearsigning, verifying a detached signature, and -/// telling whether a clearsigned file's signature is our own. The +/// One OpenPGP implementation's way of performing the operations debmagic +/// needs: clearsigning, verifying a detached signature, telling whether a +/// clearsigned file's signature is our own, and reading a key's +/// fingerprint. The /// invocation plumbing (running the command, reading stdout, error /// reporting) is shared in this module; each impl only builds commands /// and parses its tool's output. @@ -93,6 +94,9 @@ trait SignBackend { /// resolves to. Whatever the backend cannot tell counts as "no" — /// callers then re-sign. fn signed_by_us(&self, path: &Path, signas: &str) -> bool; + + /// The fingerprint of the primary key in the key file `key`. + fn key_fingerprint(&self, key: &Path) -> anyhow::Result; } /// Run `cmd` and return its captured stdout, or `None` when it cannot @@ -106,6 +110,40 @@ fn capture_stdout(cmd: Command) -> Option { .and_then(|result| result.stdout) } +/// Run `cmd` and return its stdout, failing with its stderr on a non-zero +/// exit. +fn checked_stdout(cmd: Command) -> anyhow::Result { + let program = cmd.get_program().to_string_lossy().into_owned(); + let result = crate::subprocess::command(cmd) + .capture(Capture::ALL) + .run() + .with_context(|| format!("failed to run {program}"))?; + if result.exit_code != 0 { + bail!( + "{program} failed:\n{}", + result.stderr.as_deref().unwrap_or_default().trim() + ); + } + Ok(result.stdout.unwrap_or_default()) +} + +/// The first `Fingerprint:` in `sq inspect` output, the primary key's. +fn sq_fingerprint(inspect: &str) -> Option<&str> { + inspect + .lines() + .find_map(|line| line.trim_start().strip_prefix("Fingerprint: ")) + .map(str::trim) +} + +/// The first `fpr:` record in gpg `--with-colons` output, the primary +/// key's. +fn gpg_fingerprint(colons: &str) -> Option<&str> { + colons + .lines() + .find(|line| line.starts_with("fpr:")) + .and_then(|line| line.split(':').nth(9)) +} + impl SignTool { fn backend(self, options: &SignConfig) -> Box { match self { @@ -205,18 +243,19 @@ impl SignBackend for Gpg { let mut list = Command::new("gpg"); list.args(["--list-secret-keys", "--with-colons"]) .arg(signas); - let Some(colo) = capture_stdout(list) else { - return false; - }; - // The fpr field of the secret key's primary: fpr::... - let Some(ours) = colo - .lines() - .find(|line| line.starts_with("fpr:")) - .and_then(|line| line.split(':').nth(9)) - else { + let Some(ours) = capture_stdout(list) else { return false; }; - theirs.eq_ignore_ascii_case(ours) + gpg_fingerprint(&ours).is_some_and(|ours| theirs.eq_ignore_ascii_case(ours)) + } + + fn key_fingerprint(&self, key: &Path) -> anyhow::Result { + let mut cmd = Command::new("gpg"); + cmd.args(["--show-keys", "--with-colons"]).arg(key); + let out = checked_stdout(cmd)?; + gpg_fingerprint(&out) + .map(str::to_string) + .context("gpg --show-keys found no key") } } @@ -268,15 +307,19 @@ impl SignBackend for Sequoia { let mut inspect = Command::new("sq"); inspect.args(["inspect", "--cert-email"]).arg(signas); - let Some(ours) = capture_stdout(inspect).and_then(|out| { - out.lines() - .find_map(|line| line.strip_prefix("Fingerprint: ")) - .map(str::trim) - .map(str::to_string) - }) else { + let Some(ours) = capture_stdout(inspect) else { return false; }; - theirs.eq_ignore_ascii_case(&ours) + sq_fingerprint(&ours).is_some_and(|ours| theirs.eq_ignore_ascii_case(ours)) + } + + fn key_fingerprint(&self, key: &Path) -> anyhow::Result { + let mut cmd = Command::new("sq"); + cmd.arg("inspect").arg(key); + let out = checked_stdout(cmd)?; + sq_fingerprint(&out) + .map(str::to_string) + .context("sq inspect found no key") } } @@ -364,6 +407,29 @@ impl SignBackend for Custom<'_> { }; result.exit_code == 0 } + + fn key_fingerprint(&self, key: &Path) -> anyhow::Result { + let command = self.options.fingerprint_command.as_deref().context( + "sign.tool = \"custom\" requires sign.fingerprint_command to read key fingerprints", + )?; + let out = checked_stdout(self.file_command(command, key, "")?)?; + let fingerprint = out.trim(); + if fingerprint.is_empty() { + bail!("sign.fingerprint_command printed no fingerprint"); + } + Ok(fingerprint.to_string()) + } +} + +/// The fingerprint of the primary key in the key file `key`, using the +/// configured backend. With `tool = "custom"`, `sign.fingerprint_command` +/// runs with `{file}` substituted (or the path appended) and prints it. +pub fn key_fingerprint(options: &SignConfig, key: &Path) -> anyhow::Result { + options + .tool + .backend(options) + .key_fingerprint(key) + .with_context(|| format!("reading the fingerprint of {} failed", key.display())) } /// Verify a detached signature `signature` over `file` against the @@ -492,6 +558,7 @@ pub fn sign(input: SignIntent) -> anyhow::Result<()> { sign_command: config.sign.sign_command.clone(), verify_command: config.sign.verify_command.clone(), signed_by_command: config.sign.signed_by_command.clone(), + fingerprint_command: config.sign.fingerprint_command.clone(), source: config.sign.source, notify: config.sign.notify, }; @@ -813,11 +880,8 @@ mod tests { let options = SignConfig { key: Some("mykey".into()), tool: SignTool::Gpg, - sign_command: None, - verify_command: None, - signed_by_command: None, source: SignMode::No, - notify: None, + ..Default::default() }; let control = parse_control("Maintainer: A \nChanged-By: B \n"); @@ -830,6 +894,47 @@ mod tests { assert_eq!(guess_signas(&options, &control), "A "); } + #[test] + fn fingerprints_of_the_primary_key() { + let gpg = "pub:-:4096:1:BA6932366A755776:1501330817:::-:::scSC::::::23::0:\n\ + fpr:::::::::F23C5A6CF475977595C89F51BA6932366A755776:\n\ + sub:-:4096:1:0123456789ABCDEF:1501330817::::::e::::::23:\n\ + fpr:::::::::00000000000000000000000000000000DEADBEEF:\n"; + assert_eq!( + gpg_fingerprint(gpg), + Some("F23C5A6CF475977595C89F51BA6932366A755776") + ); + let sq = "key.asc: OpenPGP Certificate.\n\n \ + Fingerprint: F23C5A6CF475977595C89F51BA6932366A755776\n \ + Public-key algo: RSA\n\n \ + Subkey: 00000000000000000000000000000000DEADBEEF\n"; + assert_eq!( + sq_fingerprint(sq), + Some("F23C5A6CF475977595C89F51BA6932366A755776") + ); + assert_eq!(gpg_fingerprint(""), None); + assert_eq!(sq_fingerprint(""), None); + } + + #[test] + fn custom_fingerprint_command_prints_it() { + let dir = std::env::temp_dir().join("debmagic-sign-test-fingerprint"); + std::fs::create_dir_all(&dir).unwrap(); + let key = dir.join("key.asc"); + std::fs::write(&key, "ABCD\n").unwrap(); + let custom = |command: Option<&str>| SignConfig { + tool: SignTool::Custom, + fingerprint_command: command.map(str::to_string), + ..Default::default() + }; + + assert_eq!(key_fingerprint(&custom(Some("cat")), &key).unwrap(), "ABCD"); + assert!(key_fingerprint(&custom(Some("true {file}")), &key).is_err()); + assert!(key_fingerprint(&custom(Some("false")), &key).is_err()); + assert!(key_fingerprint(&custom(None), &key).is_err()); + std::fs::remove_dir_all(&dir).unwrap(); + } + #[test] fn render_arg_substitutes_placeholders() { assert_eq!( diff --git a/packages/debmagic/src/test/run.rs b/packages/debmagic/src/test/run.rs index 90303cf5..249adfbb 100644 --- a/packages/debmagic/src/test/run.rs +++ b/packages/debmagic/src/test/run.rs @@ -345,7 +345,10 @@ pub async fn run_test(intent: &TestIntent) -> anyhow::Result { "no prior build metadata found; pass --distro when using --changes without a prior Invocation" ); }; - apt.archives(&distro)?; + let apt = apt + .plan(&distro, &intent.config.sign) + .await + .context("resolving the apt archives failed")?; let environment = Environment::new( driver, @@ -363,7 +366,7 @@ pub async fn run_test(intent: &TestIntent) -> anyhow::Result { ®istry, environment, &intent.config.driver, - apt, + &apt, &intent.driver_overrides, HostRootPolicy::Reset, |environment, _| { @@ -392,12 +395,12 @@ pub async fn run_test(intent: &TestIntent) -> anyhow::Result { ..Default::default() }; let distro = &claimed.environment().distro; - let archives = apt.archives(distro)?; + let archives = &claimed.apt().archives; crate::apt::install( claimed.driver(), &staged_source_dir, distro, - &archives, + archives, &install, ) .context("installing autopkgtest failed")?; From ac3483b280b65f1638e38e8c149ce28971a888b5 Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Fri, 23 Oct 2026 20:46:42 +0200 Subject: [PATCH 45/48] feat(build): add sources.list deb lines with `--archive` Options pass through to apt, e.g. trusted=yes; a signed-by key file is read on the host and copied into the environment. Packages picked from such an archive are pinned by its host. --- docs/usage/build.md | 10 +- docs/usage/config.md | 2 +- packages/debmagic-common/src/apt.rs | 285 ++++++++++++++++++++++++++-- packages/debmagic/src/apt.rs | 46 +++-- 4 files changed, 310 insertions(+), 33 deletions(-) diff --git a/docs/usage/build.md b/docs/usage/build.md index d29c33e6..f153dc7d 100644 --- a/docs/usage/build.md +++ b/docs/usage/build.md @@ -28,7 +28,7 @@ debmagic build binary --driver lxd \ | `--proposed[=]` | Use [build dependencies from `proposed`](#proposed-dependencies) | | `--experimental[=]` | Use build dependencies from [experimental](#experimental) | | `--package [=][/]` | Install a [package in a version or from an archive](#package-versions-and-archives) | -| `--archive [=]` | Add an [archive like a PPA](#extra-archives) | +| `--archive [=]` | Add an [archive](#extra-archives): `ppa:/` or a `deb` line | | `--sign` | [GPG-sign the resulting `.changes`/`.dsc`/`.buildinfo`](#signing) | | `--clean` | Run [`debian/rules clean` before building](#cleaning) | | `--debug-symbols` | [Build the automatic `-dbgsym` debug symbol packages](#building-debug-symbol-packages) | @@ -202,12 +202,13 @@ libfoo1 = { from = "proposed" } ## Extra archives -`--archive` adds an archive beyond the distro's, for now a Launchpad PPA: +`--archive` adds an archive beyond the distro's, a Launchpad PPA or a [sources.list(5)](https://manpages.debian.org/sources.list.5) `deb` line: ```shell debmagic build --archive ppa:me/stuff # all newer packages from it debmagic build --package libfoo1/ppa:me/stuff # only libfoo1 from it debmagic build --archive mine=ppa:me/stuff --package libfoo1/mine +debmagic build --archive 'mine=deb [signed-by=/home/me/mine.asc] https://example.org/debian trixie main' ``` An archive is named after its spec unless `=` names it; `--package` takes packages from it by that name. @@ -217,11 +218,16 @@ A PPA is taken for the target Ubuntu release; `ppa:/@` picks Its signing key is looked up on Launchpad and fetched from keyserver.ubuntu.com before the build starts; if either fails, nothing is built. The key is then kept in `~/.cache/debmagic/ppa-keys//.asc` and not fetched again; delete it when the PPA's key changes. +A `deb` line's options are apt's, e.g. `trusted=yes` for an unsigned repository. +A `signed-by` key file is read on the host and copied into the environment; without one, the archive must be signed by a key the environment trusts. +Picking packages from it pins them by the host of its URI, which matches any other archive on that host too. + In the [config](config.md): ```toml [apt.archives] mine = "ppa:me/stuff" +snapshot = "deb [check-valid-until=no] https://snapshot.debian.org/archive/debian/20260101T000000Z trixie main" ``` ## Building debug symbol packages diff --git a/docs/usage/config.md b/docs/usage/config.md index 086e7072..14fa3eec 100644 --- a/docs/usage/config.md +++ b/docs/usage/config.md @@ -31,7 +31,7 @@ All keys are optional. | `apt.update_age` | string | `"1d"` | `--apt-update-age` | When a persistent environment re-runs `apt-get update && upgrade`: `"now"` (every build), `"never"` (only on creation), or a maximum age like `"1d"`, `"12h"`, `"30m"`. | | `apt.proposed` | bool | `false` | `--proposed` | All newer packages from the [proposed suite](build.md#proposed-dependencies). | | `apt.experimental` | bool | `false` | `--experimental` | What nothing else provides from [experimental](build.md#experimental), for unstable and testing. | -| `apt.archives` | map | — | `--archive` | [Extra archives](build.md#extra-archives) by name, e.g. `{ mine = "ppa:me/stuff" }`. | +| `apt.archives` | map | — | `--archive` | [Extra archives](build.md#extra-archives) by name: a PPA or a `deb` line, e.g. `{ mine = "ppa:me/stuff" }`. | | `apt.packages` | map | — | `--package` | [Package settings](build.md#package-versions-and-archives) by pattern: `from` (an archive) and `version`, e.g. `{ "src:bar" = { from = "proposed" } }`. | | `environments_dir` | path | `$XDG_DATA_HOME/debmagic/environments` | — | Parent directory of Environment Host roots. Must be absolute or start with `~/`. The Environment registry is always `$XDG_DATA_HOME/debmagic/db.sqlite` (override both with `DEBMAGIC_DATA_DIR`). | | `output_dir` | path | `build` | `-o`/`--output-dir` | Where build artifacts are exported; relative paths resolve against the package root. See [Output dir](build.md#output-dir). | diff --git a/packages/debmagic-common/src/apt.rs b/packages/debmagic-common/src/apt.rs index c657ef9e..9f1db177 100644 --- a/packages/debmagic-common/src/apt.rs +++ b/packages/debmagic-common/src/apt.rs @@ -117,6 +117,167 @@ impl FromStr for Selection { } } +/// An extra archive as `--archive` and `apt.archives` take it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ArchiveSpec { + Ppa(Ppa), + Deb(DebLine), +} + +impl FromStr for ArchiveSpec { + type Err = String; + + fn from_str(spec: &str) -> Result { + match spec.starts_with("ppa:") { + true => spec.parse().map(Self::Ppa), + false => spec.parse().map(Self::Deb), + } + } +} + +/// A sources.list(5) one-line entry, +/// `deb [