diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4a5189d0..2c196eb8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -141,6 +141,10 @@ jobs: contents: read # Push of the pre-built CI image is additionally gated to main below. packages: write + env: + # Registry and Environment Host roots together, so the cached build + # tree stays registered under the same Environment id. + DEBMAGIC_DATA_DIR: /tmp/debmagic-data steps: - uses: actions/checkout@v5 - uses: Swatinem/rust-cache@v2 @@ -149,16 +153,18 @@ jobs: - name: Cache the debmagic build tree uses: actions/cache@v4 with: - path: /tmp/debmagic - key: debmagic-build-${{ hashFiles('Cargo.lock', 'debian/changelog', 'debian/control', 'debian/rules') }} - restore-keys: debmagic-build- - - name: Prune stale build trees from the restored cache + path: /tmp/debmagic-data + key: debmagic-data-${{ hashFiles('Cargo.lock', 'debian/changelog', 'debian/control', 'debian/rules') }} + restore-keys: debmagic-data- + - name: Prune Environments of other package versions from the restored cache run: | - mkdir -p /tmp/debmagic current="$(dpkg-parsechangelog -SSource)-$(dpkg-parsechangelog -SVersion)" - find /tmp/debmagic -mindepth 1 -maxdepth 1 -type d \ - ! -name "$current" ! -name "${current}-test" \ - -exec rm -rf {} + + cargo run --locked -q -p debmagic -- env list \ + | awk -v current="$current" 'NR > 1 && $4 != current { print $1 }' \ + | while read -r id; do + cargo run --locked -q -p debmagic -- env clean "$id" \ + || echo "::warning::could not prune Environment $id" + done - name: Resolve CI image tag id: image run: | diff --git a/.gitignore b/.gitignore index 0e11b667..3c3c7ab8 100644 --- a/.gitignore +++ b/.gitignore @@ -30,7 +30,8 @@ src/*/*.lock # testing /*.conf /test -/build +/build-area +/build-debmagic *.snap # ides diff --git a/Cargo.lock b/Cargo.lock index 07ccdad5..35745f02 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,6 +11,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -93,6 +99,15 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -183,6 +198,12 @@ version = "3.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.12.1" @@ -286,6 +307,12 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "colorchoice" version = "1.0.4" @@ -397,6 +424,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -416,6 +458,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -447,11 +498,20 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "ctutils" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03bb0e1cc970d482d121d9a1744999169b69a07470b3d644a7894e53fcaf4574" +dependencies = [ + "cmov", +] + [[package]] name = "deb822-derive" -version = "0.2.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b6e5cafe61e77421a090e2a33b8a2e4e2ff1b106fd906ebade111307064d981" +checksum = "e83ef29a094bcb2b7dd0f609ace7f5a34ef9a62e0731ebd350637640320a3b15" dependencies = [ "proc-macro2", "quote", @@ -459,12 +519,20 @@ dependencies = [ ] [[package]] -name = "deb822-lossless" -version = "0.2.4" +name = "deb822-fast" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812bb5c8052a89edc6d45d1bc3b3400e8186dd166e9b0a9520bfa5a2bd8477ee" +checksum = "114c474fa4cd5d6d24bb5e68b36fa4ef70f5b830e3cc14a9b66a12e71a15aeb9" dependencies = [ "deb822-derive", +] + +[[package]] +name = "deb822-lossless" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45bfeac719b1f3441c6f8c29652c60376e11eabbdbd99c53b3f14e0e8585d053" +dependencies = [ "regex", "rowan", "serde", @@ -482,16 +550,17 @@ dependencies = [ "log", "rowan", "textwrap", - "whoami", + "whoami 1.6.1", ] [[package]] name = "debian-control" -version = "0.1.39" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3367f1dc94222cb9cf16e07c782e442b1928a26dd8dcee3aa4d2dbd32ca11d21" +checksum = "99ffab14a76cf08121d4bf7fb37b40bff0985e49cf050a7cc00b402e0d31e00d" dependencies = [ "chrono", + "deb822-fast", "deb822-lossless", "debversion", "regex", @@ -523,6 +592,8 @@ dependencies = [ "serde_json", "sha1", "sha2 0.11.0", + "sqlx", + "tempfile", "test-case", "tokio", "toml", @@ -577,6 +648,7 @@ dependencies = [ "block-buffer 0.12.1", "const-oid", "crypto-common 0.2.2", + "ctutils", ] [[package]] @@ -620,12 +692,27 @@ dependencies = [ "const-random", ] +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "dunce" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" +dependencies = [ + "serde", +] + [[package]] name = "encoding_rs" version = "0.8.35" @@ -652,18 +739,71 @@ dependencies = [ "typeid", ] +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "etcetera" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de48cc4d1c1d97a20fd819def54b890cadde72ed3ad0c614822a0a433361be96" +dependencies = [ + "cfg-if", + "windows-sys 0.61.2", +] + +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + [[package]] name = "find-msvc-tools" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "645cbb3a84e60b7531617d5ae4e57f7e27308f6445f5abf653209ea76dec8dff" +[[package]] +name = "flume" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + [[package]] name = "foldhash" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -686,6 +826,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -694,6 +835,28 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + [[package]] name = "futures-io" version = "0.3.34" @@ -702,13 +865,13 @@ checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] name = "futures-macro" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 3.0.3", ] [[package]] @@ -725,9 +888,9 @@ checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-util" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-core", "futures-io", @@ -736,7 +899,6 @@ dependencies = [ "futures-task", "memchr", "pin-project-lite", - "pin-utils", "slab", ] @@ -820,7 +982,18 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "foldhash", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", ] [[package]] @@ -838,12 +1011,45 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "hashlink" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824e001ac4f3012dd16a264bec811403a67ca9deb6c102fc5049b32c4574b35f" +dependencies = [ + "hashbrown 0.16.1", +] + [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa26c720c68b866f2c96ef5c1264b3e6f473fe5d4ce61cd44bbe913e553018" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "http" version = "1.5.0" @@ -1239,15 +1445,41 @@ dependencies = [ "bitflags", "libc", "plain", - "redox_syscall", + "redox_syscall 0.9.4", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f111c8c41e7c61a49cd34e44c7619462967221a6443b0ec299e0ac30cfb9b1" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "litemap" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.29" @@ -1349,6 +1581,35 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.18", + "smallvec", + "windows-link", +] + [[package]] name = "pathdiff" version = "0.2.3" @@ -1410,12 +1671,6 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - [[package]] name = "pkg-config" version = "0.3.34" @@ -1550,6 +1805,15 @@ dependencies = [ "rand_core", ] +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + [[package]] name = "redox_syscall" version = "0.9.4" @@ -1708,6 +1972,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + [[package]] name = "rustls" version = "0.23.45" @@ -1807,6 +2084,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "security-framework" version = "3.7.0" @@ -1972,6 +2255,9 @@ name = "smallvec" version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +dependencies = [ + "serde", +] [[package]] name = "smawk" @@ -1989,12 +2275,201 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" +dependencies = [ + "lock_api", +] + +[[package]] +name = "sqlx" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "378620ccc25c62c89d8be1c819e76a88d59bdcc3304733330788948e619bfd71" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b44e85bf579a8eeb4ceaa77a3a523baf2bf0e9bac7e40f405d537b5d2d5ccb" +dependencies = [ + "base64 0.22.1", + "bytes", + "cfg-if", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.16.1", + "hashlink 0.11.1", + "indexmap", + "log", + "memchr", + "percent-encoding", + "serde", + "serde_json", + "sha2 0.10.9", + "smallvec", + "thiserror", + "tokio", + "tokio-stream", + "tracing", + "url", +] + +[[package]] +name = "sqlx-macros" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd2b84f2bc39a5705ef27ec785a11c934a41bbd4a24941e257927cddc26b60bf" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 2.0.111", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb8d96de5fdc85a5c4ec813432b523ec637e80ba98f046555f75f7908ddac7c3" +dependencies = [ + "cfg-if", + "dotenvy", + "either", + "heck", + "hex", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2 0.10.9", + "sqlx-core", + "sqlx-sqlite", + "syn 2.0.111", + "thiserror", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90b8020fe17c5f2c245bfa2505d7ef59c5604839527c740266ad2214acebea27" +dependencies = [ + "bitflags", + "byteorder", + "bytes", + "crc", + "digest 0.11.3", + "dotenvy", + "either", + "futures-core", + "futures-util", + "generic-array", + "log", + "percent-encoding", + "serde", + "sha1", + "sha2 0.11.0", + "sqlx-core", + "thiserror", + "tracing", +] + +[[package]] +name = "sqlx-postgres" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87a2bdd6e83f6b3ea525ca9fee568030508b58355a43d0b2c1674d5f79dcd65e" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags", + "byteorder", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "rand", + "serde", + "serde_json", + "sha2 0.11.0", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "whoami 2.1.3", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488e99c397a62007e4229aec669a179816339afc6d2620ca6fa420dbee2e982c" +dependencies = [ + "atoi", + "flume", + "form_urlencoded", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "sqlx-core", + "thiserror", + "tracing", + "url", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.11.1" @@ -2049,6 +2524,19 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "test-case" version = "3.3.1" @@ -2101,22 +2589,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.17" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.17" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 3.0.3", ] [[package]] @@ -2180,6 +2668,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.19" @@ -2296,10 +2795,23 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "tracing-core" version = "0.1.36" @@ -2333,6 +2845,12 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.22" @@ -2345,6 +2863,21 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "unicode-segmentation" version = "1.12.0" @@ -2399,6 +2932,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" @@ -2552,6 +3091,12 @@ dependencies = [ "wasite", ] +[[package]] +name = "whoami" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" + [[package]] name = "winapi-util" version = "0.1.11" @@ -2731,7 +3276,7 @@ checksum = "2462ea039c445496d8793d052e13787f2b90e750b833afee748e601c17621ed9" dependencies = [ "arraydeque", "encoding_rs", - "hashlink", + "hashlink 0.10.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index fcec6a2d..bd3946d9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ members = ["packages/debmagic", "packages/debmagic-common"] [workspace.package] edition = "2024" -rust-version = "1.89" +rust-version = "1.94" [workspace.dependencies] # internal @@ -20,6 +20,7 @@ glob = ">=0.3.2" libc = ">=0.2.169" serde = { version = ">=1.0.217", features = ["derive"] } serde_json = ">=1.0.139" +tempfile = ">=3.10" toml = ">=0.8" toml_edit = ">=0.22" uuid = { version = ">=1.10.0", features = ["v4", "v5"] } @@ -27,14 +28,16 @@ chrono = { version = ">=0.4.42" } regex = { version = ">=1.12.2" } reqwest = { version = ">=0.13", default-features = false, features = ["rustls", "stream"] } futures-util = "0.3" -tokio = { version = ">=1.43", features = ["macros", "rt-multi-thread"] } +tokio = { version = ">=1.43", features = ["macros", "rt-multi-thread", "sync", "time"] } debian-changelog = { version = ">=0.2.14" } -debian-control = { version = ">=0.1.39" } -deb822-lossless = { version = ">=0.2" } +debian-control = { version = ">=0.3" } +deb822-lossless = { version = ">=0.5" } ignore = { version = ">=0.4.25" } md-5 = { version = ">=0.11" } sha1 = { version = ">=0.11" } sha2 = { version = ">=0.11" } +# `sqlite` bundles libsqlite3; Debian's sqlx maps it to the system library +sqlx = { version = ">=0.9", default-features = false, features = ["runtime-tokio", "sqlite", "macros", "migrate"] } # dev dependencies test-case = { version = ">=3.3.1" } diff --git a/README.md b/README.md index 2ecba0ae..1c889883 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Modern, robust & easy [Debian](https://debian.org)/[Ubuntu](https://ubuntu.com) Debmagic unifies the the packaging experience as a single, streamlined tool. Additionally, `debmagic-pkg` allows you to define package build recipes in Python. - **Isolated builds without the setup**: `debmagic build binary` builds a `debian/`-packaged source tree in a temporary container (using LXD, Incus, Docker) -- **Fast iteration**: `--persistent`/`--incremental` reuse the environment and sync only source changes; `--shell-on-failure` and `debmagic shell` drop you right where the build broke +- **Fast iteration**: `--persistent`/`--incremental` reuse the environment and sync only source changes; a failed build prints `debmagic env shell ` - **Test & sign integrated**: `debmagic test` runs the package's autopkgtest tests in a fresh environment, `debmagic sign` GPG-signs `.changes`/`.dsc`/`.buildinfo` on the host - **Python packaging API**: replace complicated `debian/rules` Makefiles with typed Python `debian/rules.py`, with optional `dh` compatibility @@ -30,7 +30,7 @@ uvx debmagic build binary --driver docker | `debmagic build binary` | Build a binary package (`.deb`) in an isolated environment | | `debmagic build source` | Create a source package (`.dsc`) for upload (incl signing) | | `debmagic test` | Run the package's autopkgtest tests (`debian/tests/`) against a prior build | -| `debmagic shell` | Attach an interactive shell to the build environment | +| `debmagic env` | List, clean, or open a shell in a build or test environment | | `debmagic sign` | GPG-sign a `.changes` file (and its `.dsc`/`.buildinfo`) on the host | | `debmagic config` | Inspect and edit the effective `debmagic.toml` configuration | | `debmagic check` | Lint the package *(in progress)* | diff --git a/debian/control b/debian/control index 4216a09a..5567543d 100644 --- a/debian/control +++ b/debian/control @@ -21,6 +21,7 @@ Build-Depends: librust-libc-dev (>=0.2.169), librust-serde-dev (>=1.0.217), librust-serde-json-dev (>=1.0.139), + librust-tempfile-dev (>=3.10), librust-toml-dev (>=0.8), librust-toml-edit-dev (>=0.22), librust-uuid-dev (>=1.10.0), @@ -31,13 +32,20 @@ Build-Depends: librust-tokio-dev (>= 1.43), librust-test-case-dev (>=3.3.1), librust-pyo3-dev (>=0.27.2), - librust-deb822-lossless-dev (>=0.5.18), + librust-deb822-lossless-dev (>=0.5.19), librust-debian-changelog-dev (>=0.2.14), - librust-debian-control-dev (>= 0.1.39), + librust-debian-control-dev (>= 0.3.17), librust-ignore-dev (>=0.4.25), librust-md-5-dev (>= 0.11), librust-sha1-dev (>= 0.11), librust-sha2-dev (>= 0.11), + librust-sqlx-dev (>= 0.9), + librust-sqlx+runtime-tokio-dev (>= 0.9), + librust-sqlx+sqlite-dev (>= 0.9), + librust-sqlx+macros-dev (>= 0.9), + librust-sqlx+migrate-dev (>= 0.9), + libsqlite3-dev, + pkg-config, Rules-Requires-Root: no X-Style: black Standards-Version: 4.7.2 @@ -73,6 +81,7 @@ Description: dpkg build driver for debmagic rules.py packages. Package: debmagic Architecture: any Depends: + distro-info-data, ${misc:Depends}, ${shlibs:Depends}, Suggests: diff --git a/debian/rules.py b/debian/rules.py index 9c3bb05d..68f00037 100755 --- a/debian/rules.py +++ b/debian/rules.py @@ -17,13 +17,15 @@ ) # TODO: add this section to our rust debmagic module to make setting this up less annoying -os.environ.update({ - "PATH": f"/usr/share/cargo/bin:{os.environ['PATH']}", - "CARGO": "/usr/share/cargo/bin/cargo", - "CARGO_HOME": f"{pkg.base_dir}/debian/cargo_home", - "CARGO_REGISTRY": f"{pkg.base_dir}/debian/cargo_registry", - "DEB_CARGO_CRATE": f"{pkg.build_env.DEB_SOURCE}_{pkg.build_env.DEB_VERSION_UPSTREAM}" -}) +os.environ.update( + { + "PATH": f"/usr/share/cargo/bin:{os.environ['PATH']}", + "CARGO": "/usr/share/cargo/bin/cargo", + "CARGO_HOME": f"{pkg.base_dir}/debian/cargo_home", + "CARGO_REGISTRY": f"{pkg.base_dir}/debian/cargo_registry", + "DEB_CARGO_CRATE": f"{pkg.build_env.DEB_SOURCE}_{pkg.build_env.DEB_VERSION_UPSTREAM}", + } +) packages = { "debmagic": ("packages/debmagic", "debmagic"), @@ -33,10 +35,14 @@ cargo_lock = Path("Cargo.lock") cargo_lock_saved = Path("Cargo.lock.saved") + def dh_auto(build: Build, stage: str, use_destdir: bool = False): for pkg_name, (path, python_pkg_name) in packages.items(): destdir = f" --destdir debian/{pkg_name} " if use_destdir else "" - build.cmd(f"{stage} -p {pkg_name} --sourcedirectory {path} --buildsystem=pybuild {destdir} -- --name {python_pkg_name}") + build.cmd( + f"{stage} -p {pkg_name} --sourcedirectory {path} --buildsystem=pybuild {destdir} -- --name {python_pkg_name}" + ) + @dhp.override def dh_auto_configure(build: Build): @@ -68,4 +74,4 @@ def dh_auto_clean(build: Build): dh_auto(build, "dh_auto_clean") -pkg.pack() \ No newline at end of file +pkg.pack() diff --git a/docs/index.md b/docs/index.md index bff6dd8e..8b4b1055 100644 --- a/docs/index.md +++ b/docs/index.md @@ -9,7 +9,7 @@ Working with Debian packages without `debmagic` means juggling with several inde Debmagic unifies this into one CLI with useful defaults: - **Isolated builds without the setup**: `debmagic build binary` builds a `debian/`-packaged source tree in a temporary container (using LXD, Incus, Docker) -- **Fast iteration**: `--persistent`/`--incremental` reuse the environment and sync only source changes; `--shell-on-failure` and `debmagic shell` drop you right where the build broke +- **Fast iteration**: `--persistent`/`--incremental` reuse the environment and sync only source changes; analyze failures with `debmagic env shell` - **Test & sign integrated**: `debmagic test` runs the package's autopkgtest tests in a fresh environment, `debmagic sign` GPG-signs `.changes`/`.dsc`/`.buildinfo` on the host - **Package uploading**: `debmagic upload` (to `debian`, `ubuntu`, `ppa:user/repo`) pushes signed builds to archives and PPAs via scp/sftp, with configurable pre-upload checks - **Upstream tracking**: `debmagic upstream list`/`switch` replace uscan/uupdate, reading the package's existing `debian/watch` and `debian/copyright` @@ -37,9 +37,11 @@ Then dive into the pages below — start with [Getting started](usage/getting-st usage/getting-started.md usage/build.md usage/test.md +usage/env.md usage/source.md usage/upload.md usage/upstream.md +usage/changelog.md usage/config.md usage/packaging.md usage/modules/index.md diff --git a/docs/usage/build.md b/docs/usage/build.md index 412b9ca5..a6577e7a 100644 --- a/docs/usage/build.md +++ b/docs/usage/build.md @@ -22,21 +22,24 @@ debmagic build binary --driver lxd \ |---|---| | `--driver <...>` | [Build environment driver to use](#picking-a-driver) | | `--source-sync ` | [Which source files to include](#source-file-staging) | -| `--persistent` | Retain the build environment. This does not do incremental builds. | -| `--incremental` | Do incremental builds by syncing changed sources only; implies `persistent` | +| `--persistent ` | How long the build environment outlives this build: `on-failure` (default) keeps it when the build command fails, `always` keeps it after every build, `no` tears it down. A bare `--persistent` means `always` | +| `--incremental` | Do incremental builds by syncing changed sources only; forces `--persistent=always` | | `--distro ` | [Select the target distro/release](#selecting-a-distrorelease) (e.g. `trixie`, `resolute`) | -| `--proposed` | Use [build dependencies from `proposed`](#proposed-dependencies) pocket | +| `--proposed[=]` | Use [build dependencies from `proposed`](#proposed-dependencies) | +| `--experimental[=]` | Use build dependencies from [experimental](#experimental) | +| `--package [=][/]` | Install a [package in a version or from an archive](#package-versions-and-archives) | +| `--archive [=]` | Add an [archive](#extra-archives): `ppa:/` or a `deb` line | | `--sign` | [GPG-sign the resulting `.changes`/`.dsc`/`.buildinfo`](#signing) | | `--clean` | Run [`debian/rules clean` before building](#cleaning) | | `--debug-symbols` | [Build the automatic `-dbgsym` debug symbol packages](#building-debug-symbol-packages) | | `--test` | [Run the package's test suite](#running-tests) | +| `--notify` | [Desktop notification and terminal bell](#notifications) when the build finishes or signing needs a key touch | | `--apt-mirror ` | [Mirror URL](#mirror-selection) | -| `--apt-update-age ` | [When a persistent environment re-runs `apt-get update`](#apt-update-age) | +| `--apt-update-age ` | [When a persistent environment re-runs `apt-get update` and `upgrade`](#apt-update-age) | | `--source-dir ` | Directory containing the `debian/` package directory | | `--output-dir ` | Directory to put the resulting build artifacts | -| `--shell-on-failure` | On build failure, drop into an interactive shell in the build environment when stdout is a TTY | -[`debmagic shell`](#inspecting-a-failed-build) — attach an interactive shell to a build environment +[`debmagic env shell`](#inspecting-a-failed-build) — attach an interactive shell to an Environment ## Picking a driver @@ -52,24 +55,19 @@ There's no auto-detection; pick one and pass it explicitly every time (or config ## Inspecting a failed build -On failure the build environment is torn down by default. Pass `--shell-on-failure` to drop into an interactive shell inside the build environment when stdout is a TTY (destroyed on shell exit unless `--persistent` was used). - -To inspect after the run finishes, pass `--persistent` up front, then: +When the build command itself fails and this run's Persistence is `on-failure` (the default) or `always`, the Environment stays and debmagic prints `debmagic env shell `. Setup failures, and failures after the build command succeeded (export, signing), tear it down. `no` always tears it down and prints no hint. ```shell -# if you're in the package still -debmagic shell -# from the outside: -debmagic shell --source-dir /path/to/parent/of/debian/dir +debmagic env shell ``` -This attaches an interactive shell inside the still-running (or restartable) build environment, at the package's build directory. +Each `debmagic env shell` is its own shell, so you can open more than one. A bare `debmagic env shell` attaches only when this checkout has exactly one Environment. ## Mirror selection Fresh containers install their base tooling plus every `Build-Depends`, so slow mirrors directly translate into slow builds. -Pass `--apt-mirror` to use a faster mirror for build-dependency resolution after the base tooling is bootstrapped from the image's configured archives: +Pass `--apt-mirror` to replace the image's apt sources with a different mirror: ```shell debmagic build binary --driver lxd --apt-mirror http:///ubuntu ... @@ -79,17 +77,18 @@ You can persistently set this flag in `.config/debmagic/config.toml`. ## Apt update age -Fresh environments always run `apt-get update` once on creation. -When a persistent environment is reused, `--apt-update-age` decides whether the apt index is refreshed again: +`--apt-update-age` decides when `apt-get update/upgrade` run again, measured from the last upgrade: | Value | Behavior | |---|---| -| `now` | Update before every build | -| `never` | Only the initial update on creation | -| `1d` (default), `12h`, `30m`, … | Update again once the last one is older than this | +| `now` | Upgrade before every build | +| `never` | Only the initial upgrade on creation | +| `1d` (default), `12h`, `30m`, … | Upgrade again once the last one is older than this | + +The bare driver builds on the host and never upgrades it. The default `1d` matches a typical developer machine's daily apt refresh: repeated builds stay fast, while the index can't go arbitrarily stale. -Persist the setting as `apt_update_age = "1d"` in [`debmagic.toml`](config.md). +Persist the setting as `update_age = "1d"` in the `[apt]` section of [`debmagic.toml`](config.md). ## Source file staging @@ -103,33 +102,44 @@ Before building, `debmagic` stages the source tree into the build environment. | `worktree` | everything that isn't git-ignored, tracked or not | | If the source directory is not a git worktree, `tracked` and `committed` fall back to `worktree` with a warning. +The [output dir](#output-dir) is never staged. Git submodules are skipped with a note, since their contents aren't tracked by the parent repository. To persist a mode, set `source_sync_mode = "committed"` in [`debmagic.toml`](config.md). +## Output dir + +Build artifacts land in `build-area/` under the package source directory by default. +Relative `output_dir` and `output_dir_fallback` config paths resolve against the package source directory; absolute paths are used as given. +`-o` overrides the config: relative paths resolve against debmagic's current working directory, not the source directory. +debmagic marks the output dirs it creates with a [`CACHEDIR.TAG`](https://bford.info/cachedir/), so backup tools skip them. +The tag also tells debmagic's `build-area/` apart from one an upstream project ships: an existing `build-area/` without the tag that holds more than build artifacts belongs to upstream. +debmagic then uses `build-debmagic/` instead (`output_dir_fallback`), or stops with `output_dir_clash = "fail"`. + ## Repeating a build You can iterate on the same build for faster compile times. ### Persistent container -Every `debmagic build binary` invocation creates a new container by default and tears it down afterwards. -For repeated attempts against the same package and distro, add `--persistent` to retain and reuse the running environment while restaging the source tree for each build: +Every `debmagic build binary` keeps the Environment when the build command fails (`--persistent=on-failure`, the default) and tears it down when the build succeeds. Pass `--persistent` (that is, `--persistent=always`) to keep it after success too, and reuse it on the next build while restaging the source tree: ```shell debmagic build binary --driver lxd --persistent \ --source-dir . --output-dir /tmp/out ``` +`--persistent=no` discards a kept Environment and starts fresh. A later `on-failure` success also tears down an Environment a previous `always` left behind. + ### Incremental builds Use `--incremental` to retain the environment and synchronize only source changes while preserving generated files and unchanged source inodes. -This flag implies `--persistent`, and cannot be combined with `--clean yes`. +This flag forces `--persistent=always`, including over an explicit `no` or `on-failure`, and cannot be combined with `--clean yes`. The preserved build tree is kept even when the environment itself is *not* reused (e.g. a fresh CI runner where the tree was restored from a cache). Quilt patches that a previous build or shell session left applied in the build tree are unapplied with `dpkg-source --after-build` before sources are synced, so the worktree remains the source of truth and stale `.pc` state cannot break later builds. -The work dir (`temp_build_dir`, default `/tmp/debmagic`) keeps the artifacts of builds, including their `.changes` files. +The Environment's Host root lives under `environments_dir` (default `$XDG_DATA_HOME/debmagic/environments`). Build artifacts, including `.changes` files, are exported to `output_dir`. ## Selecting a distro/release @@ -140,15 +150,86 @@ If the changelog has a single unambiguous entry, omit it. `--distro` also overrides the changelog's distribution: you can use it to rebuild a package released for an older release on a newer one, or to attempt a backport. On the Bare driver the target must still match the host's os-release; pass `--bare-ignore-release` to build for a different suite anyway, with the host providing the build dependencies itself. -Suite aliases in the changelog (or via `--distro`) resolve to a concrete release: Debian `stable` / `oldstable` / `sid` (→ `unstable`), and Ubuntu `devel`. -Alias targets are updated manually when Debian/Ubuntu roll. +Suite aliases in the changelog (or via `--distro`) resolve to a concrete release: Debian `stable` / `oldstable` / `testing` / `sid` (→ `unstable`), and Ubuntu `devel`. + +The releases and their dates come from [distro-info-data](https://salsa.debian.org/debian/distro-info-data) in `/usr/share/distro-info`. +Without it, or when it predates the latest release, debmagic fetches the data into `~/.cache/debmagic/distro-info/` and refreshes that copy every 30 days. Non-Debian/Ubuntu suites (still apt/dpkg-based) are supported when declared for the active container Driver via `base_images`, e.g. `driver.docker.base_images = { "yocto:kirkstone" = "my-registry/yocto-kirkstone:latest" }`. The changelog/`--distro` value stays the bare codename (`kirkstone`). On the Bare driver, binary builds require the host `/etc/os-release` to match: built-in Debian/Ubuntu need matching `ID` and codename, other suites a matching `VERSION_CODENAME`. ## Proposed dependencies -If needed, build dependencies can be used from `-proposed`. -Pass `--proposed` to enable the proposed pocket in the build environment. +Pass `--proposed` to build against the suite the release's updates come from, like Launchpad builds for `-proposed` do: everything newer there is installed and upgraded. + +| Target | Proposed suite | +|---|---| +| Ubuntu release | `-proposed` | +| Debian testing (devel release) | `unstable` | +| Debian stable release | `-proposed-updates` | +| Debian `unstable`, `experimental` | none | + +Pass a comma-separated list to take only those packages from it, e.g. `--proposed=libfoo1,src:bar`, short for [`--package libfoo1/proposed --package src:bar/proposed`](#package-versions-and-archives). +Everything else stays with the release. + +## Experimental + +A build for Debian `experimental` installs its build dependencies from unstable. +Only when unstable can't satisfy them, the install is retried preferring the packages from experimental. + +Builds for unstable or testing do the same with `--experimental`, or take only the listed packages from it with `--experimental=` like `--proposed`. +Other releases reject it. + +## Package versions and archives + +`--package` sets how one package is installed, in apt-get's syntax: + +```shell +debmagic build --package libfoo1/proposed # from proposed +debmagic build --package src:bar=2.0-1 # all binaries of bar in version 2.0-1 +debmagic build --package libbaz1=1.2-3/proposed +``` + +The package is an [apt preferences](https://manpages.debian.org/apt_preferences) pattern: a binary package, `src:` for all binaries of a source, or a glob, each with an optional `:`. +The archive is `release` (with its updates), `proposed`, `experimental` or an [extra archive](#extra-archives); an archive packages come from is added to the environment. +A version is installed even if older than the installed one, from whichever archive has it. + +The [config](config.md) takes the same as a table: + +```toml +[apt.packages] +libfoo1 = { from = "proposed" } +"src:bar" = { version = "2.0-1" } +``` + +## Extra archives + +`--archive` adds an archive beyond the distro's, a Launchpad PPA or a [sources.list(5)](https://manpages.debian.org/sources.list.5) `deb` line: + +```shell +debmagic build --archive ppa:me/stuff # all newer packages from it +debmagic build --package libfoo1/ppa:me/stuff # only libfoo1 from it +debmagic build --archive mine=ppa:me/stuff --package libfoo1/mine +debmagic build --archive 'mine=deb [signed-by=/home/me/mine.asc] https://example.org/debian trixie main' +``` + +An archive is named after its spec unless `=` names it; `--package` takes packages from it by that name. +Once packages are picked from an archive, its other packages only upgrade installed ones. + +A PPA is taken for the target Ubuntu release; `ppa:/@` picks another, which Debian targets need. +Its signing key is looked up on Launchpad and fetched from keyserver.ubuntu.com before the build starts; if either fails, nothing is built. +The key is then kept in `~/.cache/debmagic/ppa-keys//.asc` and not fetched again; delete it when the PPA's key changes. + +A `deb` line's options are apt's, e.g. `trusted=yes` for an unsigned repository. +A `signed-by` key file is read on the host and copied into the environment; without one, the archive must be signed by a key the environment trusts. +Picking packages from it pins them by the host of its URI, which matches any other archive on that host too. + +In the [config](config.md): + +```toml +[apt.archives] +mine = "ppa:me/stuff" +snapshot = "deb [check-valid-until=no] https://snapshot.debian.org/archive/debian/20260101T000000Z trixie main" +``` ## Building debug symbol packages @@ -164,17 +245,30 @@ Or set `build_debug_symbols = true` in the [`debmagic.toml`](config.md). ## Running tests By default the build runs the package's test suite (the `test` stage of `debian/rules.py`, or `dh_auto_test` via the dh preset). -Pass `--test=false` to skip it for one invocation, or set `run_test = false` in the [`debmagic.toml`](config.md): +Pass `--test=no` to skip it for one invocation, or set `run_test = false` in the [`debmagic.toml`](config.md): ```shell -debmagic build binary --test=false +debmagic build binary --test=no ``` +Boolean flags accept `yes`/`no`, `true`/`false` and `on`/`off`; a bare flag means `yes`. + This exports `DEB_BUILD_OPTIONS=nocheck`, the standard dpkg mechanism: dpkg-buildpackage propagates it into the build, debmagic's `test` stage is skipped, and classic debhelper packages skip `dh_auto_test` as usual. +It also adds the `nocheck` build profile to `DEB_BUILD_PROFILES`, so test-only build dependencies (``) aren't installed. ## Signing `--sign` GPG-signs the resulting `.changes`/`.dsc`/`.buildinfo` after building — mainly useful for [source builds destined for Launchpad](source.md#uploading-to-launchpad), but works for binary builds too. + +Without `--sign`, source builds are signed and binary builds are not (if `sign.key`, or `sign.tool = "custom"` are configured). +`build.sign_source` and `build.sign_binary` change that per kind; `--sign` signs regardless, and `--sign=no` never does. + +```toml +[build] +sign_binary = true # also sign binary builds +sign_source = false # but not source builds +``` + Signing is debmagic's own reimplementation of `debsign` and always runs on the host with your gpg keyring: the artifacts are exported to the host output dir first, so no container or agent forwarding is involved. Children are signed first (`.dsc`, then `.buildinfo`) and the `.changes` checksums are rewritten after each, exactly like `debsign`. @@ -193,11 +287,11 @@ The `--sign` mode decides what happens to an existing signature: | Option | Config | Description | |---|---|---| -| `--sign ` | `sign.source` | Sign after building: `no`, `keep`, `auto` or `force` | +| `--sign ` | `build.sign_source`, `build.sign_binary`, `sign.source` | Sign after building: `no`, `keep`, `auto` (default) or `force`; the config decides what happens without the flag | | `--sign-key ` | `sign.key` | Key ID/fingerprint/email; defaults to the `Changed-By:`/`Maintainer:` address of the file being signed | | `--sign-tool ` | `sign.tool` | OpenPGP implementation: `gpg` (default), `sequoia` (sq), or `custom` | | `--sign-command ` | `sign.sign_command` | Custom signing command for `--sign-tool custom` (see below) | -| `--sign-notify` | `sign.notify` | Desktop notification + terminal bell just before signing, so a hardware-key touch prompt isn't missed after a long build | +| `--notify` | `sign.notify` | Desktop notification + terminal bell just before signing, so a hardware-key touch prompt isn't missed after a long build (see [Notifications](#notifications)) | A custom signing command runs without a shell and must write the clearsigned result to stdout. The file to sign is passed via the `{file}` placeholder (or, if no placeholder is used, as the last argument). @@ -212,6 +306,22 @@ Unknown placeholders are an error. Defaults can be set in [`debmagic.toml`](config.md). +## Notifications + +A desktop notification (`notify-send`) plus a terminal bell tells you when a long build needs you. +The top-level `notify` setting is the default for each step; `build.notify` and `sign.notify` override it per step: + +```toml +notify = true + +[sign] +notify = false # no touch prompt notification, but "build finished" +``` + +A build that signs notifies once, when signing starts: "build finished, signing" if `build.notify` is on, else just the touch prompt if `sign.notify` is. +An unsigned build notifies when it finishes, a failed one in any case. +`--notify[=no]` overrides all steps of one run. + ## Signing an existing build `debmagic sign` signs a `.changes` file (and its `.dsc`/`.buildinfo` children) that already exists — the same code path `--sign` uses after a build: @@ -220,11 +330,11 @@ Defaults can be set in [`debmagic.toml`](config.md). debmagic sign ../mypkg_1.0_amd64.changes ``` -Without a file argument, it locates the `.changes` via `debian/changelog` and `--output`/`-o` (default: the `output_dir` config, `build/` under the package root), preferring the source-only `_source.changes` when several match. +Without a file argument, it locates the `.changes` via `debian/changelog` and `--output`/`-o` (default: the `output_dir` config, `build-area/` under the package source directory), preferring the source-only `_source.changes` when several match. ## Cleaning -`--clean` runs `debian/rules clean` before building, like plain `dpkg-buildpackage` does unless passed `-nc`; `--clean=false` skips it even if the config file defaults to cleaning. +`--clean` runs `debian/rules clean` before building, like plain `dpkg-buildpackage` does unless passed `-nc`; `--clean=no` skips it even if the config file defaults to cleaning. Non-incremental builds already stage a clean source tree, while incremental builds preserve outputs intentionally. Enable cleaning only for packages whose `clean` target performs required setup or code generation. @@ -234,4 +344,4 @@ Instead of repeating CLI flags on every invocation, drop a [config file](config. ## Internals -- Container/device names are derived and sanitized internally (alphanumeric + hyphen, ≤63 chars for LXD/Incus) — don't try to predict or construct them yourself; use `debmagic shell` instead of `lxc`/`docker` commands directly. +- Container/device names are derived and sanitized internally (alphanumeric + hyphen, ≤63 chars for LXD/Incus) — don't try to predict or construct them yourself; use `debmagic env shell` instead of `lxc`/`docker` commands directly. diff --git a/docs/usage/changelog.md b/docs/usage/changelog.md new file mode 100644 index 00000000..1e4f4bac --- /dev/null +++ b/docs/usage/changelog.md @@ -0,0 +1,29 @@ +# Changelog + +`debmagic changelog` edits `debian/changelog`. + +```shell +debmagic changelog new # add an UNRELEASED entry on top, keeping the current version +``` + +## `changelog new` + +Prepends an entry for the current package and version, with distribution `UNRELEASED`, urgency `medium`, an empty `* ` bullet and a trailer signed by you with the current local time: + +```text +foo (1.2.3-1ubuntu2) UNRELEASED; urgency=medium + + * + + -- Larry Cow Mue, 34 Oct 2026 11:67:56 +0200 +``` + +Fill in the bullet and adjust the version in your editor. +The rest of the file is kept byte for byte. + +If the top entry already is `UNRELEASED`, `new` refuses: keep editing that entry instead. + +Your identity is taken from `DEBFULLNAME` and `DEBEMAIL`, then `NAME` and `EMAIL` (an email variable may also carry the name as `Elsa Cow `), then git's `user.name` and `user.email`. +Without one, `new` fails rather than guessing from your login and hostname. + +`--source-dir` points at a package other than the one in the current directory. diff --git a/docs/usage/config.md b/docs/usage/config.md index 781393e6..b0779ad7 100644 --- a/docs/usage/config.md +++ b/docs/usage/config.md @@ -14,6 +14,8 @@ Config files are merged in order of increasing precedence, so later files overri Only files that exist are read; missing ones are skipped. Command-line flags override whatever the merged config resolves to. +`DEBMAGIC_DATA_DIR` relocates the Environment registry (`db.sqlite`) and the default `environments_dir` together. A relative value is resolved to an absolute path against the current working directory. It is not a config file key. + ## Options All keys are optional. @@ -21,28 +23,38 @@ All keys are optional. | Key | Type | Default | CLI flag | Description | |---|---|---|---|---| | `driver.default` | enum | — | `--driver` | Build driver (`docker`, `bare`, `lxd`, `incus`) | -| `driver.persistent` | bool | `false` | `--persistent` | Keep and reuse the build environment across runs instead of tearing it down. | -| `driver.apt_mirror` | string | — | `--apt-mirror` | Mirror used for build-dependency resolution. Not used by the `bare` driver. | -| `driver.proposed` | bool | `false` | `--proposed` | Also enable the `-proposed` pocket. Not used by the `bare` driver. | -| `driver.apt_update_age` | string | `"1d"` | `--apt-update-age` | When a persistent environment re-runs `apt-get update`: `"now"` (every build), `"never"` (only on creation), or a maximum index age like `"1d"`, `"12h"`, `"30m"`. Fresh environments always update once. Not used by the `bare` driver. | +| `driver.persistent` | enum | `"on-failure"` | `--persistent` | How long the Environment outlives the claim: `"no"`, `"on-failure"`, or `"always"`. A bare `--persistent` means `always`. | | `driver.docker.base_images` | map | — | — | Base image per distro, keyed by `":"` (e.g. `"debian:trixie"`). Falls back to `docker.io/:`. For non-Debian/Ubuntu suites (e.g. `"yocto:kirkstone"`), the map entry is what makes the suite a known DistroVersion for Docker builds. | | `driver.lxd.project` | string | — | — | LXD/Incus project to use. | | `driver.lxd.base_images` | map | — | — | Base image per distro, keyed by `":"`. Falls back to the driver's default remote image. Same custom-suite registry role as Docker's map for LXD/Incus. | -| `temp_build_dir` | path | `/tmp/debmagic` | — | Where build trees are staged. | -| `output_dir` | path | `build` | `-o`/`--output-dir` | Where build artifacts are exported; relative paths resolve against the package root. | -| `incremental` | bool | `false` | `--incremental` | Retain the environment and sync only source changes, preserving generated files. Implies `persistent`; incompatible with `clean`. | +| `apt.mirror` | string | — | `--apt-mirror` | Mirror replacing the image's apt sources. | +| `apt.update_age` | string | `"1d"` | `--apt-update-age` | When a persistent environment re-runs `apt-get update && upgrade`: `"now"` (every build), `"never"` (only on creation), or a maximum age like `"1d"`, `"12h"`, `"30m"`. | +| `apt.proposed` | bool | `false` | `--proposed` | All newer packages from the [proposed suite](build.md#proposed-dependencies). | +| `apt.experimental` | bool | `false` | `--experimental` | What nothing else provides from [experimental](build.md#experimental), for unstable and testing. | +| `apt.archives` | map | — | `--archive` | [Extra archives](build.md#extra-archives) by name: a PPA or a `deb` line, e.g. `{ mine = "ppa:me/stuff" }`. | +| `apt.packages` | map | — | `--package` | [Package settings](build.md#package-versions-and-archives) by pattern: `from` (an archive) and `version`, e.g. `{ "src:bar" = { from = "proposed" } }`. | +| `environments_dir` | path | `$XDG_DATA_HOME/debmagic/environments` | — | Parent directory of Environment Host roots. Must be absolute or start with `~/`. The Environment registry is always `$XDG_DATA_HOME/debmagic/db.sqlite` (override both with `DEBMAGIC_DATA_DIR`). | +| `output_dir` | path | `build-area` | `-o`/`--output-dir` | Where build artifacts are exported; relative paths resolve against the package source directory. See [Output dir](build.md#output-dir). | +| `output_dir_fallback` | path | `build-debmagic` | — | Used instead of `output_dir` when upstream owns that directory; relative paths resolve against the package source directory. | +| `output_dir_clash` | enum | `fallback` | — | When upstream owns `output_dir`: `fallback` uses `output_dir_fallback`, `fail` stops. | +| `incremental` | bool | `false` | `--incremental` | Retain the environment and sync only source changes, preserving generated files. Forces Persistence `always`; incompatible with `clean`. | | `source_sync_mode` | enum | `tracked` | `--source-sync` | Which source files are staged (see below). | | `build_debug_symbols` | bool | `false` | `--debug-symbols` | Build the automatic `-dbgsym` debug symbol package. | -| `run_test` | bool | `true` | `--test` | Run the package's test suite during the build; `false` exports `DEB_BUILD_OPTIONS=nocheck` so tests are skipped. | -| `sign.source` | enum | `no` | `--sign` | Sign the resulting `.changes`/`.dsc`: `no` never, `keep` keeps any existing signature, `auto` skips what our key already signed, `force` always re-signs (see below). | +| `run_test` | bool | `true` | `--test` | Run the package's test suite during the build; `false` exports `DEB_BUILD_OPTIONS=nocheck` and the `nocheck` build profile, so tests and their build dependencies are skipped. | +| `sign.source` | enum | `auto` | `--sign` | How to sign the resulting `.changes`/`.dsc`/`.buildinfo`: `no` never, `keep` keeps any existing signature, `auto` skips what our key already signed, `force` always re-signs (see below). Whether a build signs at all is up to `build.sign_source`/`build.sign_binary`. | | `sign.key` | string | — | `--sign-key` | GPG key ID/email to sign with; falls back to the Changed-By/Maintainer address. | | `sign.tool` | enum | `gpg` | `--sign-tool` | OpenPGP implementation: `gpg`, `sequoia` (sq) or `custom` (uses `sign.sign_command`). | | `sign.sign_command` | string | — | `--sign-command` | Custom signing command for `sign.tool = "custom"`, run without a shell with `{file}`/`{key}`/`{email}` placeholders; writes the clearsigned result to stdout. | -| `sign.notify` | bool | `false` | `--sign-notify` | Send a desktop notification via `notify-send` just before signing, so a hardware-key touch prompt isn't missed. | +| `sign.notify` | bool | `notify` | `--notify` | Send a desktop notification via `notify-send` just before signing, so a hardware-key touch prompt isn't missed. | | `clean` | bool | `false` | `--clean` | Run `debian/rules clean` before building. Disabled by default; incompatible with `incremental`. | -| `shell_on_failure` | bool | `false` | `--shell-on-failure` | On build or test failure, drop into an interactive shell in the environment when stdout is a TTY. | +| `notify` | bool | `false` | `--notify` | Default for `build.notify`, `test.notify` and `sign.notify`. | +| `build.notify` | bool | `notify` | `--notify` | Desktop notification via `notify-send` and terminal bell when a build finishes; a build that signs notifies once, when signing starts (see [Notifications](build.md#notifications)). | +| `build.sign_source` | bool | `true` | `--sign` | Sign `build source` results without `--sign`, if a signing key is configured (see [Signing](build.md#signing)). | +| `build.sign_binary` | bool | `false` | `--sign` | Sign `build binary` results without `--sign`, if a signing key is configured. | +| `test.notify` | bool | `notify` | `--notify` | Desktop notification via `notify-send` and terminal bell when `debmagic test` finishes. | | `host_arch_variant` | string | — | `--host-arch-variant` | Build for a dpkg architecture variant (e.g. `"amd64v3"` on Ubuntu) -> `DEB_HOST_ARCH_VARIANT`. | -| `upstream.verify_signatures` | bool | `true` | `--no-signature-check` | Verify the upstream tarball's PGP signature against `debian/upstream/signing-key.asc` during `upstream switch` (see [Upstream](upstream.md#signature-verification)). | +| `upstream.signature_check` | bool | `true` | `--signature-check` | Verify the upstream tarball's PGP signature against `debian/upstream/signing-key.asc` during `upstream fetch`/`switch` (see [Upstream](upstream.md#signature-verification)). | +| `upstream.repack` | bool | `true` | `--repack` | Strip the `Files-Excluded` paths from upstream and custom tarballs during `upstream fetch`/`switch` (see [Upstream](upstream.md#repacking)). | | `upload.targets` | map | — | `debmagic upload ` | Named upload targets for [uploading](upload.md), merged field-by-field over the builtins (`ppa`, `ubuntu`, `debian`). | ### `source_sync_mode` @@ -57,13 +69,14 @@ All keys are optional. | Key | Type | Default | CLI flag | Description | |---|---|---|---|---| -| `source` | enum | `no` | `--sign` | Sign the source package (`.changes`/`.dsc`) after building: `no` never, `keep` keeps any existing signature, `auto` skips what our key already signed, `force` always re-signs. | +| `source` | enum | `auto` | `--sign` | How to sign after building: `no` never, `keep` keeps any existing signature, `auto` skips what our key already signed, `force` always re-signs. Whether a build signs at all is up to `build.sign_source`/`build.sign_binary`. | | `key` | string | — | `--sign-key` | GPG key ID/email to sign with; falls back to the Changed-By/Maintainer address. | | `tool` | enum | `gpg` | `--sign-tool` | OpenPGP implementation: `gpg` (sequoia's `sq`) or `custom` (uses `sign_command`). | -| `notify` | bool | `false` | `--sign-notify` | Desktop notification via `notify-send` before signing. | +| `notify` | bool | top-level `notify` | `--notify` | Desktop notification via `notify-send` before signing. | | `sign_command` | string | — | `--sign-command` | Custom signing command for `tool = "custom"`, like `debsign`'s `-p`. | | `verify_command` | string | — | `--verify-command` (upstream switch) | Custom verification command for `sign.tool = "custom"`, run without a shell with `{file}`/`{signature}`/`{keyring}` placeholders; without `{signature}` the signature path is appended. Falls back to `sign_command`. | | `signed_by_command` | string | — | — | Custom same-key check for `tool = "custom"` under `source = "auto"`: exit code 0 means the file's signature is ours (skip), anything else re-signs. Without it, `auto` always re-signs. | +| `fingerprint_command` | string | — | — | Custom key fingerprint reader for `tool = "custom"`, run without a shell with the `{file}` placeholder (or the key path appended): prints the primary key's fingerprint. Checks the signing keys of [PPAs](build.md#extra-archives). | Signing always runs on the host with your gpg keyring — see [Signing](build.md#signing). ### `orig_tarball` @@ -99,15 +112,19 @@ See [Uploading](upload.md) for target resolution, builtins and pre-upload checks build_debug_symbols = true clean = false +[build] +sign_binary = true # source builds sign by default, binary builds don't + [sign] -source = true key = "you@example.com or gpg key id" notify = true [driver] default = "lxd" -persistent = true -apt_mirror = "http:///ubuntu" +persistent = "always" + +[apt] +mirror = "http:///ubuntu" [driver.docker] # Optional image overrides for known Debian/Ubuntu releases, and the registry diff --git a/docs/usage/env.md b/docs/usage/env.md new file mode 100644 index 00000000..76eec86f --- /dev/null +++ b/docs/usage/env.md @@ -0,0 +1,85 @@ +# Environments + +Every build and test runs in an Environment: a container, VM or chroot that a Driver creates for one package version and distro. +debmagic remembers all Environments on this machine, so you can inspect, reuse and clean them up later. + +```shell +debmagic env list # what exists, and in what state +debmagic env shell [] # open the selected user's login shell +debmagic env exec [] -- ... # run a command directly +debmagic env clean # destroy everything Stale +debmagic env rm ... # destroy specific Environments +``` + +`env` is short for `debmagic environment`. + +## list + +Prints every Environment, after asking each Driver whether it still exists: + +| Column | Meaning | +|---|---| +| ID | Environment id, used by `shell` and `rm` | +| PURPOSE | `build` or `test` | +| DRIVER | Isolation backend | +| PACKAGE | Package name and version | +| DISTRO | Codename | +| PERSIST | Persistence of the run that created it: `no`, `on-failure`, or `always` | +| STATUS | `live`, `stale`, or `unreachable` | +| ATTACH | Number of open `debmagic env shell` sessions | +| SOURCE | Source tree path | + +An Environment is **Stale** when its Driver resource or Host root is gone, or when it has Persistence `no` and neither its creating command nor a shell is still running. +`on-failure` and `always` Environments stay `live` after their command exits. +**Unreachable** means the Driver could not be queried, so debmagic cannot tell whether the Environment still exists. + +## shell + +```shell +debmagic env shell # inside a Source tree: its only Environment +debmagic env shell # anywhere +debmagic env shell -u 1000 +``` + +The interactive command is the selected user's login shell from the Environment's passwd database. +Runs as `root` in containers by default, or as the invoking user with the bare driver; override with `--user`. + +## exec + +```shell +debmagic env exec [] -- dpkg -l 'libfoo*' +debmagic env exec -u 1000 -- sh -c 'echo $HOME' # as the build user +``` + +The command runs directly, without an implicit shell, and debmagic returns its exit code. +It gets a terminal only when debmagic has one, so pipes and redirections work. +Use `sh -c` explicitly when shell expansion is wanted. +Environment selection and `--user` work as for `shell`. + +Without an id, debmagic picks the Environment of the current Source tree. +Environments of the version in `debian/changelog` take precedence over those of earlier versions. +If there are none or several (for example a build and a test Environment), it lists them and asks for an id. + +Each call opens a new shell, so you can open several. +While a shell is open, the Environment is kept, even with Persistence `no`, and a build or test that would reuse it refuses to start instead of resetting it under you. + +## clean and rm + +Each package version gets its own Environment, so kept ones accumulate. + +`debmagic env clean` destroys all **Stale** Environments on the machine and leaves healthy ones alone. +Unreachable Environments are skipped and reported. +An Environment that a concurrent run has just recreated is skipped too. + +`debmagic env rm ...` (alias `remove`) destroys the given Environments, even healthy ones. +It refuses while a shell or the creating command is still running, and refuses Unreachable Environments. +The remaining ids are still processed when one fails. + +If the Driver fails to destroy its resource (for example `docker rm` fails), the Environment stays listed so the resource is not lost; retry once the Driver works. +`--force` removes the Environment anyway, also when it is Unreachable, and only warns if the Driver resource may have leaked. + +## Data directory + +The registry and the Host roots live under `$XDG_DATA_HOME/debmagic` (see [`environments_dir`](config.md)). +Keep this on a local filesystem. +With a network-mounted home directory, set `DEBMAGIC_DATA_DIR` to a local path. diff --git a/docs/usage/source.md b/docs/usage/source.md index 1ed4e142..18c0b91e 100644 --- a/docs/usage/source.md +++ b/docs/usage/source.md @@ -6,7 +6,7 @@ It's a target of the same [`debmagic build`](build.md) command that builds binar ## TL;DR ```shell -# be in debian package directory, output to current directory: +# be in debian package directory, output to build-area/: debmagic build source # or outside the debian package dir: @@ -28,7 +28,7 @@ debmagic build source --sign --sign-key you@example.com \ --upload ppa:your-lp-username/your-ppa # or upload an already-built .changes separately: -debmagic upload ppa:your-lp-username/your-ppa ./build/pkg_*_source.changes +debmagic upload ppa:your-lp-username/your-ppa ./build-area/pkg_*_source.changes ``` - `--sign` GPG-signs the `.dsc`/`.buildinfo`/`.changes` after building, on the host with your gpg keyring — see [Signing](build.md#signing). diff --git a/docs/usage/test.md b/docs/usage/test.md index 96393f17..7d2346dc 100644 --- a/docs/usage/test.md +++ b/docs/usage/test.md @@ -5,7 +5,7 @@ Quick reference for running a package's declared Debian autopkgtest tests with ` ## TL;DR - Entry point: `debmagic test` — runs tests from `debian/tests/control`; needs a prior `debmagic build` -- Requires a completed build in the same build root (or pass `--changes` to point at exported artifacts) +- Requires a completed binary build (looked up from the Environment registry's Invocation history, or pass `--changes`) ```shell cd your-package @@ -17,32 +17,48 @@ debmagic test --driver docker `debmagic test` installs the binary packages from a prior build and runs the package's declared autopkgtest tests (`debian/tests/control`) inside a **fresh, separate** driver-managed environment. -The test environment is never the build environment — even when `--persistent` reuses a container across runs, +The test environment is never the build environment — even when Persistence `on-failure` or `always` reuses a container across runs, the test tree is reset and the `.debs` are reinstalled each time. The driver *is* the testbed: `autopkgtest` runs with the `null` backend inside the container (or on the host for the bare driver). No `autopkgtest-virt-*` backends are used. +## Testing the archive's build + +Naming the package (`src:`) or one of its binaries in a [`--package`](build.md#package-versions-and-archives), `--proposed` or `--experimental` runs the source tree's tests against that build instead of a local one, so no prior build is needed: + +```shell +debmagic test --driver lxd --proposed=libfoo1 # proposed's build +debmagic test --driver lxd --package src:foo/release # what the release ships +debmagic test --driver lxd --package src:foo=1.2-3 # a version +``` + +All binaries of the source get the same setting, so their versions match. +The distro comes from `--distro` or the changelog, and the test output goes to `/_.test`. + ## Available options | Option | Description | |---|---| -| `--driver <...>` | Test environment driver (defaults to the driver recorded in the prior build's `environment.json`) | -| `--persistent` | Retain the test environment after the run for reattach/debug | +| `--driver <...>` | Test environment driver (defaults to the driver recorded on the prior binary-build Invocation) | +| `--persistent ` | How long the test environment outlives this run: `on-failure` (default) keeps it when the tests fail, `always` keeps it after every run, `no` tears it down. A bare `--persistent` means `always` | | `--strict` | Treat skipped tests and "no tests declared" as failures (exit code 2) | -| `--changes ` | Path to a `.changes` file whose directory supplies the built `.debs` (for pipeline use) | -| `--distro ` | Override the target distro for the test environment (defaults to the prior build's distro from `environment.json`, not the changelog) | -| `--proposed` | Enable the `-proposed` pocket in the test environment | +| `--changes ` | Path to a `.changes` file whose directory supplies the built `.debs` (for pipeline use). Overrides the artifact path only; driver and distro still default to the prior binary-build Invocation | +| `--distro ` | Override the target distro for the test environment (defaults to the prior binary-build Invocation's distro, not the changelog) | +| `--proposed[=]` | Test against packages from the [proposed suite](build.md#proposed-dependencies): all newer ones, or only the listed ones. Naming this package [tests the archive's build](#testing-the-archives-build) | +| `--experimental[=]` | Test against the listed packages from [experimental](build.md#experimental); without a list, only what nothing else provides. Naming this package [tests the archive's build](#testing-the-archives-build) | +| `--package [=][/]` | Test against a [package in a version or from an archive](build.md#package-versions-and-archives). Naming this package [tests the archive's build](#testing-the-archives-build) | +| `--archive [=]` | Add an [archive like a PPA](build.md#extra-archives), e.g. to test a PPA's build with `--package src:/` | | `--apt-mirror ` | Mirror URL (same as [`debmagic build`](build.md)) | | `--apt-update-age ` | Configure when to run `apt-get update` (same as [`debmagic build`](build.md#apt-update-age)) | | `--source-dir ` | Directory containing the `debian/` package directory | | `--allow-host-test` | Allow the bare driver, which runs autopkgtest as root on the host | -| `--shell-on-failure` | On test failure, drop into an interactive shell in the test environment when stdout is a TTY | +| `--notify` | Desktop notification and terminal bell when the run finishes | Driver-specific flags (`--driver-docker-base-image`, `--driver-lxd-*`) mirror `debmagic build`. ## Picking a driver -Use the same drivers as for builds. Pass `--driver` explicitly (or rely on the driver recorded in the prior build's `environment.json`): +Use the same drivers as for builds. Pass `--driver` explicitly (or rely on the driver recorded on the prior binary-build Invocation): | Driver | Isolation the Environment provides | |---|---| @@ -66,16 +82,21 @@ If no `debian/tests/control` exists (or it declares no tests), the run exits 0 w ## Inspecting a failed test run -On failure the test environment is torn down by default. Pass `--shell-on-failure` to drop into an interactive shell inside the test environment when stdout is a TTY (destroyed on shell exit unless `--persistent` was used). +When the tests themselves fail and this run's Persistence is `on-failure` (the default) or `always`, the Environment stays and debmagic prints `debmagic env shell `. A strict skip tears the Environment down. `no` always tears it down and prints no hint. -Test output and logs are exported to a `test/` subdirectory of the build root; the path is printed at the end of the run. +Test output and logs are exported to a `.test/` directory next to the `.changes` file (for `pkg_1.0-1_amd64.changes`, that is `pkg_1.0-1_amd64.test/`); the path is printed at the end of the run. A later run replaces that directory only if debmagic wrote it; otherwise the run fails rather than deleting it. + +```shell +debmagic env shell +``` ## Prior build required -By default `debmagic test` resolves the prior build from the build root (same layout as `debmagic shell`). If no build artifacts are found: -run `debmagic build` first +By default `debmagic test` looks up the latest successful binary-build Invocation for this Source tree whose exported `.changes` still exists. If more than one DistroVersion matches, it errors and lists them — pass `--distro`. If none are found: + +run `debmagic build binary` first -Use `--changes` to supply a `.changes` file from an exported output directory instead. +Use `--changes` to supply a `.changes` file from an exported output directory. That overrides the artifact path only. Driver and distro still come from the prior binary-build Invocation when one exists; pass `--driver` and `--distro` when it does not. Several DistroVersions still require `--distro`. ## Bare driver diff --git a/docs/usage/upload.md b/docs/usage/upload.md index 94fe9ced..96d47509 100644 --- a/docs/usage/upload.md +++ b/docs/usage/upload.md @@ -16,7 +16,7 @@ debmagic build source --sign --sign-key you@example.com --upload ppa:your-lp-use debmagic upload --sign ppa:your-lp-username/your-ppa # upload a specific .changes file to a host defined in debmagic.toml: -debmagic upload myhost ./build/pkg_1.0-1_source.changes +debmagic upload myhost ./build-area/pkg_1.0-1_source.changes ``` ## Upload targets @@ -72,7 +72,7 @@ A target's own `pre_upload_commands` run after the global ones. With dput-ng installed, it runs the pre-upload hooks the dput-ng profile of the same name enables (e.g. `ppa`), exactly as `dput` would; pass `--profile ` to use another profile. Without dput-ng, it runs every `"pre": true` hook with an emulation of the python API dput-ng provides, so unmodified dput-ng hook modules work when placed in `/usr/share/dput-ng`, `/etc/dput.d` and `~/.dput.d`. -Pass `--no-hooks` to skip the checks. +Pass `--hooks=no` to skip the checks. ## Signing before the upload diff --git a/docs/usage/upstream.md b/docs/usage/upstream.md index 914cca64..7179a827 100644 --- a/docs/usage/upstream.md +++ b/docs/usage/upstream.md @@ -1,11 +1,13 @@ # Upstream versions -`debmagic upstream` replaces `uscan`/`uupdate`: it reads the package's existing `debian/watch` and `debian/copyright` and offers two operations — checking for new upstream versions and switching the package tree to one. +`debmagic upstream` replaces `uscan`/`uupdate`: it reads the package's existing `debian/watch` and `debian/copyright` and offers three operations — checking for new upstream versions, fetching an orig tarball, and switching the package tree to one. ```shell debmagic upstream list # newest upstream version newer than the changelog's debmagic upstream list --all # every candidate, newest first debmagic upstream list --previous 3 # the 3 newest versions above the current one +debmagic upstream fetch # the orig tarball of the changelog's version +debmagic upstream fetch --from upstream --repack=no # the unmodified upstream release debmagic upstream switch latest # switch the tree to the newest version debmagic upstream switch 3.12.0 # switch to a specific version debmagic upstream switch latest --dry-run @@ -18,16 +20,38 @@ By default only versions newer than the changelog's current upstream version are The changelog's version is normalized with the watch file's `Dversion-Mangle` rules before comparing, so `+dfsg` suffixes and similar don't hide available updates. +## `upstream fetch` + +Puts the `orig` tarball of an upstream version into the package's output dir (`build-area/` by default, `-o` overrides it), where `debmagic build source` finds it. +Without a version it fetches the changelog's upstream version; MUT component tarballs are fetched along with the main one. + +An orig tarball already in the output dir is kept, and one next to the source tree (`../`) is linked into it: re-creating a tarball changes its checksums, and the archive rejects an upload whose orig differs from the one it has. +`--force` replaces it. + +`--from` picks where the tarball comes from: + +- `auto` (default): the configured [`orig_tarball.method`](#orig-tarballs-in-builds), then upstream — a version Debian or Ubuntu published must be reused as-is +- `upstream`: the release `debian/watch` points to +- `launchpad`, `ubuntu`, `debian`: the distro archives +- `custom`: the `orig_tarball.command` + +Upstream and custom tarballs are [repacked](#repacking) unless `--repack=no` (or `upstream.repack = false`) is given. +Then the tarball is kept byte for byte, and every file in it that `Files-Excluded` says must go is listed as a warning. +Distro tarballs are never repacked: the archive already holds them. + +`--dry-run` reports what would happen — method, tarball URL, repack excludes — without downloading anything. + ## `upstream switch` -Fetches the chosen version's tarball, applies the package's repack configuration, writes the `orig` tarball into the package's output dir (`build/` by default) and replaces the source tree's contents — everything except `debian/`, which is kept as-is. -The changelog is not touched; add the new version entry yourself (automated changelog handling is planned). +Fetches the chosen version's orig tarball exactly like `upstream fetch` — with the same options — and replaces the source tree's contents with it, everything except `debian/`, which is kept as-is. +The changelog is not touched; add the new version entry with [`debmagic changelog new`](changelog.md). -`--dry-run` reports what would happen — tarball URL, repack excludes, version suffix — without downloading anything. +The output dir (`build-area/` by default) is kept across the switch. +If the new version starts shipping a directory of that name, the output dir moves to `build-debmagic/` (see [Output dir](build.md#output-dir)); with `output_dir_clash = "fail"` or `-o`, the switch stops before changing anything. For a concrete version, the tarball URL is resolved without scraping the download listing: -1. the distro archives (the `launchpad` orig method) — a version Debian or Ubuntu published needs no upstream download at all +1. the distro archives (with `--from auto`) — a version Debian or Ubuntu published needs no upstream download at all 2. the watch pattern, read as a URL template: `@ANY_VERSION@` becomes the version, `@ARCHIVE_EXT@` each archive extension in turn (the extension of the project's existing orig tarball is probed first), and the constructed URLs are probed — this also finds versions that have already fallen out of the listing 3. the listing, fetched as a *structure teacher*: any listed sibling reveals the prefix/extension shape, and the requested version's URL is constructed from it — this covers patterns pure inversion rejects (multiple capture groups, regex constructs) and compression changes 4. the listing, filtered for the requested version — the last resort @@ -40,11 +64,11 @@ After downloading, the tarball's PGP signature is verified against the keyring i Verification is on by default and uses the same OpenPGP backend as signing — `gpg` by default, `sq` (Sequoia) when `sign.tool = "sequoia"`, or your custom command when `sign.tool = "custom"`: `sign.verify_command` with `{file}`, `{signature}` and `{keyring}` placeholders (without `{signature}` the signature path is appended as the last argument), falling back to `sign.sign_command`. The signature is looked up via the watch file's `Pgp-Sig-Url-Mangle` rules; without them the common suffixes (`.asc`, `.sig`, `.sign`, `.pgp`, `.gpg`) are tried next to the tarball. -If no signature exists upstream, the switch fails — disable the check per-run with `--no-signature-check` or persistently with `verify_signatures = false`: +If no signature exists upstream, the fetch fails — disable the check per-run with `--signature-check=no` or persistently with `signature_check = false`: ```toml [upstream] -verify_signatures = false +signature_check = false ``` Export the upstream key with `gpg --export --armor > debian/upstream/signing-key.asc`. @@ -56,22 +80,31 @@ Two sources are merged: - `Files-Excluded` in `debian/copyright` — the standard way to strip non-DFSG or useless files; `Files-Excluded-` works for multiple-upstream-tarball packages - the watch file's `Repack`/`Repacksuffix` options, e.g. `Repacksuffix: +dfsg` appends the suffix to the upstream version of the repacked tarball -The repacked `orig` tarball uses the canonical Debian layout: a single top-level `_/` directory, xz-compressed. +A tarball containing excluded files is re-created in the canonical Debian layout: a single top-level `-/` directory, xz-compressed. +One without excluded files is used unchanged, unless dpkg-source can't read its compression (e.g. zip or zstd). + +```toml +[upstream] +repack = false # keep upstream tarballs as-is, warning about excluded files +``` ## Watch file support `debian/watch` versions 2–5 are parsed (version 1 is rejected). -The common options work: `Source`, `Matching-Pattern` (including the `@ANY_VERSION@`/`@ARCHIVE_EXT@`/`@PACKAGE@` substitutions), `Search-Mode`, `Uversion-Mangle`, `Dversion-Mangle`, `Filename-Mangle`, `Download-Url-Mangle`, `Pgp-Mode`/`Pgp-Sig-Url-Mangle`, `Repack`, `Repacksuffix`, `Component` (for MUT packages), `Untrackable`. +The common options work: `Source`, `Matching-Pattern` (including the `@ANY_VERSION@`/`@ARCHIVE_EXT@`/`@PACKAGE@` substitutions), `Search-Mode`, `Uversion-Mangle`, `Dversion-Mangle`, `Oversion-Mangle`, `Filename-Mangle`, `Download-Url-Mangle`, `Pgp-Mode`/`Pgp-Sig-Url-Mangle`, `Repack`, `Repacksuffix`, `Component` (for MUT packages), `Untrackable`. + +In version 2–4 lines, the version field may be `debian`, or `same`/`ignore` for MUT components: components are fetched for the main tarball's version, and an `ignore` component that has no such release falls back to its newest one. +The `uupdate` update script is accepted, since `debmagic upstream switch` does its job. http(s) sources are fetched directly; ftp sources fall back to `curl` (install it when a watch file needs it). -Rare options with no debmagic equivalent yet (`Mode: git`/`svn`, `Version-Schema: group/checksum`, `Page-Mangle`, `Update-Script`, ...) produce a clear error suggesting an explicit upstream declaration instead of silently misbehaving. +Rare options with no debmagic equivalent yet (`Mode: git`/`svn`, `Version-Schema: group/checksum`, `Page-Mangle`, custom update scripts, ...) produce a clear error suggesting an explicit upstream declaration instead of silently misbehaving. ## Orig tarballs in builds `debmagic build source` needs the `orig` tarball for `3.0 (quilt)` packages and finds it without any configuration: -1. the package's output dir (`build/` by default) — where `upstream switch` puts it +1. the package's output dir (`build-area/` by default) — where `upstream switch` puts it 2. next to the source tree (`../`), the conventional location — used as-is, never written to 3. the download cache (`~/.cache/debmagic/orig///`) — populated by the `launchpad` method; a cached tarball is hardlinked into the output dir when both are on the same filesystem, copied otherwise 4. the configured fetch method — `launchpad` by default, no configuration needed: diff --git a/packages/debmagic-common/src/apt.rs b/packages/debmagic-common/src/apt.rs new file mode 100644 index 00000000..9f1db177 --- /dev/null +++ b/packages/debmagic-common/src/apt.rs @@ -0,0 +1,1127 @@ +//! What apt needs to know about a target distro: the archive layout a build +//! environment pulls from, and the features of the release's apt. + +use std::{collections::BTreeMap, str::FromStr}; + +use anyhow::bail; +use serde::{Deserialize, Serialize}; + +use crate::distro::{Distro, DistroVersion}; + +/// Debian suites without `-updates`/`-security` pockets: unstable, and +/// experimental, an add-on to it. +const DEBIAN_UNSTABLE: &[&str] = &["unstable", "sid", "experimental"]; +const DEBIAN_ARCHIVE: &str = "http://deb.debian.org/debian"; +/// Debian mirrors don't carry the security pocket. +const DEBIAN_SECURITY_ARCHIVE: &str = "http://deb.debian.org/debian-security"; +const UBUNTU_ARCHIVE: &str = "http://archive.ubuntu.com/ubuntu"; +const UBUNTU_PORTS_ARCHIVE: &str = "http://ports.ubuntu.com/ubuntu-ports"; +const RELEASE: &str = "release"; +const PROPOSED: &str = "proposed"; +const EXPERIMENTAL: &str = "experimental"; +/// The archives a package can be taken from. +const ARCHIVES: &[&str] = &[RELEASE, PROPOSED, EXPERIMENTAL]; + +/// How a build environment installs the packages matching an apt +/// preferences pattern: a binary package name, `src:` for all its +/// binaries, or a glob. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct PackageSettings { + /// The archive to take it from: `release` (with its updates), + /// `proposed` or `experimental`. + pub from: Option, + /// The version to install, even if older, from whichever archive in use + /// has it. + pub version: Option, +} + +/// [`PackageSettings`] by pattern. +pub type Packages = BTreeMap; + +/// A `[=][/]` package spec, apt-get's syntax. +pub fn parse_package(spec: &str) -> Result<(String, PackageSettings), String> { + let (package, from) = match spec.split_once('/') { + Some((package, from)) => (package, Some(from.to_string())), + None => (spec, None), + }; + let (pattern, version) = match package.split_once('=') { + Some((pattern, version)) => (pattern, Some(version.to_string())), + None => (package, None), + }; + validate_pattern(pattern)?; + let settings = PackageSettings { from, version }; + if settings == PackageSettings::default() { + return Err(format!( + "'{spec}' needs a version (=) or an archive (/)" + )); + } + if let Some(invalid) = [&settings.from, &settings.version] + .into_iter() + .flatten() + .find(|value| value.is_empty() || value.contains(char::is_whitespace)) + { + return Err(format!("invalid '{invalid}' in '{spec}'")); + } + Ok((pattern.to_string(), settings)) +} + +fn validate_pattern(pattern: &str) -> Result<(), String> { + match pattern.is_empty() || pattern.contains(char::is_whitespace) { + true => Err(format!("invalid package pattern '{pattern}'")), + false => Ok(()), + } +} + +/// Whether `pattern` names the source package `source` or one of its +/// `binaries`, globs aside. +pub fn names_package(pattern: &str, source: &str, binaries: &[String]) -> bool { + match pattern.strip_prefix("src:") { + Some(name) => without_arch(name) == source, + None => binaries + .iter() + .any(|binary| binary == without_arch(pattern)), + } +} + +fn without_arch(package: &str) -> &str { + package.split_once(':').map_or(package, |(name, _)| name) +} + +/// `--proposed`/`--experimental`: a bool, or a comma-separated list of +/// packages to take from that suite. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Selection { + Off, + /// Whatever apt prefers with the suite enabled. + All, + Only(Vec), +} + +impl FromStr for Selection { + type Err = String; + + fn from_str(value: &str) -> Result { + match value.to_ascii_lowercase().as_str() { + "y" | "yes" | "t" | "true" | "on" | "1" => Ok(Self::All), + "n" | "no" | "f" | "false" | "off" | "0" => Ok(Self::Off), + _ => { + let packages: Vec = + value.split(',').map(|p| p.trim().to_string()).collect(); + for package in &packages { + validate_pattern(package)?; + } + Ok(Self::Only(packages)) + } + } + } +} + +/// An extra archive as `--archive` and `apt.archives` take it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ArchiveSpec { + Ppa(Ppa), + Deb(DebLine), +} + +impl FromStr for ArchiveSpec { + type Err = String; + + fn from_str(spec: &str) -> Result { + match spec.starts_with("ppa:") { + true => spec.parse().map(Self::Ppa), + false => spec.parse().map(Self::Deb), + } + } +} + +/// A sources.list(5) one-line entry, +/// `deb [