diff --git a/.github/workflows/update-lockfiles.yml b/.github/workflows/update-lockfiles.yml index 3b2cdc12491e..0e41d7246407 100644 --- a/.github/workflows/update-lockfiles.yml +++ b/.github/workflows/update-lockfiles.yml @@ -62,7 +62,10 @@ jobs: - name: Regenerate lock files if: runner.os != 'Windows' run: | - python scripts/regenerate-lock-files --show-files --include-sdist + python scripts/regenerate-lock-files \ + --show-files \ + --include-sdist \ + --include-blackbox echo "CHANGES<> $GITHUB_ENV echo "CHANGES=$(git status --porcelain=v1)" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV diff --git a/requirements-test-blackbox-lock.txt b/requirements-test-blackbox-lock.txt new file mode 100644 index 000000000000..acdb388276c0 --- /dev/null +++ b/requirements-test-blackbox-lock.txt @@ -0,0 +1,319 @@ +# +# This file is autogenerated by pip-compile with Python 3.14 +# by the following command: +# +# pip-compile --allow-unsafe --generate-hashes --no-index --output-file=requirements-test-blackbox-lock.txt --unsafe-package=flit-core --unsafe-package=pip --unsafe-package=setuptools --unsafe-package=wheel requirements-test-blackbox.txt +# +anyio==4.15.0 \ + --hash=sha256:7ecd9937369ffce8bba0b5ccb9b3a9507b101b0ed50256aecfbab27e6c2acb99 \ + --hash=sha256:b5c620ed540725e2579c31b17bb995b3bf02c9281c9cace04c7d186380bab85e + # via localstub +build==1.6.0 \ + --hash=sha256:bd2c8afc603e7a2e0ce70e2ea85f0a6d02043bafbd307f5bada0f98669eca5af \ + --hash=sha256:f7aaf1ebbb79178a02ba248bb524f2176b256017e17e8e4bd4289c7b38cc2bad + # via pip-tools +cffi==2.1.1 \ + --hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \ + --hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \ + --hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \ + --hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \ + --hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \ + --hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \ + --hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \ + --hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \ + --hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \ + --hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \ + --hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \ + --hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \ + --hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \ + --hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \ + --hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \ + --hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \ + --hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \ + --hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \ + --hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \ + --hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \ + --hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \ + --hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \ + --hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \ + --hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \ + --hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \ + --hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \ + --hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \ + --hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \ + --hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \ + --hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \ + --hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \ + --hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \ + --hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \ + --hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \ + --hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \ + --hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \ + --hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \ + --hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \ + --hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \ + --hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \ + --hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \ + --hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \ + --hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \ + --hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \ + --hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \ + --hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \ + --hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \ + --hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \ + --hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \ + --hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \ + --hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \ + --hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \ + --hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \ + --hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \ + --hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \ + --hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \ + --hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \ + --hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \ + --hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \ + --hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \ + --hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \ + --hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \ + --hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \ + --hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \ + --hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \ + --hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \ + --hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \ + --hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \ + --hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \ + --hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \ + --hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \ + --hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \ + --hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \ + --hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \ + --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \ + --hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \ + --hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \ + --hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \ + --hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \ + --hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \ + --hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \ + --hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \ + --hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \ + --hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \ + --hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \ + --hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \ + --hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \ + --hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \ + --hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \ + --hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \ + --hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \ + --hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \ + --hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \ + --hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \ + --hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \ + --hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \ + --hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \ + --hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \ + --hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \ + --hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264 + # via cryptography +click==8.5.0 \ + --hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 \ + --hash=sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34 + # via pip-tools +cryptography==50.0.1 \ + --hash=sha256:01f41478cf33fc605a6a089cd56d28b45c6c0b45a1928b61797f2621a04bac71 \ + --hash=sha256:05ba322c4da95b262a212c345af888ef2c37c88c0509756ea00a0e6d68850f23 \ + --hash=sha256:16c5ecd954b3330ebfb6605eca4fd952da8bef376551d5cc264534e3770a9ee6 \ + --hash=sha256:2a93d05e34d5f67fba6f891fe85d929999baa7195e853923ea6d7576c9e68c5e \ + --hash=sha256:2b34d76a652ea2b6faf777c35df230c5637842cd904e04f16230c3f9f03e4361 \ + --hash=sha256:2ebbfb0f1fed745e91796e3e1080a1440423fdae8ece1b995a1d80883a409054 \ + --hash=sha256:30a125032e5642a21ff816e021152bd4e7e94f03eff3f4b7fca41cd22bc3110f \ + --hash=sha256:330fbb252391c596f1ae42c5754449dc924e6ad012dca8efe0d703f9f2d12ec6 \ + --hash=sha256:359e62deae718bce96170e223fdcb6357e4fbd3bb7a3a75f4430763532560e49 \ + --hash=sha256:407fe2b6db00939c05c0e945e9914238f2f0a430974839429dafc82b1ee6bee5 \ + --hash=sha256:42be3bb70596b3abe4ac097b75be223e8b3ab614a0e5de068e3dcc54d71d6149 \ + --hash=sha256:4c4188f7c0cf655be5c06342b817ed0f9595b69ffa2b12026e5353eed29dea88 \ + --hash=sha256:51593d180cf6d179bde5c5d065bed81386b1f381656ae7d042b7ffc87a9895ad \ + --hash=sha256:51afcfceb15597cf2635068e4ac9a56b2abde622edde17f37d85fd7b5306497a \ + --hash=sha256:53e279950892dc102c6b4e52af03ae5ea92fac572a1ddab78ca73a997f62b69f \ + --hash=sha256:55d16b1ef3ee0958d893a977b19777887e546c9954ea81b200c3301a864013f2 \ + --hash=sha256:5dd9bda1c12b4162f6ff568eeb5e0ff956c28d14406e875cfe8a63a2d414ff20 \ + --hash=sha256:5fe002589592ed749ce77fe0695fcbd3500dd61d7d6db5858a7544c612fa8e45 \ + --hash=sha256:5fe939deeb161024a6be98229c953b6591fef1f41214497a78fe793a244c017f \ + --hash=sha256:693c99b49bd37d0d096e4334c10232c77248c415b98d35236094cdf96d57258b \ + --hash=sha256:76de83fbd91ac49c0feaaa983d0748fd7a53176afac5fb3bf7478d244f0eb527 \ + --hash=sha256:79bf008d1f9af6071c797ad133e39915dfee7614f18f18f4db9072eb715064a3 \ + --hash=sha256:804728ce710890870f3aaa344b2e161172d258d768ac139d02cfd9092d0d94e6 \ + --hash=sha256:8921d58f426793c5f1b47f0b59575780de9a095214958d0eb37d909593db8367 \ + --hash=sha256:8df2de9102026855887e4587084f6eabd80ed0f345b8ad8a7ac27ab9bf4723e0 \ + --hash=sha256:9cb3cb952cf5a8abd50c782a98a89d71699715e802fe349704b47f2425b42a94 \ + --hash=sha256:9dde0a357190eb3b1da1bb9ab750e9c85cba82ca5977aa0836cbb94e92611239 \ + --hash=sha256:9ebcdd5519be9b652a46f507817a74591774fc3d6923ac364e4dfa64e36b291b \ + --hash=sha256:a0b1a59e3a089064a0ec309e9428c8e3ae4e161419d20ac33600767e83fc658a \ + --hash=sha256:a255449073358275b64b67d3f595f268bbef70e72b6edb65e0c70c735bf739c9 \ + --hash=sha256:a8f40ea47330e71b594a7e246898f93177c259490c63183dbaf9e571d71ed9a5 \ + --hash=sha256:ac02b07824d4d1001bd4367599f839c19cb171924c796e52c23508ac14c2c0cc \ + --hash=sha256:aed8db4f6d71c51efb89530e12d9464e7bf2923d46c3205dc794a2a93f8c0648 \ + --hash=sha256:b8f852c65863251b9e3a1b8c150ce21e59b522dbb6a7d4bc80e680d38388e986 \ + --hash=sha256:be224a65493ec5b74a158ff22a5522ce4a5ca1e543c647a3a4730d4a09e5f959 \ + --hash=sha256:ca83d00d9e69cd5eb63f2e69c3a5a59e0cecae5ae14c6ae0b35830fe3b37bad0 \ + --hash=sha256:cbf74a81765ee67413503ca6e26dcc4f6f5a519822436cc0a1b97aab6c1b8a17 \ + --hash=sha256:d63ae8f6481fec907ac0f588eee8a90aefde112c633131fe540e5711ddbb5a4e \ + --hash=sha256:e22dfed744bd4002e909464cb23d2f0b05c6f3113a79ef2e9864a53db737c733 \ + --hash=sha256:e2ca8fd1b6b4b82a1c4cb02841d0837e3c12336c2e24b520ab8ab3b969733d8f \ + --hash=sha256:e74591e283fe6eb956416c929eb58262a719fe0311fd9054c62c3350ed8760d8 \ + --hash=sha256:f74455bb086a85d5e81246412602aaa97ed095e504cd40dd261ef50be42205bf \ + --hash=sha256:fb4b9672d389c738b175c4166e78310f8a70358886aacd9173ee03a85ffdc671 \ + --hash=sha256:fc3ed7ebd2a8c96f5b166de0ab9b624996bef3b07bbeb19364dfb78222c22c80 \ + --hash=sha256:fd3718b960d0b5dd213cdf03f3bcb7000e69dda0de8b956061947ff6bcff5558 \ + --hash=sha256:ff838d62ec1bfce4f9ba7fa16f4a7b554cd8d0c299e6be37502161a660c84eef + # via + # localstub + # trustme +execnet==2.1.2 \ + --hash=sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd \ + --hash=sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec + # via pytest-xdist +httptools==0.8.0 \ + --hash=sha256:0770728beb05094c809b98e814edff5fef69d26ad7d21185f2f6d5884a0ba683 \ + --hash=sha256:0ea897f0c729581ebf72131a438a7932d9b14efef72d75ada966700cac3caaeb \ + --hash=sha256:159e9ab5f701ccd42e555a12f1ad8ff69702910fc1c996cf2bb66e5fcb7a231b \ + --hash=sha256:19d1ee275bb59ba2643ba9a3a1e51cc0c788caf2b8df506368e03f56fdd08527 \ + --hash=sha256:20b4aac66ff65f7db06a375808b78f42a94970aa22e826b3cb2b43eb09174124 \ + --hash=sha256:2a021c3a8e65cc125390d72f59b968afca3bdcaff25bd67965e0a055a14946ca \ + --hash=sha256:2c032fa028f46871ec7e1fc59fc15e8023eab3e6bbe6ece786a1611719a5d081 \ + --hash=sha256:2d689918c15a013c65ef52d9fd495d766893ab831a2c8d89f2ac5940a5df847c \ + --hash=sha256:384c17174464c8e873398b7af24f0b1f44d992c820328413951a625323155d77 \ + --hash=sha256:425f83884fd6343828d8c565f046cb72b6d19063f6924093e11bcd8e1548cd09 \ + --hash=sha256:48774d39cbb70e2b1f71f88852a3087ae1d3a1eb80482bb48c13067ab080c14f \ + --hash=sha256:52dd695b865fe96d9d2b16b64a895f3f57bf3cb064e8383cd3b5713a069e8085 \ + --hash=sha256:57278e6fa0424c42a8a3e454828ab4f0aff27b40cddf9679579b98c6dce6a376 \ + --hash=sha256:5931891fb7b441b8a3853cf1b85c82c903defce084dd5f6771ca46e31bf862c5 \ + --hash=sha256:5d7fa4ba7292c1139c0526f0b5aad507c6263c948206ea1b1cbca015c8af1b62 \ + --hash=sha256:5eb911c515b96ee44bbd861e42cbefc488681d450545b1d02127f6136e3a86f5 \ + --hash=sha256:614ceea8ea606848bece2338ac03b3ce5324bcb4be8dc7d377ed708012fa4db8 \ + --hash=sha256:6a43c9dd399758ccc0531acb0a3c4a6c299ee893ee9400e9c893b7bdcfae0681 \ + --hash=sha256:6b2a32f18d97e16e90827d7a819ffa8dbd8cc245fc4e1fa9d1095b54ef4bd999 \ + --hash=sha256:7685df791fad561384bfb139e77fde27a1ffd93134e016f95a0db424ffbf77b1 \ + --hash=sha256:7b71e7d7031928c650e1006e6c03e911bf967f7c69c011d37d541c3e7bf55005 \ + --hash=sha256:880490234c10f70a9830743097e8958d6e4b9f5a0ffc24515023afeef984054d \ + --hash=sha256:88bdd940f2b5d487b4d032c6afa5489a7dc4694410d43de3c38c4fb3af0dc45d \ + --hash=sha256:88eead8ec8680a9f146c655bc88445a325bd7921cfd8194c7337e9467282427d \ + --hash=sha256:9518c406d7b310f05adb1a37f80acabac40504a575d7c0da6d3e365c695ac20d \ + --hash=sha256:9878eb2785ba5eb70631ad269b37976f73d647955e26c91d490eb8a4edfda4ba \ + --hash=sha256:9fc1644f415372cec4f8a5be3a64183737398f10dbb1263602a036427fe75247 \ + --hash=sha256:a1afd7c9fbff0d9f5d489c4ce2768bd09c84a46ddefc7161e6aa82ae35c85745 \ + --hash=sha256:a1b4c8e7a489a0d750d91894e9a8cdc295838f1924c0ca903ae993456fddec07 \ + --hash=sha256:a3b7387147361c3fd47a0bde763c5c91b5b4cd4dc9989b8ece84ff436c99843b \ + --hash=sha256:a6f21e2a3b0067bbe7f67e34cfd16276af556e5e52f4c7503be0cb5f90e905e4 \ + --hash=sha256:b15fc622b0f869d19207c4089a501d9bcc63ca5e071ffdd2f03f922df882dcb2 \ + --hash=sha256:b205e5f5523fa039679da0dfe5a10132b2a4abeae6a86fdd1ddc035f7f836557 \ + --hash=sha256:bbb8caadb2b742d293169d2b458b5c001ef70e3158704aa3d3ef9597624c5d1d \ + --hash=sha256:bf3b6f807c8541503cecfbb8a8dffb385640d0d96102f3d112aa8740f9b7c826 \ + --hash=sha256:c08ffe3e79756e0963cbc8fe410139f38a5884874b6f2e17761bef6563fdcd9b \ + --hash=sha256:c0d726cc107fceb7d45f978483b4b70dd8caa836f5914d3434bb18628eb73813 \ + --hash=sha256:c4a9f1707e4823d54dfec6c33fa3697d302aed536ed352a7ebb5a061ddb869d0 \ + --hash=sha256:cd96f29b4bab1d42fa6e3d008711c75e0f79e94e06827330160e3a304227f150 \ + --hash=sha256:d76ad7b951387e3632c8716a9bb03ac5b45c5f16119aa409db0459520887944e \ + --hash=sha256:da684f2e1aa2ee9bdcb083f3f3a68c5956750b375bc5df864d3a5f0c42a40b77 \ + --hash=sha256:de1ed58a974e75d56560acc7e7fed01a454994429456f65209789992e41f2568 \ + --hash=sha256:de242a49b5d18e0a8776e654e9f6bf6d89f3875a5c35b425a0e7ce940feb3fd6 \ + --hash=sha256:df31ef5494f406ab6cf827b7e64a22841c6e2d654100e6a116ea15b46d02d5e8 \ + --hash=sha256:e93c227b595c6926c1acee96891dd9da4be338cfbe82e5cd3bb9d8dd7dc4ac0b \ + --hash=sha256:eb3028cca2fc0a6d720e52ef61d8ebb62fcbfeb1de56874546d858d3f25a26b7 \ + --hash=sha256:ed377e64805bdba4943c82717333f8f8603a13b09aff9cead2717c6c817fb168 \ + --hash=sha256:ef7c3c97f4311c7be57e2986629df89d49cb434dbff78eafcd48c2bff986b15a \ + --hash=sha256:f256d6ce930c52ca1cb2a960b7da03548c454e7d28b06059ad41bfe789036ce0 \ + --hash=sha256:fe2a4c95aeba2209434e7b31172da572846cae8ca0bf1e7013e61b99fbbf5e72 + # via localstub +idna==3.19 \ + --hash=sha256:5e0811a4383b21dc5838069f801c4fb62113b7447663d2530d2bd6e77b49bf15 \ + --hash=sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4 + # via + # anyio + # trustme +iniconfig==2.3.0 \ + --hash=sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730 \ + --hash=sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12 + # via pytest +localstub==0.0.3 \ + --hash=sha256:62c7ff8930a73d013c2a1dce19fbdb5e7547c50915c043afd9a191bcb1d4916a \ + --hash=sha256:db3cdf67691177d8e6c27a1af9b2205826996289ad339ca4c7c4bcd46daa1310 + # via -r requirements-test-blackbox.txt +markdown-it-py==4.2.0 \ + --hash=sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49 \ + --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a + # via rich +mdurl==0.1.2 \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba + # via markdown-it-py +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via + # build + # pytest + # wheel +pip-tools==7.6.1 \ + --hash=sha256:6111c8b4b07fd14b7223ca921485b0e96cf66e20bf94da95eeed9845f510cb8f \ + --hash=sha256:695556edeb647eb94ee8345cc7108657fdb7fb16b3876623a399b4f61bbede01 + # via -r requirements-test-blackbox.txt +pluggy==1.6.0 \ + --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ + --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 + # via pytest +pycparser==3.0 \ + --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ + --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 + # via cffi +pygments==2.21.0 \ + --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 \ + --hash=sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c + # via + # pytest + # rich +pyproject-hooks==1.2.0 \ + --hash=sha256:1e859bd5c40fae9448642dd871adf459e5e2084186e8d2c2a79a824c970da1f8 \ + --hash=sha256:9e5c6bfa8dcc30091c74b0cf803c81fdd29d94f01992a7707bc97babb1141913 + # via + # build + # pip-tools +pytest==8.4.0 \ + --hash=sha256:14d920b48472ea0dbf68e45b96cd1ffda4705f33307dcc86c676c1b5104838a6 \ + --hash=sha256:f40f825768ad76c0977cbacdf1fd37c6f7a468e460ea6a0636078f8972d4517e + # via + # -r requirements-test-blackbox.txt + # pytest-xdist +pytest-xdist==3.6.1 \ + --hash=sha256:9ed4adfb68a016610848639bb7e02c9352d5d9f03d04809919e2dafc3be4cca7 \ + --hash=sha256:ead156a4db231eec769737f57668ef58a2084a34b2e55c4a8fa20d861107300d + # via -r requirements-test-blackbox.txt +rich==15.0.0 \ + --hash=sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb \ + --hash=sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36 + # via localstub +trustme==1.2.1 \ + --hash=sha256:6528ba2bbc7f2db41f33825c8dd13e3e3eb9d334ba0f909713c8c3139f4ae47f \ + --hash=sha256:d768e5fc57c86dfc5ec9365102e9b092541cd6954b35d8c1eea01a84f35a762a + # via localstub +truststore==0.10.4 \ + --hash=sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301 \ + --hash=sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981 + # via localstub +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via anyio + +# The following packages are considered to be unsafe in a requirements file: +pip==26.2.1 \ + --hash=sha256:71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e \ + --hash=sha256:f6ad667e89a1fe78046c8f13232b247200f5258d7828f3f7883d660878e0813f + # via pip-tools +setuptools==84.0.0 \ + --hash=sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670 \ + --hash=sha256:f4695c21257f0d9b537ec2692c941d02ee143b7cc1276941349a546573b2ef73 + # via pip-tools +wheel==0.48.0 \ + --hash=sha256:3217dcc807155e45db462d7ef2431f5ddda0d7273b700d05a67b271ceb1287ab \ + --hash=sha256:94800765601e9171bf5d58d066e640662842bcedcbab982b2c90787a2c987322 + # via pip-tools diff --git a/requirements-test-blackbox.txt b/requirements-test-blackbox.txt new file mode 100644 index 000000000000..838348956847 --- /dev/null +++ b/requirements-test-blackbox.txt @@ -0,0 +1,6 @@ +pytest==8.4.0 +pytest-xdist==3.6.1 +pip-tools==7.6.1 +# localstub is a required dependency of +# blackbox tests. +localstub==0.0.3 diff --git a/scripts/regenerate-lock-files b/scripts/regenerate-lock-files index daaff106865e..108b5924d970 100755 --- a/scripts/regenerate-lock-files +++ b/scripts/regenerate-lock-files @@ -103,7 +103,7 @@ class LockFileBuilder: f.write(content) -def show_files(build_directory: Path, include_sdist: bool, include_base: bool): +def show_files(build_directory: Path, include_sdist: bool, include_base: bool, include_blackbox: bool): if include_sdist: for root, dirs, files in os.walk(build_directory / 'requirements'): for filename in files: @@ -115,6 +115,11 @@ def show_files(build_directory: Path, include_sdist: bool, include_base: bool): stemmed_filename = Path(filename).stem if stemmed_filename.endswith('-lock'): show_file(filename) + if include_blackbox: + for filename in Path.iterdir(build_directory): + stemmed_filename = Path(filename).stem + if stemmed_filename.endswith('-lock') and 'blackbox' in stemmed_filename: + show_file(filename) def show_file(path: Path): @@ -128,6 +133,7 @@ def main( should_show_files: bool, include_sdist: bool, include_base: bool, + include_blackbox: bool, ): builder = LockFileBuilder( source_directory=ROOT, @@ -191,8 +197,16 @@ def main( output=Path("requirements-docs"), allow_unsafe=True, ) + if include_blackbox: + builder.build_lock_file( + sources=[ + Path("requirements-test-blackbox.txt"), + ], + output=Path("requirements-test-blackbox"), + allow_unsafe=True, + ) if should_show_files: - show_files(build_directory, include_sdist, include_base) + show_files(build_directory, include_sdist, include_base, include_blackbox) if __name__ == "__main__": @@ -218,10 +232,16 @@ if __name__ == "__main__": '--no-include-base', action='store_false', dest='include_base' ) parser.set_defaults(include_base=False) + parser.add_argument('--include-blackbox', action='store_true') + parser.add_argument( + '--no-include-blackbox', action='store_false', dest='include_blackbox' + ) + parser.set_defaults(include_blackbox=False) args = parser.parse_args() main( args.output_directory, args.show_files, args.include_sdist, args.include_base, + args.include_blackbox, ) diff --git a/tests/blackbox/conftest.py b/tests/blackbox/conftest.py new file mode 100644 index 000000000000..fb0258207053 --- /dev/null +++ b/tests/blackbox/conftest.py @@ -0,0 +1,82 @@ +"""Shared fixtures for blackbox tests.""" + +from __future__ import annotations + +import asyncio +import os +import shutil + +import pytest + +from tests.blackbox.utils import cli_env, mock_server, run_cli +from localstub.server import HTTPResponse + + +def pytest_configure(config): + """Heuristic to verify if the CLI binary routes traffic through + HTTPS_PROXY before any test runs. + + If the proxy receives no request, the binary under test does not support + HTTPS_PROXY and all tests would be invalid — fail the session immediately. + """ + command = os.environ.get("AWS_TEST_COMMAND") or shutil.which("aws") + if command is None: + pytest.exit( + "No AWS CLI binary found: set AWS_TEST_COMMAND to its path or " + "make `aws` available on PATH.", + returncode=1, + ) + + async def _verify(): + + async with mock_server() as (server, proxy): + server.set_response_sequence([ + HTTPResponse.raw( + b'' + b"AccessDenied" + b"Test", + status=403, + headers={"Content-Type": "application/xml"}, + ), + ]) + env = cli_env(proxy) + await run_cli(command, ["s3", "ls"], env) + return len(server.requests) > 0 + + proxy_works = asyncio.run(_verify()) + if not proxy_works: + pytest.exit( + f"CLI binary {command!r} does not route traffic through " + f"HTTPS_PROXY. All blackbox tests require proxy support to " + f"prevent accidental calls to production AWS endpoints.", + returncode=1, + ) + + +@pytest.fixture +def aws_cli() -> str: + command = os.environ.get("AWS_TEST_COMMAND") or shutil.which("aws") + if command is None: + pytest.fail( + "No AWS CLI binary found: set AWS_TEST_COMMAND to its path or " + "make `aws` available on PATH." + ) + return command + + +@pytest.fixture +def aws_config(tmp_path): + """Factory fixture that writes an AWS config file and returns its path.""" + + def _make(config_dict: dict[str, dict[str, str]]) -> str: + lines = [] + for section, values in config_dict.items(): + lines.append(f"[{section}]") + for key, val in values.items(): + lines.append(f"{key} = {val}") + lines.append("") + path = tmp_path / "config" + path.write_text("\n".join(lines)) + return str(path) + + return _make diff --git a/tests/blackbox/s3_assertions.py b/tests/blackbox/s3_assertions.py new file mode 100644 index 000000000000..4c34ededcbe6 --- /dev/null +++ b/tests/blackbox/s3_assertions.py @@ -0,0 +1,698 @@ +"""Per-operation S3 assertion helpers.""" +from __future__ import annotations + +import xml.etree.ElementTree as ET +from urllib.parse import parse_qs, urlparse + + +# Known list wrapper elements in S3 XML request bodies. +_LIST_ELEMENTS = {"Tags", "TagSet", "Parts"} + + +def _check_bucket(request, bucket: str, addressing_style: str, op_name: str): + """Assert Bucket is in the correct location based on addressing style.""" + parsed_url = urlparse(request.path) + host = request.headers.get("host", "") + if bucket.startswith("arn:"): + resource = bucket.split(":", 5)[-1] + name = resource.split("/")[-1].split(":")[-1] + assert name in host, ( + f"{op_name}.Bucket: expected {name!r} (from ARN) in host, got {host!r}" + ) + elif addressing_style == "path": + assert parsed_url.path.startswith(f"/{bucket}"), ( + f"{op_name}.Bucket: expected path-style /{bucket}/..., got {parsed_url.path!r}" + ) + else: + assert bucket in host, ( + f"{op_name}.Bucket: expected {bucket!r} in host, got {host!r}" + ) + + +def _check_key(request, key: str, addressing_style: str, op_name: str): + """Assert Key is in the correct path location based on addressing style.""" + parsed_url = urlparse(request.path) + if addressing_style == "path": + assert parsed_url.path.endswith(f"/{key}"), ( + f"{op_name}.Key: expected path ending with /{key}, got {parsed_url.path!r}" + ) + else: + assert parsed_url.path == f"/{key}", ( + f"{op_name}.Key: expected /{key}, got {parsed_url.path!r}" + ) + + +def _check_params(request, params: dict, param_map: dict, op_name: str): + """Check optional params against the wire using a lookup map. + + param_map maps param_name -> (location, wire_name) where location + is "header", "headers", or "querystring". + """ + parsed_url = urlparse(request.path) + actual_qs = parse_qs(parsed_url.query) + for param_name, expected in params.items(): + if param_name not in param_map: + raise ValueError(f"{op_name}: unknown param {param_name!r}") + loc, wire = param_map[param_name] + if loc == "header": + actual = ( + request.headers.get(wire) or request.headers.get(wire.lower()) + ) + assert actual == expected, ( + f"{op_name}.{param_name}: expected {wire}={expected!r}, got {actual!r}" + ) + elif loc == "headers": + prefix = wire.lower() + assert isinstance(expected, dict) + for k, v in expected.items(): + hdr = f"{prefix}{k.lower()}" + actual = request.headers.get(hdr) + assert actual == v, ( + f"{op_name}.{param_name}[{k}]: expected {hdr}={v!r}, got {actual!r}" + ) + elif loc == "querystring": + actual_list = actual_qs.get(wire, []) + actual = actual_list[0] if actual_list else None + assert actual == expected, ( + f"{op_name}.{param_name}: expected ?{wire}={expected!r}, got {actual!r}" + ) + elif loc == "payload": + actual_body = _parse_xml_body(request.body, wire) + _assert_subset(actual_body, expected, f"{op_name}.{param_name}", + coerce_strings=True) + + +def _assert_subset(actual, expected, path: str, coerce_strings: bool = False): + """Assert expected is a subset of actual (recursive). + + When coerce_strings is True, leaf values are compared as strings. + This is used for XML body comparisons where the parser returns all + values as strings, but test authors may pass native types. + """ + if isinstance(expected, dict): + assert isinstance(actual, dict), f"{path}: expected dict, got {type(actual)}" + for k, v in expected.items(): + assert k in actual, f"{path}: missing key {k!r}, have {list(actual.keys())}" + _assert_subset(actual[k], v, f"{path}.{k}", coerce_strings) + elif isinstance(expected, list): + assert isinstance(actual, list), f"{path}: expected list, got {type(actual)}" + for i, item in enumerate(expected): + found = any(_matches(a, item, coerce_strings) for a in actual) + assert found, f"{path}[{i}]: {item!r} not found in {actual!r}" + else: + if coerce_strings: + assert str(actual) == str(expected), ( + f"{path}: expected {expected!r}, got {actual!r}" + ) + else: + assert actual == expected, ( + f"{path}: expected {expected!r}, got {actual!r}" + ) + + +def _matches(actual, expected, coerce_strings: bool = False) -> bool: + try: + _assert_subset(actual, expected, "", coerce_strings) + return True + except AssertionError: + return False + + +def _parse_xml_body(body: str | bytes, ns: str) -> dict | None: + """Parse an XML request body into a dict, handling namespace.""" + if not body: + return None + root = ET.fromstring(body if isinstance(body, str) else body.decode("utf-8")) + ns_prefix = f"{{{ns}}}" if ns else "" + return _xml_to_dict(root, ns_prefix) + + +def _xml_to_dict(element, ns_prefix: str) -> dict | str | list: + """Recursively convert XML element to dict.""" + children = list(element) + if not children: + return element.text + own_tag = element.tag + if own_tag.startswith(ns_prefix): + own_tag = own_tag[len(ns_prefix):] + child_tags = set() + for c in children: + tag = c.tag + if tag.startswith(ns_prefix): + tag = tag[len(ns_prefix):] + child_tags.add(tag) + is_list = ( + (len(child_tags) == 1 and len(children) > 1) + or own_tag in _LIST_ELEMENTS + ) + if is_list: + return [_xml_to_dict(c, ns_prefix) for c in children] + result = {} + for child in children: + tag = child.tag + if tag.startswith(ns_prefix): + tag = tag[len(ns_prefix):] + child_value = _xml_to_dict(child, ns_prefix) + if tag in result: + existing = result[tag] + if isinstance(existing, list): + existing.append(child_value) + else: + result[tag] = [existing, child_value] + else: + result[tag] = child_value + return result + + +# PutObject: PUT /{Bucket}/{Key+} +_PUT_OBJECT_PARAMS = { + "ACL": ("header", "x-amz-acl"), + "CacheControl": ("header", "Cache-Control"), + "ContentDisposition": ("header", "Content-Disposition"), + "ContentEncoding": ("header", "Content-Encoding"), + "ContentLanguage": ("header", "Content-Language"), + "ContentLength": ("header", "Content-Length"), + "ContentMD5": ("header", "Content-MD5"), + "ContentType": ("header", "Content-Type"), + "ChecksumAlgorithm": ("header", "x-amz-sdk-checksum-algorithm"), + "ChecksumCRC32": ("header", "x-amz-checksum-crc32"), + "ChecksumCRC32C": ("header", "x-amz-checksum-crc32c"), + "ChecksumCRC64NVME": ("header", "x-amz-checksum-crc64nvme"), + "ChecksumSHA1": ("header", "x-amz-checksum-sha1"), + "ChecksumSHA256": ("header", "x-amz-checksum-sha256"), + "ChecksumSHA512": ("header", "x-amz-checksum-sha512"), + "ChecksumMD5": ("header", "x-amz-checksum-md5"), + "ChecksumXXHASH64": ("header", "x-amz-checksum-xxhash64"), + "ChecksumXXHASH3": ("header", "x-amz-checksum-xxhash3"), + "ChecksumXXHASH128": ("header", "x-amz-checksum-xxhash128"), + "Expires": ("header", "Expires"), + "IfMatch": ("header", "If-Match"), + "IfNoneMatch": ("header", "If-None-Match"), + "GrantFullControl": ("header", "x-amz-grant-full-control"), + "GrantRead": ("header", "x-amz-grant-read"), + "GrantReadACP": ("header", "x-amz-grant-read-acp"), + "GrantWriteACP": ("header", "x-amz-grant-write-acp"), + "WriteOffsetBytes": ("header", "x-amz-write-offset-bytes"), + "ServerSideEncryption": ("header", "x-amz-server-side-encryption"), + "StorageClass": ("header", "x-amz-storage-class"), + "WebsiteRedirectLocation": ("header", "x-amz-website-redirect-location"), + "SSECustomerAlgorithm": ("header", "x-amz-server-side-encryption-customer-algorithm"), + "SSECustomerKey": ("header", "x-amz-server-side-encryption-customer-key"), + "SSECustomerKeyMD5": ("header", "x-amz-server-side-encryption-customer-key-MD5"), + "SSEKMSKeyId": ("header", "x-amz-server-side-encryption-aws-kms-key-id"), + "SSEKMSEncryptionContext": ("header", "x-amz-server-side-encryption-context"), + "BucketKeyEnabled": ("header", "x-amz-server-side-encryption-bucket-key-enabled"), + "RequestPayer": ("header", "x-amz-request-payer"), + "Tagging": ("header", "x-amz-tagging"), + "ObjectLockMode": ("header", "x-amz-object-lock-mode"), + "ObjectLockRetainUntilDate": ("header", "x-amz-object-lock-retain-until-date"), + "ObjectLockLegalHoldStatus": ("header", "x-amz-object-lock-legal-hold"), + "ExpectedBucketOwner": ("header", "x-amz-expected-bucket-owner"), + "Metadata": ("headers", "x-amz-meta-"), +} + + +def assert_put_object(request, Bucket: str, Key: str, addressing_style: str = "virtual", **params): + """Assert request is a PutObject (PUT /{Bucket}/{Key+}).""" + assert request.method == "PUT", ( + f"PutObject: expected PUT, got {request.method}" + ) + _check_bucket(request, Bucket, addressing_style, "PutObject") + _check_key(request, Key, addressing_style, "PutObject") + _check_params(request, params, _PUT_OBJECT_PARAMS, "PutObject") + + +# HeadObject: HEAD /{Bucket}/{Key+} +_HEAD_OBJECT_PARAMS = { + "IfMatch": ("header", "If-Match"), + "IfModifiedSince": ("header", "If-Modified-Since"), + "IfNoneMatch": ("header", "If-None-Match"), + "IfUnmodifiedSince": ("header", "If-Unmodified-Since"), + "Range": ("header", "Range"), + "SSECustomerAlgorithm": ("header", "x-amz-server-side-encryption-customer-algorithm"), + "SSECustomerKey": ("header", "x-amz-server-side-encryption-customer-key"), + "SSECustomerKeyMD5": ("header", "x-amz-server-side-encryption-customer-key-MD5"), + "RequestPayer": ("header", "x-amz-request-payer"), + "ExpectedBucketOwner": ("header", "x-amz-expected-bucket-owner"), + "ChecksumMode": ("header", "x-amz-checksum-mode"), + "ResponseCacheControl": ("querystring", "response-cache-control"), + "ResponseContentDisposition": ("querystring", "response-content-disposition"), + "ResponseContentEncoding": ("querystring", "response-content-encoding"), + "ResponseContentLanguage": ("querystring", "response-content-language"), + "ResponseContentType": ("querystring", "response-content-type"), + "ResponseExpires": ("querystring", "response-expires"), + "VersionId": ("querystring", "versionId"), + "PartNumber": ("querystring", "partNumber"), +} + + +def assert_head_object(request, Bucket: str, Key: str, addressing_style: str = "virtual", **params): + """Assert request is a HeadObject (HEAD /{Bucket}/{Key+}).""" + assert request.method == "HEAD", ( + f"HeadObject: expected HEAD, got {request.method}" + ) + _check_bucket(request, Bucket, addressing_style, "HeadObject") + _check_key(request, Key, addressing_style, "HeadObject") + _check_params(request, params, _HEAD_OBJECT_PARAMS, "HeadObject") + + +# GetObject: GET /{Bucket}/{Key+} +_GET_OBJECT_PARAMS = { + "IfMatch": ("header", "If-Match"), + "IfModifiedSince": ("header", "If-Modified-Since"), + "IfNoneMatch": ("header", "If-None-Match"), + "IfUnmodifiedSince": ("header", "If-Unmodified-Since"), + "Range": ("header", "Range"), + "SSECustomerAlgorithm": ("header", "x-amz-server-side-encryption-customer-algorithm"), + "SSECustomerKey": ("header", "x-amz-server-side-encryption-customer-key"), + "SSECustomerKeyMD5": ("header", "x-amz-server-side-encryption-customer-key-MD5"), + "RequestPayer": ("header", "x-amz-request-payer"), + "ExpectedBucketOwner": ("header", "x-amz-expected-bucket-owner"), + "ChecksumMode": ("header", "x-amz-checksum-mode"), + "ResponseCacheControl": ("querystring", "response-cache-control"), + "ResponseContentDisposition": ("querystring", "response-content-disposition"), + "ResponseContentEncoding": ("querystring", "response-content-encoding"), + "ResponseContentLanguage": ("querystring", "response-content-language"), + "ResponseContentType": ("querystring", "response-content-type"), + "ResponseExpires": ("querystring", "response-expires"), + "VersionId": ("querystring", "versionId"), + "PartNumber": ("querystring", "partNumber"), +} + + +def assert_get_object(request, Bucket: str, Key: str, addressing_style: str = "virtual", **params): + """Assert request is a GetObject (GET /{Bucket}/{Key+}).""" + assert request.method == "GET", ( + f"GetObject: expected GET, got {request.method}" + ) + _check_bucket(request, Bucket, addressing_style, "GetObject") + _check_key(request, Key, addressing_style, "GetObject") + _check_params(request, params, _GET_OBJECT_PARAMS, "GetObject") + + +# DeleteObject: DELETE /{Bucket}/{Key+} +_DELETE_OBJECT_PARAMS = { + "MFA": ("header", "x-amz-mfa"), + "RequestPayer": ("header", "x-amz-request-payer"), + "BypassGovernanceRetention": ("header", "x-amz-bypass-governance-retention"), + "ExpectedBucketOwner": ("header", "x-amz-expected-bucket-owner"), + "IfMatch": ("header", "If-Match"), + "IfMatchLastModifiedTime": ("header", "x-amz-if-match-last-modified-time"), + "IfMatchSize": ("header", "x-amz-if-match-size"), + "VersionId": ("querystring", "versionId"), +} + + +def assert_delete_object(request, Bucket: str, Key: str, addressing_style: str = "virtual", **params): + """Assert request is a DeleteObject (DELETE /{Bucket}/{Key+}).""" + assert request.method == "DELETE", ( + f"DeleteObject: expected DELETE, got {request.method}" + ) + _check_bucket(request, Bucket, addressing_style, "DeleteObject") + _check_key(request, Key, addressing_style, "DeleteObject") + _check_params(request, params, _DELETE_OBJECT_PARAMS, "DeleteObject") + + +# CopyObject: PUT /{Bucket}/{Key+} +_COPY_OBJECT_PARAMS = { + "ACL": ("header", "x-amz-acl"), + "CacheControl": ("header", "Cache-Control"), + "ChecksumAlgorithm": ("header", "x-amz-checksum-algorithm"), + "ContentDisposition": ("header", "Content-Disposition"), + "ContentEncoding": ("header", "Content-Encoding"), + "ContentLanguage": ("header", "Content-Language"), + "ContentType": ("header", "Content-Type"), + "CopySource": ("header", "x-amz-copy-source"), + "CopySourceIfMatch": ("header", "x-amz-copy-source-if-match"), + "CopySourceIfModifiedSince": ("header", "x-amz-copy-source-if-modified-since"), + "CopySourceIfNoneMatch": ("header", "x-amz-copy-source-if-none-match"), + "CopySourceIfUnmodifiedSince": ("header", "x-amz-copy-source-if-unmodified-since"), + "Expires": ("header", "Expires"), + "GrantFullControl": ("header", "x-amz-grant-full-control"), + "GrantRead": ("header", "x-amz-grant-read"), + "GrantReadACP": ("header", "x-amz-grant-read-acp"), + "GrantWriteACP": ("header", "x-amz-grant-write-acp"), + "IfMatch": ("header", "If-Match"), + "IfNoneMatch": ("header", "If-None-Match"), + "MetadataDirective": ("header", "x-amz-metadata-directive"), + "TaggingDirective": ("header", "x-amz-tagging-directive"), + "AnnotationDirective": ("header", "x-amz-object-annotation-directive"), + "ServerSideEncryption": ("header", "x-amz-server-side-encryption"), + "StorageClass": ("header", "x-amz-storage-class"), + "WebsiteRedirectLocation": ("header", "x-amz-website-redirect-location"), + "SSECustomerAlgorithm": ("header", "x-amz-server-side-encryption-customer-algorithm"), + "SSECustomerKey": ("header", "x-amz-server-side-encryption-customer-key"), + "SSECustomerKeyMD5": ("header", "x-amz-server-side-encryption-customer-key-MD5"), + "SSEKMSKeyId": ("header", "x-amz-server-side-encryption-aws-kms-key-id"), + "SSEKMSEncryptionContext": ("header", "x-amz-server-side-encryption-context"), + "BucketKeyEnabled": ("header", "x-amz-server-side-encryption-bucket-key-enabled"), + "CopySourceSSECustomerAlgorithm": ("header", "x-amz-copy-source-server-side-encryption-customer-algorithm"), + "CopySourceSSECustomerKey": ("header", "x-amz-copy-source-server-side-encryption-customer-key"), + "CopySourceSSECustomerKeyMD5": ("header", "x-amz-copy-source-server-side-encryption-customer-key-MD5"), + "RequestPayer": ("header", "x-amz-request-payer"), + "Tagging": ("header", "x-amz-tagging"), + "ObjectLockMode": ("header", "x-amz-object-lock-mode"), + "ObjectLockRetainUntilDate": ("header", "x-amz-object-lock-retain-until-date"), + "ObjectLockLegalHoldStatus": ("header", "x-amz-object-lock-legal-hold"), + "ExpectedBucketOwner": ("header", "x-amz-expected-bucket-owner"), + "ExpectedSourceBucketOwner": ("header", "x-amz-source-expected-bucket-owner"), + "Metadata": ("headers", "x-amz-meta-"), +} + + +def assert_copy_object(request, Bucket: str, Key: str, addressing_style: str = "virtual", **params): + """Assert request is a CopyObject (PUT /{Bucket}/{Key+}).""" + assert request.method == "PUT", ( + f"CopyObject: expected PUT, got {request.method}" + ) + _check_bucket(request, Bucket, addressing_style, "CopyObject") + _check_key(request, Key, addressing_style, "CopyObject") + assert ( + request.headers.get("x-amz-copy-source") is not None + or request.headers.get("x-amz-copy-source") is not None + ), f"CopyObject: required header x-amz-copy-source (CopySource) is missing" + _check_params(request, params, _COPY_OBJECT_PARAMS, "CopyObject") + + +# ListObjectsV2: GET /{Bucket}?list-type=2 +_LIST_OBJECTS_V2_PARAMS = { + "RequestPayer": ("header", "x-amz-request-payer"), + "ExpectedBucketOwner": ("header", "x-amz-expected-bucket-owner"), + "OptionalObjectAttributes": ("header", "x-amz-optional-object-attributes"), + "Delimiter": ("querystring", "delimiter"), + "EncodingType": ("querystring", "encoding-type"), + "MaxKeys": ("querystring", "max-keys"), + "Prefix": ("querystring", "prefix"), + "ContinuationToken": ("querystring", "continuation-token"), + "FetchOwner": ("querystring", "fetch-owner"), + "StartAfter": ("querystring", "start-after"), +} + + +def assert_list_objects_v2(request, Bucket: str, addressing_style: str = "virtual", **params): + """Assert request is a ListObjectsV2 (GET /{Bucket}?list-type=2).""" + assert request.method == "GET", ( + f"ListObjectsV2: expected GET, got {request.method}" + ) + _qs = parse_qs(urlparse(request.path).query) + assert _qs.get("list-type") == ["2"], ( + f"ListObjectsV2: expected ?list-type=2 in query" + ) + _check_bucket(request, Bucket, addressing_style, "ListObjectsV2") + _check_params(request, params, _LIST_OBJECTS_V2_PARAMS, "ListObjectsV2") + + +# CreateMultipartUpload: POST /{Bucket}/{Key+}?uploads +_CREATE_MULTIPART_UPLOAD_PARAMS = { + "ACL": ("header", "x-amz-acl"), + "CacheControl": ("header", "Cache-Control"), + "ContentDisposition": ("header", "Content-Disposition"), + "ContentEncoding": ("header", "Content-Encoding"), + "ContentLanguage": ("header", "Content-Language"), + "ContentType": ("header", "Content-Type"), + "Expires": ("header", "Expires"), + "GrantFullControl": ("header", "x-amz-grant-full-control"), + "GrantRead": ("header", "x-amz-grant-read"), + "GrantReadACP": ("header", "x-amz-grant-read-acp"), + "GrantWriteACP": ("header", "x-amz-grant-write-acp"), + "ServerSideEncryption": ("header", "x-amz-server-side-encryption"), + "StorageClass": ("header", "x-amz-storage-class"), + "WebsiteRedirectLocation": ("header", "x-amz-website-redirect-location"), + "SSECustomerAlgorithm": ("header", "x-amz-server-side-encryption-customer-algorithm"), + "SSECustomerKey": ("header", "x-amz-server-side-encryption-customer-key"), + "SSECustomerKeyMD5": ("header", "x-amz-server-side-encryption-customer-key-MD5"), + "SSEKMSKeyId": ("header", "x-amz-server-side-encryption-aws-kms-key-id"), + "SSEKMSEncryptionContext": ("header", "x-amz-server-side-encryption-context"), + "BucketKeyEnabled": ("header", "x-amz-server-side-encryption-bucket-key-enabled"), + "RequestPayer": ("header", "x-amz-request-payer"), + "Tagging": ("header", "x-amz-tagging"), + "ObjectLockMode": ("header", "x-amz-object-lock-mode"), + "ObjectLockRetainUntilDate": ("header", "x-amz-object-lock-retain-until-date"), + "ObjectLockLegalHoldStatus": ("header", "x-amz-object-lock-legal-hold"), + "ExpectedBucketOwner": ("header", "x-amz-expected-bucket-owner"), + "ChecksumAlgorithm": ("header", "x-amz-checksum-algorithm"), + "ChecksumType": ("header", "x-amz-checksum-type"), + "Metadata": ("headers", "x-amz-meta-"), +} + + +def assert_create_multipart_upload(request, Bucket: str, Key: str, addressing_style: str = "virtual", **params): + """Assert request is a CreateMultipartUpload (POST /{Bucket}/{Key+}?uploads).""" + assert request.method == "POST", ( + f"CreateMultipartUpload: expected POST, got {request.method}" + ) + _qs = parse_qs(urlparse(request.path).query) + assert "uploads" in urlparse(request.path).query, ( + f"CreateMultipartUpload: expected ?uploads in {request.path}" + ) + _check_bucket(request, Bucket, addressing_style, "CreateMultipartUpload") + _check_key(request, Key, addressing_style, "CreateMultipartUpload") + _check_params(request, params, _CREATE_MULTIPART_UPLOAD_PARAMS, "CreateMultipartUpload") + + +# UploadPart: PUT /{Bucket}/{Key+} +_UPLOAD_PART_PARAMS = { + "ContentLength": ("header", "Content-Length"), + "ContentMD5": ("header", "Content-MD5"), + "ChecksumAlgorithm": ("header", "x-amz-sdk-checksum-algorithm"), + "ChecksumCRC32": ("header", "x-amz-checksum-crc32"), + "ChecksumCRC32C": ("header", "x-amz-checksum-crc32c"), + "ChecksumCRC64NVME": ("header", "x-amz-checksum-crc64nvme"), + "ChecksumSHA1": ("header", "x-amz-checksum-sha1"), + "ChecksumSHA256": ("header", "x-amz-checksum-sha256"), + "ChecksumSHA512": ("header", "x-amz-checksum-sha512"), + "ChecksumMD5": ("header", "x-amz-checksum-md5"), + "ChecksumXXHASH64": ("header", "x-amz-checksum-xxhash64"), + "ChecksumXXHASH3": ("header", "x-amz-checksum-xxhash3"), + "ChecksumXXHASH128": ("header", "x-amz-checksum-xxhash128"), + "SSECustomerAlgorithm": ("header", "x-amz-server-side-encryption-customer-algorithm"), + "SSECustomerKey": ("header", "x-amz-server-side-encryption-customer-key"), + "SSECustomerKeyMD5": ("header", "x-amz-server-side-encryption-customer-key-MD5"), + "RequestPayer": ("header", "x-amz-request-payer"), + "ExpectedBucketOwner": ("header", "x-amz-expected-bucket-owner"), + "PartNumber": ("querystring", "partNumber"), + "UploadId": ("querystring", "uploadId"), +} + + +def assert_upload_part(request, Bucket: str, Key: str, addressing_style: str = "virtual", **params): + """Assert request is a UploadPart (PUT /{Bucket}/{Key+}).""" + assert request.method == "PUT", ( + f"UploadPart: expected PUT, got {request.method}" + ) + _check_bucket(request, Bucket, addressing_style, "UploadPart") + _check_key(request, Key, addressing_style, "UploadPart") + assert "partNumber" in parse_qs(urlparse(request.path).query), ( + f"UploadPart: required query param partNumber (PartNumber) is missing" + ) + assert "uploadId" in parse_qs(urlparse(request.path).query), ( + f"UploadPart: required query param uploadId (UploadId) is missing" + ) + _check_params(request, params, _UPLOAD_PART_PARAMS, "UploadPart") + + +# UploadPartCopy: PUT /{Bucket}/{Key+} +_UPLOAD_PART_COPY_PARAMS = { + "CopySource": ("header", "x-amz-copy-source"), + "CopySourceIfMatch": ("header", "x-amz-copy-source-if-match"), + "CopySourceIfModifiedSince": ("header", "x-amz-copy-source-if-modified-since"), + "CopySourceIfNoneMatch": ("header", "x-amz-copy-source-if-none-match"), + "CopySourceIfUnmodifiedSince": ("header", "x-amz-copy-source-if-unmodified-since"), + "CopySourceRange": ("header", "x-amz-copy-source-range"), + "SSECustomerAlgorithm": ("header", "x-amz-server-side-encryption-customer-algorithm"), + "SSECustomerKey": ("header", "x-amz-server-side-encryption-customer-key"), + "SSECustomerKeyMD5": ("header", "x-amz-server-side-encryption-customer-key-MD5"), + "CopySourceSSECustomerAlgorithm": ("header", "x-amz-copy-source-server-side-encryption-customer-algorithm"), + "CopySourceSSECustomerKey": ("header", "x-amz-copy-source-server-side-encryption-customer-key"), + "CopySourceSSECustomerKeyMD5": ("header", "x-amz-copy-source-server-side-encryption-customer-key-MD5"), + "RequestPayer": ("header", "x-amz-request-payer"), + "ExpectedBucketOwner": ("header", "x-amz-expected-bucket-owner"), + "ExpectedSourceBucketOwner": ("header", "x-amz-source-expected-bucket-owner"), + "PartNumber": ("querystring", "partNumber"), + "UploadId": ("querystring", "uploadId"), +} + + +def assert_upload_part_copy(request, Bucket: str, Key: str, addressing_style: str = "virtual", **params): + """Assert request is a UploadPartCopy (PUT /{Bucket}/{Key+}).""" + assert request.method == "PUT", ( + f"UploadPartCopy: expected PUT, got {request.method}" + ) + _check_bucket(request, Bucket, addressing_style, "UploadPartCopy") + _check_key(request, Key, addressing_style, "UploadPartCopy") + assert "partNumber" in parse_qs(urlparse(request.path).query), ( + f"UploadPartCopy: required query param partNumber (PartNumber) is missing" + ) + assert "uploadId" in parse_qs(urlparse(request.path).query), ( + f"UploadPartCopy: required query param uploadId (UploadId) is missing" + ) + assert ( + request.headers.get("x-amz-copy-source") is not None + or request.headers.get("x-amz-copy-source") is not None + ), f"UploadPartCopy: required header x-amz-copy-source (CopySource) is missing" + _check_params(request, params, _UPLOAD_PART_COPY_PARAMS, "UploadPartCopy") + + +# CompleteMultipartUpload: POST /{Bucket}/{Key+} +_COMPLETE_MULTIPART_UPLOAD_PARAMS = { + "ChecksumCRC32": ("header", "x-amz-checksum-crc32"), + "ChecksumCRC32C": ("header", "x-amz-checksum-crc32c"), + "ChecksumCRC64NVME": ("header", "x-amz-checksum-crc64nvme"), + "ChecksumSHA1": ("header", "x-amz-checksum-sha1"), + "ChecksumSHA256": ("header", "x-amz-checksum-sha256"), + "ChecksumSHA512": ("header", "x-amz-checksum-sha512"), + "ChecksumMD5": ("header", "x-amz-checksum-md5"), + "ChecksumXXHASH64": ("header", "x-amz-checksum-xxhash64"), + "ChecksumXXHASH3": ("header", "x-amz-checksum-xxhash3"), + "ChecksumXXHASH128": ("header", "x-amz-checksum-xxhash128"), + "ChecksumType": ("header", "x-amz-checksum-type"), + "MpuObjectSize": ("header", "x-amz-mp-object-size"), + "RequestPayer": ("header", "x-amz-request-payer"), + "ExpectedBucketOwner": ("header", "x-amz-expected-bucket-owner"), + "IfMatch": ("header", "If-Match"), + "IfNoneMatch": ("header", "If-None-Match"), + "SSECustomerAlgorithm": ("header", "x-amz-server-side-encryption-customer-algorithm"), + "SSECustomerKey": ("header", "x-amz-server-side-encryption-customer-key"), + "SSECustomerKeyMD5": ("header", "x-amz-server-side-encryption-customer-key-MD5"), + "UploadId": ("querystring", "uploadId"), + "MultipartUpload": ("payload", "http://s3.amazonaws.com/doc/2006-03-01/"), +} + + +def assert_complete_multipart_upload(request, Bucket: str, Key: str, addressing_style: str = "virtual", **params): + """Assert request is a CompleteMultipartUpload (POST /{Bucket}/{Key+}).""" + assert request.method == "POST", ( + f"CompleteMultipartUpload: expected POST, got {request.method}" + ) + _check_bucket(request, Bucket, addressing_style, "CompleteMultipartUpload") + _check_key(request, Key, addressing_style, "CompleteMultipartUpload") + assert "uploadId" in parse_qs(urlparse(request.path).query), ( + f"CompleteMultipartUpload: required query param uploadId (UploadId) is missing" + ) + _check_params(request, params, _COMPLETE_MULTIPART_UPLOAD_PARAMS, "CompleteMultipartUpload") + + +# AbortMultipartUpload: DELETE /{Bucket}/{Key+} +_ABORT_MULTIPART_UPLOAD_PARAMS = { + "RequestPayer": ("header", "x-amz-request-payer"), + "ExpectedBucketOwner": ("header", "x-amz-expected-bucket-owner"), + "IfMatchInitiatedTime": ("header", "x-amz-if-match-initiated-time"), + "UploadId": ("querystring", "uploadId"), +} + + +def assert_abort_multipart_upload(request, Bucket: str, Key: str, addressing_style: str = "virtual", **params): + """Assert request is a AbortMultipartUpload (DELETE /{Bucket}/{Key+}).""" + assert request.method == "DELETE", ( + f"AbortMultipartUpload: expected DELETE, got {request.method}" + ) + _check_bucket(request, Bucket, addressing_style, "AbortMultipartUpload") + _check_key(request, Key, addressing_style, "AbortMultipartUpload") + assert "uploadId" in parse_qs(urlparse(request.path).query), ( + f"AbortMultipartUpload: required query param uploadId (UploadId) is missing" + ) + _check_params(request, params, _ABORT_MULTIPART_UPLOAD_PARAMS, "AbortMultipartUpload") + + +# CreateBucket: PUT /{Bucket} +_CREATE_BUCKET_PARAMS = { + "ACL": ("header", "x-amz-acl"), + "GrantFullControl": ("header", "x-amz-grant-full-control"), + "GrantRead": ("header", "x-amz-grant-read"), + "GrantReadACP": ("header", "x-amz-grant-read-acp"), + "GrantWrite": ("header", "x-amz-grant-write"), + "GrantWriteACP": ("header", "x-amz-grant-write-acp"), + "ObjectLockEnabledForBucket": ("header", "x-amz-bucket-object-lock-enabled"), + "ObjectOwnership": ("header", "x-amz-object-ownership"), + "BucketNamespace": ("header", "x-amz-bucket-namespace"), + "CreateBucketConfiguration": ("payload", "http://s3.amazonaws.com/doc/2006-03-01/"), +} + + +def assert_create_bucket(request, Bucket: str, addressing_style: str = "virtual", **params): + """Assert request is a CreateBucket (PUT /{Bucket}).""" + assert request.method == "PUT", ( + f"CreateBucket: expected PUT, got {request.method}" + ) + _check_bucket(request, Bucket, addressing_style, "CreateBucket") + _check_params(request, params, _CREATE_BUCKET_PARAMS, "CreateBucket") + + +# DeleteBucket: DELETE /{Bucket} +_DELETE_BUCKET_PARAMS = { + "ExpectedBucketOwner": ("header", "x-amz-expected-bucket-owner"), +} + + +def assert_delete_bucket(request, Bucket: str, addressing_style: str = "virtual", **params): + """Assert request is a DeleteBucket (DELETE /{Bucket}).""" + assert request.method == "DELETE", ( + f"DeleteBucket: expected DELETE, got {request.method}" + ) + _check_bucket(request, Bucket, addressing_style, "DeleteBucket") + _check_params(request, params, _DELETE_BUCKET_PARAMS, "DeleteBucket") + + +# GetObjectTagging: GET /{Bucket}/{Key+}?tagging +_GET_OBJECT_TAGGING_PARAMS = { + "ExpectedBucketOwner": ("header", "x-amz-expected-bucket-owner"), + "RequestPayer": ("header", "x-amz-request-payer"), + "VersionId": ("querystring", "versionId"), +} + + +def assert_get_object_tagging(request, Bucket: str, Key: str, addressing_style: str = "virtual", **params): + """Assert request is a GetObjectTagging (GET /{Bucket}/{Key+}?tagging).""" + assert request.method == "GET", ( + f"GetObjectTagging: expected GET, got {request.method}" + ) + _qs = parse_qs(urlparse(request.path).query) + assert "tagging" in urlparse(request.path).query, ( + f"GetObjectTagging: expected ?tagging in {request.path}" + ) + _check_bucket(request, Bucket, addressing_style, "GetObjectTagging") + _check_key(request, Key, addressing_style, "GetObjectTagging") + _check_params(request, params, _GET_OBJECT_TAGGING_PARAMS, "GetObjectTagging") + + +# PutObjectTagging: PUT /{Bucket}/{Key+}?tagging +_PUT_OBJECT_TAGGING_PARAMS = { + "ContentMD5": ("header", "Content-MD5"), + "ChecksumAlgorithm": ("header", "x-amz-sdk-checksum-algorithm"), + "ExpectedBucketOwner": ("header", "x-amz-expected-bucket-owner"), + "RequestPayer": ("header", "x-amz-request-payer"), + "VersionId": ("querystring", "versionId"), + "Tagging": ("payload", "http://s3.amazonaws.com/doc/2006-03-01/"), +} + + +def assert_put_object_tagging(request, Bucket: str, Key: str, addressing_style: str = "virtual", **params): + """Assert request is a PutObjectTagging (PUT /{Bucket}/{Key+}?tagging).""" + assert request.method == "PUT", ( + f"PutObjectTagging: expected PUT, got {request.method}" + ) + _qs = parse_qs(urlparse(request.path).query) + assert "tagging" in urlparse(request.path).query, ( + f"PutObjectTagging: expected ?tagging in {request.path}" + ) + _check_bucket(request, Bucket, addressing_style, "PutObjectTagging") + _check_key(request, Key, addressing_style, "PutObjectTagging") + _check_params(request, params, _PUT_OBJECT_TAGGING_PARAMS, "PutObjectTagging") + + +# ListBuckets: GET / +_LIST_BUCKETS_PARAMS = { + "MaxBuckets": ("querystring", "max-buckets"), + "ContinuationToken": ("querystring", "continuation-token"), + "Prefix": ("querystring", "prefix"), + "BucketRegion": ("querystring", "bucket-region"), +} + + +def assert_list_buckets(request, **params): + """Assert request is a ListBuckets (GET /).""" + assert request.method == "GET", ( + f"ListBuckets: expected GET, got {request.method}" + ) + _check_params(request, params, _LIST_BUCKETS_PARAMS, "ListBuckets") + diff --git a/tests/blackbox/test_ls_command.py b/tests/blackbox/test_ls_command.py new file mode 100644 index 000000000000..31239fcd4dad --- /dev/null +++ b/tests/blackbox/test_ls_command.py @@ -0,0 +1,574 @@ +"""Blackbox tests for `aws s3 ls` command.""" + +from __future__ import annotations + +import os + +import pytest +from dateutil import parser, tz +from dateutil import parser as dateutil_parser + +from tests.blackbox.s3_assertions import ( + assert_list_buckets, + assert_list_objects_v2, +) + +from tests.blackbox.utils import ( + cli_env, + format_requests, + get_query_params, + list_objects_xml, + run_cli, + mock_server, + setup_responses, + xml_response, +) + + +def list_buckets_xml() -> str: + return ( + '' + '' + "" + "" + ) + + +@pytest.mark.asyncio +async def test_errors_out_with_extra_arguments(aws_cli: str) -> None: + env = { + **os.environ, + "AWS_ACCESS_KEY_ID": "test", + "AWS_SECRET_ACCESS_KEY": "test", + "AWS_DEFAULT_REGION": "us-east-1", + } + stdout, stderr, rc = await run_cli( + aws_cli, ["s3", "ls", "--extra-argument-foo"], env + ) + assert rc == 252 + assert b"--extra-argument-foo" in stderr + + +@pytest.mark.asyncio +async def test_operations_used_in_recursive_list(aws_cli: str) -> None: + time_utc = "2014-01-09T20:45:49.000Z" + response_xml = list_objects_xml( + contents=[ + {"Key": "foo/bar.txt", "Size": 100, "LastModified": time_utc} + ] + ) + + async with mock_server() as (server, proxy): + setup_responses(server, [xml_response(response_xml)]) + + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "ls", "s3://bucket/", "--recursive"], + cli_env(proxy), + ) + + assert rc == 0, f"stdout={stdout!r} stderr={stderr!r}" + + # Should have made exactly one request (ListObjectsV2) + assert len(server.requests) == 1 + req = server.requests[0] + assert_list_objects_v2(req, Bucket="bucket") + + params = get_query_params(req) + # Recursive listing should not include a delimiter + assert "delimiter" not in params + + # Time is stored in UTC but displayed in local timezone + time_local = dateutil_parser.parse(time_utc).astimezone(tz.tzlocal()) + expected_stdout = ( + f"{time_local.strftime('%Y-%m-%d %H:%M:%S')} 100 foo/bar.txt\n" + ) + assert stdout.decode() == expected_stdout + + +@pytest.mark.asyncio +async def test_list_buckets_use_page_size(aws_cli: str) -> None: + response_xml = list_buckets_xml() + + async with mock_server() as (server, proxy): + setup_responses(server, [xml_response(response_xml)]) + + stdout, stderr, rc = await run_cli( + aws_cli, ["s3", "ls", "--page-size", "8"], cli_env(proxy) + ) + + assert rc == 0, f"stdout={stdout!r} stderr={stderr!r}" + assert len(server.requests) == 1 + req = server.requests[0] + assert_list_buckets(req, MaxBuckets="8") + + +@pytest.mark.asyncio +async def test_operations_use_page_size(aws_cli: str) -> None: + time_utc = "2014-01-09T20:45:49.000Z" + response_xml = list_objects_xml( + contents=[ + {"Key": "foo/bar.txt", "Size": 100, "LastModified": time_utc} + ] + ) + + async with mock_server() as (server, proxy): + setup_responses(server, [xml_response(response_xml)]) + + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "ls", "s3://bucket/", "--page-size", "8"], + cli_env(proxy), + ) + + assert rc == 0, f"stdout={stdout!r} stderr={stderr!r}" + assert len(server.requests) == 1 + req = server.requests[0] + assert_list_objects_v2(req, Bucket="bucket", MaxKeys="8") + + +@pytest.mark.asyncio +async def test_operations_use_page_size_recursive(aws_cli: str) -> None: + time_utc = "2014-01-09T20:45:49.000Z" + response_xml = list_objects_xml( + contents=[ + {"Key": "foo/bar.txt", "Size": 100, "LastModified": time_utc} + ] + ) + + async with mock_server() as (server, proxy): + setup_responses(server, [xml_response(response_xml)]) + + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "ls", "s3://bucket/", "--page-size", "8", "--recursive"], + cli_env(proxy), + ) + + assert rc == 0, f"stdout={stdout!r} stderr={stderr!r}" + assert len(server.requests) == 1 + req = server.requests[0] + assert_list_objects_v2(req, Bucket="bucket", MaxKeys="8") + params = get_query_params(req) + assert "delimiter" not in params + + +@pytest.mark.asyncio +async def test_success_rc_has_prefixes_and_objects(aws_cli: str) -> None: + time_utc = "2014-01-09T20:45:49.000Z" + response_xml = list_objects_xml( + contents=[ + {"Key": "foo/bar.txt", "Size": 100, "LastModified": time_utc} + ], + common_prefixes=["foo/"], + ) + + async with mock_server() as (server, proxy): + setup_responses(server, [xml_response(response_xml)]) + + stdout, stderr, rc = await run_cli( + aws_cli, ["s3", "ls", "s3://bucket/foo"], cli_env(proxy) + ) + + assert rc == 0, f"stdout={stdout!r} stderr={stderr!r}" + + +@pytest.mark.asyncio +async def test_success_rc_has_only_prefixes(aws_cli: str) -> None: + response_xml = list_objects_xml(common_prefixes=["foo/"]) + + async with mock_server() as (server, proxy): + setup_responses(server, [xml_response(response_xml)]) + + stdout, stderr, rc = await run_cli( + aws_cli, ["s3", "ls", "s3://bucket/foo"], cli_env(proxy) + ) + + assert rc == 0, f"stdout={stdout!r} stderr={stderr!r}" + + +@pytest.mark.asyncio +async def test_success_rc_has_only_objects(aws_cli: str) -> None: + time_utc = "2014-01-09T20:45:49.000Z" + response_xml = list_objects_xml( + contents=[ + {"Key": "foo/bar.txt", "Size": 100, "LastModified": time_utc} + ] + ) + + async with mock_server() as (server, proxy): + setup_responses(server, [xml_response(response_xml)]) + + stdout, stderr, rc = await run_cli( + aws_cli, ["s3", "ls", "s3://bucket/foo"], cli_env(proxy) + ) + + assert rc == 0, f"stdout={stdout!r} stderr={stderr!r}" + + +@pytest.mark.asyncio +async def test_success_rc_with_pagination(aws_cli: str) -> None: + # Pagination should not affect a successful return code of zero, even + # if there are no results on the second page because there were + # results in previous pages. + time_utc = "2014-01-09T20:45:49.000Z" + page1_xml = list_objects_xml( + contents=[ + {"Key": "foo/bar.txt", "Size": 100, "LastModified": time_utc} + ], + common_prefixes=["foo/"], + is_truncated=True, + next_continuation_token="token123", + ) + page2_xml = list_objects_xml() + + async with mock_server() as (server, proxy): + setup_responses( + server, + [ + xml_response(page1_xml), + xml_response(page2_xml), + ], + ) + + stdout, stderr, rc = await run_cli( + aws_cli, ["s3", "ls", "s3://bucket/foo"], cli_env(proxy) + ) + + assert rc == 0, f"stdout={stdout!r} stderr={stderr!r}" + assert len(server.requests) == 2 + + +@pytest.mark.asyncio +async def test_success_rc_empty_bucket_no_key_given(aws_cli: str) -> None: + response_xml = list_objects_xml() + + async with mock_server() as (server, proxy): + setup_responses(server, [xml_response(response_xml)]) + + stdout, stderr, rc = await run_cli( + aws_cli, ["s3", "ls", "s3://bucket"], cli_env(proxy) + ) + + assert rc == 0, f"stdout={stdout!r} stderr={stderr!r}" + + +@pytest.mark.asyncio +async def test_fail_rc_no_objects_nor_prefixes(aws_cli: str) -> None: + # Empty response with no Contents or CommonPrefixes, but a key prefix was given + response_xml = list_objects_xml() + + async with mock_server() as (server, proxy): + setup_responses(server, [xml_response(response_xml)]) + + stdout, stderr, rc = await run_cli( + aws_cli, ["s3", "ls", "s3://bucket/foo"], cli_env(proxy) + ) + + assert rc == 1 + + +@pytest.mark.asyncio +async def test_human_readable_file_size(aws_cli: str) -> None: + time_utc = "2014-01-09T20:45:49.000Z" + response_xml = list_objects_xml( + contents=[ + {"Key": "onebyte.txt", "Size": 1, "LastModified": time_utc}, + {"Key": "onekilobyte.txt", "Size": 1024, "LastModified": time_utc}, + { + "Key": "onemegabyte.txt", + "Size": 1024**2, + "LastModified": time_utc, + }, + { + "Key": "onegigabyte.txt", + "Size": 1024**3, + "LastModified": time_utc, + }, + { + "Key": "oneterabyte.txt", + "Size": 1024**4, + "LastModified": time_utc, + }, + { + "Key": "onepetabyte.txt", + "Size": 1024**5, + "LastModified": time_utc, + }, + ] + ) + + async with mock_server() as (server, proxy): + setup_responses(server, [xml_response(response_xml)]) + + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "ls", "s3://bucket/", "--human-readable"], + cli_env(proxy), + ) + + assert rc == 0, f"stdout={stdout!r} stderr={stderr!r}" + stdout_str = stdout.decode() + # Time is stored in UTC timezone, but the actual time displayed + # is specific to your tzinfo, so shift the timezone to your local's. + time_local = parser.parse(time_utc).astimezone(tz.tzlocal()) + time_fmt = time_local.strftime('%Y-%m-%d %H:%M:%S') + assert f"{time_fmt} 1 Byte onebyte.txt" in stdout_str + assert f"{time_fmt} 1.0 KiB onekilobyte.txt" in stdout_str + assert f"{time_fmt} 1.0 MiB onemegabyte.txt" in stdout_str + assert f"{time_fmt} 1.0 GiB onegigabyte.txt" in stdout_str + assert f"{time_fmt} 1.0 TiB oneterabyte.txt" in stdout_str + assert f"{time_fmt} 1.0 PiB onepetabyte.txt" in stdout_str + + +@pytest.mark.asyncio +async def test_summarize(aws_cli: str) -> None: + time_utc = "2014-01-09T20:45:49.000Z" + response_xml = list_objects_xml( + contents=[ + {"Key": "onebyte.txt", "Size": 1, "LastModified": time_utc}, + {"Key": "onekilobyte.txt", "Size": 1024, "LastModified": time_utc}, + { + "Key": "onemegabyte.txt", + "Size": 1024**2, + "LastModified": time_utc, + }, + { + "Key": "onegigabyte.txt", + "Size": 1024**3, + "LastModified": time_utc, + }, + { + "Key": "oneterabyte.txt", + "Size": 1024**4, + "LastModified": time_utc, + }, + { + "Key": "onepetabyte.txt", + "Size": 1024**5, + "LastModified": time_utc, + }, + ] + ) + + async with mock_server() as (server, proxy): + setup_responses(server, [xml_response(response_xml)]) + + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "ls", "s3://bucket/", "--summarize"], + cli_env(proxy), + ) + + assert rc == 0, f"stdout={stdout!r} stderr={stderr!r}" + stdout_str = stdout.decode() + assert "Total Objects: 6\n" in stdout_str + assert "Total Size: 1127000493261825\n" in stdout_str + + +@pytest.mark.asyncio +async def test_summarize_with_human_readable(aws_cli: str) -> None: + time_utc = "2014-01-09T20:45:49.000Z" + response_xml = list_objects_xml( + contents=[ + {"Key": "onebyte.txt", "Size": 1, "LastModified": time_utc}, + {"Key": "onekilobyte.txt", "Size": 1024, "LastModified": time_utc}, + { + "Key": "onemegabyte.txt", + "Size": 1024**2, + "LastModified": time_utc, + }, + { + "Key": "onegigabyte.txt", + "Size": 1024**3, + "LastModified": time_utc, + }, + { + "Key": "oneterabyte.txt", + "Size": 1024**4, + "LastModified": time_utc, + }, + { + "Key": "onepetabyte.txt", + "Size": 1024**5, + "LastModified": time_utc, + }, + ] + ) + + async with mock_server() as (server, proxy): + setup_responses(server, [xml_response(response_xml)]) + + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "ls", "s3://bucket/", "--human-readable", "--summarize"], + cli_env(proxy), + ) + + assert rc == 0, f"stdout={stdout!r} stderr={stderr!r}" + stdout_str = stdout.decode() + assert "Total Objects: 6\n" in stdout_str + assert "Total Size: 1.0 PiB\n" in stdout_str + + +@pytest.mark.asyncio +async def test_requester_pays(aws_cli: str) -> None: + time_utc = "2014-01-09T20:45:49.000Z" + response_xml = list_objects_xml( + contents=[{"Key": "onebyte.txt", "Size": 1, "LastModified": time_utc}] + ) + + async with mock_server() as (server, proxy): + setup_responses(server, [xml_response(response_xml)]) + + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "ls", "s3://mybucket/foo/", "--request-payer", "requester"], + cli_env(proxy), + ) + + assert rc == 0, f"stdout={stdout!r} stderr={stderr!r}" + assert len(server.requests) == 1 + req = server.requests[0] + params = get_query_params(req) + assert req.headers.get("host") == "mybucket.s3.us-east-1.amazonaws.com" + assert params.get("delimiter") == ["/"] + assert params.get("prefix") == ["foo/"] + assert params.get("encoding-type") == ["url"] + assert req.headers.get("x-amz-request-payer") == "requester" + + +@pytest.mark.asyncio +async def test_requester_pays_with_no_args(aws_cli: str) -> None: + time_utc = "2014-01-09T20:45:49.000Z" + response_xml = list_objects_xml( + contents=[{"Key": "onebyte.txt", "Size": 1, "LastModified": time_utc}] + ) + + async with mock_server() as (server, proxy): + setup_responses(server, [xml_response(response_xml)]) + + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "ls", "s3://mybucket/foo/", "--request-payer"], + cli_env(proxy), + ) + + assert rc == 0, f"stdout={stdout!r} stderr={stderr!r}" + assert len(server.requests) == 1 + req = server.requests[0] + params = get_query_params(req) + assert req.headers.get("host") == "mybucket.s3.us-east-1.amazonaws.com" + assert params.get("delimiter") == ["/"] + assert params.get("prefix") == ["foo/"] + assert params.get("encoding-type") == ["url"] + assert req.headers.get("x-amz-request-payer") == "requester" + + +@pytest.mark.asyncio +async def test_accesspoint_arn(aws_cli: str) -> None: + time_utc = "2014-01-09T20:45:49.000Z" + response_xml = list_objects_xml( + contents=[{"Key": "bar.txt", "Size": 100, "LastModified": time_utc}] + ) + + async with mock_server() as (server, proxy): + setup_responses(server, [xml_response(response_xml)]) + + arn = "arn:aws:s3:us-west-2:123456789012:accesspoint/endpoint" + stdout, stderr, rc = await run_cli( + aws_cli, ["s3", "ls", f"s3://{arn}"], cli_env(proxy) + ) + + assert ( + rc == 0 + ), f"stdout={stdout!r} stderr={stderr!r}\n{format_requests(server)}" + assert len(server.requests) == 1 + req = server.requests[0] + # For accesspoint ARNs: AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com + assert ( + req.headers.get("host") + == "endpoint-123456789012.s3-accesspoint.us-west-2.amazonaws.com" + ) + + +@pytest.mark.asyncio +async def test_list_buckets_uses_bucket_name_prefix(aws_cli: str) -> None: + response_xml = list_buckets_xml() + + async with mock_server() as (server, proxy): + setup_responses(server, [xml_response(response_xml)]) + + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "ls", "--bucket-name-prefix", "myprefix"], + cli_env(proxy), + ) + + assert rc == 0, f"stdout={stdout!r} stderr={stderr!r}" + assert len(server.requests) == 1 + req = server.requests[0] + assert_list_buckets(req, Prefix="myprefix") + + +@pytest.mark.asyncio +async def test_list_buckets_uses_bucket_region(aws_cli: str) -> None: + response_xml = list_buckets_xml() + + async with mock_server() as (server, proxy): + setup_responses(server, [xml_response(response_xml)]) + + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "ls", "--bucket-region", "us-west-1"], + cli_env(proxy), + ) + + assert rc == 0, f"stdout={stdout!r} stderr={stderr!r}" + assert len(server.requests) == 1 + req = server.requests[0] + assert_list_buckets(req, BucketRegion="us-west-1") + + +@pytest.mark.asyncio +async def test_list_objects_ignores_bucket_name_prefix(aws_cli: str) -> None: + response_xml = list_objects_xml() + + async with mock_server() as (server, proxy): + setup_responses(server, [xml_response(response_xml)]) + + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "ls", "s3://mybucket", "--bucket-name-prefix", "myprefix"], + cli_env(proxy), + ) + + assert ( + rc == 0 + ), f"stdout={stdout!r} stderr={stderr!r}\n{format_requests(server)}" + assert len(server.requests) == 1 + req = server.requests[0] + params = get_query_params(req) + # --bucket-name-prefix should be ignored for object listings; + # prefix is either absent or empty on the wire + assert params.get("prefix", [""]) == [""] + + +@pytest.mark.asyncio +async def test_list_objects_ignores_bucket_region(aws_cli: str) -> None: + response_xml = list_objects_xml() + + async with mock_server() as (server, proxy): + setup_responses(server, [xml_response(response_xml)]) + + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "ls", "s3://mybucket", "--bucket-region", "us-west-1"], + cli_env(proxy), + ) + + assert rc == 0, f"stdout={stdout!r} stderr={stderr!r}" + assert len(server.requests) == 1 + req = server.requests[0] + params = get_query_params(req) + assert "bucket-region" not in params diff --git a/tests/blackbox/test_mb_command.py b/tests/blackbox/test_mb_command.py new file mode 100644 index 000000000000..bc86865e14f7 --- /dev/null +++ b/tests/blackbox/test_mb_command.py @@ -0,0 +1,300 @@ +"""Blackbox tests for `aws s3 mb` command.""" + +from __future__ import annotations + +import xml.etree.ElementTree as ET + +import pytest + +from tests.blackbox.s3_assertions import ( + assert_create_bucket, +) + +from tests.blackbox.utils import ( + cli_env, + create_bucket_response, + format_requests, + run_cli, + mock_server, + setup_responses, +) + +NS = "http://s3.amazonaws.com/doc/2006-03-01/" + + +def parse_body_xml(request) -> ET.Element | None: + if not request.body: + return None + return ET.fromstring(request.body) + + +@pytest.mark.asyncio +class TestMBCommand: + async def test_make_bucket(self, aws_cli): + async with mock_server() as (server, proxy): + setup_responses(server, [create_bucket_response()]) + env = cli_env(proxy) + stdout, stderr, rc = await run_cli( + aws_cli, ["s3", "mb", "s3://bucket"], env + ) + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_create_bucket( + server.requests[0], Bucket="bucket" + ) + + async def test_adds_location_constraint(self, aws_cli): + async with mock_server() as (server, proxy): + setup_responses(server, [create_bucket_response()]) + env = cli_env(proxy) + env["AWS_REGION"] = "us-west-2" + env["AWS_DEFAULT_REGION"] = "us-west-2" + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "mb", "s3://bucket", "--region", "us-west-2"], + env, + ) + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_create_bucket( + server.requests[0], + Bucket="bucket", + CreateBucketConfiguration={"LocationConstraint": "us-west-2"}, + ) + + async def test_location_constraint_not_added_on_us_east_1(self, aws_cli): + async with mock_server() as (server, proxy): + setup_responses(server, [create_bucket_response()]) + env = cli_env(proxy) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "mb", "s3://bucket", "--region", "us-east-1"], + env, + ) + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + req = server.requests[0] + body = parse_body_xml(req) + # Either no body or no LocationConstraint element + if body is not None: + loc = body.find(f"{{{NS}}}LocationConstraint") + assert loc is None, ( + f"LocationConstraint should not be present for us-east-1, " + f"got: {loc.text}" + ) + + async def test_nonzero_exit_if_invalid_path_provided(self, aws_cli): + async with mock_server() as (server, proxy): + env = cli_env(proxy) + stdout, stderr, rc = await run_cli( + aws_cli, ["s3", "mb", "bucket"], env + ) + assert rc == 252 + assert len(server.requests) == 0, format_requests(server) + + async def test_incompatible_with_express_directory_bucket(self, aws_cli): + async with mock_server() as (server, proxy): + env = cli_env(proxy) + stdout, stderr, rc = await run_cli( + aws_cli, ["s3", "mb", "s3://bucket--usw2-az1--x-s3/"], env + ) + error_message = b"cannot use mb command with a directory bucket." + assert rc == 252 + assert error_message in stderr.lower() + assert len(server.requests) == 0, format_requests(server) + + async def test_make_bucket_with_single_tag(self, aws_cli): + async with mock_server() as (server, proxy): + setup_responses(server, [create_bucket_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mb", + "s3://bucket", + "--tags", + "Key1", + "Value1", + "--region", + "us-west-2", + ], + cli_env(proxy), + ) + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_create_bucket( + server.requests[0], + Bucket="bucket", + CreateBucketConfiguration={ + "LocationConstraint": "us-west-2", + "Tags": [{"Key": "Key1", "Value": "Value1"}], + }, + ) + + async def test_make_bucket_with_single_tag_us_east_1(self, aws_cli): + async with mock_server() as (server, proxy): + setup_responses(server, [create_bucket_response()]) + env = cli_env(proxy) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mb", + "s3://bucket", + "--tags", + "Key1", + "Value1", + "--region", + "us-east-1", + ], + env, + ) + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_create_bucket( + server.requests[0], + Bucket="bucket", + CreateBucketConfiguration={ + "Tags": [{"Key": "Key1", "Value": "Value1"}], + }, + ) + # Should NOT have LocationConstraint for us-east-1 + req = server.requests[0] + body = parse_body_xml(req) + loc = body.find(f"{{{NS}}}LocationConstraint") + assert loc is None + + async def test_make_bucket_with_multiple_tags(self, aws_cli): + async with mock_server() as (server, proxy): + setup_responses(server, [create_bucket_response()]) + env = cli_env(proxy) + env["AWS_REGION"] = "us-west-2" + env["AWS_DEFAULT_REGION"] = "us-west-2" + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mb", + "s3://bucket", + "--tags", + "Key1", + "Value1", + "--tags", + "Key2", + "Value2", + "--region", + "us-west-2", + ], + cli_env(proxy), + ) + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_create_bucket( + server.requests[0], + Bucket="bucket", + CreateBucketConfiguration={ + "LocationConstraint": "us-west-2", + "Tags": [ + {"Key": "Key1", "Value": "Value1"}, + {"Key": "Key2", "Value": "Value2"}, + ], + }, + ) + + async def test_account_regional_namespace_bucket(self, aws_cli): + bucket = "amzn-s3-demo-bucket-111122223333-us-west-2-an" + async with mock_server() as (server, proxy): + setup_responses(server, [create_bucket_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "mb", f"s3://{bucket}", "--region", "us-west-2"], + cli_env(proxy), + ) + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_create_bucket( + server.requests[0], + Bucket=bucket, + CreateBucketConfiguration={"LocationConstraint": "us-west-2"}, + BucketNamespace="account-regional", + ) + + async def test_account_regional_namespace_bucket_us_east_1(self, aws_cli): + bucket = "my-bucket-111122223333-us-east-1-an" + async with mock_server() as (server, proxy): + setup_responses(server, [create_bucket_response()]) + env = cli_env(proxy) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "mb", f"s3://{bucket}", "--region", "us-east-1"], + env, + ) + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_create_bucket( + server.requests[0], + Bucket=bucket, + BucketNamespace="account-regional", + ) + + async def test_account_regional_namespace_short_bucket_name(self, aws_cli): + bucket = "xyz-an" + async with mock_server() as (server, proxy): + setup_responses(server, [create_bucket_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "mb", f"s3://{bucket}", "--region", "us-east-1"], + cli_env(proxy), + ) + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_create_bucket( + server.requests[0], + Bucket=bucket, + BucketNamespace="account-regional", + ) + + async def test_regular_bucket_no_namespace(self, aws_cli): + async with mock_server() as (server, proxy): + setup_responses(server, [create_bucket_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mb", + "s3://my-regular-bucket", + "--region", + "us-east-1", + ], + cli_env(proxy), + ) + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_create_bucket( + server.requests[0], Bucket="my-regular-bucket" + ) + # Should NOT have x-amz-bucket-namespace header + assert ( + server.requests[0].headers.get("x-amz-bucket-namespace") + is None + ) + + async def test_tags_with_three_arguments_fails(self, aws_cli): + async with mock_server() as (server, proxy): + env = cli_env(proxy) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mb", + "s3://bucket", + "--tags", + "Key1", + "Value1", + "ExtraArg", + ], + env, + ) + assert rc == 252 + assert len(server.requests) == 0, format_requests(server) + assert "ParamValidation" in stderr.decode() diff --git a/tests/blackbox/test_presign_command.py b/tests/blackbox/test_presign_command.py new file mode 100644 index 000000000000..688a11b72d1f --- /dev/null +++ b/tests/blackbox/test_presign_command.py @@ -0,0 +1,156 @@ +"""Blackbox tests for `aws s3 presign` command. + +NOTE: The original tests freeze time to assert exact signature values. +In blackbox mode we cannot control the binary's clock, so we assert URL +structure (hostname, path, query param keys, credential format, expires) +but NOT exact X-Amz-Signature or X-Amz-Date values. +""" + +from __future__ import annotations + +import os +import re +from urllib.parse import urlsplit + +import pytest + +from tests.blackbox.utils import run_cli + +EXPECTED_SIGV4_KEYS = { + "X-Amz-Algorithm", + "X-Amz-Credential", + "X-Amz-Date", + "X-Amz-Expires", + "X-Amz-SignedHeaders", + "X-Amz-Signature", +} + + +def presign_env() -> dict[str, str]: + """Minimal env for presign -- no proxy needed since no HTTP requests are made.""" + return { + "PATH": os.environ.get("PATH", ""), + "HOME": os.environ.get("HOME", "/tmp"), + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + "AWS_REGION": "us-east-1", + "AWS_DEFAULT_REGION": "us-east-1", + "AWS_CONFIG_FILE": "", + "AWS_SHARED_CREDENTIALS_FILE": "", + } + + +def parse_presigned_url(url: str) -> dict: + parts = urlsplit(url) + query_params = {} + for part in parts.query.split("&"): + k, v = part.split("=", 1) + query_params[k] = v + return { + "hostname": parts.netloc, + "path": parts.path, + "query_params": query_params, + } + + +def assert_sigv4_query_params( + query_params: dict, expected_expires: str = "3600" +): + assert set(query_params.keys()) == EXPECTED_SIGV4_KEYS + assert query_params["X-Amz-Algorithm"] == "AWS4-HMAC-SHA256" + # Credential format: access_key/YYYYMMDD/region/s3/aws4_request (url-encoded) + cred = query_params["X-Amz-Credential"] + assert cred.startswith("access_key%2F") + assert "%2Fus-east-1%2Fs3%2Faws4_request" in cred + # Date format: YYYYMMDDTHHMMSSZ + assert re.match(r"\d{8}T\d{6}Z", query_params["X-Amz-Date"]) + assert query_params["X-Amz-Expires"] == expected_expires + assert query_params["X-Amz-SignedHeaders"] == "host" + + +@pytest.mark.asyncio +class TestPresignCommand: + async def test_generates_a_url(self, aws_cli): + env = presign_env() + stdout, stderr, rc = await run_cli( + aws_cli, ["s3", "presign", "s3://bucket/key"], env + ) + assert rc == 0, stderr.decode() + url = stdout.decode().strip() + parsed = parse_presigned_url(url) + assert parsed["hostname"] == "bucket.s3.us-east-1.amazonaws.com" + assert parsed["path"] == "/key" + assert_sigv4_query_params(parsed["query_params"]) + + async def test_handles_non_dns_compatible_buckets(self, aws_cli): + env = presign_env() + stdout, stderr, rc = await run_cli( + aws_cli, ["s3", "presign", "s3://bucket.dots/key"], env + ) + assert rc == 0, stderr.decode() + url = stdout.decode().strip() + parsed = parse_presigned_url(url) + assert parsed["hostname"] == "s3.us-east-1.amazonaws.com" + assert parsed["path"] == "/bucket.dots/key" + assert_sigv4_query_params(parsed["query_params"]) + + async def test_handles_expires_in(self, aws_cli): + env = presign_env() + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "presign", "s3://bucket/key", "--expires-in", "1000"], + env, + ) + assert rc == 0, stderr.decode() + url = stdout.decode().strip() + parsed = parse_presigned_url(url) + assert parsed["hostname"] == "bucket.s3.us-east-1.amazonaws.com" + assert parsed["path"] == "/key" + assert_sigv4_query_params( + parsed["query_params"], expected_expires="1000" + ) + + async def test_handles_sigv4(self, aws_cli, aws_config): + """sigv4 is the default, so this should produce same structure as test_generates_a_url.""" + config_path = aws_config( + {"default": {"s3": "\n signature_version = s3v4"}} + ) + env = presign_env() + env["AWS_CONFIG_FILE"] = config_path + stdout, stderr, rc = await run_cli( + aws_cli, ["s3", "presign", "s3://bucket/key"], env + ) + assert rc == 0, stderr.decode() + url = stdout.decode().strip() + parsed = parse_presigned_url(url) + assert parsed["hostname"] == "bucket.s3.us-east-1.amazonaws.com" + assert parsed["path"] == "/key" + assert_sigv4_query_params(parsed["query_params"]) + + async def test_s3_prefix_not_needed(self, aws_cli): + env = presign_env() + stdout, stderr, rc = await run_cli( + aws_cli, ["s3", "presign", "bucket/key"], env + ) + assert rc == 0, stderr.decode() + url = stdout.decode().strip() + parsed = parse_presigned_url(url) + assert parsed["hostname"] == "bucket.s3.us-east-1.amazonaws.com" + assert parsed["path"] == "/key" + assert_sigv4_query_params(parsed["query_params"]) + + async def test_can_support_addressing_mode_path(self, aws_cli, aws_config): + config_path = aws_config( + {"default": {"s3": "\n addressing_style = path"}} + ) + env = presign_env() + env["AWS_CONFIG_FILE"] = config_path + stdout, stderr, rc = await run_cli( + aws_cli, ["s3", "presign", "s3://bucket/key"], env + ) + assert rc == 0, stderr.decode() + url = stdout.decode().strip() + parsed = parse_presigned_url(url) + assert parsed["hostname"] == "s3.us-east-1.amazonaws.com" + assert parsed["path"] == "/bucket/key" + assert_sigv4_query_params(parsed["query_params"]) diff --git a/tests/blackbox/test_rb_command.py b/tests/blackbox/test_rb_command.py new file mode 100644 index 000000000000..13283d35f73c --- /dev/null +++ b/tests/blackbox/test_rb_command.py @@ -0,0 +1,156 @@ +"""Blackbox tests for `aws s3 rb` command.""" + +from __future__ import annotations + +import pytest + +from tests.blackbox.s3_assertions import ( + assert_delete_bucket, + assert_delete_object, + assert_list_objects_v2, +) + +from tests.blackbox.utils import ( + cli_env, + delete_bucket_response, + delete_response, + error_response, + format_requests, + list_objects_xml, + run_cli, + mock_server, + setup_responses, + xml_response, +) + + +def list_objects_response_xml(keys: list[str]) -> str: + contents = [ + {"Key": k, "Size": 100, "LastModified": "2016-03-01T23:50:13.000Z"} + for k in keys + ] + return list_objects_xml(contents=contents) + + +def empty_list_response_xml() -> str: + return list_objects_xml() + + +@pytest.mark.asyncio +class TestRBCommand: + async def test_rb(self, aws_cli): + async with mock_server() as (server, proxy): + setup_responses(server, [delete_bucket_response()]) + env = cli_env(proxy) + stdout, stderr, rc = await run_cli( + aws_cli, ["s3", "rb", "s3://bucket"], env + ) + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_delete_bucket( + server.requests[0], Bucket="bucket" + ) + + async def test_rb_force_empty_bucket(self, aws_cli): + async with mock_server() as (server, proxy): + setup_responses( + server, + [ + xml_response(empty_list_response_xml()), + delete_bucket_response(), + ], + ) + env = cli_env(proxy) + stdout, stderr, rc = await run_cli( + aws_cli, ["s3", "rb", "s3://bucket", "--force"], env + ) + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + # First: ListObjectsV2 + assert_list_objects_v2( + server.requests[0], Bucket="bucket" + ) + # Second: DeleteBucket + assert_delete_bucket( + server.requests[1], Bucket="bucket" + ) + + async def test_rb_force_non_empty_bucket(self, aws_cli): + async with mock_server() as (server, proxy): + setup_responses( + server, + [ + xml_response(list_objects_response_xml(["foo"])), + delete_response(), + delete_bucket_response(), + ], + ) + env = cli_env(proxy) + stdout, stderr, rc = await run_cli( + aws_cli, ["s3", "rb", "s3://bucket", "--force"], env + ) + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + # ListObjectsV2 + assert_list_objects_v2( + server.requests[0], Bucket="bucket" + ) + # DeleteObject + assert_delete_object( + server.requests[1], Bucket="bucket", Key="foo" + ) + # DeleteBucket + assert_delete_bucket( + server.requests[2], Bucket="bucket" + ) + + async def test_rb_failed_rc(self, aws_cli): + async with mock_server() as (server, proxy): + setup_responses(server, [error_response()]) + env = cli_env(proxy) + stdout, stderr, rc = await run_cli( + aws_cli, ["s3", "rb", "s3://bucket"], env + ) + assert rc == 1 + assert b"remove_bucket failed:" in stderr + + async def test_rb_force_with_failed_rm(self, aws_cli): + async with mock_server() as (server, proxy): + setup_responses(server, [error_response()]) + env = cli_env(proxy) + stdout, stderr, rc = await run_cli( + aws_cli, ["s3", "rb", "s3://bucket", "--force"], env + ) + assert rc == 255 + assert b"remove_bucket failed:" in stderr + # Should have attempted ListObjectsV2 only (failed there) + assert len(server.requests) == 1, format_requests(server) + assert_list_objects_v2( + server.requests[0], Bucket="bucket" + ) + + async def test_nonzero_exit_if_uri_scheme_not_provided(self, aws_cli): + async with mock_server() as (server, proxy): + env = cli_env(proxy) + stdout, stderr, rc = await run_cli( + aws_cli, ["s3", "rb", "bucket"], env + ) + assert rc == 252 + assert len(server.requests) == 0, format_requests(server) + + async def test_nonzero_exit_if_key_provided(self, aws_cli): + async with mock_server() as (server, proxy): + env = cli_env(proxy) + # With --force + stdout, stderr, rc = await run_cli( + aws_cli, ["s3", "rb", "s3://bucket/key", "--force"], env + ) + assert rc == 252 + assert len(server.requests) == 0, format_requests(server) + + # Without --force + stdout, stderr, rc = await run_cli( + aws_cli, ["s3", "rb", "s3://bucket/key"], env + ) + assert rc == 252 + assert len(server.requests) == 0, format_requests(server) diff --git a/tests/blackbox/test_rm_command.py b/tests/blackbox/test_rm_command.py new file mode 100644 index 000000000000..3370a99379a6 --- /dev/null +++ b/tests/blackbox/test_rm_command.py @@ -0,0 +1,133 @@ +"""Blackbox tests for `aws s3 rm` command.""" + +from __future__ import annotations + +import pytest + +from tests.blackbox.s3_assertions import ( + assert_delete_object, + assert_list_objects_v2, +) + +from tests.blackbox.utils import ( + cli_env, + delete_response, + format_requests, + list_objects_xml, + run_cli, + mock_server, + setup_responses, + xml_response, +) + + +def list_objects_response_xml(keys: list[str]) -> str: + contents = [ + {"Key": k, "Size": 100, "LastModified": "2014-01-09T20:45:49.000Z"} + for k in keys + ] + return list_objects_xml(contents=contents) + + +@pytest.mark.asyncio +async def test_operations_used(aws_cli: str) -> None: + """rm s3://bucket/key.txt should make exactly one DELETE request.""" + async with mock_server() as (server, proxy): + setup_responses(server, [delete_response()]) + + stdout, stderr, rc = await run_cli( + aws_cli, ["s3", "rm", "s3://bucket/key.txt"], cli_env(proxy) + ) + + assert ( + rc == 0 + ), f"stdout={stdout!r} stderr={stderr!r}\n{format_requests(server)}" + assert len(server.requests) == 1 + assert_delete_object( + server.requests[0], Bucket="bucket", Key="key.txt" + ) + + +@pytest.mark.asyncio +async def test_dryrun_delete(aws_cli: str) -> None: + """rm --dryrun should not make any requests.""" + async with mock_server() as (server, proxy): + # No responses needed — no requests should be made + setup_responses(server, []) + + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "rm", "s3://bucket/key.txt", "--dryrun"], + cli_env(proxy), + ) + + assert ( + rc == 0 + ), f"stdout={stdout!r} stderr={stderr!r}\n{format_requests(server)}" + assert len(server.requests) == 0 + assert b"(dryrun) delete: s3://bucket/key.txt" in stdout + + +@pytest.mark.asyncio +async def test_delete_with_request_payer(aws_cli: str) -> None: + """rm --request-payer should send x-amz-request-payer header.""" + async with mock_server() as (server, proxy): + setup_responses(server, [delete_response()]) + + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "rm", "s3://mybucket/mykey", "--request-payer"], + cli_env(proxy), + ) + + assert ( + rc == 0 + ), f"stdout={stdout!r} stderr={stderr!r}\n{format_requests(server)}" + assert len(server.requests) == 1 + assert_delete_object( + server.requests[0], + Bucket="mybucket", + Key="mykey", + RequestPayer="requester", + ) + + +@pytest.mark.asyncio +async def test_recursive_delete_with_request_payer(aws_cli: str) -> None: + """rm --recursive --request-payer should list then delete with request-payer on both.""" + response_xml = list_objects_response_xml(["mykey"]) + + async with mock_server() as (server, proxy): + setup_responses( + server, + [ + xml_response(response_xml), + delete_response(), + ], + ) + + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "rm", "s3://mybucket/", "--recursive", "--request-payer"], + cli_env(proxy), + ) + + assert ( + rc == 0 + ), f"stdout={stdout!r} stderr={stderr!r}\n{format_requests(server)}" + assert len(server.requests) == 2 + + # First request: ListObjectsV2 + assert_list_objects_v2( + server.requests[0], + Bucket="mybucket", + RequestPayer="requester", + ) + + # Second request: DeleteObject + assert_delete_object( + server.requests[1], + Bucket="mybucket", + Key="mykey", + RequestPayer="requester", + ) diff --git a/tests/blackbox/utils.py b/tests/blackbox/utils.py new file mode 100644 index 000000000000..72873b0aee34 --- /dev/null +++ b/tests/blackbox/utils.py @@ -0,0 +1,156 @@ +"""Shared test utilities for blackbox tests.""" + +from __future__ import annotations + +import asyncio +import os +from contextlib import asynccontextmanager +from urllib.parse import parse_qs, urlparse + +from localstub.server import AsyncHTTPTestServer, HTTPResponse +from localstub.tlsproxy import AsyncTLSInterceptProxy + + +def cli_env(proxy: AsyncTLSInterceptProxy) -> dict[str, str]: + return { + "PATH": os.environ.get("PATH", ""), + "HOME": os.environ.get("HOME", "/tmp"), + "AWS_ACCESS_KEY_ID": "test", + "AWS_SECRET_ACCESS_KEY": "test", + "AWS_REGION": "us-east-1", + "AWS_DEFAULT_REGION": "us-east-1", + "AWS_MAX_ATTEMPTS": "1", + "AWS_CONFIG_FILE": "", + "AWS_SHARED_CREDENTIALS_FILE": "", + "HTTPS_PROXY": proxy.endpoint_url, + "HTTP_PROXY": proxy.endpoint_url, + "AWS_CA_BUNDLE": str(proxy.ca.ca_pem_path()), + } + + +@asynccontextmanager +async def mock_server(on_headers_received=None): + """Async context manager that yields (server, proxy) for blackbox tests. + + All CLI traffic must go through this proxy. If the binary under test + does not respect HTTPS_PROXY, requests will fail to connect (the env + has no real credentials and NO_PROXY is empty). + """ + async with ( + AsyncHTTPTestServer(on_headers_received=on_headers_received) as server, + AsyncTLSInterceptProxy(server=server) as proxy, + ): + yield server, proxy + + +def setup_responses( + server: AsyncHTTPTestServer, responses: list[HTTPResponse] +) -> None: + """Configure server to return responses in order.""" + server.set_response_sequence(responses) + + +def xml_response(xml: str) -> HTTPResponse: + return HTTPResponse.raw( + xml.encode(), headers={"Content-Type": "application/xml"} + ) + + +def format_requests(server: AsyncHTTPTestServer) -> str: + """Format captured requests for inclusion in assertion messages.""" + lines = [ + f" [{i}] {r.method} {r.path} Host={r.headers.get('host')}" + for i, r in enumerate(server.requests) + ] + return ( + f"requests({len(server.requests)}):\n" + "\n".join(lines) + if lines + else "requests(0): " + ) + + +def get_query_params(request) -> dict[str, list[str]]: + """Parse query string params from a recorded request's path.""" + parsed = urlparse(request.path) + return parse_qs(parsed.query) + + +async def run_cli( + aws_cli: str, args: list[str], env: dict, stdin: bytes | None = None +) -> tuple[bytes, bytes, int]: + proc = await asyncio.create_subprocess_exec( + aws_cli, + *args, + env=env, + stdin=asyncio.subprocess.PIPE if stdin is not None else None, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await proc.communicate(input=stdin) + return stdout, stderr, proc.returncode + + +def list_objects_xml( + contents: list[dict] | None = None, + common_prefixes: list[str] | None = None, + is_truncated: bool = False, + next_continuation_token: str | None = None, + prefix: str | None = None, +) -> str: + xml = ( + '' + '' + ) + if prefix is None: + xml += "" + else: + xml += f"{prefix}" + if is_truncated: + xml += "true" + if next_continuation_token: + xml += f"{next_continuation_token}" + if contents: + for obj in contents: + xml += "" + xml += f"{obj['Key']}" + xml += f"{obj['Size']}" + xml += f"{obj['LastModified']}" + xml += f"{obj.get('ETag', 'd41d8cd98f00b204e9800998ecf8427e')}" + if "StorageClass" in obj: + xml += f"{obj['StorageClass']}" + xml += "" + if common_prefixes: + for prefix in common_prefixes: + xml += ( + f"{prefix}" + ) + xml += "" + return xml + + +def delete_response() -> HTTPResponse: + return HTTPResponse.raw(b"", status=204) + + +def error_response( + code: str = "InternalError", + message: str = "Internal Server Error", + status: int = 500, +) -> HTTPResponse: + body = ( + '' + f"{code}{message}" + ) + return HTTPResponse.raw( + body.encode(), + status=status, + headers={"Content-Type": "application/xml"}, + ) + + +def create_bucket_response() -> HTTPResponse: + return HTTPResponse.raw(b"", status=200, headers={"Location": "/bucket"}) + + +def delete_bucket_response() -> HTTPResponse: + return HTTPResponse.raw(b"", status=204)