diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 962852370..5e4efdd50 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -148,7 +148,9 @@ jobs: run: pnpm --filter vite-task-tools exec playwright install-deps chromium - name: Run ignored tests - run: ${{ matrix.cargo_cmd }} test --target ${{ matrix.build_target }} -- --ignored + run: | + pnpm --filter vite-task-tools test + ${{ matrix.cargo_cmd }} test --target ${{ matrix.build_target }} -- --ignored # Windows tests are cross-compiled on a fast Linux runner with cargo-xwin # (clang-cl + lld-link against the xwin-downloaded MSVC CRT/Windows SDK) @@ -281,7 +283,9 @@ jobs: - name: Run ignored tests if: matrix.mode == 'ignored' - run: cargo-nextest nextest run --archive-file windows-tests.tar.zst --workspace-remap . --run-ignored ignored-only + run: | + pnpm --filter vite-task-tools test + cargo-nextest nextest run --archive-file windows-tests.tar.zst --workspace-remap . --run-ignored ignored-only test-musl: needs: detect-changes @@ -335,7 +339,9 @@ jobs: - uses: oxc-project/setup-node@f46a72f95efdc55273fcd042d61c84e723b2892c # v1.4.1 - name: Run ignored tests - run: cargo test -- --ignored + run: | + pnpm --filter vite-task-tools test + cargo test -- --ignored fmt: name: Format and Check Deps @@ -374,6 +380,9 @@ jobs: pnpm build-vite-task-client-types git diff --exit-code packages/vite-task-client/src/index.d.ts + - name: Check test utility types + run: pnpm --filter vite-task-tools check + done: runs-on: namespace-profile-linux-x64-default if: always() diff --git a/.typos.toml b/.typos.toml index e7642038d..c227ba5df 100644 --- a/.typos.toml +++ b/.typos.toml @@ -1,6 +1,7 @@ [default.extend-words] ratatui = "ratatui" PUNICODE = "PUNICODE" +EDN = "EDN" # Win32 spells `ERROR_FILENAME_EXCED_RANGE` this way. EXCED = "EXCED" diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/package.json b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/package.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/package.json @@ -0,0 +1 @@ +{} diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots.toml b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots.toml new file mode 100644 index 000000000..20bdee4a9 --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots.toml @@ -0,0 +1,514 @@ +# Windows is skipped because the PTY launcher cannot execute pnpm command shims. +[[e2e]] +name = "entries_and_associations" +cfg = "not(windows)" +ignore = true +steps = [ + [ + "remote-cache-server", + "start", + ], + [ + "cbor-http", + "POST", + "/fetch", + "--cbor", + "{\"key\": 'A', \"secondary_key\": 'S'}", + ], + [ + "cbor-http", + "POST", + "/store", + "--form-cbor", + "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'VA'}", + ], + [ + "cbor-http", + "POST", + "/store", + "--form-cbor", + "metadata={\"key\": 'A', \"secondary_key\": 'U', \"value\": 'VA'}", + ], + [ + "cbor-http", + "POST", + "/store", + "--form-cbor", + "metadata={\"key\": 'B', \"secondary_key\": 'S', \"value\": 'VB'}", + ], + { argv = [ + "cbor-http", + "POST", + "/fetch", + "--cbor", + "{\"key\": 'A', \"secondary_key\": 'S'}", + ], comment = "The exact entry survives reassignment of S to B." }, + { argv = [ + "cbor-http", + "POST", + "/fetch", + "--cbor", + "{\"key\": 'C', \"secondary_key\": 'S'}", + ], comment = "The previous fetch did not change S. Fallback returns only B's key." }, + [ + "cbor-http", + "POST", + "/store", + "--form-cbor", + "metadata={\"key\": 'A', \"secondary_key\": 'T', \"value\": 'VA2'}", + ], + [ + "cbor-http", + "POST", + "/fetch", + "--cbor", + "{\"key\": 'A', \"secondary_key\": 'missing'}", + ], + { argv = [ + "cbor-http", + "POST", + "/fetch", + "--cbor", + "{\"key\": 'C', \"secondary_key\": 'U'}", + ], comment = "An older association to A still resolves after A is replaced." }, + [ + "cbor-http", + "POST", + "/fetch", + "--cbor", + "{\"key\": 'C', \"secondary_key\": 'T'}", + ], + [ + "cbor-http", + "POST", + "/fetch", + "--cbor", + "{\"key\": 'C', \"secondary_key\": 'S'}", + ], + [ + "remote-cache-server", + "stop", + ], +] + +[[e2e]] +name = "blob_replacement" +cfg = "not(windows)" +ignore = true +steps = [ + [ + "remote-cache-server", + "start", + ], + [ + "cbor-http", + "GET", + "/blob/missing", + ], + [ + "vtt", + "write-file", + "archive.txt", + "first archive", + ], + [ + "cbor-http", + "POST", + "/store", + "--form-cbor", + "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'first'}", + "--form-file", + "blob=archive.txt", + ], + [ + "cbor-http", + "POST", + "/fetch", + "--cbor", + "{\"key\": 'A', \"secondary_key\": 'S'}", + ], + [ + "cbor-http", + "GET", + "/blob/1", + ], + { argv = [ + "cbor-http", + "POST", + "/store", + "--form-data", + "blob='second archive'", + "--form-cbor", + "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'second'}", + ], comment = "Accept blob before metadata, with neither part supplying a filename." }, + [ + "cbor-http", + "POST", + "/fetch", + "--cbor", + "{\"key\": 'A', \"secondary_key\": 'S'}", + ], + [ + "cbor-http", + "GET", + "/blob/2", + ], + { argv = [ + "cbor-http", + "GET", + "/blob/1", + ], comment = "Previously returned IDs retain their original bytes." }, + { argv = [ + "cbor-http", + "POST", + "/store", + "--form-cbor", + "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'no archive'}", + ], comment = "Omitting blob clears the association." }, + [ + "cbor-http", + "POST", + "/fetch", + "--cbor", + "{\"key\": 'A', \"secondary_key\": 'S'}", + ], + [ + "cbor-http", + "POST", + "/store", + "--form-cbor", + "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'empty archive'}", + "--form-data", + "blob=''", + ], + [ + "cbor-http", + "POST", + "/fetch", + "--cbor", + "{\"key\": 'A', \"secondary_key\": 'S'}", + ], + { argv = [ + "cbor-http", + "GET", + "/blob/3", + ], comment = "An empty blob has an ID and downloads as an empty byte string." }, + [ + "remote-cache-server", + "stop", + ], +] + +[[e2e]] +name = "opaque_binary_data" +cfg = "not(windows)" +ignore = true +steps = [ + [ + "remote-cache-server", + "start", + ], + [ + "cbor-http", + "POST", + "/store", + "--form-cbor", + "metadata={\"key\": b64'AP+A', \"secondary_key\": '', \"value\": b64'AP+A'}", + "--form-data", + "blob=b64'AP+A'", + ], + [ + "cbor-http", + "POST", + "/fetch", + "--cbor", + "{\"key\": b64'AP+A', \"secondary_key\": ''}", + ], + [ + "cbor-http", + "POST", + "/fetch", + "--cbor", + "{\"key\": '', \"secondary_key\": ''}", + ], + [ + "cbor-http", + "GET", + "/blob/1", + ], + [ + "cbor-http", + "POST", + "/store", + "--form-cbor", + "metadata={\"key\": '', \"secondary_key\": '', \"value\": ''}", + ], + [ + "cbor-http", + "POST", + "/fetch", + "--cbor", + "{\"key\": '', \"secondary_key\": ''}", + ], + [ + "remote-cache-server", + "stop", + ], +] + +[[e2e]] +name = "invalid_requests" +cfg = "not(windows)" +ignore = true +steps = [ + [ + "remote-cache-server", + "start", + ], + [ + "cbor-http", + "GET", + "/missing", + ], + [ + "cbor-http", + "POST", + "/fetch", + "--data", + "'bad'", + ], + [ + "cbor-http", + "POST", + "/fetch", + "--data", + "'bad'", + "--content-type", + "application/cbor", + ], + [ + "cbor-http", + "POST", + "/fetch", + "--cbor", + "{\"key\": \"text\", \"secondary_key\": ''}", + ], + [ + "cbor-http", + "POST", + "/fetch", + "--cbor", + "{\"key\": ''}", + ], + [ + "cbor-http", + "POST", + "/fetch", + "--cbor", + "[]", + ], + [ + "cbor-http", + "POST", + "/fetch", + "--cbor", + "{\"key\": '', \"key\": 'duplicate', \"secondary_key\": ''}", + ], + [ + "cbor-http", + "POST", + "/store", + "--form-cbor", + "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'original'}", + "--form-data", + "blob='original archive'", + ], + [ + "cbor-http", + "POST", + "/store", + "--cbor", + "{}", + ], + [ + "cbor-http", + "POST", + "/store", + "--form-data", + "blob='missing metadata'", + ], + [ + "cbor-http", + "POST", + "/store", + "--form-cbor", + "metadata={\"key\": 'A', \"secondary_key\": 'T', \"value\": \"wrong type\"}", + ], + [ + "cbor-http", + "POST", + "/store", + "--form-cbor", + "metadata={\"key\": 'A', \"secondary_key\": 'T', \"value\": 'replacement'}", + "--form-cbor", + "metadata={}", + ], + [ + "cbor-http", + "POST", + "/store", + "--form-cbor", + "metadata={\"key\": 'A', \"secondary_key\": 'T', \"value\": 'replacement'}", + "--form-data", + "blob='one'", + "--form-data", + "blob='two'", + ], + [ + "cbor-http", + "POST", + "/store", + "--form-cbor", + "metadata={}", + "--form-data", + "unexpected='part'", + ], + [ + "cbor-http", + "POST", + "/store", + "--form-data", + "metadata='wrong content type'", + ], + [ + "cbor-http", + "POST", + "/store", + "--data", + "'--boundary\\r\\nContent-Disposition: form-data; name=\"metadata\"\\r\\nContent-Type: application/cbor\\r\\n\\r\\n'", + "--content-type", + "multipart/form-data; boundary=boundary", + ], + { argv = [ + "cbor-http", + "POST", + "/fetch", + "--cbor", + "{\"key\": 'A', \"secondary_key\": 'S'}", + ], comment = "Failed stores did not replace the value or blob." }, + [ + "cbor-http", + "GET", + "/blob/1", + ], + { argv = [ + "cbor-http", + "POST", + "/fetch", + "--cbor", + "{\"key\": 'B', \"secondary_key\": 'T'}", + ], comment = "Failed stores did not publish a secondary association." }, + [ + "remote-cache-server", + "stop", + ], +] + +[[e2e]] +name = "request_limit" +cfg = "not(windows)" +ignore = true +steps = [ + [ + "remote-cache-server", + "start", + "--max-request-bytes", + "1024", + ], + [ + "cbor-http", + "POST", + "/store", + "--form-cbor", + "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'original'}", + ], + [ + "node", + "-e", + "require('node:fs').writeFileSync('large.bin', Buffer.alloc(2048))", + ], + [ + "cbor-http", + "POST", + "/store", + "--form-cbor", + "metadata={\"key\": 'A', \"secondary_key\": 'T', \"value\": 'replacement'}", + "--form-file", + "blob=large.bin", + ], + [ + "cbor-http", + "POST", + "/fetch", + "--cbor", + "{\"key\": 'A', \"secondary_key\": 'S'}", + ], + [ + "cbor-http", + "POST", + "/fetch", + "--cbor", + "{\"key\": 'B', \"secondary_key\": 'T'}", + ], + [ + "remote-cache-server", + "stop", + ], +] + +[[e2e]] +name = "daemon_lifecycle" +cfg = "not(windows)" +ignore = true +steps = [ + [ + "remote-cache-server", + "start", + "--base-path", + "/projects/test", + ], + [ + "remote-cache-server", + "start", + ], + [ + "cbor-http", + "POST", + "/store", + "--form-cbor", + "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'first daemon'}", + ], + { argv = [ + "remote-cache-server", + "stop", + ], comment = "Connect before deleting cache.url, then wait for the connection to close before restarting." }, + [ + "remote-cache-server", + "start", + ], + { argv = [ + "cbor-http", + "POST", + "/fetch", + "--cbor", + "{\"key\": 'A', \"secondary_key\": 'S'}", + ], comment = "A new daemon starts with empty state." }, + [ + "remote-cache-server", + "stop", + ], + [ + "remote-cache-server", + "stop", + ], +] diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/blob_replacement.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/blob_replacement.md new file mode 100644 index 000000000..23d025b5b --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/blob_replacement.md @@ -0,0 +1,104 @@ +# blob_replacement + +## `remote-cache-server start` + +``` +Remote cache server started +``` + +## `cbor-http GET /blob/missing` + +``` +{"status": 404, "content_type": "text/plain; charset=utf-8", "body": "Blob not found"} +``` + +## `vtt write-file archive.txt 'first archive'` + +``` +``` + +## `cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''first'\''}' --form-file blob=archive.txt` + +``` +{"status": 200, "content_type": "application/cbor", "body": {"blob_id": "1"}} +``` + +## `cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` + +``` +{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'first', "blob_id": "1"}} +``` + +## `cbor-http GET /blob/1` + +``` +{"status": 200, "content_type": "application/octet-stream", "body": 'first archive'} +``` + +## `cbor-http POST /store --form-data 'blob='\''second archive'\''' --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''second'\''}'` + +Accept blob before metadata, with neither part supplying a filename. + +``` +{"status": 200, "content_type": "application/cbor", "body": {"blob_id": "2"}} +``` + +## `cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` + +``` +{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'second', "blob_id": "2"}} +``` + +## `cbor-http GET /blob/2` + +``` +{"status": 200, "content_type": "application/octet-stream", "body": 'second archive'} +``` + +## `cbor-http GET /blob/1` + +Previously returned IDs retain their original bytes. + +``` +{"status": 200, "content_type": "application/octet-stream", "body": 'first archive'} +``` + +## `cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''no archive'\''}'` + +Omitting blob clears the association. + +``` +{"status": 200, "content_type": "application/cbor", "body": {"blob_id": null}} +``` + +## `cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` + +``` +{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'no archive', "blob_id": null}} +``` + +## `cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''empty archive'\''}' --form-data blob=''` + +``` +{"status": 200, "content_type": "application/cbor", "body": {"blob_id": "3"}} +``` + +## `cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` + +``` +{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'empty archive', "blob_id": "3"}} +``` + +## `cbor-http GET /blob/3` + +An empty blob has an ID and downloads as an empty byte string. + +``` +{"status": 200, "content_type": "application/octet-stream", "body": ''} +``` + +## `remote-cache-server stop` + +``` +Remote cache server stopped +``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/daemon_lifecycle.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/daemon_lifecycle.md new file mode 100644 index 000000000..46d141e70 --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/daemon_lifecycle.md @@ -0,0 +1,55 @@ +# daemon_lifecycle + +## `remote-cache-server start --base-path /projects/test` + +``` +Remote cache server started +``` + +## `remote-cache-server start` + +**Exit code:** 1 + +``` +Remote cache server already started (cache.url exists) +``` + +## `cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''first daemon'\''}'` + +``` +{"status": 200, "content_type": "application/cbor", "body": {"blob_id": null}} +``` + +## `remote-cache-server stop` + +Connect before deleting cache.url, then wait for the connection to close before restarting. + +``` +Remote cache server stopped +``` + +## `remote-cache-server start` + +``` +Remote cache server started +``` + +## `cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` + +A new daemon starts with empty state. + +``` +{"status": 404, "content_type": "text/plain; charset=utf-8", "body": "Entry not found"} +``` + +## `remote-cache-server stop` + +``` +Remote cache server stopped +``` + +## `remote-cache-server stop` + +``` +No remote cache endpoint found +``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/entries_and_associations.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/entries_and_associations.md new file mode 100644 index 000000000..d52e2d149 --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/entries_and_associations.md @@ -0,0 +1,85 @@ +# entries_and_associations + +## `remote-cache-server start` + +``` +Remote cache server started +``` + +## `cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` + +``` +{"status": 404, "content_type": "text/plain; charset=utf-8", "body": "Entry not found"} +``` + +## `cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''VA'\''}'` + +``` +{"status": 200, "content_type": "application/cbor", "body": {"blob_id": null}} +``` + +## `cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''U'\'', "value": '\''VA'\''}'` + +``` +{"status": 200, "content_type": "application/cbor", "body": {"blob_id": null}} +``` + +## `cbor-http POST /store --form-cbor 'metadata={"key": '\''B'\'', "secondary_key": '\''S'\'', "value": '\''VB'\''}'` + +``` +{"status": 200, "content_type": "application/cbor", "body": {"blob_id": null}} +``` + +## `cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` + +The exact entry survives reassignment of S to B. + +``` +{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'VA', "blob_id": null}} +``` + +## `cbor-http POST /fetch --cbor '{"key": '\''C'\'', "secondary_key": '\''S'\''}'` + +The previous fetch did not change S. Fallback returns only B's key. + +``` +{"status": 200, "content_type": "application/cbor", "body": {"kind": "fallback", "key": 'B'}} +``` + +## `cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''T'\'', "value": '\''VA2'\''}'` + +``` +{"status": 200, "content_type": "application/cbor", "body": {"blob_id": null}} +``` + +## `cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''missing'\''}'` + +``` +{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'VA2', "blob_id": null}} +``` + +## `cbor-http POST /fetch --cbor '{"key": '\''C'\'', "secondary_key": '\''U'\''}'` + +An older association to A still resolves after A is replaced. + +``` +{"status": 200, "content_type": "application/cbor", "body": {"kind": "fallback", "key": 'A'}} +``` + +## `cbor-http POST /fetch --cbor '{"key": '\''C'\'', "secondary_key": '\''T'\''}'` + +``` +{"status": 200, "content_type": "application/cbor", "body": {"kind": "fallback", "key": 'A'}} +``` + +## `cbor-http POST /fetch --cbor '{"key": '\''C'\'', "secondary_key": '\''S'\''}'` + +``` +{"status": 200, "content_type": "application/cbor", "body": {"kind": "fallback", "key": 'B'}} +``` + +## `remote-cache-server stop` + +``` +Remote cache server stopped +``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/invalid_requests.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/invalid_requests.md new file mode 100644 index 000000000..de39985ce --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/invalid_requests.md @@ -0,0 +1,131 @@ +# invalid_requests + +## `remote-cache-server start` + +``` +Remote cache server started +``` + +## `cbor-http GET /missing` + +``` +{"status": 404, "content_type": "text/plain; charset=utf-8", "body": "Route not found"} +``` + +## `cbor-http POST /fetch --data 'bad'` + +``` +{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Expected application/cbor"} +``` + +## `cbor-http POST /fetch --data 'bad' --content-type application/cbor` + +``` +{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Expected byte strings: key, secondary_key"} +``` + +## `cbor-http POST /fetch --cbor '{"key": "text", "secondary_key": '\'''\''}'` + +``` +{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Expected byte strings: key, secondary_key"} +``` + +## `cbor-http POST /fetch --cbor '{"key": '\'''\''}'` + +``` +{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Expected byte strings: key, secondary_key"} +``` + +## `cbor-http POST /fetch --cbor []` + +``` +{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Expected byte strings: key, secondary_key"} +``` + +## `cbor-http POST /fetch --cbor '{"key": '\'''\'', "key": '\''duplicate'\'', "secondary_key": '\'''\''}'` + +``` +{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Invalid CBOR"} +``` + +## `cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''original'\''}' --form-data 'blob='\''original archive'\'''` + +``` +{"status": 200, "content_type": "application/cbor", "body": {"blob_id": "1"}} +``` + +## `cbor-http POST /store --cbor {}` + +``` +{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Expected multipart/form-data"} +``` + +## `cbor-http POST /store --form-data 'blob='\''missing metadata'\'''` + +``` +{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Missing metadata"} +``` + +## `cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''T'\'', "value": "wrong type"}'` + +``` +{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Expected byte strings: key, secondary_key, value"} +``` + +## `cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''T'\'', "value": '\''replacement'\''}' --form-cbor metadata={}` + +``` +{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Invalid multipart body"} +``` + +## `cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''T'\'', "value": '\''replacement'\''}' --form-data blob='one' --form-data blob='two'` + +``` +{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Invalid multipart body"} +``` + +## `cbor-http POST /store --form-cbor metadata={} --form-data unexpected='part'` + +``` +{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Invalid multipart body"} +``` + +## `cbor-http POST /store --form-data 'metadata='\''wrong content type'\'''` + +``` +{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Invalid multipart body"} +``` + +## `cbor-http POST /store --data ''\''--boundary\r\nContent-Disposition: form-data; name="metadata"\r\nContent-Type: application/cbor\r\n\r\n'\''' --content-type 'multipart/form-data; boundary=boundary'` + +``` +{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Invalid multipart body"} +``` + +## `cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` + +Failed stores did not replace the value or blob. + +``` +{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'original', "blob_id": "1"}} +``` + +## `cbor-http GET /blob/1` + +``` +{"status": 200, "content_type": "application/octet-stream", "body": 'original archive'} +``` + +## `cbor-http POST /fetch --cbor '{"key": '\''B'\'', "secondary_key": '\''T'\''}'` + +Failed stores did not publish a secondary association. + +``` +{"status": 404, "content_type": "text/plain; charset=utf-8", "body": "Entry not found"} +``` + +## `remote-cache-server stop` + +``` +Remote cache server stopped +``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/opaque_binary_data.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/opaque_binary_data.md new file mode 100644 index 000000000..d13fcf0db --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/opaque_binary_data.md @@ -0,0 +1,49 @@ +# opaque_binary_data + +## `remote-cache-server start` + +``` +Remote cache server started +``` + +## `cbor-http POST /store --form-cbor 'metadata={"key": b64'\''AP+A'\'', "secondary_key": '\'''\'', "value": b64'\''AP+A'\''}' --form-data blob=b64'AP+A'` + +``` +{"status": 200, "content_type": "application/cbor", "body": {"blob_id": "1"}} +``` + +## `cbor-http POST /fetch --cbor '{"key": b64'\''AP+A'\'', "secondary_key": '\'''\''}'` + +``` +{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": b64'AP+A', "blob_id": "1"}} +``` + +## `cbor-http POST /fetch --cbor '{"key": '\'''\'', "secondary_key": '\'''\''}'` + +``` +{"status": 200, "content_type": "application/cbor", "body": {"kind": "fallback", "key": b64'AP+A'}} +``` + +## `cbor-http GET /blob/1` + +``` +{"status": 200, "content_type": "application/octet-stream", "body": b64'AP+A'} +``` + +## `cbor-http POST /store --form-cbor 'metadata={"key": '\'''\'', "secondary_key": '\'''\'', "value": '\'''\''}'` + +``` +{"status": 200, "content_type": "application/cbor", "body": {"blob_id": null}} +``` + +## `cbor-http POST /fetch --cbor '{"key": '\'''\'', "secondary_key": '\'''\''}'` + +``` +{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": '', "blob_id": null}} +``` + +## `remote-cache-server stop` + +``` +Remote cache server stopped +``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/request_limit.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/request_limit.md new file mode 100644 index 000000000..38aceb511 --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/request_limit.md @@ -0,0 +1,42 @@ +# request_limit + +## `remote-cache-server start --max-request-bytes 1024` + +``` +Remote cache server started +``` + +## `cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''original'\''}'` + +``` +{"status": 200, "content_type": "application/cbor", "body": {"blob_id": null}} +``` + +## `node -e 'require('\''node:fs'\'').writeFileSync('\''large.bin'\'', Buffer.alloc(2048))'` + +``` +``` + +## `cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''T'\'', "value": '\''replacement'\''}' --form-file blob=large.bin` + +``` +{"status": 413, "content_type": "text/plain; charset=utf-8", "body": "Request too large"} +``` + +## `cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` + +``` +{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'original', "blob_id": null}} +``` + +## `cbor-http POST /fetch --cbor '{"key": '\''B'\'', "secondary_key": '\''T'\''}'` + +``` +{"status": 404, "content_type": "text/plain; charset=utf-8", "body": "Entry not found"} +``` + +## `remote-cache-server stop` + +``` +Remote cache server stopped +``` diff --git a/packages/tools/README.md b/packages/tools/README.md index 629f20d92..438964ca5 100644 --- a/packages/tools/README.md +++ b/packages/tools/README.md @@ -1 +1,47 @@ -This package provides Node.js dependencies (oxlint, oxfmt, cross-env) used by fspy tests. +This package provides Node.js dependencies and commands for the test suites. + +## Remote cache backend + +Run `pnpm install` at the repository root to install `remote-cache-server` and `cbor-http` into this package's `node_modules/.bin`. The workspace self-dependency makes pnpm link the package's own commands there. The E2E harness already includes that directory in PATH. Both commands run TypeScript directly using the Node version in `.node-version`. + +```sh +remote-cache-server start +cbor-http POST /store --form-cbor "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'record'}" +cbor-http POST /fetch --cbor "{\"key\": 'A', \"secondary_key\": 'S'}" +remote-cache-server stop +``` + +`start` creates `cache.url` exclusively in the current directory, refusing to start if it already exists. It launches an in-memory daemon on a free loopback port and fills the file with its endpoint before returning. Each E2E case has its own directory and server state. + +`stop` connects to the endpoint, deletes `cache.url`, and reads until EOF or a connection reset confirms shutdown. The daemon watches the directory for file changes and closes its listener before closing active connections. Wait for `stop` to finish before restarting in the same directory. Deleting the file manually also requests shutdown, but a later `stop` cannot wait for the server because the endpoint is gone. The harness enforces step timeouts and skips remaining steps when one times out. The detached daemon exits after five minutes so it is cleaned up even if `stop` never runs. Use `--max-lifetime-ms` on `start` for longer tests. + +Daemon output goes to `cache.log`, and startup failures include that log. The log stays in the test's directory. + +The backend implements `POST /fetch`, `POST /store`, and `GET /blob/{blob_id}` from the [remote cache RFC](https://github.com/voidzero-dev/vite-task/blob/rfc-cloudflare-remote-cache/docs/rfcs/0001-remote-cache.md#4-http-api-mapping). Keys and values are opaque bytes. Blobs remain unchanged until the daemon exits. Blob IDs are sequential strings to keep snapshots deterministic. There is no authentication or persistent storage. + +`start --base-path /projects/test` places the API under that path and includes it in `cache.url`. Requests are limited to 64 MiB by default; use `--max-request-bytes` to change the limit. Keys, values, and blobs have no separate length limits. + +## CBOR HTTP client + +`cbor-http METHOD PATH` resolves the path against `cache.url`. An absolute HTTP URL works without that file. Request bodies use [CBOR extended diagnostic notation (EDN)](https://www.rfc-editor.org/rfc/rfc8610.html#appendix-G): `"text"` is text, `'bytes'` is a UTF-8 byte string, and `b64'AP+A'` contains arbitrary binary bytes. + +| Option | Body | +| ----------------------- | --------------------------------------------------------- | +| `--cbor EDN` | Encode EDN as CBOR with `Content-Type: application/cbor`. | +| `--data EDN` | Send an EDN byte string as raw bytes. | +| `--form-cbor NAME=EDN` | Add a CBOR multipart part. | +| `--form-data NAME=EDN` | Add an EDN byte string as a raw multipart part. | +| `--form-file NAME=FILE` | Add file contents as a raw multipart part. | +| `--content-type TYPE` | Override the request content type. | + +Multipart options can repeat and retain their order. Raw bodies and parts default to `application/octet-stream`. Parts have no filename, including binary metadata. + +Every response prints one EDN map containing `status`, `content_type`, and `body`. CBOR is decoded, text becomes a text string, and other bodies become byte strings. Byte strings use readable UTF-8 when possible and base64 EDN otherwise. HTTP error responses exit successfully so snapshots can assert their status and body. Argument errors, transport failures, and invalid CBOR responses exit unsuccessfully. + +```text +{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'record', "blob_id": null}} +``` + +Run `pnpm --filter vite-task-tools check` for type checking and `pnpm --filter vite-task-tools test` for the EDN-formatting unit test. Run `cargo test -p vt_bin --test e2e_snapshots -- remote_cache_backend --ignored` for the backend snapshots. + +The backend snapshots are skipped on Windows because the PTY launcher cannot execute pnpm command shims. diff --git a/packages/tools/package.json b/packages/tools/package.json index 22355f5c7..8dc90a5a6 100644 --- a/packages/tools/package.json +++ b/packages/tools/package.json @@ -1,14 +1,25 @@ { "name": "vite-task-tools", "private": true, + "bin": { + "cbor-http": "./src/cbor-http.ts", + "remote-cache-server": "./src/remote-cache/cli.ts" + }, "type": "module", + "scripts": { + "check": "tsc --noEmit -p tsconfig.json", + "test": "node --test src/*.test.ts" + }, "dependencies": { "@anthropic-ai/sandbox-runtime": "catalog:", + "@fastify/busboy": "catalog:", "@openai/codex": "catalog:", "@playwright/browser-chromium": "catalog:", "@vitest/browser-playwright": "catalog:", "@voidzero-dev/vite-task-client": "workspace:*", "bun": "catalog:", + "cbor-edn": "catalog:", + "cbor2": "catalog:", "cross-env": "catalog:", "deno": "catalog:", "oxfmt": "catalog:", @@ -18,5 +29,8 @@ "typescript": "7.1.0-dev.20260910.1", "vite": "catalog:", "vitest": "catalog:" + }, + "devDependencies": { + "vite-task-tools": "workspace:*" } } diff --git a/packages/tools/src/cbor-http.ts b/packages/tools/src/cbor-http.ts new file mode 100755 index 000000000..b889d7fc9 --- /dev/null +++ b/packages/tools/src/cbor-http.ts @@ -0,0 +1,108 @@ +#!/usr/bin/env node +import { parseEDN } from 'cbor-edn'; +import { decode } from 'cbor2/decoder'; +import { encode } from 'cbor2/encoder'; +import { randomUUID } from 'node:crypto'; +import { readFile } from 'node:fs/promises'; +import { parseArgs } from 'node:util'; +import { toEDN } from './edn.ts'; + +function bytes(edn: string): Uint8Array { + const value: unknown = decode(parseEDN(edn, {})); + if (!(value instanceof Uint8Array)) throw new Error('Expected an EDN byte string'); + return value; +} + +async function main(): Promise { + const { values, positionals, tokens } = parseArgs({ + allowPositionals: true, + tokens: true, + options: { + cbor: { type: 'string' }, + data: { type: 'string' }, + 'content-type': { type: 'string' }, + 'form-cbor': { type: 'string', multiple: true }, + 'form-data': { type: 'string', multiple: true }, + 'form-file': { type: 'string', multiple: true }, + }, + }); + const [method, path] = positionals; + if (positionals.length !== 2 || !method || !path) { + throw new Error( + 'Usage: cbor-http METHOD PATH [--cbor EDN | --data EDN | --form-cbor NAME=EDN | --form-data NAME=EDN | --form-file NAME=FILE]', + ); + } + let body: Buffer | undefined; + let contentType: string | undefined; + const formTokens = tokens.filter( + (token) => token.kind === 'option' && token.name.startsWith('form-'), + ); + if ( + [values.cbor !== undefined, values.data !== undefined, formTokens.length > 0].filter(Boolean) + .length > 1 + ) { + throw new Error('Choose one request body format'); + } + if (values.cbor !== undefined) { + body = Buffer.from(parseEDN(values.cbor, {})); + contentType = 'application/cbor'; + } else if (values.data !== undefined) { + body = Buffer.from(bytes(values.data)); + contentType = 'application/octet-stream'; + } else if (formTokens.length) { + const boundary = randomUUID(); + const chunks: Buffer[] = []; + for (const token of formTokens) { + if (token.kind !== 'option' || token.value === undefined) continue; + const equal = token.value.indexOf('='); + const name = token.value.slice(0, equal); + const value = token.value.slice(equal + 1); + if (equal < 1 || /["\r\n\\]/.test(name)) throw new Error('Expected multipart NAME=VALUE'); + const isCbor = token.name === 'form-cbor'; + const data = isCbor + ? parseEDN(value, {}) + : token.name === 'form-file' + ? await readFile(value) + : bytes(value); + // Do not add filenames: binary metadata is valid without one. + chunks.push( + Buffer.from( + `--${boundary}\r\nContent-Disposition: form-data; name="${name}"\r\nContent-Type: ${isCbor ? 'application/cbor' : 'application/octet-stream'}\r\n\r\n`, + ), + ); + chunks.push(Buffer.from(data), Buffer.from('\r\n')); + } + chunks.push(Buffer.from(`--${boundary}--\r\n`)); + body = Buffer.concat(chunks); + contentType = `multipart/form-data; boundary=${boundary}`; + } + contentType = values['content-type'] ?? contentType; + const url = /^https?:\/\//.test(path) + ? path + : `${(await readFile('cache.url', 'utf8')).trim().replace(/\/$/, '')}/${path.replace(/^\//, '')}`; + const response = await fetch(url, { + method, + ...(body === undefined ? {} : { body }), + ...(contentType === undefined ? {} : { headers: { 'content-type': contentType } }), + }); + const responseType = response.headers.get('content-type') ?? ''; + const raw = new Uint8Array(await response.arrayBuffer()); + const type = responseType.split(';')[0]?.trim(); + let printedBody: string; + if (type === 'application/cbor') { + decode(raw); // diagnose alone does not reject every incomplete CBOR item. + printedBody = toEDN(raw); + } else { + printedBody = toEDN(encode(type?.startsWith('text/') ? new TextDecoder().decode(raw) : raw)); + } + console.log( + `{"status": ${response.status}, "content_type": ${JSON.stringify(responseType)}, "body": ${printedBody}}`, + ); +} + +try { + await main(); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +} diff --git a/packages/tools/src/edn.test.ts b/packages/tools/src/edn.test.ts new file mode 100644 index 000000000..50f5d5721 --- /dev/null +++ b/packages/tools/src/edn.test.ts @@ -0,0 +1,17 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { parseEDN } from 'cbor-edn'; +import { decode } from 'cbor2/decoder'; +import { encode } from 'cbor2/encoder'; +import { toEDN } from './edn.ts'; + +await test('EDN formatting preserves escapes, a UTF-8 BOM, and text resembling byte strings', () => { + for (const value of [ + new TextEncoder().encode("quotes: '\"\\\n\t\0 and h'ff'"), + new Uint8Array([239, 187, 191, 65]), + { text: "h'ff'", bytes: new TextEncoder().encode('hello') }, + ]) { + const edn = toEDN(encode(value)); + assert.deepEqual(decode(parseEDN(edn, {})), value); + } +}); diff --git a/packages/tools/src/edn.ts b/packages/tools/src/edn.ts new file mode 100644 index 000000000..44919d221 --- /dev/null +++ b/packages/tools/src/edn.ts @@ -0,0 +1,24 @@ +import { diagnose } from 'cbor2/diagnostic'; + +/** Keep byte strings readable without mistaking text containing h'...' for bytes. */ +export function toEDN(bytes: Uint8Array): string { + return diagnose(bytes).replace( + /"(?:[^"\\]|\\.)*"|h'([0-9a-f]*)'/gi, + (token, hex: string | undefined) => { + if (hex === undefined) return token; + const binary = Buffer.from(hex, 'hex'); + try { + const text = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode(binary); + const escaped = JSON.stringify(text) + .slice(1, -1) + .replace(/\\.|'/g, (part) => { + if (part === '\\"') return '"'; + return part === "'" ? "\\'" : part; + }); + return `'${escaped}'`; + } catch { + return `b64'${binary.toString('base64')}'`; + } + }, + ); +} diff --git a/packages/tools/src/remote-cache/cli.ts b/packages/tools/src/remote-cache/cli.ts new file mode 100755 index 000000000..b1dec8658 --- /dev/null +++ b/packages/tools/src/remote-cache/cli.ts @@ -0,0 +1,206 @@ +#!/usr/bin/env node +import { fork } from 'node:child_process'; +import { once } from 'node:events'; +import { open, readFile, rm, watch, writeFile } from 'node:fs/promises'; +import { createConnection } from 'node:net'; +import { Readable } from 'node:stream'; +import { setTimeout } from 'node:timers/promises'; +import { parseArgs } from 'node:util'; +import { createCacheServer } from './server.ts'; + +const urlFile = 'cache.url'; +const logFile = 'cache.log'; + +async function start(args: string[]): Promise { + try { + const file = await open(urlFile, 'wx'); + await file.close(); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'EEXIST') { + throw new Error('Remote cache server already started (cache.url exists)'); + } + throw error; + } + try { + const log = await open(logFile, 'w'); + const child = fork(import.meta.filename, ['serve', ...args], { + detached: true, + stdio: ['ignore', log.fd, log.fd, 'ipc'], + }); + try { + const [message] = await once(child, 'message'); + if (message !== 'ready') throw new Error('Unexpected daemon readiness message'); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`${message}\n${await readFile(logFile, 'utf8')}`.trim()); + } finally { + await log.close(); + if (child.connected) child.disconnect(); + child.unref(); + } + } catch (error) { + await rm(urlFile, { force: true }); + throw error; + } + console.log('Remote cache server started'); +} + +async function ownsUrlFile(url: string | undefined): Promise { + try { + return (await readFile(urlFile, 'utf8')).trim() === url; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false; + throw error; + } +} + +async function serve( + maxLifetimeMs: number, + maxRequestBytes: number, + basePath: string, +): Promise { + const server = createCacheServer({ maxRequestBytes, basePath }); + const shutdown = new AbortController(); + const { signal } = shutdown; + let url: string | undefined; + + async function watchUrlFile(): Promise { + // Watch the directory so deleting the file is observable on every platform. + for await (const { filename } of watch('.', { signal })) { + if ((filename === null || filename === urlFile) && !(await ownsUrlFile(url))) return; + } + } + + async function announceReadiness(): Promise { + // Check after subscribing to file changes, so startup cannot miss a deletion. + // If the starter disappeared, nobody can use this instance. + if (!(await ownsUrlFile(url)) || !process.connected || signal.aborted) return; + process.send?.('ready'); + await setTimeout(maxLifetimeMs, undefined, { signal }); + } + + async function run(): Promise { + const listening = once(server, 'listening', { signal }); + server.listen(0, '127.0.0.1'); + await listening; + const address = server.address(); + if (!address || typeof address === 'string') throw new Error('Missing server address'); + url = `http://127.0.0.1:${address.port}${basePath}`; + // Fill the reserved file before reporting readiness. Do not recreate it if + // it was deleted during startup. + await writeFile(urlFile, `${url}\n`, { flag: 'r+' }); + await Promise.race([watchUrlFile(), announceReadiness()]); + } + + const pending = [ + once(process, 'SIGTERM', { signal }), + once(process, 'SIGINT', { signal }), + run(), + ]; + try { + await Promise.race(pending); + } finally { + shutdown.abort(); + await Promise.allSettled(pending); + // Finish file cleanup before closing the port, so a subsequent start after + // `stop` returns cannot have its URL file removed by this instance. + try { + if (await ownsUrlFile(url)) await rm(urlFile, { force: true }); + } catch (error) { + console.error(error); + process.exitCode = 1; + } finally { + if (server.listening) { + const closed = server[Symbol.asyncDispose](); + server.closeAllConnections(); + await closed; + } + } + } +} + +async function stop(): Promise { + let endpoint: string; + try { + endpoint = await readFile(urlFile, 'utf8'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + console.log('No remote cache endpoint found'); + return; + } + const url = new URL(endpoint.trim()); + const socket = createConnection({ + host: url.hostname, + port: Number(url.port), + }); + const reader = Readable.toWeb(socket).getReader(); + try { + await once(socket, 'connect'); + await rm(urlFile, { force: true }); + // Shutdown closes the listener before this connection. EOF or a reset + // confirms the server has stopped accepting connections. + const { done } = await reader.read(); + if (!done) throw new Error('Unexpected data while waiting for remote cache server shutdown'); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code !== 'ECONNREFUSED' && code !== 'ECONNRESET') throw error; + await rm(urlFile, { force: true }); + } finally { + reader.releaseLock(); + socket.destroy(); + } + console.log('Remote cache server stopped'); +} + +function positiveInteger(value: string): number { + const number = Number(value); + if (!Number.isSafeInteger(number) || number <= 0 || number > 2_147_483_647) { + throw new Error('Limits must be positive integers no greater than 2147483647'); + } + return number; +} + +async function main(): Promise { + const { values, positionals } = parseArgs({ + allowPositionals: true, + options: { + 'max-lifetime-ms': { type: 'string', default: '300000' }, + 'max-request-bytes': { type: 'string', default: String(64 * 1024 * 1024) }, + 'base-path': { type: 'string', default: '' }, + }, + }); + const lifetime = positiveInteger(values['max-lifetime-ms']); + const limit = positiveInteger(values['max-request-bytes']); + const basePath = values['base-path']; + if (basePath && !/^\/(?:[a-zA-Z0-9_-]+\/)*[a-zA-Z0-9_-]+$/.test(basePath)) { + throw new Error('Base path must contain slash-separated names without a trailing slash'); + } + if (positionals.length !== 1) throw new Error('Usage: remote-cache-server start|stop'); + switch (positionals[0]) { + case 'start': + await start([ + '--max-lifetime-ms', + String(lifetime), + '--max-request-bytes', + String(limit), + '--base-path', + basePath, + ]); + break; + case 'serve': + await serve(lifetime, limit, basePath); + break; + case 'stop': + await stop(); + break; + default: + throw new Error('Usage: remote-cache-server start|stop'); + } +} + +try { + await main(); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +} diff --git a/packages/tools/src/remote-cache/server.ts b/packages/tools/src/remote-cache/server.ts new file mode 100644 index 000000000..8ca50870e --- /dev/null +++ b/packages/tools/src/remote-cache/server.ts @@ -0,0 +1,170 @@ +import { Busboy } from '@fastify/busboy'; +import { decode } from 'cbor2/decoder'; +import { encode } from 'cbor2/encoder'; +import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'; + +interface Entry { + value: Uint8Array; + blob_id: string | null; +} + +class RequestError extends Error { + status: number; + + constructor(status: number, message: string) { + super(message); + this.status = status; + } +} + +function mediaType(value: string | undefined): string { + return value?.split(';')[0]?.trim().toLowerCase() ?? ''; +} + +function byteFields(body: Uint8Array, names: string[]): Map { + let value: unknown; + try { + // Decode from Uint8Array so byte strings stay Uint8Array, not Node Buffers + // (whose toJSON method would otherwise turn them into maps when encoded). + value = decode(new Uint8Array(body), { preferMap: true, rejectDuplicateKeys: true }); + } catch { + throw new RequestError(400, 'Invalid CBOR'); + } + if (!(value instanceof Map) || names.some((name) => !(value.get(name) instanceof Uint8Array))) { + throw new RequestError(400, `Expected byte strings: ${names.join(', ')}`); + } + return value; +} + +function readBody(request: IncomingMessage, limit: number): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + let size = 0; + request.on('data', (chunk: Buffer) => { + size += chunk.length; + if (size > limit) { + chunks.length = 0; + reject(new RequestError(413, 'Request too large')); + } else { + chunks.push(chunk); + } + }); + request.on('end', () => resolve(Buffer.concat(chunks))); + request.on('error', reject); + request.on('aborted', () => reject(new RequestError(400, 'Incomplete request'))); + }); +} + +async function readParts(body: Buffer, contentType: string): Promise> { + return new Promise((resolve, reject) => { + const parts = new Map(); + const names = new Set(); + // Both parts are binary, including metadata without a filename. + const parser = new Busboy({ + headers: { 'content-type': contentType }, + isPartAFile: () => true, + }); + parser.on('file', (name, stream, _filename, _encoding, type) => { + const expected = name === 'metadata' ? 'application/cbor' : 'application/octet-stream'; + if (!['metadata', 'blob'].includes(name) || names.has(name) || type !== expected) { + reject(new Error('Invalid multipart part')); + } + names.add(name); + const chunks: Buffer[] = []; + stream.on('data', (chunk: Buffer) => chunks.push(chunk)); + stream.on('end', () => parts.set(name, Buffer.concat(chunks))); + stream.on('error', reject); + }); + parser.on('error', reject); + parser.on('finish', () => resolve(parts)); + parser.end(body); + }); +} + +function cbor(response: ServerResponse, value: unknown): void { + response.writeHead(200, { 'content-type': 'application/cbor' }); + response.end(encode(value)); +} + +/** A disposable backend. Keys, values, and blobs remain opaque bytes. */ +export function createCacheServer({ maxRequestBytes = 64 * 1024 * 1024, basePath = '' } = {}) { + const entries = new Map(); + const associations = new Map(); + const blobs = new Map(); + let nextBlobId = 1; + + async function handle(request: IncomingMessage, response: ServerResponse): Promise { + const path = new URL(request.url ?? '/', 'http://localhost').pathname; + if (request.method === 'GET' && path.startsWith(`${basePath}/blob/`)) { + const blob = blobs.get(path.slice(`${basePath}/blob/`.length)); + if (blob === undefined) throw new RequestError(404, 'Blob not found'); + response.writeHead(200, { 'content-type': 'application/octet-stream' }); + response.end(blob); + return; + } + if (request.method !== 'POST' || ![`${basePath}/fetch`, `${basePath}/store`].includes(path)) { + throw new RequestError(404, 'Route not found'); + } + + const contentType = request.headers['content-type'] ?? ''; + const body = await readBody(request, maxRequestBytes); + if (path === `${basePath}/fetch`) { + if (mediaType(contentType) !== 'application/cbor') { + throw new RequestError(400, 'Expected application/cbor'); + } + const fields = byteFields(body, ['key', 'secondary_key']); + const key = Buffer.from(fields.get('key')!).toString('hex'); + const secondary = Buffer.from(fields.get('secondary_key')!).toString('hex'); + const entry = entries.get(key); + if (entry) { + cbor(response, { kind: 'exact', ...entry }); + } else { + const associatedKey = associations.get(secondary); + if (associatedKey === undefined || !entries.has(associatedKey)) { + throw new RequestError(404, 'Entry not found'); + } + cbor(response, { + kind: 'fallback', + key: new Uint8Array(Buffer.from(associatedKey, 'hex')), + }); + } + return; + } + + if (mediaType(contentType) !== 'multipart/form-data') { + throw new RequestError(400, 'Expected multipart/form-data'); + } + let parts: Map; + try { + parts = await readParts(body, contentType); + } catch { + throw new RequestError(400, 'Invalid multipart body'); + } + const metadata = parts.get('metadata'); + if (metadata === undefined) throw new RequestError(400, 'Missing metadata'); + const fields = byteFields(metadata, ['key', 'secondary_key', 'value']); + const key = Buffer.from(fields.get('key')!).toString('hex'); + const secondary = Buffer.from(fields.get('secondary_key')!).toString('hex'); + const blob = parts.get('blob'); + const blobId = blob === undefined ? null : String(nextBlobId++); + + // Publish only after the complete request is validated. There is no await + // between these writes, so concurrent stores cannot mix values and blobs. + if (blobId !== null) blobs.set(blobId, blob!); + entries.set(key, { value: fields.get('value')!, blob_id: blobId }); + associations.set(secondary, key); + cbor(response, { blob_id: blobId }); + } + + return createServer((request, response) => { + void handle(request, response).catch((error: unknown) => { + const known = error instanceof RequestError; + if (!known) console.error(error); + response.writeHead(known ? error.status : 500, { + 'content-type': 'text/plain; charset=utf-8', + }); + response.end(known ? error.message : 'Internal server error'); + request.resume(); + }); + }); +} diff --git a/packages/tools/tsconfig.json b/packages/tools/tsconfig.json new file mode 100644 index 000000000..9f403cc30 --- /dev/null +++ b/packages/tools/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "@tsconfig/strictest/tsconfig.json", + "compilerOptions": { + "module": "NodeNext", + "target": "ES2022", + "lib": ["ES2022"], + "types": ["node"], + "noEmit": true, + "allowImportingTsExtensions": true, + "erasableSyntaxOnly": true + }, + "include": ["src/**/*.ts"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3747b9403..c4b92e8c9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,6 +9,9 @@ catalogs: '@anthropic-ai/sandbox-runtime': specifier: 0.0.75 version: 0.0.75 + '@fastify/busboy': + specifier: 3.2.2 + version: 3.2.2 '@openai/codex': specifier: 0.154.0 version: 0.154.0 @@ -27,6 +30,12 @@ catalogs: bun: specifier: 1.4.2 version: 1.4.2 + cbor-edn: + specifier: 0.2.2 + version: 0.2.2 + cbor2: + specifier: 1.11.0 + version: 1.11.0 cross-env: specifier: ^10.1.0 version: 10.1.0 @@ -82,6 +91,9 @@ importers: '@anthropic-ai/sandbox-runtime': specifier: 'catalog:' version: 0.0.75 + '@fastify/busboy': + specifier: 'catalog:' + version: 3.2.2 '@openai/codex': specifier: 'catalog:' version: 0.154.0 @@ -97,6 +109,12 @@ importers: bun: specifier: 'catalog:' version: 1.4.2 + cbor-edn: + specifier: 'catalog:' + version: 0.2.2 + cbor2: + specifier: 'catalog:' + version: 1.11.0 cross-env: specifier: 'catalog:' version: 10.1.0 @@ -124,6 +142,10 @@ importers: vitest: specifier: 'catalog:' version: 4.1.11(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(@vitest/browser-preview@4.1.11)(vite@8.3.0(@types/node@25.0.3)) + devDependencies: + vite-task-tools: + specifier: workspace:* + version: 'link:' packages/vite-task-client: {} @@ -184,6 +206,9 @@ packages: '@epic-web/invariant@1.0.0': resolution: {integrity: sha512-lrTPqgvfFQtR/eY/qkIzp98OGdNJu0m5ji3q/nJI8v3SXkRKEnWiOxMmbvcSoAIzv/cGiuvRy57k4suKQSAdwA==} + '@fastify/busboy@3.2.2': + resolution: {integrity: sha512-yXSS27qPExaXeuLvMRMXOLtpipzfQYNjG3FkunDWKGfMYjKuhFXko9CVzqxm8jcF+lmtS9Fd89QNdh9XDjnbNg==} + '@jridgewell/sourcemap-codec@1.5.5': resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} @@ -1316,6 +1341,15 @@ packages: os: [darwin, linux, android, freebsd, win32] hasBin: true + cbor-edn@0.2.2: + resolution: {integrity: sha512-/zu3hPQmrkHtHoW1Bp73pYRlIg1R34rNiP4L87FMI2EbBf7l0cO3B2W/rBx3W8mvqDh3Qc4cJy3DcAnVaamQPQ==} + engines: {node: '>=18.7'} + hasBin: true + + cbor2@1.11.0: + resolution: {integrity: sha512-teHnpDGcDzwouFJgCJLpDY2zMaEOYvYDryd6sln67wPrCuObmwriG1NAA1w7rUIf/l++bNdts4U7WeVdYTGwLg==} + engines: {node: '>=18.7'} + chai@6.2.2: resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==} engines: {node: '>=18'} @@ -1811,6 +1845,8 @@ snapshots: '@epic-web/invariant@1.0.0': {} + '@fastify/busboy@3.2.2': {} + '@jridgewell/sourcemap-codec@1.5.5': {} '@openai/codex@0.154.0': @@ -2581,6 +2617,12 @@ snapshots: '@oven/bun-windows-aarch64': 1.4.2 '@oven/bun-windows-x64': 1.4.2 + cbor-edn@0.2.2: + dependencies: + cbor2: 1.11.0 + + cbor2@1.11.0: {} + chai@6.2.2: {} commander@12.1.0: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 99501bf21..e99a1bd85 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -10,12 +10,15 @@ allowBuilds: catalog: '@anthropic-ai/sandbox-runtime': 0.0.75 + '@fastify/busboy': 3.2.2 '@openai/codex': 0.154.0 '@tsconfig/strictest': ^2.0.8 '@types/node': 25.0.3 '@playwright/browser-chromium': 1.63.0 '@vitest/browser-playwright': 4.1.11 bun: 1.4.2 + cbor-edn: 0.2.2 + cbor2: 1.11.0 cross-env: ^10.1.0 deno: 2.9.6 husky: ^9.1.7