diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index a998dae17..5901f6a1f 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -7,6 +7,7 @@ - [WASI support](./wasi/index.md) - [Preview 2 shim](./wasi/preview2-shim.md) - [CLI helpers](./wasi/preview2-shim/helpers/cli.md) + - [Filesystem helpers](./wasi/preview2-shim/helpers/filesystem.md) - [Preview 3 shim](./wasi/preview3-shim.md) - [Performance](./performance/index.md) - [Benchmarks](./performance/benchmarks/index.md) diff --git a/docs/src/wasi/preview2-shim.md b/docs/src/wasi/preview2-shim.md index 1226f4a74..87017fa30 100644 --- a/docs/src/wasi/preview2-shim.md +++ b/docs/src/wasi/preview2-shim.md @@ -1,3 +1,4 @@ +| `@bytecodealliance/preview2-shim/helpers/filesystem` | Host-side filesystem helpers for both builds (see [Helpers](#helpers)) | # Preview 2 shim [`@bytecodealliance/preview2-shim`][npm-p2-shim] implements WASI 0.2 (`wasi:cli`, `wasi:clocks`, @@ -57,6 +58,8 @@ underscore-prefixed functions (`_setStdin`, `_setPreopens`, ...) that configure Helpers that only exist for one build are exported under that build's prefix. - [CLI helpers](./preview2-shim/helpers/cli.md): feed a guest a fixed stdin buffer in the browser. +- [Filesystem helpers](./preview2-shim/helpers/filesystem.md): write and read files under a preopen + from host code, on both builds. [npm-p2-shim]: https://www.npmjs.com/package/@bytecodealliance/preview2-shim [p2-readme]: https://github.com/bytecodealliance/jco/blob/main/packages/preview2-shim/README.md diff --git a/docs/src/wasi/preview2-shim/helpers/filesystem.md b/docs/src/wasi/preview2-shim/helpers/filesystem.md new file mode 100644 index 000000000..e2cb1c71b --- /dev/null +++ b/docs/src/wasi/preview2-shim/helpers/filesystem.md @@ -0,0 +1,94 @@ +# Preview 2 shim filesystem helpers + +[`@bytecodealliance/preview2-shim`][npm-p2-shim] ships host-side helper functions next to its WASI +implementations. This page covers the filesystem helpers; see [Preview 2 shim](../../preview2-shim.md) +for the package overview. Helpers build only on the shim's public surface and are exported from `helpers/` +subpaths, separate from the underscore-prefixed functions (`_setPreopens`, `_setFileData`, ...) +that exist to configure the shim itself. + +The filesystem helpers move bytes between host JavaScript and a preopened directory without +involving a guest. They are available under both the browser and `node` conditions: + +```js +import { readFileAt, writeFileAt } from "@bytecodealliance/preview2-shim/helpers/filesystem"; +``` + +Both helpers use only the public `wasi:filesystem/types` `Descriptor` interface, so they work with +every adapter the shim can be configured with: the Node host filesystem, the browser in-memory +adapter, OPFS, or an application-provided implementation. + +[npm-p2-shim]: https://www.npmjs.com/package/@bytecodealliance/preview2-shim + +## Which preopens the helpers see + +The helpers resolve `virtualPath` against the shim's **global** preopens, the same set a component +receives when it is instantiated without isolated preopens. Configure them with the build's +compatibility setters before calling a helper: + +- Node.js: `filesystem._setPreopens({ "/data": "/host/path" })` or `filesystem._addPreopen(...)` +- Browser: `filesystem._setPreopens({ "/data": { dir: {} } })` or `filesystem._setFileData(...)` + +A `WASIShim` created with `sandbox.preopens` or `browserFilesystem` owns an isolated preopen set +that the helpers cannot reach. Use the global preopens, or operate on that instance's descriptors +directly, when a component needs isolation. + +## `writeFileAt(virtualPath, path, data, options?)` + +Writes `data` (a `Uint8Array`) to `path`, resolved relative to the preopen mounted at +`virtualPath`. Missing intermediate directories are created. The file is truncated first unless +`options.append` is `true`, in which case the bytes are written after the existing contents. + +```js +import { filesystem } from "@bytecodealliance/preview2-shim"; +import { writeFileAt } from "@bytecodealliance/preview2-shim/helpers/filesystem"; + +filesystem._setPreopens({ "/data": { dir: {} } }); // browser in-memory adapter + +const encoder = new TextEncoder(); +writeFileAt("/data", "logs/run.txt", encoder.encode("started\n")); +writeFileAt("/data", "logs/run.txt", encoder.encode("done\n"), { append: true }); +``` + +## `readFileAt(virtualPath, path)` + +Reads the whole file at `path` under the preopen mounted at `virtualPath` and returns its contents +as a single `Uint8Array`. Large files are read in chunks and joined, so the result is independent of +the adapter's read granularity. + +```js +import { readFileAt } from "@bytecodealliance/preview2-shim/helpers/filesystem"; + +const bytes = readFileAt("/data", "logs/run.txt"); +console.log(new TextDecoder().decode(bytes)); // "started\ndone\n" +``` + +## Errors + +An unknown `virtualPath` throws a JavaScript `Error` whose message names the missing preopen. +Everything else surfaces the adapter's own WASI `error-code` string, for example `"no-entry"` when +`readFileAt` targets a file that does not exist or `"not-permitted"` when the host denies the +operation. This mirrors what a guest would observe for the same call. + +## Example: shell-style redirection + +A host that runs a CLI component and wants to capture its output file can seed the input before +instantiation and collect the result afterwards: + +```js +import { filesystem } from "@bytecodealliance/preview2-shim"; +import { WASIShim } from "@bytecodealliance/preview2-shim/instantiation"; +import { readFileAt, writeFileAt } from "@bytecodealliance/preview2-shim/helpers/filesystem"; +import { instantiate } from "./dist/transpiled/component.js"; + +filesystem._setPreopens({ "/work": { dir: {} } }); +writeFileAt("/work", "input.txt", new TextEncoder().encode("hello\n")); + +const shim = new WASIShim({ arguments: ["component", "input.txt", "output.txt"] }); +const component = await instantiate(undefined, shim.getImportObject()); +component.run.run(); + +const output = readFileAt("/work", "output.txt"); +``` + +See [Manual Wasm instantiation with WASI overrides](../../../advanced/manual-wasm-instantiation-with-wasi-overrides.md) +for how `instantiate` and `WASIShim` fit together. diff --git a/packages/preview2-shim/README.md b/packages/preview2-shim/README.md index 9f83f4c66..1f431fb64 100644 --- a/packages/preview2-shim/README.md +++ b/packages/preview2-shim/README.md @@ -74,10 +74,14 @@ The setting affects subsequent requests made through the browser HTTP shim. Call `http._setRequestStreaming(false)` to restore portable completion buffering. Host-side helper functions that build on the public shim surface are exported from `helpers/` -subpaths, separate from the underscore-prefixed configuration setters. The browser-only +subpaths, separate from the underscore-prefixed configuration setters. The `writeFileAt` and +`readFileAt` helpers (`@bytecodealliance/preview2-shim/helpers/filesystem`) move bytes between host +JavaScript and a preopened directory without a guest, on both builds. The browser-only `bufferedInputStream` helper (`@bytecodealliance/preview2-shim/browser/helpers/cli`) feeds a guest a -fixed stdin buffer. See [Preview 2 shim CLI helpers][book-cli-helpers] in the Jco book. +fixed stdin buffer. See the [filesystem helpers][book-fs-helpers] and [CLI helpers][book-cli-helpers] +pages in the Jco book. +[book-fs-helpers]: https://bytecodealliance.github.io/jco/wasi/preview2-shim/helpers/filesystem.html [book-cli-helpers]: https://bytecodealliance.github.io/jco/wasi/preview2-shim/helpers/cli.html Browser applications select storage explicitly. The bundled file-data adapter is ephemeral and must diff --git a/packages/preview2-shim/src/browser/helpers/filesystem.ts b/packages/preview2-shim/src/browser/helpers/filesystem.ts new file mode 100644 index 000000000..bf0255f1d --- /dev/null +++ b/packages/preview2-shim/src/browser/helpers/filesystem.ts @@ -0,0 +1,23 @@ +import { _getPreopens } from "../filesystem.js"; +import { + writeFileAt as writeFileAtImpl, + readFileAt as readFileAtImpl, + type WriteFileAtOptions, +} from "../../common/helpers/filesystem.js"; + +export type { WriteFileAtOptions }; + +/** Write bytes into a preopened directory's tree from host JS, without a guest. */ +export function writeFileAt( + virtualPath: string, + path: string, + data: Uint8Array, + options: WriteFileAtOptions = {}, +): void { + writeFileAtImpl(_getPreopens(), virtualPath, path, data, options); +} + +/** Read the full contents of a file under a preopened directory from host JS, without a guest. */ +export function readFileAt(virtualPath: string, path: string): Uint8Array { + return readFileAtImpl(_getPreopens(), virtualPath, path); +} diff --git a/packages/preview2-shim/src/common/helpers/filesystem.ts b/packages/preview2-shim/src/common/helpers/filesystem.ts new file mode 100644 index 000000000..247a6ddac --- /dev/null +++ b/packages/preview2-shim/src/common/helpers/filesystem.ts @@ -0,0 +1,129 @@ +import type { types as TypesNamespace } from "../../../types/filesystem.js"; + +type Descriptor = TypesNamespace.Descriptor; +type DescriptorFlags = TypesNamespace.DescriptorFlags; +type OpenFlags = TypesNamespace.OpenFlags; + +export interface WriteFileAtOptions { + append?: boolean; +} + +const READ_CHUNK_SIZE = 1024n * 1024n; + +/** + * Write bytes into a preopened directory's tree from host JS, without a guest. + * + * Only uses the public `Descriptor` interface, so it works with any adapter. + */ +export function writeFileAt( + preopens: [Descriptor, string][], + virtualPath: string, + path: string, + data: Uint8Array, + options: WriteFileAtOptions = {}, +): void { + withFileAt( + preopens, + virtualPath, + path, + { create: true, truncate: !options.append }, + { write: true }, + (file) => { + let offset = options.append ? file.stat().size : 0n; + let written = 0; + while (written < data.byteLength) { + const count = Number(file.write(data.subarray(written), offset)); + if (count <= 0) { + throw new Error(`short write to ${JSON.stringify(path)}`); + } + written += count; + offset += BigInt(count); + } + }, + ); +} + +/** + * Read the full contents of a file under a preopened directory from host JS, without a guest. + * + * Only uses the public `Descriptor` interface, so it works with any adapter. + */ +export function readFileAt( + preopens: [Descriptor, string][], + virtualPath: string, + path: string, +): Uint8Array { + return withFileAt(preopens, virtualPath, path, {}, { read: true }, (file) => { + const chunks: Uint8Array[] = []; + let length = 0; + let offset = 0n; + for (;;) { + const [bytes, eof] = file.read(READ_CHUNK_SIZE, offset); + if (bytes.byteLength > 0) { + chunks.push(bytes); + length += bytes.byteLength; + offset += BigInt(bytes.byteLength); + } + if (eof) { + break; + } + } + const result = new Uint8Array(length); + let position = 0; + for (const chunk of chunks) { + result.set(chunk, position); + position += chunk.byteLength; + } + return result; + }); +} + +/** + * Open `path` under the preopen mounted at `virtualPath`, creating missing intermediate + * directories, run `fn` on it, and release every descriptor opened along the way. + */ +function withFileAt( + preopens: [Descriptor, string][], + virtualPath: string, + path: string, + openFlags: OpenFlags, + descriptorFlags: DescriptorFlags, + fn: (file: Descriptor) => T, +): T { + const entry = preopens.find(([, guestPath]) => guestPath === virtualPath); + if (!entry) { + throw new Error(`no preopen for ${JSON.stringify(virtualPath)}`); + } + const [root] = entry; + + const segments = path.split("/").filter((segment) => segment && segment !== "."); + const fileName = segments.pop(); + if (!fileName) { + throw new Error(`invalid path: ${JSON.stringify(path)}`); + } + + // Preopen roots are owned by the caller; everything opened here is released on exit. + const opened: Descriptor[] = []; + try { + let dir: Descriptor = root; + for (const segment of segments) { + try { + dir.createDirectoryAt(segment); + } catch (error) { + if (error !== "exist") { + throw error; + } + } + dir = dir.openAt({ symlinkFollow: true }, segment, { directory: true }, {}); + opened.push(dir); + } + + const file = dir.openAt({ symlinkFollow: true }, fileName, openFlags, descriptorFlags); + opened.push(file); + return fn(file); + } finally { + for (const descriptor of opened.reverse()) { + (descriptor as Partial)[Symbol.dispose]?.(); + } + } +} diff --git a/packages/preview2-shim/src/nodejs/helpers/filesystem.ts b/packages/preview2-shim/src/nodejs/helpers/filesystem.ts new file mode 100644 index 000000000..bf0255f1d --- /dev/null +++ b/packages/preview2-shim/src/nodejs/helpers/filesystem.ts @@ -0,0 +1,23 @@ +import { _getPreopens } from "../filesystem.js"; +import { + writeFileAt as writeFileAtImpl, + readFileAt as readFileAtImpl, + type WriteFileAtOptions, +} from "../../common/helpers/filesystem.js"; + +export type { WriteFileAtOptions }; + +/** Write bytes into a preopened directory's tree from host JS, without a guest. */ +export function writeFileAt( + virtualPath: string, + path: string, + data: Uint8Array, + options: WriteFileAtOptions = {}, +): void { + writeFileAtImpl(_getPreopens(), virtualPath, path, data, options); +} + +/** Read the full contents of a file under a preopened directory from host JS, without a guest. */ +export function readFileAt(virtualPath: string, path: string): Uint8Array { + return readFileAtImpl(_getPreopens(), virtualPath, path); +} diff --git a/packages/preview2-shim/test/browser/helpers/filesystem.ts b/packages/preview2-shim/test/browser/helpers/filesystem.ts new file mode 100644 index 000000000..e1818a0e1 --- /dev/null +++ b/packages/preview2-shim/test/browser/helpers/filesystem.ts @@ -0,0 +1,70 @@ +import { suite, test, assert } from "vitest"; + +let preopenCount = 0; + +/** Mount a fresh in-memory directory at its own virtual path so concurrent tests never share a tree. */ +async function freshPreopen(): Promise { + const { _addPreopen } = await import("../../../src/browser/filesystem.js"); + const virtualPath = `/preopen-${preopenCount++}`; + _addPreopen(virtualPath, { dir: {} }); + return virtualPath; +} + +function helpers() { + return import("../../../src/browser/helpers/filesystem.js"); +} + +suite("Browser filesystem helpers", () => { + test.concurrent("writeFileAt and readFileAt round-trip bytes without a guest", async () => { + const [root, { writeFileAt, readFileAt }] = await Promise.all([freshPreopen(), helpers()]); + + writeFileAt(root, "output.txt", new Uint8Array([1, 2, 3])); + + assert.deepStrictEqual(readFileAt(root, "output.txt"), new Uint8Array([1, 2, 3])); + }); + + test.concurrent("writeFileAt creates missing intermediate directories", async () => { + const [root, { writeFileAt, readFileAt }] = await Promise.all([freshPreopen(), helpers()]); + + writeFileAt(root, "nested/dir/output.txt", new Uint8Array([4, 5])); + + assert.deepStrictEqual(readFileAt(root, "nested/dir/output.txt"), new Uint8Array([4, 5])); + }); + + test.concurrent("writeFileAt truncates by default and appends when asked", async () => { + const [root, { writeFileAt, readFileAt }] = await Promise.all([freshPreopen(), helpers()]); + + writeFileAt(root, "log.txt", new Uint8Array([1, 2, 3])); + writeFileAt(root, "log.txt", new Uint8Array([9])); + assert.deepStrictEqual(readFileAt(root, "log.txt"), new Uint8Array([9])); + + writeFileAt(root, "log.txt", new Uint8Array([1, 2]), { append: true }); + assert.deepStrictEqual(readFileAt(root, "log.txt"), new Uint8Array([9, 1, 2])); + }); + + test.concurrent("writeFileAt resolves the virtual path against the matching preopen", async () => { + const [first, second, { writeFileAt, readFileAt }] = await Promise.all([ + freshPreopen(), + freshPreopen(), + helpers(), + ]); + + writeFileAt(second, "file.txt", new Uint8Array([7])); + + assert.deepStrictEqual(readFileAt(second, "file.txt"), new Uint8Array([7])); + assert.throws(() => readFileAt(first, "file.txt")); + assert.throws(() => readFileAt("/missing", "file.txt"), /no preopen/); + }); + + test.concurrent("readFileAt reads a file larger than a single read chunk", async () => { + const [root, { writeFileAt, readFileAt }] = await Promise.all([freshPreopen(), helpers()]); + const data = new Uint8Array(2 * 1024 * 1024 + 17); + for (let i = 0; i < data.length; i++) { + data[i] = i % 256; + } + + writeFileAt(root, "big.bin", data); + + assert.deepStrictEqual(readFileAt(root, "big.bin"), data); + }); +}); diff --git a/packages/preview2-shim/test/node-filesystem-helpers.ts b/packages/preview2-shim/test/node-filesystem-helpers.ts new file mode 100644 index 000000000..13cb24bfc --- /dev/null +++ b/packages/preview2-shim/test/node-filesystem-helpers.ts @@ -0,0 +1,91 @@ +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; + +import { filesystem } from "@bytecodealliance/preview2-shim"; +import { readFileAt, writeFileAt } from "@bytecodealliance/preview2-shim/helpers/filesystem"; +import { assert, suite, test } from "vitest"; + +/** + * Run `body` against a fresh host directory mounted at its own virtual path, then remove it. + * + * Each test gets a distinct preopen, so the suite can run concurrently without tests observing + * each other's files. + */ +async function withPreopen( + body: (virtualPath: string, hostDir: string) => Promise, +): Promise { + const hostDir = await mkdtemp(join(tmpdir(), "jco-fs-helpers-")); + try { + const virtualPath = `/${basename(hostDir)}`; + filesystem._addPreopen(virtualPath, hostDir); + await body(virtualPath, hostDir); + } finally { + await rm(hostDir, { recursive: true, force: true }); + } +} + +async function hostBytes(path: string): Promise { + return new Uint8Array(await readFile(path)); +} + +suite("Node filesystem helpers", () => { + test.concurrent("writeFileAt writes through to the host directory behind the preopen", () => + withPreopen(async (root, hostDir) => { + writeFileAt(root, "output.txt", new Uint8Array([1, 2, 3])); + + assert.deepStrictEqual( + await hostBytes(join(hostDir, "output.txt")), + new Uint8Array([1, 2, 3]), + ); + assert.deepStrictEqual(readFileAt(root, "output.txt"), new Uint8Array([1, 2, 3])); + })); + + test.concurrent("writeFileAt creates missing intermediate directories and reuses existing ones", () => + withPreopen(async (root, hostDir) => { + await mkdir(join(hostDir, "existing")); + + writeFileAt(root, "existing/nested/output.txt", new Uint8Array([4, 5])); + + assert.deepStrictEqual( + await hostBytes(join(hostDir, "existing", "nested", "output.txt")), + new Uint8Array([4, 5]), + ); + })); + + test.concurrent("writeFileAt truncates by default and appends when asked", () => + withPreopen(async (root, hostDir) => { + writeFileAt(root, "log.txt", new Uint8Array([1, 2, 3])); + writeFileAt(root, "log.txt", new Uint8Array([9])); + assert.deepStrictEqual(readFileAt(root, "log.txt"), new Uint8Array([9])); + + writeFileAt(root, "log.txt", new Uint8Array([1, 2]), { append: true }); + assert.deepStrictEqual( + await hostBytes(join(hostDir, "log.txt")), + new Uint8Array([9, 1, 2]), + ); + })); + + test.concurrent("readFileAt reads a host file larger than a single read chunk", () => + withPreopen(async (root, hostDir) => { + const data = new Uint8Array(2 * 1024 * 1024 + 17); + for (let i = 0; i < data.length; i++) { + data[i] = i % 256; + } + await writeFile(join(hostDir, "big.bin"), data); + + assert.deepStrictEqual(readFileAt(root, "big.bin"), data); + })); + + test.concurrent("helpers reject unknown preopens and missing files with WASI errors", () => + withPreopen(async (root) => { + assert.throws(() => readFileAt("/missing", "file.txt"), /no preopen/); + let caught: unknown; + try { + readFileAt(root, "absent.txt"); + } catch (err) { + caught = err; + } + assert.strictEqual(caught, "no-entry"); + })); +});