From e520f231f761d771f30224096e4f972fcaaa9c62 Mon Sep 17 00:00:00 2001 From: Joel Dice Date: Thu, 20 Aug 2026 14:50:34 -0700 Subject: [PATCH] add support for the `wasm32-wasip3` target and ABI This paves the way for supporting cooperative mulithreading. Currently, we embed two copies of each library: one for p2 and one for p3, selecting which to use based on whether the target world uses any async features. In order to avoid even longer build times, I've published a pre-built CPython WASI tarball which we use by default instead of building from source. This also adds support for testing WIT maps in the test generator, which I meant to include in an earlier commit. Co-authored-by: Alex Crichton enable multithreading and add example This required updates to the CPython fork used, and also switching to a temporary fork of WASI-SDK which enables Wasm exception handling + cooperative multithreading + shared libraries. --- .github/workflows/release.yaml | 75 +---- .github/workflows/test.yaml | 76 +---- .gitignore | 13 +- CONTRIBUTING.md | 36 +-- Cargo.toml | 2 - build.rs | 284 ++++++++++-------- .../componentize_py_async_support/__init__.py | 41 +-- examples/multithreading/README.md | 44 +++ examples/multithreading/app.py | 24 ++ runtime/Cargo.toml | 2 +- runtime/src/lib.rs | 88 +++--- src/command.rs | 28 +- src/lib.rs | 176 ++++++++--- src/prelink.rs | 264 ++++++++++------ src/python.rs | 8 +- src/test.rs | 2 + src/test/echoes.rs | 1 + src/test/python_source/app.py | 2 +- src/test/tests.rs | 38 ++- src/test/wit/tests.wit | 2 +- test-generator/src/lib.rs | 44 ++- tests/bindings.rs | 19 ++ tests/componentize.rs | 75 +++-- wit/init.wit | 2 +- 24 files changed, 809 insertions(+), 537 deletions(-) create mode 100644 examples/multithreading/README.md create mode 100644 examples/multithreading/app.py diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 2d9b0bd5..44a6657f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -12,58 +12,14 @@ on: env: CARGO_TERM_COLOR: always - WASI_SDK_VERSION: 34 permissions: contents: write pull-requests: write jobs: - linux: - name: Populate cache - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: "recursive" - - - uses: actions/setup-python@v5 - with: - python-version: "3.14.0" - - - name: Install latest Rust stable toolchain - uses: dtolnay/rust-toolchain@stable - with: - targets: wasm32-wasip2 - components: clippy, rustfmt - - - uses: Swatinem/rust-cache@v2 - with: - shared-key: "rust-cache-${{ hashFiles('./Cargo.lock') }}" - cache-on-failure: false - - - name: Install WASI-SDK - shell: bash - run: | - cd /tmp - curl -LO https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux.tar.gz - tar xf wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux.tar.gz - mv wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux /opt/wasi-sdk - - - name: Build - shell: bash - run: cargo build --release - - - name: Upload CPython builddir - uses: actions/upload-artifact@v4 - with: - name: cpython-wasi - path: cpython/builddir/wasi - release: name: Build and release - needs: linux runs-on: ${{ matrix.config.os }} strategy: fail-fast: false @@ -129,10 +85,11 @@ jobs: with: python-version: "3.14.0" - - name: Install latest Rust stable toolchain - uses: dtolnay/rust-toolchain@stable + - name: Install latest Rust toolchain + # todo: switch back to stable once 1.100 is released: + uses: dtolnay/rust-toolchain@nightly with: - targets: wasm32-wasip2 ${{ matrix.config.target }} + targets: wasm32-wasip2 wasm32-wasip3 ${{ matrix.config.target }} - uses: Swatinem/rust-cache@v2 if: matrix.config.os == 'ubuntu-latest' && matrix.config.arch == 'amd64' @@ -157,36 +114,12 @@ jobs: OS=$(echo "${{ runner.os }}" | tr '[:upper:]' '[:lower:]') echo "RUNNER_OS=$OS" >> $GITHUB_ENV - - name: Download WASI-SDK - shell: bash - run: | - cd /tmp - curl -LO https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-${{ matrix.config.wasiSDK }}.tar.gz - tar xf wasi-sdk-${WASI_SDK_VERSION}.0-${{ matrix.config.wasiSDK }}.tar.gz - - - name: Install WASI-SDK - if: runner.os != 'Windows' - shell: bash - run: sudo mv /tmp/wasi-sdk-${WASI_SDK_VERSION}.0-${{ matrix.config.wasiSDK }} /opt/wasi-sdk - - - name: Install WASI-SDK on Windows - if: runner.os == 'Windows' - shell: bash - run: echo "WASI_SDK_PATH=$(cygpath -m /tmp/wasi-sdk-${WASI_SDK_VERSION}.0-${{ matrix.config.wasiSDK }})" >> ${GITHUB_ENV} - - - name: Restore CPython - uses: actions/download-artifact@v5 - with: - name: cpython-wasi - path: cpython/builddir/wasi - - name: Build wheels uses: PyO3/maturin-action@v1 with: target: ${{ matrix.config.maturin_target }} args: --release --out dist --features pyo3 manylinux: 2_28 - docker-options: -v /opt/wasi-sdk:/opt/wasi-sdk before-script-linux: | rustup target add wasm32-wasip2 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 4be7148e..18690cb6 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -14,54 +14,10 @@ concurrency: env: CARGO_TERM_COLOR: always - WASI_SDK_VERSION: 34 jobs: - linux: - name: Populate cache - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: "recursive" - - - uses: actions/setup-python@v5 - with: - python-version: "3.14.0" - - - name: Install latest Rust stable toolchain - uses: dtolnay/rust-toolchain@stable - with: - targets: wasm32-wasip2 - components: clippy, rustfmt - - - uses: Swatinem/rust-cache@v2 - with: - shared-key: "rust-cache-${{ hashFiles('./Cargo.lock') }}" - cache-on-failure: false - - - name: Install WASI-SDK - shell: bash - run: | - cd /tmp - curl -LO https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux.tar.gz - tar xf wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux.tar.gz - mv wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux /opt/wasi-sdk - - - name: Build - shell: bash - run: cargo build --release - - - name: Upload CPython builddir - uses: actions/upload-artifact@v4 - with: - name: cpython-wasi - path: cpython/builddir/wasi - test: name: Test - needs: linux strategy: fail-fast: false matrix: @@ -76,10 +32,11 @@ jobs: with: submodules: "recursive" - - name: Install latest Rust stable toolchain - uses: dtolnay/rust-toolchain@stable + - name: Install latest Rust toolchain + # todo: switch back to stable once 1.100 is released: + uses: dtolnay/rust-toolchain@nightly with: - targets: wasm32-wasip2 + targets: wasm32-wasip2 wasm32-wasip3 components: clippy, rustfmt - uses: Swatinem/rust-cache@v2 @@ -87,36 +44,13 @@ jobs: shared-key: "rust-cache-${{ hashFiles('./Cargo.lock') }}" cache-on-failure: false - - name: Download WASI-SDK - shell: bash - run: | - cd /tmp - curl -LO https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-${{ matrix.config.wasiSDK }}.tar.gz - tar xf wasi-sdk-${WASI_SDK_VERSION}.0-${{ matrix.config.wasiSDK }}.tar.gz - - - name: Install WASI-SDK - if: runner.os != 'Windows' - shell: bash - run: sudo mv /tmp/wasi-sdk-${WASI_SDK_VERSION}.0-${{ matrix.config.wasiSDK }} /opt/wasi-sdk - - - name: Install WASI-SDK on Windows - if: runner.os == 'Windows' - shell: bash - run: echo "WASI_SDK_PATH=$(cygpath -m /tmp/wasi-sdk-${WASI_SDK_VERSION}.0-${{ matrix.config.wasiSDK }})" >> ${GITHUB_ENV} - - - name: Restore CPython - uses: actions/download-artifact@v5 - with: - name: cpython-wasi - path: cpython/builddir/wasi - - name: Lint shell: bash run: bash .github/workflows/lint.sh - uses: taiki-e/install-action@v2 with: - tool: wasmtime-cli@46.0.1 + tool: wasmtime-cli@48.0.2 - uses: actions/setup-python@v5 with: diff --git a/.gitignore b/.gitignore index b69b5b0a..2a5d691c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,24 +2,17 @@ target __pycache__ .mypy_cache .venv -bacon.toml dist -examples/matrix-math/*.wasm +examples/*/*.wasm examples/matrix-math/numpy* examples/matrix-math/matrix_math examples/matrix-math/wasmtime-py -examples/http/.spin -examples/http/http.wasm examples/http/proxy examples/http/poll_loop.py -examples/http-p3/http.wasm -examples/tcp-p3/tcp.wasm -examples/tcp/tcp.wasm examples/tcp/command -examples/cli-p3/cli.wasm -examples/cli/cli.wasm examples/cli/command examples/sandbox/sandbox -examples/sandbox/sandbox.wasm +examples/multithreading/*.wasm +examples/multithreading/command wasi-sdk cpython diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0e2975f3..18cb2938 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,41 +16,35 @@ channel](https://bytecodealliance.zulipchat.com/#narrow/stream/394175-SIG-Guest- ### Prerequisites -- Tools needed to build [CPython](https://github.com/python/cpython) (Make, Clang, etc.) -- [Rust](https://rustup.rs/) stable 1.94 or later, including the `wasm32-wasip2` target +- (optional) Tools needed to build [CPython](https://github.com/python/cpython) + (Make, Clang, etc.) +- [Rust](https://rustup.rs/) stable 1.100 or later, including the + `wasm32-wasip2` and `wasm32-wasip3` targets For Rust, something like this should work once you have `rustup`: ```shell rustup update -rustup target add wasm32-wasip2 +rustup target add wasm32-wasip2 wasm32-wasip3 ``` -### Building and Running +### Building and Running Tests -First, make sure you've got all the submodules cloned. +To build the project and run the tests, use: ```shell -git submodule update --init --recursive +cargo test --release ``` -Next, install WASI-SDK 33 to `/opt/wasi-sdk` (alternatively, you can specify a -different location and reference it later using the `WASI_SDK_PATH` environment -variable). Replace `arm64-linux` with `x86_64-linux`, `arm64-macos`, -`x86_64-macos`, `arm64-windows`, or `x86_64-windows` below depending on your -architecure and OS, if necessary. +By default, `build.rs` will download and use pre-built WASI-SDK and Cpython +binaries. If you'd like to supply your own version of WASI-SDK and use it to +build CPython from source, install +[WASI-SDK](https://github.com/WebAssembly/wasi-sdk/releases) 34 or later, then +run: ```shell -curl -LO https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-33/wasi-sdk-33.0-arm64-linux.tar.gz -tar xf wasi-sdk-33.0-arm64-linux.tar.gz -sudo mv wasi-sdk-33.0-arm64-linux /opt/wasi-sdk -export WASI_SDK_PATH=/opt/wasi-sdk -``` - -Finally, build and run `componentize-py`. - -```shell -cargo run --release -- --help +rm -rf cpython +CPYTHON_BUILD_FROM_SOURCE=1 cargo test --release ``` ## Publishing Releases diff --git a/Cargo.toml b/Cargo.toml index 8eb8120c..6360055a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,6 @@ wit-parser = "0.258.0" wit-component = "0.258.0" wasmparser = "0.258.0" indexmap = "2.6.0" -bincode = "1.3.3" heck = "0.5.0" pyo3 = { version = "0.28.3", features = [ "abi3-py39", @@ -43,7 +42,6 @@ tokio = { version = "1.41.0", features = [ ] } bytes = "1.8.0" pretty_env_logger = "0.5.0" -cap-std = "3.3.0" im-rc = "15.1.0" serde = { version = "1.0.213", features = ["derive"] } toml = "0.8.19" diff --git a/build.rs b/build.rs index 5c2e65c6..56f7e95d 100644 --- a/build.rs +++ b/build.rs @@ -38,8 +38,13 @@ const CLANG_EXECUTABLE: &str = "clang"; // TODO: switch to upstream release per // https://github.com/bytecodealliance/componentize-py/issues/215 -const CPYTHON_TARBALL_URL: &str = "https://github.com/dicej/cpython/tarball/v3.14.0-wasi-sdk-30"; -const CPYTHON_TARBALL_BASE_DIR: &str = "dicej-cpython-0e13686"; +const CPYTHON_SOURCE_TARBALL_URL: &str = + "https://github.com/dicej/cpython/tarball/v3.14.0-wasi-sdk-34-v2"; +const CPYTHON_SOURCE_TARBALL_BASE_DIR: &str = "dicej-cpython-ff50203"; + +const CPYTHON_BINARY_TARBALL_URL: &str = "https://github.com/dicej/cpython/releases/download/v3.14.0-wasi-sdk-34-v2/cpython-v3.14.0-wasi-sdk-34-v2.tar.zst"; + +static TARGETS: &[&str] = &["wasip2", "wasip3"]; fn main() -> Result<()> { println!("cargo:rerun-if-changed=build.rs"); @@ -61,25 +66,38 @@ fn main() -> Result<()> { } fn stubs_for_clippy(out_dir: &Path) -> Result<()> { - println!( - "cargo:warning=using stubbed runtime, core library, and adapter for static analysis purposes..." - ); + println!("cargo:warning=using dummy embedded files for static analysis purposes..."); - let files = [ - "libcomponentize_py_runtime_sync.so.zst", - "libcomponentize_py_runtime_async.so.zst", + let libraries = [ + "libcomponentize_py_runtime.so.zst", "libpython3.14.so.zst", "libc.so.zst", + "threads/libc.so.zst", "libwasi-emulated-mman.so.zst", "libwasi-emulated-process-clocks.so.zst", "libwasi-emulated-getpid.so.zst", "libwasi-emulated-signal.so.zst", "libc++.so.zst", "libc++abi.so.zst", - "wasi_snapshot_preview1.reactor.wasm.zst", + "libunwind.so.zst", ]; - for file in files { + for file in libraries { + for target in TARGETS { + let target_dir = out_dir.join(target); + + let path = target_dir.join(file); + fs::create_dir_all(path.parent().unwrap())?; + + if !path.exists() { + Encoder::new(File::create(path)?, ZSTD_COMPRESSION_LEVEL)?.do_finish()?; + } + } + } + + let adapters = ["wasi_snapshot_preview1.reactor.wasm.zst"]; + + for file in adapters { let path = out_dir.join(file); if !path.exists() { @@ -87,20 +105,16 @@ fn stubs_for_clippy(out_dir: &Path) -> Result<()> { } } - let path = out_dir.join("python-lib.tar.zst"); - - if !path.exists() { - Builder::new(Encoder::new(File::create(path)?, ZSTD_COMPRESSION_LEVEL)?) - .into_inner()? - .do_finish()?; - } + let tarballs = ["python-lib.tar.zst", "bundled.tar.zst"]; - let path = out_dir.join("bundled.tar.zst"); + for file in tarballs { + let path = out_dir.join(file); - if !path.exists() { - Builder::new(Encoder::new(File::create(path)?, ZSTD_COMPRESSION_LEVEL)?) - .into_inner()? - .do_finish()?; + if !path.exists() { + Builder::new(Encoder::new(File::create(path)?, ZSTD_COMPRESSION_LEVEL)?) + .into_inner()? + .do_finish()?; + } } Ok(()) @@ -126,14 +140,18 @@ fn find_wasi_sdk(out_dir: &Path) -> Result { None => DEFAULT_SDK_VERSION.to_string(), }; // same as the github release pipeline, allow overriding source repo - let source = env::var("WASI_SDK_SOURCE").unwrap_or("WebAssembly".to_string()); + // + // TODO: switch back to upstream release once Wasm exceptions + shared + // library support has been merged and released. + let suffix = "-shared-library-exceptions"; + let source = env::var("WASI_SDK_SOURCE").unwrap_or("dicej".to_string()); // Wasi-sdk currently releases for x86_64 and arm64 linux, macos, and windows let (arch, os) = match (env::consts::ARCH, env::consts::OS) { pair @ ("x86_64", "linux" | "windows" | "macos") => pair, ("aarch64", os @ ("linux" | "windows" | "macos")) => ("arm64", os), _ => bail!("Unsupported platform for automatic wasi-sdk download: {} {}", env::consts::ARCH, env::consts::OS), }; - Ok(format!("https://github.com/{source}/wasi-sdk/releases/download/wasi-sdk-{version}/wasi-sdk-{version}.0-{arch}-{os}.tar.gz")) + Ok(format!("https://github.com/{source}/wasi-sdk/releases/download/wasi-sdk-{version}{suffix}/wasi-sdk-{version}.0{suffix}-{arch}-{os}.tar.gz")) })?; // get the filename from the url @@ -154,7 +172,7 @@ fn find_wasi_sdk(out_dir: &Path) -> Result { url, wasi_sdk.display() ); - fetch_extract(&url, out_dir)?; + fetch_extract_gzip(&url, out_dir)?; Ok(wasi_sdk) } @@ -164,60 +182,62 @@ fn package_all_the_things(out_dir: &Path) -> Result<()> { let wasi_sdk = find_wasi_sdk(out_dir)?; - maybe_make_cpython(&repo_dir, &wasi_sdk)?; - - let cpython_wasi_dir = repo_dir.join("cpython/builddir/wasi"); + for &target in TARGETS { + maybe_make_cpython(target, &repo_dir, &wasi_sdk)?; - make_pyo3_config(&repo_dir)?; + let cpython_wasi_dir = repo_dir.join("cpython/builddir").join(target); - make_runtime( - &repo_dir, - out_dir, - &wasi_sdk, - &cpython_wasi_dir, - false, - "libcomponentize_py_runtime_sync.so", - )?; - make_runtime( - &repo_dir, - out_dir, - &wasi_sdk, - &cpython_wasi_dir, - true, - "libcomponentize_py_runtime_async.so", - )?; + let library_dir = out_dir.join(target); + fs::create_dir_all(&library_dir)?; - let libraries = [ - "libc.so", - "libwasi-emulated-mman.so", - "libwasi-emulated-process-clocks.so", - "libwasi-emulated-getpid.so", - "libwasi-emulated-signal.so", - ]; + make_pyo3_config(target, &repo_dir)?; - for library in libraries { - compress( - &wasi_sdk.join("share/wasi-sysroot/lib/wasm32-wasip2"), - library, - out_dir, - true, + make_runtime( + target, + &repo_dir, + &library_dir, + &wasi_sdk, + &cpython_wasi_dir, )?; - } - let libraries = ["libc++.so", "libc++abi.so"]; + let threads = if target == "wasip3" { + "experimental-coop-threads/" + } else { + "" + }; + + let default_dir = &format!("share/wasi-sysroot/lib/wasm32-{target}"); + let exceptions_dir = &format!("share/wasi-sysroot/lib/wasm32-{target}/eh"); + let threads_dir = &format!("share/wasi-sysroot/{threads}lib/wasm32-{target}"); + + let libraries = [ + (default_dir, "libc.so"), + (default_dir, "libwasi-emulated-mman.so"), + (default_dir, "libwasi-emulated-process-clocks.so"), + (default_dir, "libwasi-emulated-getpid.so"), + (default_dir, "libwasi-emulated-signal.so"), + (exceptions_dir, "libc++.so"), + (exceptions_dir, "libc++abi.so"), + (exceptions_dir, "libunwind.so"), + ]; + + for (dir, library) in libraries { + compress(&wasi_sdk.join(dir), library, &library_dir, true)?; + } - for library in libraries { + let threads_library_dir = library_dir.join("threads"); + fs::create_dir_all(&threads_library_dir)?; compress( - &wasi_sdk.join("share/wasi-sysroot/lib/wasm32-wasip2/noeh"), - library, - out_dir, + &wasi_sdk.join(threads_dir), + "libc.so", + &threads_library_dir, true, )?; - } - compress(&cpython_wasi_dir, "libpython3.14.so", out_dir, true)?; + compress(&cpython_wasi_dir, "libpython3.14.so", &library_dir, true)?; + } - let path = repo_dir.join("cpython/builddir/wasi/install/lib/python3.14"); + let path = repo_dir.join("cpython/builddir/wasip2/install/lib/python3.14"); if path.exists() { let mut builder = Builder::new(Encoder::new( @@ -305,18 +325,29 @@ fn add(builder: &mut Builder, root: &Path, path: &Path) -> Result<() Ok(()) } -fn maybe_make_cpython(repo_dir: &Path, wasi_sdk: &Path) -> Result<()> { +fn maybe_make_cpython(target: &str, repo_dir: &Path, wasi_sdk: &Path) -> Result<()> { let cpython_dir = repo_dir.join("cpython"); if !cpython_dir.exists() { - let url = &env::var("CPYTHON_TARBALL_URL").unwrap_or_else(|_| CPYTHON_TARBALL_URL.into()); - let base_dir = &env::var_os("CPYTHON_TARBALL_BASE_DIR") - .unwrap_or_else(|| CPYTHON_TARBALL_BASE_DIR.into()); - println!("cargo:warning=downloading CPython source code from {url}"); - fetch_extract(url, repo_dir)?; - fs::rename(repo_dir.join(base_dir), &cpython_dir)?; + if env::var_os("CPYTHON_BUILD_FROM_SOURCE").is_some() { + let url = &env::var("CPYTHON_SOURCE_TARBALL_URL") + .unwrap_or_else(|_| CPYTHON_SOURCE_TARBALL_URL.into()); + let base_dir = &env::var_os("CPYTHON_SOURCE_TARBALL_BASE_DIR") + .unwrap_or_else(|| CPYTHON_SOURCE_TARBALL_BASE_DIR.into()); + println!("cargo:warning=downloading CPython source code from {url}"); + fetch_extract_gzip(url, repo_dir)?; + fs::rename(repo_dir.join(base_dir), &cpython_dir)?; + } else { + let url = &env::var("CPYTHON_BINARY_TARBALL_URL") + .unwrap_or_else(|_| CPYTHON_BINARY_TARBALL_URL.into()); + println!("cargo:warning=downloading CPython build from {url}"); + let dir = cpython_dir.join("builddir"); + fs::create_dir_all(&dir)?; + fetch_extract_zstd(url, &dir)?; + return Ok(()); + } } - let cpython_wasi_dir = cpython_dir.join("builddir/wasi"); + let cpython_wasi_dir = cpython_dir.join("builddir").join(target); if !cpython_wasi_dir.join("libpython3.14.so").exists() { fs::create_dir_all(&cpython_wasi_dir)?; if !cpython_wasi_dir.join("libpython3.14.a").exists() { @@ -338,9 +369,9 @@ fn maybe_make_cpython(repo_dir: &Path, wasi_sdk: &Path) -> Result<()> { } let lib_install_dir = cpython_wasi_dir.join("deps"); - build_zlib(wasi_sdk, &lib_install_dir)?; + build_zlib(target, wasi_sdk, &lib_install_dir)?; - build_sqlite(wasi_sdk, &lib_install_dir)?; + build_sqlite(target, wasi_sdk, &lib_install_dir)?; let config_guess = run(Command::new("../../config.guess").current_dir(&cpython_wasi_dir))?; @@ -358,12 +389,12 @@ fn maybe_make_cpython(repo_dir: &Path, wasi_sdk: &Path) -> Result<()> { ) .env( "CFLAGS", - format!("--target=wasm32-wasip2 -fPIC -I{dir}/deps/include"), + format!("--target=wasm32-{target} -fPIC -I{dir}/deps/include"), ) .env("WASI_SDK_PATH", wasi_sdk) .env( "LDFLAGS", - format!("--target=wasm32-wasip2 -L{dir}/deps/lib"), + format!("--target=wasm32-{target} -L{dir}/deps/lib"), ) .current_dir(&cpython_wasi_dir) .args([ @@ -388,7 +419,7 @@ fn maybe_make_cpython(repo_dir: &Path, wasi_sdk: &Path) -> Result<()> { // Link libpython3.14.so - now includes libsqlite3.a run(Command::new(wasi_sdk.join("bin/clang")) - .arg("--target=wasm32-wasip2") + .arg(format!("--target=wasm32-{target}")) .arg("-shared") .arg("-o") .arg(cpython_wasi_dir.join("libpython3.14.so")) @@ -475,9 +506,9 @@ fn run(command: &mut Command) -> Result> { } } -fn make_pyo3_config(repo_dir: &Path) -> Result<()> { +fn make_pyo3_config(target: &str, repo_dir: &Path) -> Result<()> { let out_dir = env::var("OUT_DIR")?; - let mut cpython_wasi_dir = repo_dir.join("cpython/builddir/wasi"); + let mut cpython_wasi_dir = repo_dir.join("cpython/builddir").join(target); let mut cygpath = Command::new("cygpath"); cygpath.arg("-w").arg(&cpython_wasi_dir); if let Ok(output) = cygpath.output() { @@ -497,7 +528,10 @@ fn make_pyo3_config(repo_dir: &Path) -> Result<()> { "lib_dir={}", cpython_wasi_dir.to_str().unwrap() )?; - fs::write(Path::new(&out_dir).join("pyo3-config.txt"), pyo3_config)?; + fs::write( + Path::new(&out_dir).join(target).join("pyo3-config.txt"), + pyo3_config, + )?; println!("cargo:rerun-if-changed=pyo3-config.txt"); @@ -505,37 +539,26 @@ fn make_pyo3_config(repo_dir: &Path) -> Result<()> { } fn make_runtime( + target: &str, repo_dir: &Path, out_dir: &Path, wasi_sdk: &Path, cpython_wasi_dir: &Path, - async_: bool, - name: &str, ) -> Result<()> { let mut cmd = Command::new("cargo"); cmd.current_dir(repo_dir.join("runtime")) .arg("build") - .arg("--target=wasm32-wasip2"); + .arg(format!("--target=wasm32-{target}")); if !DEBUG_RUNTIME { cmd.arg("--release"); } - if async_ { + if target == "wasip3" { cmd.arg("--features=async"); } - for (key, _) in env::vars_os() { - if key - .to_str() - .map(|key| key.starts_with("RUST") || key.starts_with("CARGO")) - .unwrap_or(false) - { - cmd.env_remove(&key); - } - } - - let target = if async_ { "async" } else { "sync" }; + cmd.env_remove("CARGO_ENCODED_RUSTFLAGS"); let clang = wasi_sdk.join(format!("bin/{CLANG_EXECUTABLE}")); cmd.env( @@ -543,6 +566,7 @@ fn make_runtime( format!( "--cfg pyo3_disable_reference_pool \ -Clink-args=-Wl,--skip-wit-component \ + -Clink-args=-Wl,--export-if-defined=__wasm_library_tls_info \ -Clink-args=-shared \ -Clink-args=-L{} \ -Clink-args=-lpython3.14 \ @@ -550,7 +574,10 @@ fn make_runtime( cpython_wasi_dir.to_str().unwrap() ), ) - .env("CARGO_TARGET_WASM32_WASIP2_LINKER", clang) + .env( + format!("CARGO_TARGET_WASM32_{}_LINKER", target.to_uppercase()), + clang, + ) .env("CARGO_TARGET_DIR", out_dir.join(target)) .env("PYO3_CONFIG_FILE", out_dir.join("pyo3-config.txt")); @@ -562,10 +589,11 @@ fn make_runtime( let build = if DEBUG_RUNTIME { "debug" } else { "release" }; let path = out_dir.join(target).join(format!( - "wasm32-wasip2/{build}/componentize_py_runtime.wasm" + "wasm32-{target}/{build}/componentize_py_runtime.wasm" )); if path.exists() { + let name = "libcomponentize_py_runtime.so"; fs::copy(&path, out_dir.join(name))?; compress(out_dir, name, out_dir, false)?; } else { @@ -575,7 +603,7 @@ fn make_runtime( Ok(()) } -fn fetch_extract(url: &str, out_dir: &Path) -> Result<()> { +fn fetch_extract_gzip(url: &str, out_dir: &Path) -> Result<()> { let response = reqwest::blocking::get(url)?; let decoder = flate2::read::GzDecoder::new(response); let mut archive = tar::Archive::new(decoder); @@ -583,7 +611,15 @@ fn fetch_extract(url: &str, out_dir: &Path) -> Result<()> { Ok(()) } -fn add_compile_envs(wasi_sdk: &Path, command: &mut Command) { +fn fetch_extract_zstd(url: &str, out_dir: &Path) -> Result<()> { + let response = reqwest::blocking::get(url)?; + let decoder = zstd::Decoder::new(response)?; + let mut archive = tar::Archive::new(decoder); + archive.unpack(out_dir)?; + Ok(()) +} + +fn add_compile_envs(target: &str, wasi_sdk: &Path, command: &mut Command) { let sysroot = wasi_sdk.join("share/wasi-sysroot"); let sysroot = sysroot.to_string_lossy(); command @@ -592,17 +628,17 @@ fn add_compile_envs(wasi_sdk: &Path, command: &mut Command) { .env("RANLIB", wasi_sdk.join("bin/ranlib")) .env( "CFLAGS", - format!("--target=wasm32-wasip2 --sysroot={sysroot} -I{sysroot}/include/wasm32-wasip2 -D_WASI_EMULATED_SIGNAL -fPIC"), + format!("--target=wasm32-{target} --sysroot={sysroot} -I{sysroot}/include/wasm32-{target} -D_WASI_EMULATED_SIGNAL -fPIC"), ) .env( "LDFLAGS", - format!("--target=wasm32-wasip2 --sysroot={sysroot} -L{sysroot}/lib -lwasi-emulated-signal") + format!("--target=wasm32-{target} --sysroot={sysroot} -L{sysroot}/lib -lwasi-emulated-signal") ); } -fn build_zlib(wasi_sdk: &Path, install_dir: &Path) -> Result<()> { +fn build_zlib(target: &str, wasi_sdk: &Path, install_dir: &Path) -> Result<()> { let out_dir = PathBuf::from(env::var("OUT_DIR")?); - fetch_extract( + fetch_extract_gzip( "https://github.com/madler/zlib/releases/download/v1.3.1/zlib-1.3.1.tar.gz", &out_dir, )?; @@ -613,7 +649,7 @@ fn build_zlib(wasi_sdk: &Path, install_dir: &Path) -> Result<()> { .ok_or_else(|| anyhow!("non-UTF8 path: {}", install_dir.display()))?; let mut configure = Command::new("./configure"); - add_compile_envs(wasi_sdk, &mut configure); + add_compile_envs(target, wasi_sdk, &mut configure); configure .current_dir(&src_dir) .arg("--static") @@ -631,7 +667,7 @@ fn build_zlib(wasi_sdk: &Path, install_dir: &Path) -> Result<()> { .ok_or_else(|| anyhow!("non-UTF8 path: {}", clang_dir.display()))?; let mut make = Command::new("make"); - add_compile_envs(wasi_sdk, &mut make); + add_compile_envs(target, wasi_sdk, &mut make); make.current_dir(src_dir) .arg(format!("AR={ar_dir}")) .arg("ARFLAGS=rcs") @@ -650,7 +686,7 @@ fn build_zlib(wasi_sdk: &Path, install_dir: &Path) -> Result<()> { /// - SQLITE_OMIT_WAL: WAL requires mmap which isn't available in WASI preview1 /// - SQLITE_OMIT_LOAD_EXTENSION: No dlopen in WASI /// - SQLITE_THREADSAFE=0: Single-threaded for WASM -fn build_sqlite(wasi_sdk: &Path, install_dir: &Path) -> Result<()> { +fn build_sqlite(target: &str, wasi_sdk: &Path, install_dir: &Path) -> Result<()> { let out_dir = PathBuf::from(env::var("OUT_DIR")?); // Check if already built @@ -661,12 +697,18 @@ fn build_sqlite(wasi_sdk: &Path, install_dir: &Path) -> Result<()> { println!("cargo:warning=Building SQLite {SQLITE_VERSION} for WASI..."); - // Download SQLite amalgamation - let url = format!("https://sqlite.org/{SQLITE_YEAR}/sqlite-autoconf-{SQLITE_VERSION}.tar.gz"); - fetch_extract(&url, &out_dir)?; - let src_dir = out_dir.join(format!("sqlite-autoconf-{SQLITE_VERSION}")); + if !src_dir.exists() { + // Download SQLite amalgamation + let url = + format!("https://sqlite.org/{SQLITE_YEAR}/sqlite-autoconf-{SQLITE_VERSION}.tar.gz"); + fetch_extract_gzip(&url, &out_dir)?; + } + + let build_dir = src_dir.join(target); + fs::create_dir_all(&build_dir)?; + // Ensure install directories exist fs::create_dir_all(install_dir.join("lib"))?; fs::create_dir_all(install_dir.join("include"))?; @@ -687,9 +729,9 @@ fn build_sqlite(wasi_sdk: &Path, install_dir: &Path) -> Result<()> { // Note: Don't set SQLITE_THREADSAFE here - let --disable-threadsafe handle it // to avoid macro redefinition warnings let sqlite_cflags = format!( - "--target=wasm32-wasip2 \ + "--target=wasm32-{target} \ --sysroot={sysroot_str} \ - -I{sysroot_str}/include/wasm32-wasip2 \ + -I{sysroot_str}/include/wasm32-{target} \ -D_WASI_EMULATED_SIGNAL \ -D_WASI_EMULATED_PROCESS_CLOCKS \ -fPIC \ @@ -702,18 +744,18 @@ fn build_sqlite(wasi_sdk: &Path, install_dir: &Path) -> Result<()> { ); // Configure SQLite - let mut configure = Command::new("./configure"); + let mut configure = Command::new("../configure"); configure - .current_dir(&src_dir) + .current_dir(&build_dir) .env("AR", wasi_sdk.join("bin/ar")) .env("CC", wasi_sdk.join("bin/clang")) .env("RANLIB", wasi_sdk.join("bin/ranlib")) .env("CFLAGS", &sqlite_cflags) .env( "LDFLAGS", - format!("--target=wasm32-wasip2 --sysroot={sysroot_str} -L{sysroot_str}/lib",), + format!("--target=wasm32-{target} --sysroot={sysroot_str} -L{sysroot_str}/lib",), ) - .arg("--host=wasm32-wasip2") + .arg(format!("--host=wasm32-{target}")) .arg(format!("--prefix={install_dir_str}")) .arg("--disable-shared") .arg("--enable-static") @@ -725,7 +767,7 @@ fn build_sqlite(wasi_sdk: &Path, install_dir: &Path) -> Result<()> { // Build only the static library (not the shell, which fails to link on WASI) let mut make = Command::new("make"); - make.current_dir(&src_dir) + make.current_dir(&build_dir) .env("AR", wasi_sdk.join("bin/ar")) .env("CC", wasi_sdk.join("bin/clang")) .env("RANLIB", wasi_sdk.join("bin/ranlib")) @@ -738,7 +780,7 @@ fn build_sqlite(wasi_sdk: &Path, install_dir: &Path) -> Result<()> { // Manual install since we didn't build everything // Copy the library fs::copy( - src_dir.join("libsqlite3.a"), + build_dir.join("libsqlite3.a"), install_dir.join("lib/libsqlite3.a"), )?; // Copy the headers diff --git a/bundled/componentize_py_async_support/__init__.py b/bundled/componentize_py_async_support/__init__.py index d4025667..6095b6f9 100644 --- a/bundled/componentize_py_async_support/__init__.py +++ b/bundled/componentize_py_async_support/__init__.py @@ -10,6 +10,7 @@ import componentize_py_runtime import subprocess +from threading import local from os import PathLike from socket import AddressFamily, AddressInfo, SocketKind, socket from concurrent.futures import Executor @@ -67,7 +68,7 @@ class _Status: async def _noop() -> None: pass - + class _Loop(asyncio.AbstractEventLoop): def __init__(self) -> None: self.running: bool = False @@ -80,13 +81,13 @@ def poll(self, future_state: _FutureState) -> None: for handle in handles: if not handle._cancelled: handle._run() - + if self.exception is not None: raise self.exception if len(handles) == 0 and len(future_state.handles) == 0: return - + def get_debug(self) -> bool: return False @@ -187,7 +188,7 @@ async def getnameinfo(self, sockaddr: tuple[str, int] | tuple[str, int, int, int], flags: int = 0) -> tuple[str, str]: raise NotImplementedError - + async def create_connection( self, protocol_factory: Callable[[], _ProtocolT], @@ -207,7 +208,7 @@ async def create_connection( interleave: int | None = None, ) -> tuple[Transport, _ProtocolT]: raise NotImplementedError - + async def create_server( self, protocol_factory: Callable[[], BaseProtocol], @@ -420,12 +421,9 @@ def default_exception_handler(self, context: dict[str, Any]) -> None: def set_debug(self, enabled: bool) -> None: raise NotImplementedError - + _future_state: ContextVar[_FutureState] = ContextVar("_future_state") _loop = _Loop() -asyncio.set_event_loop(_loop) -_loop.running = True -asyncio.events._set_running_loop(_loop) def _set_future_state(future_state: _FutureState) -> None: global _future_state @@ -448,12 +446,17 @@ async def _return_result(export_index: int, borrows: int, coroutine: Any) -> Non assert _future_state.get().pending_count > 0 _future_state.get().pending_count -= 1 +_thread_local = local() + def first_poll(export_index: int, borrows: int, coroutine: Any) -> int: """Internal function called by generated code for exported functions. This is not meant to be called by application code. """ + asyncio.set_event_loop(_loop) + _loop.running = True + asyncio.events._set_running_loop(_loop) context = Context() future_state = _FutureState(None, {}, [], 1) context.run(_set_future_state, future_state) @@ -468,12 +471,12 @@ def _poll(future_state: _FutureState) -> int: if future_state.pending_count == 0: if future_state.waitable_set is not None: componentize_py_runtime.waitable_set_drop(future_state.waitable_set) - + return _CallbackCode.EXIT else: waitable_set = future_state.waitable_set assert waitable_set is not None - componentize_py_runtime.context_set(future_state) + _thread_local.state = future_state return _CallbackCode.WAIT | (waitable_set << 4) def callback(event0: int, event1: int, event2: int) -> int: @@ -482,9 +485,9 @@ def callback(event0: int, event1: int, event2: int) -> int: This is not meant to be called by application code. """ - future_state = componentize_py_runtime.context_get() - componentize_py_runtime.context_set(None) - + future_state = _thread_local.state + _thread_local.state = None + match event0: case _Event.NONE: pass @@ -509,7 +512,7 @@ def callback(event0: int, event1: int, event2: int) -> int: raise NotImplementedError return _poll(future_state) - + async def await_result[T](result: Result[T, tuple[int, int]]) -> T: """Internal function called by generated code for imported functions. @@ -518,7 +521,7 @@ async def await_result[T](result: Result[T, tuple[int, int]]) -> T: """ global _loop global _future_state - + if isinstance(result, Ok): return result.value else: @@ -526,11 +529,11 @@ async def await_result[T](result: Result[T, tuple[int, int]]) -> T: waitable, promise = result.value future = _loop.create_future() future_state.futures[waitable] = future - + if future_state.waitable_set is None: future_state.waitable_set = componentize_py_runtime.waitable_set_new() componentize_py_runtime.waitable_join(waitable, future_state.waitable_set) - + return cast(T, componentize_py_runtime.promise_get_result(await future, promise)) async def _wrap_spawned(coroutine: Any) -> None: @@ -566,5 +569,5 @@ def spawn(coroutine: Any) -> None: global _future_state _future_state.get().pending_count += 1 - + asyncio.create_task(_wrap_spawned(coroutine)) diff --git a/examples/multithreading/README.md b/examples/multithreading/README.md new file mode 100644 index 00000000..0f04bf39 --- /dev/null +++ b/examples/multithreading/README.md @@ -0,0 +1,44 @@ +# Example: `multithreading` + +This is an example of how to use [componentize-py] and [Wasmtime] to build and +run a Python-based component targetting version `0.3.0` of the [wasi-cli] +`command` world and using multiple threads. + +[componentize-py]: https://github.com/bytecodealliance/componentize-py +[Wasmtime]: https://github.com/bytecodealliance/wasmtime +[wasi-cli]: https://github.com/WebAssembly/WASI/tree/v0.3.0/proposals/cli/wit + +## Prerequisites + +* `Wasmtime` 48.0.0 +* `componentize-py` 0.25.0 + +Below, we use [Rust](https://rustup.rs/)'s `cargo` to install `Wasmtime`. If +you don't have `cargo`, you can download and install from +https://github.com/bytecodealliance/wasmtime/releases/tag/v48.0.0. + +``` +cargo install --version 48.0.0 wasmtime-cli +pip install componentize-py==0.25.0 +``` + +## Running the demo + +``` +componentize-py -d ../../wit -w wasi:cli/command@0.3.0 componentize --target wasm32-wasip3-threads app -o app.wasm +wasmtime run -Wcomponent-model-threading app.wasm +``` + +The `wasmtime run` command above should print something like the following (the +output may vary depending on how the threads are scheduled): + +``` +thread `a` started +thread `b` started +thread `c` started +started all threads +thread `a` finished +thread `b` finished +thread `c` finished +joined all threads +``` diff --git a/examples/multithreading/app.py b/examples/multithreading/app.py new file mode 100644 index 00000000..a4ced113 --- /dev/null +++ b/examples/multithreading/app.py @@ -0,0 +1,24 @@ +import time + +from threading import Thread +from wit.exports.wasi.cli_v0_3 import run, Run + +@run.guest +class Cli(Run): + async def run(self) -> None: + threads = list(map(lambda name: Thread(target=run_thread, args=(name,)), ["a", "b", "c"])) + + for thread in threads: + thread.start() + + print("started all threads") + + for thread in threads: + thread.join() + + print("joined all threads") + +def run_thread(name: str) -> None: + print(f"thread `{name}` started") + time.sleep(1) + print(f"thread `{name}` finished") diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index e2d8576c..7f8469f4 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -13,7 +13,7 @@ anyhow = "1.0.91" once_cell = "1.20.2" pyo3 = { version="0.28.3", features = ["abi3-py312", "num-bigint"] } num-bigint = "0.4.6" -wit-bindgen = { version = "0.61.0", default-features = false, features = ["macros", "realloc"] } +wit-bindgen = { version = "0.61.0", default-features = false, features = ["macros", "realloc", "async"] } wit-dylib-ffi = { git = "https://github.com/bytecodealliance/wasm-tools", tag = "v1.258.0", features = ["async-raw"] } [features] diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index 4fa62e5b..a7c708b3 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -22,7 +22,7 @@ use { pyo3::{ Bound, IntoPyObject, Py, PyAny, PyErr, PyResult, Python, exceptions::PyAssertionError, - intern, + ffi, intern, types::{ PyAnyMethods, PyBool, PyBytes, PyBytesMethods, PyDict, PyDictMethods, PyList, PyListMethods, PyMapping, PyMappingMethods, PyModule, PyModuleMethods, PyString, @@ -31,9 +31,10 @@ use { }, std::{ alloc::{self, Layout}, + cell::Cell, iter, marker::PhantomData, - mem, slice, str, + mem, ptr, slice, str, sync::Once, }, wit_dylib_ffi::{ @@ -53,6 +54,8 @@ mod bindings { export!(MyExports); } +const CALLBACK_CODE_EXIT: u32 = 0; + static WIT: OnceCell = OnceCell::new(); static STUB_WASI: OnceCell = OnceCell::new(); static EXPORTS: OnceCell> = OnceCell::new(); @@ -1184,7 +1187,7 @@ fn do_init(app_name: String, symbols: Symbols, stub_wasi: bool) -> Result<(), St struct MyExports; impl Guest for MyExports { - fn init(app_name: String, symbols: Symbols, stub_wasi: bool) -> Result<(), String> { + async fn init(app_name: String, symbols: Symbols, stub_wasi: bool) -> Result<(), String> { let result = do_init(app_name, symbols, stub_wasi); // This tells the WASI Preview 1 component adapter to reset its state. @@ -1218,7 +1221,9 @@ struct MyInterpreter; impl MyInterpreter { fn export_call_(func: ExportFunction, cx: &mut MyCall<'_>, async_: bool) -> u32 { - Python::attach(|py| { + let gil_state = unsafe { ffi::PyGILState_Ensure() }; + + let result = Python::attach(|py| { if !*STUB_WASI.get().unwrap() { static ONCE: Once = Once::new(); ONCE.call_once(|| { @@ -1316,12 +1321,24 @@ impl MyInterpreter { release_borrows(py, mem::take(&mut cx.borrows)); - 0 + CALLBACK_CODE_EXIT } - }) + }); + + if result == CALLBACK_CODE_EXIT { + unsafe { ffi::PyGILState_Release(gil_state) }; + } else { + THREAD_STATE.with(|v| { + v.set((Some(gil_state), unsafe { ffi::PyEval_SaveThread() })); + }); + } + + result } } +std::thread_local!(static THREAD_STATE: Cell<(Option, *mut ffi::PyThreadState)> = const { Cell::new((None, ptr::null_mut())) } ); + impl Interpreter for MyInterpreter { type CallCx<'a> = MyCall<'a>; @@ -1352,7 +1369,13 @@ impl Interpreter for MyInterpreter { fn export_async_callback(event0: u32, event1: u32, event2: u32) -> u32 { #[cfg(feature = "async")] { - Python::attach(|py| { + let gil_state = THREAD_STATE.with(|v| { + let (gil_state, thread_state) = v.get(); + unsafe { ffi::PyEval_RestoreThread(thread_state) }; + gil_state + }); + + let result = Python::attach(|py| { async_::CALLBACK .get() .unwrap() @@ -1360,7 +1383,17 @@ impl Interpreter for MyInterpreter { .unwrap() .extract(py) .unwrap() - }) + }); + + if result == CALLBACK_CODE_EXIT { + unsafe { ffi::PyGILState_Release(gil_state.unwrap()) }; + } else { + THREAD_STATE.with(|v| { + v.set((gil_state, unsafe { ffi::PyEval_SaveThread() })); + }); + } + + result } #[cfg(not(feature = "async"))] { @@ -1444,7 +1477,9 @@ impl Drop for MyCall<'_> { fn drop(&mut self) { for &(ptr, layout) in &self.deferred_deallocations { unsafe { - alloc::dealloc(ptr, layout); + if layout.size() > 0 { + alloc::dealloc(ptr, layout); + } } } } @@ -2098,7 +2133,9 @@ impl Call for MyCall<'_> { .to_owned() .into_any() .unbind(); - alloc::dealloc(src, Layout::from_size_align(len, 1).unwrap()); + if len > 0 { + alloc::dealloc(src, Layout::from_size_align(len, 1).unwrap()); + } value })); true @@ -2226,34 +2263,3 @@ fn exported_resource_to_canon( } wit_dylib_ffi::export!(MyInterpreter); - -// As of this writing, recent Rust `nightly` builds include a version of the -// `libc` crate that expects `wasi-libc` to define the following global -// variables, but `wasi-libc` defines them as preprocessor constants which -// aren't visible at link time, so we need to define them somewhere. Ideally, -// we should fix this upstream, but for now we work around it: - -#[unsafe(no_mangle)] -static _CLOCK_PROCESS_CPUTIME_ID: u8 = 2; -#[unsafe(no_mangle)] -static _CLOCK_THREAD_CPUTIME_ID: u8 = 3; - -// Traditionally, `wit-bindgen` would provide a `cabi_realloc` implementation, -// but recent versions use a weak symbol trick to avoid conflicts when more than -// one `wit-bindgen` version is used, and that trick does not currently play -// nice with how we build this library. So for now, we just define it ourselves -// here: -/// # Safety -/// TODO -#[unsafe(export_name = "cabi_realloc")] -pub unsafe extern "C" fn cabi_realloc( - old_ptr: *mut u8, - old_len: usize, - align: usize, - new_size: usize, -) -> *mut u8 { - assert!(old_ptr.is_null()); - assert!(old_len == 0); - - unsafe { alloc::alloc(Layout::from_size_align(new_size, align).unwrap()) } -} diff --git a/src/command.rs b/src/command.rs index 0d9e03c7..7ca52dd6 100644 --- a/src/command.rs +++ b/src/command.rs @@ -1,5 +1,5 @@ use { - crate::{BindingsGenerator, ComponentGenerator}, + crate::{BindingsGenerator, ComponentGenerator, Target}, anyhow::{Context, Result}, clap::Parser as _, std::{ @@ -7,7 +7,8 @@ use { ffi::OsString, fs, path::{Path, PathBuf}, - process, str, + process, + str::{self, FromStr as _}, }, tokio::runtime::Runtime, }; @@ -205,6 +206,22 @@ pub struct Componentize { /// randomness is required. #[arg(short = 's', long)] pub stub_wasi: bool, + + /// Set the native target explicitly. + /// + /// By default, `componentize-py` will choose the native target + /// automatically based on whether any of the target worlds use Component + /// Model async features. If so, `wasm32-wasip3` will be used; otherwise + /// `wasm32-wasip2` will be used. This option overrides that selection. + /// + /// A value of `wasm32-wasip3-threads` is also currently accepted, in which + /// case the Component Model cooperative multithreading feature will be + /// used. That feature is experimental as of this writing; once it is + /// stablized, `componentize-py` will enable multithreading by default for + /// the `wasm32-wasip3` target, and the `wasm32-wasip3-threads` target will + /// be simply become an alias for `wasm32-wasip3`. + #[arg(long)] + pub target: Option, } #[derive(clap::Args, Debug)] @@ -335,6 +352,11 @@ fn componentize(common: Common, componentize: Componentize) -> Result<()> { intersect_world: componentize.intersect_world.as_deref(), registry_config: common.registry_config.as_deref(), default_registries: !common.no_default_registries, + target: componentize + .target + .as_deref() + .map(Target::from_str) + .transpose()?, } .generate(), )?; @@ -588,6 +610,7 @@ class Bindings(bindings.WorldExports): output: out_dir.path().join("app.wasm"), stub_wasi: false, intersect_world: None, + target: None, }; componentize(common, componentize_opts) } @@ -705,6 +728,7 @@ world lib-world { output: dir.path().join("app.wasm"), stub_wasi: false, intersect_world: None, + target: None, }, ) } diff --git a/src/lib.rs b/src/lib.rs index dd3fe44c..46ba4ccd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,12 +11,12 @@ use { std::{ borrow::Cow, collections::{HashMap, HashSet}, - fs, + fmt, fs, io::Cursor, iter, ops::Deref, path::{Path, PathBuf}, - str, + str::{self, FromStr}, }, summary::{Locations, Summary}, tar::Archive, @@ -51,6 +51,7 @@ mod test; mod util; const DEBUG_PYTHON_BINDINGS: bool = false; +const DEBUG_PRE_INIT: bool = false; /// Default bindings module name, overridable via `--bindings-module`. static DEFAULT_BINDINGS_MODULE: &str = "wit"; @@ -75,12 +76,49 @@ impl WasiView for Ctx { } } +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Multithreading { + Enabled, + Disabled, +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Target { + Wasip2, + Wasip3(Multithreading), +} + +impl FromStr for Target { + type Err = anyhow::Error; + + fn from_str(s: &str) -> anyhow::Result { + Ok(match s { + "wasm32-wasip2" => Target::Wasip2, + "wasm32-wasip3" => Target::Wasip3(Multithreading::Disabled), + "wasm32-wasip3-threads" => Target::Wasip3(Multithreading::Enabled), + _ => bail!( + "unrecognized target: `{s}`; \ + expected `wasm32-wasip2`, `wasm32-wasip3`, or `wasm32-wasip3-threads`" + ), + }) + } +} + pub struct Library { name: String, module: Vec, dl_openable: bool, } +impl fmt::Debug for Library { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("Library") + .field("name", &self.name) + .field("dl_openable", &self.dl_openable) + .finish() + } +} + #[derive(Deserialize)] struct RawComponentizePyConfig { bindings: Option, @@ -415,6 +453,7 @@ pub struct ComponentGenerator<'a> { pub intersect_world: Option<&'a str>, pub registry_config: Option<&'a Path>, pub default_registries: bool, + pub target: Option, } impl ComponentGenerator<'_> { @@ -457,8 +496,8 @@ impl ComponentGenerator<'_> { // generated code produced earlier. Assuming this step succeeds, we'll // snapshot the result and emit the snapshot as the final output. - // Remove non-existent elements from `python_path` so we don't choke on them - // later: + // Remove non-existent elements from `python_path` so we don't choke on + // them later: let python_path = &self .python_path .iter() @@ -716,13 +755,54 @@ impl ComponentGenerator<'_> { // Extract relevant metadata from the `Resolve` into a `Summary` instance, // which we'll use to generate Wasm- and Python-level bindings. + let target = if let Some(target) = self.target { + target + } else { + // Determine whether to use the WASIp2 or WASIp3 target based on + // whether the world uses any async features. + // + // TODO: Creating a temporary `Summary` is a heavyweight way to + // determine whether the world uses async features, especially since + // we will create the real one down below, but otherwise we'd have + // an ordering problem because we need to know the target before we + // call `wit_dylib::create_with_metadata` (which produces the + // metadata we'll need to create the real `Summary`), and we can't + // call that until we know the target. We should be able to extract + // the code that `Summary::try_new` uses to check for async features + // and use it without the rest of the things `Summary::try_new` + // does. + // + // Note that `componentize-go` has a reasonably succinct check for + // whether a world uses async features which we could move to + // `wit_component` and reuse from there. + let need_async = Summary::try_new( + &resolve, + &iter::once(world).collect(), + &import_interface_names, + &export_interface_names, + &HashMap::new(), + &HashMap::new(), + &HashMap::new(), + )? + .need_async(); + + if need_async { + Target::Wasip3(Multithreading::Disabled) + } else { + Target::Wasip2 + } + }; + let (mut bindings, metadata) = wit_dylib::create_with_metadata( &resolve, world, Some(&mut DylibOpts { - stack_pointer: wit_dylib::StackPointer::Global, interpreter: Some("libcomponentize_py_runtime.so".into()), async_: Default::default(), + stack_pointer: match target { + Target::Wasip2 => wit_dylib::StackPointer::Global, + Target::Wasip3(_) => wit_dylib::StackPointer::TaskContext, + }, }), ); @@ -737,6 +817,24 @@ impl ComponentGenerator<'_> { } .append_to(&mut bindings); + // Now that we know which target to use, add the bundled libraries for + // that target, as well as the bindings `wit-dylib` just generated. + // + // Note that we simply assume any libraries previously discovered in the + // Python search path are appropriate for the current target; if they + // aren't (i.e. if they use the `wasm32-wasip2` ABI when the + // `wasm32-wasip3` ABI was expected, or vice-versa), then stuff will + // break when we try to link, validate, or run the component. + let libraries = prelink::bundled_libraries(target)? + .into_iter() + .chain(libraries) + .chain(Some(Library { + name: "libcomponentize_py_bindings.so".into(), + module: bindings, + dl_openable: false, + })) + .collect::>(); + let imported_function_indexes = metadata .import_funcs .iter() @@ -788,34 +886,6 @@ impl ComponentGenerator<'_> { &stream_and_future_indexes, )?; - let need_async = summary.need_async(); - - // Now that we know whether to use the sync or async version of - // `libcomponentize_py_runtime.so`, update `libraries` accordingly. - // - // Note that we have two separate versions because older runtimes don't - // understand the new async ABI, so we only use the async version if it's - // actually needed. - let mut libraries = libraries - .into_iter() - .filter_map(|library| match (need_async, library.name.as_str()) { - (true, "libcomponentize_py_runtime_sync.so") - | (false, "libcomponentize_py_runtime_async.so") => None, - (true, "libcomponentize_py_runtime_async.so") - | (false, "libcomponentize_py_runtime_sync.so") => Some(Library { - name: "libcomponentize_py_runtime.so".into(), - ..library - }), - _ => Some(library), - }) - .collect::>(); - - libraries.push(Library { - name: "libcomponentize_py_bindings.so".into(), - module: bindings, - dl_openable: false, - }); - let component = link::link_libraries(&libraries)?; let stubbed_component = if self.stub_wasi { @@ -834,10 +904,14 @@ impl ComponentGenerator<'_> { let stderr = MemoryOutputPipe::new(10000); let mut wasi = WasiCtxBuilder::new(); - wasi.stdin(MemoryInputPipe::new(Bytes::new())) - .stdout(stdout.clone()) - .stderr(stderr.clone()) - .env("PYTHONUNBUFFERED", "1") + if DEBUG_PRE_INIT { + wasi.inherit_stdio(); + } else { + wasi.stdin(MemoryInputPipe::new(Bytes::new())) + .stdout(stdout.clone()) + .stderr(stderr.clone()); + } + wasi.env("PYTHONUNBUFFERED", "1") .env("PYTHONHOME", "/python") .preopened_dir( embedded_python_standard_lib.path(), @@ -964,6 +1038,7 @@ impl ComponentGenerator<'_> { config.wasm_component_model(true); config.wasm_component_model_async(true); config.wasm_component_model_map(true); + config.wasm_component_model_threading(true); let engine = Engine::new(&config)?; @@ -995,10 +1070,16 @@ impl ComponentGenerator<'_> { let instance = pre.instance_pre.instantiate_async(&mut store).await?; let guest = pre.indices.interface0.load(&mut store, &instance)?; - guest - .call_init(&mut store, &app_name, &symbols, stub_wasi) - .await? - .map_err(|e| anyhow!("{e}"))?; + store + .run_concurrent(async |store| { + guest + .call_init(store, app_name, symbols, stub_wasi) + .await? + .map_err(|e| anyhow!("{e}"))?; + + anyhow::Ok(()) + }) + .await??; Ok(Box::new(MyInvoker { store, instance }) as Box) } @@ -1173,6 +1254,7 @@ fn add_wasi_and_stubs( linker: &mut Linker, ) -> Result<()> { wasmtime_wasi::p2::add_to_linker_async(linker)?; + wasmtime_wasi::p3::add_to_linker(linker)?; enum Stub<'a> { Function(&'a String, &'a FunctionKind), @@ -1228,10 +1310,10 @@ fn add_wasi_and_stubs( for (interface_name, stubs) in stubs { if let Some(interface_name) = interface_name { // Note that we do _not_ stub interfaces which appear to be part of - // WASIp2 since those should be provided by the - // `wasmtime_wasi::add_to_linker_async` call above, and adding stubs - // to those same interfaces would just cause trouble. - if !is_wasip2_cli(&interface_name) + // WASIp2/p3 since those should be provided by the + // `add_to_linker{_async}` calls above, and adding stubs to those + // same interfaces would just cause trouble. + if !is_wasi_cli(&interface_name) && let Ok(mut instance) = linker.instance(&interface_name) { for stub in stubs { @@ -1313,12 +1395,12 @@ fn add_wasi_and_stubs( Ok(()) } -fn is_wasip2_cli(interface_name: &str) -> bool { +fn is_wasi_cli(interface_name: &str) -> bool { (interface_name.starts_with("wasi:cli/") || interface_name.starts_with("wasi:clocks/") || interface_name.starts_with("wasi:random/") || interface_name.starts_with("wasi:io/") || interface_name.starts_with("wasi:filesystem/") || interface_name.starts_with("wasi:sockets/")) - && interface_name.contains("@0.2.") + && (interface_name.contains("@0.2.") || interface_name.contains("@0.3.")) } diff --git a/src/prelink.rs b/src/prelink.rs index 3aac57af..a01a1cf9 100644 --- a/src/prelink.rs +++ b/src/prelink.rs @@ -14,7 +14,9 @@ use tar::Archive; use tempfile::TempDir; use zstd::Decoder; -use crate::{ComponentizePyConfig, ConfigContext, Library, RawComponentizePyConfig}; +use crate::{ + ComponentizePyConfig, ConfigContext, Library, Multithreading, RawComponentizePyConfig, Target, +}; static NATIVE_EXTENSION_SUFFIXES: &[&str] = &[".cpython-314-wasm32-wasi.so", ".abi3.so"]; @@ -57,90 +59,182 @@ pub fn embedded_helper_utils() -> Result { Ok(bundled) } -pub fn bundle_libraries(library_path: Vec<(&str, Vec)>) -> Result> { - let mut libraries = vec![ - Library { - name: "libcomponentize_py_runtime_sync.so".into(), - module: zstd::decode_all(Cursor::new(include_bytes!(concat!( - env!("OUT_DIR"), - "/libcomponentize_py_runtime_sync.so.zst" - ))))?, - dl_openable: false, - }, - Library { - name: "libcomponentize_py_runtime_async.so".into(), - module: zstd::decode_all(Cursor::new(include_bytes!(concat!( - env!("OUT_DIR"), - "/libcomponentize_py_runtime_async.so.zst" - ))))?, - dl_openable: false, - }, - Library { - name: "libpython3.14.so".into(), - module: zstd::decode_all(Cursor::new(include_bytes!(concat!( - env!("OUT_DIR"), - "/libpython3.14.so.zst" - ))))?, - dl_openable: false, - }, - Library { - name: "libc.so".into(), - module: zstd::decode_all(Cursor::new(include_bytes!(concat!( - env!("OUT_DIR"), - "/libc.so.zst" - ))))?, - dl_openable: false, - }, - Library { - name: "libwasi-emulated-mman.so".into(), - module: zstd::decode_all(Cursor::new(include_bytes!(concat!( - env!("OUT_DIR"), - "/libwasi-emulated-mman.so.zst" - ))))?, - dl_openable: false, - }, - Library { - name: "libwasi-emulated-process-clocks.so".into(), - module: zstd::decode_all(Cursor::new(include_bytes!(concat!( - env!("OUT_DIR"), - "/libwasi-emulated-process-clocks.so.zst" - ))))?, - dl_openable: false, - }, - Library { - name: "libwasi-emulated-getpid.so".into(), - module: zstd::decode_all(Cursor::new(include_bytes!(concat!( - env!("OUT_DIR"), - "/libwasi-emulated-getpid.so.zst" - ))))?, - dl_openable: false, - }, - Library { - name: "libwasi-emulated-signal.so".into(), - module: zstd::decode_all(Cursor::new(include_bytes!(concat!( - env!("OUT_DIR"), - "/libwasi-emulated-signal.so.zst" - ))))?, - dl_openable: false, - }, - Library { - name: "libc++.so".into(), - module: zstd::decode_all(Cursor::new(include_bytes!(concat!( - env!("OUT_DIR"), - "/libc++.so.zst" - ))))?, - dl_openable: false, - }, - Library { - name: "libc++abi.so".into(), - module: zstd::decode_all(Cursor::new(include_bytes!(concat!( - env!("OUT_DIR"), - "/libc++abi.so.zst" - ))))?, - dl_openable: false, - }, - ]; +pub fn bundled_libraries(target: Target) -> Result> { + Ok(match target { + Target::Wasip2 => vec![ + Library { + name: "libcomponentize_py_runtime.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip2/libcomponentize_py_runtime.so.zst" + ))))?, + dl_openable: false, + }, + Library { + name: "libpython3.14.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip2/libpython3.14.so.zst" + ))))?, + dl_openable: false, + }, + Library { + name: "libc.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip2/libc.so.zst" + ))))?, + dl_openable: false, + }, + Library { + name: "libwasi-emulated-mman.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip2/libwasi-emulated-mman.so.zst" + ))))?, + dl_openable: false, + }, + Library { + name: "libwasi-emulated-process-clocks.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip2/libwasi-emulated-process-clocks.so.zst" + ))))?, + dl_openable: false, + }, + Library { + name: "libwasi-emulated-getpid.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip2/libwasi-emulated-getpid.so.zst" + ))))?, + dl_openable: false, + }, + Library { + name: "libwasi-emulated-signal.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip2/libwasi-emulated-signal.so.zst" + ))))?, + dl_openable: false, + }, + Library { + name: "libc++.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip2/libc++.so.zst" + ))))?, + dl_openable: false, + }, + Library { + name: "libc++abi.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip2/libc++abi.so.zst" + ))))?, + dl_openable: false, + }, + Library { + name: "libunwind.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip2/libunwind.so.zst" + ))))?, + dl_openable: false, + }, + ], + Target::Wasip3(multithreading) => vec![ + Library { + name: "libcomponentize_py_runtime.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip3/libcomponentize_py_runtime.so.zst" + ))))?, + dl_openable: false, + }, + Library { + name: "libpython3.14.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip3/libpython3.14.so.zst" + ))))?, + dl_openable: false, + }, + Library { + name: "libc.so".into(), + module: zstd::decode_all(Cursor::new(match multithreading { + Multithreading::Enabled => { + include_bytes!(concat!(env!("OUT_DIR"), "/wasip3/threads/libc.so.zst")) + as &[_] + } + Multithreading::Disabled => { + include_bytes!(concat!(env!("OUT_DIR"), "/wasip3/libc.so.zst")) + } + }))?, + dl_openable: false, + }, + Library { + name: "libwasi-emulated-mman.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip3/libwasi-emulated-mman.so.zst" + ))))?, + dl_openable: false, + }, + Library { + name: "libwasi-emulated-process-clocks.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip3/libwasi-emulated-process-clocks.so.zst" + ))))?, + dl_openable: false, + }, + Library { + name: "libwasi-emulated-getpid.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip3/libwasi-emulated-getpid.so.zst" + ))))?, + dl_openable: false, + }, + Library { + name: "libwasi-emulated-signal.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip3/libwasi-emulated-signal.so.zst" + ))))?, + dl_openable: false, + }, + Library { + name: "libc++.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip3/libc++.so.zst" + ))))?, + dl_openable: false, + }, + Library { + name: "libc++abi.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip3/libc++abi.so.zst" + ))))?, + dl_openable: false, + }, + Library { + name: "libunwind.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip3/libunwind.so.zst" + ))))?, + dl_openable: false, + }, + ], + }) +} +pub fn read_libraries(library_path: Vec<(&str, Vec)>) -> Result> { + let mut libraries = Vec::new(); for (index, (path, libs)) in library_path.iter().enumerate() { for library in libs { let path = library @@ -181,7 +275,7 @@ pub fn search_for_libraries_and_configs<'a>( library_path.push((*path, libraries)); } - let libraries = bundle_libraries(library_path)?; + let libraries = read_libraries(library_path)?; // Validate the paths parsed from any componentize-py.toml files discovered // above and match them up with `module_worlds` entries. Note that we use diff --git a/src/python.rs b/src/python.rs index 39d98f1c..3b5d145f 100644 --- a/src/python.rs +++ b/src/python.rs @@ -4,21 +4,21 @@ )] use { - crate::{BindingsGenerator, ComponentGenerator}, + crate::{BindingsGenerator, ComponentGenerator, Target}, pyo3::{ Bound, PyResult, Python, exceptions::PyAssertionError, pybacked::PyBackedStr, types::{PyAnyMethods, PyModule, PyModuleMethods}, }, - std::{ffi::OsString, path::PathBuf}, + std::{ffi::OsString, path::PathBuf, str::FromStr as _}, tokio::runtime::Runtime, }; #[allow(clippy::too_many_arguments)] #[pyo3::pyfunction] #[pyo3(name = "componentize")] -#[pyo3(signature = (wit_path, worlds, features, all_features, bindings_module, python_path, module_worlds, app_name, output_path, stub_wasi, import_interface_names, export_interface_names, full_names = None, intersect_world = None, world_module = None, registry_config = None, no_default_registries = false))] +#[pyo3(signature = (wit_path, worlds, features, all_features, bindings_module, python_path, module_worlds, app_name, output_path, stub_wasi, import_interface_names, export_interface_names, full_names = None, intersect_world = None, world_module = None, registry_config = None, no_default_registries = false, target = None))] fn python_componentize( wit_path: Vec, worlds: Vec, @@ -37,6 +37,7 @@ fn python_componentize( world_module: Option<&str>, registry_config: Option, no_default_registries: bool, + target: Option<&str>, ) -> PyResult<()> { let bindings_module = crate::resolve_deprecated( bindings_module.map(str::to_owned), @@ -76,6 +77,7 @@ fn python_componentize( intersect_world, registry_config: registry_config.as_deref(), default_registries: !no_default_registries, + target: target.map(Target::from_str).transpose()?, } .generate(), ) diff --git a/src/test.rs b/src/test.rs index d5e88df4..510834de 100644 --- a/src/test.rs +++ b/src/test.rs @@ -39,6 +39,7 @@ static ENGINE: Lazy = Lazy::new(|| { config.wasm_component_model(true); config.wasm_component_model_async(true); config.wasm_component_model_map(true); + config.wasm_component_model_threading(true); Engine::new(&config).unwrap() }); @@ -87,6 +88,7 @@ async fn make_component( intersect_world, registry_config: None, default_registries: true, + target: None, } .generate() .await?; diff --git a/src/test/echoes.rs b/src/test/echoes.rs index 60a54fc8..9999ff11 100644 --- a/src/test/echoes.rs +++ b/src/test/echoes.rs @@ -204,6 +204,7 @@ impl super::Host for Host { fn add_to_linker(linker: &mut Linker) -> Result<()> { wasmtime_wasi::p2::add_to_linker_async(&mut *linker)?; + wasmtime_wasi::p3::add_to_linker(&mut *linker)?; componentize_py::test::echoes::add_to_linker::<_, HasSelf<_>>(linker, |ctx| ctx)?; Ok(()) } diff --git a/src/test/python_source/app.py b/src/test/python_source/app.py index 6d4a74fd..05ba5dff 100644 --- a/src/test/python_source/app.py +++ b/src/test/python_source/app.py @@ -259,7 +259,7 @@ def test_resource_alias(self, things: List[imports.resource_alias1.Thing]) -> Li def add(self, a: imports.resource_floats.Float, b: imports.resource_floats.Float) -> imports.resource_floats.Float: return imports.resource_floats.Float(a.get() + b.get() + 5) - def read_file(self, path: str) -> bytes: + async def read_file(self, path: str) -> bytes: try: with open(file=path, mode="rb") as f: return f.read() diff --git a/src/test/tests.rs b/src/test/tests.rs index 6929fa29..d4371494 100644 --- a/src/test/tests.rs +++ b/src/test/tests.rs @@ -133,6 +133,7 @@ impl super::Host for Host { fn add_to_linker(linker: &mut Linker) -> Result<()> { wasmtime_wasi::p2::add_to_linker_async(linker)?; + wasmtime_wasi::p3::add_to_linker(linker)?; Tests::add_to_linker::<_, HasSelf<_>>(linker, |ctx| ctx)?; foo_sdk::FooWorldUnion::add_to_linker::<_, HasSelf<_>>(linker, |ctx| ctx)?; Ok(()) @@ -1065,6 +1066,7 @@ fn componentize_app( intersect_world: None, registry_config: None, default_registries: true, + target: None, } .generate(), ) @@ -1208,14 +1210,18 @@ fn filesystem() -> Result<()> { TESTER.test_with_wasi::(wasi, |world, store, runtime| { runtime.block_on(async { - let value = world - .call_read_file(store, filename) - .await? - .map_err(|s| anyhow!("{s}"))?; + store + .run_concurrent(async |store| { + let value = world + .call_read_file(store, filename.to_string()) + .await? + .map_err(|s| anyhow!("{s}"))?; - assert_eq!(&value, message); + assert_eq!(&value, message); - Ok(()) + Ok(()) + }) + .await? }) }) } @@ -2034,14 +2040,14 @@ fn guest_instance_shared_across_functions() -> Result<()> { "#, ) .replace( - r#" def read_file(self, path: str) -> bytes: + r#" async def read_file(self, path: str) -> bytes: try: with open(file=path, mode="rb") as f: return f.read() except: raise Err(traceback.format_exc()) "#, - r#" def read_file(self, path: str) -> bytes: + r#" async def read_file(self, path: str) -> bytes: return self.stash "#, ) @@ -2053,14 +2059,18 @@ fn guest_instance_shared_across_functions() -> Result<()> { .call_test_resource_borrow_import(&mut *store, 42) .await?; - let value = world - .call_read_file(&mut *store, "unused") - .await? - .map_err(|s| anyhow!("{s}"))?; + store + .run_concurrent(async |store| { + let value = world + .call_read_file(store, "unused".into()) + .await? + .map_err(|s| anyhow!("{s}"))?; - assert_eq!(b"42".as_slice(), &value); + assert_eq!(b"42".as_slice(), &value); - Ok(()) + Ok(()) + }) + .await? }) }) } diff --git a/src/test/wit/tests.wit b/src/test/wit/tests.wit index cfe233f7..c913dd0e 100644 --- a/src/test/wit/tests.wit +++ b/src/test/wit/tests.wit @@ -234,7 +234,7 @@ world tests { export add: func(a: borrow, b: borrow) -> own; - export read-file: func(path: string) -> result, string>; + export read-file: async func(path: string) -> result, string>; export test-refcounts: func(); diff --git a/test-generator/src/lib.rs b/test-generator/src/lib.rs index 57d6e6c2..a265029c 100644 --- a/test-generator/src/lib.rs +++ b/test-generator/src/lib.rs @@ -60,10 +60,28 @@ enum Type { }, Tuple(Vec), List(Box), + Map(Box, Box), +} + +fn any_key_type() -> impl Strategy { + (0..11).prop_flat_map(move |index| match index { + 0 => Just(Type::Bool).boxed(), + 1 => Just(Type::U8).boxed(), + 2 => Just(Type::S8).boxed(), + 3 => Just(Type::U16).boxed(), + 4 => Just(Type::S16).boxed(), + 5 => Just(Type::U32).boxed(), + 6 => Just(Type::S32).boxed(), + 7 => Just(Type::U64).boxed(), + 8 => Just(Type::S64).boxed(), + 9 => Just(Type::Char).boxed(), + 10 => Just(Type::String).boxed(), + _ => unreachable!(), + }) } fn any_type(max_size: usize, next_id: Rc>) -> impl Strategy { - (0..21).prop_flat_map(move |index| match index { + (0..22).prop_flat_map(move |index| match index { 0 => Just(Type::Bool).boxed(), 1 => Just(Type::U8).boxed(), 2 => Just(Type::S8).boxed(), @@ -142,6 +160,9 @@ fn any_type(max_size: usize, next_id: Rc>) -> impl Strategy any_type(max_size, next_id.clone()) .prop_map(|ty| Type::List(Box::new(ty))) .boxed(), + 21 => (any_key_type(), any_type(max_size, next_id.clone())) + .prop_map(|(k, v)| Type::Map(Box::new(k), Box::new(v))) + .boxed(), _ => unreachable!(), }) } @@ -271,6 +292,9 @@ fn wit_type_name(wit: &mut String, ty: &Type) -> String { Type::List(ty) => { format!("list<{}>", wit_type_name(wit, ty)) } + Type::Map(k, v) => { + format!("map<{}, {}>", wit_type_name(wit, k), wit_type_name(wit, v)) + } } } @@ -326,6 +350,9 @@ fn rust_type_name(ty: &Type) -> String { Type::List(ty) => { format!("Vec<{}>", rust_type_name(ty)) } + Type::Map(k, v) => { + format!("HashMap<{}, {}>", rust_type_name(k), rust_type_name(v)) + } } } @@ -418,6 +445,12 @@ fn equality(a: &str, b: &str, ty: &Type) -> String { "{a}.len() == {b}.len() && {a}.iter().zip({b}.iter()).all(|(a, b)| {})", equality("a", "b", ty) ), + Type::Map(k, v) => format!( + "{a}.len() == {b}.len() && {a}.iter().collect::>().into_iter()\ + .zip({b}.iter().collect::>().into_iter()).all(|((ak, av), (bk, bv))| {} && {})", + equality("ak", "bk", k), + equality("av", "bv", v) + ), } } @@ -547,6 +580,13 @@ fn strategy(ty: &Type, max_list_size: usize) -> String { strategy(ty, max_list_size / 2) ) } + Type::Map(k, v) => { + format!( + "proptest::collection::hash_map({}, {}, 0..{max_list_size}.max(1))", + strategy(k, max_list_size / 2), + strategy(v, max_list_size / 2) + ) + } } } @@ -814,6 +854,7 @@ use {{ component::{{Instance, InstancePre, Linker, TypedFunc, HasSelf}}, Store, }}, + std::collections::{{HashMap, BTreeMap}}, }}; wasmtime::component::bindgen!({{ @@ -838,6 +879,7 @@ impl super::Host for Host {{ fn add_to_linker(linker: &mut Linker) -> Result<()> {{ wasmtime_wasi::p2::add_to_linker_async(&mut *linker)?; + wasmtime_wasi::p3::add_to_linker(&mut *linker)?; {PREFIX}::add_to_linker::<_, HasSelf<_>>(linker, |ctx| ctx)?; Ok(()) }} diff --git a/tests/bindings.rs b/tests/bindings.rs index c45c2afe..2102e283 100644 --- a/tests/bindings.rs +++ b/tests/bindings.rs @@ -471,6 +471,25 @@ fn lint_cli_bindings() -> anyhow::Result<()> { Ok(()) } +#[test] +fn lint_multithreading_bindings() -> anyhow::Result<()> { + let dir = tempfile::tempdir()?; + fs_extra::copy_items( + &["./examples/multithreading", "./wit"], + dir.path(), + &CopyOptions::new(), + )?; + let path = dir.path().join("multithreading"); + + generate_bindings(&path, "wasi:cli/command@0.3.0")?; + + assert!(predicate::path::is_dir().eval(&path.join("wit"))); + + mypy_check(&path, ["--strict", "-m", "app"]); + + Ok(()) +} + #[test] fn lint_cli_p3_bindings() -> anyhow::Result<()> { let dir = tempfile::tempdir()?; diff --git a/tests/componentize.rs b/tests/componentize.rs index 22d792f0..e89adca4 100644 --- a/tests/componentize.rs +++ b/tests/componentize.rs @@ -131,15 +131,38 @@ fn remote_intersector_preserves_local_default_target() -> anyhow::Result<()> { #[test] fn cli_example() -> anyhow::Result<()> { - test_cli_example("cli", "wasi:cli/command@0.2.0") + test_cli_example("cli", "wasi:cli/command@0.2.0", "Hello, world!\n", false) } #[test] fn cli_p3_example() -> anyhow::Result<()> { - test_cli_example("cli-p3", "wasi:cli/command@0.3.0") + test_cli_example("cli-p3", "wasi:cli/command@0.3.0", "Hello, world!\n", false) } -fn test_cli_example(name: &str, world: &str) -> anyhow::Result<()> { +#[test] +fn multithreading_example() -> anyhow::Result<()> { + test_cli_example( + "multithreading", + "wasi:cli/command@0.3.0", + r#"thread `a` started +thread `b` started +thread `c` started +started all threads +thread `a` finished +thread `b` finished +thread `c` finished +joined all threads +"#, + true, + ) +} + +fn test_cli_example( + name: &str, + world: &str, + stdout: &'static str, + threading: bool, +) -> anyhow::Result<()> { let dir = tempfile::tempdir()?; fs_extra::copy_items( &[format!("./examples/{name}").as_str(), "./wit"], @@ -148,28 +171,32 @@ fn test_cli_example(name: &str, world: &str) -> anyhow::Result<()> { )?; let path = dir.path().join(name); - cargo::cargo_bin_cmd!("componentize-py") - .current_dir(&path) - .args([ - "-d", - "../wit", - "-w", - world, - "componentize", - "app", - "-o", - "cli.wasm", - ]) + let mut command = cargo::cargo_bin_cmd!("componentize-py"); + command.current_dir(&path).args([ + "-d", + "../wit", + "-w", + world, + "componentize", + "app", + "-o", + "cli.wasm", + ]); + if threading { + command.arg("--target=wasm32-wasip3-threads"); + } + command .assert() .success() .stdout("Component built successfully\n"); - Command::new("wasmtime") - .current_dir(&path) - .args(["run", "-Sp3", "-Wcomponent-model-async", "cli.wasm"]) - .assert() - .success() - .stdout("Hello, world!\n"); + let mut command = Command::new("wasmtime"); + command.current_dir(&path).arg("run"); + if threading { + command.arg("-Wcomponent-model-threading"); + } + command.arg("cli.wasm"); + command.assert().success().stdout(stdout); Ok(()) } @@ -214,8 +241,7 @@ fn test_http_example(name: &str, world: &str, port: u16) -> anyhow::Result<()> { .args([ "serve", &format!("--addr=0.0.0.0:{port}"), - "-Sp3,common", - "-Wcomponent-model-async", + "-Scommon", "http.wasm", ]) .spawn()?; @@ -386,8 +412,7 @@ fn test_tcp_example(name: &str, world: &str) -> anyhow::Result<()> { .current_dir(&path) .args([ "run", - "-Sp3,inherit-network", - "-Wcomponent-model-async", + "-Sinherit-network", "tcp.wasm", &format!("127.0.0.1:{port}"), ]) diff --git a/wit/init.wit b/wit/init.wit index 7cc2f34f..8f0a97f0 100644 --- a/wit/init.wit +++ b/wit/init.wit @@ -98,6 +98,6 @@ world init { results: list, } - init: func(app-name: string, symbols: symbols, stub-wasi: bool) -> result<_, string>; + init: async func(app-name: string, symbols: symbols, stub-wasi: bool) -> result<_, string>; } }