From fa10f6949076edb67c25bff4079318626231087a Mon Sep 17 00:00:00 2001 From: William Candillon Date: Tue, 19 May 2026 10:03:41 +0200 Subject: [PATCH 01/46] Initial implementation --- apps/example/src/App.tsx | 5 + apps/example/src/Home.tsx | 4 + apps/example/src/Route.ts | 1 + .../SharedTextureMemory.tsx | 198 ++++++++++++++++++ apps/example/src/SharedTextureMemory/index.ts | 1 + packages/webgpu/android/CMakeLists.txt | 1 + .../android/cpp/AndroidPlatformContext.h | 17 ++ packages/webgpu/apple/ApplePlatformContext.h | 5 + packages/webgpu/apple/ApplePlatformContext.mm | 129 ++++++++++++ packages/webgpu/cpp/rnwgpu/PlatformContext.h | 27 +++ .../webgpu/cpp/rnwgpu/RNWebGPUManager.cpp | 4 + packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp | 39 ++++ packages/webgpu/cpp/rnwgpu/api/GPUDevice.h | 6 + .../cpp/rnwgpu/api/GPUSharedTextureMemory.cpp | 56 +++++ .../cpp/rnwgpu/api/GPUSharedTextureMemory.h | 71 +++++++ packages/webgpu/cpp/rnwgpu/api/RNWebGPU.h | 18 ++ packages/webgpu/cpp/rnwgpu/api/VideoFrame.h | 62 ++++++ .../GPUSharedTextureMemoryDescriptor.h | 62 ++++++ packages/webgpu/src/Canvas.tsx | 5 + packages/webgpu/src/index.tsx | 16 ++ packages/webgpu/src/types.ts | 39 ++++ 21 files changed, 766 insertions(+) create mode 100644 apps/example/src/SharedTextureMemory/SharedTextureMemory.tsx create mode 100644 apps/example/src/SharedTextureMemory/index.ts create mode 100644 packages/webgpu/cpp/rnwgpu/api/GPUSharedTextureMemory.cpp create mode 100644 packages/webgpu/cpp/rnwgpu/api/GPUSharedTextureMemory.h create mode 100644 packages/webgpu/cpp/rnwgpu/api/VideoFrame.h create mode 100644 packages/webgpu/cpp/rnwgpu/api/descriptors/GPUSharedTextureMemoryDescriptor.h diff --git a/apps/example/src/App.tsx b/apps/example/src/App.tsx index 667f87164..0b26ab049 100644 --- a/apps/example/src/App.tsx +++ b/apps/example/src/App.tsx @@ -36,6 +36,7 @@ import { Reanimated } from "./Reanimated"; import { AsyncStarvation } from "./Diagnostics/AsyncStarvation"; import { DeviceLostHang } from "./Diagnostics/DeviceLostHang"; import { StorageBufferVertices } from "./StorageBufferVertices"; +import { SharedTextureMemory } from "./SharedTextureMemory"; // The two lines below are needed by three.js import "fast-text-encoding"; @@ -97,6 +98,10 @@ function App() { name="StorageBufferVertices" component={StorageBufferVertices} /> + diff --git a/apps/example/src/Home.tsx b/apps/example/src/Home.tsx index 9272dfec9..2db838041 100644 --- a/apps/example/src/Home.tsx +++ b/apps/example/src/Home.tsx @@ -127,6 +127,10 @@ export const examples = [ screen: "StorageBufferVertices", title: "đź’ľ Storage Buffer Vertices", }, + { + screen: "SharedTextureMemory", + title: "🎞️ Shared Texture Memory", + }, ]; const styles = StyleSheet.create({ diff --git a/apps/example/src/Route.ts b/apps/example/src/Route.ts index 152923e1e..1f2dbf187 100644 --- a/apps/example/src/Route.ts +++ b/apps/example/src/Route.ts @@ -29,4 +29,5 @@ export type Routes = { AsyncStarvation: undefined; DeviceLostHang: undefined; StorageBufferVertices: undefined; + SharedTextureMemory: undefined; }; diff --git a/apps/example/src/SharedTextureMemory/SharedTextureMemory.tsx b/apps/example/src/SharedTextureMemory/SharedTextureMemory.tsx new file mode 100644 index 000000000..0f3fc0a2f --- /dev/null +++ b/apps/example/src/SharedTextureMemory/SharedTextureMemory.tsx @@ -0,0 +1,198 @@ +import React, { useEffect, useRef, useState } from "react"; +import { PixelRatio, Platform, StyleSheet, Text, View } from "react-native"; +import { + Canvas, + useCanvasRef, + useDevice, + type NativeCanvas, +} from "react-native-wgpu"; + +const SHADER = /* wgsl */ ` +struct VsOut { + @builtin(position) position: vec4f, + @location(0) uv: vec2f, +}; + +@vertex +fn vs_main(@builtin(vertex_index) vid: u32) -> VsOut { + // Full-screen triangle. + var positions = array( + vec2f(-1.0, -3.0), + vec2f(-1.0, 1.0), + vec2f( 3.0, 1.0), + ); + var uvs = array( + vec2f(0.0, 2.0), + vec2f(0.0, 0.0), + vec2f(2.0, 0.0), + ); + var out: VsOut; + out.position = vec4f(positions[vid], 0.0, 1.0); + out.uv = uvs[vid]; + return out; +} + +@group(0) @binding(0) var srcTex: texture_2d; +@group(0) @binding(1) var srcSampler: sampler; + +@fragment +fn fs_main(in: VsOut) -> @location(0) vec4f { + return textureSample(srcTex, srcSampler, in.uv); +} +`; + +const REQUIRED_FEATURE = + Platform.OS === "ios" + ? "shared-texture-memory-iosurface" + : "shared-texture-memory-ahardware-buffer"; + +export const SharedTextureMemory = () => { + const ref = useCanvasRef(); + const [error, setError] = useState(null); + const rafRef = useRef(null); + + // Request the shared-memory feature when constructing the device so the + // shared-texture-memory* extension is enabled. + const { device, adapter } = useDevice(undefined, { + // Cast: GPUFeatureName in @webgpu/types doesn't include the Dawn-specific + // extension name yet, but Dawn accepts it. + requiredFeatures: [REQUIRED_FEATURE as GPUFeatureName], + }); + + useEffect(() => { + if (!device) { + return; + } + if (!device.features.has(REQUIRED_FEATURE)) { + setError( + `Device is missing the '${REQUIRED_FEATURE}' feature (adapter supports: ${ + adapter + ? [...adapter.features] + .filter((f) => f.toString().startsWith("shared-")) + .join(", ") || "none" + : "n/a" + })`, + ); + return; + } + + const context = ref.current?.getContext("webgpu"); + if (!context) { + return; + } + const canvas = context.canvas as unknown as NativeCanvas; + canvas.width = canvas.clientWidth * PixelRatio.get(); + canvas.height = canvas.clientHeight * PixelRatio.get(); + const presentationFormat = navigator.gpu.getPreferredCanvasFormat(); + context.configure({ + device, + format: presentationFormat, + alphaMode: "premultiplied", + }); + + // 1. Acquire a native, GPU-shareable surface. In production this would + // come from a camera frame processor or video decoder. The test helper + // synthesizes a 256x256 RGB-gradient pattern in an IOSurface. + const frame = RNWebGPU.createTestVideoFrame(256, 256); + + // 2. Import the raw native handle into a SharedTextureMemory. + const sharedMemory = device.importSharedTextureMemory({ + handle: frame.handle, + label: "video-frame-shared-memory", + }); + + // 3. Create a regular GPUTexture that aliases the surface's pixels. + // No descriptor needed: the format/size are inferred from the surface. + const texture = sharedMemory.createTexture(); + + // 4. beginAccess declares that we're about to read or write the texture on + // the GPU timeline. `initialized: true` means "the surface already has + // meaningful pixels", which is correct for an incoming video frame. + // + // Because this example owns a *static* IOSurface (no external producer + // is writing new pixels between frames), we keep one access window open + // for the lifetime of the texture and call endAccess only on unmount. + // + // For a live camera or video feed, you'd instead wrap each frame: + // beginAccess(tex, true) -> submit -> endAccess(tex) + // around every render to hand ownership back to the producer. That's + // also where fence support (not yet wired through this binding) becomes + // important to avoid races with the producer. + if (!sharedMemory.beginAccess(texture, true)) { + setError("beginAccess() failed"); + return; + } + + const module = device.createShaderModule({ code: SHADER }); + const pipeline = device.createRenderPipeline({ + layout: "auto", + vertex: { module, entryPoint: "vs_main" }, + fragment: { + module, + entryPoint: "fs_main", + targets: [{ format: presentationFormat }], + }, + primitive: { topology: "triangle-list" }, + }); + const sampler = device.createSampler({ + magFilter: "linear", + minFilter: "linear", + }); + const bindGroup = device.createBindGroup({ + layout: pipeline.getBindGroupLayout(0), + entries: [ + { binding: 0, resource: texture.createView() }, + { binding: 1, resource: sampler }, + ], + }); + + const render = () => { + const encoder = device.createCommandEncoder(); + const pass = encoder.beginRenderPass({ + colorAttachments: [ + { + view: context.getCurrentTexture().createView(), + clearValue: { r: 0, g: 0, b: 0, a: 1 }, + loadOp: "clear", + storeOp: "store", + }, + ], + }); + pass.setPipeline(pipeline); + pass.setBindGroup(0, bindGroup); + pass.draw(3); + pass.end(); + device.queue.submit([encoder.finish()]); + context.present(); + rafRef.current = requestAnimationFrame(render); + }; + rafRef.current = requestAnimationFrame(render); + + return () => { + if (rafRef.current !== null) { + cancelAnimationFrame(rafRef.current); + } + sharedMemory.endAccess(texture); + texture.destroy(); + frame.release(); + }; + }, [device, adapter, ref]); + + if (error) { + return ( + + {error} + + ); + } + return ( + + + + ); +}; + +const styles = StyleSheet.create({ + errorContainer: { flex: 1, padding: 16, justifyContent: "center" }, + errorText: { color: "red", fontSize: 14 }, +}); diff --git a/apps/example/src/SharedTextureMemory/index.ts b/apps/example/src/SharedTextureMemory/index.ts new file mode 100644 index 000000000..235958b49 --- /dev/null +++ b/apps/example/src/SharedTextureMemory/index.ts @@ -0,0 +1 @@ +export * from "./SharedTextureMemory"; diff --git a/packages/webgpu/android/CMakeLists.txt b/packages/webgpu/android/CMakeLists.txt index 6e7488b87..fcab0baa1 100644 --- a/packages/webgpu/android/CMakeLists.txt +++ b/packages/webgpu/android/CMakeLists.txt @@ -37,6 +37,7 @@ add_library(${PACKAGE_NAME} SHARED ../cpp/rnwgpu/api/GPUCommandEncoder.cpp ../cpp/rnwgpu/api/GPUQuerySet.cpp ../cpp/rnwgpu/api/GPUTexture.cpp + ../cpp/rnwgpu/api/GPUSharedTextureMemory.cpp ../cpp/rnwgpu/api/GPURenderBundleEncoder.cpp ../cpp/rnwgpu/api/GPURenderPassEncoder.cpp ../cpp/rnwgpu/api/GPURenderPipeline.cpp diff --git a/packages/webgpu/android/cpp/AndroidPlatformContext.h b/packages/webgpu/android/cpp/AndroidPlatformContext.h index fe036bc7b..4b8e43b94 100644 --- a/packages/webgpu/android/cpp/AndroidPlatformContext.h +++ b/packages/webgpu/android/cpp/AndroidPlatformContext.h @@ -202,6 +202,23 @@ class AndroidPlatformContext : public PlatformContext { } }).detach(); } + + VideoFrameHandle loadVideoFrame(const std::string & /*path*/) override { + // TODO: implement using MediaExtractor + MediaCodec to decode the first + // frame into an AHardwareBuffer-backed Image (Android API 26+). + throw std::runtime_error( + "loadVideoFrame is not yet implemented on Android. Pass an " + "AHardwareBuffer pointer obtained elsewhere (e.g. from " + "react-native-vision-camera) directly to " + "device.importSharedTextureMemory."); + } + + VideoFrameHandle createTestVideoFrame(uint32_t /*width*/, + uint32_t /*height*/) override { + // TODO: implement using AHardwareBuffer_allocate (Android API 26+). + throw std::runtime_error( + "createTestVideoFrame is not yet implemented on Android."); + } }; } // namespace rnwgpu diff --git a/packages/webgpu/apple/ApplePlatformContext.h b/packages/webgpu/apple/ApplePlatformContext.h index 86e80b807..77c8a2a21 100644 --- a/packages/webgpu/apple/ApplePlatformContext.h +++ b/packages/webgpu/apple/ApplePlatformContext.h @@ -26,6 +26,11 @@ class ApplePlatformContext : public PlatformContext { void createImageBitmapFromDataAsync( std::span data, std::function onSuccess, std::function onError) override; + + VideoFrameHandle loadVideoFrame(const std::string &path) override; + + VideoFrameHandle createTestVideoFrame(uint32_t width, + uint32_t height) override; }; } // namespace rnwgpu diff --git a/packages/webgpu/apple/ApplePlatformContext.mm b/packages/webgpu/apple/ApplePlatformContext.mm index 8907511a0..ffad62a1e 100644 --- a/packages/webgpu/apple/ApplePlatformContext.mm +++ b/packages/webgpu/apple/ApplePlatformContext.mm @@ -2,6 +2,8 @@ #include +#import +#import #import #import #import @@ -154,4 +156,131 @@ void checkIfUsingSimulatorWithAPIValidation() { }); } +VideoFrameHandle +ApplePlatformContext::loadVideoFrame(const std::string &path) { + NSString *nsPath = [NSString stringWithUTF8String:path.c_str()]; + NSURL *url = [nsPath hasPrefix:@"file://"] + ? [NSURL URLWithString:nsPath] + : [NSURL fileURLWithPath:nsPath]; + AVURLAsset *asset = [AVURLAsset assetWithURL:url]; + + NSArray *videoTracks = + [asset tracksWithMediaType:AVMediaTypeVideo]; + if (videoTracks.count == 0) { + throw std::runtime_error("loadVideoFrame: no video track in file"); + } + AVAssetTrack *videoTrack = videoTracks.firstObject; + + NSError *error = nil; + AVAssetReader *reader = [AVAssetReader assetReaderWithAsset:asset + error:&error]; + if (error || !reader) { + throw std::runtime_error( + std::string("loadVideoFrame: AVAssetReader init failed: ") + + [[error localizedDescription] UTF8String]); + } + + NSDictionary *outputSettings = @{ + (NSString *)kCVPixelBufferPixelFormatTypeKey : + @(kCVPixelFormatType_32BGRA), + (NSString *)kCVPixelBufferIOSurfacePropertiesKey : @{}, + (NSString *)kCVPixelBufferMetalCompatibilityKey : @YES, + }; + AVAssetReaderTrackOutput *output = + [AVAssetReaderTrackOutput assetReaderTrackOutputWithTrack:videoTrack + outputSettings:outputSettings]; + output.alwaysCopiesSampleData = NO; + if (![reader canAddOutput:output]) { + throw std::runtime_error("loadVideoFrame: cannot add output"); + } + [reader addOutput:output]; + + if (![reader startReading]) { + throw std::runtime_error( + std::string("loadVideoFrame: startReading failed: ") + + [[reader.error localizedDescription] UTF8String]); + } + + CMSampleBufferRef sampleBuffer = [output copyNextSampleBuffer]; + if (!sampleBuffer) { + throw std::runtime_error("loadVideoFrame: no sample buffer"); + } + + CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); + if (!pixelBuffer) { + CFRelease(sampleBuffer); + throw std::runtime_error("loadVideoFrame: no pixel buffer"); + } + + IOSurfaceRef ioSurface = CVPixelBufferGetIOSurface(pixelBuffer); + if (!ioSurface) { + CFRelease(sampleBuffer); + throw std::runtime_error( + "loadVideoFrame: pixel buffer is not IOSurface-backed"); + } + + // Retain the IOSurface so it survives past the sample buffer's lifetime. + CFRetain(ioSurface); + + VideoFrameHandle handle; + handle.handle = (void *)ioSurface; + handle.width = static_cast(CVPixelBufferGetWidth(pixelBuffer)); + handle.height = static_cast(CVPixelBufferGetHeight(pixelBuffer)); + handle.deleter = [ioSurface]() { CFRelease(ioSurface); }; + + CFRelease(sampleBuffer); + [reader cancelReading]; + + return handle; +} + +VideoFrameHandle +ApplePlatformContext::createTestVideoFrame(uint32_t width, uint32_t height) { + NSDictionary *attrs = @{ + (NSString *)kCVPixelBufferIOSurfacePropertiesKey : @{}, + (NSString *)kCVPixelBufferMetalCompatibilityKey : @YES, + }; + CVPixelBufferRef pixelBuffer = NULL; + CVReturn err = CVPixelBufferCreate( + kCFAllocatorDefault, width, height, kCVPixelFormatType_32BGRA, + (__bridge CFDictionaryRef)attrs, &pixelBuffer); + if (err != kCVReturnSuccess || !pixelBuffer) { + throw std::runtime_error("createTestVideoFrame: CVPixelBufferCreate " + "failed"); + } + + CVPixelBufferLockBaseAddress(pixelBuffer, 0); + uint8_t *base = + static_cast(CVPixelBufferGetBaseAddress(pixelBuffer)); + size_t rowBytes = CVPixelBufferGetBytesPerRow(pixelBuffer); + for (uint32_t y = 0; y < height; ++y) { + uint8_t *row = base + y * rowBytes; + for (uint32_t x = 0; x < width; ++x) { + // RGB gradient + diagonal stripes, in BGRA byte order. + uint8_t r = static_cast((x * 255) / std::max(width - 1, 1u)); + uint8_t g = static_cast((y * 255) / std::max(height - 1, 1u)); + uint8_t b = static_cast(((x + y) & 0x20) ? 220 : 30); + row[x * 4 + 0] = b; + row[x * 4 + 1] = g; + row[x * 4 + 2] = r; + row[x * 4 + 3] = 0xFF; + } + } + CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); + + IOSurfaceRef ioSurface = CVPixelBufferGetIOSurface(pixelBuffer); + if (!ioSurface) { + CFRelease(pixelBuffer); + throw std::runtime_error( + "createTestVideoFrame: pixel buffer is not IOSurface-backed"); + } + + VideoFrameHandle handle; + handle.handle = (void *)ioSurface; + handle.width = width; + handle.height = height; + handle.deleter = [pixelBuffer]() { CFRelease(pixelBuffer); }; + return handle; +} + } // namespace rnwgpu diff --git a/packages/webgpu/cpp/rnwgpu/PlatformContext.h b/packages/webgpu/cpp/rnwgpu/PlatformContext.h index bca6a2608..9743682d4 100644 --- a/packages/webgpu/cpp/rnwgpu/PlatformContext.h +++ b/packages/webgpu/cpp/rnwgpu/PlatformContext.h @@ -17,6 +17,22 @@ struct ImageData { wgpu::TextureFormat format; }; +// A native handle to a video frame that can be imported into a +// GPUSharedTextureMemory. +// +// - handle is an IOSurfaceRef on Apple platforms +// - handle is an AHardwareBuffer* on Android +// +// The deleter is responsible for releasing the underlying backing object +// (CVPixelBuffer / AHardwareBuffer) and must be called exactly once. The +// VideoFrame JS wrapper handles this on destruction. +struct VideoFrameHandle { + void *handle = nullptr; + uint32_t width = 0; + uint32_t height = 0; + std::function deleter; +}; + class PlatformContext { public: PlatformContext() = default; @@ -41,6 +57,17 @@ class PlatformContext { createImageBitmapFromDataAsync(std::span data, std::function onSuccess, std::function onError) = 0; + + // Decode the first video frame of `path` (a local file path) into a + // native, GPU-shareable surface. Used by the SharedTextureMemory example; + // not intended as a long-term media-loading API. + virtual VideoFrameHandle loadVideoFrame(const std::string &path) = 0; + + // Create a synthetic, GPU-shareable IOSurface/AHardwareBuffer filled with a + // generated test pattern. Avoids the need to bundle a video asset for the + // SharedTextureMemory example. + virtual VideoFrameHandle createTestVideoFrame(uint32_t width, + uint32_t height) = 0; }; } // namespace rnwgpu diff --git a/packages/webgpu/cpp/rnwgpu/RNWebGPUManager.cpp b/packages/webgpu/cpp/rnwgpu/RNWebGPUManager.cpp index 5a2decc09..0cd62bbb9 100644 --- a/packages/webgpu/cpp/rnwgpu/RNWebGPUManager.cpp +++ b/packages/webgpu/cpp/rnwgpu/RNWebGPUManager.cpp @@ -31,12 +31,14 @@ #include "GPURenderPassEncoder.h" #include "GPURenderPipeline.h" #include "GPUSampler.h" +#include "GPUSharedTextureMemory.h" #include "GPUShaderModule.h" #include "GPUSupportedLimits.h" #include "GPUTexture.h" #include "GPUTextureView.h" #include "GPUUncapturedErrorEvent.h" #include "GPUValidationError.h" +#include "VideoFrame.h" // Enums #include "GPUBufferUsage.h" @@ -97,10 +99,12 @@ RNWebGPUManager::RNWebGPUManager( GPURenderPassEncoder::installConstructor(*_jsRuntime); GPURenderPipeline::installConstructor(*_jsRuntime); GPUSampler::installConstructor(*_jsRuntime); + GPUSharedTextureMemory::installConstructor(*_jsRuntime); GPUShaderModule::installConstructor(*_jsRuntime); GPUSupportedLimits::installConstructor(*_jsRuntime); GPUTexture::installConstructor(*_jsRuntime); GPUTextureView::installConstructor(*_jsRuntime); + VideoFrame::installConstructor(*_jsRuntime); // Install constant objects as plain JS objects with own properties _jsRuntime->global().setProperty(*_jsRuntime, "GPUBufferUsage", diff --git a/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp b/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp index 909c4555a..0837da3e9 100644 --- a/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp +++ b/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp @@ -238,6 +238,45 @@ std::shared_ptr GPUDevice::importExternalTexture( "GPUDevice::importExternalTexture(): Not implemented"); } +std::shared_ptr GPUDevice::importSharedTextureMemory( + std::shared_ptr descriptor) { + if (!descriptor || descriptor->handle == nullptr) { + throw std::runtime_error("GPUDevice::importSharedTextureMemory(): handle " + "must be a non-null native pointer"); + } + + wgpu::SharedTextureMemoryDescriptor desc{}; + std::string label = descriptor->label.value_or(""); + if (!label.empty()) { + desc.label = wgpu::StringView(label.c_str(), label.size()); + } + +#if defined(__APPLE__) + wgpu::SharedTextureMemoryIOSurfaceDescriptor platformDesc{}; + platformDesc.ioSurface = descriptor->handle; + platformDesc.allowStorageBinding = true; + desc.nextInChain = &platformDesc; +#elif defined(__ANDROID__) + wgpu::SharedTextureMemoryAHardwareBufferDescriptor platformDesc{}; + platformDesc.handle = descriptor->handle; + desc.nextInChain = &platformDesc; +#else + throw std::runtime_error( + "GPUDevice::importSharedTextureMemory(): unsupported platform"); +#endif + + auto memory = _instance.ImportSharedTextureMemory(&desc); + if (memory == nullptr) { + throw std::runtime_error("GPUDevice::importSharedTextureMemory(): " + "ImportSharedTextureMemory returned null - is the " + "'shared-texture-memory-iosurface' (Apple) or " + "'shared-texture-memory-ahardware-buffer' " + "(Android) feature enabled on the device?"); + } + return std::make_shared(std::move(memory), + std::move(label)); +} + async::AsyncTaskHandle GPUDevice::createComputePipelineAsync( std::shared_ptr descriptor) { wgpu::ComputePipelineDescriptor desc{}; diff --git a/packages/webgpu/cpp/rnwgpu/api/GPUDevice.h b/packages/webgpu/cpp/rnwgpu/api/GPUDevice.h index 9b0681c2f..2ab1ddd14 100644 --- a/packages/webgpu/cpp/rnwgpu/api/GPUDevice.h +++ b/packages/webgpu/cpp/rnwgpu/api/GPUDevice.h @@ -45,6 +45,8 @@ #include "GPURenderPipelineDescriptor.h" #include "GPUSampler.h" #include "GPUSamplerDescriptor.h" +#include "GPUSharedTextureMemory.h" +#include "GPUSharedTextureMemoryDescriptor.h" #include "GPUShaderModule.h" #include "GPUShaderModuleDescriptor.h" #include "GPUSupportedLimits.h" @@ -116,6 +118,8 @@ class GPUDevice : public NativeObject { std::optional> descriptor); std::shared_ptr importExternalTexture( std::shared_ptr descriptor); + std::shared_ptr importSharedTextureMemory( + std::shared_ptr descriptor); std::shared_ptr createBindGroupLayout( std::shared_ptr descriptor); std::shared_ptr @@ -169,6 +173,8 @@ class GPUDevice : public NativeObject { &GPUDevice::createSampler); installMethod(runtime, prototype, "importExternalTexture", &GPUDevice::importExternalTexture); + installMethod(runtime, prototype, "importSharedTextureMemory", + &GPUDevice::importSharedTextureMemory); installMethod(runtime, prototype, "createBindGroupLayout", &GPUDevice::createBindGroupLayout); installMethod(runtime, prototype, "createPipelineLayout", diff --git a/packages/webgpu/cpp/rnwgpu/api/GPUSharedTextureMemory.cpp b/packages/webgpu/cpp/rnwgpu/api/GPUSharedTextureMemory.cpp new file mode 100644 index 000000000..bbbd27d82 --- /dev/null +++ b/packages/webgpu/cpp/rnwgpu/api/GPUSharedTextureMemory.cpp @@ -0,0 +1,56 @@ +#include "GPUSharedTextureMemory.h" + +#include +#include +#include + +#include "Convertors.h" + +namespace rnwgpu { + +std::shared_ptr GPUSharedTextureMemory::createTexture( + std::optional> descriptor) { + if (!descriptor.has_value() || descriptor.value() == nullptr) { + auto texture = _instance.CreateTexture(); + return std::make_shared(texture, ""); + } + + wgpu::TextureDescriptor desc{}; + Convertor conv; + if (!conv(desc, descriptor.value())) { + throw std::runtime_error( + "GPUSharedTextureMemory::createTexture(): Error with " + "GPUTextureDescriptor"); + } + auto texture = _instance.CreateTexture(&desc); + return std::make_shared(texture, + descriptor.value()->label.value_or("")); +} + +bool GPUSharedTextureMemory::beginAccess(std::shared_ptr texture, + bool initialized) { + if (!texture) { + throw std::runtime_error( + "GPUSharedTextureMemory::beginAccess(): texture is null"); + } + wgpu::SharedTextureMemoryBeginAccessDescriptor desc{}; + desc.initialized = initialized; + desc.concurrentRead = false; + desc.fenceCount = 0; + desc.fences = nullptr; + desc.signaledValues = nullptr; + auto status = _instance.BeginAccess(texture->get(), &desc); + return static_cast(status); +} + +bool GPUSharedTextureMemory::endAccess(std::shared_ptr texture) { + if (!texture) { + throw std::runtime_error( + "GPUSharedTextureMemory::endAccess(): texture is null"); + } + wgpu::SharedTextureMemoryEndAccessState state{}; + auto status = _instance.EndAccess(texture->get(), &state); + return static_cast(status); +} + +} // namespace rnwgpu diff --git a/packages/webgpu/cpp/rnwgpu/api/GPUSharedTextureMemory.h b/packages/webgpu/cpp/rnwgpu/api/GPUSharedTextureMemory.h new file mode 100644 index 000000000..02b5f7c62 --- /dev/null +++ b/packages/webgpu/cpp/rnwgpu/api/GPUSharedTextureMemory.h @@ -0,0 +1,71 @@ +#pragma once + +#include +#include +#include + +#include "NativeObject.h" + +#include "webgpu/webgpu_cpp.h" + +#include "GPUTexture.h" +#include "GPUTextureDescriptor.h" + +namespace rnwgpu { + +namespace jsi = facebook::jsi; + +class GPUSharedTextureMemory : public NativeObject { +public: + static constexpr const char *CLASS_NAME = "GPUSharedTextureMemory"; + + explicit GPUSharedTextureMemory(wgpu::SharedTextureMemory instance, + std::string label) + : NativeObject(CLASS_NAME), _instance(std::move(instance)), + _label(std::move(label)) {} + +public: + std::string getBrand() { return CLASS_NAME; } + + std::shared_ptr + createTexture(std::optional> descriptor); + + // Returns true on success. Marks the shared memory as initialized so the + // texture's content is preserved (or not). Callers that want fence-based + // synchronization should pass fences via beginAccess descriptor (not yet + // exposed - we currently take the implicit/no-fence path that matches the + // most common RN use cases: still images, single-producer video frames). + bool beginAccess(std::shared_ptr texture, bool initialized); + + // Returns true on success. Drops any fences produced by end-access (we do + // not yet surface them to JS). + bool endAccess(std::shared_ptr texture); + + std::string getLabel() { return _label; } + void setLabel(const std::string &label) { + _label = label; + _instance.SetLabel(_label.c_str()); + } + + static void definePrototype(jsi::Runtime &runtime, jsi::Object &prototype) { + installGetter(runtime, prototype, "__brand", + &GPUSharedTextureMemory::getBrand); + installMethod(runtime, prototype, "createTexture", + &GPUSharedTextureMemory::createTexture); + installMethod(runtime, prototype, "beginAccess", + &GPUSharedTextureMemory::beginAccess); + installMethod(runtime, prototype, "endAccess", + &GPUSharedTextureMemory::endAccess); + installGetterSetter(runtime, prototype, "label", + &GPUSharedTextureMemory::getLabel, + &GPUSharedTextureMemory::setLabel); + } + + inline const wgpu::SharedTextureMemory get() { return _instance; } + +private: + wgpu::SharedTextureMemory _instance; + std::string _label; +}; + +} // namespace rnwgpu diff --git a/packages/webgpu/cpp/rnwgpu/api/RNWebGPU.h b/packages/webgpu/cpp/rnwgpu/api/RNWebGPU.h index 78ef5ec7b..7a59f3cbe 100644 --- a/packages/webgpu/cpp/rnwgpu/api/RNWebGPU.h +++ b/packages/webgpu/cpp/rnwgpu/api/RNWebGPU.h @@ -10,6 +10,7 @@ #include "GPUCanvasContext.h" #include "ImageBitmap.h" #include "PlatformContext.h" +#include "VideoFrame.h" #include @@ -168,6 +169,19 @@ class RNWebGPU : public NativeObject { }); } + std::shared_ptr loadVideoFrame(std::string path) { + auto frame = _platformContext->loadVideoFrame(path); + return std::make_shared(frame.handle, frame.width, frame.height, + std::move(frame.deleter)); + } + + std::shared_ptr createTestVideoFrame(double width, double height) { + auto frame = _platformContext->createTestVideoFrame( + static_cast(width), static_cast(height)); + return std::make_shared(frame.handle, frame.width, frame.height, + std::move(frame.deleter)); + } + std::shared_ptr getNativeSurface(int contextId) { auto ®istry = rnwgpu::SurfaceRegistry::getInstance(); auto info = registry.getSurfaceInfo(contextId); @@ -188,6 +202,10 @@ class RNWebGPU : public NativeObject { &RNWebGPU::getNativeSurface); installMethod(runtime, prototype, "MakeWebGPUCanvasContext", &RNWebGPU::MakeWebGPUCanvasContext); + installMethod(runtime, prototype, "loadVideoFrame", + &RNWebGPU::loadVideoFrame); + installMethod(runtime, prototype, "createTestVideoFrame", + &RNWebGPU::createTestVideoFrame); } private: diff --git a/packages/webgpu/cpp/rnwgpu/api/VideoFrame.h b/packages/webgpu/cpp/rnwgpu/api/VideoFrame.h new file mode 100644 index 000000000..af09cd127 --- /dev/null +++ b/packages/webgpu/cpp/rnwgpu/api/VideoFrame.h @@ -0,0 +1,62 @@ +#pragma once + +#include +#include +#include +#include + +#include "JSIConverter.h" +#include "NativeObject.h" + +namespace rnwgpu { + +namespace jsi = facebook::jsi; + +// VideoFrame is a small RAII wrapper around a platform-specific native handle +// (IOSurfaceRef-backed CVPixelBuffer on Apple, AHardwareBuffer on Android). +// It surfaces the raw handle as a BigInt to JS so callers can hand it to +// GPUDevice.importSharedTextureMemory, and owns a deleter so the underlying +// object stays alive until the JS object is GC'd (or release() is called). +class VideoFrame : public NativeObject { +public: + static constexpr const char *CLASS_NAME = "VideoFrame"; + + VideoFrame(void *handle, uint32_t width, uint32_t height, + std::function deleter) + : NativeObject(CLASS_NAME), _handle(handle), _width(width), + _height(height), _deleter(std::move(deleter)) {} + + ~VideoFrame() override { release(); } + + std::string getBrand() { return CLASS_NAME; } + + // The native handle (IOSurfaceRef / AHardwareBuffer*) as a uintptr_t value. + // Exposed as a BigInt on the JS side. + void *getHandle() { return _handle; } + uint32_t getWidth() { return _width; } + uint32_t getHeight() { return _height; } + + void release() { + if (_deleter) { + _deleter(); + _deleter = nullptr; + } + _handle = nullptr; + } + + static void definePrototype(jsi::Runtime &runtime, jsi::Object &prototype) { + installGetter(runtime, prototype, "__brand", &VideoFrame::getBrand); + installGetter(runtime, prototype, "handle", &VideoFrame::getHandle); + installGetter(runtime, prototype, "width", &VideoFrame::getWidth); + installGetter(runtime, prototype, "height", &VideoFrame::getHeight); + installMethod(runtime, prototype, "release", &VideoFrame::release); + } + +private: + void *_handle; + uint32_t _width; + uint32_t _height; + std::function _deleter; +}; + +} // namespace rnwgpu diff --git a/packages/webgpu/cpp/rnwgpu/api/descriptors/GPUSharedTextureMemoryDescriptor.h b/packages/webgpu/cpp/rnwgpu/api/descriptors/GPUSharedTextureMemoryDescriptor.h new file mode 100644 index 000000000..b2ff7fe9d --- /dev/null +++ b/packages/webgpu/cpp/rnwgpu/api/descriptors/GPUSharedTextureMemoryDescriptor.h @@ -0,0 +1,62 @@ +#pragma once + +#include +#include +#include + +#include "webgpu/webgpu_cpp.h" + +#include "JSIConverter.h" + +namespace jsi = facebook::jsi; + +namespace rnwgpu { + +// Descriptor for GPUDevice.importSharedTextureMemory. +// +// `handle` is the raw native handle as a uintptr_t (passed as BigInt from JS): +// - Apple platforms: IOSurfaceRef +// - Android: AHardwareBuffer* +// +// Lifetime: the caller is responsible for keeping the underlying object alive +// for as long as this shared memory is in use. The VideoFrame helper handles +// this automatically when the handle came from PlatformContext.loadVideoFrame. +struct GPUSharedTextureMemoryDescriptor { + void *handle = nullptr; + std::optional label; +}; + +} // namespace rnwgpu + +namespace rnwgpu { + +template <> +struct JSIConverter> { + static std::shared_ptr + fromJSI(jsi::Runtime &runtime, const jsi::Value &arg, bool outOfBounds) { + auto result = + std::make_shared(); + if (!outOfBounds && arg.isObject()) { + auto value = arg.getObject(runtime); + if (value.hasProperty(runtime, "handle")) { + auto prop = value.getProperty(runtime, "handle"); + result->handle = + JSIConverter::fromJSI(runtime, prop, false); + } + if (value.hasProperty(runtime, "label")) { + auto prop = value.getProperty(runtime, "label"); + result->label = JSIConverter>::fromJSI( + runtime, prop, false); + } + } + return result; + } + static jsi::Value + toJSI(jsi::Runtime & /*runtime*/, + std::shared_ptr /*arg*/) { + throw std::runtime_error( + "Invalid GPUSharedTextureMemoryDescriptor::toJSI()"); + } +}; + +} // namespace rnwgpu diff --git a/packages/webgpu/src/Canvas.tsx b/packages/webgpu/src/Canvas.tsx index 142e5de2c..c06080721 100644 --- a/packages/webgpu/src/Canvas.tsx +++ b/packages/webgpu/src/Canvas.tsx @@ -21,6 +21,11 @@ declare global { ) => RNCanvasContext; DecodeToUTF8: (buffer: NodeJS.ArrayBufferView | ArrayBuffer) => string; createImageBitmap: typeof createImageBitmap; + loadVideoFrame: (path: string) => import("./types").VideoFrame; + createTestVideoFrame: ( + width: number, + height: number, + ) => import("./types").VideoFrame; }; } diff --git a/packages/webgpu/src/index.tsx b/packages/webgpu/src/index.tsx index a497a9bf0..e98906d08 100644 --- a/packages/webgpu/src/index.tsx +++ b/packages/webgpu/src/index.tsx @@ -3,6 +3,11 @@ import type { NativeCanvas, RNCanvasContext } from "./types"; export * from "./main"; +export type { + VideoFrame, + GPUSharedTextureMemory, + GPUSharedTextureMemoryDescriptor, +} from "./types"; declare global { interface Navigator { @@ -22,8 +27,19 @@ declare global { ) => RNCanvasContext; DecodeToUTF8: (buffer: NodeJS.ArrayBufferView | ArrayBuffer) => string; createImageBitmap: typeof createImageBitmap; + loadVideoFrame: (path: string) => import("./types").VideoFrame; + createTestVideoFrame: ( + width: number, + height: number, + ) => import("./types").VideoFrame; }; + interface GPUDevice { + importSharedTextureMemory( + descriptor: import("./types").GPUSharedTextureMemoryDescriptor, + ): import("./types").GPUSharedTextureMemory; + } + // Extend createImageBitmap to accept ArrayBuffer/TypedArray (encoded image bytes) function createImageBitmap( image: ArrayBuffer | ArrayBufferView, diff --git a/packages/webgpu/src/types.ts b/packages/webgpu/src/types.ts index af4684cfa..ed9481193 100644 --- a/packages/webgpu/src/types.ts +++ b/packages/webgpu/src/types.ts @@ -18,3 +18,42 @@ export interface CanvasRef { getNativeSurface: () => NativeCanvas; whenReady: (callback: () => void) => void; } + +// A native, GPU-shareable handle to a single video frame. +// +// - handle is the raw pointer (IOSurfaceRef on Apple, AHardwareBuffer* on +// Android) encoded as a BigInt. Pass it to +// GPUDevice.importSharedTextureMemory. +// - release() drops the underlying backing object (a CVPixelBuffer on Apple). +// The frame is also released when the JS wrapper is garbage-collected; call +// release() eagerly when you know you're done. +export interface VideoFrame { + readonly handle: bigint; + readonly width: number; + readonly height: number; + release(): void; +} + +export interface GPUSharedTextureMemoryDescriptor { + // Raw native handle (IOSurfaceRef on Apple, AHardwareBuffer* on Android), + // encoded as a BigInt. The caller is responsible for keeping the underlying + // object alive for as long as the shared memory (and any textures derived + // from it) are in use. Using VideoFrame.handle handles this automatically. + handle: bigint; + label?: string; +} + +// A piece of shared GPU memory backed by a native surface. Use createTexture() +// to obtain a regular GPUTexture that aliases the surface's pixels. The +// returned texture must be bracketed by beginAccess/endAccess around any +// command-buffer submission that uses it. +export interface GPUSharedTextureMemory { + readonly __brand: "GPUSharedTextureMemory"; + label: string; + createTexture(descriptor?: GPUTextureDescriptor): GPUTexture; + // `initialized` declares whether the surface already holds meaningful pixels + // (true for an incoming video/camera frame, false if the next pass will fully + // overwrite it). + beginAccess(texture: GPUTexture, initialized: boolean): boolean; + endAccess(texture: GPUTexture): boolean; +} From cf63d0a72ff5e76d8c7abfd0ca44dcb6b9bfc925 Mon Sep 17 00:00:00 2001 From: William Candillon Date: Tue, 19 May 2026 10:27:47 +0200 Subject: [PATCH 02/46] :wrench: --- .../SharedTextureMemory.tsx | 161 ++++++++----- .../android/cpp/AndroidPlatformContext.h | 13 + packages/webgpu/apple/ApplePlatformContext.h | 5 + packages/webgpu/apple/ApplePlatformContext.mm | 11 + packages/webgpu/apple/AppleVideoPlayer.h | 20 ++ packages/webgpu/apple/AppleVideoPlayer.mm | 228 ++++++++++++++++++ packages/webgpu/cpp/rnwgpu/PlatformContext.h | 26 ++ .../webgpu/cpp/rnwgpu/RNWebGPUManager.cpp | 2 + packages/webgpu/cpp/rnwgpu/api/RNWebGPU.h | 14 ++ packages/webgpu/cpp/rnwgpu/api/VideoPlayer.h | 71 ++++++ packages/webgpu/src/Canvas.tsx | 2 + packages/webgpu/src/index.tsx | 3 + packages/webgpu/src/types.ts | 10 + 13 files changed, 511 insertions(+), 55 deletions(-) create mode 100644 packages/webgpu/apple/AppleVideoPlayer.h create mode 100644 packages/webgpu/apple/AppleVideoPlayer.mm create mode 100644 packages/webgpu/cpp/rnwgpu/api/VideoPlayer.h diff --git a/apps/example/src/SharedTextureMemory/SharedTextureMemory.tsx b/apps/example/src/SharedTextureMemory/SharedTextureMemory.tsx index 0f3fc0a2f..488b91054 100644 --- a/apps/example/src/SharedTextureMemory/SharedTextureMemory.tsx +++ b/apps/example/src/SharedTextureMemory/SharedTextureMemory.tsx @@ -4,7 +4,9 @@ import { Canvas, useCanvasRef, useDevice, + type GPUSharedTextureMemory, type NativeCanvas, + type VideoFrame, } from "react-native-wgpu"; const SHADER = /* wgsl */ ` @@ -41,37 +43,44 @@ fn fs_main(in: VsOut) -> @location(0) vec4f { } `; -const REQUIRED_FEATURE = +// On Metal, EndAccess on an IOSurface-backed SharedTextureMemory always +// produces an MTLSharedEvent fence (so the producer can wait on the GPU). Even +// though we don't currently expose the fence to JS, Dawn validates that the +// fence feature is enabled before letting EndAccess succeed. Android has the +// equivalent pairing with sync fds. +const REQUIRED_FEATURES = Platform.OS === "ios" - ? "shared-texture-memory-iosurface" - : "shared-texture-memory-ahardware-buffer"; + ? ["shared-texture-memory-iosurface", "shared-fence-mtl-shared-event"] + : [ + "shared-texture-memory-ahardware-buffer", + "shared-fence-vk-semaphore-sync-fd", + ]; export const SharedTextureMemory = () => { const ref = useCanvasRef(); const [error, setError] = useState(null); const rafRef = useRef(null); - // Request the shared-memory feature when constructing the device so the - // shared-texture-memory* extension is enabled. const { device, adapter } = useDevice(undefined, { // Cast: GPUFeatureName in @webgpu/types doesn't include the Dawn-specific - // extension name yet, but Dawn accepts it. - requiredFeatures: [REQUIRED_FEATURE as GPUFeatureName], + // extension names yet, but Dawn accepts them. + requiredFeatures: REQUIRED_FEATURES as unknown as GPUFeatureName[], }); useEffect(() => { if (!device) { return; } - if (!device.features.has(REQUIRED_FEATURE)) { + const missing = REQUIRED_FEATURES.filter((f) => !device.features.has(f)); + if (missing.length > 0) { setError( - `Device is missing the '${REQUIRED_FEATURE}' feature (adapter supports: ${ + `Device is missing required features [${missing.join(", ")}]. Adapter supports: ${ adapter ? [...adapter.features] .filter((f) => f.toString().startsWith("shared-")) .join(", ") || "none" : "n/a" - })`, + }`, ); return; } @@ -90,38 +99,14 @@ export const SharedTextureMemory = () => { alphaMode: "premultiplied", }); - // 1. Acquire a native, GPU-shareable surface. In production this would - // come from a camera frame processor or video decoder. The test helper - // synthesizes a 256x256 RGB-gradient pattern in an IOSurface. - const frame = RNWebGPU.createTestVideoFrame(256, 256); - - // 2. Import the raw native handle into a SharedTextureMemory. - const sharedMemory = device.importSharedTextureMemory({ - handle: frame.handle, - label: "video-frame-shared-memory", - }); - - // 3. Create a regular GPUTexture that aliases the surface's pixels. - // No descriptor needed: the format/size are inferred from the surface. - const texture = sharedMemory.createTexture(); - - // 4. beginAccess declares that we're about to read or write the texture on - // the GPU timeline. `initialized: true` means "the surface already has - // meaningful pixels", which is correct for an incoming video frame. - // - // Because this example owns a *static* IOSurface (no external producer - // is writing new pixels between frames), we keep one access window open - // for the lifetime of the texture and call endAccess only on unmount. - // - // For a live camera or video feed, you'd instead wrap each frame: - // beginAccess(tex, true) -> submit -> endAccess(tex) - // around every render to hand ownership back to the producer. That's - // also where fence support (not yet wired through this binding) becomes - // important to avoid races with the producer. - if (!sharedMemory.beginAccess(texture, true)) { - setError("beginAccess() failed"); - return; - } + // 1. Open the video and start playback. AVPlayer accepts local file paths + // as well as http(s):// URLs and keeps the IOSurface pool up to date + // in the background. For a fully offline demo, swap this URL for + // RNWebGPU.writeTestVideoFile() which generates a tiny mp4 on disk. + const VIDEO_URL = + "https://test-videos.co.uk/vids/bigbuckbunny/mp4/h264/1080/Big_Buck_Bunny_1080_10s_5MB.mp4"; + const player = RNWebGPU.createVideoPlayer(VIDEO_URL); + player.play(); const module = device.createShaderModule({ code: SHADER }); const pipeline = device.createRenderPipeline({ @@ -138,15 +123,77 @@ export const SharedTextureMemory = () => { magFilter: "linear", minFilter: "linear", }); - const bindGroup = device.createBindGroup({ - layout: pipeline.getBindGroupLayout(0), - entries: [ - { binding: 0, resource: texture.createView() }, - { binding: 1, resource: sampler }, - ], - }); + + // We hold the *current* frame across rAF ticks so that when the video + // hasn't produced a new frame yet (between decoder timestamps), we keep + // rendering the last one rather than dropping to a black screen. + // + // For each new IOSurface we: + // - create a SharedTextureMemory + texture + bindGroup + // - beginAccess(initialized: true) to declare "the producer has written + // these pixels and we're now sampling them" + // - sample in the shader + // - endAccess to hand ownership back to the producer + // + // We close out the previous frame's access window first. In a fence-aware + // build we'd plumb an AVPlayer fence through beginAccess/endAccess; for the + // demo we rely on AVPlayer recycling its IOSurface pool, which is safe as + // long as we end-access before letting the player reclaim the buffer. + type Bound = { + frame: VideoFrame; + memory: GPUSharedTextureMemory; + texture: GPUTexture; + bindGroup: GPUBindGroup; + }; + let current: Bound | null = null; + + const bindFrame = (frame: VideoFrame): Bound | null => { + try { + const memory = device.importSharedTextureMemory({ + handle: frame.handle, + label: "video-frame", + }); + const texture = memory.createTexture(); + if (!memory.beginAccess(texture, true)) { + texture.destroy(); + frame.release(); + return null; + } + const bindGroup = device.createBindGroup({ + layout: pipeline.getBindGroupLayout(0), + entries: [ + { binding: 0, resource: texture.createView() }, + { binding: 1, resource: sampler }, + ], + }); + return { frame, memory, texture, bindGroup }; + } catch (e) { + console.warn("[SharedTextureMemory] bindFrame failed:", e); + frame.release(); + return null; + } + }; + + const releaseBound = (b: Bound) => { + b.memory.endAccess(b.texture); + b.texture.destroy(); + b.frame.release(); + }; const render = () => { + // Pull the latest frame from the player. Null means "no new frame since + // we last asked", in which case we keep using the existing one. + const newFrame = player.copyLatestFrame(); + if (newFrame) { + const next = bindFrame(newFrame); + if (next) { + if (current) { + releaseBound(current); + } + current = next; + } + } + const encoder = device.createCommandEncoder(); const pass = encoder.beginRenderPass({ colorAttachments: [ @@ -158,9 +205,11 @@ export const SharedTextureMemory = () => { }, ], }); - pass.setPipeline(pipeline); - pass.setBindGroup(0, bindGroup); - pass.draw(3); + if (current) { + pass.setPipeline(pipeline); + pass.setBindGroup(0, current.bindGroup); + pass.draw(3); + } pass.end(); device.queue.submit([encoder.finish()]); context.present(); @@ -172,9 +221,11 @@ export const SharedTextureMemory = () => { if (rafRef.current !== null) { cancelAnimationFrame(rafRef.current); } - sharedMemory.endAccess(texture); - texture.destroy(); - frame.release(); + if (current) { + releaseBound(current); + current = null; + } + player.release(); }; }, [device, adapter, ref]); diff --git a/packages/webgpu/android/cpp/AndroidPlatformContext.h b/packages/webgpu/android/cpp/AndroidPlatformContext.h index 4b8e43b94..bf81bc398 100644 --- a/packages/webgpu/android/cpp/AndroidPlatformContext.h +++ b/packages/webgpu/android/cpp/AndroidPlatformContext.h @@ -219,6 +219,19 @@ class AndroidPlatformContext : public PlatformContext { throw std::runtime_error( "createTestVideoFrame is not yet implemented on Android."); } + + std::unique_ptr + createVideoPlayer(const std::string & /*path*/) override { + // TODO: implement using MediaCodec -> ImageReader (AHardwareBuffer mode). + throw std::runtime_error( + "createVideoPlayer is not yet implemented on Android."); + } + + std::string writeTestVideoFile() override { + // TODO: implement using MediaCodec (H.264 encoder) or MediaMuxer. + throw std::runtime_error( + "writeTestVideoFile is not yet implemented on Android."); + } }; } // namespace rnwgpu diff --git a/packages/webgpu/apple/ApplePlatformContext.h b/packages/webgpu/apple/ApplePlatformContext.h index 77c8a2a21..4d9b29d1f 100644 --- a/packages/webgpu/apple/ApplePlatformContext.h +++ b/packages/webgpu/apple/ApplePlatformContext.h @@ -31,6 +31,11 @@ class ApplePlatformContext : public PlatformContext { VideoFrameHandle createTestVideoFrame(uint32_t width, uint32_t height) override; + + std::unique_ptr + createVideoPlayer(const std::string &path) override; + + std::string writeTestVideoFile() override; }; } // namespace rnwgpu diff --git a/packages/webgpu/apple/ApplePlatformContext.mm b/packages/webgpu/apple/ApplePlatformContext.mm index ffad62a1e..959b45b8d 100644 --- a/packages/webgpu/apple/ApplePlatformContext.mm +++ b/packages/webgpu/apple/ApplePlatformContext.mm @@ -8,6 +8,8 @@ #import #import +#include "AppleVideoPlayer.h" + #include "RNWebGPUManager.h" #include "WebGPUModule.h" @@ -234,6 +236,15 @@ void checkIfUsingSimulatorWithAPIValidation() { return handle; } +std::unique_ptr +ApplePlatformContext::createVideoPlayer(const std::string &path) { + return createAppleVideoPlayer(path); +} + +std::string ApplePlatformContext::writeTestVideoFile() { + return writeAppleTestVideoFile(); +} + VideoFrameHandle ApplePlatformContext::createTestVideoFrame(uint32_t width, uint32_t height) { NSDictionary *attrs = @{ diff --git a/packages/webgpu/apple/AppleVideoPlayer.h b/packages/webgpu/apple/AppleVideoPlayer.h new file mode 100644 index 000000000..1866f2440 --- /dev/null +++ b/packages/webgpu/apple/AppleVideoPlayer.h @@ -0,0 +1,20 @@ +#pragma once + +#include "PlatformContext.h" + +#include +#include + +namespace rnwgpu { + +// Factory: creates a new IVideoPlayer backed by AVPlayer + +// AVPlayerItemVideoOutput. +std::unique_ptr +createAppleVideoPlayer(const std::string &path); + +// Generate a small procedurally-animated test video and write it to a +// temporary file. Returns the absolute path. Used by the SharedTextureMemory +// example so it doesn't need a bundled .mp4. +std::string writeAppleTestVideoFile(); + +} // namespace rnwgpu diff --git a/packages/webgpu/apple/AppleVideoPlayer.mm b/packages/webgpu/apple/AppleVideoPlayer.mm new file mode 100644 index 000000000..4c58f215c --- /dev/null +++ b/packages/webgpu/apple/AppleVideoPlayer.mm @@ -0,0 +1,228 @@ +#include "AppleVideoPlayer.h" + +#import +#import + +#include + +namespace rnwgpu { + +namespace { + +class AppleVideoPlayer : public IVideoPlayer { +public: + AppleVideoPlayer(AVPlayer *player, AVPlayerItemVideoOutput *output, + id loopObserver) + : _player(player), _output(output), _loopObserver(loopObserver) {} + + ~AppleVideoPlayer() override { + if (_loopObserver) { + [[NSNotificationCenter defaultCenter] removeObserver:_loopObserver]; + _loopObserver = nil; + } + [_player pause]; + _player = nil; + _output = nil; + } + + VideoFrameHandle copyLatestFrame() override { + CMTime currentTime = [_output itemTimeForHostTime:CACurrentMediaTime()]; + if (![_output hasNewPixelBufferForItemTime:currentTime]) { + return {}; + } + CVPixelBufferRef pixelBuffer = + [_output copyPixelBufferForItemTime:currentTime + itemTimeForDisplay:nullptr]; + if (!pixelBuffer) { + return {}; + } + + IOSurfaceRef ioSurface = CVPixelBufferGetIOSurface(pixelBuffer); + if (!ioSurface) { + CFRelease(pixelBuffer); + return {}; + } + + VideoFrameHandle handle; + handle.handle = (void *)ioSurface; + handle.width = static_cast(CVPixelBufferGetWidth(pixelBuffer)); + handle.height = static_cast(CVPixelBufferGetHeight(pixelBuffer)); + handle.deleter = [pixelBuffer]() { CFRelease(pixelBuffer); }; + return handle; + } + + void play() override { [_player play]; } + void pause() override { [_player pause]; } + +private: + AVPlayer *_player; + AVPlayerItemVideoOutput *_output; + id _loopObserver; +}; + +} // namespace + +std::unique_ptr +createAppleVideoPlayer(const std::string &path) { + NSString *nsPath = [NSString stringWithUTF8String:path.c_str()]; + NSURL *url; + if ([nsPath hasPrefix:@"http://"] || [nsPath hasPrefix:@"https://"] || + [nsPath hasPrefix:@"file://"]) { + url = [NSURL URLWithString:nsPath]; + } else { + url = [NSURL fileURLWithPath:nsPath]; + } + + AVPlayerItem *item = [AVPlayerItem playerItemWithURL:url]; + if (!item) { + throw std::runtime_error("createAppleVideoPlayer: failed to create " + "AVPlayerItem"); + } + + NSDictionary *outputSettings = @{ + (NSString *)kCVPixelBufferPixelFormatTypeKey : + @(kCVPixelFormatType_32BGRA), + (NSString *)kCVPixelBufferIOSurfacePropertiesKey : @{}, + (NSString *)kCVPixelBufferMetalCompatibilityKey : @YES, + }; + AVPlayerItemVideoOutput *output = [[AVPlayerItemVideoOutput alloc] + initWithPixelBufferAttributes:outputSettings]; + [item addOutput:output]; + + AVPlayer *player = [AVPlayer playerWithPlayerItem:item]; + player.actionAtItemEnd = AVPlayerActionAtItemEndNone; + + // Loop on end-of-stream by seeking back to zero. + __weak AVPlayer *weakPlayer = player; + id loopObserver = [[NSNotificationCenter defaultCenter] + addObserverForName:AVPlayerItemDidPlayToEndTimeNotification + object:item + queue:[NSOperationQueue mainQueue] + usingBlock:^(NSNotification * /*note*/) { + [weakPlayer seekToTime:kCMTimeZero]; + [weakPlayer play]; + }]; + + return std::make_unique(player, output, loopObserver); +} + +std::string writeAppleTestVideoFile() { + NSString *tmpDir = NSTemporaryDirectory(); + NSString *outputPath = + [tmpDir stringByAppendingPathComponent:@"rnwgpu-test-video.mp4"]; + NSURL *outputURL = [NSURL fileURLWithPath:outputPath]; + + // If the file already exists, reuse it. This makes the example zero-cost on + // subsequent runs. + if ([[NSFileManager defaultManager] fileExistsAtPath:outputPath]) { + return [outputPath UTF8String]; + } + + NSError *error = nil; + AVAssetWriter *writer = [AVAssetWriter assetWriterWithURL:outputURL + fileType:AVFileTypeMPEG4 + error:&error]; + if (error || !writer) { + throw std::runtime_error( + std::string("writeTestVideoFile: AVAssetWriter init failed: ") + + [[error localizedDescription] UTF8String]); + } + + const int kWidth = 256; + const int kHeight = 256; + const int kFps = 30; + const int kSeconds = 3; + const int kTotalFrames = kFps * kSeconds; + + NSDictionary *videoSettings = @{ + AVVideoCodecKey : AVVideoCodecTypeH264, + AVVideoWidthKey : @(kWidth), + AVVideoHeightKey : @(kHeight), + }; + AVAssetWriterInput *writerInput = + [AVAssetWriterInput assetWriterInputWithMediaType:AVMediaTypeVideo + outputSettings:videoSettings]; + writerInput.expectsMediaDataInRealTime = NO; + + NSDictionary *bufferAttrs = @{ + (NSString *)kCVPixelBufferPixelFormatTypeKey : + @(kCVPixelFormatType_32BGRA), + (NSString *)kCVPixelBufferWidthKey : @(kWidth), + (NSString *)kCVPixelBufferHeightKey : @(kHeight), + (NSString *)kCVPixelBufferIOSurfacePropertiesKey : @{}, + }; + AVAssetWriterInputPixelBufferAdaptor *adaptor = + [AVAssetWriterInputPixelBufferAdaptor + assetWriterInputPixelBufferAdaptorWithAssetWriterInput:writerInput + sourcePixelBufferAttributes:bufferAttrs]; + [writer addInput:writerInput]; + + if (![writer startWriting]) { + throw std::runtime_error( + std::string("writeTestVideoFile: startWriting failed: ") + + [[writer.error localizedDescription] UTF8String]); + } + [writer startSessionAtSourceTime:kCMTimeZero]; + + for (int i = 0; i < kTotalFrames; ++i) { + // Spin briefly if the input is not ready (the adaptor pool fills up). + while (!writerInput.isReadyForMoreMediaData) { + [NSThread sleepForTimeInterval:0.005]; + } + + CVPixelBufferRef pixelBuffer = NULL; + CVReturn err = CVPixelBufferPoolCreatePixelBuffer( + kCFAllocatorDefault, adaptor.pixelBufferPool, &pixelBuffer); + if (err != kCVReturnSuccess || !pixelBuffer) { + throw std::runtime_error("writeTestVideoFile: pool exhausted"); + } + + CVPixelBufferLockBaseAddress(pixelBuffer, 0); + uint8_t *base = + static_cast(CVPixelBufferGetBaseAddress(pixelBuffer)); + size_t rowBytes = CVPixelBufferGetBytesPerRow(pixelBuffer); + // Procedural pattern: scrolling diagonal stripes + per-frame color shift. + int phase = i * 6; + for (int y = 0; y < kHeight; ++y) { + uint8_t *row = base + y * rowBytes; + for (int x = 0; x < kWidth; ++x) { + uint8_t r = static_cast((x + phase) & 0xFF); + uint8_t g = static_cast((y + phase * 2) & 0xFF); + uint8_t b = + static_cast(((x + y + phase) & 0x40) ? 220 : 30); + row[x * 4 + 0] = b; + row[x * 4 + 1] = g; + row[x * 4 + 2] = r; + row[x * 4 + 3] = 0xFF; + } + } + CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); + + CMTime pts = CMTimeMake(i, kFps); + if (![adaptor appendPixelBuffer:pixelBuffer withPresentationTime:pts]) { + CFRelease(pixelBuffer); + throw std::runtime_error( + std::string("writeTestVideoFile: appendPixelBuffer failed: ") + + [[writer.error localizedDescription] UTF8String]); + } + CFRelease(pixelBuffer); + } + + [writerInput markAsFinished]; + dispatch_semaphore_t sem = dispatch_semaphore_create(0); + [writer finishWritingWithCompletionHandler:^{ + dispatch_semaphore_signal(sem); + }]; + dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); + + if (writer.status != AVAssetWriterStatusCompleted) { + throw std::runtime_error( + std::string("writeTestVideoFile: writer finished with status ") + + std::to_string(static_cast(writer.status)) + ": " + + [[writer.error localizedDescription] UTF8String]); + } + + return [outputPath UTF8String]; +} + +} // namespace rnwgpu diff --git a/packages/webgpu/cpp/rnwgpu/PlatformContext.h b/packages/webgpu/cpp/rnwgpu/PlatformContext.h index 9743682d4..b34b15a31 100644 --- a/packages/webgpu/cpp/rnwgpu/PlatformContext.h +++ b/packages/webgpu/cpp/rnwgpu/PlatformContext.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include "webgpu/webgpu_cpp.h" @@ -33,6 +34,21 @@ struct VideoFrameHandle { std::function deleter; }; +// Platform-implemented video source that hands out fresh IOSurface / +// AHardwareBuffer-backed frames as a video plays. +class IVideoPlayer { +public: + virtual ~IVideoPlayer() = default; + + // Returns the latest decoded frame, or an empty handle (handle == nullptr) + // when no new frame is ready yet. Each non-empty return retains its backing + // surface; the VideoFrame wrapper releases it on destruction. + virtual VideoFrameHandle copyLatestFrame() = 0; + + virtual void play() = 0; + virtual void pause() = 0; +}; + class PlatformContext { public: PlatformContext() = default; @@ -68,6 +84,16 @@ class PlatformContext { // SharedTextureMemory example. virtual VideoFrameHandle createTestVideoFrame(uint32_t width, uint32_t height) = 0; + + // Open a video file at `path` for playback. The returned player yields + // IOSurface / AHardwareBuffer-backed frames via copyLatestFrame(). + virtual std::unique_ptr + createVideoPlayer(const std::string &path) = 0; + + // Write a small procedurally-generated test video to a temporary location + // and return its absolute path. Lets the SharedTextureMemory example play + // a real decoded video without bundling an asset. + virtual std::string writeTestVideoFile() = 0; }; } // namespace rnwgpu diff --git a/packages/webgpu/cpp/rnwgpu/RNWebGPUManager.cpp b/packages/webgpu/cpp/rnwgpu/RNWebGPUManager.cpp index 0cd62bbb9..38f675d37 100644 --- a/packages/webgpu/cpp/rnwgpu/RNWebGPUManager.cpp +++ b/packages/webgpu/cpp/rnwgpu/RNWebGPUManager.cpp @@ -39,6 +39,7 @@ #include "GPUUncapturedErrorEvent.h" #include "GPUValidationError.h" #include "VideoFrame.h" +#include "VideoPlayer.h" // Enums #include "GPUBufferUsage.h" @@ -105,6 +106,7 @@ RNWebGPUManager::RNWebGPUManager( GPUTexture::installConstructor(*_jsRuntime); GPUTextureView::installConstructor(*_jsRuntime); VideoFrame::installConstructor(*_jsRuntime); + VideoPlayer::installConstructor(*_jsRuntime); // Install constant objects as plain JS objects with own properties _jsRuntime->global().setProperty(*_jsRuntime, "GPUBufferUsage", diff --git a/packages/webgpu/cpp/rnwgpu/api/RNWebGPU.h b/packages/webgpu/cpp/rnwgpu/api/RNWebGPU.h index 7a59f3cbe..c0a2a44c7 100644 --- a/packages/webgpu/cpp/rnwgpu/api/RNWebGPU.h +++ b/packages/webgpu/cpp/rnwgpu/api/RNWebGPU.h @@ -11,6 +11,7 @@ #include "ImageBitmap.h" #include "PlatformContext.h" #include "VideoFrame.h" +#include "VideoPlayer.h" #include @@ -182,6 +183,15 @@ class RNWebGPU : public NativeObject { std::move(frame.deleter)); } + std::shared_ptr createVideoPlayer(std::string path) { + auto impl = _platformContext->createVideoPlayer(path); + return std::make_shared(std::move(impl)); + } + + std::string writeTestVideoFile() { + return _platformContext->writeTestVideoFile(); + } + std::shared_ptr getNativeSurface(int contextId) { auto ®istry = rnwgpu::SurfaceRegistry::getInstance(); auto info = registry.getSurfaceInfo(contextId); @@ -206,6 +216,10 @@ class RNWebGPU : public NativeObject { &RNWebGPU::loadVideoFrame); installMethod(runtime, prototype, "createTestVideoFrame", &RNWebGPU::createTestVideoFrame); + installMethod(runtime, prototype, "createVideoPlayer", + &RNWebGPU::createVideoPlayer); + installMethod(runtime, prototype, "writeTestVideoFile", + &RNWebGPU::writeTestVideoFile); } private: diff --git a/packages/webgpu/cpp/rnwgpu/api/VideoPlayer.h b/packages/webgpu/cpp/rnwgpu/api/VideoPlayer.h new file mode 100644 index 000000000..ee8c2b7af --- /dev/null +++ b/packages/webgpu/cpp/rnwgpu/api/VideoPlayer.h @@ -0,0 +1,71 @@ +#pragma once + +#include +#include +#include +#include + +#include "NativeObject.h" +#include "PlatformContext.h" +#include "VideoFrame.h" + +namespace rnwgpu { + +namespace jsi = facebook::jsi; + +// JSI wrapper around a platform-specific IVideoPlayer. Hands out fresh +// VideoFrame handles each time the underlying decoder produces a new frame. +class VideoPlayer : public NativeObject { +public: + static constexpr const char *CLASS_NAME = "VideoPlayer"; + + explicit VideoPlayer(std::unique_ptr impl) + : NativeObject(CLASS_NAME), _impl(std::move(impl)) {} + + std::string getBrand() { return CLASS_NAME; } + + // Returns the latest decoded frame, or null if no new frame is ready yet. + // Callers should poll this from their render loop and skip rendering (or + // reuse the last frame's texture) when null. + std::variant> + copyLatestFrame() { + if (!_impl) { + return nullptr; + } + auto handle = _impl->copyLatestFrame(); + if (handle.handle == nullptr) { + return nullptr; + } + return std::make_shared(handle.handle, handle.width, + handle.height, + std::move(handle.deleter)); + } + + void play() { + if (_impl) { + _impl->play(); + } + } + + void pause() { + if (_impl) { + _impl->pause(); + } + } + + void release() { _impl.reset(); } + + static void definePrototype(jsi::Runtime &runtime, jsi::Object &prototype) { + installGetter(runtime, prototype, "__brand", &VideoPlayer::getBrand); + installMethod(runtime, prototype, "copyLatestFrame", + &VideoPlayer::copyLatestFrame); + installMethod(runtime, prototype, "play", &VideoPlayer::play); + installMethod(runtime, prototype, "pause", &VideoPlayer::pause); + installMethod(runtime, prototype, "release", &VideoPlayer::release); + } + +private: + std::unique_ptr _impl; +}; + +} // namespace rnwgpu diff --git a/packages/webgpu/src/Canvas.tsx b/packages/webgpu/src/Canvas.tsx index c06080721..ce553badb 100644 --- a/packages/webgpu/src/Canvas.tsx +++ b/packages/webgpu/src/Canvas.tsx @@ -26,6 +26,8 @@ declare global { width: number, height: number, ) => import("./types").VideoFrame; + createVideoPlayer: (path: string) => import("./types").VideoPlayer; + writeTestVideoFile: () => string; }; } diff --git a/packages/webgpu/src/index.tsx b/packages/webgpu/src/index.tsx index e98906d08..a0ab50902 100644 --- a/packages/webgpu/src/index.tsx +++ b/packages/webgpu/src/index.tsx @@ -5,6 +5,7 @@ import type { NativeCanvas, RNCanvasContext } from "./types"; export * from "./main"; export type { VideoFrame, + VideoPlayer, GPUSharedTextureMemory, GPUSharedTextureMemoryDescriptor, } from "./types"; @@ -32,6 +33,8 @@ declare global { width: number, height: number, ) => import("./types").VideoFrame; + createVideoPlayer: (path: string) => import("./types").VideoPlayer; + writeTestVideoFile: () => string; }; interface GPUDevice { diff --git a/packages/webgpu/src/types.ts b/packages/webgpu/src/types.ts index ed9481193..ff7fb6a1b 100644 --- a/packages/webgpu/src/types.ts +++ b/packages/webgpu/src/types.ts @@ -34,6 +34,16 @@ export interface VideoFrame { release(): void; } +// A handle to a decoded video stream. Poll copyLatestFrame() each render tick +// to obtain the most recently decoded frame as an IOSurface/AHardwareBuffer +// (returns null between frames so callers can skip the import work). +export interface VideoPlayer { + copyLatestFrame(): VideoFrame | null; + play(): void; + pause(): void; + release(): void; +} + export interface GPUSharedTextureMemoryDescriptor { // Raw native handle (IOSurfaceRef on Apple, AHardwareBuffer* on Android), // encoded as a BigInt. The caller is responsible for keeping the underlying From 87fb8acf7c4e8c2660e2f8eb31bcd44d60e8f892 Mon Sep 17 00:00:00 2001 From: William Candillon Date: Tue, 19 May 2026 10:43:41 +0200 Subject: [PATCH 03/46] :wrench: --- .../SharedTextureMemory.tsx | 43 ++++++++++++++++--- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/apps/example/src/SharedTextureMemory/SharedTextureMemory.tsx b/apps/example/src/SharedTextureMemory/SharedTextureMemory.tsx index 488b91054..1e92a2ce8 100644 --- a/apps/example/src/SharedTextureMemory/SharedTextureMemory.tsx +++ b/apps/example/src/SharedTextureMemory/SharedTextureMemory.tsx @@ -15,6 +15,18 @@ struct VsOut { @location(0) uv: vec2f, }; +struct Uniforms { + // Per-axis scale applied to UVs *around the center* so that the canvas + // samples a sub-rectangle of the texture matching the canvas aspect ratio. + // 'cover' fit: one axis is 1.0, the other is canvasAR / textureAR (or its + // reciprocal), whichever is < 1 — i.e. we crop on the longer axis. + uvScale: vec2f, +}; + +@group(0) @binding(0) var srcTex: texture_2d; +@group(0) @binding(1) var srcSampler: sampler; +@group(0) @binding(2) var u: Uniforms; + @vertex fn vs_main(@builtin(vertex_index) vid: u32) -> VsOut { // Full-screen triangle. @@ -34,12 +46,10 @@ fn vs_main(@builtin(vertex_index) vid: u32) -> VsOut { return out; } -@group(0) @binding(0) var srcTex: texture_2d; -@group(0) @binding(1) var srcSampler: sampler; - @fragment fn fs_main(in: VsOut) -> @location(0) vec4f { - return textureSample(srcTex, srcSampler, in.uv); + let uv = vec2f(0.5) + (in.uv - vec2f(0.5)) * u.uvScale; + return textureSample(srcTex, srcSampler, uv); } `; @@ -144,9 +154,24 @@ export const SharedTextureMemory = () => { memory: GPUSharedTextureMemory; texture: GPUTexture; bindGroup: GPUBindGroup; + uniformBuffer: GPUBuffer; }; let current: Bound | null = null; + // 'cover' fit: scale UVs around their center so the longer axis of the + // texture is cropped to match the canvas aspect ratio. + const computeUvScale = (texW: number, texH: number): [number, number] => { + const canvasAR = canvas.width / canvas.height; + const texAR = texW / texH; + if (texAR > canvasAR) { + // Texture is wider than the canvas: crop horizontally. + return [canvasAR / texAR, 1]; + } else { + // Texture is taller than (or equal to) the canvas: crop vertically. + return [1, texAR / canvasAR]; + } + }; + const bindFrame = (frame: VideoFrame): Bound | null => { try { const memory = device.importSharedTextureMemory({ @@ -159,14 +184,21 @@ export const SharedTextureMemory = () => { frame.release(); return null; } + const uniformBuffer = device.createBuffer({ + size: 16, // vec2 padded to 16-byte uniform alignment + usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, + }); + const [sx, sy] = computeUvScale(frame.width, frame.height); + device.queue.writeBuffer(uniformBuffer, 0, new Float32Array([sx, sy])); const bindGroup = device.createBindGroup({ layout: pipeline.getBindGroupLayout(0), entries: [ { binding: 0, resource: texture.createView() }, { binding: 1, resource: sampler }, + { binding: 2, resource: { buffer: uniformBuffer } }, ], }); - return { frame, memory, texture, bindGroup }; + return { frame, memory, texture, bindGroup, uniformBuffer }; } catch (e) { console.warn("[SharedTextureMemory] bindFrame failed:", e); frame.release(); @@ -177,6 +209,7 @@ export const SharedTextureMemory = () => { const releaseBound = (b: Bound) => { b.memory.endAccess(b.texture); b.texture.destroy(); + b.uniformBuffer.destroy(); b.frame.release(); }; From e9c0406c98dd313d926e9304d19b9b9001fb3077 Mon Sep 17 00:00:00 2001 From: William Candillon Date: Tue, 19 May 2026 11:16:54 +0200 Subject: [PATCH 04/46] :wrenhc: --- apps/example/src/App.tsx | 2 + .../src/ExternalTexture/ExternalTexture.tsx | 258 ++++++++++++++++++ apps/example/src/ExternalTexture/index.ts | 1 + apps/example/src/Home.tsx | 4 + apps/example/src/Route.ts | 1 + .../android/cpp/AndroidPlatformContext.h | 3 +- packages/webgpu/apple/ApplePlatformContext.h | 3 +- packages/webgpu/apple/ApplePlatformContext.mm | 5 +- packages/webgpu/apple/AppleVideoPlayer.h | 4 +- packages/webgpu/apple/AppleVideoPlayer.mm | 65 ++++- packages/webgpu/cpp/rnwgpu/PlatformContext.h | 22 +- packages/webgpu/cpp/rnwgpu/api/Convertors.h | 11 +- packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp | 135 ++++++++- .../cpp/rnwgpu/api/GPUExternalTexture.h | 25 +- packages/webgpu/cpp/rnwgpu/api/RNWebGPU.h | 14 +- packages/webgpu/cpp/rnwgpu/api/VideoFrame.h | 37 ++- packages/webgpu/cpp/rnwgpu/api/VideoPlayer.h | 4 +- .../api/descriptors/GPUBindGroupEntry.h | 5 +- .../GPUExternalTextureDescriptor.h | 47 +--- packages/webgpu/src/Canvas.tsx | 5 +- packages/webgpu/src/index.tsx | 7 +- packages/webgpu/src/types.ts | 14 + 22 files changed, 595 insertions(+), 77 deletions(-) create mode 100644 apps/example/src/ExternalTexture/ExternalTexture.tsx create mode 100644 apps/example/src/ExternalTexture/index.ts diff --git a/apps/example/src/App.tsx b/apps/example/src/App.tsx index 0b26ab049..15a550eff 100644 --- a/apps/example/src/App.tsx +++ b/apps/example/src/App.tsx @@ -37,6 +37,7 @@ import { AsyncStarvation } from "./Diagnostics/AsyncStarvation"; import { DeviceLostHang } from "./Diagnostics/DeviceLostHang"; import { StorageBufferVertices } from "./StorageBufferVertices"; import { SharedTextureMemory } from "./SharedTextureMemory"; +import { ExternalTexture } from "./ExternalTexture"; // The two lines below are needed by three.js import "fast-text-encoding"; @@ -102,6 +103,7 @@ function App() { name="SharedTextureMemory" component={SharedTextureMemory} /> + diff --git a/apps/example/src/ExternalTexture/ExternalTexture.tsx b/apps/example/src/ExternalTexture/ExternalTexture.tsx new file mode 100644 index 000000000..78273eb7e --- /dev/null +++ b/apps/example/src/ExternalTexture/ExternalTexture.tsx @@ -0,0 +1,258 @@ +import React, { useEffect, useRef, useState } from "react"; +import { PixelRatio, Platform, StyleSheet, Text, View } from "react-native"; +import { + Canvas, + useCanvasRef, + useDevice, + type NativeCanvas, +} from "react-native-wgpu"; + +// importExternalTexture is the spec-mandated path for "I have a YUV-encoded +// video/camera frame and I want to sample it in a shader without copying or +// hand-rolling YUV math". The WGSL side uses texture_external + +// textureSampleBaseClampToEdge; the driver does the planar fetch, YUV→RGB +// matrix multiply, sRGB transfer, and gamut conversion in the sampler. +// +// Bind groups for texture_external use auto layout slots like any other +// resource. WGSL doesn't expose the underlying plane textures directly. +const SHADER = /* wgsl */ ` +struct VsOut { + @builtin(position) position: vec4f, + @location(0) uv: vec2f, +}; + +struct Uniforms { + uvScale: vec2f, +}; + +@group(0) @binding(0) var srcTex: texture_external; +@group(0) @binding(1) var srcSampler: sampler; +@group(0) @binding(2) var u: Uniforms; + +@vertex +fn vs_main(@builtin(vertex_index) vid: u32) -> VsOut { + var positions = array( + vec2f(-1.0, -3.0), + vec2f(-1.0, 1.0), + vec2f( 3.0, 1.0), + ); + var uvs = array( + vec2f(0.0, 2.0), + vec2f(0.0, 0.0), + vec2f(2.0, 0.0), + ); + var out: VsOut; + out.position = vec4f(positions[vid], 0.0, 1.0); + out.uv = uvs[vid]; + return out; +} + +@fragment +fn fs_main(in: VsOut) -> @location(0) vec4f { + let uv = vec2f(0.5) + (in.uv - vec2f(0.5)) * u.uvScale; + return textureSampleBaseClampToEdge(srcTex, srcSampler, uv); +} +`; + +// We need the same shared-memory + shared-fence pair as the BGRA demo (the +// IOSurface still flows through SharedTextureMemory under the hood), plus +// dawn-multi-planar-formats so Dawn can interpret the NV12 surface as a +// biplanar texture. +const REQUIRED_FEATURES = + Platform.OS === "ios" + ? [ + "shared-texture-memory-iosurface", + "shared-fence-mtl-shared-event", + "dawn-multi-planar-formats", + ] + : [ + "shared-texture-memory-ahardware-buffer", + "shared-fence-vk-semaphore-sync-fd", + "dawn-multi-planar-formats", + ]; + +const VIDEO_URL = + "https://test-videos.co.uk/vids/bigbuckbunny/mp4/h264/1080/Big_Buck_Bunny_1080_10s_5MB.mp4"; + +export const ExternalTexture = () => { + const ref = useCanvasRef(); + const [error, setError] = useState(null); + const rafRef = useRef(null); + + const { device, adapter } = useDevice(undefined, { + requiredFeatures: REQUIRED_FEATURES as unknown as GPUFeatureName[], + }); + + useEffect(() => { + if (!device) { + return; + } + const missing = REQUIRED_FEATURES.filter((f) => !device.features.has(f)); + if (missing.length > 0) { + setError( + `Device is missing required features [${missing.join(", ")}]. Adapter supports: ${ + adapter + ? [...adapter.features] + .filter((f) => f.toString().startsWith("shared-")) + .join(", ") || "none" + : "n/a" + }`, + ); + return; + } + + const context = ref.current?.getContext("webgpu"); + if (!context) { + return; + } + const canvas = context.canvas as unknown as NativeCanvas; + canvas.width = canvas.clientWidth * PixelRatio.get(); + canvas.height = canvas.clientHeight * PixelRatio.get(); + const presentationFormat = navigator.gpu.getPreferredCanvasFormat(); + context.configure({ + device, + format: presentationFormat, + alphaMode: "premultiplied", + }); + + // Same Big Buck Bunny URL as the SharedTextureMemory demo, but ask AVPlayer + // for native NV12 instead of BGRA. Each VideoFrame now carries the YUV + // matrix + plane info that importExternalTexture needs. + const player = RNWebGPU.createVideoPlayer(VIDEO_URL, "nv12"); + player.play(); + + const module = device.createShaderModule({ code: SHADER }); + const pipeline = device.createRenderPipeline({ + layout: "auto", + vertex: { module, entryPoint: "vs_main" }, + fragment: { + module, + entryPoint: "fs_main", + targets: [{ format: presentationFormat }], + }, + primitive: { topology: "triangle-list" }, + }); + const sampler = device.createSampler({ + magFilter: "linear", + minFilter: "linear", + }); + + // One persistent uniform buffer; we rewrite its uvScale whenever a new + // frame's dimensions differ from the last one. + const uniformBuffer = device.createBuffer({ + size: 16, + usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, + }); + let lastUvScale: [number, number] | null = null; + const writeUvScale = (texW: number, texH: number) => { + const canvasAR = canvas.width / canvas.height; + const texAR = texW / texH; + const next: [number, number] = + texAR > canvasAR ? [canvasAR / texAR, 1] : [1, texAR / canvasAR]; + if ( + !lastUvScale || + lastUvScale[0] !== next[0] || + lastUvScale[1] !== next[1] + ) { + device.queue.writeBuffer(uniformBuffer, 0, new Float32Array(next)); + lastUvScale = next; + } + }; + + const render = () => { + const frame = player.copyLatestFrame(); + if (frame) { + // GPUExternalTexture is one-shot: it's valid for the submission that + // immediately follows. We rebuild it from each new frame, render, and + // let GC drop the wrapper (which runs EndAccess for us). + let externalTex: GPUExternalTexture | null = null; + try { + externalTex = device.importExternalTexture({ + source: frame, + label: "video-external", + }); + } catch (e) { + console.warn("[ExternalTexture] importExternalTexture failed:", e); + frame.release(); + } + + if (externalTex) { + writeUvScale(frame.width, frame.height); + const bindGroup = device.createBindGroup({ + layout: pipeline.getBindGroupLayout(0), + entries: [ + { binding: 0, resource: externalTex }, + { binding: 1, resource: sampler }, + { binding: 2, resource: { buffer: uniformBuffer } }, + ], + }); + + const encoder = device.createCommandEncoder(); + const pass = encoder.beginRenderPass({ + colorAttachments: [ + { + view: context.getCurrentTexture().createView(), + clearValue: { r: 0, g: 0, b: 0, a: 1 }, + loadOp: "clear", + storeOp: "store", + }, + ], + }); + pass.setPipeline(pipeline); + pass.setBindGroup(0, bindGroup); + pass.draw(3); + pass.end(); + device.queue.submit([encoder.finish()]); + context.present(); + // externalTex and frame both fall out of scope here; the underlying + // SharedTextureMemory + IOSurface release on the next GC cycle. + } + } else { + // No new frame yet (still buffering). Present a clear pass so the + // canvas doesn't show whatever was in the previous swapchain image. + const encoder = device.createCommandEncoder(); + const pass = encoder.beginRenderPass({ + colorAttachments: [ + { + view: context.getCurrentTexture().createView(), + clearValue: { r: 0, g: 0, b: 0, a: 1 }, + loadOp: "clear", + storeOp: "store", + }, + ], + }); + pass.end(); + device.queue.submit([encoder.finish()]); + context.present(); + } + rafRef.current = requestAnimationFrame(render); + }; + rafRef.current = requestAnimationFrame(render); + + return () => { + if (rafRef.current !== null) { + cancelAnimationFrame(rafRef.current); + } + uniformBuffer.destroy(); + player.release(); + }; + }, [device, adapter, ref]); + + if (error) { + return ( + + {error} + + ); + } + return ( + + + + ); +}; + +const styles = StyleSheet.create({ + errorContainer: { flex: 1, padding: 16, justifyContent: "center" }, + errorText: { color: "red", fontSize: 14 }, +}); diff --git a/apps/example/src/ExternalTexture/index.ts b/apps/example/src/ExternalTexture/index.ts new file mode 100644 index 000000000..d019021b8 --- /dev/null +++ b/apps/example/src/ExternalTexture/index.ts @@ -0,0 +1 @@ +export * from "./ExternalTexture"; diff --git a/apps/example/src/Home.tsx b/apps/example/src/Home.tsx index 2db838041..6a032d2ec 100644 --- a/apps/example/src/Home.tsx +++ b/apps/example/src/Home.tsx @@ -131,6 +131,10 @@ export const examples = [ screen: "SharedTextureMemory", title: "🎞️ Shared Texture Memory", }, + { + screen: "ExternalTexture", + title: "🟨 External Texture (YUV)", + }, ]; const styles = StyleSheet.create({ diff --git a/apps/example/src/Route.ts b/apps/example/src/Route.ts index 1f2dbf187..ce0ac6441 100644 --- a/apps/example/src/Route.ts +++ b/apps/example/src/Route.ts @@ -30,4 +30,5 @@ export type Routes = { DeviceLostHang: undefined; StorageBufferVertices: undefined; SharedTextureMemory: undefined; + ExternalTexture: undefined; }; diff --git a/packages/webgpu/android/cpp/AndroidPlatformContext.h b/packages/webgpu/android/cpp/AndroidPlatformContext.h index bf81bc398..5f8a2577b 100644 --- a/packages/webgpu/android/cpp/AndroidPlatformContext.h +++ b/packages/webgpu/android/cpp/AndroidPlatformContext.h @@ -221,7 +221,8 @@ class AndroidPlatformContext : public PlatformContext { } std::unique_ptr - createVideoPlayer(const std::string & /*path*/) override { + createVideoPlayer(const std::string & /*path*/, + VideoPixelFormat /*format*/) override { // TODO: implement using MediaCodec -> ImageReader (AHardwareBuffer mode). throw std::runtime_error( "createVideoPlayer is not yet implemented on Android."); diff --git a/packages/webgpu/apple/ApplePlatformContext.h b/packages/webgpu/apple/ApplePlatformContext.h index 4d9b29d1f..5d4c78ec5 100644 --- a/packages/webgpu/apple/ApplePlatformContext.h +++ b/packages/webgpu/apple/ApplePlatformContext.h @@ -33,7 +33,8 @@ class ApplePlatformContext : public PlatformContext { uint32_t height) override; std::unique_ptr - createVideoPlayer(const std::string &path) override; + createVideoPlayer(const std::string &path, + VideoPixelFormat format) override; std::string writeTestVideoFile() override; }; diff --git a/packages/webgpu/apple/ApplePlatformContext.mm b/packages/webgpu/apple/ApplePlatformContext.mm index 959b45b8d..69280e5a1 100644 --- a/packages/webgpu/apple/ApplePlatformContext.mm +++ b/packages/webgpu/apple/ApplePlatformContext.mm @@ -237,8 +237,9 @@ void checkIfUsingSimulatorWithAPIValidation() { } std::unique_ptr -ApplePlatformContext::createVideoPlayer(const std::string &path) { - return createAppleVideoPlayer(path); +ApplePlatformContext::createVideoPlayer(const std::string &path, + VideoPixelFormat format) { + return createAppleVideoPlayer(path, format); } std::string ApplePlatformContext::writeTestVideoFile() { diff --git a/packages/webgpu/apple/AppleVideoPlayer.h b/packages/webgpu/apple/AppleVideoPlayer.h index 1866f2440..1962cc6c8 100644 --- a/packages/webgpu/apple/AppleVideoPlayer.h +++ b/packages/webgpu/apple/AppleVideoPlayer.h @@ -8,9 +8,9 @@ namespace rnwgpu { // Factory: creates a new IVideoPlayer backed by AVPlayer + -// AVPlayerItemVideoOutput. +// AVPlayerItemVideoOutput. `format` selects the surface layout. std::unique_ptr -createAppleVideoPlayer(const std::string &path); +createAppleVideoPlayer(const std::string &path, VideoPixelFormat format); // Generate a small procedurally-animated test video and write it to a // temporary file. Returns the absolute path. Used by the SharedTextureMemory diff --git a/packages/webgpu/apple/AppleVideoPlayer.mm b/packages/webgpu/apple/AppleVideoPlayer.mm index 4c58f215c..8a79012b1 100644 --- a/packages/webgpu/apple/AppleVideoPlayer.mm +++ b/packages/webgpu/apple/AppleVideoPlayer.mm @@ -9,11 +9,52 @@ namespace { +// 3x4 row-major matrices mapping [Y, U, V, 1] to linear RGB. +// Limited-range (video range) means luma is 16..235, chroma is 16..240 (8-bit). +// Reference: https://en.wikipedia.org/wiki/YCbCr (BT.601 / BT.709). +static constexpr float kBT709LimitedToLinearRGB[12] = { + 1.164383f, 0.000000f, 1.792741f, -0.972945f, // + 1.164383f, -0.213249f, -0.532909f, 0.301517f, // + 1.164383f, 2.112402f, 0.000000f, -1.133402f, // +}; +static constexpr float kBT601LimitedToLinearRGB[12] = { + 1.164383f, 0.000000f, 1.596027f, -0.874202f, // + 1.164383f, -0.391762f, -0.812968f, 0.531668f, // + 1.164383f, 2.017232f, 0.000000f, -1.085631f, // +}; +static constexpr float kBT2020LimitedToLinearRGB[12] = { + 1.164383f, 0.000000f, 1.678674f, -0.915688f, // + 1.164383f, -0.187326f, -0.650424f, 0.347459f, // + 1.164383f, 2.141772f, 0.000000f, -1.148145f, // +}; + +// Pick the right YUV→RGB matrix from the pixel buffer's color attachments. +// Falls back to BT.709 limited range (the right call for ≥720p H.264, which +// is what AVPlayer hands us for Big Buck Bunny and most streamed media). +static void fillYuvMatrix(CVPixelBufferRef pixelBuffer, float out[12]) { + CFTypeRef matrixKey = CVBufferGetAttachment( + pixelBuffer, kCVImageBufferYCbCrMatrixKey, nullptr); + const float *src = kBT709LimitedToLinearRGB; + if (matrixKey) { + auto matrix = (CFStringRef)matrixKey; + if (CFEqual(matrix, kCVImageBufferYCbCrMatrix_ITU_R_601_4) || + CFEqual(matrix, kCVImageBufferYCbCrMatrix_SMPTE_240M_1995)) { + src = kBT601LimitedToLinearRGB; + } else if (CFEqual(matrix, kCVImageBufferYCbCrMatrix_ITU_R_2020)) { + src = kBT2020LimitedToLinearRGB; + } + } + for (int i = 0; i < 12; ++i) { + out[i] = src[i]; + } +} + class AppleVideoPlayer : public IVideoPlayer { public: AppleVideoPlayer(AVPlayer *player, AVPlayerItemVideoOutput *output, - id loopObserver) - : _player(player), _output(output), _loopObserver(loopObserver) {} + id loopObserver, VideoPixelFormat pixelFormat) + : _player(player), _output(output), _loopObserver(loopObserver), + _pixelFormat(pixelFormat) {} ~AppleVideoPlayer() override { if (_loopObserver) { @@ -47,6 +88,10 @@ VideoFrameHandle copyLatestFrame() override { handle.handle = (void *)ioSurface; handle.width = static_cast(CVPixelBufferGetWidth(pixelBuffer)); handle.height = static_cast(CVPixelBufferGetHeight(pixelBuffer)); + handle.pixelFormat = _pixelFormat; + if (_pixelFormat == VideoPixelFormat::NV12) { + fillYuvMatrix(pixelBuffer, handle.yuvToRgbMatrix); + } handle.deleter = [pixelBuffer]() { CFRelease(pixelBuffer); }; return handle; } @@ -58,12 +103,13 @@ VideoFrameHandle copyLatestFrame() override { AVPlayer *_player; AVPlayerItemVideoOutput *_output; id _loopObserver; + VideoPixelFormat _pixelFormat; }; } // namespace std::unique_ptr -createAppleVideoPlayer(const std::string &path) { +createAppleVideoPlayer(const std::string &path, VideoPixelFormat format) { NSString *nsPath = [NSString stringWithUTF8String:path.c_str()]; NSURL *url; if ([nsPath hasPrefix:@"http://"] || [nsPath hasPrefix:@"https://"] || @@ -79,9 +125,15 @@ VideoFrameHandle copyLatestFrame() override { "AVPlayerItem"); } + // NV12 (kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) lets us hand the + // IOSurface straight to Dawn as a multi-planar texture for + // importExternalTexture. BGRA is the "decode + convert" path for the + // single-plane SharedTextureMemory demo. + OSType pixelFormat = format == VideoPixelFormat::NV12 + ? kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + : kCVPixelFormatType_32BGRA; NSDictionary *outputSettings = @{ - (NSString *)kCVPixelBufferPixelFormatTypeKey : - @(kCVPixelFormatType_32BGRA), + (NSString *)kCVPixelBufferPixelFormatTypeKey : @(pixelFormat), (NSString *)kCVPixelBufferIOSurfacePropertiesKey : @{}, (NSString *)kCVPixelBufferMetalCompatibilityKey : @YES, }; @@ -103,7 +155,8 @@ VideoFrameHandle copyLatestFrame() override { [weakPlayer play]; }]; - return std::make_unique(player, output, loopObserver); + return std::make_unique(player, output, loopObserver, + format); } std::string writeAppleTestVideoFile() { diff --git a/packages/webgpu/cpp/rnwgpu/PlatformContext.h b/packages/webgpu/cpp/rnwgpu/PlatformContext.h index b34b15a31..f46543484 100644 --- a/packages/webgpu/cpp/rnwgpu/PlatformContext.h +++ b/packages/webgpu/cpp/rnwgpu/PlatformContext.h @@ -18,6 +18,16 @@ struct ImageData { wgpu::TextureFormat format; }; +// Pixel layout of a VideoFrame. Determines whether the underlying surface is +// a single RGBA plane or a biplanar Y / CbCr pair. +enum class VideoPixelFormat { + // Single-plane 8-bit BGRA (default; what RGBA-style sampling expects). + BGRA8, + // Biplanar 4:2:0 8-bit Y + interleaved CbCr (NV12). Used for the + // importExternalTexture path; needs the YUV→RGB conversion matrix below. + NV12, +}; + // A native handle to a video frame that can be imported into a // GPUSharedTextureMemory. // @@ -31,6 +41,12 @@ struct VideoFrameHandle { void *handle = nullptr; uint32_t width = 0; uint32_t height = 0; + VideoPixelFormat pixelFormat = VideoPixelFormat::BGRA8; + // 3x4 row-major matrix mapping [Y, U, V, 1] → linear [R, G, B]. Pre-computed + // at decode time from CVPixelBuffer attachments (kCVImageBufferYCbCrMatrixKey + // + range), with a BT.709 limited-range default. Only meaningful when + // pixelFormat == NV12. + float yuvToRgbMatrix[12] = {}; std::function deleter; }; @@ -87,8 +103,12 @@ class PlatformContext { // Open a video file at `path` for playback. The returned player yields // IOSurface / AHardwareBuffer-backed frames via copyLatestFrame(). + // + // `format` selects the requested pixel layout. BGRA8 is the easiest target + // for a regular sampled GPUTexture; NV12 is the right shape for the + // importExternalTexture path (zero-copy biplanar YUV). virtual std::unique_ptr - createVideoPlayer(const std::string &path) = 0; + createVideoPlayer(const std::string &path, VideoPixelFormat format) = 0; // Write a small procedurally-generated test video to a temporary location // and return its absolute path. Lets the SharedTextureMemory example play diff --git a/packages/webgpu/cpp/rnwgpu/api/Convertors.h b/packages/webgpu/cpp/rnwgpu/api/Convertors.h index e168afcba..8d4e29bd1 100644 --- a/packages/webgpu/cpp/rnwgpu/api/Convertors.h +++ b/packages/webgpu/cpp/rnwgpu/api/Convertors.h @@ -729,7 +729,16 @@ class Convertor { out.buffer = buffer->get(); return true; } - // Not external textures at the moment + if (in.externalTexture != nullptr) { + // External textures bind via a chained struct rather than a direct field + // on BindGroupEntry. The chained struct must outlive the + // BindGroupEntry until Device::CreateBindGroup returns, so we allocate + // it on the Convertor's arena. + auto *chain = Allocate(); + chain->externalTexture = in.externalTexture->get(); + out.nextInChain = chain; + return true; + } return false; } diff --git a/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp b/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp index 0837da3e9..967351cb0 100644 --- a/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp +++ b/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp @@ -232,10 +232,143 @@ std::shared_ptr GPUDevice::createPipelineLayout( _instance.CreatePipelineLayout(&desc), descriptor->label.value_or("")); } +// Identity gamut (BT.709 -> sRGB, same primaries) as a 3x3 column-major matrix. +static const float kIdentityGamutMatrix[9] = { + 1.0f, 0.0f, 0.0f, // + 0.0f, 1.0f, 0.0f, // + 0.0f, 0.0f, 1.0f, // +}; + +// Piecewise gamma transfer-function parameters Dawn expects: +// for |x| < D: y = sign(x) * (C * |x| + F) +// else : y = sign(x) * (pow(A * |x| + B, G) + E) +// sRGB decode (encoded -> linear). +static const float kSrgbDecodeParams[7] = { + 2.4f, // G + 1.0f / 1.055f, // A + 0.055f / 1.055f, // B + 1.0f / 12.92f, // C + 0.04045f, // D + 0.0f, // E + 0.0f, // F +}; +// sRGB encode (linear -> encoded). +static const float kSrgbEncodeParams[7] = { + 1.0f / 2.4f, // G + 1.055f, // A + 0.0f, // B + 12.92f, // C + 0.0031308f, // D + -0.055f, // E + 0.0f, // F +}; + std::shared_ptr GPUDevice::importExternalTexture( std::shared_ptr descriptor) { + if (!descriptor || !descriptor->source) { + throw std::runtime_error( + "GPUDevice::importExternalTexture(): descriptor.source (VideoFrame) " + "is required"); + } + const auto &source = descriptor->source; + const auto &frame = source->handle(); + if (frame.handle == nullptr) { + throw std::runtime_error( + "GPUDevice::importExternalTexture(): VideoFrame has been released"); + } + +#if defined(__APPLE__) + // 1. Import the IOSurface as SharedTextureMemory. For NV12 surfaces this + // yields a biplanar texture; for BGRA, a single-plane one. + wgpu::SharedTextureMemoryDescriptor memDesc{}; + std::string label = descriptor->label.value_or("external-texture"); + if (!label.empty()) { + memDesc.label = wgpu::StringView(label.c_str(), label.size()); + } + wgpu::SharedTextureMemoryIOSurfaceDescriptor platformDesc{}; + platformDesc.ioSurface = frame.handle; + // ExternalTexture views are sampled-only; storage binding isn't needed and + // for biplanar formats it would fail validation. + platformDesc.allowStorageBinding = false; + memDesc.nextInChain = &platformDesc; + auto memory = _instance.ImportSharedTextureMemory(&memDesc); + if (memory == nullptr) { + throw std::runtime_error( + "GPUDevice::importExternalTexture(): ImportSharedTextureMemory " + "returned null. Is 'shared-texture-memory-iosurface' enabled?"); + } + + // 2. Create the texture from the surface. We pass the right format + // explicitly so Dawn picks the multi-planar variant on NV12. + bool isYuv = frame.pixelFormat == VideoPixelFormat::NV12; + auto texture = memory.CreateTexture(); + if (texture == nullptr) { + throw std::runtime_error( + "GPUDevice::importExternalTexture(): CreateTexture returned null"); + } + + // 3. Begin access on the underlying memory. The matching EndAccess runs in + // the GPUExternalTexture destructor. + wgpu::SharedTextureMemoryBeginAccessDescriptor begin{}; + begin.initialized = true; + begin.concurrentRead = false; + if (!memory.BeginAccess(texture, &begin)) { + throw std::runtime_error( + "GPUDevice::importExternalTexture(): BeginAccess failed"); + } + + // 4. Build plane views. For NV12 we need plane0 = R8 luma and plane1 = RG8 + // chroma; for BGRA we only set plane0. + wgpu::TextureView plane0; + wgpu::TextureView plane1; + { + wgpu::TextureViewDescriptor v{}; + v.aspect = isYuv ? wgpu::TextureAspect::Plane0Only + : wgpu::TextureAspect::All; + plane0 = texture.CreateView(&v); + } + if (isYuv) { + wgpu::TextureViewDescriptor v{}; + v.aspect = wgpu::TextureAspect::Plane1Only; + plane1 = texture.CreateView(&v); + } + + // 5. Build the ExternalTextureDescriptor. We hand Dawn explicit YUV→RGB and + // sRGB transfer-function parameters so the sampler does the full color + // conversion in hardware. + wgpu::ExternalTextureDescriptor extDesc{}; + if (!label.empty()) { + extDesc.label = wgpu::StringView(label.c_str(), label.size()); + } + extDesc.plane0 = plane0; + if (isYuv) { + extDesc.plane1 = plane1; + extDesc.yuvToRgbConversionMatrix = frame.yuvToRgbMatrix; + extDesc.srcTransferFunctionParameters = kSrgbDecodeParams; + extDesc.dstTransferFunctionParameters = kSrgbEncodeParams; + extDesc.gamutConversionMatrix = kIdentityGamutMatrix; + } + extDesc.cropOrigin = {0, 0}; + extDesc.cropSize = {frame.width, frame.height}; + extDesc.apparentSize = {frame.width, frame.height}; + + auto external = _instance.CreateExternalTexture(&extDesc); + if (external == nullptr) { + wgpu::SharedTextureMemoryEndAccessState state{}; + (void)memory.EndAccess(texture, &state); + throw std::runtime_error( + "GPUDevice::importExternalTexture(): CreateExternalTexture returned " + "null"); + } + + return std::make_shared( + std::move(external), std::move(memory), std::move(texture), + std::move(descriptor->source), std::move(label)); +#else throw std::runtime_error( - "GPUDevice::importExternalTexture(): Not implemented"); + "GPUDevice::importExternalTexture(): not yet implemented on this " + "platform"); +#endif } std::shared_ptr GPUDevice::importSharedTextureMemory( diff --git a/packages/webgpu/cpp/rnwgpu/api/GPUExternalTexture.h b/packages/webgpu/cpp/rnwgpu/api/GPUExternalTexture.h index 9be5efe6f..71e23f3b7 100644 --- a/packages/webgpu/cpp/rnwgpu/api/GPUExternalTexture.h +++ b/packages/webgpu/cpp/rnwgpu/api/GPUExternalTexture.h @@ -1,10 +1,13 @@ #pragma once +#include #include +#include #include "Unions.h" #include "NativeObject.h" +#include "VideoFrame.h" #include "webgpu/webgpu_cpp.h" @@ -16,8 +19,23 @@ class GPUExternalTexture : public NativeObject { public: static constexpr const char *CLASS_NAME = "GPUExternalTexture"; - explicit GPUExternalTexture(wgpu::ExternalTexture instance, std::string label) - : NativeObject(CLASS_NAME), _instance(instance), _label(label) {} + // Construct from an already-built wgpu::ExternalTexture plus the underlying + // shared-memory resources we need to keep alive. The wrapper takes ownership + // of the SharedTextureMemory + Texture and calls EndAccess on destruction so + // the producer (e.g. AVPlayer) can reclaim the IOSurface. + GPUExternalTexture(wgpu::ExternalTexture instance, + wgpu::SharedTextureMemory memory, wgpu::Texture texture, + std::shared_ptr source, std::string label) + : NativeObject(CLASS_NAME), _instance(std::move(instance)), + _memory(std::move(memory)), _texture(std::move(texture)), + _source(std::move(source)), _label(std::move(label)) {} + + ~GPUExternalTexture() override { + if (_memory && _texture) { + wgpu::SharedTextureMemoryEndAccessState state{}; + (void)_memory.EndAccess(_texture, &state); + } + } public: std::string getBrand() { return CLASS_NAME; } @@ -39,6 +57,9 @@ class GPUExternalTexture : public NativeObject { private: wgpu::ExternalTexture _instance; + wgpu::SharedTextureMemory _memory; + wgpu::Texture _texture; + std::shared_ptr _source; std::string _label; }; diff --git a/packages/webgpu/cpp/rnwgpu/api/RNWebGPU.h b/packages/webgpu/cpp/rnwgpu/api/RNWebGPU.h index c0a2a44c7..c7457adc2 100644 --- a/packages/webgpu/cpp/rnwgpu/api/RNWebGPU.h +++ b/packages/webgpu/cpp/rnwgpu/api/RNWebGPU.h @@ -172,19 +172,21 @@ class RNWebGPU : public NativeObject { std::shared_ptr loadVideoFrame(std::string path) { auto frame = _platformContext->loadVideoFrame(path); - return std::make_shared(frame.handle, frame.width, frame.height, - std::move(frame.deleter)); + return std::make_shared(std::move(frame)); } std::shared_ptr createTestVideoFrame(double width, double height) { auto frame = _platformContext->createTestVideoFrame( static_cast(width), static_cast(height)); - return std::make_shared(frame.handle, frame.width, frame.height, - std::move(frame.deleter)); + return std::make_shared(std::move(frame)); } - std::shared_ptr createVideoPlayer(std::string path) { - auto impl = _platformContext->createVideoPlayer(path); + std::shared_ptr + createVideoPlayer(std::string path, std::optional pixelFormat) { + auto format = (pixelFormat && pixelFormat.value() == "nv12") + ? VideoPixelFormat::NV12 + : VideoPixelFormat::BGRA8; + auto impl = _platformContext->createVideoPlayer(path, format); return std::make_shared(std::move(impl)); } diff --git a/packages/webgpu/cpp/rnwgpu/api/VideoFrame.h b/packages/webgpu/cpp/rnwgpu/api/VideoFrame.h index af09cd127..f001285e0 100644 --- a/packages/webgpu/cpp/rnwgpu/api/VideoFrame.h +++ b/packages/webgpu/cpp/rnwgpu/api/VideoFrame.h @@ -7,6 +7,7 @@ #include "JSIConverter.h" #include "NativeObject.h" +#include "PlatformContext.h" namespace rnwgpu { @@ -21,10 +22,8 @@ class VideoFrame : public NativeObject { public: static constexpr const char *CLASS_NAME = "VideoFrame"; - VideoFrame(void *handle, uint32_t width, uint32_t height, - std::function deleter) - : NativeObject(CLASS_NAME), _handle(handle), _width(width), - _height(height), _deleter(std::move(deleter)) {} + explicit VideoFrame(VideoFrameHandle handle) + : NativeObject(CLASS_NAME), _handle(std::move(handle)) {} ~VideoFrame() override { release(); } @@ -32,16 +31,25 @@ class VideoFrame : public NativeObject { // The native handle (IOSurfaceRef / AHardwareBuffer*) as a uintptr_t value. // Exposed as a BigInt on the JS side. - void *getHandle() { return _handle; } - uint32_t getWidth() { return _width; } - uint32_t getHeight() { return _height; } + void *getHandle() { return _handle.handle; } + uint32_t getWidth() { return _handle.width; } + uint32_t getHeight() { return _handle.height; } + + // Pixel format as a JS-visible string: "bgra8" | "nv12". + std::string getPixelFormat() { + return _handle.pixelFormat == VideoPixelFormat::NV12 ? "nv12" : "bgra8"; + } + + // Direct access to the underlying handle, for use by importExternalTexture / + // importSharedTextureMemory inside the C++ layer (not exposed to JS). + const VideoFrameHandle &handle() const { return _handle; } void release() { - if (_deleter) { - _deleter(); - _deleter = nullptr; + if (_handle.deleter) { + _handle.deleter(); + _handle.deleter = nullptr; } - _handle = nullptr; + _handle.handle = nullptr; } static void definePrototype(jsi::Runtime &runtime, jsi::Object &prototype) { @@ -49,14 +57,13 @@ class VideoFrame : public NativeObject { installGetter(runtime, prototype, "handle", &VideoFrame::getHandle); installGetter(runtime, prototype, "width", &VideoFrame::getWidth); installGetter(runtime, prototype, "height", &VideoFrame::getHeight); + installGetter(runtime, prototype, "pixelFormat", + &VideoFrame::getPixelFormat); installMethod(runtime, prototype, "release", &VideoFrame::release); } private: - void *_handle; - uint32_t _width; - uint32_t _height; - std::function _deleter; + VideoFrameHandle _handle; }; } // namespace rnwgpu diff --git a/packages/webgpu/cpp/rnwgpu/api/VideoPlayer.h b/packages/webgpu/cpp/rnwgpu/api/VideoPlayer.h index ee8c2b7af..b97552aca 100644 --- a/packages/webgpu/cpp/rnwgpu/api/VideoPlayer.h +++ b/packages/webgpu/cpp/rnwgpu/api/VideoPlayer.h @@ -36,9 +36,7 @@ class VideoPlayer : public NativeObject { if (handle.handle == nullptr) { return nullptr; } - return std::make_shared(handle.handle, handle.width, - handle.height, - std::move(handle.deleter)); + return std::make_shared(std::move(handle)); } void play() { diff --git a/packages/webgpu/cpp/rnwgpu/api/descriptors/GPUBindGroupEntry.h b/packages/webgpu/cpp/rnwgpu/api/descriptors/GPUBindGroupEntry.h index 73e54dc43..af6ebfda1 100644 --- a/packages/webgpu/cpp/rnwgpu/api/descriptors/GPUBindGroupEntry.h +++ b/packages/webgpu/cpp/rnwgpu/api/descriptors/GPUBindGroupEntry.h @@ -21,7 +21,7 @@ struct GPUBindGroupEntry { std::shared_ptr sampler = nullptr; std::shared_ptr textureView = nullptr; std::shared_ptr buffer = nullptr; - // external textures + std::shared_ptr externalTexture = nullptr; }; } // namespace rnwgpu @@ -46,6 +46,9 @@ template <> struct JSIConverter> { } else if (obj.hasNativeState(runtime)) { result->textureView = obj.getNativeState(runtime); + } else if (obj.hasNativeState(runtime)) { + result->externalTexture = + obj.getNativeState(runtime); } else if (obj.hasNativeState(runtime)) { // Support passing GPUBuffer directly as resource (auto-wrap in // GPUBufferBinding) diff --git a/packages/webgpu/cpp/rnwgpu/api/descriptors/GPUExternalTextureDescriptor.h b/packages/webgpu/cpp/rnwgpu/api/descriptors/GPUExternalTextureDescriptor.h index fba6721ee..da7affc9b 100644 --- a/packages/webgpu/cpp/rnwgpu/api/descriptors/GPUExternalTextureDescriptor.h +++ b/packages/webgpu/cpp/rnwgpu/api/descriptors/GPUExternalTextureDescriptor.h @@ -3,35 +3,25 @@ #include #include #include -#include #include "webgpu/webgpu_cpp.h" -#include "Convertors.h" - #include "JSIConverter.h" -#include "WGPULogger.h" +#include "VideoFrame.h" namespace jsi = facebook::jsi; namespace rnwgpu { +// Mirror of GPUExternalTextureDescriptor from the WebGPU spec, but with our +// VideoFrame as the (only) supported source. We don't expose colorSpace yet; +// the C++ side picks dst-sRGB and identity gamut, which is the right default +// for "render this video frame to a regular sRGB framebuffer". struct GPUExternalTextureDescriptor { - // std::variant, - // std::shared_ptr> - // source; // | HTMLVideoElement | VideoFrame - // std::optional colorSpace; // PredefinedColorSpace - std::optional label; // string + std::shared_ptr source; + std::optional label; }; -static bool conv(wgpu::ExternalTextureDescriptor &out, - const std::shared_ptr &in) { - // TODO: implement - // return conv(out.source, in->source) && conv(out.colorSpace, in->colorSpace) - // && - // return conv(out.label, in->label); - return false; -} } // namespace rnwgpu namespace rnwgpu { @@ -40,23 +30,15 @@ template <> struct JSIConverter> { static std::shared_ptr fromJSI(jsi::Runtime &runtime, const jsi::Value &arg, bool outOfBounds) { - auto result = std::make_unique(); + auto result = std::make_shared(); if (!outOfBounds && arg.isObject()) { auto value = arg.getObject(runtime); if (value.hasProperty(runtime, "source")) { auto prop = value.getProperty(runtime, "source"); - // result->source = JSIConverter< - // std::variant, - // std::shared_ptr>>::fromJSI(runtime, - // prop, - // false); - } - if (value.hasProperty(runtime, "colorSpace")) { - auto prop = value.getProperty(runtime, "colorSpace"); - if (!prop.isUndefined()) { - // result->colorSpace = - // JSIConverter>::fromJSI( - // runtime, prop, false); + if (!prop.isUndefined() && !prop.isNull()) { + result->source = + JSIConverter>::fromJSI( + runtime, prop, false); } } if (value.hasProperty(runtime, "label")) { @@ -67,12 +49,11 @@ struct JSIConverter> { } } } - return result; } static jsi::Value - toJSI(jsi::Runtime &runtime, - std::shared_ptr arg) { + toJSI(jsi::Runtime & /*runtime*/, + std::shared_ptr /*arg*/) { throw std::runtime_error("Invalid GPUExternalTextureDescriptor::toJSI()"); } }; diff --git a/packages/webgpu/src/Canvas.tsx b/packages/webgpu/src/Canvas.tsx index ce553badb..ef4cba14d 100644 --- a/packages/webgpu/src/Canvas.tsx +++ b/packages/webgpu/src/Canvas.tsx @@ -26,7 +26,10 @@ declare global { width: number, height: number, ) => import("./types").VideoFrame; - createVideoPlayer: (path: string) => import("./types").VideoPlayer; + createVideoPlayer: ( + path: string, + pixelFormat?: import("./types").VideoPixelFormat, + ) => import("./types").VideoPlayer; writeTestVideoFile: () => string; }; } diff --git a/packages/webgpu/src/index.tsx b/packages/webgpu/src/index.tsx index a0ab50902..2244239fc 100644 --- a/packages/webgpu/src/index.tsx +++ b/packages/webgpu/src/index.tsx @@ -6,6 +6,8 @@ export * from "./main"; export type { VideoFrame, VideoPlayer, + VideoPixelFormat, + CreateVideoPlayerOptions, GPUSharedTextureMemory, GPUSharedTextureMemoryDescriptor, } from "./types"; @@ -33,7 +35,10 @@ declare global { width: number, height: number, ) => import("./types").VideoFrame; - createVideoPlayer: (path: string) => import("./types").VideoPlayer; + createVideoPlayer: ( + path: string, + pixelFormat?: import("./types").VideoPixelFormat, + ) => import("./types").VideoPlayer; writeTestVideoFile: () => string; }; diff --git a/packages/webgpu/src/types.ts b/packages/webgpu/src/types.ts index ff7fb6a1b..19ffc71a9 100644 --- a/packages/webgpu/src/types.ts +++ b/packages/webgpu/src/types.ts @@ -19,11 +19,16 @@ export interface CanvasRef { whenReady: (callback: () => void) => void; } +export type VideoPixelFormat = "bgra8" | "nv12"; + // A native, GPU-shareable handle to a single video frame. // // - handle is the raw pointer (IOSurfaceRef on Apple, AHardwareBuffer* on // Android) encoded as a BigInt. Pass it to // GPUDevice.importSharedTextureMemory. +// - pixelFormat describes the surface layout: 'bgra8' for a sampled +// GPUTexture; 'nv12' (biplanar Y + CbCr) for the importExternalTexture +// path. // - release() drops the underlying backing object (a CVPixelBuffer on Apple). // The frame is also released when the JS wrapper is garbage-collected; call // release() eagerly when you know you're done. @@ -31,6 +36,7 @@ export interface VideoFrame { readonly handle: bigint; readonly width: number; readonly height: number; + readonly pixelFormat: VideoPixelFormat; release(): void; } @@ -44,6 +50,14 @@ export interface VideoPlayer { release(): void; } +export interface CreateVideoPlayerOptions { + // 'bgra8' (default): emit a single-plane BGRA surface, suitable for + // SharedTextureMemory and a regular sampled GPUTexture. + // 'nv12': emit biplanar Y + CbCr surfaces, suitable for + // GPUDevice.importExternalTexture. + pixelFormat?: VideoPixelFormat; +} + export interface GPUSharedTextureMemoryDescriptor { // Raw native handle (IOSurfaceRef on Apple, AHardwareBuffer* on Android), // encoded as a BigInt. The caller is responsible for keeping the underlying From 69dcd30c97527e1a75b40814f4906a1b2061f44d Mon Sep 17 00:00:00 2001 From: William Candillon Date: Tue, 19 May 2026 11:31:55 +0200 Subject: [PATCH 05/46] :wrench: --- .../src/ExternalTexture/ExternalTexture.tsx | 84 +++++++++---------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/apps/example/src/ExternalTexture/ExternalTexture.tsx b/apps/example/src/ExternalTexture/ExternalTexture.tsx index 78273eb7e..e68255270 100644 --- a/apps/example/src/ExternalTexture/ExternalTexture.tsx +++ b/apps/example/src/ExternalTexture/ExternalTexture.tsx @@ -5,6 +5,7 @@ import { useCanvasRef, useDevice, type NativeCanvas, + type VideoFrame, } from "react-native-wgpu"; // importExternalTexture is the spec-mandated path for "I have a YUV-encoded @@ -159,25 +160,50 @@ export const ExternalTexture = () => { } }; + // The video plays at ~24fps but we tick at the display's 60Hz, so most rAF + // ticks have no new frame from AVPlayer. Hold the latest VideoFrame across + // ticks and re-import an ExternalTexture from it on the "no new frame" + // ticks — this is what stops the canvas from flashing black ~2/3 of the + // time. AVPlayer's pool is several buffers deep so holding one back like + // this doesn't stall decoding. + let currentFrame: VideoFrame | null = null; + const render = () => { - const frame = player.copyLatestFrame(); - if (frame) { - // GPUExternalTexture is one-shot: it's valid for the submission that - // immediately follows. We rebuild it from each new frame, render, and - // let GC drop the wrapper (which runs EndAccess for us). + const newFrame = player.copyLatestFrame(); + if (newFrame) { + if (currentFrame) { + currentFrame.release(); + } + currentFrame = newFrame; + } + + const encoder = device.createCommandEncoder(); + const pass = encoder.beginRenderPass({ + colorAttachments: [ + { + view: context.getCurrentTexture().createView(), + clearValue: { r: 0, g: 0, b: 0, a: 1 }, + loadOp: "clear", + storeOp: "store", + }, + ], + }); + + if (currentFrame) { + // GPUExternalTexture expires after each submit, so we rebuild one + // every tick — even when sampling the same VideoFrame as last tick. let externalTex: GPUExternalTexture | null = null; try { externalTex = device.importExternalTexture({ - source: frame, + source: currentFrame, label: "video-external", }); } catch (e) { console.warn("[ExternalTexture] importExternalTexture failed:", e); - frame.release(); } if (externalTex) { - writeUvScale(frame.width, frame.height); + writeUvScale(currentFrame.width, currentFrame.height); const bindGroup = device.createBindGroup({ layout: pipeline.getBindGroupLayout(0), entries: [ @@ -186,45 +212,15 @@ export const ExternalTexture = () => { { binding: 2, resource: { buffer: uniformBuffer } }, ], }); - - const encoder = device.createCommandEncoder(); - const pass = encoder.beginRenderPass({ - colorAttachments: [ - { - view: context.getCurrentTexture().createView(), - clearValue: { r: 0, g: 0, b: 0, a: 1 }, - loadOp: "clear", - storeOp: "store", - }, - ], - }); pass.setPipeline(pipeline); pass.setBindGroup(0, bindGroup); pass.draw(3); - pass.end(); - device.queue.submit([encoder.finish()]); - context.present(); - // externalTex and frame both fall out of scope here; the underlying - // SharedTextureMemory + IOSurface release on the next GC cycle. } - } else { - // No new frame yet (still buffering). Present a clear pass so the - // canvas doesn't show whatever was in the previous swapchain image. - const encoder = device.createCommandEncoder(); - const pass = encoder.beginRenderPass({ - colorAttachments: [ - { - view: context.getCurrentTexture().createView(), - clearValue: { r: 0, g: 0, b: 0, a: 1 }, - loadOp: "clear", - storeOp: "store", - }, - ], - }); - pass.end(); - device.queue.submit([encoder.finish()]); - context.present(); } + + pass.end(); + device.queue.submit([encoder.finish()]); + context.present(); rafRef.current = requestAnimationFrame(render); }; rafRef.current = requestAnimationFrame(render); @@ -233,6 +229,10 @@ export const ExternalTexture = () => { if (rafRef.current !== null) { cancelAnimationFrame(rafRef.current); } + if (currentFrame) { + currentFrame.release(); + currentFrame = null; + } uniformBuffer.destroy(); player.release(); }; From be9b29f8819486abef2b8ba55148209a3fdff858 Mon Sep 17 00:00:00 2001 From: William Candillon Date: Tue, 19 May 2026 14:58:28 +0200 Subject: [PATCH 06/46] :wrench: --- apps/example/ios/Podfile.lock | 162 +++++++- apps/example/package.json | 8 +- apps/example/src/App.tsx | 10 +- apps/example/src/Home.tsx | 4 + apps/example/src/Route.ts | 1 + .../example/src/VisionCamera/VisionCamera.tsx | 376 ++++++++++++++++++ apps/example/src/VisionCamera/index.ts | 1 + .../android/cpp/AndroidPlatformContext.h | 7 + packages/webgpu/apple/ApplePlatformContext.h | 2 + packages/webgpu/apple/ApplePlatformContext.mm | 4 + packages/webgpu/apple/AppleVideoPlayer.h | 11 + packages/webgpu/apple/AppleVideoPlayer.mm | 72 +++- packages/webgpu/cpp/rnwgpu/PlatformContext.h | 16 + .../webgpu/cpp/rnwgpu/RNWebGPUManager.cpp | 11 + packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp | 14 + packages/webgpu/cpp/rnwgpu/api/GPUDevice.h | 9 + packages/webgpu/cpp/rnwgpu/api/RNWebGPU.h | 10 + packages/webgpu/package.json | 2 +- packages/webgpu/src/Canvas.tsx | 3 + .../reanimated/registerWebGPUForReanimated.ts | 69 +++- packages/webgpu/src/index.tsx | 12 + packages/webgpu/src/main/index.tsx | 46 ++- yarn.lock | 207 +++++----- 23 files changed, 895 insertions(+), 162 deletions(-) create mode 100644 apps/example/src/VisionCamera/VisionCamera.tsx create mode 100644 apps/example/src/VisionCamera/index.ts diff --git a/apps/example/ios/Podfile.lock b/apps/example/ios/Podfile.lock index f9bc9b29a..3d55cd5e2 100644 --- a/apps/example/ios/Podfile.lock +++ b/apps/example/ios/Podfile.lock @@ -8,6 +8,65 @@ PODS: - hermes-engine (0.81.4): - hermes-engine/Pre-built (= 0.81.4) - hermes-engine/Pre-built (0.81.4) + - NitroImage (0.14.0): + - boost + - DoubleConversion + - fast_float + - fmt + - glog + - hermes-engine + - NitroModules + - RCT-Folly + - RCT-Folly/Fabric + - RCTRequired + - RCTTypeSafety + - React-callinvoker + - React-Core + - React-debug + - React-Fabric + - React-featureflags + - React-graphics + - React-ImageManager + - React-jsi + - React-NativeModulesApple + - React-RCTFabric + - React-renderercss + - React-rendererdebug + - React-utils + - ReactCodegen + - ReactCommon/turbomodule/bridging + - ReactCommon/turbomodule/core + - SocketRocket + - Yoga + - NitroModules (0.35.6): + - boost + - DoubleConversion + - fast_float + - fmt + - glog + - hermes-engine + - RCT-Folly + - RCT-Folly/Fabric + - RCTRequired + - RCTTypeSafety + - React-callinvoker + - React-Core + - React-debug + - React-Fabric + - React-featureflags + - React-graphics + - React-ImageManager + - React-jsi + - React-NativeModulesApple + - React-RCTFabric + - React-renderercss + - React-rendererdebug + - React-utils + - ReactCodegen + - ReactCommon/turbomodule/bridging + - ReactCommon/turbomodule/core + - SocketRocket + - Yoga - RCT-Folly (2024.11.18.00): - boost - DoubleConversion @@ -2460,7 +2519,7 @@ PODS: - ReactCommon/turbomodule/core - SocketRocket - Yoga - - RNReanimated (4.2.1): + - RNReanimated (4.3.1): - boost - DoubleConversion - fast_float @@ -2487,11 +2546,12 @@ PODS: - ReactCodegen - ReactCommon/turbomodule/bridging - ReactCommon/turbomodule/core - - RNReanimated/reanimated (= 4.2.1) + - RNReanimated/apple (= 4.3.1) + - RNReanimated/common (= 4.3.1) - RNWorklets - SocketRocket - Yoga - - RNReanimated/reanimated (4.2.1): + - RNReanimated/apple (4.3.1): - boost - DoubleConversion - fast_float @@ -2518,11 +2578,10 @@ PODS: - ReactCodegen - ReactCommon/turbomodule/bridging - ReactCommon/turbomodule/core - - RNReanimated/reanimated/apple (= 4.2.1) - RNWorklets - SocketRocket - Yoga - - RNReanimated/reanimated/apple (4.2.1): + - RNReanimated/common (4.3.1): - boost - DoubleConversion - fast_float @@ -2552,7 +2611,7 @@ PODS: - RNWorklets - SocketRocket - Yoga - - RNWorklets (0.7.2): + - RNWorklets (0.8.3): - boost - DoubleConversion - fast_float @@ -2579,10 +2638,11 @@ PODS: - ReactCodegen - ReactCommon/turbomodule/bridging - ReactCommon/turbomodule/core - - RNWorklets/worklets (= 0.7.2) + - RNWorklets/apple (= 0.8.3) + - RNWorklets/common (= 0.8.3) - SocketRocket - Yoga - - RNWorklets/worklets (0.7.2): + - RNWorklets/apple (0.8.3): - boost - DoubleConversion - fast_float @@ -2609,10 +2669,9 @@ PODS: - ReactCodegen - ReactCommon/turbomodule/bridging - ReactCommon/turbomodule/core - - RNWorklets/worklets/apple (= 0.7.2) - SocketRocket - Yoga - - RNWorklets/worklets/apple (0.7.2): + - RNWorklets/common (0.8.3): - boost - DoubleConversion - fast_float @@ -2642,6 +2701,69 @@ PODS: - SocketRocket - Yoga - SocketRocket (0.7.1) + - VisionCamera (5.0.9): + - boost + - DoubleConversion + - fast_float + - fmt + - glog + - hermes-engine + - NitroImage + - NitroModules + - RCT-Folly + - RCT-Folly/Fabric + - RCTRequired + - RCTTypeSafety + - React-callinvoker + - React-Core + - React-debug + - React-Fabric + - React-featureflags + - React-graphics + - React-ImageManager + - React-jsi + - React-NativeModulesApple + - React-RCTFabric + - React-renderercss + - React-rendererdebug + - React-utils + - ReactCodegen + - ReactCommon/turbomodule/bridging + - ReactCommon/turbomodule/core + - SocketRocket + - Yoga + - VisionCameraWorklets (5.0.9): + - boost + - DoubleConversion + - fast_float + - fmt + - glog + - hermes-engine + - NitroModules + - RCT-Folly + - RCT-Folly/Fabric + - RCTRequired + - RCTTypeSafety + - React-callinvoker + - React-Core + - React-debug + - React-Fabric + - React-featureflags + - React-graphics + - React-ImageManager + - React-jsi + - React-NativeModulesApple + - React-RCTFabric + - React-renderercss + - React-rendererdebug + - React-utils + - ReactCodegen + - ReactCommon/turbomodule/bridging + - ReactCommon/turbomodule/core + - RNWorklets + - SocketRocket + - VisionCamera + - Yoga - Yoga (0.0.0) DEPENDENCIES: @@ -2652,6 +2774,8 @@ DEPENDENCIES: - fmt (from `../../../node_modules/react-native/third-party-podspecs/fmt.podspec`) - glog (from `../../../node_modules/react-native/third-party-podspecs/glog.podspec`) - hermes-engine (from `../../../node_modules/react-native/sdks/hermes-engine/hermes-engine.podspec`) + - NitroImage (from `../../../node_modules/react-native-nitro-image`) + - NitroModules (from `../../../node_modules/react-native-nitro-modules`) - RCT-Folly (from `../../../node_modules/react-native/third-party-podspecs/RCT-Folly.podspec`) - RCTDeprecation (from `../../../node_modules/react-native/ReactApple/Libraries/RCTFoundation/RCTDeprecation`) - RCTRequired (from `../../../node_modules/react-native/Libraries/Required`) @@ -2726,6 +2850,8 @@ DEPENDENCIES: - RNReanimated (from `../../../node_modules/react-native-reanimated`) - RNWorklets (from `../../../node_modules/react-native-worklets`) - SocketRocket (~> 0.7.1) + - VisionCamera (from `../../../node_modules/react-native-vision-camera`) + - VisionCameraWorklets (from `../../../node_modules/react-native-vision-camera-worklets`) - Yoga (from `../../../node_modules/react-native/ReactCommon/yoga`) SPEC REPOS: @@ -2748,6 +2874,10 @@ EXTERNAL SOURCES: hermes-engine: :podspec: "../../../node_modules/react-native/sdks/hermes-engine/hermes-engine.podspec" :tag: hermes-2025-07-07-RNv0.81.0-e0fc67142ec0763c6b6153ca2bf96df815539782 + NitroImage: + :path: "../../../node_modules/react-native-nitro-image" + NitroModules: + :path: "../../../node_modules/react-native-nitro-modules" RCT-Folly: :podspec: "../../../node_modules/react-native/third-party-podspecs/RCT-Folly.podspec" RCTDeprecation: @@ -2892,6 +3022,10 @@ EXTERNAL SOURCES: :path: "../../../node_modules/react-native-reanimated" RNWorklets: :path: "../../../node_modules/react-native-worklets" + VisionCamera: + :path: "../../../node_modules/react-native-vision-camera" + VisionCameraWorklets: + :path: "../../../node_modules/react-native-vision-camera-worklets" Yoga: :path: "../../../node_modules/react-native/ReactCommon/yoga" @@ -2903,6 +3037,8 @@ SPEC CHECKSUMS: fmt: 530618a01105dae0fa3a2f27c81ae11fa8f67eac glog: 5683914934d5b6e4240e497e0f4a3b42d1854183 hermes-engine: 35c763d57c9832d0eef764316ca1c4d043581394 + NitroImage: 4ffcf183d975de179ae1662b7c3b4b3b37747c7e + NitroModules: 70e2ff77a0b718e47d5bccd837fefe8a9aa20f50 RCT-Folly: b29feb752b08042c62badaef7d453f3bb5e6ae23 RCTDeprecation: c0ed3249a97243002615517dff789bf4666cf585 RCTRequired: 58719f5124f9267b5f9649c08bf23d9aea845b23 @@ -2973,9 +3109,11 @@ SPEC CHECKSUMS: ReactTestApp-DevSupport: 9b7bbba5e8fed998e763809171d9906a1375f9d3 ReactTestApp-Resources: 1bd9ff10e4c24f2ad87101a32023721ae923bccf RNGestureHandler: e37bdb684df1ac17c7e1d8f71a3311b2793c186b - RNReanimated: 464375ff2caa801358547c44eca894ff0bf68e74 - RNWorklets: ee58e869ea579800ec5f2f1cb6ae195fd3537546 + RNReanimated: 9d012d4031abc9df896f8a82f9928eb2b9eae417 + RNWorklets: 0da2552f9ff5d17506918a692304110cfebb9f0a SocketRocket: d4aabe649be1e368d1318fdf28a022d714d65748 + VisionCamera: 433061777a7a2f4436be5369ccaba0f24e0ff7b3 + VisionCameraWorklets: 29d29366e8216700971ecead05136cbf5400f010 Yoga: a3ed390a19db0459bd6839823a6ac6d9c6db198d PODFILE CHECKSUM: 22a8651333bf096f67ca333598bd33455d994c1f diff --git a/apps/example/package.json b/apps/example/package.json index bc1e36625..37cecb719 100644 --- a/apps/example/package.json +++ b/apps/example/package.json @@ -33,11 +33,15 @@ "react-native": "0.81.4", "react-native-gesture-handler": "^2.28.0", "react-native-macos": "^0.79.0", - "react-native-reanimated": "4.2.1", + "react-native-nitro-image": "^0.14.0", + "react-native-nitro-modules": "^0.35.6", + "react-native-reanimated": "4.3.1", "react-native-safe-area-context": "^5.4.0", + "react-native-vision-camera": "^5.0.9", + "react-native-vision-camera-worklets": "^5.0.9", "react-native-web": "^0.21.2", "react-native-wgpu": "*", - "react-native-worklets": "0.7.2", + "react-native-worklets": "0.8.3", "teapot": "^1.0.0", "three": "0.172.0", "typegpu": "^0.3.2", diff --git a/apps/example/src/App.tsx b/apps/example/src/App.tsx index 15a550eff..0646dadb4 100644 --- a/apps/example/src/App.tsx +++ b/apps/example/src/App.tsx @@ -38,6 +38,7 @@ import { DeviceLostHang } from "./Diagnostics/DeviceLostHang"; import { StorageBufferVertices } from "./StorageBufferVertices"; import { SharedTextureMemory } from "./SharedTextureMemory"; import { ExternalTexture } from "./ExternalTexture"; +import { VisionCamera } from "./VisionCamera"; // The two lines below are needed by three.js import "fast-text-encoding"; @@ -48,9 +49,9 @@ const Stack = createStackNavigator(); function App() { const assets = useAssets(); - if (assets === null) { - return null; - } + // if (assets === null) { + // return null; + // } return ( @@ -89,7 +90,7 @@ function App() { - {(props) => } + {(props) => (assets ? : null)} @@ -104,6 +105,7 @@ function App() { component={SharedTextureMemory} /> + diff --git a/apps/example/src/Home.tsx b/apps/example/src/Home.tsx index 6a032d2ec..66489c5c0 100644 --- a/apps/example/src/Home.tsx +++ b/apps/example/src/Home.tsx @@ -135,6 +135,10 @@ export const examples = [ screen: "ExternalTexture", title: "🟨 External Texture (YUV)", }, + { + screen: "VisionCamera", + title: "đź“· VisionCamera integration", + }, ]; const styles = StyleSheet.create({ diff --git a/apps/example/src/Route.ts b/apps/example/src/Route.ts index ce0ac6441..33c920f29 100644 --- a/apps/example/src/Route.ts +++ b/apps/example/src/Route.ts @@ -31,4 +31,5 @@ export type Routes = { StorageBufferVertices: undefined; SharedTextureMemory: undefined; ExternalTexture: undefined; + VisionCamera: undefined; }; diff --git a/apps/example/src/VisionCamera/VisionCamera.tsx b/apps/example/src/VisionCamera/VisionCamera.tsx new file mode 100644 index 000000000..2ff5a9555 --- /dev/null +++ b/apps/example/src/VisionCamera/VisionCamera.tsx @@ -0,0 +1,376 @@ +import React, { useEffect } from "react"; +import { + Linking, + PixelRatio, + Platform, + StyleSheet, + Text, + TouchableOpacity, + View, +} from "react-native"; +import { + Canvas, + useCanvasRef, + useDevice, + type NativeCanvas, + type RNCanvasContext, +} from "react-native-wgpu"; +import { + useCamera, + useCameraDevices, + useCameraPermission, + useFrameOutput, +} from "react-native-vision-camera"; + +// Camera frame → SharedTextureMemory (NV12 biplanar) → GPUExternalTexture → +// textureSampleBaseClampToEdge with hardware YUV/sRGB conversion → chromatic +// aberration in WGSL. +// +// Everything past frame arrival runs on Vision Camera's worklet runtime. +// react-native-wgpu's `registerWebGPUForReanimated` (loaded from main on +// startup) registers a Worklets custom serializer for GPUDevice / canvas / +// pipeline / sampler / buffer, so the closure references below auto-box on +// the way into the worklet and auto-unbox on the way out. + +const SHADER = /* wgsl */ ` +struct VsOut { + @builtin(position) position: vec4f, + @location(0) uv: vec2f, +}; + +struct Uniforms { + // x, y: 'cover'-fit UV scale around (0.5, 0.5). + // z: chromatic aberration offset in UV units. + // w: unused, padding. + scaleAndAberration: vec4f, +}; + +@group(0) @binding(0) var srcTex: texture_external; +@group(0) @binding(1) var srcSampler: sampler; +@group(0) @binding(2) var u: Uniforms; + +@vertex +fn vs_main(@builtin(vertex_index) vid: u32) -> VsOut { + var positions = array( + vec2f(-1.0, -3.0), + vec2f(-1.0, 1.0), + vec2f( 3.0, 1.0), + ); + var uvs = array( + vec2f(0.0, 2.0), + vec2f(0.0, 0.0), + vec2f(2.0, 0.0), + ); + var out: VsOut; + out.position = vec4f(positions[vid], 0.0, 1.0); + out.uv = uvs[vid]; + return out; +} + +@fragment +fn fs_main(in: VsOut) -> @location(0) vec4f { + let uvScale = u.scaleAndAberration.xy; + let aberration = u.scaleAndAberration.z; + let uv = vec2f(0.5) + (in.uv - vec2f(0.5)) * uvScale; + // RGB split: sample red shifted right, blue shifted left, green centered. + let r = textureSampleBaseClampToEdge(srcTex, srcSampler, uv + vec2f( aberration, 0.0)).r; + let g = textureSampleBaseClampToEdge(srcTex, srcSampler, uv).g; + let b = textureSampleBaseClampToEdge(srcTex, srcSampler, uv + vec2f(-aberration, 0.0)).b; + return vec4f(r, g, b, 1.0); +} +`; + +const REQUIRED_FEATURES = + Platform.OS === "ios" + ? [ + "shared-texture-memory-iosurface", + "shared-fence-mtl-shared-event", + "dawn-multi-planar-formats", + ] + : [ + "shared-texture-memory-ahardware-buffer", + "shared-fence-vk-semaphore-sync-fd", + "dawn-multi-planar-formats", + ]; + +const ABERRATION_STRENGTH = 0.006; + +export const VisionCamera = () => { + const { hasPermission, requestPermission } = useCameraPermission(); + useEffect(() => { + if (!hasPermission) { + requestPermission(); + } + }, [hasPermission, requestPermission]); + + if (!hasPermission) { + return ( + + + Camera access is required. Grant it in Settings or tap below. + + Linking.openSettings()} + style={styles.permissionButton} + > + Open Settings + + + ); + } + return ; +}; + +const CameraView = () => { + const ref = useCanvasRef(); + const { device, adapter } = useDevice(undefined, { + requiredFeatures: REQUIRED_FEATURES as unknown as GPUFeatureName[], + }); + const devices = useCameraDevices(); + // Pick back camera if available, otherwise front, otherwise anything. The + // iOS simulator returns an empty list since there are no cameras, in which + // case we surface a clear error rather than letting useCamera throw. + const cameraDevice = React.useMemo( + () => + devices.find((d) => d.position === "back") ?? + devices.find((d) => d.position === "front") ?? + devices[0], + [devices], + ); + + const [pipelineState, setPipelineState] = React.useState<{ + pipeline: GPURenderPipeline; + sampler: GPUSampler; + uniformBuffer: GPUBuffer; + context: RNCanvasContext; + canvasWidth: number; + canvasHeight: number; + } | null>(null); + const [error, setError] = React.useState(null); + + // Initialize pipeline once device + canvas are both ready. + useEffect(() => { + if (!device || pipelineState) { + return; + } + const missing = REQUIRED_FEATURES.filter((f) => !device.features.has(f)); + if (missing.length > 0) { + setError( + `Device missing features [${missing.join(", ")}]. Adapter: ${ + adapter + ? [...adapter.features] + .filter((f) => f.toString().startsWith("shared-")) + .join(", ") || "none" + : "n/a" + }`, + ); + return; + } + const context = ref.current?.getContext("webgpu"); + if (!context) { + return; + } + const canvas = context.canvas as unknown as NativeCanvas; + canvas.width = canvas.clientWidth * PixelRatio.get(); + canvas.height = canvas.clientHeight * PixelRatio.get(); + const presentationFormat = navigator.gpu.getPreferredCanvasFormat(); + context.configure({ + device, + format: presentationFormat, + alphaMode: "premultiplied", + }); + + const module = device.createShaderModule({ code: SHADER }); + const pipeline = device.createRenderPipeline({ + layout: "auto", + vertex: { module, entryPoint: "vs_main" }, + fragment: { + module, + entryPoint: "fs_main", + targets: [{ format: presentationFormat }], + }, + primitive: { topology: "triangle-list" }, + }); + const sampler = device.createSampler({ + magFilter: "linear", + minFilter: "linear", + }); + const uniformBuffer = device.createBuffer({ + size: 16, + usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, + }); + setPipelineState({ + pipeline, + sampler, + uniformBuffer, + context, + canvasWidth: canvas.width, + canvasHeight: canvas.height, + }); + }, [device, adapter, ref, pipelineState]); + + // Build the frame processor worklet. Captured WebGPU objects flow into the + // worklet runtime via the registerWebGPUForReanimated custom serializer. + // Diagnostic: log once on the *first call only* by capturing a plain bool + // box. Worklets serializes the box once on closure creation, so flipping + // .seen mutates the worklet-side copy; we use it strictly to avoid spamming + // metro logs and don't rely on main-thread visibility. + const logBox = React.useMemo(() => ({ seen: false }), []); + const frameOutput = useFrameOutput({ + pixelFormat: "native", // zero-copy, gives us NV12 IOSurfaces on iOS + onFrame: (frame) => { + "worklet"; + if (!logBox.seen) { + logBox.seen = true; + + console.log( + "[VisionCamera] worklet first frame, hasPipeline=" + + String(pipelineState != null) + + " hasDevice=" + + String(device != null) + + " frame=" + + String(frame.width) + + "x" + + String(frame.height), + ); + } + if (!pipelineState || !device) { + frame.dispose(); + return; + } + const { + pipeline, + sampler, + uniformBuffer, + context, + canvasWidth, + canvasHeight, + } = pipelineState; + const nativeBuffer = frame.getNativeBuffer(); + try { + let videoFrame; + try { + // Call createVideoFrameFromNativeBuffer on the device, not on the + // RNWebGPU global — `device` is already box-able across worklet + // runtimes via the WebGPU custom serializer (proven by the + // Reanimated demo); RNWebGPU is a main-runtime-only global. + videoFrame = device.createVideoFrameFromNativeBuffer( + nativeBuffer.pointer, + ); + } catch (e) { + console.warn( + "[VisionCamera] createVideoFrameFromNativeBuffer threw: " + + String(e), + ); + throw e; + } + try { + // Compute cover-fit uvScale based on frame & canvas aspect ratios. + // On most phones the back camera is landscape (e.g. 1920x1080) and + // the canvas is portrait, so the y-axis gets cropped. + const canvasAR = canvasWidth / canvasHeight; + const frameAR = videoFrame.width / videoFrame.height; + let sx = 1; + let sy = 1; + if (frameAR > canvasAR) { + sx = canvasAR / frameAR; + } else { + sy = frameAR / canvasAR; + } + device.queue.writeBuffer( + uniformBuffer, + 0, + new Float32Array([sx, sy, ABERRATION_STRENGTH, 0]), + ); + + const externalTex = device.importExternalTexture({ + source: videoFrame, + label: "camera-frame", + }); + const bindGroup = device.createBindGroup({ + layout: pipeline.getBindGroupLayout(0), + entries: [ + { binding: 0, resource: externalTex }, + { binding: 1, resource: sampler }, + { binding: 2, resource: { buffer: uniformBuffer } }, + ], + }); + + const encoder = device.createCommandEncoder(); + const pass = encoder.beginRenderPass({ + colorAttachments: [ + { + view: context.getCurrentTexture().createView(), + clearValue: { r: 0, g: 0, b: 0, a: 1 }, + loadOp: "clear", + storeOp: "store", + }, + ], + }); + pass.setPipeline(pipeline); + pass.setBindGroup(0, bindGroup); + pass.draw(3); + pass.end(); + device.queue.submit([encoder.finish()]); + context.present(); + } finally { + videoFrame.release(); + } + } finally { + nativeBuffer.release(); + frame.dispose(); + } + }, + }); + + // We have to call useCamera unconditionally (hook order). Pass a stub + // device when none exists so the hook doesn't throw, but keep isActive + // false so it never tries to start the session. + useCamera({ + isActive: pipelineState != null && cameraDevice != null, + device: cameraDevice as NonNullable, + outputs: [frameOutput], + }); + + if (error) { + return ( + + {error} + + ); + } + if (cameraDevice == null) { + return ( + + + No camera available. This screen needs a physical device with a camera + (the iOS Simulator does not have one). + + + ); + } + return ( + + + + ); +}; + +const styles = StyleSheet.create({ + errorContainer: { flex: 1, padding: 16, justifyContent: "center" }, + errorText: { color: "red", fontSize: 14 }, + permissionContainer: { + flex: 1, + padding: 24, + justifyContent: "center", + alignItems: "center", + }, + permissionText: { fontSize: 16, textAlign: "center", marginBottom: 16 }, + permissionButton: { + backgroundColor: "#007AFF", + paddingHorizontal: 24, + paddingVertical: 12, + borderRadius: 8, + }, + permissionButtonText: { color: "white", fontSize: 16, fontWeight: "600" }, +}); diff --git a/apps/example/src/VisionCamera/index.ts b/apps/example/src/VisionCamera/index.ts new file mode 100644 index 000000000..fc62066eb --- /dev/null +++ b/apps/example/src/VisionCamera/index.ts @@ -0,0 +1 @@ +export * from "./VisionCamera"; diff --git a/packages/webgpu/android/cpp/AndroidPlatformContext.h b/packages/webgpu/android/cpp/AndroidPlatformContext.h index 5f8a2577b..7c273be33 100644 --- a/packages/webgpu/android/cpp/AndroidPlatformContext.h +++ b/packages/webgpu/android/cpp/AndroidPlatformContext.h @@ -233,6 +233,13 @@ class AndroidPlatformContext : public PlatformContext { throw std::runtime_error( "writeTestVideoFile is not yet implemented on Android."); } + + VideoFrameHandle wrapNativeBuffer(void * /*pointer*/) override { + // TODO: AHardwareBuffer_acquire + extract dimensions, format, color + // metadata. + throw std::runtime_error( + "wrapNativeBuffer is not yet implemented on Android."); + } }; } // namespace rnwgpu diff --git a/packages/webgpu/apple/ApplePlatformContext.h b/packages/webgpu/apple/ApplePlatformContext.h index 5d4c78ec5..6536663c4 100644 --- a/packages/webgpu/apple/ApplePlatformContext.h +++ b/packages/webgpu/apple/ApplePlatformContext.h @@ -37,6 +37,8 @@ class ApplePlatformContext : public PlatformContext { VideoPixelFormat format) override; std::string writeTestVideoFile() override; + + VideoFrameHandle wrapNativeBuffer(void *pointer) override; }; } // namespace rnwgpu diff --git a/packages/webgpu/apple/ApplePlatformContext.mm b/packages/webgpu/apple/ApplePlatformContext.mm index 69280e5a1..594337cfc 100644 --- a/packages/webgpu/apple/ApplePlatformContext.mm +++ b/packages/webgpu/apple/ApplePlatformContext.mm @@ -246,6 +246,10 @@ void checkIfUsingSimulatorWithAPIValidation() { return writeAppleTestVideoFile(); } +VideoFrameHandle ApplePlatformContext::wrapNativeBuffer(void *pointer) { + return wrapCVPixelBuffer(static_cast(pointer)); +} + VideoFrameHandle ApplePlatformContext::createTestVideoFrame(uint32_t width, uint32_t height) { NSDictionary *attrs = @{ diff --git a/packages/webgpu/apple/AppleVideoPlayer.h b/packages/webgpu/apple/AppleVideoPlayer.h index 1962cc6c8..554d69d76 100644 --- a/packages/webgpu/apple/AppleVideoPlayer.h +++ b/packages/webgpu/apple/AppleVideoPlayer.h @@ -5,6 +5,10 @@ #include #include +#ifdef __OBJC__ +#import +#endif + namespace rnwgpu { // Factory: creates a new IVideoPlayer backed by AVPlayer + @@ -17,4 +21,11 @@ createAppleVideoPlayer(const std::string &path, VideoPixelFormat format); // example so it doesn't need a bundled .mp4. std::string writeAppleTestVideoFile(); +#ifdef __OBJC__ +// Build a VideoFrameHandle from an existing CVPixelBuffer. CFRetains the +// pixel buffer so the caller can release their reference immediately. Reads +// IOSurface, dimensions, pixel format, and YUV matrix off the buffer. +VideoFrameHandle wrapCVPixelBuffer(CVPixelBufferRef pixelBuffer); +#endif + } // namespace rnwgpu diff --git a/packages/webgpu/apple/AppleVideoPlayer.mm b/packages/webgpu/apple/AppleVideoPlayer.mm index 8a79012b1..64e713fd5 100644 --- a/packages/webgpu/apple/AppleVideoPlayer.mm +++ b/packages/webgpu/apple/AppleVideoPlayer.mm @@ -49,12 +49,24 @@ static void fillYuvMatrix(CVPixelBufferRef pixelBuffer, float out[12]) { } } +// Map a CVPixelBuffer's pixel format to our VideoPixelFormat enum. +static VideoPixelFormat pixelFormatFromCVPixelBuffer( + CVPixelBufferRef pixelBuffer) { + OSType type = CVPixelBufferGetPixelFormatType(pixelBuffer); + switch (type) { + case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: + case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange: + return VideoPixelFormat::NV12; + default: + return VideoPixelFormat::BGRA8; + } +} + class AppleVideoPlayer : public IVideoPlayer { public: AppleVideoPlayer(AVPlayer *player, AVPlayerItemVideoOutput *output, - id loopObserver, VideoPixelFormat pixelFormat) - : _player(player), _output(output), _loopObserver(loopObserver), - _pixelFormat(pixelFormat) {} + id loopObserver) + : _player(player), _output(output), _loopObserver(loopObserver) {} ~AppleVideoPlayer() override { if (_loopObserver) { @@ -71,29 +83,23 @@ VideoFrameHandle copyLatestFrame() override { if (![_output hasNewPixelBufferForItemTime:currentTime]) { return {}; } + // copyPixelBufferForItemTime returns a +1 retained CVPixelBuffer; we then + // hand it to wrapCVPixelBuffer which adds another retain. Balance with a + // CFRelease here so we don't leak. CVPixelBufferRef pixelBuffer = [_output copyPixelBufferForItemTime:currentTime itemTimeForDisplay:nullptr]; if (!pixelBuffer) { return {}; } - - IOSurfaceRef ioSurface = CVPixelBufferGetIOSurface(pixelBuffer); - if (!ioSurface) { + try { + auto handle = wrapCVPixelBuffer(pixelBuffer); CFRelease(pixelBuffer); - return {}; - } - - VideoFrameHandle handle; - handle.handle = (void *)ioSurface; - handle.width = static_cast(CVPixelBufferGetWidth(pixelBuffer)); - handle.height = static_cast(CVPixelBufferGetHeight(pixelBuffer)); - handle.pixelFormat = _pixelFormat; - if (_pixelFormat == VideoPixelFormat::NV12) { - fillYuvMatrix(pixelBuffer, handle.yuvToRgbMatrix); + return handle; + } catch (...) { + CFRelease(pixelBuffer); + throw; } - handle.deleter = [pixelBuffer]() { CFRelease(pixelBuffer); }; - return handle; } void play() override { [_player play]; } @@ -103,11 +109,36 @@ VideoFrameHandle copyLatestFrame() override { AVPlayer *_player; AVPlayerItemVideoOutput *_output; id _loopObserver; - VideoPixelFormat _pixelFormat; }; } // namespace +VideoFrameHandle wrapCVPixelBuffer(CVPixelBufferRef pixelBuffer) { + if (!pixelBuffer) { + throw std::runtime_error("wrapCVPixelBuffer: pointer is null"); + } + IOSurfaceRef ioSurface = CVPixelBufferGetIOSurface(pixelBuffer); + if (!ioSurface) { + throw std::runtime_error( + "wrapCVPixelBuffer: pixel buffer is not IOSurface-backed (was the " + "camera/video pipeline configured for Metal/IOSurface output?)"); + } + + // Retain the pixel buffer so the caller can release theirs immediately. + CFRetain(pixelBuffer); + + VideoFrameHandle handle; + handle.handle = (void *)ioSurface; + handle.width = static_cast(CVPixelBufferGetWidth(pixelBuffer)); + handle.height = static_cast(CVPixelBufferGetHeight(pixelBuffer)); + handle.pixelFormat = pixelFormatFromCVPixelBuffer(pixelBuffer); + if (handle.pixelFormat == VideoPixelFormat::NV12) { + fillYuvMatrix(pixelBuffer, handle.yuvToRgbMatrix); + } + handle.deleter = [pixelBuffer]() { CFRelease(pixelBuffer); }; + return handle; +} + std::unique_ptr createAppleVideoPlayer(const std::string &path, VideoPixelFormat format) { NSString *nsPath = [NSString stringWithUTF8String:path.c_str()]; @@ -155,8 +186,7 @@ VideoFrameHandle copyLatestFrame() override { [weakPlayer play]; }]; - return std::make_unique(player, output, loopObserver, - format); + return std::make_unique(player, output, loopObserver); } std::string writeAppleTestVideoFile() { diff --git a/packages/webgpu/cpp/rnwgpu/PlatformContext.h b/packages/webgpu/cpp/rnwgpu/PlatformContext.h index f46543484..cdab10da4 100644 --- a/packages/webgpu/cpp/rnwgpu/PlatformContext.h +++ b/packages/webgpu/cpp/rnwgpu/PlatformContext.h @@ -70,6 +70,14 @@ class PlatformContext { PlatformContext() = default; virtual ~PlatformContext() = default; + // Singleton-style accessor so leaf classes (e.g. GPUDevice) can reach the + // platform context without threading it through every constructor. Set by + // RNWebGPUManager at startup. + static std::shared_ptr &global() { + static std::shared_ptr instance; + return instance; + } + virtual wgpu::Surface makeSurface(wgpu::Instance instance, void *surface, int width, int height) = 0; virtual ImageData createImageBitmap(std::string blobId, double offset, @@ -110,6 +118,14 @@ class PlatformContext { virtual std::unique_ptr createVideoPlayer(const std::string &path, VideoPixelFormat format) = 0; + // Wrap a CVPixelBufferRef (Apple) or AHardwareBuffer* (Android) pointer + // obtained from another library (typically VisionCamera's + // Frame.getNativeBuffer().pointer) as one of our VideoFrame handles. + // + // We CFRetain / AHardwareBuffer_acquire on the way in, so callers can + // safely release their own reference immediately after. + virtual VideoFrameHandle wrapNativeBuffer(void *pointer) = 0; + // Write a small procedurally-generated test video to a temporary location // and return its absolute path. Lets the SharedTextureMemory example play // a real decoded video without bundling an asset. diff --git a/packages/webgpu/cpp/rnwgpu/RNWebGPUManager.cpp b/packages/webgpu/cpp/rnwgpu/RNWebGPUManager.cpp index 38f675d37..6b8a28cb3 100644 --- a/packages/webgpu/cpp/rnwgpu/RNWebGPUManager.cpp +++ b/packages/webgpu/cpp/rnwgpu/RNWebGPUManager.cpp @@ -63,10 +63,21 @@ RNWebGPUManager::RNWebGPUManager( // Register main runtime for RuntimeAwareCache BaseRuntimeAwareCache::setMainJsRuntime(_jsRuntime); + // Expose the platform context for leaf classes that need it (e.g. + // GPUDevice::createVideoFrameFromNativeBuffer) without threading it through + // every constructor. + PlatformContext::global() = _platformContext; + auto gpu = std::make_shared(*_jsRuntime); auto rnWebGPU = std::make_shared(gpu, _platformContext, _jsCallInvoker); _gpu = gpu->get(); + + // RNWebGPU needs its brand registered in NativeObjectRegistry so the boxing + // path can install the prototype on worklet runtimes. installConstructor + // does that registration but also sets globalThis.RNWebGPU = ctor, so we + // call it FIRST and then overwrite the global with the actual instance. + RNWebGPU::installConstructor(*_jsRuntime); _jsRuntime->global().setProperty(*_jsRuntime, "RNWebGPU", RNWebGPU::create(*_jsRuntime, rnWebGPU)); diff --git a/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp b/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp index 967351cb0..5b95ba010 100644 --- a/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp +++ b/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp @@ -8,6 +8,7 @@ #include "Convertors.h" #include "JSIConverter.h" +#include "PlatformContext.h" #include "GPUFeatures.h" #include "GPUInternalError.h" @@ -371,6 +372,19 @@ std::shared_ptr GPUDevice::importExternalTexture( #endif } +std::shared_ptr +GPUDevice::createVideoFrameFromNativeBuffer(uint64_t pointer) { + auto platformContext = PlatformContext::global(); + if (!platformContext) { + throw std::runtime_error( + "GPUDevice::createVideoFrameFromNativeBuffer(): PlatformContext is " + "not initialized"); + } + auto handle = + platformContext->wrapNativeBuffer(reinterpret_cast(pointer)); + return std::make_shared(std::move(handle)); +} + std::shared_ptr GPUDevice::importSharedTextureMemory( std::shared_ptr descriptor) { if (!descriptor || descriptor->handle == nullptr) { diff --git a/packages/webgpu/cpp/rnwgpu/api/GPUDevice.h b/packages/webgpu/cpp/rnwgpu/api/GPUDevice.h index 2ab1ddd14..28add6e0c 100644 --- a/packages/webgpu/cpp/rnwgpu/api/GPUDevice.h +++ b/packages/webgpu/cpp/rnwgpu/api/GPUDevice.h @@ -53,6 +53,7 @@ #include "GPUTexture.h" #include "GPUTextureDescriptor.h" #include "GPUUncapturedErrorEvent.h" +#include "VideoFrame.h" namespace rnwgpu { @@ -120,6 +121,12 @@ class GPUDevice : public NativeObject { std::shared_ptr descriptor); std::shared_ptr importSharedTextureMemory( std::shared_ptr descriptor); + // Wrap a CVPixelBufferRef / AHardwareBuffer* pointer (as a BigInt) into a + // VideoFrame. Mirrors RNWebGPU.createVideoFrameFromNativeBuffer but is + // reachable from worklet runtimes since GPUDevice is already serialized + // across the worklet boundary via the WebGPU custom serializer. + std::shared_ptr + createVideoFrameFromNativeBuffer(uint64_t pointer); std::shared_ptr createBindGroupLayout( std::shared_ptr descriptor); std::shared_ptr @@ -175,6 +182,8 @@ class GPUDevice : public NativeObject { &GPUDevice::importExternalTexture); installMethod(runtime, prototype, "importSharedTextureMemory", &GPUDevice::importSharedTextureMemory); + installMethod(runtime, prototype, "createVideoFrameFromNativeBuffer", + &GPUDevice::createVideoFrameFromNativeBuffer); installMethod(runtime, prototype, "createBindGroupLayout", &GPUDevice::createBindGroupLayout); installMethod(runtime, prototype, "createPipelineLayout", diff --git a/packages/webgpu/cpp/rnwgpu/api/RNWebGPU.h b/packages/webgpu/cpp/rnwgpu/api/RNWebGPU.h index c7457adc2..273f7065d 100644 --- a/packages/webgpu/cpp/rnwgpu/api/RNWebGPU.h +++ b/packages/webgpu/cpp/rnwgpu/api/RNWebGPU.h @@ -181,6 +181,14 @@ class RNWebGPU : public NativeObject { return std::make_shared(std::move(frame)); } + // Wrap a CVPixelBufferRef / AHardwareBuffer* pointer (typed as void* via + // BigInt on the JS side) into one of our VideoFrames. The native side + // CFRetains / acquires so the caller can release immediately. + std::shared_ptr createVideoFrameFromNativeBuffer(void *pointer) { + auto handle = _platformContext->wrapNativeBuffer(pointer); + return std::make_shared(std::move(handle)); + } + std::shared_ptr createVideoPlayer(std::string path, std::optional pixelFormat) { auto format = (pixelFormat && pixelFormat.value() == "nv12") @@ -218,6 +226,8 @@ class RNWebGPU : public NativeObject { &RNWebGPU::loadVideoFrame); installMethod(runtime, prototype, "createTestVideoFrame", &RNWebGPU::createTestVideoFrame); + installMethod(runtime, prototype, "createVideoFrameFromNativeBuffer", + &RNWebGPU::createVideoFrameFromNativeBuffer); installMethod(runtime, prototype, "createVideoPlayer", &RNWebGPU::createVideoPlayer); installMethod(runtime, prototype, "writeTestVideoFile", diff --git a/packages/webgpu/package.json b/packages/webgpu/package.json index bda3bc9a3..69060631e 100644 --- a/packages/webgpu/package.json +++ b/packages/webgpu/package.json @@ -84,7 +84,7 @@ "react-native-builder-bob": "^0.23.2", "react-native-reanimated": "^4.2.1", "react-native-web": "^0.21.2", - "react-native-worklets": "^0.7.0", + "react-native-worklets": "^0.8.3", "rimraf": "^5.0.7", "seedrandom": "^3.0.5", "teapot": "^1.0.0", diff --git a/packages/webgpu/src/Canvas.tsx b/packages/webgpu/src/Canvas.tsx index ef4cba14d..1dc8676fa 100644 --- a/packages/webgpu/src/Canvas.tsx +++ b/packages/webgpu/src/Canvas.tsx @@ -26,6 +26,9 @@ declare global { width: number, height: number, ) => import("./types").VideoFrame; + createVideoFrameFromNativeBuffer: ( + pointer: bigint, + ) => import("./types").VideoFrame; createVideoPlayer: ( path: string, pixelFormat?: import("./types").VideoPixelFormat, diff --git a/packages/webgpu/src/external/reanimated/registerWebGPUForReanimated.ts b/packages/webgpu/src/external/reanimated/registerWebGPUForReanimated.ts index 04feb3933..c3e9b8e6c 100644 --- a/packages/webgpu/src/external/reanimated/registerWebGPUForReanimated.ts +++ b/packages/webgpu/src/external/reanimated/registerWebGPUForReanimated.ts @@ -1,10 +1,14 @@ -// Declare global WebGPU worklet helper functions (installed by native module) -declare function __webgpuIsWebGPUObject(obj: unknown): boolean; +// Declare global WebGPU worklet helper functions (installed by native module +// on the main JS runtime only — secondary worklet runtimes like Vision +// Camera's frame-processor thread do NOT have these on their globalThis). declare function __webgpuBox(obj: object): { unbox: () => object; __boxedWebGPU: true; }; +// eslint-disable-next-line no-console +console.log("[WebGPU] registerWebGPUForReanimated module loaded"); + let isRegistered = false; /** @@ -22,22 +26,77 @@ export const registerWebGPUForReanimated = () => { try { const { registerCustomSerializable } = require("react-native-worklets"); + // eslint-disable-next-line no-console + console.log( + "[WebGPU] registering custom serializer (v2: __brand-getter check)", + ); + registerCustomSerializable({ name: "WebGPU", + // `determine` is invoked by Worklets in *both* runtimes during the + // serialization handshake, so its body must not rely on any globals + // beyond the JS built-ins. We inline the WebGPU-object check (native + // state + Symbol.toStringTag on prototype, same as the native + // __webgpuIsWebGPUObject helper) here so the function works on the + // main runtime and on every worklet runtime — including ones we don't + // own (Vision Camera, etc.). determine: (value: object): value is object => { "worklet"; - return __webgpuIsWebGPUObject(value); + if (value == null || typeof value !== "object") { + return false; + } + if ((value as { __boxedWebGPU?: boolean }).__boxedWebGPU === true) { + // eslint-disable-next-line no-console + console.log("[WebGPU determine] matched boxed object"); + return true; + } + const proto = Object.getPrototypeOf(value); + if (proto == null) { + return false; + } + const desc = Object.getOwnPropertyDescriptor(proto, "__brand"); + const matched = desc != null && typeof desc.get === "function"; + // Skip plain JS objects — they're handled by Worklets natively and + // would spam the log with every captured plain object. + if (proto !== Object.prototype) { + let brand: string | undefined; + try { + brand = desc?.get?.call(value) as string | undefined; + } catch { + brand = ""; + } + // eslint-disable-next-line no-console + console.log( + "[WebGPU determine] matched=" + + String(matched) + + " brand=" + + String(brand) + + " protoCtor=" + + String((proto.constructor as { name?: string })?.name), + ); + } + return matched; }, + // `pack` runs on the source runtime, which is always the main JS + // runtime in our setup (you can't create raw WebGPU objects inside a + // worklet), so __webgpuBox is available here. pack: (value: object) => { "worklet"; return __webgpuBox(value); }, + // `unpack` runs on the destination runtime and just calls the + // BoxedWebGPUObject's own `unbox` method — no globals needed. unpack: (boxed: { unbox: () => object }) => { "worklet"; return boxed.unbox(); }, }); - } catch { - // react-native-worklets not installed, skip registration + // eslint-disable-next-line no-console + console.log("[WebGPU] registerCustomSerializable call returned OK"); + } catch (e) { + // eslint-disable-next-line no-console + console.warn( + "[WebGPU] registerCustomSerializable threw: " + String(e), + ); } }; diff --git a/packages/webgpu/src/index.tsx b/packages/webgpu/src/index.tsx index 2244239fc..d42298069 100644 --- a/packages/webgpu/src/index.tsx +++ b/packages/webgpu/src/index.tsx @@ -35,6 +35,12 @@ declare global { width: number, height: number, ) => import("./types").VideoFrame; + // Wrap a NativeBuffer.pointer (CVPixelBufferRef on iOS / AHardwareBuffer* + // on Android) into a VideoFrame. Matches the shape used by libraries that + // emit NativeBuffer (e.g. react-native-vision-camera). + createVideoFrameFromNativeBuffer: ( + pointer: bigint, + ) => import("./types").VideoFrame; createVideoPlayer: ( path: string, pixelFormat?: import("./types").VideoPixelFormat, @@ -46,6 +52,12 @@ declare global { importSharedTextureMemory( descriptor: import("./types").GPUSharedTextureMemoryDescriptor, ): import("./types").GPUSharedTextureMemory; + // Wrap a NativeBuffer.pointer into a VideoFrame. Reachable from worklet + // runtimes (e.g. Vision Camera frame processors) because GPUDevice is + // serialized across worklet boundaries via the WebGPU custom serializer. + createVideoFrameFromNativeBuffer( + pointer: bigint, + ): import("./types").VideoFrame; } // Extend createImageBitmap to accept ArrayBuffer/TypedArray (encoded image bytes) diff --git a/packages/webgpu/src/main/index.tsx b/packages/webgpu/src/main/index.tsx index df21bd780..7fedf163e 100644 --- a/packages/webgpu/src/main/index.tsx +++ b/packages/webgpu/src/main/index.tsx @@ -12,27 +12,37 @@ const _installOk = WebGPUModule.install(); registerWebGPUForReanimated(); -if (typeof RNWebGPU !== "undefined") { - if (!navigator) { - // @ts-expect-error Navigation object is more complex than this, setting it to an empty object to add gpu property - navigator = { - gpu: RNWebGPU.gpu, - userAgent: "react-native", - }; - } else { - navigator.gpu = RNWebGPU.gpu; - if (typeof navigator.userAgent !== "string") { - try { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - Hermes navigator may not include a userAgent, align with the polyfill if needed - navigator.userAgent = "react-native"; - } catch { - // navigator.userAgent can be read-only; ignore if assignment fails +// `RNWebGPU` is only fully populated in the main JS runtime where +// WebGPUModule.install() was called against. When this bundle re-evaluates in +// secondary worklet runtimes (Reanimated UI, Vision Camera frame processor, +// etc.) `RNWebGPU` may either be undefined or be a stripped-down stub. Guard +// every member access so a missing property doesn't take down the whole +// module graph (which would surface as `Cannot read property 'bind' of +// undefined` + every downstream import returning undefined). +if (typeof RNWebGPU !== "undefined" && RNWebGPU != null) { + if (RNWebGPU.gpu) { + if (!navigator) { + // @ts-expect-error Navigation object is more complex than this, setting it to an empty object to add gpu property + navigator = { + gpu: RNWebGPU.gpu, + userAgent: "react-native", + }; + } else { + navigator.gpu = RNWebGPU.gpu; + if (typeof navigator.userAgent !== "string") { + try { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore - Hermes navigator may not include a userAgent, align with the polyfill if needed + navigator.userAgent = "react-native"; + } catch { + // navigator.userAgent can be read-only; ignore if assignment fails + } } } } - global.createImageBitmap = - global.createImageBitmap ?? RNWebGPU.createImageBitmap.bind(RNWebGPU); + if (!global.createImageBitmap && typeof RNWebGPU.createImageBitmap === "function") { + global.createImageBitmap = RNWebGPU.createImageBitmap.bind(RNWebGPU); + } } else { console.warn( `[react-native-wgpu] install() returned ${_installOk} but RNWebGPU global is not available`, diff --git a/yarn.lock b/yarn.lock index f6a59461c..a1fd1235a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -95,7 +95,7 @@ __metadata: languageName: node linkType: hard -"@babel/helper-create-class-features-plugin@npm:^7.18.6, @babel/helper-create-class-features-plugin@npm:^7.27.1, @babel/helper-create-class-features-plugin@npm:^7.28.6": +"@babel/helper-create-class-features-plugin@npm:^7.18.6, @babel/helper-create-class-features-plugin@npm:^7.28.6": version: 7.28.6 resolution: "@babel/helper-create-class-features-plugin@npm:7.28.6" dependencies: @@ -718,7 +718,7 @@ __metadata: languageName: node linkType: hard -"@babel/plugin-transform-arrow-functions@npm:7.27.1, @babel/plugin-transform-arrow-functions@npm:^7.0.0, @babel/plugin-transform-arrow-functions@npm:^7.24.7, @babel/plugin-transform-arrow-functions@npm:^7.27.1": +"@babel/plugin-transform-arrow-functions@npm:^7.0.0, @babel/plugin-transform-arrow-functions@npm:^7.24.7, @babel/plugin-transform-arrow-functions@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-arrow-functions@npm:7.27.1" dependencies: @@ -777,19 +777,7 @@ __metadata: languageName: node linkType: hard -"@babel/plugin-transform-class-properties@npm:7.27.1": - version: 7.27.1 - resolution: "@babel/plugin-transform-class-properties@npm:7.27.1" - dependencies: - "@babel/helper-create-class-features-plugin": ^7.27.1 - "@babel/helper-plugin-utils": ^7.27.1 - peerDependencies: - "@babel/core": ^7.0.0-0 - checksum: 475a6e5a9454912fe1bdc171941976ca10ea4e707675d671cdb5ce6b6761d84d1791ac61b6bca81a2e5f6430cb7b9d8e4b2392404110e69c28207a754e196294 - languageName: node - linkType: hard - -"@babel/plugin-transform-class-properties@npm:^7.25.4, @babel/plugin-transform-class-properties@npm:^7.28.6": +"@babel/plugin-transform-class-properties@npm:^7.25.4, @babel/plugin-transform-class-properties@npm:^7.27.1, @babel/plugin-transform-class-properties@npm:^7.28.6": version: 7.28.6 resolution: "@babel/plugin-transform-class-properties@npm:7.28.6" dependencies: @@ -813,23 +801,7 @@ __metadata: languageName: node linkType: hard -"@babel/plugin-transform-classes@npm:7.28.4": - version: 7.28.4 - resolution: "@babel/plugin-transform-classes@npm:7.28.4" - dependencies: - "@babel/helper-annotate-as-pure": ^7.27.3 - "@babel/helper-compilation-targets": ^7.27.2 - "@babel/helper-globals": ^7.28.0 - "@babel/helper-plugin-utils": ^7.27.1 - "@babel/helper-replace-supers": ^7.27.1 - "@babel/traverse": ^7.28.4 - peerDependencies: - "@babel/core": ^7.0.0-0 - checksum: f412e00c86584a9094cc0a2f3dd181b8108a4dced477d609c5406beddd5bf79d05a7ea74db508dc4dcb37172f042d5ef98d3d6311ade61c7ea6fbbbb70f5ec29 - languageName: node - linkType: hard - -"@babel/plugin-transform-classes@npm:^7.0.0, @babel/plugin-transform-classes@npm:^7.25.4, @babel/plugin-transform-classes@npm:^7.28.6": +"@babel/plugin-transform-classes@npm:^7.0.0, @babel/plugin-transform-classes@npm:^7.25.4, @babel/plugin-transform-classes@npm:^7.28.4, @babel/plugin-transform-classes@npm:^7.28.6": version: 7.28.6 resolution: "@babel/plugin-transform-classes@npm:7.28.6" dependencies: @@ -1103,18 +1075,7 @@ __metadata: languageName: node linkType: hard -"@babel/plugin-transform-nullish-coalescing-operator@npm:7.27.1": - version: 7.27.1 - resolution: "@babel/plugin-transform-nullish-coalescing-operator@npm:7.27.1" - dependencies: - "@babel/helper-plugin-utils": ^7.27.1 - peerDependencies: - "@babel/core": ^7.0.0-0 - checksum: 1c6b3730748782d2178cc30f5cc37be7d7666148260f3f2dfc43999908bdd319bdfebaaf19cf04ac1f9dee0f7081093d3fa730cda5ae1b34bcd73ce406a78be7 - languageName: node - linkType: hard - -"@babel/plugin-transform-nullish-coalescing-operator@npm:^7.24.7, @babel/plugin-transform-nullish-coalescing-operator@npm:^7.28.6": +"@babel/plugin-transform-nullish-coalescing-operator@npm:^7.24.7, @babel/plugin-transform-nullish-coalescing-operator@npm:^7.27.1, @babel/plugin-transform-nullish-coalescing-operator@npm:^7.28.6": version: 7.28.6 resolution: "@babel/plugin-transform-nullish-coalescing-operator@npm:7.28.6" dependencies: @@ -1174,18 +1135,6 @@ __metadata: languageName: node linkType: hard -"@babel/plugin-transform-optional-chaining@npm:7.27.1": - version: 7.27.1 - resolution: "@babel/plugin-transform-optional-chaining@npm:7.27.1" - dependencies: - "@babel/helper-plugin-utils": ^7.27.1 - "@babel/helper-skip-transparent-expression-wrappers": ^7.27.1 - peerDependencies: - "@babel/core": ^7.0.0-0 - checksum: c4428d31f182d724db6f10575669aad3dbccceb0dea26aa9071fa89f11b3456278da3097fcc78937639a13c105a82cd452dc0218ce51abdbcf7626a013b928a5 - languageName: node - linkType: hard - "@babel/plugin-transform-optional-chaining@npm:^7.24.8, @babel/plugin-transform-optional-chaining@npm:^7.27.1, @babel/plugin-transform-optional-chaining@npm:^7.28.6": version: 7.28.6 resolution: "@babel/plugin-transform-optional-chaining@npm:7.28.6" @@ -1366,7 +1315,7 @@ __metadata: languageName: node linkType: hard -"@babel/plugin-transform-shorthand-properties@npm:7.27.1, @babel/plugin-transform-shorthand-properties@npm:^7.0.0, @babel/plugin-transform-shorthand-properties@npm:^7.24.7, @babel/plugin-transform-shorthand-properties@npm:^7.27.1": +"@babel/plugin-transform-shorthand-properties@npm:^7.0.0, @babel/plugin-transform-shorthand-properties@npm:^7.24.7, @babel/plugin-transform-shorthand-properties@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-shorthand-properties@npm:7.27.1" dependencies: @@ -1400,7 +1349,7 @@ __metadata: languageName: node linkType: hard -"@babel/plugin-transform-template-literals@npm:7.27.1, @babel/plugin-transform-template-literals@npm:^7.27.1": +"@babel/plugin-transform-template-literals@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-template-literals@npm:7.27.1" dependencies: @@ -1422,7 +1371,7 @@ __metadata: languageName: node linkType: hard -"@babel/plugin-transform-typescript@npm:^7.25.2, @babel/plugin-transform-typescript@npm:^7.27.1, @babel/plugin-transform-typescript@npm:^7.28.5, @babel/plugin-transform-typescript@npm:^7.5.0": +"@babel/plugin-transform-typescript@npm:^7.25.2, @babel/plugin-transform-typescript@npm:^7.28.5, @babel/plugin-transform-typescript@npm:^7.5.0": version: 7.28.6 resolution: "@babel/plugin-transform-typescript@npm:7.28.6" dependencies: @@ -1460,7 +1409,7 @@ __metadata: languageName: node linkType: hard -"@babel/plugin-transform-unicode-regex@npm:7.27.1, @babel/plugin-transform-unicode-regex@npm:^7.0.0, @babel/plugin-transform-unicode-regex@npm:^7.24.7, @babel/plugin-transform-unicode-regex@npm:^7.27.1": +"@babel/plugin-transform-unicode-regex@npm:^7.0.0, @babel/plugin-transform-unicode-regex@npm:^7.24.7, @babel/plugin-transform-unicode-regex@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-unicode-regex@npm:7.27.1" dependencies: @@ -1606,22 +1555,7 @@ __metadata: languageName: node linkType: hard -"@babel/preset-typescript@npm:7.27.1": - version: 7.27.1 - resolution: "@babel/preset-typescript@npm:7.27.1" - dependencies: - "@babel/helper-plugin-utils": ^7.27.1 - "@babel/helper-validator-option": ^7.27.1 - "@babel/plugin-syntax-jsx": ^7.27.1 - "@babel/plugin-transform-modules-commonjs": ^7.27.1 - "@babel/plugin-transform-typescript": ^7.27.1 - peerDependencies: - "@babel/core": ^7.0.0-0 - checksum: 38020f1b23e88ec4fbffd5737da455d8939244bddfb48a2516aef93fb5947bd9163fb807ce6eff3e43fa5ffe9113aa131305fef0fb5053998410bbfcfe6ce0ec - languageName: node - linkType: hard - -"@babel/preset-typescript@npm:^7.13.0, @babel/preset-typescript@npm:^7.17.12": +"@babel/preset-typescript@npm:^7.13.0, @babel/preset-typescript@npm:^7.17.12, @babel/preset-typescript@npm:^7.27.1": version: 7.28.5 resolution: "@babel/preset-typescript@npm:7.28.5" dependencies: @@ -1669,7 +1603,7 @@ __metadata: languageName: node linkType: hard -"@babel/traverse--for-generate-function-map@npm:@babel/traverse@^7.25.3, @babel/traverse@npm:^7.20.0, @babel/traverse@npm:^7.25.3, @babel/traverse@npm:^7.27.1, @babel/traverse@npm:^7.28.4, @babel/traverse@npm:^7.28.5, @babel/traverse@npm:^7.28.6": +"@babel/traverse--for-generate-function-map@npm:@babel/traverse@^7.25.3, @babel/traverse@npm:^7.20.0, @babel/traverse@npm:^7.25.3, @babel/traverse@npm:^7.27.1, @babel/traverse@npm:^7.28.5, @babel/traverse@npm:^7.28.6": version: 7.28.6 resolution: "@babel/traverse@npm:7.28.6" dependencies: @@ -4965,12 +4899,16 @@ __metadata: react-native: 0.81.4 react-native-gesture-handler: ^2.28.0 react-native-macos: ^0.79.0 - react-native-reanimated: 4.2.1 + react-native-nitro-image: ^0.14.0 + react-native-nitro-modules: ^0.35.6 + react-native-reanimated: 4.3.1 react-native-safe-area-context: ^5.4.0 react-native-test-app: 4.4.10 + react-native-vision-camera: ^5.0.9 + react-native-vision-camera-worklets: ^5.0.9 react-native-web: ^0.21.2 react-native-wgpu: "*" - react-native-worklets: 0.7.2 + react-native-worklets: 0.8.3 react-test-renderer: 18.2.0 teapot: ^1.0.0 three: 0.172.0 @@ -6282,7 +6220,7 @@ __metadata: languageName: node linkType: hard -"convert-source-map@npm:2.0.0, convert-source-map@npm:^2.0.0": +"convert-source-map@npm:^2.0.0": version: 2.0.0 resolution: "convert-source-map@npm:2.0.0" checksum: 63ae9933be5a2b8d4509daca5124e20c14d023c820258e484e32dc324d34c2754e71297c94a05784064ad27615037ef677e3f0c00469fb55f409d2bb21261035 @@ -12714,6 +12652,16 @@ __metadata: languageName: node linkType: hard +"react-native-is-edge-to-edge@npm:^1.3.1": + version: 1.3.1 + resolution: "react-native-is-edge-to-edge@npm:1.3.1" + peerDependencies: + react: "*" + react-native: "*" + checksum: dc82d54e0bf8f89208a538bb0d14e4891af6efae27ed5b7b21be683a72c38c5219ab9be1ea9bd40aa1c905d481174e649d0b71aeceaa9946e6c707f251568282 + languageName: node + linkType: hard + "react-native-macos@npm:^0.79.0": version: 0.79.1 resolution: "react-native-macos@npm:0.79.1" @@ -12769,7 +12717,42 @@ __metadata: languageName: node linkType: hard -"react-native-reanimated@npm:4.2.1, react-native-reanimated@npm:^4.2.1": +"react-native-nitro-image@npm:^0.14.0": + version: 0.14.0 + resolution: "react-native-nitro-image@npm:0.14.0" + peerDependencies: + react: "*" + react-native: "*" + react-native-nitro-modules: "*" + checksum: ea5d19665bdffc8bb38fd5fef78a7d9c5f3f033cf0e0b6c224a39f718b863f3de74d776765eb7f5440743e985e2ea6c9ab210a1545947b0dcf02570408af5877 + languageName: node + linkType: hard + +"react-native-nitro-modules@npm:^0.35.6": + version: 0.35.6 + resolution: "react-native-nitro-modules@npm:0.35.6" + peerDependencies: + react: "*" + react-native: "*" + checksum: b057dc606c4717bff2447cff3efdeebf5cd7342fa11b3f51cfd420ccce1224d0b4be5999ea63c9856721520f1cbb50fba9372109f49a4c8b14202d7ab4d3b37e + languageName: node + linkType: hard + +"react-native-reanimated@npm:4.3.1": + version: 4.3.1 + resolution: "react-native-reanimated@npm:4.3.1" + dependencies: + react-native-is-edge-to-edge: ^1.3.1 + semver: ^7.7.3 + peerDependencies: + react: "*" + react-native: 0.81 - 0.85 + react-native-worklets: 0.8.x + checksum: 2649672f72dbd52aa612e5c9bcf8ab2dc256d614ed3cd21cd44ba97b26eefd30e0be629f5aec6ba209ccc07aad46831866b69f6bf51fb9708d49431bd608507a + languageName: node + linkType: hard + +"react-native-reanimated@npm:^4.2.1": version: 4.2.1 resolution: "react-native-reanimated@npm:4.2.1" dependencies: @@ -12858,6 +12841,31 @@ __metadata: languageName: node linkType: hard +"react-native-vision-camera-worklets@npm:^5.0.9": + version: 5.0.9 + resolution: "react-native-vision-camera-worklets@npm:5.0.9" + peerDependencies: + react: "*" + react-native: "*" + react-native-nitro-modules: "*" + react-native-vision-camera: "*" + react-native-worklets: "*" + checksum: e7133fb013efb51bd78da707176a8834f668f117e0b98da7c01eef7c090c30d0fcd42bf534330691d41b6ad27d3221b4565e4721953b2d74fb0071432da7bef4 + languageName: node + linkType: hard + +"react-native-vision-camera@npm:^5.0.9": + version: 5.0.9 + resolution: "react-native-vision-camera@npm:5.0.9" + peerDependencies: + react: "*" + react-native: "*" + react-native-nitro-image: "*" + react-native-nitro-modules: "*" + checksum: 34423f0f4c71ae9f6ce3eb22a2622ac0491f7808776d90d1e0e005b1adee4e1119b5688115f739467942b6952665e86b7f3ae13b9a6f649298fa1528aeb6c0ef + languageName: node + linkType: hard + "react-native-web@npm:^0.21.2": version: 0.21.2 resolution: "react-native-web@npm:0.21.2" @@ -12918,7 +12926,7 @@ __metadata: react-native-builder-bob: ^0.23.2 react-native-reanimated: ^4.2.1 react-native-web: ^0.21.2 - react-native-worklets: ^0.7.0 + react-native-worklets: ^0.8.3 rimraf: ^5.0.7 seedrandom: ^3.0.5 teapot: ^1.0.0 @@ -12941,26 +12949,27 @@ __metadata: languageName: unknown linkType: soft -"react-native-worklets@npm:0.7.2, react-native-worklets@npm:^0.7.0": - version: 0.7.2 - resolution: "react-native-worklets@npm:0.7.2" - dependencies: - "@babel/plugin-transform-arrow-functions": 7.27.1 - "@babel/plugin-transform-class-properties": 7.27.1 - "@babel/plugin-transform-classes": 7.28.4 - "@babel/plugin-transform-nullish-coalescing-operator": 7.27.1 - "@babel/plugin-transform-optional-chaining": 7.27.1 - "@babel/plugin-transform-shorthand-properties": 7.27.1 - "@babel/plugin-transform-template-literals": 7.27.1 - "@babel/plugin-transform-unicode-regex": 7.27.1 - "@babel/preset-typescript": 7.27.1 - convert-source-map: 2.0.0 - semver: 7.7.3 +"react-native-worklets@npm:0.8.3, react-native-worklets@npm:^0.8.3": + version: 0.8.3 + resolution: "react-native-worklets@npm:0.8.3" + dependencies: + "@babel/plugin-transform-arrow-functions": ^7.27.1 + "@babel/plugin-transform-class-properties": ^7.27.1 + "@babel/plugin-transform-classes": ^7.28.4 + "@babel/plugin-transform-nullish-coalescing-operator": ^7.27.1 + "@babel/plugin-transform-optional-chaining": ^7.27.1 + "@babel/plugin-transform-shorthand-properties": ^7.27.1 + "@babel/plugin-transform-template-literals": ^7.27.1 + "@babel/plugin-transform-unicode-regex": ^7.27.1 + "@babel/preset-typescript": ^7.27.1 + convert-source-map: ^2.0.0 + semver: ^7.7.3 peerDependencies: "@babel/core": "*" + "@react-native/metro-config": "*" react: "*" - react-native: "*" - checksum: a1d220499c40e44286b4b5f97e6e0483ec6f84cda7f87db56e4c5c49b0991d749b9087d03915a7dfb8d5aa892308d9ba8661403e48a5fad8396e91b7b35228a2 + react-native: 0.81 - 0.85 + checksum: a44e50fc0ec765612df3f6b6104444d28e94634f3642b689df2638c5c9187ddadd7069953bb9d9a68ca253422e6630db36ffcfee80eea5f71f73cc99539307fd languageName: node linkType: hard From 7c26343c269666ddd58ecdc108bc2215afe5ca37 Mon Sep 17 00:00:00 2001 From: William Candillon Date: Wed, 20 May 2026 08:50:10 +0200 Subject: [PATCH 07/46] :wrench: --- apps/example/ios/Podfile.lock | 24 ++++++++++++------------ apps/example/package.json | 2 +- yarn.lock | 28 ++++++++++++++++++++++++++-- 3 files changed, 39 insertions(+), 15 deletions(-) diff --git a/apps/example/ios/Podfile.lock b/apps/example/ios/Podfile.lock index e21cd966e..ddd0b7c14 100644 --- a/apps/example/ios/Podfile.lock +++ b/apps/example/ios/Podfile.lock @@ -2519,7 +2519,7 @@ PODS: - ReactCommon/turbomodule/core - SocketRocket - Yoga - - RNReanimated (4.2.1): + - RNReanimated (4.3.1): - boost - DoubleConversion - fast_float @@ -2546,11 +2546,12 @@ PODS: - ReactCodegen - ReactCommon/turbomodule/bridging - ReactCommon/turbomodule/core - - RNReanimated/reanimated (= 4.2.1) + - RNReanimated/apple (= 4.3.1) + - RNReanimated/common (= 4.3.1) - RNWorklets - SocketRocket - Yoga - - RNReanimated/reanimated (4.2.1): + - RNReanimated/apple (4.3.1): - boost - DoubleConversion - fast_float @@ -2577,11 +2578,10 @@ PODS: - ReactCodegen - ReactCommon/turbomodule/bridging - ReactCommon/turbomodule/core - - RNReanimated/reanimated/apple (= 4.2.1) - RNWorklets - SocketRocket - Yoga - - RNReanimated/reanimated/apple (4.2.1): + - RNReanimated/common (4.3.1): - boost - DoubleConversion - fast_float @@ -2611,7 +2611,7 @@ PODS: - RNWorklets - SocketRocket - Yoga - - RNWorklets (0.7.2): + - RNWorklets (0.8.3): - boost - DoubleConversion - fast_float @@ -2638,10 +2638,11 @@ PODS: - ReactCodegen - ReactCommon/turbomodule/bridging - ReactCommon/turbomodule/core - - RNWorklets/worklets (= 0.7.2) + - RNWorklets/apple (= 0.8.3) + - RNWorklets/common (= 0.8.3) - SocketRocket - Yoga - - RNWorklets/worklets (0.7.2): + - RNWorklets/apple (0.8.3): - boost - DoubleConversion - fast_float @@ -2668,10 +2669,9 @@ PODS: - ReactCodegen - ReactCommon/turbomodule/bridging - ReactCommon/turbomodule/core - - RNWorklets/worklets/apple (= 0.7.2) - SocketRocket - Yoga - - RNWorklets/worklets/apple (0.7.2): + - RNWorklets/common (0.8.3): - boost - DoubleConversion - fast_float @@ -3074,8 +3074,8 @@ SPEC CHECKSUMS: ReactTestApp-DevSupport: 9b7bbba5e8fed998e763809171d9906a1375f9d3 ReactTestApp-Resources: 1bd9ff10e4c24f2ad87101a32023721ae923bccf RNGestureHandler: e37bdb684df1ac17c7e1d8f71a3311b2793c186b - RNReanimated: 464375ff2caa801358547c44eca894ff0bf68e74 - RNWorklets: ee58e869ea579800ec5f2f1cb6ae195fd3537546 + RNReanimated: 9d012d4031abc9df896f8a82f9928eb2b9eae417 + RNWorklets: 0da2552f9ff5d17506918a692304110cfebb9f0a SocketRocket: d4aabe649be1e368d1318fdf28a022d714d65748 VisionCamera: 8c913d0cb2c868f779035fb69a1e0ab69e10f1c3 Yoga: a3ed390a19db0459bd6839823a6ac6d9c6db198d diff --git a/apps/example/package.json b/apps/example/package.json index 63d22f2d4..754be2f96 100644 --- a/apps/example/package.json +++ b/apps/example/package.json @@ -35,7 +35,7 @@ "react-native-macos": "^0.79.0", "react-native-nitro-image": "0.14.0", "react-native-nitro-modules": "0.35.7", - "react-native-reanimated": "4.2.1", + "react-native-reanimated": "4.3.1", "react-native-safe-area-context": "^5.4.0", "react-native-vision-camera": "5.0.10", "react-native-web": "^0.21.2", diff --git a/yarn.lock b/yarn.lock index 5b379b701..ea326947f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4901,7 +4901,7 @@ __metadata: react-native-macos: ^0.79.0 react-native-nitro-image: 0.14.0 react-native-nitro-modules: 0.35.7 - react-native-reanimated: 4.2.1 + react-native-reanimated: 4.3.1 react-native-safe-area-context: ^5.4.0 react-native-test-app: 4.4.10 react-native-vision-camera: 5.0.10 @@ -12651,6 +12651,16 @@ __metadata: languageName: node linkType: hard +"react-native-is-edge-to-edge@npm:^1.3.1": + version: 1.3.1 + resolution: "react-native-is-edge-to-edge@npm:1.3.1" + peerDependencies: + react: "*" + react-native: "*" + checksum: dc82d54e0bf8f89208a538bb0d14e4891af6efae27ed5b7b21be683a72c38c5219ab9be1ea9bd40aa1c905d481174e649d0b71aeceaa9946e6c707f251568282 + languageName: node + linkType: hard + "react-native-macos@npm:^0.79.0": version: 0.79.1 resolution: "react-native-macos@npm:0.79.1" @@ -12727,7 +12737,21 @@ __metadata: languageName: node linkType: hard -"react-native-reanimated@npm:4.2.1, react-native-reanimated@npm:^4.2.1": +"react-native-reanimated@npm:4.3.1": + version: 4.3.1 + resolution: "react-native-reanimated@npm:4.3.1" + dependencies: + react-native-is-edge-to-edge: ^1.3.1 + semver: ^7.7.3 + peerDependencies: + react: "*" + react-native: 0.81 - 0.85 + react-native-worklets: 0.8.x + checksum: 2649672f72dbd52aa612e5c9bcf8ab2dc256d614ed3cd21cd44ba97b26eefd30e0be629f5aec6ba209ccc07aad46831866b69f6bf51fb9708d49431bd608507a + languageName: node + linkType: hard + +"react-native-reanimated@npm:^4.2.1": version: 4.2.1 resolution: "react-native-reanimated@npm:4.2.1" dependencies: From 6f9b11a8cd49e6aee538ca7c39228aabf80fac6a Mon Sep 17 00:00:00 2001 From: William Candillon Date: Wed, 20 May 2026 10:04:10 +0200 Subject: [PATCH 08/46] :wrench: --- apps/example/ios/Podfile.lock | 36 +++++++++++++++++++++++++++++++++++ apps/example/package.json | 1 + yarn.lock | 14 ++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/apps/example/ios/Podfile.lock b/apps/example/ios/Podfile.lock index ddd0b7c14..fd5ba968c 100644 --- a/apps/example/ios/Podfile.lock +++ b/apps/example/ios/Podfile.lock @@ -2732,6 +2732,38 @@ PODS: - ReactCommon/turbomodule/core - SocketRocket - Yoga + - VisionCameraWorklets (5.0.10): + - boost + - DoubleConversion + - fast_float + - fmt + - glog + - hermes-engine + - NitroModules + - RCT-Folly + - RCT-Folly/Fabric + - RCTRequired + - RCTTypeSafety + - React-callinvoker + - React-Core + - React-debug + - React-Fabric + - React-featureflags + - React-graphics + - React-ImageManager + - React-jsi + - React-NativeModulesApple + - React-RCTFabric + - React-renderercss + - React-rendererdebug + - React-utils + - ReactCodegen + - ReactCommon/turbomodule/bridging + - ReactCommon/turbomodule/core + - RNWorklets + - SocketRocket + - VisionCamera + - Yoga - Yoga (0.0.0) DEPENDENCIES: @@ -2819,6 +2851,7 @@ DEPENDENCIES: - RNWorklets (from `../../../node_modules/react-native-worklets`) - SocketRocket (~> 0.7.1) - VisionCamera (from `../../../node_modules/react-native-vision-camera`) + - VisionCameraWorklets (from `../../../node_modules/react-native-vision-camera-worklets`) - Yoga (from `../../../node_modules/react-native/ReactCommon/yoga`) SPEC REPOS: @@ -2991,6 +3024,8 @@ EXTERNAL SOURCES: :path: "../../../node_modules/react-native-worklets" VisionCamera: :path: "../../../node_modules/react-native-vision-camera" + VisionCameraWorklets: + :path: "../../../node_modules/react-native-vision-camera-worklets" Yoga: :path: "../../../node_modules/react-native/ReactCommon/yoga" @@ -3078,6 +3113,7 @@ SPEC CHECKSUMS: RNWorklets: 0da2552f9ff5d17506918a692304110cfebb9f0a SocketRocket: d4aabe649be1e368d1318fdf28a022d714d65748 VisionCamera: 8c913d0cb2c868f779035fb69a1e0ab69e10f1c3 + VisionCameraWorklets: cc88e3e6d7304e2c00a95cae4f728aec5a6758c1 Yoga: a3ed390a19db0459bd6839823a6ac6d9c6db198d PODFILE CHECKSUM: 22a8651333bf096f67ca333598bd33455d994c1f diff --git a/apps/example/package.json b/apps/example/package.json index 754be2f96..5b27dd785 100644 --- a/apps/example/package.json +++ b/apps/example/package.json @@ -38,6 +38,7 @@ "react-native-reanimated": "4.3.1", "react-native-safe-area-context": "^5.4.0", "react-native-vision-camera": "5.0.10", + "react-native-vision-camera-worklets": "5.0.10", "react-native-web": "^0.21.2", "react-native-wgpu": "*", "react-native-worklets": "0.8.3", diff --git a/yarn.lock b/yarn.lock index ea326947f..d4cd4f246 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4905,6 +4905,7 @@ __metadata: react-native-safe-area-context: ^5.4.0 react-native-test-app: 4.4.10 react-native-vision-camera: 5.0.10 + react-native-vision-camera-worklets: 5.0.10 react-native-web: ^0.21.2 react-native-wgpu: "*" react-native-worklets: 0.8.3 @@ -12840,6 +12841,19 @@ __metadata: languageName: node linkType: hard +"react-native-vision-camera-worklets@npm:5.0.10": + version: 5.0.10 + resolution: "react-native-vision-camera-worklets@npm:5.0.10" + peerDependencies: + react: "*" + react-native: "*" + react-native-nitro-modules: "*" + react-native-vision-camera: "*" + react-native-worklets: "*" + checksum: 0e864993c90593d8ce0ec1a3eac990b29e75b58be556a1c113cb303e3ca9c25f072fd202369109fdabf3562d23c3edb1f05405b8934b2515ef09aa35a85378a9 + languageName: node + linkType: hard + "react-native-vision-camera@npm:5.0.10": version: 5.0.10 resolution: "react-native-vision-camera@npm:5.0.10" From bb740562fe71e9565b0c789927cd7087fcb20cb4 Mon Sep 17 00:00:00 2001 From: William Candillon Date: Wed, 20 May 2026 11:14:40 +0200 Subject: [PATCH 09/46] :wrench: --- .../example/src/VisionCamera/VisionCamera.tsx | 19 ++---- .../android/cpp/AndroidPlatformContext.h | 34 ++++++++-- packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp | 68 +++++++++++++++++++ packages/webgpu/cpp/rnwgpu/api/GPUFeatures.h | 3 + packages/webgpu/cpp/rnwgpu/api/RnFeatures.h | 7 +- .../cpp/rnwgpu/api/descriptors/Unions.h | 5 ++ 6 files changed, 116 insertions(+), 20 deletions(-) diff --git a/apps/example/src/VisionCamera/VisionCamera.tsx b/apps/example/src/VisionCamera/VisionCamera.tsx index 2ff5a9555..66684ff88 100644 --- a/apps/example/src/VisionCamera/VisionCamera.tsx +++ b/apps/example/src/VisionCamera/VisionCamera.tsx @@ -2,7 +2,6 @@ import React, { useEffect } from "react"; import { Linking, PixelRatio, - Platform, StyleSheet, Text, TouchableOpacity, @@ -80,18 +79,10 @@ fn fs_main(in: VsOut) -> @location(0) vec4f { } `; -const REQUIRED_FEATURES = - Platform.OS === "ios" - ? [ - "shared-texture-memory-iosurface", - "shared-fence-mtl-shared-event", - "dawn-multi-planar-formats", - ] - : [ - "shared-texture-memory-ahardware-buffer", - "shared-fence-vk-semaphore-sync-fd", - "dawn-multi-planar-formats", - ]; +const REQUIRED_FEATURES: GPUFeatureName[] = [ + "rnwebgpu/shared-texture-memory" as GPUFeatureName, + "dawn-multi-planar-formats" as GPUFeatureName, +]; const ABERRATION_STRENGTH = 0.006; @@ -124,7 +115,7 @@ export const VisionCamera = () => { const CameraView = () => { const ref = useCanvasRef(); const { device, adapter } = useDevice(undefined, { - requiredFeatures: REQUIRED_FEATURES as unknown as GPUFeatureName[], + requiredFeatures: REQUIRED_FEATURES, }); const devices = useCameraDevices(); // Pick back camera if available, otherwise front, otherwise anything. The diff --git a/packages/webgpu/android/cpp/AndroidPlatformContext.h b/packages/webgpu/android/cpp/AndroidPlatformContext.h index cff19c448..080b1bb5e 100644 --- a/packages/webgpu/android/cpp/AndroidPlatformContext.h +++ b/packages/webgpu/android/cpp/AndroidPlatformContext.h @@ -293,11 +293,35 @@ class AndroidPlatformContext : public PlatformContext { "writeTestVideoFile is not yet implemented on Android."); } - VideoFrameHandle wrapNativeBuffer(void * /*pointer*/) override { - // TODO: AHardwareBuffer_acquire + extract dimensions, format, color - // metadata. - throw std::runtime_error( - "wrapNativeBuffer is not yet implemented on Android."); + VideoFrameHandle wrapNativeBuffer(void *pointer) override { + if (!pointer) { + throw std::runtime_error("wrapNativeBuffer: pointer is null"); + } + auto *buffer = static_cast(pointer); + + AHardwareBuffer_Desc desc = {}; + AHardwareBuffer_describe(buffer, &desc); + + AHardwareBuffer_acquire(buffer); + + VideoFrameHandle handle; + handle.handle = static_cast(buffer); + handle.width = desc.width; + handle.height = desc.height; + // YUV / opaque formats route through Vulkan's SamplerYcbcrConversion via + // Dawn's OpaqueYCbCrAndroidForExternalTexture path. Single-plane RGBA AHBs + // take the plain BGRA8 path (sampled as a regular 2D texture). + switch (desc.format) { + case AHARDWAREBUFFER_FORMAT_Y8Cb8Cr8_420: + case AHARDWAREBUFFER_FORMAT_YCbCr_P010: + handle.pixelFormat = VideoPixelFormat::NV12; + break; + default: + handle.pixelFormat = VideoPixelFormat::BGRA8; + break; + } + handle.deleter = [buffer]() { AHardwareBuffer_release(buffer); }; + return handle; } }; diff --git a/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp b/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp index 82e6f6f5c..9d11edfd2 100644 --- a/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp +++ b/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp @@ -363,6 +363,74 @@ std::shared_ptr GPUDevice::importExternalTexture( "null"); } + return std::make_shared( + std::move(external), std::move(memory), std::move(texture), + std::move(descriptor->source), std::move(label)); +#elif defined(__ANDROID__) + // 1. Import the AHardwareBuffer as SharedTextureMemory. For YUV AHBs this + // yields a Dawn texture in the implementation-defined OpaqueYCbCrAndroid + // format; for RGBA AHBs, a regular single-plane texture. + wgpu::SharedTextureMemoryDescriptor memDesc{}; + std::string label = descriptor->label.value_or("external-texture"); + if (!label.empty()) { + memDesc.label = wgpu::StringView(label.c_str(), label.size()); + } + wgpu::SharedTextureMemoryAHardwareBufferDescriptor platformDesc{}; + platformDesc.handle = frame.handle; + memDesc.nextInChain = &platformDesc; + auto memory = _instance.ImportSharedTextureMemory(&memDesc); + if (memory == nullptr) { + throw std::runtime_error( + "GPUDevice::importExternalTexture(): ImportSharedTextureMemory " + "returned null. Is 'shared-texture-memory-ahardware-buffer' enabled?"); + } + + // 2. Create the texture. No descriptor: Dawn picks the right format + // (OpaqueYCbCrAndroid for YUV, R8 / RGBA8 / ... for color AHBs). + auto texture = memory.CreateTexture(); + if (texture == nullptr) { + throw std::runtime_error( + "GPUDevice::importExternalTexture(): CreateTexture returned null"); + } + + // 3. Begin access. Vulkan requires us to advertise the incoming VkImage + // layout (UNDEFINED is fine for the first acquisition of an AHB whose + // contents we expect Dawn to read as-is). + wgpu::SharedTextureMemoryBeginAccessDescriptor begin{}; + begin.initialized = true; + begin.concurrentRead = false; + wgpu::SharedTextureMemoryVkImageLayoutBeginState beginLayout{}; + begin.nextInChain = &beginLayout; + if (!memory.BeginAccess(texture, &begin)) { + throw std::runtime_error( + "GPUDevice::importExternalTexture(): BeginAccess failed"); + } + + // 4. Build the ExternalTextureDescriptor. Unlike iOS we do *not* split + // planes or pass an explicit YUV→RGB matrix: when the underlying texture + // is OpaqueYCbCrAndroid, Dawn routes sampling through a Vulkan + // SamplerYcbcrConversion that does the conversion implicitly, driven by + // the AHB's own format metadata. This is the "passthrough external + // texture" pattern from Dawn's tests + // (utils::MakePassthroughExternalTexture). + wgpu::ExternalTextureDescriptor extDesc{}; + if (!label.empty()) { + extDesc.label = wgpu::StringView(label.c_str(), label.size()); + } + extDesc.plane0 = texture.CreateView(); + extDesc.cropOrigin = {0, 0}; + extDesc.cropSize = {frame.width, frame.height}; + extDesc.apparentSize = {frame.width, frame.height}; + + auto external = _instance.CreateExternalTexture(&extDesc); + if (external == nullptr) { + wgpu::SharedTextureMemoryEndAccessState state{}; + (void)memory.EndAccess(texture, &state); + throw std::runtime_error( + "GPUDevice::importExternalTexture(): CreateExternalTexture returned " + "null"); + } + return std::make_shared( std::move(external), std::move(memory), std::move(texture), std::move(descriptor->source), std::move(label)); diff --git a/packages/webgpu/cpp/rnwgpu/api/GPUFeatures.h b/packages/webgpu/cpp/rnwgpu/api/GPUFeatures.h index c574355ac..a4faa3d98 100644 --- a/packages/webgpu/cpp/rnwgpu/api/GPUFeatures.h +++ b/packages/webgpu/cpp/rnwgpu/api/GPUFeatures.h @@ -188,6 +188,9 @@ static void convertEnumToJSUnion(wgpu::FeatureName inEnum, case wgpu::FeatureName::YCbCrVulkanSamplers: *outUnion = "ycbcr-vulkan-samplers"; break; + case wgpu::FeatureName::OpaqueYCbCrAndroidForExternalTexture: + *outUnion = "opaque-ycbcr-android-for-external-texture"; + break; case wgpu::FeatureName::ShaderModuleCompilationOptions: *outUnion = "shader-module-compilation-options"; break; diff --git a/packages/webgpu/cpp/rnwgpu/api/RnFeatures.h b/packages/webgpu/cpp/rnwgpu/api/RnFeatures.h index 4c60fba23..49120c78f 100644 --- a/packages/webgpu/cpp/rnwgpu/api/RnFeatures.h +++ b/packages/webgpu/cpp/rnwgpu/api/RnFeatures.h @@ -24,8 +24,13 @@ inline std::vector rnSharedTextureMemoryBackingFeatures() { return {wgpu::FeatureName::SharedTextureMemoryIOSurface, wgpu::FeatureName::SharedFenceMTLSharedEvent}; #elif defined(__ANDROID__) + // OpaqueYCbCrAndroidForExternalTexture is the Vulkan-side equivalent of what + // we get "for free" through IOSurface biplanar textures on Metal: it lets + // CreateExternalTexture wrap an AHB-backed YCbCr texture and have sampling + // route through a SamplerYcbcrConversion implicitly. return {wgpu::FeatureName::SharedTextureMemoryAHardwareBuffer, - wgpu::FeatureName::SharedFenceSyncFD}; + wgpu::FeatureName::SharedFenceSyncFD, + wgpu::FeatureName::OpaqueYCbCrAndroidForExternalTexture}; #else return {}; #endif diff --git a/packages/webgpu/cpp/rnwgpu/api/descriptors/Unions.h b/packages/webgpu/cpp/rnwgpu/api/descriptors/Unions.h index fef82e9c3..bf3958295 100644 --- a/packages/webgpu/cpp/rnwgpu/api/descriptors/Unions.h +++ b/packages/webgpu/cpp/rnwgpu/api/descriptors/Unions.h @@ -527,6 +527,8 @@ inline void convertJSUnionToEnum(const std::string &inUnion, *outEnum = wgpu::FeatureName::StaticSamplers; } else if (inUnion == "ycbcr-vulkan-samplers") { *outEnum = wgpu::FeatureName::YCbCrVulkanSamplers; + } else if (inUnion == "opaque-ycbcr-android-for-external-texture") { + *outEnum = wgpu::FeatureName::OpaqueYCbCrAndroidForExternalTexture; } else if (inUnion == "shader-module-compilation-options") { *outEnum = wgpu::FeatureName::ShaderModuleCompilationOptions; } else if (inUnion == "dawn-load-resolve-texture") { @@ -718,6 +720,9 @@ inline void convertEnumToJSUnion(wgpu::FeatureName inEnum, case wgpu::FeatureName::YCbCrVulkanSamplers: *outUnion = "ycbcr-vulkan-samplers"; break; + case wgpu::FeatureName::OpaqueYCbCrAndroidForExternalTexture: + *outUnion = "opaque-ycbcr-android-for-external-texture"; + break; case wgpu::FeatureName::ShaderModuleCompilationOptions: *outUnion = "shader-module-compilation-options"; break; From 026c2f218b2d7ec81a8df9cde4043b66f31a4c7f Mon Sep 17 00:00:00 2001 From: William Candillon Date: Wed, 20 May 2026 14:07:35 +0200 Subject: [PATCH 10/46] :wrench: --- .../example/src/VisionCamera/VisionCamera.tsx | 109 ++++++++++++++++-- packages/webgpu/cpp/rnwgpu/api/GPU.cpp | 16 +++ packages/webgpu/cpp/rnwgpu/api/RnFeatures.h | 7 +- 3 files changed, 118 insertions(+), 14 deletions(-) diff --git a/apps/example/src/VisionCamera/VisionCamera.tsx b/apps/example/src/VisionCamera/VisionCamera.tsx index 66684ff88..4122bfc01 100644 --- a/apps/example/src/VisionCamera/VisionCamera.tsx +++ b/apps/example/src/VisionCamera/VisionCamera.tsx @@ -2,6 +2,7 @@ import React, { useEffect } from "react"; import { Linking, PixelRatio, + Platform, StyleSheet, Text, TouchableOpacity, @@ -10,7 +11,6 @@ import { import { Canvas, useCanvasRef, - useDevice, type NativeCanvas, type RNCanvasContext, } from "react-native-wgpu"; @@ -84,6 +84,15 @@ const REQUIRED_FEATURES: GPUFeatureName[] = [ "dawn-multi-planar-formats" as GPUFeatureName, ]; +// Android-only feature, gates Dawn's "wrap a YCbCr AHB as a GPUExternalTexture +// with implicit SamplerYcbcrConversion" path. Without it our native +// `importExternalTexture` flow on Android can't produce a usable external +// texture from a camera frame. We probe the adapter for it and surface a +// clear error if the device's Vulkan driver doesn't advertise it (e.g. some +// Android-Desktop / Chromebook configurations). +const OPAQUE_YCBCR_EXT = + "opaque-ycbcr-android-for-external-texture" as GPUFeatureName; + const ABERRATION_STRENGTH = 0.006; export const VisionCamera = () => { @@ -114,9 +123,69 @@ export const VisionCamera = () => { const CameraView = () => { const ref = useCanvasRef(); - const { device, adapter } = useDevice(undefined, { - requiredFeatures: REQUIRED_FEATURES, - }); + const [gpu, setGpu] = React.useState<{ + adapter: GPUAdapter; + device: GPUDevice; + } | null>(null); + const [deviceError, setDeviceError] = React.useState(null); + React.useEffect(() => { + let cancelled = false; + (async () => { + try { + const adapter = await navigator.gpu.requestAdapter(); + if (!adapter) { + throw new Error("requestAdapter returned null"); + } + const adapterFeatures = [...adapter.features].sort(); + console.log( + "[VisionCamera] adapter features (" + + adapterFeatures.length + + "): " + + adapterFeatures.join(", "), + ); + const hasOpaqueYCbCrExt = + Platform.OS !== "android" || adapter.features.has(OPAQUE_YCBCR_EXT); + if (Platform.OS === "android" && !hasOpaqueYCbCrExt) { + throw new Error( + "This Android device's Vulkan driver doesn't advertise " + + "opaque-ycbcr-android-for-external-texture. Camera-frame import " + + "as a GPUExternalTexture isn't supported here. (This is a " + + "device/driver limitation, not a code issue.)", + ); + } + const featuresToRequest: GPUFeatureName[] = [ + ...REQUIRED_FEATURES, + ...(Platform.OS === "android" ? [OPAQUE_YCBCR_EXT] : []), + ]; + console.log( + "[VisionCamera] requesting device with features: " + + featuresToRequest.join(", "), + ); + const device = await adapter.requestDevice({ + requiredFeatures: featuresToRequest, + }); + if (cancelled) { + return; + } + console.log( + "[VisionCamera] device created, features: " + + [...device.features].sort().join(", "), + ); + setGpu({ adapter, device }); + } catch (e) { + if (cancelled) { + return; + } + console.warn("[VisionCamera] device creation failed: " + String(e)); + setDeviceError(String(e)); + } + })(); + return () => { + cancelled = true; + }; + }, []); + const device = gpu?.device ?? null; + const adapter = gpu?.adapter ?? null; const devices = useCameraDevices(); // Pick back camera if available, otherwise front, otherwise anything. The // iOS simulator returns an empty list since there are no cameras, in which @@ -274,10 +343,18 @@ const CameraView = () => { new Float32Array([sx, sy, ABERRATION_STRENGTH, 0]), ); - const externalTex = device.importExternalTexture({ - source: videoFrame, - label: "camera-frame", - }); + let externalTex; + try { + externalTex = device.importExternalTexture({ + source: videoFrame, + label: "camera-frame", + }); + } catch (e) { + console.warn( + "[VisionCamera] importExternalTexture threw: " + String(e), + ); + throw e; + } const bindGroup = device.createBindGroup({ layout: pipeline.getBindGroupLayout(0), entries: [ @@ -323,6 +400,15 @@ const CameraView = () => { outputs: [frameOutput], }); + if (deviceError) { + return ( + + + Device creation failed: {deviceError} + + + ); + } if (error) { return ( @@ -330,6 +416,13 @@ const CameraView = () => { ); } + if (!device) { + return ( + + Waiting for GPU device... + + ); + } if (cameraDevice == null) { return ( diff --git a/packages/webgpu/cpp/rnwgpu/api/GPU.cpp b/packages/webgpu/cpp/rnwgpu/api/GPU.cpp index 764a9aa32..bd11bf399 100644 --- a/packages/webgpu/cpp/rnwgpu/api/GPU.cpp +++ b/packages/webgpu/cpp/rnwgpu/api/GPU.cpp @@ -20,6 +20,22 @@ GPU::GPU(jsi::Runtime &runtime) : NativeObject(CLASS_NAME) { wgpu::InstanceLimits limits{.timedWaitAnyMaxCount = 64}; instanceDesc.requiredLimits = &limits; + + // Expose Dawn's experimental adapter features. Several features needed by + // our Android external-texture path (YCbCrVulkanSamplers, StaticSamplers, + // and eventually OpaqueYCbCrAndroidForExternalTexture once it's wired up in + // the Vulkan backend) are tagged Experimental in Dawn's feature table and + // are otherwise hidden from adapter.features. The toggle only unhides + // features; application code still has to list each one in + // requiredFeatures. + static const char *const kEnabledToggles[] = { + "expose_wgsl_experimental_features", + }; + wgpu::DawnTogglesDescriptor toggles; + toggles.enabledToggleCount = std::size(kEnabledToggles); + toggles.enabledToggles = kEnabledToggles; + instanceDesc.nextInChain = &toggles; + _instance = wgpu::CreateInstance(&instanceDesc); auto dispatcher = std::make_shared(runtime); diff --git a/packages/webgpu/cpp/rnwgpu/api/RnFeatures.h b/packages/webgpu/cpp/rnwgpu/api/RnFeatures.h index 49120c78f..4c60fba23 100644 --- a/packages/webgpu/cpp/rnwgpu/api/RnFeatures.h +++ b/packages/webgpu/cpp/rnwgpu/api/RnFeatures.h @@ -24,13 +24,8 @@ inline std::vector rnSharedTextureMemoryBackingFeatures() { return {wgpu::FeatureName::SharedTextureMemoryIOSurface, wgpu::FeatureName::SharedFenceMTLSharedEvent}; #elif defined(__ANDROID__) - // OpaqueYCbCrAndroidForExternalTexture is the Vulkan-side equivalent of what - // we get "for free" through IOSurface biplanar textures on Metal: it lets - // CreateExternalTexture wrap an AHB-backed YCbCr texture and have sampling - // route through a SamplerYcbcrConversion implicitly. return {wgpu::FeatureName::SharedTextureMemoryAHardwareBuffer, - wgpu::FeatureName::SharedFenceSyncFD, - wgpu::FeatureName::OpaqueYCbCrAndroidForExternalTexture}; + wgpu::FeatureName::SharedFenceSyncFD}; #else return {}; #endif From 054f90adb5baacbc972eda6181a7eade5e4f08da Mon Sep 17 00:00:00 2001 From: William Candillon Date: Wed, 20 May 2026 14:10:30 +0200 Subject: [PATCH 11/46] :wrench: --- .../src/ExternalTexture/ExternalTexture.tsx | 25 +++++++------------ .../reanimated/registerWebGPUForReanimated.ts | 12 +++------ packages/webgpu/src/index.tsx | 2 +- packages/webgpu/src/main/index.tsx | 5 +++- 4 files changed, 17 insertions(+), 27 deletions(-) diff --git a/apps/example/src/ExternalTexture/ExternalTexture.tsx b/apps/example/src/ExternalTexture/ExternalTexture.tsx index e68255270..b7218f0b7 100644 --- a/apps/example/src/ExternalTexture/ExternalTexture.tsx +++ b/apps/example/src/ExternalTexture/ExternalTexture.tsx @@ -1,5 +1,5 @@ import React, { useEffect, useRef, useState } from "react"; -import { PixelRatio, Platform, StyleSheet, Text, View } from "react-native"; +import { PixelRatio, StyleSheet, Text, View } from "react-native"; import { Canvas, useCanvasRef, @@ -55,22 +55,15 @@ fn fs_main(in: VsOut) -> @location(0) vec4f { } `; -// We need the same shared-memory + shared-fence pair as the BGRA demo (the -// IOSurface still flows through SharedTextureMemory under the hood), plus +// rnwebgpu/shared-texture-memory is our umbrella that expands to the +// platform's shared-memory + shared-fence pair (the IOSurface / AHB still +// flows through SharedTextureMemory under the hood). Plus // dawn-multi-planar-formats so Dawn can interpret the NV12 surface as a // biplanar texture. -const REQUIRED_FEATURES = - Platform.OS === "ios" - ? [ - "shared-texture-memory-iosurface", - "shared-fence-mtl-shared-event", - "dawn-multi-planar-formats", - ] - : [ - "shared-texture-memory-ahardware-buffer", - "shared-fence-vk-semaphore-sync-fd", - "dawn-multi-planar-formats", - ]; +const REQUIRED_FEATURES: GPUFeatureName[] = [ + "rnwebgpu/shared-texture-memory" as GPUFeatureName, + "dawn-multi-planar-formats" as GPUFeatureName, +]; const VIDEO_URL = "https://test-videos.co.uk/vids/bigbuckbunny/mp4/h264/1080/Big_Buck_Bunny_1080_10s_5MB.mp4"; @@ -81,7 +74,7 @@ export const ExternalTexture = () => { const rafRef = useRef(null); const { device, adapter } = useDevice(undefined, { - requiredFeatures: REQUIRED_FEATURES as unknown as GPUFeatureName[], + requiredFeatures: REQUIRED_FEATURES, }); useEffect(() => { diff --git a/packages/webgpu/src/external/reanimated/registerWebGPUForReanimated.ts b/packages/webgpu/src/external/reanimated/registerWebGPUForReanimated.ts index c3e9b8e6c..c8169e552 100644 --- a/packages/webgpu/src/external/reanimated/registerWebGPUForReanimated.ts +++ b/packages/webgpu/src/external/reanimated/registerWebGPUForReanimated.ts @@ -6,7 +6,6 @@ declare function __webgpuBox(obj: object): { __boxedWebGPU: true; }; -// eslint-disable-next-line no-console console.log("[WebGPU] registerWebGPUForReanimated module loaded"); let isRegistered = false; @@ -26,7 +25,6 @@ export const registerWebGPUForReanimated = () => { try { const { registerCustomSerializable } = require("react-native-worklets"); - // eslint-disable-next-line no-console console.log( "[WebGPU] registering custom serializer (v2: __brand-getter check)", ); @@ -46,7 +44,6 @@ export const registerWebGPUForReanimated = () => { return false; } if ((value as { __boxedWebGPU?: boolean }).__boxedWebGPU === true) { - // eslint-disable-next-line no-console console.log("[WebGPU determine] matched boxed object"); return true; } @@ -65,7 +62,7 @@ export const registerWebGPUForReanimated = () => { } catch { brand = ""; } - // eslint-disable-next-line no-console + console.log( "[WebGPU determine] matched=" + String(matched) + @@ -91,12 +88,9 @@ export const registerWebGPUForReanimated = () => { return boxed.unbox(); }, }); - // eslint-disable-next-line no-console + console.log("[WebGPU] registerCustomSerializable call returned OK"); } catch (e) { - // eslint-disable-next-line no-console - console.warn( - "[WebGPU] registerCustomSerializable threw: " + String(e), - ); + console.warn("[WebGPU] registerCustomSerializable threw: " + String(e)); } }; diff --git a/packages/webgpu/src/index.tsx b/packages/webgpu/src/index.tsx index 5bc843c60..1e696e273 100644 --- a/packages/webgpu/src/index.tsx +++ b/packages/webgpu/src/index.tsx @@ -44,7 +44,7 @@ declare global { createVideoFrameFromNativeBuffer: (pointer: bigint) => VideoFrame; createVideoPlayer: ( path: string, - pixelFormat?: import("./types").VideoPixelFormat, + pixelFormat?: VideoPixelFormat, ) => VideoPlayer; writeTestVideoFile: () => string; }; diff --git a/packages/webgpu/src/main/index.tsx b/packages/webgpu/src/main/index.tsx index 7fedf163e..5f504d5b2 100644 --- a/packages/webgpu/src/main/index.tsx +++ b/packages/webgpu/src/main/index.tsx @@ -40,7 +40,10 @@ if (typeof RNWebGPU !== "undefined" && RNWebGPU != null) { } } } - if (!global.createImageBitmap && typeof RNWebGPU.createImageBitmap === "function") { + if ( + !global.createImageBitmap && + typeof RNWebGPU.createImageBitmap === "function" + ) { global.createImageBitmap = RNWebGPU.createImageBitmap.bind(RNWebGPU); } } else { From b7cf5ce65193c56e5b4265fb5c22e6eaf0e1efa4 Mon Sep 17 00:00:00 2001 From: William Candillon Date: Thu, 21 May 2026 17:58:50 +0200 Subject: [PATCH 12/46] :wrench: --- packages/webgpu/cpp/rnwgpu/api/GPU.cpp | 14 +++++++------ packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp | 21 +++++++++++++++++--- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/packages/webgpu/cpp/rnwgpu/api/GPU.cpp b/packages/webgpu/cpp/rnwgpu/api/GPU.cpp index bd11bf399..b9d2a46bd 100644 --- a/packages/webgpu/cpp/rnwgpu/api/GPU.cpp +++ b/packages/webgpu/cpp/rnwgpu/api/GPU.cpp @@ -22,13 +22,15 @@ GPU::GPU(jsi::Runtime &runtime) : NativeObject(CLASS_NAME) { instanceDesc.requiredLimits = &limits; // Expose Dawn's experimental adapter features. Several features needed by - // our Android external-texture path (YCbCrVulkanSamplers, StaticSamplers, - // and eventually OpaqueYCbCrAndroidForExternalTexture once it's wired up in - // the Vulkan backend) are tagged Experimental in Dawn's feature table and - // are otherwise hidden from adapter.features. The toggle only unhides - // features; application code still has to list each one in - // requiredFeatures. + // our Android external-texture path (YCbCrVulkanSamplers, + // OpaqueYCbCrAndroidForExternalTexture) are tagged Experimental in Dawn's + // feature table and are otherwise filtered out of adapter.features by + // PhysicalDeviceBase::GetSupportedFeatures. The allow_unsafe_apis toggle + // disables that filter so the features become visible; application code + // still has to list each one in requiredFeatures. expose_wgsl_experimental_features + // is the parallel toggle for WGSL language features. static const char *const kEnabledToggles[] = { + "allow_unsafe_apis", "expose_wgsl_experimental_features", }; wgpu::DawnTogglesDescriptor toggles; diff --git a/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp b/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp index 5803223da..4a0fdba0c 100644 --- a/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp +++ b/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp @@ -410,9 +410,21 @@ std::shared_ptr GPUDevice::importExternalTexture( // planes or pass an explicit YUV→RGB matrix: when the underlying texture // is OpaqueYCbCrAndroid, Dawn routes sampling through a Vulkan // SamplerYcbcrConversion that does the conversion implicitly, driven by - // the AHB's own format metadata. This is the "passthrough external - // texture" pattern from Dawn's tests - // (utils::MakePassthroughExternalTexture). + // the AHB's own format metadata. We still must pass noop gamut/transfer + // arrays: Dawn's ComputeExternalTextureParams unconditionally dereferences + // gamutConversionMatrix / src/dstTransferFunctionParameters (see + // externals/dawn/.../ExternalTexture.cpp), so leaving them null produces a + // silent black sample. Identity transfer = TransferFunctionToArray of + // kEOTF_Identity ({g=1,a=1,rest=0}); identity gamut = 3x3 identity. + static const float kIdentityTransferParams[7] = { + 1.0f, // G + 1.0f, // A + 0.0f, // B + 0.0f, // C + 0.0f, // D + 0.0f, // E + 0.0f, // F + }; wgpu::ExternalTextureDescriptor extDesc{}; if (!label.empty()) { extDesc.label = wgpu::StringView(label.c_str(), label.size()); @@ -421,6 +433,9 @@ std::shared_ptr GPUDevice::importExternalTexture( extDesc.cropOrigin = {0, 0}; extDesc.cropSize = {frame.width, frame.height}; extDesc.apparentSize = {frame.width, frame.height}; + extDesc.gamutConversionMatrix = kIdentityGamutMatrix; + extDesc.srcTransferFunctionParameters = kIdentityTransferParams; + extDesc.dstTransferFunctionParameters = kIdentityTransferParams; auto external = _instance.CreateExternalTexture(&extDesc); if (external == nullptr) { From f4db13470ec1e4f302d03447fbe230dba5c33a25 Mon Sep 17 00:00:00 2001 From: William Candillon Date: Fri, 22 May 2026 12:09:14 +0200 Subject: [PATCH 13/46] :wrench: --- .../src/VisionCamera/EffectToolbar.tsx | 73 +++++++++++ .../example/src/VisionCamera/VisionCamera.tsx | 123 +++++++++++++++--- apps/example/src/VisionCamera/features.ts | 39 ++++++ 3 files changed, 216 insertions(+), 19 deletions(-) create mode 100644 apps/example/src/VisionCamera/EffectToolbar.tsx create mode 100644 apps/example/src/VisionCamera/features.ts diff --git a/apps/example/src/VisionCamera/EffectToolbar.tsx b/apps/example/src/VisionCamera/EffectToolbar.tsx new file mode 100644 index 000000000..15ca54665 --- /dev/null +++ b/apps/example/src/VisionCamera/EffectToolbar.tsx @@ -0,0 +1,73 @@ +import React from "react"; +import { Pressable, ScrollView, StyleSheet, Text, View } from "react-native"; + +import { FEATURES, type Modes } from "./features"; + +type Props = { + modes: Modes; + onCycle: (key: keyof Modes, optionsCount: number) => void; +}; + +export const EffectToolbar = ({ modes, onCycle }: Props) => { + return ( + + + {FEATURES.map((f) => ( + onCycle(f.key, f.labels.length)} + style={({ pressed }) => [ + styles.button, + pressed && styles.buttonPressed, + ]} + > + {f.title} + {f.labels[modes[f.key]]} + + ))} + + + ); +}; + +const styles = StyleSheet.create({ + toolbar: { + position: "absolute", + left: 0, + right: 0, + top: 60, + }, + toolbarContent: { + paddingHorizontal: 12, + gap: 8, + }, + button: { + backgroundColor: "rgba(0,0,0,0.55)", + borderColor: "rgba(255,255,255,0.18)", + borderWidth: 1, + borderRadius: 14, + paddingHorizontal: 12, + paddingVertical: 8, + minWidth: 84, + }, + buttonPressed: { + backgroundColor: "rgba(255,255,255,0.18)", + }, + buttonTitle: { + color: "rgba(255,255,255,0.65)", + fontSize: 11, + fontWeight: "500", + letterSpacing: 0.4, + textTransform: "uppercase", + }, + buttonValue: { + color: "white", + fontSize: 15, + fontWeight: "600", + marginTop: 2, + }, +}); diff --git a/apps/example/src/VisionCamera/VisionCamera.tsx b/apps/example/src/VisionCamera/VisionCamera.tsx index 4122bfc01..67807eb89 100644 --- a/apps/example/src/VisionCamera/VisionCamera.tsx +++ b/apps/example/src/VisionCamera/VisionCamera.tsx @@ -21,6 +21,14 @@ import { useFrameOutput, } from "react-native-vision-camera"; +import { EffectToolbar } from "./EffectToolbar"; +import { + ABERRATION_STRENGTHS, + INITIAL_MODES, + PIXELATE_BLOCKS, + type Modes, +} from "./features"; + // Camera frame → SharedTextureMemory (NV12 biplanar) → GPUExternalTexture → // textureSampleBaseClampToEdge with hardware YUV/sRGB conversion → chromatic // aberration in WGSL. @@ -39,9 +47,13 @@ struct VsOut { struct Uniforms { // x, y: 'cover'-fit UV scale around (0.5, 0.5). - // z: chromatic aberration offset in UV units. - // w: unused, padding. - scaleAndAberration: vec4f, + // z: chromatic aberration offset in UV units (0 disables). + // w: pixelate block size in UV units (0 disables). + params: vec4f, + // x: effect (0 off, 1 gray, 2 sepia, 3 invert, 4 vibrant) + // y: tint (0 off, 1 warm, 2 cool) + // z: vignette (0 off, 1 on) + modes: vec4u, }; @group(0) @binding(0) var srcTex: texture_external; @@ -66,16 +78,75 @@ fn vs_main(@builtin(vertex_index) vid: u32) -> VsOut { return out; } +fn sampleAt(uv: vec2f, block: f32) -> vec4f { + var sampleUv = uv; + if (block > 0.0) { + // Snap to a grid in cover-fit UV space; the linear sampler still gives a + // crisp blocky look because the snap collapses neighborhoods. + sampleUv = (floor(uv / block) + vec2f(0.5)) * block; + } + return textureSampleBaseClampToEdge(srcTex, srcSampler, sampleUv); +} + +fn applyEffect(rgb: vec3f, mode: u32) -> vec3f { + if (mode == 1u) { + let l = dot(rgb, vec3f(0.2126, 0.7152, 0.0722)); + return vec3f(l); + } + if (mode == 2u) { + return vec3f( + dot(rgb, vec3f(0.393, 0.769, 0.189)), + dot(rgb, vec3f(0.349, 0.686, 0.168)), + dot(rgb, vec3f(0.272, 0.534, 0.131)) + ); + } + if (mode == 3u) { + return vec3f(1.0) - rgb; + } + if (mode == 4u) { + let l = dot(rgb, vec3f(0.2126, 0.7152, 0.0722)); + let sat = mix(vec3f(l), rgb, 1.55); + return clamp((sat - 0.5) * 1.18 + 0.5, vec3f(0.0), vec3f(1.0)); + } + return rgb; +} + +fn applyTint(rgb: vec3f, mode: u32) -> vec3f { + if (mode == 1u) { + return clamp(rgb * vec3f(1.10, 1.02, 0.86), vec3f(0.0), vec3f(1.0)); + } + if (mode == 2u) { + return clamp(rgb * vec3f(0.86, 0.98, 1.16), vec3f(0.0), vec3f(1.0)); + } + return rgb; +} + @fragment fn fs_main(in: VsOut) -> @location(0) vec4f { - let uvScale = u.scaleAndAberration.xy; - let aberration = u.scaleAndAberration.z; + let uvScale = u.params.xy; + let aberration = u.params.z; + let pixelate = u.params.w; + let effect = u.modes.x; + let tint = u.modes.y; + let vignette = u.modes.z; + let uv = vec2f(0.5) + (in.uv - vec2f(0.5)) * uvScale; // RGB split: sample red shifted right, blue shifted left, green centered. - let r = textureSampleBaseClampToEdge(srcTex, srcSampler, uv + vec2f( aberration, 0.0)).r; - let g = textureSampleBaseClampToEdge(srcTex, srcSampler, uv).g; - let b = textureSampleBaseClampToEdge(srcTex, srcSampler, uv + vec2f(-aberration, 0.0)).b; - return vec4f(r, g, b, 1.0); + let r = sampleAt(uv + vec2f( aberration, 0.0), pixelate).r; + let g = sampleAt(uv, pixelate).g; + let b = sampleAt(uv + vec2f(-aberration, 0.0), pixelate).b; + var color = vec3f(r, g, b); + + color = applyEffect(color, effect); + color = applyTint(color, tint); + + if (vignette == 1u) { + let d = distance(in.uv, vec2f(0.5)); + let v = 1.0 - smoothstep(0.35, 0.85, d); + color = color * v; + } + + return vec4f(color, 1.0); } `; @@ -93,8 +164,6 @@ const REQUIRED_FEATURES: GPUFeatureName[] = [ const OPAQUE_YCBCR_EXT = "opaque-ycbcr-android-for-external-texture" as GPUFeatureName; -const ABERRATION_STRENGTH = 0.006; - export const VisionCamera = () => { const { hasPermission, requestPermission } = useCameraPermission(); useEffect(() => { @@ -207,6 +276,10 @@ const CameraView = () => { canvasHeight: number; } | null>(null); const [error, setError] = React.useState(null); + const [modes, setModes] = React.useState(INITIAL_MODES); + const cycle = React.useCallback((key: keyof Modes, optionsCount: number) => { + setModes((prev) => ({ ...prev, [key]: (prev[key] + 1) % optionsCount })); + }, []); // Initialize pipeline once device + canvas are both ready. useEffect(() => { @@ -256,7 +329,7 @@ const CameraView = () => { minFilter: "linear", }); const uniformBuffer = device.createBuffer({ - size: 16, + size: 32, usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, }); setPipelineState({ @@ -337,11 +410,20 @@ const CameraView = () => { } else { sy = frameAR / canvasAR; } - device.queue.writeBuffer( - uniformBuffer, - 0, - new Float32Array([sx, sy, ABERRATION_STRENGTH, 0]), - ); + // 32-byte uniform: vec4f params + vec4u modes. Built on a single + // ArrayBuffer so the f32/u32 halves go up in one writeBuffer call. + const uniformData = new ArrayBuffer(32); + const uniformF32 = new Float32Array(uniformData); + const uniformU32 = new Uint32Array(uniformData); + uniformF32[0] = sx; + uniformF32[1] = sy; + uniformF32[2] = ABERRATION_STRENGTHS[modes.aberration] ?? 0; + uniformF32[3] = PIXELATE_BLOCKS[modes.pixelate] ?? 0; + uniformU32[4] = modes.effect; + uniformU32[5] = modes.tint; + uniformU32[6] = modes.vignette; + uniformU32[7] = 0; + device.queue.writeBuffer(uniformBuffer, 0, uniformData); let externalTex; try { @@ -434,13 +516,16 @@ const CameraView = () => { ); } return ( - - + + + ); }; const styles = StyleSheet.create({ + root: { flex: 1, backgroundColor: "black" }, + canvas: { flex: 1 }, errorContainer: { flex: 1, padding: 16, justifyContent: "center" }, errorText: { color: "red", fontSize: 14 }, permissionContainer: { diff --git a/apps/example/src/VisionCamera/features.ts b/apps/example/src/VisionCamera/features.ts new file mode 100644 index 000000000..c2a67eb68 --- /dev/null +++ b/apps/example/src/VisionCamera/features.ts @@ -0,0 +1,39 @@ +const EFFECT_LABELS = ["Off", "Gray", "Sepia", "Invert", "Vibrant"] as const; +const TINT_LABELS = ["Off", "Warm", "Cool"] as const; +const ABERRATION_LABELS = ["Off", "Soft", "Strong"] as const; +const TOGGLE_LABELS = ["Off", "On"] as const; + +export type Modes = { + effect: number; + tint: number; + aberration: number; + vignette: number; + pixelate: number; +}; + +export const INITIAL_MODES: Modes = { + effect: 0, + tint: 0, + aberration: 1, + vignette: 0, + pixelate: 0, +}; + +// Aberration strength in UV units per level. Soft matches the original demo. +export const ABERRATION_STRENGTHS = [0.0, 0.006, 0.018] as const; +// Block size in UV units per pixelate level. Larger value, chunkier pixels. +export const PIXELATE_BLOCKS = [0.0, 0.02] as const; + +export type Feature = { + title: string; + key: keyof Modes; + labels: readonly string[]; +}; + +export const FEATURES: Feature[] = [ + { title: "Effect", key: "effect", labels: EFFECT_LABELS }, + { title: "Tint", key: "tint", labels: TINT_LABELS }, + { title: "Aberration", key: "aberration", labels: ABERRATION_LABELS }, + { title: "Vignette", key: "vignette", labels: TOGGLE_LABELS }, + { title: "Pixelate", key: "pixelate", labels: TOGGLE_LABELS }, +]; From 9a283c600aa2769119b6e8a059b3a92c0aa9b709 Mon Sep 17 00:00:00 2001 From: William Candillon Date: Fri, 22 May 2026 14:10:52 +0200 Subject: [PATCH 14/46] :wrench: --- .../example/src/VisionCamera/VisionCamera.tsx | 314 +++++++++++++++++- apps/example/src/VisionCamera/blurShaders.ts | 141 ++++++++ apps/example/src/VisionCamera/features.ts | 9 + 3 files changed, 448 insertions(+), 16 deletions(-) create mode 100644 apps/example/src/VisionCamera/blurShaders.ts diff --git a/apps/example/src/VisionCamera/VisionCamera.tsx b/apps/example/src/VisionCamera/VisionCamera.tsx index 67807eb89..db629f12a 100644 --- a/apps/example/src/VisionCamera/VisionCamera.tsx +++ b/apps/example/src/VisionCamera/VisionCamera.tsx @@ -21,9 +21,11 @@ import { useFrameOutput, } from "react-native-vision-camera"; +import { BLUR_SHADER, PREPASS_SHADER } from "./blurShaders"; import { EffectToolbar } from "./EffectToolbar"; import { ABERRATION_STRENGTHS, + BLUR_ITERATIONS, INITIAL_MODES, PIXELATE_BLOCKS, type Modes, @@ -50,15 +52,18 @@ struct Uniforms { // z: chromatic aberration offset in UV units (0 disables). // w: pixelate block size in UV units (0 disables). params: vec4f, - // x: effect (0 off, 1 gray, 2 sepia, 3 invert, 4 vibrant) - // y: tint (0 off, 1 warm, 2 cool) + // x: effect (0 off, 1 gray, 2 sepia, 3 invert, 4 vibrant) + // y: tint (0 off, 1 warm, 2 cool) // z: vignette (0 off, 1 on) + // w: blurMode (0 off, 1 strong - blurred everywhere (prepass bakes + // cover-fit), 2 overlay - blurred backdrop + sharp card) modes: vec4u, }; @group(0) @binding(0) var srcTex: texture_external; @group(0) @binding(1) var srcSampler: sampler; @group(0) @binding(2) var u: Uniforms; +@group(0) @binding(3) var blurredTex: texture_2d; @vertex fn vs_main(@builtin(vertex_index) vid: u32) -> VsOut { @@ -78,14 +83,40 @@ fn vs_main(@builtin(vertex_index) vid: u32) -> VsOut { return out; } -fn sampleAt(uv: vec2f, block: f32) -> vec4f { - var sampleUv = uv; - if (block > 0.0) { - // Snap to a grid in cover-fit UV space; the linear sampler still gives a - // crisp blocky look because the snap collapses neighborhoods. - sampleUv = (floor(uv / block) + vec2f(0.5)) * block; +fn snap(uv: vec2f, block: f32) -> vec2f { + if (block <= 0.0) { + return uv; } - return textureSampleBaseClampToEdge(srcTex, srcSampler, sampleUv); + return (floor(uv / block) + vec2f(0.5)) * block; +} + +fn sampleExternal(uv: vec2f, block: f32) -> vec4f { + return textureSampleBaseClampToEdge(srcTex, srcSampler, snap(uv, block)); +} + +fn sampleBlurred(uv: vec2f, block: f32) -> vec4f { + return textureSampleLevel( + blurredTex, + srcSampler, + clamp(snap(uv, block), vec2f(0.0), vec2f(1.0)), + 0.0, + ); +} + +// RGB-split sample of the external camera with cover-fit + optional pixelate. +fn rgbSplitExternal(uv: vec2f, aberration: f32, block: f32) -> vec3f { + let r = sampleExternal(uv + vec2f( aberration, 0.0), block).r; + let g = sampleExternal(uv, block).g; + let b = sampleExternal(uv + vec2f(-aberration, 0.0), block).b; + return vec3f(r, g, b); +} + +// RGB-split sample of the pre-blurred 2D texture (cover-fit baked in). +fn rgbSplitBlurred(uv: vec2f, aberration: f32, block: f32) -> vec3f { + let r = sampleBlurred(uv + vec2f( aberration, 0.0), block).r; + let g = sampleBlurred(uv, block).g; + let b = sampleBlurred(uv + vec2f(-aberration, 0.0), block).b; + return vec3f(r, g, b); } fn applyEffect(rgb: vec3f, mode: u32) -> vec3f { @@ -129,13 +160,38 @@ fn fs_main(in: VsOut) -> @location(0) vec4f { let effect = u.modes.x; let tint = u.modes.y; let vignette = u.modes.z; + let blurMode = u.modes.w; - let uv = vec2f(0.5) + (in.uv - vec2f(0.5)) * uvScale; - // RGB split: sample red shifted right, blue shifted left, green centered. - let r = sampleAt(uv + vec2f( aberration, 0.0), pixelate).r; - let g = sampleAt(uv, pixelate).g; - let b = sampleAt(uv + vec2f(-aberration, 0.0), pixelate).b; - var color = vec3f(r, g, b); + // Overlay card geometry. NDC-space rect, the camera image is cover-fit + // inside the card (same uvScale as the off path), the blurred backdrop + // fills outside. + let overlayPadding = 0.08; + let edgeAA = 0.004; + + var color: vec3f; + if (blurMode == 1u) { + // Strong: prepass already baked cover-fit, so feed in.uv straight in. + color = rgbSplitBlurred(in.uv, aberration, pixelate); + } else if (blurMode == 2u) { + // Overlay: per-fragment edge factor 0 strictly inside the card, + // 1 strictly outside, smoothstep band for AA. Uniform within the branch + // (blurMode came from the uniform buffer), so calling + // textureSampleBaseClampToEdge on the external texture is allowed. + let cardHalf = vec2f(0.5 - overlayPadding); + let p = abs(in.uv - vec2f(0.5)); + let edgeDist = max(p.x - cardHalf.x, p.y - cardHalf.y); + let outside = smoothstep(-edgeAA, edgeAA, edgeDist); + + let cardUv = (in.uv - vec2f(overlayPadding)) / + (1.0 - 2.0 * overlayPadding); + let sharpUv = vec2f(0.5) + (cardUv - vec2f(0.5)) * uvScale; + let sharp = rgbSplitExternal(sharpUv, aberration, pixelate); + let backdrop = rgbSplitBlurred(in.uv, aberration, pixelate); + color = mix(sharp, backdrop, outside); + } else { + let uv = vec2f(0.5) + (in.uv - vec2f(0.5)) * uvScale; + color = rgbSplitExternal(uv, aberration, pixelate); + } color = applyEffect(color, effect); color = applyTint(color, tint); @@ -164,6 +220,17 @@ const REQUIRED_FEATURES: GPUFeatureName[] = [ const OPAQUE_YCBCR_EXT = "opaque-ycbcr-android-for-external-texture" as GPUFeatureName; +// Blur infrastructure. Mirrors the ExternalTexture demo: prepass writes the +// cover-fit camera image into a 1/4-res rgba8unorm, the separable box-blur +// compute pings between two storage textures, and the main pass linearly +// upsamples the final result so the effective sigma is ~4x the per-iteration +// kernel. +const BLUR_SCALE = 4; +const BLUR_FILTER_SIZE = 31; +const BLUR_TILE_DIM = 128; +const BLUR_BATCH = 4; +const BLUR_BLOCK_DIM = BLUR_TILE_DIM - BLUR_FILTER_SIZE; + export const VisionCamera = () => { const { hasPermission, requestPermission } = useCameraPermission(); useEffect(() => { @@ -274,6 +341,17 @@ const CameraView = () => { context: RNCanvasContext; canvasWidth: number; canvasHeight: number; + prepassPipeline: GPURenderPipeline; + prepassUniformBuffer: GPUBuffer; + blurPipeline: GPUComputePipeline; + blurConstants: GPUBindGroup; + blurBindGroup0: GPUBindGroup; + blurBindGroup1: GPUBindGroup; + blurBindGroup2: GPUBindGroup; + blurSrcTexture: GPUTexture; + blurredView: GPUTextureView; + blurWidth: number; + blurHeight: number; } | null>(null); const [error, setError] = React.useState(null); const [modes, setModes] = React.useState(INITIAL_MODES); @@ -332,6 +410,118 @@ const CameraView = () => { size: 32, usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, }); + + // ----- Blur infrastructure (matches ExternalTexture's "Blur" chain) ---- + const blurWidth = Math.max( + BLUR_TILE_DIM, + Math.ceil(canvas.width / BLUR_SCALE), + ); + const blurHeight = Math.max( + BLUR_TILE_DIM, + Math.ceil(canvas.height / BLUR_SCALE), + ); + + const prepassModule = device.createShaderModule({ code: PREPASS_SHADER }); + const prepassPipeline = device.createRenderPipeline({ + layout: "auto", + vertex: { module: prepassModule, entryPoint: "vs_main" }, + fragment: { + module: prepassModule, + entryPoint: "fs_main", + targets: [{ format: "rgba8unorm" }], + }, + primitive: { topology: "triangle-list" }, + }); + const prepassUniformBuffer = device.createBuffer({ + size: 16, + usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, + }); + + const blurSrcTexture = device.createTexture({ + size: [blurWidth, blurHeight], + format: "rgba8unorm", + usage: + GPUTextureUsage.RENDER_ATTACHMENT | GPUTextureUsage.TEXTURE_BINDING, + }); + const blurPing = [0, 1].map(() => + device.createTexture({ + size: [blurWidth, blurHeight], + format: "rgba8unorm", + usage: + GPUTextureUsage.STORAGE_BINDING | GPUTextureUsage.TEXTURE_BINDING, + }), + ); + + const blurPipeline = device.createComputePipeline({ + layout: "auto", + compute: { + module: device.createShaderModule({ code: BLUR_SHADER }), + entryPoint: "main", + }, + }); + + const flip0Buffer = device.createBuffer({ + size: 4, + mappedAtCreation: true, + usage: GPUBufferUsage.UNIFORM, + }); + new Uint32Array(flip0Buffer.getMappedRange())[0] = 0; + flip0Buffer.unmap(); + const flip1Buffer = device.createBuffer({ + size: 4, + mappedAtCreation: true, + usage: GPUBufferUsage.UNIFORM, + }); + new Uint32Array(flip1Buffer.getMappedRange())[0] = 1; + flip1Buffer.unmap(); + + const blurParamsBuffer = device.createBuffer({ + size: 8, + usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, + }); + device.queue.writeBuffer( + blurParamsBuffer, + 0, + new Uint32Array([BLUR_FILTER_SIZE + 1, BLUR_BLOCK_DIM]), + ); + + const blurConstants = device.createBindGroup({ + layout: blurPipeline.getBindGroupLayout(0), + entries: [ + { binding: 0, resource: sampler }, + { binding: 1, resource: { buffer: blurParamsBuffer } }, + ], + }); + // H: blurSrcTexture -> blurPing[0] + const blurBindGroup0 = device.createBindGroup({ + layout: blurPipeline.getBindGroupLayout(1), + entries: [ + { binding: 1, resource: blurSrcTexture.createView() }, + { binding: 2, resource: blurPing[0].createView() }, + { binding: 3, resource: { buffer: flip0Buffer } }, + ], + }); + // V: blurPing[0] -> blurPing[1] + const blurBindGroup1 = device.createBindGroup({ + layout: blurPipeline.getBindGroupLayout(1), + entries: [ + { binding: 1, resource: blurPing[0].createView() }, + { binding: 2, resource: blurPing[1].createView() }, + { binding: 3, resource: { buffer: flip1Buffer } }, + ], + }); + // H (iteration N>=2): blurPing[1] -> blurPing[0] + const blurBindGroup2 = device.createBindGroup({ + layout: blurPipeline.getBindGroupLayout(1), + entries: [ + { binding: 1, resource: blurPing[1].createView() }, + { binding: 2, resource: blurPing[0].createView() }, + { binding: 3, resource: { buffer: flip0Buffer } }, + ], + }); + // Final iteration's V pass always lands in blurPing[1]. + const blurredView = blurPing[1].createView(); + setPipelineState({ pipeline, sampler, @@ -339,6 +529,17 @@ const CameraView = () => { context, canvasWidth: canvas.width, canvasHeight: canvas.height, + prepassPipeline, + prepassUniformBuffer, + blurPipeline, + blurConstants, + blurBindGroup0, + blurBindGroup1, + blurBindGroup2, + blurSrcTexture, + blurredView, + blurWidth, + blurHeight, }); }, [device, adapter, ref, pipelineState]); @@ -378,7 +579,20 @@ const CameraView = () => { context, canvasWidth, canvasHeight, + prepassPipeline, + prepassUniformBuffer, + blurPipeline, + blurConstants, + blurBindGroup0, + blurBindGroup1, + blurBindGroup2, + blurSrcTexture, + blurredView, + blurWidth, + blurHeight, } = pipelineState; + const blurIterations = BLUR_ITERATIONS[modes.blur] ?? 0; + const blurMode = blurIterations > 0 ? modes.blur : 0; const nativeBuffer = frame.getNativeBuffer(); try { let videoFrame; @@ -422,7 +636,7 @@ const CameraView = () => { uniformU32[4] = modes.effect; uniformU32[5] = modes.tint; uniformU32[6] = modes.vignette; - uniformU32[7] = 0; + uniformU32[7] = blurMode; device.queue.writeBuffer(uniformBuffer, 0, uniformData); let externalTex; @@ -443,10 +657,78 @@ const CameraView = () => { { binding: 0, resource: externalTex }, { binding: 1, resource: sampler }, { binding: 2, resource: { buffer: uniformBuffer } }, + { binding: 3, resource: blurredView }, ], }); const encoder = device.createCommandEncoder(); + + if (blurIterations > 0) { + // Prepass: cover-fit external (YUV) -> rgba8unorm at 1/4 res. + device.queue.writeBuffer( + prepassUniformBuffer, + 0, + new Float32Array([ + videoFrame.width, + videoFrame.height, + canvasWidth, + canvasHeight, + ]), + ); + const prepassBindGroup = device.createBindGroup({ + layout: prepassPipeline.getBindGroupLayout(0), + entries: [ + { binding: 0, resource: externalTex }, + { binding: 1, resource: sampler }, + { binding: 2, resource: { buffer: prepassUniformBuffer } }, + ], + }); + const prepass = encoder.beginRenderPass({ + colorAttachments: [ + { + view: blurSrcTexture.createView(), + clearValue: { r: 0, g: 0, b: 0, a: 1 }, + loadOp: "clear", + storeOp: "store", + }, + ], + }); + prepass.setPipeline(prepassPipeline); + prepass.setBindGroup(0, prepassBindGroup); + prepass.draw(3); + prepass.end(); + + // Separable box-blur compute. Iteration 1 reads from + // blurSrcTexture; subsequent iterations ping-pong between + // blurPing[1] -> blurPing[0] -> blurPing[1]. + const compute = encoder.beginComputePass(); + compute.setPipeline(blurPipeline); + compute.setBindGroup(0, blurConstants); + compute.setBindGroup(1, blurBindGroup0); + compute.dispatchWorkgroups( + Math.ceil(blurWidth / BLUR_BLOCK_DIM), + Math.ceil(blurHeight / BLUR_BATCH), + ); + compute.setBindGroup(1, blurBindGroup1); + compute.dispatchWorkgroups( + Math.ceil(blurHeight / BLUR_BLOCK_DIM), + Math.ceil(blurWidth / BLUR_BATCH), + ); + for (let i = 0; i < blurIterations - 1; i++) { + compute.setBindGroup(1, blurBindGroup2); + compute.dispatchWorkgroups( + Math.ceil(blurWidth / BLUR_BLOCK_DIM), + Math.ceil(blurHeight / BLUR_BATCH), + ); + compute.setBindGroup(1, blurBindGroup1); + compute.dispatchWorkgroups( + Math.ceil(blurHeight / BLUR_BLOCK_DIM), + Math.ceil(blurWidth / BLUR_BATCH), + ); + } + compute.end(); + } + const pass = encoder.beginRenderPass({ colorAttachments: [ { diff --git a/apps/example/src/VisionCamera/blurShaders.ts b/apps/example/src/VisionCamera/blurShaders.ts new file mode 100644 index 000000000..e589bff07 --- /dev/null +++ b/apps/example/src/VisionCamera/blurShaders.ts @@ -0,0 +1,141 @@ +// Two shaders that together feed the Blur effect with a true large-kernel +// gaussian, computed once on the GPU per frame and resampled by the main pass. +// +// 1. PREPASS_SHADER. Renders the external (YUV) camera texture into a regular +// rgba8unorm texture, cover-projected at 1/4 canvas resolution. This solves +// two problems at once: +// a. texture_external can only be sampled with textureSampleBaseClampToEdge +// and cannot be bound to a compute pipeline that expects texture_2d, +// so we need a fixup pass anyway. +// b. The blur runs at 1/4 res, and the main fragment's linear sampler +// upsamples for free, which makes the effective kernel ~4x wider than +// what we actually compute. Big blur, cheap. +// +// 2. BLUR_SHADER. The tile-based separable box-blur compute shader from the +// WebGPU samples repo, used unmodified. One workgroup loads a 128 x 4 tile +// into shared memory, then 32 threads write filterDim-averaged outputs. +// A `flip` uniform swaps x/y so the same shader does both axes. Iterating +// H-V passes (the variance adds) approximates a gaussian. + +export const PREPASS_SHADER = /* wgsl */ ` +struct PrepassUniforms { + texSize: vec2f, + canvasSize: vec2f, +}; + +@group(0) @binding(0) var srcTex: texture_external; +@group(0) @binding(1) var srcSampler: sampler; +@group(0) @binding(2) var u: PrepassUniforms; + +struct VsOut { + @builtin(position) position: vec4f, + @location(0) uv: vec2f, +}; + +@vertex +fn vs_main(@builtin(vertex_index) vid: u32) -> VsOut { + var positions = array( + vec2f(-1.0, -3.0), + vec2f(-1.0, 1.0), + vec2f( 3.0, 1.0), + ); + var uvs = array( + vec2f(0.0, 2.0), + vec2f(0.0, 0.0), + vec2f(2.0, 0.0), + ); + var out: VsOut; + out.position = vec4f(positions[vid], 0.0, 1.0); + out.uv = uvs[vid]; + return out; +} + +@fragment +fn fs_main(in: VsOut) -> @location(0) vec4f { + // Cover-fit, same math the worklet sends via uvScale to the main pass so + // the blurred image lines up with the unblurred path. + let canvasAR = u.canvasSize.x / u.canvasSize.y; + let texAR = u.texSize.x / u.texSize.y; + var scale: vec2f; + if (texAR > canvasAR) { + scale = vec2f(canvasAR / texAR, 1.0); + } else { + scale = vec2f(1.0, texAR / canvasAR); + } + let uv = vec2f(0.5) + (in.uv - vec2f(0.5)) * scale; + let c = textureSampleBaseClampToEdge( + srcTex, + srcSampler, + clamp(uv, vec2f(0.0), vec2f(1.0)), + ); + return vec4f(c.rgb, 1.0); +} +`; + +export const BLUR_SHADER = /* wgsl */ ` +struct Params { + filterDim: i32, + blockDim: u32, +} + +@group(0) @binding(0) var samp: sampler; +@group(0) @binding(1) var params: Params; +@group(1) @binding(1) var inputTex: texture_2d; +@group(1) @binding(2) var outputTex: texture_storage_2d; + +struct Flip { + value: u32, +} +@group(1) @binding(3) var flip: Flip; + +var tile: array, 4>; + +@compute @workgroup_size(32, 1, 1) +fn main( + @builtin(workgroup_id) WorkGroupID: vec3u, + @builtin(local_invocation_id) LocalInvocationID: vec3u, +) { + let filterOffset = (params.filterDim - 1) / 2; + let dims = vec2i(textureDimensions(inputTex, 0)); + let baseIndex = vec2i(WorkGroupID.xy * vec2(params.blockDim, 4u) + + LocalInvocationID.xy * vec2(4u, 1u)) + - vec2(filterOffset, 0); + + for (var r = 0; r < 4; r++) { + for (var c = 0; c < 4; c++) { + var loadIndex = baseIndex + vec2(c, r); + if (flip.value != 0u) { + loadIndex = loadIndex.yx; + } + tile[r][4u * LocalInvocationID.x + u32(c)] = textureSampleLevel( + inputTex, + samp, + (vec2f(loadIndex) + vec2f(0.5)) / vec2f(dims), + 0.0, + ).rgb; + } + } + + workgroupBarrier(); + + for (var r = 0; r < 4; r++) { + for (var c = 0; c < 4; c++) { + var writeIndex = baseIndex + vec2(c, r); + if (flip.value != 0u) { + writeIndex = writeIndex.yx; + } + let center = i32(4u * LocalInvocationID.x) + c; + if (center >= filterOffset && + center < 128 - filterOffset && + all(writeIndex < dims)) { + var acc = vec3f(0.0); + for (var f = 0; f < params.filterDim; f++) { + let i = center + f - filterOffset; + acc = acc + (1.0 / f32(params.filterDim)) * tile[r][i]; + } + textureStore(outputTex, writeIndex, vec4f(acc, 1.0)); + } + } + } +} +`; diff --git a/apps/example/src/VisionCamera/features.ts b/apps/example/src/VisionCamera/features.ts index c2a67eb68..7e8ed57a8 100644 --- a/apps/example/src/VisionCamera/features.ts +++ b/apps/example/src/VisionCamera/features.ts @@ -1,12 +1,14 @@ const EFFECT_LABELS = ["Off", "Gray", "Sepia", "Invert", "Vibrant"] as const; const TINT_LABELS = ["Off", "Warm", "Cool"] as const; const ABERRATION_LABELS = ["Off", "Soft", "Strong"] as const; +const BLUR_LABELS = ["Off", "Strong", "Overlay"] as const; const TOGGLE_LABELS = ["Off", "On"] as const; export type Modes = { effect: number; tint: number; aberration: number; + blur: number; vignette: number; pixelate: number; }; @@ -15,6 +17,7 @@ export const INITIAL_MODES: Modes = { effect: 0, tint: 0, aberration: 1, + blur: 0, vignette: 0, pixelate: 0, }; @@ -23,6 +26,11 @@ export const INITIAL_MODES: Modes = { export const ABERRATION_STRENGTHS = [0.0, 0.006, 0.018] as const; // Block size in UV units per pixelate level. Larger value, chunkier pixels. export const PIXELATE_BLOCKS = [0.0, 0.02] as const; +// H-V iteration count per blur level. Each iteration is a separable box pass +// at 1/4 canvas res; variance adds, so the effective sigma grows as sqrt(N). +// Strong and Overlay share the same iteration count, the difference is in the +// fragment shader (full-canvas blurred vs. card-shaped sharp inset). +export const BLUR_ITERATIONS = [0, 3, 3] as const; export type Feature = { title: string; @@ -34,6 +42,7 @@ export const FEATURES: Feature[] = [ { title: "Effect", key: "effect", labels: EFFECT_LABELS }, { title: "Tint", key: "tint", labels: TINT_LABELS }, { title: "Aberration", key: "aberration", labels: ABERRATION_LABELS }, + { title: "Blur", key: "blur", labels: BLUR_LABELS }, { title: "Vignette", key: "vignette", labels: TOGGLE_LABELS }, { title: "Pixelate", key: "pixelate", labels: TOGGLE_LABELS }, ]; From c7a116f898bca787a63aed2c469ef47f11cc1398 Mon Sep 17 00:00:00 2001 From: William Candillon Date: Fri, 22 May 2026 14:44:43 +0200 Subject: [PATCH 15/46] :wrench: --- apps/example/src/App.tsx | 2 + .../example/src/ChromeSphere/ChromeSphere.tsx | 856 ++++++++++++++++++ apps/example/src/ChromeSphere/geometry.ts | 169 ++++ apps/example/src/ChromeSphere/index.ts | 1 + apps/example/src/ChromeSphere/shader.ts | 94 ++ apps/example/src/Home.tsx | 4 + apps/example/src/Route.ts | 1 + 7 files changed, 1127 insertions(+) create mode 100644 apps/example/src/ChromeSphere/ChromeSphere.tsx create mode 100644 apps/example/src/ChromeSphere/geometry.ts create mode 100644 apps/example/src/ChromeSphere/index.ts create mode 100644 apps/example/src/ChromeSphere/shader.ts diff --git a/apps/example/src/App.tsx b/apps/example/src/App.tsx index 0646dadb4..917429ce0 100644 --- a/apps/example/src/App.tsx +++ b/apps/example/src/App.tsx @@ -39,6 +39,7 @@ import { StorageBufferVertices } from "./StorageBufferVertices"; import { SharedTextureMemory } from "./SharedTextureMemory"; import { ExternalTexture } from "./ExternalTexture"; import { VisionCamera } from "./VisionCamera"; +import { ChromeSphere } from "./ChromeSphere"; // The two lines below are needed by three.js import "fast-text-encoding"; @@ -106,6 +107,7 @@ function App() { /> + diff --git a/apps/example/src/ChromeSphere/ChromeSphere.tsx b/apps/example/src/ChromeSphere/ChromeSphere.tsx new file mode 100644 index 000000000..bcbf847cd --- /dev/null +++ b/apps/example/src/ChromeSphere/ChromeSphere.tsx @@ -0,0 +1,856 @@ +/* eslint-disable prefer-destructuring */ +import React, { useEffect } from "react"; +import { + Linking, + PixelRatio, + Platform, + StyleSheet, + Text, + TouchableOpacity, + View, +} from "react-native"; +import { + Canvas, + useCanvasRef, + type NativeCanvas, + type RNCanvasContext, +} from "react-native-wgpu"; +import { + useCamera, + useCameraDevices, + useCameraPermission, + useFrameOutput, +} from "react-native-vision-camera"; + +import { + generateCube, + generateSphere, + generateTorus, + NORMAL_OFFSET, + POSITION_OFFSET, + VERTEX_STRIDE_BYTES, +} from "./geometry"; +import { SHADER } from "./shader"; + +// All matrix math runs inside the Vision Camera frame-processor worklet, so it +// has to be implemented with worklet-friendly helpers. wgpu-matrix calls +// `mat4.create` / `vec3.create` internally and those aren't marked as +// worklets, so we inline what we need here. Conventions match wgpu-matrix: +// column-major 4x4 stored as Float32Array(16), index = col * 4 + row, +// right-handed view space, perspective maps z to [0, 1] (WebGPU clip space). + +const setIdentity = (out: Float32Array) => { + "worklet"; + out[0] = 1; + out[1] = 0; + out[2] = 0; + out[3] = 0; + out[4] = 0; + out[5] = 1; + out[6] = 0; + out[7] = 0; + out[8] = 0; + out[9] = 0; + out[10] = 1; + out[11] = 0; + out[12] = 0; + out[13] = 0; + out[14] = 0; + out[15] = 1; +}; + +// dst = m * Rx(angle). Safe with dst === m: only columns 1 and 2 change, and +// we snapshot them into locals before writing. +const applyRotateX = (m: Float32Array, angle: number, dst: Float32Array) => { + "worklet"; + const c = Math.cos(angle); + const s = Math.sin(angle); + const m01 = m[4], + m11 = m[5], + m21 = m[6], + m31 = m[7]; + const m02 = m[8], + m12 = m[9], + m22 = m[10], + m32 = m[11]; + if (dst !== m) { + dst[0] = m[0]; + dst[1] = m[1]; + dst[2] = m[2]; + dst[3] = m[3]; + dst[12] = m[12]; + dst[13] = m[13]; + dst[14] = m[14]; + dst[15] = m[15]; + } + dst[4] = c * m01 + s * m02; + dst[5] = c * m11 + s * m12; + dst[6] = c * m21 + s * m22; + dst[7] = c * m31 + s * m32; + dst[8] = -s * m01 + c * m02; + dst[9] = -s * m11 + c * m12; + dst[10] = -s * m21 + c * m22; + dst[11] = -s * m31 + c * m32; +}; + +// dst = m * Ry(angle). Safe with dst === m. +const applyRotateY = (m: Float32Array, angle: number, dst: Float32Array) => { + "worklet"; + const c = Math.cos(angle); + const s = Math.sin(angle); + const m00 = m[0], + m10 = m[1], + m20 = m[2], + m30 = m[3]; + const m02 = m[8], + m12 = m[9], + m22 = m[10], + m32 = m[11]; + if (dst !== m) { + dst[4] = m[4]; + dst[5] = m[5]; + dst[6] = m[6]; + dst[7] = m[7]; + dst[12] = m[12]; + dst[13] = m[13]; + dst[14] = m[14]; + dst[15] = m[15]; + } + dst[0] = c * m00 - s * m02; + dst[1] = c * m10 - s * m12; + dst[2] = c * m20 - s * m22; + dst[3] = c * m30 - s * m32; + dst[8] = s * m00 + c * m02; + dst[9] = s * m10 + c * m12; + dst[10] = s * m20 + c * m22; + dst[11] = s * m30 + c * m32; +}; + +// dst = m * Rz(angle). Safe with dst === m. +const applyRotateZ = (m: Float32Array, angle: number, dst: Float32Array) => { + "worklet"; + const c = Math.cos(angle); + const s = Math.sin(angle); + const m00 = m[0], + m10 = m[1], + m20 = m[2], + m30 = m[3]; + const m01 = m[4], + m11 = m[5], + m21 = m[6], + m31 = m[7]; + if (dst !== m) { + dst[8] = m[8]; + dst[9] = m[9]; + dst[10] = m[10]; + dst[11] = m[11]; + dst[12] = m[12]; + dst[13] = m[13]; + dst[14] = m[14]; + dst[15] = m[15]; + } + dst[0] = c * m00 + s * m01; + dst[1] = c * m10 + s * m11; + dst[2] = c * m20 + s * m21; + dst[3] = c * m30 + s * m31; + dst[4] = -s * m00 + c * m01; + dst[5] = -s * m10 + c * m11; + dst[6] = -s * m20 + c * m21; + dst[7] = -s * m30 + c * m31; +}; + +// dst = m * translate(tx, ty, tz). Only column 3 changes; safe with dst === m +// because we read the old col0/1/2 entries and the old col3 first. +const applyTranslate = ( + m: Float32Array, + tx: number, + ty: number, + tz: number, + dst: Float32Array, +) => { + "worklet"; + const c0r0 = m[0], + c0r1 = m[1], + c0r2 = m[2], + c0r3 = m[3]; + const c1r0 = m[4], + c1r1 = m[5], + c1r2 = m[6], + c1r3 = m[7]; + const c2r0 = m[8], + c2r1 = m[9], + c2r2 = m[10], + c2r3 = m[11]; + const c3r0 = m[12], + c3r1 = m[13], + c3r2 = m[14], + c3r3 = m[15]; + if (dst !== m) { + dst[0] = c0r0; + dst[1] = c0r1; + dst[2] = c0r2; + dst[3] = c0r3; + dst[4] = c1r0; + dst[5] = c1r1; + dst[6] = c1r2; + dst[7] = c1r3; + dst[8] = c2r0; + dst[9] = c2r1; + dst[10] = c2r2; + dst[11] = c2r3; + } + dst[12] = c0r0 * tx + c1r0 * ty + c2r0 * tz + c3r0; + dst[13] = c0r1 * tx + c1r1 * ty + c2r1 * tz + c3r1; + dst[14] = c0r2 * tx + c1r2 * ty + c2r2 * tz + c3r2; + dst[15] = c0r3 * tx + c1r3 * ty + c2r3 * tz + c3r3; +}; + +// WebGPU-style perspective: right-handed, output z mapped to [0, 1]. fovy is +// the vertical field of view in radians. +const setPerspective = ( + out: Float32Array, + fovy: number, + aspect: number, + near: number, + far: number, +) => { + "worklet"; + const f = 1 / Math.tan(fovy * 0.5); + const rangeInv = 1 / (near - far); + out[0] = f / aspect; + out[1] = 0; + out[2] = 0; + out[3] = 0; + out[4] = 0; + out[5] = f; + out[6] = 0; + out[7] = 0; + out[8] = 0; + out[9] = 0; + out[10] = far * rangeInv; + out[11] = -1; + out[12] = 0; + out[13] = 0; + out[14] = far * near * rangeInv; + out[15] = 0; +}; + +// View matrix (RH). Camera looks down -z in its local frame. Same layout +// wgpu-matrix's lookAt produces, with the translation column carrying the +// negative basis-dot-eye terms. +const setLookAt = ( + out: Float32Array, + ex: number, + ey: number, + ez: number, + tx: number, + ty: number, + tz: number, + ux: number, + uy: number, + uz: number, +) => { + "worklet"; + let zx = ex - tx; + let zy = ey - ty; + let zz = ez - tz; + const zLen = Math.hypot(zx, zy, zz); + zx /= zLen; + zy /= zLen; + zz /= zLen; + + let xx = uy * zz - uz * zy; + let xy = uz * zx - ux * zz; + let xz = ux * zy - uy * zx; + const xLen = Math.hypot(xx, xy, xz); + xx /= xLen; + xy /= xLen; + xz /= xLen; + + const yx = zy * xz - zz * xy; + const yy = zz * xx - zx * xz; + const yz = zx * xy - zy * xx; + + out[0] = xx; + out[1] = yx; + out[2] = zx; + out[3] = 0; + out[4] = xy; + out[5] = yy; + out[6] = zy; + out[7] = 0; + out[8] = xz; + out[9] = yz; + out[10] = zz; + out[11] = 0; + out[12] = -(xx * ex + xy * ey + xz * ez); + out[13] = -(yx * ex + yy * ey + yz * ez); + out[14] = -(zx * ex + zy * ey + zz * ez); + out[15] = 1; +}; + +// dst = a * b. Safe with dst === a or dst === b (snapshots both fully first). +const multiplyMat4 = (a: Float32Array, b: Float32Array, dst: Float32Array) => { + "worklet"; + const a00 = a[0], + a10 = a[1], + a20 = a[2], + a30 = a[3]; + const a01 = a[4], + a11 = a[5], + a21 = a[6], + a31 = a[7]; + const a02 = a[8], + a12 = a[9], + a22 = a[10], + a32 = a[11]; + const a03 = a[12], + a13 = a[13], + a23 = a[14], + a33 = a[15]; + const b00 = b[0], + b10 = b[1], + b20 = b[2], + b30 = b[3]; + const b01 = b[4], + b11 = b[5], + b21 = b[6], + b31 = b[7]; + const b02 = b[8], + b12 = b[9], + b22 = b[10], + b32 = b[11]; + const b03 = b[12], + b13 = b[13], + b23 = b[14], + b33 = b[15]; + + dst[0] = a00 * b00 + a01 * b10 + a02 * b20 + a03 * b30; + dst[1] = a10 * b00 + a11 * b10 + a12 * b20 + a13 * b30; + dst[2] = a20 * b00 + a21 * b10 + a22 * b20 + a23 * b30; + dst[3] = a30 * b00 + a31 * b10 + a32 * b20 + a33 * b30; + dst[4] = a00 * b01 + a01 * b11 + a02 * b21 + a03 * b31; + dst[5] = a10 * b01 + a11 * b11 + a12 * b21 + a13 * b31; + dst[6] = a20 * b01 + a21 * b11 + a22 * b21 + a23 * b31; + dst[7] = a30 * b01 + a31 * b11 + a32 * b21 + a33 * b31; + dst[8] = a00 * b02 + a01 * b12 + a02 * b22 + a03 * b32; + dst[9] = a10 * b02 + a11 * b12 + a12 * b22 + a13 * b32; + dst[10] = a20 * b02 + a21 * b12 + a22 * b22 + a23 * b32; + dst[11] = a30 * b02 + a31 * b12 + a32 * b22 + a33 * b32; + dst[12] = a00 * b03 + a01 * b13 + a02 * b23 + a03 * b33; + dst[13] = a10 * b03 + a11 * b13 + a12 * b23 + a13 * b33; + dst[14] = a20 * b03 + a21 * b13 + a22 * b23 + a23 * b33; + dst[15] = a30 * b03 + a31 * b13 + a32 * b23 + a33 * b33; +}; + +// The 3D variant of the VisionCamera demo. Reuses the same shared-texture-memory +// pipeline to import camera frames as GPUExternalTextures, but instead of +// applying 2D effects, it samples the camera as a spherical environment map on +// a chrome sphere, an orbiting cube, and a torus. + +const REQUIRED_FEATURES: GPUFeatureName[] = [ + "rnwebgpu/shared-texture-memory" as GPUFeatureName, + "dawn-multi-planar-formats" as GPUFeatureName, +]; + +// Android-only feature; same probe as VisionCamera.tsx. Without it Dawn can't +// wrap a YCbCr AHB as a GPUExternalTexture. +const OPAQUE_YCBCR_EXT = + "opaque-ycbcr-android-for-external-texture" as GPUFeatureName; + +const DEPTH_FORMAT: GPUTextureFormat = "depth24plus"; + +// Scene UBO layout. mat4(64) + vec4(16) + vec4(16) = 96 bytes; pad to 16-byte +// alignment for safety. +const SCENE_UBO_SIZE = 96; +const SCENE_UBO_FLOATS = SCENE_UBO_SIZE / 4; +const OBJECT_UBO_SIZE = 64; // mat4 + +type Shape = { + vertexBuffer: GPUBuffer; + indexBuffer: GPUBuffer; + indexCount: number; + uniformBuffer: GPUBuffer; + // Returns a model matrix for time t (seconds). + modelAt: (t: number, out: Float32Array) => void; +}; + +export const ChromeSphere = () => { + const { hasPermission, requestPermission } = useCameraPermission(); + useEffect(() => { + if (!hasPermission) { + requestPermission(); + } + }, [hasPermission, requestPermission]); + + if (!hasPermission) { + return ( + + + Camera access is required. Grant it in Settings or tap below. + + Linking.openSettings()} + style={styles.permissionButton} + > + Open Settings + + + ); + } + return ; +}; + +const SceneView = () => { + const ref = useCanvasRef(); + const [gpu, setGpu] = React.useState<{ + adapter: GPUAdapter; + device: GPUDevice; + } | null>(null); + const [deviceError, setDeviceError] = React.useState(null); + React.useEffect(() => { + let cancelled = false; + (async () => { + try { + const adapter = await navigator.gpu.requestAdapter(); + if (!adapter) { + throw new Error("requestAdapter returned null"); + } + const hasOpaqueYCbCrExt = + Platform.OS !== "android" || adapter.features.has(OPAQUE_YCBCR_EXT); + if (Platform.OS === "android" && !hasOpaqueYCbCrExt) { + throw new Error( + "This Android device's Vulkan driver doesn't advertise " + + "opaque-ycbcr-android-for-external-texture. Camera-frame import " + + "as a GPUExternalTexture isn't supported here.", + ); + } + const featuresToRequest: GPUFeatureName[] = [ + ...REQUIRED_FEATURES, + ...(Platform.OS === "android" ? [OPAQUE_YCBCR_EXT] : []), + ]; + const device = await adapter.requestDevice({ + requiredFeatures: featuresToRequest, + }); + if (cancelled) { + return; + } + setGpu({ adapter, device }); + } catch (e) { + if (cancelled) { + return; + } + setDeviceError(String(e)); + } + })(); + return () => { + cancelled = true; + }; + }, []); + const device = gpu?.device ?? null; + const adapter = gpu?.adapter ?? null; + const devices = useCameraDevices(); + const cameraDevice = React.useMemo( + () => + devices.find((d) => d.position === "back") ?? + devices.find((d) => d.position === "front") ?? + devices[0], + [devices], + ); + + const [pipelineState, setPipelineState] = React.useState<{ + pipeline: GPURenderPipeline; + sampler: GPUSampler; + sceneUniformBuffer: GPUBuffer; + depthView: GPUTextureView; + context: RNCanvasContext; + canvasWidth: number; + canvasHeight: number; + shapes: Shape[]; + // Pre-allocated scratch so the worklet doesn't allocate per frame. These + // get mutated each tick, which is safe because the worklet sees its own + // copy after closure serialization. + view: Float32Array; + proj: Float32Array; + viewProj: Float32Array; + sceneData: Float32Array; + modelScratch: Float32Array; + } | null>(null); + const [error, setError] = React.useState(null); + + useEffect(() => { + if (!device || pipelineState) { + return; + } + const missing = REQUIRED_FEATURES.filter((f) => !device.features.has(f)); + if (missing.length > 0) { + setError( + `Device missing features [${missing.join(", ")}]. Adapter: ${ + adapter + ? [...adapter.features] + .filter((f) => f.toString().startsWith("shared-")) + .join(", ") || "none" + : "n/a" + }`, + ); + return; + } + const context = ref.current?.getContext("webgpu"); + if (!context) { + return; + } + const canvas = context.canvas as unknown as NativeCanvas; + canvas.width = canvas.clientWidth * PixelRatio.get(); + canvas.height = canvas.clientHeight * PixelRatio.get(); + const presentationFormat = navigator.gpu.getPreferredCanvasFormat(); + context.configure({ + device, + format: presentationFormat, + alphaMode: "premultiplied", + }); + + const module = device.createShaderModule({ code: SHADER }); + const pipeline = device.createRenderPipeline({ + layout: "auto", + vertex: { + module, + entryPoint: "vs_main", + buffers: [ + { + arrayStride: VERTEX_STRIDE_BYTES, + attributes: [ + { + shaderLocation: 0, + offset: POSITION_OFFSET, + format: "float32x3", + }, + { + shaderLocation: 1, + offset: NORMAL_OFFSET, + format: "float32x3", + }, + ], + }, + ], + }, + fragment: { + module, + entryPoint: "fs_main", + targets: [{ format: presentationFormat }], + }, + primitive: { topology: "triangle-list", cullMode: "back" }, + depthStencil: { + depthCompare: "less", + depthWriteEnabled: true, + format: DEPTH_FORMAT, + }, + }); + const sampler = device.createSampler({ + magFilter: "linear", + minFilter: "linear", + }); + const sceneUniformBuffer = device.createBuffer({ + size: SCENE_UBO_SIZE, + usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, + }); + const depthTexture = device.createTexture({ + size: [canvas.width, canvas.height], + format: DEPTH_FORMAT, + usage: GPUTextureUsage.RENDER_ATTACHMENT, + }); + + const sphereMesh = generateSphere(1.0, 48, 64); + const cubeMesh = generateCube(0.9); + const torusMesh = generateTorus(0.55, 0.18, 48, 24); + + const buildShape = ( + mesh: { vertices: Float32Array; indices: Uint16Array }, + modelAt: (t: number, out: Float32Array) => void, + ): Shape => { + const vertexBuffer = device.createBuffer({ + size: mesh.vertices.byteLength, + usage: GPUBufferUsage.VERTEX | GPUBufferUsage.COPY_DST, + }); + device.queue.writeBuffer(vertexBuffer, 0, mesh.vertices); + // Index buffers must be sized to a multiple of 4 bytes; pad Uint16Array + // by one extra index if needed. + const indexByteLength = (mesh.indices.byteLength + 3) & ~3; + const indexBuffer = device.createBuffer({ + size: indexByteLength, + usage: GPUBufferUsage.INDEX | GPUBufferUsage.COPY_DST, + }); + device.queue.writeBuffer(indexBuffer, 0, mesh.indices); + const uniformBuffer = device.createBuffer({ + size: OBJECT_UBO_SIZE, + usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, + }); + // The full bind group is built per-frame in the worklet because + // GPUExternalTexture is single-use; scene + object buffers above stay + // alive across frames and feed back into that per-frame build. + return { + vertexBuffer, + indexBuffer, + indexCount: mesh.indices.length, + uniformBuffer, + modelAt, + }; + }; + + // Layout: sphere center stage, cube and torus orbiting on a tilted ring + // 1.9 units out. All three counter-rotate so motion reads as a clear + // composition rather than a swirl. These run inside the frame worklet. + const shapes: Shape[] = [ + buildShape(sphereMesh, (t, out) => { + "worklet"; + setIdentity(out); + applyRotateY(out, t * 0.25, out); + }), + buildShape(cubeMesh, (t, out) => { + "worklet"; + setIdentity(out); + applyRotateY(out, t * 0.35, out); + applyTranslate(out, 1.9, 0.4, 0.0, out); + applyRotateY(out, t * -0.9, out); + applyRotateX(out, t * 0.6, out); + }), + buildShape(torusMesh, (t, out) => { + "worklet"; + setIdentity(out); + applyRotateY(out, t * 0.35 + Math.PI, out); + applyTranslate(out, 1.9, -0.3, 0.0, out); + applyRotateX(out, t * 0.5, out); + applyRotateZ(out, t * 0.3, out); + }), + ]; + + setPipelineState({ + pipeline, + sampler, + sceneUniformBuffer, + depthView: depthTexture.createView(), + context, + canvasWidth: canvas.width, + canvasHeight: canvas.height, + shapes, + view: new Float32Array(16), + proj: new Float32Array(16), + viewProj: new Float32Array(16), + sceneData: new Float32Array(SCENE_UBO_FLOATS), + modelScratch: new Float32Array(16), + }); + }, [device, adapter, ref, pipelineState]); + + const startTimeRef = React.useRef(performance.now()); + const logBox = React.useMemo(() => ({ seen: false }), []); + + const frameOutput = useFrameOutput({ + pixelFormat: "native", + onFrame: (frame) => { + "worklet"; + if (!logBox.seen) { + logBox.seen = true; + console.log( + "[ChromeSphere] worklet first frame, hasPipeline=" + + String(pipelineState != null) + + " frame=" + + String(frame.width) + + "x" + + String(frame.height), + ); + } + if (!pipelineState || !device) { + frame.dispose(); + return; + } + const { + pipeline, + sampler, + sceneUniformBuffer, + depthView, + context, + canvasWidth, + canvasHeight, + shapes, + view, + proj, + viewProj, + sceneData, + modelScratch, + } = pipelineState; + const nativeBuffer = frame.getNativeBuffer(); + try { + const videoFrame = device.createVideoFrameFromNativeBuffer( + nativeBuffer.pointer, + ); + try { + const t = (performance.now() - startTimeRef.current) / 1000; + + // Orbit the eye around the origin, looking at it. Small bob in y + // so reflections shift along the polar axis too. + const orbitR = 5.2; + const ex = Math.cos(t * 0.15) * orbitR; + const ey = 1.2 + Math.sin(t * 0.2) * 0.4; + const ez = Math.sin(t * 0.15) * orbitR; + // Bias the look target toward the orbiters so they read as the + // composition's center of attention. + setLookAt(view, ex, ey, ez, 0.6, 0.0, 0.0, 0, 1, 0); + setPerspective( + proj, + Math.PI / 4, + canvasWidth / canvasHeight, + 0.1, + 100, + ); + multiplyMat4(proj, view, viewProj); + + sceneData.set(viewProj, 0); + sceneData[16] = ex; + sceneData[17] = ey; + sceneData[18] = ez; + sceneData[19] = 0; + // Key light rises and slowly drifts so the chrome specular sweeps + // across the silhouettes. + const ldRawX = Math.cos(t * 0.3) * 0.6; + const ldRawY = 0.8; + const ldRawZ = Math.sin(t * 0.3) * 0.6; + const ldLen = Math.hypot(ldRawX, ldRawY, ldRawZ); + sceneData[20] = ldRawX / ldLen; + sceneData[21] = ldRawY / ldLen; + sceneData[22] = ldRawZ / ldLen; + sceneData[23] = 0; + device.queue.writeBuffer(sceneUniformBuffer, 0, sceneData); + + for (const shape of shapes) { + shape.modelAt(t, modelScratch); + device.queue.writeBuffer(shape.uniformBuffer, 0, modelScratch); + } + + let externalTex; + try { + externalTex = device.importExternalTexture({ + source: videoFrame, + label: "chrome-env", + }); + } catch (e) { + console.warn( + "[ChromeSphere] importExternalTexture threw: " + String(e), + ); + throw e; + } + + const encoder = device.createCommandEncoder(); + const pass = encoder.beginRenderPass({ + colorAttachments: [ + { + view: context.getCurrentTexture().createView(), + clearValue: { r: 0.02, g: 0.02, b: 0.03, a: 1 }, + loadOp: "clear", + storeOp: "store", + }, + ], + depthStencilAttachment: { + view: depthView, + depthClearValue: 1.0, + depthLoadOp: "clear", + depthStoreOp: "store", + }, + }); + pass.setPipeline(pipeline); + for (const shape of shapes) { + // The external texture is bound per-shape (one bind group per + // shape, the only difference being the per-object uniform), so + // we rebuild the bind group each frame to splice in this frame's + // externalTex alongside the cached scene + object buffers. + const bindGroup = device.createBindGroup({ + layout: pipeline.getBindGroupLayout(0), + entries: [ + { binding: 0, resource: { buffer: sceneUniformBuffer } }, + { binding: 1, resource: { buffer: shape.uniformBuffer } }, + { binding: 2, resource: externalTex }, + { binding: 3, resource: sampler }, + ], + }); + pass.setBindGroup(0, bindGroup); + pass.setVertexBuffer(0, shape.vertexBuffer); + pass.setIndexBuffer(shape.indexBuffer, "uint16"); + pass.drawIndexed(shape.indexCount); + } + pass.end(); + device.queue.submit([encoder.finish()]); + context.present(); + } finally { + videoFrame.release(); + } + } finally { + nativeBuffer.release(); + frame.dispose(); + } + }, + }); + + useCamera({ + isActive: pipelineState != null && cameraDevice != null, + device: cameraDevice as NonNullable, + outputs: [frameOutput], + }); + + if (deviceError) { + return ( + + + Device creation failed: {deviceError} + + + ); + } + if (error) { + return ( + + {error} + + ); + } + if (!device) { + return ( + + Waiting for GPU device... + + ); + } + if (cameraDevice == null) { + return ( + + + No camera available. This screen needs a physical device with a camera + (the iOS Simulator does not have one). + + + ); + } + return ( + + + + ); +}; + +const styles = StyleSheet.create({ + root: { flex: 1, backgroundColor: "black" }, + canvas: { flex: 1 }, + errorContainer: { flex: 1, padding: 16, justifyContent: "center" }, + errorText: { color: "red", fontSize: 14 }, + permissionContainer: { + flex: 1, + padding: 24, + justifyContent: "center", + alignItems: "center", + }, + permissionText: { fontSize: 16, textAlign: "center", marginBottom: 16 }, + permissionButton: { + backgroundColor: "#007AFF", + paddingHorizontal: 24, + paddingVertical: 12, + borderRadius: 8, + }, + permissionButtonText: { color: "white", fontSize: 16, fontWeight: "600" }, +}); diff --git a/apps/example/src/ChromeSphere/geometry.ts b/apps/example/src/ChromeSphere/geometry.ts new file mode 100644 index 000000000..c5d2e64c8 --- /dev/null +++ b/apps/example/src/ChromeSphere/geometry.ts @@ -0,0 +1,169 @@ +// Tiny mesh generators for the chrome scene. Each returns interleaved +// position+normal vertices and a triangle index list. The shader uses the +// upper-left 3x3 of the model matrix to transform normals, so all shapes +// here must be built with unit-length normals (no non-uniform scale at +// generation time). + +export type Mesh = { + vertices: Float32Array; // [px, py, pz, nx, ny, nz] * N + indices: Uint16Array; +}; + +const FLOATS_PER_VERTEX = 6; + +export const VERTEX_STRIDE_BYTES = FLOATS_PER_VERTEX * 4; +export const POSITION_OFFSET = 0; +export const NORMAL_OFFSET = 12; + +export function generateSphere( + radius: number, + latBands: number, + lonBands: number, +): Mesh { + const verts: number[] = []; + const idx: number[] = []; + for (let lat = 0; lat <= latBands; lat++) { + const theta = (lat * Math.PI) / latBands; + const sinTheta = Math.sin(theta); + const cosTheta = Math.cos(theta); + for (let lon = 0; lon <= lonBands; lon++) { + const phi = (lon * 2 * Math.PI) / lonBands; + const sinPhi = Math.sin(phi); + const cosPhi = Math.cos(phi); + const nx = cosPhi * sinTheta; + const ny = cosTheta; + const nz = sinPhi * sinTheta; + verts.push(nx * radius, ny * radius, nz * radius, nx, ny, nz); + } + } + const stride = lonBands + 1; + for (let lat = 0; lat < latBands; lat++) { + for (let lon = 0; lon < lonBands; lon++) { + const a = lat * stride + lon; + const b = a + stride; + idx.push(a, b, a + 1, b, b + 1, a + 1); + } + } + return { + vertices: new Float32Array(verts), + indices: new Uint16Array(idx), + }; +} + +export function generateCube(size: number): Mesh { + const s = size / 2; + // Each face has its own 4 vertices so face normals stay flat (no smoothing + // across edges, which would defeat the cube-as-mirror look). + const faces: { + n: [number, number, number]; + corners: [number, number, number][]; + }[] = [ + { + n: [1, 0, 0], + corners: [ + [s, -s, -s], + [s, s, -s], + [s, s, s], + [s, -s, s], + ], + }, + { + n: [-1, 0, 0], + corners: [ + [-s, -s, s], + [-s, s, s], + [-s, s, -s], + [-s, -s, -s], + ], + }, + { + n: [0, 1, 0], + corners: [ + [-s, s, -s], + [-s, s, s], + [s, s, s], + [s, s, -s], + ], + }, + { + n: [0, -1, 0], + corners: [ + [-s, -s, s], + [-s, -s, -s], + [s, -s, -s], + [s, -s, s], + ], + }, + { + n: [0, 0, 1], + corners: [ + [-s, -s, s], + [s, -s, s], + [s, s, s], + [-s, s, s], + ], + }, + { + n: [0, 0, -1], + corners: [ + [s, -s, -s], + [-s, -s, -s], + [-s, s, -s], + [s, s, -s], + ], + }, + ]; + const verts: number[] = []; + const idx: number[] = []; + faces.forEach((face, fi) => { + face.corners.forEach((c) => { + verts.push(c[0], c[1], c[2], face.n[0], face.n[1], face.n[2]); + }); + const base = fi * 4; + idx.push(base, base + 1, base + 2, base, base + 2, base + 3); + }); + return { + vertices: new Float32Array(verts), + indices: new Uint16Array(idx), + }; +} + +export function generateTorus( + majorRadius: number, + minorRadius: number, + majorSegments: number, + minorSegments: number, +): Mesh { + const verts: number[] = []; + const idx: number[] = []; + for (let i = 0; i <= majorSegments; i++) { + const u = (i / majorSegments) * 2 * Math.PI; + const cu = Math.cos(u); + const su = Math.sin(u); + for (let j = 0; j <= minorSegments; j++) { + const v = (j / minorSegments) * 2 * Math.PI; + const cv = Math.cos(v); + const sv = Math.sin(v); + const x = (majorRadius + minorRadius * cv) * cu; + const y = minorRadius * sv; + const z = (majorRadius + minorRadius * cv) * su; + // Outward-from-minor-circle normal, unit length by construction. + const nx = cv * cu; + const ny = sv; + const nz = cv * su; + verts.push(x, y, z, nx, ny, nz); + } + } + const stride = minorSegments + 1; + for (let i = 0; i < majorSegments; i++) { + for (let j = 0; j < minorSegments; j++) { + const a = i * stride + j; + const b = a + stride; + idx.push(a, b, a + 1, b, b + 1, a + 1); + } + } + return { + vertices: new Float32Array(verts), + indices: new Uint16Array(idx), + }; +} diff --git a/apps/example/src/ChromeSphere/index.ts b/apps/example/src/ChromeSphere/index.ts new file mode 100644 index 000000000..5f32bab2e --- /dev/null +++ b/apps/example/src/ChromeSphere/index.ts @@ -0,0 +1 @@ +export * from "./ChromeSphere"; diff --git a/apps/example/src/ChromeSphere/shader.ts b/apps/example/src/ChromeSphere/shader.ts new file mode 100644 index 000000000..6f945f803 --- /dev/null +++ b/apps/example/src/ChromeSphere/shader.ts @@ -0,0 +1,94 @@ +// Chrome-style env-map reflection driven by the live camera feed. +// +// For each fragment we compute the reflection vector around the world-space +// normal, convert it to spherical (theta, phi) → uv, and sample the external +// camera texture. textureSampleBaseClampToEdge is the only sampling call +// allowed on texture_external, but it's all we need: spherical uvs land +// inside [0, 1]^2 so no wrap mode tricks are required. +// +// The visible result is the classic CGI chrome ball: your face wraps around +// the sphere as if the camera image were the ambient environment, and the +// cube + torus reflect distorted copies of the same scene. + +export const SHADER = /* wgsl */ ` +struct Scene { + viewProj: mat4x4f, + cameraPos: vec4f, // xyz = world-space eye, w unused + lightDir: vec4f, // xyz = unit vector toward light, w unused +}; + +struct Object { + model: mat4x4f, +}; + +@group(0) @binding(0) var scene: Scene; +@group(0) @binding(1) var obj: Object; +@group(0) @binding(2) var srcTex: texture_external; +@group(0) @binding(3) var srcSampler: sampler; + +struct VsIn { + @location(0) position: vec3f, + @location(1) normal: vec3f, +}; + +struct VsOut { + @builtin(position) clipPos: vec4f, + @location(0) worldPos: vec3f, + @location(1) worldNormal: vec3f, +}; + +@vertex +fn vs_main(in: VsIn) -> VsOut { + let world = obj.model * vec4f(in.position, 1.0); + // Rotation-only transforms: normal matrix is the upper-left 3x3 of model. + // No non-uniform scale anywhere in the scene, so inverse-transpose collapses + // to this. + let normalMat = mat3x3f( + obj.model[0].xyz, + obj.model[1].xyz, + obj.model[2].xyz, + ); + var out: VsOut; + out.clipPos = scene.viewProj * world; + out.worldPos = world.xyz; + out.worldNormal = normalize(normalMat * in.normal); + return out; +} + +const PI: f32 = 3.14159265359; + +fn sphericalUv(dir: vec3f) -> vec2f { + // dir is treated as a direction from the chrome surface to the + // environment. theta wraps around the y-axis, phi runs pole to pole. + let theta = atan2(dir.z, dir.x); + let phi = acos(clamp(dir.y, -1.0, 1.0)); + return vec2f((theta + PI) / (2.0 * PI), phi / PI); +} + +@fragment +fn fs_main(in: VsOut) -> @location(0) vec4f { + let n = normalize(in.worldNormal); + let v = normalize(scene.cameraPos.xyz - in.worldPos); + // reflect() expects the incident vector (from light/source to surface), so + // negate the view direction. + let r = normalize(reflect(-v, n)); + + let uv = sphericalUv(r); + let env = textureSampleBaseClampToEdge(srcTex, srcSampler, uv).rgb; + + // Schlick-ish Fresnel: chrome reflectance is high everywhere, but boost a + // bit at grazing angles so silhouettes catch the light. + let cosTheta = max(dot(n, v), 0.0); + let fresnel = pow(1.0 - cosTheta, 3.0); + let tint = vec3f(0.96, 0.97, 1.00); // subtle cool cast, very polished chrome + var color = env * tint + vec3f(fresnel) * 0.18; + + // Single directional specular highlight so the chrome doesn't look like a + // pure billboard. Halfway vector with a tight exponent. + let h = normalize(scene.lightDir.xyz + v); + let spec = pow(max(dot(n, h), 0.0), 96.0); + color = color + vec3f(spec) * 1.2; + + return vec4f(color, 1.0); +} +`; diff --git a/apps/example/src/Home.tsx b/apps/example/src/Home.tsx index 66489c5c0..d43f0822e 100644 --- a/apps/example/src/Home.tsx +++ b/apps/example/src/Home.tsx @@ -139,6 +139,10 @@ export const examples = [ screen: "VisionCamera", title: "đź“· VisionCamera integration", }, + { + screen: "ChromeSphere", + title: "🪩 Chrome Sphere (camera env map)", + }, ]; const styles = StyleSheet.create({ diff --git a/apps/example/src/Route.ts b/apps/example/src/Route.ts index 33c920f29..57f09c992 100644 --- a/apps/example/src/Route.ts +++ b/apps/example/src/Route.ts @@ -32,4 +32,5 @@ export type Routes = { SharedTextureMemory: undefined; ExternalTexture: undefined; VisionCamera: undefined; + ChromeSphere: undefined; }; From 34c0ff325f65a888318e02d43f791e8632abd45c Mon Sep 17 00:00:00 2001 From: William Candillon Date: Fri, 22 May 2026 15:00:32 +0200 Subject: [PATCH 16/46] :wrench: --- .../example/src/ChromeSphere/ChromeSphere.tsx | 406 ++++++++++++------ apps/example/src/ChromeSphere/geometry.ts | 125 +----- apps/example/src/ChromeSphere/shader.ts | 43 ++ 3 files changed, 311 insertions(+), 263 deletions(-) diff --git a/apps/example/src/ChromeSphere/ChromeSphere.tsx b/apps/example/src/ChromeSphere/ChromeSphere.tsx index bcbf847cd..287962d6f 100644 --- a/apps/example/src/ChromeSphere/ChromeSphere.tsx +++ b/apps/example/src/ChromeSphere/ChromeSphere.tsx @@ -22,15 +22,15 @@ import { useFrameOutput, } from "react-native-vision-camera"; +import { BLUR_SHADER, PREPASS_SHADER } from "../VisionCamera/blurShaders"; + import { - generateCube, generateSphere, - generateTorus, NORMAL_OFFSET, POSITION_OFFSET, VERTEX_STRIDE_BYTES, } from "./geometry"; -import { SHADER } from "./shader"; +import { BACKDROP_SHADER, SHADER } from "./shader"; // All matrix math runs inside the Vision Camera frame-processor worklet, so it // has to be implemented with worklet-friendly helpers. wgpu-matrix calls @@ -59,40 +59,6 @@ const setIdentity = (out: Float32Array) => { out[15] = 1; }; -// dst = m * Rx(angle). Safe with dst === m: only columns 1 and 2 change, and -// we snapshot them into locals before writing. -const applyRotateX = (m: Float32Array, angle: number, dst: Float32Array) => { - "worklet"; - const c = Math.cos(angle); - const s = Math.sin(angle); - const m01 = m[4], - m11 = m[5], - m21 = m[6], - m31 = m[7]; - const m02 = m[8], - m12 = m[9], - m22 = m[10], - m32 = m[11]; - if (dst !== m) { - dst[0] = m[0]; - dst[1] = m[1]; - dst[2] = m[2]; - dst[3] = m[3]; - dst[12] = m[12]; - dst[13] = m[13]; - dst[14] = m[14]; - dst[15] = m[15]; - } - dst[4] = c * m01 + s * m02; - dst[5] = c * m11 + s * m12; - dst[6] = c * m21 + s * m22; - dst[7] = c * m31 + s * m32; - dst[8] = -s * m01 + c * m02; - dst[9] = -s * m11 + c * m12; - dst[10] = -s * m21 + c * m22; - dst[11] = -s * m31 + c * m32; -}; - // dst = m * Ry(angle). Safe with dst === m. const applyRotateY = (m: Float32Array, angle: number, dst: Float32Array) => { "worklet"; @@ -126,85 +92,6 @@ const applyRotateY = (m: Float32Array, angle: number, dst: Float32Array) => { dst[11] = s * m30 + c * m32; }; -// dst = m * Rz(angle). Safe with dst === m. -const applyRotateZ = (m: Float32Array, angle: number, dst: Float32Array) => { - "worklet"; - const c = Math.cos(angle); - const s = Math.sin(angle); - const m00 = m[0], - m10 = m[1], - m20 = m[2], - m30 = m[3]; - const m01 = m[4], - m11 = m[5], - m21 = m[6], - m31 = m[7]; - if (dst !== m) { - dst[8] = m[8]; - dst[9] = m[9]; - dst[10] = m[10]; - dst[11] = m[11]; - dst[12] = m[12]; - dst[13] = m[13]; - dst[14] = m[14]; - dst[15] = m[15]; - } - dst[0] = c * m00 + s * m01; - dst[1] = c * m10 + s * m11; - dst[2] = c * m20 + s * m21; - dst[3] = c * m30 + s * m31; - dst[4] = -s * m00 + c * m01; - dst[5] = -s * m10 + c * m11; - dst[6] = -s * m20 + c * m21; - dst[7] = -s * m30 + c * m31; -}; - -// dst = m * translate(tx, ty, tz). Only column 3 changes; safe with dst === m -// because we read the old col0/1/2 entries and the old col3 first. -const applyTranslate = ( - m: Float32Array, - tx: number, - ty: number, - tz: number, - dst: Float32Array, -) => { - "worklet"; - const c0r0 = m[0], - c0r1 = m[1], - c0r2 = m[2], - c0r3 = m[3]; - const c1r0 = m[4], - c1r1 = m[5], - c1r2 = m[6], - c1r3 = m[7]; - const c2r0 = m[8], - c2r1 = m[9], - c2r2 = m[10], - c2r3 = m[11]; - const c3r0 = m[12], - c3r1 = m[13], - c3r2 = m[14], - c3r3 = m[15]; - if (dst !== m) { - dst[0] = c0r0; - dst[1] = c0r1; - dst[2] = c0r2; - dst[3] = c0r3; - dst[4] = c1r0; - dst[5] = c1r1; - dst[6] = c1r2; - dst[7] = c1r3; - dst[8] = c2r0; - dst[9] = c2r1; - dst[10] = c2r2; - dst[11] = c2r3; - } - dst[12] = c0r0 * tx + c1r0 * ty + c2r0 * tz + c3r0; - dst[13] = c0r1 * tx + c1r1 * ty + c2r1 * tz + c3r1; - dst[14] = c0r2 * tx + c1r2 * ty + c2r2 * tz + c3r2; - dst[15] = c0r3 * tx + c1r3 * ty + c2r3 * tz + c3r3; -}; - // WebGPU-style perspective: right-handed, output z mapped to [0, 1]. fovy is // the vertical field of view in radians. const setPerspective = ( @@ -360,6 +247,16 @@ const OPAQUE_YCBCR_EXT = const DEPTH_FORMAT: GPUTextureFormat = "depth24plus"; +// Backdrop blur tuning. Matches VisionCamera's "Strong" preset: prepass to a +// 1/4-res rgba8unorm, then 3 H-V iterations of the tile-based box blur. The +// final result is sampled by BACKDROP_SHADER as a fullscreen backdrop. +const BLUR_SCALE = 4; +const BLUR_FILTER_SIZE = 31; +const BLUR_TILE_DIM = 128; +const BLUR_BATCH = 4; +const BLUR_BLOCK_DIM = BLUR_TILE_DIM - BLUR_FILTER_SIZE; +const BLUR_ITERATIONS = 3; + // Scene UBO layout. mat4(64) + vec4(16) + vec4(16) = 96 bytes; pad to 16-byte // alignment for safety. const SCENE_UBO_SIZE = 96; @@ -475,6 +372,19 @@ const SceneView = () => { viewProj: Float32Array; sceneData: Float32Array; modelScratch: Float32Array; + // Blurred-camera backdrop infrastructure. + backdropPipeline: GPURenderPipeline; + backdropBindGroup: GPUBindGroup; + prepassPipeline: GPURenderPipeline; + prepassUniformBuffer: GPUBuffer; + blurPipeline: GPUComputePipeline; + blurConstants: GPUBindGroup; + blurBindGroup0: GPUBindGroup; + blurBindGroup1: GPUBindGroup; + blurBindGroup2: GPUBindGroup; + blurSrcTexture: GPUTexture; + blurWidth: number; + blurHeight: number; } | null>(null); const [error, setError] = React.useState(null); @@ -560,8 +470,6 @@ const SceneView = () => { }); const sphereMesh = generateSphere(1.0, 48, 64); - const cubeMesh = generateCube(0.9); - const torusMesh = generateTorus(0.55, 0.18, 48, 24); const buildShape = ( mesh: { vertices: Float32Array; indices: Uint16Array }, @@ -596,33 +504,156 @@ const SceneView = () => { }; }; - // Layout: sphere center stage, cube and torus orbiting on a tilted ring - // 1.9 units out. All three counter-rotate so motion reads as a clear - // composition rather than a swirl. These run inside the frame worklet. + // Single chrome sphere center stage with a slow Y-rotation so the + // reflection drifts even when the orbit camera is between key positions. + // Runs inside the frame worklet. const shapes: Shape[] = [ buildShape(sphereMesh, (t, out) => { "worklet"; setIdentity(out); applyRotateY(out, t * 0.25, out); }), - buildShape(cubeMesh, (t, out) => { - "worklet"; - setIdentity(out); - applyRotateY(out, t * 0.35, out); - applyTranslate(out, 1.9, 0.4, 0.0, out); - applyRotateY(out, t * -0.9, out); - applyRotateX(out, t * 0.6, out); - }), - buildShape(torusMesh, (t, out) => { - "worklet"; - setIdentity(out); - applyRotateY(out, t * 0.35 + Math.PI, out); - applyTranslate(out, 1.9, -0.3, 0.0, out); - applyRotateX(out, t * 0.5, out); - applyRotateZ(out, t * 0.3, out); - }), ]; + // ----- Backdrop blur infrastructure (same as VisionCamera "Strong") --- + const blurWidth = Math.max( + BLUR_TILE_DIM, + Math.ceil(canvas.width / BLUR_SCALE), + ); + const blurHeight = Math.max( + BLUR_TILE_DIM, + Math.ceil(canvas.height / BLUR_SCALE), + ); + + const prepassModule = device.createShaderModule({ code: PREPASS_SHADER }); + const prepassPipeline = device.createRenderPipeline({ + layout: "auto", + vertex: { module: prepassModule, entryPoint: "vs_main" }, + fragment: { + module: prepassModule, + entryPoint: "fs_main", + targets: [{ format: "rgba8unorm" }], + }, + primitive: { topology: "triangle-list" }, + }); + const prepassUniformBuffer = device.createBuffer({ + size: 16, + usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, + }); + + const blurSrcTexture = device.createTexture({ + size: [blurWidth, blurHeight], + format: "rgba8unorm", + usage: + GPUTextureUsage.RENDER_ATTACHMENT | GPUTextureUsage.TEXTURE_BINDING, + }); + const blurPing = [0, 1].map(() => + device.createTexture({ + size: [blurWidth, blurHeight], + format: "rgba8unorm", + usage: + GPUTextureUsage.STORAGE_BINDING | GPUTextureUsage.TEXTURE_BINDING, + }), + ); + + const blurPipeline = device.createComputePipeline({ + layout: "auto", + compute: { + module: device.createShaderModule({ code: BLUR_SHADER }), + entryPoint: "main", + }, + }); + + const flip0Buffer = device.createBuffer({ + size: 4, + mappedAtCreation: true, + usage: GPUBufferUsage.UNIFORM, + }); + new Uint32Array(flip0Buffer.getMappedRange())[0] = 0; + flip0Buffer.unmap(); + const flip1Buffer = device.createBuffer({ + size: 4, + mappedAtCreation: true, + usage: GPUBufferUsage.UNIFORM, + }); + new Uint32Array(flip1Buffer.getMappedRange())[0] = 1; + flip1Buffer.unmap(); + + const blurParamsBuffer = device.createBuffer({ + size: 8, + usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, + }); + device.queue.writeBuffer( + blurParamsBuffer, + 0, + new Uint32Array([BLUR_FILTER_SIZE + 1, BLUR_BLOCK_DIM]), + ); + + const blurConstants = device.createBindGroup({ + layout: blurPipeline.getBindGroupLayout(0), + entries: [ + { binding: 0, resource: sampler }, + { binding: 1, resource: { buffer: blurParamsBuffer } }, + ], + }); + // H: blurSrcTexture -> blurPing[0] + const blurBindGroup0 = device.createBindGroup({ + layout: blurPipeline.getBindGroupLayout(1), + entries: [ + { binding: 1, resource: blurSrcTexture.createView() }, + { binding: 2, resource: blurPing[0].createView() }, + { binding: 3, resource: { buffer: flip0Buffer } }, + ], + }); + // V: blurPing[0] -> blurPing[1] + const blurBindGroup1 = device.createBindGroup({ + layout: blurPipeline.getBindGroupLayout(1), + entries: [ + { binding: 1, resource: blurPing[0].createView() }, + { binding: 2, resource: blurPing[1].createView() }, + { binding: 3, resource: { buffer: flip1Buffer } }, + ], + }); + // H (iteration N>=2): blurPing[1] -> blurPing[0] + const blurBindGroup2 = device.createBindGroup({ + layout: blurPipeline.getBindGroupLayout(1), + entries: [ + { binding: 1, resource: blurPing[1].createView() }, + { binding: 2, resource: blurPing[0].createView() }, + { binding: 3, resource: { buffer: flip0Buffer } }, + ], + }); + // Final iteration's V pass always lands in blurPing[1]. + const blurredView = blurPing[1].createView(); + + // Backdrop pipeline: shares the render pass with the chrome sphere, so + // it must declare a matching depth-stencil layout. depthCompare always / + // depthWriteEnabled false means it draws unconditionally and never + // disturbs depth for the subsequent sphere draw. + const backdropModule = device.createShaderModule({ code: BACKDROP_SHADER }); + const backdropPipeline = device.createRenderPipeline({ + layout: "auto", + vertex: { module: backdropModule, entryPoint: "vs_main" }, + fragment: { + module: backdropModule, + entryPoint: "fs_main", + targets: [{ format: presentationFormat }], + }, + primitive: { topology: "triangle-list" }, + depthStencil: { + depthCompare: "always", + depthWriteEnabled: false, + format: DEPTH_FORMAT, + }, + }); + const backdropBindGroup = device.createBindGroup({ + layout: backdropPipeline.getBindGroupLayout(0), + entries: [ + { binding: 0, resource: blurredView }, + { binding: 1, resource: sampler }, + ], + }); + setPipelineState({ pipeline, sampler, @@ -637,6 +668,18 @@ const SceneView = () => { viewProj: new Float32Array(16), sceneData: new Float32Array(SCENE_UBO_FLOATS), modelScratch: new Float32Array(16), + backdropPipeline, + backdropBindGroup, + prepassPipeline, + prepassUniformBuffer, + blurPipeline, + blurConstants, + blurBindGroup0, + blurBindGroup1, + blurBindGroup2, + blurSrcTexture, + blurWidth, + blurHeight, }); }, [device, adapter, ref, pipelineState]); @@ -676,6 +719,18 @@ const SceneView = () => { viewProj, sceneData, modelScratch, + backdropPipeline, + backdropBindGroup, + prepassPipeline, + prepassUniformBuffer, + blurPipeline, + blurConstants, + blurBindGroup0, + blurBindGroup1, + blurBindGroup2, + blurSrcTexture, + blurWidth, + blurHeight, } = pipelineState; const nativeBuffer = frame.getNativeBuffer(); try { @@ -691,9 +746,7 @@ const SceneView = () => { const ex = Math.cos(t * 0.15) * orbitR; const ey = 1.2 + Math.sin(t * 0.2) * 0.4; const ez = Math.sin(t * 0.15) * orbitR; - // Bias the look target toward the orbiters so they read as the - // composition's center of attention. - setLookAt(view, ex, ey, ez, 0.6, 0.0, 0.0, 0, 1, 0); + setLookAt(view, ex, ey, ez, 0.0, 0.0, 0.0, 0, 1, 0); setPerspective( proj, Math.PI / 4, @@ -739,11 +792,74 @@ const SceneView = () => { } const encoder = device.createCommandEncoder(); + + // ---- Backdrop blur (prepass + 3 H-V iterations at 1/4 res) ---- + device.queue.writeBuffer( + prepassUniformBuffer, + 0, + new Float32Array([ + videoFrame.width, + videoFrame.height, + canvasWidth, + canvasHeight, + ]), + ); + const prepassBindGroup = device.createBindGroup({ + layout: prepassPipeline.getBindGroupLayout(0), + entries: [ + { binding: 0, resource: externalTex }, + { binding: 1, resource: sampler }, + { binding: 2, resource: { buffer: prepassUniformBuffer } }, + ], + }); + const prepass = encoder.beginRenderPass({ + colorAttachments: [ + { + view: blurSrcTexture.createView(), + clearValue: { r: 0, g: 0, b: 0, a: 1 }, + loadOp: "clear", + storeOp: "store", + }, + ], + }); + prepass.setPipeline(prepassPipeline); + prepass.setBindGroup(0, prepassBindGroup); + prepass.draw(3); + prepass.end(); + + const compute = encoder.beginComputePass(); + compute.setPipeline(blurPipeline); + compute.setBindGroup(0, blurConstants); + compute.setBindGroup(1, blurBindGroup0); + compute.dispatchWorkgroups( + Math.ceil(blurWidth / BLUR_BLOCK_DIM), + Math.ceil(blurHeight / BLUR_BATCH), + ); + compute.setBindGroup(1, blurBindGroup1); + compute.dispatchWorkgroups( + Math.ceil(blurHeight / BLUR_BLOCK_DIM), + Math.ceil(blurWidth / BLUR_BATCH), + ); + for (let i = 0; i < BLUR_ITERATIONS - 1; i++) { + compute.setBindGroup(1, blurBindGroup2); + compute.dispatchWorkgroups( + Math.ceil(blurWidth / BLUR_BLOCK_DIM), + Math.ceil(blurHeight / BLUR_BATCH), + ); + compute.setBindGroup(1, blurBindGroup1); + compute.dispatchWorkgroups( + Math.ceil(blurHeight / BLUR_BLOCK_DIM), + Math.ceil(blurWidth / BLUR_BATCH), + ); + } + compute.end(); + + // ---- Main scene pass: backdrop first (no depth write), then sphere const pass = encoder.beginRenderPass({ colorAttachments: [ { view: context.getCurrentTexture().createView(), - clearValue: { r: 0.02, g: 0.02, b: 0.03, a: 1 }, + clearValue: { r: 0, g: 0, b: 0, a: 1 }, loadOp: "clear", storeOp: "store", }, @@ -755,6 +871,14 @@ const SceneView = () => { depthStoreOp: "store", }, }); + + // Backdrop pipeline has depthCompare: "always" and writes nothing + // to depth, so the subsequent sphere draw still sees a clean depth + // buffer with the canvas-clear far value. + pass.setPipeline(backdropPipeline); + pass.setBindGroup(0, backdropBindGroup); + pass.draw(3); + pass.setPipeline(pipeline); for (const shape of shapes) { // The external texture is bound per-shape (one bind group per diff --git a/apps/example/src/ChromeSphere/geometry.ts b/apps/example/src/ChromeSphere/geometry.ts index c5d2e64c8..1ede29fcd 100644 --- a/apps/example/src/ChromeSphere/geometry.ts +++ b/apps/example/src/ChromeSphere/geometry.ts @@ -1,8 +1,7 @@ -// Tiny mesh generators for the chrome scene. Each returns interleaved +// Sphere mesh generator for the chrome scene. Returns interleaved // position+normal vertices and a triangle index list. The shader uses the -// upper-left 3x3 of the model matrix to transform normals, so all shapes -// here must be built with unit-length normals (no non-uniform scale at -// generation time). +// upper-left 3x3 of the model matrix to transform normals, so the sphere is +// built with unit-length normals (no non-uniform scale at generation time). export type Mesh = { vertices: Float32Array; // [px, py, pz, nx, ny, nz] * N @@ -49,121 +48,3 @@ export function generateSphere( indices: new Uint16Array(idx), }; } - -export function generateCube(size: number): Mesh { - const s = size / 2; - // Each face has its own 4 vertices so face normals stay flat (no smoothing - // across edges, which would defeat the cube-as-mirror look). - const faces: { - n: [number, number, number]; - corners: [number, number, number][]; - }[] = [ - { - n: [1, 0, 0], - corners: [ - [s, -s, -s], - [s, s, -s], - [s, s, s], - [s, -s, s], - ], - }, - { - n: [-1, 0, 0], - corners: [ - [-s, -s, s], - [-s, s, s], - [-s, s, -s], - [-s, -s, -s], - ], - }, - { - n: [0, 1, 0], - corners: [ - [-s, s, -s], - [-s, s, s], - [s, s, s], - [s, s, -s], - ], - }, - { - n: [0, -1, 0], - corners: [ - [-s, -s, s], - [-s, -s, -s], - [s, -s, -s], - [s, -s, s], - ], - }, - { - n: [0, 0, 1], - corners: [ - [-s, -s, s], - [s, -s, s], - [s, s, s], - [-s, s, s], - ], - }, - { - n: [0, 0, -1], - corners: [ - [s, -s, -s], - [-s, -s, -s], - [-s, s, -s], - [s, s, -s], - ], - }, - ]; - const verts: number[] = []; - const idx: number[] = []; - faces.forEach((face, fi) => { - face.corners.forEach((c) => { - verts.push(c[0], c[1], c[2], face.n[0], face.n[1], face.n[2]); - }); - const base = fi * 4; - idx.push(base, base + 1, base + 2, base, base + 2, base + 3); - }); - return { - vertices: new Float32Array(verts), - indices: new Uint16Array(idx), - }; -} - -export function generateTorus( - majorRadius: number, - minorRadius: number, - majorSegments: number, - minorSegments: number, -): Mesh { - const verts: number[] = []; - const idx: number[] = []; - for (let i = 0; i <= majorSegments; i++) { - const u = (i / majorSegments) * 2 * Math.PI; - const cu = Math.cos(u); - const su = Math.sin(u); - for (let j = 0; j <= minorSegments; j++) { - const v = (j / minorSegments) * 2 * Math.PI; - const cv = Math.cos(v); - const sv = Math.sin(v); - const x = (majorRadius + minorRadius * cv) * cu; - const y = minorRadius * sv; - const z = (majorRadius + minorRadius * cv) * su; - // Outward-from-minor-circle normal, unit length by construction. - const nx = cv * cu; - const ny = sv; - const nz = cv * su; - verts.push(x, y, z, nx, ny, nz); - } - } - const stride = minorSegments + 1; - for (let i = 0; i < majorSegments; i++) { - for (let j = 0; j < minorSegments; j++) { - const a = i * stride + j; - const b = a + stride; - idx.push(a, b, a + 1, b, b + 1, a + 1); - } - } - return { - vertices: new Float32Array(verts), - indices: new Uint16Array(idx), - }; -} diff --git a/apps/example/src/ChromeSphere/shader.ts b/apps/example/src/ChromeSphere/shader.ts index 6f945f803..425606aaf 100644 --- a/apps/example/src/ChromeSphere/shader.ts +++ b/apps/example/src/ChromeSphere/shader.ts @@ -1,3 +1,46 @@ +// Backdrop shader. Fullscreen triangle that samples the pre-blurred camera +// image (cover-fit baked in by the prepass), dims it ~50%, and adds a soft +// vignette so the chrome sphere stays the focal point of the scene. +export const BACKDROP_SHADER = /* wgsl */ ` +@group(0) @binding(0) var src: texture_2d; +@group(0) @binding(1) var samp: sampler; + +struct VsOut { + @builtin(position) position: vec4f, + @location(0) uv: vec2f, +}; + +@vertex +fn vs_main(@builtin(vertex_index) vid: u32) -> VsOut { + var positions = array( + vec2f(-1.0, -3.0), + vec2f(-1.0, 1.0), + vec2f( 3.0, 1.0), + ); + var uvs = array( + vec2f(0.0, 2.0), + vec2f(0.0, 0.0), + vec2f(2.0, 0.0), + ); + var out: VsOut; + out.position = vec4f(positions[vid], 0.0, 1.0); + out.uv = uvs[vid]; + return out; +} + +@fragment +fn fs_main(in: VsOut) -> @location(0) vec4f { + var c = textureSampleLevel(src, samp, in.uv, 0.0).rgb; + // Subdue the room: half-brightness + soft vignette darkening the edges by + // another ~40%. Keeps a sense of ambient lighting without competing with + // the chrome reflection. + c = c * 0.55; + let d = distance(in.uv, vec2f(0.5)); + let v = 1.0 - smoothstep(0.4, 0.95, d) * 0.45; + return vec4f(c * v, 1.0); +} +`; + // Chrome-style env-map reflection driven by the live camera feed. // // For each fragment we compute the reflection vector around the world-space From 6e36091f5a2567c87df4a2d650453c3922751c15 Mon Sep 17 00:00:00 2001 From: William Candillon Date: Fri, 22 May 2026 16:25:56 +0200 Subject: [PATCH 17/46] :wrench: --- apps/example/src/ThreeJS/CameraHelmet.tsx | 456 ++++++++++++++++++ apps/example/src/ThreeJS/List.tsx | 4 + apps/example/src/ThreeJS/Routes.ts | 1 + apps/example/src/ThreeJS/cameraEnvShader.ts | 41 ++ .../ThreeJS/components/makeWebGPURenderer.ts | 10 +- apps/example/src/ThreeJS/index.tsx | 8 + 6 files changed, 519 insertions(+), 1 deletion(-) create mode 100644 apps/example/src/ThreeJS/CameraHelmet.tsx create mode 100644 apps/example/src/ThreeJS/cameraEnvShader.ts diff --git a/apps/example/src/ThreeJS/CameraHelmet.tsx b/apps/example/src/ThreeJS/CameraHelmet.tsx new file mode 100644 index 000000000..fdb538358 --- /dev/null +++ b/apps/example/src/ThreeJS/CameraHelmet.tsx @@ -0,0 +1,456 @@ +import React, { useEffect, useRef, useState } from "react"; +import { + Linking, + Platform, + StyleSheet, + Text, + TouchableOpacity, + View, +} from "react-native"; +import type { CanvasRef } from "react-native-wgpu"; +import { Canvas } from "react-native-wgpu"; +import { + useCamera, + useCameraDevices, + useCameraPermission, + useFrameOutput, +} from "react-native-vision-camera"; +import * as THREE from "three"; + +import { useGLTF } from "./assets/AssetManager"; +import { makeWebGPURenderer } from "./components/makeWebGPURenderer"; +import { CAMERA_ENV_SHADER } from "./cameraEnvShader"; + +// Live camera as a three.js environment map. The GLTF helmet renders with +// three.js' WebGPURenderer; its env map is a THREE.ExternalTexture wrapping a +// GPUTexture we own. A Vision Camera frame-processor worklet writes each +// camera frame into that GPUTexture via its own render pass. Three.js and the +// worklet share a single GPUDevice (the one three.js creates internally), so +// the queue ordering between "write env" and "sample env" is automatic. + +// Equirectangular panorama aspect (2:1). Sized larger than the 1280x720 +// source so the backdrop reads sharp on retina screens; the equirect→cube +// pass downsamples cleanly. Cube face dimension matches ENV_HEIGHT so the +// cubemap doesn't waste detail. +const ENV_WIDTH = 2048; +const ENV_HEIGHT = 1024; + +// Vision Camera + react-native-wgpu both want these features for the external +// texture path. dawn-multi-planar-formats lets Dawn interpret NV12 buffers. +const REQUIRED_FEATURES: GPUFeatureName[] = [ + "rnwebgpu/shared-texture-memory" as GPUFeatureName, + "dawn-multi-planar-formats" as GPUFeatureName, +]; + +const OPAQUE_YCBCR_EXT = + "opaque-ycbcr-android-for-external-texture" as GPUFeatureName; + +export const CameraHelmet = () => { + const { hasPermission, requestPermission } = useCameraPermission(); + useEffect(() => { + if (!hasPermission) { + requestPermission(); + } + }, [hasPermission, requestPermission]); + + if (!hasPermission) { + return ( + + + Camera access is required. Grant it in Settings or tap below. + + Linking.openSettings()} + style={styles.permissionButton} + > + Open Settings + + + ); + } + return ; +}; + +const Scene = () => { + useEffect(() => { + console.log("[CameraHelmet] Scene mounted"); + return () => console.log("[CameraHelmet] Scene unmounted"); + }, []); + const ref = useRef(null); + const gltf = useGLTF(require("./assets/helmet/DamagedHelmet.gltf")); + + const devices = useCameraDevices(); + const cameraDevice = React.useMemo( + () => + devices.find((d) => d.position === "back") ?? + devices.find((d) => d.position === "front") ?? + devices[0], + [devices], + ); + + const [pipelineState, setPipelineState] = useState<{ + device: GPUDevice; + cameraPipeline: GPURenderPipeline; + cameraSampler: GPUSampler; + envTexture: GPUTexture; + envTextureView: GPUTextureView; + } | null>(null); + const [error, setError] = useState(null); + const [device, setDevice] = useState(null); + + // Acquire the GPU device on its own effect. By the time the async adapter + + // device requests resolve, the Canvas component has been rendered and its + // ref populated, so the main setup effect (gated on `device`) can grab the + // GPUCanvasContext synchronously. Same two-effect pattern as + // VisionCamera.tsx / ChromeSphere.tsx. + useEffect(() => { + console.log("[CameraHelmet] device-acquisition effect fired"); + let cancelled = false; + (async () => { + try { + const adapter = await navigator.gpu.requestAdapter(); + if (!adapter) { + throw new Error("requestAdapter returned null"); + } + const requiredFeatures = [...adapter.features] as GPUFeatureName[]; + const missing = REQUIRED_FEATURES.filter( + (f) => !adapter.features.has(f), + ); + const needsAndroidExt = + Platform.OS === "android" && !adapter.features.has(OPAQUE_YCBCR_EXT); + if (missing.length > 0 || needsAndroidExt) { + throw new Error( + "Adapter doesn't advertise the features the Vision Camera " + + "external-texture path needs: " + + `${[...missing, needsAndroidExt ? OPAQUE_YCBCR_EXT : null] + .filter(Boolean) + .join(", ")}.`, + ); + } + const d = await adapter.requestDevice({ requiredFeatures }); + console.log( + "[CameraHelmet] device acquired, features: " + + [...d.features].sort().join(", "), + ); + if (cancelled) { + d.destroy(); + return; + } + setDevice(d); + } catch (e) { + if (cancelled) { + return; + } + console.warn("[CameraHelmet] device acquisition failed: " + String(e)); + setError(String(e)); + } + })(); + return () => { + cancelled = true; + }; + }, []); + + // Note: pipelineState is intentionally not in the deps array. Including it + // would re-run the effect when we call setPipelineState below — React would + // run the cleanup (which calls setAnimationLoop(null)) and then the effect + // would bail on the pipelineState guard, leaving us with no render loop. + // The effect only needs to fire once, when `device` transitions to set. + + useEffect(() => { + console.log( + "[CameraHelmet] setup effect fired, device=" + + String(device != null) + + " gltf=" + + String(gltf != null), + ); + if (!device || !gltf) { + return; + } + const context = ref.current?.getContext("webgpu"); + if (!context) { + console.log( + "[CameraHelmet] no webgpu context yet (ref.current=" + + String(ref.current != null) + + ") — bailing this effect run", + ); + return; + } + let cancelled = false; + let renderer: THREE.WebGPURenderer | null = null; + + console.log("[CameraHelmet] context acquired, building three.js scene"); + (async () => { + try { + const { width, height } = context.canvas; + console.log( + "[CameraHelmet] canvas size = " + + String(width) + + "x" + + String(height), + ); + + renderer = makeWebGPURenderer(context, { device }); + renderer.toneMapping = THREE.ACESFilmicToneMapping; + await renderer.init(); + console.log("[CameraHelmet] three.js renderer init complete"); + if (cancelled) { + return; + } + + // Env GPUTexture: render target on our side, sampleable on three's + // side. rgba8unorm + RENDER_ATTACHMENT|TEXTURE_BINDING lets the + // single resource pivot between the two roles via implicit barriers. + const envTexture = device.createTexture({ + size: [ENV_WIDTH, ENV_HEIGHT], + format: "rgba8unorm", + usage: + GPUTextureUsage.RENDER_ATTACHMENT | GPUTextureUsage.TEXTURE_BINDING, + }); + + // Camera prepass pipeline. Output format matches the env texture so + // it can be the render target. + const module = device.createShaderModule({ code: CAMERA_ENV_SHADER }); + const cameraPipeline = device.createRenderPipeline({ + layout: "auto", + vertex: { module, entryPoint: "vs_main" }, + fragment: { + module, + entryPoint: "fs_main", + targets: [{ format: "rgba8unorm" }], + }, + primitive: { topology: "triangle-list" }, + }); + const cameraSampler = device.createSampler({ + magFilter: "linear", + minFilter: "linear", + }); + + // THREE.ExternalTexture bridges our GPUTexture into three.js as a + // sampleable 2D equirect. We never set scene.background or + // material.envMap to this directly: three.js' CubeMapNode would only + // run the equirect→cubemap conversion once and cache it, which means + // we'd be stuck sampling whatever was in our env texture on frame 1 + // (= black, before the worklet ever wrote). + const envExternalTexture = new THREE.ExternalTexture(envTexture); + envExternalTexture.mapping = THREE.EquirectangularReflectionMapping; + envExternalTexture.colorSpace = THREE.SRGBColorSpace; + (envExternalTexture as unknown as { image: unknown }).image = { + width: ENV_WIDTH, + height: ENV_HEIGHT, + }; + envExternalTexture.needsUpdate = true; + + // Allocate the cubemap once. Each frame we'll call + // cubeRT.fromEquirectangularTexture(renderer, envExternalTexture) to + // refresh the cube faces from the equirect's *current* contents. + // That's the same code path CubeMapNode uses internally, but we + // drive it on every tick instead of letting three.js cache it. + const cubeRT = new THREE.CubeRenderTarget(ENV_HEIGHT); + cubeRT.texture.mapping = THREE.CubeReflectionMapping; + cubeRT.texture.colorSpace = THREE.SRGBColorSpace; + + const scene = new THREE.Scene(); + scene.background = cubeRT.texture; + // Swap the GLTF helmet's PBR materials for MeshBasicMaterial backed + // by our cubemap. Same env path that already works for the sphere — + // no PMREM, no per-frame regeneration headaches — and the helmet's + // geometry becomes a chrome shell reflecting the camera. Original + // GLTF textures (albedo, normal, etc.) are dropped on purpose; + // they'd compete with the reflection for the metal look. + const chromeMaterial = new THREE.MeshBasicMaterial({ + envMap: cubeRT.texture, + }); + gltf.scene.traverse((child) => { + if ((child as THREE.Mesh).isMesh) { + (child as THREE.Mesh).material = chromeMaterial; + } + }); + scene.add(gltf.scene); + + // Drive the perspective from min(width, height) so the helmet keeps + // a consistent on-screen size in both orientations. three.js' + // PerspectiveCamera takes a *vertical* FOV; on portrait canvases we + // derive vFov from a fixed horizontal FOV so the wider dimension + // never under-frames the helmet. + const aspect = width / height; + const baseFov = 45; + let vFov = baseFov; + if (aspect < 1) { + const hFovRad = (baseFov * Math.PI) / 180; + const vFovRad = 2 * Math.atan(Math.tan(hFovRad / 2) / aspect); + vFov = (vFovRad * 180) / Math.PI; + } + const camera = new THREE.PerspectiveCamera(vFov, aspect, 0.25, 20); + camera.position.set(0, 0.5, 3); + + const clock = new THREE.Clock(); + let frameCount = 0; + const animate = () => { + const elapsed = clock.getElapsedTime(); + const distance = 3; + camera.position.x = Math.sin(elapsed * 0.4) * distance; + camera.position.z = Math.cos(elapsed * 0.4) * distance; + camera.position.y = 0.5; + camera.lookAt(0, 0, 0); + // Refresh the cubemap from the (worklet-updated) equirect before + // rendering the scene. The conversion does 6 fullscreen draws into + // the cube faces; pipelines are reused across calls. + cubeRT.fromEquirectangularTexture(renderer!, envExternalTexture); + renderer!.render(scene, camera); + context.present(); + frameCount++; + if (frameCount === 1) { + console.log("[CameraHelmet] first three.js frame rendered"); + } + }; + renderer.setAnimationLoop(animate); + console.log("[CameraHelmet] animation loop started"); + + setPipelineState({ + device, + cameraPipeline, + cameraSampler, + envTexture, + envTextureView: envTexture.createView(), + }); + console.log("[CameraHelmet] pipelineState set, camera will activate"); + } catch (e) { + if (cancelled) { + return; + } + console.warn("[CameraHelmet] setup failed: " + String(e)); + setError(String(e)); + } + })(); + + return () => { + console.log("[CameraHelmet] setup-effect cleanup"); + cancelled = true; + if (renderer) { + renderer.setAnimationLoop(null); + } + }; + }, [device, gltf]); + + // Frame processor worklet: copy the camera frame into envTexture each tick. + // The single device.queue is shared with three.js, so the helmet pass on + // the next rAF tick samples this frame's write. + const logBox = React.useMemo(() => ({ count: 0 }), []); + const frameOutput = useFrameOutput({ + pixelFormat: "native", + onFrame: (frame) => { + "worklet"; + logBox.count += 1; + if (logBox.count === 1) { + console.log( + "[CameraHelmet] worklet first frame, hasPipeline=" + + String(pipelineState != null) + + " frame=" + + String(frame.width) + + "x" + + String(frame.height), + ); + } + if (!pipelineState) { + frame.dispose(); + return; + } + const { + device: gpuDevice, + cameraPipeline, + cameraSampler, + envTextureView, + } = pipelineState; + const nativeBuffer = frame.getNativeBuffer(); + try { + const videoFrame = gpuDevice.createVideoFrameFromNativeBuffer( + nativeBuffer.pointer, + ); + try { + const externalTex = gpuDevice.importExternalTexture({ + source: videoFrame, + label: "camera-helmet-env", + }); + const bindGroup = gpuDevice.createBindGroup({ + layout: cameraPipeline.getBindGroupLayout(0), + entries: [ + { binding: 0, resource: externalTex }, + { binding: 1, resource: cameraSampler }, + ], + }); + const encoder = gpuDevice.createCommandEncoder(); + const pass = encoder.beginRenderPass({ + colorAttachments: [ + { + view: envTextureView, + clearValue: { r: 0, g: 0, b: 0, a: 1 }, + loadOp: "clear", + storeOp: "store", + }, + ], + }); + pass.setPipeline(cameraPipeline); + pass.setBindGroup(0, bindGroup); + pass.draw(3); + pass.end(); + gpuDevice.queue.submit([encoder.finish()]); + } finally { + videoFrame.release(); + } + } finally { + nativeBuffer.release(); + frame.dispose(); + } + }, + }); + + useCamera({ + isActive: pipelineState != null && cameraDevice != null, + device: cameraDevice as NonNullable, + outputs: [frameOutput], + }); + + if (error) { + return ( + + {error} + + ); + } + if (cameraDevice == null) { + return ( + + + No camera available. This screen needs a physical device with a camera + (the iOS Simulator doesn't have one). + + + ); + } + return ( + + + + ); +}; + +const styles = StyleSheet.create({ + root: { flex: 1, backgroundColor: "black" }, + canvas: { flex: 1 }, + errorContainer: { flex: 1, padding: 16, justifyContent: "center" }, + errorText: { color: "red", fontSize: 14 }, + permissionContainer: { + flex: 1, + padding: 24, + justifyContent: "center", + alignItems: "center", + }, + permissionText: { fontSize: 16, textAlign: "center", marginBottom: 16 }, + permissionButton: { + backgroundColor: "#007AFF", + paddingHorizontal: 24, + paddingVertical: 12, + borderRadius: 8, + }, + permissionButtonText: { color: "white", fontSize: 16, fontWeight: "600" }, +}); diff --git a/apps/example/src/ThreeJS/List.tsx b/apps/example/src/ThreeJS/List.tsx index 9601abb71..972522e87 100644 --- a/apps/example/src/ThreeJS/List.tsx +++ b/apps/example/src/ThreeJS/List.tsx @@ -23,6 +23,10 @@ export const examples = [ screen: "Helmet", title: "⛑️ Helmet", }, + { + screen: "CameraHelmet", + title: "đź“· Camera Env Sphere", + }, { screen: "PostProcessing", title: "🪄 Post Processing Effects", diff --git a/apps/example/src/ThreeJS/Routes.ts b/apps/example/src/ThreeJS/Routes.ts index cb76ac65f..aa78c2240 100644 --- a/apps/example/src/ThreeJS/Routes.ts +++ b/apps/example/src/ThreeJS/Routes.ts @@ -2,6 +2,7 @@ export type Routes = { List: undefined; Cube: undefined; Helmet: undefined; + CameraHelmet: undefined; Backdrop: undefined; InstancedMesh: undefined; Fiber: undefined; diff --git a/apps/example/src/ThreeJS/cameraEnvShader.ts b/apps/example/src/ThreeJS/cameraEnvShader.ts new file mode 100644 index 000000000..8c9892e7d --- /dev/null +++ b/apps/example/src/ThreeJS/cameraEnvShader.ts @@ -0,0 +1,41 @@ +// Tiny "copy camera frame into an rgba8unorm texture" shader. The output +// texture is then wrapped in a THREE.ExternalTexture and used as an +// equirectangular environment map by three.js' WebGPURenderer. +// +// The camera image is stretched to fill the 2:1 env texture, which when +// sampled equirectangularly produces a panorama wrap of the camera view +// around the helmet — your face becomes the world. + +export const CAMERA_ENV_SHADER = /* wgsl */ ` +struct VsOut { + @builtin(position) position: vec4f, + @location(0) uv: vec2f, +}; + +@group(0) @binding(0) var srcTex: texture_external; +@group(0) @binding(1) var srcSampler: sampler; + +@vertex +fn vs_main(@builtin(vertex_index) vid: u32) -> VsOut { + var positions = array( + vec2f(-1.0, -3.0), + vec2f(-1.0, 1.0), + vec2f( 3.0, 1.0), + ); + var uvs = array( + vec2f(0.0, 2.0), + vec2f(0.0, 0.0), + vec2f(2.0, 0.0), + ); + var out: VsOut; + out.position = vec4f(positions[vid], 0.0, 1.0); + out.uv = uvs[vid]; + return out; +} + +@fragment +fn fs_main(in: VsOut) -> @location(0) vec4f { + let c = textureSampleBaseClampToEdge(srcTex, srcSampler, in.uv); + return vec4f(c.rgb, 1.0); +} +`; diff --git a/apps/example/src/ThreeJS/components/makeWebGPURenderer.ts b/apps/example/src/ThreeJS/components/makeWebGPURenderer.ts index 39c4bf399..a17a728b5 100644 --- a/apps/example/src/ThreeJS/components/makeWebGPURenderer.ts +++ b/apps/example/src/ThreeJS/components/makeWebGPURenderer.ts @@ -60,7 +60,10 @@ export class ReactNativeCanvas { export const makeWebGPURenderer = ( context: GPUCanvasContext, - { antialias = true }: { antialias?: boolean } = {}, + { + antialias = true, + device, + }: { antialias?: boolean; device?: GPUDevice } = {}, ) => new THREE.WebGPURenderer({ antialias, @@ -68,4 +71,9 @@ export const makeWebGPURenderer = ( // @ts-expect-error canvas: new ReactNativeCanvas(context.canvas), context, + // When supplied, three.js skips its own adapter/device acquisition and + // uses this device. Lets callers request custom features (e.g. Dawn's + // shared-texture-memory) that three.js doesn't include in its default + // GPUFeatureName enum walk. + ...(device ? { device } : {}), }); diff --git a/apps/example/src/ThreeJS/index.tsx b/apps/example/src/ThreeJS/index.tsx index 244c43d71..1a7535404 100644 --- a/apps/example/src/ThreeJS/index.tsx +++ b/apps/example/src/ThreeJS/index.tsx @@ -6,6 +6,7 @@ import { Cube } from "./Cube"; import type { Routes } from "./Routes"; import { List } from "./List"; import { Helmet } from "./Helmet"; +import { CameraHelmet } from "./CameraHelmet"; import { Backdrop } from "./Backdrop"; import { InstancedMesh } from "./InstancedMesh"; import { Fiber } from "./Fiber"; @@ -73,6 +74,13 @@ export const ThreeJS = () => { title: "⛑️ Helmet", }} /> + Date: Fri, 22 May 2026 17:02:13 +0200 Subject: [PATCH 18/46] :wrech: --- apps/example/src/ThreeJS/CameraHelmet.tsx | 41 +++++++++++++++---- apps/example/src/ThreeJS/cameraEnvShader.ts | 6 ++- .../ThreeJS/components/makeWebGPURenderer.ts | 4 +- 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/apps/example/src/ThreeJS/CameraHelmet.tsx b/apps/example/src/ThreeJS/CameraHelmet.tsx index fdb538358..e39805c23 100644 --- a/apps/example/src/ThreeJS/CameraHelmet.tsx +++ b/apps/example/src/ThreeJS/CameraHelmet.tsx @@ -10,10 +10,13 @@ import { import type { CanvasRef } from "react-native-wgpu"; import { Canvas } from "react-native-wgpu"; import { + CommonResolutions, + NativePreviewView, useCamera, useCameraDevices, useCameraPermission, useFrameOutput, + usePreviewOutput, } from "react-native-vision-camera"; import * as THREE from "three"; @@ -78,6 +81,11 @@ const Scene = () => { }, []); const ref = useRef(null); const gltf = useGLTF(require("./assets/helmet/DamagedHelmet.gltf")); + // Live camera preview, rendered as a native view behind the WebGPU canvas. + // The worklet still writes the camera into our env texture for the helmet + // reflection, but the *backdrop* now comes straight from this native + // preview — no detour through equirect/cubemap, no quality loss. + const previewOutput = usePreviewOutput(); const devices = useCameraDevices(); const cameraDevice = React.useMemo( @@ -189,8 +197,12 @@ const Scene = () => { String(height), ); - renderer = makeWebGPURenderer(context, { device }); + // alpha:true configures the canvas with premultiplied alpha mode, so + // pixels outside the helmet stay transparent and the native camera + // preview behind the canvas shows through. + renderer = makeWebGPURenderer(context, { device, alpha: true }); renderer.toneMapping = THREE.ACESFilmicToneMapping; + renderer.setClearColor(0x000000, 0); await renderer.init(); console.log("[CameraHelmet] three.js renderer init complete"); if (cancelled) { @@ -250,7 +262,8 @@ const Scene = () => { cubeRT.texture.colorSpace = THREE.SRGBColorSpace; const scene = new THREE.Scene(); - scene.background = cubeRT.texture; + // No scene.background — the canvas is alpha-cleared and the native + // camera preview View sits behind it (see JSX below). // Swap the GLTF helmet's PBR materials for MeshBasicMaterial backed // by our cubemap. Same env path that already works for the sphere — // no PMREM, no per-frame regeneration headaches — and the helmet's @@ -281,17 +294,20 @@ const Scene = () => { vFov = (vFovRad * 180) / Math.PI; } const camera = new THREE.PerspectiveCamera(vFov, aspect, 0.25, 20); - camera.position.set(0, 0.5, 3); + camera.position.set(0, 0, 3); const clock = new THREE.Clock(); + const distance = 3; let frameCount = 0; const animate = () => { + // Slow time-based orbit around the helmet, matching the three.js + // env-map reference demo. const elapsed = clock.getElapsedTime(); - const distance = 3; camera.position.x = Math.sin(elapsed * 0.4) * distance; camera.position.z = Math.cos(elapsed * 0.4) * distance; - camera.position.y = 0.5; + camera.position.y = 0; camera.lookAt(0, 0, 0); + // Refresh the cubemap from the (worklet-updated) equirect before // rendering the scene. The conversion does 6 fullscreen draws into // the cube faces; pipelines are reused across calls. @@ -338,6 +354,10 @@ const Scene = () => { const logBox = React.useMemo(() => ({ count: 0 }), []); const frameOutput = useFrameOutput({ pixelFormat: "native", + // Request 4K (UHD). Source resolution is the hard ceiling on backdrop + // sharpness; UHD ~9x the pixel count of the default 720p. The worklet's + // copy pass is cheap so this is mostly a memory-bandwidth bump. + targetResolution: CommonResolutions.UHD_16_9, onFrame: (frame) => { "worklet"; logBox.count += 1; @@ -407,7 +427,7 @@ const Scene = () => { useCamera({ isActive: pipelineState != null && cameraDevice != null, device: cameraDevice as NonNullable, - outputs: [frameOutput], + outputs: [frameOutput, previewOutput], }); if (error) { @@ -429,14 +449,19 @@ const Scene = () => { } return ( - + + ); }; const styles = StyleSheet.create({ root: { flex: 1, backgroundColor: "black" }, - canvas: { flex: 1 }, + // Transparent canvas overlaid on the native camera preview view. + canvas: { ...StyleSheet.absoluteFillObject, backgroundColor: "transparent" }, errorContainer: { flex: 1, padding: 16, justifyContent: "center" }, errorText: { color: "red", fontSize: 14 }, permissionContainer: { diff --git a/apps/example/src/ThreeJS/cameraEnvShader.ts b/apps/example/src/ThreeJS/cameraEnvShader.ts index 8c9892e7d..3a196b3c7 100644 --- a/apps/example/src/ThreeJS/cameraEnvShader.ts +++ b/apps/example/src/ThreeJS/cameraEnvShader.ts @@ -35,7 +35,11 @@ fn vs_main(@builtin(vertex_index) vid: u32) -> VsOut { @fragment fn fs_main(in: VsOut) -> @location(0) vec4f { - let c = textureSampleBaseClampToEdge(srcTex, srcSampler, in.uv); + // Rotate the camera 90° CW so VisionCamera's landscape sensor frame is + // upright when the device is held in portrait. (Vertical/horizontal flips + // of this are easy to swap in if a particular device needs them.) + let rotatedUv = vec2f(in.uv.y, 1.0 - in.uv.x); + let c = textureSampleBaseClampToEdge(srcTex, srcSampler, rotatedUv); return vec4f(c.rgb, 1.0); } `; diff --git a/apps/example/src/ThreeJS/components/makeWebGPURenderer.ts b/apps/example/src/ThreeJS/components/makeWebGPURenderer.ts index a17a728b5..f6729b3c9 100644 --- a/apps/example/src/ThreeJS/components/makeWebGPURenderer.ts +++ b/apps/example/src/ThreeJS/components/makeWebGPURenderer.ts @@ -63,10 +63,12 @@ export const makeWebGPURenderer = ( { antialias = true, device, - }: { antialias?: boolean; device?: GPUDevice } = {}, + alpha = false, + }: { antialias?: boolean; device?: GPUDevice; alpha?: boolean } = {}, ) => new THREE.WebGPURenderer({ antialias, + alpha, // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error canvas: new ReactNativeCanvas(context.canvas), From f9073a2d7dc2956312d6b7195480e2d208d067ec Mon Sep 17 00:00:00 2001 From: William Candillon Date: Fri, 22 May 2026 17:15:19 +0200 Subject: [PATCH 19/46] :wrench: --- apps/example/src/ThreeJS/CameraHelmet.tsx | 127 +++++++++++++++++--- apps/example/src/ThreeJS/cameraEnvShader.ts | 11 +- 2 files changed, 116 insertions(+), 22 deletions(-) diff --git a/apps/example/src/ThreeJS/CameraHelmet.tsx b/apps/example/src/ThreeJS/CameraHelmet.tsx index e39805c23..83f894603 100644 --- a/apps/example/src/ThreeJS/CameraHelmet.tsx +++ b/apps/example/src/ThreeJS/CameraHelmet.tsx @@ -12,11 +12,15 @@ import { Canvas } from "react-native-wgpu"; import { CommonResolutions, NativePreviewView, - useCamera, useCameraDevices, useCameraPermission, useFrameOutput, usePreviewOutput, + VisionCamera as VisionCameraFactory, +} from "react-native-vision-camera"; +import type { + CameraController, + CameraSession, } from "react-native-vision-camera"; import * as THREE from "three"; @@ -87,12 +91,17 @@ const Scene = () => { // preview — no detour through equirect/cubemap, no quality loss. const previewOutput = usePreviewOutput(); + // Two cameras at once: the back camera feeds the native preview backdrop, + // the front camera feeds the helmet's environment map (so you see yourself + // reflected in the chrome). Requires multi-cam capable hardware (iPhone + // XS+ / most modern Android flagships). const devices = useCameraDevices(); - const cameraDevice = React.useMemo( - () => - devices.find((d) => d.position === "back") ?? - devices.find((d) => d.position === "front") ?? - devices[0], + const backDevice = React.useMemo( + () => devices.find((d) => d.position === "back"), + [devices], + ); + const frontDevice = React.useMemo( + () => devices.find((d) => d.position === "front"), [devices], ); @@ -354,10 +363,10 @@ const Scene = () => { const logBox = React.useMemo(() => ({ count: 0 }), []); const frameOutput = useFrameOutput({ pixelFormat: "native", - // Request 4K (UHD). Source resolution is the hard ceiling on backdrop - // sharpness; UHD ~9x the pixel count of the default 720p. The worklet's - // copy pass is cheap so this is mostly a memory-bandwidth bump. - targetResolution: CommonResolutions.UHD_16_9, + // 720p front-cam frames are plenty for the helmet's reflection — it's a + // small on-screen area. Keeping this low matters more in a multi-cam + // session, where both cameras share AVFoundation's bandwidth budget. + targetResolution: CommonResolutions.HD_16_9, onFrame: (frame) => { "worklet"; logBox.count += 1; @@ -424,11 +433,93 @@ const Scene = () => { }, }); - useCamera({ - isActive: pipelineState != null && cameraDevice != null, - device: cameraDevice as NonNullable, - outputs: [frameOutput, previewOutput], - }); + // ---- Multi-cam session ------------------------------------------------ + // useCamera always sets enableMultiCamSupport=false, so we drop down to + // the imperative API to drive two camera connections from a single + // session: front → frameOutput (helmet env), back → previewOutput + // (backdrop View). + const [session, setSession] = useState(null); + useEffect(() => { + if (!VisionCameraFactory.supportsMultiCamSessions) { + setError( + "This device doesn't support multi-cam sessions. Need an iPhone XS " + + "or newer / a comparable Android flagship.", + ); + return; + } + let cancelled = false; + let created: CameraSession | null = null; + (async () => { + const s = await VisionCameraFactory.createCameraSession(true); + if (cancelled) { + s.dispose(); + return; + } + created = s; + setSession(s); + })(); + return () => { + cancelled = true; + created?.stop(); + created?.dispose(); + }; + }, []); + + // Configure the session with two connections once everything is ready. + // We wait on pipelineState too because the worklet (which receives the + // front cam frames) only has somewhere to write once the env texture + + // camera-copy pipeline exist. + useEffect(() => { + if (!session || !backDevice || !frontDevice || !pipelineState) { + return; + } + console.log("[CameraHelmet] configuring multi-cam session"); + let cancelled = false; + let controllers: CameraController[] = []; + (async () => { + try { + controllers = await session.configure( + [ + { + input: backDevice, + outputs: [{ output: previewOutput, mirrorMode: "auto" }], + constraints: [], + }, + { + input: frontDevice, + outputs: [{ output: frameOutput, mirrorMode: "auto" }], + constraints: [], + }, + ], + {}, + ); + if (cancelled) { + controllers.forEach((c) => c.dispose()); + return; + } + console.log("[CameraHelmet] session configured, starting"); + session.start(); + } catch (e) { + if (cancelled) { + return; + } + console.warn("[CameraHelmet] session configure failed: " + String(e)); + setError(String(e)); + } + })(); + return () => { + cancelled = true; + session.stop(); + controllers.forEach((c) => c.dispose()); + }; + }, [ + session, + backDevice, + frontDevice, + previewOutput, + frameOutput, + pipelineState, + ]); if (error) { return ( @@ -437,12 +528,12 @@ const Scene = () => { ); } - if (cameraDevice == null) { + if (backDevice == null || frontDevice == null) { return ( - No camera available. This screen needs a physical device with a camera - (the iOS Simulator doesn't have one). + Need both a back and a front camera. The iOS Simulator has none, and + some devices expose only one. ); diff --git a/apps/example/src/ThreeJS/cameraEnvShader.ts b/apps/example/src/ThreeJS/cameraEnvShader.ts index 3a196b3c7..df38ecb96 100644 --- a/apps/example/src/ThreeJS/cameraEnvShader.ts +++ b/apps/example/src/ThreeJS/cameraEnvShader.ts @@ -35,10 +35,13 @@ fn vs_main(@builtin(vertex_index) vid: u32) -> VsOut { @fragment fn fs_main(in: VsOut) -> @location(0) vec4f { - // Rotate the camera 90° CW so VisionCamera's landscape sensor frame is - // upright when the device is held in portrait. (Vertical/horizontal flips - // of this are easy to swap in if a particular device needs them.) - let rotatedUv = vec2f(in.uv.y, 1.0 - in.uv.x); + // Front camera: iOS delivers landscape-orientation frames with the + // horizontal axis already mirrored (selfie convention). To bring those + // upright in the equirect we (a) compensate for the horizontal mirror + // by sampling at (1-x) and (b) rotate 90° CCW with V flipped, giving + // (1-v, 1-u). Equivalent to the 90° CW back-cam mapping (v, 1-u) with + // its U axis pre-flipped to undo the mirror. + let rotatedUv = vec2f(1.0 - in.uv.y, 1.0 - in.uv.x); let c = textureSampleBaseClampToEdge(srcTex, srcSampler, rotatedUv); return vec4f(c.rgb, 1.0); } From 2de307b7a36ac12ab8648ad0be4d86168bc94e0c Mon Sep 17 00:00:00 2001 From: William Candillon Date: Fri, 22 May 2026 17:32:56 +0200 Subject: [PATCH 20/46] :wrench: --- apps/example/src/ThreeJS/CameraHelmet.tsx | 80 +++- apps/example/src/ThreeJS/CameraSpheres.tsx | 489 +++++++++++++++++++++ apps/example/src/ThreeJS/List.tsx | 4 + apps/example/src/ThreeJS/Routes.ts | 1 + apps/example/src/ThreeJS/index.tsx | 8 + 5 files changed, 562 insertions(+), 20 deletions(-) create mode 100644 apps/example/src/ThreeJS/CameraSpheres.tsx diff --git a/apps/example/src/ThreeJS/CameraHelmet.tsx b/apps/example/src/ThreeJS/CameraHelmet.tsx index 83f894603..58b72c4fe 100644 --- a/apps/example/src/ThreeJS/CameraHelmet.tsx +++ b/apps/example/src/ThreeJS/CameraHelmet.tsx @@ -35,12 +35,21 @@ import { CAMERA_ENV_SHADER } from "./cameraEnvShader"; // worklet share a single GPUDevice (the one three.js creates internally), so // the queue ordering between "write env" and "sample env" is automatic. -// Equirectangular panorama aspect (2:1). Sized larger than the 1280x720 -// source so the backdrop reads sharp on retina screens; the equirect→cube -// pass downsamples cleanly. Cube face dimension matches ENV_HEIGHT so the -// cubemap doesn't waste detail. -const ENV_WIDTH = 2048; -const ENV_HEIGHT = 1024; +// Equirectangular panorama aspect (2:1). Front cam delivers 720p so the +// env texture doesn't gain anything from going much larger than that. +// Cube face dimension matches ENV_HEIGHT. Each frame we do (env write + 6 +// cube faces + optional mipmap chain), so this knob drives most of the +// per-frame GPU cost. +const ENV_WIDTH = 1024; +const ENV_HEIGHT = 512; + +// PBR helmet (uses the GLTF's MeshStandardMaterial textures + cubemap +// reflection via per-material envMap, with the cubemap mipmapped each +// frame so rough surfaces sample blurrier mips). Set to `false` to fall +// back to a flat MeshBasicMaterial chrome shell — drops the PBR fragment +// cost and the every-frame mipmap regeneration of the cubemap, which is +// the bulk of the perf cost on phone GPUs. +const USE_PBR = false; // Vision Camera + react-native-wgpu both want these features for the external // texture path. dawn-multi-planar-formats lets Dawn interpret NV12 buffers. @@ -269,24 +278,55 @@ const Scene = () => { const cubeRT = new THREE.CubeRenderTarget(ENV_HEIGHT); cubeRT.texture.mapping = THREE.CubeReflectionMapping; cubeRT.texture.colorSpace = THREE.SRGBColorSpace; + if (USE_PBR) { + // PBR mode: generate mipmaps so MeshStandardMaterial's + // roughness-aware envMap sample picks softer mips for rough + // surfaces. Not true PMREM (no GGX importance sampling) but a + // cheap approximation; adds an auto-mipmap regen per frame. + cubeRT.texture.generateMipmaps = true; + cubeRT.texture.minFilter = THREE.LinearMipmapLinearFilter; + cubeRT.texture.magFilter = THREE.LinearFilter; + } const scene = new THREE.Scene(); // No scene.background — the canvas is alpha-cleared and the native // camera preview View sits behind it (see JSX below). - // Swap the GLTF helmet's PBR materials for MeshBasicMaterial backed - // by our cubemap. Same env path that already works for the sphere — - // no PMREM, no per-frame regeneration headaches — and the helmet's - // geometry becomes a chrome shell reflecting the camera. Original - // GLTF textures (albedo, normal, etc.) are dropped on purpose; - // they'd compete with the reflection for the metal look. - const chromeMaterial = new THREE.MeshBasicMaterial({ - envMap: cubeRT.texture, - }); - gltf.scene.traverse((child) => { - if ((child as THREE.Mesh).isMesh) { - (child as THREE.Mesh).material = chromeMaterial; - } - }); + if (USE_PBR) { + // Keep the GLTF's MeshStandardMaterial intact (albedo / normal / + // metalRoughness / AO from the original textures) and just plug + // our cubemap into each material's envMap. The PBR shader then + // combines the helmet's surface detail with the live front-camera + // reflection. + gltf.scene.traverse((child) => { + const mesh = child as THREE.Mesh; + if (!mesh.isMesh) { + return; + } + const mats = Array.isArray(mesh.material) + ? mesh.material + : [mesh.material]; + for (const m of mats) { + const std = m as THREE.MeshStandardMaterial; + std.envMap = cubeRT.texture; + std.envMapIntensity = 1.0; + std.needsUpdate = true; + } + }); + } else { + // Non-PBR mode: swap every mesh's material for a single + // MeshBasicMaterial that just samples the cubemap. Loses surface + // detail (no albedo / normal / roughness) but gives a fast + // chrome-shell look that's a good 5–10x cheaper on fragments. + const chrome = new THREE.MeshBasicMaterial({ + envMap: cubeRT.texture, + }); + gltf.scene.traverse((child) => { + const mesh = child as THREE.Mesh; + if (mesh.isMesh) { + mesh.material = chrome; + } + }); + } scene.add(gltf.scene); // Drive the perspective from min(width, height) so the helmet keeps diff --git a/apps/example/src/ThreeJS/CameraSpheres.tsx b/apps/example/src/ThreeJS/CameraSpheres.tsx new file mode 100644 index 000000000..f3d7dedf3 --- /dev/null +++ b/apps/example/src/ThreeJS/CameraSpheres.tsx @@ -0,0 +1,489 @@ +import React, { useEffect, useRef, useState } from "react"; +import { + Linking, + Platform, + StyleSheet, + Text, + TouchableOpacity, + View, +} from "react-native"; +import type { CanvasRef } from "react-native-wgpu"; +import { Canvas } from "react-native-wgpu"; +import { + CommonResolutions, + NativePreviewView, + useCameraDevices, + useCameraPermission, + useFrameOutput, + usePreviewOutput, + VisionCamera as VisionCameraFactory, +} from "react-native-vision-camera"; +import type { + CameraController, + CameraSession, +} from "react-native-vision-camera"; +import * as THREE from "three"; + +import { makeWebGPURenderer } from "./components/makeWebGPURenderer"; +import { CAMERA_ENV_SHADER } from "./cameraEnvShader"; + +// Sibling of CameraHelmet but with three procedural chrome spheres in place +// of the GLTF helmet. Multi-cam setup: back camera renders behind the canvas +// as a native preview view, front camera feeds the cubemap that the spheres +// reflect. No PBR (no GLTF assets), so we skip mipmap generation entirely — +// the chrome look stays sharp on all surfaces. + +const ENV_WIDTH = 1024; +const ENV_HEIGHT = 512; + +const REQUIRED_FEATURES: GPUFeatureName[] = [ + "rnwebgpu/shared-texture-memory" as GPUFeatureName, + "dawn-multi-planar-formats" as GPUFeatureName, +]; + +const OPAQUE_YCBCR_EXT = + "opaque-ycbcr-android-for-external-texture" as GPUFeatureName; + +// Three big chrome spheres swirling around the origin, inspired by three.js' +// stereo-effects demo (which uses InstancedMesh + per-frame matrix updates). +// Even at 3 instances the InstancedMesh path is still nice because all +// three render in a single draw call. +const BEAD_COUNT = 3; +const BEAD_RADIUS = 0.55; +// XY radius of the swirl. Spheres orbit evenly spaced around the origin. +const SWIRL_RADIUS = 1.8; + +export const CameraSpheres = () => { + const { hasPermission, requestPermission } = useCameraPermission(); + useEffect(() => { + if (!hasPermission) { + requestPermission(); + } + }, [hasPermission, requestPermission]); + + if (!hasPermission) { + return ( + + + Camera access is required. Grant it in Settings or tap below. + + Linking.openSettings()} + style={styles.permissionButton} + > + Open Settings + + + ); + } + return ; +}; + +const Scene = () => { + useEffect(() => { + console.log("[CameraSpheres] Scene mounted"); + return () => console.log("[CameraSpheres] Scene unmounted"); + }, []); + const ref = useRef(null); + const previewOutput = usePreviewOutput(); + + const devices = useCameraDevices(); + const backDevice = React.useMemo( + () => devices.find((d) => d.position === "back"), + [devices], + ); + const frontDevice = React.useMemo( + () => devices.find((d) => d.position === "front"), + [devices], + ); + + const [pipelineState, setPipelineState] = useState<{ + device: GPUDevice; + cameraPipeline: GPURenderPipeline; + cameraSampler: GPUSampler; + envTexture: GPUTexture; + envTextureView: GPUTextureView; + } | null>(null); + const [error, setError] = useState(null); + const [device, setDevice] = useState(null); + + useEffect(() => { + let cancelled = false; + (async () => { + try { + const adapter = await navigator.gpu.requestAdapter(); + if (!adapter) { + throw new Error("requestAdapter returned null"); + } + const requiredFeatures = [...adapter.features] as GPUFeatureName[]; + const missing = REQUIRED_FEATURES.filter( + (f) => !adapter.features.has(f), + ); + const needsAndroidExt = + Platform.OS === "android" && !adapter.features.has(OPAQUE_YCBCR_EXT); + if (missing.length > 0 || needsAndroidExt) { + throw new Error( + "Adapter doesn't advertise the features the Vision Camera " + + "external-texture path needs: " + + `${[...missing, needsAndroidExt ? OPAQUE_YCBCR_EXT : null] + .filter(Boolean) + .join(", ")}.`, + ); + } + const d = await adapter.requestDevice({ requiredFeatures }); + if (cancelled) { + d.destroy(); + return; + } + setDevice(d); + } catch (e) { + if (cancelled) { + return; + } + console.warn("[CameraSpheres] device acquisition failed: " + String(e)); + setError(String(e)); + } + })(); + return () => { + cancelled = true; + }; + }, []); + + useEffect(() => { + if (!device) { + return; + } + const context = ref.current?.getContext("webgpu"); + if (!context) { + return; + } + let cancelled = false; + let renderer: THREE.WebGPURenderer | null = null; + + (async () => { + try { + const { width, height } = context.canvas; + + renderer = makeWebGPURenderer(context, { device, alpha: true }); + renderer.toneMapping = THREE.ACESFilmicToneMapping; + renderer.setClearColor(0x000000, 0); + await renderer.init(); + if (cancelled) { + return; + } + + const envTexture = device.createTexture({ + size: [ENV_WIDTH, ENV_HEIGHT], + format: "rgba8unorm", + usage: + GPUTextureUsage.RENDER_ATTACHMENT | GPUTextureUsage.TEXTURE_BINDING, + }); + + const module = device.createShaderModule({ code: CAMERA_ENV_SHADER }); + const cameraPipeline = device.createRenderPipeline({ + layout: "auto", + vertex: { module, entryPoint: "vs_main" }, + fragment: { + module, + entryPoint: "fs_main", + targets: [{ format: "rgba8unorm" }], + }, + primitive: { topology: "triangle-list" }, + }); + const cameraSampler = device.createSampler({ + magFilter: "linear", + minFilter: "linear", + }); + + const envExternalTexture = new THREE.ExternalTexture(envTexture); + envExternalTexture.mapping = THREE.EquirectangularReflectionMapping; + envExternalTexture.colorSpace = THREE.SRGBColorSpace; + (envExternalTexture as unknown as { image: unknown }).image = { + width: ENV_WIDTH, + height: ENV_HEIGHT, + }; + envExternalTexture.needsUpdate = true; + + // Cube target refreshed per frame from the equirect — same dynamic + // env trick as CameraHelmet. No mipmap chain: the spheres use + // MeshBasicMaterial which samples mip 0 unconditionally, so the + // auto-mipmap regeneration would be pure waste. + const cubeRT = new THREE.CubeRenderTarget(ENV_HEIGHT); + cubeRT.texture.mapping = THREE.CubeReflectionMapping; + cubeRT.texture.colorSpace = THREE.SRGBColorSpace; + + const scene = new THREE.Scene(); + // Single InstancedMesh: all three spheres in one draw call. Mesh + // tessellation matters more here than in the swarm version because + // each sphere is bigger on screen, so 48x32 segments instead of + // 16x8 — smoother silhouettes against the panorama backdrop. + const chrome = new THREE.MeshBasicMaterial({ envMap: cubeRT.texture }); + const geometry = new THREE.SphereGeometry(BEAD_RADIUS, 48, 32); + const beads = new THREE.InstancedMesh(geometry, chrome, BEAD_COUNT); + beads.instanceMatrix.setUsage(THREE.DynamicDrawUsage); + scene.add(beads); + + // Seed each instance's matrix to identity. The animate loop + // overwrites the translation column each frame; scale and rotation + // stay as identity (= sphere radius set by BEAD_RADIUS above). + const dummy = new THREE.Object3D(); + for (let i = 0; i < BEAD_COUNT; i++) { + dummy.updateMatrix(); + beads.setMatrixAt(i, dummy.matrix); + } + beads.instanceMatrix.needsUpdate = true; + const beadPos = new THREE.Vector3(); + + const aspect = width / height; + const baseFov = 60; + let vFov = baseFov; + if (aspect < 1) { + const hFovRad = (baseFov * Math.PI) / 180; + const vFovRad = 2 * Math.atan(Math.tan(hFovRad / 2) / aspect); + vFov = (vFovRad * 180) / Math.PI; + } + const camera = new THREE.PerspectiveCamera(vFov, aspect, 0.25, 20); + camera.position.set(0, 0, 4); + + const clock = new THREE.Clock(); + const animate = () => { + // Slow rotation of the three spheres around the origin in the XY + // plane. Phase offsets are evenly spaced (2Ď€/3 apart) so the + // spheres form a rotating equilateral triangle, never overlapping. + const elapsed = clock.getElapsedTime() * 0.4; + for (let i = 0; i < BEAD_COUNT; i++) { + const angle = elapsed + (i * 2 * Math.PI) / BEAD_COUNT; + beadPos.set( + SWIRL_RADIUS * Math.cos(angle), + SWIRL_RADIUS * Math.sin(angle), + 0, + ); + beads.getMatrixAt(i, dummy.matrix); + dummy.matrix.setPosition(beadPos); + beads.setMatrixAt(i, dummy.matrix); + } + beads.instanceMatrix.needsUpdate = true; + + cubeRT.fromEquirectangularTexture(renderer!, envExternalTexture); + renderer!.render(scene, camera); + context.present(); + }; + renderer.setAnimationLoop(animate); + + setPipelineState({ + device, + cameraPipeline, + cameraSampler, + envTexture, + envTextureView: envTexture.createView(), + }); + } catch (e) { + if (cancelled) { + return; + } + console.warn("[CameraSpheres] setup failed: " + String(e)); + setError(String(e)); + } + })(); + + return () => { + cancelled = true; + if (renderer) { + renderer.setAnimationLoop(null); + } + }; + }, [device]); + + const logBox = React.useMemo(() => ({ count: 0 }), []); + const frameOutput = useFrameOutput({ + pixelFormat: "native", + targetResolution: CommonResolutions.HD_16_9, + onFrame: (frame) => { + "worklet"; + logBox.count += 1; + if (logBox.count === 1) { + console.log( + "[CameraSpheres] worklet first frame, frame=" + + String(frame.width) + + "x" + + String(frame.height), + ); + } + if (!pipelineState) { + frame.dispose(); + return; + } + const { + device: gpuDevice, + cameraPipeline, + cameraSampler, + envTextureView, + } = pipelineState; + const nativeBuffer = frame.getNativeBuffer(); + try { + const videoFrame = gpuDevice.createVideoFrameFromNativeBuffer( + nativeBuffer.pointer, + ); + try { + const externalTex = gpuDevice.importExternalTexture({ + source: videoFrame, + label: "camera-spheres-env", + }); + const bindGroup = gpuDevice.createBindGroup({ + layout: cameraPipeline.getBindGroupLayout(0), + entries: [ + { binding: 0, resource: externalTex }, + { binding: 1, resource: cameraSampler }, + ], + }); + const encoder = gpuDevice.createCommandEncoder(); + const pass = encoder.beginRenderPass({ + colorAttachments: [ + { + view: envTextureView, + clearValue: { r: 0, g: 0, b: 0, a: 1 }, + loadOp: "clear", + storeOp: "store", + }, + ], + }); + pass.setPipeline(cameraPipeline); + pass.setBindGroup(0, bindGroup); + pass.draw(3); + pass.end(); + gpuDevice.queue.submit([encoder.finish()]); + } finally { + videoFrame.release(); + } + } finally { + nativeBuffer.release(); + frame.dispose(); + } + }, + }); + + const [session, setSession] = useState(null); + useEffect(() => { + if (!VisionCameraFactory.supportsMultiCamSessions) { + setError( + "This device doesn't support multi-cam sessions. Need an iPhone XS " + + "or newer / a comparable Android flagship.", + ); + return; + } + let cancelled = false; + let created: CameraSession | null = null; + (async () => { + const s = await VisionCameraFactory.createCameraSession(true); + if (cancelled) { + s.dispose(); + return; + } + created = s; + setSession(s); + })(); + return () => { + cancelled = true; + created?.stop(); + created?.dispose(); + }; + }, []); + + useEffect(() => { + if (!session || !backDevice || !frontDevice || !pipelineState) { + return; + } + let cancelled = false; + let controllers: CameraController[] = []; + (async () => { + try { + controllers = await session.configure( + [ + { + input: backDevice, + outputs: [{ output: previewOutput, mirrorMode: "auto" }], + constraints: [], + }, + { + input: frontDevice, + outputs: [{ output: frameOutput, mirrorMode: "auto" }], + constraints: [], + }, + ], + {}, + ); + if (cancelled) { + controllers.forEach((c) => c.dispose()); + return; + } + session.start(); + } catch (e) { + if (cancelled) { + return; + } + console.warn("[CameraSpheres] session configure failed: " + String(e)); + setError(String(e)); + } + })(); + return () => { + cancelled = true; + session.stop(); + controllers.forEach((c) => c.dispose()); + }; + }, [ + session, + backDevice, + frontDevice, + previewOutput, + frameOutput, + pipelineState, + ]); + + if (error) { + return ( + + {error} + + ); + } + if (backDevice == null || frontDevice == null) { + return ( + + + Need both a back and a front camera. The iOS Simulator has none, and + some devices expose only one. + + + ); + } + return ( + + + + + ); +}; + +const styles = StyleSheet.create({ + root: { flex: 1, backgroundColor: "black" }, + canvas: { ...StyleSheet.absoluteFillObject, backgroundColor: "transparent" }, + errorContainer: { flex: 1, padding: 16, justifyContent: "center" }, + errorText: { color: "red", fontSize: 14 }, + permissionContainer: { + flex: 1, + padding: 24, + justifyContent: "center", + alignItems: "center", + }, + permissionText: { fontSize: 16, textAlign: "center", marginBottom: 16 }, + permissionButton: { + backgroundColor: "#007AFF", + paddingHorizontal: 24, + paddingVertical: 12, + borderRadius: 8, + }, + permissionButtonText: { color: "white", fontSize: 16, fontWeight: "600" }, +}); diff --git a/apps/example/src/ThreeJS/List.tsx b/apps/example/src/ThreeJS/List.tsx index 972522e87..ad3547fdb 100644 --- a/apps/example/src/ThreeJS/List.tsx +++ b/apps/example/src/ThreeJS/List.tsx @@ -27,6 +27,10 @@ export const examples = [ screen: "CameraHelmet", title: "đź“· Camera Env Sphere", }, + { + screen: "CameraSpheres", + title: "đź“· Camera Env Spheres", + }, { screen: "PostProcessing", title: "🪄 Post Processing Effects", diff --git a/apps/example/src/ThreeJS/Routes.ts b/apps/example/src/ThreeJS/Routes.ts index aa78c2240..68971cef5 100644 --- a/apps/example/src/ThreeJS/Routes.ts +++ b/apps/example/src/ThreeJS/Routes.ts @@ -3,6 +3,7 @@ export type Routes = { Cube: undefined; Helmet: undefined; CameraHelmet: undefined; + CameraSpheres: undefined; Backdrop: undefined; InstancedMesh: undefined; Fiber: undefined; diff --git a/apps/example/src/ThreeJS/index.tsx b/apps/example/src/ThreeJS/index.tsx index 1a7535404..e679d88e4 100644 --- a/apps/example/src/ThreeJS/index.tsx +++ b/apps/example/src/ThreeJS/index.tsx @@ -7,6 +7,7 @@ import type { Routes } from "./Routes"; import { List } from "./List"; import { Helmet } from "./Helmet"; import { CameraHelmet } from "./CameraHelmet"; +import { CameraSpheres } from "./CameraSpheres"; import { Backdrop } from "./Backdrop"; import { InstancedMesh } from "./InstancedMesh"; import { Fiber } from "./Fiber"; @@ -81,6 +82,13 @@ export const ThreeJS = () => { title: "đź“· Camera Env Sphere", }} /> + Date: Fri, 22 May 2026 18:39:12 +0000 Subject: [PATCH 21/46] CameraHelmet: build the env cubemap with THREE.CubeCamera Replace the per-frame fromEquirectangularTexture blit with a real THREE.CubeCamera that renders a layer-1-only sky sphere wearing the worklet-updated camera feed. The reflection now picks up any scene geometry placed on layer 1, not just the panorama itself. https://claude.ai/code/session_01Sgh5mWoT9XBAYQotQFf94g --- apps/example/src/ThreeJS/CameraHelmet.tsx | 48 ++++++++++++++++------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/apps/example/src/ThreeJS/CameraHelmet.tsx b/apps/example/src/ThreeJS/CameraHelmet.tsx index 58b72c4fe..e642935ef 100644 --- a/apps/example/src/ThreeJS/CameraHelmet.tsx +++ b/apps/example/src/ThreeJS/CameraHelmet.tsx @@ -256,13 +256,11 @@ const Scene = () => { }); // THREE.ExternalTexture bridges our GPUTexture into three.js as a - // sampleable 2D equirect. We never set scene.background or - // material.envMap to this directly: three.js' CubeMapNode would only - // run the equirect→cubemap conversion once and cache it, which means - // we'd be stuck sampling whatever was in our env texture on frame 1 - // (= black, before the worklet ever wrote). + // sampleable 2D texture. SphereGeometry's default UVs already lay + // out 0..1 the same way an equirect does (u around, v pole-to-pole), + // so plain UV mapping wraps the panorama correctly when we use this + // as a regular .map on the inside-out sky sphere below. const envExternalTexture = new THREE.ExternalTexture(envTexture); - envExternalTexture.mapping = THREE.EquirectangularReflectionMapping; envExternalTexture.colorSpace = THREE.SRGBColorSpace; (envExternalTexture as unknown as { image: unknown }).image = { width: ENV_WIDTH, @@ -270,11 +268,11 @@ const Scene = () => { }; envExternalTexture.needsUpdate = true; - // Allocate the cubemap once. Each frame we'll call - // cubeRT.fromEquirectangularTexture(renderer, envExternalTexture) to - // refresh the cube faces from the equirect's *current* contents. - // That's the same code path CubeMapNode uses internally, but we - // drive it on every tick instead of letting three.js cache it. + // The cubemap is rendered each frame by THREE.CubeCamera (six + // perspective passes into the six faces) instead of being blitted + // from an equirect. CubeCamera renders the actual scene, so the + // reflection picks up any other geometry we add — not just the sky + // sphere that carries the camera feed. const cubeRT = new THREE.CubeRenderTarget(ENV_HEIGHT); cubeRT.texture.mapping = THREE.CubeReflectionMapping; cubeRT.texture.colorSpace = THREE.SRGBColorSpace; @@ -291,6 +289,24 @@ const Scene = () => { const scene = new THREE.Scene(); // No scene.background — the canvas is alpha-cleared and the native // camera preview View sits behind it (see JSX below). + + // Layer split: main camera sees layer 0 (helmet only) so the native + // preview View remains visible everywhere else; CubeCamera sees + // layer 1 (the sky sphere) so the helmet never reflects itself. + const ENV_LAYER = 1; + const sky = new THREE.Mesh( + new THREE.SphereGeometry(50, 64, 32), + new THREE.MeshBasicMaterial({ + map: envExternalTexture, + side: THREE.BackSide, + }), + ); + sky.layers.set(ENV_LAYER); + scene.add(sky); + + const cubeCamera = new THREE.CubeCamera(0.1, 100, cubeRT); + cubeCamera.layers.set(ENV_LAYER); + scene.add(cubeCamera); if (USE_PBR) { // Keep the GLTF's MeshStandardMaterial intact (albedo / normal / // metalRoughness / AO from the original textures) and just plug @@ -357,10 +373,12 @@ const Scene = () => { camera.position.y = 0; camera.lookAt(0, 0, 0); - // Refresh the cubemap from the (worklet-updated) equirect before - // rendering the scene. The conversion does 6 fullscreen draws into - // the cube faces; pipelines are reused across calls. - cubeRT.fromEquirectangularTexture(renderer!, envExternalTexture); + // Refresh the cubemap by rendering the env-layer of the scene + // (= the sky sphere wearing the worklet-updated camera feed) from + // six perspectives. Costlier than a equirect blit but captures + // every layer-1 object, so adding scene props would also show up + // in the reflection. + cubeCamera.update(renderer!, scene); renderer!.render(scene, camera); context.present(); frameCount++; From 3eba2c81fa7bd96fc1cea7385d826f90b200b22c Mon Sep 17 00:00:00 2001 From: William Candillon Date: Fri, 22 May 2026 23:03:59 +0200 Subject: [PATCH 22/46] :wrench: --- apps/example/src/ThreeJS/CameraHelmet.tsx | 145 ++++++++++++++-------- 1 file changed, 93 insertions(+), 52 deletions(-) diff --git a/apps/example/src/ThreeJS/CameraHelmet.tsx b/apps/example/src/ThreeJS/CameraHelmet.tsx index e642935ef..adfd43ed8 100644 --- a/apps/example/src/ThreeJS/CameraHelmet.tsx +++ b/apps/example/src/ThreeJS/CameraHelmet.tsx @@ -43,13 +43,12 @@ import { CAMERA_ENV_SHADER } from "./cameraEnvShader"; const ENV_WIDTH = 1024; const ENV_HEIGHT = 512; -// PBR helmet (uses the GLTF's MeshStandardMaterial textures + cubemap -// reflection via per-material envMap, with the cubemap mipmapped each -// frame so rough surfaces sample blurrier mips). Set to `false` to fall -// back to a flat MeshBasicMaterial chrome shell — drops the PBR fragment -// cost and the every-frame mipmap regeneration of the cubemap, which is -// the bulk of the perf cost on phone GPUs. -const USE_PBR = false; +// PBR mode is runtime-toggleable from a button in the JSX below. PBR uses +// the GLTF's MeshStandardMaterial textures (albedo / normal / metalRoughness +// / AO) plus a cubemap envMap for the live reflection; "chrome" mode swaps +// every mesh for a single MeshBasicMaterial that just samples the cubemap. +// Chrome is roughly 5-10x cheaper on fragments and also skips the per-frame +// cubemap mipmap regen the PBR roughness lookup needs. // Vision Camera + react-native-wgpu both want these features for the external // texture path. dawn-multi-planar-formats lets Dawn interpret NV12 buffers. @@ -124,6 +123,21 @@ const Scene = () => { const [error, setError] = useState(null); const [device, setDevice] = useState(null); + // PBR is the default. Toggling to chrome swaps every helmet material for + // a single MeshBasicMaterial that just samples the cubemap. usePBRRef + // shadows the state so the (one-shot) setup effect can read the *current* + // value when it first applies materials, even if the user has already + // toggled before three.js finished initializing. + const [usePBR, setUsePBR] = useState(true); + const usePBRRef = useRef(true); + const applyPBRFnRef = useRef<((pbr: boolean) => void) | null>(null); + const togglePBR = () => { + const next = !usePBRRef.current; + usePBRRef.current = next; + setUsePBR(next); + applyPBRFnRef.current?.(next); + }; + // Acquire the GPU device on its own effect. By the time the async adapter + // device requests resolve, the Canvas component has been rendered and its // ref populated, so the main setup effect (gated on `device`) can grab the @@ -276,15 +290,14 @@ const Scene = () => { const cubeRT = new THREE.CubeRenderTarget(ENV_HEIGHT); cubeRT.texture.mapping = THREE.CubeReflectionMapping; cubeRT.texture.colorSpace = THREE.SRGBColorSpace; - if (USE_PBR) { - // PBR mode: generate mipmaps so MeshStandardMaterial's - // roughness-aware envMap sample picks softer mips for rough - // surfaces. Not true PMREM (no GGX importance sampling) but a - // cheap approximation; adds an auto-mipmap regen per frame. - cubeRT.texture.generateMipmaps = true; - cubeRT.texture.minFilter = THREE.LinearMipmapLinearFilter; - cubeRT.texture.magFilter = THREE.LinearFilter; - } + // Always-on mipmaps: MeshStandardMaterial's roughness-aware envMap + // sample picks softer mips for rough surfaces (not true PMREM since + // there's no GGX importance sampling, but a cheap approximation). + // Chrome mode pays the regen cost too, but doesn't suffer from it + // since it samples mip 0 only. + cubeRT.texture.generateMipmaps = true; + cubeRT.texture.minFilter = THREE.LinearMipmapLinearFilter; + cubeRT.texture.magFilter = THREE.LinearFilter; const scene = new THREE.Scene(); // No scene.background — the canvas is alpha-cleared and the native @@ -307,42 +320,48 @@ const Scene = () => { const cubeCamera = new THREE.CubeCamera(0.1, 100, cubeRT); cubeCamera.layers.set(ENV_LAYER); scene.add(cubeCamera); - if (USE_PBR) { - // Keep the GLTF's MeshStandardMaterial intact (albedo / normal / - // metalRoughness / AO from the original textures) and just plug - // our cubemap into each material's envMap. The PBR shader then - // combines the helmet's surface detail with the live front-camera - // reflection. - gltf.scene.traverse((child) => { - const mesh = child as THREE.Mesh; - if (!mesh.isMesh) { - return; - } - const mats = Array.isArray(mesh.material) - ? mesh.material - : [mesh.material]; - for (const m of mats) { - const std = m as THREE.MeshStandardMaterial; - std.envMap = cubeRT.texture; - std.envMapIntensity = 1.0; - std.needsUpdate = true; - } - }); - } else { - // Non-PBR mode: swap every mesh's material for a single - // MeshBasicMaterial that just samples the cubemap. Loses surface - // detail (no albedo / normal / roughness) but gives a fast - // chrome-shell look that's a good 5–10x cheaper on fragments. - const chrome = new THREE.MeshBasicMaterial({ - envMap: cubeRT.texture, - }); - gltf.scene.traverse((child) => { - const mesh = child as THREE.Mesh; - if (mesh.isMesh) { - mesh.material = chrome; - } - }); - } + + // PBR path: keep the GLTF's MeshStandardMaterial intact (albedo / + // normal / metalRoughness / AO from the original textures) and plug + // our live cubemap into each material's envMap. We capture the + // originals so the chrome toggle can swap back and forth without + // losing them. + const pbrMaterials = new Map< + THREE.Mesh, + THREE.Material | THREE.Material[] + >(); + gltf.scene.traverse((child) => { + const mesh = child as THREE.Mesh; + if (!mesh.isMesh) { + return; + } + pbrMaterials.set(mesh, mesh.material); + const mats = Array.isArray(mesh.material) + ? mesh.material + : [mesh.material]; + for (const m of mats) { + const std = m as THREE.MeshStandardMaterial; + std.envMap = cubeRT.texture; + std.envMapIntensity = 1.0; + std.needsUpdate = true; + } + }); + + // Chrome path: every mesh shares a single MeshBasicMaterial that + // just samples the cubemap. No surface detail, but ~5-10x cheaper + // fragment cost, a useful A/B against PBR on the same scene. + const chromeMaterial = new THREE.MeshBasicMaterial({ + envMap: cubeRT.texture, + }); + + const applyPBR = (pbr: boolean) => { + for (const [mesh, original] of pbrMaterials) { + mesh.material = pbr ? original : chromeMaterial; + } + }; + applyPBR(usePBRRef.current); + applyPBRFnRef.current = applyPBR; + scene.add(gltf.scene); // Drive the perspective from min(width, height) so the helmet keeps @@ -409,6 +428,7 @@ const Scene = () => { return () => { console.log("[CameraHelmet] setup-effect cleanup"); cancelled = true; + applyPBRFnRef.current = null; if (renderer) { renderer.setAnimationLoop(null); } @@ -603,6 +623,15 @@ const Scene = () => { style={StyleSheet.absoluteFill} /> + + + {usePBR ? "PBR" : "Chrome"} + + ); }; @@ -627,4 +656,16 @@ const styles = StyleSheet.create({ borderRadius: 8, }, permissionButtonText: { color: "white", fontSize: 16, fontWeight: "600" }, + toggleButton: { + position: "absolute", + top: 60, + right: 16, + backgroundColor: "rgba(0, 0, 0, 0.6)", + paddingHorizontal: 16, + paddingVertical: 10, + borderRadius: 20, + minWidth: 88, + alignItems: "center", + }, + toggleButtonText: { color: "white", fontSize: 14, fontWeight: "600" }, }); From bd849b622cad3c7199a105062cca3075fa9a5944 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 27 May 2026 13:42:06 +0000 Subject: [PATCH 23/46] CameraHelmet: reflect the front cam as a planar screen, not a panorama MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The front camera is a narrow-FOV image, not a 360° environment, so wrapping it equirectangularly produces an obviously fake "your face is the world" look. Treat the frame as a virtual screen at the viewer's location instead: a billboarded plane sized at the camera's natural 9:16 aspect that tracks the orbit camera each frame. The CubeCamera at the helmet's center bakes that plane into the cube faces, so surfaces facing the viewer pick up the user's face the way a real chrome object would, while a soft hemisphere- gradient backdrop fills the rest so grazing reflections don't fall to black. Env texture changes from 1024Ă—512 (stretched equirect) to 540Ă—960 (matches the rotated frame's aspect — no more distortion). --- apps/example/src/ThreeJS/CameraHelmet.tsx | 104 +++++++++++++++----- apps/example/src/ThreeJS/cameraEnvShader.ts | 13 +-- 2 files changed, 89 insertions(+), 28 deletions(-) diff --git a/apps/example/src/ThreeJS/CameraHelmet.tsx b/apps/example/src/ThreeJS/CameraHelmet.tsx index adfd43ed8..0a18ff02b 100644 --- a/apps/example/src/ThreeJS/CameraHelmet.tsx +++ b/apps/example/src/ThreeJS/CameraHelmet.tsx @@ -34,14 +34,23 @@ import { CAMERA_ENV_SHADER } from "./cameraEnvShader"; // camera frame into that GPUTexture via its own render pass. Three.js and the // worklet share a single GPUDevice (the one three.js creates internally), so // the queue ordering between "write env" and "sample env" is automatic. - -// Equirectangular panorama aspect (2:1). Front cam delivers 720p so the -// env texture doesn't gain anything from going much larger than that. -// Cube face dimension matches ENV_HEIGHT. Each frame we do (env write + 6 -// cube faces + optional mipmap chain), so this knob drives most of the -// per-frame GPU cost. -const ENV_WIDTH = 1024; -const ENV_HEIGHT = 512; +// +// The front camera is *not* a panorama — it's a narrow-FOV planar image. So +// rather than stretching it across a full equirect (which produces an +// obviously fake "your face wrapped around the world" look), we treat the +// frame as a virtual screen sitting at the viewer's location. The CubeCamera +// at the helmet's center bakes that screen into the cube faces, so reflective +// surfaces facing the viewer pick up the camera content (as a real chrome +// object would), while grazing / back-facing reflections fall to a soft +// hemisphere gradient. + +// 9:16 portrait — front cam delivers 16:9 landscape and the shader rotates it +// 90° to selfie-upright, so this aspect lets the rotated frame fill the env +// texture with no stretching. Cube face dimension matches ENV_HEIGHT. Each +// frame we do (env write + 6 cube faces + optional mipmap chain), so this +// knob drives most of the per-frame GPU cost. +const ENV_WIDTH = 540; +const ENV_HEIGHT = 960; // PBR mode is runtime-toggleable from a button in the JSX below. PBR uses // the GLTF's MeshStandardMaterial textures (albedo / normal / metalRoughness @@ -270,10 +279,8 @@ const Scene = () => { }); // THREE.ExternalTexture bridges our GPUTexture into three.js as a - // sampleable 2D texture. SphereGeometry's default UVs already lay - // out 0..1 the same way an equirect does (u around, v pole-to-pole), - // so plain UV mapping wraps the panorama correctly when we use this - // as a regular .map on the inside-out sky sphere below. + // sampleable 2D texture. Used below as the .map of a billboarded + // plane that represents the viewer's screen inside the env layer. const envExternalTexture = new THREE.ExternalTexture(envTexture); envExternalTexture.colorSpace = THREE.SRGBColorSpace; (envExternalTexture as unknown as { image: unknown }).image = { @@ -305,17 +312,61 @@ const Scene = () => { // Layer split: main camera sees layer 0 (helmet only) so the native // preview View remains visible everywhere else; CubeCamera sees - // layer 1 (the sky sphere) so the helmet never reflects itself. + // layer 1 (the reflection screen + gradient backdrop) so the helmet + // never reflects itself. const ENV_LAYER = 1; - const sky = new THREE.Mesh( - new THREE.SphereGeometry(50, 64, 32), + + // Soft hemisphere gradient (cool top, warm-dark bottom) baked into a + // 1Ă—64 DataTexture and wrapped on a back-side sphere. Without it the + // cube faces that the reflection screen doesn't cover would be the + // renderer's transparent-black clear, leaving the helmet pitch black + // anywhere it isn't reflecting the user. + const gradientHeight = 64; + const gradientPixels = new Uint8Array(gradientHeight * 4); + for (let i = 0; i < gradientHeight; i++) { + const t = i / (gradientHeight - 1); + gradientPixels[i * 4 + 0] = Math.round(THREE.MathUtils.lerp(48, 110, t)); + gradientPixels[i * 4 + 1] = Math.round(THREE.MathUtils.lerp(38, 130, t)); + gradientPixels[i * 4 + 2] = Math.round(THREE.MathUtils.lerp(40, 170, t)); + gradientPixels[i * 4 + 3] = 255; + } + const gradientTex = new THREE.DataTexture( + gradientPixels, + 1, + gradientHeight, + THREE.RGBAFormat, + ); + gradientTex.colorSpace = THREE.SRGBColorSpace; + gradientTex.needsUpdate = true; + const skyDome = new THREE.Mesh( + new THREE.SphereGeometry(50, 32, 16), new THREE.MeshBasicMaterial({ - map: envExternalTexture, + map: gradientTex, side: THREE.BackSide, }), ); - sky.layers.set(ENV_LAYER); - scene.add(sky); + skyDome.layers.set(ENV_LAYER); + scene.add(skyDome); + + // Virtual "screen" carrying the live camera frame. Sized at the env + // texture's natural 9:16 aspect; positioned each frame along the + // orbit camera's view ray (behind the camera so the screen subtends + // a sensible solid angle from the helmet's POV). The CubeCamera at + // the helmet's center bakes this plane into the cube faces, so the + // helmet's surfaces facing the viewer reflect the user's face. + const screenAspect = ENV_WIDTH / ENV_HEIGHT; + const screenHeight = 4; + const screenWidth = screenHeight * screenAspect; + const reflectionScreen = new THREE.Mesh( + new THREE.PlaneGeometry(screenWidth, screenHeight), + new THREE.MeshBasicMaterial({ + map: envExternalTexture, + side: THREE.DoubleSide, + toneMapped: false, + }), + ); + reflectionScreen.layers.set(ENV_LAYER); + scene.add(reflectionScreen); const cubeCamera = new THREE.CubeCamera(0.1, 100, cubeRT); cubeCamera.layers.set(ENV_LAYER); @@ -382,6 +433,7 @@ const Scene = () => { const clock = new THREE.Clock(); const distance = 3; + const screenDistance = 4.5; let frameCount = 0; const animate = () => { // Slow time-based orbit around the helmet, matching the three.js @@ -392,11 +444,19 @@ const Scene = () => { camera.position.y = 0; camera.lookAt(0, 0, 0); + // Park the reflection screen along the view ray, slightly behind + // the orbit camera, facing the helmet. Doing this each frame keeps + // the user's face on the helmet surface that's currently visible + // to the viewer, no matter where the orbit lands. + reflectionScreen.position + .copy(camera.position) + .setLength(screenDistance); + reflectionScreen.lookAt(0, 0, 0); + // Refresh the cubemap by rendering the env-layer of the scene - // (= the sky sphere wearing the worklet-updated camera feed) from - // six perspectives. Costlier than a equirect blit but captures - // every layer-1 object, so adding scene props would also show up - // in the reflection. + // (reflection screen + gradient dome) from six perspectives. + // Costlier than an equirect blit but captures every layer-1 + // object, so adding scene props would also show up in reflections. cubeCamera.update(renderer!, scene); renderer!.render(scene, camera); context.present(); diff --git a/apps/example/src/ThreeJS/cameraEnvShader.ts b/apps/example/src/ThreeJS/cameraEnvShader.ts index df38ecb96..2222ef444 100644 --- a/apps/example/src/ThreeJS/cameraEnvShader.ts +++ b/apps/example/src/ThreeJS/cameraEnvShader.ts @@ -1,10 +1,11 @@ // Tiny "copy camera frame into an rgba8unorm texture" shader. The output -// texture is then wrapped in a THREE.ExternalTexture and used as an -// equirectangular environment map by three.js' WebGPURenderer. -// -// The camera image is stretched to fill the 2:1 env texture, which when -// sampled equirectangularly produces a panorama wrap of the camera view -// around the helmet — your face becomes the world. +// texture is then wrapped in a THREE.ExternalTexture and mapped onto a +// billboarded plane that three.js' CubeCamera bakes into the helmet's +// envMap — i.e. it acts as a virtual screen at the viewer's location +// rather than a 360° panorama. The destination texture's aspect (9:16) is +// chosen to match the camera frame's post-rotation aspect so no stretching +// happens here; the fullscreen triangle just rotates+mirrors the source to +// selfie-upright. export const CAMERA_ENV_SHADER = /* wgsl */ ` struct VsOut { From 55c8fafab63c4175877cb9f69ad9dda21407a53b Mon Sep 17 00:00:00 2001 From: William Candillon Date: Wed, 27 May 2026 16:24:03 +0200 Subject: [PATCH 24/46] :wrench: --- apps/example/src/ThreeJS/CameraHelmet.tsx | 50 ++++++++++++++++------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/apps/example/src/ThreeJS/CameraHelmet.tsx b/apps/example/src/ThreeJS/CameraHelmet.tsx index adfd43ed8..da7c2b0d8 100644 --- a/apps/example/src/ThreeJS/CameraHelmet.tsx +++ b/apps/example/src/ThreeJS/CameraHelmet.tsx @@ -43,6 +43,14 @@ import { CAMERA_ENV_SHADER } from "./cameraEnvShader"; const ENV_WIDTH = 1024; const ENV_HEIGHT = 512; +// Cube face size is mode-dependent. Chrome is a pure mirror reflection, so it +// wants the equirect's full resolution. PBR samples roughness-selected mips +// (helmet's metal-roughness keeps most surfaces in the 0.3-0.6 range, i.e. +// mip 2-3), so a 128 cube + its short mip chain is visually indistinguishable +// from 512 while cutting cubemap fill rate ~16x and mip-regen cost with it. +const CUBE_SIZE_PBR = 128; +const CUBE_SIZE_CHROME = 512; + // PBR mode is runtime-toggleable from a button in the JSX below. PBR uses // the GLTF's MeshStandardMaterial textures (albedo / normal / metalRoughness // / AO) plus a cubemap envMap for the live reflection; "chrome" mode swaps @@ -287,17 +295,29 @@ const Scene = () => { // from an equirect. CubeCamera renders the actual scene, so the // reflection picks up any other geometry we add — not just the sky // sphere that carries the camera feed. - const cubeRT = new THREE.CubeRenderTarget(ENV_HEIGHT); - cubeRT.texture.mapping = THREE.CubeReflectionMapping; - cubeRT.texture.colorSpace = THREE.SRGBColorSpace; - // Always-on mipmaps: MeshStandardMaterial's roughness-aware envMap - // sample picks softer mips for rough surfaces (not true PMREM since - // there's no GGX importance sampling, but a cheap approximation). - // Chrome mode pays the regen cost too, but doesn't suffer from it - // since it samples mip 0 only. - cubeRT.texture.generateMipmaps = true; - cubeRT.texture.minFilter = THREE.LinearMipmapLinearFilter; - cubeRT.texture.magFilter = THREE.LinearFilter; + // Two cube targets, one per mode. We swap which one CubeCamera writes + // into on toggle (resizing a single target via setSize doesn't fully + // reallocate cleanly on the WebGPU path, the reflection comes back + // blurry). Only the active one gets updated each frame so the cost + // stays paid for one mode at a time. + const makeCubeRT = (size: number) => { + const rt = new THREE.CubeRenderTarget(size); + rt.texture.mapping = THREE.CubeReflectionMapping; + rt.texture.colorSpace = THREE.SRGBColorSpace; + // Mipmaps only matter for PBR (roughness-aware sample). Chrome + // samples mip 0 only, so we skip the regen cost on the 512 target. + const wantsMips = size <= CUBE_SIZE_PBR; + rt.texture.generateMipmaps = wantsMips; + rt.texture.minFilter = wantsMips + ? THREE.LinearMipmapLinearFilter + : THREE.LinearFilter; + rt.texture.magFilter = THREE.LinearFilter; + return rt; + }; + const cubeRTPbr = makeCubeRT(CUBE_SIZE_PBR); + const cubeRTChrome = makeCubeRT(CUBE_SIZE_CHROME); + const activeCubeRT = () => + usePBRRef.current ? cubeRTPbr : cubeRTChrome; const scene = new THREE.Scene(); // No scene.background — the canvas is alpha-cleared and the native @@ -317,7 +337,7 @@ const Scene = () => { sky.layers.set(ENV_LAYER); scene.add(sky); - const cubeCamera = new THREE.CubeCamera(0.1, 100, cubeRT); + const cubeCamera = new THREE.CubeCamera(0.1, 100, activeCubeRT()); cubeCamera.layers.set(ENV_LAYER); scene.add(cubeCamera); @@ -341,7 +361,7 @@ const Scene = () => { : [mesh.material]; for (const m of mats) { const std = m as THREE.MeshStandardMaterial; - std.envMap = cubeRT.texture; + std.envMap = cubeRTPbr.texture; std.envMapIntensity = 1.0; std.needsUpdate = true; } @@ -351,10 +371,12 @@ const Scene = () => { // just samples the cubemap. No surface detail, but ~5-10x cheaper // fragment cost, a useful A/B against PBR on the same scene. const chromeMaterial = new THREE.MeshBasicMaterial({ - envMap: cubeRT.texture, + envMap: cubeRTChrome.texture, }); const applyPBR = (pbr: boolean) => { + (cubeCamera as unknown as { renderTarget: THREE.CubeRenderTarget }) + .renderTarget = pbr ? cubeRTPbr : cubeRTChrome; for (const [mesh, original] of pbrMaterials) { mesh.material = pbr ? original : chromeMaterial; } From c0b6f78d98825f186cf3c496cfa80fdc79593365 Mon Sep 17 00:00:00 2001 From: William Candillon Date: Wed, 27 May 2026 16:33:23 +0200 Subject: [PATCH 25/46] :wrench: --- apps/example/src/ThreeJS/CameraHelmet.tsx | 138 +++++++++------------- 1 file changed, 56 insertions(+), 82 deletions(-) diff --git a/apps/example/src/ThreeJS/CameraHelmet.tsx b/apps/example/src/ThreeJS/CameraHelmet.tsx index 0a18ff02b..074ad2fc8 100644 --- a/apps/example/src/ThreeJS/CameraHelmet.tsx +++ b/apps/example/src/ThreeJS/CameraHelmet.tsx @@ -35,14 +35,13 @@ import { CAMERA_ENV_SHADER } from "./cameraEnvShader"; // worklet share a single GPUDevice (the one three.js creates internally), so // the queue ordering between "write env" and "sample env" is automatic. // -// The front camera is *not* a panorama — it's a narrow-FOV planar image. So -// rather than stretching it across a full equirect (which produces an -// obviously fake "your face wrapped around the world" look), we treat the -// frame as a virtual screen sitting at the viewer's location. The CubeCamera -// at the helmet's center bakes that screen into the cube faces, so reflective -// surfaces facing the viewer pick up the camera content (as a real chrome -// object would), while grazing / back-facing reflections fall to a soft -// hemisphere gradient. +// The front-camera frame is wrapped around the inside of a large back-side +// sphere centered on the helmet, so the CubeCamera at the helmet's middle +// samples camera content in every direction. The helmet's reflections +// therefore fully cover the helmet and never reveal the frame's edges. The +// trade-off is the obvious "face wrapped around the world" look on grazing +// reflections; that's chosen deliberately, since full coverage matters more +// than the seam artifact for this demo. // 9:16 portrait — front cam delivers 16:9 landscape and the shader rotates it // 90° to selfie-upright, so this aspect lets the rotated frame fill the env @@ -52,6 +51,14 @@ import { CAMERA_ENV_SHADER } from "./cameraEnvShader"; const ENV_WIDTH = 540; const ENV_HEIGHT = 960; +// Cube face size is mode-dependent. Chrome is a pure mirror reflection, so it +// wants the env's full resolution. PBR samples roughness-selected mips (the +// helmet's metal-roughness keeps most surfaces in the 0.3-0.6 range, i.e. +// mip 2-3), so a 128 cube + its short mip chain is visually indistinguishable +// from 512 while cutting cubemap fill rate ~16x and mip-regen cost with it. +const CUBE_SIZE_PBR = 128; +const CUBE_SIZE_CHROME = 512; + // PBR mode is runtime-toggleable from a button in the JSX below. PBR uses // the GLTF's MeshStandardMaterial textures (albedo / normal / metalRoughness // / AO) plus a cubemap envMap for the live reflection; "chrome" mode swaps @@ -294,17 +301,29 @@ const Scene = () => { // from an equirect. CubeCamera renders the actual scene, so the // reflection picks up any other geometry we add — not just the sky // sphere that carries the camera feed. - const cubeRT = new THREE.CubeRenderTarget(ENV_HEIGHT); - cubeRT.texture.mapping = THREE.CubeReflectionMapping; - cubeRT.texture.colorSpace = THREE.SRGBColorSpace; - // Always-on mipmaps: MeshStandardMaterial's roughness-aware envMap - // sample picks softer mips for rough surfaces (not true PMREM since - // there's no GGX importance sampling, but a cheap approximation). - // Chrome mode pays the regen cost too, but doesn't suffer from it - // since it samples mip 0 only. - cubeRT.texture.generateMipmaps = true; - cubeRT.texture.minFilter = THREE.LinearMipmapLinearFilter; - cubeRT.texture.magFilter = THREE.LinearFilter; + // Two cube targets, one per mode. We swap which one CubeCamera writes + // into on toggle (resizing a single target via setSize doesn't fully + // reallocate cleanly on the WebGPU path, the reflection comes back + // blurry). Only the active one gets updated each frame so the cost + // stays paid for one mode at a time. + const makeCubeRT = (size: number) => { + const rt = new THREE.CubeRenderTarget(size); + rt.texture.mapping = THREE.CubeReflectionMapping; + rt.texture.colorSpace = THREE.SRGBColorSpace; + // Mipmaps only matter for PBR (roughness-aware sample). Chrome + // samples mip 0 only, so we skip the regen cost on the 512 target. + const wantsMips = size <= CUBE_SIZE_PBR; + rt.texture.generateMipmaps = wantsMips; + rt.texture.minFilter = wantsMips + ? THREE.LinearMipmapLinearFilter + : THREE.LinearFilter; + rt.texture.magFilter = THREE.LinearFilter; + return rt; + }; + const cubeRTPbr = makeCubeRT(CUBE_SIZE_PBR); + const cubeRTChrome = makeCubeRT(CUBE_SIZE_CHROME); + const activeCubeRT = () => + usePBRRef.current ? cubeRTPbr : cubeRTChrome; const scene = new THREE.Scene(); // No scene.background — the canvas is alpha-cleared and the native @@ -316,59 +335,22 @@ const Scene = () => { // never reflects itself. const ENV_LAYER = 1; - // Soft hemisphere gradient (cool top, warm-dark bottom) baked into a - // 1Ă—64 DataTexture and wrapped on a back-side sphere. Without it the - // cube faces that the reflection screen doesn't cover would be the - // renderer's transparent-black clear, leaving the helmet pitch black - // anywhere it isn't reflecting the user. - const gradientHeight = 64; - const gradientPixels = new Uint8Array(gradientHeight * 4); - for (let i = 0; i < gradientHeight; i++) { - const t = i / (gradientHeight - 1); - gradientPixels[i * 4 + 0] = Math.round(THREE.MathUtils.lerp(48, 110, t)); - gradientPixels[i * 4 + 1] = Math.round(THREE.MathUtils.lerp(38, 130, t)); - gradientPixels[i * 4 + 2] = Math.round(THREE.MathUtils.lerp(40, 170, t)); - gradientPixels[i * 4 + 3] = 255; - } - const gradientTex = new THREE.DataTexture( - gradientPixels, - 1, - gradientHeight, - THREE.RGBAFormat, - ); - gradientTex.colorSpace = THREE.SRGBColorSpace; - gradientTex.needsUpdate = true; - const skyDome = new THREE.Mesh( - new THREE.SphereGeometry(50, 32, 16), - new THREE.MeshBasicMaterial({ - map: gradientTex, - side: THREE.BackSide, - }), - ); - skyDome.layers.set(ENV_LAYER); - scene.add(skyDome); - - // Virtual "screen" carrying the live camera frame. Sized at the env - // texture's natural 9:16 aspect; positioned each frame along the - // orbit camera's view ray (behind the camera so the screen subtends - // a sensible solid angle from the helmet's POV). The CubeCamera at - // the helmet's center bakes this plane into the cube faces, so the - // helmet's surfaces facing the viewer reflect the user's face. - const screenAspect = ENV_WIDTH / ENV_HEIGHT; - const screenHeight = 4; - const screenWidth = screenHeight * screenAspect; - const reflectionScreen = new THREE.Mesh( - new THREE.PlaneGeometry(screenWidth, screenHeight), + // Live camera frame wrapped around the inside of a large back-side + // sphere. The CubeCamera at the helmet's center sees the camera in + // every direction, so reflections fully cover the helmet without + // any visible frame edges. + const envSphere = new THREE.Mesh( + new THREE.SphereGeometry(50, 64, 32), new THREE.MeshBasicMaterial({ map: envExternalTexture, - side: THREE.DoubleSide, + side: THREE.BackSide, toneMapped: false, }), ); - reflectionScreen.layers.set(ENV_LAYER); - scene.add(reflectionScreen); + envSphere.layers.set(ENV_LAYER); + scene.add(envSphere); - const cubeCamera = new THREE.CubeCamera(0.1, 100, cubeRT); + const cubeCamera = new THREE.CubeCamera(0.1, 100, activeCubeRT()); cubeCamera.layers.set(ENV_LAYER); scene.add(cubeCamera); @@ -392,7 +374,7 @@ const Scene = () => { : [mesh.material]; for (const m of mats) { const std = m as THREE.MeshStandardMaterial; - std.envMap = cubeRT.texture; + std.envMap = cubeRTPbr.texture; std.envMapIntensity = 1.0; std.needsUpdate = true; } @@ -402,10 +384,12 @@ const Scene = () => { // just samples the cubemap. No surface detail, but ~5-10x cheaper // fragment cost, a useful A/B against PBR on the same scene. const chromeMaterial = new THREE.MeshBasicMaterial({ - envMap: cubeRT.texture, + envMap: cubeRTChrome.texture, }); const applyPBR = (pbr: boolean) => { + (cubeCamera as unknown as { renderTarget: THREE.CubeRenderTarget }) + .renderTarget = pbr ? cubeRTPbr : cubeRTChrome; for (const [mesh, original] of pbrMaterials) { mesh.material = pbr ? original : chromeMaterial; } @@ -433,7 +417,6 @@ const Scene = () => { const clock = new THREE.Clock(); const distance = 3; - const screenDistance = 4.5; let frameCount = 0; const animate = () => { // Slow time-based orbit around the helmet, matching the three.js @@ -444,19 +427,10 @@ const Scene = () => { camera.position.y = 0; camera.lookAt(0, 0, 0); - // Park the reflection screen along the view ray, slightly behind - // the orbit camera, facing the helmet. Doing this each frame keeps - // the user's face on the helmet surface that's currently visible - // to the viewer, no matter where the orbit lands. - reflectionScreen.position - .copy(camera.position) - .setLength(screenDistance); - reflectionScreen.lookAt(0, 0, 0); - - // Refresh the cubemap by rendering the env-layer of the scene - // (reflection screen + gradient dome) from six perspectives. - // Costlier than an equirect blit but captures every layer-1 - // object, so adding scene props would also show up in reflections. + // Refresh the cubemap by rendering the env-layer (camera sphere) + // from six perspectives. Costlier than an equirect blit but lets + // us add other layer-1 props later that would also show up in + // reflections. cubeCamera.update(renderer!, scene); renderer!.render(scene, camera); context.present(); From a1b60e12caa86865a0fc8a4c59f355fbb3fde75f Mon Sep 17 00:00:00 2001 From: William Candillon Date: Wed, 27 May 2026 16:36:16 +0200 Subject: [PATCH 26/46] :wrench: --- apps/example/src/ThreeJS/CameraHelmet.tsx | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/apps/example/src/ThreeJS/CameraHelmet.tsx b/apps/example/src/ThreeJS/CameraHelmet.tsx index 074ad2fc8..03620619a 100644 --- a/apps/example/src/ThreeJS/CameraHelmet.tsx +++ b/apps/example/src/ThreeJS/CameraHelmet.tsx @@ -383,8 +383,15 @@ const Scene = () => { // Chrome path: every mesh shares a single MeshBasicMaterial that // just samples the cubemap. No surface detail, but ~5-10x cheaper // fragment cost, a useful A/B against PBR on the same scene. + // MeshBasicMaterial samples the envMap at full intensity (no Fresnel + // / roughness attenuation like PBR), then ACES tone-maps the result. + // Bright camera regions land in ACES's compressed range and read as + // washed-out. Dimming the base color multiplies the env sample down + // before tone mapping; ~70% matches a real chrome surface's + // reflectance and brings the brightness back in line with PBR. const chromeMaterial = new THREE.MeshBasicMaterial({ envMap: cubeRTChrome.texture, + color: 0xb0b0b0, }); const applyPBR = (pbr: boolean) => { From 7c66119dcd3e74c40f665d82bd5022b81e598e08 Mon Sep 17 00:00:00 2001 From: William Candillon Date: Sun, 31 May 2026 20:36:38 +0200 Subject: [PATCH 27/46] :wrench: --- .../src/ExternalTexture/ExternalTexture.tsx | 4 +-- .../SharedTextureMemory.tsx | 10 +++---- apps/example/src/ThreeJS/CameraHelmet.tsx | 9 +++---- packages/webgpu/src/index.tsx | 27 ++++++++++--------- packages/webgpu/src/types.ts | 21 ++++++++++----- 5 files changed, 40 insertions(+), 31 deletions(-) diff --git a/apps/example/src/ExternalTexture/ExternalTexture.tsx b/apps/example/src/ExternalTexture/ExternalTexture.tsx index b7218f0b7..2b2504b4c 100644 --- a/apps/example/src/ExternalTexture/ExternalTexture.tsx +++ b/apps/example/src/ExternalTexture/ExternalTexture.tsx @@ -5,7 +5,7 @@ import { useCanvasRef, useDevice, type NativeCanvas, - type VideoFrame, + type NativeVideoFrame, } from "react-native-wgpu"; // importExternalTexture is the spec-mandated path for "I have a YUV-encoded @@ -159,7 +159,7 @@ export const ExternalTexture = () => { // ticks — this is what stops the canvas from flashing black ~2/3 of the // time. AVPlayer's pool is several buffers deep so holding one back like // this doesn't stall decoding. - let currentFrame: VideoFrame | null = null; + let currentFrame: NativeVideoFrame | null = null; const render = () => { const newFrame = player.copyLatestFrame(); diff --git a/apps/example/src/SharedTextureMemory/SharedTextureMemory.tsx b/apps/example/src/SharedTextureMemory/SharedTextureMemory.tsx index 7805cb7ff..e91c69367 100644 --- a/apps/example/src/SharedTextureMemory/SharedTextureMemory.tsx +++ b/apps/example/src/SharedTextureMemory/SharedTextureMemory.tsx @@ -6,7 +6,7 @@ import { useDevice, type GPUSharedTextureMemory, type NativeCanvas, - type VideoFrame, + type NativeVideoFrame, } from "react-native-wgpu"; const SHADER = /* wgsl */ ` @@ -103,7 +103,7 @@ export const SharedTextureMemory = () => { // from copyLatestFrame() as "keep showing the previous frame", which means // a one-shot source renders correctly without any other change. interface FrameSource { - copyLatestFrame(): VideoFrame | null; + copyLatestFrame(): NativeVideoFrame | null; release(): void; } let source: FrameSource; @@ -117,7 +117,7 @@ export const SharedTextureMemory = () => { release: () => player.release(), }; } else { - let pending: VideoFrame | null = RNWebGPU.createTestVideoFrame( + let pending: NativeVideoFrame | null = RNWebGPU.createTestVideoFrame( 1024, 1024, ); @@ -168,7 +168,7 @@ export const SharedTextureMemory = () => { // demo we rely on AVPlayer recycling its IOSurface pool, which is safe as // long as we end-access before letting the player reclaim the buffer. type Bound = { - frame: VideoFrame; + frame: NativeVideoFrame; memory: GPUSharedTextureMemory; texture: GPUTexture; bindGroup: GPUBindGroup; @@ -190,7 +190,7 @@ export const SharedTextureMemory = () => { } }; - const bindFrame = (frame: VideoFrame): Bound | null => { + const bindFrame = (frame: NativeVideoFrame): Bound | null => { try { const memory = device.importSharedTextureMemory({ handle: frame.handle, diff --git a/apps/example/src/ThreeJS/CameraHelmet.tsx b/apps/example/src/ThreeJS/CameraHelmet.tsx index 03620619a..0d48d0c47 100644 --- a/apps/example/src/ThreeJS/CameraHelmet.tsx +++ b/apps/example/src/ThreeJS/CameraHelmet.tsx @@ -395,8 +395,9 @@ const Scene = () => { }); const applyPBR = (pbr: boolean) => { - (cubeCamera as unknown as { renderTarget: THREE.CubeRenderTarget }) - .renderTarget = pbr ? cubeRTPbr : cubeRTChrome; + ( + cubeCamera as unknown as { renderTarget: THREE.CubeRenderTarget } + ).renderTarget = pbr ? cubeRTPbr : cubeRTChrome; for (const [mesh, original] of pbrMaterials) { mesh.material = pbr ? original : chromeMaterial; } @@ -669,9 +670,7 @@ const Scene = () => { style={styles.toggleButton} activeOpacity={0.8} > - - {usePBR ? "PBR" : "Chrome"} - + {usePBR ? "PBR" : "Chrome"} ); diff --git a/packages/webgpu/src/index.tsx b/packages/webgpu/src/index.tsx index 1e696e273..09d2877c3 100644 --- a/packages/webgpu/src/index.tsx +++ b/packages/webgpu/src/index.tsx @@ -5,14 +5,15 @@ import type { NativeCanvas, RNCanvasContext, VideoPlayer, - VideoFrame, + NativeVideoFrame, + NativeVideoPixelFormat, } from "./types"; export * from "./main"; export type { - VideoFrame, + NativeVideoFrame, VideoPlayer, - VideoPixelFormat, + NativeVideoPixelFormat, CreateVideoPlayerOptions, GPUSharedTextureMemory, GPUSharedTextureMemoryDescriptor, @@ -36,15 +37,15 @@ declare global { ) => RNCanvasContext; DecodeToUTF8: (buffer: NodeJS.ArrayBufferView | ArrayBuffer) => string; createImageBitmap: typeof createImageBitmap; - loadVideoFrame: (path: string) => VideoFrame; - createTestVideoFrame: (width: number, height: number) => VideoFrame; + loadVideoFrame: (path: string) => NativeVideoFrame; + createTestVideoFrame: (width: number, height: number) => NativeVideoFrame; // Wrap a NativeBuffer.pointer (CVPixelBufferRef on iOS / AHardwareBuffer* - // on Android) into a VideoFrame. Matches the shape used by libraries that - // emit NativeBuffer (e.g. react-native-vision-camera). - createVideoFrameFromNativeBuffer: (pointer: bigint) => VideoFrame; + // on Android) into a NativeVideoFrame. Matches the shape used by libraries + // that emit NativeBuffer (e.g. react-native-vision-camera). + createVideoFrameFromNativeBuffer: (pointer: bigint) => NativeVideoFrame; createVideoPlayer: ( path: string, - pixelFormat?: VideoPixelFormat, + pixelFormat?: NativeVideoPixelFormat, ) => VideoPlayer; writeTestVideoFile: () => string; }; @@ -53,10 +54,10 @@ declare global { importSharedTextureMemory( descriptor: GPUSharedTextureMemoryDescriptor, ): GPUSharedTextureMemory; - // Wrap a NativeBuffer.pointer into a VideoFrame. Reachable from worklet - // runtimes (e.g. Vision Camera frame processors) because GPUDevice is - // serialized across worklet boundaries via the WebGPU custom serializer. - createVideoFrameFromNativeBuffer(pointer: bigint): VideoFrame; + // Wrap a NativeBuffer.pointer into a NativeVideoFrame. Reachable from + // worklet runtimes (e.g. Vision Camera frame processors) because GPUDevice + // is serialized across worklet boundaries via the WebGPU custom serializer. + createVideoFrameFromNativeBuffer(pointer: bigint): NativeVideoFrame; } // Extend createImageBitmap to accept ArrayBuffer/TypedArray (encoded image bytes) diff --git a/packages/webgpu/src/types.ts b/packages/webgpu/src/types.ts index a6037a381..a982763a5 100644 --- a/packages/webgpu/src/types.ts +++ b/packages/webgpu/src/types.ts @@ -18,10 +18,18 @@ export interface CanvasRef { getNativeSurface: () => NativeCanvas; } -export type VideoPixelFormat = "bgra8" | "nv12"; +// Pixel layout of a NativeVideoFrame. NOT the WebCodecs `VideoPixelFormat` +// enum — these are the two native surface layouts we support, lower-cased to +// avoid being mistaken for the spec values ("NV12", "BGRA", …). +export type NativeVideoPixelFormat = "bgra8" | "nv12"; // A native, GPU-shareable handle to a single video frame. // +// NOT the WebCodecs `VideoFrame`: there is no `close()`/`format`/`timestamp`, +// the surface is referenced by a raw native pointer, and disposal is +// `release()`. Named with a `Native` prefix so it doesn't shadow the global +// WebCodecs type or imply spec semantics it doesn't have. +// // - handle is the raw pointer (IOSurfaceRef on Apple, AHardwareBuffer* on // Android) encoded as a BigInt. Pass it to // GPUDevice.importSharedTextureMemory. @@ -31,11 +39,11 @@ export type VideoPixelFormat = "bgra8" | "nv12"; // - release() drops the underlying backing object (a CVPixelBuffer on Apple). // The frame is also released when the JS wrapper is garbage-collected; call // release() eagerly when you know you're done. -export interface VideoFrame { +export interface NativeVideoFrame { readonly handle: bigint; readonly width: number; readonly height: number; - readonly pixelFormat: VideoPixelFormat; + readonly pixelFormat: NativeVideoPixelFormat; release(): void; } @@ -43,7 +51,7 @@ export interface VideoFrame { // to obtain the most recently decoded frame as an IOSurface/AHardwareBuffer // (returns null between frames so callers can skip the import work). export interface VideoPlayer { - copyLatestFrame(): VideoFrame | null; + copyLatestFrame(): NativeVideoFrame | null; play(): void; pause(): void; release(): void; @@ -54,14 +62,15 @@ export interface CreateVideoPlayerOptions { // SharedTextureMemory and a regular sampled GPUTexture. // 'nv12': emit biplanar Y + CbCr surfaces, suitable for // GPUDevice.importExternalTexture. - pixelFormat?: VideoPixelFormat; + pixelFormat?: NativeVideoPixelFormat; } export interface GPUSharedTextureMemoryDescriptor { // Raw native handle (IOSurfaceRef on Apple, AHardwareBuffer* on Android), // encoded as a BigInt. The caller is responsible for keeping the underlying // object alive for as long as the shared memory (and any textures derived - // from it) are in use. Using VideoFrame.handle handles this automatically. + // from it) are in use. Using NativeVideoFrame.handle handles this + // automatically. handle: bigint; label?: string; } From b0ac50cdb5be1546c4878a5b0fdc1e6eab0ee037 Mon Sep 17 00:00:00 2001 From: William Candillon Date: Mon, 1 Jun 2026 09:16:49 +0200 Subject: [PATCH 28/46] :wrench: --- .../reanimated/registerWebGPUForReanimated.ts | 71 +++---------------- 1 file changed, 9 insertions(+), 62 deletions(-) diff --git a/packages/webgpu/src/external/reanimated/registerWebGPUForReanimated.ts b/packages/webgpu/src/external/reanimated/registerWebGPUForReanimated.ts index c8169e552..e331b731a 100644 --- a/packages/webgpu/src/external/reanimated/registerWebGPUForReanimated.ts +++ b/packages/webgpu/src/external/reanimated/registerWebGPUForReanimated.ts @@ -1,12 +1,8 @@ -// Declare global WebGPU worklet helper functions (installed by native module -// on the main JS runtime only — secondary worklet runtimes like Vision -// Camera's frame-processor thread do NOT have these on their globalThis). -declare function __webgpuBox(obj: object): { - unbox: () => object; - __boxedWebGPU: true; -}; - -console.log("[WebGPU] registerWebGPUForReanimated module loaded"); +// Declare global WebGPU worklet helper functions (installed by native module) +declare function __webgpuIsWebGPUObject(obj: unknown): boolean; +declare function __webgpuBox( + obj: object +): { unbox: () => object; __boxedWebGPU: true }; let isRegistered = false; @@ -23,74 +19,25 @@ export const registerWebGPUForReanimated = () => { isRegistered = true; try { + // eslint-disable-next-line @typescript-eslint/no-var-requires const { registerCustomSerializable } = require("react-native-worklets"); - console.log( - "[WebGPU] registering custom serializer (v2: __brand-getter check)", - ); - registerCustomSerializable({ name: "WebGPU", - // `determine` is invoked by Worklets in *both* runtimes during the - // serialization handshake, so its body must not rely on any globals - // beyond the JS built-ins. We inline the WebGPU-object check (native - // state + Symbol.toStringTag on prototype, same as the native - // __webgpuIsWebGPUObject helper) here so the function works on the - // main runtime and on every worklet runtime — including ones we don't - // own (Vision Camera, etc.). determine: (value: object): value is object => { "worklet"; - if (value == null || typeof value !== "object") { - return false; - } - if ((value as { __boxedWebGPU?: boolean }).__boxedWebGPU === true) { - console.log("[WebGPU determine] matched boxed object"); - return true; - } - const proto = Object.getPrototypeOf(value); - if (proto == null) { - return false; - } - const desc = Object.getOwnPropertyDescriptor(proto, "__brand"); - const matched = desc != null && typeof desc.get === "function"; - // Skip plain JS objects — they're handled by Worklets natively and - // would spam the log with every captured plain object. - if (proto !== Object.prototype) { - let brand: string | undefined; - try { - brand = desc?.get?.call(value) as string | undefined; - } catch { - brand = ""; - } - - console.log( - "[WebGPU determine] matched=" + - String(matched) + - " brand=" + - String(brand) + - " protoCtor=" + - String((proto.constructor as { name?: string })?.name), - ); - } - return matched; + return __webgpuIsWebGPUObject(value); }, - // `pack` runs on the source runtime, which is always the main JS - // runtime in our setup (you can't create raw WebGPU objects inside a - // worklet), so __webgpuBox is available here. pack: (value: object) => { "worklet"; return __webgpuBox(value); }, - // `unpack` runs on the destination runtime and just calls the - // BoxedWebGPUObject's own `unbox` method — no globals needed. unpack: (boxed: { unbox: () => object }) => { "worklet"; return boxed.unbox(); }, }); - - console.log("[WebGPU] registerCustomSerializable call returned OK"); - } catch (e) { - console.warn("[WebGPU] registerCustomSerializable threw: " + String(e)); + } catch { + // react-native-worklets not installed, skip registration } }; From 2075e97d400a054db17fa5b9754357f8ffbddb95 Mon Sep 17 00:00:00 2001 From: William Candillon Date: Mon, 1 Jun 2026 10:09:26 +0200 Subject: [PATCH 29/46] :wrench: --- .../external/reanimated/registerWebGPUForReanimated.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/webgpu/src/external/reanimated/registerWebGPUForReanimated.ts b/packages/webgpu/src/external/reanimated/registerWebGPUForReanimated.ts index e331b731a..04feb3933 100644 --- a/packages/webgpu/src/external/reanimated/registerWebGPUForReanimated.ts +++ b/packages/webgpu/src/external/reanimated/registerWebGPUForReanimated.ts @@ -1,8 +1,9 @@ // Declare global WebGPU worklet helper functions (installed by native module) declare function __webgpuIsWebGPUObject(obj: unknown): boolean; -declare function __webgpuBox( - obj: object -): { unbox: () => object; __boxedWebGPU: true }; +declare function __webgpuBox(obj: object): { + unbox: () => object; + __boxedWebGPU: true; +}; let isRegistered = false; @@ -19,7 +20,6 @@ export const registerWebGPUForReanimated = () => { isRegistered = true; try { - // eslint-disable-next-line @typescript-eslint/no-var-requires const { registerCustomSerializable } = require("react-native-worklets"); registerCustomSerializable({ From f89f7084beadee1739e02fd43f20e5bb145cca1a Mon Sep 17 00:00:00 2001 From: William Candillon Date: Mon, 1 Jun 2026 10:42:50 +0200 Subject: [PATCH 30/46] :wrench: --- packages/webgpu/src/main/index.tsx | 49 +++++++++++------------------- 1 file changed, 18 insertions(+), 31 deletions(-) diff --git a/packages/webgpu/src/main/index.tsx b/packages/webgpu/src/main/index.tsx index 5f504d5b2..df21bd780 100644 --- a/packages/webgpu/src/main/index.tsx +++ b/packages/webgpu/src/main/index.tsx @@ -12,40 +12,27 @@ const _installOk = WebGPUModule.install(); registerWebGPUForReanimated(); -// `RNWebGPU` is only fully populated in the main JS runtime where -// WebGPUModule.install() was called against. When this bundle re-evaluates in -// secondary worklet runtimes (Reanimated UI, Vision Camera frame processor, -// etc.) `RNWebGPU` may either be undefined or be a stripped-down stub. Guard -// every member access so a missing property doesn't take down the whole -// module graph (which would surface as `Cannot read property 'bind' of -// undefined` + every downstream import returning undefined). -if (typeof RNWebGPU !== "undefined" && RNWebGPU != null) { - if (RNWebGPU.gpu) { - if (!navigator) { - // @ts-expect-error Navigation object is more complex than this, setting it to an empty object to add gpu property - navigator = { - gpu: RNWebGPU.gpu, - userAgent: "react-native", - }; - } else { - navigator.gpu = RNWebGPU.gpu; - if (typeof navigator.userAgent !== "string") { - try { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - Hermes navigator may not include a userAgent, align with the polyfill if needed - navigator.userAgent = "react-native"; - } catch { - // navigator.userAgent can be read-only; ignore if assignment fails - } +if (typeof RNWebGPU !== "undefined") { + if (!navigator) { + // @ts-expect-error Navigation object is more complex than this, setting it to an empty object to add gpu property + navigator = { + gpu: RNWebGPU.gpu, + userAgent: "react-native", + }; + } else { + navigator.gpu = RNWebGPU.gpu; + if (typeof navigator.userAgent !== "string") { + try { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore - Hermes navigator may not include a userAgent, align with the polyfill if needed + navigator.userAgent = "react-native"; + } catch { + // navigator.userAgent can be read-only; ignore if assignment fails } } } - if ( - !global.createImageBitmap && - typeof RNWebGPU.createImageBitmap === "function" - ) { - global.createImageBitmap = RNWebGPU.createImageBitmap.bind(RNWebGPU); - } + global.createImageBitmap = + global.createImageBitmap ?? RNWebGPU.createImageBitmap.bind(RNWebGPU); } else { console.warn( `[react-native-wgpu] install() returned ${_installOk} but RNWebGPU global is not available`, From 85ecdc24b724d85ce82f380cddaa3d077bf5ca97 Mon Sep 17 00:00:00 2001 From: William Candillon Date: Mon, 1 Jun 2026 10:52:41 +0200 Subject: [PATCH 31/46] :wrench: --- apps/example/src/App.tsx | 12 +- .../example/src/ChromeSphere/ChromeSphere.tsx | 980 ------------------ apps/example/src/ChromeSphere/geometry.ts | 50 - apps/example/src/ChromeSphere/index.ts | 1 - apps/example/src/ChromeSphere/shader.ts | 137 --- .../src/ExternalTexture/ExternalTexture.tsx | 251 ----- apps/example/src/ExternalTexture/index.ts | 1 - apps/example/src/Home.tsx | 8 - apps/example/src/Route.ts | 2 - apps/example/src/ThreeJS/CameraHelmet.tsx | 711 ------------- apps/example/src/ThreeJS/CameraSpheres.tsx | 489 --------- apps/example/src/ThreeJS/List.tsx | 8 - apps/example/src/ThreeJS/Routes.ts | 2 - apps/example/src/ThreeJS/cameraEnvShader.ts | 49 - .../ThreeJS/components/makeWebGPURenderer.ts | 12 +- apps/example/src/ThreeJS/index.tsx | 16 - .../example/src/VisionCamera/VisionCamera.tsx | 4 +- apps/example/src/VisionCamera/features.ts | 2 +- 18 files changed, 8 insertions(+), 2727 deletions(-) delete mode 100644 apps/example/src/ChromeSphere/ChromeSphere.tsx delete mode 100644 apps/example/src/ChromeSphere/geometry.ts delete mode 100644 apps/example/src/ChromeSphere/index.ts delete mode 100644 apps/example/src/ChromeSphere/shader.ts delete mode 100644 apps/example/src/ExternalTexture/ExternalTexture.tsx delete mode 100644 apps/example/src/ExternalTexture/index.ts delete mode 100644 apps/example/src/ThreeJS/CameraHelmet.tsx delete mode 100644 apps/example/src/ThreeJS/CameraSpheres.tsx delete mode 100644 apps/example/src/ThreeJS/cameraEnvShader.ts diff --git a/apps/example/src/App.tsx b/apps/example/src/App.tsx index 917429ce0..9ccebcadd 100644 --- a/apps/example/src/App.tsx +++ b/apps/example/src/App.tsx @@ -37,9 +37,7 @@ import { AsyncStarvation } from "./Diagnostics/AsyncStarvation"; import { DeviceLostHang } from "./Diagnostics/DeviceLostHang"; import { StorageBufferVertices } from "./StorageBufferVertices"; import { SharedTextureMemory } from "./SharedTextureMemory"; -import { ExternalTexture } from "./ExternalTexture"; import { VisionCamera } from "./VisionCamera"; -import { ChromeSphere } from "./ChromeSphere"; // The two lines below are needed by three.js import "fast-text-encoding"; @@ -50,9 +48,9 @@ const Stack = createStackNavigator(); function App() { const assets = useAssets(); - // if (assets === null) { - // return null; - // } + if (assets === null) { + return null; + } return ( @@ -91,7 +89,7 @@ function App() { - {(props) => (assets ? : null)} + {(props) => } @@ -105,9 +103,7 @@ function App() { name="SharedTextureMemory" component={SharedTextureMemory} /> - - diff --git a/apps/example/src/ChromeSphere/ChromeSphere.tsx b/apps/example/src/ChromeSphere/ChromeSphere.tsx deleted file mode 100644 index 287962d6f..000000000 --- a/apps/example/src/ChromeSphere/ChromeSphere.tsx +++ /dev/null @@ -1,980 +0,0 @@ -/* eslint-disable prefer-destructuring */ -import React, { useEffect } from "react"; -import { - Linking, - PixelRatio, - Platform, - StyleSheet, - Text, - TouchableOpacity, - View, -} from "react-native"; -import { - Canvas, - useCanvasRef, - type NativeCanvas, - type RNCanvasContext, -} from "react-native-wgpu"; -import { - useCamera, - useCameraDevices, - useCameraPermission, - useFrameOutput, -} from "react-native-vision-camera"; - -import { BLUR_SHADER, PREPASS_SHADER } from "../VisionCamera/blurShaders"; - -import { - generateSphere, - NORMAL_OFFSET, - POSITION_OFFSET, - VERTEX_STRIDE_BYTES, -} from "./geometry"; -import { BACKDROP_SHADER, SHADER } from "./shader"; - -// All matrix math runs inside the Vision Camera frame-processor worklet, so it -// has to be implemented with worklet-friendly helpers. wgpu-matrix calls -// `mat4.create` / `vec3.create` internally and those aren't marked as -// worklets, so we inline what we need here. Conventions match wgpu-matrix: -// column-major 4x4 stored as Float32Array(16), index = col * 4 + row, -// right-handed view space, perspective maps z to [0, 1] (WebGPU clip space). - -const setIdentity = (out: Float32Array) => { - "worklet"; - out[0] = 1; - out[1] = 0; - out[2] = 0; - out[3] = 0; - out[4] = 0; - out[5] = 1; - out[6] = 0; - out[7] = 0; - out[8] = 0; - out[9] = 0; - out[10] = 1; - out[11] = 0; - out[12] = 0; - out[13] = 0; - out[14] = 0; - out[15] = 1; -}; - -// dst = m * Ry(angle). Safe with dst === m. -const applyRotateY = (m: Float32Array, angle: number, dst: Float32Array) => { - "worklet"; - const c = Math.cos(angle); - const s = Math.sin(angle); - const m00 = m[0], - m10 = m[1], - m20 = m[2], - m30 = m[3]; - const m02 = m[8], - m12 = m[9], - m22 = m[10], - m32 = m[11]; - if (dst !== m) { - dst[4] = m[4]; - dst[5] = m[5]; - dst[6] = m[6]; - dst[7] = m[7]; - dst[12] = m[12]; - dst[13] = m[13]; - dst[14] = m[14]; - dst[15] = m[15]; - } - dst[0] = c * m00 - s * m02; - dst[1] = c * m10 - s * m12; - dst[2] = c * m20 - s * m22; - dst[3] = c * m30 - s * m32; - dst[8] = s * m00 + c * m02; - dst[9] = s * m10 + c * m12; - dst[10] = s * m20 + c * m22; - dst[11] = s * m30 + c * m32; -}; - -// WebGPU-style perspective: right-handed, output z mapped to [0, 1]. fovy is -// the vertical field of view in radians. -const setPerspective = ( - out: Float32Array, - fovy: number, - aspect: number, - near: number, - far: number, -) => { - "worklet"; - const f = 1 / Math.tan(fovy * 0.5); - const rangeInv = 1 / (near - far); - out[0] = f / aspect; - out[1] = 0; - out[2] = 0; - out[3] = 0; - out[4] = 0; - out[5] = f; - out[6] = 0; - out[7] = 0; - out[8] = 0; - out[9] = 0; - out[10] = far * rangeInv; - out[11] = -1; - out[12] = 0; - out[13] = 0; - out[14] = far * near * rangeInv; - out[15] = 0; -}; - -// View matrix (RH). Camera looks down -z in its local frame. Same layout -// wgpu-matrix's lookAt produces, with the translation column carrying the -// negative basis-dot-eye terms. -const setLookAt = ( - out: Float32Array, - ex: number, - ey: number, - ez: number, - tx: number, - ty: number, - tz: number, - ux: number, - uy: number, - uz: number, -) => { - "worklet"; - let zx = ex - tx; - let zy = ey - ty; - let zz = ez - tz; - const zLen = Math.hypot(zx, zy, zz); - zx /= zLen; - zy /= zLen; - zz /= zLen; - - let xx = uy * zz - uz * zy; - let xy = uz * zx - ux * zz; - let xz = ux * zy - uy * zx; - const xLen = Math.hypot(xx, xy, xz); - xx /= xLen; - xy /= xLen; - xz /= xLen; - - const yx = zy * xz - zz * xy; - const yy = zz * xx - zx * xz; - const yz = zx * xy - zy * xx; - - out[0] = xx; - out[1] = yx; - out[2] = zx; - out[3] = 0; - out[4] = xy; - out[5] = yy; - out[6] = zy; - out[7] = 0; - out[8] = xz; - out[9] = yz; - out[10] = zz; - out[11] = 0; - out[12] = -(xx * ex + xy * ey + xz * ez); - out[13] = -(yx * ex + yy * ey + yz * ez); - out[14] = -(zx * ex + zy * ey + zz * ez); - out[15] = 1; -}; - -// dst = a * b. Safe with dst === a or dst === b (snapshots both fully first). -const multiplyMat4 = (a: Float32Array, b: Float32Array, dst: Float32Array) => { - "worklet"; - const a00 = a[0], - a10 = a[1], - a20 = a[2], - a30 = a[3]; - const a01 = a[4], - a11 = a[5], - a21 = a[6], - a31 = a[7]; - const a02 = a[8], - a12 = a[9], - a22 = a[10], - a32 = a[11]; - const a03 = a[12], - a13 = a[13], - a23 = a[14], - a33 = a[15]; - const b00 = b[0], - b10 = b[1], - b20 = b[2], - b30 = b[3]; - const b01 = b[4], - b11 = b[5], - b21 = b[6], - b31 = b[7]; - const b02 = b[8], - b12 = b[9], - b22 = b[10], - b32 = b[11]; - const b03 = b[12], - b13 = b[13], - b23 = b[14], - b33 = b[15]; - - dst[0] = a00 * b00 + a01 * b10 + a02 * b20 + a03 * b30; - dst[1] = a10 * b00 + a11 * b10 + a12 * b20 + a13 * b30; - dst[2] = a20 * b00 + a21 * b10 + a22 * b20 + a23 * b30; - dst[3] = a30 * b00 + a31 * b10 + a32 * b20 + a33 * b30; - dst[4] = a00 * b01 + a01 * b11 + a02 * b21 + a03 * b31; - dst[5] = a10 * b01 + a11 * b11 + a12 * b21 + a13 * b31; - dst[6] = a20 * b01 + a21 * b11 + a22 * b21 + a23 * b31; - dst[7] = a30 * b01 + a31 * b11 + a32 * b21 + a33 * b31; - dst[8] = a00 * b02 + a01 * b12 + a02 * b22 + a03 * b32; - dst[9] = a10 * b02 + a11 * b12 + a12 * b22 + a13 * b32; - dst[10] = a20 * b02 + a21 * b12 + a22 * b22 + a23 * b32; - dst[11] = a30 * b02 + a31 * b12 + a32 * b22 + a33 * b32; - dst[12] = a00 * b03 + a01 * b13 + a02 * b23 + a03 * b33; - dst[13] = a10 * b03 + a11 * b13 + a12 * b23 + a13 * b33; - dst[14] = a20 * b03 + a21 * b13 + a22 * b23 + a23 * b33; - dst[15] = a30 * b03 + a31 * b13 + a32 * b23 + a33 * b33; -}; - -// The 3D variant of the VisionCamera demo. Reuses the same shared-texture-memory -// pipeline to import camera frames as GPUExternalTextures, but instead of -// applying 2D effects, it samples the camera as a spherical environment map on -// a chrome sphere, an orbiting cube, and a torus. - -const REQUIRED_FEATURES: GPUFeatureName[] = [ - "rnwebgpu/shared-texture-memory" as GPUFeatureName, - "dawn-multi-planar-formats" as GPUFeatureName, -]; - -// Android-only feature; same probe as VisionCamera.tsx. Without it Dawn can't -// wrap a YCbCr AHB as a GPUExternalTexture. -const OPAQUE_YCBCR_EXT = - "opaque-ycbcr-android-for-external-texture" as GPUFeatureName; - -const DEPTH_FORMAT: GPUTextureFormat = "depth24plus"; - -// Backdrop blur tuning. Matches VisionCamera's "Strong" preset: prepass to a -// 1/4-res rgba8unorm, then 3 H-V iterations of the tile-based box blur. The -// final result is sampled by BACKDROP_SHADER as a fullscreen backdrop. -const BLUR_SCALE = 4; -const BLUR_FILTER_SIZE = 31; -const BLUR_TILE_DIM = 128; -const BLUR_BATCH = 4; -const BLUR_BLOCK_DIM = BLUR_TILE_DIM - BLUR_FILTER_SIZE; -const BLUR_ITERATIONS = 3; - -// Scene UBO layout. mat4(64) + vec4(16) + vec4(16) = 96 bytes; pad to 16-byte -// alignment for safety. -const SCENE_UBO_SIZE = 96; -const SCENE_UBO_FLOATS = SCENE_UBO_SIZE / 4; -const OBJECT_UBO_SIZE = 64; // mat4 - -type Shape = { - vertexBuffer: GPUBuffer; - indexBuffer: GPUBuffer; - indexCount: number; - uniformBuffer: GPUBuffer; - // Returns a model matrix for time t (seconds). - modelAt: (t: number, out: Float32Array) => void; -}; - -export const ChromeSphere = () => { - const { hasPermission, requestPermission } = useCameraPermission(); - useEffect(() => { - if (!hasPermission) { - requestPermission(); - } - }, [hasPermission, requestPermission]); - - if (!hasPermission) { - return ( - - - Camera access is required. Grant it in Settings or tap below. - - Linking.openSettings()} - style={styles.permissionButton} - > - Open Settings - - - ); - } - return ; -}; - -const SceneView = () => { - const ref = useCanvasRef(); - const [gpu, setGpu] = React.useState<{ - adapter: GPUAdapter; - device: GPUDevice; - } | null>(null); - const [deviceError, setDeviceError] = React.useState(null); - React.useEffect(() => { - let cancelled = false; - (async () => { - try { - const adapter = await navigator.gpu.requestAdapter(); - if (!adapter) { - throw new Error("requestAdapter returned null"); - } - const hasOpaqueYCbCrExt = - Platform.OS !== "android" || adapter.features.has(OPAQUE_YCBCR_EXT); - if (Platform.OS === "android" && !hasOpaqueYCbCrExt) { - throw new Error( - "This Android device's Vulkan driver doesn't advertise " + - "opaque-ycbcr-android-for-external-texture. Camera-frame import " + - "as a GPUExternalTexture isn't supported here.", - ); - } - const featuresToRequest: GPUFeatureName[] = [ - ...REQUIRED_FEATURES, - ...(Platform.OS === "android" ? [OPAQUE_YCBCR_EXT] : []), - ]; - const device = await adapter.requestDevice({ - requiredFeatures: featuresToRequest, - }); - if (cancelled) { - return; - } - setGpu({ adapter, device }); - } catch (e) { - if (cancelled) { - return; - } - setDeviceError(String(e)); - } - })(); - return () => { - cancelled = true; - }; - }, []); - const device = gpu?.device ?? null; - const adapter = gpu?.adapter ?? null; - const devices = useCameraDevices(); - const cameraDevice = React.useMemo( - () => - devices.find((d) => d.position === "back") ?? - devices.find((d) => d.position === "front") ?? - devices[0], - [devices], - ); - - const [pipelineState, setPipelineState] = React.useState<{ - pipeline: GPURenderPipeline; - sampler: GPUSampler; - sceneUniformBuffer: GPUBuffer; - depthView: GPUTextureView; - context: RNCanvasContext; - canvasWidth: number; - canvasHeight: number; - shapes: Shape[]; - // Pre-allocated scratch so the worklet doesn't allocate per frame. These - // get mutated each tick, which is safe because the worklet sees its own - // copy after closure serialization. - view: Float32Array; - proj: Float32Array; - viewProj: Float32Array; - sceneData: Float32Array; - modelScratch: Float32Array; - // Blurred-camera backdrop infrastructure. - backdropPipeline: GPURenderPipeline; - backdropBindGroup: GPUBindGroup; - prepassPipeline: GPURenderPipeline; - prepassUniformBuffer: GPUBuffer; - blurPipeline: GPUComputePipeline; - blurConstants: GPUBindGroup; - blurBindGroup0: GPUBindGroup; - blurBindGroup1: GPUBindGroup; - blurBindGroup2: GPUBindGroup; - blurSrcTexture: GPUTexture; - blurWidth: number; - blurHeight: number; - } | null>(null); - const [error, setError] = React.useState(null); - - useEffect(() => { - if (!device || pipelineState) { - return; - } - const missing = REQUIRED_FEATURES.filter((f) => !device.features.has(f)); - if (missing.length > 0) { - setError( - `Device missing features [${missing.join(", ")}]. Adapter: ${ - adapter - ? [...adapter.features] - .filter((f) => f.toString().startsWith("shared-")) - .join(", ") || "none" - : "n/a" - }`, - ); - return; - } - const context = ref.current?.getContext("webgpu"); - if (!context) { - return; - } - const canvas = context.canvas as unknown as NativeCanvas; - canvas.width = canvas.clientWidth * PixelRatio.get(); - canvas.height = canvas.clientHeight * PixelRatio.get(); - const presentationFormat = navigator.gpu.getPreferredCanvasFormat(); - context.configure({ - device, - format: presentationFormat, - alphaMode: "premultiplied", - }); - - const module = device.createShaderModule({ code: SHADER }); - const pipeline = device.createRenderPipeline({ - layout: "auto", - vertex: { - module, - entryPoint: "vs_main", - buffers: [ - { - arrayStride: VERTEX_STRIDE_BYTES, - attributes: [ - { - shaderLocation: 0, - offset: POSITION_OFFSET, - format: "float32x3", - }, - { - shaderLocation: 1, - offset: NORMAL_OFFSET, - format: "float32x3", - }, - ], - }, - ], - }, - fragment: { - module, - entryPoint: "fs_main", - targets: [{ format: presentationFormat }], - }, - primitive: { topology: "triangle-list", cullMode: "back" }, - depthStencil: { - depthCompare: "less", - depthWriteEnabled: true, - format: DEPTH_FORMAT, - }, - }); - const sampler = device.createSampler({ - magFilter: "linear", - minFilter: "linear", - }); - const sceneUniformBuffer = device.createBuffer({ - size: SCENE_UBO_SIZE, - usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, - }); - const depthTexture = device.createTexture({ - size: [canvas.width, canvas.height], - format: DEPTH_FORMAT, - usage: GPUTextureUsage.RENDER_ATTACHMENT, - }); - - const sphereMesh = generateSphere(1.0, 48, 64); - - const buildShape = ( - mesh: { vertices: Float32Array; indices: Uint16Array }, - modelAt: (t: number, out: Float32Array) => void, - ): Shape => { - const vertexBuffer = device.createBuffer({ - size: mesh.vertices.byteLength, - usage: GPUBufferUsage.VERTEX | GPUBufferUsage.COPY_DST, - }); - device.queue.writeBuffer(vertexBuffer, 0, mesh.vertices); - // Index buffers must be sized to a multiple of 4 bytes; pad Uint16Array - // by one extra index if needed. - const indexByteLength = (mesh.indices.byteLength + 3) & ~3; - const indexBuffer = device.createBuffer({ - size: indexByteLength, - usage: GPUBufferUsage.INDEX | GPUBufferUsage.COPY_DST, - }); - device.queue.writeBuffer(indexBuffer, 0, mesh.indices); - const uniformBuffer = device.createBuffer({ - size: OBJECT_UBO_SIZE, - usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, - }); - // The full bind group is built per-frame in the worklet because - // GPUExternalTexture is single-use; scene + object buffers above stay - // alive across frames and feed back into that per-frame build. - return { - vertexBuffer, - indexBuffer, - indexCount: mesh.indices.length, - uniformBuffer, - modelAt, - }; - }; - - // Single chrome sphere center stage with a slow Y-rotation so the - // reflection drifts even when the orbit camera is between key positions. - // Runs inside the frame worklet. - const shapes: Shape[] = [ - buildShape(sphereMesh, (t, out) => { - "worklet"; - setIdentity(out); - applyRotateY(out, t * 0.25, out); - }), - ]; - - // ----- Backdrop blur infrastructure (same as VisionCamera "Strong") --- - const blurWidth = Math.max( - BLUR_TILE_DIM, - Math.ceil(canvas.width / BLUR_SCALE), - ); - const blurHeight = Math.max( - BLUR_TILE_DIM, - Math.ceil(canvas.height / BLUR_SCALE), - ); - - const prepassModule = device.createShaderModule({ code: PREPASS_SHADER }); - const prepassPipeline = device.createRenderPipeline({ - layout: "auto", - vertex: { module: prepassModule, entryPoint: "vs_main" }, - fragment: { - module: prepassModule, - entryPoint: "fs_main", - targets: [{ format: "rgba8unorm" }], - }, - primitive: { topology: "triangle-list" }, - }); - const prepassUniformBuffer = device.createBuffer({ - size: 16, - usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, - }); - - const blurSrcTexture = device.createTexture({ - size: [blurWidth, blurHeight], - format: "rgba8unorm", - usage: - GPUTextureUsage.RENDER_ATTACHMENT | GPUTextureUsage.TEXTURE_BINDING, - }); - const blurPing = [0, 1].map(() => - device.createTexture({ - size: [blurWidth, blurHeight], - format: "rgba8unorm", - usage: - GPUTextureUsage.STORAGE_BINDING | GPUTextureUsage.TEXTURE_BINDING, - }), - ); - - const blurPipeline = device.createComputePipeline({ - layout: "auto", - compute: { - module: device.createShaderModule({ code: BLUR_SHADER }), - entryPoint: "main", - }, - }); - - const flip0Buffer = device.createBuffer({ - size: 4, - mappedAtCreation: true, - usage: GPUBufferUsage.UNIFORM, - }); - new Uint32Array(flip0Buffer.getMappedRange())[0] = 0; - flip0Buffer.unmap(); - const flip1Buffer = device.createBuffer({ - size: 4, - mappedAtCreation: true, - usage: GPUBufferUsage.UNIFORM, - }); - new Uint32Array(flip1Buffer.getMappedRange())[0] = 1; - flip1Buffer.unmap(); - - const blurParamsBuffer = device.createBuffer({ - size: 8, - usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, - }); - device.queue.writeBuffer( - blurParamsBuffer, - 0, - new Uint32Array([BLUR_FILTER_SIZE + 1, BLUR_BLOCK_DIM]), - ); - - const blurConstants = device.createBindGroup({ - layout: blurPipeline.getBindGroupLayout(0), - entries: [ - { binding: 0, resource: sampler }, - { binding: 1, resource: { buffer: blurParamsBuffer } }, - ], - }); - // H: blurSrcTexture -> blurPing[0] - const blurBindGroup0 = device.createBindGroup({ - layout: blurPipeline.getBindGroupLayout(1), - entries: [ - { binding: 1, resource: blurSrcTexture.createView() }, - { binding: 2, resource: blurPing[0].createView() }, - { binding: 3, resource: { buffer: flip0Buffer } }, - ], - }); - // V: blurPing[0] -> blurPing[1] - const blurBindGroup1 = device.createBindGroup({ - layout: blurPipeline.getBindGroupLayout(1), - entries: [ - { binding: 1, resource: blurPing[0].createView() }, - { binding: 2, resource: blurPing[1].createView() }, - { binding: 3, resource: { buffer: flip1Buffer } }, - ], - }); - // H (iteration N>=2): blurPing[1] -> blurPing[0] - const blurBindGroup2 = device.createBindGroup({ - layout: blurPipeline.getBindGroupLayout(1), - entries: [ - { binding: 1, resource: blurPing[1].createView() }, - { binding: 2, resource: blurPing[0].createView() }, - { binding: 3, resource: { buffer: flip0Buffer } }, - ], - }); - // Final iteration's V pass always lands in blurPing[1]. - const blurredView = blurPing[1].createView(); - - // Backdrop pipeline: shares the render pass with the chrome sphere, so - // it must declare a matching depth-stencil layout. depthCompare always / - // depthWriteEnabled false means it draws unconditionally and never - // disturbs depth for the subsequent sphere draw. - const backdropModule = device.createShaderModule({ code: BACKDROP_SHADER }); - const backdropPipeline = device.createRenderPipeline({ - layout: "auto", - vertex: { module: backdropModule, entryPoint: "vs_main" }, - fragment: { - module: backdropModule, - entryPoint: "fs_main", - targets: [{ format: presentationFormat }], - }, - primitive: { topology: "triangle-list" }, - depthStencil: { - depthCompare: "always", - depthWriteEnabled: false, - format: DEPTH_FORMAT, - }, - }); - const backdropBindGroup = device.createBindGroup({ - layout: backdropPipeline.getBindGroupLayout(0), - entries: [ - { binding: 0, resource: blurredView }, - { binding: 1, resource: sampler }, - ], - }); - - setPipelineState({ - pipeline, - sampler, - sceneUniformBuffer, - depthView: depthTexture.createView(), - context, - canvasWidth: canvas.width, - canvasHeight: canvas.height, - shapes, - view: new Float32Array(16), - proj: new Float32Array(16), - viewProj: new Float32Array(16), - sceneData: new Float32Array(SCENE_UBO_FLOATS), - modelScratch: new Float32Array(16), - backdropPipeline, - backdropBindGroup, - prepassPipeline, - prepassUniformBuffer, - blurPipeline, - blurConstants, - blurBindGroup0, - blurBindGroup1, - blurBindGroup2, - blurSrcTexture, - blurWidth, - blurHeight, - }); - }, [device, adapter, ref, pipelineState]); - - const startTimeRef = React.useRef(performance.now()); - const logBox = React.useMemo(() => ({ seen: false }), []); - - const frameOutput = useFrameOutput({ - pixelFormat: "native", - onFrame: (frame) => { - "worklet"; - if (!logBox.seen) { - logBox.seen = true; - console.log( - "[ChromeSphere] worklet first frame, hasPipeline=" + - String(pipelineState != null) + - " frame=" + - String(frame.width) + - "x" + - String(frame.height), - ); - } - if (!pipelineState || !device) { - frame.dispose(); - return; - } - const { - pipeline, - sampler, - sceneUniformBuffer, - depthView, - context, - canvasWidth, - canvasHeight, - shapes, - view, - proj, - viewProj, - sceneData, - modelScratch, - backdropPipeline, - backdropBindGroup, - prepassPipeline, - prepassUniformBuffer, - blurPipeline, - blurConstants, - blurBindGroup0, - blurBindGroup1, - blurBindGroup2, - blurSrcTexture, - blurWidth, - blurHeight, - } = pipelineState; - const nativeBuffer = frame.getNativeBuffer(); - try { - const videoFrame = device.createVideoFrameFromNativeBuffer( - nativeBuffer.pointer, - ); - try { - const t = (performance.now() - startTimeRef.current) / 1000; - - // Orbit the eye around the origin, looking at it. Small bob in y - // so reflections shift along the polar axis too. - const orbitR = 5.2; - const ex = Math.cos(t * 0.15) * orbitR; - const ey = 1.2 + Math.sin(t * 0.2) * 0.4; - const ez = Math.sin(t * 0.15) * orbitR; - setLookAt(view, ex, ey, ez, 0.0, 0.0, 0.0, 0, 1, 0); - setPerspective( - proj, - Math.PI / 4, - canvasWidth / canvasHeight, - 0.1, - 100, - ); - multiplyMat4(proj, view, viewProj); - - sceneData.set(viewProj, 0); - sceneData[16] = ex; - sceneData[17] = ey; - sceneData[18] = ez; - sceneData[19] = 0; - // Key light rises and slowly drifts so the chrome specular sweeps - // across the silhouettes. - const ldRawX = Math.cos(t * 0.3) * 0.6; - const ldRawY = 0.8; - const ldRawZ = Math.sin(t * 0.3) * 0.6; - const ldLen = Math.hypot(ldRawX, ldRawY, ldRawZ); - sceneData[20] = ldRawX / ldLen; - sceneData[21] = ldRawY / ldLen; - sceneData[22] = ldRawZ / ldLen; - sceneData[23] = 0; - device.queue.writeBuffer(sceneUniformBuffer, 0, sceneData); - - for (const shape of shapes) { - shape.modelAt(t, modelScratch); - device.queue.writeBuffer(shape.uniformBuffer, 0, modelScratch); - } - - let externalTex; - try { - externalTex = device.importExternalTexture({ - source: videoFrame, - label: "chrome-env", - }); - } catch (e) { - console.warn( - "[ChromeSphere] importExternalTexture threw: " + String(e), - ); - throw e; - } - - const encoder = device.createCommandEncoder(); - - // ---- Backdrop blur (prepass + 3 H-V iterations at 1/4 res) ---- - device.queue.writeBuffer( - prepassUniformBuffer, - 0, - new Float32Array([ - videoFrame.width, - videoFrame.height, - canvasWidth, - canvasHeight, - ]), - ); - const prepassBindGroup = device.createBindGroup({ - layout: prepassPipeline.getBindGroupLayout(0), - entries: [ - { binding: 0, resource: externalTex }, - { binding: 1, resource: sampler }, - { binding: 2, resource: { buffer: prepassUniformBuffer } }, - ], - }); - const prepass = encoder.beginRenderPass({ - colorAttachments: [ - { - view: blurSrcTexture.createView(), - clearValue: { r: 0, g: 0, b: 0, a: 1 }, - loadOp: "clear", - storeOp: "store", - }, - ], - }); - prepass.setPipeline(prepassPipeline); - prepass.setBindGroup(0, prepassBindGroup); - prepass.draw(3); - prepass.end(); - - const compute = encoder.beginComputePass(); - compute.setPipeline(blurPipeline); - compute.setBindGroup(0, blurConstants); - compute.setBindGroup(1, blurBindGroup0); - compute.dispatchWorkgroups( - Math.ceil(blurWidth / BLUR_BLOCK_DIM), - Math.ceil(blurHeight / BLUR_BATCH), - ); - compute.setBindGroup(1, blurBindGroup1); - compute.dispatchWorkgroups( - Math.ceil(blurHeight / BLUR_BLOCK_DIM), - Math.ceil(blurWidth / BLUR_BATCH), - ); - for (let i = 0; i < BLUR_ITERATIONS - 1; i++) { - compute.setBindGroup(1, blurBindGroup2); - compute.dispatchWorkgroups( - Math.ceil(blurWidth / BLUR_BLOCK_DIM), - Math.ceil(blurHeight / BLUR_BATCH), - ); - compute.setBindGroup(1, blurBindGroup1); - compute.dispatchWorkgroups( - Math.ceil(blurHeight / BLUR_BLOCK_DIM), - Math.ceil(blurWidth / BLUR_BATCH), - ); - } - compute.end(); - - // ---- Main scene pass: backdrop first (no depth write), then sphere - const pass = encoder.beginRenderPass({ - colorAttachments: [ - { - view: context.getCurrentTexture().createView(), - clearValue: { r: 0, g: 0, b: 0, a: 1 }, - loadOp: "clear", - storeOp: "store", - }, - ], - depthStencilAttachment: { - view: depthView, - depthClearValue: 1.0, - depthLoadOp: "clear", - depthStoreOp: "store", - }, - }); - - // Backdrop pipeline has depthCompare: "always" and writes nothing - // to depth, so the subsequent sphere draw still sees a clean depth - // buffer with the canvas-clear far value. - pass.setPipeline(backdropPipeline); - pass.setBindGroup(0, backdropBindGroup); - pass.draw(3); - - pass.setPipeline(pipeline); - for (const shape of shapes) { - // The external texture is bound per-shape (one bind group per - // shape, the only difference being the per-object uniform), so - // we rebuild the bind group each frame to splice in this frame's - // externalTex alongside the cached scene + object buffers. - const bindGroup = device.createBindGroup({ - layout: pipeline.getBindGroupLayout(0), - entries: [ - { binding: 0, resource: { buffer: sceneUniformBuffer } }, - { binding: 1, resource: { buffer: shape.uniformBuffer } }, - { binding: 2, resource: externalTex }, - { binding: 3, resource: sampler }, - ], - }); - pass.setBindGroup(0, bindGroup); - pass.setVertexBuffer(0, shape.vertexBuffer); - pass.setIndexBuffer(shape.indexBuffer, "uint16"); - pass.drawIndexed(shape.indexCount); - } - pass.end(); - device.queue.submit([encoder.finish()]); - context.present(); - } finally { - videoFrame.release(); - } - } finally { - nativeBuffer.release(); - frame.dispose(); - } - }, - }); - - useCamera({ - isActive: pipelineState != null && cameraDevice != null, - device: cameraDevice as NonNullable, - outputs: [frameOutput], - }); - - if (deviceError) { - return ( - - - Device creation failed: {deviceError} - - - ); - } - if (error) { - return ( - - {error} - - ); - } - if (!device) { - return ( - - Waiting for GPU device... - - ); - } - if (cameraDevice == null) { - return ( - - - No camera available. This screen needs a physical device with a camera - (the iOS Simulator does not have one). - - - ); - } - return ( - - - - ); -}; - -const styles = StyleSheet.create({ - root: { flex: 1, backgroundColor: "black" }, - canvas: { flex: 1 }, - errorContainer: { flex: 1, padding: 16, justifyContent: "center" }, - errorText: { color: "red", fontSize: 14 }, - permissionContainer: { - flex: 1, - padding: 24, - justifyContent: "center", - alignItems: "center", - }, - permissionText: { fontSize: 16, textAlign: "center", marginBottom: 16 }, - permissionButton: { - backgroundColor: "#007AFF", - paddingHorizontal: 24, - paddingVertical: 12, - borderRadius: 8, - }, - permissionButtonText: { color: "white", fontSize: 16, fontWeight: "600" }, -}); diff --git a/apps/example/src/ChromeSphere/geometry.ts b/apps/example/src/ChromeSphere/geometry.ts deleted file mode 100644 index 1ede29fcd..000000000 --- a/apps/example/src/ChromeSphere/geometry.ts +++ /dev/null @@ -1,50 +0,0 @@ -// Sphere mesh generator for the chrome scene. Returns interleaved -// position+normal vertices and a triangle index list. The shader uses the -// upper-left 3x3 of the model matrix to transform normals, so the sphere is -// built with unit-length normals (no non-uniform scale at generation time). - -export type Mesh = { - vertices: Float32Array; // [px, py, pz, nx, ny, nz] * N - indices: Uint16Array; -}; - -const FLOATS_PER_VERTEX = 6; - -export const VERTEX_STRIDE_BYTES = FLOATS_PER_VERTEX * 4; -export const POSITION_OFFSET = 0; -export const NORMAL_OFFSET = 12; - -export function generateSphere( - radius: number, - latBands: number, - lonBands: number, -): Mesh { - const verts: number[] = []; - const idx: number[] = []; - for (let lat = 0; lat <= latBands; lat++) { - const theta = (lat * Math.PI) / latBands; - const sinTheta = Math.sin(theta); - const cosTheta = Math.cos(theta); - for (let lon = 0; lon <= lonBands; lon++) { - const phi = (lon * 2 * Math.PI) / lonBands; - const sinPhi = Math.sin(phi); - const cosPhi = Math.cos(phi); - const nx = cosPhi * sinTheta; - const ny = cosTheta; - const nz = sinPhi * sinTheta; - verts.push(nx * radius, ny * radius, nz * radius, nx, ny, nz); - } - } - const stride = lonBands + 1; - for (let lat = 0; lat < latBands; lat++) { - for (let lon = 0; lon < lonBands; lon++) { - const a = lat * stride + lon; - const b = a + stride; - idx.push(a, b, a + 1, b, b + 1, a + 1); - } - } - return { - vertices: new Float32Array(verts), - indices: new Uint16Array(idx), - }; -} diff --git a/apps/example/src/ChromeSphere/index.ts b/apps/example/src/ChromeSphere/index.ts deleted file mode 100644 index 5f32bab2e..000000000 --- a/apps/example/src/ChromeSphere/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "./ChromeSphere"; diff --git a/apps/example/src/ChromeSphere/shader.ts b/apps/example/src/ChromeSphere/shader.ts deleted file mode 100644 index 425606aaf..000000000 --- a/apps/example/src/ChromeSphere/shader.ts +++ /dev/null @@ -1,137 +0,0 @@ -// Backdrop shader. Fullscreen triangle that samples the pre-blurred camera -// image (cover-fit baked in by the prepass), dims it ~50%, and adds a soft -// vignette so the chrome sphere stays the focal point of the scene. -export const BACKDROP_SHADER = /* wgsl */ ` -@group(0) @binding(0) var src: texture_2d; -@group(0) @binding(1) var samp: sampler; - -struct VsOut { - @builtin(position) position: vec4f, - @location(0) uv: vec2f, -}; - -@vertex -fn vs_main(@builtin(vertex_index) vid: u32) -> VsOut { - var positions = array( - vec2f(-1.0, -3.0), - vec2f(-1.0, 1.0), - vec2f( 3.0, 1.0), - ); - var uvs = array( - vec2f(0.0, 2.0), - vec2f(0.0, 0.0), - vec2f(2.0, 0.0), - ); - var out: VsOut; - out.position = vec4f(positions[vid], 0.0, 1.0); - out.uv = uvs[vid]; - return out; -} - -@fragment -fn fs_main(in: VsOut) -> @location(0) vec4f { - var c = textureSampleLevel(src, samp, in.uv, 0.0).rgb; - // Subdue the room: half-brightness + soft vignette darkening the edges by - // another ~40%. Keeps a sense of ambient lighting without competing with - // the chrome reflection. - c = c * 0.55; - let d = distance(in.uv, vec2f(0.5)); - let v = 1.0 - smoothstep(0.4, 0.95, d) * 0.45; - return vec4f(c * v, 1.0); -} -`; - -// Chrome-style env-map reflection driven by the live camera feed. -// -// For each fragment we compute the reflection vector around the world-space -// normal, convert it to spherical (theta, phi) → uv, and sample the external -// camera texture. textureSampleBaseClampToEdge is the only sampling call -// allowed on texture_external, but it's all we need: spherical uvs land -// inside [0, 1]^2 so no wrap mode tricks are required. -// -// The visible result is the classic CGI chrome ball: your face wraps around -// the sphere as if the camera image were the ambient environment, and the -// cube + torus reflect distorted copies of the same scene. - -export const SHADER = /* wgsl */ ` -struct Scene { - viewProj: mat4x4f, - cameraPos: vec4f, // xyz = world-space eye, w unused - lightDir: vec4f, // xyz = unit vector toward light, w unused -}; - -struct Object { - model: mat4x4f, -}; - -@group(0) @binding(0) var scene: Scene; -@group(0) @binding(1) var obj: Object; -@group(0) @binding(2) var srcTex: texture_external; -@group(0) @binding(3) var srcSampler: sampler; - -struct VsIn { - @location(0) position: vec3f, - @location(1) normal: vec3f, -}; - -struct VsOut { - @builtin(position) clipPos: vec4f, - @location(0) worldPos: vec3f, - @location(1) worldNormal: vec3f, -}; - -@vertex -fn vs_main(in: VsIn) -> VsOut { - let world = obj.model * vec4f(in.position, 1.0); - // Rotation-only transforms: normal matrix is the upper-left 3x3 of model. - // No non-uniform scale anywhere in the scene, so inverse-transpose collapses - // to this. - let normalMat = mat3x3f( - obj.model[0].xyz, - obj.model[1].xyz, - obj.model[2].xyz, - ); - var out: VsOut; - out.clipPos = scene.viewProj * world; - out.worldPos = world.xyz; - out.worldNormal = normalize(normalMat * in.normal); - return out; -} - -const PI: f32 = 3.14159265359; - -fn sphericalUv(dir: vec3f) -> vec2f { - // dir is treated as a direction from the chrome surface to the - // environment. theta wraps around the y-axis, phi runs pole to pole. - let theta = atan2(dir.z, dir.x); - let phi = acos(clamp(dir.y, -1.0, 1.0)); - return vec2f((theta + PI) / (2.0 * PI), phi / PI); -} - -@fragment -fn fs_main(in: VsOut) -> @location(0) vec4f { - let n = normalize(in.worldNormal); - let v = normalize(scene.cameraPos.xyz - in.worldPos); - // reflect() expects the incident vector (from light/source to surface), so - // negate the view direction. - let r = normalize(reflect(-v, n)); - - let uv = sphericalUv(r); - let env = textureSampleBaseClampToEdge(srcTex, srcSampler, uv).rgb; - - // Schlick-ish Fresnel: chrome reflectance is high everywhere, but boost a - // bit at grazing angles so silhouettes catch the light. - let cosTheta = max(dot(n, v), 0.0); - let fresnel = pow(1.0 - cosTheta, 3.0); - let tint = vec3f(0.96, 0.97, 1.00); // subtle cool cast, very polished chrome - var color = env * tint + vec3f(fresnel) * 0.18; - - // Single directional specular highlight so the chrome doesn't look like a - // pure billboard. Halfway vector with a tight exponent. - let h = normalize(scene.lightDir.xyz + v); - let spec = pow(max(dot(n, h), 0.0), 96.0); - color = color + vec3f(spec) * 1.2; - - return vec4f(color, 1.0); -} -`; diff --git a/apps/example/src/ExternalTexture/ExternalTexture.tsx b/apps/example/src/ExternalTexture/ExternalTexture.tsx deleted file mode 100644 index 2b2504b4c..000000000 --- a/apps/example/src/ExternalTexture/ExternalTexture.tsx +++ /dev/null @@ -1,251 +0,0 @@ -import React, { useEffect, useRef, useState } from "react"; -import { PixelRatio, StyleSheet, Text, View } from "react-native"; -import { - Canvas, - useCanvasRef, - useDevice, - type NativeCanvas, - type NativeVideoFrame, -} from "react-native-wgpu"; - -// importExternalTexture is the spec-mandated path for "I have a YUV-encoded -// video/camera frame and I want to sample it in a shader without copying or -// hand-rolling YUV math". The WGSL side uses texture_external + -// textureSampleBaseClampToEdge; the driver does the planar fetch, YUV→RGB -// matrix multiply, sRGB transfer, and gamut conversion in the sampler. -// -// Bind groups for texture_external use auto layout slots like any other -// resource. WGSL doesn't expose the underlying plane textures directly. -const SHADER = /* wgsl */ ` -struct VsOut { - @builtin(position) position: vec4f, - @location(0) uv: vec2f, -}; - -struct Uniforms { - uvScale: vec2f, -}; - -@group(0) @binding(0) var srcTex: texture_external; -@group(0) @binding(1) var srcSampler: sampler; -@group(0) @binding(2) var u: Uniforms; - -@vertex -fn vs_main(@builtin(vertex_index) vid: u32) -> VsOut { - var positions = array( - vec2f(-1.0, -3.0), - vec2f(-1.0, 1.0), - vec2f( 3.0, 1.0), - ); - var uvs = array( - vec2f(0.0, 2.0), - vec2f(0.0, 0.0), - vec2f(2.0, 0.0), - ); - var out: VsOut; - out.position = vec4f(positions[vid], 0.0, 1.0); - out.uv = uvs[vid]; - return out; -} - -@fragment -fn fs_main(in: VsOut) -> @location(0) vec4f { - let uv = vec2f(0.5) + (in.uv - vec2f(0.5)) * u.uvScale; - return textureSampleBaseClampToEdge(srcTex, srcSampler, uv); -} -`; - -// rnwebgpu/shared-texture-memory is our umbrella that expands to the -// platform's shared-memory + shared-fence pair (the IOSurface / AHB still -// flows through SharedTextureMemory under the hood). Plus -// dawn-multi-planar-formats so Dawn can interpret the NV12 surface as a -// biplanar texture. -const REQUIRED_FEATURES: GPUFeatureName[] = [ - "rnwebgpu/shared-texture-memory" as GPUFeatureName, - "dawn-multi-planar-formats" as GPUFeatureName, -]; - -const VIDEO_URL = - "https://test-videos.co.uk/vids/bigbuckbunny/mp4/h264/1080/Big_Buck_Bunny_1080_10s_5MB.mp4"; - -export const ExternalTexture = () => { - const ref = useCanvasRef(); - const [error, setError] = useState(null); - const rafRef = useRef(null); - - const { device, adapter } = useDevice(undefined, { - requiredFeatures: REQUIRED_FEATURES, - }); - - useEffect(() => { - if (!device) { - return; - } - const missing = REQUIRED_FEATURES.filter((f) => !device.features.has(f)); - if (missing.length > 0) { - setError( - `Device is missing required features [${missing.join(", ")}]. Adapter supports: ${ - adapter - ? [...adapter.features] - .filter((f) => f.toString().startsWith("shared-")) - .join(", ") || "none" - : "n/a" - }`, - ); - return; - } - - const context = ref.current?.getContext("webgpu"); - if (!context) { - return; - } - const canvas = context.canvas as unknown as NativeCanvas; - canvas.width = canvas.clientWidth * PixelRatio.get(); - canvas.height = canvas.clientHeight * PixelRatio.get(); - const presentationFormat = navigator.gpu.getPreferredCanvasFormat(); - context.configure({ - device, - format: presentationFormat, - alphaMode: "premultiplied", - }); - - // Same Big Buck Bunny URL as the SharedTextureMemory demo, but ask AVPlayer - // for native NV12 instead of BGRA. Each VideoFrame now carries the YUV - // matrix + plane info that importExternalTexture needs. - const player = RNWebGPU.createVideoPlayer(VIDEO_URL, "nv12"); - player.play(); - - const module = device.createShaderModule({ code: SHADER }); - const pipeline = device.createRenderPipeline({ - layout: "auto", - vertex: { module, entryPoint: "vs_main" }, - fragment: { - module, - entryPoint: "fs_main", - targets: [{ format: presentationFormat }], - }, - primitive: { topology: "triangle-list" }, - }); - const sampler = device.createSampler({ - magFilter: "linear", - minFilter: "linear", - }); - - // One persistent uniform buffer; we rewrite its uvScale whenever a new - // frame's dimensions differ from the last one. - const uniformBuffer = device.createBuffer({ - size: 16, - usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, - }); - let lastUvScale: [number, number] | null = null; - const writeUvScale = (texW: number, texH: number) => { - const canvasAR = canvas.width / canvas.height; - const texAR = texW / texH; - const next: [number, number] = - texAR > canvasAR ? [canvasAR / texAR, 1] : [1, texAR / canvasAR]; - if ( - !lastUvScale || - lastUvScale[0] !== next[0] || - lastUvScale[1] !== next[1] - ) { - device.queue.writeBuffer(uniformBuffer, 0, new Float32Array(next)); - lastUvScale = next; - } - }; - - // The video plays at ~24fps but we tick at the display's 60Hz, so most rAF - // ticks have no new frame from AVPlayer. Hold the latest VideoFrame across - // ticks and re-import an ExternalTexture from it on the "no new frame" - // ticks — this is what stops the canvas from flashing black ~2/3 of the - // time. AVPlayer's pool is several buffers deep so holding one back like - // this doesn't stall decoding. - let currentFrame: NativeVideoFrame | null = null; - - const render = () => { - const newFrame = player.copyLatestFrame(); - if (newFrame) { - if (currentFrame) { - currentFrame.release(); - } - currentFrame = newFrame; - } - - const encoder = device.createCommandEncoder(); - const pass = encoder.beginRenderPass({ - colorAttachments: [ - { - view: context.getCurrentTexture().createView(), - clearValue: { r: 0, g: 0, b: 0, a: 1 }, - loadOp: "clear", - storeOp: "store", - }, - ], - }); - - if (currentFrame) { - // GPUExternalTexture expires after each submit, so we rebuild one - // every tick — even when sampling the same VideoFrame as last tick. - let externalTex: GPUExternalTexture | null = null; - try { - externalTex = device.importExternalTexture({ - source: currentFrame, - label: "video-external", - }); - } catch (e) { - console.warn("[ExternalTexture] importExternalTexture failed:", e); - } - - if (externalTex) { - writeUvScale(currentFrame.width, currentFrame.height); - const bindGroup = device.createBindGroup({ - layout: pipeline.getBindGroupLayout(0), - entries: [ - { binding: 0, resource: externalTex }, - { binding: 1, resource: sampler }, - { binding: 2, resource: { buffer: uniformBuffer } }, - ], - }); - pass.setPipeline(pipeline); - pass.setBindGroup(0, bindGroup); - pass.draw(3); - } - } - - pass.end(); - device.queue.submit([encoder.finish()]); - context.present(); - rafRef.current = requestAnimationFrame(render); - }; - rafRef.current = requestAnimationFrame(render); - - return () => { - if (rafRef.current !== null) { - cancelAnimationFrame(rafRef.current); - } - if (currentFrame) { - currentFrame.release(); - currentFrame = null; - } - uniformBuffer.destroy(); - player.release(); - }; - }, [device, adapter, ref]); - - if (error) { - return ( - - {error} - - ); - } - return ( - - - - ); -}; - -const styles = StyleSheet.create({ - errorContainer: { flex: 1, padding: 16, justifyContent: "center" }, - errorText: { color: "red", fontSize: 14 }, -}); diff --git a/apps/example/src/ExternalTexture/index.ts b/apps/example/src/ExternalTexture/index.ts deleted file mode 100644 index d019021b8..000000000 --- a/apps/example/src/ExternalTexture/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "./ExternalTexture"; diff --git a/apps/example/src/Home.tsx b/apps/example/src/Home.tsx index d43f0822e..c1321105f 100644 --- a/apps/example/src/Home.tsx +++ b/apps/example/src/Home.tsx @@ -131,18 +131,10 @@ export const examples = [ screen: "SharedTextureMemory", title: "🎞️ Shared Texture Memory", }, - { - screen: "ExternalTexture", - title: "🟨 External Texture (YUV)", - }, { screen: "VisionCamera", title: "đź“· VisionCamera integration", }, - { - screen: "ChromeSphere", - title: "🪩 Chrome Sphere (camera env map)", - }, ]; const styles = StyleSheet.create({ diff --git a/apps/example/src/Route.ts b/apps/example/src/Route.ts index 57f09c992..d3068a5d2 100644 --- a/apps/example/src/Route.ts +++ b/apps/example/src/Route.ts @@ -30,7 +30,5 @@ export type Routes = { DeviceLostHang: undefined; StorageBufferVertices: undefined; SharedTextureMemory: undefined; - ExternalTexture: undefined; VisionCamera: undefined; - ChromeSphere: undefined; }; diff --git a/apps/example/src/ThreeJS/CameraHelmet.tsx b/apps/example/src/ThreeJS/CameraHelmet.tsx deleted file mode 100644 index 0d48d0c47..000000000 --- a/apps/example/src/ThreeJS/CameraHelmet.tsx +++ /dev/null @@ -1,711 +0,0 @@ -import React, { useEffect, useRef, useState } from "react"; -import { - Linking, - Platform, - StyleSheet, - Text, - TouchableOpacity, - View, -} from "react-native"; -import type { CanvasRef } from "react-native-wgpu"; -import { Canvas } from "react-native-wgpu"; -import { - CommonResolutions, - NativePreviewView, - useCameraDevices, - useCameraPermission, - useFrameOutput, - usePreviewOutput, - VisionCamera as VisionCameraFactory, -} from "react-native-vision-camera"; -import type { - CameraController, - CameraSession, -} from "react-native-vision-camera"; -import * as THREE from "three"; - -import { useGLTF } from "./assets/AssetManager"; -import { makeWebGPURenderer } from "./components/makeWebGPURenderer"; -import { CAMERA_ENV_SHADER } from "./cameraEnvShader"; - -// Live camera as a three.js environment map. The GLTF helmet renders with -// three.js' WebGPURenderer; its env map is a THREE.ExternalTexture wrapping a -// GPUTexture we own. A Vision Camera frame-processor worklet writes each -// camera frame into that GPUTexture via its own render pass. Three.js and the -// worklet share a single GPUDevice (the one three.js creates internally), so -// the queue ordering between "write env" and "sample env" is automatic. -// -// The front-camera frame is wrapped around the inside of a large back-side -// sphere centered on the helmet, so the CubeCamera at the helmet's middle -// samples camera content in every direction. The helmet's reflections -// therefore fully cover the helmet and never reveal the frame's edges. The -// trade-off is the obvious "face wrapped around the world" look on grazing -// reflections; that's chosen deliberately, since full coverage matters more -// than the seam artifact for this demo. - -// 9:16 portrait — front cam delivers 16:9 landscape and the shader rotates it -// 90° to selfie-upright, so this aspect lets the rotated frame fill the env -// texture with no stretching. Cube face dimension matches ENV_HEIGHT. Each -// frame we do (env write + 6 cube faces + optional mipmap chain), so this -// knob drives most of the per-frame GPU cost. -const ENV_WIDTH = 540; -const ENV_HEIGHT = 960; - -// Cube face size is mode-dependent. Chrome is a pure mirror reflection, so it -// wants the env's full resolution. PBR samples roughness-selected mips (the -// helmet's metal-roughness keeps most surfaces in the 0.3-0.6 range, i.e. -// mip 2-3), so a 128 cube + its short mip chain is visually indistinguishable -// from 512 while cutting cubemap fill rate ~16x and mip-regen cost with it. -const CUBE_SIZE_PBR = 128; -const CUBE_SIZE_CHROME = 512; - -// PBR mode is runtime-toggleable from a button in the JSX below. PBR uses -// the GLTF's MeshStandardMaterial textures (albedo / normal / metalRoughness -// / AO) plus a cubemap envMap for the live reflection; "chrome" mode swaps -// every mesh for a single MeshBasicMaterial that just samples the cubemap. -// Chrome is roughly 5-10x cheaper on fragments and also skips the per-frame -// cubemap mipmap regen the PBR roughness lookup needs. - -// Vision Camera + react-native-wgpu both want these features for the external -// texture path. dawn-multi-planar-formats lets Dawn interpret NV12 buffers. -const REQUIRED_FEATURES: GPUFeatureName[] = [ - "rnwebgpu/shared-texture-memory" as GPUFeatureName, - "dawn-multi-planar-formats" as GPUFeatureName, -]; - -const OPAQUE_YCBCR_EXT = - "opaque-ycbcr-android-for-external-texture" as GPUFeatureName; - -export const CameraHelmet = () => { - const { hasPermission, requestPermission } = useCameraPermission(); - useEffect(() => { - if (!hasPermission) { - requestPermission(); - } - }, [hasPermission, requestPermission]); - - if (!hasPermission) { - return ( - - - Camera access is required. Grant it in Settings or tap below. - - Linking.openSettings()} - style={styles.permissionButton} - > - Open Settings - - - ); - } - return ; -}; - -const Scene = () => { - useEffect(() => { - console.log("[CameraHelmet] Scene mounted"); - return () => console.log("[CameraHelmet] Scene unmounted"); - }, []); - const ref = useRef(null); - const gltf = useGLTF(require("./assets/helmet/DamagedHelmet.gltf")); - // Live camera preview, rendered as a native view behind the WebGPU canvas. - // The worklet still writes the camera into our env texture for the helmet - // reflection, but the *backdrop* now comes straight from this native - // preview — no detour through equirect/cubemap, no quality loss. - const previewOutput = usePreviewOutput(); - - // Two cameras at once: the back camera feeds the native preview backdrop, - // the front camera feeds the helmet's environment map (so you see yourself - // reflected in the chrome). Requires multi-cam capable hardware (iPhone - // XS+ / most modern Android flagships). - const devices = useCameraDevices(); - const backDevice = React.useMemo( - () => devices.find((d) => d.position === "back"), - [devices], - ); - const frontDevice = React.useMemo( - () => devices.find((d) => d.position === "front"), - [devices], - ); - - const [pipelineState, setPipelineState] = useState<{ - device: GPUDevice; - cameraPipeline: GPURenderPipeline; - cameraSampler: GPUSampler; - envTexture: GPUTexture; - envTextureView: GPUTextureView; - } | null>(null); - const [error, setError] = useState(null); - const [device, setDevice] = useState(null); - - // PBR is the default. Toggling to chrome swaps every helmet material for - // a single MeshBasicMaterial that just samples the cubemap. usePBRRef - // shadows the state so the (one-shot) setup effect can read the *current* - // value when it first applies materials, even if the user has already - // toggled before three.js finished initializing. - const [usePBR, setUsePBR] = useState(true); - const usePBRRef = useRef(true); - const applyPBRFnRef = useRef<((pbr: boolean) => void) | null>(null); - const togglePBR = () => { - const next = !usePBRRef.current; - usePBRRef.current = next; - setUsePBR(next); - applyPBRFnRef.current?.(next); - }; - - // Acquire the GPU device on its own effect. By the time the async adapter + - // device requests resolve, the Canvas component has been rendered and its - // ref populated, so the main setup effect (gated on `device`) can grab the - // GPUCanvasContext synchronously. Same two-effect pattern as - // VisionCamera.tsx / ChromeSphere.tsx. - useEffect(() => { - console.log("[CameraHelmet] device-acquisition effect fired"); - let cancelled = false; - (async () => { - try { - const adapter = await navigator.gpu.requestAdapter(); - if (!adapter) { - throw new Error("requestAdapter returned null"); - } - const requiredFeatures = [...adapter.features] as GPUFeatureName[]; - const missing = REQUIRED_FEATURES.filter( - (f) => !adapter.features.has(f), - ); - const needsAndroidExt = - Platform.OS === "android" && !adapter.features.has(OPAQUE_YCBCR_EXT); - if (missing.length > 0 || needsAndroidExt) { - throw new Error( - "Adapter doesn't advertise the features the Vision Camera " + - "external-texture path needs: " + - `${[...missing, needsAndroidExt ? OPAQUE_YCBCR_EXT : null] - .filter(Boolean) - .join(", ")}.`, - ); - } - const d = await adapter.requestDevice({ requiredFeatures }); - console.log( - "[CameraHelmet] device acquired, features: " + - [...d.features].sort().join(", "), - ); - if (cancelled) { - d.destroy(); - return; - } - setDevice(d); - } catch (e) { - if (cancelled) { - return; - } - console.warn("[CameraHelmet] device acquisition failed: " + String(e)); - setError(String(e)); - } - })(); - return () => { - cancelled = true; - }; - }, []); - - // Note: pipelineState is intentionally not in the deps array. Including it - // would re-run the effect when we call setPipelineState below — React would - // run the cleanup (which calls setAnimationLoop(null)) and then the effect - // would bail on the pipelineState guard, leaving us with no render loop. - // The effect only needs to fire once, when `device` transitions to set. - - useEffect(() => { - console.log( - "[CameraHelmet] setup effect fired, device=" + - String(device != null) + - " gltf=" + - String(gltf != null), - ); - if (!device || !gltf) { - return; - } - const context = ref.current?.getContext("webgpu"); - if (!context) { - console.log( - "[CameraHelmet] no webgpu context yet (ref.current=" + - String(ref.current != null) + - ") — bailing this effect run", - ); - return; - } - let cancelled = false; - let renderer: THREE.WebGPURenderer | null = null; - - console.log("[CameraHelmet] context acquired, building three.js scene"); - (async () => { - try { - const { width, height } = context.canvas; - console.log( - "[CameraHelmet] canvas size = " + - String(width) + - "x" + - String(height), - ); - - // alpha:true configures the canvas with premultiplied alpha mode, so - // pixels outside the helmet stay transparent and the native camera - // preview behind the canvas shows through. - renderer = makeWebGPURenderer(context, { device, alpha: true }); - renderer.toneMapping = THREE.ACESFilmicToneMapping; - renderer.setClearColor(0x000000, 0); - await renderer.init(); - console.log("[CameraHelmet] three.js renderer init complete"); - if (cancelled) { - return; - } - - // Env GPUTexture: render target on our side, sampleable on three's - // side. rgba8unorm + RENDER_ATTACHMENT|TEXTURE_BINDING lets the - // single resource pivot between the two roles via implicit barriers. - const envTexture = device.createTexture({ - size: [ENV_WIDTH, ENV_HEIGHT], - format: "rgba8unorm", - usage: - GPUTextureUsage.RENDER_ATTACHMENT | GPUTextureUsage.TEXTURE_BINDING, - }); - - // Camera prepass pipeline. Output format matches the env texture so - // it can be the render target. - const module = device.createShaderModule({ code: CAMERA_ENV_SHADER }); - const cameraPipeline = device.createRenderPipeline({ - layout: "auto", - vertex: { module, entryPoint: "vs_main" }, - fragment: { - module, - entryPoint: "fs_main", - targets: [{ format: "rgba8unorm" }], - }, - primitive: { topology: "triangle-list" }, - }); - const cameraSampler = device.createSampler({ - magFilter: "linear", - minFilter: "linear", - }); - - // THREE.ExternalTexture bridges our GPUTexture into three.js as a - // sampleable 2D texture. Used below as the .map of a billboarded - // plane that represents the viewer's screen inside the env layer. - const envExternalTexture = new THREE.ExternalTexture(envTexture); - envExternalTexture.colorSpace = THREE.SRGBColorSpace; - (envExternalTexture as unknown as { image: unknown }).image = { - width: ENV_WIDTH, - height: ENV_HEIGHT, - }; - envExternalTexture.needsUpdate = true; - - // The cubemap is rendered each frame by THREE.CubeCamera (six - // perspective passes into the six faces) instead of being blitted - // from an equirect. CubeCamera renders the actual scene, so the - // reflection picks up any other geometry we add — not just the sky - // sphere that carries the camera feed. - // Two cube targets, one per mode. We swap which one CubeCamera writes - // into on toggle (resizing a single target via setSize doesn't fully - // reallocate cleanly on the WebGPU path, the reflection comes back - // blurry). Only the active one gets updated each frame so the cost - // stays paid for one mode at a time. - const makeCubeRT = (size: number) => { - const rt = new THREE.CubeRenderTarget(size); - rt.texture.mapping = THREE.CubeReflectionMapping; - rt.texture.colorSpace = THREE.SRGBColorSpace; - // Mipmaps only matter for PBR (roughness-aware sample). Chrome - // samples mip 0 only, so we skip the regen cost on the 512 target. - const wantsMips = size <= CUBE_SIZE_PBR; - rt.texture.generateMipmaps = wantsMips; - rt.texture.minFilter = wantsMips - ? THREE.LinearMipmapLinearFilter - : THREE.LinearFilter; - rt.texture.magFilter = THREE.LinearFilter; - return rt; - }; - const cubeRTPbr = makeCubeRT(CUBE_SIZE_PBR); - const cubeRTChrome = makeCubeRT(CUBE_SIZE_CHROME); - const activeCubeRT = () => - usePBRRef.current ? cubeRTPbr : cubeRTChrome; - - const scene = new THREE.Scene(); - // No scene.background — the canvas is alpha-cleared and the native - // camera preview View sits behind it (see JSX below). - - // Layer split: main camera sees layer 0 (helmet only) so the native - // preview View remains visible everywhere else; CubeCamera sees - // layer 1 (the reflection screen + gradient backdrop) so the helmet - // never reflects itself. - const ENV_LAYER = 1; - - // Live camera frame wrapped around the inside of a large back-side - // sphere. The CubeCamera at the helmet's center sees the camera in - // every direction, so reflections fully cover the helmet without - // any visible frame edges. - const envSphere = new THREE.Mesh( - new THREE.SphereGeometry(50, 64, 32), - new THREE.MeshBasicMaterial({ - map: envExternalTexture, - side: THREE.BackSide, - toneMapped: false, - }), - ); - envSphere.layers.set(ENV_LAYER); - scene.add(envSphere); - - const cubeCamera = new THREE.CubeCamera(0.1, 100, activeCubeRT()); - cubeCamera.layers.set(ENV_LAYER); - scene.add(cubeCamera); - - // PBR path: keep the GLTF's MeshStandardMaterial intact (albedo / - // normal / metalRoughness / AO from the original textures) and plug - // our live cubemap into each material's envMap. We capture the - // originals so the chrome toggle can swap back and forth without - // losing them. - const pbrMaterials = new Map< - THREE.Mesh, - THREE.Material | THREE.Material[] - >(); - gltf.scene.traverse((child) => { - const mesh = child as THREE.Mesh; - if (!mesh.isMesh) { - return; - } - pbrMaterials.set(mesh, mesh.material); - const mats = Array.isArray(mesh.material) - ? mesh.material - : [mesh.material]; - for (const m of mats) { - const std = m as THREE.MeshStandardMaterial; - std.envMap = cubeRTPbr.texture; - std.envMapIntensity = 1.0; - std.needsUpdate = true; - } - }); - - // Chrome path: every mesh shares a single MeshBasicMaterial that - // just samples the cubemap. No surface detail, but ~5-10x cheaper - // fragment cost, a useful A/B against PBR on the same scene. - // MeshBasicMaterial samples the envMap at full intensity (no Fresnel - // / roughness attenuation like PBR), then ACES tone-maps the result. - // Bright camera regions land in ACES's compressed range and read as - // washed-out. Dimming the base color multiplies the env sample down - // before tone mapping; ~70% matches a real chrome surface's - // reflectance and brings the brightness back in line with PBR. - const chromeMaterial = new THREE.MeshBasicMaterial({ - envMap: cubeRTChrome.texture, - color: 0xb0b0b0, - }); - - const applyPBR = (pbr: boolean) => { - ( - cubeCamera as unknown as { renderTarget: THREE.CubeRenderTarget } - ).renderTarget = pbr ? cubeRTPbr : cubeRTChrome; - for (const [mesh, original] of pbrMaterials) { - mesh.material = pbr ? original : chromeMaterial; - } - }; - applyPBR(usePBRRef.current); - applyPBRFnRef.current = applyPBR; - - scene.add(gltf.scene); - - // Drive the perspective from min(width, height) so the helmet keeps - // a consistent on-screen size in both orientations. three.js' - // PerspectiveCamera takes a *vertical* FOV; on portrait canvases we - // derive vFov from a fixed horizontal FOV so the wider dimension - // never under-frames the helmet. - const aspect = width / height; - const baseFov = 45; - let vFov = baseFov; - if (aspect < 1) { - const hFovRad = (baseFov * Math.PI) / 180; - const vFovRad = 2 * Math.atan(Math.tan(hFovRad / 2) / aspect); - vFov = (vFovRad * 180) / Math.PI; - } - const camera = new THREE.PerspectiveCamera(vFov, aspect, 0.25, 20); - camera.position.set(0, 0, 3); - - const clock = new THREE.Clock(); - const distance = 3; - let frameCount = 0; - const animate = () => { - // Slow time-based orbit around the helmet, matching the three.js - // env-map reference demo. - const elapsed = clock.getElapsedTime(); - camera.position.x = Math.sin(elapsed * 0.4) * distance; - camera.position.z = Math.cos(elapsed * 0.4) * distance; - camera.position.y = 0; - camera.lookAt(0, 0, 0); - - // Refresh the cubemap by rendering the env-layer (camera sphere) - // from six perspectives. Costlier than an equirect blit but lets - // us add other layer-1 props later that would also show up in - // reflections. - cubeCamera.update(renderer!, scene); - renderer!.render(scene, camera); - context.present(); - frameCount++; - if (frameCount === 1) { - console.log("[CameraHelmet] first three.js frame rendered"); - } - }; - renderer.setAnimationLoop(animate); - console.log("[CameraHelmet] animation loop started"); - - setPipelineState({ - device, - cameraPipeline, - cameraSampler, - envTexture, - envTextureView: envTexture.createView(), - }); - console.log("[CameraHelmet] pipelineState set, camera will activate"); - } catch (e) { - if (cancelled) { - return; - } - console.warn("[CameraHelmet] setup failed: " + String(e)); - setError(String(e)); - } - })(); - - return () => { - console.log("[CameraHelmet] setup-effect cleanup"); - cancelled = true; - applyPBRFnRef.current = null; - if (renderer) { - renderer.setAnimationLoop(null); - } - }; - }, [device, gltf]); - - // Frame processor worklet: copy the camera frame into envTexture each tick. - // The single device.queue is shared with three.js, so the helmet pass on - // the next rAF tick samples this frame's write. - const logBox = React.useMemo(() => ({ count: 0 }), []); - const frameOutput = useFrameOutput({ - pixelFormat: "native", - // 720p front-cam frames are plenty for the helmet's reflection — it's a - // small on-screen area. Keeping this low matters more in a multi-cam - // session, where both cameras share AVFoundation's bandwidth budget. - targetResolution: CommonResolutions.HD_16_9, - onFrame: (frame) => { - "worklet"; - logBox.count += 1; - if (logBox.count === 1) { - console.log( - "[CameraHelmet] worklet first frame, hasPipeline=" + - String(pipelineState != null) + - " frame=" + - String(frame.width) + - "x" + - String(frame.height), - ); - } - if (!pipelineState) { - frame.dispose(); - return; - } - const { - device: gpuDevice, - cameraPipeline, - cameraSampler, - envTextureView, - } = pipelineState; - const nativeBuffer = frame.getNativeBuffer(); - try { - const videoFrame = gpuDevice.createVideoFrameFromNativeBuffer( - nativeBuffer.pointer, - ); - try { - const externalTex = gpuDevice.importExternalTexture({ - source: videoFrame, - label: "camera-helmet-env", - }); - const bindGroup = gpuDevice.createBindGroup({ - layout: cameraPipeline.getBindGroupLayout(0), - entries: [ - { binding: 0, resource: externalTex }, - { binding: 1, resource: cameraSampler }, - ], - }); - const encoder = gpuDevice.createCommandEncoder(); - const pass = encoder.beginRenderPass({ - colorAttachments: [ - { - view: envTextureView, - clearValue: { r: 0, g: 0, b: 0, a: 1 }, - loadOp: "clear", - storeOp: "store", - }, - ], - }); - pass.setPipeline(cameraPipeline); - pass.setBindGroup(0, bindGroup); - pass.draw(3); - pass.end(); - gpuDevice.queue.submit([encoder.finish()]); - } finally { - videoFrame.release(); - } - } finally { - nativeBuffer.release(); - frame.dispose(); - } - }, - }); - - // ---- Multi-cam session ------------------------------------------------ - // useCamera always sets enableMultiCamSupport=false, so we drop down to - // the imperative API to drive two camera connections from a single - // session: front → frameOutput (helmet env), back → previewOutput - // (backdrop View). - const [session, setSession] = useState(null); - useEffect(() => { - if (!VisionCameraFactory.supportsMultiCamSessions) { - setError( - "This device doesn't support multi-cam sessions. Need an iPhone XS " + - "or newer / a comparable Android flagship.", - ); - return; - } - let cancelled = false; - let created: CameraSession | null = null; - (async () => { - const s = await VisionCameraFactory.createCameraSession(true); - if (cancelled) { - s.dispose(); - return; - } - created = s; - setSession(s); - })(); - return () => { - cancelled = true; - created?.stop(); - created?.dispose(); - }; - }, []); - - // Configure the session with two connections once everything is ready. - // We wait on pipelineState too because the worklet (which receives the - // front cam frames) only has somewhere to write once the env texture + - // camera-copy pipeline exist. - useEffect(() => { - if (!session || !backDevice || !frontDevice || !pipelineState) { - return; - } - console.log("[CameraHelmet] configuring multi-cam session"); - let cancelled = false; - let controllers: CameraController[] = []; - (async () => { - try { - controllers = await session.configure( - [ - { - input: backDevice, - outputs: [{ output: previewOutput, mirrorMode: "auto" }], - constraints: [], - }, - { - input: frontDevice, - outputs: [{ output: frameOutput, mirrorMode: "auto" }], - constraints: [], - }, - ], - {}, - ); - if (cancelled) { - controllers.forEach((c) => c.dispose()); - return; - } - console.log("[CameraHelmet] session configured, starting"); - session.start(); - } catch (e) { - if (cancelled) { - return; - } - console.warn("[CameraHelmet] session configure failed: " + String(e)); - setError(String(e)); - } - })(); - return () => { - cancelled = true; - session.stop(); - controllers.forEach((c) => c.dispose()); - }; - }, [ - session, - backDevice, - frontDevice, - previewOutput, - frameOutput, - pipelineState, - ]); - - if (error) { - return ( - - {error} - - ); - } - if (backDevice == null || frontDevice == null) { - return ( - - - Need both a back and a front camera. The iOS Simulator has none, and - some devices expose only one. - - - ); - } - return ( - - - - - {usePBR ? "PBR" : "Chrome"} - - - ); -}; - -const styles = StyleSheet.create({ - root: { flex: 1, backgroundColor: "black" }, - // Transparent canvas overlaid on the native camera preview view. - canvas: { ...StyleSheet.absoluteFillObject, backgroundColor: "transparent" }, - errorContainer: { flex: 1, padding: 16, justifyContent: "center" }, - errorText: { color: "red", fontSize: 14 }, - permissionContainer: { - flex: 1, - padding: 24, - justifyContent: "center", - alignItems: "center", - }, - permissionText: { fontSize: 16, textAlign: "center", marginBottom: 16 }, - permissionButton: { - backgroundColor: "#007AFF", - paddingHorizontal: 24, - paddingVertical: 12, - borderRadius: 8, - }, - permissionButtonText: { color: "white", fontSize: 16, fontWeight: "600" }, - toggleButton: { - position: "absolute", - top: 60, - right: 16, - backgroundColor: "rgba(0, 0, 0, 0.6)", - paddingHorizontal: 16, - paddingVertical: 10, - borderRadius: 20, - minWidth: 88, - alignItems: "center", - }, - toggleButtonText: { color: "white", fontSize: 14, fontWeight: "600" }, -}); diff --git a/apps/example/src/ThreeJS/CameraSpheres.tsx b/apps/example/src/ThreeJS/CameraSpheres.tsx deleted file mode 100644 index f3d7dedf3..000000000 --- a/apps/example/src/ThreeJS/CameraSpheres.tsx +++ /dev/null @@ -1,489 +0,0 @@ -import React, { useEffect, useRef, useState } from "react"; -import { - Linking, - Platform, - StyleSheet, - Text, - TouchableOpacity, - View, -} from "react-native"; -import type { CanvasRef } from "react-native-wgpu"; -import { Canvas } from "react-native-wgpu"; -import { - CommonResolutions, - NativePreviewView, - useCameraDevices, - useCameraPermission, - useFrameOutput, - usePreviewOutput, - VisionCamera as VisionCameraFactory, -} from "react-native-vision-camera"; -import type { - CameraController, - CameraSession, -} from "react-native-vision-camera"; -import * as THREE from "three"; - -import { makeWebGPURenderer } from "./components/makeWebGPURenderer"; -import { CAMERA_ENV_SHADER } from "./cameraEnvShader"; - -// Sibling of CameraHelmet but with three procedural chrome spheres in place -// of the GLTF helmet. Multi-cam setup: back camera renders behind the canvas -// as a native preview view, front camera feeds the cubemap that the spheres -// reflect. No PBR (no GLTF assets), so we skip mipmap generation entirely — -// the chrome look stays sharp on all surfaces. - -const ENV_WIDTH = 1024; -const ENV_HEIGHT = 512; - -const REQUIRED_FEATURES: GPUFeatureName[] = [ - "rnwebgpu/shared-texture-memory" as GPUFeatureName, - "dawn-multi-planar-formats" as GPUFeatureName, -]; - -const OPAQUE_YCBCR_EXT = - "opaque-ycbcr-android-for-external-texture" as GPUFeatureName; - -// Three big chrome spheres swirling around the origin, inspired by three.js' -// stereo-effects demo (which uses InstancedMesh + per-frame matrix updates). -// Even at 3 instances the InstancedMesh path is still nice because all -// three render in a single draw call. -const BEAD_COUNT = 3; -const BEAD_RADIUS = 0.55; -// XY radius of the swirl. Spheres orbit evenly spaced around the origin. -const SWIRL_RADIUS = 1.8; - -export const CameraSpheres = () => { - const { hasPermission, requestPermission } = useCameraPermission(); - useEffect(() => { - if (!hasPermission) { - requestPermission(); - } - }, [hasPermission, requestPermission]); - - if (!hasPermission) { - return ( - - - Camera access is required. Grant it in Settings or tap below. - - Linking.openSettings()} - style={styles.permissionButton} - > - Open Settings - - - ); - } - return ; -}; - -const Scene = () => { - useEffect(() => { - console.log("[CameraSpheres] Scene mounted"); - return () => console.log("[CameraSpheres] Scene unmounted"); - }, []); - const ref = useRef(null); - const previewOutput = usePreviewOutput(); - - const devices = useCameraDevices(); - const backDevice = React.useMemo( - () => devices.find((d) => d.position === "back"), - [devices], - ); - const frontDevice = React.useMemo( - () => devices.find((d) => d.position === "front"), - [devices], - ); - - const [pipelineState, setPipelineState] = useState<{ - device: GPUDevice; - cameraPipeline: GPURenderPipeline; - cameraSampler: GPUSampler; - envTexture: GPUTexture; - envTextureView: GPUTextureView; - } | null>(null); - const [error, setError] = useState(null); - const [device, setDevice] = useState(null); - - useEffect(() => { - let cancelled = false; - (async () => { - try { - const adapter = await navigator.gpu.requestAdapter(); - if (!adapter) { - throw new Error("requestAdapter returned null"); - } - const requiredFeatures = [...adapter.features] as GPUFeatureName[]; - const missing = REQUIRED_FEATURES.filter( - (f) => !adapter.features.has(f), - ); - const needsAndroidExt = - Platform.OS === "android" && !adapter.features.has(OPAQUE_YCBCR_EXT); - if (missing.length > 0 || needsAndroidExt) { - throw new Error( - "Adapter doesn't advertise the features the Vision Camera " + - "external-texture path needs: " + - `${[...missing, needsAndroidExt ? OPAQUE_YCBCR_EXT : null] - .filter(Boolean) - .join(", ")}.`, - ); - } - const d = await adapter.requestDevice({ requiredFeatures }); - if (cancelled) { - d.destroy(); - return; - } - setDevice(d); - } catch (e) { - if (cancelled) { - return; - } - console.warn("[CameraSpheres] device acquisition failed: " + String(e)); - setError(String(e)); - } - })(); - return () => { - cancelled = true; - }; - }, []); - - useEffect(() => { - if (!device) { - return; - } - const context = ref.current?.getContext("webgpu"); - if (!context) { - return; - } - let cancelled = false; - let renderer: THREE.WebGPURenderer | null = null; - - (async () => { - try { - const { width, height } = context.canvas; - - renderer = makeWebGPURenderer(context, { device, alpha: true }); - renderer.toneMapping = THREE.ACESFilmicToneMapping; - renderer.setClearColor(0x000000, 0); - await renderer.init(); - if (cancelled) { - return; - } - - const envTexture = device.createTexture({ - size: [ENV_WIDTH, ENV_HEIGHT], - format: "rgba8unorm", - usage: - GPUTextureUsage.RENDER_ATTACHMENT | GPUTextureUsage.TEXTURE_BINDING, - }); - - const module = device.createShaderModule({ code: CAMERA_ENV_SHADER }); - const cameraPipeline = device.createRenderPipeline({ - layout: "auto", - vertex: { module, entryPoint: "vs_main" }, - fragment: { - module, - entryPoint: "fs_main", - targets: [{ format: "rgba8unorm" }], - }, - primitive: { topology: "triangle-list" }, - }); - const cameraSampler = device.createSampler({ - magFilter: "linear", - minFilter: "linear", - }); - - const envExternalTexture = new THREE.ExternalTexture(envTexture); - envExternalTexture.mapping = THREE.EquirectangularReflectionMapping; - envExternalTexture.colorSpace = THREE.SRGBColorSpace; - (envExternalTexture as unknown as { image: unknown }).image = { - width: ENV_WIDTH, - height: ENV_HEIGHT, - }; - envExternalTexture.needsUpdate = true; - - // Cube target refreshed per frame from the equirect — same dynamic - // env trick as CameraHelmet. No mipmap chain: the spheres use - // MeshBasicMaterial which samples mip 0 unconditionally, so the - // auto-mipmap regeneration would be pure waste. - const cubeRT = new THREE.CubeRenderTarget(ENV_HEIGHT); - cubeRT.texture.mapping = THREE.CubeReflectionMapping; - cubeRT.texture.colorSpace = THREE.SRGBColorSpace; - - const scene = new THREE.Scene(); - // Single InstancedMesh: all three spheres in one draw call. Mesh - // tessellation matters more here than in the swarm version because - // each sphere is bigger on screen, so 48x32 segments instead of - // 16x8 — smoother silhouettes against the panorama backdrop. - const chrome = new THREE.MeshBasicMaterial({ envMap: cubeRT.texture }); - const geometry = new THREE.SphereGeometry(BEAD_RADIUS, 48, 32); - const beads = new THREE.InstancedMesh(geometry, chrome, BEAD_COUNT); - beads.instanceMatrix.setUsage(THREE.DynamicDrawUsage); - scene.add(beads); - - // Seed each instance's matrix to identity. The animate loop - // overwrites the translation column each frame; scale and rotation - // stay as identity (= sphere radius set by BEAD_RADIUS above). - const dummy = new THREE.Object3D(); - for (let i = 0; i < BEAD_COUNT; i++) { - dummy.updateMatrix(); - beads.setMatrixAt(i, dummy.matrix); - } - beads.instanceMatrix.needsUpdate = true; - const beadPos = new THREE.Vector3(); - - const aspect = width / height; - const baseFov = 60; - let vFov = baseFov; - if (aspect < 1) { - const hFovRad = (baseFov * Math.PI) / 180; - const vFovRad = 2 * Math.atan(Math.tan(hFovRad / 2) / aspect); - vFov = (vFovRad * 180) / Math.PI; - } - const camera = new THREE.PerspectiveCamera(vFov, aspect, 0.25, 20); - camera.position.set(0, 0, 4); - - const clock = new THREE.Clock(); - const animate = () => { - // Slow rotation of the three spheres around the origin in the XY - // plane. Phase offsets are evenly spaced (2Ď€/3 apart) so the - // spheres form a rotating equilateral triangle, never overlapping. - const elapsed = clock.getElapsedTime() * 0.4; - for (let i = 0; i < BEAD_COUNT; i++) { - const angle = elapsed + (i * 2 * Math.PI) / BEAD_COUNT; - beadPos.set( - SWIRL_RADIUS * Math.cos(angle), - SWIRL_RADIUS * Math.sin(angle), - 0, - ); - beads.getMatrixAt(i, dummy.matrix); - dummy.matrix.setPosition(beadPos); - beads.setMatrixAt(i, dummy.matrix); - } - beads.instanceMatrix.needsUpdate = true; - - cubeRT.fromEquirectangularTexture(renderer!, envExternalTexture); - renderer!.render(scene, camera); - context.present(); - }; - renderer.setAnimationLoop(animate); - - setPipelineState({ - device, - cameraPipeline, - cameraSampler, - envTexture, - envTextureView: envTexture.createView(), - }); - } catch (e) { - if (cancelled) { - return; - } - console.warn("[CameraSpheres] setup failed: " + String(e)); - setError(String(e)); - } - })(); - - return () => { - cancelled = true; - if (renderer) { - renderer.setAnimationLoop(null); - } - }; - }, [device]); - - const logBox = React.useMemo(() => ({ count: 0 }), []); - const frameOutput = useFrameOutput({ - pixelFormat: "native", - targetResolution: CommonResolutions.HD_16_9, - onFrame: (frame) => { - "worklet"; - logBox.count += 1; - if (logBox.count === 1) { - console.log( - "[CameraSpheres] worklet first frame, frame=" + - String(frame.width) + - "x" + - String(frame.height), - ); - } - if (!pipelineState) { - frame.dispose(); - return; - } - const { - device: gpuDevice, - cameraPipeline, - cameraSampler, - envTextureView, - } = pipelineState; - const nativeBuffer = frame.getNativeBuffer(); - try { - const videoFrame = gpuDevice.createVideoFrameFromNativeBuffer( - nativeBuffer.pointer, - ); - try { - const externalTex = gpuDevice.importExternalTexture({ - source: videoFrame, - label: "camera-spheres-env", - }); - const bindGroup = gpuDevice.createBindGroup({ - layout: cameraPipeline.getBindGroupLayout(0), - entries: [ - { binding: 0, resource: externalTex }, - { binding: 1, resource: cameraSampler }, - ], - }); - const encoder = gpuDevice.createCommandEncoder(); - const pass = encoder.beginRenderPass({ - colorAttachments: [ - { - view: envTextureView, - clearValue: { r: 0, g: 0, b: 0, a: 1 }, - loadOp: "clear", - storeOp: "store", - }, - ], - }); - pass.setPipeline(cameraPipeline); - pass.setBindGroup(0, bindGroup); - pass.draw(3); - pass.end(); - gpuDevice.queue.submit([encoder.finish()]); - } finally { - videoFrame.release(); - } - } finally { - nativeBuffer.release(); - frame.dispose(); - } - }, - }); - - const [session, setSession] = useState(null); - useEffect(() => { - if (!VisionCameraFactory.supportsMultiCamSessions) { - setError( - "This device doesn't support multi-cam sessions. Need an iPhone XS " + - "or newer / a comparable Android flagship.", - ); - return; - } - let cancelled = false; - let created: CameraSession | null = null; - (async () => { - const s = await VisionCameraFactory.createCameraSession(true); - if (cancelled) { - s.dispose(); - return; - } - created = s; - setSession(s); - })(); - return () => { - cancelled = true; - created?.stop(); - created?.dispose(); - }; - }, []); - - useEffect(() => { - if (!session || !backDevice || !frontDevice || !pipelineState) { - return; - } - let cancelled = false; - let controllers: CameraController[] = []; - (async () => { - try { - controllers = await session.configure( - [ - { - input: backDevice, - outputs: [{ output: previewOutput, mirrorMode: "auto" }], - constraints: [], - }, - { - input: frontDevice, - outputs: [{ output: frameOutput, mirrorMode: "auto" }], - constraints: [], - }, - ], - {}, - ); - if (cancelled) { - controllers.forEach((c) => c.dispose()); - return; - } - session.start(); - } catch (e) { - if (cancelled) { - return; - } - console.warn("[CameraSpheres] session configure failed: " + String(e)); - setError(String(e)); - } - })(); - return () => { - cancelled = true; - session.stop(); - controllers.forEach((c) => c.dispose()); - }; - }, [ - session, - backDevice, - frontDevice, - previewOutput, - frameOutput, - pipelineState, - ]); - - if (error) { - return ( - - {error} - - ); - } - if (backDevice == null || frontDevice == null) { - return ( - - - Need both a back and a front camera. The iOS Simulator has none, and - some devices expose only one. - - - ); - } - return ( - - - - - ); -}; - -const styles = StyleSheet.create({ - root: { flex: 1, backgroundColor: "black" }, - canvas: { ...StyleSheet.absoluteFillObject, backgroundColor: "transparent" }, - errorContainer: { flex: 1, padding: 16, justifyContent: "center" }, - errorText: { color: "red", fontSize: 14 }, - permissionContainer: { - flex: 1, - padding: 24, - justifyContent: "center", - alignItems: "center", - }, - permissionText: { fontSize: 16, textAlign: "center", marginBottom: 16 }, - permissionButton: { - backgroundColor: "#007AFF", - paddingHorizontal: 24, - paddingVertical: 12, - borderRadius: 8, - }, - permissionButtonText: { color: "white", fontSize: 16, fontWeight: "600" }, -}); diff --git a/apps/example/src/ThreeJS/List.tsx b/apps/example/src/ThreeJS/List.tsx index ad3547fdb..9601abb71 100644 --- a/apps/example/src/ThreeJS/List.tsx +++ b/apps/example/src/ThreeJS/List.tsx @@ -23,14 +23,6 @@ export const examples = [ screen: "Helmet", title: "⛑️ Helmet", }, - { - screen: "CameraHelmet", - title: "đź“· Camera Env Sphere", - }, - { - screen: "CameraSpheres", - title: "đź“· Camera Env Spheres", - }, { screen: "PostProcessing", title: "🪄 Post Processing Effects", diff --git a/apps/example/src/ThreeJS/Routes.ts b/apps/example/src/ThreeJS/Routes.ts index 68971cef5..cb76ac65f 100644 --- a/apps/example/src/ThreeJS/Routes.ts +++ b/apps/example/src/ThreeJS/Routes.ts @@ -2,8 +2,6 @@ export type Routes = { List: undefined; Cube: undefined; Helmet: undefined; - CameraHelmet: undefined; - CameraSpheres: undefined; Backdrop: undefined; InstancedMesh: undefined; Fiber: undefined; diff --git a/apps/example/src/ThreeJS/cameraEnvShader.ts b/apps/example/src/ThreeJS/cameraEnvShader.ts deleted file mode 100644 index 2222ef444..000000000 --- a/apps/example/src/ThreeJS/cameraEnvShader.ts +++ /dev/null @@ -1,49 +0,0 @@ -// Tiny "copy camera frame into an rgba8unorm texture" shader. The output -// texture is then wrapped in a THREE.ExternalTexture and mapped onto a -// billboarded plane that three.js' CubeCamera bakes into the helmet's -// envMap — i.e. it acts as a virtual screen at the viewer's location -// rather than a 360° panorama. The destination texture's aspect (9:16) is -// chosen to match the camera frame's post-rotation aspect so no stretching -// happens here; the fullscreen triangle just rotates+mirrors the source to -// selfie-upright. - -export const CAMERA_ENV_SHADER = /* wgsl */ ` -struct VsOut { - @builtin(position) position: vec4f, - @location(0) uv: vec2f, -}; - -@group(0) @binding(0) var srcTex: texture_external; -@group(0) @binding(1) var srcSampler: sampler; - -@vertex -fn vs_main(@builtin(vertex_index) vid: u32) -> VsOut { - var positions = array( - vec2f(-1.0, -3.0), - vec2f(-1.0, 1.0), - vec2f( 3.0, 1.0), - ); - var uvs = array( - vec2f(0.0, 2.0), - vec2f(0.0, 0.0), - vec2f(2.0, 0.0), - ); - var out: VsOut; - out.position = vec4f(positions[vid], 0.0, 1.0); - out.uv = uvs[vid]; - return out; -} - -@fragment -fn fs_main(in: VsOut) -> @location(0) vec4f { - // Front camera: iOS delivers landscape-orientation frames with the - // horizontal axis already mirrored (selfie convention). To bring those - // upright in the equirect we (a) compensate for the horizontal mirror - // by sampling at (1-x) and (b) rotate 90° CCW with V flipped, giving - // (1-v, 1-u). Equivalent to the 90° CW back-cam mapping (v, 1-u) with - // its U axis pre-flipped to undo the mirror. - let rotatedUv = vec2f(1.0 - in.uv.y, 1.0 - in.uv.x); - let c = textureSampleBaseClampToEdge(srcTex, srcSampler, rotatedUv); - return vec4f(c.rgb, 1.0); -} -`; diff --git a/apps/example/src/ThreeJS/components/makeWebGPURenderer.ts b/apps/example/src/ThreeJS/components/makeWebGPURenderer.ts index f6729b3c9..39c4bf399 100644 --- a/apps/example/src/ThreeJS/components/makeWebGPURenderer.ts +++ b/apps/example/src/ThreeJS/components/makeWebGPURenderer.ts @@ -60,22 +60,12 @@ export class ReactNativeCanvas { export const makeWebGPURenderer = ( context: GPUCanvasContext, - { - antialias = true, - device, - alpha = false, - }: { antialias?: boolean; device?: GPUDevice; alpha?: boolean } = {}, + { antialias = true }: { antialias?: boolean } = {}, ) => new THREE.WebGPURenderer({ antialias, - alpha, // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error canvas: new ReactNativeCanvas(context.canvas), context, - // When supplied, three.js skips its own adapter/device acquisition and - // uses this device. Lets callers request custom features (e.g. Dawn's - // shared-texture-memory) that three.js doesn't include in its default - // GPUFeatureName enum walk. - ...(device ? { device } : {}), }); diff --git a/apps/example/src/ThreeJS/index.tsx b/apps/example/src/ThreeJS/index.tsx index e679d88e4..244c43d71 100644 --- a/apps/example/src/ThreeJS/index.tsx +++ b/apps/example/src/ThreeJS/index.tsx @@ -6,8 +6,6 @@ import { Cube } from "./Cube"; import type { Routes } from "./Routes"; import { List } from "./List"; import { Helmet } from "./Helmet"; -import { CameraHelmet } from "./CameraHelmet"; -import { CameraSpheres } from "./CameraSpheres"; import { Backdrop } from "./Backdrop"; import { InstancedMesh } from "./InstancedMesh"; import { Fiber } from "./Fiber"; @@ -75,20 +73,6 @@ export const ThreeJS = () => { title: "⛑️ Helmet", }} /> - - { usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, }); - // ----- Blur infrastructure (matches ExternalTexture's "Blur" chain) ---- + // ----- Blur infrastructure ----- const blurWidth = Math.max( BLUR_TILE_DIM, Math.ceil(canvas.width / BLUR_SCALE), diff --git a/apps/example/src/VisionCamera/features.ts b/apps/example/src/VisionCamera/features.ts index 7e8ed57a8..eb46bc224 100644 --- a/apps/example/src/VisionCamera/features.ts +++ b/apps/example/src/VisionCamera/features.ts @@ -16,7 +16,7 @@ export type Modes = { export const INITIAL_MODES: Modes = { effect: 0, tint: 0, - aberration: 1, + aberration: 0, blur: 0, vignette: 0, pixelate: 0, From 8adcf2dba685e549a06c05eac943d531771efb5d Mon Sep 17 00:00:00 2001 From: William Candillon Date: Mon, 1 Jun 2026 11:41:44 +0200 Subject: [PATCH 32/46] :wrench: --- .../example/src/VisionCamera/VisionCamera.tsx | 88 ++++++++++++-- apps/example/src/VisionCamera/blurShaders.ts | 9 +- packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp | 113 ++++++++++++++++-- .../GPUExternalTextureDescriptor.h | 21 ++++ packages/webgpu/src/index.tsx | 10 ++ 5 files changed, 214 insertions(+), 27 deletions(-) diff --git a/apps/example/src/VisionCamera/VisionCamera.tsx b/apps/example/src/VisionCamera/VisionCamera.tsx index aa3c8e1ce..155a80f4e 100644 --- a/apps/example/src/VisionCamera/VisionCamera.tsx +++ b/apps/example/src/VisionCamera/VisionCamera.tsx @@ -91,7 +91,9 @@ fn snap(uv: vec2f, block: f32) -> vec2f { } fn sampleExternal(uv: vec2f, block: f32) -> vec4f { - return textureSampleBaseClampToEdge(srcTex, srcSampler, snap(uv, block)); + return cameraDecode( + textureSampleBaseClampToEdge(srcTex, srcSampler, cameraCoord(snap(uv, block))), + ); } fn sampleBlurred(uv: vec2f, block: f32) -> vec4f { @@ -206,6 +208,47 @@ fn fs_main(in: VsOut) -> @location(0) vec4f { } `; +// Android delivers camera frames as an external-format (opaque) YUV buffer. +// Dawn copies the AHardwareBuffer's suggested YCbCr model verbatim, and on the +// devices we've seen that resolves to identity, so the external sample comes +// back as raw [Y, Cb, Cr]. After Dawn's rotation the frame is also mirrored on +// both axes relative to the canvas (Android buffer origin), so we flip X and Y +// here. iOS goes through the native two-plane path, which already converts and +// orients, so this correction is Android-only. The prelude is prepended to +// every shader module that samples the camera (main pass + blur prepass). +const CAMERA_PRELUDE = /* wgsl */ ` +const CAMERA_IS_YUV: bool = ${Platform.OS === "android"}; +const CAMERA_FLIP_X: bool = ${Platform.OS === "android"}; +const CAMERA_FLIP_Y: bool = ${Platform.OS === "android"}; + +fn cameraCoord(uv: vec2f) -> vec2f { + var c = uv; + if (CAMERA_FLIP_X) { + c.x = 1.0 - c.x; + } + if (CAMERA_FLIP_Y) { + c.y = 1.0 - c.y; + } + return c; +} + +// BT.709 limited-range YUV -> RGB. The sampled channels are [Y, Cb, Cr] when +// the driver's YCbCr conversion is identity (the Android opaque path); a no-op +// passthrough otherwise. +fn cameraDecode(c: vec4f) -> vec4f { + if (!CAMERA_IS_YUV) { + return c; + } + let y = c.r - 0.0627451; + let cb = c.g - 0.5; + let cr = c.b - 0.5; + let r = 1.164384 * y + 1.792741 * cr; + let g = 1.164384 * y - 0.213249 * cb - 0.532909 * cr; + let b = 1.164384 * y + 2.112402 * cb; + return vec4f(clamp(vec3f(r, g, b), vec3f(0.0), vec3f(1.0)), 1.0); +} +`; + const REQUIRED_FEATURES: GPUFeatureName[] = [ "rnwebgpu/shared-texture-memory" as GPUFeatureName, "dawn-multi-planar-formats" as GPUFeatureName, @@ -391,7 +434,9 @@ const CameraView = () => { alphaMode: "premultiplied", }); - const module = device.createShaderModule({ code: SHADER }); + const module = device.createShaderModule({ + code: CAMERA_PRELUDE + SHADER, + }); const pipeline = device.createRenderPipeline({ layout: "auto", vertex: { module, entryPoint: "vs_main" }, @@ -421,7 +466,9 @@ const CameraView = () => { Math.ceil(canvas.height / BLUR_SCALE), ); - const prepassModule = device.createShaderModule({ code: PREPASS_SHADER }); + const prepassModule = device.createShaderModule({ + code: CAMERA_PRELUDE + PREPASS_SHADER, + }); const prepassPipeline = device.createRenderPipeline({ layout: "auto", vertex: { module: prepassModule, entryPoint: "vs_main" }, @@ -565,7 +612,11 @@ const CameraView = () => { " frame=" + String(frame.width) + "x" + - String(frame.height), + String(frame.height) + + " orientation=" + + String(frame.orientation) + + " isMirrored=" + + String(frame.isMirrored), ); } if (!pipelineState || !device) { @@ -612,11 +663,29 @@ const CameraView = () => { throw e; } try { + // Orientation. The sensor buffer is landscape; frame.orientation is + // the rotation needed to bring it upright. We hand that to Dawn via + // importExternalTexture's `rotation`, which de-rotates the frame into + // the portrait canvas. The residual vertical flip (Android buffer + // Y-origin) and YUV->RGB are corrected in the shader (CAMERA_PRELUDE). + let rotationDeg: 0 | 90 | 180 | 270 = 0; + if (frame.orientation === "right") { + rotationDeg = 90; + } else if (frame.orientation === "down") { + rotationDeg = 180; + } else if (frame.orientation === "left") { + rotationDeg = 270; + } + // A 90/270 rotation swaps the displayed width & height, so cover-fit + // uses the post-rotation dimensions. + const rotated = rotationDeg === 90 || rotationDeg === 270; + const dispW = rotated ? videoFrame.height : videoFrame.width; + const dispH = rotated ? videoFrame.width : videoFrame.height; // Compute cover-fit uvScale based on frame & canvas aspect ratios. // On most phones the back camera is landscape (e.g. 1920x1080) and // the canvas is portrait, so the y-axis gets cropped. const canvasAR = canvasWidth / canvasHeight; - const frameAR = videoFrame.width / videoFrame.height; + const frameAR = dispW / dispH; let sx = 1; let sy = 1; if (frameAR > canvasAR) { @@ -644,6 +713,8 @@ const CameraView = () => { externalTex = device.importExternalTexture({ source: videoFrame, label: "camera-frame", + rotation: rotationDeg, + mirrored: frame.isMirrored, }); } catch (e) { console.warn( @@ -668,12 +739,7 @@ const CameraView = () => { device.queue.writeBuffer( prepassUniformBuffer, 0, - new Float32Array([ - videoFrame.width, - videoFrame.height, - canvasWidth, - canvasHeight, - ]), + new Float32Array([dispW, dispH, canvasWidth, canvasHeight]), ); const prepassBindGroup = device.createBindGroup({ layout: prepassPipeline.getBindGroupLayout(0), diff --git a/apps/example/src/VisionCamera/blurShaders.ts b/apps/example/src/VisionCamera/blurShaders.ts index e589bff07..eddbfb8d7 100644 --- a/apps/example/src/VisionCamera/blurShaders.ts +++ b/apps/example/src/VisionCamera/blurShaders.ts @@ -63,11 +63,14 @@ fn fs_main(in: VsOut) -> @location(0) vec4f { scale = vec2f(1.0, texAR / canvasAR); } let uv = vec2f(0.5) + (in.uv - vec2f(0.5)) * scale; - let c = textureSampleBaseClampToEdge( + // cameraCoord (vertical flip) + cameraDecode (YUV->RGB) come from + // CAMERA_PRELUDE, prepended when this module is compiled. They are no-ops on + // iOS and handle the Android opaque-YUV case. + let c = cameraDecode(textureSampleBaseClampToEdge( srcTex, srcSampler, - clamp(uv, vec2f(0.0), vec2f(1.0)), - ); + cameraCoord(clamp(uv, vec2f(0.0), vec2f(1.0))), + )); return vec4f(c.rgb, 1.0); } `; diff --git a/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp b/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp index 4a0fdba0c..a19bbf856 100644 --- a/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp +++ b/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp @@ -1,5 +1,6 @@ #include "GPUDevice.h" +#include #include #include #include @@ -265,6 +266,55 @@ static const float kSrgbEncodeParams[7] = { 0.0f, // F }; +// BT.709 limited-range YUV -> R'G'B' as a 3x4 row-major matrix mapping +// [Y, Cb, Cr, 1]. Same values the Apple NV12 path computes from the +// CVPixelBuffer; used for Android buffers that arrive as a *defined* biplanar +// format (where we split the planes and convert ourselves) rather than an +// opaque external-format AHB. Camera streams are limited-range BT.709 in the +// overwhelming majority of cases; full-range / BT.601 would need different +// coefficients (refine from the buffer's suggested range if it matters). +[[maybe_unused]] static const float kBT709LimitedToRgb[12] = { + 1.164383f, 0.000000f, 1.792741f, -0.972945f, // + 1.164383f, -0.213249f, -0.532909f, 0.301517f, // + 1.164383f, 2.112402f, 0.000000f, -1.133402f, // +}; + +// True for the multi-planar Y + CbCr formats whose planes we can view as +// Plane0Only (luma) / Plane1Only (chroma) and convert with an explicit matrix. +// Excludes OpaqueYCbCrAndroid (external format, no plane views) and triplanar +// formats (would need a third plane). Only referenced on Android. +[[maybe_unused]] static bool isBiplanarYuvFormat(wgpu::TextureFormat format) { + switch (format) { + case wgpu::TextureFormat::R8BG8Biplanar420Unorm: + case wgpu::TextureFormat::R8BG8Biplanar422Unorm: + case wgpu::TextureFormat::R8BG8Biplanar444Unorm: + case wgpu::TextureFormat::R10X6BG10X6Biplanar420Unorm: + case wgpu::TextureFormat::R10X6BG10X6Biplanar422Unorm: + case wgpu::TextureFormat::R10X6BG10X6Biplanar444Unorm: + return true; + default: + return false; + } +} + +// Map a rotation in degrees (0 / 90 / 180 / 270) to Dawn's enum. Anything that +// isn't a clean multiple of 90 snaps to the nearest quadrant; Dawn only +// supports those four steps for external textures. +static wgpu::ExternalTextureRotation +toExternalTextureRotation(double degrees) { + int quadrant = static_cast(std::lround(degrees / 90.0)) & 3; + switch (quadrant) { + case 1: + return wgpu::ExternalTextureRotation::Rotate90Degrees; + case 2: + return wgpu::ExternalTextureRotation::Rotate180Degrees; + case 3: + return wgpu::ExternalTextureRotation::Rotate270Degrees; + default: + return wgpu::ExternalTextureRotation::Rotate0Degrees; + } +} + std::shared_ptr GPUDevice::importExternalTexture( std::shared_ptr descriptor) { if (!descriptor || !descriptor->source) { @@ -353,6 +403,9 @@ std::shared_ptr GPUDevice::importExternalTexture( extDesc.cropOrigin = {0, 0}; extDesc.cropSize = {frame.width, frame.height}; extDesc.apparentSize = {frame.width, frame.height}; + extDesc.mirrored = descriptor->mirrored.value_or(false); + extDesc.rotation = + toExternalTextureRotation(descriptor->rotation.value_or(0)); auto external = _instance.CreateExternalTexture(&extDesc); if (external == nullptr) { @@ -406,16 +459,26 @@ std::shared_ptr GPUDevice::importExternalTexture( "GPUDevice::importExternalTexture(): BeginAccess failed"); } - // 4. Build the ExternalTextureDescriptor. Unlike iOS we do *not* split - // planes or pass an explicit YUV→RGB matrix: when the underlying texture - // is OpaqueYCbCrAndroid, Dawn routes sampling through a Vulkan - // SamplerYcbcrConversion that does the conversion implicitly, driven by - // the AHB's own format metadata. We still must pass noop gamut/transfer - // arrays: Dawn's ComputeExternalTextureParams unconditionally dereferences - // gamutConversionMatrix / src/dstTransferFunctionParameters (see - // externals/dawn/.../ExternalTexture.cpp), so leaving them null produces a - // silent black sample. Identity transfer = TransferFunctionToArray of - // kEOTF_Identity ({g=1,a=1,rest=0}); identity gamut = 3x3 identity. + // 4. Build the ExternalTextureDescriptor. There are two cases depending on + // how Dawn imported the AHB (see SharedTextureMemoryVk.cpp): + // + // a. *External* format (camera buffers whose layout has no Vulkan + // equivalent) -> OpaqueYCbCrAndroid, a single opaque plane. Sampling + // routes through a Vulkan SamplerYcbcrConversion whose model Dawn + // copies verbatim from the AHB's suggestedYcbcrModel. We pass a single + // plane + identity transfer and let that conversion (if any) run. NOTE: + // when the driver reports RGB_IDENTITY the sample comes back as raw + // Y/Cb/Cr; there is no public hook to override the model on this path. + // + // b. *Defined* biplanar format (e.g. R8BG8Biplanar420Unorm, exposed by the + // dawn-multi-planar-formats feature) -> we split Plane0Only (luma) / + // Plane1Only (chroma) and hand Dawn an explicit BT.709 matrix + sRGB + // transfer, exactly like the iOS NV12 path. This makes numPlanes == 2 + // so the matrix is actually applied (the single-plane branch in Dawn's + // Tint transform ignores yuvToRgbConversionMatrix). + // + // Either way we must pass non-null gamut/transfer arrays: + // ComputeExternalTextureParams dereferences them unconditionally. static const float kIdentityTransferParams[7] = { 1.0f, // G 1.0f, // A @@ -425,17 +488,41 @@ std::shared_ptr GPUDevice::importExternalTexture( 0.0f, // E 0.0f, // F }; + const bool isBiplanar = + frame.pixelFormat == VideoPixelFormat::NV12 && + isBiplanarYuvFormat(texture.GetFormat()); + + wgpu::TextureView plane0; + wgpu::TextureView plane1; wgpu::ExternalTextureDescriptor extDesc{}; if (!label.empty()) { extDesc.label = wgpu::StringView(label.c_str(), label.size()); } - extDesc.plane0 = texture.CreateView(); extDesc.cropOrigin = {0, 0}; extDesc.cropSize = {frame.width, frame.height}; extDesc.apparentSize = {frame.width, frame.height}; extDesc.gamutConversionMatrix = kIdentityGamutMatrix; - extDesc.srcTransferFunctionParameters = kIdentityTransferParams; - extDesc.dstTransferFunctionParameters = kIdentityTransferParams; + if (isBiplanar) { + wgpu::TextureViewDescriptor v0{}; + v0.aspect = wgpu::TextureAspect::Plane0Only; + plane0 = texture.CreateView(&v0); + wgpu::TextureViewDescriptor v1{}; + v1.aspect = wgpu::TextureAspect::Plane1Only; + plane1 = texture.CreateView(&v1); + extDesc.plane0 = plane0; + extDesc.plane1 = plane1; + extDesc.yuvToRgbConversionMatrix = kBT709LimitedToRgb; + extDesc.srcTransferFunctionParameters = kSrgbDecodeParams; + extDesc.dstTransferFunctionParameters = kSrgbEncodeParams; + } else { + plane0 = texture.CreateView(); + extDesc.plane0 = plane0; + extDesc.srcTransferFunctionParameters = kIdentityTransferParams; + extDesc.dstTransferFunctionParameters = kIdentityTransferParams; + } + extDesc.mirrored = descriptor->mirrored.value_or(false); + extDesc.rotation = + toExternalTextureRotation(descriptor->rotation.value_or(0)); auto external = _instance.CreateExternalTexture(&extDesc); if (external == nullptr) { diff --git a/packages/webgpu/cpp/rnwgpu/api/descriptors/GPUExternalTextureDescriptor.h b/packages/webgpu/cpp/rnwgpu/api/descriptors/GPUExternalTextureDescriptor.h index da7affc9b..0cb5ecebc 100644 --- a/packages/webgpu/cpp/rnwgpu/api/descriptors/GPUExternalTextureDescriptor.h +++ b/packages/webgpu/cpp/rnwgpu/api/descriptors/GPUExternalTextureDescriptor.h @@ -17,9 +17,18 @@ namespace rnwgpu { // VideoFrame as the (only) supported source. We don't expose colorSpace yet; // the C++ side picks dst-sRGB and identity gamut, which is the right default // for "render this video frame to a regular sRGB framebuffer". +// +// `rotation` / `mirrored` are a non-spec extension: camera frames (e.g. from +// VisionCamera) arrive in the sensor's native orientation, which differs +// between iOS (CVPixelBuffer) and Android (AHardwareBuffer). Dawn's +// ExternalTextureDescriptor can bake a rotation + horizontal mirror into the +// sampling transform, so the shader sees an upright frame without any extra +// passes. `rotation` is in degrees and must be one of 0 / 90 / 180 / 270. struct GPUExternalTextureDescriptor { std::shared_ptr source; std::optional label; + std::optional rotation; + std::optional mirrored; }; } // namespace rnwgpu @@ -48,6 +57,18 @@ struct JSIConverter> { runtime, prop, false); } } + if (value.hasProperty(runtime, "rotation")) { + auto prop = value.getProperty(runtime, "rotation"); + if (prop.isNumber()) { + result->rotation = prop.asNumber(); + } + } + if (value.hasProperty(runtime, "mirrored")) { + auto prop = value.getProperty(runtime, "mirrored"); + if (prop.isBool()) { + result->mirrored = prop.getBool(); + } + } } return result; } diff --git a/packages/webgpu/src/index.tsx b/packages/webgpu/src/index.tsx index 09d2877c3..6f716ed72 100644 --- a/packages/webgpu/src/index.tsx +++ b/packages/webgpu/src/index.tsx @@ -60,6 +60,16 @@ declare global { createVideoFrameFromNativeBuffer(pointer: bigint): NativeVideoFrame; } + // Non-spec extension: camera frames arrive in the sensor's native + // orientation, which differs between iOS and Android. `rotation` (degrees, + // one of 0/90/180/270) and `mirrored` (horizontal flip) are baked into the + // sampling transform by Dawn, so the shader sees an upright frame. Maps + // directly onto VisionCamera's `frame.orientation` / `frame.isMirrored`. + interface GPUExternalTextureDescriptor { + rotation?: 0 | 90 | 180 | 270; + mirrored?: boolean; + } + // Extend createImageBitmap to accept ArrayBuffer/TypedArray (encoded image bytes) function createImageBitmap( image: ArrayBuffer | ArrayBufferView, From be66dda514ae135ef01ca1e13f581dadbb63beb2 Mon Sep 17 00:00:00 2001 From: William Candillon Date: Mon, 1 Jun 2026 12:07:38 +0200 Subject: [PATCH 33/46] :wrench: --- apps/example/app.json | 4 +- apps/example/src/App.tsx | 5 + apps/example/src/Home.tsx | 4 + .../ImportExternalTexture.tsx | 293 ++++++++++++++++++ .../src/ImportExternalTexture/index.ts | 1 + apps/example/src/Route.ts | 1 + packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp | 35 ++- .../__tests__/ImportExternalTexture.spec.ts | 146 +++++++++ 8 files changed, 477 insertions(+), 12 deletions(-) create mode 100644 apps/example/src/ImportExternalTexture/ImportExternalTexture.tsx create mode 100644 apps/example/src/ImportExternalTexture/index.ts create mode 100644 packages/webgpu/src/__tests__/ImportExternalTexture.spec.ts diff --git a/apps/example/app.json b/apps/example/app.json index 73135e124..defa402c0 100644 --- a/apps/example/app.json +++ b/apps/example/app.json @@ -30,7 +30,9 @@ "metalAPIValidation": false, "infoPlist": { "NSCameraUsageDescription": "$(PRODUCT_NAME) needs access to your Camera.", - "NSMicrophoneUsageDescription": "$(PRODUCT_NAME) needs access to your Microphone." + "NSMicrophoneUsageDescription": "$(PRODUCT_NAME) needs access to your Microphone.", + "NSLocalNetworkUsageDescription": "$(PRODUCT_NAME) needs local network access to connect to the Metro dev server.", + "NSBonjourServices": ["_http._tcp."] } }, "android": { diff --git a/apps/example/src/App.tsx b/apps/example/src/App.tsx index 9ccebcadd..f355f2bf7 100644 --- a/apps/example/src/App.tsx +++ b/apps/example/src/App.tsx @@ -37,6 +37,7 @@ import { AsyncStarvation } from "./Diagnostics/AsyncStarvation"; import { DeviceLostHang } from "./Diagnostics/DeviceLostHang"; import { StorageBufferVertices } from "./StorageBufferVertices"; import { SharedTextureMemory } from "./SharedTextureMemory"; +import { ImportExternalTexture } from "./ImportExternalTexture"; import { VisionCamera } from "./VisionCamera"; // The two lines below are needed by three.js @@ -103,6 +104,10 @@ function App() { name="SharedTextureMemory" component={SharedTextureMemory} /> + diff --git a/apps/example/src/Home.tsx b/apps/example/src/Home.tsx index c1321105f..a3a63731e 100644 --- a/apps/example/src/Home.tsx +++ b/apps/example/src/Home.tsx @@ -131,6 +131,10 @@ export const examples = [ screen: "SharedTextureMemory", title: "🎞️ Shared Texture Memory", }, + { + screen: "ImportExternalTexture", + title: "🎬 Import External Texture", + }, { screen: "VisionCamera", title: "đź“· VisionCamera integration", diff --git a/apps/example/src/ImportExternalTexture/ImportExternalTexture.tsx b/apps/example/src/ImportExternalTexture/ImportExternalTexture.tsx new file mode 100644 index 000000000..3b0e7b7a4 --- /dev/null +++ b/apps/example/src/ImportExternalTexture/ImportExternalTexture.tsx @@ -0,0 +1,293 @@ +import React, { useEffect, useRef, useState } from "react"; +import { PixelRatio, Platform, StyleSheet, Text, View } from "react-native"; +import { + Canvas, + useCanvasRef, + useDevice, + type NativeCanvas, + type NativeVideoFrame, +} from "react-native-wgpu"; + +// This is the SharedTextureMemory demo, rewritten to use +// GPUDevice.importExternalTexture instead of the manual +// importSharedTextureMemory + createTexture + beginAccess/endAccess dance. +// +// The visible result is identical (the same video frame stretched 'cover' over +// the canvas), but the sampling path differs: the source is bound as a +// `texture_external` and read with textureSampleBaseClampToEdge, and Dawn owns +// the shared-memory begin/end-access lifecycle internally. A GPUExternalTexture +// expires once the queue work that used it is submitted, so we re-import one +// every frame. +const SHADER = /* wgsl */ ` +struct VsOut { + @builtin(position) position: vec4f, + @location(0) uv: vec2f, +}; + +struct Uniforms { + // Per-axis scale applied to UVs *around the center* so that the canvas + // samples a sub-rectangle of the texture matching the canvas aspect ratio. + // 'cover' fit: one axis is 1.0, the other is canvasAR / textureAR (or its + // reciprocal), whichever is < 1 — i.e. we crop on the longer axis. + uvScale: vec2f, +}; + +@group(0) @binding(0) var srcTex: texture_external; +@group(0) @binding(1) var srcSampler: sampler; +@group(0) @binding(2) var u: Uniforms; + +@vertex +fn vs_main(@builtin(vertex_index) vid: u32) -> VsOut { + // Full-screen triangle. + var positions = array( + vec2f(-1.0, -3.0), + vec2f(-1.0, 1.0), + vec2f( 3.0, 1.0), + ); + var uvs = array( + vec2f(0.0, 2.0), + vec2f(0.0, 0.0), + vec2f(2.0, 0.0), + ); + var out: VsOut; + out.position = vec4f(positions[vid], 0.0, 1.0); + out.uv = uvs[vid]; + return out; +} + +@fragment +fn fs_main(in: VsOut) -> @location(0) vec4f { + let uv = vec2f(0.5) + (in.uv - vec2f(0.5)) * u.uvScale; + return textureSampleBaseClampToEdge(srcTex, srcSampler, uv); +} +`; + +const REQUIRED_FEATURES = ["rnwebgpu/shared-texture-memory" as GPUFeatureName]; + +export const ImportExternalTexture = () => { + const ref = useCanvasRef(); + const [error, setError] = useState(null); + const rafRef = useRef(null); + + const { device, adapter } = useDevice(undefined, { + requiredFeatures: REQUIRED_FEATURES, + }); + + useEffect(() => { + if (!device) { + return; + } + const missing = REQUIRED_FEATURES.filter((f) => !device.features.has(f)); + if (missing.length > 0) { + setError( + `Device is missing required features [${missing.join(", ")}]. Adapter supports: ${ + adapter + ? [...adapter.features] + .filter((f) => f.toString().startsWith("shared-")) + .join(", ") || "none" + : "n/a" + }`, + ); + return; + } + + const context = ref.current?.getContext("webgpu"); + if (!context) { + return; + } + const canvas = context.canvas as unknown as NativeCanvas; + canvas.width = canvas.clientWidth * PixelRatio.get(); + canvas.height = canvas.clientHeight * PixelRatio.get(); + const presentationFormat = navigator.gpu.getPreferredCanvasFormat(); + context.configure({ + device, + format: presentationFormat, + alphaMode: "premultiplied", + }); + + // Pick a frame source per platform. On iOS we use AVPlayer to stream a + // real video; on Android we don't have a video pipeline yet, so we fall + // back to a single synthetic IOSurface/AHardwareBuffer frame produced by + // RNWebGPU.createTestVideoFrame. The rAF loop below treats a null return + // from copyLatestFrame() as "keep showing the previous frame", which means + // a one-shot source renders correctly without any other change. + interface FrameSource { + copyLatestFrame(): NativeVideoFrame | null; + release(): void; + } + let source: FrameSource; + if (Platform.OS === "ios") { + const VIDEO_URL = + "https://test-videos.co.uk/vids/bigbuckbunny/mp4/h264/1080/Big_Buck_Bunny_1080_10s_5MB.mp4"; + const player = RNWebGPU.createVideoPlayer(VIDEO_URL); + player.play(); + source = { + copyLatestFrame: () => player.copyLatestFrame(), + release: () => player.release(), + }; + } else { + let pending: NativeVideoFrame | null = RNWebGPU.createTestVideoFrame( + 1024, + 1024, + ); + source = { + copyLatestFrame: () => { + const f = pending; + pending = null; + return f; + }, + release: () => { + if (pending) { + pending.release(); + pending = null; + } + }, + }; + } + + const module = device.createShaderModule({ code: SHADER }); + const pipeline = device.createRenderPipeline({ + layout: "auto", + vertex: { module, entryPoint: "vs_main" }, + fragment: { + module, + entryPoint: "fs_main", + targets: [{ format: presentationFormat }], + }, + primitive: { topology: "triangle-list" }, + }); + const sampler = device.createSampler({ + magFilter: "linear", + minFilter: "linear", + }); + // One persistent uniform buffer; rewritten whenever the frame dimensions + // change. + const uniformBuffer = device.createBuffer({ + size: 16, // vec2 padded to 16-byte uniform alignment + usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, + }); + + // 'cover' fit: scale UVs around their center so the longer axis of the + // texture is cropped to match the canvas aspect ratio. + const computeUvScale = (texW: number, texH: number): [number, number] => { + const canvasAR = canvas.width / canvas.height; + const texAR = texW / texH; + if (texAR > canvasAR) { + // Texture is wider than the canvas: crop horizontally. + return [canvasAR / texAR, 1]; + } else { + // Texture is taller than (or equal to) the canvas: crop vertically. + return [1, texAR / canvasAR]; + } + }; + + // Hold the current frame across rAF ticks so that when the video hasn't + // produced a new frame yet (between decoder timestamps), we keep rendering + // the last one rather than dropping to a black screen. + let currentFrame: NativeVideoFrame | null = null; + let lastDims: [number, number] | null = null; + + const render = () => { + const newFrame = source.copyLatestFrame(); + if (newFrame) { + if (currentFrame) { + currentFrame.release(); + } + currentFrame = newFrame; + } + + const encoder = device.createCommandEncoder(); + const pass = encoder.beginRenderPass({ + colorAttachments: [ + { + view: context.getCurrentTexture().createView(), + clearValue: { r: 0, g: 0, b: 0, a: 1 }, + loadOp: "clear", + storeOp: "store", + }, + ], + }); + + if (currentFrame) { + // A GPUExternalTexture expires after each submit, so re-import one + // every tick — even when sampling the same frame as last tick. Dawn + // owns the shared-memory begin/end-access window internally. + let externalTex: GPUExternalTexture | null = null; + try { + externalTex = device.importExternalTexture({ + source: currentFrame, + label: "video-frame", + }); + } catch (e) { + console.warn("[ImportExternalTexture] import failed:", e); + } + + if (externalTex) { + if ( + !lastDims || + lastDims[0] !== currentFrame.width || + lastDims[1] !== currentFrame.height + ) { + const [sx, sy] = computeUvScale( + currentFrame.width, + currentFrame.height, + ); + device.queue.writeBuffer( + uniformBuffer, + 0, + new Float32Array([sx, sy]), + ); + lastDims = [currentFrame.width, currentFrame.height]; + } + const bindGroup = device.createBindGroup({ + layout: pipeline.getBindGroupLayout(0), + entries: [ + { binding: 0, resource: externalTex }, + { binding: 1, resource: sampler }, + { binding: 2, resource: { buffer: uniformBuffer } }, + ], + }); + pass.setPipeline(pipeline); + pass.setBindGroup(0, bindGroup); + pass.draw(3); + } + } + + pass.end(); + device.queue.submit([encoder.finish()]); + context.present(); + rafRef.current = requestAnimationFrame(render); + }; + rafRef.current = requestAnimationFrame(render); + + return () => { + if (rafRef.current !== null) { + cancelAnimationFrame(rafRef.current); + } + if (currentFrame) { + currentFrame.release(); + currentFrame = null; + } + uniformBuffer.destroy(); + source.release(); + }; + }, [device, adapter, ref]); + + if (error) { + return ( + + {error} + + ); + } + return ( + + + + ); +}; + +const styles = StyleSheet.create({ + errorContainer: { flex: 1, padding: 16, justifyContent: "center" }, + errorText: { color: "red", fontSize: 14 }, +}); diff --git a/apps/example/src/ImportExternalTexture/index.ts b/apps/example/src/ImportExternalTexture/index.ts new file mode 100644 index 000000000..d08addd8a --- /dev/null +++ b/apps/example/src/ImportExternalTexture/index.ts @@ -0,0 +1 @@ +export * from "./ImportExternalTexture"; diff --git a/apps/example/src/Route.ts b/apps/example/src/Route.ts index d3068a5d2..5b381edee 100644 --- a/apps/example/src/Route.ts +++ b/apps/example/src/Route.ts @@ -30,5 +30,6 @@ export type Routes = { DeviceLostHang: undefined; StorageBufferVertices: undefined; SharedTextureMemory: undefined; + ImportExternalTexture: undefined; VisionCamera: undefined; }; diff --git a/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp b/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp index a19bbf856..8188cd05e 100644 --- a/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp +++ b/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp @@ -266,6 +266,22 @@ static const float kSrgbEncodeParams[7] = { 0.0f, // F }; +// Identity transfer (y = x). Used when the sampled surface is already in the +// render target's color space: a single-plane BGRA IOSurface, or the Android +// opaque-YCbCr path where the Vulkan sampler already produced RGB. Dawn +// dereferences the transfer-function arrays unconditionally +// (ComputeExternalTextureParams), so these must be non-null even when no +// conversion is wanted. +static const float kIdentityTransferParams[7] = { + 1.0f, // G + 1.0f, // A + 0.0f, // B + 0.0f, // C + 0.0f, // D + 0.0f, // E + 0.0f, // F +}; + // BT.709 limited-range YUV -> R'G'B' as a 3x4 row-major matrix mapping // [Y, Cb, Cr, 1]. Same values the Apple NV12 path computes from the // CVPixelBuffer; used for Android buffers that arrive as a *defined* biplanar @@ -393,12 +409,17 @@ std::shared_ptr GPUDevice::importExternalTexture( extDesc.label = wgpu::StringView(label.c_str(), label.size()); } extDesc.plane0 = plane0; + extDesc.gamutConversionMatrix = kIdentityGamutMatrix; if (isYuv) { extDesc.plane1 = plane1; extDesc.yuvToRgbConversionMatrix = frame.yuvToRgbMatrix; extDesc.srcTransferFunctionParameters = kSrgbDecodeParams; extDesc.dstTransferFunctionParameters = kSrgbEncodeParams; - extDesc.gamutConversionMatrix = kIdentityGamutMatrix; + } else { + // BGRA is already RGB in the target color space; pass it through. Dawn + // dereferences these arrays unconditionally, so they must be non-null. + extDesc.srcTransferFunctionParameters = kIdentityTransferParams; + extDesc.dstTransferFunctionParameters = kIdentityTransferParams; } extDesc.cropOrigin = {0, 0}; extDesc.cropSize = {frame.width, frame.height}; @@ -478,16 +499,8 @@ std::shared_ptr GPUDevice::importExternalTexture( // Tint transform ignores yuvToRgbConversionMatrix). // // Either way we must pass non-null gamut/transfer arrays: - // ComputeExternalTextureParams dereferences them unconditionally. - static const float kIdentityTransferParams[7] = { - 1.0f, // G - 1.0f, // A - 0.0f, // B - 0.0f, // C - 0.0f, // D - 0.0f, // E - 0.0f, // F - }; + // ComputeExternalTextureParams dereferences them unconditionally + // (kIdentityTransferParams is defined at file scope). const bool isBiplanar = frame.pixelFormat == VideoPixelFormat::NV12 && isBiplanarYuvFormat(texture.GetFormat()); diff --git a/packages/webgpu/src/__tests__/ImportExternalTexture.spec.ts b/packages/webgpu/src/__tests__/ImportExternalTexture.spec.ts new file mode 100644 index 000000000..ad15319d6 --- /dev/null +++ b/packages/webgpu/src/__tests__/ImportExternalTexture.spec.ts @@ -0,0 +1,146 @@ +import { checkImage, client, encodeImage } from "./setup"; + +interface BitmapData { + data: number[]; + width: number; + height: number; + format: string; +} + +type EvalResult = + | { kind: "skip"; reason: string } + | { kind: "fail"; reason: string } + | ({ kind: "ok" } & BitmapData); + +describe("ImportExternalTexture", () => { + it("imports a test frame as an external texture and samples it", async () => { + const result = await client.eval, EvalResult>( + ({ device, gpu, ctx, canvas }) => { + // The umbrella feature backs importExternalTexture's IOSurface / + // AHardwareBuffer import. The Chrome reference run and fallback adapters + // won't advertise it, so that's the *only* legitimate skip. Anything + // past this point is a real failure. + const FEATURE = "rnwebgpu/shared-texture-memory"; + if (!device.features.has(FEATURE as GPUFeatureName)) { + return { + kind: "skip", + reason: `${FEATURE} not enabled on this device`, + }; + } + if (typeof RNWebGPU?.createTestVideoFrame !== "function") { + return { + kind: "skip", + reason: "RNWebGPU.createTestVideoFrame is unavailable", + }; + } + + try { + const frame = RNWebGPU.createTestVideoFrame(256, 256); + // Unlike importSharedTextureMemory, there is no createTexture / + // beginAccess / endAccess to manage: the GPUExternalTexture owns the + // shared-memory access window and keeps the frame alive internally. + const externalTexture = device.importExternalTexture({ + // createTestVideoFrame returns our NativeVideoFrame; the native + // importExternalTexture binding accepts it, but the spec type wants + // a WebCodecs VideoFrame, so cast to satisfy the signature. + source: frame as unknown as VideoFrame, + label: "test-frame", + }); + + const module = device.createShaderModule({ + code: /* wgsl */ ` + struct VsOut { + @builtin(position) position: vec4f, + @location(0) uv: vec2f, + }; + + @vertex fn vs(@builtin(vertex_index) vid: u32) -> VsOut { + var positions = array( + vec2f(-1.0, -3.0), + vec2f(-1.0, 1.0), + vec2f( 3.0, 1.0), + ); + var uvs = array( + vec2f(0.0, 2.0), + vec2f(0.0, 0.0), + vec2f(2.0, 0.0), + ); + var out: VsOut; + out.position = vec4f(positions[vid], 0.0, 1.0); + out.uv = uvs[vid]; + return out; + } + + @group(0) @binding(0) var srcTex: texture_external; + @group(0) @binding(1) var srcSampler: sampler; + + @fragment fn fs(in: VsOut) -> @location(0) vec4f { + return textureSampleBaseClampToEdge(srcTex, srcSampler, in.uv); + } + `, + }); + const pipeline = device.createRenderPipeline({ + layout: "auto", + vertex: { module, entryPoint: "vs" }, + fragment: { + module, + entryPoint: "fs", + targets: [{ format: gpu.getPreferredCanvasFormat() }], + }, + primitive: { topology: "triangle-list" }, + }); + const sampler = device.createSampler({ + magFilter: "linear", + minFilter: "linear", + }); + const bindGroup = device.createBindGroup({ + layout: pipeline.getBindGroupLayout(0), + entries: [ + { binding: 0, resource: externalTexture }, + { binding: 1, resource: sampler }, + ], + }); + + const encoder = device.createCommandEncoder(); + const pass = encoder.beginRenderPass({ + colorAttachments: [ + { + view: ctx.getCurrentTexture().createView(), + clearValue: { r: 0, g: 0, b: 0, a: 1 }, + loadOp: "clear", + storeOp: "store", + }, + ], + }); + pass.setPipeline(pipeline); + pass.setBindGroup(0, bindGroup); + pass.draw(3); + pass.end(); + device.queue.submit([encoder.finish()]); + + return canvas.getImageData().then((image: BitmapData) => { + // Safe to release now: all GPU work referencing the frame is + // submitted and the pixels have been read back. + frame.release(); + return { kind: "ok" as const, ...image }; + }); + } catch (e) { + return { + kind: "fail", + reason: `${(e as Error).message ?? e}`, + }; + } + }, + ); + + if (result.kind === "skip") { + console.log(`ImportExternalTexture: skipping (${result.reason})`); + return; + } + if (result.kind === "fail") { + throw new Error(`ImportExternalTexture: ${result.reason}`); + } + const image = encodeImage(result); + checkImage(image, "snapshots/import-external-texture.png"); + }); +}); From 9be78c7140c8ee57afcc927ddb69ad1459864e5d Mon Sep 17 00:00:00 2001 From: William Candillon Date: Mon, 1 Jun 2026 13:15:42 +0200 Subject: [PATCH 34/46] :wrench: --- .../example/src/VisionCamera/VisionCamera.tsx | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/apps/example/src/VisionCamera/VisionCamera.tsx b/apps/example/src/VisionCamera/VisionCamera.tsx index 155a80f4e..d88e80fee 100644 --- a/apps/example/src/VisionCamera/VisionCamera.tsx +++ b/apps/example/src/VisionCamera/VisionCamera.tsx @@ -209,13 +209,18 @@ fn fs_main(in: VsOut) -> @location(0) vec4f { `; // Android delivers camera frames as an external-format (opaque) YUV buffer. -// Dawn copies the AHardwareBuffer's suggested YCbCr model verbatim, and on the -// devices we've seen that resolves to identity, so the external sample comes -// back as raw [Y, Cb, Cr]. After Dawn's rotation the frame is also mirrored on -// both axes relative to the canvas (Android buffer origin), so we flip X and Y -// here. iOS goes through the native two-plane path, which already converts and -// orients, so this correction is Android-only. The prelude is prepended to -// every shader module that samples the camera (main pass + blur prepass). +// Dawn's OpaqueYCbCrAndroidForExternalTexture path samples it through a Vulkan +// YCbCr conversion that is hard-coded to RGB_IDENTITY (Dawn's +// SamplerVk.cpp::GetYCbCrForTextureView; see crbug.com/497675620), so the +// external sample comes back as raw [Y, Cb, Cr] on *every* device — this is by +// design, not a driver quirk — and we do the BT.709 YUV->RGB ourselves below. +// (Dawn's own SharedTextureMemoryOpaqueYCbCrAndroidForExternalTexture +// .NoopSampleY8Cb8Cr8AHB test asserts the same raw passthrough.) The frame also +// comes out mirrored on both axes relative to the canvas (Android buffer +// origin), so we flip X and Y. iOS goes through the native two-plane path, +// which already converts and orients, so this correction is Android-only. The +// prelude is prepended to every shader module that samples the camera (main +// pass + blur prepass). const CAMERA_PRELUDE = /* wgsl */ ` const CAMERA_IS_YUV: bool = ${Platform.OS === "android"}; const CAMERA_FLIP_X: bool = ${Platform.OS === "android"}; @@ -232,9 +237,9 @@ fn cameraCoord(uv: vec2f) -> vec2f { return c; } -// BT.709 limited-range YUV -> RGB. The sampled channels are [Y, Cb, Cr] when -// the driver's YCbCr conversion is identity (the Android opaque path); a no-op -// passthrough otherwise. +// BT.709 limited-range YUV -> RGB. On the Android opaque path the sampled +// channels are always raw [Y, Cb, Cr] (Dawn forces an RGB_IDENTITY Vulkan +// conversion); a no-op passthrough on every other platform. fn cameraDecode(c: vec4f) -> vec4f { if (!CAMERA_IS_YUV) { return c; From 77f0e9cb16def20ffe61637985a772485722ae17 Mon Sep 17 00:00:00 2001 From: William Candillon Date: Mon, 1 Jun 2026 13:24:14 +0200 Subject: [PATCH 35/46] :wrench: --- .../example/src/VisionCamera/VisionCamera.tsx | 236 +----------------- apps/example/src/VisionCamera/shaders.ts | 219 ++++++++++++++++ 2 files changed, 232 insertions(+), 223 deletions(-) create mode 100644 apps/example/src/VisionCamera/shaders.ts diff --git a/apps/example/src/VisionCamera/VisionCamera.tsx b/apps/example/src/VisionCamera/VisionCamera.tsx index d88e80fee..6092c5969 100644 --- a/apps/example/src/VisionCamera/VisionCamera.tsx +++ b/apps/example/src/VisionCamera/VisionCamera.tsx @@ -1,4 +1,4 @@ -import React, { useEffect } from "react"; +import React, { useCallback, useEffect, useMemo, useState } from "react"; import { Linking, PixelRatio, @@ -30,6 +30,7 @@ import { PIXELATE_BLOCKS, type Modes, } from "./features"; +import { CAMERA_PRELUDE, SHADER } from "./shaders"; // Camera frame → SharedTextureMemory (NV12 biplanar) → GPUExternalTexture → // textureSampleBaseClampToEdge with hardware YUV/sRGB conversion → chromatic @@ -40,219 +41,8 @@ import { // startup) registers a Worklets custom serializer for GPUDevice / canvas / // pipeline / sampler / buffer, so the closure references below auto-box on // the way into the worklet and auto-unbox on the way out. - -const SHADER = /* wgsl */ ` -struct VsOut { - @builtin(position) position: vec4f, - @location(0) uv: vec2f, -}; - -struct Uniforms { - // x, y: 'cover'-fit UV scale around (0.5, 0.5). - // z: chromatic aberration offset in UV units (0 disables). - // w: pixelate block size in UV units (0 disables). - params: vec4f, - // x: effect (0 off, 1 gray, 2 sepia, 3 invert, 4 vibrant) - // y: tint (0 off, 1 warm, 2 cool) - // z: vignette (0 off, 1 on) - // w: blurMode (0 off, 1 strong - blurred everywhere (prepass bakes - // cover-fit), 2 overlay - blurred backdrop + sharp card) - modes: vec4u, -}; - -@group(0) @binding(0) var srcTex: texture_external; -@group(0) @binding(1) var srcSampler: sampler; -@group(0) @binding(2) var u: Uniforms; -@group(0) @binding(3) var blurredTex: texture_2d; - -@vertex -fn vs_main(@builtin(vertex_index) vid: u32) -> VsOut { - var positions = array( - vec2f(-1.0, -3.0), - vec2f(-1.0, 1.0), - vec2f( 3.0, 1.0), - ); - var uvs = array( - vec2f(0.0, 2.0), - vec2f(0.0, 0.0), - vec2f(2.0, 0.0), - ); - var out: VsOut; - out.position = vec4f(positions[vid], 0.0, 1.0); - out.uv = uvs[vid]; - return out; -} - -fn snap(uv: vec2f, block: f32) -> vec2f { - if (block <= 0.0) { - return uv; - } - return (floor(uv / block) + vec2f(0.5)) * block; -} - -fn sampleExternal(uv: vec2f, block: f32) -> vec4f { - return cameraDecode( - textureSampleBaseClampToEdge(srcTex, srcSampler, cameraCoord(snap(uv, block))), - ); -} - -fn sampleBlurred(uv: vec2f, block: f32) -> vec4f { - return textureSampleLevel( - blurredTex, - srcSampler, - clamp(snap(uv, block), vec2f(0.0), vec2f(1.0)), - 0.0, - ); -} - -// RGB-split sample of the external camera with cover-fit + optional pixelate. -fn rgbSplitExternal(uv: vec2f, aberration: f32, block: f32) -> vec3f { - let r = sampleExternal(uv + vec2f( aberration, 0.0), block).r; - let g = sampleExternal(uv, block).g; - let b = sampleExternal(uv + vec2f(-aberration, 0.0), block).b; - return vec3f(r, g, b); -} - -// RGB-split sample of the pre-blurred 2D texture (cover-fit baked in). -fn rgbSplitBlurred(uv: vec2f, aberration: f32, block: f32) -> vec3f { - let r = sampleBlurred(uv + vec2f( aberration, 0.0), block).r; - let g = sampleBlurred(uv, block).g; - let b = sampleBlurred(uv + vec2f(-aberration, 0.0), block).b; - return vec3f(r, g, b); -} - -fn applyEffect(rgb: vec3f, mode: u32) -> vec3f { - if (mode == 1u) { - let l = dot(rgb, vec3f(0.2126, 0.7152, 0.0722)); - return vec3f(l); - } - if (mode == 2u) { - return vec3f( - dot(rgb, vec3f(0.393, 0.769, 0.189)), - dot(rgb, vec3f(0.349, 0.686, 0.168)), - dot(rgb, vec3f(0.272, 0.534, 0.131)) - ); - } - if (mode == 3u) { - return vec3f(1.0) - rgb; - } - if (mode == 4u) { - let l = dot(rgb, vec3f(0.2126, 0.7152, 0.0722)); - let sat = mix(vec3f(l), rgb, 1.55); - return clamp((sat - 0.5) * 1.18 + 0.5, vec3f(0.0), vec3f(1.0)); - } - return rgb; -} - -fn applyTint(rgb: vec3f, mode: u32) -> vec3f { - if (mode == 1u) { - return clamp(rgb * vec3f(1.10, 1.02, 0.86), vec3f(0.0), vec3f(1.0)); - } - if (mode == 2u) { - return clamp(rgb * vec3f(0.86, 0.98, 1.16), vec3f(0.0), vec3f(1.0)); - } - return rgb; -} - -@fragment -fn fs_main(in: VsOut) -> @location(0) vec4f { - let uvScale = u.params.xy; - let aberration = u.params.z; - let pixelate = u.params.w; - let effect = u.modes.x; - let tint = u.modes.y; - let vignette = u.modes.z; - let blurMode = u.modes.w; - - // Overlay card geometry. NDC-space rect, the camera image is cover-fit - // inside the card (same uvScale as the off path), the blurred backdrop - // fills outside. - let overlayPadding = 0.08; - let edgeAA = 0.004; - - var color: vec3f; - if (blurMode == 1u) { - // Strong: prepass already baked cover-fit, so feed in.uv straight in. - color = rgbSplitBlurred(in.uv, aberration, pixelate); - } else if (blurMode == 2u) { - // Overlay: per-fragment edge factor 0 strictly inside the card, - // 1 strictly outside, smoothstep band for AA. Uniform within the branch - // (blurMode came from the uniform buffer), so calling - // textureSampleBaseClampToEdge on the external texture is allowed. - let cardHalf = vec2f(0.5 - overlayPadding); - let p = abs(in.uv - vec2f(0.5)); - let edgeDist = max(p.x - cardHalf.x, p.y - cardHalf.y); - let outside = smoothstep(-edgeAA, edgeAA, edgeDist); - - let cardUv = (in.uv - vec2f(overlayPadding)) / - (1.0 - 2.0 * overlayPadding); - let sharpUv = vec2f(0.5) + (cardUv - vec2f(0.5)) * uvScale; - let sharp = rgbSplitExternal(sharpUv, aberration, pixelate); - let backdrop = rgbSplitBlurred(in.uv, aberration, pixelate); - color = mix(sharp, backdrop, outside); - } else { - let uv = vec2f(0.5) + (in.uv - vec2f(0.5)) * uvScale; - color = rgbSplitExternal(uv, aberration, pixelate); - } - - color = applyEffect(color, effect); - color = applyTint(color, tint); - - if (vignette == 1u) { - let d = distance(in.uv, vec2f(0.5)); - let v = 1.0 - smoothstep(0.35, 0.85, d); - color = color * v; - } - - return vec4f(color, 1.0); -} -`; - -// Android delivers camera frames as an external-format (opaque) YUV buffer. -// Dawn's OpaqueYCbCrAndroidForExternalTexture path samples it through a Vulkan -// YCbCr conversion that is hard-coded to RGB_IDENTITY (Dawn's -// SamplerVk.cpp::GetYCbCrForTextureView; see crbug.com/497675620), so the -// external sample comes back as raw [Y, Cb, Cr] on *every* device — this is by -// design, not a driver quirk — and we do the BT.709 YUV->RGB ourselves below. -// (Dawn's own SharedTextureMemoryOpaqueYCbCrAndroidForExternalTexture -// .NoopSampleY8Cb8Cr8AHB test asserts the same raw passthrough.) The frame also -// comes out mirrored on both axes relative to the canvas (Android buffer -// origin), so we flip X and Y. iOS goes through the native two-plane path, -// which already converts and orients, so this correction is Android-only. The -// prelude is prepended to every shader module that samples the camera (main -// pass + blur prepass). -const CAMERA_PRELUDE = /* wgsl */ ` -const CAMERA_IS_YUV: bool = ${Platform.OS === "android"}; -const CAMERA_FLIP_X: bool = ${Platform.OS === "android"}; -const CAMERA_FLIP_Y: bool = ${Platform.OS === "android"}; - -fn cameraCoord(uv: vec2f) -> vec2f { - var c = uv; - if (CAMERA_FLIP_X) { - c.x = 1.0 - c.x; - } - if (CAMERA_FLIP_Y) { - c.y = 1.0 - c.y; - } - return c; -} - -// BT.709 limited-range YUV -> RGB. On the Android opaque path the sampled -// channels are always raw [Y, Cb, Cr] (Dawn forces an RGB_IDENTITY Vulkan -// conversion); a no-op passthrough on every other platform. -fn cameraDecode(c: vec4f) -> vec4f { - if (!CAMERA_IS_YUV) { - return c; - } - let y = c.r - 0.0627451; - let cb = c.g - 0.5; - let cr = c.b - 0.5; - let r = 1.164384 * y + 1.792741 * cr; - let g = 1.164384 * y - 0.213249 * cb - 0.532909 * cr; - let b = 1.164384 * y + 2.112402 * cb; - return vec4f(clamp(vec3f(r, g, b), vec3f(0.0), vec3f(1.0)), 1.0); -} -`; +// +// The WGSL (SHADER + CAMERA_PRELUDE) lives in ./shaders. const REQUIRED_FEATURES: GPUFeatureName[] = [ "rnwebgpu/shared-texture-memory" as GPUFeatureName, @@ -307,12 +97,12 @@ export const VisionCamera = () => { const CameraView = () => { const ref = useCanvasRef(); - const [gpu, setGpu] = React.useState<{ + const [gpu, setGpu] = useState<{ adapter: GPUAdapter; device: GPUDevice; } | null>(null); - const [deviceError, setDeviceError] = React.useState(null); - React.useEffect(() => { + const [deviceError, setDeviceError] = useState(null); + useEffect(() => { let cancelled = false; (async () => { try { @@ -374,7 +164,7 @@ const CameraView = () => { // Pick back camera if available, otherwise front, otherwise anything. The // iOS simulator returns an empty list since there are no cameras, in which // case we surface a clear error rather than letting useCamera throw. - const cameraDevice = React.useMemo( + const cameraDevice = useMemo( () => devices.find((d) => d.position === "back") ?? devices.find((d) => d.position === "front") ?? @@ -382,7 +172,7 @@ const CameraView = () => { [devices], ); - const [pipelineState, setPipelineState] = React.useState<{ + const [pipelineState, setPipelineState] = useState<{ pipeline: GPURenderPipeline; sampler: GPUSampler; uniformBuffer: GPUBuffer; @@ -401,9 +191,9 @@ const CameraView = () => { blurWidth: number; blurHeight: number; } | null>(null); - const [error, setError] = React.useState(null); - const [modes, setModes] = React.useState(INITIAL_MODES); - const cycle = React.useCallback((key: keyof Modes, optionsCount: number) => { + const [error, setError] = useState(null); + const [modes, setModes] = useState(INITIAL_MODES); + const cycle = useCallback((key: keyof Modes, optionsCount: number) => { setModes((prev) => ({ ...prev, [key]: (prev[key] + 1) % optionsCount })); }, []); @@ -601,7 +391,7 @@ const CameraView = () => { // box. Worklets serializes the box once on closure creation, so flipping // .seen mutates the worklet-side copy; we use it strictly to avoid spamming // metro logs and don't rely on main-thread visibility. - const logBox = React.useMemo(() => ({ seen: false }), []); + const logBox = useMemo(() => ({ seen: false }), []); const frameOutput = useFrameOutput({ pixelFormat: "native", // zero-copy, gives us NV12 IOSurfaces on iOS onFrame: (frame) => { diff --git a/apps/example/src/VisionCamera/shaders.ts b/apps/example/src/VisionCamera/shaders.ts new file mode 100644 index 000000000..e93520107 --- /dev/null +++ b/apps/example/src/VisionCamera/shaders.ts @@ -0,0 +1,219 @@ +import { Platform } from "react-native"; + +// Main camera-effects shader: samples the imported external texture (camera +// frame) with cover-fit + optional chromatic aberration / pixelate, optionally +// mixes in the pre-blurred backdrop, then applies effect / tint / vignette. +// `cameraCoord` / `cameraDecode` come from CAMERA_PRELUDE, which is prepended +// at shader-module creation time. +export const SHADER = /* wgsl */ ` +struct VsOut { + @builtin(position) position: vec4f, + @location(0) uv: vec2f, +}; + +struct Uniforms { + // x, y: 'cover'-fit UV scale around (0.5, 0.5). + // z: chromatic aberration offset in UV units (0 disables). + // w: pixelate block size in UV units (0 disables). + params: vec4f, + // x: effect (0 off, 1 gray, 2 sepia, 3 invert, 4 vibrant) + // y: tint (0 off, 1 warm, 2 cool) + // z: vignette (0 off, 1 on) + // w: blurMode (0 off, 1 strong - blurred everywhere (prepass bakes + // cover-fit), 2 overlay - blurred backdrop + sharp card) + modes: vec4u, +}; + +@group(0) @binding(0) var srcTex: texture_external; +@group(0) @binding(1) var srcSampler: sampler; +@group(0) @binding(2) var u: Uniforms; +@group(0) @binding(3) var blurredTex: texture_2d; + +@vertex +fn vs_main(@builtin(vertex_index) vid: u32) -> VsOut { + var positions = array( + vec2f(-1.0, -3.0), + vec2f(-1.0, 1.0), + vec2f( 3.0, 1.0), + ); + var uvs = array( + vec2f(0.0, 2.0), + vec2f(0.0, 0.0), + vec2f(2.0, 0.0), + ); + var out: VsOut; + out.position = vec4f(positions[vid], 0.0, 1.0); + out.uv = uvs[vid]; + return out; +} + +fn snap(uv: vec2f, block: f32) -> vec2f { + if (block <= 0.0) { + return uv; + } + return (floor(uv / block) + vec2f(0.5)) * block; +} + +fn sampleExternal(uv: vec2f, block: f32) -> vec4f { + return cameraDecode( + textureSampleBaseClampToEdge(srcTex, srcSampler, cameraCoord(snap(uv, block))), + ); +} + +fn sampleBlurred(uv: vec2f, block: f32) -> vec4f { + return textureSampleLevel( + blurredTex, + srcSampler, + clamp(snap(uv, block), vec2f(0.0), vec2f(1.0)), + 0.0, + ); +} + +// RGB-split sample of the external camera with cover-fit + optional pixelate. +fn rgbSplitExternal(uv: vec2f, aberration: f32, block: f32) -> vec3f { + let r = sampleExternal(uv + vec2f( aberration, 0.0), block).r; + let g = sampleExternal(uv, block).g; + let b = sampleExternal(uv + vec2f(-aberration, 0.0), block).b; + return vec3f(r, g, b); +} + +// RGB-split sample of the pre-blurred 2D texture (cover-fit baked in). +fn rgbSplitBlurred(uv: vec2f, aberration: f32, block: f32) -> vec3f { + let r = sampleBlurred(uv + vec2f( aberration, 0.0), block).r; + let g = sampleBlurred(uv, block).g; + let b = sampleBlurred(uv + vec2f(-aberration, 0.0), block).b; + return vec3f(r, g, b); +} + +fn applyEffect(rgb: vec3f, mode: u32) -> vec3f { + if (mode == 1u) { + let l = dot(rgb, vec3f(0.2126, 0.7152, 0.0722)); + return vec3f(l); + } + if (mode == 2u) { + return vec3f( + dot(rgb, vec3f(0.393, 0.769, 0.189)), + dot(rgb, vec3f(0.349, 0.686, 0.168)), + dot(rgb, vec3f(0.272, 0.534, 0.131)) + ); + } + if (mode == 3u) { + return vec3f(1.0) - rgb; + } + if (mode == 4u) { + let l = dot(rgb, vec3f(0.2126, 0.7152, 0.0722)); + let sat = mix(vec3f(l), rgb, 1.55); + return clamp((sat - 0.5) * 1.18 + 0.5, vec3f(0.0), vec3f(1.0)); + } + return rgb; +} + +fn applyTint(rgb: vec3f, mode: u32) -> vec3f { + if (mode == 1u) { + return clamp(rgb * vec3f(1.10, 1.02, 0.86), vec3f(0.0), vec3f(1.0)); + } + if (mode == 2u) { + return clamp(rgb * vec3f(0.86, 0.98, 1.16), vec3f(0.0), vec3f(1.0)); + } + return rgb; +} + +@fragment +fn fs_main(in: VsOut) -> @location(0) vec4f { + let uvScale = u.params.xy; + let aberration = u.params.z; + let pixelate = u.params.w; + let effect = u.modes.x; + let tint = u.modes.y; + let vignette = u.modes.z; + let blurMode = u.modes.w; + + // Overlay card geometry. NDC-space rect, the camera image is cover-fit + // inside the card (same uvScale as the off path), the blurred backdrop + // fills outside. + let overlayPadding = 0.08; + let edgeAA = 0.004; + + var color: vec3f; + if (blurMode == 1u) { + // Strong: prepass already baked cover-fit, so feed in.uv straight in. + color = rgbSplitBlurred(in.uv, aberration, pixelate); + } else if (blurMode == 2u) { + // Overlay: per-fragment edge factor 0 strictly inside the card, + // 1 strictly outside, smoothstep band for AA. Uniform within the branch + // (blurMode came from the uniform buffer), so calling + // textureSampleBaseClampToEdge on the external texture is allowed. + let cardHalf = vec2f(0.5 - overlayPadding); + let p = abs(in.uv - vec2f(0.5)); + let edgeDist = max(p.x - cardHalf.x, p.y - cardHalf.y); + let outside = smoothstep(-edgeAA, edgeAA, edgeDist); + + let cardUv = (in.uv - vec2f(overlayPadding)) / + (1.0 - 2.0 * overlayPadding); + let sharpUv = vec2f(0.5) + (cardUv - vec2f(0.5)) * uvScale; + let sharp = rgbSplitExternal(sharpUv, aberration, pixelate); + let backdrop = rgbSplitBlurred(in.uv, aberration, pixelate); + color = mix(sharp, backdrop, outside); + } else { + let uv = vec2f(0.5) + (in.uv - vec2f(0.5)) * uvScale; + color = rgbSplitExternal(uv, aberration, pixelate); + } + + color = applyEffect(color, effect); + color = applyTint(color, tint); + + if (vignette == 1u) { + let d = distance(in.uv, vec2f(0.5)); + let v = 1.0 - smoothstep(0.35, 0.85, d); + color = color * v; + } + + return vec4f(color, 1.0); +} +`; + +// Android delivers camera frames as an external-format (opaque) YUV buffer. +// Dawn's OpaqueYCbCrAndroidForExternalTexture path samples it through a Vulkan +// YCbCr conversion that is hard-coded to RGB_IDENTITY (Dawn's +// SamplerVk.cpp::GetYCbCrForTextureView; see crbug.com/497675620), so the +// external sample comes back as raw [Y, Cb, Cr] on *every* device — this is by +// design, not a driver quirk — and we do the BT.709 YUV->RGB ourselves below. +// (Dawn's own SharedTextureMemoryOpaqueYCbCrAndroidForExternalTexture +// .NoopSampleY8Cb8Cr8AHB test asserts the same raw passthrough.) The frame also +// comes out mirrored on both axes relative to the canvas (Android buffer +// origin), so we flip X and Y. iOS goes through the native two-plane path, +// which already converts and orients, so this correction is Android-only. The +// prelude is prepended to every shader module that samples the camera (main +// pass + blur prepass). +export const CAMERA_PRELUDE = /* wgsl */ ` +const CAMERA_IS_YUV: bool = ${Platform.OS === "android"}; +const CAMERA_FLIP_X: bool = ${Platform.OS === "android"}; +const CAMERA_FLIP_Y: bool = ${Platform.OS === "android"}; + +fn cameraCoord(uv: vec2f) -> vec2f { + var c = uv; + if (CAMERA_FLIP_X) { + c.x = 1.0 - c.x; + } + if (CAMERA_FLIP_Y) { + c.y = 1.0 - c.y; + } + return c; +} + +// BT.709 limited-range YUV -> RGB. On the Android opaque path the sampled +// channels are always raw [Y, Cb, Cr] (Dawn forces an RGB_IDENTITY Vulkan +// conversion); a no-op passthrough on every other platform. +fn cameraDecode(c: vec4f) -> vec4f { + if (!CAMERA_IS_YUV) { + return c; + } + let y = c.r - 0.0627451; + let cb = c.g - 0.5; + let cr = c.b - 0.5; + let r = 1.164384 * y + 1.792741 * cr; + let g = 1.164384 * y - 0.213249 * cb - 0.532909 * cr; + let b = 1.164384 * y + 2.112402 * cb; + return vec4f(clamp(vec3f(r, g, b), vec3f(0.0), vec3f(1.0)), 1.0); +} +`; From 322f92568f356274bf040fcef7abd9af545fc776 Mon Sep 17 00:00:00 2001 From: William Candillon Date: Mon, 1 Jun 2026 13:55:56 +0200 Subject: [PATCH 36/46] :wrench: --- .../ImportExternalTexture.tsx | 76 ++++++++----------- .../SharedTextureMemory.tsx | 2 +- apps/example/src/Tests.tsx | 2 +- .../example/src/VisionCamera/VisionCamera.tsx | 2 +- packages/webgpu/README.md | 10 +-- packages/webgpu/cpp/rnwgpu/api/GPUAdapter.cpp | 2 +- packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp | 2 +- packages/webgpu/cpp/rnwgpu/api/RnFeatures.h | 12 +-- .../api/descriptors/GPUDeviceDescriptor.h | 4 +- .../__tests__/ImportExternalTexture.spec.ts | 2 +- .../src/__tests__/SharedTextureMemory.spec.ts | 2 +- 11 files changed, 53 insertions(+), 63 deletions(-) diff --git a/apps/example/src/ImportExternalTexture/ImportExternalTexture.tsx b/apps/example/src/ImportExternalTexture/ImportExternalTexture.tsx index 3b0e7b7a4..577b221dc 100644 --- a/apps/example/src/ImportExternalTexture/ImportExternalTexture.tsx +++ b/apps/example/src/ImportExternalTexture/ImportExternalTexture.tsx @@ -6,6 +6,7 @@ import { useDevice, type NativeCanvas, type NativeVideoFrame, + type VideoPlayer, } from "react-native-wgpu"; // This is the SharedTextureMemory demo, rewritten to use @@ -62,7 +63,12 @@ fn fs_main(in: VsOut) -> @location(0) vec4f { } `; -const REQUIRED_FEATURES = ["rnwebgpu/shared-texture-memory" as GPUFeatureName]; +// We never call importSharedTextureMemory directly here, but +// importExternalTexture is implemented on top of it natively (it imports the +// frame's IOSurface / AHardwareBuffer as shared texture memory, then wraps that +// as an external texture). So the device must enable this umbrella feature, or +// the import throws "ImportSharedTextureMemory returned null". +const REQUIRED_FEATURES = ["rnwebgpu/native-texture" as GPUFeatureName]; export const ImportExternalTexture = () => { const ref = useCanvasRef(); @@ -105,44 +111,20 @@ export const ImportExternalTexture = () => { alphaMode: "premultiplied", }); - // Pick a frame source per platform. On iOS we use AVPlayer to stream a - // real video; on Android we don't have a video pipeline yet, so we fall - // back to a single synthetic IOSurface/AHardwareBuffer frame produced by - // RNWebGPU.createTestVideoFrame. The rAF loop below treats a null return - // from copyLatestFrame() as "keep showing the previous frame", which means - // a one-shot source renders correctly without any other change. - interface FrameSource { - copyLatestFrame(): NativeVideoFrame | null; - release(): void; - } - let source: FrameSource; + // Pick a frame source per platform. On iOS we stream a real video via + // AVPlayer; elsewhere we don't have a video pipeline yet, so we use a + // single synthetic IOSurface/AHardwareBuffer frame. A VideoPlayer exposes + // copyLatestFrame() (a fresh frame each tick) while a NativeVideoFrame does + // not — the render loop tells them apart with that property. + let source: VideoPlayer | NativeVideoFrame; if (Platform.OS === "ios") { const VIDEO_URL = "https://test-videos.co.uk/vids/bigbuckbunny/mp4/h264/1080/Big_Buck_Bunny_1080_10s_5MB.mp4"; const player = RNWebGPU.createVideoPlayer(VIDEO_URL); player.play(); - source = { - copyLatestFrame: () => player.copyLatestFrame(), - release: () => player.release(), - }; + source = player; } else { - let pending: NativeVideoFrame | null = RNWebGPU.createTestVideoFrame( - 1024, - 1024, - ); - source = { - copyLatestFrame: () => { - const f = pending; - pending = null; - return f; - }, - release: () => { - if (pending) { - pending.release(); - pending = null; - } - }, - }; + source = RNWebGPU.createTestVideoFrame(1024, 1024); } const module = device.createShaderModule({ code: SHADER }); @@ -181,19 +163,23 @@ export const ImportExternalTexture = () => { } }; - // Hold the current frame across rAF ticks so that when the video hasn't - // produced a new frame yet (between decoder timestamps), we keep rendering - // the last one rather than dropping to a black screen. - let currentFrame: NativeVideoFrame | null = null; + // Hold the current frame across rAF ticks. For a VideoPlayer we pull the + // latest frame each tick (keeping the previous one when the decoder hasn't + // produced a new one yet, to avoid a black flash); for a one-shot + // NativeVideoFrame we just keep re-importing the same frame. + let currentFrame: NativeVideoFrame | null = + "copyLatestFrame" in source ? null : source; let lastDims: [number, number] | null = null; const render = () => { - const newFrame = source.copyLatestFrame(); - if (newFrame) { - if (currentFrame) { - currentFrame.release(); + if ("copyLatestFrame" in source) { + const newFrame = source.copyLatestFrame(); + if (newFrame) { + if (currentFrame) { + currentFrame.release(); + } + currentFrame = newFrame; } - currentFrame = newFrame; } const encoder = device.createCommandEncoder(); @@ -269,7 +255,11 @@ export const ImportExternalTexture = () => { currentFrame = null; } uniformBuffer.destroy(); - source.release(); + // For the player, release it; the one-shot frame was released above as + // currentFrame (same object), so don't double-release it here. + if ("copyLatestFrame" in source) { + source.release(); + } }; }, [device, adapter, ref]); diff --git a/apps/example/src/SharedTextureMemory/SharedTextureMemory.tsx b/apps/example/src/SharedTextureMemory/SharedTextureMemory.tsx index e91c69367..4ce1f315c 100644 --- a/apps/example/src/SharedTextureMemory/SharedTextureMemory.tsx +++ b/apps/example/src/SharedTextureMemory/SharedTextureMemory.tsx @@ -53,7 +53,7 @@ fn fs_main(in: VsOut) -> @location(0) vec4f { } `; -const REQUIRED_FEATURES = ["rnwebgpu/shared-texture-memory" as GPUFeatureName]; +const REQUIRED_FEATURES = ["rnwebgpu/native-texture" as GPUFeatureName]; export const SharedTextureMemory = () => { const ref = useCanvasRef(); diff --git a/apps/example/src/Tests.tsx b/apps/example/src/Tests.tsx index c6ae7ae1b..63372ff3e 100644 --- a/apps/example/src/Tests.tsx +++ b/apps/example/src/Tests.tsx @@ -17,7 +17,7 @@ const presentationFormat = navigator.gpu.getPreferredCanvasFormat(); // Umbrella feature owned by react-native-wgpu: expands to the platform's // Dawn feature pair natively and appears on adapter.features when supported. const OPTIONAL_SHARED_TEXTURE_MEMORY_FEATURES = [ - "rnwebgpu/shared-texture-memory" as GPUFeatureName, + "rnwebgpu/native-texture" as GPUFeatureName, ]; export const Tests = ({ assets: { di3D, saturn, moon } }: AssetProps) => { diff --git a/apps/example/src/VisionCamera/VisionCamera.tsx b/apps/example/src/VisionCamera/VisionCamera.tsx index 6092c5969..dcb652477 100644 --- a/apps/example/src/VisionCamera/VisionCamera.tsx +++ b/apps/example/src/VisionCamera/VisionCamera.tsx @@ -45,7 +45,7 @@ import { CAMERA_PRELUDE, SHADER } from "./shaders"; // The WGSL (SHADER + CAMERA_PRELUDE) lives in ./shaders. const REQUIRED_FEATURES: GPUFeatureName[] = [ - "rnwebgpu/shared-texture-memory" as GPUFeatureName, + "rnwebgpu/native-texture" as GPUFeatureName, "dawn-multi-planar-formats" as GPUFeatureName, ]; diff --git a/packages/webgpu/README.md b/packages/webgpu/README.md index b3ec7eb8e..57379699e 100644 --- a/packages/webgpu/README.md +++ b/packages/webgpu/README.md @@ -226,19 +226,19 @@ device.queue.copyExternalImageToTexture( React Native WebGPU exposes Dawn's `SharedTextureMemory` so you can import a native pixel surface (an `IOSurface`-backed `CVPixelBuffer` on iOS, an `AHardwareBuffer` on Android) as a sampleable `GPUTexture` without copying pixels through the CPU. This is the path you want for camera frames, video frames, or anything coming out of a hardware producer. -We expose a single umbrella feature name, `"rnwebgpu/shared-texture-memory"`. Request it at device creation. +We expose a single umbrella feature name, `"rnwebgpu/native-texture"`. Request it at device creation. ```tsx -import type { VideoFrame } from "react-native-wgpu"; +import type { NativeVideoFrame } from "react-native-wgpu"; -const FEATURE = "rnwebgpu/shared-texture-memory" as GPUFeatureName; +const FEATURE = "rnwebgpu/native-texture" as GPUFeatureName; const adapter = await navigator.gpu.requestAdapter(); const requiredFeatures = adapter!.features.has(FEATURE) ? [FEATURE] : []; const device = await adapter!.requestDevice({ requiredFeatures }); -// `frame` here is a VideoFrame whose .handle is the native surface -// (IOSurfaceRef / AHardwareBuffer*). VideoFrames are produced by helpers +// `frame` here is a NativeVideoFrame whose .handle is the native surface +// (IOSurfaceRef / AHardwareBuffer*). NativeVideoFrames are produced by helpers // like RNWebGPU.createVideoPlayer or RNWebGPU.createTestVideoFrame, or by // any third-party module that hands you a compatible native pointer. const memory = device.importSharedTextureMemory({ diff --git a/packages/webgpu/cpp/rnwgpu/api/GPUAdapter.cpp b/packages/webgpu/cpp/rnwgpu/api/GPUAdapter.cpp index 57f77b625..c816f6328 100644 --- a/packages/webgpu/cpp/rnwgpu/api/GPUAdapter.cpp +++ b/packages/webgpu/cpp/rnwgpu/api/GPUAdapter.cpp @@ -178,7 +178,7 @@ std::unordered_set GPUAdapter::getFeatures() { result.insert(name); } } - maybeSynthesizeRnSharedTextureMemoryFeature(enabled, result); + maybeSynthesizeRnNativeTextureFeature(enabled, result); return result; } diff --git a/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp b/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp index 8188cd05e..a5e54be6d 100644 --- a/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp +++ b/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp @@ -772,7 +772,7 @@ std::unordered_set GPUDevice::getFeatures() { convertEnumToJSUnion(feature, &name); result.insert(name); } - maybeSynthesizeRnSharedTextureMemoryFeature(enabled, result); + maybeSynthesizeRnNativeTextureFeature(enabled, result); return result; } diff --git a/packages/webgpu/cpp/rnwgpu/api/RnFeatures.h b/packages/webgpu/cpp/rnwgpu/api/RnFeatures.h index 4c60fba23..4577ace66 100644 --- a/packages/webgpu/cpp/rnwgpu/api/RnFeatures.h +++ b/packages/webgpu/cpp/rnwgpu/api/RnFeatures.h @@ -12,14 +12,14 @@ namespace rnwgpu { // platform-specific pair of Dawn features. The prefix is intentional: this // string is not part of the WebGPU spec, it is our API surface for the // "import a native surface as a sampleable texture" capability. -inline constexpr const char *kRnSharedTextureMemoryFeature = - "rnwebgpu/shared-texture-memory"; +inline constexpr const char *kRnNativeTextureFeature = + "rnwebgpu/native-texture"; // Dawn features that back the umbrella on the current platform. Empty on // platforms where the capability is not available, in which case the umbrella // behaves as a no-op (it won't appear in adapter.features and asking for it // in requiredFeatures expands to nothing). -inline std::vector rnSharedTextureMemoryBackingFeatures() { +inline std::vector rnNativeTextureBackingFeatures() { #if defined(__APPLE__) return {wgpu::FeatureName::SharedTextureMemoryIOSurface, wgpu::FeatureName::SharedFenceMTLSharedEvent}; @@ -35,10 +35,10 @@ inline std::vector rnSharedTextureMemoryBackingFeatures() { // umbrella name to `out`. Used by adapter.features / device.features so JS // callers can see (and call .has on) the same name they pass in. inline void -maybeSynthesizeRnSharedTextureMemoryFeature( +maybeSynthesizeRnNativeTextureFeature( const std::unordered_set &enabled, std::unordered_set &out) { - auto backing = rnSharedTextureMemoryBackingFeatures(); + auto backing = rnNativeTextureBackingFeatures(); if (backing.empty()) { return; } @@ -47,7 +47,7 @@ maybeSynthesizeRnSharedTextureMemoryFeature( return; } } - out.insert(kRnSharedTextureMemoryFeature); + out.insert(kRnNativeTextureFeature); } } // namespace rnwgpu diff --git a/packages/webgpu/cpp/rnwgpu/api/descriptors/GPUDeviceDescriptor.h b/packages/webgpu/cpp/rnwgpu/api/descriptors/GPUDeviceDescriptor.h index 7e531807c..e18f73e16 100644 --- a/packages/webgpu/cpp/rnwgpu/api/descriptors/GPUDeviceDescriptor.h +++ b/packages/webgpu/cpp/rnwgpu/api/descriptors/GPUDeviceDescriptor.h @@ -50,8 +50,8 @@ template <> struct JSIConverter> { auto str = elementValue.asString(runtime).utf8(runtime); // Expand react-native-wgpu's umbrella feature into the platform's // backing Dawn features before they reach RequestDevice. - if (str == kRnSharedTextureMemoryFeature) { - for (auto f : rnSharedTextureMemoryBackingFeatures()) { + if (str == kRnNativeTextureFeature) { + for (auto f : rnNativeTextureBackingFeatures()) { vector.emplace_back(f); } continue; diff --git a/packages/webgpu/src/__tests__/ImportExternalTexture.spec.ts b/packages/webgpu/src/__tests__/ImportExternalTexture.spec.ts index ad15319d6..e506240aa 100644 --- a/packages/webgpu/src/__tests__/ImportExternalTexture.spec.ts +++ b/packages/webgpu/src/__tests__/ImportExternalTexture.spec.ts @@ -20,7 +20,7 @@ describe("ImportExternalTexture", () => { // AHardwareBuffer import. The Chrome reference run and fallback adapters // won't advertise it, so that's the *only* legitimate skip. Anything // past this point is a real failure. - const FEATURE = "rnwebgpu/shared-texture-memory"; + const FEATURE = "rnwebgpu/native-texture"; if (!device.features.has(FEATURE as GPUFeatureName)) { return { kind: "skip", diff --git a/packages/webgpu/src/__tests__/SharedTextureMemory.spec.ts b/packages/webgpu/src/__tests__/SharedTextureMemory.spec.ts index 2e1e3fad4..797be07a8 100644 --- a/packages/webgpu/src/__tests__/SharedTextureMemory.spec.ts +++ b/packages/webgpu/src/__tests__/SharedTextureMemory.spec.ts @@ -20,7 +20,7 @@ describe("SharedTextureMemory", () => { // reference run and fallback adapters won't advertise it. Anything // else past this point is a real failure and must surface as a test // failure, not a silent skip. - const FEATURE = "rnwebgpu/shared-texture-memory"; + const FEATURE = "rnwebgpu/native-texture"; if (!device.features.has(FEATURE as GPUFeatureName)) { return { kind: "skip", From dd5d61f92ee636b6f13dd6252df9412cb0db30c5 Mon Sep 17 00:00:00 2001 From: William Candillon Date: Mon, 1 Jun 2026 15:08:50 +0200 Subject: [PATCH 37/46] :wrench: --- packages/webgpu/cpp/rnwgpu/api/Convertors.h | 33 ++-- .../src/__tests__/ExternalTexture.spec.ts | 157 ++++++++++++++++++ .../src/__tests__/SharedTextureMemory.spec.ts | 143 ++++++++++++++++ 3 files changed, 323 insertions(+), 10 deletions(-) diff --git a/packages/webgpu/cpp/rnwgpu/api/Convertors.h b/packages/webgpu/cpp/rnwgpu/api/Convertors.h index 8d4e29bd1..2422d5020 100644 --- a/packages/webgpu/cpp/rnwgpu/api/Convertors.h +++ b/packages/webgpu/cpp/rnwgpu/api/Convertors.h @@ -252,13 +252,24 @@ class Convertor { [[nodiscard]] bool Convert(wgpu::BindGroupLayoutEntry &out, const GPUBindGroupLayoutEntry &in) { - return Convert(out.binding, in.binding) && - Convert(out.visibility, in.visibility) && - Convert(out.buffer, in.buffer) && Convert(out.sampler, in.sampler) && - Convert(out.texture, in.texture) && - Convert(out.storageTexture, in.storageTexture); - // no external textures here - //&& Convert(out.externalTexture, in.externalTexture); + out = {}; + if (!Convert(out.binding, in.binding) || + !Convert(out.visibility, in.visibility) || + !Convert(out.buffer, in.buffer) || !Convert(out.sampler, in.sampler) || + !Convert(out.texture, in.texture) || + !Convert(out.storageTexture, in.storageTexture)) { + return false; + } + if (in.externalTexture.has_value() && + in.externalTexture.value() != nullptr) { + // External texture layouts bind via a chained struct rather than a + // direct field on BindGroupLayoutEntry. The chained struct must outlive + // the BindGroupLayoutEntry until Device::CreateBindGroupLayout returns, + // so we allocate it on the Convertor's arena. + auto *chain = Allocate(); + out.nextInChain = chain; + } + return true; } [[nodiscard]] bool Convert(wgpu::BlendComponent &out, @@ -422,9 +433,11 @@ class Convertor { } [[nodiscard]] bool Convert(wgpu::ExternalTextureBindingLayout &out, - const GPUExternalTextureBindingLayout &in) { - // no external textures at the moment - return false; + const GPUExternalTextureBindingLayout & /*in*/) { + // ExternalTextureBindingLayout carries no fields of its own; its presence + // (as a chained struct) is what marks the entry as an external texture. + out = {}; + return true; } [[nodiscard]] bool Convert(wgpu::ConstantEntry &out, const std::string &key, diff --git a/packages/webgpu/src/__tests__/ExternalTexture.spec.ts b/packages/webgpu/src/__tests__/ExternalTexture.spec.ts index 47976ec4b..6151abc31 100644 --- a/packages/webgpu/src/__tests__/ExternalTexture.spec.ts +++ b/packages/webgpu/src/__tests__/ExternalTexture.spec.ts @@ -1,5 +1,17 @@ import { checkImage, client, encodeImage } from "./setup"; +interface BitmapData { + data: number[]; + width: number; + height: number; + format: string; +} + +type EvalResult = + | { kind: "skip"; reason: string } + | { kind: "fail"; reason: string } + | ({ kind: "ok" } & BitmapData); + describe("External Textures", () => { it("Simple (1)", async () => { const result = await client.eval( @@ -405,4 +417,149 @@ describe("External Textures", () => { // This test catches the bug where std::optional was checked incorrectly checkImage(image, "snapshots/f2.png"); }); + + // Regression test for createBindGroupLayout with a `texture_external` entry. + // Every other path uses layout: "auto", which never builds an explicit + // BindGroupLayoutEntry for an external texture, so the native conversion that + // chains ExternalTextureBindingLayout went untested. Here we build the layout + // ourselves (externalTexture: {}) and render a GPUExternalTexture through it. + // Device-only: needs a native frame, so it skips on the web reference. + it("samples a GPUExternalTexture through an explicit bind group layout", async () => { + const result = await client.eval, EvalResult>( + ({ device, gpu, ctx, canvas }) => { + const FEATURE = "rnwebgpu/native-texture"; + if (!device.features.has(FEATURE as GPUFeatureName)) { + return { + kind: "skip", + reason: `${FEATURE} not enabled on this device`, + }; + } + if (typeof RNWebGPU?.createTestVideoFrame !== "function") { + return { + kind: "skip", + reason: "RNWebGPU.createTestVideoFrame is unavailable", + }; + } + + try { + const frame = RNWebGPU.createTestVideoFrame(256, 256); + const externalTexture = device.importExternalTexture({ + // createTestVideoFrame returns our NativeVideoFrame; the native + // binding accepts it, but the spec type wants a WebCodecs + // VideoFrame, so cast to satisfy the signature. + source: frame as unknown as VideoFrame, + label: "test-frame", + }); + + const module = device.createShaderModule({ + code: /* wgsl */ ` + struct VsOut { + @builtin(position) position: vec4f, + @location(0) uv: vec2f, + }; + + @vertex fn vs(@builtin(vertex_index) vid: u32) -> VsOut { + var positions = array( + vec2f(-1.0, -3.0), + vec2f(-1.0, 1.0), + vec2f( 3.0, 1.0), + ); + var uvs = array( + vec2f(0.0, 2.0), + vec2f(0.0, 0.0), + vec2f(2.0, 0.0), + ); + var out: VsOut; + out.position = vec4f(positions[vid], 0.0, 1.0); + out.uv = uvs[vid]; + return out; + } + + @group(0) @binding(0) var srcTex: texture_external; + @group(0) @binding(1) var srcSampler: sampler; + + @fragment fn fs(in: VsOut) -> @location(0) vec4f { + return textureSampleBaseClampToEdge(srcTex, srcSampler, in.uv); + } + `, + }); + // The whole point of the test: an explicit external-texture layout. + const bindGroupLayout = device.createBindGroupLayout({ + entries: [ + { + binding: 0, + visibility: GPUShaderStage.FRAGMENT, + externalTexture: {}, + }, + { binding: 1, visibility: GPUShaderStage.FRAGMENT, sampler: {} }, + ], + }); + const pipeline = device.createRenderPipeline({ + layout: device.createPipelineLayout({ + bindGroupLayouts: [bindGroupLayout], + }), + vertex: { module, entryPoint: "vs" }, + fragment: { + module, + entryPoint: "fs", + targets: [{ format: gpu.getPreferredCanvasFormat() }], + }, + primitive: { topology: "triangle-list" }, + }); + const sampler = device.createSampler({ + magFilter: "linear", + minFilter: "linear", + }); + const bindGroup = device.createBindGroup({ + layout: bindGroupLayout, + entries: [ + { binding: 0, resource: externalTexture }, + { binding: 1, resource: sampler }, + ], + }); + + const encoder = device.createCommandEncoder(); + const pass = encoder.beginRenderPass({ + colorAttachments: [ + { + view: ctx.getCurrentTexture().createView(), + clearValue: { r: 0, g: 0, b: 0, a: 1 }, + loadOp: "clear", + storeOp: "store", + }, + ], + }); + pass.setPipeline(pipeline); + pass.setBindGroup(0, bindGroup); + pass.draw(3); + pass.end(); + device.queue.submit([encoder.finish()]); + + return canvas.getImageData().then((image: BitmapData) => { + frame.release(); + return { kind: "ok" as const, ...image }; + }); + } catch (e) { + return { + kind: "fail", + reason: `${(e as Error).message ?? e}`, + }; + } + }, + ); + + if (result.kind === "skip") { + console.log( + `ExternalTexture (explicit layout): skipping (${result.reason})`, + ); + return; + } + if (result.kind === "fail") { + throw new Error(`ExternalTexture (explicit layout): ${result.reason}`); + } + const image = encodeImage(result); + // Same render as the auto-layout path in ImportExternalTexture.spec.ts, so + // it must match that snapshot bit-for-bit. + checkImage(image, "snapshots/import-external-texture.png"); + }); }); diff --git a/packages/webgpu/src/__tests__/SharedTextureMemory.spec.ts b/packages/webgpu/src/__tests__/SharedTextureMemory.spec.ts index 797be07a8..325f95932 100644 --- a/packages/webgpu/src/__tests__/SharedTextureMemory.spec.ts +++ b/packages/webgpu/src/__tests__/SharedTextureMemory.spec.ts @@ -145,4 +145,147 @@ describe("SharedTextureMemory", () => { const image = encodeImage(result); checkImage(image, "snapshots/shared-texture-memory.png"); }); + + // Same as above, but with an *explicit* bind group layout + // (device.createBindGroupLayout + createPipelineLayout) instead of + // layout: "auto". This exercises the native BindGroupLayoutEntry conversion + // path, which "auto" layouts bypass entirely. + it("samples a shared texture through an explicit bind group layout", async () => { + const result = await client.eval, EvalResult>( + ({ device, gpu, ctx, canvas }) => { + const FEATURE = "rnwebgpu/native-texture"; + if (!device.features.has(FEATURE as GPUFeatureName)) { + return { + kind: "skip", + reason: `${FEATURE} not enabled on this device`, + }; + } + if (typeof RNWebGPU?.createTestVideoFrame !== "function") { + return { + kind: "skip", + reason: "RNWebGPU.createTestVideoFrame is unavailable", + }; + } + + try { + const frame = RNWebGPU.createTestVideoFrame(256, 256); + const memory = device.importSharedTextureMemory({ + handle: frame.handle, + label: "test-frame", + }); + const texture = memory.createTexture(); + if (!memory.beginAccess(texture, true)) { + frame.release(); + return { kind: "fail", reason: `beginAccess returned false` }; + } + + const module = device.createShaderModule({ + code: /* wgsl */ ` + struct VsOut { + @builtin(position) position: vec4f, + @location(0) uv: vec2f, + }; + + @vertex fn vs(@builtin(vertex_index) vid: u32) -> VsOut { + var positions = array( + vec2f(-1.0, -3.0), + vec2f(-1.0, 1.0), + vec2f( 3.0, 1.0), + ); + var uvs = array( + vec2f(0.0, 2.0), + vec2f(0.0, 0.0), + vec2f(2.0, 0.0), + ); + var out: VsOut; + out.position = vec4f(positions[vid], 0.0, 1.0); + out.uv = uvs[vid]; + return out; + } + + @group(0) @binding(0) var srcTex: texture_2d; + @group(0) @binding(1) var srcSampler: sampler; + + @fragment fn fs(in: VsOut) -> @location(0) vec4f { + return textureSample(srcTex, srcSampler, in.uv); + } + `, + }); + const bindGroupLayout = device.createBindGroupLayout({ + entries: [ + { binding: 0, visibility: GPUShaderStage.FRAGMENT, texture: {} }, + { binding: 1, visibility: GPUShaderStage.FRAGMENT, sampler: {} }, + ], + }); + const pipeline = device.createRenderPipeline({ + layout: device.createPipelineLayout({ + bindGroupLayouts: [bindGroupLayout], + }), + vertex: { module, entryPoint: "vs" }, + fragment: { + module, + entryPoint: "fs", + targets: [{ format: gpu.getPreferredCanvasFormat() }], + }, + primitive: { topology: "triangle-list" }, + }); + const sampler = device.createSampler({ + magFilter: "linear", + minFilter: "linear", + }); + const bindGroup = device.createBindGroup({ + layout: bindGroupLayout, + entries: [ + { binding: 0, resource: texture.createView() }, + { binding: 1, resource: sampler }, + ], + }); + + const encoder = device.createCommandEncoder(); + const pass = encoder.beginRenderPass({ + colorAttachments: [ + { + view: ctx.getCurrentTexture().createView(), + clearValue: { r: 0, g: 0, b: 0, a: 1 }, + loadOp: "clear", + storeOp: "store", + }, + ], + }); + pass.setPipeline(pipeline); + pass.setBindGroup(0, bindGroup); + pass.draw(3); + pass.end(); + device.queue.submit([encoder.finish()]); + + return canvas.getImageData().then((image: BitmapData) => { + memory.endAccess(texture); + texture.destroy(); + frame.release(); + return { kind: "ok" as const, ...image }; + }); + } catch (e) { + return { + kind: "fail", + reason: `${(e as Error).message ?? e}`, + }; + } + }, + ); + + if (result.kind === "skip") { + console.log( + `SharedTextureMemory (explicit layout): skipping (${result.reason})`, + ); + return; + } + if (result.kind === "fail") { + throw new Error( + `SharedTextureMemory (explicit layout): ${result.reason}`, + ); + } + const image = encodeImage(result); + // Identical render to the auto-layout case above. + checkImage(image, "snapshots/shared-texture-memory.png"); + }); }); From 6f5a00ea20eaceccb2a1435bf093a428abab0278 Mon Sep 17 00:00:00 2001 From: William Candillon Date: Mon, 1 Jun 2026 17:07:04 +0200 Subject: [PATCH 38/46] :wrench: --- packages/webgpu/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/webgpu/package.json b/packages/webgpu/package.json index 961528fb3..07a48b53e 100644 --- a/packages/webgpu/package.json +++ b/packages/webgpu/package.json @@ -1,6 +1,6 @@ { "name": "react-native-wgpu", - "version": "0.5.12", + "version": "0.5.13", "description": "React Native WebGPU", "main": "lib/commonjs/index", "module": "lib/module/index", From 5b229874de99c7b70eea8fd6799056337dc6878c Mon Sep 17 00:00:00 2001 From: William Candillon Date: Mon, 1 Jun 2026 17:15:09 +0200 Subject: [PATCH 39/46] :wrench: --- .github/workflows/ci.yml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 847ff4bcb..f081b3f0b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,20 @@ jobs: with: github_token: ${{ secrets.GITHUB_TOKEN }} + # `git clean` (checkout clean: true) wipes node_modules/Pods/build so the + # native side rebuilds fresh, but Metro's Haste/transform cache lives + # outside the repo (in $TMPDIR) and survives between runs. After a + # reanimated/worklets bump that stale cache serves JS built by the old + # Babel transform, which shows up as "mismatch between native worklet + # version and JS one". Clear it so the freshly-installed worklets aren't + # shadowed by stale transforms. + - name: Reset React Native caches + run: | + watchman watch-del-all 2>/dev/null || true + TMP="$(node -e 'console.log(require("os").tmpdir())')" + rm -rf "$TMP"/metro-* "$TMP"/haste-map-* 2>/dev/null || true + rm -rf /tmp/metro-* /tmp/haste-map-* 2>/dev/null || true + - name: Lint files run: yarn lint @@ -45,7 +59,10 @@ jobs: - name: Start Package Manager working-directory: apps/example - run: CI=true yarn start & + # --reset-cache: force Metro to re-transform from the freshly-installed + # node_modules instead of reusing the cache that survives `git clean`. + # This is the key guard against the native/JS worklet version mismatch. + run: CI=true yarn start --reset-cache & - name: Build example for iOS working-directory: apps/example/ios @@ -85,6 +102,9 @@ jobs: - name: Install and launch app on Simulator run: | + # Remove any previous build so a stale binary never lingers on the + # persistent self-hosted simulator. + xcrun simctl uninstall booted com.microsoft.ReactTestApp || true xcrun simctl install booted apps/example/ios/build/Build/Products/Debug-iphonesimulator/ReactTestApp.app xcrun simctl launch booted com.microsoft.ReactTestApp From 0600883038efa3299da1d690249ef47a53bba471 Mon Sep 17 00:00:00 2001 From: William Candillon Date: Mon, 1 Jun 2026 18:16:49 +0200 Subject: [PATCH 40/46] :wrench: --- README.md | 51 ++- .../ImportExternalTexture.tsx | 13 +- .../example/src/VisionCamera/VisionCamera.tsx | 4 + packages/webgpu/README.md | 41 +++ packages/webgpu/android/CMakeLists.txt | 1 + packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp | 322 +---------------- .../cpp/rnwgpu/api/GPUExternalTexture.cpp | 335 ++++++++++++++++++ .../cpp/rnwgpu/api/GPUExternalTexture.h | 32 +- .../src/__tests__/ExternalTexture.spec.ts | 3 + .../__tests__/ImportExternalTexture.spec.ts | 3 + packages/webgpu/src/index.tsx | 11 + 11 files changed, 485 insertions(+), 331 deletions(-) create mode 100644 packages/webgpu/cpp/rnwgpu/api/GPUExternalTexture.cpp diff --git a/README.md b/README.md index b3ec7eb8e..37038b65d 100644 --- a/README.md +++ b/README.md @@ -226,19 +226,19 @@ device.queue.copyExternalImageToTexture( React Native WebGPU exposes Dawn's `SharedTextureMemory` so you can import a native pixel surface (an `IOSurface`-backed `CVPixelBuffer` on iOS, an `AHardwareBuffer` on Android) as a sampleable `GPUTexture` without copying pixels through the CPU. This is the path you want for camera frames, video frames, or anything coming out of a hardware producer. -We expose a single umbrella feature name, `"rnwebgpu/shared-texture-memory"`. Request it at device creation. +We expose a single umbrella feature name, `"rnwebgpu/native-texture"`. Request it at device creation. ```tsx -import type { VideoFrame } from "react-native-wgpu"; +import type { NativeVideoFrame } from "react-native-wgpu"; -const FEATURE = "rnwebgpu/shared-texture-memory" as GPUFeatureName; +const FEATURE = "rnwebgpu/native-texture" as GPUFeatureName; const adapter = await navigator.gpu.requestAdapter(); const requiredFeatures = adapter!.features.has(FEATURE) ? [FEATURE] : []; const device = await adapter!.requestDevice({ requiredFeatures }); -// `frame` here is a VideoFrame whose .handle is the native surface -// (IOSurfaceRef / AHardwareBuffer*). VideoFrames are produced by helpers +// `frame` here is a NativeVideoFrame whose .handle is the native surface +// (IOSurfaceRef / AHardwareBuffer*). NativeVideoFrames are produced by helpers // like RNWebGPU.createVideoPlayer or RNWebGPU.createTestVideoFrame, or by // any third-party module that hands you a compatible native pointer. const memory = device.importSharedTextureMemory({ @@ -257,6 +257,47 @@ frame.release(); `beginAccess`/`endAccess` bracket the GPU's read window on the shared surface. Pass `initialized: true` when the producer has already written meaningful pixels (the typical video/camera case) and `false` when the next pass will fully overwrite the texture. +### Importing External Textures + +`GPUDevice.importExternalTexture` is the higher-level path for sampling a native surface. Instead of managing `SharedTextureMemory` + `createTexture` + `beginAccess`/`endAccess` yourself, you hand it a `NativeVideoFrame` and get back a `GPUExternalTexture` you bind as a `texture_external` and read with `textureSampleBaseClampToEdge`. Dawn does the YUV→RGB conversion in hardware for biplanar (NV12) surfaces, so this is the path you want for camera and video frames. It uses the same `"rnwebgpu/native-texture"` feature. + +```tsx +const FEATURE = "rnwebgpu/native-texture" as GPUFeatureName; +const adapter = await navigator.gpu.requestAdapter(); +const requiredFeatures = adapter!.features.has(FEATURE) ? [FEATURE] : []; +const device = await adapter!.requestDevice({ requiredFeatures }); + +const render = () => { + // A GPUExternalTexture expires once the queue work that used it is submitted, + // so re-import one every frame. + const externalTexture = device.importExternalTexture({ + source: frame, // a NativeVideoFrame + label: "video-frame", + }); + + const bindGroup = device.createBindGroup({ + layout: pipeline.getBindGroupLayout(0), + entries: [ + { binding: 0, resource: externalTexture }, + { binding: 1, resource: sampler }, + ], + }); + + // ... encode a pass that samples `externalTexture`, then: + device.queue.submit([encoder.finish()]); + + // Release the surface's access window right after the submit that sampled it. + externalTexture.destroy(); + context.present(); +}; +``` + +Camera frames arrive in the sensor's native orientation, so `importExternalTexture` also accepts non-spec `rotation` (`0` | `90` | `180` | `270`, in degrees) and `mirrored` (horizontal flip) options. Dawn bakes them into the sampling transform, so the shader sees an upright frame. They map directly onto VisionCamera's `frame.orientation` / `frame.isMirrored`. + +#### Calling `destroy()` + +A `GPUExternalTexture` keeps an open access window on the underlying native surface until the wrapper is destroyed. On the Web `importExternalTexture` is core and the lifetime is handled for you; here the window is tied to the JavaScript object's lifetime. Call `externalTexture.destroy()` right after the `queue.submit()` that sampled it (never before) to release the surface back to its producer immediately. `destroy()` is idempotent, and the surface is also released when the object is garbage-collected, but relying on GC can starve a producer's buffer pool (e.g. an `AVPlayer`'s recycled `IOSurface`s) and pile up GPU resources, so prefer the explicit call in a render loop. + ### Reanimated Integration React Native WebGPU supports running WebGPU rendering on the UI thread using [React Native Reanimated](https://docs.swmansion.com/react-native-reanimated/) and [React Native Worklets](https://github.com/margelo/react-native-worklets). diff --git a/apps/example/src/ImportExternalTexture/ImportExternalTexture.tsx b/apps/example/src/ImportExternalTexture/ImportExternalTexture.tsx index 577b221dc..b8cb82d8c 100644 --- a/apps/example/src/ImportExternalTexture/ImportExternalTexture.tsx +++ b/apps/example/src/ImportExternalTexture/ImportExternalTexture.tsx @@ -194,11 +194,13 @@ export const ImportExternalTexture = () => { ], }); + // A GPUExternalTexture expires after each submit, so re-import one every + // tick, even when sampling the same frame as last tick. Unlike on the + // web, the shared-memory begin/end-access window is tied to this + // wrapper's lifetime, so we destroy() it right after submit (below) to + // release the surface promptly instead of waiting for GC. + let externalTex: GPUExternalTexture | null = null; if (currentFrame) { - // A GPUExternalTexture expires after each submit, so re-import one - // every tick — even when sampling the same frame as last tick. Dawn - // owns the shared-memory begin/end-access window internally. - let externalTex: GPUExternalTexture | null = null; try { externalTex = device.importExternalTexture({ source: currentFrame, @@ -241,6 +243,9 @@ export const ImportExternalTexture = () => { pass.end(); device.queue.submit([encoder.finish()]); + // Now that the work sampling it has been submitted, end the external + // texture's access window so the frame's surface is released promptly. + externalTex?.destroy(); context.present(); rafRef.current = requestAnimationFrame(render); }; diff --git a/apps/example/src/VisionCamera/VisionCamera.tsx b/apps/example/src/VisionCamera/VisionCamera.tsx index dcb652477..6b5152eb4 100644 --- a/apps/example/src/VisionCamera/VisionCamera.tsx +++ b/apps/example/src/VisionCamera/VisionCamera.tsx @@ -605,6 +605,10 @@ const CameraView = () => { pass.draw(3); pass.end(); device.queue.submit([encoder.finish()]); + // The work sampling it is submitted, so end the external texture's + // access window now to release the camera frame's surface promptly + // (don't wait for GC, which would starve the frame buffer pool). + externalTex.destroy(); context.present(); } finally { videoFrame.release(); diff --git a/packages/webgpu/README.md b/packages/webgpu/README.md index 57379699e..37038b65d 100644 --- a/packages/webgpu/README.md +++ b/packages/webgpu/README.md @@ -257,6 +257,47 @@ frame.release(); `beginAccess`/`endAccess` bracket the GPU's read window on the shared surface. Pass `initialized: true` when the producer has already written meaningful pixels (the typical video/camera case) and `false` when the next pass will fully overwrite the texture. +### Importing External Textures + +`GPUDevice.importExternalTexture` is the higher-level path for sampling a native surface. Instead of managing `SharedTextureMemory` + `createTexture` + `beginAccess`/`endAccess` yourself, you hand it a `NativeVideoFrame` and get back a `GPUExternalTexture` you bind as a `texture_external` and read with `textureSampleBaseClampToEdge`. Dawn does the YUV→RGB conversion in hardware for biplanar (NV12) surfaces, so this is the path you want for camera and video frames. It uses the same `"rnwebgpu/native-texture"` feature. + +```tsx +const FEATURE = "rnwebgpu/native-texture" as GPUFeatureName; +const adapter = await navigator.gpu.requestAdapter(); +const requiredFeatures = adapter!.features.has(FEATURE) ? [FEATURE] : []; +const device = await adapter!.requestDevice({ requiredFeatures }); + +const render = () => { + // A GPUExternalTexture expires once the queue work that used it is submitted, + // so re-import one every frame. + const externalTexture = device.importExternalTexture({ + source: frame, // a NativeVideoFrame + label: "video-frame", + }); + + const bindGroup = device.createBindGroup({ + layout: pipeline.getBindGroupLayout(0), + entries: [ + { binding: 0, resource: externalTexture }, + { binding: 1, resource: sampler }, + ], + }); + + // ... encode a pass that samples `externalTexture`, then: + device.queue.submit([encoder.finish()]); + + // Release the surface's access window right after the submit that sampled it. + externalTexture.destroy(); + context.present(); +}; +``` + +Camera frames arrive in the sensor's native orientation, so `importExternalTexture` also accepts non-spec `rotation` (`0` | `90` | `180` | `270`, in degrees) and `mirrored` (horizontal flip) options. Dawn bakes them into the sampling transform, so the shader sees an upright frame. They map directly onto VisionCamera's `frame.orientation` / `frame.isMirrored`. + +#### Calling `destroy()` + +A `GPUExternalTexture` keeps an open access window on the underlying native surface until the wrapper is destroyed. On the Web `importExternalTexture` is core and the lifetime is handled for you; here the window is tied to the JavaScript object's lifetime. Call `externalTexture.destroy()` right after the `queue.submit()` that sampled it (never before) to release the surface back to its producer immediately. `destroy()` is idempotent, and the surface is also released when the object is garbage-collected, but relying on GC can starve a producer's buffer pool (e.g. an `AVPlayer`'s recycled `IOSurface`s) and pile up GPU resources, so prefer the explicit call in a render loop. + ### Reanimated Integration React Native WebGPU supports running WebGPU rendering on the UI thread using [React Native Reanimated](https://docs.swmansion.com/react-native-reanimated/) and [React Native Worklets](https://github.com/margelo/react-native-worklets). diff --git a/packages/webgpu/android/CMakeLists.txt b/packages/webgpu/android/CMakeLists.txt index fcab0baa1..3ef33bb7b 100644 --- a/packages/webgpu/android/CMakeLists.txt +++ b/packages/webgpu/android/CMakeLists.txt @@ -38,6 +38,7 @@ add_library(${PACKAGE_NAME} SHARED ../cpp/rnwgpu/api/GPUQuerySet.cpp ../cpp/rnwgpu/api/GPUTexture.cpp ../cpp/rnwgpu/api/GPUSharedTextureMemory.cpp + ../cpp/rnwgpu/api/GPUExternalTexture.cpp ../cpp/rnwgpu/api/GPURenderBundleEncoder.cpp ../cpp/rnwgpu/api/GPURenderPassEncoder.cpp ../cpp/rnwgpu/api/GPURenderPipeline.cpp diff --git a/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp b/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp index a5e54be6d..18df1d07b 100644 --- a/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp +++ b/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp @@ -1,6 +1,5 @@ #include "GPUDevice.h" -#include #include #include #include @@ -235,325 +234,12 @@ std::shared_ptr GPUDevice::createPipelineLayout( _instance.CreatePipelineLayout(&desc), descriptor->label.value_or("")); } -// Identity gamut (BT.709 -> sRGB, same primaries) as a 3x3 column-major matrix. -static const float kIdentityGamutMatrix[9] = { - 1.0f, 0.0f, 0.0f, // - 0.0f, 1.0f, 0.0f, // - 0.0f, 0.0f, 1.0f, // -}; - -// Piecewise gamma transfer-function parameters Dawn expects: -// for |x| < D: y = sign(x) * (C * |x| + F) -// else : y = sign(x) * (pow(A * |x| + B, G) + E) -// sRGB decode (encoded -> linear). -static const float kSrgbDecodeParams[7] = { - 2.4f, // G - 1.0f / 1.055f, // A - 0.055f / 1.055f, // B - 1.0f / 12.92f, // C - 0.04045f, // D - 0.0f, // E - 0.0f, // F -}; -// sRGB encode (linear -> encoded). -static const float kSrgbEncodeParams[7] = { - 1.0f / 2.4f, // G - 1.055f, // A - 0.0f, // B - 12.92f, // C - 0.0031308f, // D - -0.055f, // E - 0.0f, // F -}; - -// Identity transfer (y = x). Used when the sampled surface is already in the -// render target's color space: a single-plane BGRA IOSurface, or the Android -// opaque-YCbCr path where the Vulkan sampler already produced RGB. Dawn -// dereferences the transfer-function arrays unconditionally -// (ComputeExternalTextureParams), so these must be non-null even when no -// conversion is wanted. -static const float kIdentityTransferParams[7] = { - 1.0f, // G - 1.0f, // A - 0.0f, // B - 0.0f, // C - 0.0f, // D - 0.0f, // E - 0.0f, // F -}; - -// BT.709 limited-range YUV -> R'G'B' as a 3x4 row-major matrix mapping -// [Y, Cb, Cr, 1]. Same values the Apple NV12 path computes from the -// CVPixelBuffer; used for Android buffers that arrive as a *defined* biplanar -// format (where we split the planes and convert ourselves) rather than an -// opaque external-format AHB. Camera streams are limited-range BT.709 in the -// overwhelming majority of cases; full-range / BT.601 would need different -// coefficients (refine from the buffer's suggested range if it matters). -[[maybe_unused]] static const float kBT709LimitedToRgb[12] = { - 1.164383f, 0.000000f, 1.792741f, -0.972945f, // - 1.164383f, -0.213249f, -0.532909f, 0.301517f, // - 1.164383f, 2.112402f, 0.000000f, -1.133402f, // -}; - -// True for the multi-planar Y + CbCr formats whose planes we can view as -// Plane0Only (luma) / Plane1Only (chroma) and convert with an explicit matrix. -// Excludes OpaqueYCbCrAndroid (external format, no plane views) and triplanar -// formats (would need a third plane). Only referenced on Android. -[[maybe_unused]] static bool isBiplanarYuvFormat(wgpu::TextureFormat format) { - switch (format) { - case wgpu::TextureFormat::R8BG8Biplanar420Unorm: - case wgpu::TextureFormat::R8BG8Biplanar422Unorm: - case wgpu::TextureFormat::R8BG8Biplanar444Unorm: - case wgpu::TextureFormat::R10X6BG10X6Biplanar420Unorm: - case wgpu::TextureFormat::R10X6BG10X6Biplanar422Unorm: - case wgpu::TextureFormat::R10X6BG10X6Biplanar444Unorm: - return true; - default: - return false; - } -} - -// Map a rotation in degrees (0 / 90 / 180 / 270) to Dawn's enum. Anything that -// isn't a clean multiple of 90 snaps to the nearest quadrant; Dawn only -// supports those four steps for external textures. -static wgpu::ExternalTextureRotation -toExternalTextureRotation(double degrees) { - int quadrant = static_cast(std::lround(degrees / 90.0)) & 3; - switch (quadrant) { - case 1: - return wgpu::ExternalTextureRotation::Rotate90Degrees; - case 2: - return wgpu::ExternalTextureRotation::Rotate180Degrees; - case 3: - return wgpu::ExternalTextureRotation::Rotate270Degrees; - default: - return wgpu::ExternalTextureRotation::Rotate0Degrees; - } -} - std::shared_ptr GPUDevice::importExternalTexture( std::shared_ptr descriptor) { - if (!descriptor || !descriptor->source) { - throw std::runtime_error( - "GPUDevice::importExternalTexture(): descriptor.source (VideoFrame) " - "is required"); - } - const auto &source = descriptor->source; - const auto &frame = source->handle(); - if (frame.handle == nullptr) { - throw std::runtime_error( - "GPUDevice::importExternalTexture(): VideoFrame has been released"); - } - -#if defined(__APPLE__) - // 1. Import the IOSurface as SharedTextureMemory. For NV12 surfaces this - // yields a biplanar texture; for BGRA, a single-plane one. - wgpu::SharedTextureMemoryDescriptor memDesc{}; - std::string label = descriptor->label.value_or("external-texture"); - if (!label.empty()) { - memDesc.label = wgpu::StringView(label.c_str(), label.size()); - } - wgpu::SharedTextureMemoryIOSurfaceDescriptor platformDesc{}; - platformDesc.ioSurface = frame.handle; - // ExternalTexture views are sampled-only; storage binding isn't needed and - // for biplanar formats it would fail validation. - platformDesc.allowStorageBinding = false; - memDesc.nextInChain = &platformDesc; - auto memory = _instance.ImportSharedTextureMemory(&memDesc); - if (memory == nullptr) { - throw std::runtime_error( - "GPUDevice::importExternalTexture(): ImportSharedTextureMemory " - "returned null. Is 'shared-texture-memory-iosurface' enabled?"); - } - - // 2. Create the texture from the surface. We pass the right format - // explicitly so Dawn picks the multi-planar variant on NV12. - bool isYuv = frame.pixelFormat == VideoPixelFormat::NV12; - auto texture = memory.CreateTexture(); - if (texture == nullptr) { - throw std::runtime_error( - "GPUDevice::importExternalTexture(): CreateTexture returned null"); - } - - // 3. Begin access on the underlying memory. The matching EndAccess runs in - // the GPUExternalTexture destructor. - wgpu::SharedTextureMemoryBeginAccessDescriptor begin{}; - begin.initialized = true; - begin.concurrentRead = false; - if (!memory.BeginAccess(texture, &begin)) { - throw std::runtime_error( - "GPUDevice::importExternalTexture(): BeginAccess failed"); - } - - // 4. Build plane views. For NV12 we need plane0 = R8 luma and plane1 = RG8 - // chroma; for BGRA we only set plane0. - wgpu::TextureView plane0; - wgpu::TextureView plane1; - { - wgpu::TextureViewDescriptor v{}; - v.aspect = isYuv ? wgpu::TextureAspect::Plane0Only - : wgpu::TextureAspect::All; - plane0 = texture.CreateView(&v); - } - if (isYuv) { - wgpu::TextureViewDescriptor v{}; - v.aspect = wgpu::TextureAspect::Plane1Only; - plane1 = texture.CreateView(&v); - } - - // 5. Build the ExternalTextureDescriptor. We hand Dawn explicit YUV→RGB and - // sRGB transfer-function parameters so the sampler does the full color - // conversion in hardware. - wgpu::ExternalTextureDescriptor extDesc{}; - if (!label.empty()) { - extDesc.label = wgpu::StringView(label.c_str(), label.size()); - } - extDesc.plane0 = plane0; - extDesc.gamutConversionMatrix = kIdentityGamutMatrix; - if (isYuv) { - extDesc.plane1 = plane1; - extDesc.yuvToRgbConversionMatrix = frame.yuvToRgbMatrix; - extDesc.srcTransferFunctionParameters = kSrgbDecodeParams; - extDesc.dstTransferFunctionParameters = kSrgbEncodeParams; - } else { - // BGRA is already RGB in the target color space; pass it through. Dawn - // dereferences these arrays unconditionally, so they must be non-null. - extDesc.srcTransferFunctionParameters = kIdentityTransferParams; - extDesc.dstTransferFunctionParameters = kIdentityTransferParams; - } - extDesc.cropOrigin = {0, 0}; - extDesc.cropSize = {frame.width, frame.height}; - extDesc.apparentSize = {frame.width, frame.height}; - extDesc.mirrored = descriptor->mirrored.value_or(false); - extDesc.rotation = - toExternalTextureRotation(descriptor->rotation.value_or(0)); - - auto external = _instance.CreateExternalTexture(&extDesc); - if (external == nullptr) { - wgpu::SharedTextureMemoryEndAccessState state{}; - (void)memory.EndAccess(texture, &state); - throw std::runtime_error( - "GPUDevice::importExternalTexture(): CreateExternalTexture returned " - "null"); - } - - return std::make_shared( - std::move(external), std::move(memory), std::move(texture), - std::move(descriptor->source), std::move(label)); -#elif defined(__ANDROID__) - // 1. Import the AHardwareBuffer as SharedTextureMemory. For YUV AHBs this - // yields a Dawn texture in the implementation-defined OpaqueYCbCrAndroid - // format; for RGBA AHBs, a regular single-plane texture. - wgpu::SharedTextureMemoryDescriptor memDesc{}; - std::string label = descriptor->label.value_or("external-texture"); - if (!label.empty()) { - memDesc.label = wgpu::StringView(label.c_str(), label.size()); - } - wgpu::SharedTextureMemoryAHardwareBufferDescriptor platformDesc{}; - platformDesc.handle = frame.handle; - memDesc.nextInChain = &platformDesc; - auto memory = _instance.ImportSharedTextureMemory(&memDesc); - if (memory == nullptr) { - throw std::runtime_error( - "GPUDevice::importExternalTexture(): ImportSharedTextureMemory " - "returned null. Is 'shared-texture-memory-ahardware-buffer' enabled?"); - } - - // 2. Create the texture. No descriptor: Dawn picks the right format - // (OpaqueYCbCrAndroid for YUV, R8 / RGBA8 / ... for color AHBs). - auto texture = memory.CreateTexture(); - if (texture == nullptr) { - throw std::runtime_error( - "GPUDevice::importExternalTexture(): CreateTexture returned null"); - } - - // 3. Begin access. Vulkan requires us to advertise the incoming VkImage - // layout (UNDEFINED is fine for the first acquisition of an AHB whose - // contents we expect Dawn to read as-is). - wgpu::SharedTextureMemoryBeginAccessDescriptor begin{}; - begin.initialized = true; - begin.concurrentRead = false; - wgpu::SharedTextureMemoryVkImageLayoutBeginState beginLayout{}; - begin.nextInChain = &beginLayout; - if (!memory.BeginAccess(texture, &begin)) { - throw std::runtime_error( - "GPUDevice::importExternalTexture(): BeginAccess failed"); - } - - // 4. Build the ExternalTextureDescriptor. There are two cases depending on - // how Dawn imported the AHB (see SharedTextureMemoryVk.cpp): - // - // a. *External* format (camera buffers whose layout has no Vulkan - // equivalent) -> OpaqueYCbCrAndroid, a single opaque plane. Sampling - // routes through a Vulkan SamplerYcbcrConversion whose model Dawn - // copies verbatim from the AHB's suggestedYcbcrModel. We pass a single - // plane + identity transfer and let that conversion (if any) run. NOTE: - // when the driver reports RGB_IDENTITY the sample comes back as raw - // Y/Cb/Cr; there is no public hook to override the model on this path. - // - // b. *Defined* biplanar format (e.g. R8BG8Biplanar420Unorm, exposed by the - // dawn-multi-planar-formats feature) -> we split Plane0Only (luma) / - // Plane1Only (chroma) and hand Dawn an explicit BT.709 matrix + sRGB - // transfer, exactly like the iOS NV12 path. This makes numPlanes == 2 - // so the matrix is actually applied (the single-plane branch in Dawn's - // Tint transform ignores yuvToRgbConversionMatrix). - // - // Either way we must pass non-null gamut/transfer arrays: - // ComputeExternalTextureParams dereferences them unconditionally - // (kIdentityTransferParams is defined at file scope). - const bool isBiplanar = - frame.pixelFormat == VideoPixelFormat::NV12 && - isBiplanarYuvFormat(texture.GetFormat()); - - wgpu::TextureView plane0; - wgpu::TextureView plane1; - wgpu::ExternalTextureDescriptor extDesc{}; - if (!label.empty()) { - extDesc.label = wgpu::StringView(label.c_str(), label.size()); - } - extDesc.cropOrigin = {0, 0}; - extDesc.cropSize = {frame.width, frame.height}; - extDesc.apparentSize = {frame.width, frame.height}; - extDesc.gamutConversionMatrix = kIdentityGamutMatrix; - if (isBiplanar) { - wgpu::TextureViewDescriptor v0{}; - v0.aspect = wgpu::TextureAspect::Plane0Only; - plane0 = texture.CreateView(&v0); - wgpu::TextureViewDescriptor v1{}; - v1.aspect = wgpu::TextureAspect::Plane1Only; - plane1 = texture.CreateView(&v1); - extDesc.plane0 = plane0; - extDesc.plane1 = plane1; - extDesc.yuvToRgbConversionMatrix = kBT709LimitedToRgb; - extDesc.srcTransferFunctionParameters = kSrgbDecodeParams; - extDesc.dstTransferFunctionParameters = kSrgbEncodeParams; - } else { - plane0 = texture.CreateView(); - extDesc.plane0 = plane0; - extDesc.srcTransferFunctionParameters = kIdentityTransferParams; - extDesc.dstTransferFunctionParameters = kIdentityTransferParams; - } - extDesc.mirrored = descriptor->mirrored.value_or(false); - extDesc.rotation = - toExternalTextureRotation(descriptor->rotation.value_or(0)); - - auto external = _instance.CreateExternalTexture(&extDesc); - if (external == nullptr) { - wgpu::SharedTextureMemoryEndAccessState state{}; - (void)memory.EndAccess(texture, &state); - throw std::runtime_error( - "GPUDevice::importExternalTexture(): CreateExternalTexture returned " - "null"); - } - - return std::make_shared( - std::move(external), std::move(memory), std::move(texture), - std::move(descriptor->source), std::move(label)); -#else - throw std::runtime_error( - "GPUDevice::importExternalTexture(): not yet implemented on this " - "platform"); -#endif + // The import / begin-access / descriptor-build logic, plus the matching + // EndAccess, all live on GPUExternalTexture so the begin/end lifecycle stays + // in one translation unit (see GPUExternalTexture.cpp). + return GPUExternalTexture::Create(_instance, std::move(descriptor)); } std::shared_ptr diff --git a/packages/webgpu/cpp/rnwgpu/api/GPUExternalTexture.cpp b/packages/webgpu/cpp/rnwgpu/api/GPUExternalTexture.cpp new file mode 100644 index 000000000..255cd6f80 --- /dev/null +++ b/packages/webgpu/cpp/rnwgpu/api/GPUExternalTexture.cpp @@ -0,0 +1,335 @@ +#include "GPUExternalTexture.h" + +#include +#include +#include +#include + +#include "GPUExternalTextureDescriptor.h" + +namespace rnwgpu { + +// Identity gamut (BT.709 -> sRGB, same primaries) as a 3x3 column-major matrix. +static const float kIdentityGamutMatrix[9] = { + 1.0f, 0.0f, 0.0f, // + 0.0f, 1.0f, 0.0f, // + 0.0f, 0.0f, 1.0f, // +}; + +// Piecewise gamma transfer-function parameters Dawn expects: +// for |x| < D: y = sign(x) * (C * |x| + F) +// else : y = sign(x) * (pow(A * |x| + B, G) + E) +// sRGB decode (encoded -> linear). +static const float kSrgbDecodeParams[7] = { + 2.4f, // G + 1.0f / 1.055f, // A + 0.055f / 1.055f, // B + 1.0f / 12.92f, // C + 0.04045f, // D + 0.0f, // E + 0.0f, // F +}; +// sRGB encode (linear -> encoded). +static const float kSrgbEncodeParams[7] = { + 1.0f / 2.4f, // G + 1.055f, // A + 0.0f, // B + 12.92f, // C + 0.0031308f, // D + -0.055f, // E + 0.0f, // F +}; + +// Identity transfer (y = x). Used when the sampled surface is already in the +// render target's color space: a single-plane BGRA IOSurface, or the Android +// opaque-YCbCr path where the Vulkan sampler already produced RGB. Dawn +// dereferences the transfer-function arrays unconditionally +// (ComputeExternalTextureParams), so these must be non-null even when no +// conversion is wanted. +static const float kIdentityTransferParams[7] = { + 1.0f, // G + 1.0f, // A + 0.0f, // B + 0.0f, // C + 0.0f, // D + 0.0f, // E + 0.0f, // F +}; + +// BT.709 limited-range YUV -> R'G'B' as a 3x4 row-major matrix mapping +// [Y, Cb, Cr, 1] to gamma-encoded R'G'B' (NOT linear; the sRGB decode in +// srcTransferFunctionParameters linearizes afterwards). Same values the Apple +// NV12 path computes from the CVPixelBuffer; used for Android buffers that +// arrive as a *defined* biplanar format (where we split the planes and convert +// ourselves) rather than an opaque external-format AHB. Camera streams are +// limited-range BT.709 in the overwhelming majority of cases; full-range / +// BT.601 would need different coefficients (refine from the buffer's suggested +// range if it matters). +[[maybe_unused]] static const float kBT709LimitedToRgb[12] = { + 1.164383f, 0.000000f, 1.792741f, -0.972945f, // + 1.164383f, -0.213249f, -0.532909f, 0.301517f, // + 1.164383f, 2.112402f, 0.000000f, -1.133402f, // +}; + +// True for the multi-planar Y + CbCr formats whose planes we can view as +// Plane0Only (luma) / Plane1Only (chroma) and convert with an explicit matrix. +// Excludes OpaqueYCbCrAndroid (external format, no plane views) and triplanar +// formats (would need a third plane). Only referenced on Android. +[[maybe_unused]] static bool isBiplanarYuvFormat(wgpu::TextureFormat format) { + switch (format) { + case wgpu::TextureFormat::R8BG8Biplanar420Unorm: + case wgpu::TextureFormat::R8BG8Biplanar422Unorm: + case wgpu::TextureFormat::R8BG8Biplanar444Unorm: + case wgpu::TextureFormat::R10X6BG10X6Biplanar420Unorm: + case wgpu::TextureFormat::R10X6BG10X6Biplanar422Unorm: + case wgpu::TextureFormat::R10X6BG10X6Biplanar444Unorm: + return true; + default: + return false; + } +} + +// Map a rotation in degrees (0 / 90 / 180 / 270) to Dawn's enum. Anything that +// isn't a clean multiple of 90 snaps to the nearest quadrant; Dawn only +// supports those four steps for external textures. +static wgpu::ExternalTextureRotation +toExternalTextureRotation(double degrees) { + int quadrant = static_cast(std::lround(degrees / 90.0)) & 3; + switch (quadrant) { + case 1: + return wgpu::ExternalTextureRotation::Rotate90Degrees; + case 2: + return wgpu::ExternalTextureRotation::Rotate180Degrees; + case 3: + return wgpu::ExternalTextureRotation::Rotate270Degrees; + default: + return wgpu::ExternalTextureRotation::Rotate0Degrees; + } +} + +std::shared_ptr GPUExternalTexture::Create( + wgpu::Device device, + std::shared_ptr descriptor) { + if (!descriptor || !descriptor->source) { + throw std::runtime_error( + "GPUExternalTexture::Create(): descriptor.source (VideoFrame) " + "is required"); + } + const auto &source = descriptor->source; + const auto &frame = source->handle(); + if (frame.handle == nullptr) { + throw std::runtime_error( + "GPUExternalTexture::Create(): VideoFrame has been released"); + } + +#if defined(__APPLE__) + // 1. Import the IOSurface as SharedTextureMemory. For NV12 surfaces this + // yields a biplanar texture; for BGRA, a single-plane one. + wgpu::SharedTextureMemoryDescriptor memDesc{}; + std::string label = descriptor->label.value_or("external-texture"); + if (!label.empty()) { + memDesc.label = wgpu::StringView(label.c_str(), label.size()); + } + wgpu::SharedTextureMemoryIOSurfaceDescriptor platformDesc{}; + platformDesc.ioSurface = frame.handle; + // ExternalTexture views are sampled-only; storage binding isn't needed and + // for biplanar formats it would fail validation. + platformDesc.allowStorageBinding = false; + memDesc.nextInChain = &platformDesc; + auto memory = device.ImportSharedTextureMemory(&memDesc); + if (memory == nullptr) { + throw std::runtime_error( + "GPUExternalTexture::Create(): ImportSharedTextureMemory " + "returned null. Is 'shared-texture-memory-iosurface' enabled?"); + } + + // 2. Create the texture from the surface. We pass the right format + // explicitly so Dawn picks the multi-planar variant on NV12. + bool isYuv = frame.pixelFormat == VideoPixelFormat::NV12; + auto texture = memory.CreateTexture(); + if (texture == nullptr) { + throw std::runtime_error( + "GPUExternalTexture::Create(): CreateTexture returned null"); + } + + // 3. Begin access on the underlying memory. The matching EndAccess runs when + // the GPUExternalTexture is destroyed (explicitly via destroy() or at GC). + wgpu::SharedTextureMemoryBeginAccessDescriptor begin{}; + begin.initialized = true; + begin.concurrentRead = false; + if (!memory.BeginAccess(texture, &begin)) { + throw std::runtime_error( + "GPUExternalTexture::Create(): BeginAccess failed"); + } + + // 4. Build plane views. For NV12 we need plane0 = R8 luma and plane1 = RG8 + // chroma; for BGRA we only set plane0. + wgpu::TextureView plane0; + wgpu::TextureView plane1; + { + wgpu::TextureViewDescriptor v{}; + v.aspect = + isYuv ? wgpu::TextureAspect::Plane0Only : wgpu::TextureAspect::All; + plane0 = texture.CreateView(&v); + } + if (isYuv) { + wgpu::TextureViewDescriptor v{}; + v.aspect = wgpu::TextureAspect::Plane1Only; + plane1 = texture.CreateView(&v); + } + + // 5. Build the ExternalTextureDescriptor. We hand Dawn explicit YUV→RGB and + // sRGB transfer-function parameters so the sampler does the full color + // conversion in hardware. + wgpu::ExternalTextureDescriptor extDesc{}; + if (!label.empty()) { + extDesc.label = wgpu::StringView(label.c_str(), label.size()); + } + extDesc.plane0 = plane0; + extDesc.gamutConversionMatrix = kIdentityGamutMatrix; + if (isYuv) { + extDesc.plane1 = plane1; + extDesc.yuvToRgbConversionMatrix = frame.yuvToRgbMatrix; + extDesc.srcTransferFunctionParameters = kSrgbDecodeParams; + extDesc.dstTransferFunctionParameters = kSrgbEncodeParams; + } else { + // BGRA is already RGB in the target color space; pass it through. Dawn + // dereferences these arrays unconditionally, so they must be non-null. + extDesc.srcTransferFunctionParameters = kIdentityTransferParams; + extDesc.dstTransferFunctionParameters = kIdentityTransferParams; + } + extDesc.cropOrigin = {0, 0}; + extDesc.cropSize = {frame.width, frame.height}; + extDesc.apparentSize = {frame.width, frame.height}; + extDesc.mirrored = descriptor->mirrored.value_or(false); + extDesc.rotation = + toExternalTextureRotation(descriptor->rotation.value_or(0)); + + auto external = device.CreateExternalTexture(&extDesc); + if (external == nullptr) { + wgpu::SharedTextureMemoryEndAccessState state{}; + (void)memory.EndAccess(texture, &state); + throw std::runtime_error( + "GPUExternalTexture::Create(): CreateExternalTexture returned " + "null"); + } + + return std::make_shared( + std::move(external), std::move(memory), std::move(texture), + std::move(descriptor->source), std::move(label)); +#elif defined(__ANDROID__) + // 1. Import the AHardwareBuffer as SharedTextureMemory. For YUV AHBs this + // yields a Dawn texture in the implementation-defined OpaqueYCbCrAndroid + // format; for RGBA AHBs, a regular single-plane texture. + wgpu::SharedTextureMemoryDescriptor memDesc{}; + std::string label = descriptor->label.value_or("external-texture"); + if (!label.empty()) { + memDesc.label = wgpu::StringView(label.c_str(), label.size()); + } + wgpu::SharedTextureMemoryAHardwareBufferDescriptor platformDesc{}; + platformDesc.handle = frame.handle; + memDesc.nextInChain = &platformDesc; + auto memory = device.ImportSharedTextureMemory(&memDesc); + if (memory == nullptr) { + throw std::runtime_error( + "GPUExternalTexture::Create(): ImportSharedTextureMemory " + "returned null. Is 'shared-texture-memory-ahardware-buffer' enabled?"); + } + + // 2. Create the texture. No descriptor: Dawn picks the right format + // (OpaqueYCbCrAndroid for YUV, R8 / RGBA8 / ... for color AHBs). + auto texture = memory.CreateTexture(); + if (texture == nullptr) { + throw std::runtime_error( + "GPUExternalTexture::Create(): CreateTexture returned null"); + } + + // 3. Begin access. Vulkan requires us to advertise the incoming VkImage + // layout (UNDEFINED is fine for the first acquisition of an AHB whose + // contents we expect Dawn to read as-is). + wgpu::SharedTextureMemoryBeginAccessDescriptor begin{}; + begin.initialized = true; + begin.concurrentRead = false; + wgpu::SharedTextureMemoryVkImageLayoutBeginState beginLayout{}; + begin.nextInChain = &beginLayout; + if (!memory.BeginAccess(texture, &begin)) { + throw std::runtime_error( + "GPUExternalTexture::Create(): BeginAccess failed"); + } + + // 4. Build the ExternalTextureDescriptor. There are two cases depending on + // how Dawn imported the AHB (see SharedTextureMemoryVk.cpp): + // + // a. *External* format (camera buffers whose layout has no Vulkan + // equivalent) -> OpaqueYCbCrAndroid, a single opaque plane. Sampling + // routes through a Vulkan SamplerYcbcrConversion whose model Dawn + // copies verbatim from the AHB's suggestedYcbcrModel. We pass a single + // plane + identity transfer and let that conversion (if any) run. NOTE: + // when the driver reports RGB_IDENTITY the sample comes back as raw + // Y/Cb/Cr; there is no public hook to override the model on this path. + // + // b. *Defined* biplanar format (e.g. R8BG8Biplanar420Unorm, exposed by the + // dawn-multi-planar-formats feature) -> we split Plane0Only (luma) / + // Plane1Only (chroma) and hand Dawn an explicit BT.709 matrix + sRGB + // transfer, exactly like the iOS NV12 path. This makes numPlanes == 2 + // so the matrix is actually applied (the single-plane branch in Dawn's + // Tint transform ignores yuvToRgbConversionMatrix). + // + // Either way we must pass non-null gamut/transfer arrays: + // ComputeExternalTextureParams dereferences them unconditionally + // (kIdentityTransferParams is defined at file scope). + const bool isBiplanar = frame.pixelFormat == VideoPixelFormat::NV12 && + isBiplanarYuvFormat(texture.GetFormat()); + + wgpu::TextureView plane0; + wgpu::TextureView plane1; + wgpu::ExternalTextureDescriptor extDesc{}; + if (!label.empty()) { + extDesc.label = wgpu::StringView(label.c_str(), label.size()); + } + extDesc.cropOrigin = {0, 0}; + extDesc.cropSize = {frame.width, frame.height}; + extDesc.apparentSize = {frame.width, frame.height}; + extDesc.gamutConversionMatrix = kIdentityGamutMatrix; + if (isBiplanar) { + wgpu::TextureViewDescriptor v0{}; + v0.aspect = wgpu::TextureAspect::Plane0Only; + plane0 = texture.CreateView(&v0); + wgpu::TextureViewDescriptor v1{}; + v1.aspect = wgpu::TextureAspect::Plane1Only; + plane1 = texture.CreateView(&v1); + extDesc.plane0 = plane0; + extDesc.plane1 = plane1; + extDesc.yuvToRgbConversionMatrix = kBT709LimitedToRgb; + extDesc.srcTransferFunctionParameters = kSrgbDecodeParams; + extDesc.dstTransferFunctionParameters = kSrgbEncodeParams; + } else { + plane0 = texture.CreateView(); + extDesc.plane0 = plane0; + extDesc.srcTransferFunctionParameters = kIdentityTransferParams; + extDesc.dstTransferFunctionParameters = kIdentityTransferParams; + } + extDesc.mirrored = descriptor->mirrored.value_or(false); + extDesc.rotation = + toExternalTextureRotation(descriptor->rotation.value_or(0)); + + auto external = device.CreateExternalTexture(&extDesc); + if (external == nullptr) { + wgpu::SharedTextureMemoryEndAccessState state{}; + (void)memory.EndAccess(texture, &state); + throw std::runtime_error( + "GPUExternalTexture::Create(): CreateExternalTexture returned " + "null"); + } + + return std::make_shared( + std::move(external), std::move(memory), std::move(texture), + std::move(descriptor->source), std::move(label)); +#else + throw std::runtime_error( + "GPUExternalTexture::Create(): not yet implemented on this " + "platform"); +#endif +} + +} // namespace rnwgpu diff --git a/packages/webgpu/cpp/rnwgpu/api/GPUExternalTexture.h b/packages/webgpu/cpp/rnwgpu/api/GPUExternalTexture.h index 71e23f3b7..217ab45d9 100644 --- a/packages/webgpu/cpp/rnwgpu/api/GPUExternalTexture.h +++ b/packages/webgpu/cpp/rnwgpu/api/GPUExternalTexture.h @@ -15,10 +15,21 @@ namespace rnwgpu { namespace jsi = facebook::jsi; +struct GPUExternalTextureDescriptor; + class GPUExternalTexture : public NativeObject { public: static constexpr const char *CLASS_NAME = "GPUExternalTexture"; + // Import a VideoFrame (via descriptor.source) as a GPUExternalTexture on + // `device`: imports the native surface as SharedTextureMemory, begins access, + // and wraps the resulting wgpu::ExternalTexture together with the resources + // whose lifetime it owns. The matching EndAccess runs in destroy() / the + // destructor. Defined in GPUExternalTexture.cpp. + static std::shared_ptr + Create(wgpu::Device device, + std::shared_ptr descriptor); + // Construct from an already-built wgpu::ExternalTexture plus the underlying // shared-memory resources we need to keep alive. The wrapper takes ownership // of the SharedTextureMemory + Texture and calls EndAccess on destruction so @@ -30,16 +41,28 @@ class GPUExternalTexture : public NativeObject { _memory(std::move(memory)), _texture(std::move(texture)), _source(std::move(source)), _label(std::move(label)) {} - ~GPUExternalTexture() override { + ~GPUExternalTexture() override { destroy(); } + +public: + std::string getBrand() { return CLASS_NAME; } + + // End the shared-memory access window and release the underlying resources. + // Idempotent: safe to call more than once, and the destructor calls it as a + // garbage-collection fallback. Call it right after the queue.submit() that + // sampled this texture (never before): a GPUExternalTexture's access window + // is owned by this wrapper's lifetime, not by submit, so without an explicit + // destroy() the producer's surface (e.g. an AVPlayer IOSurface) stays claimed + // until GC runs. EndAccess is the designed post-submit call: Dawn keeps the + // texture alive for in-flight GPU work via the fences it returns. + void destroy() { if (_memory && _texture) { wgpu::SharedTextureMemoryEndAccessState state{}; (void)_memory.EndAccess(_texture, &state); } + _texture = nullptr; + _memory = nullptr; } -public: - std::string getBrand() { return CLASS_NAME; } - std::string getLabel() { return _label; } void setLabel(const std::string &label) { _label = label; @@ -51,6 +74,7 @@ class GPUExternalTexture : public NativeObject { installGetterSetter(runtime, prototype, "label", &GPUExternalTexture::getLabel, &GPUExternalTexture::setLabel); + installMethod(runtime, prototype, "destroy", &GPUExternalTexture::destroy); } inline const wgpu::ExternalTexture get() { return _instance; } diff --git a/packages/webgpu/src/__tests__/ExternalTexture.spec.ts b/packages/webgpu/src/__tests__/ExternalTexture.spec.ts index 6151abc31..712fec9bb 100644 --- a/packages/webgpu/src/__tests__/ExternalTexture.spec.ts +++ b/packages/webgpu/src/__tests__/ExternalTexture.spec.ts @@ -534,6 +534,9 @@ describe("External Textures", () => { pass.draw(3); pass.end(); device.queue.submit([encoder.finish()]); + // End the external texture's shared-memory access window now that the + // work sampling it is submitted, rather than waiting for GC. + externalTexture.destroy(); return canvas.getImageData().then((image: BitmapData) => { frame.release(); diff --git a/packages/webgpu/src/__tests__/ImportExternalTexture.spec.ts b/packages/webgpu/src/__tests__/ImportExternalTexture.spec.ts index e506240aa..376d26d10 100644 --- a/packages/webgpu/src/__tests__/ImportExternalTexture.spec.ts +++ b/packages/webgpu/src/__tests__/ImportExternalTexture.spec.ts @@ -117,6 +117,9 @@ describe("ImportExternalTexture", () => { pass.draw(3); pass.end(); device.queue.submit([encoder.finish()]); + // End the external texture's shared-memory access window now that the + // work sampling it is submitted, rather than waiting for GC. + externalTexture.destroy(); return canvas.getImageData().then((image: BitmapData) => { // Safe to release now: all GPU work referencing the frame is diff --git a/packages/webgpu/src/index.tsx b/packages/webgpu/src/index.tsx index 6f716ed72..3e5e819ac 100644 --- a/packages/webgpu/src/index.tsx +++ b/packages/webgpu/src/index.tsx @@ -70,6 +70,17 @@ declare global { mirrored?: boolean; } + // Non-spec extension: a GPUExternalTexture imported from a native surface + // holds an open shared-memory access window on that surface until this + // wrapper is destroyed. Call destroy() right after the queue.submit() that + // sampled it (never before) to release the surface back to its producer + // immediately, instead of waiting for garbage collection. Forgetting to call + // it is not fatal (GC still cleans up), but it can starve a producer's buffer + // pool (e.g. a camera/video player) and pile up GPU resources. + interface GPUExternalTexture { + destroy(): void; + } + // Extend createImageBitmap to accept ArrayBuffer/TypedArray (encoded image bytes) function createImageBitmap( image: ArrayBuffer | ArrayBufferView, From 542caba704c3da6494e79256924abb8d1f9ae3fb Mon Sep 17 00:00:00 2001 From: William Candillon Date: Mon, 1 Jun 2026 22:19:57 +0200 Subject: [PATCH 41/46] :wrench: --- .../snapshots/import-external-texture-web.png | Bin 0 -> 87271 bytes .../snapshots/import-external-texture.png | Bin 0 -> 87271 bytes 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 packages/webgpu/src/__tests__/snapshots/import-external-texture-web.png create mode 100644 packages/webgpu/src/__tests__/snapshots/import-external-texture.png diff --git a/packages/webgpu/src/__tests__/snapshots/import-external-texture-web.png b/packages/webgpu/src/__tests__/snapshots/import-external-texture-web.png new file mode 100644 index 0000000000000000000000000000000000000000..a15e25e3a9bcbe218a173d2b6915cad7035b0501 GIT binary patch literal 87271 zcmcItd0Z3M_U{ub)~Hdb;u8fI3N94%RRz~rmx>!$aqDx5OMTi`6|hvWYUWW3X=-t) zP;H^cty1@;?7`E7LMP98>7};tKij_4z8(A*(PzlEm;N&M^t8!u&W-y^ z`PIzNUjF$%eQ$OD;ImG5yL}baH*Gq3@;93JDP^AQOOM&R6aiNgb2^SIOnjy1n+4Tx z{QK@dd(#i-!(WZ~{vW^n_v?4wc>f>QPxARcvO87V9#tCtQnb^7e`LP*BY!vg78PuY z*G%R2x9pbmY|jp#;Y7apiR>3gR?FN$@>hfW5m_bfDEX(|9r^uYWejz|I7T+1=CEUC zf>dJjq}2Voy2|a6kE74$AJ@x?LEk3NnU4NLrGJ=AcM-R3 z6Yp4|#rlsjT`a zC2wDT@7%%Cti;*#_hv+C^0E|O$4&;GU74S_!_8~rJ4;Hk6TIB~*B?25DQia`uhE^? zwqNz3Lgcx=V_L>;)zda(Hm-dVa<*Ucudf9z&Zr8Kd{mH3w~y~r8FK5~@~Hz)%vfy5 zR>XUG&Ut6avivcfW7gj5s(RYZ^Gei;;=F{p^F8lX?QrYkwegelmvR$=(t2h@_e=Tp z<&Bq1C*&vgIe0QKS|jo|l}(LQ5uauyy7@e5-xjSXPZ8eHw;NieCd++vn|QSH=n@Y% z#pJ;!L~BEI@jE8;g$1ooGvrR_j~40GdCP$TUC_e4{Kr@Q3HLkX-M(#`KJW=&wJONB zn<5@Qk*iiA!|i}i`uX#o-4wZ5@VR*YynCDY2~(#|&5wBJJ{W(+vSrJP7rk@eEq=m* z!-o%-9y?i{FuDJ*p+kr2m^(K#$*`!Bw_sJmZ7kCt9W5crQC)d?5b?H?usn6LJbpsoTeolDu9`dl#l-%*d&0-?;sbPR%yyOBO+8Az)@TQ} zTfZ3nUMHMF{o+WKbZ!&tpvNuH*mrI|KT1s*>m)QeeZzyGD!6r9;owBJYK->!IgwxM z_jcjoFCDUW^^6|}5nXvfk*~4IUsCn{&ml5Bflu7vA)EW{4($k^EklNEcG5qPiNV_N z!NOuwbB?&|E;v|Xx-;l(60*=EAireRDkMT7cM9RQalk_>rKY(CvXPou#yuD_0c7pk zQTCA74R9RWrses{zP{@elQ!{)+#Uw;LNySrPOrSK=$U7S;iFHiNQNRzNb3VAA2f$^C?=Z(7cr^ zS*U;K$3f&Rj$@M}{**h3sP7INZ1%GJ#%VrZcygT^?nr+EFWVn`VnB3{R0X{%G;igE z>;i!7OxGL|7C?z$X47<(Ymajok@N6H%mXj`ecOh|{N}(u99THrZdUXmgyVQ!e`lepsS36rJ{OrH^OIeO)a zmHko*ULNh8G4$50uBx(bqR7I1_ZEhx*UmLvoRE{?-wtYBVSDdoRAoi}6PM-UF zLP>1Yk`<+SA`d8krHh6yj#~j)fi^~w+v345^(%2)8II+a!q-WkeU%Zg`=x-W(T82Vh?a2~l9^a;|wD}f4*2Yl43 zpU1=DnJMuk@2Q81#!UEZsq;5f@9h5M$(o)Sx4NKTAoB7E9*KVZ;ls;|Cr&~?G+^z_ z!h50G7^t)*#)~>3dz?KxEPCYt^xH@GY8N=1F} zjk*sf4OCjp+PmSr4-_BKCsO;a50oTPaZIZESq9ZJKlog!_kd>wHMQX#7q3ANH6bBM z;pK%M>h0S@qYn0ihuWjZNWb-KkHEsFPR+?mNCbo|TzGF^#yNzLK7Bg5dG9y5z;?oKFZl3LWjYiisNnRdaB6P>9tc3W)9q6COmFDHjE!1;;NSrC zR~}{zJ9lmux}|t=&oxh&#RRXPxi;VkaXg&v8J{Q$?+BGEc$m?p2?{8B6};RlaO|)y z?a@QaOY-#VrU1OJ$!m z%JoDHF(>($3H@{)HrGqX*K6>CQ%i0cJ->-bV6zCcuQ5ouqsm3bG=(d7bj7#V=bAPv zO;LEUsaMy4r@|{EISfy+cAFqf42Ok$JeO4wWk73>H`ZLDr2OaoC z84gd$Wq11Pa59O1fEnUn$3JrE>578L0Rmkmmk*#e!-fy;$}4K40LLe4nZ_w)UP8Aw zhx^p3pOx)(mH#x+TWeP}iPC)W$Pw}-`RNksP}86S=n)*Fl%)khLz-kgJn_hpLQ$i} z*Z++x+i?ht&7@Jv%7P%Q5FG&!^~4O(UP{?taYFP=Rhi_Gv$29h|1eGNz3_eaIP#L?n&>%8JB7d)fbKYJ!3OqJ@GJ|oGI`~#D!$zh%L1_QeVL-JeRkndVv8j9+OmO zGzChRZTgSK*T2`~w8Is`c^;k4loj}xeECIit_{v{jMv_+E(^yAQQ0h|X&u+8v6;h< zL&wp(HD&4^ys}J=sNVD-KUXGJnskvJc%3M@;K+4ACFSAt1)N@Vv!Oc3f&9`TFG?!I z>GL?f=w@`3e_L=>i7R{=$8D|MEA-sXGW;>w_~FeFK~*-50A9 zP16+616xT&jZ$d+JpTTSFZvPVj{Op}8-jYRL7L zUN`Zo)u`NrDtQ;rfA3}KYMXTbJ}OP2s^6*d@xq0@ObAJiHuN4%D z>G0w#&#;)`z1y#fg__vno(uP(AL=FzxqUmTZ*tLVepjNGFI$d&Y3D7P$y0OUXK&oQ z&ybf8pP-n#(K7_v;QDlOpLl=CiV`o+c0K&&+%G7iI}A^YJA~T*VigIZAk;Zn4mHsd z1?mzkdaXy`<)NtGB6C8H!XGsgZrr;tLzj&j2P@ii^52ho4&i>9Y!T`&+O zMZKYGb4Q{ZdahMTA~%u8F=}H)xjW-a;gXyRRrb$pa z?a?kUfcd>pZ};zop5KZU%ZsxoqMsIW>sIukeh5pgd`%}cX)5G;pMF*wpeG{oAMK1$ z=@;Tytz@zlP8_Cj1e3jiu$PXJb&ZHNtRx$fn(~x zaiA*o<&w(%cGiL-WZsxN{T|mq^o_z4C3=PGa3F6c z-e)Vd5Z)s~TS|Ludzo(@W%!<1rX^ow?IJk&gjJrVX1T97H?`10oF$ZwQ9!^)8sLv`?k;ypY(g~D|`r)Z4u}_0M z4hC?re$=8DJzk+At6HhL2{E&gEWi5WXE8rq=$o}>@H*p%bl>J%^0?kVwbb9Ic~wjfqUYE*nq@a_G`q~vXcp9{ z(G0y75TU-c>_N;Dqf6PQM_~F|y2`Gg4oBu)zpWpw)vv)CwItibXnV07H<+`VZ!Txg zSs-M32{b>E`*B%U1IrwJU* zyc*QZYei5lI--Oab#+u{t*-hI;>5ROtqo8pWmgE$Ub`YSn^gcYO8O@>vj!YEy%yM_~^ z@?LBhGCj2%J5)-f?Q#YV%i zM>$E#rEgj29}eS@gG*vuIhK0lxf^E!I@{C}em_=-=EoVXxoXpwj-ysN&1|v@{K`^I zFtOX7DAsIl$r*z9DE)xj z)bOOk5SfIXo2U`qrlA~ky8RCAZtnA`=_4_*4`WuZe?QJ}-N)CSx5ylI_`y`ad=qihLcS+O_O!r3E>WGJi#anL@`_Hu#>28Zz=M zS=ZfaUFNhV86`u%6PBNnN9FWA4jyPTXLE;wl~kQW1mLSO9i+xVSdt{;4NKnMrj=MB zTB*^fx%jpiDkIv;#~SsQIKAk8Z`rd*?$QXoeO$(ZO?`jv*7l>%!WT>0wABpU`Xjx) ze4#!f>|cGioog4^{>y)C%==&B(eKnxHsA2__v-=bbeYid?~Zt<_d!;W&o$%bOUAvg z^$25V%cw;!o;j^Ti(ZJC0}U;D?hFSSwdmzCz8Olay%`QPdgbcD03Y>K6*C0_GCh@e zy4_NldVAxGZDxLM)PjvKrdU|J&?WVznH!xkWK12D9GG)M@;CJsK51dfH(Pg`>AB-o zH)IwPbcIT-zAj10h+4k9Bp3CWA^vD@rk_vb*~2dkwWBf3zwxM_kJr~;L_OP1^pDmg zK_8_jYFsLfcSHQ~a*#z7d``YjcKCDz#9Ucb@;=$3hg(JpX-=g+4y@{(Z59?!4sweNPJs z`fz!1~kMKcSDV)(VFb+Tm@=CL(STdl8XL6?ahr=?6Mk)65p{-_ouJu_kI~b;3W&O8D^`@WZXZVg zTVG@l(|<~$SD^k}#Ek09wDKj0L>~UAKbK*mOhB!Ci!;s+ZS`xk_j;L8VtPO<$7V=E z9Bj%V&~C$lN?71bt@^4U+_hlWE%iCGjKKbM40p}KlQ7CtrOAW{NG6?;xB+argy685 z0U(wbE9@GTPGk(C{T%d(&;!lwVDlw~$+KE>dpE9F6d`<8t(I663}RQUZrLd?cgBE3 z7O?d@!IZCqbVdjxodGs~7&7c!6oYr6*WjM#k2xQ5So6}+5hJTG1{|B^X-V)Z4~97I zoQq_kO8@tIkg5l;a(g4G2MDC1$h3V6L3Fl4_%+j4spl0K%}G2uO1TJ~WC`3FD?CrE zm%VNJQn>NB3~!njtc@IO&1_hh2~LyBq#s6@{A>x04POrIZ0>MhdN*Sbffb?TZ-7ws zL@YUxHJ`e31{naC_XLR^*{M{$=W~~Ac`^ZR%?Rx{kOHy$8wHp~303u?AB9mMVscKIZCX?S`-1KVIEaP^Ap!~cwIIAJJZ)c=-oJ-r;?7sHLLh2Cp8W6{E2BoFz zRnp1U^qXf|*+MRVr&0@|vaZ?mJEI~Hg|-pU9M9ZY*b>PQ4!;lQPwS(t! z)_Pp0eDX7Zs2Vdwr9aIwe$&>mlX*)8}v!Bl`m_Hy+88f%SEH(0Gmrl7g?5oq2kcdJE^KKfYddrrI@|-aS~mpaafLpP2!X zejesc9|#6`72Bq#_oDjZI;b!9^i*h9f84jvI%y?6RA2MhxBZ4XQCqiyKWB}Ii1>hb zcJkB7wdol{=%c`AMl8rI;4|A*nSij$2=D=)*;e2~d}c(091DDATPY7QNRj>?PXkzERz)z9;A5(J*{E3(qH5UXVbCne z92h*2wj}awB}pLOhi*t;1vV)Z=#~`&o77gJLu^up77le%TX7VzNs$&Luu1Le0f48#( zr8s&{x6Q+Yp`8o9b|;WoWK?ki(#?cyHZ5+YwKNrkR1Q%YGS>?!ZZg&8)Xk*WGO0`{FVB^7(SgQsnE7ju}7kv?byYiQ(`=D7r%XP#8-wJVp3M4oaLT~AMFWcGFF&s8H!dl>}m1fpm_iNK9+*&eC@JZWe*6a{|jof%K392UzgEP-< z&Adoy^dRW}6-j<`#a5qcBs1&s@xpU|CUla&EI27bXNe|B7ICvk0`j`p5~UP$Lzg(u zsYW8UW+zVXIoI??EN~s#3*X)FR=bI}pufr_Kw%dnM!(7=Pq`$z>*>mhNSxvQ7&X2^ z_AI!&;LHd)k|vp!%qNz>>O|2Ey*P4lsMR~^3$!({H5c(>v?pLR6*ESO&ofD^eiYsC zzfRiDEVk3=cr?{mS_G<#NVZ+TS%@<$RhCvjcj<(D4@>9G3XQwJA0v+>{F?r8M%^_i zNHxsMxspq+D8zAQGJa54|DJ0Cy|S_*+{5DP&%e z(huI{;W}}uq@@+&51c!zH{%~nAH@>;Y&BEI>K|OkP<>5QO{ydm>nUlDadSpp4pxZ5 z4T}o-);XH79V5ooq8lTenr?7)Rf^9WR=CK9>3ibHHk4#ETsfqQ!YJXUa5t)dO!;IZGb5 zYq6$AjDs~Mik#$=+Of`Hk(Ji*5E#hg_KZ~h-5OmiR=Lx$ls>jCR)oGWmRf?vmya#fqP$HsX+?_scdvV=eecooNWiG zG58tzV&d6s(=1Fh6P=Igk!G(xvL80q4qX=Mts)h;czRXU!D8)f*D#=%TVte-)4=%L za0DwwkFJ{C87GNUC-GMGV+V(|&I{#cj@ds}!Jt*YL(o-QnK87~QB;3kj1P ztNMmv0U&A@u9OMsND(P=eURQY>RQAnaPKehaY%JjA<$DZurl$fw;>4)#gN6LnVRV2 zpX~#n4|7rGbh8-ljZrMZ6yPlzL~h5LiAuK1^0FnvvQx3**IU;a3L)6G=b%<%999~NC6(#A~a zWmI)>i#@%?pa&Fwm2Q23%eP`ybDBIf?Y6-Lh-p1DiY3p3cIa7R^+Rl$} z%XaPCze8!l-j#KaB?i)8P&kZ)0?4LdWtpV2(HG$rWhgC50)ppa$nw&3PDlH*s$GjUU#?_2g@K;dvUmU=hP! z2o%v+h(@@+2TiblU~74{^9-8UsG0}UF@n>uq!0!Yr{D!Axjhpk_;>^wp5C*vq~c%%!?@zJZGu?hAp@^uSHFwoQ~|C&25{hArfLzDAv{FzHf51^Y?P{jUZz4dCMx zIgp4Nup7Zz6Z_%CQ+Dba_}Wmii!R{Y=_=xfx}2lkctvd9!HL?NxsFfBh9|)b!SsfU zNiH#_p1AbjEulC>oP)-wHs)8D&MNRGN(1S8MI)QIKMv3_BX(L3>nh2{nrpQJXd!k! zvKh7s9ScqWxk4pNI6}StCcrajaFbkza}Oi#LE{*X7#~eCRLSFty(t3ob}n~n8Wl@^ z?izs76%~7TW6V5fn?}SEhADV6&xyvnh4p{K2~qMQQL5j|-!y4TO(%xXv)aiKR zk-jpz<|0;%j;k<@bjj&RMXHO`c)chz8fqdzRjbhn$GKe(J(f5_{6T6|g=wX0EDG(A z62B(LaK0j19EW;p1msLUU*JtawP&JheD>h zB1)pI_{J5)8NI?*D|y;VO@-RdqA$pM+gA1{+Ys>Pf%H#*ukF_M|9*b_n5GAenO_^! z=goiRE_Cy3V-{IF^rYf=_|}`7EDD971pT9h*1S~pv$uuT3|i8W7Nx$u8_rv2azlri zmqnBB+q80^yE1I)xu%cCQ#qI#QHDkx?3YX)1xa4_X}~7U%gvjJv{AQ|pkW!Py;1Nr zVwR6-IT~Do3a8A>7l!YY76>}MOqDcU1npA{PqK{P*Q5nyQ=Ych09k9q;WuSs*{ z>OWm$P=M%qjIUF*cdIqQWCYM7g8fGJm@^9WHa*a=A*KaQ=r!Yrt%f3g(Bp~FGi5pI z=Wm&{O5+CkZ^pOhWm|nrr!l_q-;fAfeoQx!BxBe)D9@C&>7!&v>j=K~*YVU6P~=s! z<4YKBBqJKl6rBrR?iIKNQF&qD0Ouf90K^rI{-a|+!0P%>Ce(TpF--pyV z5)#q$poRDLWt`iM&<4do&t7QvE?>T~B=0RdJ#IBPCDTs&h4kJ+djjXvy893JbHZe za-^9Ju2&Yda>dGiDFwo>)i3o?1%ZC4hSADlW??aBq_Q@hJg*?Z7iMw+bGib5!sBaY zHMO&4U0cb+ZJ=``y(e;x#C>NJvO>Dj3xs9#0*mx?xF);PIN%DbsiG)Vn&E zptqFAS5MtKka*iA2Bq8ks6ZG8kr}b@Z|+D`a&hAMBx!#|NT5+DXRk zarlyUn>PauS@aeXs{mB1No~y2he6bqA^K zI4#OxbOvUNz5_|{GMC& zjF$M-d+g}@rasuW<9|0ltX}!iN5@Z`h^hH|@JLh=Y3#3w?qV{T*lwB2jEAj9ekRvP zc7rhHz(`j+3a+)S*GBcLUQjjL)EC%)jJzX`nR+(v)Qbhsj@p+`4#i0t<*?{Hu&87* zP9iBts7^9&YYT-DN9MY+EOB`3jXCG_^Ts6;x9w`kVoSyI;)e5f>4Kn%hV^~MCKGQr zK^i)zx1OkS82OI+r<@?IwA+(7r$?4SVOf`#q)1Arjs-+b!O3(cJjGgNwOtg$*x*l0 zg?qn>Bv3G&24bJYr^#aa@hvcWe`fyM(j#HhXT4RH4^LPijd)L)tR zOHR&#gqh|STx88yoz5&2iRq}i1au-mC(=BO9Uc82vITvCi6{bQhnyCQs7R6&i~^q*Je#*g6aK0f<21b)AFhG&y++ z=VsU6++=uQu$@ioFiz0bBa05emna>h-bWV24AHOUc-^P|%qGJ%SE)eJl^TmgIVOw_ z1B-6Q8r}yJpUcUUSTTymxw{IZ`I`;buzFE%i11Ueo{mz8opKUXcJYOHePxx-=b4NA zsJc=k@x!_6M*&8~uV4*K#11*V535BV#~BJ-bKYEJd^V1H4`ULlBJy&AplF;gVmq9Y z*cbOgMYfbxx^xBolXveEm%e=Wl{QPnicafBL{I2)W89UK`)RtX@$u-teB7Scwrt=( zkDrSzgDk&!Cagzz3^RD+T$Cmu z-pnoy8+*GeGZrE{!LuFAHZi?XVfq`=Io<&um^p%qQb=%r5uiq7ED^os7oh&YT(0 zaWONO7-V`!hcVN1fr6aH%*o*OHCg9a=<#jiekJPAVGWwuGIL95c2+hTDAL=It=N(1 z29%p&RY@W@kq0!FMiqO5FNeW8u)}Xx!C)Oo7?>u>A!E>Bof!C_!8*G_ zeG$X?q3Q;b`0N887_5UTf=KfCqW;+rbp>rZ_g?cPrhac|D8qxgVER)3& z3k6%R=;bCxHjXH)*SWvkd5dQ9)EqP%XrCc3AwB`k5DL*Hc=YMyj!1STUY_lG_yJwI zILk9EW_WKj2uBgQxaY!sXpFI&H01W}C^SRI?@ILYWy|}gNTC(GhfwRmESScRLe?pk zM5A?_+1}h{Ha;(`Umq`YgM*P8y9@2zn3|l*SDi2(Jf$X2LpCdxTy#Jq%w=)_**K!8 z9u3oS!Jhvkvu+kngRT;T6HUat<WtlFI`!%zLMdEUPdrz`5b? zg9oLMS;?G zz9JF%zI_XOny0zAcOD+o>eqm9vXd@Vnnc2!B$si4ZURfZ1Q8ZItkY&B#Icw3u{t54 zy9HxMJrl=jBl|lTxjAAF3McW2v>rBiGMUg~&f{2kH$9LPrhta2C1vFBe`tIgUuRmS z6UVNKV3V2_EaPylL`N+cnc^Voz!c4bQx^oLe4P!$TnOXbzAeUe>e`a~bFDiQ#yU8; z&9lkElv!I^P8@~Vn4;m%zJ7i^+A)(avf@E9)SnqPxMIcfVrJ-7?~q%!q7R{&gnYh6 zeVOT_>cJ-7j)AbbOkM`&oQ%<&kv|L}zr%M5vP+ZHrKhPw-9R|V$)ug#`m!@%&!9mL zpLkj{yr$+77y#2=M`um-AE&AQ?5WVs9!?)NO`m4!w}(1^u4L0r!{G|qK=Koet`nhF z0ymb3FbnF%-_8Z=D1w)?z&d` z@k^sNQ53y%8hvCy=q{?$U(j6W;E0H4$NyMb^y8N{nlmtQtt7vxHd3xutvN=Crv(J8 zomqH~VS)+jFtasA8e%uq&M^AyUT9$2#85YyAu9omEeB?r&}+I%{alkAt0-^g*c4*7 zm8HD1H8TM*+{)lRY~zRneZs(x&|1wj1snh#Sb4>jd36U%vSIkm$g=Iq%BhKRBC=aBBkP%9(f$}V_f|(jCnh2kvLF)1^awQ`BDFUDxB8u%YS$rkr{z`=C^RL*MyG?6*+E_j z@!x{TQkNW#RCH##`mLr4FHLhwo^mn9$FSjp@gCr3n?{9`NjPV~K+kUND%bc=BVAK{ zZ91Q04^Kigu``{7cLi9JC6&6gY%k8?d9zL4;iPNg`yhf$mtuwJ-MHFoLH)5FHxE+p zj|G`sj0uy9%qt+TyX16~O#GNmq~bWhqp{HQKYaaXR8Cy5R?;T$lf$Kd(8Rm{`-1qr zZTpg2^K;r=o&C$)MSFJne)Ue}lt2Fy6EnPHZ%_cKjDD6wPFtZ#@bF-|)(({JaD!Kh z(6x3>m#t7R%>%ch^W#zJ57ZyyG`YD%qgIqKFCJqG&4WzH{Ojbb zl5B;)AM`S!Hzc|&=*maRtYKr>$uREimQBrm4XDSJ;GTf%6RDPO@%^EbXou&6si_nxy?srvP%k&tJJJp2Qp6%pE4a^QVE`l$;uhDMi6ubuK>ta-j75V!yFTE$^_i;yhhS0Pp7`GC6DmfD;Mq!akS7CI^dZf>a5u=Y| zp5-Wgh3{|}HncQkSTxhqI=Uic=&f5_RX58Xh410Dona6H|wCD*B|O7(T0FNN-K)V0+3S;#sC zOQ<&$37%my&F$1X##l$@2!<`(lURroqSBn}m8i!VR1NpV4H;86(X0t;0FUbIRoS>; za*^N-4sdGhE}(gn(v9JO84S4zO511dVCxgxyXTrGOdA;=!=rnySBU)gqe?XDj>>Wa zRa+#998#4#+1#b)?v0KJ_m}XpyKGi4S>|j6;*Mcd?Rr@RPJ`YDCrZMkVxd8uYs-7X z8hIHCP5j~!=f@RNXRHDo5Q#k7#3R;2)OrW>dX*tva?~< z6EjZ{sS`~={G5P*#rr~{`@sl0^cK0LuNmNW&n6jnc4;Xy=Zi#$aGqh*MIvJm-LFk2 zhkRnQTm5o<&aC^t;M~(JI-wOdc~+|y;%P|qwfbw<#j9q-V+i8xnW;J`cvVCT>5ilO z9No{u3G}sqUs?jUW<-kI5Pj*~*{J0uxy)2bP~Hgv9ZqmwB2T}t#VCP8T1KsXjnqDp zpHIM%wzJ30+SkzQFG*DK2yb3 z>T_HP%Qd~qa!qqtI`lR~hXz1{0pb%qf?6{b;|{7OTIuBvS$esMrI)XS-gDy1OuhdP zcGP8fmx`#%;~P+yr55V)&ISw=G-VrA01r1{M8mLC#E3Q+h7_hqreWmh-H~c5#MD8Wqam2gqIHWCGX~Lm+TqAcO{dI}a$0eJ-y!bRFZh;9m$&d6xLtB2~L?OjM1Hi8JuXUeRquBGc%)ZolEvQ zwWC7jB{V%kVlRz#Oi+9W2~%ESo6Aq4dp6Er%VGgd?Hi{(TuzY#iK>bwu&!NFwC)c( zjUsDC?nti54cVxRr*Q6cy%ff}AH$m~4W_m07x4+%JP$}(vT$9gON>b=D=u_x0x;p( z{9MzyOv&2*coQXQ;f2Q^UdrPcU*Nqr{D(AQu6Y%&c35Ri})& zUgRU|iZ`^+HI0Zx`jGe_dgg~Z*97;0G+klB8r@Z7K1_E$iZ>n|qc#`TufvMbotb13 zhSL!h^rCqGltaB|ba^u9Zy7^>>-*-v(zeL{{yk0aJVIx8Z5wJz_~hHKzBzZ{;J`~) zeyK?_cc>jpD`z{@s6WP0t1u4XNm1wL1fkJWw)w6>EO9TKCGL5$P5-t*#Uhr}Yn!#3 zz$j5tnkQ&En=DmouFf{~q2LAnO!(l4KTK5GVvxi9$^MvF&$Lb~ zCI`|zp#9$IBat}|pv!!`RiI#+m*l``?B-pku39YC&Xzz62-Dh#!A07fEUlZ}8QRD1 z?aMf~8x0Za)5*=V7a|fbU%s*=?=7342EU2gI*GUJ?H$R`J?_xChs#B>kflox_8tsm zd?&%k=oX=KfQA^GJPA7{QP0lUdKzvG46%`Z>(?HEKZrPY&Kx(XHvpMaLxG6M>Ois6 z-G%OU;?cq0{c)O8xoIE~;Tm+3d6SH(7@5w7{{TEqU*g;8Kq%GR{RnWbI3VDq@MRw$K z-FE%OU063f31dN1L)R7>@(^{2BYOld%VUtc&7m4jd8?$qu>6}*_c_MmE)sUNfXs$z z@i-y6V3T2ramMu4_B%Y zk$L}*VIwxb&q{dMganS};<#a2JrGRQW&*{;L~l3mvFsNJ*LE6D-P zu-%Pu=|luuZLF5-cLJhWA+E8k_TEEQP0A*AX4B@di5}V%R?+f#sK3*(oI2r&|C|sJ}IcscogFE6IB}rV<-K#LUW$JY}l^fP|^l$=H=rg9v%J zHVCIYWGsT{UPudAc}Q4b^frOv1qk&dQR4NF)=9E5gTCm0hgjpI15pPW_nL>IWG2x?#!I z+8ijs>KYv$ksi0ev{V!OyzJs`jKL_gHs++Wr=ey}o!RL)<#m&3eVG{Kbp;v@)#{r( z2U=Tz)O7{r?jRf3BmDxT7x5)ZvrQwE&|!)hpI?WQC!4}d@Oe{z##nvLMTycN zn%-Kw3S(VAnYfuQK~tIpp4K_NEcv(~Xn??IL@A+~`0IasXS}qmaQDJn^LM}Uam)>t z0eyDQgY{loS8wy=6TNMw}K4O8L@dS^PDXL9*94UK5{)BTT?K7amLx9(JLAX#w3V9-LMBz zykd1_czumb%Jh-VQHazb!_Gx9x4aUy4827Kk(@21FeaMO1Tkb=#33_uNiau^(XS~L zSp@n@vV+5B2AG9Hf*7$b?Zs$VW)r)B%m~nk49#<9-aVP;Ia|czQ0qDxi*1?btbh-wQ$TAg zBRe(8_-9f5@NS-g0W;H>5!tNNNwg*#lnfgxqVW1!Wod<6Ik~A(kUWH7D;u{K!ZMrK zg#&y%jSQ;{f5yI8VK~hW+s-C`V*f~F1dQR%6^cg?YHN&ceSSurCj1M2(Bo}oQ%|%^ zKH%eLWVFg0G!Bv@5Ba2YJ^SP=;J6@ zCPp_3t5n-7!U9VQmm4(}*8ig^(2(S(KsS1`tU|61ZE6%w=# zZR%+)d^gZIgfZ|fWd#y4Q~J7K>v3)R5ry@-Ax=bfPRo;tuuPnr9#(l5O@wTm`Z!{{ zF`SP?db5bYY|)~6opkzQL!JU=TqF8tRf0uWJ6hxA@7IG-o6h&Ns>CKlrtMp3)=QR} z^>pDeU5Q??Lc>TkaO*t8QX#hcnNrcDW(e0t)tkN+QG1TN(O literal 0 HcmV?d00001 diff --git a/packages/webgpu/src/__tests__/snapshots/import-external-texture.png b/packages/webgpu/src/__tests__/snapshots/import-external-texture.png new file mode 100644 index 0000000000000000000000000000000000000000..a15e25e3a9bcbe218a173d2b6915cad7035b0501 GIT binary patch literal 87271 zcmcItd0Z3M_U{ub)~Hdb;u8fI3N94%RRz~rmx>!$aqDx5OMTi`6|hvWYUWW3X=-t) zP;H^cty1@;?7`E7LMP98>7};tKij_4z8(A*(PzlEm;N&M^t8!u&W-y^ z`PIzNUjF$%eQ$OD;ImG5yL}baH*Gq3@;93JDP^AQOOM&R6aiNgb2^SIOnjy1n+4Tx z{QK@dd(#i-!(WZ~{vW^n_v?4wc>f>QPxARcvO87V9#tCtQnb^7e`LP*BY!vg78PuY z*G%R2x9pbmY|jp#;Y7apiR>3gR?FN$@>hfW5m_bfDEX(|9r^uYWejz|I7T+1=CEUC zf>dJjq}2Voy2|a6kE74$AJ@x?LEk3NnU4NLrGJ=AcM-R3 z6Yp4|#rlsjT`a zC2wDT@7%%Cti;*#_hv+C^0E|O$4&;GU74S_!_8~rJ4;Hk6TIB~*B?25DQia`uhE^? zwqNz3Lgcx=V_L>;)zda(Hm-dVa<*Ucudf9z&Zr8Kd{mH3w~y~r8FK5~@~Hz)%vfy5 zR>XUG&Ut6avivcfW7gj5s(RYZ^Gei;;=F{p^F8lX?QrYkwegelmvR$=(t2h@_e=Tp z<&Bq1C*&vgIe0QKS|jo|l}(LQ5uauyy7@e5-xjSXPZ8eHw;NieCd++vn|QSH=n@Y% z#pJ;!L~BEI@jE8;g$1ooGvrR_j~40GdCP$TUC_e4{Kr@Q3HLkX-M(#`KJW=&wJONB zn<5@Qk*iiA!|i}i`uX#o-4wZ5@VR*YynCDY2~(#|&5wBJJ{W(+vSrJP7rk@eEq=m* z!-o%-9y?i{FuDJ*p+kr2m^(K#$*`!Bw_sJmZ7kCt9W5crQC)d?5b?H?usn6LJbpsoTeolDu9`dl#l-%*d&0-?;sbPR%yyOBO+8Az)@TQ} zTfZ3nUMHMF{o+WKbZ!&tpvNuH*mrI|KT1s*>m)QeeZzyGD!6r9;owBJYK->!IgwxM z_jcjoFCDUW^^6|}5nXvfk*~4IUsCn{&ml5Bflu7vA)EW{4($k^EklNEcG5qPiNV_N z!NOuwbB?&|E;v|Xx-;l(60*=EAireRDkMT7cM9RQalk_>rKY(CvXPou#yuD_0c7pk zQTCA74R9RWrses{zP{@elQ!{)+#Uw;LNySrPOrSK=$U7S;iFHiNQNRzNb3VAA2f$^C?=Z(7cr^ zS*U;K$3f&Rj$@M}{**h3sP7INZ1%GJ#%VrZcygT^?nr+EFWVn`VnB3{R0X{%G;igE z>;i!7OxGL|7C?z$X47<(Ymajok@N6H%mXj`ecOh|{N}(u99THrZdUXmgyVQ!e`lepsS36rJ{OrH^OIeO)a zmHko*ULNh8G4$50uBx(bqR7I1_ZEhx*UmLvoRE{?-wtYBVSDdoRAoi}6PM-UF zLP>1Yk`<+SA`d8krHh6yj#~j)fi^~w+v345^(%2)8II+a!q-WkeU%Zg`=x-W(T82Vh?a2~l9^a;|wD}f4*2Yl43 zpU1=DnJMuk@2Q81#!UEZsq;5f@9h5M$(o)Sx4NKTAoB7E9*KVZ;ls;|Cr&~?G+^z_ z!h50G7^t)*#)~>3dz?KxEPCYt^xH@GY8N=1F} zjk*sf4OCjp+PmSr4-_BKCsO;a50oTPaZIZESq9ZJKlog!_kd>wHMQX#7q3ANH6bBM z;pK%M>h0S@qYn0ihuWjZNWb-KkHEsFPR+?mNCbo|TzGF^#yNzLK7Bg5dG9y5z;?oKFZl3LWjYiisNnRdaB6P>9tc3W)9q6COmFDHjE!1;;NSrC zR~}{zJ9lmux}|t=&oxh&#RRXPxi;VkaXg&v8J{Q$?+BGEc$m?p2?{8B6};RlaO|)y z?a@QaOY-#VrU1OJ$!m z%JoDHF(>($3H@{)HrGqX*K6>CQ%i0cJ->-bV6zCcuQ5ouqsm3bG=(d7bj7#V=bAPv zO;LEUsaMy4r@|{EISfy+cAFqf42Ok$JeO4wWk73>H`ZLDr2OaoC z84gd$Wq11Pa59O1fEnUn$3JrE>578L0Rmkmmk*#e!-fy;$}4K40LLe4nZ_w)UP8Aw zhx^p3pOx)(mH#x+TWeP}iPC)W$Pw}-`RNksP}86S=n)*Fl%)khLz-kgJn_hpLQ$i} z*Z++x+i?ht&7@Jv%7P%Q5FG&!^~4O(UP{?taYFP=Rhi_Gv$29h|1eGNz3_eaIP#L?n&>%8JB7d)fbKYJ!3OqJ@GJ|oGI`~#D!$zh%L1_QeVL-JeRkndVv8j9+OmO zGzChRZTgSK*T2`~w8Is`c^;k4loj}xeECIit_{v{jMv_+E(^yAQQ0h|X&u+8v6;h< zL&wp(HD&4^ys}J=sNVD-KUXGJnskvJc%3M@;K+4ACFSAt1)N@Vv!Oc3f&9`TFG?!I z>GL?f=w@`3e_L=>i7R{=$8D|MEA-sXGW;>w_~FeFK~*-50A9 zP16+616xT&jZ$d+JpTTSFZvPVj{Op}8-jYRL7L zUN`Zo)u`NrDtQ;rfA3}KYMXTbJ}OP2s^6*d@xq0@ObAJiHuN4%D z>G0w#&#;)`z1y#fg__vno(uP(AL=FzxqUmTZ*tLVepjNGFI$d&Y3D7P$y0OUXK&oQ z&ybf8pP-n#(K7_v;QDlOpLl=CiV`o+c0K&&+%G7iI}A^YJA~T*VigIZAk;Zn4mHsd z1?mzkdaXy`<)NtGB6C8H!XGsgZrr;tLzj&j2P@ii^52ho4&i>9Y!T`&+O zMZKYGb4Q{ZdahMTA~%u8F=}H)xjW-a;gXyRRrb$pa z?a?kUfcd>pZ};zop5KZU%ZsxoqMsIW>sIukeh5pgd`%}cX)5G;pMF*wpeG{oAMK1$ z=@;Tytz@zlP8_Cj1e3jiu$PXJb&ZHNtRx$fn(~x zaiA*o<&w(%cGiL-WZsxN{T|mq^o_z4C3=PGa3F6c z-e)Vd5Z)s~TS|Ludzo(@W%!<1rX^ow?IJk&gjJrVX1T97H?`10oF$ZwQ9!^)8sLv`?k;ypY(g~D|`r)Z4u}_0M z4hC?re$=8DJzk+At6HhL2{E&gEWi5WXE8rq=$o}>@H*p%bl>J%^0?kVwbb9Ic~wjfqUYE*nq@a_G`q~vXcp9{ z(G0y75TU-c>_N;Dqf6PQM_~F|y2`Gg4oBu)zpWpw)vv)CwItibXnV07H<+`VZ!Txg zSs-M32{b>E`*B%U1IrwJU* zyc*QZYei5lI--Oab#+u{t*-hI;>5ROtqo8pWmgE$Ub`YSn^gcYO8O@>vj!YEy%yM_~^ z@?LBhGCj2%J5)-f?Q#YV%i zM>$E#rEgj29}eS@gG*vuIhK0lxf^E!I@{C}em_=-=EoVXxoXpwj-ysN&1|v@{K`^I zFtOX7DAsIl$r*z9DE)xj z)bOOk5SfIXo2U`qrlA~ky8RCAZtnA`=_4_*4`WuZe?QJ}-N)CSx5ylI_`y`ad=qihLcS+O_O!r3E>WGJi#anL@`_Hu#>28Zz=M zS=ZfaUFNhV86`u%6PBNnN9FWA4jyPTXLE;wl~kQW1mLSO9i+xVSdt{;4NKnMrj=MB zTB*^fx%jpiDkIv;#~SsQIKAk8Z`rd*?$QXoeO$(ZO?`jv*7l>%!WT>0wABpU`Xjx) ze4#!f>|cGioog4^{>y)C%==&B(eKnxHsA2__v-=bbeYid?~Zt<_d!;W&o$%bOUAvg z^$25V%cw;!o;j^Ti(ZJC0}U;D?hFSSwdmzCz8Olay%`QPdgbcD03Y>K6*C0_GCh@e zy4_NldVAxGZDxLM)PjvKrdU|J&?WVznH!xkWK12D9GG)M@;CJsK51dfH(Pg`>AB-o zH)IwPbcIT-zAj10h+4k9Bp3CWA^vD@rk_vb*~2dkwWBf3zwxM_kJr~;L_OP1^pDmg zK_8_jYFsLfcSHQ~a*#z7d``YjcKCDz#9Ucb@;=$3hg(JpX-=g+4y@{(Z59?!4sweNPJs z`fz!1~kMKcSDV)(VFb+Tm@=CL(STdl8XL6?ahr=?6Mk)65p{-_ouJu_kI~b;3W&O8D^`@WZXZVg zTVG@l(|<~$SD^k}#Ek09wDKj0L>~UAKbK*mOhB!Ci!;s+ZS`xk_j;L8VtPO<$7V=E z9Bj%V&~C$lN?71bt@^4U+_hlWE%iCGjKKbM40p}KlQ7CtrOAW{NG6?;xB+argy685 z0U(wbE9@GTPGk(C{T%d(&;!lwVDlw~$+KE>dpE9F6d`<8t(I663}RQUZrLd?cgBE3 z7O?d@!IZCqbVdjxodGs~7&7c!6oYr6*WjM#k2xQ5So6}+5hJTG1{|B^X-V)Z4~97I zoQq_kO8@tIkg5l;a(g4G2MDC1$h3V6L3Fl4_%+j4spl0K%}G2uO1TJ~WC`3FD?CrE zm%VNJQn>NB3~!njtc@IO&1_hh2~LyBq#s6@{A>x04POrIZ0>MhdN*Sbffb?TZ-7ws zL@YUxHJ`e31{naC_XLR^*{M{$=W~~Ac`^ZR%?Rx{kOHy$8wHp~303u?AB9mMVscKIZCX?S`-1KVIEaP^Ap!~cwIIAJJZ)c=-oJ-r;?7sHLLh2Cp8W6{E2BoFz zRnp1U^qXf|*+MRVr&0@|vaZ?mJEI~Hg|-pU9M9ZY*b>PQ4!;lQPwS(t! z)_Pp0eDX7Zs2Vdwr9aIwe$&>mlX*)8}v!Bl`m_Hy+88f%SEH(0Gmrl7g?5oq2kcdJE^KKfYddrrI@|-aS~mpaafLpP2!X zejesc9|#6`72Bq#_oDjZI;b!9^i*h9f84jvI%y?6RA2MhxBZ4XQCqiyKWB}Ii1>hb zcJkB7wdol{=%c`AMl8rI;4|A*nSij$2=D=)*;e2~d}c(091DDATPY7QNRj>?PXkzERz)z9;A5(J*{E3(qH5UXVbCne z92h*2wj}awB}pLOhi*t;1vV)Z=#~`&o77gJLu^up77le%TX7VzNs$&Luu1Le0f48#( zr8s&{x6Q+Yp`8o9b|;WoWK?ki(#?cyHZ5+YwKNrkR1Q%YGS>?!ZZg&8)Xk*WGO0`{FVB^7(SgQsnE7ju}7kv?byYiQ(`=D7r%XP#8-wJVp3M4oaLT~AMFWcGFF&s8H!dl>}m1fpm_iNK9+*&eC@JZWe*6a{|jof%K392UzgEP-< z&Adoy^dRW}6-j<`#a5qcBs1&s@xpU|CUla&EI27bXNe|B7ICvk0`j`p5~UP$Lzg(u zsYW8UW+zVXIoI??EN~s#3*X)FR=bI}pufr_Kw%dnM!(7=Pq`$z>*>mhNSxvQ7&X2^ z_AI!&;LHd)k|vp!%qNz>>O|2Ey*P4lsMR~^3$!({H5c(>v?pLR6*ESO&ofD^eiYsC zzfRiDEVk3=cr?{mS_G<#NVZ+TS%@<$RhCvjcj<(D4@>9G3XQwJA0v+>{F?r8M%^_i zNHxsMxspq+D8zAQGJa54|DJ0Cy|S_*+{5DP&%e z(huI{;W}}uq@@+&51c!zH{%~nAH@>;Y&BEI>K|OkP<>5QO{ydm>nUlDadSpp4pxZ5 z4T}o-);XH79V5ooq8lTenr?7)Rf^9WR=CK9>3ibHHk4#ETsfqQ!YJXUa5t)dO!;IZGb5 zYq6$AjDs~Mik#$=+Of`Hk(Ji*5E#hg_KZ~h-5OmiR=Lx$ls>jCR)oGWmRf?vmya#fqP$HsX+?_scdvV=eecooNWiG zG58tzV&d6s(=1Fh6P=Igk!G(xvL80q4qX=Mts)h;czRXU!D8)f*D#=%TVte-)4=%L za0DwwkFJ{C87GNUC-GMGV+V(|&I{#cj@ds}!Jt*YL(o-QnK87~QB;3kj1P ztNMmv0U&A@u9OMsND(P=eURQY>RQAnaPKehaY%JjA<$DZurl$fw;>4)#gN6LnVRV2 zpX~#n4|7rGbh8-ljZrMZ6yPlzL~h5LiAuK1^0FnvvQx3**IU;a3L)6G=b%<%999~NC6(#A~a zWmI)>i#@%?pa&Fwm2Q23%eP`ybDBIf?Y6-Lh-p1DiY3p3cIa7R^+Rl$} z%XaPCze8!l-j#KaB?i)8P&kZ)0?4LdWtpV2(HG$rWhgC50)ppa$nw&3PDlH*s$GjUU#?_2g@K;dvUmU=hP! z2o%v+h(@@+2TiblU~74{^9-8UsG0}UF@n>uq!0!Yr{D!Axjhpk_;>^wp5C*vq~c%%!?@zJZGu?hAp@^uSHFwoQ~|C&25{hArfLzDAv{FzHf51^Y?P{jUZz4dCMx zIgp4Nup7Zz6Z_%CQ+Dba_}Wmii!R{Y=_=xfx}2lkctvd9!HL?NxsFfBh9|)b!SsfU zNiH#_p1AbjEulC>oP)-wHs)8D&MNRGN(1S8MI)QIKMv3_BX(L3>nh2{nrpQJXd!k! zvKh7s9ScqWxk4pNI6}StCcrajaFbkza}Oi#LE{*X7#~eCRLSFty(t3ob}n~n8Wl@^ z?izs76%~7TW6V5fn?}SEhADV6&xyvnh4p{K2~qMQQL5j|-!y4TO(%xXv)aiKR zk-jpz<|0;%j;k<@bjj&RMXHO`c)chz8fqdzRjbhn$GKe(J(f5_{6T6|g=wX0EDG(A z62B(LaK0j19EW;p1msLUU*JtawP&JheD>h zB1)pI_{J5)8NI?*D|y;VO@-RdqA$pM+gA1{+Ys>Pf%H#*ukF_M|9*b_n5GAenO_^! z=goiRE_Cy3V-{IF^rYf=_|}`7EDD971pT9h*1S~pv$uuT3|i8W7Nx$u8_rv2azlri zmqnBB+q80^yE1I)xu%cCQ#qI#QHDkx?3YX)1xa4_X}~7U%gvjJv{AQ|pkW!Py;1Nr zVwR6-IT~Do3a8A>7l!YY76>}MOqDcU1npA{PqK{P*Q5nyQ=Ych09k9q;WuSs*{ z>OWm$P=M%qjIUF*cdIqQWCYM7g8fGJm@^9WHa*a=A*KaQ=r!Yrt%f3g(Bp~FGi5pI z=Wm&{O5+CkZ^pOhWm|nrr!l_q-;fAfeoQx!BxBe)D9@C&>7!&v>j=K~*YVU6P~=s! z<4YKBBqJKl6rBrR?iIKNQF&qD0Ouf90K^rI{-a|+!0P%>Ce(TpF--pyV z5)#q$poRDLWt`iM&<4do&t7QvE?>T~B=0RdJ#IBPCDTs&h4kJ+djjXvy893JbHZe za-^9Ju2&Yda>dGiDFwo>)i3o?1%ZC4hSADlW??aBq_Q@hJg*?Z7iMw+bGib5!sBaY zHMO&4U0cb+ZJ=``y(e;x#C>NJvO>Dj3xs9#0*mx?xF);PIN%DbsiG)Vn&E zptqFAS5MtKka*iA2Bq8ks6ZG8kr}b@Z|+D`a&hAMBx!#|NT5+DXRk zarlyUn>PauS@aeXs{mB1No~y2he6bqA^K zI4#OxbOvUNz5_|{GMC& zjF$M-d+g}@rasuW<9|0ltX}!iN5@Z`h^hH|@JLh=Y3#3w?qV{T*lwB2jEAj9ekRvP zc7rhHz(`j+3a+)S*GBcLUQjjL)EC%)jJzX`nR+(v)Qbhsj@p+`4#i0t<*?{Hu&87* zP9iBts7^9&YYT-DN9MY+EOB`3jXCG_^Ts6;x9w`kVoSyI;)e5f>4Kn%hV^~MCKGQr zK^i)zx1OkS82OI+r<@?IwA+(7r$?4SVOf`#q)1Arjs-+b!O3(cJjGgNwOtg$*x*l0 zg?qn>Bv3G&24bJYr^#aa@hvcWe`fyM(j#HhXT4RH4^LPijd)L)tR zOHR&#gqh|STx88yoz5&2iRq}i1au-mC(=BO9Uc82vITvCi6{bQhnyCQs7R6&i~^q*Je#*g6aK0f<21b)AFhG&y++ z=VsU6++=uQu$@ioFiz0bBa05emna>h-bWV24AHOUc-^P|%qGJ%SE)eJl^TmgIVOw_ z1B-6Q8r}yJpUcUUSTTymxw{IZ`I`;buzFE%i11Ueo{mz8opKUXcJYOHePxx-=b4NA zsJc=k@x!_6M*&8~uV4*K#11*V535BV#~BJ-bKYEJd^V1H4`ULlBJy&AplF;gVmq9Y z*cbOgMYfbxx^xBolXveEm%e=Wl{QPnicafBL{I2)W89UK`)RtX@$u-teB7Scwrt=( zkDrSzgDk&!Cagzz3^RD+T$Cmu z-pnoy8+*GeGZrE{!LuFAHZi?XVfq`=Io<&um^p%qQb=%r5uiq7ED^os7oh&YT(0 zaWONO7-V`!hcVN1fr6aH%*o*OHCg9a=<#jiekJPAVGWwuGIL95c2+hTDAL=It=N(1 z29%p&RY@W@kq0!FMiqO5FNeW8u)}Xx!C)Oo7?>u>A!E>Bof!C_!8*G_ zeG$X?q3Q;b`0N887_5UTf=KfCqW;+rbp>rZ_g?cPrhac|D8qxgVER)3& z3k6%R=;bCxHjXH)*SWvkd5dQ9)EqP%XrCc3AwB`k5DL*Hc=YMyj!1STUY_lG_yJwI zILk9EW_WKj2uBgQxaY!sXpFI&H01W}C^SRI?@ILYWy|}gNTC(GhfwRmESScRLe?pk zM5A?_+1}h{Ha;(`Umq`YgM*P8y9@2zn3|l*SDi2(Jf$X2LpCdxTy#Jq%w=)_**K!8 z9u3oS!Jhvkvu+kngRT;T6HUat<WtlFI`!%zLMdEUPdrz`5b? zg9oLMS;?G zz9JF%zI_XOny0zAcOD+o>eqm9vXd@Vnnc2!B$si4ZURfZ1Q8ZItkY&B#Icw3u{t54 zy9HxMJrl=jBl|lTxjAAF3McW2v>rBiGMUg~&f{2kH$9LPrhta2C1vFBe`tIgUuRmS z6UVNKV3V2_EaPylL`N+cnc^Voz!c4bQx^oLe4P!$TnOXbzAeUe>e`a~bFDiQ#yU8; z&9lkElv!I^P8@~Vn4;m%zJ7i^+A)(avf@E9)SnqPxMIcfVrJ-7?~q%!q7R{&gnYh6 zeVOT_>cJ-7j)AbbOkM`&oQ%<&kv|L}zr%M5vP+ZHrKhPw-9R|V$)ug#`m!@%&!9mL zpLkj{yr$+77y#2=M`um-AE&AQ?5WVs9!?)NO`m4!w}(1^u4L0r!{G|qK=Koet`nhF z0ymb3FbnF%-_8Z=D1w)?z&d` z@k^sNQ53y%8hvCy=q{?$U(j6W;E0H4$NyMb^y8N{nlmtQtt7vxHd3xutvN=Crv(J8 zomqH~VS)+jFtasA8e%uq&M^AyUT9$2#85YyAu9omEeB?r&}+I%{alkAt0-^g*c4*7 zm8HD1H8TM*+{)lRY~zRneZs(x&|1wj1snh#Sb4>jd36U%vSIkm$g=Iq%BhKRBC=aBBkP%9(f$}V_f|(jCnh2kvLF)1^awQ`BDFUDxB8u%YS$rkr{z`=C^RL*MyG?6*+E_j z@!x{TQkNW#RCH##`mLr4FHLhwo^mn9$FSjp@gCr3n?{9`NjPV~K+kUND%bc=BVAK{ zZ91Q04^Kigu``{7cLi9JC6&6gY%k8?d9zL4;iPNg`yhf$mtuwJ-MHFoLH)5FHxE+p zj|G`sj0uy9%qt+TyX16~O#GNmq~bWhqp{HQKYaaXR8Cy5R?;T$lf$Kd(8Rm{`-1qr zZTpg2^K;r=o&C$)MSFJne)Ue}lt2Fy6EnPHZ%_cKjDD6wPFtZ#@bF-|)(({JaD!Kh z(6x3>m#t7R%>%ch^W#zJ57ZyyG`YD%qgIqKFCJqG&4WzH{Ojbb zl5B;)AM`S!Hzc|&=*maRtYKr>$uREimQBrm4XDSJ;GTf%6RDPO@%^EbXou&6si_nxy?srvP%k&tJJJp2Qp6%pE4a^QVE`l$;uhDMi6ubuK>ta-j75V!yFTE$^_i;yhhS0Pp7`GC6DmfD;Mq!akS7CI^dZf>a5u=Y| zp5-Wgh3{|}HncQkSTxhqI=Uic=&f5_RX58Xh410Dona6H|wCD*B|O7(T0FNN-K)V0+3S;#sC zOQ<&$37%my&F$1X##l$@2!<`(lURroqSBn}m8i!VR1NpV4H;86(X0t;0FUbIRoS>; za*^N-4sdGhE}(gn(v9JO84S4zO511dVCxgxyXTrGOdA;=!=rnySBU)gqe?XDj>>Wa zRa+#998#4#+1#b)?v0KJ_m}XpyKGi4S>|j6;*Mcd?Rr@RPJ`YDCrZMkVxd8uYs-7X z8hIHCP5j~!=f@RNXRHDo5Q#k7#3R;2)OrW>dX*tva?~< z6EjZ{sS`~={G5P*#rr~{`@sl0^cK0LuNmNW&n6jnc4;Xy=Zi#$aGqh*MIvJm-LFk2 zhkRnQTm5o<&aC^t;M~(JI-wOdc~+|y;%P|qwfbw<#j9q-V+i8xnW;J`cvVCT>5ilO z9No{u3G}sqUs?jUW<-kI5Pj*~*{J0uxy)2bP~Hgv9ZqmwB2T}t#VCP8T1KsXjnqDp zpHIM%wzJ30+SkzQFG*DK2yb3 z>T_HP%Qd~qa!qqtI`lR~hXz1{0pb%qf?6{b;|{7OTIuBvS$esMrI)XS-gDy1OuhdP zcGP8fmx`#%;~P+yr55V)&ISw=G-VrA01r1{M8mLC#E3Q+h7_hqreWmh-H~c5#MD8Wqam2gqIHWCGX~Lm+TqAcO{dI}a$0eJ-y!bRFZh;9m$&d6xLtB2~L?OjM1Hi8JuXUeRquBGc%)ZolEvQ zwWC7jB{V%kVlRz#Oi+9W2~%ESo6Aq4dp6Er%VGgd?Hi{(TuzY#iK>bwu&!NFwC)c( zjUsDC?nti54cVxRr*Q6cy%ff}AH$m~4W_m07x4+%JP$}(vT$9gON>b=D=u_x0x;p( z{9MzyOv&2*coQXQ;f2Q^UdrPcU*Nqr{D(AQu6Y%&c35Ri})& zUgRU|iZ`^+HI0Zx`jGe_dgg~Z*97;0G+klB8r@Z7K1_E$iZ>n|qc#`TufvMbotb13 zhSL!h^rCqGltaB|ba^u9Zy7^>>-*-v(zeL{{yk0aJVIx8Z5wJz_~hHKzBzZ{;J`~) zeyK?_cc>jpD`z{@s6WP0t1u4XNm1wL1fkJWw)w6>EO9TKCGL5$P5-t*#Uhr}Yn!#3 zz$j5tnkQ&En=DmouFf{~q2LAnO!(l4KTK5GVvxi9$^MvF&$Lb~ zCI`|zp#9$IBat}|pv!!`RiI#+m*l``?B-pku39YC&Xzz62-Dh#!A07fEUlZ}8QRD1 z?aMf~8x0Za)5*=V7a|fbU%s*=?=7342EU2gI*GUJ?H$R`J?_xChs#B>kflox_8tsm zd?&%k=oX=KfQA^GJPA7{QP0lUdKzvG46%`Z>(?HEKZrPY&Kx(XHvpMaLxG6M>Ois6 z-G%OU;?cq0{c)O8xoIE~;Tm+3d6SH(7@5w7{{TEqU*g;8Kq%GR{RnWbI3VDq@MRw$K z-FE%OU063f31dN1L)R7>@(^{2BYOld%VUtc&7m4jd8?$qu>6}*_c_MmE)sUNfXs$z z@i-y6V3T2ramMu4_B%Y zk$L}*VIwxb&q{dMganS};<#a2JrGRQW&*{;L~l3mvFsNJ*LE6D-P zu-%Pu=|luuZLF5-cLJhWA+E8k_TEEQP0A*AX4B@di5}V%R?+f#sK3*(oI2r&|C|sJ}IcscogFE6IB}rV<-K#LUW$JY}l^fP|^l$=H=rg9v%J zHVCIYWGsT{UPudAc}Q4b^frOv1qk&dQR4NF)=9E5gTCm0hgjpI15pPW_nL>IWG2x?#!I z+8ijs>KYv$ksi0ev{V!OyzJs`jKL_gHs++Wr=ey}o!RL)<#m&3eVG{Kbp;v@)#{r( z2U=Tz)O7{r?jRf3BmDxT7x5)ZvrQwE&|!)hpI?WQC!4}d@Oe{z##nvLMTycN zn%-Kw3S(VAnYfuQK~tIpp4K_NEcv(~Xn??IL@A+~`0IasXS}qmaQDJn^LM}Uam)>t z0eyDQgY{loS8wy=6TNMw}K4O8L@dS^PDXL9*94UK5{)BTT?K7amLx9(JLAX#w3V9-LMBz zykd1_czumb%Jh-VQHazb!_Gx9x4aUy4827Kk(@21FeaMO1Tkb=#33_uNiau^(XS~L zSp@n@vV+5B2AG9Hf*7$b?Zs$VW)r)B%m~nk49#<9-aVP;Ia|czQ0qDxi*1?btbh-wQ$TAg zBRe(8_-9f5@NS-g0W;H>5!tNNNwg*#lnfgxqVW1!Wod<6Ik~A(kUWH7D;u{K!ZMrK zg#&y%jSQ;{f5yI8VK~hW+s-C`V*f~F1dQR%6^cg?YHN&ceSSurCj1M2(Bo}oQ%|%^ zKH%eLWVFg0G!Bv@5Ba2YJ^SP=;J6@ zCPp_3t5n-7!U9VQmm4(}*8ig^(2(S(KsS1`tU|61ZE6%w=# zZR%+)d^gZIgfZ|fWd#y4Q~J7K>v3)R5ry@-Ax=bfPRo;tuuPnr9#(l5O@wTm`Z!{{ zF`SP?db5bYY|)~6opkzQL!JU=TqF8tRf0uWJ6hxA@7IG-o6h&Ns>CKlrtMp3)=QR} z^>pDeU5Q??Lc>TkaO*t8QX#hcnNrcDW(e0t)tkN+QG1TN(O literal 0 HcmV?d00001 From d67b3b2f8311dfe2925ff3d32ee8003ef5540fcc Mon Sep 17 00:00:00 2001 From: William Candillon Date: Tue, 2 Jun 2026 05:52:47 +0200 Subject: [PATCH 42/46] :wrench: --- apps/example/app.json | 4 +--- packages/webgpu/apple/AppleVideoPlayer.mm | 19 +++++++++++-------- packages/webgpu/cpp/rnwgpu/api/GPU.cpp | 15 ++++++++++++--- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/apps/example/app.json b/apps/example/app.json index defa402c0..73135e124 100644 --- a/apps/example/app.json +++ b/apps/example/app.json @@ -30,9 +30,7 @@ "metalAPIValidation": false, "infoPlist": { "NSCameraUsageDescription": "$(PRODUCT_NAME) needs access to your Camera.", - "NSMicrophoneUsageDescription": "$(PRODUCT_NAME) needs access to your Microphone.", - "NSLocalNetworkUsageDescription": "$(PRODUCT_NAME) needs local network access to connect to the Metro dev server.", - "NSBonjourServices": ["_http._tcp."] + "NSMicrophoneUsageDescription": "$(PRODUCT_NAME) needs access to your Microphone." } }, "android": { diff --git a/packages/webgpu/apple/AppleVideoPlayer.mm b/packages/webgpu/apple/AppleVideoPlayer.mm index 64e713fd5..a06144dc7 100644 --- a/packages/webgpu/apple/AppleVideoPlayer.mm +++ b/packages/webgpu/apple/AppleVideoPlayer.mm @@ -9,20 +9,23 @@ namespace { -// 3x4 row-major matrices mapping [Y, U, V, 1] to linear RGB. -// Limited-range (video range) means luma is 16..235, chroma is 16..240 (8-bit). +// 3x4 row-major matrices mapping [Y, Cb, Cr, 1] to gamma-encoded R'G'B' (NOT +// linear): YCbCr is derived from gamma-encoded RGB, so this conversion stays in +// the encoded domain. The sRGB decode in srcTransferFunctionParameters +// linearizes afterward (see GPUExternalTexture.cpp). Limited-range (video +// range) means luma is 16..235, chroma is 16..240 (8-bit). // Reference: https://en.wikipedia.org/wiki/YCbCr (BT.601 / BT.709). -static constexpr float kBT709LimitedToLinearRGB[12] = { +static constexpr float kBT709LimitedToRgb[12] = { 1.164383f, 0.000000f, 1.792741f, -0.972945f, // 1.164383f, -0.213249f, -0.532909f, 0.301517f, // 1.164383f, 2.112402f, 0.000000f, -1.133402f, // }; -static constexpr float kBT601LimitedToLinearRGB[12] = { +static constexpr float kBT601LimitedToRgb[12] = { 1.164383f, 0.000000f, 1.596027f, -0.874202f, // 1.164383f, -0.391762f, -0.812968f, 0.531668f, // 1.164383f, 2.017232f, 0.000000f, -1.085631f, // }; -static constexpr float kBT2020LimitedToLinearRGB[12] = { +static constexpr float kBT2020LimitedToRgb[12] = { 1.164383f, 0.000000f, 1.678674f, -0.915688f, // 1.164383f, -0.187326f, -0.650424f, 0.347459f, // 1.164383f, 2.141772f, 0.000000f, -1.148145f, // @@ -34,14 +37,14 @@ static void fillYuvMatrix(CVPixelBufferRef pixelBuffer, float out[12]) { CFTypeRef matrixKey = CVBufferGetAttachment( pixelBuffer, kCVImageBufferYCbCrMatrixKey, nullptr); - const float *src = kBT709LimitedToLinearRGB; + const float *src = kBT709LimitedToRgb; if (matrixKey) { auto matrix = (CFStringRef)matrixKey; if (CFEqual(matrix, kCVImageBufferYCbCrMatrix_ITU_R_601_4) || CFEqual(matrix, kCVImageBufferYCbCrMatrix_SMPTE_240M_1995)) { - src = kBT601LimitedToLinearRGB; + src = kBT601LimitedToRgb; } else if (CFEqual(matrix, kCVImageBufferYCbCrMatrix_ITU_R_2020)) { - src = kBT2020LimitedToLinearRGB; + src = kBT2020LimitedToRgb; } } for (int i = 0; i < 12; ++i) { diff --git a/packages/webgpu/cpp/rnwgpu/api/GPU.cpp b/packages/webgpu/cpp/rnwgpu/api/GPU.cpp index b9d2a46bd..11530f4da 100644 --- a/packages/webgpu/cpp/rnwgpu/api/GPU.cpp +++ b/packages/webgpu/cpp/rnwgpu/api/GPU.cpp @@ -26,9 +26,18 @@ GPU::GPU(jsi::Runtime &runtime) : NativeObject(CLASS_NAME) { // OpaqueYCbCrAndroidForExternalTexture) are tagged Experimental in Dawn's // feature table and are otherwise filtered out of adapter.features by // PhysicalDeviceBase::GetSupportedFeatures. The allow_unsafe_apis toggle - // disables that filter so the features become visible; application code - // still has to list each one in requiredFeatures. expose_wgsl_experimental_features - // is the parallel toggle for WGSL language features. + // disables that filter so the features become visible; + // expose_wgsl_experimental_features is the parallel toggle for WGSL language + // features. + // + // Trade-off: these are instance-level Dawn toggles, so they apply to every + // adapter/device created from this instance, not just the external-texture + // path. There is no finer-grained, per-feature mechanism to surface these. + // The exposure is acceptable because the toggle only *un-hides* experimental + // features in adapter.features; it does not enable any of them. Nothing + // experimental becomes active unless application code explicitly lists that + // feature in requiredFeatures at device creation, so the default behavior of + // a device that asks for no experimental features is unchanged. static const char *const kEnabledToggles[] = { "allow_unsafe_apis", "expose_wgsl_experimental_features", From 27b99d805d44b82f1740acd82ce0f343a7919852 Mon Sep 17 00:00:00 2001 From: William Candillon Date: Tue, 2 Jun 2026 06:03:28 +0200 Subject: [PATCH 43/46] :wrench: --- README.md | 9 ++++++++- packages/webgpu/README.md | 9 ++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 37038b65d..581a9e1d0 100644 --- a/README.md +++ b/README.md @@ -259,7 +259,14 @@ frame.release(); ### Importing External Textures -`GPUDevice.importExternalTexture` is the higher-level path for sampling a native surface. Instead of managing `SharedTextureMemory` + `createTexture` + `beginAccess`/`endAccess` yourself, you hand it a `NativeVideoFrame` and get back a `GPUExternalTexture` you bind as a `texture_external` and read with `textureSampleBaseClampToEdge`. Dawn does the YUV→RGB conversion in hardware for biplanar (NV12) surfaces, so this is the path you want for camera and video frames. It uses the same `"rnwebgpu/native-texture"` feature. +`GPUDevice.importExternalTexture` is the higher-level path for sampling a native surface. You hand it a `NativeVideoFrame` and get back a `GPUExternalTexture` that you bind as a `texture_external` and read with `textureSampleBaseClampToEdge`. It does two things for you on top of `SharedTextureMemory`: + +- **Color conversion.** Camera and video surfaces are usually biplanar YUV (NV12), not RGB. An external texture carries the YUV→RGB matrix and the source/destination color-space transfer functions, so on the supported paths the sampler returns ready-to-use RGB in hardware. With raw `SharedTextureMemory` you would sample the luma/chroma planes and do that conversion by hand in WGSL. This is the main reason to prefer it for camera and video frames. +- **Lifecycle.** It owns the `SharedTextureMemory` + `createTexture` + `beginAccess`/`endAccess` sequence internally, so you just import the frame and `destroy()` the result. + +It uses the same `"rnwebgpu/native-texture"` feature. + +> **Android note:** the hardware YUV→RGB conversion is fully automatic on iOS (NV12 `IOSurface`). On Android, camera frames arrive as an _opaque_ YCbCr `AHardwareBuffer`, and Dawn's Vulkan path forces an identity (`RGB_IDENTITY`) sampler conversion, so the external sample comes back as raw `[Y, Cb, Cr]`. You still get the zero-copy import and the rotation/mirror transform, but you need to apply the YUV→RGB matrix yourself in the shader. See the `CAMERA_PRELUDE` in the [VisionCamera example](/apps/example/src/VisionCamera/shaders.ts) for a ready-made BT.709 decode. ```tsx const FEATURE = "rnwebgpu/native-texture" as GPUFeatureName; diff --git a/packages/webgpu/README.md b/packages/webgpu/README.md index 37038b65d..581a9e1d0 100644 --- a/packages/webgpu/README.md +++ b/packages/webgpu/README.md @@ -259,7 +259,14 @@ frame.release(); ### Importing External Textures -`GPUDevice.importExternalTexture` is the higher-level path for sampling a native surface. Instead of managing `SharedTextureMemory` + `createTexture` + `beginAccess`/`endAccess` yourself, you hand it a `NativeVideoFrame` and get back a `GPUExternalTexture` you bind as a `texture_external` and read with `textureSampleBaseClampToEdge`. Dawn does the YUV→RGB conversion in hardware for biplanar (NV12) surfaces, so this is the path you want for camera and video frames. It uses the same `"rnwebgpu/native-texture"` feature. +`GPUDevice.importExternalTexture` is the higher-level path for sampling a native surface. You hand it a `NativeVideoFrame` and get back a `GPUExternalTexture` that you bind as a `texture_external` and read with `textureSampleBaseClampToEdge`. It does two things for you on top of `SharedTextureMemory`: + +- **Color conversion.** Camera and video surfaces are usually biplanar YUV (NV12), not RGB. An external texture carries the YUV→RGB matrix and the source/destination color-space transfer functions, so on the supported paths the sampler returns ready-to-use RGB in hardware. With raw `SharedTextureMemory` you would sample the luma/chroma planes and do that conversion by hand in WGSL. This is the main reason to prefer it for camera and video frames. +- **Lifecycle.** It owns the `SharedTextureMemory` + `createTexture` + `beginAccess`/`endAccess` sequence internally, so you just import the frame and `destroy()` the result. + +It uses the same `"rnwebgpu/native-texture"` feature. + +> **Android note:** the hardware YUV→RGB conversion is fully automatic on iOS (NV12 `IOSurface`). On Android, camera frames arrive as an _opaque_ YCbCr `AHardwareBuffer`, and Dawn's Vulkan path forces an identity (`RGB_IDENTITY`) sampler conversion, so the external sample comes back as raw `[Y, Cb, Cr]`. You still get the zero-copy import and the rotation/mirror transform, but you need to apply the YUV→RGB matrix yourself in the shader. See the `CAMERA_PRELUDE` in the [VisionCamera example](/apps/example/src/VisionCamera/shaders.ts) for a ready-made BT.709 decode. ```tsx const FEATURE = "rnwebgpu/native-texture" as GPUFeatureName; From 2eeea5856dc3047acccfb81dea9618bd42fb8400 Mon Sep 17 00:00:00 2001 From: William Candillon Date: Tue, 2 Jun 2026 06:18:01 +0200 Subject: [PATCH 44/46] :wrench: --- packages/webgpu/cpp/rnwgpu/api/VideoFrame.h | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/webgpu/cpp/rnwgpu/api/VideoFrame.h b/packages/webgpu/cpp/rnwgpu/api/VideoFrame.h index f001285e0..9446e83ae 100644 --- a/packages/webgpu/cpp/rnwgpu/api/VideoFrame.h +++ b/packages/webgpu/cpp/rnwgpu/api/VideoFrame.h @@ -20,7 +20,14 @@ namespace jsi = facebook::jsi; // object stays alive until the JS object is GC'd (or release() is called). class VideoFrame : public NativeObject { public: - static constexpr const char *CLASS_NAME = "VideoFrame"; + // JS-facing brand. installConstructor() exposes this as globalThis[CLASS_NAME] + // and keys the NativeObjectRegistry / worklet serializer off it, so it must + // NOT be "VideoFrame": that would install our constructor over the WebCodecs + // `VideoFrame` global and shadow it at runtime. "NativeVideoFrame" matches the + // public TypeScript type (see src/types.ts) and keeps the two namespaces + // distinct. The C++ class stays `VideoFrame` for brevity; only the brand is + // namespaced. + static constexpr const char *CLASS_NAME = "NativeVideoFrame"; explicit VideoFrame(VideoFrameHandle handle) : NativeObject(CLASS_NAME), _handle(std::move(handle)) {} From 942b4d51dbc3aef714d21ed32698cfda6485843b Mon Sep 17 00:00:00 2001 From: William Candillon Date: Tue, 2 Jun 2026 07:22:30 +0200 Subject: [PATCH 45/46] =?UTF-8?q?chore(=F0=9F=A7=B5):=20worklet-reachable?= =?UTF-8?q?=20native-buffer=20factory=20via=20RNWebGPU=20capture=20(#376)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/example/src/VisionCamera/VisionCamera.tsx | 18 +++++++++++++----- packages/webgpu/cpp/rnwgpu/PlatformContext.h | 8 -------- packages/webgpu/cpp/rnwgpu/RNWebGPUManager.cpp | 5 ----- packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp | 14 -------------- packages/webgpu/cpp/rnwgpu/api/GPUDevice.h | 9 --------- packages/webgpu/src/index.tsx | 4 ---- 6 files changed, 13 insertions(+), 45 deletions(-) diff --git a/apps/example/src/VisionCamera/VisionCamera.tsx b/apps/example/src/VisionCamera/VisionCamera.tsx index 6b5152eb4..c4adcfaa0 100644 --- a/apps/example/src/VisionCamera/VisionCamera.tsx +++ b/apps/example/src/VisionCamera/VisionCamera.tsx @@ -160,6 +160,13 @@ const CameraView = () => { }, []); const device = gpu?.device ?? null; const adapter = gpu?.adapter ?? null; + // Capture the RNWebGPU singleton into a local so the frame-processor worklet + // closes over it. RNWebGPU is a registered, boxable WebGPU NativeObject, so + // the Worklets custom serializer ships it across the worklet boundary the same + // way it does `device` (it is NOT installed as a global on worklet runtimes). + // This is what lets us call the interop factory off RNWebGPU, where the native + // platform context already lives, instead of off `device`. + const rnwgpu = RNWebGPU; const devices = useCameraDevices(); // Pick back camera if available, otherwise front, otherwise anything. The // iOS simulator returns an empty list since there are no cameras, in which @@ -443,11 +450,12 @@ const CameraView = () => { try { let videoFrame; try { - // Call createVideoFrameFromNativeBuffer on the device, not on the - // RNWebGPU global — `device` is already box-able across worklet - // runtimes via the WebGPU custom serializer (proven by the - // Reanimated demo); RNWebGPU is a main-runtime-only global. - videoFrame = device.createVideoFrameFromNativeBuffer( + // Call createVideoFrameFromNativeBuffer on the captured RNWebGPU + // singleton (see `rnwgpu` above). It rides into this worklet via the + // WebGPU custom serializer, same as `device`, so the factory can live + // on RNWebGPU where the native platform context already is — no + // GPUDevice-level workaround and no PlatformContext global singleton. + videoFrame = rnwgpu.createVideoFrameFromNativeBuffer( nativeBuffer.pointer, ); } catch (e) { diff --git a/packages/webgpu/cpp/rnwgpu/PlatformContext.h b/packages/webgpu/cpp/rnwgpu/PlatformContext.h index cdab10da4..fec049256 100644 --- a/packages/webgpu/cpp/rnwgpu/PlatformContext.h +++ b/packages/webgpu/cpp/rnwgpu/PlatformContext.h @@ -70,14 +70,6 @@ class PlatformContext { PlatformContext() = default; virtual ~PlatformContext() = default; - // Singleton-style accessor so leaf classes (e.g. GPUDevice) can reach the - // platform context without threading it through every constructor. Set by - // RNWebGPUManager at startup. - static std::shared_ptr &global() { - static std::shared_ptr instance; - return instance; - } - virtual wgpu::Surface makeSurface(wgpu::Instance instance, void *surface, int width, int height) = 0; virtual ImageData createImageBitmap(std::string blobId, double offset, diff --git a/packages/webgpu/cpp/rnwgpu/RNWebGPUManager.cpp b/packages/webgpu/cpp/rnwgpu/RNWebGPUManager.cpp index 6b8a28cb3..f874c14e6 100644 --- a/packages/webgpu/cpp/rnwgpu/RNWebGPUManager.cpp +++ b/packages/webgpu/cpp/rnwgpu/RNWebGPUManager.cpp @@ -63,11 +63,6 @@ RNWebGPUManager::RNWebGPUManager( // Register main runtime for RuntimeAwareCache BaseRuntimeAwareCache::setMainJsRuntime(_jsRuntime); - // Expose the platform context for leaf classes that need it (e.g. - // GPUDevice::createVideoFrameFromNativeBuffer) without threading it through - // every constructor. - PlatformContext::global() = _platformContext; - auto gpu = std::make_shared(*_jsRuntime); auto rnWebGPU = std::make_shared(gpu, _platformContext, _jsCallInvoker); diff --git a/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp b/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp index 18df1d07b..4d6c92ffa 100644 --- a/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp +++ b/packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp @@ -8,7 +8,6 @@ #include "Convertors.h" #include "JSIConverter.h" -#include "PlatformContext.h" #include "GPUFeatures.h" #include "GPUInternalError.h" @@ -242,19 +241,6 @@ std::shared_ptr GPUDevice::importExternalTexture( return GPUExternalTexture::Create(_instance, std::move(descriptor)); } -std::shared_ptr -GPUDevice::createVideoFrameFromNativeBuffer(uint64_t pointer) { - auto platformContext = PlatformContext::global(); - if (!platformContext) { - throw std::runtime_error( - "GPUDevice::createVideoFrameFromNativeBuffer(): PlatformContext is " - "not initialized"); - } - auto handle = - platformContext->wrapNativeBuffer(reinterpret_cast(pointer)); - return std::make_shared(std::move(handle)); -} - std::shared_ptr GPUDevice::importSharedTextureMemory( std::shared_ptr descriptor) { if (!descriptor || descriptor->handle == nullptr) { diff --git a/packages/webgpu/cpp/rnwgpu/api/GPUDevice.h b/packages/webgpu/cpp/rnwgpu/api/GPUDevice.h index 28add6e0c..2ab1ddd14 100644 --- a/packages/webgpu/cpp/rnwgpu/api/GPUDevice.h +++ b/packages/webgpu/cpp/rnwgpu/api/GPUDevice.h @@ -53,7 +53,6 @@ #include "GPUTexture.h" #include "GPUTextureDescriptor.h" #include "GPUUncapturedErrorEvent.h" -#include "VideoFrame.h" namespace rnwgpu { @@ -121,12 +120,6 @@ class GPUDevice : public NativeObject { std::shared_ptr descriptor); std::shared_ptr importSharedTextureMemory( std::shared_ptr descriptor); - // Wrap a CVPixelBufferRef / AHardwareBuffer* pointer (as a BigInt) into a - // VideoFrame. Mirrors RNWebGPU.createVideoFrameFromNativeBuffer but is - // reachable from worklet runtimes since GPUDevice is already serialized - // across the worklet boundary via the WebGPU custom serializer. - std::shared_ptr - createVideoFrameFromNativeBuffer(uint64_t pointer); std::shared_ptr createBindGroupLayout( std::shared_ptr descriptor); std::shared_ptr @@ -182,8 +175,6 @@ class GPUDevice : public NativeObject { &GPUDevice::importExternalTexture); installMethod(runtime, prototype, "importSharedTextureMemory", &GPUDevice::importSharedTextureMemory); - installMethod(runtime, prototype, "createVideoFrameFromNativeBuffer", - &GPUDevice::createVideoFrameFromNativeBuffer); installMethod(runtime, prototype, "createBindGroupLayout", &GPUDevice::createBindGroupLayout); installMethod(runtime, prototype, "createPipelineLayout", diff --git a/packages/webgpu/src/index.tsx b/packages/webgpu/src/index.tsx index 3e5e819ac..6ac631a48 100644 --- a/packages/webgpu/src/index.tsx +++ b/packages/webgpu/src/index.tsx @@ -54,10 +54,6 @@ declare global { importSharedTextureMemory( descriptor: GPUSharedTextureMemoryDescriptor, ): GPUSharedTextureMemory; - // Wrap a NativeBuffer.pointer into a NativeVideoFrame. Reachable from - // worklet runtimes (e.g. Vision Camera frame processors) because GPUDevice - // is serialized across worklet boundaries via the WebGPU custom serializer. - createVideoFrameFromNativeBuffer(pointer: bigint): NativeVideoFrame; } // Non-spec extension: camera frames arrive in the sensor's native From 1692605ee320d4a15b569230889f3e882eaff5ef Mon Sep 17 00:00:00 2001 From: William Candillon Date: Tue, 2 Jun 2026 08:11:48 +0200 Subject: [PATCH 46/46] :wrench: --- README.md | 19 +++++---- .../ImportExternalTexture.tsx | 25 +++++------ .../SharedTextureMemory.tsx | 5 +++ apps/example/src/Tests.tsx | 15 +++---- packages/webgpu/README.md | 19 +++++---- packages/webgpu/cpp/rnwgpu/api/GPUAdapter.cpp | 42 +++++++++++++++++++ 6 files changed, 85 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 581a9e1d0..8eeb1cba1 100644 --- a/README.md +++ b/README.md @@ -226,16 +226,18 @@ device.queue.copyExternalImageToTexture( React Native WebGPU exposes Dawn's `SharedTextureMemory` so you can import a native pixel surface (an `IOSurface`-backed `CVPixelBuffer` on iOS, an `AHardwareBuffer` on Android) as a sampleable `GPUTexture` without copying pixels through the CPU. This is the path you want for camera frames, video frames, or anything coming out of a hardware producer. -We expose a single umbrella feature name, `"rnwebgpu/native-texture"`. Request it at device creation. +Like `importExternalTexture` on the web, this is **enabled by default**, there is nothing to request at device creation. The only thing to check is that the device supports it before importing. It always does on iOS/macOS; it can be missing on some Android drivers and emulators. ```tsx import type { NativeVideoFrame } from "react-native-wgpu"; -const FEATURE = "rnwebgpu/native-texture" as GPUFeatureName; - const adapter = await navigator.gpu.requestAdapter(); -const requiredFeatures = adapter!.features.has(FEATURE) ? [FEATURE] : []; -const device = await adapter!.requestDevice({ requiredFeatures }); +const device = await adapter!.requestDevice(); + +// On by default when supported; this is the only check you need. +if (!device.features.has("rnwebgpu/native-texture" as GPUFeatureName)) { + return; // rare: some Android drivers/emulators can't import native surfaces +} // `frame` here is a NativeVideoFrame whose .handle is the native surface // (IOSurfaceRef / AHardwareBuffer*). NativeVideoFrames are produced by helpers @@ -264,15 +266,14 @@ frame.release(); - **Color conversion.** Camera and video surfaces are usually biplanar YUV (NV12), not RGB. An external texture carries the YUV→RGB matrix and the source/destination color-space transfer functions, so on the supported paths the sampler returns ready-to-use RGB in hardware. With raw `SharedTextureMemory` you would sample the luma/chroma planes and do that conversion by hand in WGSL. This is the main reason to prefer it for camera and video frames. - **Lifecycle.** It owns the `SharedTextureMemory` + `createTexture` + `beginAccess`/`endAccess` sequence internally, so you just import the frame and `destroy()` the result. -It uses the same `"rnwebgpu/native-texture"` feature. +It builds on the same default-on capability as Shared Texture Memory above, so feature-detect the device the same way before importing. > **Android note:** the hardware YUV→RGB conversion is fully automatic on iOS (NV12 `IOSurface`). On Android, camera frames arrive as an _opaque_ YCbCr `AHardwareBuffer`, and Dawn's Vulkan path forces an identity (`RGB_IDENTITY`) sampler conversion, so the external sample comes back as raw `[Y, Cb, Cr]`. You still get the zero-copy import and the rotation/mirror transform, but you need to apply the YUV→RGB matrix yourself in the shader. See the `CAMERA_PRELUDE` in the [VisionCamera example](/apps/example/src/VisionCamera/shaders.ts) for a ready-made BT.709 decode. ```tsx -const FEATURE = "rnwebgpu/native-texture" as GPUFeatureName; const adapter = await navigator.gpu.requestAdapter(); -const requiredFeatures = adapter!.features.has(FEATURE) ? [FEATURE] : []; -const device = await adapter!.requestDevice({ requiredFeatures }); +const device = await adapter!.requestDevice(); +// Feature-detect as shown above before importing on unsupported hardware. const render = () => { // A GPUExternalTexture expires once the queue work that used it is submitted, diff --git a/apps/example/src/ImportExternalTexture/ImportExternalTexture.tsx b/apps/example/src/ImportExternalTexture/ImportExternalTexture.tsx index b8cb82d8c..f8399ee8a 100644 --- a/apps/example/src/ImportExternalTexture/ImportExternalTexture.tsx +++ b/apps/example/src/ImportExternalTexture/ImportExternalTexture.tsx @@ -63,30 +63,31 @@ fn fs_main(in: VsOut) -> @location(0) vec4f { } `; -// We never call importSharedTextureMemory directly here, but -// importExternalTexture is implemented on top of it natively (it imports the -// frame's IOSurface / AHardwareBuffer as shared texture memory, then wraps that -// as an external texture). So the device must enable this umbrella feature, or -// the import throws "ImportSharedTextureMemory returned null". -const REQUIRED_FEATURES = ["rnwebgpu/native-texture" as GPUFeatureName]; +// importExternalTexture is backed by the "rnwebgpu/native-texture" umbrella +// feature (it imports the frame's IOSurface / AHardwareBuffer as shared texture +// memory, then wraps that as an external texture). Like importExternalTexture on +// the web, that capability is now enabled by default: requestDevice / useDevice +// turns it on automatically whenever the adapter supports it, so we don't pass +// anything in requiredFeatures. We keep the name only to feature-detect and +// degrade gracefully on hardware that doesn't support it. +const FEATURE = "rnwebgpu/native-texture" as GPUFeatureName; export const ImportExternalTexture = () => { const ref = useCanvasRef(); const [error, setError] = useState(null); const rafRef = useRef(null); - const { device, adapter } = useDevice(undefined, { - requiredFeatures: REQUIRED_FEATURES, - }); + const { device, adapter } = useDevice(); useEffect(() => { if (!device) { return; } - const missing = REQUIRED_FEATURES.filter((f) => !device.features.has(f)); - if (missing.length > 0) { + // native-texture is auto-enabled when supported; if it is still missing, + // this hardware/driver can't back importExternalTexture. + if (!device.features.has(FEATURE)) { setError( - `Device is missing required features [${missing.join(", ")}]. Adapter supports: ${ + `This device doesn't support ${FEATURE} (importExternalTexture). Adapter supports: ${ adapter ? [...adapter.features] .filter((f) => f.toString().startsWith("shared-")) diff --git a/apps/example/src/SharedTextureMemory/SharedTextureMemory.tsx b/apps/example/src/SharedTextureMemory/SharedTextureMemory.tsx index 4ce1f315c..b5627cc43 100644 --- a/apps/example/src/SharedTextureMemory/SharedTextureMemory.tsx +++ b/apps/example/src/SharedTextureMemory/SharedTextureMemory.tsx @@ -53,6 +53,11 @@ fn fs_main(in: VsOut) -> @location(0) vec4f { } `; +// This screen keeps the explicit feature request for documentation. As of the +// default-on change, "rnwebgpu/native-texture" is enabled automatically by +// requestDevice / useDevice whenever the adapter supports it (like +// importExternalTexture on the web), so passing it in requiredFeatures is +// optional. We still list it here to show how to gate on the capability. const REQUIRED_FEATURES = ["rnwebgpu/native-texture" as GPUFeatureName]; export const SharedTextureMemory = () => { diff --git a/apps/example/src/Tests.tsx b/apps/example/src/Tests.tsx index 63372ff3e..fc9ebf635 100644 --- a/apps/example/src/Tests.tsx +++ b/apps/example/src/Tests.tsx @@ -14,12 +14,6 @@ export const CI = process.env.CI === "true"; const { width } = Dimensions.get("window"); const presentationFormat = navigator.gpu.getPreferredCanvasFormat(); -// Umbrella feature owned by react-native-wgpu: expands to the platform's -// Dawn feature pair natively and appears on adapter.features when supported. -const OPTIONAL_SHARED_TEXTURE_MEMORY_FEATURES = [ - "rnwebgpu/native-texture" as GPUFeatureName, -]; - export const Tests = ({ assets: { di3D, saturn, moon } }: AssetProps) => { const [texture, setTexture] = useState(null); const [adapter, setAdapter] = useState(null); @@ -33,10 +27,11 @@ export const Tests = ({ assets: { di3D, saturn, moon } }: AssetProps) => { if (!a) { throw new Error("No appropriate GPUAdapter found."); } - const requiredFeatures = OPTIONAL_SHARED_TEXTURE_MEMORY_FEATURES.filter( - (f) => a.features.has(f), - ); - const d = await a.requestDevice({ requiredFeatures }); + // "rnwebgpu/native-texture" is enabled by default whenever the adapter + // supports it, so the shared-texture / importExternalTexture specs get + // the capability without requesting it here. Specs that need it still + // gate on device.features.has(...) and skip where it is unavailable. + const d = await a.requestDevice(); if (!d) { throw new Error("No appropriate GPUDevice found."); } diff --git a/packages/webgpu/README.md b/packages/webgpu/README.md index 581a9e1d0..8eeb1cba1 100644 --- a/packages/webgpu/README.md +++ b/packages/webgpu/README.md @@ -226,16 +226,18 @@ device.queue.copyExternalImageToTexture( React Native WebGPU exposes Dawn's `SharedTextureMemory` so you can import a native pixel surface (an `IOSurface`-backed `CVPixelBuffer` on iOS, an `AHardwareBuffer` on Android) as a sampleable `GPUTexture` without copying pixels through the CPU. This is the path you want for camera frames, video frames, or anything coming out of a hardware producer. -We expose a single umbrella feature name, `"rnwebgpu/native-texture"`. Request it at device creation. +Like `importExternalTexture` on the web, this is **enabled by default**, there is nothing to request at device creation. The only thing to check is that the device supports it before importing. It always does on iOS/macOS; it can be missing on some Android drivers and emulators. ```tsx import type { NativeVideoFrame } from "react-native-wgpu"; -const FEATURE = "rnwebgpu/native-texture" as GPUFeatureName; - const adapter = await navigator.gpu.requestAdapter(); -const requiredFeatures = adapter!.features.has(FEATURE) ? [FEATURE] : []; -const device = await adapter!.requestDevice({ requiredFeatures }); +const device = await adapter!.requestDevice(); + +// On by default when supported; this is the only check you need. +if (!device.features.has("rnwebgpu/native-texture" as GPUFeatureName)) { + return; // rare: some Android drivers/emulators can't import native surfaces +} // `frame` here is a NativeVideoFrame whose .handle is the native surface // (IOSurfaceRef / AHardwareBuffer*). NativeVideoFrames are produced by helpers @@ -264,15 +266,14 @@ frame.release(); - **Color conversion.** Camera and video surfaces are usually biplanar YUV (NV12), not RGB. An external texture carries the YUV→RGB matrix and the source/destination color-space transfer functions, so on the supported paths the sampler returns ready-to-use RGB in hardware. With raw `SharedTextureMemory` you would sample the luma/chroma planes and do that conversion by hand in WGSL. This is the main reason to prefer it for camera and video frames. - **Lifecycle.** It owns the `SharedTextureMemory` + `createTexture` + `beginAccess`/`endAccess` sequence internally, so you just import the frame and `destroy()` the result. -It uses the same `"rnwebgpu/native-texture"` feature. +It builds on the same default-on capability as Shared Texture Memory above, so feature-detect the device the same way before importing. > **Android note:** the hardware YUV→RGB conversion is fully automatic on iOS (NV12 `IOSurface`). On Android, camera frames arrive as an _opaque_ YCbCr `AHardwareBuffer`, and Dawn's Vulkan path forces an identity (`RGB_IDENTITY`) sampler conversion, so the external sample comes back as raw `[Y, Cb, Cr]`. You still get the zero-copy import and the rotation/mirror transform, but you need to apply the YUV→RGB matrix yourself in the shader. See the `CAMERA_PRELUDE` in the [VisionCamera example](/apps/example/src/VisionCamera/shaders.ts) for a ready-made BT.709 decode. ```tsx -const FEATURE = "rnwebgpu/native-texture" as GPUFeatureName; const adapter = await navigator.gpu.requestAdapter(); -const requiredFeatures = adapter!.features.has(FEATURE) ? [FEATURE] : []; -const device = await adapter!.requestDevice({ requiredFeatures }); +const device = await adapter!.requestDevice(); +// Feature-detect as shown above before importing on unsupported hardware. const render = () => { // A GPUExternalTexture expires once the queue work that used it is submitted, diff --git a/packages/webgpu/cpp/rnwgpu/api/GPUAdapter.cpp b/packages/webgpu/cpp/rnwgpu/api/GPUAdapter.cpp index c816f6328..7bdb21840 100644 --- a/packages/webgpu/cpp/rnwgpu/api/GPUAdapter.cpp +++ b/packages/webgpu/cpp/rnwgpu/api/GPUAdapter.cpp @@ -1,5 +1,6 @@ #include "GPUAdapter.h" +#include #include #include #include @@ -18,6 +19,47 @@ namespace rnwgpu { async::AsyncTaskHandle GPUAdapter::requestDevice( std::optional> descriptor) { + // Enable the react-native-wgpu "native-texture" umbrella by default, mirroring + // the web where importExternalTexture is core and needs no feature request. + // We append the umbrella's backing Dawn features to requiredFeatures so the + // capability is on without the caller listing it. Two rules keep this safe: + // - All-or-nothing: only inject when the adapter supports *every* backing + // feature (same semantics as maybeSynthesizeRnNativeTextureFeature). On a + // web/fallback adapter the backing set is empty or unsupported, so this is + // a no-op and device creation is unaffected. + // - Requesting a feature the adapter doesn't support makes RequestDevice + // fail, hence the support check below. + // Callers can still pass "rnwebgpu/native-texture" explicitly; the dedupe + // keeps that idempotent. + { + auto backing = rnNativeTextureBackingFeatures(); + if (!backing.empty()) { + wgpu::SupportedFeatures supported; + _instance.GetFeatures(&supported); + std::unordered_set supportedSet( + supported.features, supported.features + supported.featureCount); + bool allSupported = std::all_of( + backing.begin(), backing.end(), + [&](wgpu::FeatureName f) { return supportedSet.count(f) > 0; }); + if (allSupported) { + if (!descriptor.has_value()) { + descriptor = std::make_shared(); + } + auto &desc = descriptor.value(); + if (!desc->requiredFeatures.has_value()) { + desc->requiredFeatures = std::vector{}; + } + auto &features = desc->requiredFeatures.value(); + for (auto f : backing) { + if (std::find(features.begin(), features.end(), f) == + features.end()) { + features.push_back(f); + } + } + } + } + } + wgpu::DeviceDescriptor aDescriptor; Convertor conv; if (!conv(aDescriptor, descriptor)) {