diff --git a/.github/workflows/ci-build-test.yml b/.github/workflows/ci-build-test.yml index f49426673..be60b22f3 100644 --- a/.github/workflows/ci-build-test.yml +++ b/.github/workflows/ci-build-test.yml @@ -91,7 +91,7 @@ jobs: - name: Run renderer tests (non-async suites) # The sweep takes 26-30 minutes on the hosted macOS runners and grows with # every render suite; leave headroom so a slow runner is not a red build. - timeout-minutes: 45 + timeout-minutes: 60 env: CI: true UNTOLD_PSNR_THRESHOLD: "33.5" # tweak per suite/scene diff --git a/BuildChecks/MetalKernelsStaleCheck/checkStaleKernels.swift b/BuildChecks/MetalKernelsStaleCheck/checkStaleKernels.swift new file mode 100644 index 000000000..684f11b2b --- /dev/null +++ b/BuildChecks/MetalKernelsStaleCheck/checkStaleKernels.swift @@ -0,0 +1,72 @@ +// +// checkStaleKernels.swift +// MetalKernelsStaleCheck +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +// SwiftPM does not compile UntoldEngineKernels.metal; the engine target ships prebuilt +// .metallib resources instead (see buildkernels.sh and Package.swift), so an edit to the +// .metal source has no effect on `swift build` and produces no warning on its own. +// UntoldEngineKernels.metal #includes every file in ../Shaders, so the compiled kernel +// actually depends on all of them too โ€” the hash below covers the kernels file and every +// Shaders/*.metal file for that reason, not just the file SwiftPM excludes by name. This +// plugin runs on every build plan and compares that hash against the one buildkernels.sh +// stamped the last time it rebuilt the metallibs, so a stale shader binary is surfaced +// instead of running silently. + +import CryptoKit +import Foundation +import PackagePlugin + +@main +struct MetalKernelsStaleCheck: BuildToolPlugin { + /// `Target.directoryURL` needs PackageDescription 6.1; this package is on 6.0, so + /// `.directory` (deprecated `Path`) is the only way to locate the target on this + /// tools version. SwiftPM invokes this method through the `BuildToolPlugin` + /// existential at runtime rather than our code calling it directly, so marking it + /// deprecated silences the unavoidable warning here without anyone needing to call + /// a deprecated entry point themselves. + @available(*, deprecated) + func createBuildCommands(context _: PluginContext, target: Target) async throws -> [Command] { + let engineDir = URL(fileURLWithPath: target.directory.string) + let kernelsDir = engineDir.appending(path: "UntoldEngineKernels") + let shadersDir = engineDir.appending(path: "Shaders") + let metalSource = kernelsDir.appending(path: "UntoldEngineKernels.metal") + let hashFile = kernelsDir.appending(path: ".metalsource.sha256") + + guard let kernelsData = try? Data(contentsOf: metalSource) else { + return [] + } + + let shaderFileNames = (try? FileManager.default.contentsOfDirectory(atPath: shadersDir.path))? + .filter { $0.hasSuffix(".metal") } + .sorted() ?? [] + + var combined = kernelsData + for name in shaderFileNames { + guard let data = try? Data(contentsOf: shadersDir.appending(path: name)) else { return [] } + combined.append(data) + } + + let currentHash = SHA256.hash(data: combined).map { String(format: "%02x", $0) }.joined() + let storedHash = (try? String(contentsOf: hashFile, encoding: .utf8))? + .trimmingCharacters(in: .whitespacesAndNewlines) + + if storedHash != currentHash { + Diagnostics.warning(""" + UntoldEngineKernels.metal or one of the Shaders/*.metal files it includes has \ + changed since the shipped .metallib resources were built (or \ + \(hashFile.lastPathComponent) is missing). `swift build` does not recompile \ + Metal shaders, so this build is running against stale shader binaries. Run \ + ./buildkernels.sh from the package root, then commit the updated .metallib \ + files and \(hashFile.lastPathComponent). + """) + } + + return [] + } +} diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dbf6e5fe..30daec4b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,29 @@ # Changelog +## v0.23.0 - 2026-10-12 +### ๐Ÿž Fixes +- [Patch] Start a slot of objects as no reference (#1313) (6456434โ€ฆ) +- [Bugfix] Export the material a slot linked to the object shows, not the mesh's default (#1320) (6b7b664โ€ฆ) +- [Bugfix] Test occlusion from the camera the depth pyramid was built with (#1323) (45823f6โ€ฆ) +- [Bugfix] Keep a model's hidden parts hidden at its coarse LOD levels (#1324) (c33fca8โ€ฆ) +- [Bugfix] Detect pip --break-system-packages support in make testrenderer (#1328) (23d66a7โ€ฆ) +- [Bugfix] Use single-bounce reflectance for IBL diffuse energy conservation (#1329) (a2b154dโ€ฆ) +- [Bugfix] Report animation clips that match zero joints on the bound skeleton (#1330) (1085580โ€ฆ) +- [Bugfix] Shade each XR eye from its own position (#1285) (0b2863dโ€ฆ) +- [Bugfix] Glass keeps its place against its frame and in a headset (#1331) (144863aโ€ฆ) +### ๐Ÿ“š Docs +- [Docs] Add a getting-started tutorial for Jolt Physics in XR (#1327) (e544326โ€ฆ) +### ๐Ÿš€ Features +- [Feature] Bake procedural Blender materials into repeating textures (#1307) (e8ed7adโ€ฆ) +- [Feature] Glass: a material's transmission shows what is behind it and keeps its reflections (#1308) (da580eeโ€ฆ) +- [Feature] Select LOD levels by the size of the entity on screen (#1322) (6446393โ€ฆ) +- [Feature] Specular anti-aliasing in the model pass (#1332) (99de731โ€ฆ) +### โšก Performance +- [Performance] The render passes read the scene through a snapshot (#1321) (8b695d6โ€ฆ) +### โœจ Enhancement +- [Enhancement] Add animation playback helpers for synchronized gameplay (#1316) (ec141a0โ€ฆ) +### ๐Ÿงน Chores / Tests +- Remove hardcoded AWS/CloudFront streaming URLs from demos and tests (#1317) (4ae99caโ€ฆ) +- [Test] Keep the reader of the component release test reading for the whole run (#1312) (9f3c637โ€ฆ) ## v0.22.0 - 2026-10-04 ### ๐Ÿž Fixes - [Patch] Remember failed texture writes per export, not per Blender session (#1263) (fa706bfโ€ฆ) diff --git a/Package.swift b/Package.swift index f7a5f034e..a8d0e7be1 100644 --- a/Package.swift +++ b/Package.swift @@ -100,6 +100,11 @@ let package = Package( exclude: ["LICENSE.md", "README.md"], publicHeadersPath: "include" ), + .plugin( + name: "MetalKernelsStaleCheck", + capability: .buildTool(), + path: "BuildChecks/MetalKernelsStaleCheck" + ), .target( name: "UntoldEngine", dependencies: ["CShaderTypes"], @@ -130,7 +135,8 @@ let package = Package( // iOS UI stack (only if some targets import UIKit) .linkedFramework("UIKit", .when(platforms: [.iOS])), - ] + ], + plugins: ["MetalKernelsStaleCheck"] ), // The cook's mesh work. A module of its own so that UntoldEngine does not import diff --git a/README.md b/README.md index 1913a639c..a978b8588 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@

- Logo + Untold Engine Logo

@@ -39,7 +39,7 @@ and input, so you can build your own interactive spatial and XR worlds. Creator & Lead Developer: https://www.haroldserrano.com -[![CoolSaber demo](docs/images/coolsaber-thumbnail.png)](https://youtu.be/4JWZf_NFddY) +[![State of the Engine 2026](docs/images/state-of-the-engine-2026-thumbnail.png)](https://youtu.be/AnN2NBRKl8E) --- @@ -64,6 +64,21 @@ workflow, product experience, and customer-specific behavior. --- +## Try the Untold Editor + +The Untold Editor is a companion app for building scenes visually โ€” placing +entities, setting up lighting and cameras, and authoring `.untoldscene` files +that load straight into your app with `loadUntoldScene(named:)`. It's +optional โ€” nothing in Untold Engine requires it โ€” but it's the fastest way to +get a scene on screen without hand-writing setup code. + +The Editor is now stable enough for regular use. Grab the latest build from +the [Releases page](https://github.com/untoldengine/UntoldEditor/releases). + +![untold-editor](docs/images/editor-main-highlight.png) + +--- + ## Watch It in Action โ€” Apple Vision Pro Demos | Demo | Description | @@ -100,7 +115,7 @@ Clone the repository and launch the Starter Demo: ```bash git clone https://github.com/untoldengine/UntoldEngine.git cd UntoldEngine -git checkout v0.22.0 +git checkout v0.23.0 swift run StarterDemo ``` @@ -122,21 +137,6 @@ Engine as a package dependency. --- -## Try the Untold Editor - -The Untold Editor is a companion app for building scenes visually โ€” placing -entities, setting up lighting and cameras, and authoring `.untoldscene` files -that load straight into your app with `loadUntoldScene(named:)`. It's -optional โ€” nothing in Untold Engine requires it โ€” but it's the fastest way to -get a scene on screen without hand-writing setup code. - -The Editor is now stable enough for regular use. Grab the latest build from -the [Releases page](https://github.com/untoldengine/UntoldEditor/releases). - -![untold-editor](docs/images/editor-main-highlight.png) - ---- - ## Getting Started To create your own XR, 3D, or spatial visualization app using Untold Engine, @@ -159,20 +159,23 @@ start with the documentation path that matches where you are: If your team is evaluating Untold Engine and needs an engine capability that is not currently available, see [Commercial Use, Sponsored Features, and Support](COMMERCIAL.md). -## Core Direction +## Our Direction + +### Vision โ€” Why We Exist + +> An engine that empowers creativity and never stands in its way. -**Vision:** An engine that empowers creativity and never stands in its way. +### Mission โ€” What We Commit To -**Mission:** Build a reliable engine that lets developers focus on creating, -not troubleshooting the tools they depend on. +> Build a reliable engine that lets developers focus on creating, not troubleshooting the tools they depend on. -Untold Engine is built around three focused goals: +### Engineering Principles โ€” How We Operate -- **Spatial Engine First** โ€” Designed for spatial computing applications. LOD, geometry streaming, and static batching exist to support large, real-world-scale environments where presence and performance both matter. +**Reliability ยท Testing ยท Simplicity ยท Compatibility ยท Developer Experience ยท Performance** -- **XR / visionOS Support** โ€” Spatial input, AR workflows, and Vision Pro support are functional today and expanding with each release. +These principles guide how we design, build, test, and maintain Untold Engine. -- **Metal-First Architecture** โ€” The rendering layer stays close to Metal to maintain performance, shader control, and pipeline flexibility without abstraction layers getting in the way. +Read our [Engineering Principles](docs/Contributor/EngineeringPrinciples.md) for the standards that guide our development decisions. --- @@ -219,6 +222,7 @@ Untold Engine is built around three focused goals: - [Async Loading](docs/API/UsingAsyncLoading.md) - [Spatial Input](docs/API/UsingSpatialInput.md) - [XR Immersion Modes](docs/API/UsingXRImmersionMode.md) +- [The Vision Pro Rendering Profile, As Measured](docs/API/VisionProRenderingProfile.md) - [Light Portals](docs/API/UsingLightPortals.md) - [Rendering Extensions](docs/API/UsingRenderingExtensions.md) - [Rendering Extension Examples](Examples/RenderingExtensions/README.md) diff --git a/Sources/CShaderTypes/ShaderTypes.h b/Sources/CShaderTypes/ShaderTypes.h index 599d73072..be6842331 100644 --- a/Sources/CShaderTypes/ShaderTypes.h +++ b/Sources/CShaderTypes/ShaderTypes.h @@ -42,6 +42,12 @@ typedef struct matrix_float3x3 normalMatrix; matrix_float4x4 modelMatrix; simd_float3 cameraPosition; + // For the velocity the opaque pass writes per pixel: this frame's view-projection without + // its jitter, the previous frame's (of the same eye), and the model matrix the entity was + // drawn with the previous frame. + matrix_float4x4 unjitteredViewProjectionMatrix; + matrix_float4x4 prevViewProjectionMatrix; + matrix_float4x4 prevModelMatrix; } Uniforms; @@ -140,6 +146,17 @@ typedef enum{ modelPassFragmentSTScaleIndex, modelPassFragmentPOMQualityIndex, modelPassFragmentNormalIsPackedXYIndex, + // Per-fragment indirect lighting: the light pass's inputs the geometry shares. + modelPassFragmentCSMUniformsIndex, + modelPassFragmentLightParamsIndex, + modelPassFragmentIBLParamIndex, + modelPassFragmentIBLRotationAngleIndex, + // Per-fragment lighting: the light quad's other inputs, when the geometry lights itself whole. + modelPassFragmentPointLightsIndex, + modelPassFragmentSpotLightsIndex, + modelPassFragmentAreaLightsIndex, + modelPassFragmentPointShadowUniformIndex, + modelPassFragmentSpotShadowUniformIndex, }ModelPassFragmentBufferIndices; typedef enum{ @@ -361,10 +378,38 @@ typedef struct{ }MLDecodeParams; +/// Function constants of the model fragment shader (`fragmentModelShader`). +typedef enum{ + /// bool: whether a draw through the pipeline may discard fragments (an alpha-masked or + /// blended material, a dither fade). False compiles the discards out, which lets the + /// GPU resolve visibility before it shades: hidden fragments are never shaded. + modelFunctionConstantMayDiscard = 0, + /// True evaluates the image-based light and the sun's cascade shadow once per fragment + /// and writes them into the G-buffer (the emissive attachment: indirect light added to + /// the emissive, the shadow factor in alpha); the light pass then adds them from there + /// instead of computing them once per sample. `setRendering(.perFragmentIndirectLighting)`. + modelFunctionConstantIndirectInGBuffer = 3, + /// True lights the surface whole once per fragment โ€” the direct light of the sun, the + /// point, spot and area lights with their shadows, over the indirect light of + /// `modelFunctionConstantIndirectInGBuffer`, which it needs โ€” and writes the lit colour to + /// the attachment the light pass would write, so that pass stands down. + /// `setRendering(.perFragmentLighting)`. + modelFunctionConstantLitInGBuffer = 5 +}ModelFunctionConstants; + +/// Function constants of `fragmentLightShaderTBDR`. The indices share one space with the +/// model shader's across the library, so they continue its numbering. +typedef enum{ + /// True takes the indirect light and the sun's shadow from the G-buffer, where the + /// geometry wrote them once per fragment (`modelFunctionConstantIndirectInGBuffer`). + lightFunctionConstantIndirectFromGBuffer = 4 +}LightFunctionConstants; + typedef enum{ prePassGizmoBufferIndex, prePassPassthroughBufferIndex, - prePassSSAOEnabledIndex + prePassSSAOEnabledIndex, + prePassGaussianLayerIndex // bool: the Gaussian pass drew its layer this frame }PrePassBufferIndices; typedef enum{ @@ -407,6 +452,16 @@ typedef enum{ modelPassNormalTextureIndex, modelPassHeightTextureIndex, modelPassEmissiveTextureIndex, + // Per-fragment indirect lighting: the light pass's textures the geometry shares. + modelPassIBLIrradianceTextureIndex, + modelPassIBLSpecularTextureIndex, + modelPassIBLBRDFTextureIndex, + modelPassCSMShadowTextureIndex, + // Per-fragment lighting: the light quad's other textures, when the geometry lights itself whole. + modelPassAreaLTCMagTextureIndex, + modelPassAreaLTCMatTextureIndex, + modelPassPointShadowTextureIndex, + modelPassSpotShadowTextureIndex, }ModelPassTextureIndices; typedef enum{ @@ -492,6 +547,9 @@ typedef enum { typedef enum { tonemapOperatorACES = 0, tonemapOperatorAgX = 1, + /// No tone map: the scene's linear colour as it is, for a mixed-immersion space where + /// the passthrough sets the display's response and the compositor applies it. + tonemapOperatorNone = 2, } TonemapOperatorID; // The Gaussian pass's layer on the look pass: a splat pixel is display-referred already @@ -502,13 +560,25 @@ typedef enum { typedef enum { lookPassSplatCoverageTextureIndex = 3, // texture(0..2) above: the splat layer, premultiplied lookPassGizmoTextureIndex = 4, + lookPassDepthTextureIndex = 5, // the opaque depth, when the look pass presents the frame itself } LookPassSplatTextureIndices; typedef enum { lookPassSplatMaskIndex = 15, // starts after TonemapSelectBufferIndices (14) lookPassGizmoOverrideIndex = 16, // bool: the gizmo layer overrides pixels (the editor, not game mode) + lookPassEncodingModeIndex = 17, // int: the display's encoding, when the look pass presents the frame itself + lookPassColorPrimariesIndex = 18, // int: ColorPrimariesID, the primaries the scene is shown with } LookPassSplatBufferIndices; +/// The primaries the look pass shows the scene with: its sRGB colour as it is, or the colour +/// taken as Display P3 coordinates and shown with the panel's wider primaries, as RealityKit +/// shows a mixed-immersion scene (its colours measure as the P3โ†’sRGB matrix of their sRGB +/// values, to a count). +typedef enum { + colorPrimariesSRGB = 0, + colorPrimariesDisplayP3 = 1, +} ColorPrimariesID; + typedef enum{ colorCorrectionPassColorTextureIndex, colorCorrectionPassTemperatureIndex, @@ -559,6 +629,8 @@ typedef enum{ ssaoPassFrustumIndex, ssaoPassReverseZIndex, ssaoPassProjScaleIndex, + ssaoPassRateMapDataIndex, // rasterization_rate_map_data of the eye's rate map (XR foveation; a never-read placeholder otherwise) + ssaoPassRateMapSizesIndex, // float4: the map's screen size in xy, the depth texture's size in zw; x <= 0 when there is no map }SSAOBufferIndices; typedef enum{ @@ -767,7 +839,9 @@ typedef enum{ hzbCullPassViewportIndex, hzbCullPassMipCountIndex, hzbCullPassReverseZIndex, - hzbCullPassOcclusionBiasIndex + hzbCullPassOcclusionBiasIndex, + hzbCullPassRateMapDataIndex, // rasterization_rate_map_data the pyramid was drawn through (XR foveation; a never-read placeholder otherwise) + hzbCullPassRateMapSizesIndex // float4: the map's screen size in xy, the pyramid's base size in zw; x <= 0 when there is no map }HZBOcclusionCullingBufferIndices; typedef enum{ @@ -800,6 +874,8 @@ typedef enum{ gaussianCullHZBReverseZIndex, gaussianCullHZBOcclusionBiasIndex, gaussianCullHZBValidIndex, + gaussianCullHZBRateMapDataIndex, // rasterization_rate_map_data the pyramid was drawn through (XR foveation; a never-read placeholder otherwise) + gaussianCullHZBRateMapSizesIndex, // float4: the map's screen size in xy, the pyramid's base size in zw; x <= 0 when there is no map }GaussianDepthBufferIndices; typedef enum{ @@ -1034,7 +1110,8 @@ typedef struct{ uint32_t uniformQuotas; // GaussianDebugOptions.disableScreenWeightedQuotas: screenArea = splatCount, the uniform quota rule uint32_t paged; // 0: every record resident; 1: a paged entity (the cull writes the demand table and lists resident ranks only, the fused pass reads the page pool); 2: demand only (a warming tier: write the demand word and return) uint32_t rangeCount; // entries of ranges[] (gaussianChunkCullRangesIndex, GaussianChunkTreeCull); a thread whose chunkIndex falls in none of them skips the per-chunk test. 0 (GaussianDebugOptions.disableTreeSkip, or a file without a tree) skips the check entirely, as before the tree was wired in -}GaussianChunkCullConstants; // 192 bytes (176 + rangeCount, padded to the struct's 16-byte alignment) + simd_float4 hzbRateMapSizes; // the rasterization rate map the HZB was drawn through (XR foveation): its screen size in xy, the pyramid's base size in zw; x <= 0 when there is none and the rate map buffer is a never-read placeholder +}GaussianChunkCullConstants; // 208 bytes (180 + hzbRateMapSizes at 192, padded to the struct's 16-byte alignment) /// One span of GaussianChunkTreeCull.visibleChunkRanges (Swift): chunk indices [firstChunk, /// firstChunk + chunkCount) the tree walk could not rule out this frame. Every chunk still gets @@ -1062,6 +1139,7 @@ typedef enum{ gaussianChunkCullLevelStateIndex = 11, // GaussianChunkLevelState[chunkCount], persistent, written by gaussianComputeChunkQuotas (a never-read stand-in without coarse levels) gaussianChunkCullLevelConstantsIndex = 12, // GaussianChunkLevelConstants (setBytes); hasCoarse = 0 keeps every path as it was gaussianChunkCullRangesIndex = 13, // GaussianChunkCullRange[rangeCount] (setBytes; a never-read stand-in when rangeCount == 0) + gaussianChunkCullRateMapDataIndex = 14, // rasterization_rate_map_data the HZB was drawn through (XR foveation; read only when hzbRateMapSizes.x > 0, a placeholder otherwise) }GaussianChunkCullBufferIndices; typedef enum{ @@ -1299,6 +1377,7 @@ typedef enum{ gaussianChunkPreprocessCoarseTableIndex, // 17: GaussianChunkDecodeConstants[levelCount ร— chunkCount], the coarse rows, level-major gaussianChunkPreprocessLevelStateIndex, // 18: GaussianChunkLevelState[chunkCount] as gaussianComputeChunkQuotas left it this frame gaussianChunkPreprocessLevelConstantsIndex, // 19: GaussianChunkLevelConstants (setBytes) + gaussianChunkPreprocessRateMapDataIndex, // 20: rasterization_rate_map_data the HZB was drawn through (XR foveation; read only when the cull constants' hzbRateMapSizes.x > 0, a placeholder otherwise) }GaussianChunkPreprocessBufferIndices; typedef enum{ @@ -1354,6 +1433,40 @@ typedef enum{ fxaaPassSplatCoverageTextureIndex = 1 }FXAATextureIndices; +// MARK: - Temporal anti-aliasing (TAAShader.metal) + +/// The resolve's inputs: the eye's unjittered matrices for this frame and the frame its +/// history holds, and the rasterization rate maps of both when the frames were foveated +/// (XR): the history is laid out in its own frame's physical space, so a pixel goes +/// physical โ†’ screen โ†’ world โ†’ previous screen โ†’ previous physical before the history is read. +typedef struct { + matrix_float4x4 invViewProjection; // this frame, unjittered, scene root included + matrix_float4x4 prevViewProjection; // the history's frame, unjittered, scene root included + simd_float4 rateMapSizes; // this frame's map: screen size xy, physical size zw; x <= 0 when drawn uniformly + simd_float4 prevRateMapSizes; // the history frame's map, same layout + simd_float2 physicalSize; // the colour, depth and history textures' size in pixels + float historyWeight; // share of the clipped history in the output (0.9) + uint32_t historyValid; // 0 on the first frame, after a resize or a mode change: the output is the current frame + uint32_t reverseZ; + float clipGamma; // the clip box is the 3ร—3 mean ยฑ clipGamma ร— standard deviation (1.25) + uint32_t velocityValid; // 1 when the opaque pass wrote the frame's velocity texture (per-object motion) + uint32_t pad1; +} TAAConstants; // 176 bytes + +typedef enum{ + taaPassConstantsIndex = 0, // TAAConstants + taaPassRateMapDataIndex, // rasterization_rate_map_data of this frame (read only when rateMapSizes.x > 0; a placeholder otherwise) + taaPassPrevRateMapDataIndex, // rasterization_rate_map_data of the history's frame (read only when prevRateMapSizes.x > 0) + taaPassEncodingModeIndex // int: the present encoding (1 = linear to sRGB), for the resolve that presents +}TAABufferIndices; + +typedef enum{ + taaPassColorTextureIndex = 0, // this frame's look output + taaPassHistoryTextureIndex, // the previous resolve of this eye + taaPassDepthTextureIndex, // this frame's opaque depth + taaPassVelocityTextureIndex // this frame's velocity (NDC, this minus previous), when velocityValid +}TAATextureIndices; + typedef enum{ smaaPassTexelSizeIndex, smaaPassEdgeThresholdIndex, diff --git a/Sources/Demos/ShowcaseDemo/AppDelegate.swift b/Sources/Demos/ShowcaseDemo/AppDelegate.swift index 19c5272b5..f80d4e5a2 100644 --- a/Sources/Demos/ShowcaseDemo/AppDelegate.swift +++ b/Sources/Demos/ShowcaseDemo/AppDelegate.swift @@ -21,7 +21,7 @@ @MainActor final class AppDelegate: NSObject, NSApplicationDelegate { private enum Constants { - static let appVersion = "0.22.0" + static let appVersion = "0.23.0" static let defaultWindowSize = NSSize(width: 1920, height: 1080) static let minimumWindowSize = NSSize(width: 640, height: 480) } diff --git a/Sources/Sandbox/AppDelegate.swift b/Sources/Sandbox/AppDelegate.swift index 6f88e3e0e..0d66a37ee 100644 --- a/Sources/Sandbox/AppDelegate.swift +++ b/Sources/Sandbox/AppDelegate.swift @@ -20,7 +20,7 @@ @MainActor final class AppDelegate: NSObject, NSApplicationDelegate { private enum Constants { - static let appVersion = "0.22.0" + static let appVersion = "0.23.0" static let windowSize = NSSize(width: 1600, height: 900) } diff --git a/Sources/UntoldEngine/Animation/TransformTrack.swift b/Sources/UntoldEngine/Animation/TransformTrack.swift new file mode 100644 index 000000000..4ffca86a1 --- /dev/null +++ b/Sources/UntoldEngine/Animation/TransformTrack.swift @@ -0,0 +1,144 @@ +// +// TransformTrack.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Foundation +import simd + +/// How a track moves from one key to the next. +public enum TrackEasing: Sendable, Equatable { + /// At a constant rate. + case linear + /// Slowly away from the key, faster towards the next. + case easeIn + /// Fast away from the key, slowing into the next. + case easeOut + /// Slowly away and slowly in (a smoothstep). + case easeInOut + + /// The progress through a segment, 0 to 1, for a fraction of its duration. + public func progress(_ fraction: Float) -> Float { + let t = min(max(fraction, 0), 1) + switch self { + case .linear: + return t + case .easeIn: + return t * t + case .easeOut: + return 1 - (1 - t) * (1 - t) + case .easeInOut: + return t * t * (3 - 2 * t) + } + } +} + +/// A transform at a time on a track, and how the track leaves it for the next key. +public struct TransformKey: Sendable, Equatable { + /// Seconds from the track's start. + public var time: TimeInterval + public var position: simd_float3 + public var rotation: simd_quatf + public var scale: simd_float3 + /// How the track moves from this key to the next. + public var easing: TrackEasing + + public init( + time: TimeInterval, + position: simd_float3, + rotation: simd_quatf = simd_quatf(angle: 0, axis: simd_float3(0, 1, 0)), + scale: simd_float3 = simd_float3(repeating: 1), + easing: TrackEasing = .linear + ) { + self.time = time + self.position = position + self.rotation = rotation + self.scale = scale + self.easing = easing + } +} + +/// A position, rotation and scale sampled from a track. +public struct TransformPose: Sendable, Equatable { + public var position: simd_float3 + public var rotation: simd_quatf + public var scale: simd_float3 + + public init(position: simd_float3, rotation: simd_quatf, scale: simd_float3) { + self.position = position + self.rotation = rotation + self.scale = scale + } +} + +/// Transform keys sampled at an absolute time: a frame's pose is a function of the frame's +/// presentation time alone, with no state carried over from the frame before, so a frame the +/// display skipped lands where the timeline says and the frames after it step by one period +/// again. Sample it at `FrameTime.presentationTime` less the time the track started. +public struct TransformTrack: Sendable, Equatable { + /// The keys in time order. + public let keys: [TransformKey] + /// Whether the track starts again after its last key, or holds the last key. + public let loops: Bool + + public init(keys: [TransformKey], loops: Bool = false) { + self.keys = keys.sorted { $0.time < $1.time } + self.loops = loops + } + + /// The time of the last key: the length of one pass. + public var duration: TimeInterval { keys.last?.time ?? 0 } + + /// The pose `time` seconds into the track: before the first key the first key's, after the + /// last the last's (or, looping, the pose that far into the next pass). Nil without keys. + public func pose(at time: TimeInterval) -> TransformPose? { + guard let first = keys.first, let last = keys.last else { return nil } + var t = time + if loops, duration > 0 { + t = t.truncatingRemainder(dividingBy: duration) + if t < 0 { t += duration } + } + if t <= first.time { return pose(of: first) } + if t >= last.time { return pose(of: last) } + + // The segment the time falls in: the last key at or before it, and the one after. + var index = 0 + while index + 1 < keys.count, keys[index + 1].time <= t { + index += 1 + } + let from = keys[index] + let to = keys[index + 1] + let span = to.time - from.time + let fraction = span > 0 ? Float((t - from.time) / span) : 1 + let s = from.easing.progress(fraction) + return TransformPose( + position: simd_mix(from.position, to.position, simd_float3(repeating: s)), + rotation: simd_slerp(from.rotation, to.rotation, s), + scale: simd_mix(from.scale, to.scale, simd_float3(repeating: s)) + ) + } + + /// The pose for a frame: the track's time is the frame's presentation time less + /// `startedAt`, the presentation time of the frame the track started in. + public func pose(for frame: FrameTime, startedAt: TimeInterval) -> TransformPose? { + pose(at: frame.presentationTime - startedAt) + } + + private func pose(of key: TransformKey) -> TransformPose { + TransformPose(position: key.position, rotation: key.rotation, scale: key.scale) + } +} + +/// Poses an entity from a track at a frame's presentation time: its position, rotation and +/// scale. `startedAt` is the presentation time of the frame the track started in. +public func applyTransformTrack(entityId: EntityID, track: TransformTrack, startedAt: TimeInterval, frame: FrameTime) { + guard let pose = track.pose(for: frame, startedAt: startedAt) else { return } + translateTo(entityId: entityId, position: pose.position) + rotateTo(entityId: entityId, rotation: pose.rotation) + scaleTo(entityId: entityId, scale: pose.scale) +} diff --git a/Sources/UntoldEngine/BuildSystem/BuildSystem.swift b/Sources/UntoldEngine/BuildSystem/BuildSystem.swift index 89fd5f68f..83ba9f51b 100644 --- a/Sources/UntoldEngine/BuildSystem/BuildSystem.swift +++ b/Sources/UntoldEngine/BuildSystem/BuildSystem.swift @@ -272,12 +272,9 @@ import Foundation /// Validate that an existing project has the expected structure public func isValidProjectStructure(settings: BuildSettings) -> Bool { let projectDir = settings.outputPath.appendingPathComponent(settings.projectName) - let gameDataDir = projectDir - .appendingPathComponent("Sources") - .appendingPathComponent(settings.projectName) - .appendingPathComponent("GameData") + let layout = ProjectLayout.detect(projectRoot: projectDir, projectName: settings.projectName) - return FileManager.default.fileExists(atPath: gameDataDir.path) + return FileManager.default.fileExists(atPath: layout.gameDataDirectory.path) } /// Update only game data in an existing project (preserves custom code) @@ -293,10 +290,8 @@ import Foundation // 2. Validate project structure let projectDir = settings.outputPath.appendingPathComponent(settings.projectName) - let gameDataDir = projectDir - .appendingPathComponent("Sources") - .appendingPathComponent(settings.projectName) - .appendingPathComponent("GameData") + let layout = ProjectLayout.detect(projectRoot: projectDir, projectName: settings.projectName) + let gameDataDir = layout.gameDataDirectory // If GameData doesn't exist, create the directory structure if !FileManager.default.fileExists(atPath: gameDataDir.path) { @@ -437,14 +432,16 @@ import Foundation try content.write(to: fileURL, atomically: true, encoding: .utf8) } - // Create GameData directory structure - // Both single-platform and multi-platform use Sources/{ProjectName}/GameData - let gameDataDir = projectDir - .appendingPathComponent("Sources") - .appendingPathComponent(settings.projectName) - .appendingPathComponent("GameData") + // Create GameData, Resources and Docs at the project root. + let layout = ProjectLayout(kind: .rootLevel, projectRoot: projectDir, projectName: settings.projectName) - try createGameDataDirectories(at: gameDataDir) + try createGameDataDirectories(at: layout.gameDataDirectory) + if let resourcesDirectory = layout.resourcesDirectory { + try fileManager.createDirectory(at: resourcesDirectory, withIntermediateDirectories: true) + } + if let docsDirectory = layout.docsDirectory { + try fileManager.createDirectory(at: docsDirectory, withIntermediateDirectories: true) + } Logger.log(message: "๐Ÿ“„ Created \(filesToWrite.count) files from embedded templates") @@ -457,11 +454,8 @@ import Foundation private func bundleGameData(to projectDir: URL, settings: BuildSettings) async throws -> [String] { var bundledAssets: [String] = [] - // Both single-platform and multi-platform use Sources/{ProjectName}/GameData - let gameDataDir = projectDir - .appendingPathComponent("Sources") - .appendingPathComponent(settings.projectName) - .appendingPathComponent("GameData") + let layout = ProjectLayout.detect(projectRoot: projectDir, projectName: settings.projectName) + let gameDataDir = layout.gameDataDirectory // Create subdirectories try createGameDataDirectories(at: gameDataDir) @@ -642,6 +636,135 @@ import Foundation ) } + // MARK: - Project Layout Conversion + + /// What `convertProjectLayout` did, in order. + public struct ProjectConversionResult { + public let projectRoot: URL + public let log: [String] + } + + /// Moves a project from the legacy layout (`GameData` nested under + /// `Sources//GameData`) to the root-level one (`GameData`, `Resources` and + /// `Docs` at the project root, next to `Sources`): moves `GameData` itself, creates + /// empty `Resources`/`Docs` folders, writes a `.gitignore` if the project doesn't have + /// one yet, rewrites `project.yml` and regenerates the Xcode project, and checks that + /// every pack's `sourceAsset` still resolves afterwards. + /// + /// That last check is not a rewrite: `sourceAsset` is relative to the pack's own + /// folder (see `UntoldPackData.resolvedSourceAssetURL`), and moving `GameData` moves + /// every pack inside it together with whatever source sits beside it, so the + /// relationship between a pack and its source survives the move unchanged. Sources + /// that happen to sit inside `GameData` today are not relocated into `Resources` here + /// -- that is a separate, judgment-heavy migration (shared sources, naming collisions) + /// better done deliberately than as a side effect of a layout move. + /// + /// Refuses to run if the project is already on the root-level layout, or if the + /// legacy `GameData` doesn't exist where expected. + @discardableResult + public func convertProjectLayout( + projectRoot: URL, + projectName: String, + progress: ((String) -> Void)? = nil + ) throws -> ProjectConversionResult { + let fileManager = FileManager.default + var log: [String] = [] + func note(_ message: String) { + log.append(message) + progress?(message) + } + + guard fileManager.fileExists(atPath: projectRoot.path) else { + throw BuildError.projectNotFound("Project not found at \(projectRoot.path)") + } + + let layout = ProjectLayout.detect(projectRoot: projectRoot, projectName: projectName) + guard layout.kind == .legacy else { + throw BuildError.invalidProjectStructure("\(projectRoot.path) is already on the root-level layout") + } + guard fileManager.fileExists(atPath: layout.gameDataDirectory.path) else { + throw BuildError.invalidProjectStructure("No GameData folder found at \(layout.gameDataDirectory.path)") + } + + let rootLayout = ProjectLayout(kind: .rootLevel, projectRoot: projectRoot, projectName: projectName) + + try fileManager.moveItem(at: layout.gameDataDirectory, to: rootLayout.gameDataDirectory) + note("๐Ÿ“ฆ Moved GameData: \(layout.gameDataDirectory.path) -> \(rootLayout.gameDataDirectory.path)") + + // Everything below here can fail (disk I/O, the xcodegen subprocess). If it does, + // move GameData back so the project is left in the same, retryable legacy state it + // started in rather than a half-converted one (GameData at the root, project.yml + // still pointing at the old path, or vice versa). + do { + if let resourcesDirectory = rootLayout.resourcesDirectory, !fileManager.fileExists(atPath: resourcesDirectory.path) { + try fileManager.createDirectory(at: resourcesDirectory, withIntermediateDirectories: true) + note("๐Ÿ“ Created \(resourcesDirectory.path)") + } + if let docsDirectory = rootLayout.docsDirectory, !fileManager.fileExists(atPath: docsDirectory.path) { + try fileManager.createDirectory(at: docsDirectory, withIntermediateDirectories: true) + note("๐Ÿ“ Created \(docsDirectory.path)") + } + + let gitignorePath = projectRoot.appendingPathComponent(".gitignore") + if !fileManager.fileExists(atPath: gitignorePath.path) { + try BuildTemplates.gitignore.write(to: gitignorePath, atomically: true, encoding: .utf8) + note("๐Ÿ“ Wrote .gitignore") + } + + let projectYMLPath = projectRoot.appendingPathComponent("project.yml") + let legacyGameDataPath = "Sources/\(projectName)/GameData" + if let yaml = try? String(contentsOf: projectYMLPath, encoding: .utf8) { + let converted = XcodeGenProjectSpec.convertingToRootLevelGameData(inYAML: yaml, projectName: projectName) + if converted != yaml { + try converted.write(to: projectYMLPath, atomically: true, encoding: .utf8) + note("๐Ÿ“ Rewrote project.yml") + if Self.xcodeGenPath() != nil { + try runXcodeGen(in: projectRoot) + note("โš™๏ธ Regenerated Xcode project") + } else { + note("โš ๏ธ xcodegen not found; run `xcodegen generate` in the project folder to finish regenerating the Xcode project") + } + } else if yaml.contains(legacyGameDataPath) { + // The rewrite is a textual match against exactly what the generator + // produces; a hand-edited project.yml (different quoting, reordered + // keys, ...) can still reference the legacy path without matching that + // shape. Say so explicitly instead of reporting "already converted". + note("โš ๏ธ project.yml still references \(legacyGameDataPath) but didn't match the expected pattern (hand-edited project.yml?); update it to reference \"GameData\" and run `xcodegen generate` manually") + } else { + note("โ„น๏ธ project.yml already matched the root-level layout") + } + } else { + note("โš ๏ธ No project.yml found; skipped Xcode project regeneration") + } + } catch { + try? fileManager.moveItem(at: rootLayout.gameDataDirectory, to: layout.gameDataDirectory) + note("โ†ฉ๏ธ Rolled back: moved GameData back to \(layout.gameDataDirectory.path) after a failure converting project.yml") + throw error + } + + let packURLs = (fileManager.enumerator(at: rootLayout.gameDataDirectory, includingPropertiesForKeys: nil)? + .compactMap { $0 as? URL } ?? []) + .filter { $0.pathExtension.lowercased() == "untoldpack" } + var unresolvedSources = 0 + for packURL in packURLs { + guard let pack = loadUntoldPack(url: packURL), pack.sourceAsset != nil else { continue } + if let resolved = pack.resolvedSourceAssetURL(packURL: packURL), fileManager.fileExists(atPath: resolved.path) { + continue + } + unresolvedSources += 1 + note("โš ๏ธ \(packURL.lastPathComponent): sourceAsset \"\(pack.sourceAsset ?? "")\" does not resolve to an existing file") + } + if packURLs.isEmpty { + note("โ„น๏ธ No .untoldpack manifests found under GameData") + } else if unresolvedSources == 0 { + note("โœ… \(packURLs.count) pack manifest(s) checked; sourceAsset still resolves for all of them") + } else { + note("โš ๏ธ \(unresolvedSources) of \(packURLs.count) pack manifest(s) have a sourceAsset that no longer resolves") + } + + return ProjectConversionResult(projectRoot: projectRoot, log: log) + } + // MARK: - Template Processing private func copyDirectory(from source: URL, to destination: URL) throws { @@ -730,9 +853,7 @@ import Foundation // MARK: - Asset Bundling Helpers private func createGameDataDirectories(at gameDataDir: URL) throws { - let subdirs = ["Scenes", "Scripts", "Models", "StreamModels", "Animations", "Gaussians", "Textures", "Shaders", "LUT"] - - for subdir in subdirs { + for subdir in ProjectLayout.gameDataCategories { let dirURL = gameDataDir.appendingPathComponent(subdir) try FileManager.default.createDirectory(at: dirURL, withIntermediateDirectories: true) } @@ -988,9 +1109,8 @@ import Foundation /// Clear contents of GameData directory while preserving the directory structure private func clearGameDataDirectory(at gameDataDir: URL) throws { let fileManager = FileManager.default - let subdirs = ["Scenes", "Scripts", "Models", "StreamModels", "Animations", "Gaussians", "Textures", "Shaders", "LUT"] - for subdir in subdirs { + for subdir in ProjectLayout.gameDataCategories { let dirURL = gameDataDir.appendingPathComponent(subdir) // Remove directory if it exists diff --git a/Sources/UntoldEngine/BuildSystem/BuildTemplates.swift b/Sources/UntoldEngine/BuildSystem/BuildTemplates.swift index 1fd1d7c4e..0a68a8241 100644 --- a/Sources/UntoldEngine/BuildSystem/BuildTemplates.swift +++ b/Sources/UntoldEngine/BuildSystem/BuildTemplates.swift @@ -53,6 +53,9 @@ import Foundation {{CODE_COMPONENTS_PACKAGE_DEPENDENCIES}} .product(name: "UntoldEngineShaderSupport", package: "UntoldEngine") ], + path: ".", + exclude: ["Resources", "Docs", "README.md", "project.yml", ".gitignore", "{{PROJECT_NAME}}.xcodeproj"], + sources: ["Sources/{{PROJECT_NAME}}"], resources: [ .process("GameData") ], @@ -1460,41 +1463,29 @@ import Foundation ## ๐ŸŽฎ About - This is a macOS game project built with UntoldEngine. The project includes all your game data (scenes, scripts, assets) bundled and ready to run. + This is a game project built with UntoldEngine. ## ๐Ÿš€ Quick Start - ### Build and Run - - ```bash - swift build - swift run - ``` - - ### Open in Xcode - - ```bash - open Package.swift - ``` - - Then press `Cmd+R` to build and run. + Open `{{PROJECT_NAME}}.xcodeproj` (macOS projects can also `swift build && swift run`, or `open Package.swift`), then press `Cmd+R` to build and run. ## ๐Ÿ“ Project Structure ``` {{PROJECT_NAME}}/ - โ”œโ”€โ”€ Package.swift # Swift Package configuration - โ”œโ”€โ”€ README.md + โ”œโ”€โ”€ .gitignore + โ”œโ”€โ”€ README.md + โ”œโ”€โ”€ project.yml # xcodegen spec; regenerate with: xcodegen generate + โ”œโ”€โ”€ {{PROJECT_NAME}}.xcodeproj # generated from project.yml -- do not edit directly + โ”œโ”€โ”€ GameData/ # bundled at build time; everything the engine reads + โ”‚ โ”œโ”€โ”€ Scenes/ # .untoldscene files + โ”‚ โ”œโ”€โ”€ Scripts/ # USC scripts (.uscript) + โ”‚ โ”œโ”€โ”€ Models/ # cooked models (.untoldpack) + โ”‚ โ”œโ”€โ”€ Textures/ Animations/ Gaussians/ StreamModels/ Shaders/ LUT/ + โ”œโ”€โ”€ Resources/ # artist sources (.blend, .usd, references) -- never bundled + โ”œโ”€โ”€ Docs/ # design notes, decisions -- whatever you want written down โ””โ”€โ”€ Sources/ - โ””โ”€โ”€ {{PROJECT_NAME}}/ - โ”œโ”€โ”€ main.swift # Entry point - โ”œโ”€โ”€ GameRuntime.swift # Game loader - โ””โ”€โ”€ GameData/ # Bundled game content - โ”œโ”€โ”€ Scenes/ # Scene files (.json) - โ”œโ”€โ”€ Scripts/ # USC scripts (.uscript) - โ”œโ”€โ”€ Models/ # 3D models - โ”œโ”€โ”€ Textures/ # Texture files - โ””โ”€โ”€ Shaders/ # Compiled Metal shaders + โ””โ”€โ”€ {{PROJECT_NAME}}/ # app code ``` ## ๐ŸŽฏ Requirements @@ -1505,9 +1496,9 @@ import Foundation ## ๐Ÿ“ Notes - - This project uses UntoldEngine as a Swift Package dependency from GitHub - - All game data is bundled as resources - - USC scripts are loaded and executed at runtime + - This project uses UntoldEngine as a Swift Package dependency from GitHub. + - `GameData` is bundled as a resource folder; nothing under `Resources/` or `Docs/` ships in the built app. + - USC scripts are loaded and executed at runtime. ## ๐Ÿ”— UntoldEngine @@ -1518,6 +1509,19 @@ import Foundation Built with โค๏ธ using UntoldEngine """ + // MARK: - .gitignore Template + + static let gitignore = """ + *.xcodeproj/ + .build/ + .swiftpm/ + DerivedData/ + xcuserdata/ + .DS_Store + *.blend1 + + """ + // MARK: - Multi-Platform Templates /// Unified GameScene template that works across all platforms @@ -1798,6 +1802,7 @@ import Foundation case .macOS: return [ "Package.swift": packageSwift, + ".gitignore": gitignore, "README.md": readmeMd, "Sources/{{PROJECT_NAME}}/AppDelegate.swift": appDelegateSwift, "Sources/{{PROJECT_NAME}}/GameScene.swift": macOSGameSceneSwift, @@ -1808,6 +1813,7 @@ import Foundation ] case .iOS: return [ + ".gitignore": gitignore, "README.md": readmeMd, "Sources/{{PROJECT_NAME}}/AppDelegate.swift": iOSAppDelegateSwift, "Sources/{{PROJECT_NAME}}/GameScene.swift": iOSGameSceneSwift, @@ -1819,6 +1825,7 @@ import Foundation case .visionOS: // VisionOS uses SwiftUI App structure with CompositorLayer return [ + ".gitignore": gitignore, "README.md": readmeMd, "Sources/{{PROJECT_NAME}}/GameScene.swift": visionOSGameSceneSwift, "Sources/{{PROJECT_NAME}}/GameSceneUtils.swift": visionOSGameSceneUtilsSwift, @@ -1916,6 +1923,7 @@ import Foundation /// Get template files for iOS AR specifically static func getTemplateFilesForIOSAR() -> [String: String] { [ + ".gitignore": gitignore, "README.md": readmeMd, "Sources/{{PROJECT_NAME}}/AppDelegate.swift": iOSAppDelegateSwift, "Sources/{{PROJECT_NAME}}/GameScene.swift": iOSARGameSceneSwift, @@ -1930,6 +1938,7 @@ import Foundation /// Structure: Platform-specific folders + Sources/{ProjectName}/ for shared code static func getTemplateFilesForMultiPlatform() -> [String: String] { [ + ".gitignore": gitignore, "README.md": readmeMd, // macOS target - platform-specific files diff --git a/Sources/UntoldEngine/BuildSystem/ProjectLayout.swift b/Sources/UntoldEngine/BuildSystem/ProjectLayout.swift new file mode 100644 index 000000000..55b9f23df --- /dev/null +++ b/Sources/UntoldEngine/BuildSystem/ProjectLayout.swift @@ -0,0 +1,115 @@ +// +// ProjectLayout.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Foundation + +#if os(macOS) + + /// Where a project's `GameData` lives, and which of the two folder shapes a + /// project on disk is using. Projects created before the root-level layout + /// landed keep `GameData` nested under `Sources//GameData`; new + /// projects (and ones moved over with `untoldengine convert`) hold `GameData`, + /// `Resources` and `Docs` at the project root, next to `Sources`. + /// + /// Everything that locates `GameData` -- the build system, `update`, `assets + /// install`, the editor -- goes through this type instead of rebuilding the + /// path shape itself, which is how the category list and the `Sources// + /// GameData` prefix ended up duplicated across the build system, the CLI and the + /// docs in the first place. + public struct ProjectLayout: Sendable, Equatable { + public enum Kind: String, Codable, Sendable { + /// `GameData` nested under `Sources//GameData`. + case legacy + /// `GameData`, `Resources` and `Docs` at the project root. + case rootLevel + } + + public let kind: Kind + public let projectRoot: URL + public let projectName: String + + public init(kind: Kind, projectRoot: URL, projectName: String) { + self.kind = kind + self.projectRoot = projectRoot + self.projectName = projectName + } + + /// The category folders created under `GameData`. Kept in one place so + /// `createGameDataDirectories` and `clearGameDataDirectory` can't drift + /// from each other, or from what `bundleGameData` actually writes into. + public static let gameDataCategories = [ + "Scenes", "Scripts", "Models", "StreamModels", "Animations", "Gaussians", "Textures", "Shaders", "LUT", + ] + + public var gameDataDirectory: URL { + switch kind { + case .legacy: + return projectRoot + .appendingPathComponent("Sources") + .appendingPathComponent(projectName) + .appendingPathComponent("GameData") + case .rootLevel: + return projectRoot.appendingPathComponent("GameData") + } + } + + /// Where artist sources (`.blend`, `.usd`, reference images) live. Only + /// meaningful for the root-level layout -- a legacy project has no home + /// for them, which is the problem this layout exists to fix. + public var resourcesDirectory: URL? { + kind == .rootLevel ? projectRoot.appendingPathComponent("Resources") : nil + } + + /// Where project documentation lives. Only meaningful for the root-level layout. + public var docsDirectory: URL? { + kind == .rootLevel ? projectRoot.appendingPathComponent("Docs") : nil + } + + public var sourcesDirectory: URL { + projectRoot.appendingPathComponent("Sources") + } + + /// Detects which layout a project on disk is using by checking for a + /// root-level `GameData` first, falling back to the legacy nested + /// location. Defaults to `.legacy` if neither exists yet -- a project + /// about to be created explicitly chooses its own layout instead of + /// calling this (see `BuildSystem.createProjectFromEmbeddedTemplates`). + public static func detect(projectRoot: URL, projectName: String) -> ProjectLayout { + let rootGameData = projectRoot.appendingPathComponent("GameData") + if FileManager.default.fileExists(atPath: rootGameData.path) { + return ProjectLayout(kind: .rootLevel, projectRoot: projectRoot, projectName: projectName) + } + return ProjectLayout(kind: .legacy, projectRoot: projectRoot, projectName: projectName) + } + + /// Infers the project root, project name, and layout kind from a `GameData` + /// directory's own location -- the inverse of `gameDataDirectory`. For a caller + /// that only has `assetBasePath` (the `GameData` URL itself), not the project + /// folder it came from, e.g. the editor after it has opened a project. + /// + /// Distinguishes the two shapes structurally, not by checking disk: `GameData` + /// sitting directly inside a `Sources` folder is the legacy nested shape + /// (`/Sources//GameData`); anywhere else, `GameData`'s own parent + /// is the project root (`/GameData`). + public static func detect(gameDataDirectory: URL) -> ProjectLayout { + let projectNameDirectory = gameDataDirectory.deletingLastPathComponent() + let sourcesDirectory = projectNameDirectory.deletingLastPathComponent() + if sourcesDirectory.lastPathComponent == "Sources" { + return ProjectLayout( + kind: .legacy, + projectRoot: sourcesDirectory.deletingLastPathComponent(), + projectName: projectNameDirectory.lastPathComponent + ) + } + return ProjectLayout(kind: .rootLevel, projectRoot: projectNameDirectory, projectName: projectNameDirectory.lastPathComponent) + } + } + +#endif // os(macOS) diff --git a/Sources/UntoldEngine/BuildSystem/XcodeGenProjectSpec.swift b/Sources/UntoldEngine/BuildSystem/XcodeGenProjectSpec.swift index f08e2a210..34a07665c 100644 --- a/Sources/UntoldEngine/BuildSystem/XcodeGenProjectSpec.swift +++ b/Sources/UntoldEngine/BuildSystem/XcodeGenProjectSpec.swift @@ -87,6 +87,50 @@ import Foundation return output.joined(separator: "\n") } + /// Rewrites a `project.yml` generated before `GameData` moved to the project root + /// (see `BuildSystem.createProjectFromEmbeddedTemplates`), pointing its `GameData` + /// folder reference(s) at the root instead of `Sources//GameData`. Used by + /// `BuildSystem.convertProjectLayout`. A no-op (returns the input unchanged) when the + /// legacy shape isn't present, so it's safe to call on a spec that's already converted. + static func convertingToRootLevelGameData(inYAML yaml: String, projectName: String) -> String { + let lines = yaml.components(separatedBy: "\n") + let sourcesLine = "- path: Sources" + let excludesLine = "excludes:" + let excludedEntryLine = "- \"\(projectName)/GameData\"" + let gameDataLine = "- path: Sources/\(projectName)/GameData" + + var output: [String] = [] + var index = 0 + while index < lines.count { + let line = lines[index] + let trimmed = line.trimmingCharacters(in: .whitespaces) + + // The single-platform spec excluded GameData from the plain "Sources" entry + // because GameData used to be nested inside it; it's a root-level sibling of + // Sources now, so there's nothing left to exclude. + if trimmed == sourcesLine, + index + 2 < lines.count, + lines[index + 1].trimmingCharacters(in: .whitespaces) == excludesLine, + lines[index + 2].trimmingCharacters(in: .whitespaces) == excludedEntryLine + { + output.append(line) + index += 3 + continue + } + + if trimmed == gameDataLine { + let indentation = String(line.prefix(while: { $0 == " " || $0 == "\t" })) + output.append("\(indentation)- path: GameData") + index += 1 + continue + } + + output.append(line) + index += 1 + } + return output.joined(separator: "\n") + } + /// Generate project.yml YAML content from build settings static func generateYAML(settings: BuildSettings) throws -> String { let isMultiPlatform = settings.target.platforms.count > 1 @@ -154,12 +198,11 @@ import Foundation // Build sources section. Storyboards under Base.lproj are compiled // from the Sources tree; do not add Base.lproj as a raw folder // resource or the app bundle will contain an uncompiled storyboard. + // GameData lives at the project root, a sibling of Sources, not nested inside it. let sourcesSection = """ sources: - path: Sources - excludes: - - "\(settings.projectName)/GameData" - - path: Sources/\(settings.projectName)/GameData + - path: GameData type: folder buildPhase: resources """ @@ -251,7 +294,7 @@ import Foundation sources: - path: \(settings.projectName) macOS - path: Sources/\(settings.projectName)\(componentSourcesIfEnabled) - - path: Sources/\(settings.projectName)/GameData + - path: GameData type: folder buildPhase: resources \(dependenciesWithKit) @@ -280,7 +323,7 @@ import Foundation sources: - path: \(settings.projectName) iOS - path: Sources/\(settings.projectName)\(componentSourcesIfEnabled) - - path: Sources/\(settings.projectName)/GameData + - path: GameData type: folder buildPhase: resources dependencies: @@ -314,7 +357,7 @@ import Foundation sources: - path: \(settings.projectName) iOS AR - path: Sources/\(settings.projectName)\(componentSourcesIfEnabled) - - path: Sources/\(settings.projectName)/GameData + - path: GameData type: folder buildPhase: resources dependencies: @@ -344,7 +387,7 @@ import Foundation sources: - path: \(settings.projectName) visionOS - path: Sources/\(settings.projectName)\(componentSourcesIfEnabled) - - path: Sources/\(settings.projectName)/GameData + - path: GameData type: folder buildPhase: resources dependencies: diff --git a/Sources/UntoldEngine/ECS/EngineExtensions.swift b/Sources/UntoldEngine/ECS/EngineExtensions.swift index 019948701..e333213d9 100644 --- a/Sources/UntoldEngine/ECS/EngineExtensions.swift +++ b/Sources/UntoldEngine/ECS/EngineExtensions.swift @@ -17,19 +17,24 @@ public struct EngineExtensionUpdateContext: Sendable { public let frameIndex: UInt64 public let currentEye: Int public let isPrimaryEye: Bool + /// The frame's predicted presentation time, its delta from the frame before and the + /// refreshes skipped between them; `.zero` for a context built outside a frame. + public let frameTime: FrameTime public init( viewport: SIMD2, immersionStyle: UntoldImmersionMode, frameIndex: UInt64, currentEye: Int, - isPrimaryEye: Bool + isPrimaryEye: Bool, + frameTime: FrameTime = .zero ) { self.viewport = viewport self.immersionStyle = immersionStyle self.frameIndex = frameIndex self.currentEye = currentEye self.isPrimaryEye = isPrimaryEye + self.frameTime = frameTime } } diff --git a/Sources/UntoldEngine/Mesh/Mesh.swift b/Sources/UntoldEngine/Mesh/Mesh.swift index 5316029a6..fc80f0886 100644 --- a/Sources/UntoldEngine/Mesh/Mesh.swift +++ b/Sources/UntoldEngine/Mesh/Mesh.swift @@ -850,7 +850,7 @@ public struct Material { /// contrast to work with unless that slice is stretched back out first. public var heightRemapMin: Float = 0.0 public var heightRemapMax: Float = 1.0 - public var heightEnabled: Bool = true + public var heightEnabled: Bool = false /// Texture presence flags public var hasNormalMap: Bool { @@ -895,6 +895,27 @@ public struct Material { alphaMode == .blend || transmitsLight } + /// What a polished non-metal lets cross at one face seen head on: one minus the + /// dielectric reflectance of 0.04 that TransparencyShader.metal takes as F0 + /// (`environmentReflectance(float3(0.04), ...)`), collapsed to its head-on value since + /// there is no view angle or roughness per pixel here. Keep the two in step. + static let dielectricHeadOnTransmittance: Float = 1 - 0.04 + + /// Whether a pane is mostly itself (its tint, its frost, its reflections) rather than + /// what is behind it: less than half of the light crosses it. A headset's compositor + /// reprojects a pixel by the depth it finds, so such a pane writes its own depth there, + /// while a clear pane leaves the depth of what is seen through it. Mirrors the share of + /// light that crosses in TransparencyShader.metal. + public var paneShowsItself: Bool { + let coverage = baseColorValue.w + let t = min(max((roughnessValue - GLASS_CLEAR_UP_TO_ROUGHNESS) / (GLASS_FROSTED_FROM_ROUGHNESS - GLASS_CLEAR_UP_TO_ROUGHNESS), 0), 1) + let clearShare = 1 - t * t * (3 - 2 * t) + let brightness = max(baseColorValue.x, max(baseColorValue.y, baseColorValue.z)).squareRoot() + let crosses = (1 - metallicValue) * brightness * Self.dielectricHeadOnTransmittance + let through = (1 - coverage) + coverage * transmission * clearShare * crosses + return through < 0.5 + } + public var stScale: Float = 1.0 @available(*, deprecated, message: "Material name is no longer used for initialization.") @@ -1039,6 +1060,12 @@ public struct Material { heightMidlevel = runtimeMaterial.heightMidlevel heightRemapMin = runtimeMaterial.heightRemapMin heightRemapMax = runtimeMaterial.heightRemapMax + // A height texture baked/embedded in the source asset means the author + // intended POM; a blank material later given a height texture through the + // editor should stay off until explicitly toggled (see issue #1360). + if heightTexture != nil { + heightEnabled = true + } let alphaModeBits = runtimeMaterial.flags & 0b11 alphaMode = MaterialAlphaMode(rawValue: Int32(alphaModeBits)) ?? .opaque @@ -1122,6 +1149,12 @@ public struct Material { if let displacementScale = mdlMaterial.property(with: .displacementScale)?.floatValue { heightScale = displacementScale } + // A height texture baked/embedded in the source asset means the author + // intended POM; a blank material later given a height texture through the + // editor should stay off until explicitly toggled (see issue #1360). + if heightTex != nil { + heightEnabled = true + } baseColorSourceDimensions = baseColorDims normalSourceDimensions = normalDims diff --git a/Sources/UntoldEngine/Profiling/EngineBuild.swift b/Sources/UntoldEngine/Profiling/EngineBuild.swift new file mode 100644 index 000000000..2ba06aba9 --- /dev/null +++ b/Sources/UntoldEngine/Profiling/EngineBuild.swift @@ -0,0 +1,22 @@ +// +// EngineBuild.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +/// How the engine module itself was compiled, for a measurement to say so: an app's Debug +/// configuration builds the package at `-Onone`, and a frame that fits comfortably when the +/// engine is optimized is then bound by the CPU encoding it (measured on a headset: the +/// opaque draws took five times longer to encode, and the GPU downclocked while it waited). +public enum EngineBuild { + /// Whether the engine module was compiled with the optimizer on. Read here, inside the + /// module and never inlined, so it reports the engine's own build and not the caller's. + @inline(never) + public static var isOptimized: Bool { + !_isDebugAssertConfiguration() + } +} diff --git a/Sources/UntoldEngine/Profiling/EngineStatsMonitor.swift b/Sources/UntoldEngine/Profiling/EngineStatsMonitor.swift index cb39d9993..d296cd59b 100644 --- a/Sources/UntoldEngine/Profiling/EngineStatsMonitor.swift +++ b/Sources/UntoldEngine/Profiling/EngineStatsMonitor.swift @@ -11,6 +11,17 @@ import Foundation import QuartzCore +/// The time the GPU spent on a frame's command buffer beyond the previous frame's: the part +/// of its span (`gpuStartTime` to `gpuEndTime`) after the previous command buffer ended. +/// Consecutive frames overlap on the GPU โ€” the next frame's first passes run while the last +/// passes of the previous one finish โ€” so the plain span of a command buffer also counts the +/// previous frame's tail, and on a saturated GPU reads far above the time the frame costs. +/// Summed over a run, this measure is the GPU's time per frame. A command buffer that ended +/// before the previous one counts nothing. +public func gpuTimeExcludingOverlap(gpuStartTime: CFTimeInterval, gpuEndTime: CFTimeInterval, previousGPUEndTime: CFTimeInterval) -> CFTimeInterval { + max(0, gpuEndTime - max(gpuStartTime, previousGPUEndTime)) +} + public enum EngineStatsLoggingProfile: Equatable { case compact case verbose @@ -32,6 +43,7 @@ public final class EngineStatsMonitor: @unchecked Sendable { private var _latestGPUExecutionMs: Double = 0.0 private var _latestGPUFrameCadenceMs: Double = 0.0 private var _lastGPUCompletionTime: Double = 0.0 // 0 = no completion seen yet + private var _lastGPUCommandBufferEndTime: CFTimeInterval = 0.0 // gpuEndTime of the last frame recorded // 30-frame rolling average for smoothed CPU frame time private let kSmoothingWindow = 30 @@ -162,6 +174,23 @@ public final class EngineStatsMonitor: @unchecked Sendable { #endif } + /// Records a frame's command buffer from its GPU timestamps, as the time the GPU gave + /// it beyond the previous frame's: `gpuTimeExcludingOverlap`. Safe to call from any thread. + public func recordGPUCompletion(gpuStartTime: CFTimeInterval, gpuEndTime: CFTimeInterval) { + #if ENGINE_STATS_ENABLED + lock.lock() + let previousEnd = _lastGPUCommandBufferEndTime + _lastGPUCommandBufferEndTime = max(previousEnd, gpuEndTime) + lock.unlock() + recordGPUCompletion(executionMs: gpuTimeExcludingOverlap( + gpuStartTime: gpuStartTime, gpuEndTime: gpuEndTime, previousGPUEndTime: previousEnd + ) * 1000.0) + #else + _ = gpuStartTime + _ = gpuEndTime + #endif + } + /// Publishes the current frame so API readers can safely consume it next frame. public func completeFrame() { #if ENGINE_STATS_ENABLED diff --git a/Sources/UntoldEngine/Profiling/EngineStatsSnapshot.swift b/Sources/UntoldEngine/Profiling/EngineStatsSnapshot.swift index 338f8f86c..0ceb1821a 100644 --- a/Sources/UntoldEngine/Profiling/EngineStatsSnapshot.swift +++ b/Sources/UntoldEngine/Profiling/EngineStatsSnapshot.swift @@ -15,7 +15,10 @@ public struct EngineTimingStats { public var frameTotalMs: Double = 0.0 /// CPU frame time averaged over the last 30 frames โ€” use this for smooth FPS display. public var smoothedFrameMs: Double = 0.0 - /// True GPU execution time for this frame (gpuEndTime - gpuStartTime from MTLCommandBuffer). + /// The GPU's time on this frame's command buffer beyond the previous frame's (consecutive + /// frames overlap on the GPU, so the command buffer's own span would count the previous + /// frame's tail too; see `gpuTimeExcludingOverlap`). On a saturated GPU this is the time + /// the GPU takes per frame. public var gpuExecutionMs: Double = 0.0 /// Wall-clock interval between successive GPU frame completions โ€” use this for true GPU FPS. public var gpuFrameCadenceMs: Double = 0.0 @@ -25,6 +28,17 @@ public struct EngineTimingStats { public var encodeMs: Double = 0.0 public var submitMs: Double = 0.0 public var cullingMs: Double = 0.0 + /// XR: the time the render loop waited for the compositor's optimal input time between + /// its update phase and its submission phase โ€” the frame's pacing, not the CPU's work. + /// `encodeMs` is then the submission phase: the drawables' command buffers encoded and + /// committed. The CPU's work in a frame is `updateMs` + `encodeMs`. + public var xrWaitMs: Double = 0.0 + /// Milliseconds from the previous frame's predicted presentation to this one's: the + /// display's period on a steady cadence, two periods after a skipped frame. + public var presentationDeltaMs: Double = 0.0 + /// Refreshes between the previous frame and this one that showed the previous frame again + /// (`FrameTime.skippedFrames`). + public var skippedFrames: Int = 0 public var streamingRegionMs: Double = 0.0 public var geometryStreamingMs: Double = 0.0 public var batchingTickMs: Double = 0.0 @@ -41,10 +55,13 @@ public struct EngineTimingStats { encodeMs: Double = 0.0, submitMs: Double = 0.0, cullingMs: Double = 0.0, + xrWaitMs: Double = 0.0, streamingRegionMs: Double = 0.0, geometryStreamingMs: Double = 0.0, batchingTickMs: Double = 0.0, - batchingRebuildMs: Double = 0.0 + batchingRebuildMs: Double = 0.0, + presentationDeltaMs: Double = 0.0, + skippedFrames: Int = 0 ) { self.frameTotalMs = frameTotalMs self.smoothedFrameMs = smoothedFrameMs @@ -56,10 +73,13 @@ public struct EngineTimingStats { self.encodeMs = encodeMs self.submitMs = submitMs self.cullingMs = cullingMs + self.xrWaitMs = xrWaitMs self.streamingRegionMs = streamingRegionMs self.geometryStreamingMs = geometryStreamingMs self.batchingTickMs = batchingTickMs self.batchingRebuildMs = batchingRebuildMs + self.presentationDeltaMs = presentationDeltaMs + self.skippedFrames = skippedFrames } } diff --git a/Sources/UntoldEngine/Profiling/GPUPassTiming.swift b/Sources/UntoldEngine/Profiling/GPUPassTiming.swift new file mode 100644 index 000000000..d7b919eb7 --- /dev/null +++ b/Sources/UntoldEngine/Profiling/GPUPassTiming.swift @@ -0,0 +1,313 @@ +// +// GPUPassTiming.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Foundation +import Metal +import QuartzCore + +/// GPU time per pass, measured by the GPU itself. A frame's passes write timestamps into a +/// counter sample buffer the frame's command buffer carries: a render pass at the start and +/// end of its vertex stage and of its fragment stage, a compute or blit pass at its bounds. +/// The buffer is resolved when the command buffer completes, and each pass's time is averaged +/// over the recent frames (the same label twice in a frame, one per eye, is summed). Off by +/// default: `setRendering(.gpuPassTiming(.enabled))`. Nothing is measured on a GPU without +/// stage-boundary sampling, and `averages()` stays empty. +public final class GPUPassTiming: @unchecked Sendable { + public static let shared = GPUPassTiming() + + /// One pass's time per frame, in milliseconds: a render pass's vertex and fragment stages; + /// a compute or blit pass has only `fragmentMs`, the whole pass. + public struct PassTime: Sendable, Equatable { + public let label: String + public let vertexMs: Double + public let fragmentMs: Double + public var totalMs: Double { + vertexMs + fragmentMs + } + } + + /// Samples per frame the buffer holds: four per render pass, two per compute or blit pass. + private static let samplesPerFrame = 512 + /// Frames in flight whose buffers may still be unresolved. + private static let ringSize = 4 + /// The weight of a new frame in the averages. + private static let smoothing = 0.1 + + private struct Pass { + let label: String + let firstSample: Int + let isRenderPass: Bool + } + + /// A ring slot's frame. Its sample buffer is a Metal object the completed handler reads + /// only once the GPU has finished writing it, and the slot's in-flight flag keeps the + /// encoder away from it until then; the frame crosses into the handler as a value. + private struct Frame: @unchecked Sendable { + let buffer: MTLCounterSampleBuffer + var nextSample = 0 + var passes: [Pass] = [] + var inFlight = false + } + + private let lock = NSLock() + private var enabledFlag = false + private var ring: [Frame] = [] + private var encoding: Int? // the ring slot of the frame being encoded + private var averageTimes: [String: (vertex: Double, fragment: Double)] = [:] + private var lastFrameTimes: [String: (vertex: Double, fragment: Double)] = [:] + private var lastFrameIndex = 0 + private var warnedUnsupported = false + + // GPU ticks to seconds, from two readings of the device's clocks far enough apart. + private var calibrationStart: (media: Double, gpu: MTLTimestamp)? + private var secondsPerTick: Double? + + private init() {} + + /// Whether passes are being timed. + public var isEnabled: Bool { + lock.lock() + defer { lock.unlock() } + return enabledFlag + } + + public func setEnabled(_ enabled: Bool) { + lock.lock() + enabledFlag = enabled + if !enabled { + averageTimes = [:] + lastFrameTimes = [:] + encoding = nil + } + lock.unlock() + } + + /// The last resolved frame's passes, heaviest first, with a number that changes with each + /// resolved frame, for a caller that averages frames its own way. Empty when off. + public func lastFrame() -> (index: Int, passes: [PassTime]) { + lock.lock() + defer { lock.unlock() } + let passes = lastFrameTimes + .map { PassTime(label: $0.key, vertexMs: $0.value.vertex, fragmentMs: $0.value.fragment) } + .sorted { $0.totalMs > $1.totalMs } + return (lastFrameIndex, passes) + } + + /// Each pass's average time over the recent frames, heaviest first. Empty when off. + public func averages() -> [PassTime] { + lock.lock() + defer { lock.unlock() } + return averageTimes + .map { PassTime(label: $0.key, vertexMs: $0.value.vertex, fragmentMs: $0.value.fragment) } + .sorted { $0.totalMs > $1.totalMs } + } + + // MARK: - Frame + + /// Starts timing a frame's passes: the frame's command buffer has just been created. + public func beginFrame(device: MTLDevice) { + lock.lock() + defer { lock.unlock() } + encoding = nil + guard enabledFlag else { return } + if ring.isEmpty, !makeRing(device: device) { return } + calibrate(device: device) + guard let slot = ring.indices.first(where: { !ring[$0].inFlight }) else { return } + ring[slot].nextSample = 0 + ring[slot].passes = [] + encoding = slot + } + + /// Ends the frame: the command buffer resolves the timestamps when it completes. + public func endFrame(_ commandBuffer: MTLCommandBuffer) { + lock.lock() + guard let slot = encoding, !ring[slot].passes.isEmpty else { + encoding = nil + lock.unlock() + return + } + encoding = nil + ring[slot].inFlight = true + let frame = ring[slot] + lock.unlock() + commandBuffer.addCompletedHandler { [weak self] cb in + self?.resolve(frame, slot: slot, gpuSeconds: cb.gpuEndTime - cb.gpuStartTime) + } + } + + // MARK: - Passes + + /// Times a render pass about to be encoded with this descriptor: its vertex and fragment + /// stages. The same label in one frame (one pass per eye) adds up. + public func attach(_ descriptor: MTLRenderPassDescriptor, _ label: String) { + lock.lock() + defer { lock.unlock() } + guard let slot = encoding, ring[slot].nextSample + 4 <= Self.samplesPerFrame else { return } + let first = ring[slot].nextSample + guard let attachment = descriptor.sampleBufferAttachments[0] else { return } + attachment.sampleBuffer = ring[slot].buffer + attachment.startOfVertexSampleIndex = first + attachment.endOfVertexSampleIndex = first + 1 + attachment.startOfFragmentSampleIndex = first + 2 + attachment.endOfFragmentSampleIndex = first + 3 + ring[slot].passes.append(Pass(label: label, firstSample: first, isRenderPass: true)) + ring[slot].nextSample += 4 + } + + /// A compute encoder for this command buffer, timed as a whole under `label` when the + /// frame is being timed; a plain encoder otherwise. + public func makeComputeEncoder(_ commandBuffer: MTLCommandBuffer, _ label: String) -> MTLComputeCommandEncoder? { + lock.lock() + guard let slot = encoding, ring[slot].nextSample + 2 <= Self.samplesPerFrame else { + lock.unlock() + return commandBuffer.makeComputeCommandEncoder() + } + let first = ring[slot].nextSample + let descriptor = MTLComputePassDescriptor() + guard let attachment = descriptor.sampleBufferAttachments[0] else { + lock.unlock() + return commandBuffer.makeComputeCommandEncoder() + } + attachment.sampleBuffer = ring[slot].buffer + attachment.startOfEncoderSampleIndex = first + attachment.endOfEncoderSampleIndex = first + 1 + ring[slot].passes.append(Pass(label: label, firstSample: first, isRenderPass: false)) + ring[slot].nextSample += 2 + lock.unlock() + return commandBuffer.makeComputeCommandEncoder(descriptor: descriptor) + } + + /// A blit encoder for this command buffer, timed as a whole under `label` when the frame + /// is being timed; a plain encoder otherwise. + public func makeBlitEncoder(_ commandBuffer: MTLCommandBuffer, _ label: String) -> MTLBlitCommandEncoder? { + lock.lock() + guard let slot = encoding, ring[slot].nextSample + 2 <= Self.samplesPerFrame else { + lock.unlock() + return commandBuffer.makeBlitCommandEncoder() + } + let first = ring[slot].nextSample + let descriptor = MTLBlitPassDescriptor() + guard let attachment = descriptor.sampleBufferAttachments[0] else { + lock.unlock() + return commandBuffer.makeBlitCommandEncoder() + } + attachment.sampleBuffer = ring[slot].buffer + attachment.startOfEncoderSampleIndex = first + attachment.endOfEncoderSampleIndex = first + 1 + ring[slot].passes.append(Pass(label: label, firstSample: first, isRenderPass: false)) + ring[slot].nextSample += 2 + lock.unlock() + return commandBuffer.makeBlitCommandEncoder(descriptor: descriptor) + } + + // MARK: - Resolution + + private func makeRing(device: MTLDevice) -> Bool { + guard device.supportsCounterSampling(.atStageBoundary), + let counterSet = device.counterSets?.first(where: { $0.name == MTLCommonCounterSet.timestamp.rawValue }) + else { + if !warnedUnsupported { + warnedUnsupported = true + Logger.logWarning(message: "GPU pass timing: this GPU does not sample timestamps at stage boundaries; passes are not timed") + } + enabledFlag = false + return false + } + let descriptor = MTLCounterSampleBufferDescriptor() + descriptor.counterSet = counterSet + descriptor.storageMode = .shared + descriptor.sampleCount = Self.samplesPerFrame + descriptor.label = "GPU pass timing" + var frames: [Frame] = [] + for _ in 0 ..< Self.ringSize { + guard let buffer = try? device.makeCounterSampleBuffer(descriptor: descriptor) else { + Logger.logWarning(message: "GPU pass timing: no counter sample buffer; passes are not timed") + enabledFlag = false + return false + } + frames.append(Frame(buffer: buffer)) + } + ring = frames + return true + } + + /// The GPU clock against the media clock, read once at the start and again after a + /// quarter of a second: GPU timestamps come in ticks whose length the device does not say. + private func calibrate(device: MTLDevice) { + guard secondsPerTick == nil else { return } + let gpu = device.sampleTimestamps().gpu + let media = CACurrentMediaTime() + guard let start = calibrationStart else { + calibrationStart = (media, gpu) + return + } + let elapsed = media - start.media + guard elapsed >= 0.25, gpu > start.gpu else { return } + secondsPerTick = elapsed / Double(gpu - start.gpu) + } + + private func resolve(_ frame: Frame, slot: Int, gpuSeconds: Double) { + defer { + lock.lock() + if slot < ring.count { ring[slot].inFlight = false } + lock.unlock() + } + guard frame.nextSample > 0, + let data = try? frame.buffer.resolveCounterRange(0 ..< frame.nextSample) + else { return } + let stamps: [UInt64] = data.withUnsafeBytes { bytes in + let results = bytes.bindMemory(to: MTLCounterResultTimestamp.self) + return results.map(\.timestamp) + } + guard stamps.count >= frame.nextSample else { return } + + lock.lock() + let secondsPerTick = secondsPerTick + lock.unlock() + // Until the clocks are calibrated, the frame's own span sets the scale. + let valid = stamps.filter { $0 != MTLCounterErrorValue && $0 != 0 } + guard let first = valid.min(), let last = valid.max(), last > first else { return } + let msPerTick = (secondsPerTick ?? (gpuSeconds / Double(last - first))) * 1000.0 + + func span(_ start: Int, _ end: Int) -> Double { + let a = stamps[start], b = stamps[end] + guard a != MTLCounterErrorValue, b != MTLCounterErrorValue, a != 0, b != 0, b >= a else { return 0 } + return Double(b - a) * msPerTick + } + var frameTimes: [String: (vertex: Double, fragment: Double)] = [:] + for pass in frame.passes { + let vertex = pass.isRenderPass ? span(pass.firstSample, pass.firstSample + 1) : 0 + let fragment = pass.isRenderPass + ? span(pass.firstSample + 2, pass.firstSample + 3) + : span(pass.firstSample, pass.firstSample + 1) + let before = frameTimes[pass.label] ?? (0, 0) + frameTimes[pass.label] = (before.vertex + vertex, before.fragment + fragment) + } + + lock.lock() + guard enabledFlag else { + lock.unlock() + return + } + lastFrameTimes = frameTimes + lastFrameIndex &+= 1 + let weight = Self.smoothing + for label in Set(averageTimes.keys).union(frameTimes.keys) { + let now = frameTimes[label] ?? (0, 0) + guard let old = averageTimes[label] else { + averageTimes[label] = now + continue + } + averageTimes[label] = (old.vertex + (now.vertex - old.vertex) * weight, + old.fragment + (now.fragment - old.fragment) * weight) + } + lock.unlock() + } +} diff --git a/Sources/UntoldEngine/Renderer/FrameTime.swift b/Sources/UntoldEngine/Renderer/FrameTime.swift new file mode 100644 index 000000000..d747fedcd --- /dev/null +++ b/Sources/UntoldEngine/Renderer/FrameTime.swift @@ -0,0 +1,127 @@ +// +// FrameTime.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Foundation + +/// When the frame being prepared is predicted to reach the display, and how far that is from +/// the frame before. +/// +/// A headset frame is drawn with the head pose predicted for its presentation instant, so an +/// object posed for `presentationTime` keeps its place against the world while the head moves, +/// and one moving along a path steps by the display's period from one frame to the next +/// instead of by the varying time between two game updates. A frame the display skipped lands +/// where the timeline says, and the frames after it step by one period again. +/// +/// `presentationTime` is in the `CACurrentMediaTime()` base, which `ProcessInfo.systemUptime` +/// and ARKit's anchor timestamps share, so a game may keep its own timelines in that base. +public struct FrameTime: Sendable, Equatable { + /// Seconds at which the frame is predicted to be shown. + public let presentationTime: TimeInterval + /// Seconds from the previous frame's presentation to this one's: one period on a steady + /// cadence, more after a skipped frame, 0 on the first frame. + public let deltaTime: TimeInterval + /// The display's period as the loop knows it: the host's nominal frame interval, or the + /// shortest spacing between presentations seen so far (0 until one has been seen). + public let period: TimeInterval + /// Refreshes between the previous frame and this one at which the display showed the + /// previous frame again because no new one was ready. 0 on a steady cadence. + public let skippedFrames: Int + /// The engine's count of frames prepared, from 1. + public let frameIndex: UInt64 + + public init( + presentationTime: TimeInterval, + deltaTime: TimeInterval, + period: TimeInterval, + skippedFrames: Int, + frameIndex: UInt64 + ) { + self.presentationTime = presentationTime + self.deltaTime = deltaTime + self.period = period + self.skippedFrames = skippedFrames + self.frameIndex = frameIndex + } + + /// Whether the display repeated the previous frame before this one. + public var wasFrameSkipped: Bool { skippedFrames > 0 } + + /// No frame yet: every field zero. + public static let zero = FrameTime(presentationTime: 0, deltaTime: 0, period: 0, skippedFrames: 0, frameIndex: 0) +} + +/// Makes each frame's `FrameTime` from the presentation times a host loop predicts: the delta +/// between consecutive presentations, the period (the host's nominal one, or the shortest +/// spacing seen) and the refreshes skipped between two frames. +public struct FrameClock: Sendable, Equatable { + /// The most recent frame, or nil before the first. + public private(set) var current: FrameTime? + /// The period the host declares (1 / the view's preferred frames per second), or nil to + /// learn it from the presentation times. + public var nominalPeriod: TimeInterval? + private var learnedPeriod: TimeInterval? + private var frameIndex: UInt64 = 0 + + /// A spacing shorter than this is two frames for one refresh, not a period. + static let shortestPeriod: TimeInterval = 1.0 / 250.0 + /// A spacing longer than this is a stall, not a period. + static let longestPeriod: TimeInterval = 1.0 / 24.0 + + public init(nominalPeriod: TimeInterval? = nil) { + self.nominalPeriod = nominalPeriod + } + + /// The display's period as the clock knows it, 0 before it knows one. + public var period: TimeInterval { nominalPeriod ?? learnedPeriod ?? 0 } + + /// Advances to a frame predicted to be shown at `presentationTime` and returns its time. + @discardableResult + public mutating func advance(toPresentation presentationTime: TimeInterval) -> FrameTime { + frameIndex &+= 1 + guard let previous = current else { + let frame = FrameTime(presentationTime: presentationTime, deltaTime: 0, period: period, skippedFrames: 0, frameIndex: frameIndex) + current = frame + return frame + } + + // A presentation before the previous one is a clock going backwards: no time passes. + let delta = max(0, presentationTime - previous.presentationTime) + if nominalPeriod == nil, delta >= Self.shortestPeriod, delta <= Self.longestPeriod { + learnedPeriod = min(learnedPeriod ?? delta, delta) + } + + let period = period + var skipped = 0 + if period > 0, delta > 0 { + // The refreshes the delta spans, less the one every frame takes. + let refreshes = min(delta / period, 1_000_000).rounded() + skipped = max(0, Int(refreshes) - 1) + } + + let frame = FrameTime(presentationTime: presentationTime, deltaTime: delta, period: period, skippedFrames: skipped, frameIndex: frameIndex) + current = frame + return frame + } + + /// Advances to a frame the loop estimates `lead` ahead of `now`, for a host with no + /// predicted presentation time (an MTKView's draw): the delta is then the spacing of the + /// draws on the wall clock. + @discardableResult + public mutating func advance(now: TimeInterval, lead: TimeInterval) -> FrameTime { + advance(toPresentation: now + lead) + } + + /// Forgets every frame: the next one is a first frame again. + public mutating func reset() { + current = nil + learnedPeriod = nil + frameIndex = 0 + } +} diff --git a/Sources/UntoldEngine/Renderer/ObjectMotion.swift b/Sources/UntoldEngine/Renderer/ObjectMotion.swift new file mode 100644 index 000000000..fe349cd6f --- /dev/null +++ b/Sources/UntoldEngine/Renderer/ObjectMotion.swift @@ -0,0 +1,65 @@ +// +// ObjectMotion.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Foundation +import simd + +/// Where each entity was the frame before: the world transform it was drawn with, for the +/// velocity the opaque pass writes per pixel (the temporal resolve finds a moving surface's +/// history where the surface was, not where the camera's motion alone would put it). +public final class ObjectMotion: @unchecked Sendable { + public static let shared = ObjectMotion() + + private struct Entry { + var frame: UInt64 + var previous: simd_float4x4 + var current: simd_float4x4 + } + + private let lock = NSLock() + private var entries: [EntityID: Entry] = [:] + private var frame: UInt64 = 1 + + /// Once per frame, before its passes: a new frame begins. Both eyes of a headset frame + /// draw within the same one, so the second eye sees the same previous transforms. + public func beginFrame() { + lock.lock() + frame &+= 1 + // Entities not drawn for a while are forgotten (they come back as newly seen). + if frame % 600 == 0 { + let stale = frame &- 300 + entries = entries.filter { $0.value.frame >= stale } + } + lock.unlock() + } + + /// The entity's world transform of the previous frame โ€” `current` on its first sighting, + /// so it does not move โ€” and `current` recorded as this frame's for the next. + public func previousWorldTransform(of entityId: EntityID, current: simd_float4x4) -> simd_float4x4 { + lock.lock() + defer { lock.unlock() } + if var entry = entries[entityId] { + if entry.frame != frame { + entry = Entry(frame: frame, previous: entry.current, current: current) + entries[entityId] = entry + } + return entry.previous + } + entries[entityId] = Entry(frame: frame, previous: current, current: current) + return current + } + + /// Forgets every entity: the next frame moves nothing. + public func reset() { + lock.lock() + entries.removeAll() + lock.unlock() + } +} diff --git a/Sources/UntoldEngine/Renderer/Pipelines/RenderPipeLines.swift b/Sources/UntoldEngine/Renderer/Pipelines/RenderPipeLines.swift index 7fc744d59..5d0834162 100644 --- a/Sources/UntoldEngine/Renderer/Pipelines/RenderPipeLines.swift +++ b/Sources/UntoldEngine/Renderer/Pipelines/RenderPipeLines.swift @@ -7,6 +7,7 @@ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +import CShaderTypes import MetalKit public struct RenderPipeline { @@ -255,6 +256,7 @@ public func CreateTilePipeline( tileShader: String, tileShaderLibrary: RenderShaderLibraryReference = .engine, colorFormats: [MTLPixelFormat], + rasterSampleCount: Int = 1, name: String ) -> RenderPipeline? { #if targetEnvironment(simulator) @@ -281,6 +283,7 @@ public func CreateTilePipeline( pipelineDescriptor.label = name pipelineDescriptor.tileFunction = tileFunction pipelineDescriptor.threadgroupSizeMatchesTileSize = true + pipelineDescriptor.rasterSampleCount = max(1, rasterSampleCount) for (index, format) in colorFormats.enumerated() { pipelineDescriptor.colorAttachments[index].pixelFormat = format @@ -296,7 +299,8 @@ public func CreateTilePipeline( pipelineState: pipelineState, depthState: nil, success: true, - name: name + name: name, + rasterSampleCount: max(1, rasterSampleCount) ) } catch { @@ -367,7 +371,24 @@ private var wf: WorkingColorFormats { /// pipeline. Attachments 0-4 are the G-buffer targets written by geometry draws. /// Attachment 5 is the lit output (deferredColorMap) that only the light sub-pass /// writes โ€” geometry must not touch it, so writeMask is disabled for slot 5. +/// The general model pipeline: its fragment function keeps the discards an alpha-masked or +/// blended material or a dither fade needs. public func InitModelPipeline() -> RenderPipeline? { + makeModelPipeline(mayDiscard: true, name: "Model Pipeline") +} + +/// The model pipeline for the opaque draws that never discard, compiled without the +/// discards: the GPU then resolves visibility before it shades, and a fragment another +/// surface covers costs nothing. The pass picks it per draw (`combinedModelLightExecution`). +public func InitModelOpaquePipeline() -> RenderPipeline? { + makeModelPipeline(mayDiscard: false, name: "Model Pipeline (opaque)") +} + +private func makeModelPipeline(mayDiscard: Bool, name: String) -> RenderPipeline? { + let indirectInGBuffer = renderInfo.perFragmentIndirectLighting + let litInGBuffer = renderInfo.perFragmentLightingWanted + renderInfo.perFragmentIndirectLightingBuilt = indirectInGBuffer + renderInfo.perFragmentLightingBuilt = litInGBuffer guard let library = renderInfo.library else { handleError(.metalLibraryNotFound) return nil @@ -376,33 +397,52 @@ public func InitModelPipeline() -> RenderPipeline? { handleError(.shaderCreationFailed, "vertexModelShader") return nil } - guard let fragmentFunction = library.makeFunction(name: "fragmentModelShader") else { - handleError(.shaderCreationFailed, "fragmentModelShader") + let constants = MTLFunctionConstantValues() + var mayDiscardValue = mayDiscard + constants.setConstantValue(&mayDiscardValue, type: .bool, index: Int(modelFunctionConstantMayDiscard.rawValue)) + var indirectValue = indirectInGBuffer + constants.setConstantValue(&indirectValue, type: .bool, index: Int(modelFunctionConstantIndirectInGBuffer.rawValue)) + var litValue = litInGBuffer + constants.setConstantValue(&litValue, type: .bool, index: Int(modelFunctionConstantLitInGBuffer.rawValue)) + let fragmentFunction: MTLFunction + do { + fragmentFunction = try library.makeFunction(name: "fragmentModelShader", constantValues: constants) + } catch { + handleError(.shaderCreationFailed, "fragmentModelShader (\(name)): \(error.localizedDescription)") return nil } let desc = MTLRenderPipelineDescriptor() - desc.label = "Model Pipeline" + desc.label = name desc.vertexFunction = vertexFunction desc.fragmentFunction = fragmentFunction desc.vertexDescriptor = createModelVertexDescriptor() desc.depthAttachmentPixelFormat = renderInfo.depthPixelFormat desc.rasterSampleCount = max(1, renderInfo.opaqueSampleCount) - // G-buffer outputs: geometry writes to these. - let gbufferFormats: [MTLPixelFormat] = [ - wf.gBufferAlbedo, wf.gBufferNormal, wf.gBufferPosition, - wf.gBufferMaterial, wf.gBufferEmissive, - ] - for (i, fmt) in gbufferFormats.enumerated() { - desc.colorAttachments[i].pixelFormat = fmt - desc.colorAttachments[i].writeMask = .all + if litInGBuffer { + // The forward pass: the lit colour alone, in the light pass's slot + // (`forwardRenderPassDescriptor`), and the velocity; the shader's G-buffer outputs + // are compiled out. + desc.colorAttachments[5].pixelFormat = wf.sceneColor + desc.colorAttachments[5].writeMask = .all + desc.colorAttachments[6].pixelFormat = .rg16Float + desc.colorAttachments[6].writeMask = .all + } else { + // G-buffer outputs: geometry writes to these. + let gbufferFormats: [MTLPixelFormat] = [ + wf.gBufferAlbedo, wf.gBufferNormal, wf.gBufferPosition, + wf.gBufferMaterial, wf.gBufferEmissive, + ] + for (i, fmt) in gbufferFormats.enumerated() { + desc.colorAttachments[i].pixelFormat = fmt + desc.colorAttachments[i].writeMask = .all + } + // Attachment 5: deferredColorMap โ€” the light quad writes it; the geometry must not. + desc.colorAttachments[5].pixelFormat = wf.sceneColor + desc.colorAttachments[5].writeMask = [] } - // Attachment 5: deferredColorMap โ€” geometry must not write here. - desc.colorAttachments[5].pixelFormat = wf.sceneColor - desc.colorAttachments[5].writeMask = [] - let depthDesc = MTLDepthStencilDescriptor() depthDesc.depthCompareFunction = sceneDepthCompareFunction(.lessEqual) depthDesc.isDepthWriteEnabled = true @@ -414,11 +454,11 @@ public func InitModelPipeline() -> RenderPipeline? { pipelineState: pipelineState, depthState: depthState, success: true, - name: "Model Pipeline", + name: name, rasterSampleCount: desc.rasterSampleCount ) } catch { - handleError(.pipelineStateCreationFailed, "Model Pipeline") + handleError(.pipelineStateCreationFailed, name) return nil } } @@ -450,6 +490,14 @@ public func InitLightPipeline() -> RenderPipeline? { name: "Light Pipeline (Simulator)" ) #else + return makeLightPipeline(name: "Light Pipeline (TBDR)") + #endif +} + +#if !targetEnvironment(simulator) + /// The TBDR light quad from `fragmentLightShaderTBDR`, specialised by function constant for + /// where the indirect light comes from. + private func makeLightPipeline(name: String) -> RenderPipeline? { guard let library = renderInfo.library else { handleError(.metalLibraryNotFound) return nil @@ -458,13 +506,19 @@ public func InitLightPipeline() -> RenderPipeline? { handleError(.shaderCreationFailed, "vertexLightShader") return nil } - guard let fragmentFunction = library.makeFunction(name: "fragmentLightShaderTBDR") else { - handleError(.shaderCreationFailed, "fragmentLightShaderTBDR") + let constants = MTLFunctionConstantValues() + var indirectValue = renderInfo.perFragmentIndirectLighting + constants.setConstantValue(&indirectValue, type: .bool, index: Int(lightFunctionConstantIndirectFromGBuffer.rawValue)) + let fragmentFunction: MTLFunction + do { + fragmentFunction = try library.makeFunction(name: "fragmentLightShaderTBDR", constantValues: constants) + } catch { + handleError(.shaderCreationFailed, "fragmentLightShaderTBDR (\(name)): \(error.localizedDescription)") return nil } let desc = MTLRenderPipelineDescriptor() - desc.label = "Light Pipeline (TBDR)" + desc.label = name desc.vertexFunction = vertexFunction desc.fragmentFunction = fragmentFunction desc.vertexDescriptor = createLightVertexDescriptor() @@ -495,14 +549,33 @@ public func InitLightPipeline() -> RenderPipeline? { pipelineState: pipelineState, depthState: depthState, success: true, - name: "Light Pipeline (TBDR)", + name: name, rasterSampleCount: desc.rasterSampleCount ) } catch { - handleError(.pipelineStateCreationFailed, "Light Pipeline (TBDR): \(error.localizedDescription)") + handleError(.pipelineStateCreationFailed, "\(name): \(error.localizedDescription)") return nil } - #endif + } +#endif + +/// The TBDR light pass as a tile stage (`lightTileShaderTBDR`): the same inputs and the same +/// attachments as the light quad, dispatched over the tiles of the G-buffer encoder. Nil where +/// tile shading is unavailable (the simulator; GPUs before Apple family 4; a paravirtualized +/// GPU, which compiles the stage's pipeline and renders it black), when the quad runs. +public func InitLightTilePipeline() -> RenderPipeline? { + guard let device = renderInfo.device, device.supportsFamily(.apple4), + !device.name.localizedCaseInsensitiveContains("paravirtual") + else { return nil } + return CreateTilePipeline( + tileShader: "lightTileShaderTBDR", + colorFormats: [ + wf.gBufferAlbedo, wf.gBufferNormal, wf.gBufferPosition, + wf.gBufferMaterial, wf.gBufferEmissive, wf.sceneColor, + ], + rasterSampleCount: max(1, renderInfo.opaqueSampleCount), + name: "Light Tile Pipeline (TBDR)" + ) } // MARK: Geometry pipeline @@ -932,12 +1005,28 @@ public func InitLookPipeline() -> RenderPipeline? { fragmentShader: "fragmentLookShader", vertexDescriptor: createPostProcessVertexDescriptor(), colorFormats: [wf.lookOutput], - depthFormat: renderInfo.depthPixelFormat, + // The look pass draws into its own descriptor, colour alone: no depth to match. + depthFormat: .invalid, depthEnabled: false, name: "Look Pipeline" ) } +/// The look pass that presents as it grades: the look output, the frame encoded for the +/// display and the opaque depth, in one pass, in place of the output transform pass. +public func InitLookPresentPipeline() -> RenderPipeline? { + CreatePipeline( + vertexShader: "vertexLookShader", + fragmentShader: "fragmentLookPresentShader", + vertexDescriptor: createPostProcessVertexDescriptor(), + colorFormats: [wf.lookOutput, renderInfo.presentColorPixelFormat], + depthFormat: renderInfo.presentDepthPixelFormat, + depthCompareFunction: .always, + depthEnabled: true, + name: "Look Pipeline (presents)" + ) +} + public func InitOutputTransformPipeline() -> RenderPipeline? { CreatePipeline( vertexShader: "vertexOutputTransformShader", @@ -963,6 +1052,35 @@ public func InitFXAAPipeline() -> RenderPipeline? { ) } +/// The temporal resolve: the look output and the eye's history into the eye's next history, +/// one colour target of the look format, which the output transform then presents. +public func InitTAAPipeline() -> RenderPipeline? { + CreatePipeline( + vertexShader: "vertexTAAShader", + fragmentShader: "fragmentTAAShader", + vertexDescriptor: createPostProcessVertexDescriptor(), + colorFormats: [renderInfo.colorPipeline.working.lookOutput], + depthFormat: .invalid, + depthEnabled: false, + name: "TAA Pipeline" + ) +} + +/// The temporal resolve that presents as it resolves: the history, the frame encoded for the +/// display and the opaque depth, in one pass, in place of the output transform pass. +public func InitTAAPresentPipeline() -> RenderPipeline? { + CreatePipeline( + vertexShader: "vertexTAAShader", + fragmentShader: "fragmentTAAPresentShader", + vertexDescriptor: createPostProcessVertexDescriptor(), + colorFormats: [renderInfo.colorPipeline.working.lookOutput, renderInfo.presentColorPixelFormat], + depthFormat: renderInfo.presentDepthPixelFormat, + depthCompareFunction: .always, + depthEnabled: true, + name: "TAA Pipeline (presents)" + ) +} + public func InitFXAAEdgeDebugPipeline() -> RenderPipeline? { CreatePipeline( vertexShader: "vertexFXAAShader", @@ -1145,7 +1263,9 @@ public func DefaultPipeLines() -> [(RenderPipelineType, RenderPipelineInitBlock) (.sky, InitSkyPipeline), (.shadow, InitShadowPipeline), (.model, InitModelPipeline), + (.modelOpaque, InitModelOpaquePipeline), (.light, InitLightPipeline), + (.lightTile, InitLightTilePipeline), (.geometry, InitGeometryPipeline), (.highlight, InitHighlightPipeline), (.lightVisual, InitLightVisualPipeline), @@ -1174,8 +1294,11 @@ public func DefaultPipeLines() -> [(RenderPipelineType, RenderPipelineInitBlock) (.gaussianTBDRPostprocess, InitGaussianTBDRPostprocessPipeline), (.spatialDebug, InitSpatialDebugPipeline), (.look, InitLookPipeline), + (.lookPresent, InitLookPresentPipeline), (.fxaa, InitFXAAPipeline), (.fxaaEdgeDebug, InitFXAAEdgeDebugPipeline), + (.taa, InitTAAPipeline), + (.taaPresent, InitTAAPresentPipeline), (.smaaEdges, InitSMAAEdgesPipeline), (.smaaBlendWeights, InitSMAABlendWeightsPipeline), (.smaaNeighborhood, InitSMAANeighborhoodPipeline), diff --git a/Sources/UntoldEngine/Renderer/Pipelines/RenderPipelineType.swift b/Sources/UntoldEngine/Renderer/Pipelines/RenderPipelineType.swift index 0d935727f..2047274d3 100644 --- a/Sources/UntoldEngine/Renderer/Pipelines/RenderPipelineType.swift +++ b/Sources/UntoldEngine/Renderer/Pipelines/RenderPipelineType.swift @@ -25,7 +25,12 @@ public extension RenderPipelineType { static let sky: RenderPipelineType = "sky" static let shadow: RenderPipelineType = "shadow" static let model: RenderPipelineType = "model" + /// The model pipeline compiled without fragment discards, for the opaque draws that never + /// need them: the GPU resolves visibility before shading them. + static let modelOpaque: RenderPipelineType = "modelOpaque" static let light: RenderPipelineType = "light" + /// The TBDR light pass as a tile stage: lights each pixel once per colour it holds. + static let lightTile: RenderPipelineType = "lightTile" static let geometry: RenderPipelineType = "geometry" static let highlight: RenderPipelineType = "highlight" static let lightVisual: RenderPipelineType = "lightVisual" @@ -57,6 +62,11 @@ public extension RenderPipelineType { static let outputTransform: RenderPipelineType = "outputTransform" static let fxaa: RenderPipelineType = "fxaa" static let fxaaEdgeDebug: RenderPipelineType = "fxaaEdgeDebug" + static let taa: RenderPipelineType = "taa" + /// The temporal resolve that also presents the frame (history, display colour, depth). + static let taaPresent: RenderPipelineType = "taaPresent" + /// The look pass that also presents the frame (look output, display colour, depth). + static let lookPresent: RenderPipelineType = "lookPresent" static let smaaEdges: RenderPipelineType = "smaaEdges" static let smaaBlendWeights: RenderPipelineType = "smaaBlendWeights" static let smaaNeighborhood: RenderPipelineType = "smaaNeighborhood" diff --git a/Sources/UntoldEngine/Renderer/PostProcessRenderPasses.swift b/Sources/UntoldEngine/Renderer/PostProcessRenderPasses.swift index 26dad5b94..c7b835648 100644 --- a/Sources/UntoldEngine/Renderer/PostProcessRenderPasses.swift +++ b/Sources/UntoldEngine/Renderer/PostProcessRenderPasses.swift @@ -31,6 +31,7 @@ func executePostProcess(postProcessPipeline: RenderPipeline, uCommandBuffer: MTL let pipelineName: String = postProcessPipeline.name! // set your encoder here + GPUPassTiming.shared.attach(renderPassDescriptor, "Post") guard let renderEncoder = uCommandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else { @@ -94,6 +95,7 @@ func executeIBLPreFilterPass(uCommandBuffer: MTLCommandBuffer, _ envTexture: MTL renderPassDescriptor.colorAttachments[2].storeAction = MTLStoreAction.store // set your encoder here + GPUPassTiming.shared.attach(renderPassDescriptor, "IBL") if let renderEncoder = uCommandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) { renderEncoder.setRenderPipelineState(iblPrefilterPipeline.pipelineState!) @@ -137,6 +139,7 @@ func executeIBLPreFilterPass(uCommandBuffer: MTLCommandBuffer, _ envTexture: MTL renderPassDescriptor.colorAttachments[0].loadAction = .dontCare renderPassDescriptor.colorAttachments[0].storeAction = .store + GPUPassTiming.shared.attach(renderPassDescriptor, "IBL") guard let renderEncoder = uCommandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else { continue } @@ -188,6 +191,33 @@ public func executeXRIBLCubePreFilterPass( return false } + // The probe's harmonics first, for the irradiance (see xrProbeSHProjection). + guard let probeSH = target.probeSH else { + handleError(.renderPassCreationFailed, "XR IBL Cube Pre-Filter Pass: the target has no harmonics buffer") + return false + } + if !xrProbeSHPipeline.success, let library = renderInfo.library { + createComputePipeline( + into: &xrProbeSHPipeline, + device: renderInfo.device, + library: library, + functionName: "xrProbeSHProjection", + pipelineName: "XR Probe SH Projection" + ) + } + guard xrProbeSHPipeline.success, let projectionState = xrProbeSHPipeline.pipelineState, + let projectionEncoder = commandBuffer.makeComputeCommandEncoder() + else { + handleError(.pipelineStateNulled, "XR Probe SH Projection") + return false + } + projectionEncoder.label = "XR Probe SH Projection" + projectionEncoder.setComputePipelineState(projectionState) + projectionEncoder.setTexture(environmentCubeTexture, index: 0) + projectionEncoder.setBuffer(probeSH, offset: 0, index: 0) + projectionEncoder.dispatchThreadgroups(MTLSize(width: 1, height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1)) + projectionEncoder.endEncoding() + let renderPassDescriptor = MTLRenderPassDescriptor() renderPassDescriptor.renderTargetWidth = target.irradianceMap.width renderPassDescriptor.renderTargetHeight = target.irradianceMap.height @@ -204,6 +234,7 @@ public func executeXRIBLCubePreFilterPass( renderPassDescriptor.colorAttachments[2].loadAction = .dontCare renderPassDescriptor.colorAttachments[2].storeAction = .store + GPUPassTiming.shared.attach(renderPassDescriptor, "IBL") guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else { return false } @@ -214,6 +245,7 @@ public func executeXRIBLCubePreFilterPass( renderEncoder.setVertexBuffer(bufferResources.quadVerticesBuffer, offset: 0, index: 0) renderEncoder.setVertexBuffer(bufferResources.quadTexCoordsBuffer, offset: 0, index: 1) renderEncoder.setFragmentTexture(environmentCubeTexture, index: 0) + renderEncoder.setFragmentBuffer(probeSH, offset: 0, index: 1) renderEncoder.drawIndexedPrimitivesTracked( type: .triangle, indexCount: 6, @@ -235,6 +267,7 @@ public func executeXRIBLCubePreFilterPass( mipRenderPassDescriptor.colorAttachments[0].loadAction = .dontCare mipRenderPassDescriptor.colorAttachments[0].storeAction = .store + GPUPassTiming.shared.attach(mipRenderPassDescriptor, "IBL") guard let mipEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: mipRenderPassDescriptor) else { continue } diff --git a/Sources/UntoldEngine/Renderer/RenderExtensionModelSurface.swift b/Sources/UntoldEngine/Renderer/RenderExtensionModelSurface.swift index 83f2deaba..e80f6d21e 100644 --- a/Sources/UntoldEngine/Renderer/RenderExtensionModelSurface.swift +++ b/Sources/UntoldEngine/Renderer/RenderExtensionModelSurface.swift @@ -258,6 +258,7 @@ public extension RenderPassContext { renderEncoder.label = label renderEncoder.pushDebugGroup(label) + applyXRFoveationViewport(renderEncoder) renderEncoder.setRenderPipelineState(pipelineState) renderEncoder.setDepthStencilState(pipeline.depthState) diff --git a/Sources/UntoldEngine/Renderer/RenderExtensions.swift b/Sources/UntoldEngine/Renderer/RenderExtensions.swift index 4a80f0203..259554574 100644 --- a/Sources/UntoldEngine/Renderer/RenderExtensions.swift +++ b/Sources/UntoldEngine/Renderer/RenderExtensions.swift @@ -2239,6 +2239,13 @@ public final class RenderExtensionRegistry: @unchecked Sendable { return renderExtension } + /// Whether any render extension is registered, whatever its state. + public var hasRegisteredExtensions: Bool { + lock.lock() + defer { lock.unlock() } + return !extensionsByID.isEmpty + } + func setRegisteredExtensionOrder(_ orderedIDs: [String]) { lifecycleLock.lock() defer { lifecycleLock.unlock() } diff --git a/Sources/UntoldEngine/Renderer/RenderInitializer.swift b/Sources/UntoldEngine/Renderer/RenderInitializer.swift index b7e38243d..f8744a75d 100644 --- a/Sources/UntoldEngine/Renderer/RenderInitializer.swift +++ b/Sources/UntoldEngine/Renderer/RenderInitializer.swift @@ -118,11 +118,15 @@ func requestedOpaqueSampleCount() -> Int { return 1 #else switch antiAliasingMode { - case .msaa: - let preferredSampleCount = 4 + case .msaa, .msaaTaa: + // The multisample count asked for, or the largest below it the device has. guard let device = renderInfo.device else { return 1 } - return device.supportsTextureSampleCount(preferredSampleCount) ? preferredSampleCount : 1 - case .none, .fxaa, .smaa: + var sampleCount = renderInfo.preferredMultisampleCount + while sampleCount > 1, !device.supportsTextureSampleCount(sampleCount) { + sampleCount /= 2 + } + return max(1, sampleCount) + case .none, .fxaa, .smaa, .taa: return 1 } #endif @@ -141,9 +145,15 @@ func updateOpaquePipelinesForSampleCount() { if let modelPipeline = InitModelPipeline() { PipelineManager.shared.update(rendererPipeLine: modelPipeline, forType: .model) } + if let modelOpaquePipeline = InitModelOpaquePipeline() { + PipelineManager.shared.update(rendererPipeLine: modelOpaquePipeline, forType: .modelOpaque) + } if let lightPipeline = InitLightPipeline() { PipelineManager.shared.update(rendererPipeLine: lightPipeline, forType: .light) } + if let lightTilePipeline = InitLightTilePipeline() { + PipelineManager.shared.update(rendererPipeLine: lightTilePipeline, forType: .lightTile) + } } func updateOpaqueSampleCountForCurrentState() { @@ -156,6 +166,15 @@ func updateOpaqueSampleCountForCurrentState() { } let newSampleCount = effectiveOpaqueSampleCount() + // A per-fragment lighting setting changed: the model and light pipelines are built for + // one value of each. + if renderInfo.perFragmentIndirectLightingBuilt != renderInfo.perFragmentIndirectLighting + || renderInfo.perFragmentLightingBuilt != renderInfo.perFragmentLightingWanted, + renderInfo.opaqueSampleCount == newSampleCount + { + updateOpaquePipelinesForSampleCount() + return + } guard renderInfo.opaqueSampleCount != newSampleCount else { return } renderInfo.opaqueSampleCount = newSampleCount @@ -494,6 +513,40 @@ func initRenderPassDescriptors() { renderInfo.offscreenRenderPassDescriptor.depthAttachment.resolveTexture = textureResources.depthMap } + // The forward pass's descriptor (the geometry lighting itself per fragment): the lit + // colour alone, in the light pass's slot, and the depth, resolved as above. + let forwardDescriptor = MTLRenderPassDescriptor() + forwardDescriptor.renderTargetWidth = Int(renderInfo.viewPort.x) + forwardDescriptor.renderTargetHeight = Int(renderInfo.viewPort.y) + configureAttachment( + descriptor: forwardDescriptor.colorAttachments[5], + texture: opaqueColorTexture, + loadAction: .clear, + storeAction: usesMSAA ? .multisampleResolve : .store, + clearColor: MTLClearColorMake(0.0, 0.0, 0.0, 0.0) + ) + configureAttachment( + descriptor: forwardDescriptor.depthAttachment, + texture: opaqueDepthTexture, + loadAction: .clear, + storeAction: usesMSAA ? .multisampleResolve : .store, + clearDepth: sceneDepthClearValue() + ) + // Its velocity, attachment 6 (the lit colour keeps the light pass's slot, 5). + configureAttachment( + descriptor: forwardDescriptor.colorAttachments[6], + texture: usesMSAA ? textureResources.msaaVelocityMap : textureResources.velocityMap, + loadAction: .clear, + storeAction: usesMSAA ? .multisampleResolve : .store, + clearColor: MTLClearColorMake(0.0, 0.0, 0.0, 0.0) + ) + if usesMSAA { + forwardDescriptor.colorAttachments[5].resolveTexture = textureResources.deferredColorMap + forwardDescriptor.colorAttachments[6].resolveTexture = textureResources.velocityMap + forwardDescriptor.depthAttachment.resolveTexture = textureResources.depthMap + } + renderInfo.forwardRenderPassDescriptor = forwardDescriptor + // Deferred Render Pass renderInfo.deferredRenderPassDescriptor = createRenderPassDescriptor( width: Int(renderInfo.viewPort.x), @@ -534,6 +587,15 @@ func initRenderPassDescriptors() { depthAttachment: (textureResources.depthMap, .dontCare, .store, sceneDepthClearValue()) ) + // The look pass: its colour alone (see RenderInfo.lookRenderPassDescriptor). + renderInfo.lookRenderPassDescriptor = createRenderPassDescriptor( + width: Int(renderInfo.viewPort.x), + height: Int(renderInfo.viewPort.y), + colorAttachments: [ + (textureResources.lookTexture, .clear, .store, MTLClearColorMake(0.0, 0.0, 0.0, 0.0)), + ] + ) + // Gizmo Render Pass renderInfo.gizmoRenderPassDescriptor = createRenderPassDescriptor( width: Int(renderInfo.viewPort.x), @@ -795,6 +857,32 @@ func initTextureResources() { textureResources.msaaDepthMap = nil } + // The velocity the opaque pass writes per pixel in its forward mode, for the temporal + // resolve: the surface's motion since the previous frame, in NDC. + textureResources.velocityMap = createTexture( + device: renderInfo.device, + label: "Velocity Texture", + pixelFormat: .rg16Float, + width: viewportWidth, + height: viewportHeight, + usage: [.shaderRead, .renderTarget], + storageMode: .private + ) + if renderInfo.opaqueSampleCount > 1 { + textureResources.msaaVelocityMap = createTexture( + device: renderInfo.device, + label: "MSAA Velocity Texture", + pixelFormat: .rg16Float, + width: viewportWidth, + height: viewportHeight, + usage: [.renderTarget], + storageMode: .memoryless, + sampleCount: renderInfo.opaqueSampleCount + ) + } else { + textureResources.msaaVelocityMap = nil + } + // HZB uses a dedicated source depth so wireframe channels can contribute // filled occluder depth without changing visible scene color. textureResources.hzbSourceDepthMap = createTexture( @@ -859,7 +947,7 @@ func initTextureResources() { pixelFormat: wf.sceneColor, width: viewportWidth, height: viewportHeight, - usage: [.shaderRead, .renderTarget, .shaderWrite], + usage: [.shaderRead, .renderTarget], // no shaderWrite: it would forbid the GPU's lossless compression storageMode: .shared ) @@ -942,7 +1030,7 @@ func initTextureResources() { pixelFormat: wf.environment, width: viewportWidth, height: viewportHeight, - usage: [.shaderRead, .renderTarget, .shaderWrite], + usage: [.shaderRead, .renderTarget], // no shaderWrite: it would forbid the GPU's lossless compression storageMode: .shared ) @@ -963,7 +1051,7 @@ func initTextureResources() { pixelFormat: wf.sceneComposite, width: viewportWidth, height: viewportHeight, - usage: [.shaderRead, .renderTarget, .shaderWrite], + usage: [.shaderRead, .renderTarget], // no shaderWrite: it would forbid the GPU's lossless compression storageMode: .shared ) @@ -973,7 +1061,7 @@ func initTextureResources() { pixelFormat: wf.lookOutput, width: viewportWidth, height: viewportHeight, - usage: [.shaderRead, .renderTarget, .shaderWrite], + usage: [.shaderRead, .renderTarget], // no shaderWrite: it would forbid the GPU's lossless compression storageMode: .shared ) @@ -983,7 +1071,7 @@ func initTextureResources() { pixelFormat: wf.lookOutput, width: viewportWidth, height: viewportHeight, - usage: [.shaderRead, .renderTarget, .shaderWrite], + usage: [.shaderRead, .renderTarget], // no shaderWrite: it would forbid the GPU's lossless compression storageMode: .shared ) diff --git a/Sources/UntoldEngine/Renderer/RenderPasses+MuscleDebug.swift b/Sources/UntoldEngine/Renderer/RenderPasses+MuscleDebug.swift index 3cdeef819..be1ff968a 100644 --- a/Sources/UntoldEngine/Renderer/RenderPasses+MuscleDebug.swift +++ b/Sources/UntoldEngine/Renderer/RenderPasses+MuscleDebug.swift @@ -155,6 +155,7 @@ public extension RenderPasses { } renderEncoder.label = "Muscle Debug Pass" renderEncoder.pushDebugGroup("Muscle Debug Pass") + applyXRFoveationViewport(renderEncoder) renderEncoder.setRenderPipelineState(pipelineState) // The cages live under the skin, so never depth-test them. In visionOS // mixed immersion the compositor drops pixels that wrote no depth, so @@ -164,7 +165,7 @@ public extension RenderPasses { ) { renderEncoder.setDepthStencilState(depthState) } - renderEncoder.waitForFence(renderInfo.fence, before: .vertex) + renderEncoder.waitForFence(renderInfo.fence, before: .fragment) renderEncoder.setCullMode(.none) var viewMatrix = SceneRootTransform.shared.effectiveViewMatrix(cameraComponent.viewSpace) diff --git a/Sources/UntoldEngine/Renderer/RenderPasses.swift b/Sources/UntoldEngine/Renderer/RenderPasses.swift index 8256831bf..3401a3d78 100644 --- a/Sources/UntoldEngine/Renderer/RenderPasses.swift +++ b/Sources/UntoldEngine/Renderer/RenderPasses.swift @@ -101,7 +101,16 @@ public enum RenderPasses { return shouldLog } + /// The depth bias of a glass draw. A pane modelled into its frame shares a plane with + /// the frame and fights it for depth along its edges at every move of the camera, so + /// glass draws a few units of depth resolution behind where it is, and two more per + /// unit of slope, and the frame wins wherever the two coincide. The sign follows the + /// depth convention: with reverse Z the near plane is 1 and away is down. + private static let glassDepthBias: Float = 4 + private static let glassDepthSlopeScale: Float = 2 + @inline(__always) + private static func getOrCreateTransparencyXRDepthWriteState(device: MTLDevice) -> MTLDepthStencilState? { runtimeState.lock.lock() if let cached = runtimeState.transparencyXRDepthWriteState { @@ -281,6 +290,10 @@ public enum RenderPasses { let viewMatrix: simd_float4x4 let cameraPosition: simd_float3 let projectionMatrix = renderInfo.perspectiveSpace + /// For the velocity each draw writes: this frame's view-projection without the jitter + /// and the previous frame's, of the eye being drawn. + let unjitteredViewProjection: simd_float4x4 + let prevViewProjection: simd_float4x4 let pomQuality = RenderPasses.currentPOMQualityUniform() let batchingEnabled = BatchingSystem.shared.isEnabled() let lodFadesEnabled = LODConfig.shared.enableFadeTransitions @@ -292,6 +305,8 @@ public enum RenderPasses { init(cameraComponent: CameraComponent) { viewMatrix = SceneRootTransform.shared.effectiveViewMatrix(cameraComponent.viewSpace) cameraPosition = shadingCameraPosition(cameraComponent) + unjitteredViewProjection = simd_mul(TemporalAntiAliasing.shared.unjitteredProjection, viewMatrix) + prevViewProjection = TemporalAntiAliasing.shared.previousViewProjection(fallback: unjitteredViewProjection) } /// The entity's LOD component while LOD fades are on; nil otherwise, and for an @@ -334,6 +349,14 @@ public enum RenderPasses { ] } + /// Whether a draw with these material parameters can never discard a fragment โ€” an opaque + /// material, neither masked nor blended, with no dither fade running โ€” and so goes through + /// the model pipeline compiled without discards. + @inline(__always) + private static func modelDrawNeverDiscards(_ parameters: MaterialParametersUniform) -> Bool { + parameters.alphaMode == 0 && parameters.lodDither.y <= 0.5 + } + @inline(__always) private static func applyLODDither( draw: LODDitherDraw, @@ -512,6 +535,8 @@ public enum RenderPasses { } public static let copyOpaqueDepthForHZBExecution: RenderPassExecution = { commandBuffer in + // The copy feeds the depth pyramid alone; no pyramid, no copy. + guard HZBOcclusionCulling.pyramidWanted() else { return } guard let sourceDepth = textureResources.depthMap, let hzbSourceDepth = textureResources.hzbSourceDepthMap else { return } @@ -520,7 +545,7 @@ public enum RenderPasses { let height = min(sourceDepth.height, hzbSourceDepth.height) guard width > 0, height > 0 else { return } - guard let blitEncoder = commandBuffer.makeBlitCommandEncoder() else { return } + guard let blitEncoder = GPUPassTiming.shared.makeBlitEncoder(commandBuffer, "DepthCopy") else { return } blitEncoder.label = "Copy Opaque Depth for HZB" blitEncoder.copy( from: sourceDepth, @@ -534,6 +559,66 @@ public enum RenderPasses { blitEncoder.endEncoding() } + // MARK: - Tile light stage + + /// Pipeline states Metal refused a pass for (their imageblock did not fit the GPU's tile + /// memory at the pass's sample count); the quad lights those from then on. + private nonisolated(unsafe) static var refusedTileLightStates: [ObjectIdentifier] = [] + private nonisolated(unsafe) static var loggedTileLightStage: ObjectIdentifier? + + /// The smallest tile Metal lays a pass with an imageblock out in, in pixels (16ร—16; the + /// pass's tile is whatever larger size Metal picks). An imageblock that does not fit the + /// GPU's tile memory even at this size cannot be asked for: Metal refuses the encoder, and + /// under API validation it asserts instead. + private static let smallestTilePixels = 16 * 16 + + /// The tile light stage's pipeline for a combined pass, when it runs, with the descriptor + /// told the imageblock's bytes per sample; the tile size is left to Metal, which picks one + /// the imageblock fits (the encoder reports it). Nil runs the quad: the stage is off, has + /// no pipeline (the simulator, a GPU without tile shading), its imageblock at the pass's + /// sample count exceeds the tile memory, or Metal refused it before. + static func tileLightStage(for descriptor: MTLRenderPassDescriptor) -> MTLRenderPipelineState? { + descriptor.tileWidth = 0 + descriptor.tileHeight = 0 + descriptor.imageblockSampleLength = 0 + #if targetEnvironment(simulator) + return nil + #else + guard renderInfo.tileLightStageEnabled, + !renderInfo.perFragmentIndirectLightingBuilt, // the stage would light the indirect part again + !renderInfo.perFragmentLightingBuilt, // the geometry lit itself; nothing for a stage to do + let pipeline = PipelineManager.shared.renderPipelinesByType[.lightTile], + pipeline.success, + let state = pipeline.pipelineState, + !refusedTileLightStates.contains(ObjectIdentifier(state)) + else { + return nil + } + // Metal lays the imageblock out in 16-byte strides per sample. + let bytesPerSample = (state.imageblockSampleLength + 15) / 16 * 16 + let tileMemoryNeeded = bytesPerSample * max(1, renderInfo.opaqueSampleCount) * smallestTilePixels + guard tileMemoryNeeded <= renderInfo.device.maxThreadgroupMemoryLength else { + refuseTileLightStage(state, reason: "the stage's imageblock needs \(tileMemoryNeeded) bytes in a 16ร—16 tile, the GPU's tile memory holds \(renderInfo.device.maxThreadgroupMemoryLength)") + return nil + } + descriptor.imageblockSampleLength = state.imageblockSampleLength + return state + #endif + } + + /// Records that a pass with this pipeline's imageblock cannot run, once. + static func refuseTileLightStage(_ state: MTLRenderPipelineState, reason: String = "Metal refused the pass with the stage's imageblock") { + refusedTileLightStates.append(ObjectIdentifier(state)) + Logger.logWarning(message: "Tile light stage: \(reason) (\(state.imageblockSampleLength) bytes per sample, \(renderInfo.opaqueSampleCount) sample(s)); lighting with the quad") + } + + /// Logs the tile size Metal chose, once per pipeline state. + static func logTileLightStage(_ state: MTLRenderPipelineState, encoder: MTLRenderCommandEncoder) { + guard loggedTileLightStage != ObjectIdentifier(state) else { return } + loggedTileLightStage = ObjectIdentifier(state) + print("โœ“ Tile light stage: \(encoder.tileWidth)ร—\(encoder.tileHeight) tiles, \(state.imageblockSampleLength) bytes per sample, \(renderInfo.opaqueSampleCount) sample(s)") + } + // MARK: - Shadow entity cache /// Guards one-time subscription setup for the shadow entity cache. @@ -1054,6 +1139,7 @@ public enum RenderPasses { encoderDescriptor.colorAttachments[0].storeAction = MTLStoreAction.store encoderDescriptor.colorAttachments[0].loadAction = loadAction + GPUPassTiming.shared.attach(encoderDescriptor, "Grid") guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: encoderDescriptor) else { handleError(.renderPassCreationFailed, "Grid Pass") @@ -1067,6 +1153,7 @@ public enum RenderPasses { } renderEncoder.label = "Grid Pass" + applyXRFoveationViewport(renderEncoder) renderEncoder.pushDebugGroup("Grid Pass") @@ -1145,6 +1232,7 @@ public enum RenderPasses { encoderDescriptor.colorAttachments[0].storeAction = MTLStoreAction.store encoderDescriptor.colorAttachments[0].loadAction = MTLLoadAction.clear + GPUPassTiming.shared.attach(encoderDescriptor, "Sky") guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: encoderDescriptor) else { handleError(.renderPassCreationFailed, "Sky Pass") @@ -1158,6 +1246,7 @@ public enum RenderPasses { } renderEncoder.label = "Sky Pass" + applyXRFoveationViewport(renderEncoder) renderEncoder.pushDebugGroup("Sky Pass") @@ -1215,6 +1304,7 @@ public enum RenderPasses { encoderDescriptor.colorAttachments[0].storeAction = MTLStoreAction.store encoderDescriptor.colorAttachments[0].loadAction = MTLLoadAction.clear + GPUPassTiming.shared.attach(encoderDescriptor, "Environment") guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: encoderDescriptor) else { handleError(.renderPassCreationFailed, "Environment Pass") @@ -1228,6 +1318,7 @@ public enum RenderPasses { } renderEncoder.label = "Environment Pass" + applyXRFoveationViewport(renderEncoder) renderEncoder.pushDebugGroup("Environment Pass") @@ -1324,13 +1415,14 @@ public enum RenderPasses { let projectionMatrix = renderInfo.perspectiveSpace // Render each cascade into its own depth array slice. - for cascadeIdx in 0 ..< csmCascadeCount { + for cascadeIdx in 0 ..< shadowSystem.cascadeCount { guard cascadeIdx < renderInfo.csmRenderPassDescriptors.count else { break } let descriptor = renderInfo.csmRenderPassDescriptors[cascadeIdx] descriptor.depthAttachment.loadAction = .clear descriptor.depthAttachment.storeAction = .store descriptor.depthAttachment.clearDepth = 1.0 + GPUPassTiming.shared.attach(descriptor, "Shadows") guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: descriptor) else { handleError(.renderPassCreationFailed, "Shadow Cascade \(cascadeIdx)") continue @@ -1340,7 +1432,7 @@ public enum RenderPasses { renderEncoder.pushDebugGroup("Shadow Cascade \(cascadeIdx)") renderEncoder.setRenderPipelineState(shadowPipeline.pipelineState!) renderEncoder.setDepthStencilState(shadowPipeline.depthState!) - renderEncoder.waitForFence(renderInfo.fence, before: .vertex) + renderEncoder.waitForFence(renderInfo.fence, before: .fragment) renderEncoder.setDepthBias(0.0015, slopeScale: 0.75, clamp: 0.02) renderEncoder.setViewport( MTLViewport(originX: 0, originY: 0, @@ -1436,13 +1528,14 @@ public enum RenderPasses { batchUniforms.cameraPosition = shadingCameraPosition(cameraComponent) batchUniforms.projectionMatrix = renderInfo.perspectiveSpace - for cascadeIdx in 0 ..< csmCascadeCount { + for cascadeIdx in 0 ..< shadowSystem.cascadeCount { guard cascadeIdx < renderInfo.csmRenderPassDescriptors.count else { break } // Batched geometry adds into the existing cascade depth slice (already written by shadowExecution). let descriptor = renderInfo.csmRenderPassDescriptors[cascadeIdx] descriptor.depthAttachment.loadAction = .load + GPUPassTiming.shared.attach(descriptor, "Shadows") guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: descriptor) else { handleError(.renderPassCreationFailed, "Batched Shadow Cascade \(cascadeIdx)") continue @@ -1452,7 +1545,7 @@ public enum RenderPasses { renderEncoder.pushDebugGroup("Batched Shadow Cascade \(cascadeIdx)") renderEncoder.setRenderPipelineState(shadowPipeline.pipelineState!) renderEncoder.setDepthStencilState(shadowPipeline.depthState!) - renderEncoder.waitForFence(renderInfo.fence, before: .vertex) + renderEncoder.waitForFence(renderInfo.fence, before: .fragment) renderEncoder.setDepthBias(0.0015, slopeScale: 0.75, clamp: 0.02) renderEncoder.setViewport( MTLViewport(originX: 0, originY: 0, @@ -1531,6 +1624,7 @@ public enum RenderPasses { return } + GPUPassTiming.shared.attach(descriptor, "SpotShadow") guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: descriptor) else { handleError(.renderPassCreationFailed, "Spot Shadow") return @@ -1540,7 +1634,7 @@ public enum RenderPasses { renderEncoder.pushDebugGroup("Spot Shadow") renderEncoder.setRenderPipelineState(shadowPipeline.pipelineState!) renderEncoder.setDepthStencilState(shadowPipeline.depthState!) - renderEncoder.waitForFence(renderInfo.fence, before: .vertex) + renderEncoder.waitForFence(renderInfo.fence, before: .fragment) renderEncoder.setDepthBias(0.0015, slopeScale: 0.75, clamp: 0.02) renderEncoder.setViewport( MTLViewport(originX: 0, originY: 0, @@ -1670,6 +1764,7 @@ public enum RenderPasses { descriptor.depthAttachment.clearDepth = 1.0 descriptor.depthAttachment.slice = face + GPUPassTiming.shared.attach(descriptor, "PointShadow") guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: descriptor) else { handleError(.renderPassCreationFailed, "Point Shadow") return @@ -1679,7 +1774,7 @@ public enum RenderPasses { renderEncoder.pushDebugGroup("Point Shadow Face \(face)") renderEncoder.setRenderPipelineState(shadowPipeline.pipelineState!) renderEncoder.setDepthStencilState(shadowPipeline.depthState!) - renderEncoder.waitForFence(renderInfo.fence, before: .vertex) + renderEncoder.waitForFence(renderInfo.fence, before: .fragment) renderEncoder.setDepthBias(0.002, slopeScale: 0.9, clamp: 0.03) renderEncoder.setViewport( MTLViewport(originX: 0, originY: 0, @@ -1783,36 +1878,39 @@ public enum RenderPasses { return } - guard let encoderDescriptor = renderInfo.offscreenRenderPassDescriptor else { + guard let encoderDescriptor = renderInfo.opaquePassDescriptor else { handleError(.renderPassCreationFailed, "Offscreen render pass descriptor not initialized") return } - encoderDescriptor.colorAttachments[Int(colorTarget.rawValue)] - .loadAction = .clear - encoderDescriptor.colorAttachments[Int(normalTarget.rawValue)] - .loadAction = .clear - encoderDescriptor.colorAttachments[Int(positionTarget.rawValue)] - .loadAction = .clear - encoderDescriptor.colorAttachments[Int(materialTarget.rawValue)].loadAction = .clear - encoderDescriptor.colorAttachments[Int(emissiveTarget.rawValue)].loadAction = .clear + if !renderInfo.perFragmentLightingBuilt { + encoderDescriptor.colorAttachments[Int(colorTarget.rawValue)] + .loadAction = .clear + encoderDescriptor.colorAttachments[Int(normalTarget.rawValue)] + .loadAction = .clear + encoderDescriptor.colorAttachments[Int(positionTarget.rawValue)] + .loadAction = .clear + encoderDescriptor.colorAttachments[Int(materialTarget.rawValue)].loadAction = .clear + encoderDescriptor.colorAttachments[Int(emissiveTarget.rawValue)].loadAction = .clear - encoderDescriptor.colorAttachments[Int(colorTarget.rawValue)] - .storeAction = .store + encoderDescriptor.colorAttachments[Int(colorTarget.rawValue)] + .storeAction = .store - encoderDescriptor.colorAttachments[Int(normalTarget.rawValue)] - .storeAction = .store - encoderDescriptor.colorAttachments[Int(positionTarget.rawValue)] - .storeAction = .store - encoderDescriptor.colorAttachments[Int(materialTarget.rawValue)] - .storeAction = .store - encoderDescriptor.colorAttachments[Int(emissiveTarget.rawValue)] - .storeAction = .store + encoderDescriptor.colorAttachments[Int(normalTarget.rawValue)] + .storeAction = .store + encoderDescriptor.colorAttachments[Int(positionTarget.rawValue)] + .storeAction = .store + encoderDescriptor.colorAttachments[Int(materialTarget.rawValue)] + .storeAction = .store + encoderDescriptor.colorAttachments[Int(emissiveTarget.rawValue)] + .storeAction = .store + } encoderDescriptor.depthAttachment.storeAction = .store encoderDescriptor.depthAttachment.loadAction = .clear encoderDescriptor.depthAttachment.clearDepth = sceneDepthClearValue() + GPUPassTiming.shared.attach(encoderDescriptor, "Model") guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: encoderDescriptor) else { handleError(.renderPassCreationFailed, "Model Pass") @@ -1826,13 +1924,14 @@ public enum RenderPasses { } renderEncoder.label = "Model Pass" + applyXRFoveationViewport(renderEncoder) renderEncoder.pushDebugGroup("Model Pass") renderEncoder.setRenderPipelineState(modelPipeline.pipelineState!) renderEncoder.setDepthStencilState(modelPipeline.depthState) - renderEncoder.waitForFence(renderInfo.fence, before: .vertex) + renderEncoder.waitForFence(renderInfo.fence, before: .fragment) // What every draw of the pass shares, read once. let pass = OpaquePassState(cameraComponent: cameraComponent) @@ -2068,19 +2167,22 @@ public enum RenderPasses { return } - guard let encoderDescriptor = renderInfo.offscreenRenderPassDescriptor else { + guard let encoderDescriptor = renderInfo.opaquePassDescriptor else { handleError(.renderPassCreationFailed, "Offscreen render pass descriptor not initialized") return } // Reuse existing offscreen descriptor (already configured by modelExecution) - encoderDescriptor.colorAttachments[Int(colorTarget.rawValue)].loadAction = .load - encoderDescriptor.colorAttachments[Int(normalTarget.rawValue)].loadAction = .load - encoderDescriptor.colorAttachments[Int(positionTarget.rawValue)].loadAction = .load - encoderDescriptor.colorAttachments[Int(materialTarget.rawValue)].loadAction = .load - encoderDescriptor.colorAttachments[Int(emissiveTarget.rawValue)].loadAction = .load + if !renderInfo.perFragmentLightingBuilt { + encoderDescriptor.colorAttachments[Int(colorTarget.rawValue)].loadAction = .load + encoderDescriptor.colorAttachments[Int(normalTarget.rawValue)].loadAction = .load + encoderDescriptor.colorAttachments[Int(positionTarget.rawValue)].loadAction = .load + encoderDescriptor.colorAttachments[Int(materialTarget.rawValue)].loadAction = .load + encoderDescriptor.colorAttachments[Int(emissiveTarget.rawValue)].loadAction = .load + } encoderDescriptor.depthAttachment.loadAction = .load + GPUPassTiming.shared.attach(encoderDescriptor, "Model") guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: encoderDescriptor) else { handleError(.renderPassCreationFailed, "Batched Model Pass") @@ -2093,11 +2195,12 @@ public enum RenderPasses { } renderEncoder.label = "Batched Model Pass" + applyXRFoveationViewport(renderEncoder) renderEncoder.pushDebugGroup("Batched Model Pass") renderEncoder.setRenderPipelineState(modelPipeline.pipelineState!) renderEncoder.setDepthStencilState(modelPipeline.depthState) - renderEncoder.waitForFence(renderInfo.fence, before: .vertex) + renderEncoder.waitForFence(renderInfo.fence, before: .fragment) // Create identity uniform (batched vertices are already in world space) var batchUniforms = Uniforms() @@ -2289,11 +2392,25 @@ public enum RenderPasses { return } - guard let encoderDescriptor = renderInfo.offscreenRenderPassDescriptor else { + // The forward descriptor when the geometry lights itself per fragment (the lit colour + // and the depth alone), the G-buffer's otherwise: whichever the pipelines were built for. + let forwardPass = renderInfo.perFragmentLightingBuilt + guard let encoderDescriptor = renderInfo.opaquePassDescriptor else { handleError(.renderPassCreationFailed, "Combined G-buffer + Light Pass descriptor not initialized") return } + // The model pipeline compiled without discards, for the draws that never need them (an + // opaque material, no fade running): the GPU resolves their visibility before shading. + // Nil runs every draw through the general pipeline. + let plainModelPipeline: RenderPipeline? = renderInfo.opaqueHiddenSurfaceRemovalEnabled + ? PipelineManager.shared.renderPipelinesByType[.modelOpaque].flatMap { $0.success && $0.pipelineState != nil ? $0 : nil } + : nil + renderInfo.opaqueDrawsPlain = 0 + renderInfo.opaqueDrawsDiscarding = 0 + // Which of the two the encoder holds; the general one is bound first below. + var plainBound = false + // G-buffer slots (0-4): cleared at start, normally discarded at end. // Lit output (5): cleared at start, stored for downstream passes. // The simulator always stores the G-buffer: its light pass samples it as @@ -2303,9 +2420,11 @@ public enum RenderPasses { #else let gBufferStoreAction: MTLStoreAction = renderInfo.gBufferDebugStorageEnabled ? .store : .dontCare #endif - for i in 0 ..< 5 { - encoderDescriptor.colorAttachments[i].loadAction = .clear - encoderDescriptor.colorAttachments[i].storeAction = gBufferStoreAction + if !forwardPass { + for i in 0 ..< 5 { + encoderDescriptor.colorAttachments[i].loadAction = .clear + encoderDescriptor.colorAttachments[i].storeAction = gBufferStoreAction + } } encoderDescriptor.colorAttachments[5].loadAction = .clear encoderDescriptor.colorAttachments[5].storeAction = renderInfo.opaqueSampleCount > 1 ? .multisampleResolve : .store @@ -2313,10 +2432,28 @@ public enum RenderPasses { encoderDescriptor.depthAttachment.storeAction = renderInfo.opaqueSampleCount > 1 ? .multisampleResolve : .store encoderDescriptor.depthAttachment.clearDepth = sceneDepthClearValue() - guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: encoderDescriptor) else { + // The light pass as a tile stage, when it runs: its pipeline, with the descriptor told + // the imageblock's size before the encoder exists. A pass Metal refuses with it (the + // imageblock does not fit the GPU's tile memory at this sample count) runs the quad, + // this frame and after. + var tileLightState = forwardPass ? nil : tileLightStage(for: encoderDescriptor) + GPUPassTiming.shared.attach(encoderDescriptor, forwardPass ? "Forward" : "G-Buffer") + var encoder = commandBuffer.makeRenderCommandEncoder(descriptor: encoderDescriptor) + if encoder == nil, let refused = tileLightState { + refuseTileLightStage(refused) + tileLightState = nil + encoderDescriptor.imageblockSampleLength = 0 + encoder = commandBuffer.makeRenderCommandEncoder(descriptor: encoderDescriptor) + } + guard let renderEncoder = encoder else { + renderInfo.tileLightStageUsed = false handleError(.renderPassCreationFailed, "Combined G-buffer + Light Pass") return } + renderInfo.tileLightStageUsed = tileLightState != nil + if let tileLightState { + logTileLightStage(tileLightState, encoder: renderEncoder) + } // Both paths end this encoder explicitly (the simulator path ends it // mid-function and continues in a second one), so the defer is a @@ -2331,8 +2468,12 @@ public enum RenderPasses { } } - renderEncoder.label = "G-Buffer + Light Pass (TBDR)" - renderEncoder.pushDebugGroup("G-Buffer + Light Pass (TBDR)") + renderEncoder.label = forwardPass ? "Forward Pass" : "G-Buffer + Light Pass (TBDR)" + renderEncoder.pushDebugGroup(forwardPass ? "Forward Pass" : "G-Buffer + Light Pass (TBDR)") + // Through the eye's rasterization rate map when there is one (XR foveation): the + // geometry in the map's screen space, the light quad over the whole of it, the + // G-buffer read back from the tile under each fragment either way. + applyXRFoveationViewport(renderEncoder) // What every draw of the pass shares, read once. let pass = OpaquePassState(cameraComponent: cameraComponent) @@ -2340,10 +2481,61 @@ public enum RenderPasses { let viewMatrix = pass.viewMatrix var renderModes = SceneChannelRenderModeMemo() + // The light pass's inputs, which the geometry shares when it lights per fragment: the + // indirect light and the sun's shadow then go into the G-buffer once per fragment and + // the quad adds them from there instead of computing them once per sample. + var csmUniforms = shadowSystem.makeUniforms() + csmUniforms.cameraViewMatrix = viewMatrix + csmUniforms.lightSpaceMatrices.0 = SceneRootTransform.shared.effectiveLightMatrix(csmUniforms.lightSpaceMatrices.0) + csmUniforms.lightSpaceMatrices.1 = SceneRootTransform.shared.effectiveLightMatrix(csmUniforms.lightSpaceMatrices.1) + csmUniforms.lightSpaceMatrices.2 = SceneRootTransform.shared.effectiveLightMatrix(csmUniforms.lightSpaceMatrices.2) + let environmentLighting = resolveCurrentEnvironmentLighting() + var lightParams = getDirectionalLightParameters() + var brdfParameters = IBLParamsUniform() + brdfParameters.applyIBL = environmentLighting.applyIBL + brdfParameters.ambientIntensity = environmentLighting.ambientIntensity + var lightPassRotationAngle = envRotationAngle + if renderInfo.perFragmentIndirectLightingBuilt { + renderEncoder.setFragmentTexture(environmentLighting.irradianceMap, index: Int(modelPassIBLIrradianceTextureIndex.rawValue)) + renderEncoder.setFragmentTexture(environmentLighting.specularMap, index: Int(modelPassIBLSpecularTextureIndex.rawValue)) + renderEncoder.setFragmentTexture(environmentLighting.brdfMap, index: Int(modelPassIBLBRDFTextureIndex.rawValue)) + renderEncoder.setFragmentTexture(textureResources.csmShadowMap, index: Int(modelPassCSMShadowTextureIndex.rawValue)) + renderEncoder.setFragmentBytes(&csmUniforms, length: MemoryLayout.stride, index: Int(modelPassFragmentCSMUniformsIndex.rawValue)) + renderEncoder.setFragmentBytes(&lightParams, length: MemoryLayout.stride, index: Int(modelPassFragmentLightParamsIndex.rawValue)) + renderEncoder.setFragmentBytes(&brdfParameters, length: MemoryLayout.stride, index: Int(modelPassFragmentIBLParamIndex.rawValue)) + renderEncoder.setFragmentBytes(&lightPassRotationAngle, length: MemoryLayout.stride, index: Int(modelPassFragmentIBLRotationAngleIndex.rawValue)) + } + // The light pass's other inputs โ€” the point, spot and area lights and their shadows โ€” + // uploaded once for whichever lights with them: the geometry, when it lights itself + // whole per fragment and the quad stands down, or the quad. + let headerSize = 16 + let pointLights = uploadLights(buffer: bufferResources.pointLightBuffer, lights: getPointLights(), maxCount: 1024, headerSize: headerSize, labelForErrors: "Point Lights") + let spotLights = uploadLights(buffer: bufferResources.spotLightBuffer, lights: getSpotLights(), maxCount: 1024, headerSize: headerSize, labelForErrors: "Spot Lights") + let areaLights = uploadLights(buffer: bufferResources.areaLightBuffer, lights: getAreaLights(), maxCount: 1024, headerSize: headerSize, labelForErrors: "Area Lights") + var pointShadowUniforms = pointShadowState.makeUniforms() + var spotShadowUniforms = spotShadowState.makeUniforms() + if renderInfo.perFragmentLightingBuilt { + if let pointLights { + renderEncoder.setFragmentBuffer(pointLights, offset: 0, index: Int(modelPassFragmentPointLightsIndex.rawValue)) + } + if let spotLights { + renderEncoder.setFragmentBuffer(spotLights, offset: 0, index: Int(modelPassFragmentSpotLightsIndex.rawValue)) + } + if let areaLights { + renderEncoder.setFragmentBuffer(areaLights, offset: 0, index: Int(modelPassFragmentAreaLightsIndex.rawValue)) + } + renderEncoder.setFragmentBytes(&pointShadowUniforms, length: MemoryLayout.stride, index: Int(modelPassFragmentPointShadowUniformIndex.rawValue)) + renderEncoder.setFragmentBytes(&spotShadowUniforms, length: MemoryLayout.stride, index: Int(modelPassFragmentSpotShadowUniformIndex.rawValue)) + renderEncoder.setFragmentTexture(textureResources.areaTextureLTCMag, index: Int(modelPassAreaLTCMagTextureIndex.rawValue)) + renderEncoder.setFragmentTexture(textureResources.areaTextureLTCMat, index: Int(modelPassAreaLTCMatTextureIndex.rawValue)) + renderEncoder.setFragmentTexture(textureResources.pointShadowCubeMap, index: Int(modelPassPointShadowTextureIndex.rawValue)) + renderEncoder.setFragmentTexture(textureResources.spotShadowMap, index: Int(modelPassSpotShadowTextureIndex.rawValue)) + } + // โ”€โ”€ Sub-pass 1: Unbatched geometry โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ renderEncoder.setRenderPipelineState(modelPipeline.pipelineState!) renderEncoder.setDepthStencilState(modelPipeline.depthState) - renderEncoder.waitForFence(renderInfo.fence, before: .vertex) + renderEncoder.waitForFence(renderInfo.fence, before: .fragment) for entityId in visibleEntityIds { guard let entity = sceneSnapshot.entity(entityId) else { continue } @@ -2387,6 +2579,12 @@ public enum RenderPasses { modelUniforms.modelMatrix = modelMatrix modelUniforms.cameraPosition = pass.cameraPosition modelUniforms.projectionMatrix = pass.projectionMatrix + // Where the entity was the previous frame, for the velocity. + modelUniforms.unjitteredViewProjectionMatrix = pass.unjitteredViewProjection + modelUniforms.prevViewProjectionMatrix = pass.prevViewProjection + modelUniforms.prevModelMatrix = simd_mul( + ObjectMotion.shared.previousWorldTransform(of: entityId, current: worldTransformComponent.space), mesh.localSpace + ) renderEncoder.setVertexBytes(&modelUniforms, length: MemoryLayout.stride, index: Int(modelPassUniformIndex.rawValue)) @@ -2443,6 +2641,13 @@ public enum RenderPasses { applyTileRepresentationDither(fade: tileRepresentationFade, materialParameters: &materialParameters) applyMeshFadeDither(fade: meshFade, materialParameters: &materialParameters) + let plain = plainModelPipeline != nil && modelDrawNeverDiscards(materialParameters) + if plain != plainBound { + renderEncoder.setRenderPipelineState((plain ? plainModelPipeline! : modelPipeline).pipelineState!) + plainBound = plain + } + if plain { renderInfo.opaqueDrawsPlain += 1 } else { renderInfo.opaqueDrawsDiscarding += 1 } + renderEncoder.setFragmentBytes(&materialParameters, length: MemoryLayout.stride, index: Int(modelPassFragmentMaterialParameterIndex.rawValue)) renderEncoder.setFragmentTexture(material.normal.texture, index: Int(modelPassNormalTextureIndex.rawValue)) renderEncoder.setFragmentSamplerState(material.normal.sampler, index: Int(modelPassNormalSamplerIndex.rawValue)) @@ -2475,6 +2680,10 @@ public enum RenderPasses { batchUniforms.normalMatrix = matrix3x3_upper_left(modelMatrix).inverse.transpose batchUniforms.cameraPosition = shadingCameraPosition(cameraComponent) batchUniforms.projectionMatrix = renderInfo.perspectiveSpace + // A static batch moves with the camera alone. + batchUniforms.unjitteredViewProjectionMatrix = pass.unjitteredViewProjection + batchUniforms.prevViewProjectionMatrix = pass.prevViewProjection + batchUniforms.prevModelMatrix = matrix_identity_float4x4 for batchGroup in visibleBatchGroups { guard let positionBuffer = batchGroup.positionBuffer, @@ -2541,6 +2750,13 @@ public enum RenderPasses { applyLODDebugColorOverride(batchGroup: batchGroup, materialParameters: &materialParameters) applyStreamingTierDebugColorOverride(batchMaterial: material, materialParameters: &materialParameters) + let plain = plainModelPipeline != nil && modelDrawNeverDiscards(materialParameters) + if plain != plainBound { + renderEncoder.setRenderPipelineState((plain ? plainModelPipeline! : modelPipeline).pipelineState!) + plainBound = plain + } + if plain { renderInfo.opaqueDrawsPlain += 1 } else { renderInfo.opaqueDrawsDiscarding += 1 } + renderEncoder.setFragmentBytes(&materialParameters, length: MemoryLayout.stride, index: Int(modelPassFragmentMaterialParameterIndex.rawValue)) renderEncoder.setFragmentTexture(material.normal.texture, index: Int(modelPassNormalTextureIndex.rawValue)) renderEncoder.setFragmentSamplerState(material.normal.sampler, index: Int(modelPassNormalSamplerIndex.rawValue)) @@ -2562,6 +2778,15 @@ public enum RenderPasses { } // โ”€โ”€ Sub-pass 2: Lighting quad โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + // When the geometry lit itself whole per fragment, the lit attachment holds the frame + // already and the quad stands down: the pass ends here, on device and in the simulator. + if renderInfo.perFragmentLightingBuilt { + renderEncoder.updateFence(renderInfo.fence, after: .fragment) + renderEncoder.popDebugGroup() + renderEncoder.endEncoding() + combinedEncoderClosed = true + return + } // Device: reads G-buffer from tile memory via [[color(N)]] framebuffer fetch // in the same encoder โ€” no G-buffer textures are bound. // Simulator: framebuffer fetch is unsupported, so end the G-buffer encoder @@ -2585,6 +2810,7 @@ public enum RenderPasses { lightPassDescriptor.depthAttachment.loadAction = .load lightPassDescriptor.depthAttachment.storeAction = .store + GPUPassTiming.shared.attach(lightPassDescriptor, "Light") guard let lightEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: lightPassDescriptor) else { handleError(.renderPassCreationFailed, "Light Pass (Simulator)") return @@ -2610,88 +2836,66 @@ public enum RenderPasses { let lightQuadEncoder: MTLRenderCommandEncoder = renderEncoder #endif - lightQuadEncoder.setRenderPipelineState(lightPipeline.pipelineState!) - if let depthState = lightPipeline.depthState { - lightQuadEncoder.setDepthStencilState(depthState) + // The tile stage lights each pixel once per colour it holds; the quad lights every + // sample. Same encoder and the same inputs either way; only the binding differs. + let lights = LightStageBinder(encoder: lightQuadEncoder, tile: tileLightState != nil) + if let tileLightState { + lightQuadEncoder.setRenderPipelineState(tileLightState) + } else { + lightQuadEncoder.setRenderPipelineState(lightPipeline.pipelineState!) + if let depthState = lightPipeline.depthState { + lightQuadEncoder.setDepthStencilState(depthState) + } + lightQuadEncoder.setVertexBuffer(bufferResources.quadVerticesBuffer, offset: 0, index: 0) + lightQuadEncoder.setVertexBuffer(bufferResources.quadTexCoordsBuffer, offset: 0, index: 1) } - lightQuadEncoder.setVertexBuffer(bufferResources.quadVerticesBuffer, offset: 0, index: 0) - lightQuadEncoder.setVertexBuffer(bufferResources.quadTexCoordsBuffer, offset: 0, index: 1) - var effectiveCamPos = shadingCameraPosition(cameraComponent) - lightQuadEncoder.setFragmentBytes(&effectiveCamPos, length: MemoryLayout.stride, index: Int(lightPassCameraPositionIndex.rawValue)) + lights.setBytes(&effectiveCamPos, length: MemoryLayout.stride, index: Int(lightPassCameraPositionIndex.rawValue)) - var csmUniforms = shadowSystem.makeUniforms() - csmUniforms.cameraViewMatrix = viewMatrix - csmUniforms.lightSpaceMatrices.0 = SceneRootTransform.shared.effectiveLightMatrix(csmUniforms.lightSpaceMatrices.0) - csmUniforms.lightSpaceMatrices.1 = SceneRootTransform.shared.effectiveLightMatrix(csmUniforms.lightSpaceMatrices.1) - csmUniforms.lightSpaceMatrices.2 = SceneRootTransform.shared.effectiveLightMatrix(csmUniforms.lightSpaceMatrices.2) - lightQuadEncoder.setFragmentBytes(&csmUniforms, length: MemoryLayout.stride, index: Int(lightPassLightOrthoViewMatrixIndex.rawValue)) + lights.setBytes(&csmUniforms, length: MemoryLayout.stride, index: Int(lightPassLightOrthoViewMatrixIndex.rawValue)) - lightQuadEncoder.setFragmentTexture(textureResources.csmShadowMap, index: Int(lightPassShadowTextureIndex.rawValue)) - lightQuadEncoder.setFragmentTexture(textureResources.pointShadowCubeMap, index: Int(lightPassPointShadowTextureIndex.rawValue)) - var pointShadowUniforms = pointShadowState.makeUniforms() - lightQuadEncoder.setFragmentBytes(&pointShadowUniforms, length: MemoryLayout.stride, index: Int(lightPassPointShadowUniformIndex.rawValue)) - lightQuadEncoder.setFragmentTexture(textureResources.spotShadowMap, index: Int(lightPassSpotShadowTextureIndex.rawValue)) - var spotShadowUniforms = spotShadowState.makeUniforms() - lightQuadEncoder.setFragmentBytes(&spotShadowUniforms, length: MemoryLayout.stride, index: Int(lightPassSpotShadowUniformIndex.rawValue)) - let environmentLighting = resolveCurrentEnvironmentLighting() - lightQuadEncoder.setFragmentTexture(environmentLighting.irradianceMap, index: Int(lightPassIBLIrradianceTextureIndex.rawValue)) - lightQuadEncoder.setFragmentTexture(environmentLighting.specularMap, index: Int(lightPassIBLSpecularTextureIndex.rawValue)) - lightQuadEncoder.setFragmentTexture(environmentLighting.brdfMap, index: Int(lightPassIBLBRDFMapTextureIndex.rawValue)) - lightQuadEncoder.setFragmentTexture(textureResources.areaTextureLTCMag, index: Int(lightPassAreaLTCMagTextureIndex.rawValue)) - lightQuadEncoder.setFragmentTexture(textureResources.areaTextureLTCMat, index: Int(lightPassAreaLTCMatTextureIndex.rawValue)) - - var lightParams = getDirectionalLightParameters() - lightQuadEncoder.setFragmentBytes(&lightParams, length: MemoryLayout.stride, index: Int(lightPassLightParamsIndex.rawValue)) + lights.setTexture(textureResources.csmShadowMap, index: Int(lightPassShadowTextureIndex.rawValue)) + lights.setTexture(textureResources.pointShadowCubeMap, index: Int(lightPassPointShadowTextureIndex.rawValue)) + lights.setBytes(&pointShadowUniforms, length: MemoryLayout.stride, index: Int(lightPassPointShadowUniformIndex.rawValue)) + lights.setTexture(textureResources.spotShadowMap, index: Int(lightPassSpotShadowTextureIndex.rawValue)) + lights.setBytes(&spotShadowUniforms, length: MemoryLayout.stride, index: Int(lightPassSpotShadowUniformIndex.rawValue)) + lights.setTexture(environmentLighting.irradianceMap, index: Int(lightPassIBLIrradianceTextureIndex.rawValue)) + lights.setTexture(environmentLighting.specularMap, index: Int(lightPassIBLSpecularTextureIndex.rawValue)) + lights.setTexture(environmentLighting.brdfMap, index: Int(lightPassIBLBRDFMapTextureIndex.rawValue)) + lights.setTexture(textureResources.areaTextureLTCMag, index: Int(lightPassAreaLTCMagTextureIndex.rawValue)) + lights.setTexture(textureResources.areaTextureLTCMat, index: Int(lightPassAreaLTCMatTextureIndex.rawValue)) - let headerSize = 16 - _ = uploadAndBindLights( - buffer: bufferResources.pointLightBuffer, - lights: getPointLights(), - maxCount: 1024, - headerSize: headerSize, - encoder: lightQuadEncoder, - bufferIndex: Int(lightPassPointLightsIndex.rawValue), - labelForErrors: "Point Lights" - ) - _ = uploadAndBindLights( - buffer: bufferResources.spotLightBuffer, - lights: getSpotLights(), - maxCount: 1024, - headerSize: headerSize, - encoder: lightQuadEncoder, - bufferIndex: Int(lightPassSpotLightsIndex.rawValue), - labelForErrors: "Spot Lights" - ) - _ = uploadAndBindLights( - buffer: bufferResources.areaLightBuffer, - lights: getAreaLights(), - maxCount: 1024, - headerSize: headerSize, - encoder: lightQuadEncoder, - bufferIndex: Int(lightPassAreaLightsIndex.rawValue), - labelForErrors: "Area Lights" - ) + lights.setBytes(&lightParams, length: MemoryLayout.stride, index: Int(lightPassLightParamsIndex.rawValue)) - var brdfParameters = IBLParamsUniform() - brdfParameters.applyIBL = environmentLighting.applyIBL - brdfParameters.ambientIntensity = environmentLighting.ambientIntensity - lightQuadEncoder.setFragmentBytes(&brdfParameters, length: MemoryLayout.stride, index: Int(lightPassIBLParamIndex.rawValue)) + if let pointLights { + lights.setBuffer(pointLights, offset: 0, index: Int(lightPassPointLightsIndex.rawValue)) + } + if let spotLights { + lights.setBuffer(spotLights, offset: 0, index: Int(lightPassSpotLightsIndex.rawValue)) + } + if let areaLights { + lights.setBuffer(areaLights, offset: 0, index: Int(lightPassAreaLightsIndex.rawValue)) + } - var lightPassRotationAngle = envRotationAngle - lightQuadEncoder.setFragmentBytes(&lightPassRotationAngle, length: MemoryLayout.stride, index: Int(lightPassIBLRotationAngleIndex.rawValue)) + lights.setBytes(&brdfParameters, length: MemoryLayout.stride, index: Int(lightPassIBLParamIndex.rawValue)) + lights.setBytes(&lightPassRotationAngle, length: MemoryLayout.stride, index: Int(lightPassIBLRotationAngleIndex.rawValue)) var isGameMode = gameMode - lightQuadEncoder.setFragmentBytes(&isGameMode, length: MemoryLayout.size, index: Int(lightPassGameModeIndex.rawValue)) + lights.setBytes(&isGameMode, length: MemoryLayout.size, index: Int(lightPassGameModeIndex.rawValue)) - lightQuadEncoder.drawIndexedPrimitivesTracked( - type: .triangle, - indexCount: 6, - indexType: .uint16, - indexBuffer: bufferResources.quadIndexBuffer!, - indexBufferOffset: 0 - ) + if tileLightState != nil { + // One thread per pixel of the tile Metal chose for this pass. + lightQuadEncoder.dispatchThreadsPerTile(MTLSize(width: lightQuadEncoder.tileWidth, height: lightQuadEncoder.tileHeight, depth: 1)) + } else { + lightQuadEncoder.drawIndexedPrimitivesTracked( + type: .triangle, + indexCount: 6, + indexType: .uint16, + indexBuffer: bufferResources.quadIndexBuffer!, + indexBufferOffset: 0 + ) + } lightQuadEncoder.updateFence(renderInfo.fence, after: .fragment) // Ends the shared TBDR encoder on device, the standalone light encoder @@ -2730,6 +2934,7 @@ public enum RenderPasses { renderPassDescriptor.colorAttachments[0].storeAction = MTLStoreAction.store // set your encoder here + GPUPassTiming.shared.attach(renderPassDescriptor, "SSAO") guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else { @@ -2749,7 +2954,7 @@ public enum RenderPasses { renderEncoder.setRenderPipelineState(ssaoPipeline.pipelineState!) // renderEncoder.setDepthStencilState(ssaoPipeline.depthState) - renderEncoder.waitForFence(renderInfo.fence, before: .vertex) + renderEncoder.waitForFence(renderInfo.fence, before: .fragment) renderEncoder.setVertexBuffer(bufferResources.quadVerticesBuffer, offset: 0, index: 0) renderEncoder.setVertexBuffer(bufferResources.quadTexCoordsBuffer, offset: 0, index: 1) @@ -2791,6 +2996,14 @@ public enum RenderPasses { var projScale = simd_float2(renderInfo.perspectiveSpace.columns.0.x, renderInfo.perspectiveSpace.columns.1.y) renderEncoder.setFragmentBytes(&projScale, length: MemoryLayout.stride, index: Int(ssaoPassProjScaleIndex.rawValue)) + // The eye's rasterization rate map (XR foveation): the depth was drawn through it, so + // the view positions are rebuilt from where its texels sit on screen, not in the texture. + bindRateMap( + renderInfo.xrFoveation?.rateMapData, + toFragmentsOf: renderEncoder, + dataIndex: Int(ssaoPassRateMapDataIndex.rawValue), + sizesIndex: Int(ssaoPassRateMapSizesIndex.rawValue) + ) // set the draw command renderEncoder.drawIndexedPrimitivesTracked( @@ -2830,6 +3043,7 @@ public enum RenderPasses { renderInfo.ssaoRenderPassDescriptor.colorAttachments[Int(colorTarget.rawValue)].loadAction = .load // set your encoder here + GPUPassTiming.shared.attach(renderPassDescriptor, "SSAO") guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else { @@ -2849,7 +3063,7 @@ public enum RenderPasses { renderEncoder.setRenderPipelineState(ssaoBlurPipeline.pipelineState!) // renderEncoder.setDepthStencilState(ssaoBlurPipeline.depthState) - renderEncoder.waitForFence(renderInfo.fence, before: .vertex) + renderEncoder.waitForFence(renderInfo.fence, before: .fragment) renderEncoder.setVertexBuffer(bufferResources.quadVerticesBuffer, offset: 0, index: 0) renderEncoder.setVertexBuffer(bufferResources.quadTexCoordsBuffer, offset: 0, index: 1) @@ -2957,6 +3171,7 @@ public enum RenderPasses { return } + GPUPassTiming.shared.attach(renderPassDescriptor, "SSAO") guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else { handleError(.renderPassCreationFailed, "SSAO Low-Res Pass") return @@ -2971,7 +3186,7 @@ public enum RenderPasses { renderEncoder.pushDebugGroup("SSAO Low-Res Pass") renderEncoder.setRenderPipelineState(ssaoPipeline.pipelineState!) - renderEncoder.waitForFence(renderInfo.fence, before: .vertex) + renderEncoder.waitForFence(renderInfo.fence, before: .fragment) renderEncoder.setVertexBuffer(bufferResources.quadVerticesBuffer, offset: 0, index: 0) renderEncoder.setVertexBuffer(bufferResources.quadTexCoordsBuffer, offset: 0, index: 1) @@ -2991,6 +3206,14 @@ public enum RenderPasses { var projScale = simd_float2(renderInfo.perspectiveSpace.columns.0.x, renderInfo.perspectiveSpace.columns.1.y) renderEncoder.setFragmentBytes(&projScale, length: MemoryLayout.stride, index: Int(ssaoPassProjScaleIndex.rawValue)) + // The eye's rasterization rate map (XR foveation): the depth was drawn through it, so + // the view positions are rebuilt from where its texels sit on screen, not in the texture. + bindRateMap( + renderInfo.xrFoveation?.rateMapData, + toFragmentsOf: renderEncoder, + dataIndex: Int(ssaoPassRateMapDataIndex.rawValue), + sizesIndex: Int(ssaoPassRateMapSizesIndex.rawValue) + ) // SSAO properties renderEncoder.setFragmentBytes(&SSAOParams.shared.radius, length: MemoryLayout.stride, index: Int(ssaoPassRadiusIndex.rawValue)) @@ -3051,6 +3274,7 @@ public enum RenderPasses { direction: simd_float2, label: String ) { + GPUPassTiming.shared.attach(destinationDescriptor, "SSAO") guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: destinationDescriptor) else { handleError(.renderPassCreationFailed, label) return @@ -3066,7 +3290,7 @@ public enum RenderPasses { renderEncoder.setRenderPipelineState(pipeline.pipelineState!) // renderEncoder.setDepthStencilState(pipeline.depthState) - renderEncoder.waitForFence(renderInfo.fence, before: .vertex) + renderEncoder.waitForFence(renderInfo.fence, before: .fragment) renderEncoder.setVertexBuffer(bufferResources.quadVerticesBuffer, offset: 0, index: 0) renderEncoder.setVertexBuffer(bufferResources.quadTexCoordsBuffer, offset: 0, index: 1) @@ -3116,6 +3340,7 @@ public enum RenderPasses { return } + GPUPassTiming.shared.attach(renderPassDescriptor, "SSAO") guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else { handleError(.renderPassCreationFailed, "SSAO Blur Pass") return @@ -3130,7 +3355,7 @@ public enum RenderPasses { renderEncoder.pushDebugGroup("SSAO Simple Blur Pass") renderEncoder.setRenderPipelineState(ssaoBlurPipeline.pipelineState!) - renderEncoder.waitForFence(renderInfo.fence, before: .vertex) + renderEncoder.waitForFence(renderInfo.fence, before: .fragment) renderEncoder.setVertexBuffer(bufferResources.quadVerticesBuffer, offset: 0, index: 0) renderEncoder.setVertexBuffer(bufferResources.quadTexCoordsBuffer, offset: 0, index: 1) @@ -3167,6 +3392,7 @@ public enum RenderPasses { return } + GPUPassTiming.shared.attach(renderPassDescriptor, "SSAO") guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else { handleError(.renderPassCreationFailed, "SSAO Upsample Pass") return @@ -3181,7 +3407,7 @@ public enum RenderPasses { renderEncoder.pushDebugGroup("SSAO Upsample Pass") renderEncoder.setRenderPipelineState(upsamplePipeline.pipelineState!) - renderEncoder.waitForFence(renderInfo.fence, before: .vertex) + renderEncoder.waitForFence(renderInfo.fence, before: .fragment) renderEncoder.setVertexBuffer(bufferResources.quadVerticesBuffer, offset: 0, index: 0) renderEncoder.setVertexBuffer(bufferResources.quadTexCoordsBuffer, offset: 0, index: 1) @@ -3256,6 +3482,7 @@ public enum RenderPasses { renderPassDescriptor.depthAttachment.storeAction = .store // set your encoder here + GPUPassTiming.shared.attach(renderPassDescriptor, "Light") guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else { @@ -3274,7 +3501,7 @@ public enum RenderPasses { renderEncoder.pushDebugGroup("Light Pass") renderEncoder.setRenderPipelineState(lightPipeline.pipelineState!) - renderEncoder.waitForFence(renderInfo.fence, before: .vertex) + renderEncoder.waitForFence(renderInfo.fence, before: .fragment) renderEncoder.setVertexBuffer(bufferResources.quadVerticesBuffer, offset: 0, index: 0) renderEncoder.setVertexBuffer(bufferResources.quadTexCoordsBuffer, offset: 0, index: 1) @@ -3429,6 +3656,9 @@ public enum RenderPasses { renderEncoder.updateFence(renderInfo.fence, after: .fragment) } // end _legacyLightExecution_DEAD + /// Whether the first skipped pre-composite frame has been logged. + private nonisolated(unsafe) static var loggedPreCompositeBypass = false + public static let preCompositeExecution: RenderPassExecution = { commandBuffer in guard let preCompositePipeline = PipelineManager.shared.renderPipelinesByType[.preComposite] else { handleError(.pipelineStateNulled, "preCompositePipeline is nil") @@ -3478,7 +3708,44 @@ public enum RenderPasses { // Load Gaussian texture so it isn't cleared renderInfo.gaussianRenderPassDescriptor?.colorAttachments[0].loadAction = .load + // In XR passthrough nothing lies behind the scene, so with no splat, ambient-occlusion + // or gizmo layer to join and the lit view wanted, the composite is the lit scene itself + // and this full-screen pass would copy one texture into another (two frames' worth of + // bandwidth per eye at the headset's resolution). The look pass reads the lit scene + // directly instead. A registered render extension may draw into the composite, which + // keeps the pass. + let gameFrame = gameMode + let finalColorTexture: MTLTexture? = gameFrame + ? (bypassPostProcessing + ? (renderInfo.deferredRenderPassDescriptor?.colorAttachments[0].texture + ?? renderInfo.postProcessRenderPassDescriptor?.colorAttachments[0].texture) + : renderInfo.postProcessRenderPassDescriptor?.colorAttachments[0].texture) + : renderInfo.deferredRenderPassDescriptor?.colorAttachments[0].texture + // The splat layer holds something only on a frame the Gaussian pass drew for splat + // entities: the pass draws the working set's entities, none when its order is empty. + let gaussianFrameSlot = min(renderInfo.currentInFlightFrameSlot, maxInFlightCommandBuffers - 1) + let splatLayerPresent = renderInfo.gaussianCoverageWritten + && textureResources.gaussianColorMap != nil + && !GaussianSharedWorkingSet.shared.entityOrder(slot: gaussianFrameSlot).isEmpty + let copyOnly = renderInfo.preCompositeBypassEnabled + && gameFrame + && renderInfo.immersionStyle == .mixed + && !SSAOParams.shared.enabled + && !splatLayerPresent + && renderDebugViewMode == .lit + && !RenderExtensionRegistry.shared.hasRegisteredExtensions + if copyOnly, let finalColorTexture { + renderInfo.preCompositeBypassSource = finalColorTexture + if !RenderPasses.loggedPreCompositeBypass { + RenderPasses.loggedPreCompositeBypass = true + print("โœ“ Pre-composite pass skipped: a passthrough frame with no layer to join; the look pass reads the lit scene") + } + return + } + renderInfo.preCompositeBypassSource = nil + // set your encoder here + GPUPassTiming.shared.attach(renderPassDescriptor, "PreComposite") guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else { @@ -3498,7 +3765,7 @@ public enum RenderPasses { renderEncoder.setRenderPipelineState(preCompositePipeline.pipelineState!) - renderEncoder.waitForFence(renderInfo.fence, before: .vertex) + renderEncoder.waitForFence(renderInfo.fence, before: .fragment) renderEncoder.setVertexBuffer(bufferResources.quadVerticesBuffer, offset: 0, index: 0) renderEncoder.setVertexBuffer(bufferResources.quadTexCoordsBuffer, offset: 0, index: 1) @@ -3518,11 +3785,6 @@ public enum RenderPasses { index: Int(prePassDepthTextureIndex.rawValue) ) } else { - let postProcessColorTexture = renderInfo.postProcessRenderPassDescriptor?.colorAttachments[0].texture - let finalColorTexture = bypassPostProcessing - ? (renderInfo.deferredRenderPassDescriptor?.colorAttachments[0].texture ?? postProcessColorTexture) - : postProcessColorTexture - renderEncoder.setFragmentTexture( finalColorTexture, index: Int(prePassFinalTextureIndex.rawValue) @@ -3556,6 +3818,11 @@ public enum RenderPasses { var ssaoEnabled = SSAOParams.shared.enabled renderEncoder.setFragmentBytes(&ssaoEnabled, length: MemoryLayout.stride, index: Int(prePassSSAOEnabledIndex.rawValue)) + // The splat layer is read only on a frame the Gaussian pass drew it (see + // gaussianExecution): on any other it holds an earlier frame, or nothing at all. + var gaussianLayerPresent = renderInfo.gaussianCoverageWritten && textureResources.gaussianColorMap != nil + renderEncoder.setFragmentBytes(&gaussianLayerPresent, length: MemoryLayout.stride, index: Int(prePassGaussianLayerIndex.rawValue)) + // set the draw command renderEncoder.drawIndexedPrimitivesTracked( @@ -3585,6 +3852,80 @@ public enum RenderPasses { return } + let effectiveCameraPosition = shadingCameraPosition(cameraComponent) + + // What every draw of the pass shares, read once. + let sceneSnapshot = RenderSceneSnapshot() + let viewMatrix = SceneRootTransform.shared.effectiveViewMatrix(cameraComponent.viewSpace) + let projectionMatrix = renderInfo.perspectiveSpace + let batchingEnabled = BatchingSystem.shared.isEnabled() + let colorsByLOD = SpatialDebugVisualization.shared.colorRenderablesByLOD + let colorsByStreamingTier = SpatialDebugVisualization.shared.colorRenderablesByStreamingTier + var renderModes = SceneChannelRenderModeMemo() + + // Build and sort transparent draw items back-to-front for correct alpha blending. + var transparentEntities: [( + entityId: EntityID, + render: RenderComponent, + world: WorldTransformComponent, + deformation: DeformationComponent?, + hasSkeleton: Bool, + distanceSq: Float + )] = [] + + for entityId in visibleEntityIds { + guard let entity = sceneSnapshot.entity(entityId) else { continue } + let renderMode = renderModes.mode(of: sceneSnapshot.sceneChannels(of: entity)) + if renderMode == .hidden || renderMode == .wireframe { continue } + + if batchingEnabled, BatchingSystem.shared.isBatched(entityId: entityId) { + continue + } + + if !entity.traits.isDisjoint(with: [.sceneCamera, .camera, .gizmo, .light]) { continue } + // Blend submeshes go with the colour (there is no dither path here). + if meshSkipsColor(entity, in: sceneSnapshot) { continue } + + guard let renderComponent = sceneSnapshot.render(of: entity) else { + continue + } + guard let worldTransformComponent = sceneSnapshot.worldTransform(of: entity) else { + continue + } + + let hasTransparentSubmesh = renderComponent.mesh.contains { mesh in + mesh.submeshes.contains { submesh in + submesh.material?.hasTransparency ?? false + } + } + + if !hasTransparentSubmesh { + continue + } + + let worldPosition = simd_float3( + worldTransformComponent.space.columns.3.x, + worldTransformComponent.space.columns.3.y, + worldTransformComponent.space.columns.3.z + ) + let distanceSq = simd_length_squared(effectiveCameraPosition - worldPosition) + transparentEntities.append(( + entityId: entityId, + render: renderComponent, + world: worldTransformComponent, + deformation: sceneSnapshot.deformation(of: entity), + hasSkeleton: entity.traits.contains(.skeleton), + distanceSq: distanceSq + )) + } + + transparentEntities.sort { $0.distanceSq > $1.distanceSq } + + // No transparent surface, no pass: the encoder would load and store the lit colour and + // the depth of the whole frame to draw nothing, as the wireframe and occluder passes + // also refuse. + guard !transparentEntities.isEmpty else { return } + guard let encoderDescriptor = renderInfo.deferredRenderPassDescriptor else { handleError(.renderPassCreationFailed, "Deferred render pass descriptor not initialized") return @@ -3598,6 +3939,7 @@ public enum RenderPasses { encoderDescriptor.depthAttachment.loadAction = .load encoderDescriptor.depthAttachment.storeAction = .store + GPUPassTiming.shared.attach(encoderDescriptor, "Transparency") guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: encoderDescriptor) else { handleError(.renderPassCreationFailed, "Transparency Pass") @@ -3610,6 +3952,7 @@ public enum RenderPasses { } renderEncoder.label = "Transparency Pass" + applyXRFoveationViewport(renderEncoder) renderEncoder.pushDebugGroup("Transparency Pass") renderEncoder.setRenderPipelineState(transparencyPipeline.pipelineState!) @@ -3625,7 +3968,7 @@ public enum RenderPasses { } else { renderEncoder.setDepthStencilState(transparencyPipeline.depthState) } - renderEncoder.waitForFence(renderInfo.fence, before: .vertex) + renderEncoder.waitForFence(renderInfo.fence, before: .fragment) var transpCSM = shadowSystem.makeUniforms() transpCSM.cameraViewMatrix = SceneRootTransform.shared.effectiveViewMatrix(cameraComponent.viewSpace) @@ -3725,75 +4068,6 @@ public enum RenderPasses { index: Int(transparencyPassIBLRotationAngleIndex.rawValue) ) - let effectiveCameraPosition = shadingCameraPosition(cameraComponent) - - // What every draw of the pass shares, read once. - let sceneSnapshot = RenderSceneSnapshot() - let viewMatrix = SceneRootTransform.shared.effectiveViewMatrix(cameraComponent.viewSpace) - let projectionMatrix = renderInfo.perspectiveSpace - let batchingEnabled = BatchingSystem.shared.isEnabled() - let colorsByLOD = SpatialDebugVisualization.shared.colorRenderablesByLOD - let colorsByStreamingTier = SpatialDebugVisualization.shared.colorRenderablesByStreamingTier - var renderModes = SceneChannelRenderModeMemo() - - // Build and sort transparent draw items back-to-front for correct alpha blending. - var transparentEntities: [( - entityId: EntityID, - render: RenderComponent, - world: WorldTransformComponent, - deformation: DeformationComponent?, - hasSkeleton: Bool, - distanceSq: Float - )] = [] - - for entityId in visibleEntityIds { - guard let entity = sceneSnapshot.entity(entityId) else { continue } - let renderMode = renderModes.mode(of: sceneSnapshot.sceneChannels(of: entity)) - if renderMode == .hidden || renderMode == .wireframe { continue } - - if batchingEnabled, BatchingSystem.shared.isBatched(entityId: entityId) { - continue - } - - if !entity.traits.isDisjoint(with: [.sceneCamera, .camera, .gizmo, .light]) { continue } - // Blend submeshes go with the colour (there is no dither path here). - if meshSkipsColor(entity, in: sceneSnapshot) { continue } - - guard let renderComponent = sceneSnapshot.render(of: entity) else { - continue - } - guard let worldTransformComponent = sceneSnapshot.worldTransform(of: entity) else { - continue - } - - let hasTransparentSubmesh = renderComponent.mesh.contains { mesh in - mesh.submeshes.contains { submesh in - submesh.material?.hasTransparency ?? false - } - } - - if !hasTransparentSubmesh { - continue - } - - let worldPosition = simd_float3( - worldTransformComponent.space.columns.3.x, - worldTransformComponent.space.columns.3.y, - worldTransformComponent.space.columns.3.z - ) - let distanceSq = simd_length_squared(effectiveCameraPosition - worldPosition) - transparentEntities.append(( - entityId: entityId, - render: renderComponent, - world: worldTransformComponent, - deformation: sceneSnapshot.deformation(of: entity), - hasSkeleton: entity.traits.contains(.skeleton), - distanceSq: distanceSq - )) - } - - transparentEntities.sort { $0.distanceSq > $1.distanceSq } - for entry in transparentEntities { let entityId = entry.entityId let renderComponent = entry.render @@ -3927,6 +4201,27 @@ public enum RenderPasses { // in no order, and the far side of a pane must not come out over its near // side. The other materials are drawn in one go, as before. let isGlass = material.transmitsLight + // Glass draws a hair behind where it is (glassDepthBias): a pane modelled + // into its frame shares a plane with the frame, and the frame must win + // wherever the two coincide, or the edge flickers at every move of the camera. + if isGlass { + let away: Float = renderInfo.reverseZEnabled ? -1 : 1 + renderEncoder.setDepthBias(away * glassDepthBias, slopeScale: away * glassDepthSlopeScale, clamp: 0) + } else { + renderEncoder.setDepthBias(0, slopeScale: 0, clamp: 0) + } + // A headset's compositor reprojects every pixel by the depth it finds. A pane + // that mostly shows itself (dark, frosted) writes its depth, so its tint and + // reflections stay on the pane; a clear pane leaves the depth of what is seen + // through it, which is what the eye follows. The other blended materials keep + // the pass's state; passthrough already writes depth for every transparent draw. + if renderInfo.isXRStereoMode, renderInfo.immersionStyle != .mixed { + if isGlass, material.paneShowsItself, let writeState = getOrCreateTransparencyXRDepthWriteState(device: renderInfo.device) { + renderEncoder.setDepthStencilState(writeState) + } else { + renderEncoder.setDepthStencilState(transparencyPipeline.depthState) + } + } let firstGo = isGlass ? transparencyPassFarFaces : transparencyPassEveryFace for go in 0 ..< (isGlass ? 2 : 1) { var faces = Int32((go == 0 ? firstGo : transparencyPassNearFaces).rawValue) @@ -4004,7 +4299,11 @@ public enum RenderPasses { descriptor.depthAttachment.texture = depthTexture descriptor.depthAttachment.loadAction = .load descriptor.depthAttachment.storeAction = .store + // The depth was drawn through the eye's rasterization rate map when there is one + // (XR foveation); so is what this pass adds to it. + attachXRFoveation(renderInfo.xrFoveation?.rateMap, to: descriptor) + GPUPassTiming.shared.attach(descriptor, "Occluders") guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: descriptor) else { handleError(.renderPassCreationFailed, "Mesh Occluder Shell Pass") return @@ -4017,10 +4316,11 @@ public enum RenderPasses { } renderEncoder.label = "Mesh Occluder Shell Pass" + applyXRFoveationViewport(renderEncoder) renderEncoder.pushDebugGroup("Mesh Occluder Shell Pass") renderEncoder.setRenderPipelineState(pipelineState) renderEncoder.setDepthStencilState(pipeline.depthState) - renderEncoder.waitForFence(renderInfo.fence, before: .vertex) + renderEncoder.waitForFence(renderInfo.fence, before: .fragment) renderEncoder.setCullMode(.none) renderEncoder.setTriangleFillMode(.fill) @@ -4142,7 +4442,11 @@ public enum RenderPasses { descriptor.depthAttachment.texture = depthTexture descriptor.depthAttachment.loadAction = .load descriptor.depthAttachment.storeAction = .store + // The depth was drawn through the eye's rasterization rate map when there is one + // (XR foveation); so is what this pass adds to it. + attachXRFoveation(renderInfo.xrFoveation?.rateMap, to: descriptor) + GPUPassTiming.shared.attach(descriptor, "Occluders") guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: descriptor) else { handleError(.renderPassCreationFailed, "Wireframe Occlusion Depth Pass") return @@ -4155,10 +4459,11 @@ public enum RenderPasses { } renderEncoder.label = "Wireframe Occlusion Depth Pass" + applyXRFoveationViewport(renderEncoder) renderEncoder.pushDebugGroup("Wireframe Occlusion Depth Pass") renderEncoder.setRenderPipelineState(pipelineState) renderEncoder.setDepthStencilState(pipeline.depthState) - renderEncoder.waitForFence(renderInfo.fence, before: .vertex) + renderEncoder.waitForFence(renderInfo.fence, before: .fragment) renderEncoder.setCullMode(.none) renderEncoder.setTriangleFillMode(.fill) @@ -4327,6 +4632,7 @@ public enum RenderPasses { encoderDescriptor.depthAttachment.loadAction = .load encoderDescriptor.depthAttachment.storeAction = .store + GPUPassTiming.shared.attach(encoderDescriptor, "Wireframe") guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: encoderDescriptor) else { handleError(.renderPassCreationFailed, "Wireframe Pass") @@ -4339,6 +4645,7 @@ public enum RenderPasses { } renderEncoder.label = "Wireframe Pass" + applyXRFoveationViewport(renderEncoder) renderEncoder.pushDebugGroup("Wireframe Pass") renderEncoder.setRenderPipelineState(wireframePipelineState) @@ -4355,7 +4662,7 @@ public enum RenderPasses { } else { renderEncoder.setDepthStencilState(wireframePipeline.depthState) } - renderEncoder.waitForFence(renderInfo.fence, before: .vertex) + renderEncoder.waitForFence(renderInfo.fence, before: .fragment) renderEncoder.setCullMode(.none) var wireframeColor = wireframeSettings.color @@ -4708,6 +5015,7 @@ public enum RenderPasses { encoderDescriptor.depthAttachment.loadAction = .load encoderDescriptor.depthAttachment.storeAction = .store + GPUPassTiming.shared.attach(encoderDescriptor, "DebugBounds") guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: encoderDescriptor) else { handleError(.renderPassCreationFailed, "Spatial Debug Bounds Pass") return @@ -4719,13 +5027,14 @@ public enum RenderPasses { } renderEncoder.label = "Spatial Debug Bounds Pass" + applyXRFoveationViewport(renderEncoder) renderEncoder.pushDebugGroup("Spatial Debug Bounds Pass") renderEncoder.setRenderPipelineState(spatialDebugPipeline.pipelineState!) if let depthState = spatialDebugPipeline.depthState { renderEncoder.setDepthStencilState(depthState) } - renderEncoder.waitForFence(renderInfo.fence, before: .vertex) + renderEncoder.waitForFence(renderInfo.fence, before: .fragment) renderEncoder.setCullMode(.none) var viewMatrix = SceneRootTransform.shared.effectiveViewMatrix(cameraComponent.viewSpace) @@ -4788,6 +5097,17 @@ public enum RenderPasses { _ = commandBuffer return #else + // A frame without a splat entity has nothing to draw. The pass is skipped whole โ€” + // the depth snapshot, the tile initialise, the empty draw, the resolve โ€” and its + // layer is left as it is: the composite, the look and the anti-aliasing passes read + // it only when `gaussianCoverageWritten` says this frame drew it. The order is the + // preprocess's (an empty one when the scene has no splat entity), stale only while + // the asset-loading gate skips the preprocess, when the draw below handles it. + let workingSet = GaussianSharedWorkingSet.shared + let gaussianFrameSlot = min(renderInfo.currentInFlightFrameSlot, maxInFlightCommandBuffers - 1) + let entities = workingSet.entityOrder(slot: gaussianFrameSlot) + guard !entities.isEmpty else { return } + EngineProfiler.shared.beginScope(.gaussianDraw) defer { EngineProfiler.shared.endScope(.gaussianDraw) } @@ -4861,12 +5181,14 @@ public enum RenderPasses { } } + GPUPassTiming.shared.attach(renderPassDescriptor, "Gaussian") guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else { handleError(.renderPassCreationFailed, "Gaussian Pass") return } renderEncoder.label = "Gaussian Pass" + applyXRFoveationViewport(renderEncoder) renderInfo.gaussianCoverageWritten = true renderEncoder.pushDebugGroup("Gaussian Pass") @@ -4910,12 +5232,8 @@ public enum RenderPasses { // vertex stage indexes, written per uniform ring index so the two eyes of a stereo // frame keep their own matrices; an entity that has gone gets zero matrices, which // makes its stale records fail the vertex stage's w test and draw nothing. - let workingSet = GaussianSharedWorkingSet.shared - let gaussianFrameSlot = min(renderInfo.currentInFlightFrameSlot, maxInFlightCommandBuffers - 1) let uniformIndex = min(currentUniformBufferIndex(), totalPerMeshUniformBuffers() - 1) - let entities = workingSet.entityOrder(slot: gaussianFrameSlot) - if !entities.isEmpty, - let sortedKeys = workingSet.keys(slot: gaussianFrameSlot), + if let sortedKeys = workingSet.keys(slot: gaussianFrameSlot), let sharedRecords = workingSet.records(slot: gaussianFrameSlot), let sharedVisibleSet = workingSet.visibleSet(slot: gaussianFrameSlot), let entityConstants = workingSet.entityConstants(uniformIndex: uniformIndex) @@ -4988,10 +5306,15 @@ public enum RenderPasses { #endif } + /// A full-screen pass of `pipeline` from `source` into `destination`. It draws into the + /// post-process descriptor, which carries the frame's depth loaded and stored, unless + /// `descriptor` names another: a pipeline built with no depth attachment (the look pass) + /// brings its own, colour alone. static func executePostProcess( _ pipeline: RenderPipeline, source: MTLTexture, destination: MTLTexture, + descriptor: MTLRenderPassDescriptor? = nil, customization: @escaping (_ encoder: MTLRenderCommandEncoder) -> Void ) -> (MTLCommandBuffer) -> Void { { commandBuffer in @@ -5000,18 +5323,20 @@ public enum RenderPasses { return } - renderInfo.postProcessRenderPassDescriptor.colorAttachments[0].texture = destination - - renderInfo.postProcessRenderPassDescriptor.colorAttachments[0].loadAction = .clear - renderInfo.postProcessRenderPassDescriptor.colorAttachments[0].storeAction = .store - renderInfo.postProcessRenderPassDescriptor.depthAttachment.loadAction = .load - - guard let renderPassDescriptor = renderInfo.postProcessRenderPassDescriptor else { + guard let renderPassDescriptor = descriptor ?? renderInfo.postProcessRenderPassDescriptor else { handleError(.renderPassCreationFailed, "Post-process render pass descriptor not initialized") return } + renderPassDescriptor.colorAttachments[0].texture = destination + renderPassDescriptor.colorAttachments[0].loadAction = .clear + renderPassDescriptor.colorAttachments[0].storeAction = .store + if renderPassDescriptor.depthAttachment.texture != nil { + renderPassDescriptor.depthAttachment.loadAction = .load + } + // set your encoder here + GPUPassTiming.shared.attach(renderPassDescriptor, "Post") guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else { @@ -5031,7 +5356,7 @@ public enum RenderPasses { renderEncoder.setRenderPipelineState(pipeline.pipelineState!) - renderEncoder.waitForFence(renderInfo.fence, before: .vertex) + renderEncoder.waitForFence(renderInfo.fence, before: .fragment) renderEncoder.setVertexBuffer(bufferResources.quadVerticesBuffer, offset: 0, index: 0) renderEncoder.setVertexBuffer(bufferResources.quadTexCoordsBuffer, offset: 0, index: 1) @@ -5063,19 +5388,19 @@ extension float4x4 { } } +/// Writes the lights into the buffer (a uint4 count header, then the records, capped to +/// `maxCount`) and returns it, or nil without a buffer. @inline(__always) -private func uploadAndBindLights( +private func uploadLights( buffer: MTLBuffer?, lights: [T], maxCount: Int, headerSize: Int = 16, // uint4 header - encoder: MTLRenderCommandEncoder, - bufferIndex: Int, labelForErrors: String -) -> Bool { +) -> MTLBuffer? { guard let buf = buffer else { print("Missing buffer for \(labelForErrors)") - return false + return nil } // Cap to buffer capacity @@ -5091,12 +5416,56 @@ private func uploadAndBindLights( lights.withUnsafeBytes { raw in dst.copyMemory(from: raw.baseAddress!, byteCount: nBytes) } + return buf +} - // Bind +private func uploadAndBindLights( + buffer: MTLBuffer?, + lights: [some Any], + maxCount: Int, + headerSize: Int = 16, // uint4 header + encoder: MTLRenderCommandEncoder, + bufferIndex: Int, + labelForErrors: String +) -> Bool { + guard let buf = uploadLights(buffer: buffer, lights: lights, maxCount: maxCount, headerSize: headerSize, labelForErrors: labelForErrors) else { + return false + } encoder.setFragmentBuffer(buf, offset: 0, index: bufferIndex) return true } +/// Binds the light pass's inputs to the stage that lights: the quad's fragment function or +/// the tile kernel. Both take the same indices. +struct LightStageBinder { + let encoder: MTLRenderCommandEncoder + let tile: Bool + + func setBytes(_ bytes: UnsafeRawPointer, length: Int, index: Int) { + if tile { + encoder.setTileBytes(bytes, length: length, index: index) + } else { + encoder.setFragmentBytes(bytes, length: length, index: index) + } + } + + func setTexture(_ texture: MTLTexture?, index: Int) { + if tile { + encoder.setTileTexture(texture, index: index) + } else { + encoder.setFragmentTexture(texture, index: index) + } + } + + func setBuffer(_ buffer: MTLBuffer?, offset: Int, index: Int) { + if tile { + encoder.setTileBuffer(buffer, offset: offset, index: index) + } else { + encoder.setFragmentBuffer(buffer, offset: offset, index: index) + } + } +} + /// Returns true when the entity's AABB is farther than maxDistance from the camera. /// Uses closest-point-on-AABB distance so large meshes near the camera are never wrongly excluded. /// maxDistance == 0 disables culling (always returns false). diff --git a/Sources/UntoldEngine/Renderer/RenderResources.swift b/Sources/UntoldEngine/Renderer/RenderResources.swift index d30c80743..714cd45fb 100644 --- a/Sources/UntoldEngine/Renderer/RenderResources.swift +++ b/Sources/UntoldEngine/Renderer/RenderResources.swift @@ -29,8 +29,17 @@ public struct RenderInfo { public var renderPassDescriptor: MTLRenderPassDescriptor! public var environmentRenderPassDescriptor: MTLRenderPassDescriptor! public var offscreenRenderPassDescriptor: MTLRenderPassDescriptor! + /// The opaque pass's descriptor when the geometry lights itself per fragment: the lit colour + /// alone, in the light pass's slot, and the depth โ€” nothing else for the imageblock to hold, + /// so the GPU runs the pass in its largest tiles. The G-buffer's descriptor above otherwise. + public var forwardRenderPassDescriptor: MTLRenderPassDescriptor! public var gaussianRenderPassDescriptor: MTLRenderPassDescriptor! public var postProcessRenderPassDescriptor: MTLRenderPassDescriptor! + /// The look pass's descriptor: its colour alone, no depth attachment. The post-process + /// descriptor carries the frame's depth for the passes that read it; the look pass does + /// not, and a depth attachment loaded and stored for nothing is a full-size read and + /// write per eye. + public var lookRenderPassDescriptor: MTLRenderPassDescriptor! // CSM: one render pass descriptor per cascade (3 total), each targeting a separate array slice. public var csmRenderPassDescriptors: [MTLRenderPassDescriptor] = [] public var pointShadowRenderPassDescriptors: [MTLRenderPassDescriptor] = [] @@ -63,9 +72,95 @@ public struct RenderInfo { /// coverage: false on the simulator and on a frame the pass skipped, when the anti-aliasing /// passes must not read the map. public var gaussianCoverageWritten: Bool = false + /// The texture the look pass reads instead of `sceneCompositeTexture` on a frame the + /// pre-composite pass was skipped (see `RenderPasses.preCompositeExecution`): in XR + /// passthrough the composite is the lit scene itself whenever no layer joins it, so the + /// pass would copy a full-screen texture into another. Nil on a frame the pass ran. + public var preCompositeBypassSource: MTLTexture? + /// Whether the pre-composite pass may be skipped on a frame it would only copy. On by + /// default; `setRendering(.preCompositeBypass(.disabled))` runs the pass on every frame. + public var preCompositeBypassEnabled: Bool = true public var hzbDebugMipLevel: Int = 0 public var gBufferDebugStorageEnabled: Bool = false public var opaqueSampleCount: Int = 1 + /// Whether the TBDR light pass runs as a tile stage, lighting each pixel once per colour it + /// holds, rather than as a quad that lights every sample. On by default where tile shading + /// exists; `setRendering(.tileLightStage(.disabled))` runs the quad, as a diagnostic. + public var tileLightStageEnabled: Bool = true + /// The samples per pixel the multisampled anti-aliasing modes ask for (2, 4 or 8; the + /// device may grant fewer). 4 by default; `setRendering(.multisampleCount(2))` halves the + /// G-buffer's samples, which under the tile light stage is what lets the lighting run once + /// per pixel on GPUs whose tile memory does not hold four samples of the G-buffer. + public var preferredMultisampleCount: Int = 4 + /// Whether the last combined G-buffer pass lit through the tile stage (false after a frame + /// that ran the quad: the setting off, no tile pipeline, the simulator). + public var tileLightStageUsed: Bool = false + /// Whether opaque draws that never discard go through the model pipeline compiled without + /// discards, so the GPU resolves visibility before shading them. On by default; + /// `setRendering(.opaqueHiddenSurfaceRemoval(.disabled))` draws everything through the + /// general pipeline, as a diagnostic. + public var opaqueHiddenSurfaceRemovalEnabled: Bool = true + /// The last combined pass's draws by pipeline: plain (no discards) and general. Per pass, + /// so per eye in XR. + public var opaqueDrawsPlain: Int = 0 + public var opaqueDrawsDiscarding: Int = 0 + /// The render path a platform starts on (`setRendering(.renderPath(_:))` chooses another): + /// forward on visionOS, where the geometry lighting itself once per fragment is the measured + /// match for RealityKit's cost under MSAA and the rasterization rate maps; deferred on every + /// other platform, the path the engine was built around โ€” the lighting cost scales with the + /// lit pixels, the tile light stage, the effects that read the G-buffer. + public static let defaultRenderPath: RenderPath = { + #if os(visionOS) + return .forward + #else + return .deferred + #endif + }() + + /// Whether the geometry evaluates the image-based light and the sun's cascade shadow once + /// per fragment and carries them in the G-buffer for the light pass to add, instead of the + /// light pass computing them once per sample. On where `defaultRenderPath` is hybrid or + /// forward; the pipelines follow it on the next frame (`perFragmentIndirectLightingBuilt` + /// is what they were built with). + public var perFragmentIndirectLighting: Bool = RenderInfo.defaultRenderPath != .deferred + public var perFragmentIndirectLightingBuilt: Bool = false + /// Whether the geometry lights itself whole once per fragment โ€” the direct light of the sun, + /// the point, spot and area lights with their shadows, over the indirect light above, which + /// it needs โ€” and writes the lit colour, so the light quad, which lights once per sample, + /// stands down. On where `defaultRenderPath` is forward; the pipelines follow it on the + /// next frame (`perFragmentLightingBuilt` is what they were built with). + public var perFragmentLighting: Bool = RenderInfo.defaultRenderPath == .forward + public var perFragmentLightingBuilt: Bool = false + /// Whether the app chose a render path (`setRendering(.renderPath(_:))`, or the two settings + /// above one by one) rather than running on the platform's default. The XR loop adopts the + /// headset's path at its start when it has not (`adoptXRRenderPathUnlessChosen`). + public var renderPathChosen: Bool = false + /// What the model pipelines are to light per fragment: the setting, which needs the + /// per-fragment indirect light, and stands aside for the G-buffer debug views, which read + /// what the forward pass does not write. + var perFragmentLightingWanted: Bool { + perFragmentIndirectLighting && perFragmentLighting && !gBufferDebugStorageEnabled + } + + /// The opaque pass's descriptor for the pipelines as built: the forward one when the + /// geometry lights itself, the G-buffer's otherwise. + var opaquePassDescriptor: MTLRenderPassDescriptor? { + perFragmentLightingBuilt ? forwardRenderPassDescriptor : offscreenRenderPassDescriptor + } + + /// The objects the frame's culling put to the occlusion test: entities with their + /// children, so a model's parts count as one. With `HZBOcclusionCulling.minimumEntities` + /// it decides whether the frame's depth is worth a pyramid. + public var hzbCandidateCount: Int = 0 + /// Whether the output transform pass follows the temporal resolve directly in the frame's + /// graph (nothing registered between them), so the resolve may present the frame itself + /// and the output pass stand down. Set when the graph is built. + public var temporalResolveMayPresent: Bool = false + /// The same for the look pass, on a frame without a pass after it (no anti-aliasing pass + /// after the look, nothing registered before the output): it then presents the frame + /// itself, and `lookPresentedThisFrame` says it did. + public var lookMayPresent: Bool = false + public var lookPresentedThisFrame: Bool = false // XR stereo-aware HZB culling // When true, depth and HZB pyramids are maintained per eye and occlusion @@ -87,6 +182,10 @@ public struct RenderInfo { /// shadingCameraPosition so each eye gets its own view vector (specular highlights, /// Fresnel), while streaming, LOD and culling keep the head-centre camera position. public var xrEyeCameraPosition: simd_float3? + /// The rasterization rate map of the eye being drawn (visionOS foveation), with the + /// viewport and the decoder data that go with it; nil when the frame is drawn uniformly. + /// `viewPort` stays the size of the textures, the map's physical size. See XRFoveation.swift. + public var xrFoveation: XRFoveationFrame? } /// Where an eye is, from its view matrix (the inverse of the eye's camera transform). @@ -176,6 +275,9 @@ public struct BufferResources { var reduceScanCapacity: Int = 0 /// Whether a frame with more bounding boxes than the scan takes was reported. var reportedReduceScanOverflow: Bool = false + + /// Bound where a shader declares a rasterization rate map it will not read (XRFoveation.swift). + var rateMapPlaceholder: MTLBuffer? } public struct TripleBufferResources { @@ -209,6 +311,10 @@ public struct TextureResources { public var materialMap: MTLTexture? public var emissiveMap: MTLTexture? public var depthMap: MTLTexture? + /// The velocity the opaque pass writes per pixel in its forward mode (NDC, this frame minus + /// the previous), for the temporal resolve; the multisampled one it draws into, resolved. + public var velocityMap: MTLTexture? + public var msaaVelocityMap: MTLTexture? public var msaaColorMap: MTLTexture? public var msaaNormalMap: MTLTexture? public var msaaPositionMap: MTLTexture? @@ -328,9 +434,14 @@ public struct HZBPyramidFrame: Sendable { public var viewProjection: simd_float4x4 /// Where that frame's camera stood, in world space. public var cameraPosition: simd_float3 + /// The rasterization rate map that frame was drawn through (XR foveation), nil when it was + /// drawn uniformly: the pyramid is laid out in the map's physical space, so a screen rect or + /// point the test projects must go through the map before the pyramid is sampled there. + public var rateMapData: XRRasterizationRateMapData? - public init(viewProjection: simd_float4x4, cameraPosition: simd_float3) { + public init(viewProjection: simd_float4x4, cameraPosition: simd_float3, rateMapData: XRRasterizationRateMapData? = nil) { self.viewProjection = viewProjection self.cameraPosition = cameraPosition + self.rateMapData = rateMapData } } diff --git a/Sources/UntoldEngine/Renderer/RenderSceneSnapshot.swift b/Sources/UntoldEngine/Renderer/RenderSceneSnapshot.swift index b707dcacf..3178ede92 100644 --- a/Sources/UntoldEngine/Renderer/RenderSceneSnapshot.swift +++ b/Sources/UntoldEngine/Renderer/RenderSceneSnapshot.swift @@ -169,6 +169,18 @@ struct RenderSceneSnapshot { component(.tileLODTag, pools.tileLODTag, of: entity) } + /// The object `entity` is drawn as a part of: its parent in the scenegraph when it has + /// one, else itself. A model loaded from a file is one entity per node under the entity + /// the model was set on, so the parts of one model share this. + @inline(__always) + func object(of entity: Entity) -> EntityID { + guard entities[entity.index].mask.test(ids.scenegraph), + let scenegraph = pools.scenegraph?.component(at: entity.index, as: ScenegraphComponent.self), + scenegraph.parent != .invalid + else { return entity.entityId } + return scenegraph.parent + } + /// The scene channels of `entity`: those of its channel component, or the ones an /// entity without the component gets from its name. @inline(__always) @@ -208,6 +220,7 @@ struct RenderSceneSnapshot { let deformation = getComponentId(for: DeformationComponent.self) let skeleton = getComponentId(for: SkeletonComponent.self) let tileLODTag = getComponentId(for: TileLODTagComponent.self) + let scenegraph = getComponentId(for: ScenegraphComponent.self) @inline(__always) func traits(of mask: ComponentMask) -> RenderEntityTraits { @@ -245,6 +258,7 @@ struct RenderSceneSnapshot { let tileRepresentationFade: ComponentPool? let deformation: ComponentPool? let tileLODTag: ComponentPool? + let scenegraph: ComponentPool? init(ids: ComponentIds, scene: Scene) { render = scene.componentPool[ids.render] @@ -257,6 +271,7 @@ struct RenderSceneSnapshot { tileRepresentationFade = scene.componentPool[ids.tileRepresentationFade] deformation = scene.componentPool[ids.deformation] tileLODTag = scene.componentPool[ids.tileLODTag] + scenegraph = scene.componentPool[ids.scenegraph] } } } diff --git a/Sources/UntoldEngine/Renderer/RuntimeEnvironmentLighting.swift b/Sources/UntoldEngine/Renderer/RuntimeEnvironmentLighting.swift index 3b0715c89..3d3772147 100644 --- a/Sources/UntoldEngine/Renderer/RuntimeEnvironmentLighting.swift +++ b/Sources/UntoldEngine/Renderer/RuntimeEnvironmentLighting.swift @@ -60,6 +60,9 @@ public struct RuntimeEnvironmentLightingTextureSet { public var irradianceMap: MTLTexture public var specularMap: MTLTexture public var brdfMap: MTLTexture + /// The probe's first nine spherical harmonics (nine float4s), which the headset's room + /// probe pre-filter projects and reads the irradiance from. + public var probeSH: MTLBuffer? } public struct ResolvedEnvironmentLighting { @@ -108,10 +111,14 @@ public func makeRuntimeEnvironmentLightingTextureSet(labelPrefix: String) -> Run storageMode: .private ) else { return nil } + let probeSH = currentRenderInfo.device.makeBuffer(length: 9 * MemoryLayout>.stride, options: .storageModeShared) + probeSH?.label = "\(labelPrefix) Probe Harmonics" + return RuntimeEnvironmentLightingTextureSet( irradianceMap: irradianceMap, specularMap: specularMap, - brdfMap: brdfMap + brdfMap: brdfMap, + probeSH: probeSH ) } diff --git a/Sources/UntoldEngine/Renderer/TemporalAntiAliasing.swift b/Sources/UntoldEngine/Renderer/TemporalAntiAliasing.swift new file mode 100644 index 000000000..0c17c498e --- /dev/null +++ b/Sources/UntoldEngine/Renderer/TemporalAntiAliasing.swift @@ -0,0 +1,348 @@ +// +// TemporalAntiAliasing.swift +// UntoldEngine +// +// Temporal anti-aliasing: the projection is jittered by a sub-pixel amount each frame, and +// the resolve pass after the look pass blends the frame with the eye's reprojected, clipped +// history (TAAShader.metal). Sub-pixel detail a single sample flickers on โ€” thin wires, fine +// normal maps, the periphery of a foveated frame where a pixel spans several screen pixels โ€” +// averages out over frames. The reprojection follows the camera through the depth buffer and, +// on the forward path, each pixel's own motion from the opaque pass; on the deferred and hybrid +// paths what moves on its own is covered by the neighbourhood clip alone. +// +// State is per eye (XR renders the two eyes through the same passes): a history texture +// ping-pong, the unjittered view-projection of the frame the history holds, and that frame's +// rasterization rate map when it was foveated โ€” the history is laid out in that map's +// physical space, so the resolve maps through both frames' maps. +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import CShaderTypes +import Metal +import simd + +/// Tunables of the temporal resolve. +public final class TAAParams: @unchecked Sendable { + public static let shared = TAAParams() + private let lock = NSLock() + private var historyWeightValue: Float = 0.9 + private var clipGammaValue: Float = 1.25 + private var jitterScaleValue: Float = 1 + + /// Scales the sub-pixel jitter: 1 is the full half-pixel sequence, 0 draws every frame + /// from the pixel centre (the resolve still runs, as a diagnostic of what the jitter + /// itself changes). Clamped to [0, 1]. + public var jitterScale: Float { + get { lock.withLock { jitterScaleValue } } + set { lock.withLock { jitterScaleValue = min(max(newValue, 0), 1) } } + } + + /// The share of the (clipped) history in each resolved pixel: higher converges further + /// and reacts slower. Clamped to [0, 0.98]; the default is 0.9. + public var historyWeight: Float { + get { lock.withLock { historyWeightValue } } + set { lock.withLock { historyWeightValue = min(max(newValue, 0), 0.98) } } + } + + /// How far the history may stray from a pixel's 3ร—3 neighbourhood: the clip box is the + /// neighbourhood's mean ยฑ this many standard deviations. Lower rejects ghosting sooner, + /// higher keeps pixel-thin features (wires, text) through the frames that miss them. + /// Clamped to [0.5, 3]; the default is 1.25. + public var clipGamma: Float { + get { lock.withLock { clipGammaValue } } + set { lock.withLock { clipGammaValue = min(max(newValue, 0.5), 3) } } + } +} + +/// Sub-pixel offsets, in pixels, of the Halton (2, 3) sequence centred on the pixel: eight +/// of them, so the average over a short window sits at the pixel centre. +let taaJitterSequence: [simd_float2] = (1 ... 8).map { index in + func halton(_ index: Int, _ base: Int) -> Float { + var result: Float = 0 + var fraction: Float = 1 + var i = index + while i > 0 { + fraction /= Float(base) + result += fraction * Float(i % base) + i /= base + } + return result + } + return simd_float2(halton(index, 2) - 0.5, halton(index, 3) - 0.5) +} + +/// Whether a mode runs the temporal resolve. +extension AntiAliasingMode { + var isTemporal: Bool { + switch self { + case .taa, .msaaTaa: + return true + case .none, .fxaa, .smaa, .msaa: + return false + } + } +} + +public final class TemporalAntiAliasing: @unchecked Sendable { + public static let shared = TemporalAntiAliasing() + + /// What one eye carries from frame to frame. + struct EyeHistory { + var textures: [MTLTexture] = [] + var readIndex = 0 + var viewProjection = matrix_identity_float4x4 + var rateMapData: XRRasterizationRateMapData? + var valid = false + } + + private(set) var eyes: [EyeHistory] = [EyeHistory(), EyeHistory()] + private var jitterIndex = 0 + /// The jitter currently added to `renderInfo.perspectiveSpace`, in NDC, so it can be + /// taken out before the next one goes in (the macOS projection persists across frames). + private(set) var appliedJitter = simd_float2.zero + /// The jitter of this frame in pixels of the screen, for diagnostics and tests. + private(set) var currentJitterPixels = simd_float2.zero + private var loggedResolve = false + /// False while a secondary target is drawn (XR: a capture drawable of a recording or a + /// stream, seen from its own pose): the resolve outputs the frame as is and leaves the + /// eye's history, matrices and map untouched for the built-in display's next frame. + public var historyUpdatesEnabled = true + /// The texture the last resolve wrote: the resolved frame, which is also the eye's new + /// history. The output transform presents it (`outputTransformRenderPass`); writing it a + /// second time into `antiAliasingTexture` would cost a full-screen write per eye. + public private(set) var resolvedTexture: MTLTexture? + /// Whether the last resolve also presented the frame โ€” wrote the display's colour and depth + /// textures โ€” so the output transform pass stands down for it. The resolve presents when the + /// graph put nothing between it and the output (`renderInfo.temporalResolveMayPresent`), the + /// view is the lit one, and the present textures exist at its size. + public private(set) var presentedThisFrame = false + + private var eyeIndex: Int { + renderInfo.isXRStereoMode ? min(max(renderInfo.currentEye, 0), 1) : 0 + } + + /// The projection was set afresh (a resize, a new eye): nothing of the old jitter is in it. + func projectionDidChange() { + appliedJitter = .zero + } + + /// Before a frame's graph is built: takes last frame's jitter out of the projection and + /// puts this frame's in โ€” or just takes it out when the mode is not temporal. The offset + /// lives in the projection's third column: for a perspective projection, whose clip w is + /// the view-space depth, it shifts every point by a constant amount in NDC. + func applyJitter() { + // A new frame: nothing has presented yet (the resolve says so when it does). + presentedThisFrame = false + var projection = renderInfo.perspectiveSpace + projection.columns.2.x -= appliedJitter.x + projection.columns.2.y -= appliedJitter.y + appliedJitter = .zero + currentJitterPixels = .zero + + if antiAliasingMode.isTemporal, let screen = screenSize(), screen.x > 0, screen.y > 0 { + let pixels = taaJitterSequence[jitterIndex % taaJitterSequence.count] * TAAParams.shared.jitterScale + jitterIndex += 1 + let ndc = simd_float2(2 * pixels.x / screen.x, 2 * pixels.y / screen.y) + projection.columns.2.x += ndc.x + projection.columns.2.y += ndc.y + appliedJitter = ndc + currentJitterPixels = pixels + } + renderInfo.perspectiveSpace = projection + } + + /// The size of the screen the projection maps to: the rate map's under foveation, the + /// textures' otherwise. + private func screenSize() -> simd_float2? { + if let foveation = renderInfo.xrFoveation { + return foveation.screenSize + } + return renderInfo.viewPort + } + + /// The previous frame's view-projection of the eye being drawn (unjittered, scene root + /// included), which the resolve reprojects the history with, or `fallback` when the eye has + /// no history: what the opaque pass measures a surface's velocity against. + func previousViewProjection(fallback: simd_float4x4) -> simd_float4x4 { + let eye = eyeIndex + return eyes[eye].valid && historyUpdatesEnabled ? eyes[eye].viewProjection : fallback + } + + /// This frame's projection without its jitter. + var unjitteredProjection: simd_float4x4 { + var projection = renderInfo.perspectiveSpace + projection.columns.2.x -= appliedJitter.x + projection.columns.2.y -= appliedJitter.y + return projection + } + + /// Forgets every eye's history: the next resolve of each eye outputs its frame as is. + func invalidateHistory() { + for index in eyes.indices { + eyes[index].valid = false + } + } + + /// The eye's history textures at the current viewport, made or remade when needed (a + /// remake invalidates the history). + private func historyTextures(for eye: Int) -> [MTLTexture]? { + let width = max(1, Int(renderInfo.viewPort.x)) + let height = max(1, Int(renderInfo.viewPort.y)) + if eyes[eye].textures.count == 2, eyes[eye].textures[0].width == width, eyes[eye].textures[0].height == height { + return eyes[eye].textures + } + guard let device = renderInfo.device else { return nil } + var textures: [MTLTexture] = [] + for slot in 0 ..< 2 { + guard let texture = createTexture( + device: device, + label: "TAA History Eye \(eye) \(slot)", + pixelFormat: renderInfo.colorPipeline.working.lookOutput, + width: width, + height: height, + usage: [.shaderRead, .renderTarget], + storageMode: .private + ) else { return nil } + textures.append(texture) + } + eyes[eye].textures = textures + eyes[eye].readIndex = 0 + eyes[eye].valid = false + return textures + } + + /// The resolve: the look output and the eye's history into the eye's other history + /// texture (`resolvedTexture` afterwards), then the frame's matrices and map become the + /// history's. + func encodeResolve(_ commandBuffer: MTLCommandBuffer) { + guard let source = textureResources.lookTexture, + let depth = textureResources.depthMap, + let pipeline = PipelineManager.shared.renderPipelinesByType[.taa], + pipeline.success, + let pipelineState = pipeline.pipelineState, + let camera = CameraSystem.shared.activeCamera, + let cameraComponent = scene.get(component: CameraComponent.self, for: camera) + else { + handleError(.renderPassCreationFailed, "TAA Pass: missing texture, pipeline or camera") + return + } + presentedThisFrame = false + let eye = eyeIndex + guard let textures = historyTextures(for: eye) else { + handleError(.renderPassCreationFailed, "TAA Pass: history textures") + return + } + if !loggedResolve { + loggedResolve = true + let foveated = renderInfo.xrFoveation != nil ? "rate-map reprojection, rate-weighted history" : "uniform reprojection" + print("โœ“ TAA resolve: \(source.width)ร—\(source.height), history weight \(TAAParams.shared.historyWeight), clip gamma \(TAAParams.shared.clipGamma) (variance clip), \(foveated)") + } + let readTexture = textures[eyes[eye].readIndex] + let writeTexture = textures[1 - eyes[eye].readIndex] + + let view = SceneRootTransform.shared.effectiveViewMatrix(cameraComponent.viewSpace) + let viewProjection = simd_mul(unjitteredProjection, view) + let currentRateMap = renderInfo.xrFoveation?.rateMapData + var constants = TAAConstants() + constants.invViewProjection = simd_inverse(viewProjection) + constants.prevViewProjection = eyes[eye].viewProjection + constants.rateMapSizes = rateMapSizes(currentRateMap) + constants.prevRateMapSizes = rateMapSizes(eyes[eye].rateMapData) + constants.physicalSize = simd_float2(Float(source.width), Float(source.height)) + constants.historyWeight = TAAParams.shared.historyWeight + constants.clipGamma = TAAParams.shared.clipGamma + constants.historyValid = eyes[eye].valid && historyUpdatesEnabled ? 1 : 0 + constants.reverseZ = renderInfo.reverseZEnabled ? 1 : 0 + // The opaque pass wrote the frame's velocity in its forward mode: a surface that moves + // on its own has its history where it was. + let velocity = renderInfo.perFragmentLightingBuilt ? textureResources.velocityMap : nil + constants.velocityValid = velocity != nil ? 1 : 0 + + let descriptor = MTLRenderPassDescriptor() + descriptor.colorAttachments[0].texture = writeTexture + descriptor.colorAttachments[0].loadAction = .dontCare + descriptor.colorAttachments[0].storeAction = .store + + // Presenting here, when the output transform would only copy the resolve: the display's + // colour and depth textures become the pass's second colour attachment and depth. + let presentDescriptor = renderInfo.renderPassDescriptor + let presentColor = presentDescriptor?.colorAttachments[0].texture + let presentDepth = presentDescriptor?.depthAttachment.texture + let presentPipeline = PipelineManager.shared.renderPipelinesByType[.taaPresent] + let presents = renderInfo.temporalResolveMayPresent + && renderDebugViewMode == .lit + && presentColor != nil && presentDepth != nil + && presentColor?.width == source.width && presentColor?.height == source.height + && presentPipeline?.success == true && presentPipeline?.pipelineState != nil + if presents, let presentDescriptor { + descriptor.colorAttachments[1].texture = presentColor + descriptor.colorAttachments[1].loadAction = presentDescriptor.colorAttachments[0].loadAction + descriptor.colorAttachments[1].clearColor = presentDescriptor.colorAttachments[0].clearColor + descriptor.colorAttachments[1].storeAction = .store + descriptor.depthAttachment.texture = presentDepth + descriptor.depthAttachment.loadAction = presentDescriptor.depthAttachment.loadAction + descriptor.depthAttachment.clearDepth = presentDescriptor.depthAttachment.clearDepth + descriptor.depthAttachment.storeAction = .store + } + + GPUPassTiming.shared.attach(descriptor, "TAA") + guard let encoder = commandBuffer.makeRenderCommandEncoder(descriptor: descriptor) else { + handleError(.renderPassCreationFailed, "TAA Pass") + return + } + encoder.label = presents ? "TAA Pass (presents)" : "TAA Pass" + encoder.pushDebugGroup("TAA Pass") + if presents, let presentPipeline, let presentState = presentPipeline.pipelineState { + encoder.setRenderPipelineState(presentState) + if let depthState = presentPipeline.depthState { + encoder.setDepthStencilState(depthState) + } + var encodingMode = renderInfo.colorPipeline.present.encodingMode.rawValue + encoder.setFragmentBytes(&encodingMode, length: MemoryLayout.stride, index: Int(taaPassEncodingModeIndex.rawValue)) + } else { + encoder.setRenderPipelineState(pipelineState) + } + encoder.waitForFence(renderInfo.fence, before: .fragment) + encoder.setVertexBuffer(bufferResources.quadVerticesBuffer, offset: 0, index: 0) + encoder.setVertexBuffer(bufferResources.quadTexCoordsBuffer, offset: 0, index: 1) + encoder.setFragmentTexture(source, index: Int(taaPassColorTextureIndex.rawValue)) + encoder.setFragmentTexture(readTexture, index: Int(taaPassHistoryTextureIndex.rawValue)) + encoder.setFragmentTexture(depth, index: Int(taaPassDepthTextureIndex.rawValue)) + encoder.setFragmentTexture(velocity ?? depth, index: Int(taaPassVelocityTextureIndex.rawValue)) + encoder.setFragmentBytes(&constants, length: MemoryLayout.stride, index: Int(taaPassConstantsIndex.rawValue)) + // The maps' sizes are in the constants; the shader reads a map's parameters only when + // its sizes say there is one, so a placeholder stands in for a uniform frame. + let placeholder = rateMapPlaceholderBuffer() + encoder.setFragmentBuffer(currentRateMap?.buffer ?? placeholder, offset: 0, index: Int(taaPassRateMapDataIndex.rawValue)) + encoder.setFragmentBuffer(eyes[eye].rateMapData?.buffer ?? placeholder, offset: 0, index: Int(taaPassPrevRateMapDataIndex.rawValue)) + encoder.drawIndexedPrimitivesTracked( + type: .triangle, + indexCount: 6, + indexType: .uint16, + indexBuffer: bufferResources.quadIndexBuffer!, + indexBufferOffset: 0 + ) + encoder.updateFence(renderInfo.fence, after: .fragment) + encoder.popDebugGroup() + encoder.endEncoding() + resolvedTexture = writeTexture + presentedThisFrame = presents + + // A secondary target wrote the spare history texture, which the next primary resolve + // overwrites anyway; the one it reads, and the frame's matrices and map, are untouched. + guard historyUpdatesEnabled else { return } + eyes[eye].readIndex = 1 - eyes[eye].readIndex + eyes[eye].viewProjection = viewProjection + eyes[eye].rateMapData = currentRateMap + eyes[eye].valid = true + } +} + +/// The graph's temporal resolve pass, in the slot FXAA or SMAA otherwise take. +let taaRenderPass: RenderPasses.RenderPassExecution = { commandBuffer in + TemporalAntiAliasing.shared.encodeResolve(commandBuffer) +} diff --git a/Sources/UntoldEngine/Renderer/UntoldEngine.swift b/Sources/UntoldEngine/Renderer/UntoldEngine.swift index df278b6fa..38d13928b 100644 --- a/Sources/UntoldEngine/Renderer/UntoldEngine.swift +++ b/Sources/UntoldEngine/Renderer/UntoldEngine.swift @@ -23,8 +23,21 @@ public class UntoldRenderer: NSObject, MTKViewDelegate { public let metalView: MTKView var gameUpdateCallback: ((_ deltaTime: Float) -> Void)? + var frameUpdateCallback: ((FrameTime) -> Void)? var handleInputCallback: (() -> Void)? + /// Makes each frame's `FrameTime` from the presentation time the host loop predicts. + var frameClock = FrameClock() + + /// A host that drives `draw(in:)` from its own display link sets the link's target + /// timestamp here before each draw; the frame is then posed for that instant instead of + /// one period ahead of the draw. The frame consumes it. + public var predictedPresentationTime: TimeInterval? + + /// The frame being prepared or drawn: its predicted presentation time, the delta from the + /// frame before and whether the display skipped a frame between them. + public var frameTime: FrameTime { currentFrameTime } + let frameEvents = FrameEventDispatcher() private var configuration: UntoldRendererConfig @@ -137,11 +150,27 @@ public class UntoldRenderer: NSObject, MTKViewDelegate { return renderer } + /// Installs the game's per-frame callbacks. `gameUpdate` receives the seconds from the + /// previous frame's presentation to this one's; the frame's own time is `frameTime` (or + /// the `currentFrameTime` global) while it runs. Replaces a `frameUpdate` callback. public func setupCallbacks( gameUpdate: @escaping (_ deltaTime: Float) -> Void, handleInput: @escaping () -> Void ) { gameUpdateCallback = gameUpdate + frameUpdateCallback = nil + handleInputCallback = handleInput + } + + /// Installs the game's per-frame callbacks with the frame's time: the predicted + /// presentation time to pose a moving object for, the delta from the frame before and + /// the refreshes skipped between them. Replaces a `gameUpdate` callback. + public func setupCallbacks( + frameUpdate: @escaping (_ frame: FrameTime) -> Void, + handleInput: @escaping () -> Void + ) { + frameUpdateCallback = frameUpdate + gameUpdateCallback = nil handleInputCallback = handleInput } @@ -164,6 +193,7 @@ public class UntoldRenderer: NSObject, MTKViewDelegate { initSizeableResources() // TODO: Find a better name function shadowSystem = ShadowSystem() + applyShadowSoftnessSetBeforeSystem() pointShadowState = PointShadowState() spotShadowState = SpotShadowState() @@ -181,25 +211,37 @@ public class UntoldRenderer: NSObject, MTKViewDelegate { initScriptingSystem() - // Create defautls objects. + // The default objects: a camera, which every path needs (on the headset it carries the + // head pose), and a sun, which the configuration decides and the headset by default + // does not get: in passthrough the room's light is the ARKit probe's, and a sun the room + // does not have would light and shadow the scene until the app replaced it; in full + // immersion the environment carries the light, and a sun is the app's to add. + createDefaultEntities(sun: configuration.defaultSun ?? !renderInfo.isXRStereoMode) + + #if os(visionOS) + BatchingSystem.shared.applyRuntimeBatchingTuning(.visionOSBalanced) + #else + BatchingSystem.shared.applyRuntimeBatchingTuning(.macOSBalanced) + #endif + + Logger.log(message: "Untold Engine Starting. Version 0.23.0") + } + + /// The default camera, made active, and with `sun` the default directional light, made + /// active too: what a scene has before the app adds anything. + @discardableResult + func createDefaultEntities(sun: Bool) -> (camera: EntityID, sun: EntityID?) { let gameCamera = createEntity() setEntityName(entityId: gameCamera, name: "Main Camera") createGameCamera(entityId: gameCamera) + setCamera(.active(gameCamera)) + guard sun else { return (gameCamera, nil) } let light = createEntity() setEntityName(entityId: light, name: "Directional Light") createDirLight(entityId: light) - - setCamera(.active(gameCamera)) setDirectionalLight(.active(light)) - - #if os(visionOS) - BatchingSystem.shared.applyRuntimeBatchingTuning(.visionOSBalanced) - #else - BatchingSystem.shared.applyRuntimeBatchingTuning(.macOSBalanced) - #endif - - Logger.log(message: "Untold Engine Starting. Version 0.22.0") + return (gameCamera, light) } public func initSizeableResources() { @@ -221,38 +263,43 @@ public class UntoldRenderer: NSObject, MTKViewDelegate { reinitSSAOTextures() } - func calculateDeltaTime() { + /// The period assumed for a loop that neither predicts a presentation time nor names its + /// display's rate. + static let defaultFramePeriod: TimeInterval = 1.0 / 60.0 + + /// Advances the frame clock to the frame's predicted presentation time โ€” the host's, or + /// one period ahead of now for a loop that predicts none โ€” and derives the frame's delta + /// from it: the time between two presentations, not between two updates, so a path + /// stepped by the delta moves by the display's period from one frame to the next. + /// `nominalPeriod` is the host display's frame interval; nil lets the clock learn it from + /// the presentation times. + func calculateDeltaTime(presentationTime: TimeInterval?, nominalPeriod: TimeInterval?) { if !firstUpdateCall { - // init the time properties for the update - - timeSinceLastUpdate = 0.0 - - timeSinceLastUpdatePreviousTime = CACurrentMediaTime() - + // The first frame after a reset: no time has passed. + frameClock.reset() firstUpdateCall = true - - // init fps time properties frameCount = 0 timePassedSinceLastFrame = 0.0 + } + let frame: FrameTime + if let presentationTime { + frameClock.nominalPeriod = nominalPeriod + frame = frameClock.advance(toPresentation: presentationTime) } else { - // figure out the time since we last we drew - let currentTime: TimeInterval = CACurrentMediaTime() - - timeSinceLastUpdate = Float(currentTime - timeSinceLastUpdatePreviousTime) - - // keep track of the time interval between draws - timeSinceLastUpdatePreviousTime = currentTime - - // get fps - timePassedSinceLastFrame += Float(timeSinceLastUpdate) - - if timePassedSinceLastFrame > 0.1 { - // let fps:Float=Float(frameCount)/timePassedSinceLastFrame + let period = nominalPeriod ?? Self.defaultFramePeriod + frameClock.nominalPeriod = period + frame = frameClock.advance(now: CACurrentMediaTime(), lead: period) + } + currentFrameTime = frame + timeSinceLastUpdate = Float(frame.deltaTime) + timeSinceLastUpdatePreviousTime = frame.presentationTime - frameCount = 0 - timePassedSinceLastFrame = 0.0 - } + // fps bookkeeping + timePassedSinceLastFrame += Float(frame.deltaTime) + if timePassedSinceLastFrame > 0.1 { + frameCount = 0 + timePassedSinceLastFrame = 0.0 } } @@ -294,9 +341,12 @@ public class UntoldRenderer: NSObject, MTKViewDelegate { let drawStats = RenderStatsCollector.shared.snapshot() let tileRenderCosts = auditVisibleTileRenderCosts() let memStats = MemoryBudgetManager.shared.getStats() + let frame = currentFrameTime EngineStatsMonitor.shared.update { snapshot in snapshot.timing.frameTotalMs = frameTotalMs + snapshot.timing.presentationDeltaMs = frame.deltaTime * 1000.0 + snapshot.timing.skippedFrames = frame.skippedFrames snapshot.render.drawCallsTotal = drawStats.drawCallsTotal snapshot.render.drawCallsOpaque = drawStats.drawCallsOpaque @@ -463,8 +513,13 @@ public class UntoldRenderer: NSObject, MTKViewDelegate { #endif } + /// Runs one frame: the simulation posed for `presentationTime` โ€” the instant the host + /// loop predicts the frame will be shown, or one period ahead of now when nil โ€” then the + /// render hooks. `nominalPeriod` is the host display's frame interval, if it knows one. @discardableResult - private func runFrame(beforeRender: (() -> Void)? = nil, + private func runFrame(presentationTime: TimeInterval? = nil, + nominalPeriod: TimeInterval? = nil, + beforeRender: (() -> Void)? = nil, render: (() -> Void)? = nil, afterRender: (() -> Void)? = nil, statsLifecycle: StatsLifecycleMode = .internalManaged) -> Bool @@ -588,7 +643,7 @@ public class UntoldRenderer: NSObject, MTKViewDelegate { let updateStart = CACurrentMediaTime() #endif EngineProfiler.shared.beginScope(.update) - calculateDeltaTime() + calculateDeltaTime(presentationTime: presentationTime, nominalPeriod: nominalPeriod) traverseSceneGraph() handleInputCallback?() @@ -657,12 +712,13 @@ public class UntoldRenderer: NSObject, MTKViewDelegate { // user game update gameUpdateCallback?(timeSinceLastUpdate) + frameUpdateCallback?(currentFrameTime) } // Per-frame view update event โ€” fires every frame regardless of gameMode // (RealityKit SceneEvents.Update semantics) if frameEvents.hasSubscribers { - frameEvents.dispatch(UpdateEvent(deltaTime: TimeInterval(timeSinceLastUpdate), renderer: self)) + frameEvents.dispatch(UpdateEvent(deltaTime: TimeInterval(timeSinceLastUpdate), renderer: self, frameTime: currentFrameTime)) } EngineProfiler.shared.endScope(.update) #if ENGINE_STATS_ENABLED @@ -704,7 +760,14 @@ public class UntoldRenderer: NSObject, MTKViewDelegate { // In XR, UntoldEngineXR.renderNewFrame() dispatches this to the main thread // separately, since its runLoop() runs on the compositor render thread. ProgressiveAssetLoader.shared.tick() + // The host's display link target, if it gave one; otherwise the frame is posed one + // of the view's periods ahead of now. + let presentationTime = predictedPresentationTime + predictedPresentationTime = nil + let period = 1.0 / Double(max(view.preferredFramesPerSecond, 1)) _ = runFrame( + presentationTime: presentationTime, + nominalPeriod: period, beforeRender: { [weak self] in self?.delegate?.willDraw(in: view) }, render: { [weak self] in guard let self else { return } @@ -727,6 +790,7 @@ public class UntoldRenderer: NSObject, MTKViewDelegate { : matrixPerspectiveRightHand(fovyRadians: degreesToRadians(degrees: fov), aspectRatio: aspect, nearZ: near, farZ: far) renderInfo.perspectiveSpace = projectionMatrix + TemporalAntiAliasing.shared.projectionDidChange() let viewPortSize: simd_float2 = simd_make_float2(Float(size.width), Float(size.height)) renderInfo.viewPort = viewPortSize @@ -778,14 +842,17 @@ public class UntoldRenderer: NSObject, MTKViewDelegate { return renderer } + /// Runs the frame's simulation posed for `presentationTime`, the instant the host loop + /// predicts the frame will be shown (nil: one period ahead of now). @discardableResult - public func updateXR(useExternalStatsLifecycle: Bool = false) -> Bool { - runFrame(statsLifecycle: useExternalStatsLifecycle ? .externalManaged : .internalManaged) + public func updateXR(presentationTime: TimeInterval? = nil, useExternalStatsLifecycle: Bool = false) -> Bool { + ObjectMotion.shared.beginFrame() + return runFrame(presentationTime: presentationTime, statsLifecycle: useExternalStatsLifecycle ? .externalManaged : .internalManaged) } @discardableResult - public func updateXR(render: @escaping () -> Void, useExternalStatsLifecycle: Bool = false) -> Bool { - runFrame(render: render, statsLifecycle: useExternalStatsLifecycle ? .externalManaged : .internalManaged) + public func updateXR(presentationTime: TimeInterval? = nil, render: @escaping () -> Void, useExternalStatsLifecycle: Bool = false) -> Bool { + runFrame(presentationTime: presentationTime, render: render, statsLifecycle: useExternalStatsLifecycle ? .externalManaged : .internalManaged) } /// Syncs the physical headset world position into the ECS camera components used by @@ -830,6 +897,7 @@ public class UntoldRenderer: NSObject, MTKViewDelegate { eyeIndex: Int ) { renderInfo.perspectiveSpace = projectionMatrix + TemporalAntiAliasing.shared.projectionDidChange() guard let camera = CameraSystem.shared.activeCamera, let cameraComponent = scene.get(component: CameraComponent.self, for: camera) else { handleError(.noActiveCamera) diff --git a/Sources/UntoldEngine/Renderer/UntoldRendererConfig.swift b/Sources/UntoldEngine/Renderer/UntoldRendererConfig.swift index b27557650..72fd9a268 100644 --- a/Sources/UntoldEngine/Renderer/UntoldRendererConfig.swift +++ b/Sources/UntoldEngine/Renderer/UntoldRendererConfig.swift @@ -16,17 +16,25 @@ public struct UntoldRendererConfig { var updateRenderingSystemCallback: UpdateRenderingSystemCallback var updateXRRenderingSystemCallback: UpdateXRRenderingSystemCallback? + /// Whether the scene starts with a directional light, made active, beside its camera. + /// Unset, the platform decides: a sun on every path but the headset, where the + /// passthrough room is lit by its probe and the immersive environment by its own light, + /// so a sun is the app's to add. A project sets it to keep a sun on the headset (for + /// shadows its environment cannot cast) or to start without one elsewhere. + public var defaultSun: Bool? public init( metalView: MTKView? = nil, initPipelineBlocks: [(RenderPipelineType, RenderPipelineInitBlock)], updateRenderingSystemCallback: @escaping UpdateRenderingSystemCallback, - updateXRRenderingSystemCallback: UpdateXRRenderingSystemCallback? = nil + updateXRRenderingSystemCallback: UpdateXRRenderingSystemCallback? = nil, + defaultSun: Bool? = nil ) { self.metalView = metalView initRenderPipelineBlocks = initPipelineBlocks self.updateRenderingSystemCallback = updateRenderingSystemCallback self.updateXRRenderingSystemCallback = updateXRRenderingSystemCallback + self.defaultSun = defaultSun } } diff --git a/Sources/UntoldEngine/Renderer/XRFoveation.swift b/Sources/UntoldEngine/Renderer/XRFoveation.swift new file mode 100644 index 000000000..c591ff6b5 --- /dev/null +++ b/Sources/UntoldEngine/Renderer/XRFoveation.swift @@ -0,0 +1,183 @@ +// +// XRFoveation.swift +// UntoldEngine +// +// Variable-rate rasterization in XR. With foveation enabled in its layer configuration, the +// visionOS compositor hands the engine, per eye and per frame, a rasterization rate map and +// colour and depth textures of the map's physical size โ€” smaller than the screen the eye +// sees. The GPU draws through the map, dense where the eyes look and coarse in the periphery, +// and the compositor unwarps the texture on the way to the display; a layer configured this +// way also accepts a render quality above the platform default, which a uniform layer does not. +// +// What this asks of the renderer: every pass that rasterizes scene geometry at drawable size +// attaches the eye's map to its descriptor and sets its viewport in the map's screen space +// (`applyXRFoveationToSceneRenderPassDescriptors`, `applyXRFoveationViewport`). The full-screen +// passes that follow โ€” SSAO, post-processing, anti-aliasing, the output transform into the +// compositor's texture โ€” read and write the physical textures one pixel to one and never see +// the map; it is already baked into the layout of what they copy. Two kinds of shader do have +// to know: one that turns a texel's position back into a direction in space (SSAO rebuilds view +// positions from depth) and the occlusion culls, which project boxes and splats to the screen +// and sample a depth pyramid that is laid out in the map's physical space. Both decode the +// map's parameters (`rasterization_rate_map_decoder`, bound by `bindRateMap`), and the pyramid +// carries the map it was drawn through into the next frame's culls (`HZBPyramidFrame`). +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Metal +import simd + +/// A rasterization rate map's parameter data in a buffer the shaders decode, with the two sizes +/// a decode needs. Immutable once made, so it can travel with the depth pyramid built through +/// it (`HZBPyramidFrame.rateMapData`) into the culls of the frame after. +public final class XRRasterizationRateMapData: @unchecked Sendable { + /// The map's parameters, as `copyParameterData(buffer:offset:)` wrote them at offset 0. + public let buffer: MTLBuffer + /// The map's screen size in pixels: the space viewports and clip-space positions live in. + public let screenSize: simd_float2 + /// The size in pixels of the textures drawn through the map (layer 0). + public let physicalSize: simd_float2 + + public init(buffer: MTLBuffer, screenSize: simd_float2, physicalSize: simd_float2) { + self.buffer = buffer + self.screenSize = screenSize + self.physicalSize = physicalSize + } + + /// Copies `rateMap`'s parameters into `buffer`, which must be shared and at least + /// `rateMap.parameterDataSizeAndAlign.size` bytes long; nil when it is shorter. + public convenience init?(rateMap: MTLRasterizationRateMap, buffer: MTLBuffer) { + let size = rateMap.parameterDataSizeAndAlign.size + guard size > 0, buffer.length >= size else { return nil } + rateMap.copyParameterData(buffer: buffer, offset: 0) + let screen = rateMap.screenSize + let physical = rateMap.physicalSize(layer: 0) + self.init( + buffer: buffer, + screenSize: simd_float2(Float(screen.width), Float(screen.height)), + physicalSize: simd_float2(Float(physical.width), Float(physical.height)) + ) + } + + /// What the shaders bind next to the buffer: the screen size in xy, the physical size in + /// zw. A zero x is the "no map" word (`rateMapSizes(nil)`). + public var sizes: simd_float4 { + simd_float4(screenSize.x, screenSize.y, physicalSize.x, physicalSize.y) + } +} + +/// The rate map of the eye being drawn, as `UntoldEngineXR` hands it to the renderer for each +/// eye of each frame (`renderInfo.xrFoveation`); nil when the frame is drawn uniformly โ€” the +/// layer configured without foveation, the simulator, or `.foveatedRendering(.disabled)`. +public struct XRFoveationFrame { + public let rateMap: MTLRasterizationRateMap + /// The eye's viewport in the map's screen space (`view.textureMap.viewport`), which every + /// encoder drawing through the map sets explicitly: the default is the physical texture's. + public let viewport: MTLViewport + public let rateMapData: XRRasterizationRateMapData + + public init(rateMap: MTLRasterizationRateMap, viewport: MTLViewport, rateMapData: XRRasterizationRateMapData) { + self.rateMap = rateMap + self.viewport = viewport + self.rateMapData = rateMapData + } + + public var screenSize: simd_float2 { + rateMapData.screenSize + } + + public var physicalSize: simd_float2 { + rateMapData.physicalSize + } +} + +/// Attaches the current eye's rate map to the descriptors of the passes that rasterize the scene +/// at drawable size โ€” the G-buffer + light pass, the deferred colour the transparency, wireframe +/// and debug passes draw into, the environment, sky or grid background, the splats and the +/// gizmos โ€” or detaches it when the frame has none. The full-screen passes keep their +/// descriptors untouched: they copy physical pixels one to one. Called for every eye once its +/// graph is built, since a rebuild may have recreated the descriptors. +func applyXRFoveationToSceneRenderPassDescriptors() { + let rateMap = renderInfo.xrFoveation?.rateMap + for descriptor in [ + renderInfo.offscreenRenderPassDescriptor, + renderInfo.forwardRenderPassDescriptor, + renderInfo.deferredRenderPassDescriptor, + renderInfo.environmentRenderPassDescriptor, + renderInfo.gaussianRenderPassDescriptor, + renderInfo.gizmoRenderPassDescriptor, + ] { + guard let descriptor else { continue } + attachXRFoveation(rateMap, to: descriptor) + } +} + +/// Attaches a rate map to a descriptor whose attachments are physical-size textures, or +/// detaches it. A descriptor's render target size is read in *screen* pixels while a map is +/// attached (measured: a target size equal to the physical size clips the scene to the +/// top-left physical-size corner of the screen), so the size is left at 0 โ€” Metal then takes +/// the attachments' own size โ€” and put back to the texture size without a map, which is what +/// `createRenderPassDescriptor` sets and the memoryless attachments rely on. +func attachXRFoveation(_ rateMap: MTLRasterizationRateMap?, to descriptor: MTLRenderPassDescriptor) { + descriptor.rasterizationRateMap = rateMap + if rateMap != nil { + descriptor.renderTargetWidth = 0 + descriptor.renderTargetHeight = 0 + } else if let viewPort = renderInfo.viewPort, descriptor.renderTargetWidth == 0 { + descriptor.renderTargetWidth = Int(viewPort.x) + descriptor.renderTargetHeight = Int(viewPort.y) + } +} + +/// Sets the eye's screen-space viewport on an encoder that draws through the rate map; nothing +/// outside foveated XR, where the default viewport, the texture's own size, is the right one. +/// (Measured: a viewport of the physical size under a map clips the scene the same way a +/// physical-size render target does; the map's screen size covers the whole texture.) +@inline(__always) +func applyXRFoveationViewport(_ encoder: MTLRenderCommandEncoder) { + guard let foveation = renderInfo.xrFoveation else { return } + encoder.setViewport(foveation.viewport) +} + +/// The sizes word of a map, or the "no map" word the shaders test for. +@inline(__always) +func rateMapSizes(_ data: XRRasterizationRateMapData?) -> simd_float4 { + data?.sizes ?? .zero +} + +/// The rate map the current depth pyramid was drawn through, for the culls that sample it. +@inline(__always) +func hzbPyramidRateMapData() -> XRRasterizationRateMapData? { + renderInfo.hzbFrame?.rateMapData +} + +/// Binds a map's parameters and sizes to a kernel, or a placeholder and the "no map" word when +/// there is none: Metal wants every declared buffer bound, and the kernels read the parameters +/// only when `sizes.x > 0`. +func bindRateMap(_ data: XRRasterizationRateMapData?, to encoder: MTLComputeCommandEncoder, dataIndex: Int, sizesIndex: Int) { + var sizes = rateMapSizes(data) + encoder.setBuffer(data?.buffer ?? rateMapPlaceholderBuffer(), offset: 0, index: dataIndex) + encoder.setBytes(&sizes, length: MemoryLayout.stride, index: sizesIndex) +} + +/// The fragment-stage form of `bindRateMap(_:to:dataIndex:sizesIndex:)`. +func bindRateMap(_ data: XRRasterizationRateMapData?, toFragmentsOf encoder: MTLRenderCommandEncoder, dataIndex: Int, sizesIndex: Int) { + var sizes = rateMapSizes(data) + encoder.setFragmentBuffer(data?.buffer ?? rateMapPlaceholderBuffer(), offset: 0, index: dataIndex) + encoder.setFragmentBytes(&sizes, length: MemoryLayout.stride, index: sizesIndex) +} + +/// A small shared buffer bound wherever a shader declares a rate map it will not read. +func rateMapPlaceholderBuffer() -> MTLBuffer? { + if let buffer = bufferResources.rateMapPlaceholder { + return buffer + } + guard let device = renderInfo.device else { return nil } + let buffer = device.makeBuffer(length: 256, options: .storageModeShared) + buffer?.label = "Rate Map Placeholder" + bufferResources.rateMapPlaceholder = buffer + return buffer +} diff --git a/Sources/UntoldEngine/Renderer/XRFramePacing.swift b/Sources/UntoldEngine/Renderer/XRFramePacing.swift new file mode 100644 index 000000000..b7e0c3a09 --- /dev/null +++ b/Sources/UntoldEngine/Renderer/XRFramePacing.swift @@ -0,0 +1,149 @@ +// +// XRFramePacing.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Foundation + +/// How the XR loop paces its frames against the display's refreshes. A frame whose GPU work +/// does not fit one refresh is shown a refresh late, sometimes, which hops; rendered every +/// other refresh it is shown two refreshes apart, evenly, posed for that instant, with the +/// compositor reprojecting the head in between. See `RenderingProperty.framePacing`. +public enum XRFramePacing: Sendable, Equatable { + /// Every refresh, whatever the frames' cost. + case fullRate + /// Every refresh while the frames meet the deadline; every other one while they do not. + case halfRateWhenLate + /// Every other refresh, always. + case halfRate +} + +/// The half-rate decision from each second's deadline measurements. Enter when, with no wait +/// for the optimal input time left to skip, the display skipped a refresh for one frame in +/// ten or more, or one frame in ten or more missed its presentation (finishing past the +/// compositor's deadline by more than the deadline's margin before the presentation, less a +/// safety), for two seconds running: fewer hops than that are fewer than the steps the half +/// rate would add. Leave when the rendered frames' mean GPU work has fitted 80% of the +/// refresh for two seconds running. While active, the refreshes +/// between two frames follow the measured GPU cost: two, or more when a frame's typical work +/// (the mean span plus a fifth) with its encode does not fit two. The mean, not the longest +/// span: a GPU used on half the refreshes runs at a lower clock, and its longest spans would +/// never earn back the shorter cadence that would raise the clock again. +public struct XRHalfRatePolicy: Equatable, Sendable { + /// Whether the frames are being paced to every other refresh (or more). + public private(set) var active = false + /// Refreshes between two frames while active: the fewest the frame's typical GPU work + /// with its encode fits, two at least, four at most. + public private(set) var cadenceRefreshes = 2 + /// Refreshes after the compositor's predicted presentation at which the paced frames are + /// shown, from how far past their deadline they finish beyond the deadline's margin before + /// the presentation: the pose aims there. + public private(set) var landingRefreshes = 0 + private(set) var lateSeconds = 0 + private(set) var fitSeconds = 0 + /// Seconds running in which a shorter cadence than the current one would have done. + private(set) var shorterSeconds = 0 + + public init() {} + + /// What a second's measurements changed. + public enum Change: Equatable, Sendable { + case none + case entered + case left + } + + /// The share of a refresh the rendered frames' GPU work must fit for the pacing to end. + public static let fitShare = 0.8 + /// The CPU encode allowed for between the frame's query and its GPU work, in milliseconds, + /// when the cadence is chosen. + public static let encodeAllowanceMs = 3.0 + /// The share above the mean GPU span the cadence allows for the longer frames. + public static let spanMargin = 0.2 + public static let mostRefreshes = 4 + /// How far past a cadence's boundary the typical frame must be for the cadence to change, + /// in milliseconds: a frame on the boundary would otherwise change it every few seconds. + public static let cadenceDeadbandMs = 1.0 + /// The part of the deadline's margin before the presentation not counted on, in + /// milliseconds: a frame past the deadline by less than the margin less this is taken as + /// shown on time. + public static let marginSafetyMs = 2.0 + + /// How far past the deadline a frame may finish and still be taken as shown on time. + public static func usableMarginMs(deadlineMarginMs: Double) -> Double { + max(0, deadlineMarginMs - marginSafetyMs) + } + + /// A second of frames ended: `frames` rendered, `missed` of them past the deadline by more + /// than its usable margin, `skipped` refreshes the display showed no new frame at beyond + /// the cadence, the frames past the deadline by `meanOverrunMs` on average, their GPU + /// spans `meanGPUDurationMs` on average, the display's refresh `periodMs`, the deadline + /// `deadlineMarginMs` before the presentation, and whether the loop still has a wait it + /// could skip first. + public mutating func secondEnded(frames: Int, missed: Int, skipped: Int, meanOverrunMs: Double, meanGPUDurationMs: Double, periodMs: Double, deadlineMarginMs: Double, waitStillToSkip: Bool) -> Change { + guard frames > 0 else { return .none } + let hopping = skipped * 10 >= frames || missed * 10 >= frames + if !active { + cadenceRefreshes = 2 + landingRefreshes = 0 + if !waitStillToSkip, hopping { + lateSeconds += 1 + } else { + lateSeconds = 0 + } + if lateSeconds >= 2 { + active = true + lateSeconds = 0 + return .entered + } + return .none + } + if periodMs > 0 { + // Where the frames land: a frame finished past its deadline by more than the + // deadline's usable margin is shown at the next refresh the rest reaches; one + // within the margin makes its predicted one. + let beyond = meanOverrunMs - Self.usableMarginMs(deadlineMarginMs: deadlineMarginMs) + landingRefreshes = beyond > 0 ? min(Self.mostRefreshes - 1, Int((beyond / periodMs).rounded(.up))) : 0 + // The cadence rises at once when the frame's work needs more refreshes, and falls + // only after two seconds running that fit the shorter one: a cadence that changed + // every second would itself be a hop. Both ways with a deadband past the boundary. + let typical = meanGPUDurationMs * (1 + Self.spanMargin) + Self.encodeAllowanceMs + let current = Double(cadenceRefreshes) * periodMs + if typical > current + Self.cadenceDeadbandMs { + cadenceRefreshes = min(Self.mostRefreshes, Int(((typical - Self.cadenceDeadbandMs) / periodMs).rounded(.up))) + shorterSeconds = 0 + } else if cadenceRefreshes > 2, typical < current - periodMs - Self.cadenceDeadbandMs { + shorterSeconds += 1 + if shorterSeconds >= 2 { + cadenceRefreshes = max(2, Int(((typical + Self.cadenceDeadbandMs) / periodMs).rounded(.up))) + shorterSeconds = 0 + } + } else { + shorterSeconds = 0 + } + } + if periodMs > 0, meanGPUDurationMs < Self.fitShare * periodMs { + fitSeconds += 1 + } else { + fitSeconds = 0 + } + if fitSeconds >= 2 { + active = false + fitSeconds = 0 + shorterSeconds = 0 + cadenceRefreshes = 2 + landingRefreshes = 0 + return .left + } + return .none + } + + public mutating func reset() { + self = XRHalfRatePolicy() + } +} diff --git a/Sources/UntoldEngine/Scenes/SceneSerializer.swift b/Sources/UntoldEngine/Scenes/SceneSerializer.swift index 53e026d6a..1d0106ade 100644 --- a/Sources/UntoldEngine/Scenes/SceneSerializer.swift +++ b/Sources/UntoldEngine/Scenes/SceneSerializer.swift @@ -117,6 +117,8 @@ enum AntiAliasingModeData: String, Codable { case fxaa case smaa case msaa + case taa + case msaaTaa } struct FXAAData: Codable { @@ -778,6 +780,10 @@ private func applyDeserializedMaterialData(entityId: EntityID, entityData: Entit if let heightEnabled = materialData.heightEnabled { updateMaterialHeightEnabled(entityId: entityId, heightEnabled: heightEnabled) + } else if materialData.heightURL != nil { + // Legacy scenes saved before `heightEnabled` existed relied on POM + // activating implicitly whenever a height texture was present. + updateMaterialHeightEnabled(entityId: entityId, heightEnabled: true) } if let heightRemapMin = materialData.heightRemapMin { @@ -997,10 +1003,10 @@ public func serializeScene() -> SceneData { } // Kinetic properties - entityData.mass = getMass(entityId: entityId) - entityData.hasKineticComponent = hasComponent(entityId: entityId, componentType: KineticComponent.self) + entityData.mass = entityData.hasKineticComponent ? getMass(entityId: entityId) : 0.0 + // Dir Light properties let hasDirLight: Bool = hasComponent(entityId: entityId, componentType: DirectionalLightComponent.self) @@ -1384,6 +1390,10 @@ public func serializeScene() -> SceneData { antiAliasingModeData = .smaa case .msaa: antiAliasingModeData = .msaa + case .taa: + antiAliasingModeData = .taa + case .msaaTaa: + antiAliasingModeData = .msaaTaa } sceneData.antiAliasing = AntiAliasingData( mode: antiAliasingModeData, @@ -1696,6 +1706,10 @@ public func deserializeScene( antiAliasingMode = .smaa case .msaa: antiAliasingMode = .msaa + case .taa: + antiAliasingMode = .taa + case .msaaTaa: + antiAliasingMode = .msaaTaa } FXAAParams.shared.subpixelQuality = antiAliasing.fxaa.subpixelQuality diff --git a/Sources/UntoldEngine/Shaders/BitonicSort.metal b/Sources/UntoldEngine/Shaders/BitonicSort.metal index 7b04e7c53..babdf2d4e 100644 --- a/Sources/UntoldEngine/Shaders/BitonicSort.metal +++ b/Sources/UntoldEngine/Shaders/BitonicSort.metal @@ -19,6 +19,7 @@ #include "../../CShaderTypes/ShaderTypes.h" #include "ShaderStructs.h" #include "ShadersUtils.h" +#include "HZBOcclusion.h" using namespace metal; @@ -73,14 +74,18 @@ kernel void gaussianFinalizeVisibleSet( // (one thread per resident splat, head-centre view) and gaussianChunkDecodePreprocess // (GaussianChunkPreprocess.metal, one threadgroup per visible chunk, either eye) so the two // paths keep exactly the same splats: centre inside the guard-banded clip volume, then -// optionally not behind the previous frame's HZB. +// optionally not behind the previous frame's HZB. `hzbRateMap` and `hzbRateMapSizes` are the +// rasterization rate map the pyramid was drawn through, if any (HZBOcclusion.h): the centre's +// screen UV is moved into the pyramid's physical layout before the sample. inline bool gaussianClipCentrePassesCull( float4 centerClip, float clipGuardBand, uint hzbReverseZ, float hzbOcclusionBias, uint hzbValid, - texture2d hzbDepthPyramid) + texture2d hzbDepthPyramid, + constant rasterization_rate_map_data *hzbRateMap, + float4 hzbRateMapSizes) { if (centerClip.w <= 0.0f) return false; @@ -100,6 +105,9 @@ inline bool gaussianClipCentrePassesCull( // source of truth for partial occlusion. if (hzbValid != 0u) { float2 uv = float2(ndc.x * 0.5f + 0.5f, 1.0f - (ndc.y * 0.5f + 0.5f)); + if (hzbRateMapActive(hzbRateMapSizes)) { + uv = hzbScreenUVToPhysicalUV(hzbRateMap, hzbRateMapSizes, uv); + } float splatDepth = clamp(centerClip.z / centerClip.w, 0.0f, 1.0f); constexpr sampler pointSampler(coord::normalized, address::clamp_to_edge, filter::nearest); float hzbDepth = hzbDepthPyramid.sample(pointSampler, uv, level(0)).x; @@ -120,12 +128,14 @@ inline bool gaussianSplatPassesCull( uint hzbReverseZ, float hzbOcclusionBias, uint hzbValid, - texture2d hzbDepthPyramid) + texture2d hzbDepthPyramid, + constant rasterization_rate_map_data *hzbRateMap, + float4 hzbRateMapSizes) { float4 centerClip = uniforms.projectionMatrix * uniforms.modelViewMatrix * float4(position, 1.0f); - return gaussianClipCentrePassesCull(centerClip, clipGuardBand, hzbReverseZ, hzbOcclusionBias, hzbValid, hzbDepthPyramid); + return gaussianClipCentrePassesCull(centerClip, clipGuardBand, hzbReverseZ, hzbOcclusionBias, hzbValid, hzbDepthPyramid, hzbRateMap, hzbRateMapSizes); } // One thread per resident splat: the whole-buffer cull for entities without a chunk table @@ -141,11 +151,13 @@ kernel void gaussianFrustumCull( constant uint &hzbReverseZ [[buffer(gaussianCullHZBReverseZIndex)]], constant float &hzbOcclusionBias [[buffer(gaussianCullHZBOcclusionBiasIndex)]], constant uint &hzbValid [[buffer(gaussianCullHZBValidIndex)]], + constant rasterization_rate_map_data *hzbRateMap [[buffer(gaussianCullHZBRateMapDataIndex)]], + constant float4 &hzbRateMapSizes [[buffer(gaussianCullHZBRateMapSizesIndex)]], texture2d hzbDepthPyramid [[texture(gaussianCullHZBDepthPyramidTextureIndex)]], uint index [[thread_position_in_grid]]) { if (index >= numOfSplats) return; - if (!gaussianSplatPassesCull(splats[index].position, uniforms, clipGuardBand, hzbReverseZ, hzbOcclusionBias, hzbValid, hzbDepthPyramid)) return; + if (!gaussianSplatPassesCull(splats[index].position, uniforms, clipGuardBand, hzbReverseZ, hzbOcclusionBias, hzbValid, hzbDepthPyramid, hzbRateMap, hzbRateMapSizes)) return; uint writeIndex = atomic_fetch_add_explicit(visibleCount, 1u, memory_order_relaxed); visibleIndices[writeIndex] = index; diff --git a/Sources/UntoldEngine/Shaders/GaussianChunkCull.metal b/Sources/UntoldEngine/Shaders/GaussianChunkCull.metal index 49e4e3a2a..0d1602f75 100644 --- a/Sources/UntoldEngine/Shaders/GaussianChunkCull.metal +++ b/Sources/UntoldEngine/Shaders/GaussianChunkCull.metal @@ -93,6 +93,8 @@ static inline bool gaussianBoxPassesClipPlanes( // the HZB. The occlusion test is the mesh cull's (HZBOcclusion.h): the box's screen rect at the // mip that covers it, 5x5 samples, nearest box depth against the farthest sample plus the bias. // `area` is the box's clipped screen area in this view, 0 when the view rejects or occludes it. +// `hzbRateMap` is the rasterization rate map the pyramid was drawn through when it was +// (params.hzbRateMapSizes, HZBOcclusion.h): the rect is moved into the pyramid's layout. static inline bool gaussianChunkVisibleInView( const float3 boxMin, const float3 boxMax, @@ -100,6 +102,7 @@ static inline bool gaussianChunkVisibleInView( constant GaussianChunkCullConstants ¶ms, const uint hzbValid, texture2d hzbDepthPyramid, + constant rasterization_rate_map_data *hzbRateMap, thread float &area) { if (!gaussianBoxPassesClipPlanes(boxMin, boxMax, viewProjection, params.clipGuardBand, area)) { @@ -116,6 +119,7 @@ static inline bool gaussianChunkVisibleInView( if (!projectAABBToScreenRect(0.5f * (boxMin + boxMax), 0.5f * (boxMax - boxMin), viewProjection, reverseZ, uvMin, uvMax, nearDepth)) { return true; } + hzbRemapRectToPhysical(hzbRateMap, params.hzbRateMapSizes, uvMin, uvMax); if (hzbRectIsOccluded(hzbDepthPyramid, uvMin, uvMax, nearDepth, params.viewport, params.hzbMipCount, reverseZ, params.hzbOcclusionBias)) { area = 0.0f; return false; @@ -189,6 +193,7 @@ kernel void gaussianChunkCull( const device GaussianChunkLevelState *levelState [[buffer(gaussianChunkCullLevelStateIndex)]], constant GaussianChunkLevelConstants &lvl [[buffer(gaussianChunkCullLevelConstantsIndex)]], const device GaussianChunkCullRange *ranges [[buffer(gaussianChunkCullRangesIndex)]], + constant rasterization_rate_map_data *hzbRateMap [[buffer(gaussianChunkCullRateMapDataIndex)]], texture2d hzbDepthPyramid [[texture(gaussianChunkCullHZBDepthPyramidTextureIndex)]], uint chunkIndex [[thread_position_in_grid]]) { @@ -227,9 +232,9 @@ kernel void gaussianChunkCull( const uint hzbValidForView0 = params.viewCount > 1u ? 0u : params.hzbValid; float area0 = 0.0f; float area1 = 0.0f; - const bool keep0 = gaussianChunkVisibleInView(boxMin, boxMax, params.viewProjection0, params, hzbValidForView0, hzbDepthPyramid, area0); + const bool keep0 = gaussianChunkVisibleInView(boxMin, boxMax, params.viewProjection0, params, hzbValidForView0, hzbDepthPyramid, hzbRateMap, area0); const bool keep1 = params.viewCount > 1u - ? gaussianChunkVisibleInView(boxMin, boxMax, params.viewProjection1, params, params.hzbValid, hzbDepthPyramid, area1) + ? gaussianChunkVisibleInView(boxMin, boxMax, params.viewProjection1, params, params.hzbValid, hzbDepthPyramid, hzbRateMap, area1) : false; const bool seen = keep0 || keep1; const float area = clamp(max(keep0 ? area0 : 0.0f, keep1 ? area1 : 0.0f), kGaussianScreenAreaMin, limit * limit); diff --git a/Sources/UntoldEngine/Shaders/GaussianChunkPreprocess.metal b/Sources/UntoldEngine/Shaders/GaussianChunkPreprocess.metal index c55a82bad..790b4435a 100644 --- a/Sources/UntoldEngine/Shaders/GaussianChunkPreprocess.metal +++ b/Sources/UntoldEngine/Shaders/GaussianChunkPreprocess.metal @@ -60,16 +60,17 @@ inline bool gaussianChunkSplatPassesCull( float3 position, constant Uniforms &uniforms, constant GaussianChunkCullConstants ¶ms, - texture2d hzbDepthPyramid) + texture2d hzbDepthPyramid, + constant rasterization_rate_map_data *hzbRateMap) { if (params.viewCount <= 1u) { - return gaussianSplatPassesCull(position, uniforms, params.clipGuardBand, params.hzbReverseZ, params.hzbOcclusionBias, params.hzbValid, hzbDepthPyramid); + return gaussianSplatPassesCull(position, uniforms, params.clipGuardBand, params.hzbReverseZ, params.hzbOcclusionBias, params.hzbValid, hzbDepthPyramid, hzbRateMap, params.hzbRateMapSizes); } const float4 local = float4(position, 1.0f); - if (gaussianClipCentrePassesCull(params.viewProjection0 * local, params.clipGuardBand, params.hzbReverseZ, params.hzbOcclusionBias, 0u, hzbDepthPyramid)) { + if (gaussianClipCentrePassesCull(params.viewProjection0 * local, params.clipGuardBand, params.hzbReverseZ, params.hzbOcclusionBias, 0u, hzbDepthPyramid, hzbRateMap, params.hzbRateMapSizes)) { return true; } - return gaussianClipCentrePassesCull(params.viewProjection1 * local, params.clipGuardBand, params.hzbReverseZ, params.hzbOcclusionBias, params.hzbValid, hzbDepthPyramid); + return gaussianClipCentrePassesCull(params.viewProjection1 * local, params.clipGuardBand, params.hzbReverseZ, params.hzbOcclusionBias, params.hzbValid, hzbDepthPyramid, hzbRateMap, params.hzbRateMapSizes); } kernel void gaussianChunkDecodePreprocess( @@ -93,6 +94,7 @@ kernel void gaussianChunkDecodePreprocess( const device GaussianChunkDecodeConstants *coarseTable [[buffer(gaussianChunkPreprocessCoarseTableIndex)]], const device GaussianChunkLevelState *levelState [[buffer(gaussianChunkPreprocessLevelStateIndex)]], constant GaussianChunkLevelConstants &lvl [[buffer(gaussianChunkPreprocessLevelConstantsIndex)]], + constant rasterization_rate_map_data *hzbRateMap [[buffer(gaussianChunkPreprocessRateMapDataIndex)]], texture2d hzbDepthPyramid [[texture(gaussianChunkPreprocessHZBDepthPyramidTextureIndex)]], uint chunkSlot [[threadgroup_position_in_grid]], uint localIndex [[thread_position_in_threadgroup]], @@ -161,7 +163,7 @@ kernel void gaussianChunkDecodePreprocess( // bit-identical to what the whole-buffer path loaded, and the covariance and colour // pass through half precision the way EncodedGaussianSplat stores them. const float3 centerLocal = mix(aabbMin, aabbMax, gaussianUnpack11_10_11(record.x)); - if (!gaussianChunkSplatPassesCull(centerLocal, uniforms, cull, hzbDepthPyramid)) { + if (!gaussianChunkSplatPassesCull(centerLocal, uniforms, cull, hzbDepthPyramid, hzbRateMap)) { continue; } diff --git a/Sources/UntoldEngine/Shaders/Gaussians.metal b/Sources/UntoldEngine/Shaders/Gaussians.metal index 9f5d06d78..e97b846c2 100644 --- a/Sources/UntoldEngine/Shaders/Gaussians.metal +++ b/Sources/UntoldEngine/Shaders/Gaussians.metal @@ -621,15 +621,6 @@ inline float calcPowerFromConic(float3 conic, float2 d) 2.0f * conic.y * d.x * d.y); } -inline float2 calcScreenSpaceDelta(float2 pixelPos, - float2 centerPixel, - float projYSign) -{ - float2 d = pixelPos - centerPixel; - d.y *= projYSign; - return d; -} - typedef struct { half4 color [[raster_order_group(0)]]; @@ -693,20 +684,20 @@ vertex GaussianOutData vertexGaussianTBDRShader( return out; } - const float projYSign = -1.0f; - float2 centerNDC = centerClip.xy / centerClip.w; - float2 centerUV = centerNDC * float2(0.5f, 0.5f * projYSign) + 0.5f; - out.coordxy = centerUV * viewport; - out.conic = record.conicAndOpacity.xyz; // Tight, rotated quad along the ellipse's true principal axes (see // computeInverseCovarianceConic) instead of an axis-aligned bounding box. The axes live - // in the pixel frame of the conic (y down, as coordxy and the fragment's position), while - // NDC y points up: the y offset flips sign on the way, or a tilted splat's quad is the - // mirror image of its ellipse and the fragment falloff gets clipped to their overlap. + // in the pixel frame of the conic (y down), while NDC y points up: the y offset flips + // sign on the way, or a tilted splat's quad is the mirror image of its ellipse and the + // fragment falloff gets clipped to their overlap. The offset itself goes to the fragment + // as a varying: interpolated across the quad it is the fragment's own offset from the + // centre in the conic's frame, whatever the rasterizer did to the quad's pixels โ€” under + // a rasterization rate map (XR foveation) the fragment's position is in the physical + // texture, not on the screen the centre was projected to. float2 pixelOffset = quad.x * record.axes.xy + quad.y * record.axes.zw; float2 ndcOffset = pixelOffset * float2(2.0f, -2.0f) / viewport; + out.pixelDelta = pixelOffset; out.position = centerClip; out.position.xy += ndcOffset * centerClip.w; out.color = record.color.xyz; @@ -745,8 +736,10 @@ fragment GaussianTBDRFragmentStore fragmentGaussianTBDRShader( // (gaussianAdaptiveSigma sigma out) โ€” has negligible alpha. Rejecting those tail fragments // here means they never pay for the depth-texture read at all, on top of never reaching // the blend math below. - const float projYSign = 1.0f; - float2 d = calcScreenSpaceDelta(in.position.xy, in.coordxy, projYSign); + // The fragment's offset from the splat's centre in the conic's pixel frame, from the + // quad's corners (see vertexGaussianTBDRShader) rather than from in.position, which a + // rasterization rate map (XR foveation) places in the physical texture, not on screen. + float2 d = in.pixelDelta; float power = calcPowerFromConic(in.conic, d); float falloff = exp(power); diff --git a/Sources/UntoldEngine/Shaders/GizmoShader.metal b/Sources/UntoldEngine/Shaders/GizmoShader.metal index a516f316b..5c4e35c70 100644 --- a/Sources/UntoldEngine/Shaders/GizmoShader.metal +++ b/Sources/UntoldEngine/Shaders/GizmoShader.metal @@ -19,7 +19,6 @@ vertex VertexOutModel vertexGizmoShader( float4 position = in.position; - out.vPosition = position; out.position = uniforms.projectionMatrix * uniforms.modelViewMatrix * position; return out; diff --git a/Sources/UntoldEngine/Shaders/HZBCompute.metal b/Sources/UntoldEngine/Shaders/HZBCompute.metal index 9cfea93a4..0ad63f72d 100644 --- a/Sources/UntoldEngine/Shaders/HZBCompute.metal +++ b/Sources/UntoldEngine/Shaders/HZBCompute.metal @@ -80,6 +80,8 @@ kernel void hzbCullVisibleEntities( constant uint &mipCount [[buffer(hzbCullPassMipCountIndex)]], constant uint &reverseZ [[buffer(hzbCullPassReverseZIndex)]], constant float &occlusionBias [[buffer(hzbCullPassOcclusionBiasIndex)]], + constant rasterization_rate_map_data *rateMap [[buffer(hzbCullPassRateMapDataIndex)]], + constant float4 &rateMapSizes [[buffer(hzbCullPassRateMapSizesIndex)]], texture2d hzbDepthPyramid [[texture(hzbCullPassDepthPyramidTextureIndex)]], uint tid [[thread_position_in_grid]] ) { @@ -99,7 +101,9 @@ kernel void hzbCullVisibleEntities( return; } - // Shared with the Gaussian chunk cull โ€” see HZBOcclusion.h. + // Shared with the Gaussian chunk cull โ€” see HZBOcclusion.h. A pyramid drawn through a + // rasterization rate map (XR foveation) holds the rect elsewhere than its screen UV. + hzbRemapRectToPhysical(rateMap, rateMapSizes, uvMin, uvMax); bool isOccluded = hzbRectIsOccluded(hzbDepthPyramid, uvMin, uvMax, nearDepth, viewport, mipCount, reverseZ != 0u, occlusionBias); if (!isOccluded) { uint dst = atomic_fetch_add_explicit(outVisibleCount, 1u, memory_order_relaxed); diff --git a/Sources/UntoldEngine/Shaders/HZBOcclusion.h b/Sources/UntoldEngine/Shaders/HZBOcclusion.h index 0e7c2dcb1..550428736 100644 --- a/Sources/UntoldEngine/Shaders/HZBOcclusion.h +++ b/Sources/UntoldEngine/Shaders/HZBOcclusion.h @@ -80,6 +80,41 @@ static inline bool projectAABBToScreenRect( return true; } +/// The rasterization rate map a depth pyramid was drawn through, when its frame was (visionOS +/// foveation): `rateMapSizes.xy` is the map's screen size and `.zw` the pyramid's base size, in +/// pixels, and `rateMapSizes.x <= 0` says there is none, in which case `rateMap` is a placeholder +/// that is never read. Such a pyramid is not laid out over the screen the boxes above project +/// to โ€” dense where the eyes look, sparse in the periphery โ€” so a screen UV must be moved to +/// where the map put it before the pyramid is sampled there. +static inline bool hzbRateMapActive(const float4 rateMapSizes) { + return rateMapSizes.x > 0.0f && rateMapSizes.z > 0.0f; +} + +static inline float2 hzbScreenUVToPhysicalUV( + constant rasterization_rate_map_data *rateMap, + const float4 rateMapSizes, + const float2 uv +) { + rasterization_rate_map_decoder decoder(*rateMap); + const float2 physical = decoder.map_screen_to_physical_coordinates(uv * rateMapSizes.xy); + return clamp(physical / rateMapSizes.zw, 0.0f, 1.0f); +} + +/// Moves a screen rect from `projectAABBToScreenRect` into the pyramid's physical layout; the +/// map is monotonic along each axis, so the two corners are enough. Nothing without a map. +static inline void hzbRemapRectToPhysical( + constant rasterization_rate_map_data *rateMap, + const float4 rateMapSizes, + thread float2 &uvMin, + thread float2 &uvMax +) { + if (!hzbRateMapActive(rateMapSizes)) { + return; + } + uvMin = hzbScreenUVToPhysicalUV(rateMap, rateMapSizes, uvMin); + uvMax = hzbScreenUVToPhysicalUV(rateMap, rateMapSizes, uvMax); +} + /// True when the rect `uvMin..uvMax` whose nearest depth is `nearDepth` lies behind the HZB /// everywhere it is sampled. The mip is the one whose texel spans the rect; a 5x5 grid across /// the rect keeps porous occluders (window frames, glass assemblies) from falsely culling what diff --git a/Sources/UntoldEngine/Shaders/LightShader.metal b/Sources/UntoldEngine/Shaders/LightShader.metal index cfc71e5cb..96ea155d8 100644 --- a/Sources/UntoldEngine/Shaders/LightShader.metal +++ b/Sources/UntoldEngine/Shaders/LightShader.metal @@ -14,6 +14,10 @@ #include "ShadersUtils.h" using namespace metal; +/// Per-fragment indirect lighting: the indirect light and the sun's shadow come from the +/// G-buffer, where the geometry wrote them once per fragment, instead of once per sample here. +constant bool kLightIndirectFromGBuffer [[function_constant(lightFunctionConstantIndirectFromGBuffer)]]; + // Normal-offset shadows: how many of the cascade's own world-space texels to push the // sampled point along the surface normal before the light-space lookup. This moves the // tested point off the surface instead of only fudging the depth comparison, so it stays @@ -62,12 +66,26 @@ float sampleCSMCascade( // into this cascade's normalized depth units so blended edges stay aligned. float cascadeDepthSpan = max(csm.cascadeDepthSpans[cascade], 1.0e-6); float normalizedBias = worldBias / cascadeDepthSpan; + // The taps: the whole Poisson disk, or every second or fourth point of it, which keeps + // the spread of the disk with fewer compares (each a hardware-filtered one). Each count + // is its own loop with a constant trip count, which the compiler unrolls; one loop over a + // count read from the uniforms ran a quarter slower at 16. + float2 tapScale = texelSize * cascadeFilterRadius; float shadow = 0.0; + if (csm.shadowFilterTaps <= 4.0) { + for (int i = 0; i < 4; ++i) { + shadow += shadowArray.sample_compare(shadowSampler, proj.xy + poissonDisk[i * 4] * tapScale, cascade, proj.z - normalizedBias); + } + return shadow / 4.0; + } + if (csm.shadowFilterTaps <= 8.0) { + for (int i = 0; i < 8; ++i) { + shadow += shadowArray.sample_compare(shadowSampler, proj.xy + poissonDisk[i * 2] * tapScale, cascade, proj.z - normalizedBias); + } + return shadow / 8.0; + } for (int i = 0; i < 16; ++i) { - float2 offset = poissonDisk[i] * texelSize * cascadeFilterRadius; - shadow += shadowArray.sample_compare( - shadowSampler, proj.xy + offset, cascade, proj.z - normalizedBias - ); + shadow += shadowArray.sample_compare(shadowSampler, proj.xy + poissonDisk[i] * tapScale, cascade, proj.z - normalizedBias); } return shadow / 16.0; } @@ -287,8 +305,12 @@ EnvironmentLight computeIBLParts(texture2d irradianceTexture, //compute ibl ambient contribution // The angle to the surface, whichever side of it is seen (see specularIBL). float NoV = min(abs(dot(normalMap.xyz, viewVector)), 1.0); - - float3 irradiance=diffuseIBL(normalMap.xyz, irradianceTexture, float3(0.0,1.0,0.0),degreesToRadians(iblRotationAngle)); + // The rotation of the environment, in radians, once; the lookup table's two values at + // this angle and roughness, once, for the reflection and the share it takes from diffuse. + float rotation = degreesToRadians(iblRotationAngle); + float2 brdf = brdfIntegration(NoV, roughness, iblBRDFTexture); + + float3 irradiance=diffuseIBL(normalMap.xyz, irradianceTexture, float3(0.0,1.0,0.0), rotation); float3 diffuse = irradiance*inBaseColor.rgb; @@ -297,7 +319,8 @@ EnvironmentLight computeIBLParts(texture2d irradianceTexture, // grows when the surface is seen at a slant. float3 f0 = mix(0.04, inBaseColor.rgb, metallic); - float3 specular=specularIBL(f0, roughness, normalMap.xyz, viewVector, specularTexture, iblBRDFTexture,float3(0.0,1.0,0.0),degreesToRadians(iblRotationAngle)); + float3 specular = blurredEnvironment(reflect(-viewVector, normalMap.xyz), roughness, specularTexture, float3(0.0,1.0,0.0), rotation) + * environmentReflectance(f0, brdf); // Every surface reflects the environment, a non-metal too: that is the sheen of // polished plastic, glass or lacquer. What it reflects it does not scatter, so the @@ -305,7 +328,7 @@ EnvironmentLight computeIBLParts(texture2d irradianceTexture, // single-bounce share only: the multi-bounce energy compensation in // environmentReflectance exists to correct the specular term's own approximation, // not to measure how much the surface diverts away from diffuse scattering. - float3 reflected = singleBounceReflectance(f0, roughness, NoV, iblBRDFTexture); + float3 reflected = singleBounceReflectance(f0, brdf); EnvironmentLight light; light.diff = max(1.0 - reflected, 0.0) * (1.0 - metallic) * diffuse; @@ -618,8 +641,11 @@ fragment float4 fragmentLightShader(VertexCompositeOutput vertexOut [[stage_in]] color.diff = brdf.diff * (half3)lights.color * (half)lights.intensity; color.spec = brdf.spec*lights.color*lights.intensity; - // Compute shadow using cascaded shadow maps - float shadow = computeCSMShadow(csmShadowArray, csmUniforms, verticesInWorldSpace.xyz, surfaceNormal, lightRayDirection); + // Compute shadow using cascaded shadow maps, only with a sun (see modelShader.metal). + float shadow = 1.0; + if (lights.intensity != 0.0) { + shadow = computeCSMShadow(csmShadowArray, csmUniforms, verticesInWorldSpace.xyz, surfaceNormal, lightRayDirection); + } // shadows affect directional light for now color.diff = color.diff*(half)shadow; @@ -712,93 +738,53 @@ fragment float4 fragmentLightShader(VertexCompositeOutput vertexOut [[stage_in]] } -// TBDR light pass: reads G-buffer from tile memory via framebuffer fetch ([[color(N)]]). -// The geometry and lighting sub-passes share one MTLRenderCommandEncoder, so the -// G-buffer data never leaves the GPU tile. Outputs lit color to attachment 5. -fragment TBDRLightOutput fragmentLightShaderTBDR( - VertexCompositeOutput vertexOut [[stage_in]], - // G-buffer โ€” read directly from tile memory, no texture2d binding needed - half4 gbAlbedo [[color(0)]], - half4 gbNormal [[color(1)]], - float4 gbPosition [[color(2)]], - half4 gbMaterial [[color(3)]], - half4 gbEmissive [[color(4)]], - // Shadow and IBL textures (still come from main memory) - depth2d_array csmShadowArray [[texture(lightPassShadowTextureIndex)]], - texture2d irradianceTexture [[texture(lightPassIBLIrradianceTextureIndex)]], - texture2d specularTexture [[texture(lightPassIBLSpecularTextureIndex)]], - texture2d iblBRDFTexture [[texture(lightPassIBLBRDFMapTextureIndex)]], - texture2d ltcMagTexture [[texture(lightPassAreaLTCMagTextureIndex)]], - texture2d ltcMatTexture [[texture(lightPassAreaLTCMatTextureIndex)]], - depthcube pointShadowCube [[texture(lightPassPointShadowTextureIndex)]], - depth2d spotShadowMap [[texture(lightPassSpotShadowTextureIndex)]], - // Uniform buffers - constant CSMUniforms &csmUniforms [[buffer(lightPassLightOrthoViewMatrixIndex)]], - constant simd_float3 &cameraPosition [[buffer(lightPassCameraPositionIndex)]], - constant LightParameters &lights [[buffer(lightPassLightParamsIndex)]], - constant PointLightBlock &plBlock [[buffer(lightPassPointLightsIndex)]], - constant SpotLightBlock &slBlock [[buffer(lightPassSpotLightsIndex)]], - constant IBLParamsUniform &iblParam [[buffer(lightPassIBLParamIndex)]], - constant AreaLightBlock &alBlock [[buffer(lightPassAreaLightsIndex)]], - constant float &iblRotationAngle [[buffer(lightPassIBLRotationAngleIndex)]], - constant bool &isGameMode [[buffer(lightPassGameModeIndex)]], - constant PointShadowUniforms &pointShadowUniforms [[buffer(lightPassPointShadowUniformIndex)]], - constant SpotShadowUniforms &spotShadowUniforms [[buffer(lightPassSpotShadowUniformIndex)]], - uint sampleID [[sample_id]] +// The direct light on a surface: the sun, through the shadow factor given (its cascade +// shadow, evaluated by the caller), then the point, spot and area lights with theirs. Shared +// by the light pass below, which lights once per sample from the G-buffer, and the geometry +// when it lights itself once per fragment (`fragmentModelShader`, `kModelLitInGBuffer`). +float3 computeDirectLighting( + float4 worldPosition, + float3 viewVector, + float3 normal, + float3 albedo, + float roughness, + float metallic, + float sunShadow, + constant LightParameters &lights, + constant PointLightBlock &plBlock, + constant SpotLightBlock &slBlock, + constant AreaLightBlock &alBlock, + texture2d ltcMagTexture, + texture2d ltcMatTexture, + depthcube pointShadowCube, + depth2d spotShadowMap, + constant PointShadowUniforms &pointShadowUniforms, + constant SpotShadowUniforms &spotShadowUniforms ) { - (void)sampleID; - // Background pixels were cleared to (0,0,0,0). Skip lighting entirely โ€” - // normalize(float3(0)) is undefined, and the IBL/shadow paths would produce - // garbage that can corrupt the environment composite downstream. - if (gbAlbedo.a <= 0.0h) { - TBDRLightOutput out; - out.litColor = float4(0.0); - return out; - } - - float3 lightRayDirection = normalize(lights.direction); - - float4 albedo = float4(gbAlbedo); - float4 verticesInWorldSpace = gbPosition; - float3 surfaceNormal = normalize(float3(gbNormal.xyz)); - float roughness = float(gbMaterial.r); - float metallic = float(gbMaterial.g); - float3 emissive = float3(gbEmissive.rgb); - - // SSAO is disabled in the TBDR path; will be revisited once SSAO is - // restructured as a tile kernel running inside this same encoder. - float ambientOcclusion = 1.0; - - float3 viewVector = normalize(cameraPosition - verticesInWorldSpace.xyz); - - float3 indirectLighting = computeIBLContribution( - irradianceTexture, specularTexture, iblBRDFTexture, - iblRotationAngle, iblParam, - albedo, surfaceNormal, viewVector, roughness, metallic - ); - indirectLighting *= ambientOcclusion * iblParam.ambientIntensity; - - LightContribution brdf = computeBRDF(lightRayDirection, viewVector, surfaceNormal, albedo.rgb, float3(1.0), roughness, metallic); - LightContribution color; - color.diff = brdf.diff * (half3)lights.color * (half)lights.intensity; - color.spec = brdf.spec * lights.color * lights.intensity; - - float shadow = computeCSMShadow(csmShadowArray, csmUniforms, verticesInWorldSpace.xyz, surfaceNormal, lightRayDirection); - color.diff *= (half)shadow; - color.spec *= shadow; + // A sun at no intensity adds nothing, so its BRDF is not evaluated: the branch is on a + // per-frame constant, uniform across the draw. Any other intensity takes the old path, + // in the old order of products. + if (lights.intensity != 0.0) { + float3 lightRayDirection = normalize(lights.direction); + LightContribution brdf = computeBRDF(lightRayDirection, viewVector, normal, albedo, float3(1.0), roughness, metallic); + color.diff = brdf.diff * (half3)lights.color * (half)lights.intensity; + color.spec = brdf.spec * lights.color * lights.intensity; + color.diff *= (half)sunShadow; + color.spec *= sunShadow; + } LightContribution pointColor; uint lightCount = min(plBlock.count.x, MAX_POINT_LIGHTS); for (uint i = 0; i < lightCount; ++i) { - LightContribution pl = computePointLightContribution(plBlock.lights[i], verticesInWorldSpace, viewVector, surfaceNormal, albedo.rgb, roughness, metallic); + LightContribution pl = computePointLightContribution(plBlock.lights[i], worldPosition, viewVector, normal, albedo, roughness, metallic); if (pointShadowUniforms.enabled > 0.5 && int(i) == pointShadowUniforms.lightIndex) { float pointShadow = computePointShadow( pointShadowCube, pointShadowUniforms, plBlock.lights[i].attenuation.x < 0.0 ? plBlock.lights[i].radius : 0.0, - verticesInWorldSpace.xyz, - surfaceNormal + worldPosition.xyz, + normal ); pl.diff *= (half)pointShadow; pl.spec *= pointShadow; @@ -812,14 +798,14 @@ fragment TBDRLightOutput fragmentLightShaderTBDR( LightContribution spotLightColor; uint spotLightCount = min(slBlock.count.x, MAX_POINT_LIGHTS); for (uint i = 0; i < spotLightCount; ++i) { - LightContribution sl = computeSpotLightContribution(slBlock.lights[i], verticesInWorldSpace, viewVector, surfaceNormal, albedo.rgb, roughness, metallic); + LightContribution sl = computeSpotLightContribution(slBlock.lights[i], worldPosition, viewVector, normal, albedo, roughness, metallic); if (spotShadowUniforms.enabled > 0.5 && int(i) == spotShadowUniforms.lightIndex) { float spotShadow = computeSpotShadow( spotShadowMap, spotShadowUniforms, slBlock.lights[i], - verticesInWorldSpace.xyz, - surfaceNormal + worldPosition.xyz, + normal ); sl.diff *= (half)spotShadow; sl.spec *= spotShadow; @@ -833,16 +819,196 @@ fragment TBDRLightOutput fragmentLightShaderTBDR( LightContribution areaLightColor; uint areaLightCount = min(alBlock.count.x, MAX_POINT_LIGHTS); for (uint i = 0; i < areaLightCount; ++i) { - LightContribution al = evaluateAreaLight(alBlock.lights[i], verticesInWorldSpace, viewVector, surfaceNormal, ltcMatTexture, ltcMagTexture, albedo.rgb, roughness, metallic); + LightContribution al = evaluateAreaLight(alBlock.lights[i], worldPosition, viewVector, normal, ltcMatTexture, ltcMagTexture, albedo, roughness, metallic); areaLightColor.diff += al.diff; areaLightColor.spec += al.spec; } color.diff += areaLightColor.diff; color.spec += areaLightColor.spec; - float3 finalRGB = (float3)color.diff + color.spec + indirectLighting + emissive; + return (float3)color.diff + color.spec; +} + +// The lighting of one G-buffer record: a pixel's surface facts in, its lit colour out +// (alpha carries the coverage the geometry wrote). Shared by the per-sample fragment +// pass and the tile stage below. +static inline float4 lightTBDRSample( + half4 gbAlbedo, + half4 gbNormal, + float4 gbPosition, + half4 gbMaterial, + half4 gbEmissive, + depth2d_array csmShadowArray, + texture2d irradianceTexture, + texture2d specularTexture, + texture2d iblBRDFTexture, + texture2d ltcMagTexture, + texture2d ltcMatTexture, + depthcube pointShadowCube, + depth2d spotShadowMap, + constant CSMUniforms &csmUniforms, + constant simd_float3 &cameraPosition, + constant LightParameters &lights, + constant PointLightBlock &plBlock, + constant SpotLightBlock &slBlock, + constant IBLParamsUniform &iblParam, + constant AreaLightBlock &alBlock, + constant float &iblRotationAngle, + constant PointShadowUniforms &pointShadowUniforms, + constant SpotShadowUniforms &spotShadowUniforms, + // The indirect light and the sun's shadow are in the G-buffer already (the emissive + // carries the indirect light, its alpha the shadow factor), written once per fragment. + bool indirectFromGBuffer +) { + // Background pixels were cleared to (0,0,0,0). Skip lighting entirely โ€” + // normalize(float3(0)) is undefined, and the IBL/shadow paths would produce + // garbage that can corrupt the environment composite downstream. + if (gbAlbedo.a <= 0.0h) { + return float4(0.0); + } + + float3 lightRayDirection = normalize(lights.direction); + + float4 albedo = float4(gbAlbedo); + float4 verticesInWorldSpace = gbPosition; + float3 surfaceNormal = normalize(float3(gbNormal.xyz)); + float roughness = float(gbMaterial.r); + float metallic = float(gbMaterial.g); + float3 emissive = float3(gbEmissive.rgb); + + // SSAO is disabled in the TBDR path; will be revisited once SSAO is + // restructured as a tile kernel running inside this same encoder. + float ambientOcclusion = 1.0; + + float3 viewVector = normalize(cameraPosition - verticesInWorldSpace.xyz); + + float3 indirectLighting = float3(0.0); + if (!indirectFromGBuffer) { + indirectLighting = computeIBLContribution( + irradianceTexture, specularTexture, iblBRDFTexture, + iblRotationAngle, iblParam, + albedo, surfaceNormal, viewVector, roughness, metallic + ); + indirectLighting *= ambientOcclusion * iblParam.ambientIntensity; + } + + // The sun's shadow: the one the geometry wrote, else the cascades, only with a sun. + float shadow = 1.0; + if (indirectFromGBuffer) { + shadow = float(gbEmissive.a); + } else if (lights.intensity != 0.0) { + shadow = computeCSMShadow(csmShadowArray, csmUniforms, verticesInWorldSpace.xyz, surfaceNormal, lightRayDirection); + } + float3 direct = computeDirectLighting( + verticesInWorldSpace, viewVector, surfaceNormal, albedo.rgb, roughness, metallic, shadow, + lights, plBlock, slBlock, alBlock, ltcMagTexture, ltcMatTexture, pointShadowCube, spotShadowMap, + pointShadowUniforms, spotShadowUniforms + ); + + float3 finalRGB = direct + indirectLighting + emissive; + return float4(finalRGB, albedo.a); +} +// TBDR light pass as a full-screen quad: reads the G-buffer from tile memory via framebuffer +// fetch ([[color(N)]]) in the encoder the geometry drew into, one sample at a time โ€” under +// MSAA every sample of every pixel is lit. Kept for devices without tile shading and as the +// fallback of the tile stage below. Outputs the lit colour to attachment 5. +fragment TBDRLightOutput fragmentLightShaderTBDR( + VertexCompositeOutput vertexOut [[stage_in]], + // G-buffer โ€” read directly from tile memory, no texture2d binding needed + half4 gbAlbedo [[color(0)]], + half4 gbNormal [[color(1)]], + float4 gbPosition [[color(2)]], + half4 gbMaterial [[color(3)]], + half4 gbEmissive [[color(4)]], + // Shadow and IBL textures (still come from main memory) + depth2d_array csmShadowArray [[texture(lightPassShadowTextureIndex)]], + texture2d irradianceTexture [[texture(lightPassIBLIrradianceTextureIndex)]], + texture2d specularTexture [[texture(lightPassIBLSpecularTextureIndex)]], + texture2d iblBRDFTexture [[texture(lightPassIBLBRDFMapTextureIndex)]], + texture2d ltcMagTexture [[texture(lightPassAreaLTCMagTextureIndex)]], + texture2d ltcMatTexture [[texture(lightPassAreaLTCMatTextureIndex)]], + depthcube pointShadowCube [[texture(lightPassPointShadowTextureIndex)]], + depth2d spotShadowMap [[texture(lightPassSpotShadowTextureIndex)]], + // Uniform buffers + constant CSMUniforms &csmUniforms [[buffer(lightPassLightOrthoViewMatrixIndex)]], + constant simd_float3 &cameraPosition [[buffer(lightPassCameraPositionIndex)]], + constant LightParameters &lights [[buffer(lightPassLightParamsIndex)]], + constant PointLightBlock &plBlock [[buffer(lightPassPointLightsIndex)]], + constant SpotLightBlock &slBlock [[buffer(lightPassSpotLightsIndex)]], + constant IBLParamsUniform &iblParam [[buffer(lightPassIBLParamIndex)]], + constant AreaLightBlock &alBlock [[buffer(lightPassAreaLightsIndex)]], + constant float &iblRotationAngle [[buffer(lightPassIBLRotationAngleIndex)]], + constant bool &isGameMode [[buffer(lightPassGameModeIndex)]], + constant PointShadowUniforms &pointShadowUniforms [[buffer(lightPassPointShadowUniformIndex)]], + constant SpotShadowUniforms &spotShadowUniforms [[buffer(lightPassSpotShadowUniformIndex)]], + uint sampleID [[sample_id]] +) { + (void)sampleID; + (void)isGameMode; TBDRLightOutput out; - out.litColor = float4(finalRGB, albedo.a); + out.litColor = lightTBDRSample( + gbAlbedo, gbNormal, gbPosition, gbMaterial, gbEmissive, + csmShadowArray, irradianceTexture, specularTexture, iblBRDFTexture, ltcMagTexture, ltcMatTexture, + pointShadowCube, spotShadowMap, + csmUniforms, cameraPosition, lights, plBlock, slBlock, iblParam, alBlock, iblRotationAngle, + pointShadowUniforms, spotShadowUniforms, + kLightIndirectFromGBuffer + ); return out; } + +// The G-buffer as the tile holds it, one record per colour a pixel has: a pixel the geometry +// covered whole holds one record for all its samples; a pixel on an edge holds one per distinct +// sample value, each with its coverage mask. +struct TBDRTileGBuffer { + half4 albedo [[color(0)]]; + half4 normal [[color(1)]]; + float4 position [[color(2)]]; + half4 material [[color(3)]]; + half4 emissive [[color(4)]]; + float4 lit [[color(5)]]; +}; + +// TBDR light pass as a tile stage: lights every pixel of the tile once per colour the pixel +// holds, not once per sample. Under MSAA the geometry writes the same value to every sample +// a fragment covers, so a pixel inside a surface is lit once however many samples it has; +// only a pixel on an edge is lit per distinct value, and the resolve then averages them by +// coverage exactly as it averaged the per-sample lighting of the quad. Same encoder, same +// tile memory, same inputs as the quad. +kernel void lightTileShaderTBDR( + imageblock tile, + ushort2 pixel [[thread_position_in_threadgroup]], + depth2d_array csmShadowArray [[texture(lightPassShadowTextureIndex)]], + texture2d irradianceTexture [[texture(lightPassIBLIrradianceTextureIndex)]], + texture2d specularTexture [[texture(lightPassIBLSpecularTextureIndex)]], + texture2d iblBRDFTexture [[texture(lightPassIBLBRDFMapTextureIndex)]], + texture2d ltcMagTexture [[texture(lightPassAreaLTCMagTextureIndex)]], + texture2d ltcMatTexture [[texture(lightPassAreaLTCMatTextureIndex)]], + depthcube pointShadowCube [[texture(lightPassPointShadowTextureIndex)]], + depth2d spotShadowMap [[texture(lightPassSpotShadowTextureIndex)]], + constant CSMUniforms &csmUniforms [[buffer(lightPassLightOrthoViewMatrixIndex)]], + constant simd_float3 &cameraPosition [[buffer(lightPassCameraPositionIndex)]], + constant LightParameters &lights [[buffer(lightPassLightParamsIndex)]], + constant PointLightBlock &plBlock [[buffer(lightPassPointLightsIndex)]], + constant SpotLightBlock &slBlock [[buffer(lightPassSpotLightsIndex)]], + constant IBLParamsUniform &iblParam [[buffer(lightPassIBLParamIndex)]], + constant AreaLightBlock &alBlock [[buffer(lightPassAreaLightsIndex)]], + constant float &iblRotationAngle [[buffer(lightPassIBLRotationAngleIndex)]], + constant PointShadowUniforms &pointShadowUniforms [[buffer(lightPassPointShadowUniformIndex)]], + constant SpotShadowUniforms &spotShadowUniforms [[buffer(lightPassSpotShadowUniformIndex)]] +) { + const ushort colors = tile.get_num_colors(pixel); + for (ushort color = 0; color < colors; ++color) { + TBDRTileGBuffer record = tile.read(pixel, color, imageblock_data_rate::color); + record.lit = lightTBDRSample( + record.albedo, record.normal, record.position, record.material, record.emissive, + csmShadowArray, irradianceTexture, specularTexture, iblBRDFTexture, ltcMagTexture, ltcMatTexture, + pointShadowCube, spotShadowMap, + csmUniforms, cameraPosition, lights, plBlock, slBlock, iblParam, alBlock, iblRotationAngle, + pointShadowUniforms, spotShadowUniforms, + false // the stage stands aside when the geometry lights per fragment + ); + tile.write(record, pixel, color, imageblock_data_rate::color); + } +} diff --git a/Sources/UntoldEngine/Shaders/LookShader.metal b/Sources/UntoldEngine/Shaders/LookShader.metal index 7bba6cad3..d2e5d5601 100644 --- a/Sources/UntoldEngine/Shaders/LookShader.metal +++ b/Sources/UntoldEngine/Shaders/LookShader.metal @@ -79,29 +79,32 @@ inline float3 sampleColorGradeLUT( return gradeLUTTexture.sample(gradeLUTSampler, t).rgb; } -fragment float4 fragmentLookShader( - VertexCompositeOutput in [[stage_in]], - texture2d sceneTexture [[texture(0)]], - texture2d colorLUTTexture [[texture(lookPassColorLUTTextureIndex)]], - texture3d colorGradeLUTTexture [[texture(colorGradeLUTTextureIndex)]], - constant float &brightness [[buffer(colorGradingPassBrightnessIndex)]], - constant float &contrast [[buffer(colorGradingPassContrastIndex)]], - constant float &saturation [[buffer(colorGradingPassSaturationIndex)]], - constant float &exposure [[buffer(colorGradingPassExposureIndex)]], - constant float3 &whiteBalanceCoeffs [[buffer(colorGradingWhiteBalanceCoeffsIndex)]], - constant bool &enabled [[buffer(colorGradingPassEnabledIndex)]], - constant bool &colorLUTEnabled [[buffer(colorLUTEnabledIndex)]], - constant float &colorLUTShaperMinStops [[buffer(colorLUTShaperMinStopsIndex)]], - constant float &colorLUTShaperMaxStops [[buffer(colorLUTShaperMaxStopsIndex)]], - constant int &colorLUTSize [[buffer(colorLUTSizeIndex)]], - constant bool &colorGradeLUTEnabled [[buffer(colorGradeLUTEnabledIndex)]], - constant float3 &colorGradeLUTDomainMin [[buffer(colorGradeLUTDomainMinIndex)]], - constant float3 &colorGradeLUTDomainMax [[buffer(colorGradeLUTDomainMaxIndex)]], - constant int &tonemapOperator [[buffer(tonemapOperatorSelectIndex)]], - texture2d splatLayer [[texture(lookPassSplatCoverageTextureIndex)]], - constant int &splatMask [[buffer(lookPassSplatMaskIndex)]], - texture2d gizmoTexture [[texture(lookPassGizmoTextureIndex)]], - constant bool &gizmoOverrides [[buffer(lookPassGizmoOverrideIndex)]] +// The look pass's colour, the one the output transform presents: shared by the fragment +// function that writes it alone and the one that presents it too. +static inline float4 lookColor( + VertexCompositeOutput in, + texture2d sceneTexture, + texture2d colorLUTTexture, + texture3d colorGradeLUTTexture, + constant float &brightness, + constant float &contrast, + constant float &saturation, + constant float &exposure, + constant float3 &whiteBalanceCoeffs, + constant bool &enabled, + constant bool &colorLUTEnabled, + constant float &colorLUTShaperMinStops, + constant float &colorLUTShaperMaxStops, + constant int &colorLUTSize, + constant bool &colorGradeLUTEnabled, + constant float3 &colorGradeLUTDomainMin, + constant float3 &colorGradeLUTDomainMax, + constant int &tonemapOperator, + constant int &colorPrimaries, + texture2d splatLayer, + constant int &splatMask, + texture2d gizmoTexture, + constant bool &gizmoOverrides ) { constexpr sampler s(min_filter::linear, mag_filter::linear, address::clamp_to_edge); float4 sceneSample = sceneTexture.sample(s, in.uvCoords); @@ -148,6 +151,10 @@ fragment float4 fragmentLookShader( // the source scene's full View Transform (including its own tonemap // curve), so applying ACES on top would double-compress the image. color = sampleColorLUT(max(color, 0.0), colorLUTTexture, colorLUTShaperMinStops, colorLUTShaperMaxStops, colorLUTSize); + } else if (tonemapOperator == tonemapOperatorNone) { + // No tone map: a mixed-immersion space shows the scene's linear colour next to the + // passthrough, whose response the compositor applies to both. + color = max(color, 0.0); } else if (tonemapOperator == tonemapOperatorAgX) { // Tone map ONCE, here. color = agxToneMapping(max(color, 0.0)); @@ -162,6 +169,98 @@ fragment float4 fragmentLookShader( color = sampleColorGradeLUT(color, colorGradeLUTTexture, colorGradeLUTDomainMin, colorGradeLUTDomainMax); } + // Shown with the display's P3 primaries: the colour taken as Display P3 coordinates and + // converted to the sRGB the layer is encoded in, which widens every saturated colour, as + // RealityKit shows a mixed-immersion scene. A channel that goes negative is outside sRGB and + // clips to black, as it does there. + if (colorPrimaries == colorPrimariesDisplayP3) { + const float3x3 p3ToSRGB = float3x3(float3(1.2249, -0.0420, -0.0197), + float3(-0.2247, 1.0419, -0.0786), + float3(0.0, 0.0, 1.0983)); + color = max(p3ToSRGB * color, 0.0); + } + return float4(splatLinear + color * (1.0 - splat), sceneSample.a); } +fragment float4 fragmentLookShader( + VertexCompositeOutput in [[stage_in]], + texture2d sceneTexture [[texture(0)]], + texture2d colorLUTTexture [[texture(lookPassColorLUTTextureIndex)]], + texture3d colorGradeLUTTexture [[texture(colorGradeLUTTextureIndex)]], + constant float &brightness [[buffer(colorGradingPassBrightnessIndex)]], + constant float &contrast [[buffer(colorGradingPassContrastIndex)]], + constant float &saturation [[buffer(colorGradingPassSaturationIndex)]], + constant float &exposure [[buffer(colorGradingPassExposureIndex)]], + constant float3 &whiteBalanceCoeffs [[buffer(colorGradingWhiteBalanceCoeffsIndex)]], + constant bool &enabled [[buffer(colorGradingPassEnabledIndex)]], + constant bool &colorLUTEnabled [[buffer(colorLUTEnabledIndex)]], + constant float &colorLUTShaperMinStops [[buffer(colorLUTShaperMinStopsIndex)]], + constant float &colorLUTShaperMaxStops [[buffer(colorLUTShaperMaxStopsIndex)]], + constant int &colorLUTSize [[buffer(colorLUTSizeIndex)]], + constant bool &colorGradeLUTEnabled [[buffer(colorGradeLUTEnabledIndex)]], + constant float3 &colorGradeLUTDomainMin [[buffer(colorGradeLUTDomainMinIndex)]], + constant float3 &colorGradeLUTDomainMax [[buffer(colorGradeLUTDomainMaxIndex)]], + constant int &tonemapOperator [[buffer(tonemapOperatorSelectIndex)]], + constant int &colorPrimaries [[buffer(lookPassColorPrimariesIndex)]], + texture2d splatLayer [[texture(lookPassSplatCoverageTextureIndex)]], + constant int &splatMask [[buffer(lookPassSplatMaskIndex)]], + texture2d gizmoTexture [[texture(lookPassGizmoTextureIndex)]], + constant bool &gizmoOverrides [[buffer(lookPassGizmoOverrideIndex)]] +) { + return lookColor(in, sceneTexture, colorLUTTexture, colorGradeLUTTexture, brightness, contrast, saturation, exposure, whiteBalanceCoeffs, enabled, colorLUTEnabled, colorLUTShaperMinStops, colorLUTShaperMaxStops, colorLUTSize, colorGradeLUTEnabled, colorGradeLUTDomainMin, colorGradeLUTDomainMax, tonemapOperator, colorPrimaries, splatLayer, splatMask, gizmoTexture, gizmoOverrides); +} + +static inline float3 lookLinearToSRGB(float3 x) { + const float3 lo = x * 12.92; + const float3 hi = 1.055 * pow(max(x, 0.0), float3(1.0 / 2.4)) - 0.055; + return select(lo, hi, x > 0.0031308); +} + +struct LookPresentFragmentOut { + float4 color [[color(0)]]; // the look output + float4 present [[color(1)]]; // the frame, encoded for the display + float depth [[depth(any)]]; // the opaque depth, for the compositor +}; + +// The look pass that presents as it grades, in place of the output transform pass: the look +// output, the frame encoded for the display and the opaque depth (its clear value kept finite: +// the compositor reprojects by view distance and drops a pixel at infinity). +fragment LookPresentFragmentOut fragmentLookPresentShader( + VertexCompositeOutput in [[stage_in]], + texture2d sceneTexture [[texture(0)]], + texture2d colorLUTTexture [[texture(lookPassColorLUTTextureIndex)]], + texture3d colorGradeLUTTexture [[texture(colorGradeLUTTextureIndex)]], + constant float &brightness [[buffer(colorGradingPassBrightnessIndex)]], + constant float &contrast [[buffer(colorGradingPassContrastIndex)]], + constant float &saturation [[buffer(colorGradingPassSaturationIndex)]], + constant float &exposure [[buffer(colorGradingPassExposureIndex)]], + constant float3 &whiteBalanceCoeffs [[buffer(colorGradingWhiteBalanceCoeffsIndex)]], + constant bool &enabled [[buffer(colorGradingPassEnabledIndex)]], + constant bool &colorLUTEnabled [[buffer(colorLUTEnabledIndex)]], + constant float &colorLUTShaperMinStops [[buffer(colorLUTShaperMinStopsIndex)]], + constant float &colorLUTShaperMaxStops [[buffer(colorLUTShaperMaxStopsIndex)]], + constant int &colorLUTSize [[buffer(colorLUTSizeIndex)]], + constant bool &colorGradeLUTEnabled [[buffer(colorGradeLUTEnabledIndex)]], + constant float3 &colorGradeLUTDomainMin [[buffer(colorGradeLUTDomainMinIndex)]], + constant float3 &colorGradeLUTDomainMax [[buffer(colorGradeLUTDomainMaxIndex)]], + constant int &tonemapOperator [[buffer(tonemapOperatorSelectIndex)]], + constant int &colorPrimaries [[buffer(lookPassColorPrimariesIndex)]], + texture2d splatLayer [[texture(lookPassSplatCoverageTextureIndex)]], + constant int &splatMask [[buffer(lookPassSplatMaskIndex)]], + texture2d gizmoTexture [[texture(lookPassGizmoTextureIndex)]], + constant bool &gizmoOverrides [[buffer(lookPassGizmoOverrideIndex)]], + depth2d depthTexture [[texture(lookPassDepthTextureIndex)]], + constant int &encodingMode [[buffer(lookPassEncodingModeIndex)]] +) { + constexpr sampler depthSampler(min_filter::nearest, mag_filter::nearest, address::clamp_to_edge); + LookPresentFragmentOut out; + out.color = lookColor(in, sceneTexture, colorLUTTexture, colorGradeLUTTexture, brightness, contrast, saturation, exposure, whiteBalanceCoeffs, enabled, colorLUTEnabled, colorLUTShaperMinStops, colorLUTShaperMaxStops, colorLUTSize, colorGradeLUTEnabled, colorGradeLUTDomainMin, colorGradeLUTDomainMax, tonemapOperator, colorPrimaries, splatLayer, splatMask, gizmoTexture, gizmoOverrides); + float3 c = max(out.color.rgb, 0.0); + if (encodingMode == 1) { + c = lookLinearToSRGB(c); + } + out.present = float4(c, out.color.a); + out.depth = max(depthTexture.sample(depthSampler, in.uvCoords), 1e-4); + return out; +} diff --git a/Sources/UntoldEngine/Shaders/SSAOShader.metal b/Sources/UntoldEngine/Shaders/SSAOShader.metal index d93b867ab..7df4249a8 100644 --- a/Sources/UntoldEngine/Shaders/SSAOShader.metal +++ b/Sources/UntoldEngine/Shaders/SSAOShader.metal @@ -34,6 +34,19 @@ static float3 reconstructViewPos(float2 uv, float linearDepth, float2 projScale) return float3(xy, linearDepth); } +// The screen UV a depth texel was projected to: its own UV, unless the eye was drawn through +// a rasterization rate map (XR foveation, rateMapSizes.x > 0: the map's screen size in xy, the +// depth texture's size in zw). Such a texture is dense where the eyes look and sparse in the +// periphery, so a view position rebuilt from the texel's own UV would bend the geometry there +// and shade false creases; the map's decoder says where the texel sits on screen. +static float2 screenUVOfTexel(float2 uv, constant rasterization_rate_map_data *rateMap, float4 rateMapSizes) { + if (rateMapSizes.x <= 0.0 || rateMapSizes.z <= 0.0) { + return uv; + } + rasterization_rate_map_decoder decoder(*rateMap); + return decoder.map_physical_to_screen_coordinates(uv * rateMapSizes.zw) / rateMapSizes.xy; +} + fragment float4 fragmentSSAOShader(VertexCompositeOutput vertexOut [[stage_in]], depth2d depthTexture [[texture(ssaoDepthTextureIndex)]], constant float &radius [[buffer(ssaoPassRadiusIndex)]], @@ -43,7 +56,9 @@ fragment float4 fragmentSSAOShader(VertexCompositeOutput vertexOut [[stage_in]], constant float2 &viewPort [[buffer(ssaoPassViewPortIndex)]], constant float2 &frustumPlanes [[buffer(ssaoPassFrustumIndex)]], constant bool &reverseZ [[buffer(ssaoPassReverseZIndex)]], - constant float2 &projScale [[buffer(ssaoPassProjScaleIndex)]] + constant float2 &projScale [[buffer(ssaoPassProjScaleIndex)]], + constant rasterization_rate_map_data *rateMap [[buffer(ssaoPassRateMapDataIndex)]], + constant float4 &rateMapSizes [[buffer(ssaoPassRateMapSizesIndex)]] ) { if (!enabled){ @@ -62,7 +77,7 @@ fragment float4 fragmentSSAOShader(VertexCompositeOutput vertexOut [[stage_in]], float2 texelSize = 1.0 / max(viewPort, float2(1.0)); float pixelRadius = clamp(radius * 220.0 / max(centerDepth, 0.001), 2.0, 48.0); - float3 centerPosition = reconstructViewPos(vertexOut.uvCoords, centerDepth, projScale); + float3 centerPosition = reconstructViewPos(screenUVOfTexel(vertexOut.uvCoords, rateMap, rateMapSizes), centerDepth, projScale); // No normal G-buffer is available here โ€” normalMap/positionMap are memoryless // (TBDR tile-only) in normal rendering and can't be bound to a later, separate // pass. Derive the surface normal from the depth buffer itself via screen-space @@ -94,7 +109,7 @@ fragment float4 fragmentSSAOShader(VertexCompositeOutput vertexOut [[stage_in]], } float sampleLinearDepth = linearizeDepth(sampleRawDepth, frustumPlanes.x, frustumPlanes.y, reverseZ); - float3 samplePosition = reconstructViewPos(sampleUV, sampleLinearDepth, projScale); + float3 samplePosition = reconstructViewPos(screenUVOfTexel(sampleUV, rateMap, rateMapSizes), sampleLinearDepth, projScale); float3 toSample = samplePosition - centerPosition; float sampleDistance = length(toSample); if (sampleDistance < 1e-5) { diff --git a/Sources/UntoldEngine/Shaders/ShaderStructs.h b/Sources/UntoldEngine/Shaders/ShaderStructs.h index 8a5d939c1..a87561c12 100644 --- a/Sources/UntoldEngine/Shaders/ShaderStructs.h +++ b/Sources/UntoldEngine/Shaders/ShaderStructs.h @@ -82,17 +82,18 @@ typedef struct{ }VertexInModel; +// What the model vertex stage hands the fragment stage, kept to what the fragment reads: on +// Apple GPUs every interpolated float is written to the parameter buffer by the tiler and read +// back per tile, so the layout is a vertex-stage cost in itself (measured: the stage halves +// with the material's inputs gone). typedef struct{ float4 position [[position]]; - float4 shadowCoords; - float4 vPosition; - float4 verticesInMVSpace; - float3 normalVectorInMVSpace; - float3 normal; - float3 tbNormal; //used in tbn - float4 tangent; + float3 worldPosition; + float3 tbNormal; // the world-space normal, the N of the TBN frame + float4 tangent; // the world-space tangent, its handedness in w float2 uvCoords; - + float3 currentClipXYW; // this frame's clip position without the jitter: x, y and w (no z), for the velocity + float3 prevClipXYW; // the previous frame's clip position of the same vertex: x, y and w }VertexOutModel; typedef struct { @@ -153,15 +154,6 @@ typedef struct } FragmentModelOut; -typedef struct -{ - float4 color [[color(0)]]; - float4 normals [[color(1)]]; - float4 positions [[color(2)]]; - float4 material [[color(3)]]; - float4 emmisive [[color(4)]]; -} GBufferOut; - typedef struct{ float4 position [[attribute(modelPassVerticesIndex)]]; @@ -187,10 +179,10 @@ typedef struct{ float3 cov2; float3 conic; float3 color; - float2 coordxy; + float2 pixelDelta; // the vertex's offset from the splat's centre in the conic's pixel frame (y down) float alpha; bool valid; - + }GaussianOutData; // Cascaded Shadow Maps: 3 cascades shared between both eyes in XR. @@ -212,6 +204,7 @@ struct CSMUniforms { float shadowSoftnessFar; // Poisson PCF radius in texels at the shadow distance float shadowSoftnessDepthScale; // 0 = fixed near radius, 1 = full near-to-far ramp float shadowSoftnessEnabled; + float shadowFilterTaps; // Poisson taps per cascade sample: 16 (the disk), 8 or 4 of it }; struct SpotShadowUniforms { diff --git a/Sources/UntoldEngine/Shaders/ShadersUtils.h b/Sources/UntoldEngine/Shaders/ShadersUtils.h index a0ee3ccd6..b2cf2d16b 100644 --- a/Sources/UntoldEngine/Shaders/ShadersUtils.h +++ b/Sources/UntoldEngine/Shaders/ShadersUtils.h @@ -165,8 +165,11 @@ float4 specularImportanceMap(float2 texCoords, texture2d environmentTextu float4 BRDFIntegrationMap(float roughness, float NoV); +float2 brdfIntegration(float NoV, float roughness, texture2d brdfMap); +float3 environmentReflectance(float3 F0, float2 brdf); float3 environmentReflectance(float3 F0, float roughness, float NoV, texture2d brdfMap); +float3 singleBounceReflectance(float3 F0, float2 brdf); float3 singleBounceReflectance(float3 F0, float roughness, float NoV, texture2d brdfMap); float3 blurredEnvironment(float3 direction, float roughness, texture2d specularMap, float3 rotationAxis, float rotationAngle); @@ -231,6 +234,40 @@ LightContribution evaluateAreaLight(constant AreaLightUniform &light, float roughness, float metallic); +float computePointShadow(depthcube shadowCube, + constant PointShadowUniforms &pointShadow, + float sourceRadius, + float3 worldPos, + float3 normal); + +float computeSpotShadow(depth2d shadowMap, + constant SpotShadowUniforms &spotShadow, + constant SpotLightUniform &light, + float3 worldPos, + float3 normal); + +// The direct light on a surface: the sun, through the shadow factor given (its cascade +// shadow, evaluated by the caller), then the point, spot and area lights with theirs. Shared +// by the light pass, which lights once per sample from the G-buffer, and the geometry when it +// lights itself once per fragment. +float3 computeDirectLighting(float4 worldPosition, + float3 viewVector, + float3 normal, + float3 albedo, + float roughness, + float metallic, + float sunShadow, + constant LightParameters &lights, + constant PointLightBlock &plBlock, + constant SpotLightBlock &slBlock, + constant AreaLightBlock &alBlock, + texture2d ltcMagTexture, + texture2d ltcMatTexture, + depthcube pointShadowCube, + depth2d spotShadowMap, + constant PointShadowUniforms &pointShadowUniforms, + constant SpotShadowUniforms &spotShadowUniforms); + float3 ACESFilmicToneMapping(float3 x); // Filmic/Uncharted 2 Tone Mapping Function diff --git a/Sources/UntoldEngine/Shaders/ShadersUtils.metal b/Sources/UntoldEngine/Shaders/ShadersUtils.metal index c4a416591..d5d01ab5f 100644 --- a/Sources/UntoldEngine/Shaders/ShadersUtils.metal +++ b/Sources/UntoldEngine/Shaders/ShadersUtils.metal @@ -315,7 +315,9 @@ float2 equirectUVFromCubeDirection(float3 v){ const float2 biases {0.5f,0.5f}; //assume +z is forward. for -x forward, use atan2(v.z,v.x) - float2 uv=float2(atan2(-v.x, v.z),asin(-v.y))*scales+biases; + // The latitude's asin is clamped: a reflection of two unit vectors can round a hair past + // 1 at the poles, and asin of that is NaN. + float2 uv=float2(atan2(-v.x, v.z),asin(clamp(-v.y, -1.0, 1.0)))*scales+biases; return uv; } @@ -498,11 +500,10 @@ float4 BRDFIntegrationMap(float roughness, float NoV){ } -// The share of the light around it that a surface reflects in a single bounce off its -// microfacets: F0 head on (4 % for a non-metal, its own color for a metal), more when it -// is seen at a slant. This is the share diffuse scattering gives up to specular, before -// any later-bounce energy compensation. -float3 singleBounceReflectance(float3 F0, float roughness, float NoV, texture2d brdfMap) { +// The lookup table's two values for a surface seen at an angle (NoV) with a roughness: the +// scale and the bias the reflectance at that slant puts on F0. Read once per pixel and +// handed to the two reflectances below. +float2 brdfIntegration(float NoV, float roughness, texture2d brdfMap) { // The lookup table is read right at its edges (a surface seen head on, a roughness // of 1), so it clamps: wrapping there reads the far edge and leaves a white spot. @@ -511,23 +512,26 @@ float3 singleBounceReflectance(float3 F0, float roughness, float NoV, texture2d< mip_filter::none, address::clamp_to_edge); - float4 brdfIntegration=brdfMap.sample(s,float2(NoV,roughness)); + return brdfMap.sample(s, float2(NoV, roughness)).xy; +} + +// The share of the light around it that a surface reflects in a single bounce off its +// microfacets: F0 head on (4 % for a non-metal, its own color for a metal), more when it +// is seen at a slant. This is the share diffuse scattering gives up to specular, before +// any later-bounce energy compensation. `brdf` is the lookup table's pair for the surface. +float3 singleBounceReflectance(float3 F0, float2 brdf) { + return F0 * brdf.x + brdf.y; +} - return F0 * float(brdfIntegration.x) + float(brdfIntegration.y); +float3 singleBounceReflectance(float3 F0, float roughness, float NoV, texture2d brdfMap) { + return singleBounceReflectance(F0, brdfIntegration(NoV, roughness, brdfMap)); } // The share of the light around it that a surface reflects: F0 head on (4 % for a // non-metal, its own color for a metal), more when it is seen at a slant. -float3 environmentReflectance(float3 F0, float roughness, float NoV, texture2d brdfMap) { +float3 environmentReflectance(float3 F0, float2 brdf) { - constexpr sampler s(coord::normalized, - filter::linear, - mip_filter::none, - address::clamp_to_edge); - - float4 brdfIntegration=brdfMap.sample(s,float2(NoV,roughness)); - - float3 reflectedInOneBounce = F0 * float(brdfIntegration.x) + float(brdfIntegration.y); + float3 reflectedInOneBounce = F0 * brdf.x + brdf.y; // A rough surface bounces part of the light between its own bumps before it lets it // go. The table counts a single bounce and takes the rest for lost: up to two thirds @@ -535,12 +539,16 @@ float3 environmentReflectance(float3 F0, float roughness, float NoV, texture2d brdfMap) { + return environmentReflectance(F0, brdfIntegration(NoV, roughness, brdfMap)); +} + // The environment in a direction, as blurred as a roughness makes it. float3 blurredEnvironment(float3 direction, float roughness, texture2d specularMap, float3 rotationAxis, float rotationAngle) { @@ -556,8 +564,14 @@ float3 blurredEnvironment(float3 direction, float roughness, texture2d sp int mipCount=6; - //Rotate the direction - float3 rotatedDirection=normalize(rotateDirection(direction, rotationAxis, rotationAngle)); + // Rotate the direction, unless the environment is not rotated (the headset's room probe + // never is): the angle is a per-frame constant, so the branch is uniform. Either way the + // direction comes out unit length, which the equirectangular lookup's asin needs: the + // simulator's light pass hands in a G-buffer normal as stored. + float3 rotatedDirection = normalize(direction); + if (rotationAngle != 0.0) { + rotatedDirection = rotateDirection(direction, rotationAxis, rotationAngle); + } float2 uv = equirectUVFromCubeDirection(rotatedDirection); float mipLevel = roughness * float(mipCount - 1); @@ -586,8 +600,11 @@ float3 diffuseIBL(float3 normal, texture2d irradianceMap, float3 rotation mip_filter::linear, address::repeat); - //Rotate the normal - float3 rotatedNormal=rotateDirection(normal, rotationAxis, rotationAngle); + // Rotate the normal, unless the environment is not rotated (see blurredEnvironment). + float3 rotatedNormal = normalize(normal); + if (rotationAngle != 0.0) { + rotatedNormal = rotateDirection(normal, rotationAxis, rotationAngle); + } float2 uv = equirectUVFromCubeDirection(rotatedNormal); return float3(irradianceMap.sample(s,uv).rgb); } diff --git a/Sources/UntoldEngine/Shaders/TAAShader.metal b/Sources/UntoldEngine/Shaders/TAAShader.metal new file mode 100644 index 000000000..191c0c7d3 --- /dev/null +++ b/Sources/UntoldEngine/Shaders/TAAShader.metal @@ -0,0 +1,286 @@ +// +// TAAShader.metal +// UntoldEngine +// +// Temporal anti-aliasing resolve. The frame was drawn with the projection jittered by a +// sub-pixel amount (TemporalAntiAliasing.swift); this pass blends the jittered frame with the +// eye's history, reprojected through the depth buffer and the two frames' unjittered +// view-projections, after clipping the history to the colour range of the pixel's 3ร—3 +// neighbourhood so that what moved or was uncovered does not ghost. Over frames the jitter +// averages sub-pixel detail โ€” thin wires, fine normal maps โ€” that a single sample flickers on. +// +// Under a rasterization rate map (XR foveation) the colour, depth and history textures are in +// physical space, not the screen the geometry was projected to, and each frame's map differs: +// a pixel is mapped physical โ†’ screen (this frame's map) โ†’ world โ†’ screen (previous frame, +// previous matrices) โ†’ physical (the history frame's map) before the history is sampled. +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +#include +#include "../../CShaderTypes/ShaderTypes.h" +#include "ShaderStructs.h" +using namespace metal; + +vertex VertexCompositeOutput vertexTAAShader(VertexCompositeIn in [[stage_in]]) { + VertexCompositeOutput out; + out.position = float4(float3(in.position), 1.0); + out.uvCoords = in.uvCoords; + return out; +} + +// The resolved pixel, written once: it is the eye's next history and what the output +// transform presents (TemporalAntiAliasing.resolvedTexture), so a full-screen write is spared. +struct TAAFragmentOut { + float4 color [[color(0)]]; +}; + +// Luma-chroma space for the neighbourhood clip: a box in YCoCg hugs the local colours more +// tightly than one in RGB, so less ghosting gets through on moving edges. +static inline float3 taaRGBToYCoCg(float3 c) { + return float3(0.25 * c.r + 0.5 * c.g + 0.25 * c.b, + 0.5 * c.r - 0.5 * c.b, + -0.25 * c.r + 0.5 * c.g - 0.25 * c.b); +} + +static inline float3 taaYCoCgToRGB(float3 c) { + return float3(c.x + c.y - c.z, c.x + c.z, c.x - c.y - c.z); +} + +// Moves `history` towards `current` until it lies inside the box [lo, hi]: the direction the +// history came from is kept, only its distance is cut, which avoids the colour shifts a plain +// clamp introduces (Karis 2014). +static inline float3 taaClipToBox(float3 lo, float3 hi, float3 current, float3 history) { + const float3 center = 0.5 * (hi + lo); + const float3 extent = 0.5 * (hi - lo) + 1e-4; + const float3 offset = history - center; + const float3 unit = abs(offset / extent); + const float maxUnit = max(unit.x, max(unit.y, unit.z)); + return maxUnit > 1.0 ? center + offset / maxUnit : history; +} + +// Where a texel of this frame's textures sits on the screen, in pixels. +static inline float2 taaScreenFromPhysical(float2 physical, float4 sizes, constant rasterization_rate_map_data *rateMap) { + if (sizes.x <= 0.0) { + return physical; + } + rasterization_rate_map_decoder decoder(*rateMap); + return decoder.map_physical_to_screen_coordinates(physical); +} + +// Where a screen position of the history's frame sits in the history texture, in pixels. +static inline float2 taaPhysicalFromScreen(float2 screen, float4 sizes, constant rasterization_rate_map_data *rateMap) { + if (sizes.x <= 0.0) { + return screen; + } + rasterization_rate_map_decoder decoder(*rateMap); + return decoder.map_screen_to_physical_coordinates(screen); +} + +// The history at a fractional position, Catmull-Rom filtered with nine bilinear taps (Jimenez +// 2016): resampling the history every frame with a bilinear filter smears it a little more +// each time, which under a moving head or a moving rate map shows as a soft halo that settles +// only when everything stands still; the cubic keeps it sharp. +static inline float4 taaSampleHistory(texture2d history, sampler linearSampler, float2 uv, float2 size) { + const float2 texel = 1.0 / size; + const float2 position = uv * size; + const float2 center = floor(position - 0.5) + 0.5; + const float2 f = position - center; + const float2 w0 = f * (-0.5 + f * (1.0 - 0.5 * f)); + const float2 w1 = 1.0 + f * f * (-2.5 + 1.5 * f); + const float2 w2 = f * (0.5 + f * (2.0 - 1.5 * f)); + const float2 w3 = f * f * (-0.5 + 0.5 * f); + const float2 w12 = w1 + w2; + const float2 offset12 = w2 / w12; + const float2 uv0 = (center - 1.0) * texel; + const float2 uv3 = (center + 2.0) * texel; + const float2 uv12 = (center + offset12) * texel; + // The cubic's nine bilinear taps less the four corners, whose weights are a few + // thousandths at most; the five kept are renormalised. Four fewer fetches per pixel. + float4 result = float4(0.0); + result += history.sample(linearSampler, float2(uv12.x, uv0.y)) * w12.x * w0.y; + result += history.sample(linearSampler, float2(uv0.x, uv12.y)) * w0.x * w12.y; + result += history.sample(linearSampler, float2(uv12.x, uv12.y)) * w12.x * w12.y; + result += history.sample(linearSampler, float2(uv3.x, uv12.y)) * w3.x * w12.y; + result += history.sample(linearSampler, float2(uv12.x, uv3.y)) * w12.x * w3.y; + const float kept = w12.x * w0.y + w0.x * w12.y + w12.x * w12.y + w3.x * w12.y + w12.x * w3.y; + return max(result / max(kept, 1e-4), 0.0); +} + +// The resolve of one pixel: this frame's look output and the eye's history to the eye's new +// history. Shared by the fragment function that writes the history alone and the one that +// also presents it. +static inline float4 taaResolveColor( + VertexCompositeOutput in, + texture2d colorTexture, + texture2d historyTexture, + depth2d depthTexture, + texture2d velocityTexture, + constant TAAConstants &taa, + constant rasterization_rate_map_data *rateMap, + constant rasterization_rate_map_data *prevRateMap +) { + constexpr sampler linearSampler(min_filter::linear, mag_filter::linear, mip_filter::none, address::clamp_to_edge); + constexpr sampler pointSampler(min_filter::nearest, mag_filter::nearest, mip_filter::none, address::clamp_to_edge); + + const float2 uv = in.uvCoords; + const float2 texel = 1.0 / max(taa.physicalSize, float2(1.0)); + const float4 current = colorTexture.sample(pointSampler, uv); + + // A texel at the clear depth has no position: nothing opaque was drawn there โ€” the + // background, or a transparent quad over it โ€” and reprojecting it from the far plane + // would fetch the wrong history and ghost or wipe it under a moving camera; such a pixel + // keeps its own frame, and in passthrough most of the frame is such pixels. It leaves + // here, before the rate-map decodes and the reprojection: nothing below reads the + // neighbouring lanes (the maps' local rates come from a second decode a texel over, not + // from derivatives across the quad), so a lane that leaves early is safe. + const float depth = depthTexture.sample(pointSampler, uv); + const bool cleared = (taa.reverseZ != 0u) ? (depth <= 1e-6) : (depth >= 1.0 - 1e-6); + if (taa.historyValid == 0u || cleared) { + return current; + } + + // This pixel's position in space, from its depth, and where it was on the previous + // frame's screen and in the history texture. + const float2 physical = in.position.xy; + const float2 screen = taaScreenFromPhysical(physical, taa.rateMapSizes, rateMap); + const float2 screenSize = taa.rateMapSizes.x > 0.0 ? taa.rateMapSizes.xy : taa.physicalSize; + const float2 ndc = float2(screen.x / screenSize.x * 2.0 - 1.0, 1.0 - screen.y / screenSize.y * 2.0); + float4 world = taa.invViewProjection * float4(ndc, depth, 1.0); + const bool worldValid = abs(world.w) >= 1e-12; + world /= worldValid ? world.w : 1.0; + + // Where the pixel was on the previous frame's screen: by the surface's own motion when + // the opaque pass wrote it (a surface that moves on its own carries its history with it), + // by the camera's motion alone otherwise. + const float4 prevClip = taa.prevViewProjection * world; + const bool velocityKnown = taa.velocityValid != 0u; + const bool prevInFront = velocityKnown || prevClip.w > 0.0; + const float2 prevNdc = velocityKnown + ? ndc - velocityTexture.sample(pointSampler, uv).xy + : prevClip.xy / (prevInFront ? prevClip.w : 1.0); + const float2 prevScreenSize = taa.prevRateMapSizes.x > 0.0 ? taa.prevRateMapSizes.xy : taa.physicalSize; + const float2 prevScreen = float2((prevNdc.x * 0.5 + 0.5) * prevScreenSize.x, (1.0 - (prevNdc.y * 0.5 + 0.5)) * prevScreenSize.y); + const float2 prevPhysical = taaPhysicalFromScreen(prevScreen, taa.prevRateMapSizes, prevRateMap); + + if (!worldValid || !prevInFront) { + return current; + } + const float2 historyUV = prevPhysical * texel; + if (any(historyUV < 0.0) || any(historyUV > 1.0)) { + return current; + } + + // The colour box of the 3ร—3 neighbourhood, in YCoCg: the mean ยฑ ฮณยทฯƒ (variance clipping, + // Salvi 2016) rather than the hard min/max. A feature a pixel wide โ€” a wire, a line of + // text โ€” is rasterized in some jittered frames and not others; a min/max box of a frame + // that missed it excludes its colour and the clip wipes it from the history, so it flickers. + // The statistical box still reaches towards it through the frames that miss it. + float3 m1 = float3(0.0); + float3 m2 = float3(0.0); + for (int y = -1; y <= 1; ++y) { + for (int x = -1; x <= 1; ++x) { + const float3 c = taaRGBToYCoCg(colorTexture.sample(pointSampler, uv + float2(x, y) * texel).rgb); + m1 += c; + m2 += c * c; + } + } + const float3 mean = m1 / 9.0; + const float3 sigma = sqrt(max(m2 / 9.0 - mean * mean, 0.0)); + const float3 lo = mean - taa.clipGamma * sigma; + const float3 hi = mean + taa.clipGamma * sigma; + + float weight = taa.historyWeight; + if (taa.rateMapSizes.x > 0.0 || taa.prevRateMapSizes.x > 0.0) { + // Under foveation the history may be coarser here than this frame: the spot was in + // the periphery of the previous map (one texel spanning several screen pixels) and + // the gaze has moved onto it. Trusting such a history at full weight shows the sharp + // frame with a blurry shadow that fades over twenty frames; its weight is cut by how + // much coarser it was, measured from the two maps' local rates (texels per screen + // pixel), per axis: a map is coarser in one axis than the other wherever its rows and + // columns differ, so the ratio is taken on the axis where the history is coarsest + // relative to this frame. Both rates are read as texels per screen pixel over the + // same screen interval, one pixel on from this pixel's screen position (or back, + // at either screen's far edge), each from its own map a second time: the maps' rows + // and columns are separable, so one decode gives both axes. The same interval for + // both keeps the ratio exactly 1 wherever the two maps agree, zone boundaries + // included. Not from derivatives across the quad: the pixels that left above would + // leave those undefined. Where the maps differ, the band before a boundary weighs a + // little differently than the quad's span did; it is at most a pixel wide. + const float2 delta = select(float2(1.0), float2(-1.0), + screen + 1.0 >= screenSize || prevScreen + 1.0 >= prevScreenSize); + const float2 currentRate = abs(taaPhysicalFromScreen(screen + delta, taa.rateMapSizes, rateMap) - physical); + const float2 prevRate = abs(taaPhysicalFromScreen(prevScreen + delta, taa.prevRateMapSizes, prevRateMap) - prevPhysical); + const float2 ratio = prevRate / max(currentRate, float2(1e-4)); + weight *= saturate(min(ratio.x, ratio.y)); + } + + const float4 history = taaSampleHistory(historyTexture, linearSampler, historyUV, taa.physicalSize); + // A history with next to no surface in it โ€” the transparent background a surface has just + // moved over โ€” carries no weight: blended whole, it fades the surface's leading edge toward + // the room behind it (its colour is black, which sits inside any dark surface's colour box, + // so the clip lets it through). The weight is whole again from a quarter of the frame's + // coverage up, so an edge the frame covers a little more of keeps its history. + weight *= saturate(4.0 * history.a / max(current.a, 1e-4)); + const float3 clipped = taaYCoCgToRGB(taaClipToBox(lo, hi, taaRGBToYCoCg(current.rgb), taaRGBToYCoCg(history.rgb))); + const float3 blended = mix(current.rgb, clipped, weight); + const float alpha = mix(current.a, history.a, weight); + return float4(blended, alpha); +} + +fragment TAAFragmentOut fragmentTAAShader( + VertexCompositeOutput in [[stage_in]], + texture2d colorTexture [[texture(taaPassColorTextureIndex)]], + texture2d historyTexture [[texture(taaPassHistoryTextureIndex)]], + depth2d depthTexture [[texture(taaPassDepthTextureIndex)]], + texture2d velocityTexture [[texture(taaPassVelocityTextureIndex)]], + constant TAAConstants &taa [[buffer(taaPassConstantsIndex)]], + constant rasterization_rate_map_data *rateMap [[buffer(taaPassRateMapDataIndex)]], + constant rasterization_rate_map_data *prevRateMap [[buffer(taaPassPrevRateMapDataIndex)]] +) { + TAAFragmentOut out; + out.color = taaResolveColor(in, colorTexture, historyTexture, depthTexture, velocityTexture, taa, rateMap, prevRateMap); + return out; +} + +static inline float3 taaLinearToSRGB(float3 x) { + const float3 lo = x * 12.92; + const float3 hi = 1.055 * pow(max(x, 0.0), float3(1.0 / 2.4)) - 0.055; + return select(lo, hi, x > 0.0031308); +} + +struct TAAPresentFragmentOut { + float4 color [[color(0)]]; // the eye's new history + float4 present [[color(1)]]; // the frame, encoded for the display + float depth [[depth(any)]]; // the opaque depth, for the compositor's reprojection +}; + +// The resolve that also presents: what the output transform pass would do with the resolved +// frame, in the same pass, which spares a full-screen pass per eye. Reverse-Z sky pixels carry +// the clear depth 0, which the visionOS compositor's reprojection divides by; a finite far +// depth stands in for it, as in the output transform pass. +fragment TAAPresentFragmentOut fragmentTAAPresentShader( + VertexCompositeOutput in [[stage_in]], + texture2d colorTexture [[texture(taaPassColorTextureIndex)]], + texture2d historyTexture [[texture(taaPassHistoryTextureIndex)]], + depth2d depthTexture [[texture(taaPassDepthTextureIndex)]], + texture2d velocityTexture [[texture(taaPassVelocityTextureIndex)]], + constant TAAConstants &taa [[buffer(taaPassConstantsIndex)]], + constant rasterization_rate_map_data *rateMap [[buffer(taaPassRateMapDataIndex)]], + constant rasterization_rate_map_data *prevRateMap [[buffer(taaPassPrevRateMapDataIndex)]], + constant int &encodingMode [[buffer(taaPassEncodingModeIndex)]] +) { + constexpr sampler depthSampler(min_filter::nearest, mag_filter::nearest, address::clamp_to_edge); + TAAPresentFragmentOut out; + out.color = taaResolveColor(in, colorTexture, historyTexture, depthTexture, velocityTexture, taa, rateMap, prevRateMap); + float3 c = max(out.color.rgb, 0.0); + if (encodingMode == 1) { + c = taaLinearToSRGB(c); + } + out.present = float4(c, out.color.a); + out.depth = max(depthTexture.sample(depthSampler, in.uvCoords), 1e-4); + return out; +} diff --git a/Sources/UntoldEngine/Shaders/TransparencyShader.metal b/Sources/UntoldEngine/Shaders/TransparencyShader.metal index 2a208bf4e..f34818ad9 100644 --- a/Sources/UntoldEngine/Shaders/TransparencyShader.metal +++ b/Sources/UntoldEngine/Shaders/TransparencyShader.metal @@ -73,12 +73,23 @@ fragment TransparencyOutput fragmentTransparencyShader( discard_fragment(); } - float4 verticesInWorldSpace = uniforms.modelMatrix * in.vPosition; + float4 verticesInWorldSpace = float4(in.worldPosition, 1.0); float3 viewVector = normalize(cameraPosition - verticesInWorldSpace.xyz); // Glass is drawn in two goes (see TransparencyPassFaces): this one keeps the faces // turned away from the viewer, or the ones turned towards the viewer. - bool turnedAway = dot(normalize(uniforms.normalMatrix * in.normal), viewVector) < 0.0; + // Which side of the face is seen is decided by the face itself, not by the shading + // normal: a smooth normal bends towards the side faces at the edge of a pane, and + // from a little to the left or to the right (the two eyes of a headset) it would put + // the same edge pixels on the far side for one eye and on the near side for the + // other. The derivative normal's sign follows the winding on screen; the vertex + // normal picks the outward one of the two, so winding and mirrored transforms still + // do not matter. A degenerate face (no derivatives) falls back to the shading normal. + float3 shadingNormal = normalize(in.tbNormal); + float3 faceNormal = cross(dfdx(verticesInWorldSpace.xyz), dfdy(verticesInWorldSpace.xyz)); + faceNormal = dot(faceNormal, shadingNormal) < 0.0 ? -faceNormal : faceNormal; + float faceLength = length(faceNormal); + bool turnedAway = dot(faceLength > 0.0 ? faceNormal / faceLength : shadingNormal, viewVector) < 0.0; if ((faces == transparencyPassFarFaces && !turnedAway) || (faces == transparencyPassNearFaces && turnedAway)) { discard_fragment(); } @@ -100,10 +111,12 @@ fragment TransparencyOutput fragmentTransparencyShader( float3 normal = hasNormal ? normalize(TBN * normalMap) - : normalize(uniforms.normalMatrix * in.normal); + : normalize(in.tbNormal); // A face of glass is lit on the side the viewer sees: the far face of a pane // reflects from inside it what its near face reflects from outside. - if (faces != transparencyPassEveryFace && turnedAway) { + // The shading normal of a pane faces the viewer whichever go draws the face: the + // face's side and the smooth normal can disagree at an edge. + if (faces != transparencyPassEveryFace && dot(normal, viewVector) < 0.0) { normal = -normal; } @@ -135,9 +148,13 @@ fragment TransparencyOutput fragmentTransparencyShader( totalLight.diff = brdf.diff * (half3)lights.color * (half)lights.intensity; totalLight.spec = brdf.spec * lights.color * lights.intensity; - float shadow = computeCSMShadow(csmShadowArray, csmUniforms, verticesInWorldSpace.xyz, normal, lightDirection); - totalLight.diff *= (half)shadow; - totalLight.spec *= shadow; + // The sun's shadow only with a sun (see modelShader.metal's emissiveWithIndirect). + float shadow = 1.0; + if (lights.intensity != 0.0) { + shadow = computeCSMShadow(csmShadowArray, csmUniforms, verticesInWorldSpace.xyz, normal, lightDirection); + totalLight.diff *= (half)shadow; + totalLight.spec *= shadow; + } uint pointLightCount = min(plBlock.count.x, MAX_POINT_LIGHTS); for (uint i = 0; i < pointLightCount; ++i) { diff --git a/Sources/UntoldEngine/Shaders/iblPreFilterShaders.metal b/Sources/UntoldEngine/Shaders/iblPreFilterShaders.metal index aef57128b..445de6617 100644 --- a/Sources/UntoldEngine/Shaders/iblPreFilterShaders.metal +++ b/Sources/UntoldEngine/Shaders/iblPreFilterShaders.metal @@ -56,27 +56,101 @@ static float3 xrIBLNormalFromEquirectUV(float2 texCoords) { return float3(sin(thetaN) * cos(phiN), sin(thetaN) * sin(phiN), cos(thetaN)); } -static float4 diffuseImportanceMapCube(float2 texCoords, texturecube environmentTexture) { - constexpr sampler s(coord::normalized, - filter::linear, - mip_filter::none, - address::clamp_to_edge); +// The direction a texel of the maps stands for, in the world. The maps are read with a +// world direction through equirectUVFromCubeDirection (y up, the azimuth from x and z) and +// written here from the texel's direction in the pre-filter's own frame (z the pole, the +// azimuth from x and y); the two compose to a fixed turn of the axes, d = (โˆ’n.z, โˆ’n.x, โˆ’n.y) +// for a world direction n. The source is read with the texel's world direction, so that the +// probe's +y face, the ceiling, lights what faces up. Without it each face lit the wrong side +// (the ceiling lit the left of a sphere, what was behind the wearer lit its underside, and a +// chrome ball reflected a wall for the room behind), and the static maps only got away with +// it because an equirect image reads the same whichever way its azimuth is turned. +static float3 xrIBLWorldDirection(float3 d) { + return float3(-d.y, -d.z, -d.x); +} - constexpr uint sampleCount = 128u; - float3 normal = xrIBLNormalFromEquirectUV(texCoords); - float3x3 normalSpace = getNormalSpace(normal); - float3 result = float3(0.0); +// The probe is eight texels a face. Importance sampling it, 128 samples a texel of the maps, +// left the maps with the structure of the sample set: with a window's worth of light in one +// texel, whether none, one or two of the samples land in it turns with the normal, and a white +// sphere showed ridges and spirals. The maps are convolved instead: the irradiance from the +// probe's first nine spherical harmonics, which hold light this smooth exactly, and the rougher +// specular mips by a fixed quadrature of directions, which the normal does not turn. +constant uint kXRProbeQuadratureCount = 1024u; + +// The k-th of a Fibonacci set of directions over the sphere, in the pre-filter's frame. +static float3 xrProbeQuadratureDirection(uint k, uint count) { + float z = 1.0 - (2.0 * (float(k) + 0.5)) / float(count); + float r = sqrt(max(1.0 - z * z, 0.0)); + float phi = 2.399963229728653 * float(k); + return float3(r * cos(phi), r * sin(phi), z); +} - for (uint n = 1u; n <= sampleCount; n++) { - float2 p = hammersley(n, sampleCount); - float theta = asin(sqrt(p.y)); - float phi = 2.0 * M_PI_F * p.x; - float3 pos = float3(sin(theta) * cos(phi), sin(theta) * sin(phi), cos(theta)); - float3 posGlob = normalize(normalSpace * pos); - result += environmentTexture.sample(s, posGlob).rgb; +// The real spherical-harmonic basis to the second band. +static void xrProbeSHBasis(float3 d, thread float *y) { + y[0] = 0.282095; + y[1] = 0.488603 * d.y; + y[2] = 0.488603 * d.z; + y[3] = 0.488603 * d.x; + y[4] = 1.092548 * d.x * d.y; + y[5] = 1.092548 * d.y * d.z; + y[6] = 0.315392 * (3.0 * d.z * d.z - 1.0); + y[7] = 1.092548 * d.x * d.z; + y[8] = 0.546274 * (d.x * d.x - d.y * d.y); +} + +// Projects the probe onto its first nine spherical harmonics: one threadgroup, each thread a +// share of the quadrature directions, reduced across the group into nine float4s (rgb and a +// spare). +kernel void xrProbeSHProjection(texturecube environmentTexture [[texture(0)]], + device float4 *sh [[buffer(0)]], + uint threadIndex [[thread_index_in_threadgroup]], + uint threads [[threads_per_threadgroup]], + uint simdLane [[thread_index_in_simdgroup]], + uint simdIndex [[simdgroup_index_in_threadgroup]], + uint simdCount [[simdgroups_per_threadgroup]]) { + constexpr sampler s(coord::normalized, filter::linear, mip_filter::none, address::clamp_to_edge); + threadgroup float3 partial[32][9]; + float3 sum[9]; + for (uint i = 0u; i < 9u; i++) { + sum[i] = float3(0.0); } + for (uint k = threadIndex; k < kXRProbeQuadratureCount; k += threads) { + float3 d = xrProbeQuadratureDirection(k, kXRProbeQuadratureCount); + float3 radiance = environmentTexture.sample(s, xrIBLWorldDirection(d)).rgb; + float y[9]; + xrProbeSHBasis(d, y); + for (uint i = 0u; i < 9u; i++) { + sum[i] += radiance * y[i]; + } + } + for (uint i = 0u; i < 9u; i++) { + float3 total = float3(simd_sum(sum[i].x), simd_sum(sum[i].y), simd_sum(sum[i].z)); + if (simdLane == 0u) { + partial[simdIndex][i] = total; + } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + if (threadIndex == 0u) { + const float weight = 4.0 * M_PI_F / float(kXRProbeQuadratureCount); + for (uint i = 0u; i < 9u; i++) { + float3 total = float3(0.0); + for (uint g = 0u; g < simdCount; g++) { + total += partial[g][i]; + } + sh[i] = float4(total * weight, 0.0); + } + } +} - return float4(result / float(sampleCount), 1.0); +// The irradiance over ฯ€ at a normal, from the harmonics: Ramamoorthi and Hanrahan's +// convolution, the bands weighted ฯ€, 2ฯ€/3 and ฯ€/4, taken over ฯ€ as the map stores it. +static float3 xrProbeIrradiance(float3 normal, constant float4 *sh) { + float y[9]; + xrProbeSHBasis(normal, y); + float3 e = sh[0].rgb * y[0]; + e += (2.0 / 3.0) * (sh[1].rgb * y[1] + sh[2].rgb * y[2] + sh[3].rgb * y[3]); + e += 0.25 * (sh[4].rgb * y[4] + sh[5].rgb * y[5] + sh[6].rgb * y[6] + sh[7].rgb * y[7] + sh[8].rgb * y[8]); + return max(e, 0.0); } static float4 specularImportanceMapCube(float2 texCoords, texturecube environmentTexture, float roughness) { @@ -85,8 +159,27 @@ static float4 specularImportanceMapCube(float2 texCoords, texturecube env mip_filter::none, address::clamp_to_edge); - uint sampleCount = roughness < 0.001 ? 1u : 128u; float3 normal = xrIBLNormalFromEquirectUV(texCoords); + if (roughness >= 0.25) { + // The rougher mips by the quadrature: the GGX lobe about the normal, which stands in + // for the view (the split sum), weighs each direction. + float3 convolved = float3(0.0); + float convolvedWeight = 0.0; + for (uint k = 0u; k < kXRProbeQuadratureCount; k++) { + float3 L = xrProbeQuadratureDirection(k, kXRProbeQuadratureCount); + float NoL = dot(normal, L); + if (NoL <= 0.0) { + continue; + } + float3 H = normalize(L + normal); + float weight = distributionGGX(max(dot(normal, H), 0.0), roughness) * NoL; + convolved += environmentTexture.sample(s, xrIBLWorldDirection(L)).rgb * weight; + convolvedWeight += weight; + } + return float4(convolvedWeight > 0.0 ? convolved / convolvedWeight : float3(0.0), 1.0); + } + + uint sampleCount = roughness < 0.001 ? 1u : 128u; float3 result = float3(0.0); float totalWeight = 0.0; @@ -96,7 +189,7 @@ static float4 specularImportanceMapCube(float2 texCoords, texturecube env float3 L = normalize(2.0 * dot(normal, H) * H - normal); float NoL = max(dot(normal, L), 0.0); if (NoL > 0.0) { - result += environmentTexture.sample(s, L).rgb * NoL; + result += environmentTexture.sample(s, xrIBLWorldDirection(L)).rgb * NoL; totalWeight += NoL; } } @@ -106,9 +199,10 @@ static float4 specularImportanceMapCube(float2 texCoords, texturecube env } fragment IBLFragmentOut fragmentXRIBLCubePreFilterShader(VertexCompositeOutput in [[stage_in]], - texturecube environmentTexture [[texture(0)]]) { + texturecube environmentTexture [[texture(0)]], + constant float4 *probeSH [[buffer(1)]]) { IBLFragmentOut out; - out.irradiance = diffuseImportanceMapCube(in.uvCoords, environmentTexture); + out.irradiance = float4(xrProbeIrradiance(xrIBLNormalFromEquirectUV(in.uvCoords), probeSH), 1.0); out.specular = specularImportanceMapCube(in.uvCoords, environmentTexture, 0.0); // See fragmentIBLPreFilterShader: one row per roughness, as specularIBL reads it. out.brdfMap = BRDFIntegrationMap(in.uvCoords.y, in.uvCoords.x); diff --git a/Sources/UntoldEngine/Shaders/modelShader.metal b/Sources/UntoldEngine/Shaders/modelShader.metal index d32a4d2e5..1c68932e2 100644 --- a/Sources/UntoldEngine/Shaders/modelShader.metal +++ b/Sources/UntoldEngine/Shaders/modelShader.metal @@ -163,11 +163,16 @@ vertex VertexOutModel vertexModelShader( weights.w * (jointMatrices[joints.w] * normals)); } - out.vPosition = position; + float4 worldPosition = uniforms.modelMatrix * position; + out.worldPosition = worldPosition.xyz; out.position = uniforms.projectionMatrix * uniforms.modelViewMatrix * position; - out.normal = normals.xyz; - //out.shadowCoords = lightOrthoView * uniforms.modelMatrix * position; out.uvCoords = in.uv; + // Where the vertex is this frame and was the previous one, on the unjittered screens, for + // the velocity the fragment writes (a skinned vertex takes this frame's pose for both). + float4 currentClip = uniforms.unjitteredViewProjectionMatrix * worldPosition; + float4 prevClip = uniforms.prevViewProjectionMatrix * uniforms.prevModelMatrix * position; + out.currentClipXYW = currentClip.xyw; + out.prevClipXYW = prevClip.xyw; // Compute TBN. Tangents are directions, not normals, so transform them with the model // matrix linear part. The inverse-transpose normal matrix is only correct for normals. @@ -239,6 +244,95 @@ vertex MeshOccluderShellVertexOut vertexMeshOccluderShellShader( } +// Whether a draw through this pipeline may discard fragments: an alpha-masked or blended +// material, or a dither fade. The plain opaque pipeline compiles this false, and with the +// discards gone the GPU resolves visibility before it shades โ€” a fragment another surface +// covers is never shaded. The general pipeline keeps them for the draws that need them. +constant bool kModelMayDiscard [[function_constant(modelFunctionConstantMayDiscard)]]; +/// Per-fragment indirect lighting: the image-based light and the sun's cascade shadow, +/// evaluated here once per fragment and carried in the emissive attachment (indirect light +/// added to the emissive, the shadow factor in alpha) for the light pass to add, instead of +/// once per sample there (`setRendering(.perFragmentIndirectLighting)`). +constant bool kModelIndirectInGBuffer [[function_constant(modelFunctionConstantIndirectInGBuffer)]]; +/// Per-fragment lighting: the surface lit whole here, once per fragment โ€” the direct light of +/// the sun, the point, spot and area lights with their shadows, over the indirect light above, +/// which it needs โ€” and written to the lit attachment, the one the light pass would write; that +/// pass, which lights once per sample, then stands down (`setRendering(.perFragmentLighting)`). +constant bool kModelLitInGBuffer [[function_constant(modelFunctionConstantLitInGBuffer)]]; +/// The sun's parameters are bound for either of the two. +constant bool kModelSunNeeded = kModelIndirectInGBuffer || kModelLitInGBuffer; +/// The G-buffer is written when the light pass is to read it, that is unless the geometry +/// lights itself. +constant bool kModelGBufferOut = !kModelLitInGBuffer; + +/// What the model fragment stage writes: the G-buffer for the light pass to read from the +/// tile, or โ€” the geometry lighting itself โ€” the lit colour alone, in the attachment the +/// light pass would write, so the pass holds nothing else in its imageblock and the GPU runs +/// it in its largest tiles. +typedef struct { + float4 color [[color(0), function_constant(kModelGBufferOut)]]; + float4 normals [[color(1), function_constant(kModelGBufferOut)]]; + float4 positions [[color(2), function_constant(kModelGBufferOut)]]; + float4 material [[color(3), function_constant(kModelGBufferOut)]]; + float4 emmisive [[color(4), function_constant(kModelGBufferOut)]]; + float4 lit [[color(5), function_constant(kModelLitInGBuffer)]]; + // The pixel's motion since the previous frame, in NDC (this frame minus the previous), for + // the temporal resolve to find the history of a surface that moves on its own. + float2 velocity [[color(6), function_constant(kModelLitInGBuffer)]]; +} GBufferOut; + +/// The pixel's motion in NDC: this frame's unjittered position minus the previous frame's, each +/// given as its clip x, y and w (the third component is w, not z). A vertex behind the previous +/// eye (w <= 0) has no previous position worth a history: no motion. +static inline float2 modelVelocity(float3 currentClipXYW, float3 prevClipXYW) { + const float currentW = currentClipXYW.z; + const float prevW = prevClipXYW.z; + if (currentW <= 1e-6 || prevW <= 1e-6) { + return float2(0.0); + } + return currentClipXYW.xy / currentW - prevClipXYW.xy / prevW; +} + +/// The emissive attachment's value with the indirect light and the sun's shadow folded in. +static inline float4 emissiveWithIndirect( + float3 emissive, float4 baseColor, float3 normal, float3 worldPosition, float roughness, float metallic, + float3 viewVector, + texture2d iblIrradiance, texture2d iblSpecular, texture2d iblBRDF, + depth2d_array csmShadowArray, + constant CSMUniforms &csmUniforms, constant LightParameters &lightParams, + constant IBLParamsUniform &iblParam, constant float &iblRotationAngle +) { + float3 indirect = computeIBLContribution(iblIrradiance, iblSpecular, iblBRDF, iblRotationAngle, iblParam, + baseColor, normal, viewVector, roughness, metallic) * iblParam.ambientIntensity; + // The sun's shadow only ever multiplies the sun's light, so a sun at no intensity needs + // no shadow taps: the branch is on a per-frame constant, uniform across the draw. + float shadow = 1.0; + if (lightParams.intensity != 0.0) { + shadow = computeCSMShadow(csmShadowArray, csmUniforms, worldPosition, normal, normalize(lightParams.direction)); + } + return float4(emissive + indirect, shadow); +} + +/// The lit attachment's value: the direct light over the emissive attachment's value, which +/// already holds the emissive with the indirect light, and the sun's shadow in its alpha. The +/// alpha is the coverage the light pass would have written. +static inline float4 litWithDirect( + float4 emissiveAndShadow, float4 baseColor, float3 normal, float4 worldPosition, float roughness, float metallic, + float3 viewVector, + constant LightParameters &lightParams, + constant PointLightBlock &plBlock, constant SpotLightBlock &slBlock, constant AreaLightBlock &alBlock, + texture2d ltcMagTexture, texture2d ltcMatTexture, + depthcube pointShadowCube, depth2d spotShadowMap, + constant PointShadowUniforms &pointShadowUniforms, constant SpotShadowUniforms &spotShadowUniforms +) { + float3 direct = computeDirectLighting( + worldPosition, viewVector, normal, baseColor.rgb, roughness, metallic, emissiveAndShadow.a, + lightParams, plBlock, slBlock, alBlock, ltcMagTexture, ltcMatTexture, pointShadowCube, spotShadowMap, + pointShadowUniforms, spotShadowUniforms + ); + return float4(direct + emissiveAndShadow.rgb, baseColor.a); +} + fragment GBufferOut fragmentModelShader(VertexOutModel in [[stage_in]], constant Uniforms & uniforms [[ buffer(modelPassFragmentUniformIndex) ]], texture2d baseColor [[texture(modelPassBaseTextureIndex)]], @@ -255,7 +349,26 @@ fragment GBufferOut fragmentModelShader(VertexOutModel in [[stage_in]], sampler heightSampler [[sampler(modelPassHeightSamplerIndex)]], constant float &stScale [[buffer(modelPassFragmentSTScaleIndex)]], constant POMQualityUniform &pomQuality [[buffer(modelPassFragmentPOMQualityIndex)]], - constant bool &normalIsPackedXY [[buffer(modelPassFragmentNormalIsPackedXYIndex)]]) + constant bool &normalIsPackedXY [[buffer(modelPassFragmentNormalIsPackedXYIndex)]], + // Per-fragment indirect lighting, bound only with it on. + texture2d iblIrradiance [[texture(modelPassIBLIrradianceTextureIndex), function_constant(kModelIndirectInGBuffer)]], + texture2d iblSpecular [[texture(modelPassIBLSpecularTextureIndex), function_constant(kModelIndirectInGBuffer)]], + texture2d iblBRDF [[texture(modelPassIBLBRDFTextureIndex), function_constant(kModelIndirectInGBuffer)]], + depth2d_array csmShadowArray [[texture(modelPassCSMShadowTextureIndex), function_constant(kModelIndirectInGBuffer)]], + constant CSMUniforms &csmUniforms [[buffer(modelPassFragmentCSMUniformsIndex), function_constant(kModelIndirectInGBuffer)]], + constant LightParameters &lightParams [[buffer(modelPassFragmentLightParamsIndex), function_constant(kModelSunNeeded)]], + constant IBLParamsUniform &iblParam [[buffer(modelPassFragmentIBLParamIndex), function_constant(kModelIndirectInGBuffer)]], + constant float &iblRotationAngle [[buffer(modelPassFragmentIBLRotationAngleIndex), function_constant(kModelIndirectInGBuffer)]], + // Per-fragment lighting, bound only with it on. + constant PointLightBlock &plBlock [[buffer(modelPassFragmentPointLightsIndex), function_constant(kModelLitInGBuffer)]], + constant SpotLightBlock &slBlock [[buffer(modelPassFragmentSpotLightsIndex), function_constant(kModelLitInGBuffer)]], + constant AreaLightBlock &alBlock [[buffer(modelPassFragmentAreaLightsIndex), function_constant(kModelLitInGBuffer)]], + constant PointShadowUniforms &pointShadowUniforms [[buffer(modelPassFragmentPointShadowUniformIndex), function_constant(kModelLitInGBuffer)]], + constant SpotShadowUniforms &spotShadowUniforms [[buffer(modelPassFragmentSpotShadowUniformIndex), function_constant(kModelLitInGBuffer)]], + texture2d ltcMagTexture [[texture(modelPassAreaLTCMagTextureIndex), function_constant(kModelLitInGBuffer)]], + texture2d ltcMatTexture [[texture(modelPassAreaLTCMatTextureIndex), function_constant(kModelLitInGBuffer)]], + depthcube pointShadowCube [[texture(modelPassPointShadowTextureIndex), function_constant(kModelLitInGBuffer)]], + depth2d spotShadowMap [[texture(modelPassSpotShadowTextureIndex), function_constant(kModelLitInGBuffer)]]) { // Base Color and Normal Maps: Linear filtering, mipmaps, repeat wrapping @@ -285,13 +398,24 @@ fragment GBufferOut fragmentModelShader(VertexOutModel in [[stage_in]], float2 stDx = dfdx(st); float2 stDy = dfdy(st); - float4 verticesInWorldSpace=uniforms.modelMatrix*in.vPosition; - float3 normalVectorInWorldSpace=uniforms.normalMatrix*in.normal; + float4 verticesInWorldSpace = float4(in.worldPosition, 1.0); + float3 normalVectorInWorldSpace = in.tbNormal; //construct tbn matrix TBN (world-space basis built in the vertex stage) simd_float3 N=normalize(in.tbNormal); simd_float3 T=normalize(in.tangent.xyz); + // The model pass does not cull, so an open mesh shows its back faces โ€” a dart's flight, a + // leaf, a sheet seen from behind. Those arrive with the front side's normal, pointing away + // from the viewer, and shade as if unlit (one wing of a flight dark, the next one lit). + // A face whose normal points away from the viewer is lit with it turned around. The test + // is geometric rather than [[front_facing]]: the pass sets no winding, and assets arrive + // wound either way. + if (dot(N, uniforms.cameraPosition - verticesInWorldSpace.xyz) < 0.0) { + normalVectorInWorldSpace = -normalVectorInWorldSpace; + N = -N; + } + //B = (N x T) * T.w simd_float3 B=cross(N, T)*in.tangent.w; simd_float3x3 TBN=simd_float3x3(T,B,N); @@ -354,102 +478,148 @@ fragment GBufferOut fragmentModelShader(VertexOutModel in [[stage_in]], // pixel-to-pixel movement near the height field's silhouette-like transitions. `hasHeight` // is uniform across the whole draw call, so branching between the two sampling forms here // is safe for the implicit-derivative branch. - float4 sampledColor = hasHeight - ? baseColor.sample(baseColorSampler, sampleUV, gradient2d(stDx, stDy)) - : baseColor.sample(baseColorSampler, st, bias(0.25f)); - + // + // Each map is sampled only when the material has it. The presence flags are per-draw + // constants, so each branch is uniform across the draw and the implicit-derivative sample + // inside it is as safe as the hasHeight one; a part without the map paid the sample on a + // nil texture before and threw the value away. // A textured material whose base color factor was never set (all zeros) shows its // texture untinted. Without a texture, zero is a color like any other: black. bool isBaseColorUnset = materialParameter.hasTexture.x == 1 && all(materialParameter.baseColor.rgb < 0.001); float3 tint = isBaseColorUnset ? float3(1.0) : materialParameter.baseColor.rgb; - float4 inBaseColor = (materialParameter.hasTexture.x == 1) - ? float4(sampledColor.rgb * tint, sampledColor.a * materialParameter.baseColor.a) - : float4(tint, materialParameter.baseColor.a); + float4 inBaseColor = float4(tint, materialParameter.baseColor.a); + if (materialParameter.hasTexture.x == 1) { + float4 sampledColor = hasHeight + ? baseColor.sample(baseColorSampler, sampleUV, gradient2d(stDx, stDy)) + : baseColor.sample(baseColorSampler, st, bias(0.25f)); + inBaseColor = float4(sampledColor.rgb * tint, sampledColor.a * materialParameter.baseColor.a); + } - if (materialParameter.alphaMode == 2) { - discard_fragment(); - } + if (kModelMayDiscard) { + if (materialParameter.alphaMode == 2) { + discard_fragment(); + } - if (materialParameter.alphaMode == 1 && inBaseColor.a < materialParameter.alphaCutoff) { - discard_fragment(); - } + if (materialParameter.alphaMode == 1 && inBaseColor.a < materialParameter.alphaCutoff) { + discard_fragment(); + } - float lodDitherMode = materialParameter.lodDither.y; - if (lodDitherMode > 0.5) { - float threshold = clamp(materialParameter.lodDither.x, 0.0, 1.0); - float dither = lodBayerThreshold(in.position.xy); + float lodDitherMode = materialParameter.lodDither.y; + if (lodDitherMode > 0.5) { + float threshold = clamp(materialParameter.lodDither.x, 0.0, 1.0); + float dither = lodBayerThreshold(in.position.xy); - if (lodDitherMode < 1.5) { - if (dither >= threshold) { + if (lodDitherMode < 1.5) { + if (dither >= threshold) { + discard_fragment(); + } + } else if (dither < threshold) { discard_fragment(); } - } else if (dither < threshold) { - discard_fragment(); } } float passthroughAlpha = clamp(materialParameter.passthroughAlpha, 0.0, 1.0); - //normal map is in Tangent space - float4 normalSample = hasHeight - ? normalTexture.sample(normalSampler, sampleUV, gradient2d(stDx, stDy)) - : normalTexture.sample(normalSampler, st, bias(0.25f)); - - // Two on-disk encodings: a direct 3-component tangent-space normal (RGB, [0,1] -> - // [-1,1]), or astcenc's `-normal` packing (RGB=X, A=Y) used by texbake.py to get a - // better error metric for vector data โ€” see NativeTexFlags.normalPackedXY. Z is - // reconstructed from the unit-length constraint for the packed case. - // [0,1] -> [-1,1] first, unit length after: normalizing the stored color instead - // shortens it before the remap and leaves every normal tilted (a flat texel, (0.5, - // 0.5, 1), came out 22 degrees off). - float3 normalMapStandard = normalSample.rgb * 2.0 - 1.0; - - float2 packedXY = normalSample.ga * 2.0 - 1.0; - float3 normalMapPackedXY = float3(packedXY, sqrt(saturate(1.0 - dot(packedXY, packedXY)))); - - float3 normalMap = normalIsPackedXY ? normalMapPackedXY : normalMapStandard; - normalMap = applyNormalStrength(normalMap, materialParameter.normalScale); - - //convert to normal map to world space??? - normalMap=(hasNormal==false)?normalize(normalVectorInWorldSpace):normalize(TBN*normalMap); - - float4 roughnessSample = hasHeight - ? roughnessTexture.sample(materialSampler, sampleUV, gradient2d(stDx, stDy)) - : roughnessTexture.sample(materialSampler, st, bias(0.25f)); - float roughness=(materialParameter.hasTexture.y==1) - ? selectTextureChannel(roughnessSample, materialParameter.textureChannels.x) * materialParameter.roughness - : materialParameter.roughness; + // The shading normal: the vertex normal, or the normal map's, in tangent space, brought + // to world space. + float3 normalMap = normalize(normalVectorInWorldSpace); + if (hasNormal) { + float4 normalSample = hasHeight + ? normalTexture.sample(normalSampler, sampleUV, gradient2d(stDx, stDy)) + : normalTexture.sample(normalSampler, st, bias(0.25f)); + + // Two on-disk encodings: a direct 3-component tangent-space normal (RGB, [0,1] -> + // [-1,1]), or astcenc's `-normal` packing (RGB=X, A=Y) used by texbake.py to get a + // better error metric for vector data โ€” see NativeTexFlags.normalPackedXY. Z is + // reconstructed from the unit-length constraint for the packed case. + // [0,1] -> [-1,1] first, unit length after: normalizing the stored color instead + // shortens it before the remap and leaves every normal tilted (a flat texel, (0.5, + // 0.5, 1), came out 22 degrees off). + float3 normalMapStandard = normalSample.rgb * 2.0 - 1.0; + + float2 packedXY = normalSample.ga * 2.0 - 1.0; + float3 normalMapPackedXY = float3(packedXY, sqrt(saturate(1.0 - dot(packedXY, packedXY)))); + + float3 tangentNormal = normalIsPackedXY ? normalMapPackedXY : normalMapStandard; + tangentNormal = applyNormalStrength(tangentNormal, materialParameter.normalScale); + normalMap = normalize(TBN * tangentNormal); + } + + float roughness = materialParameter.roughness; + if (materialParameter.hasTexture.y == 1) { + float4 roughnessSample = hasHeight + ? roughnessTexture.sample(materialSampler, sampleUV, gradient2d(stDx, stDy)) + : roughnessTexture.sample(materialSampler, st, bias(0.25f)); + roughness = selectTextureChannel(roughnessSample, materialParameter.textureChannels.x) * materialParameter.roughness; + } roughness=clamp(roughness, 0.045, 1.0); - float4 metallicSample = hasHeight - ? metallicTexture.sample(materialSampler, sampleUV, gradient2d(stDx, stDy)) - : metallicTexture.sample(materialSampler, st, bias(0.25f)); - float metallic=(materialParameter.hasTexture.z==1) - ? selectTextureChannel(metallicSample, materialParameter.textureChannels.y) * materialParameter.metallic - : materialParameter.metallic; + // Specular anti-aliasing. Normal detail finer than a pixel (a dart's knurl, + // sisal fibres, a thin wire's curvature) gives a tight specular lobe a + // different peak every frame as the view moves, which reads as sparkle; + // multisampling resolves coverage, not this. Widen the lobe by the + // screen-space variance of the shading normal (Kaplanyan 2016, Tokuyoshi & + // Kaplanyan 2019) so the G-buffer carries a roughness the light pass can + // shade stably. sigma^2 = 0.25 and a 0.18 kernel clamp are the values + // Unreal and Frostbite ship; a flat surface sees no change. + { + float3 dndx = dfdx(normalMap); + float3 dndy = dfdy(normalMap); + float normalVariance = 0.25 * (dot(dndx, dndx) + dot(dndy, dndy)); + float kernelRoughness2 = min(2.0 * normalVariance, 0.18); + roughness = sqrt(roughness * roughness + kernelRoughness2); + } + + float metallic = materialParameter.metallic; + if (materialParameter.hasTexture.z == 1) { + float4 metallicSample = hasHeight + ? metallicTexture.sample(materialSampler, sampleUV, gradient2d(stDx, stDy)) + : metallicTexture.sample(materialSampler, st, bias(0.25f)); + metallic = selectTextureChannel(metallicSample, materialParameter.textureChannels.y) * materialParameter.metallic; + } metallic=clamp(metallic, 0.0, 1.0); // The light the surface gives off: the emissive color, times the emissive texture when // the material has one (the texture carries the color, the factor its strength). It // shares the base color's sampler: both wrap the same way over the same coordinates. - float4 emissiveSample = hasHeight - ? emissiveTexture.sample(baseColorSampler, sampleUV, gradient2d(stDx, stDy)) - : emissiveTexture.sample(baseColorSampler, st, bias(0.25f)); - float3 emissive = (materialParameter.hasEmissiveTexture == 1) - ? materialParameter.emmissive * emissiveSample.rgb - : materialParameter.emmissive; + float3 emissive = materialParameter.emmissive; + if (materialParameter.hasEmissiveTexture == 1) { + float4 emissiveSample = hasHeight + ? emissiveTexture.sample(baseColorSampler, sampleUV, gradient2d(stDx, stDy)) + : emissiveTexture.sample(baseColorSampler, st, bias(0.25f)); + emissive = materialParameter.emmissive * emissiveSample.rgb; + } float4 color=inBaseColor; - gBufferOut.color = float4(color.rgb, passthroughAlpha); - gBufferOut.normals=float4(normalMap,0.0); - gBufferOut.positions=verticesInWorldSpace; - // .b/.a carry POM debug data (raw height sample, uv-offset magnitude) for - // RenderDebugViewMode.heightDebug / .pomOffsetDebug โ€” see RenderingSystem.swift. - gBufferOut.material=float4(roughness, metallic, pomHeightSample, pomOffsetMagnitude); - gBufferOut.emmisive = float4(emissive, 1.0); + float4 base = float4(color.rgb, passthroughAlpha); + float4 emissiveOut = float4(emissive, 1.0); + // The view vector once, for the indirect and the direct light alike. + float3 viewVector = normalize(uniforms.cameraPosition - verticesInWorldSpace.xyz); + if (kModelIndirectInGBuffer) { + emissiveOut = emissiveWithIndirect( + emissive, base, normalMap, verticesInWorldSpace.xyz, roughness, metallic, viewVector, + iblIrradiance, iblSpecular, iblBRDF, csmShadowArray, csmUniforms, lightParams, iblParam, iblRotationAngle); + } + if (kModelGBufferOut) { + gBufferOut.color = base; + gBufferOut.normals = float4(normalMap, 0.0); + gBufferOut.positions = verticesInWorldSpace; + // .b/.a carry POM debug data (raw height sample, uv-offset magnitude) for + // RenderDebugViewMode.heightDebug / .pomOffsetDebug โ€” see RenderingSystem.swift. + gBufferOut.material = float4(roughness, metallic, pomHeightSample, pomOffsetMagnitude); + gBufferOut.emmisive = emissiveOut; + } + if (kModelLitInGBuffer) { + gBufferOut.lit = litWithDirect( + emissiveOut, base, normalMap, verticesInWorldSpace, roughness, metallic, + viewVector, lightParams, plBlock, slBlock, alBlock, ltcMagTexture, ltcMatTexture, + pointShadowCube, spotShadowMap, pointShadowUniforms, spotShadowUniforms); + gBufferOut.velocity = modelVelocity(in.currentClipXYW, in.prevClipXYW); + } return gBufferOut; diff --git a/Sources/UntoldEngine/Shaders/preCompShader.metal b/Sources/UntoldEngine/Shaders/preCompShader.metal index 72d6d1c16..838452135 100644 --- a/Sources/UntoldEngine/Shaders/preCompShader.metal +++ b/Sources/UntoldEngine/Shaders/preCompShader.metal @@ -32,34 +32,42 @@ fragment float4 fragmentPreCompositeShader(VertexCompositeOutput vertexOut [[sta texture2d ssaoTexture [[texture(prePassSSAOTextureIndex)]], constant bool &isGameMode [[ buffer(prePassGizmoBufferIndex) ]], constant bool &isPassthrough [[buffer(prePassPassthroughBufferIndex)]], - constant bool &ssaoEnabled [[buffer(prePassSSAOEnabledIndex)]]) { + constant bool &ssaoEnabled [[buffer(prePassSSAOEnabledIndex)]], + constant bool &gaussianLayerPresent [[buffer(prePassGaussianLayerIndex)]]) { constexpr sampler s(min_filter::linear, mag_filter::linear); - // Sample textures - float4 envColor = envTexture.sample(s, vertexOut.uvCoords); + // This pass is bound by memory bandwidth: every layer it samples is a full-screen read. + // Only the layers the frame has are read โ€” the environment where it shows (passthrough + // has nothing behind the scene), the splat layer on a frame the Gaussian pass drew it + // (otherwise the texture holds an earlier frame, or nothing), the gizmo layer in the editor. float4 modelColor = finalTexture.sample(s, vertexOut.uvCoords); - float4 gaussianColor = gaussianTexture.sample(s, vertexOut.uvCoords); - // Un-premultiply, decode, premultiply again: a pixel the splats cover whole shows the - // trained colour; a partial cover blends with the scene in linear. - if (gaussianColor.a > kSplatLayerAlphaFloor) { - float3 straight = gaussianColor.rgb / gaussianColor.a; - gaussianColor.rgb = splatSRGBToLinear(straight) * gaussianColor.a; + float4 background = float4(0.0, 0.0, 0.0, 0.0); + if (isPassthrough == false) { + background = envTexture.sample(s, vertexOut.uvCoords); } if (ssaoEnabled) { float ao = ssaoTexture.sample(s, vertexOut.uvCoords).r; modelColor.rgb *= ao; } - + // Composite scene color over environment using alpha. // This guarantees transparent pixels blend against environment instead of black. - float4 background = (isPassthrough == false) ? envColor : float4(0.0, 0.0, 0.0, 0.0); float4 baseColor = modelColor + background * (1.0 - modelColor.a); - - // Composite Gaussians (occlusion handled during Gaussian rendering) - baseColor.rgb = gaussianColor.rgb + baseColor.rgb * (1.0 - gaussianColor.a); - baseColor.a = gaussianColor.a + baseColor.a * (1.0 - gaussianColor.a); + + if (gaussianLayerPresent) { + float4 gaussianColor = gaussianTexture.sample(s, vertexOut.uvCoords); + // Un-premultiply, decode, premultiply again: a pixel the splats cover whole shows the + // trained colour; a partial cover blends with the scene in linear. + if (gaussianColor.a > kSplatLayerAlphaFloor) { + float3 straight = gaussianColor.rgb / gaussianColor.a; + gaussianColor.rgb = splatSRGBToLinear(straight) * gaussianColor.a; + } + // Composite Gaussians (occlusion handled during Gaussian rendering) + baseColor.rgb = gaussianColor.rgb + baseColor.rgb * (1.0 - gaussianColor.a); + baseColor.a = gaussianColor.a + baseColor.a * (1.0 - gaussianColor.a); + } // Non-passthrough modes (full immersion, macOS, AR) present an opaque layer: // the environment already fills every background pixel, so alpha must be 1.0 @@ -69,12 +77,12 @@ fragment float4 fragmentPreCompositeShader(VertexCompositeOutput vertexOut [[sta baseColor.a = 1.0; } - // Sample gizmo - float3 gizmoColor = gizmoTexture.sample(s, vertexOut.uvCoords).rgb; - float gizmoLumen = getLuminance(gizmoColor); - - if (!isGameMode && gizmoLumen > 0.1) { - return float4(gizmoColor, 1.0); + // The editor's gizmo layer overrides the pixel wherever it is lit. + if (!isGameMode) { + float3 gizmoColor = gizmoTexture.sample(s, vertexOut.uvCoords).rgb; + if (getLuminance(gizmoColor) > 0.1) { + return float4(gizmoColor, 1.0); + } } return baseColor; diff --git a/Sources/UntoldEngine/Systems/CullingSystem.swift b/Sources/UntoldEngine/Systems/CullingSystem.swift index 040abdb5d..39b116ffa 100644 --- a/Sources/UntoldEngine/Systems/CullingSystem.swift +++ b/Sources/UntoldEngine/Systems/CullingSystem.swift @@ -357,6 +357,13 @@ func initFrustumCulllingCompute() { } public func buildHZBDepthPyramid(_ commandBuffer: MTLCommandBuffer, eyeIndex: Int? = nil) { + // No pyramid for a frame whose depth could hide nothing next frame (the test is off, or + // the scene has too few mesh entities and no splats): the build and the depth copy that + // feeds it are the frame's cost for nothing, and a stale pyramid must not count as valid. + guard HZBOcclusionCulling.pyramidWanted() else { + renderInfo.hzbIsValid = false + return + } // Per-eye stereo path for XR // Current XR stereo rendering builds the shared mono HZB once after both eyes // are rendered, so mixed-mode opaque-depth snapshotting is handled by the @@ -379,7 +386,7 @@ public func buildHZBDepthPyramid(_ commandBuffer: MTLCommandBuffer, eyeIndex: In var mipLevel = UInt32(level) var sourceDimensions = simd_uint2(UInt32(sourceWidth), UInt32(sourceHeight)) var reverseZFlag: UInt32 = renderInfo.reverseZEnabled ? 1 : 0 - let computeEncoder = commandBuffer.makeComputeCommandEncoder()! + let computeEncoder = GPUPassTiming.shared.makeComputeEncoder(commandBuffer, "HZB")! computeEncoder.label = "HZB Build Eye\(ei) Mip \(level)" computeEncoder.setComputePipelineState(pipelineState) computeEncoder.setBytes(&mipLevel, length: MemoryLayout.stride, index: Int(hzbBuildPassMipLevelIndex.rawValue)) @@ -447,7 +454,7 @@ public func buildHZBDepthPyramid(_ commandBuffer: MTLCommandBuffer, eyeIndex: In ) var reverseZFlag: UInt32 = renderInfo.reverseZEnabled ? 1 : 0 - let computeEncoder: MTLComputeCommandEncoder = commandBuffer.makeComputeCommandEncoder()! + let computeEncoder: MTLComputeCommandEncoder = GPUPassTiming.shared.makeComputeEncoder(commandBuffer, "HZB")! computeEncoder.label = "HZB Build Mip \(level)" computeEncoder.setComputePipelineState(pipelineState) computeEncoder.setBytes(&mipLevel, length: MemoryLayout.stride, index: Int(hzbBuildPassMipLevelIndex.rawValue)) @@ -536,7 +543,7 @@ func executeHZBOcclusionCulling( // drift of the depths themselves before an object is called occluded. var occlusionBias: Float = 0.02 - let computeEncoder: MTLComputeCommandEncoder = commandBuffer.makeComputeCommandEncoder()! + let computeEncoder: MTLComputeCommandEncoder = GPUPassTiming.shared.makeComputeEncoder(commandBuffer, "Culling")! computeEncoder.label = "HZB Occlusion Culling pass" computeEncoder.setComputePipelineState(pipelineState) computeEncoder.setBuffer(outputVisibilityBuffer, offset: 0, index: Int(hzbCullPassEntityAABBIndex.rawValue)) @@ -548,6 +555,14 @@ func executeHZBOcclusionCulling( computeEncoder.setBytes(&mipCount, length: MemoryLayout.stride, index: Int(hzbCullPassMipCountIndex.rawValue)) computeEncoder.setBytes(&reverseZFlag, length: MemoryLayout.stride, index: Int(hzbCullPassReverseZIndex.rawValue)) computeEncoder.setBytes(&occlusionBias, length: MemoryLayout.stride, index: Int(hzbCullPassOcclusionBiasIndex.rawValue)) + // The rasterization rate map the pyramid's frame was drawn through (XR foveation): the + // rects are moved into the pyramid's physical layout before it is sampled. + bindRateMap( + pyramidFrame.rateMapData, + to: computeEncoder, + dataIndex: Int(hzbCullPassRateMapDataIndex.rawValue), + sizesIndex: Int(hzbCullPassRateMapSizesIndex.rawValue) + ) computeEncoder.setTexture(hzbDepthPyramid, index: Int(hzbCullPassDepthPyramidTextureIndex.rawValue)) let tew = pipelineState.threadExecutionWidth @@ -635,13 +650,13 @@ public func executeFrustumCulling(_ commandBuffer: MTLCommandBuffer) { frustumWritePointer[i] = simd_float4(frustum.planes[i].n, frustum.planes[i].d) } - let frustumReadBuffer = frustumTripleBuffer.bufferForRead(frame: submitFrameIndex) - let sceneSnapshot = RenderSceneSnapshot() let isXRStereoMode = renderInfo.isXRStereoMode var entityAABBContainer: [EntityAABB] = [] entityAABBContainer.reserveCapacity(sceneSnapshot.entityCapacity) // Pre-allocate to avoid reallocation churn + // The objects the entities belong to (an entity with its children), for the pyramid's gate. + var objects = Set() let smallObjectCulling = SmallObjectCulling.forCurrentFrame() @@ -677,6 +692,8 @@ public func executeFrustumCulling(_ commandBuffer: MTLCommandBuffer) { } } + objects.insert(sceneSnapshot.object(of: entity)) + // In XR stereo mode, use a single AABB per entity. The tighter sub-AABBs // produced by segmentation amplify temporal false-culling from 90 Hz head- // tracking drift and cause segment-level occlusion popping. The unsegmented @@ -706,6 +723,7 @@ public func executeFrustumCulling(_ commandBuffer: MTLCommandBuffer) { } let count = entityAABBContainer.count + let objectCount = objects.count // ensure we have enough capacity entityAABBTripleBuffer.ensureCapacity(count) @@ -721,27 +739,28 @@ public func executeFrustumCulling(_ commandBuffer: MTLCommandBuffer) { return } - let computeEncoder: MTLComputeCommandEncoder = commandBuffer.makeComputeCommandEncoder()! + let computeEncoder: MTLComputeCommandEncoder = GPUPassTiming.shared.makeComputeEncoder(commandBuffer, "Culling")! computeEncoder.label = "Frustum Culling pass" computeEncoder.setComputePipelineState(frustumCullingPipeline.pipelineState!) - computeEncoder.setBuffer(frustumReadBuffer, offset: 0, index: Int(frustumCullingPassPlanesIndex.rawValue)) + computeEncoder.setBuffer(frustumWriteBuffer, offset: 0, index: Int(frustumCullingPassPlanesIndex.rawValue)) - // write current frame's data + // This frame's list and planes, in this frame's slot of the rings, which the frame's own + // compute reads: the rings keep the frames in flight off each other's slots, they do not + // put a frame a slot behind its own data. Reading the slot before (the frame before's + // list, with this frame's count) left the entities at the end of a list that had just + // shrunk untested, so a board's face vanished on the frames its smallest parts dropped out. let entityAABBWriteBuffer = entityAABBTripleBuffer.bufferForWrite(frame: submitFrameIndex) entityAABBContainer.withUnsafeBytes { src in entityAABBWriteBuffer.contents().copyMemory(from: src.baseAddress!, byteCount: src.count) } - // pick the buffer the gpu should read - let entityAABBReadBuffer = entityAABBTripleBuffer.bufferForRead(frame: submitFrameIndex) - let candidateVisibilityBuffer = hzbCandidateVisibilityTriple.bufferForWrite(frame: submitFrameIndex) - computeEncoder.setBuffer(entityAABBReadBuffer, offset: 0, index: Int(frustumCullingPassObjectIndex.rawValue)) + computeEncoder.setBuffer(entityAABBWriteBuffer, offset: 0, index: Int(frustumCullingPassObjectIndex.rawValue)) var count32 = UInt32(count) computeEncoder.setBytes(&count32, length: MemoryLayout.stride, index: Int(frustumCullingPassObjectCountIndex.rawValue)) @@ -789,6 +808,7 @@ public func executeFrustumCulling(_ commandBuffer: MTLCommandBuffer) { // few centimetres apart, the frames a camera step. The per-eye pyramids // (hzbDepthPyramidEye) are allocated, but nothing renders the per-eye depth they // would be built from. + renderInfo.hzbCandidateCount = objectCount let ran0 = executeHZBOcclusionCulling( commandBuffer, cameraPosition: eyePosition(ofView: SceneRootTransform.shared.effectiveViewMatrix(renderInfo.xrEye0View)), @@ -845,6 +865,7 @@ public func executeFrustumCulling(_ commandBuffer: MTLCommandBuffer) { } else { // Mono path (macOS / non-stereo) + renderInfo.hzbCandidateCount = objectCount let didRunOcclusion = executeHZBOcclusionCulling( commandBuffer, cameraPosition: eyePosition(ofView: effectiveViewMatrix), @@ -970,12 +991,12 @@ func executeReduceScanFrustumCulling(_ commandBuffer: MTLCommandBuffer) { frustumWritePointer[i] = simd_float4(frustum.planes[i].n, frustum.planes[i].d) } - let frustumReadBuffer = frustumTripleBuffer.bufferForRead(frame: submitFrameIndex) - let sceneSnapshot = RenderSceneSnapshot() var entityAABBContainer: [EntityAABB] = [] entityAABBContainer.reserveCapacity(sceneSnapshot.entityCapacity) // Pre-allocate to avoid reallocation churn + // The objects the entities belong to (an entity with its children), for the pyramid's gate. + var objects = Set() let smallObjectCulling = SmallObjectCulling.forCurrentFrame() @@ -1011,6 +1032,8 @@ func executeReduceScanFrustumCulling(_ commandBuffer: MTLCommandBuffer) { } } + objects.insert(sceneSnapshot.object(of: entity)) + // get object AABB โ€” elongated meshes are split into segments. let segments = makeSegmentedEntityAABBs( localMin: localTransformComponent.boundingBox.min, @@ -1023,6 +1046,7 @@ func executeReduceScanFrustumCulling(_ commandBuffer: MTLCommandBuffer) { } let count = entityAABBContainer.count + let objectCount = objects.count // ensure we have enough capacity entityAABBTripleBuffer.ensureCapacity(count) @@ -1049,30 +1073,31 @@ func executeReduceScanFrustumCulling(_ commandBuffer: MTLCommandBuffer) { ensureReduceScanCapacity(scannedCount) let numBlocks = Int32((scannedCount + Int(BLOCK_SIZE) - 1) / Int(BLOCK_SIZE)) - // write current frame's data + // This frame's list and planes, in this frame's slot of the rings, which the frame's own + // compute reads: the rings keep the frames in flight off each other's slots, they do not + // put a frame a slot behind its own data. Reading the slot before (the frame before's + // list, with this frame's count) left the entities at the end of a list that had just + // shrunk untested, so a board's face vanished on the frames its smallest parts dropped out. let entityAABBWriteBuffer = entityAABBTripleBuffer.bufferForWrite(frame: submitFrameIndex) entityAABBContainer.withUnsafeBytes { src in entityAABBWriteBuffer.contents().copyMemory(from: src.baseAddress!, byteCount: src.count) } - // pick the buffer the gpu should read - let entityAABBReadBuffer = entityAABBTripleBuffer.bufferForRead(frame: submitFrameIndex) - let candidateVisibilityBuffer = hzbCandidateVisibilityTriple.bufferForWrite(frame: submitFrameIndex) var count32 = UInt32(scannedCount) // Mark visible launch do { - let computeEncoderMarkVisible: MTLComputeCommandEncoder = commandBuffer.makeComputeCommandEncoder()! + let computeEncoderMarkVisible: MTLComputeCommandEncoder = GPUPassTiming.shared.makeComputeEncoder(commandBuffer, "Culling")! computeEncoderMarkVisible.label = "Mark Visible Pass" computeEncoderMarkVisible.setComputePipelineState(reduceScanMarkVisiblePipeline.pipelineState!) - computeEncoderMarkVisible.setBuffer(frustumReadBuffer, offset: 0, index: Int(markVisibilityPassFrustumIndex.rawValue)) - computeEncoderMarkVisible.setBuffer(entityAABBReadBuffer, offset: 0, index: Int(markVisibilityPassEntityAABBIndex.rawValue)) + computeEncoderMarkVisible.setBuffer(frustumWriteBuffer, offset: 0, index: Int(markVisibilityPassFrustumIndex.rawValue)) + computeEncoderMarkVisible.setBuffer(entityAABBWriteBuffer, offset: 0, index: Int(markVisibilityPassEntityAABBIndex.rawValue)) computeEncoderMarkVisible.setBytes(&count32, length: MemoryLayout.stride, index: Int(markVisibilityPassEntityAABBCountIndex.rawValue)) computeEncoderMarkVisible.setBuffer(bufferResources.reduceScanFlags, offset: 0, index: Int(markVisibilityPassFlagIndex.rawValue)) @@ -1085,7 +1110,7 @@ func executeReduceScanFrustumCulling(_ commandBuffer: MTLCommandBuffer) { // scan local do { - let computeEncoderLocalScan: MTLComputeCommandEncoder = commandBuffer.makeComputeCommandEncoder()! + let computeEncoderLocalScan: MTLComputeCommandEncoder = GPUPassTiming.shared.makeComputeEncoder(commandBuffer, "Culling")! computeEncoderLocalScan.label = "Local Scan Pass" @@ -1107,7 +1132,7 @@ func executeReduceScanFrustumCulling(_ commandBuffer: MTLCommandBuffer) { // scan block do { - let computeEncoderBlockScan: MTLComputeCommandEncoder = commandBuffer.makeComputeCommandEncoder()! + let computeEncoderBlockScan: MTLComputeCommandEncoder = GPUPassTiming.shared.makeComputeEncoder(commandBuffer, "Culling")! computeEncoderBlockScan.label = "Block Scan Pass" @@ -1132,7 +1157,7 @@ func executeReduceScanFrustumCulling(_ commandBuffer: MTLCommandBuffer) { // Compact and stream do { - let computeEncoderCompact: MTLComputeCommandEncoder = commandBuffer.makeComputeCommandEncoder()! + let computeEncoderCompact: MTLComputeCommandEncoder = GPUPassTiming.shared.makeComputeEncoder(commandBuffer, "Culling")! computeEncoderCompact.label = "Compact and Stream Pass" @@ -1141,7 +1166,7 @@ func executeReduceScanFrustumCulling(_ commandBuffer: MTLCommandBuffer) { computeEncoderCompact.setBuffer(bufferResources.reduceScanFlags, offset: 0, index: Int(compactPassFlagsIndex.rawValue)) computeEncoderCompact.setBuffer(bufferResources.reduceScanIndices, offset: 0, index: Int(compactPassIndicesIndex.rawValue)) computeEncoderCompact.setBuffer(bufferResources.reduceScanBlockOffsets, offset: 0, index: Int(compactPassBlockOffsetIndex.rawValue)) - computeEncoderCompact.setBuffer(entityAABBReadBuffer, offset: 0, index: Int(compactPassEntityAABBIndex.rawValue)) + computeEncoderCompact.setBuffer(entityAABBWriteBuffer, offset: 0, index: Int(compactPassEntityAABBIndex.rawValue)) computeEncoderCompact.setBytes(&count32, length: MemoryLayout.stride, index: Int(compactPassCountIndex.rawValue)) @@ -1159,6 +1184,7 @@ func executeReduceScanFrustumCulling(_ commandBuffer: MTLCommandBuffer) { let finalVisibleCountBuffer = visibleCountTriple.bufferForWrite(frame: submitFrameIndex) let finalVisibilityBuffer = visibilityTriple.bufferForWrite(frame: submitFrameIndex) + renderInfo.hzbCandidateCount = objectCount let didRunOcclusion = executeHZBOcclusionCulling( commandBuffer, cameraPosition: eyePosition(ofView: effectiveViewMatrix), @@ -1235,6 +1261,31 @@ public enum HZBOcclusionCulling { /// draws what the frustum keeps. In scene units; 0.5 by default. Set it with /// `setRendering(.occlusionCullingMaxCameraStep(_:))`. nonisolated(unsafe) static var maxCameraStep: Float = 0.5 + + /// The fewest objects in the culling set for which the test, and the depth pyramid it + /// reads, built from a copy of the frame's depth every frame, are worth their cost: below + /// it nothing can hide anything. An object is an entity with its children: a model loaded + /// from a file is one entity per node under the entity it was set on, and the test + /// never hides one part of a model behind another, their bounds overlapping. 2 by + /// default, since one object hides nothing but itself. A frame with splat entities keeps + /// the pyramid regardless: their cull reads it. Set it with + /// `setRendering(.occlusionCullingMinimumEntities(_:))`. + nonisolated(unsafe) static var minimumEntities: Int = 2 + + /// Whether the frame's depth is worth a pyramid: the test is on, and the scene has enough + /// objects or any splats for it to hide something next frame. + static func pyramidWanted() -> Bool { + guard maxCameraStep > 0 else { return false } + if renderInfo.hzbCandidateCount >= minimumEntities { return true } + let slot = min(renderInfo.currentInFlightFrameSlot, maxInFlightCommandBuffers - 1) + return !GaussianSharedWorkingSet.shared.entityOrder(slot: slot).isEmpty + } +} + +/// The fewest objects (entities with their children) for which the occlusion test runs and +/// its depth pyramid is built. +public func getOcclusionCullingMinimumEntities() -> Int { + HZBOcclusionCulling.minimumEntities } /// The farthest the camera may move between two frames for the occlusion test to run. @@ -1257,9 +1308,11 @@ extension HZBPyramidFrame { let cameraComponent = scene.get(component: CameraComponent.self, for: camera) else { return nil } let view = SceneRootTransform.shared.effectiveViewMatrix(cameraComponent.viewSpace) + // In stereo the last eye drawn: its view, projection and rate map alike. return HZBPyramidFrame( viewProjection: simd_mul(renderInfo.perspectiveSpace, view), - cameraPosition: eyePosition(ofView: view) + cameraPosition: eyePosition(ofView: view), + rateMapData: renderInfo.xrFoveation?.rateMapData ) } } diff --git a/Sources/UntoldEngine/Systems/FrameEvents.swift b/Sources/UntoldEngine/Systems/FrameEvents.swift index cefe60ac2..2196feacd 100644 --- a/Sources/UntoldEngine/Systems/FrameEvents.swift +++ b/Sources/UntoldEngine/Systems/FrameEvents.swift @@ -13,10 +13,18 @@ import Foundation /// Per-frame update event, delivered once per rendered frame regardless of `gameMode`. /// Mirrors RealityKit's `SceneEvents.Update`. public struct UpdateEvent { - /// Seconds elapsed since the previous frame. + /// Seconds elapsed since the previous frame: from its presentation to this one's. public let deltaTime: TimeInterval /// The renderer that produced this frame. public let renderer: UntoldRenderer + /// The frame's predicted presentation time, its delta and the refreshes skipped before it. + public let frameTime: FrameTime + + init(deltaTime: TimeInterval, renderer: UntoldRenderer, frameTime: FrameTime = .zero) { + self.deltaTime = deltaTime + self.renderer = renderer + self.frameTime = frameTime + } } /// Token returned by update subscriptions. Call `cancel()` to stop receiving events. diff --git a/Sources/UntoldEngine/Systems/GaussianChunkCull.swift b/Sources/UntoldEngine/Systems/GaussianChunkCull.swift index 93c307986..bf425a541 100644 --- a/Sources/UntoldEngine/Systems/GaussianChunkCull.swift +++ b/Sources/UntoldEngine/Systems/GaussianChunkCull.swift @@ -1033,6 +1033,9 @@ func gaussianChunkCullConstants( constants.hzbValid = hzbValid ? 1 : 0 constants.hzbReverseZ = renderInfo.reverseZEnabled ? 1 : 0 constants.hzbMipCount = UInt32(max(0, renderInfo.hzbMipCount)) + // The rasterization rate map the pyramid was drawn through (XR foveation), only with a + // pyramid to sample: the "no map" word otherwise, and the kernels never touch the buffer. + constants.hzbRateMapSizes = hzbValid ? rateMapSizes(hzbPyramidRateMapData()) : .zero constants.forceAllVisible = forceAllVisible ? 1 : 0 constants.uniformQuotas = uniformQuotas ? 1 : 0 constants.paged = paged @@ -1255,6 +1258,12 @@ private func encodeGaussianChunkCullDispatch( } else { encoder.setBytes(&gpuRanges, length: MemoryLayout.stride * gpuRanges.count, index: Int(gaussianChunkCullRangesIndex.rawValue)) } + // The pyramid's rate map (XR foveation), read only when the constants' hzbRateMapSizes say so. + encoder.setBuffer( + hzbPyramidRateMapData()?.buffer ?? chunkTable.constantsBuffer, + offset: 0, + index: Int(gaussianChunkCullRateMapDataIndex.rawValue) + ) encoder.setTexture(hzbTexture, index: Int(gaussianChunkCullHZBDepthPyramidTextureIndex.rawValue)) let tew = pipelines.cull.threadExecutionWidth let block = max(min(256, pipelines.cull.maxTotalThreadsPerThreadgroup) / tew * tew, tew) @@ -1530,6 +1539,12 @@ func encodeGaussianChunkDecodePreprocess( encoder.setBuffer(inputs.levels?.coarseTable ?? inputs.chunkTable.constantsBuffer, offset: 0, index: Int(gaussianChunkPreprocessCoarseTableIndex.rawValue)) encoder.setBuffer(inputs.levels?.levelState ?? inputs.chunkTable.constantsBuffer, offset: 0, index: Int(gaussianChunkPreprocessLevelStateIndex.rawValue)) encoder.setBytes(&inputs.levelConstants, length: MemoryLayout.stride, index: Int(gaussianChunkPreprocessLevelConstantsIndex.rawValue)) + // The pyramid's rate map (XR foveation), read only when the cull constants' hzbRateMapSizes say so. + encoder.setBuffer( + hzbPyramidRateMapData()?.buffer ?? inputs.chunkTable.constantsBuffer, + offset: 0, + index: Int(gaussianChunkPreprocessRateMapDataIndex.rawValue) + ) encoder.setTexture(inputs.hzbTexture, index: Int(gaussianChunkPreprocessHZBDepthPyramidTextureIndex.rawValue)) let threads = threadsPerThreadgroup ?? gaussianChunkPreprocessThreadsPerThreadgroup(chunkTable: inputs.chunkTable, pipelineState: pipelineState) diff --git a/Sources/UntoldEngine/Systems/GaussianSystem.swift b/Sources/UntoldEngine/Systems/GaussianSystem.swift index f80679111..587f75bcd 100644 --- a/Sources/UntoldEngine/Systems/GaussianSystem.swift +++ b/Sources/UntoldEngine/Systems/GaussianSystem.swift @@ -128,6 +128,14 @@ private func bindGaussianSplatCullInputs( encoder.setBytes(&inputs.hzbReverseZ, length: MemoryLayout.stride, index: Int(gaussianCullHZBReverseZIndex.rawValue)) encoder.setBytes(&inputs.hzbOcclusionBias, length: MemoryLayout.stride, index: Int(gaussianCullHZBOcclusionBiasIndex.rawValue)) encoder.setBytes(&inputs.hzbValid, length: MemoryLayout.stride, index: Int(gaussianCullHZBValidIndex.rawValue)) + // The rasterization rate map the pyramid was drawn through (XR foveation), with a pyramid + // to sample; the "no map" word otherwise. + bindRateMap( + inputs.hzbValid != 0 ? hzbPyramidRateMapData() : nil, + to: encoder, + dataIndex: Int(gaussianCullHZBRateMapDataIndex.rawValue), + sizesIndex: Int(gaussianCullHZBRateMapSizesIndex.rawValue) + ) encoder.setTexture(hzbTexture, index: Int(gaussianCullHZBDepthPyramidTextureIndex.rawValue)) } @@ -813,10 +821,15 @@ public func executeGaussianPreprocess(_ commandBuffer: MTLCommandBuffer) { let transformId = getComponentId(for: WorldTransformComponent.self) let gaussianId = getComponentId(for: GaussianComponent.self) let entities = queryEntitiesWithComponentIds([transformId, gaussianId], in: scene) - guard !entities.isEmpty else { return } - let workingSet = GaussianSharedWorkingSet.shared let frameSlot = min(renderInfo.currentInFlightFrameSlot, maxInFlightCommandBuffers - 1) + guard !entities.isEmpty else { + // Nothing stamps this slot's records this frame, so they are whatever an earlier scene + // left: an empty order tells the draw pass to skip the frame instead of drawing them. + workingSet.setEntityOrder([], slot: frameSlot) + return + } + guard let sharedKeys = workingSet.keys(slot: frameSlot), let sharedRecords = workingSet.records(slot: frameSlot), let sharedVisibleSet = workingSet.visibleSet(slot: frameSlot) diff --git a/Sources/UntoldEngine/Systems/GraphBuilder.swift b/Sources/UntoldEngine/Systems/GraphBuilder.swift index df194fc4a..76774a7aa 100644 --- a/Sources/UntoldEngine/Systems/GraphBuilder.swift +++ b/Sources/UntoldEngine/Systems/GraphBuilder.swift @@ -217,6 +217,9 @@ public struct RenderGraphBuildContext: Sendable { public let frameIndex: UInt64 public let currentEye: Int public let isPrimaryEye: Bool + /// The frame's predicted presentation time, its delta from the frame before and the + /// refreshes skipped between them (`deltaTime` is its delta as a Float). + public let frameTime: FrameTime public init( viewport: SIMD2, @@ -224,7 +227,8 @@ public struct RenderGraphBuildContext: Sendable { deltaTime: Float = 0, frameIndex: UInt64 = 0, currentEye: Int, - isPrimaryEye: Bool = true + isPrimaryEye: Bool = true, + frameTime: FrameTime = .zero ) { self.viewport = viewport self.immersionStyle = immersionStyle @@ -232,6 +236,7 @@ public struct RenderGraphBuildContext: Sendable { self.frameIndex = frameIndex self.currentEye = currentEye self.isPrimaryEye = isPrimaryEye + self.frameTime = frameTime } } @@ -252,6 +257,9 @@ public struct RenderPassContext { public let computePipelines: ComputePipelineAccess public let renderPipelines: RenderPipelineAccess public let sceneRenderTargets: SceneRenderTargetAccess + /// The frame's predicted presentation time, its delta from the frame before and the + /// refreshes skipped between them (`deltaTime` is its delta as a Float). + public let frameTime: FrameTime public init( commandBuffer: MTLCommandBuffer, @@ -269,7 +277,8 @@ public struct RenderPassContext { resources: RenderResourceAccess = RenderResourceAccess(), computePipelines: ComputePipelineAccess = ComputePipelineAccess(), renderPipelines: RenderPipelineAccess = RenderPipelineAccess(), - sceneRenderTargets: SceneRenderTargetAccess = SceneRenderTargetAccess() + sceneRenderTargets: SceneRenderTargetAccess = SceneRenderTargetAccess(), + frameTime: FrameTime = .zero ) { self.commandBuffer = commandBuffer self.device = device @@ -287,6 +296,7 @@ public struct RenderPassContext { self.computePipelines = computePipelines self.renderPipelines = renderPipelines self.sceneRenderTargets = sceneRenderTargets + self.frameTime = frameTime } } @@ -861,7 +871,8 @@ func makeRenderGraphBuildContext() -> RenderGraphBuildContext { deltaTime: timeSinceLastUpdate ?? 0, frameIndex: renderInfo.frameIndex, currentEye: renderInfo.currentEye, - isPrimaryEye: isPrimaryRenderEye() + isPrimaryEye: isPrimaryRenderEye(), + frameTime: currentFrameTime ) } @@ -909,7 +920,8 @@ func makeRenderPassContext( resources: resources, computePipelines: ComputePipelineAccess(), renderPipelines: RenderPipelineAccess(), - sceneRenderTargets: sceneRenderTargets + sceneRenderTargets: sceneRenderTargets, + frameTime: currentFrameTime ) } @@ -922,7 +934,8 @@ func makeEngineExtensionUpdateContext() -> EngineExtensionUpdateContext { immersionStyle: renderInfo.immersionStyle, frameIndex: renderInfo.frameIndex, currentEye: renderInfo.currentEye, - isPrimaryEye: isPrimaryRenderEye() + isPrimaryEye: isPrimaryRenderEye(), + frameTime: currentFrameTime ) } @@ -949,6 +962,9 @@ func makeSceneRenderTargetAccess( return nil } encoder.label = label + // The copied descriptor carries the eye's rasterization rate map (XR foveation); the + // viewport that goes with it is the eye's, not the texture's. + applyXRFoveationViewport(encoder) return encoder } } diff --git a/Sources/UntoldEngine/Systems/RegistrationSystem.swift b/Sources/UntoldEngine/Systems/RegistrationSystem.swift index b1c06ba6a..ad2fd9565 100644 --- a/Sources/UntoldEngine/Systems/RegistrationSystem.swift +++ b/Sources/UntoldEngine/Systems/RegistrationSystem.swift @@ -2482,6 +2482,19 @@ public struct UntoldPackData: Decodable { /// levels from the finest (see `UntoldMeshLODCooker` in the UntoldEngineMeshCook /// module). Nil in a pack cooked without them. public let lodChains: [String: [UntoldPackLODLevelEntry]]? + + /// Resolves `sourceAsset` to a URL, given the pack's own file location. `sourceAsset` + /// is always relative to the pack's directory, like each model's `path` -- a bare + /// filename (the exporter's original "source sits beside the pack" convention) is + /// just the degenerate case of that with no intermediate directories, so the same + /// resolution works whether the source still sits beside the pack or lives in a + /// separate tree the pack's own folder doesn't share (e.g. `../../../Resources/...`). + public func resolvedSourceAssetURL(packURL: URL) -> URL? { + guard let sourceAsset else { return nil } + return packURL.deletingLastPathComponent() + .appendingPathComponent(sourceAsset) + .standardizedFileURL + } } /// One simplified level of a pack model: a `.untold` with the model's entities and diff --git a/Sources/UntoldEngine/Systems/RenderingSystem.swift b/Sources/UntoldEngine/Systems/RenderingSystem.swift index 42da9e94d..06b69cf95 100644 --- a/Sources/UntoldEngine/Systems/RenderingSystem.swift +++ b/Sources/UntoldEngine/Systems/RenderingSystem.swift @@ -23,6 +23,7 @@ public typealias UpdateXRRenderingSystemCallback = (RenderingSystemContext) -> V @MainActor func UpdateRenderingSystem(in view: MTKView) { + ObjectMotion.shared.beginFrame() // Snapshot loading gate once per frame. While loading, keep rendering from the // last-known-good visible list and skip ECS traversal to avoid race conditions. let loading = AssetLoadingGate.shared.isLoadingAny @@ -40,6 +41,7 @@ func UpdateRenderingSystem(in view: MTKView) { let renderTotalStart = CACurrentMediaTime() #endif renderInfo.lastCommandBuffer = commandBuffer + GPUPassTiming.shared.beginFrame(device: renderInfo.device) renderInfo.currentInFlightFrameSlot = acquireUniformFrameSlot() // Always refresh the scene-root matrices so that effectiveCameraPosition() and @@ -94,7 +96,12 @@ func UpdateRenderingSystem(in view: MTKView) { commandBuffer.label = "Rendering Command Buffer" do { + // This frame's sub-pixel jitter (temporal anti-aliasing), before anything reads the projection. + TemporalAntiAliasing.shared.applyJitter() let graph = try buildExecutableGameModeGraph() + // A rasterization rate map set on renderInfo.xrFoveation (the foveation tests + // draw through one on the Mac) goes on the scene passes here as in XR. + applyXRFoveationToSceneRenderPassDescriptors() #if ENGINE_STATS_ENABLED let encodeStart = CACurrentMediaTime() @@ -127,8 +134,7 @@ func UpdateRenderingSystem(in view: MTKView) { let visibleEntityIdsAtSubmission = visibleEntityIds commandBuffer.addCompletedHandler { cb in #if ENGINE_STATS_ENABLED - let gpuExecutionMs = (cb.gpuEndTime - cb.gpuStartTime) * 1000.0 - EngineStatsMonitor.shared.recordGPUCompletion(executionMs: gpuExecutionMs) + EngineStatsMonitor.shared.recordGPUCompletion(gpuStartTime: cb.gpuStartTime, gpuEndTime: cb.gpuEndTime) #endif // Signal that this command buffer slot is now available commandBufferSemaphore.signal() @@ -137,6 +143,7 @@ func UpdateRenderingSystem(in view: MTKView) { MemoryBudgetManager.shared.markUsed(entityIds: visibleEntityIdsAtSubmission) } + GPUPassTiming.shared.endFrame(commandBuffer) #if ENGINE_STATS_ENABLED let submitStart = CACurrentMediaTime() #endif @@ -168,8 +175,14 @@ func UpdateXRRenderingSystem(commandBuffer: MTLCommandBuffer, passDescriptor: MT commandBuffer.label = "XR Rendering Command Buffer" do { + // This eye's sub-pixel jitter (temporal anti-aliasing), before anything reads the projection. + TemporalAntiAliasing.shared.applyJitter() let graph = try buildExecutableGameModeGraph() + // The eye's rasterization rate map (visionOS foveation) on the passes that draw the + // scene, after the build: a rebuild may have recreated their descriptors. + applyXRFoveationToSceneRenderPassDescriptors() + #if ENGINE_STATS_ENABLED let encodeStart = shouldRecordStatsInThisCallback ? CACurrentMediaTime() : 0.0 #endif @@ -199,8 +212,7 @@ func UpdateXRRenderingSystem(commandBuffer: MTLCommandBuffer, passDescriptor: MT commandBuffer.addCompletedHandler { cb in #if ENGINE_STATS_ENABLED if renderInfo.immersionStyle == .ar { - let gpuExecutionMs = (cb.gpuEndTime - cb.gpuStartTime) * 1000.0 - EngineStatsMonitor.shared.recordGPUCompletion(executionMs: gpuExecutionMs) + EngineStatsMonitor.shared.recordGPUCompletion(gpuStartTime: cb.gpuStartTime, gpuEndTime: cb.gpuEndTime) } #endif needsFinalizeDestroys = true @@ -256,6 +268,7 @@ let gameModeReservedPassIDs: Set = [ "look", "fxaa", "fxaaEdgeDebug", + "taa", "smaaEdges", "smaaBlendWeights", "smaaNeighborhood", @@ -526,6 +539,10 @@ private func buildGameModeGraphWithCompilation() throws -> CompiledRenderGraphRe let fxaaPass = RenderPass(id: "fxaa", dependencies: [lookPass.id], execute: fxaaRenderPass) builder.addPass(fxaaPass) outputDependency = fxaaPass.id + case .taa, .msaaTaa: + let taaPass = RenderPass(id: "taa", dependencies: [lookPass.id], execute: taaRenderPass) + builder.addPass(taaPass) + outputDependency = taaPass.id case .smaa: let smaaEdgesPass = RenderPass(id: "smaaEdges", dependencies: [lookPass.id], execute: smaaEdgesRenderPass) builder.addPass(smaaEdgesPass) @@ -550,6 +567,11 @@ private func buildGameModeGraphWithCompilation() throws -> CompiledRenderGraphRe } let beforeOutputID = builder.resolveStage(.beforeOutput, after: outputDependency) ?? outputDependency + // With nothing between the temporal resolve and the output, the resolve presents the frame + // itself (TemporalAntiAliasing.encodeResolve) and the output pass stands down. + renderInfo.temporalResolveMayPresent = antiAliasingMode.isTemporal && beforeOutputID == outputDependency + // The same for the look pass when it is the last pass before the output. + renderInfo.lookMayPresent = outputDependency == lookPass.id && beforeOutputID == outputDependency let outputPass = RenderPass( id: "outputTransform", @@ -882,6 +904,12 @@ func colorGradingCustomization(encoder: MTLRenderCommandEncoder) { length: MemoryLayout.stride, index: Int(tonemapOperatorSelectIndex.rawValue) ) + var colorPrimaries = TonemapParams.shared.primaries.shaderValue + encoder.setFragmentBytes( + &colorPrimaries, + length: MemoryLayout.stride, + index: Int(lookPassColorPrimariesIndex.rawValue) + ) // The Gaussian pass's layer keeps splat pixels out of the grade and the tone map: they are // display-referred already, and a partly covered pixel has only the scene behind the splats @@ -1368,6 +1396,70 @@ private func debugSourceTexture(for mode: RenderDebugViewMode) -> MTLTexture? { } } +/// The look pass presenting the frame itself, when the output transform would only copy it +/// (`renderInfo.lookMayPresent`): the display's colour and depth textures become the pass's +/// second colour attachment and depth, as the temporal resolve does. Returns whether it ran; +/// false leaves the frame to the look pass proper and the output transform. +private func lookPresentingPass(source: MTLTexture, destination: MTLTexture) -> (MTLCommandBuffer) -> Bool { + { commandBuffer in + let presentDescriptor = renderInfo.renderPassDescriptor + let presentColor = presentDescriptor?.colorAttachments[0].texture + let presentDepth = presentDescriptor?.depthAttachment.texture + guard renderInfo.lookMayPresent, + let presentDescriptor, let presentColor, let presentDepth, + presentColor.width == destination.width, presentColor.height == destination.height, + let sourceDepth = textureResources.depthMap, + let pipeline = PipelineManager.shared.renderPipelinesByType[.lookPresent], + pipeline.success, let pipelineState = pipeline.pipelineState + else { return false } + + let descriptor = MTLRenderPassDescriptor() + descriptor.colorAttachments[0].texture = destination + descriptor.colorAttachments[0].loadAction = .dontCare + descriptor.colorAttachments[0].storeAction = .store + descriptor.colorAttachments[1].texture = presentColor + descriptor.colorAttachments[1].loadAction = presentDescriptor.colorAttachments[0].loadAction + descriptor.colorAttachments[1].clearColor = presentDescriptor.colorAttachments[0].clearColor + descriptor.colorAttachments[1].storeAction = .store + descriptor.depthAttachment.texture = presentDepth + descriptor.depthAttachment.loadAction = presentDescriptor.depthAttachment.loadAction + descriptor.depthAttachment.clearDepth = presentDescriptor.depthAttachment.clearDepth + descriptor.depthAttachment.storeAction = .store + + GPUPassTiming.shared.attach(descriptor, "Post") + guard let encoder = commandBuffer.makeRenderCommandEncoder(descriptor: descriptor) else { + handleError(.renderPassCreationFailed, "Look Pass (presents)") + return false + } + encoder.label = "Look Pass (presents)" + encoder.pushDebugGroup("Look Pass (presents)") + encoder.setRenderPipelineState(pipelineState) + if let depthState = pipeline.depthState { + encoder.setDepthStencilState(depthState) + } + encoder.waitForFence(renderInfo.fence, before: .fragment) + encoder.setVertexBuffer(bufferResources.quadVerticesBuffer, offset: 0, index: 0) + encoder.setVertexBuffer(bufferResources.quadTexCoordsBuffer, offset: 0, index: 1) + encoder.setFragmentTexture(source, index: 0) + colorGradingCustomization(encoder: encoder) + encoder.setFragmentTexture(sourceDepth, index: Int(lookPassDepthTextureIndex.rawValue)) + var encodingMode = renderInfo.colorPipeline.present.encodingMode.rawValue + encoder.setFragmentBytes(&encodingMode, length: MemoryLayout.stride, index: Int(lookPassEncodingModeIndex.rawValue)) + encoder.drawIndexedPrimitivesTracked( + type: .triangle, + indexCount: 6, + indexType: .uint16, + indexBuffer: bufferResources.quadIndexBuffer!, + indexBufferOffset: 0 + ) + encoder.updateFence(renderInfo.fence, after: .fragment) + encoder.popDebugGroup() + encoder.endEncoding() + renderInfo.lookPresentedThisFrame = true + return true + } +} + private func lookPassShouldRenderLitOutput(for mode: RenderDebugViewMode) -> Bool { switch mode { case .lit, .fxaaEdgeDebug, .smaaEdges, .smaaBlend, .smaaDifference, .occlusionDebug, .postTonemapOutput: @@ -1380,16 +1472,21 @@ private func lookPassShouldRenderLitOutput(for mode: RenderDebugViewMode) -> Boo public let lookRenderPass: RenderPasses.RenderPassExecution = { commandBuffer in let viewMode = renderDebugViewMode + renderInfo.lookPresentedThisFrame = false guard let destinationTexture = textureResources.lookTexture else { handleError(.renderPassCreationFailed, "Look Pass: destination texture is nil") return } if lookPassShouldRenderLitOutput(for: viewMode) { - guard let sourceTexture = textureResources.sceneCompositeTexture else { + // The composite, or the lit scene itself on a frame the pre-composite pass was skipped. + guard let sourceTexture = renderInfo.preCompositeBypassSource ?? textureResources.sceneCompositeTexture else { handleError(.renderPassCreationFailed, "Look Pass: source texture is nil") return } + if lookPresentingPass(source: sourceTexture, destination: destinationTexture)(commandBuffer) { + return + } guard let pipeline = PipelineManager.shared.renderPipelinesByType[.look] else { handleError(.pipelineStateNulled, "Look Pipeline is nil") return @@ -1403,6 +1500,7 @@ public let lookRenderPass: RenderPasses.RenderPassExecution = { commandBuffer in pipeline, source: sourceTexture, destination: destinationTexture, + descriptor: renderInfo.lookRenderPassDescriptor, customization: colorGradingCustomization )(commandBuffer) return @@ -1457,6 +1555,13 @@ public let lookRenderPass: RenderPasses.RenderPassExecution = { commandBuffer in } public let outputTransformRenderPass: RenderPasses.RenderPassExecution = { commandBuffer in + // The temporal resolve, or the look pass, presented the frame already. + if antiAliasingMode.isTemporal, TemporalAntiAliasing.shared.presentedThisFrame { + return + } + if renderInfo.lookPresentedThisFrame { + return + } let sourceTexture: MTLTexture? switch renderDebugViewMode { case .fxaaEdgeDebug: @@ -1468,9 +1573,14 @@ public let outputTransformRenderPass: RenderPasses.RenderPassExecution = { comma case .smaaDifference: sourceTexture = textureResources.smaaBlendTexture default: - sourceTexture = antiAliasingMode.usesPostLookPass - ? textureResources.antiAliasingTexture - : textureResources.lookTexture + if antiAliasingMode.isTemporal { + // The temporal resolve writes its result once, as the eye's new history. + sourceTexture = TemporalAntiAliasing.shared.resolvedTexture ?? textureResources.lookTexture + } else { + sourceTexture = antiAliasingMode.usesPostLookPass + ? textureResources.antiAliasingTexture + : textureResources.lookTexture + } } guard let sourceTexture else { handleError(.renderPassCreationFailed, "Output Transform Pass: source texture is nil") @@ -1498,6 +1608,7 @@ public let outputTransformRenderPass: RenderPasses.RenderPassExecution = { comma // Do not override clearColor here: the XR layer sets it per immersion mode // (UntoldEngineXR.swift), and the fullscreen quad overwrites every pixel anyway. + GPUPassTiming.shared.attach(renderPassDescriptor, "Output") guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else { handleError(.renderPassCreationFailed, "Output Transform Pass: encoder creation failed") return diff --git a/Sources/UntoldEngine/Systems/ShadowSystem.swift b/Sources/UntoldEngine/Systems/ShadowSystem.swift index 27e36b306..10fbc85a9 100644 --- a/Sources/UntoldEngine/Systems/ShadowSystem.swift +++ b/Sources/UntoldEngine/Systems/ShadowSystem.swift @@ -40,19 +40,31 @@ public struct ShadowSoftnessSettings: Equatable, Sendable { public var farRadiusTexels: Float public var depthScale: Float public var xrRadiusScale: Float + /// The compares per cascade sample: the whole 16-point Poisson disk, or 8 or 4 of its + /// points. Fewer taps cost less per fragment and leave a grainier penumbra, which a + /// temporal resolve smooths. + public var filterTaps: Int + /// The same on a headset, 8 by default: the frame is drawn twice at a high resolution + /// under a temporal resolve, and the filter was a fifth of the opaque fragment stage + /// there with the whole disk (measured: 1.5 ms of an 11 ms frame). + public var xrFilterTaps: Int public init( enabled: Bool = true, nearRadiusTexels: Float = 2.0, farRadiusTexels: Float = 5.0, depthScale: Float = 1.0, - xrRadiusScale: Float = 1.35 + xrRadiusScale: Float = 1.35, + filterTaps: Int = 16, + xrFilterTaps: Int = 8 ) { self.enabled = enabled self.nearRadiusTexels = nearRadiusTexels self.farRadiusTexels = farRadiusTexels self.depthScale = depthScale self.xrRadiusScale = xrRadiusScale + self.filterTaps = filterTaps + self.xrFilterTaps = xrFilterTaps } } @@ -72,6 +84,7 @@ struct CSMUniforms { var shadowSoftnessFar: Float = 2.25 var shadowSoftnessDepthScale: Float = 1.0 var shadowSoftnessEnabled: Float = 1.0 + var shadowFilterTaps: Float = 16.0 } struct SpotShadowUniforms { @@ -273,6 +286,17 @@ struct ShadowSystem { var cascadeBlendStarts: [Float] = Array(repeating: 0, count: csmCascadeCount) var softnessSettings: ShadowSoftnessSettings = .init() var isActive: Bool = false + /// The cascades the sun's shadow is drawn in and read from, one to `csmCascadeCount`: + /// what `setRendering(.shadowCascades(_:))` asked for, else the platform's default: one + /// on the headset, where the scene is a room or a tabletop and the second cascade's + /// draw of every caster and the cross-fade's double taps cost a frame's budget for a + /// distance it never shows; every cascade elsewhere. `maxShadowCastingDistance` sets how + /// far the cascades reach, so one cascade stays sharp on a small scene. + nonisolated(unsafe) static var cascadesWanted: Int? + var cascadeCount: Int { + let wanted = Self.cascadesWanted ?? (renderInfo.isXRStereoMode ? 1 : csmCascadeCount) + return min(max(wanted, 1), csmCascadeCount) + } /// Legacy accessor used by callers that only need to know "is there a shadow?" var dirLightSpaceMatrix: simd_float4x4? { @@ -315,14 +339,20 @@ struct ShadowSystem { return max(cameraNear, cascadeBlendStart(cascadeIdx: cascadeIdx - 1, splits: splits, blendFraction: blendFraction)) } - /// Fills the 3 fixed GPU slots from a per-cascade array, padding unused slots - /// (beyond csmCascadeCount) with `fallback` so the shader's cascadeCount field - /// controls which slots are actually read. - private static func pack3(_ values: [T], fallback: T) -> (T, T, T) { + /// A light-space matrix that lands every point outside the shadow map's [0, 1] range + /// (at 10, 10, 10 with w 1), where the shader samples no map and takes the point as lit. + static let nowhere = simd_float4x4(columns: ( + simd_float4(0, 0, 0, 0), simd_float4(0, 0, 0, 0), simd_float4(0, 0, 0, 0), simd_float4(10, 10, 10, 1) + )) + + /// Fills the 3 fixed GPU slots from a per-cascade array, padding the slots past the + /// `count` cascades in use with `fallback`, so the shader's cascadeCount field controls + /// which slots are actually read. + private static func pack3(_ values: [T], count: Int, fallback: T) -> (T, T, T) { ( values[0], - csmCascadeCount > 1 ? values[1] : fallback, - csmCascadeCount > 2 ? values[2] : fallback + count > 1 && values.count > 1 ? values[1] : fallback, + count > 2 && values.count > 2 ? values[2] : fallback ) } @@ -331,18 +361,24 @@ struct ShadowSystem { /// left as identity/zero so the shader's cascadeCount field controls which are read. func makeUniforms() -> CSMUniforms { var u = CSMUniforms() - u.lightSpaceMatrices = Self.pack3(cascadeLightSpaceMatrices, fallback: matrix_identity_float4x4) - u.cascadeSplits = Self.pack3(cascadeSplitDistances, fallback: 0) - u.cascadeCount = Int32(csmCascadeCount) - u.cascadeWorldTexelSizes = Self.pack3(cascadeWorldTexelSizes, fallback: cascadeWorldTexelSizes[0]) - u.cascadeDepthSpans = Self.pack3(cascadeDepthSpans, fallback: cascadeDepthSpans[0]) - u.cascadeBlendStarts = Self.pack3(cascadeBlendStarts, fallback: cascadeBlendStarts[0]) + let count = cascadeCount + // A sun that casts no shadow (none active, or castsShadow off) draws no cascades, and + // the maps hold whatever was drawn last: its matrices put every point outside the + // maps, which the shader reads as lit, instead of the stale shadows. + let matrices = isActive ? cascadeLightSpaceMatrices : Array(repeating: Self.nowhere, count: cascadeLightSpaceMatrices.count) + u.lightSpaceMatrices = Self.pack3(matrices, count: count, fallback: matrix_identity_float4x4) + u.cascadeSplits = Self.pack3(cascadeSplitDistances, count: count, fallback: 0) + u.cascadeCount = Int32(count) + u.cascadeWorldTexelSizes = Self.pack3(cascadeWorldTexelSizes, count: count, fallback: cascadeWorldTexelSizes[0]) + u.cascadeDepthSpans = Self.pack3(cascadeDepthSpans, count: count, fallback: cascadeDepthSpans[0]) + u.cascadeBlendStarts = Self.pack3(cascadeBlendStarts, count: count, fallback: cascadeBlendStarts[0]) let softness = Self.sanitizedSoftnessSettings(softnessSettings) let xrScale = renderInfo.isXRStereoMode ? softness.xrRadiusScale : 1.0 u.shadowSoftnessNear = softness.nearRadiusTexels * xrScale u.shadowSoftnessFar = softness.farRadiusTexels * xrScale u.shadowSoftnessDepthScale = softness.depthScale u.shadowSoftnessEnabled = softness.enabled ? 1.0 : 0.0 + u.shadowFilterTaps = Float(renderInfo.isXRStereoMode ? softness.xrFilterTaps : softness.filterTaps) return u } @@ -350,7 +386,7 @@ struct ShadowSystem { softnessSettings = Self.sanitizedSoftnessSettings(settings) } - private static func sanitizedSoftnessSettings(_ settings: ShadowSoftnessSettings) -> ShadowSoftnessSettings { + static func sanitizedSoftnessSettings(_ settings: ShadowSoftnessSettings) -> ShadowSoftnessSettings { let nearRadius = simd_clamp(settings.nearRadiusTexels, 0.25, 8.0) let farRadius = simd_clamp(settings.farRadiusTexels, nearRadius, 12.0) let depthScale = simd_clamp(settings.depthScale, 0.0, 2.0) @@ -360,10 +396,20 @@ struct ShadowSystem { nearRadiusTexels: nearRadius, farRadiusTexels: farRadius, depthScale: depthScale, - xrRadiusScale: xrRadiusScale + xrRadiusScale: xrRadiusScale, + filterTaps: sanitizedFilterTaps(settings.filterTaps), + xrFilterTaps: sanitizedFilterTaps(settings.xrFilterTaps) ) } + /// 16, 8 or 4 taps: the disk, or every second or fourth of its points. + private static func sanitizedFilterTaps(_ count: Int) -> Int { + if count >= 16 { + return 16 + } + return count >= 8 ? 8 : 4 + } + /// Main per-frame update โ€” computes one tight ortho matrix per cascade. mutating func updateCascades() { // In XR stereo mode the shadow map is shared between both eyes. @@ -383,6 +429,9 @@ struct ShadowSystem { return } guard directionalLight.castsShadow else { return } + // A sun at no intensity lights nothing, and the shading multiplies its shadow by that + // intensity: its cascades would be drawn, two depth maps a frame, for nothing. + guard (scene.get(component: LightComponent.self, for: lightEntity)?.intensity ?? 0) != 0 else { return } // Rotate into visual-world space to match the root-rotation-aware corners/caster bounds // below -- the light entity's own transform is scene-local, same as every other entity. @@ -426,8 +475,9 @@ struct ShadowSystem { // Practical split scheme (blend of log and uniform, ฮป=0.5). let splits = computeCascadeSplits(cameraNear: near, cameraFar: cameraFar) - cascadeSplitDistances = splits - for i in 0 ..< csmCascadeCount { + // The arrays keep every slot; the ones past the cascades in use repeat the last split. + cascadeSplitDistances = splits + Array(repeating: splits.last ?? cameraFar, count: csmCascadeCount - splits.count) + for i in 0 ..< cascadeCount { cascadeBlendStarts[i] = Self.cascadeBlendStart( cascadeIdx: i, splits: splits, blendFraction: csmCascadeBlendFraction ) @@ -442,7 +492,7 @@ struct ShadowSystem { lightSpaceBounds(for: $0, lightView: lightView) } - for i in 0 ..< csmCascadeCount { + for i in 0 ..< cascadeCount { let cascadeFar = splits[i] let cascadeNear = Self.cascadeNearDistance( cascadeIdx: i, @@ -544,8 +594,8 @@ struct ShadowSystem { private func computeCascadeSplits(cameraNear: Float, cameraFar: Float) -> [Float] { let lambda: Float = 0.5 var splits: [Float] = [] - for i in 1 ... csmCascadeCount { - let fi = Float(i) / Float(csmCascadeCount) + for i in 1 ... cascadeCount { + let fi = Float(i) / Float(cascadeCount) let logSplit = cameraNear * pow(cameraFar / cameraNear, fi) let uniformSplit = cameraNear + (cameraFar - cameraNear) * fi splits.append(lambda * logSplit + (1.0 - lambda) * uniformSplit) @@ -681,12 +731,34 @@ struct ShadowSystem { } } +/// Sets the sun shadow's softness. Before the shadow system exists (before the renderer is +/// created, or between two sessions) the setting is kept and given to the system when the +/// renderer creates it, so a setting made with no space open is neither lost nor a crash. public func setShadowSoftness(_ settings: ShadowSoftnessSettings) { + guard shadowSystem != nil else { + shadowSoftnessBeforeSystem = settings + return + } shadowSystem.setSoftness(settings) } +/// The sun shadow's softness: the system's, or, before it exists, the setting kept for it, +/// as the system will hold it. public func getShadowSoftness() -> ShadowSoftnessSettings { - shadowSystem.softnessSettings + if let system = shadowSystem { + return system.softnessSettings + } + if let kept = shadowSoftnessBeforeSystem { + return ShadowSystem.sanitizedSoftnessSettings(kept) + } + return ShadowSystem().softnessSettings +} + +/// Gives a shadow system just created the softness set before it existed, if any. +func applyShadowSoftnessSetBeforeSystem() { + guard let settings = shadowSoftnessBeforeSystem else { return } + shadowSoftnessBeforeSystem = nil + shadowSystem?.setSoftness(settings) } /// Current value set via `setRendering(.maxShadowCastingDistance(_:))`. diff --git a/Sources/UntoldEngine/Systems/SmallObjectCulling.swift b/Sources/UntoldEngine/Systems/SmallObjectCulling.swift index c37c72b0e..2f8241454 100644 --- a/Sources/UntoldEngine/Systems/SmallObjectCulling.swift +++ b/Sources/UntoldEngine/Systems/SmallObjectCulling.swift @@ -57,10 +57,13 @@ struct SmallObjectCulling { guard let viewPort, viewPort.y > 0, projection.columns.3.w == 0, projection.columns.1.y > 0 else { return nil } + // Pixels of the screen the eye sees: under a rasterization rate map (XR foveation) the + // textures are smaller than that screen, and a size in their pixels would cull early. + let screenHeight = renderInfo.xrFoveation?.screenSize.y ?? viewPort.y return SmallObjectCulling( cameraPosition: SceneRootTransform.shared.effectiveCameraPosition(cameraComponent.localPosition), minimumPixels: pixels, - viewportHeight: viewPort.y, + viewportHeight: screenHeight, tanHalfFovY: 1 / projection.columns.1.y ) } diff --git a/Sources/UntoldEngine/UntoldEngineKernels/.metalsource.sha256 b/Sources/UntoldEngine/UntoldEngineKernels/.metalsource.sha256 new file mode 100644 index 000000000..ecbb08296 --- /dev/null +++ b/Sources/UntoldEngine/UntoldEngineKernels/.metalsource.sha256 @@ -0,0 +1 @@ +a9b77a22d8a409ac18447f39f1b1557430df52388bd6df244125014ec699e9cc diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-ios.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-ios.metallib index 9aff1ed38..6948599b3 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-ios.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-ios.metallib differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-iossim.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-iossim.metallib index 9a5833052..84cc1104f 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-iossim.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-iossim.metallib differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvos.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvos.metallib index 377f3686f..394ef80ef 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvos.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvos.metallib differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvossim.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvossim.metallib index a6a063407..7c8b6458a 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvossim.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvossim.metallib differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xros.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xros.metallib index d3a5c09a2..20fb0d81d 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xros.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xros.metallib differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xrossim.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xrossim.metallib index faa1b7335..41ed46c1b 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xrossim.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xrossim.metallib differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels.metal b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels.metal index 23a16dff4..1d4773887 100644 --- a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels.metal +++ b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels.metal @@ -50,6 +50,7 @@ using namespace metal; #include "../Shaders/spatialDebugShader.metal" #include "../Shaders/FXAAShader.metal" #include "../Shaders/SMAAShader.metal" +#include "../Shaders/TAAShader.metal" // Gaussian kernels #include "../Shaders/BitonicSort.metal" #include "../Shaders/DeviceRadixSort.metal" diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels.metallib index 1d5726d7b..fbe5c0029 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels.metallib differ diff --git a/Sources/UntoldEngine/Utils/EngineSettingsAPI.swift b/Sources/UntoldEngine/Utils/EngineSettingsAPI.swift index dea53a774..1401ddee9 100644 --- a/Sources/UntoldEngine/Utils/EngineSettingsAPI.swift +++ b/Sources/UntoldEngine/Utils/EngineSettingsAPI.swift @@ -71,6 +71,13 @@ public enum RenderingProperty: Sendable { /// existing texel budget on it instead, sharpening shadow edges without needing a larger /// shadow map. See `getMaxShadowCastingDistance()` to read the current value. case maxShadowCastingDistance(Float) + /// The cascades the sun's shadow is drawn in and read from, 1 or 2. Each cascade draws + /// every caster once more into its own 2048ยฒ depth map, and the band where two meet + /// samples both. The headset's default is one: a room or a tabletop never shows the + /// distance the second covers, and with `maxShadowCastingDistance` set to the scene's + /// size one cascade is sharp on it. Elsewhere the default is two. See + /// `getShadowCascades()`. + case shadowCascades(Int) /// Objects whose bounds are under this many pixels tall on screen are not drawn and /// cast no shadow. A scene with tens of thousands of small parts shows most of them /// at a pixel or less from any distance, and each would still cost a draw. The size @@ -86,6 +93,99 @@ public enum RenderingProperty: Sendable { /// frustum keeps. In scene units; the default is 0.5. See /// `getOcclusionCullingMaxCameraStep()` to read the current value. case occlusionCullingMaxCameraStep(Float) + /// The fewest objects in the culling set for which the occlusion test runs and the depth + /// pyramid it reads is built and its depth copied, every frame. An entity with its + /// children is one object: the parts of a model loaded from a file hide nothing but + /// themselves. 2 by default: one object hides nothing but itself. A scene with splats + /// keeps the pyramid regardless. + case occlusionCullingMinimumEntities(Int) + /// Whether the XR frame is drawn through the rasterization rate maps the visionOS + /// compositor provides when the layer is configured with foveation enabled + /// (`LayerRenderer.Configuration.isFoveationEnabled`): dense where the eyes look, coarse + /// in the periphery, and the only configuration in which the compositor accepts a render + /// quality above the platform default. Enabled by default; a layer configured without + /// foveation provides no maps and draws uniformly whatever this says. Disabling it in a + /// foveated layer is a diagnostic: the frame is then drawn uniformly into textures the + /// compositor still unwarps, so the picture is distorted. See + /// `getFoveatedRenderingEnabled()` to read the current value. + case foveatedRendering(RenderingToggle) + /// Whether the XR loop waits for the compositor's optimal input time before it queries the + /// head pose and encodes the frame. Waiting gives the freshest head pose; not waiting + /// starts the GPU about the wait's length earlier, so a frame whose GPU work would end + /// after the rendering deadline may make it, with the head pose predicted from earlier for + /// the compositor's reprojection to cover. `.automatic`, the default, waits while the + /// frames meet the deadline, stops when a second had late frames, and resumes once the + /// measured slack would cover the wait again. See `getOptimalInputWaitMode()`. + case optimalInputWait(XRInputWaitMode) + /// How the XR loop paces its frames against the display's refreshes: every refresh; every + /// other refresh while the frames run late (the default), so a frame whose GPU work does + /// not fit one refresh is shown two refreshes apart, evenly, and posed for that instant, + /// instead of landing a refresh late now and then, which hops; or every other refresh + /// always. The compositor reprojects the head between the frames. See `getFramePacing()`. + case framePacing(XRFramePacing) + /// Whether the pre-composite pass is skipped on a frame it would only copy the lit scene + /// (XR passthrough with no environment, splat, ambient-occlusion or gizmo layer to join; + /// the lit debug view; no render extension). Enabled by default; disabling it runs the + /// pass on every frame, as a diagnostic. + case preCompositeBypass(RenderingToggle) + /// Whether the deferred light pass runs as a tile stage that lights each pixel once per + /// colour it holds, instead of a quad that lights every multisample. Enabled by default + /// where tile shading exists; disabling it runs the quad, as a diagnostic. + case tileLightStage(RenderingToggle) + /// The samples per pixel of the multisampled anti-aliasing modes: 2, 4 (the default) or 8, + /// or fewer when the device has no such count. Takes effect on the next frame. + case multisampleCount(Int) + /// Whether opaque draws that never discard a fragment (an opaque material, no fade + /// running) use the model pipeline compiled without discards, which lets the GPU resolve + /// visibility before shading: a fragment another surface covers is never shaded. Enabled + /// by default; disabling it draws everything through the general pipeline, as a diagnostic. + case opaqueHiddenSurfaceRemoval(RenderingToggle) + /// GPU time per pass, measured by the GPU at each pass's stage boundaries and averaged over + /// the recent frames: `GPUPassTiming.shared.averages()`. Off by default. + case gpuPassTiming(RenderingToggle) + /// Whether the geometry evaluates the image-based light and the sun's cascade shadow once + /// per fragment and carries them in the G-buffer for the light pass to add, instead of the + /// light pass computing them once per multisample. On where the platform's default path is + /// hybrid or forward (`RenderInfo.defaultRenderPath`: visionOS), off elsewhere; the same + /// picture, less lighting work under MSAA. + case perFragmentIndirectLighting(RenderingToggle) + /// Whether the geometry lights itself whole once per fragment โ€” the direct light of the sun, + /// the point, spot and area lights with their shadows, over the indirect light โ€” and writes + /// the lit colour, so the light quad, which lights once per multisample, stands down: the + /// forward way to shade. On where the platform's default path is forward (visionOS), off + /// elsewhere, and needs `perFragmentIndirectLighting`. The same picture but at the + /// multisampled edges, where the quad lit each sample and the geometry lights each pixel once. + case perFragmentLighting(RenderingToggle) + /// Where the opaque geometry is shaded (see `RenderPath`): sets `perFragmentIndirectLighting` + /// and `perFragmentLighting` together, so a scene says "forward" or "deferred" instead of + /// composing the two. Forward by default on visionOS, deferred on macOS and iOS + /// (`RenderInfo.defaultRenderPath`). `getRenderPath()` reads the path the two make. + case renderPath(RenderPath) +} + +/// Where the opaque geometry is shaded. +public enum RenderPath: String, Sendable, Codable, Equatable, CaseIterable { + /// The geometry writes its surface to the G-buffer and the light pass lights it, once per + /// multisample from a quad or once per pixel through the tile light stage. The path for a + /// scene of many lights, and for the effects that read the G-buffer: the ambient occlusion, + /// the debug views. The default on macOS and iOS. + case deferred + /// Deferred, with the image-based light and the sun's shadow evaluated by the geometry once + /// per fragment and carried in the G-buffer; the light pass adds the direct lights. + case hybrid + /// The geometry lights itself whole once per fragment and writes the lit colour; the light + /// quad stands down and the G-buffer shrinks to that colour. One attachment, the samples in + /// tile memory: the headset's path, and visionOS's default. The effects that read the + /// G-buffer stand down with it; an object moving on its own has its motion written for the + /// temporal resolve, which the other paths leave to the resolve's neighbourhood clip. + case forward +} + +/// The render path the two per-fragment settings make: forward with both on, hybrid with the +/// indirect light alone, deferred otherwise (the direct light per fragment needs the indirect). +public func getRenderPath() -> RenderPath { + guard renderInfo.perFragmentIndirectLighting else { return .deferred } + return renderInfo.perFragmentLighting ? .forward : .hybrid } public enum RenderingToggle: Sendable { @@ -93,6 +193,17 @@ public enum RenderingToggle: Sendable { case disabled } +/// When the XR loop waits for the compositor's optimal input time before encoding the frame. +/// See `RenderingProperty.optimalInputWait`. +public enum XRInputWaitMode: Sendable, Equatable { + /// Every frame: the freshest head pose, the latest GPU start. + case always + /// Never: the frame is encoded as soon as the update is done. + case never + /// By the measured deadline: waiting while the frames make it, not while they do not. + case automatic +} + public enum RenderExtensionProperty: Sendable { case register(any RenderExtension) case unregister(String) @@ -144,13 +255,93 @@ public func setRendering(_ property: RenderingProperty) { applyRenderExtensionProperty(property) case let .maxShadowCastingDistance(distance): RenderPasses.maxShadowCastingDistance = max(distance, 0.001) + case let .shadowCascades(count): + ShadowSystem.cascadesWanted = count case let .smallObjectCulling(pixels): SmallObjectCulling.minimumPixels = pixels.isFinite ? max(pixels, 0) : 0 + case let .occlusionCullingMinimumEntities(entities): + HZBOcclusionCulling.minimumEntities = max(0, entities) case let .occlusionCullingMaxCameraStep(step): HZBOcclusionCulling.maxCameraStep = step.isFinite ? max(step, 0) : 0 + case .foveatedRendering(.enabled): + xrFoveatedRenderingEnabled = true + case .foveatedRendering(.disabled): + xrFoveatedRenderingEnabled = false + case let .optimalInputWait(mode): + xrOptimalInputWaitMode = mode + case let .framePacing(pacing): + xrFramePacing = pacing + case .preCompositeBypass(.enabled): + renderInfo.preCompositeBypassEnabled = true + case .preCompositeBypass(.disabled): + renderInfo.preCompositeBypassEnabled = false + case .tileLightStage(.enabled): + renderInfo.tileLightStageEnabled = true + case .tileLightStage(.disabled): + renderInfo.tileLightStageEnabled = false + case let .multisampleCount(count): + renderInfo.preferredMultisampleCount = [2, 4, 8].contains(count) ? count : 4 + case .opaqueHiddenSurfaceRemoval(.enabled): + renderInfo.opaqueHiddenSurfaceRemovalEnabled = true + case .opaqueHiddenSurfaceRemoval(.disabled): + renderInfo.opaqueHiddenSurfaceRemovalEnabled = false + case .gpuPassTiming(.enabled): + GPUPassTiming.shared.setEnabled(true) + case .gpuPassTiming(.disabled): + GPUPassTiming.shared.setEnabled(false) + case .perFragmentIndirectLighting(.enabled): + renderInfo.perFragmentIndirectLighting = true + renderInfo.renderPathChosen = true + case .perFragmentIndirectLighting(.disabled): + renderInfo.perFragmentIndirectLighting = false + renderInfo.renderPathChosen = true + case .perFragmentLighting(.enabled): + renderInfo.perFragmentLighting = true + renderInfo.renderPathChosen = true + case .perFragmentLighting(.disabled): + renderInfo.perFragmentLighting = false + renderInfo.renderPathChosen = true + case let .renderPath(path): + renderInfo.perFragmentIndirectLighting = path != .deferred + renderInfo.perFragmentLighting = path == .forward + renderInfo.renderPathChosen = true } } +/// The XR loop's render path when the app chose none: forward, the headset's path, wherever +/// the frame is rendered โ€” on the headset itself, where it is the platform's default already, +/// or on a Mac rendering a remote immersive space, which starts deferred like any Mac app. +/// `UntoldEngineXR` calls it as it initialises; a path the app set stays, and the adoption is +/// not counted as the app's choice. +public func adoptXRRenderPathUnlessChosen() { + guard !renderInfo.renderPathChosen else { return } + renderInfo.perFragmentIndirectLighting = true + renderInfo.perFragmentLighting = true +} + +/// Whether the XR frame draws through the compositor's rasterization rate maps when the layer +/// provides them. See `RenderingProperty.foveatedRendering`. +public func getFoveatedRenderingEnabled() -> Bool { + xrFoveatedRenderingEnabled +} + +/// When the XR loop waits for the compositor's optimal input time before it encodes the +/// frame. See `RenderingProperty.optimalInputWait`. +/// The cascades the sun's shadow is drawn in and read from. +public func getShadowCascades() -> Int { + shadowSystem?.cascadeCount ?? min(max(ShadowSystem.cascadesWanted ?? csmCascadeCount, 1), csmCascadeCount) +} + +public func getOptimalInputWaitMode() -> XRInputWaitMode { + xrOptimalInputWaitMode +} + +/// How the XR loop paces its frames against the display's refreshes. See +/// `RenderingProperty.framePacing`. +public func getFramePacing() -> XRFramePacing { + xrFramePacing +} + public enum RenderResourceQuery { case texture(String) } @@ -290,6 +481,8 @@ public enum PostFXProperty: Sendable { case colorLUT(ColorLUTProperty) case colorGradeLUT(ColorGradeLUTProperty) case tonemapOperator(TonemapOperator) + /// The primaries the look pass shows the scene with (see `ColorPrimaries`). + case colorPrimaries(ColorPrimaries) case colorCorrection(ColorCorrectionProperty) case bloomThreshold(BloomThresholdProperty) case bloomComposite(BloomCompositeProperty) @@ -383,6 +576,8 @@ public func setPostFX(_ property: PostFXProperty) { applyColorGradeLUTProperty(property) case let .tonemapOperator(op): TonemapParams.shared.operator = op + case let .colorPrimaries(primaries): + TonemapParams.shared.primaries = primaries case let .colorCorrection(property): applyColorCorrectionProperty(property) case let .bloomThreshold(property): diff --git a/Sources/UntoldEngine/Utils/FuncUtils.swift b/Sources/UntoldEngine/Utils/FuncUtils.swift index dbfe86ef9..33107a41e 100644 --- a/Sources/UntoldEngine/Utils/FuncUtils.swift +++ b/Sources/UntoldEngine/Utils/FuncUtils.swift @@ -1590,7 +1590,7 @@ public func updateMaterialHeightRemapMax(entityId: EntityID, heightRemapMax: Flo /// Enables/disables Parallax Occlusion Mapping for a material that has a height texture, /// without discarding the texture assignment itself. public func getMaterialHeightEnabled(entityId: EntityID, meshIndex: Int = 0, submeshIndex: Int = 0) -> Bool { - getMaterial(entityId: entityId, meshIndex: meshIndex, submeshIndex: submeshIndex)?.heightEnabled ?? true + getMaterial(entityId: entityId, meshIndex: meshIndex, submeshIndex: submeshIndex)?.heightEnabled ?? false } public func updateMaterialHeightEnabled(entityId: EntityID, heightEnabled: Bool, meshIndex: Int = 0, submeshIndex: Int = 0) { diff --git a/Sources/UntoldEngine/Utils/Globals.swift b/Sources/UntoldEngine/Utils/Globals.swift index 1a92d1f3a..d2652de44 100644 --- a/Sources/UntoldEngine/Utils/Globals.swift +++ b/Sources/UntoldEngine/Utils/Globals.swift @@ -55,6 +55,7 @@ private final class CoreRuntimeGlobals: @unchecked Sendable { var reduceScanScatterCompactedPipeline = ComputePipeline() var hzbBuildPyramidPipeline = ComputePipeline() var hzbOcclusionCullingPipeline = ComputePipeline() + var xrProbeSHPipeline = ComputePipeline() var gaussianResetVisibleCountPipeline = ComputePipeline() var gaussianFinalizeVisibleSetPipeline = ComputePipeline() var gaussianFrustumCullPipeline = ComputePipeline() @@ -304,6 +305,13 @@ public var timeSinceLastUpdate: Float! { set { RuntimeGlobalsStore.shared.timeSinceLastUpdate = newValue } } +/// The frame being prepared: its predicted presentation time, the delta from the frame +/// before and whether the display skipped a frame between them. `.zero` before the first. +public internal(set) var currentFrameTime: FrameTime { + get { RuntimeGlobalsStore.shared.currentFrameTime } + set { RuntimeGlobalsStore.shared.currentFrameTime = newValue } +} + var firstUpdateCall: Bool { get { RuntimeGlobalsStore.shared.firstUpdateCall } set { RuntimeGlobalsStore.shared.firstUpdateCall = newValue } @@ -536,6 +544,27 @@ var hzbOcclusionCullingPipeline: ComputePipeline { } } +var xrProbeSHPipeline: ComputePipeline { + get { + let state = CoreRuntimeGlobals.shared + state.lock.lock() + defer { state.lock.unlock() } + return state.xrProbeSHPipeline + } + set { + let state = CoreRuntimeGlobals.shared + state.lock.lock() + state.xrProbeSHPipeline = newValue + state.lock.unlock() + } + _modify { + let state = CoreRuntimeGlobals.shared + state.lock.lock() + defer { state.lock.unlock() } + yield &state.xrProbeSHPipeline + } +} + var gaussianDecodePipeline: ComputePipeline { get { let state = CoreRuntimeGlobals.shared @@ -1074,6 +1103,7 @@ private final class RuntimeGlobalsStore: @unchecked Sendable { private var globalEntityCounterValue: UInt32 = 0 private var timeSinceLastUpdatePreviousTimeValue: TimeInterval? private var timeSinceLastUpdateValue: Float? + private var currentFrameTimeValue: FrameTime = .zero private var firstUpdateCallValue: Bool = false private var frameCountValue: Int = 0 private var timePassedSinceLastFrameValue: Float = 0.0 @@ -1103,6 +1133,10 @@ private final class RuntimeGlobalsStore: @unchecked Sendable { private var activeEntityValue: EntityID = .invalid private var enableEngineMetricsValue: Bool = false private var bypassPostProcessingValue: Bool = false + private var xrFoveatedRenderingEnabledValue: Bool = true + private var xrOptimalInputWaitModeValue: XRInputWaitMode = .automatic + private var xrFramePacingValue: XRFramePacing = .halfRateWhenLate + private var shadowSoftnessBeforeSystemValue: ShadowSoftnessSettings? private var antiAliasingModeValue: AntiAliasingMode = .fxaa private var renderDebugViewModeValue: RenderDebugViewMode = .lit private var pomQualitySettingsValue: POMQualitySettings = .platformDefault @@ -1172,6 +1206,20 @@ private final class RuntimeGlobalsStore: @unchecked Sendable { } } + var currentFrameTime: FrameTime { + get { + lock.lock() + let value = currentFrameTimeValue + lock.unlock() + return value + } + set { + lock.lock() + currentFrameTimeValue = newValue + lock.unlock() + } + } + var firstUpdateCall: Bool { get { lock.lock() @@ -1594,6 +1642,62 @@ private final class RuntimeGlobalsStore: @unchecked Sendable { } } + var xrFoveatedRenderingEnabled: Bool { + get { + lock.lock() + let value = xrFoveatedRenderingEnabledValue + lock.unlock() + return value + } + set { + lock.lock() + xrFoveatedRenderingEnabledValue = newValue + lock.unlock() + } + } + + var xrOptimalInputWaitMode: XRInputWaitMode { + get { + lock.lock() + let value = xrOptimalInputWaitModeValue + lock.unlock() + return value + } + set { + lock.lock() + xrOptimalInputWaitModeValue = newValue + lock.unlock() + } + } + + var xrFramePacing: XRFramePacing { + get { + lock.lock() + let value = xrFramePacingValue + lock.unlock() + return value + } + set { + lock.lock() + xrFramePacingValue = newValue + lock.unlock() + } + } + + var shadowSoftnessBeforeSystem: ShadowSoftnessSettings? { + get { + lock.lock() + let value = shadowSoftnessBeforeSystemValue + lock.unlock() + return value + } + set { + lock.lock() + shadowSoftnessBeforeSystemValue = newValue + lock.unlock() + } + } + var antiAliasingMode: AntiAliasingMode { get { lock.lock() @@ -2093,16 +2197,54 @@ public var bypassPostProcessing: Bool { set { RuntimeGlobalsStore.shared.bypassPostProcessing = newValue } } +/// Whether the XR frame draws through the rasterization rate maps the compositor provides +/// (visionOS foveation), the default. See `RenderingProperty.foveatedRendering`. +public var xrFoveatedRenderingEnabled: Bool { + get { RuntimeGlobalsStore.shared.xrFoveatedRenderingEnabled } + set { RuntimeGlobalsStore.shared.xrFoveatedRenderingEnabled = newValue } +} + +/// When the XR loop waits for the compositor's optimal input time before it queries the head +/// pose and encodes the frame: always, never, or by the measured deadline (the default). See +/// `RenderingProperty.optimalInputWait`. +public var xrOptimalInputWaitMode: XRInputWaitMode { + get { RuntimeGlobalsStore.shared.xrOptimalInputWaitMode } + set { RuntimeGlobalsStore.shared.xrOptimalInputWaitMode = newValue } +} + +/// How the XR loop paces its frames against the display's refreshes: every one, every other +/// one while the frames run late (the default), or every other one always. See +/// `RenderingProperty.framePacing`. +public var xrFramePacing: XRFramePacing { + get { RuntimeGlobalsStore.shared.xrFramePacing } + set { RuntimeGlobalsStore.shared.xrFramePacing = newValue } +} + +/// The sun shadow's softness set while no shadow system existed, for the system the renderer +/// creates next. See `setShadowSoftness`. +var shadowSoftnessBeforeSystem: ShadowSoftnessSettings? { + get { RuntimeGlobalsStore.shared.shadowSoftnessBeforeSystem } + set { RuntimeGlobalsStore.shared.shadowSoftnessBeforeSystem = newValue } +} + /// Selects the active anti-aliasing pass inserted after the look pass. public enum AntiAliasingMode: Sendable { case none case fxaa case smaa case msaa + /// Temporal: the projection jittered by a sub-pixel amount each frame and the frame + /// blended with its reprojected history after the look pass (TemporalAntiAliasing.swift). + /// Averages the sub-pixel detail a single sample flickers on. An object moving on its own + /// has its motion written by the forward path's opaque pass; on the deferred and hybrid + /// paths it relies on the neighbourhood clip (see `RenderPath`). + case taa + /// 4ร— MSAA on the G-buffer and the temporal resolve on top. + case msaaTaa var usesPostLookPass: Bool { switch self { - case .fxaa, .smaa: + case .fxaa, .smaa, .taa, .msaaTaa: return true case .none, .msaa: return false @@ -2324,11 +2466,37 @@ public final class ColorGradeLUTParams: @unchecked Sendable { public enum TonemapOperator: String, Sendable, Codable, Equatable { case aces case agx + /// No tone map: the scene's linear colour as it is. For a mixed-immersion space, where + /// the passthrough sets the display's response and the compositor applies it to the + /// scene as well; a filmic curve there compresses the object's whites and lifts its + /// blacks against the room (measured on a dartboard: whites at 87 of 255 against the + /// room's 113 for the same object, blacks at 15 against 9). + case none var shaderValue: Int32 { switch self { case .aces: Int32(tonemapOperatorACES.rawValue) case .agx: Int32(tonemapOperatorAgX.rawValue) + case .none: Int32(tonemapOperatorNone.rawValue) + } + } +} + +/// The primaries the look pass shows the scene with. +public enum ColorPrimaries: String, Sendable, Codable, Equatable { + /// The scene's sRGB colour as it is. + case sRGB + /// The colour taken as Display P3 coordinates: every saturated colour widened to the + /// panel's primaries, as RealityKit shows a mixed-immersion scene (its swatches measure as + /// the P3โ†’sRGB matrix of their sRGB values, to a count), so a scene shown next to one + /// matches it. A channel the matrix takes negative is outside sRGB and clips to black, as + /// it does there. Set with `setPostFX(.colorPrimaries(.displayP3))`. + case displayP3 + + var shaderValue: Int32 { + switch self { + case .sRGB: Int32(colorPrimariesSRGB.rawValue) + case .displayP3: Int32(colorPrimariesDisplayP3.rawValue) } } } @@ -2341,6 +2509,7 @@ public final class TonemapParams: @unchecked Sendable { // setPostFX(.tonemapOperator(.agx)) for scenes that want to match it. private let lock = NSLock() private var _operator: TonemapOperator = .aces + private var _primaries: ColorPrimaries = .sRGB public var `operator`: TonemapOperator { get { @@ -2355,9 +2524,24 @@ public final class TonemapParams: @unchecked Sendable { } } + /// The primaries the look pass shows the scene with; sRGB unless set. + public var primaries: ColorPrimaries { + get { + lock.lock() + defer { lock.unlock() } + return _primaries + } + set { + lock.lock() + _primaries = newValue + lock.unlock() + } + } + public func resetToDefaults() { lock.lock() _operator = .aces + _primaries = .sRGB lock.unlock() } } diff --git a/Sources/UntoldEngine/Utils/TripleBuffer.swift b/Sources/UntoldEngine/Utils/TripleBuffer.swift index e54169999..30d4308ab 100644 --- a/Sources/UntoldEngine/Utils/TripleBuffer.swift +++ b/Sources/UntoldEngine/Utils/TripleBuffer.swift @@ -59,16 +59,16 @@ public final class TripleBuffer { frame % inFlight } - @inline(__always) private func readIndex(_ frame: Int) -> Int { - (frame + inFlight - 1) % inFlight - } - @inline(__always) public func bufferForWrite(frame: Int) -> MTLBuffer { buffers[writeIndex(frame)] } + /// The slot the frame's GPU work reads: the one the frame wrote. The ring exists so that + /// the frames in flight each have a slot of their own, not to stagger a frame's reads a + /// slot behind its writes (which handed a frame the frame before's data under this + /// frame's count). @inline(__always) public func bufferForRead(frame: Int) -> MTLBuffer { - frame == 0 ? buffers[writeIndex(frame)] : buffers[readIndex(frame)] + buffers[writeIndex(frame)] } } diff --git a/Sources/UntoldEngineAR/UntoldEngineAR.swift b/Sources/UntoldEngineAR/UntoldEngineAR.swift index 3fad071bf..d73f97873 100644 --- a/Sources/UntoldEngineAR/UntoldEngineAR.swift +++ b/Sources/UntoldEngineAR/UntoldEngineAR.swift @@ -85,6 +85,14 @@ renderer?.setupCallbacks(gameUpdate: gameUpdate, handleInput: handleInput) } + /// Installs the game's per-frame callbacks with the frame's time (see `FrameTime`). + public func setupCallbacks( + frameUpdate: @escaping (_ frame: FrameTime) -> Void, + handleInput: @escaping () -> Void + ) { + renderer?.setupCallbacks(frameUpdate: frameUpdate, handleInput: handleInput) + } + public func drawRectResized(size: CGSize) { renderInfo.viewPort = simd_float2(Float(size.width), Float(size.height)) viewportSizeDidChange = true @@ -131,7 +139,10 @@ updateARStates(currentFrame: currentFrame) EngineProfiler.shared.endScope(.renderPrep) - renderer.updateXR() + // No presentation time is predicted for an MTKView: the frame is posed one of the + // view's periods ahead of now. + let period = 1.0 / Double(max(view.preferredFramesPerSecond, 1)) + renderer.updateXR(presentationTime: CACurrentMediaTime() + period) if let renderPassDescriptor = view.currentRenderPassDescriptor { renderInfo.renderPassDescriptor = renderPassDescriptor diff --git a/Sources/UntoldEngineXR/UntoldEngineXR.swift b/Sources/UntoldEngineXR/UntoldEngineXR.swift index b32d7c122..0aa7d18de 100644 --- a/Sources/UntoldEngineXR/UntoldEngineXR.swift +++ b/Sources/UntoldEngineXR/UntoldEngineXR.swift @@ -69,6 +69,179 @@ // Reuse render pass descriptors to avoid allocation churn (2 eyes ร— 90 FPS = 180 allocs/sec) private let passDescriptorLeft = MTLRenderPassDescriptor() private let passDescriptorRight = MTLRenderPassDescriptor() + + // Foveation: the parameter data of each eye's rasterization rate map, copied every frame + // (the maps follow the gaze) into a ring one slot deeper than the frames in flight โ€” + // the depth pyramid built on a frame carries its last eye's data into the culls of the + // frame after (HZBPyramidFrame.rateMapData), which may still run when this frame's + // slot comes round again with only the usual in-flight depth. + private var rateMapParameterBuffers: [[MTLBuffer?]] = Array(repeating: [nil, nil], count: maxInFlightCommandBuffers + 1) + private var rateMapFrameCounter = 0 + private var loggedFoveationState: Bool? + private var loggedCaptureTargetSkip = false + private var loggedDrawableCount = 0 + /// Whether the compositor's frame timing budget has been printed (once per space). + private var loggedFrameTimingBudget = false + /// Whether the compositor clock's base has been checked against the media time (once). + private var loggedClockBases = false + /// Frames whose drawable's presentation time differed from the one predicted at the + /// update: the game posed those frames for a slightly different instant than the head. + public private(set) var presentationTimeRevisions = 0 + private var lastPresentationRevisionLogTime: CFTimeInterval = 0 + /// Frames whose GPU work finished after the compositor's rendering deadline. The deadline + /// sits well before the presentation (15 ms on the headset), so a frame a few + /// milliseconds past it is still shown on time; one past that margin has the display + /// show the previous frame again and this one a refresh late, posed for an instant + /// already past โ€” a hop in a moving object that no counter of the loop's pacing sees. + public var deadlineMisses: Int { deadlineLock.withLock { deadlineMissCount } } + /// The longest a frame's GPU work finished after its deadline, in milliseconds. + public var worstDeadlineOverrunMs: Double { deadlineLock.withLock { worstOverrunMs } } + private let deadlineLock = NSLock() + private var deadlineMissCount = 0 + private var worstOverrunMs = 0.0 + private var lastDeadlineLogTime: CFTimeInterval = 0 + /// The last second's frames: how many, how many late, their worst and summed overruns, + /// the least slack any had before the deadline, and the longest wait skipped. + private var secondFrames = 0 + private var secondLate = 0 + private var secondWorstMs = 0.0 + private var secondOverrunSumMs = 0.0 + private var secondLeastSlackMs = Double.infinity + private var secondSkippedWaitMaxMs = 0.0 + private var secondGPUSumMs = 0.0 + /// The frames' GPU time after the frame before finished: the span of a frame's command + /// buffer starts when the GPU takes it, which with frames queued is before the frame + /// before is done, so the span overstates the frame's own work. + private var secondOwnGPUSumMs = 0.0 + private var lastGPUEndTime: CFTimeInterval = 0 + private var secondPacedWaitSumMs = 0.0 + private var secondPacedWaits = 0 + /// The last second's frames past the deadline by more than its usable margin before the + /// presentation (shown a refresh late), and the refreshes the display skipped beyond + /// the pacing's cadence, from the spacing of the predicted presentations. + private var secondMissed = 0 + private var secondSkipped = 0 + /// The automatic mode's current choice: waiting for the optimal input time, or not. + private var automaticWaits = true + /// The half-rate pacing: its policy, whether this frame is paced, the last second's + /// longest GPU span, and its anchors: the last rendered frame's presentation instant, + /// the refresh grid the frames are taken on, and the instant the last paced frame was + /// taken at. + private var halfRate = XRHalfRatePolicy() + private var pacingActiveNow = false + private var secondMaxGPUMs = 0.0 + private var lastRenderedPresentation: LayerRenderer.Clock.Instant? + /// When the compositor handed the loop its last frame, and the pacing's own grid: the + /// instant it started from, on which the paced queries fall every cadence. + private var lastFrameHandedAt: LayerRenderer.Clock.Instant? + private var pacingOrigin: LayerRenderer.Clock.Instant? + /// The last frame's predicted presentation, and the last second's spacing between + /// consecutive ones: its sum, shortest and longest, the evenness of the frames shown. + private var lastPredictedPresentation: LayerRenderer.Clock.Instant? + private var secondSpacingSumMs = 0.0 + private var secondSpacingMinMs = Double.infinity + private var secondSpacingMaxMs = 0.0 + private var secondSpacings = 0 + /// The last second's loop: how long the query for a frame blocked, how long the loop + /// took from one handed frame to the next query, how far after the query the compositor + /// predicted the presentation, and how far before it the rendering deadline sat. + private var secondQueryBlockSumMs = 0.0 + private var secondLoopSumMs = 0.0 + private var secondPredictedAheadSumMs = 0.0 + private var secondDeadlineMarginSumMs = 0.0 + private var secondTimedFrames = 0 + /// Whether the loop is rendering every other refresh at the moment. + public var halfRatePacingActive: Bool { deadlineLock.withLock { pacingActiveNow } } + /// Whether the last frame waited for the compositor's optimal input time: the setting, + /// or in the automatic mode what the measured deadline allowed. + public var optimalInputWaitActive: Bool { deadlineLock.withLock { lastFrameWaited } } + private var lastFrameWaited = true + + /// Whether this frame waits for the optimal input time, by the setting and, in the + /// automatic mode, by what the last seconds' deadlines allowed. `skippedWaitMs` is how + /// long the wait would be, kept for the automatic mode to know what resuming costs. + private func waitsForOptimalInput(skippedWaitMs: Double) -> Bool { + let waits: Bool + deadlineLock.lock() + switch xrOptimalInputWaitMode { + case .always: waits = true + case .never: waits = false + case .automatic: waits = automaticWaits + } + if !waits { + secondSkippedWaitMaxMs = max(secondSkippedWaitMaxMs, skippedWaitMs) + } + lastFrameWaited = waits + deadlineLock.unlock() + return waits + } + + /// The half-rate pacing, before the next compositor frame is taken. With the pacing + /// active, by the setting or by the policy when the frames run late, the queries fall + /// on the pacing's own grid: the instant it started from, every `cadenceRefreshes` + /// refreshes, exactly. The compositor hands out a frame on demand and predicts its + /// presentation a fixed time after the query, so queries spaced an exact number of + /// refreshes apart are shown an exact number of refreshes apart. (A wait aimed at the + /// previous frame's presentation instant made every cadence three refreshes, the + /// compositor predicting three ahead; a wait of a cadence less a lead spaced the frames + /// unevenly.) A query up to a quarter refresh late still takes its slot. Nothing waits + /// for the GPU: the next frame's work queues behind the previous one's, so the GPU + /// stays busy and keeps its clock. Every frame taken is submitted: a frame taken and + /// let go is never released by the compositor, and three of them abort the app. + private func paceForHalfRate() { + let active: Bool + let cadence: Int + deadlineLock.lock() + switch xrFramePacing { + case .fullRate: active = false + case .halfRate: active = true + case .halfRateWhenLate: active = halfRate.active + } + cadence = halfRate.cadenceRefreshes + pacingActiveNow = active + deadlineLock.unlock() + guard active, let handed = lastFrameHandedAt else { + pacingOrigin = nil + return + } + let period = currentFrameTime.period + guard period > 0 else { return } + let clock = LayerRenderer.Clock() + guard let origin = pacingOrigin else { + pacingOrigin = handed + return + } + let step = Double(cadence) * period + let elapsed = Self.seconds(origin.duration(to: clock.now)) + let slots = max(1, ((elapsed - 0.25 * period) / step).rounded(.up)) + let target = origin.advanced(by: .seconds(slots * step)) + let waitMs = max(0, Self.milliseconds(clock.now.duration(to: target))) + if waitMs > 0 { + clock.wait(until: target, tolerance: .zero) + } + deadlineLock.lock() + secondPacedWaitSumMs += waitMs + secondPacedWaits += 1 + deadlineLock.unlock() + } + + /// How far past the compositor's predicted presentation the frame is posed for: the + /// refreshes the paced frames were measured to land after it, from how far past their + /// deadline they finish; nothing without the pacing. + private func presentationLeadSeconds() -> TimeInterval { + deadlineLock.withLock { pacingActiveNow ? Double(halfRate.landingRefreshes) * currentFrameTime.period : 0 } + } + /// Whether the layer's depth format and the drawable's depth range have been printed. + private var loggedDepthSetup = false + /// The layer's depth format and the drawable's depth range (far, near), once a drawable + /// has been seen: what the depth test can tell apart at a distance. + public private(set) var depthSetupDescription = "" + + private static func milliseconds(_ duration: Duration) -> Double { + let parts = duration.components + return Double(parts.seconds) * 1000.0 + Double(parts.attoseconds) / 1e15 + } + private let spatialGestureRecognizer = XRSpatialGestureRecognizer() private let xrEnvironmentLightingSystem = XREnvironmentLightingSystem() private var runtimeLightingModeObserverId: UUID? @@ -113,6 +286,10 @@ @MainActor public func initUntoldXR(device: MTLDevice, commandQueue: MTLCommandQueue, layerRenderer: LayerRenderer) { + // The headset's render path, forward, wherever this loop renders: the platform's + // default on visionOS, and what a Mac rendering a remote immersive space takes too, + // unless the game chose a path. + adoptXRRenderPathUnlessChosen() configureSpatialEventBridge() configurePSVR2TrackingObserver() registerRuntimeLightingModeObserverIfNeeded() @@ -148,6 +325,16 @@ renderer?.setupCallbacks(gameUpdate: gameUpdate, handleInput: handleInput) } + /// Installs the game's per-frame callbacks with the frame's time: the instant the + /// compositor predicts the frame will be shown, which the head pose is predicted for + /// too, so an object posed for it keeps step with the head. + public func setupCallbacks( + frameUpdate: @escaping (_ frame: FrameTime) -> Void, + handleInput: @escaping () -> Void + ) { + renderer?.setupCallbacks(frameUpdate: frameUpdate, handleInput: handleInput) + } + public func enqueueSpatialInputSnapshot(_ snapshot: XRSpatialInputSnapshot) { InputSystem.shared.enqueueXRSpatialSnapshot(snapshot) } @@ -548,11 +735,50 @@ guard let layerRenderer else { return } guard isRunning() else { return } + // The half-rate pacing waits here, before a frame is taken, so that the frames + // rendered land on every other refresh, evenly, when one refresh is not enough for + // a frame's GPU work; the display shows the previous frame again meanwhile. + paceForHalfRate() + + let queryStart = LayerRenderer.Clock().now guard let frame = layerRenderer.queryNextFrame() else { return } + let handedAt = LayerRenderer.Clock().now + let loopMs = lastFrameHandedAt.map { Self.milliseconds($0.duration(to: queryStart)) } ?? 0 + lastFrameHandedAt = handedAt // 2. Call predictTiming to get the predicted render deadlines for code guard let timing = frame.predictTiming() else { return } + // The instant the compositor predicts the frame will be shown, in the media-time + // base: the game poses the frame for it in the update phase, and the head is posed + // for the same instant at submission (the drawable's frameTiming, normally this + // very value; notePresentationRevision counts the frames where it is not). + let compositorPresentationTime = Self.mediaTime(of: timing.presentationTime) + let predictedPresentationTime = compositorPresentationTime + presentationLeadSeconds() + logClockBasesOnce() + if let last = lastPredictedPresentation { + let spacingMs = Self.milliseconds(last.duration(to: timing.presentationTime)) + deadlineLock.lock() + secondSpacingSumMs += spacingMs + secondSpacingMinMs = min(secondSpacingMinMs, spacingMs) + secondSpacingMaxMs = max(secondSpacingMaxMs, spacingMs) + secondSpacings += 1 + let periodMs = currentFrameTime.period * 1000.0 + if periodMs > 0 { + let expected = pacingActiveNow ? halfRate.cadenceRefreshes : 1 + secondSkipped += max(0, Int((spacingMs / periodMs).rounded()) - expected) + } + deadlineLock.unlock() + } + lastPredictedPresentation = timing.presentationTime + deadlineLock.lock() + secondQueryBlockSumMs += Self.milliseconds(queryStart.duration(to: handedAt)) + secondLoopSumMs += loopMs + secondPredictedAheadSumMs += Self.milliseconds(handedAt.duration(to: timing.presentationTime)) + secondDeadlineMarginSumMs += Self.milliseconds(timing.renderingDeadline.duration(to: timing.presentationTime)) + secondTimedFrames += 1 + deadlineLock.unlock() + // 3. Call startUpdate() to mark the start of the update phase frame.startUpdate() #if ENGINE_STATS_ENABLED @@ -594,14 +820,36 @@ syncStreamingCameraPosition() if !loading { - renderer.updateXR(useExternalStatsLifecycle: true) + renderer.updateXR(presentationTime: predictedPresentationTime, useExternalStatsLifecycle: true) } // 6. Call endupdate() to mark the end of the update phase frame.endUpdate() // 7. Call wait(until:tolerace) to puase your render loop until the optimal rendering time - LayerRenderer.Clock().wait(until: timing.optimalInputTime, tolerance: .zero) + #if ENGINE_STATS_ENABLED + let xrWaitStart = CACurrentMediaTime() + #endif + // The wait for the compositor's optimal input time: the head pose is queried as late + // as the frame allows. Without it the frame is encoded now, so the GPU starts about + // the wait's length earlier and the head is predicted from earlier; the automatic + // mode waits while the frames meet the deadline and not while they do not. + let waitLengthMs = max(0, Self.milliseconds(LayerRenderer.Clock().now.duration(to: timing.optimalInputTime))) + if waitsForOptimalInput(skippedWaitMs: waitLengthMs) { + LayerRenderer.Clock().wait(until: timing.optimalInputTime, tolerance: .zero) + } + #if ENGINE_STATS_ENABLED + let xrWaitMs = (CACurrentMediaTime() - xrWaitStart) * 1000.0 + EngineStatsMonitor.shared.update { snapshot in + snapshot.timing.xrWaitMs = xrWaitMs + } + if !loggedFrameTimingBudget { + loggedFrameTimingBudget = true + let toDeadline = timing.optimalInputTime.duration(to: timing.renderingDeadline) + let toPresentation = timing.optimalInputTime.duration(to: timing.presentationTime) + print(String(format: "XR frame timing: optimal input to rendering deadline %.2f ms, to presentation %.2f ms", Self.milliseconds(toDeadline), Self.milliseconds(toPresentation))) + } + #endif // The compositor or app state can transition while waiting. If not running anymore, // skip submission entirely to avoid using an invalid frame. @@ -609,6 +857,9 @@ // 8. Call startSubmission() to mark the start of submission phase frame.startSubmission() + #if ENGINE_STATS_ENABLED + let xrSubmissionStart = CACurrentMediaTime() + #endif var shouldEndSubmission = true defer { if shouldEndSubmission, layerRenderer.state == .running { @@ -616,69 +867,158 @@ } } - // 9. Encode any drawing commands that depend on the device position or orientation - guard let drawable = frame.queryDrawable() else { + // 9. Encode any drawing commands that depend on the device position or orientation. + // visionOS 26 hands a frame one drawable per target (the built-in display, and a + // capture target while the view is recorded or streamed); a layer with foveation + // or a render quality set aborts the app if the single-drawable query is used. + let drawables: [LayerRenderer.Drawable] + if #available(visionOS 26.0, *) { + drawables = frame.queryDrawables() + } else { + drawables = frame.queryDrawable().map { [$0] } ?? [] + } + guard !drawables.isEmpty else { #if ENGINE_STATS_ENABLED renderer.finalizeXRStatsAndMonitors(frameStartTime: xrFrameStartTime) #else renderer.finalizeXRStatsAndMonitors(frameStartTime: 0.0) #endif - // queryDrawable() can fail during shutdown/teardown; in that case the + // No drawables (shutdown/teardown, or the compositor cancelled the frame): the // frame is already invalid and endSubmission() must not be called. shouldEndSubmission = false return } - - // 10. Fetch the predicted device anchor from ARKit using the frameTiming information, and - // apply the anchor to your frame - let presentationInstant = drawable.frameTiming.presentationTime - let presentationTimeCA: TimeInterval = compositorInstantToCATime(presentationInstant) - var deviceAnchor = queryDeviceAnchorIfTrackingRunning(atTimestamp: presentationTimeCA) - - // If predicted-time query misses, retry at "now" to reduce one-frame anchor gaps. - if deviceAnchor == nil { - let nowTimestamp = CACurrentMediaTime() - if let retryAnchor = queryDeviceAnchorIfTrackingRunning(atTimestamp: nowTimestamp) { - deviceAnchor = retryAnchor - } - } - - // Use current anchor if valid, otherwise fall back to last valid anchor. - // This prevents "Presenting a drawable without a device anchor" while maintaining - // stable rendering. We must always present the drawable once we have it. - if let anchor = deviceAnchor { - if missingAnchorFrameCount > 0 { - print("โœ“ XR device anchor recovered after \(missingAnchorFrameCount) missing frame(s)") - } - missingAnchorFrameCount = 0 - lastValidDeviceAnchor = anchor - drawable.deviceAnchor = anchor - } else if let cachedAnchor = lastValidDeviceAnchor { - missingAnchorFrameCount += 1 - if shouldLogAnchorDiagnostics() { - printXRAnchorDiagnostics(message: "XR device anchor missing; using cached anchor") - } - drawable.deviceAnchor = cachedAnchor - } else { - missingAnchorFrameCount += 1 - if shouldLogAnchorDiagnostics() { - printXRAnchorDiagnostics(message: "XR device anchor missing; no cached anchor available") + if drawables.count != loggedDrawableCount { + loggedDrawableCount = drawables.count + let targets = drawables.map { drawable -> String in + if #available(visionOS 26.0, *) { + return drawable.target == .builtIn ? "built-in" : "capture" + } + return "built-in" } + print("XR: \(drawables.count) drawable(s) per frame (\(targets.joined(separator: ", ")))") } - // Note: If we have no cached anchor either, drawable.deviceAnchor remains nil - // and the render loop will skip rendering but still present (required by compositor) // Re-check loading state right before render pass to catch any mutations that started // after our initial snapshot. This provides a second layer of protection against races. let loadingNow = AssetLoadingGate.shared.isLoadingAny let effectiveLoading = loading || loadingNow - executeXRSystemPass(frame: frame, drawable: drawable, loading: effectiveLoading) + // The frame's presentation time is checked on its first drawable (the built-in + // display); a capture drawable shares the timing. + var presentationChecked = false + for drawable in drawables { + // 10. Fetch the predicted device anchor from ARKit using the frameTiming information, and + // apply the anchor to your frame + let presentationInstant = drawable.frameTiming.presentationTime + let presentationTimeCA: TimeInterval = Self.mediaTime(of: presentationInstant) + if !presentationChecked { + presentationChecked = true + notePresentationRevision(predicted: compositorPresentationTime, final: presentationTimeCA) + } + var deviceAnchor = queryDeviceAnchorIfTrackingRunning(atTimestamp: presentationTimeCA) + + // If predicted-time query misses, retry at "now" to reduce one-frame anchor gaps. + if deviceAnchor == nil { + let nowTimestamp = CACurrentMediaTime() + if let retryAnchor = queryDeviceAnchorIfTrackingRunning(atTimestamp: nowTimestamp) { + deviceAnchor = retryAnchor + } + } + + // Use current anchor if valid, otherwise fall back to last valid anchor. + // This prevents "Presenting a drawable without a device anchor" while maintaining + // stable rendering. We must always present the drawable once we have it. + if let anchor = deviceAnchor { + if missingAnchorFrameCount > 0 { + print("โœ“ XR device anchor recovered after \(missingAnchorFrameCount) missing frame(s)") + } + missingAnchorFrameCount = 0 + lastValidDeviceAnchor = anchor + drawable.deviceAnchor = anchor + } else if let cachedAnchor = lastValidDeviceAnchor { + missingAnchorFrameCount += 1 + if shouldLogAnchorDiagnostics() { + printXRAnchorDiagnostics(message: "XR device anchor missing; using cached anchor") + } + drawable.deviceAnchor = cachedAnchor + } else { + missingAnchorFrameCount += 1 + if shouldLogAnchorDiagnostics() { + printXRAnchorDiagnostics(message: "XR device anchor missing; no cached anchor available") + } + } + // Note: If we have no cached anchor either, drawable.deviceAnchor remains nil + // and the render loop will skip rendering but still present (required by compositor) + + if drawsScene(into: drawable) { + // A capture drawable (a recording or a stream of the view) is drawn from + // its own pose, so it must not advance the frame-to-frame state the + // built-in display relies on: the temporal history and the depth pyramid. + var primary = true + if #available(visionOS 26.0, *) { + primary = drawable.target == .builtIn + } + executeXRSystemPass(frame: frame, drawable: drawable, loading: effectiveLoading, primary: primary) + } else { + presentCleared(drawable) + } + } #if ENGINE_STATS_ENABLED + let xrEncodeMs = (CACurrentMediaTime() - xrSubmissionStart) * 1000.0 + EngineStatsMonitor.shared.update { snapshot in + snapshot.timing.encodeMs = xrEncodeMs + } renderer.finalizeXRStatsAndMonitors(frameStartTime: xrFrameStartTime) #else renderer.finalizeXRStatsAndMonitors(frameStartTime: 0.0) #endif + lastRenderedPresentation = timing.presentationTime + } + + /// Whether the scene is drawn into a drawable. The built-in display always; another + /// target (a capture of the view for recording or streaming) only while its textures are + /// the size the frame's resources are allocated at โ€” every sizeable resource is sized + /// to one viewport, and a second size would reallocate them twice a frame. + private func drawsScene(into drawable: LayerRenderer.Drawable) -> Bool { + if #available(visionOS 26.0, *), drawable.target != .builtIn, + let texture = drawable.colorTextures.first, + renderInfo.viewPort != simd_float2(Float(texture.width), Float(texture.height)) + { + if !loggedCaptureTargetSkip { + loggedCaptureTargetSkip = true + print("XR: a \(texture.width)ร—\(texture.height) capture drawable differs from the \(Int(renderInfo.viewPort.x))ร—\(Int(renderInfo.viewPort.y)) frame resources; presenting it cleared") + } + return false + } + return true + } + + /// Presents a drawable with its colour cleared (the immersion mode's alpha) and its depth + /// cleared, without drawing the scene: the compositor wants every drawable it handed out + /// presented. + private func presentCleared(_ drawable: LayerRenderer.Drawable) { + guard let commandBuffer = renderInfo.commandQueue.makeCommandBuffer() else { return } + commandBuffer.label = "XR Cleared Drawable" + for viewIndex in drawable.views.indices + where viewIndex < drawable.colorTextures.count && viewIndex < drawable.depthTextures.count + { + let descriptor = MTLRenderPassDescriptor() + descriptor.colorAttachments[0].texture = drawable.colorTextures[viewIndex] + descriptor.colorAttachments[0].loadAction = .clear + descriptor.colorAttachments[0].storeAction = .store + descriptor.colorAttachments[0].clearColor = MTLClearColor(red: 0, green: 0, blue: 0, alpha: Double(getAlphaForImmersionMode())) + descriptor.depthAttachment.texture = drawable.depthTextures[viewIndex] + descriptor.depthAttachment.loadAction = .clear + descriptor.depthAttachment.storeAction = .store + descriptor.depthAttachment.clearDepth = 0.0 + if viewIndex < drawable.rasterizationRateMaps.count { + descriptor.rasterizationRateMap = drawable.rasterizationRateMaps[viewIndex] + } + commandBuffer.makeRenderCommandEncoder(descriptor: descriptor)?.endEncoding() + } + drawable.encodePresent(commandBuffer: commandBuffer) + commandBuffer.commit() } private func updateSpatialInputState() { @@ -756,7 +1096,82 @@ spatialGestureRecognizer.resetAllSpatialInteractionTracking() } - func executeXRSystemPass(frame _: LayerRenderer.Frame, drawable: LayerRenderer.Drawable, loading: Bool) { + /// The rasterization rate map the compositor made for a view, as the renderer draws + /// through it, or nil when the frame is drawn uniformly: the layer configured without + /// foveation (no maps; also the simulator) or `setRendering(.foveatedRendering(.disabled))`. + /// The map's parameters go into this frame's slot of the ring for the shaders that decode + /// them; the viewport is the view's in the map's screen space. + private func makeFoveationFrame( + drawable: LayerRenderer.Drawable, + viewport: MTLViewport, + viewIndex: Int, + slot: Int + ) -> XRFoveationFrame? { + let rateMaps = drawable.rasterizationRateMaps + let available = xrFoveatedRenderingEnabled && viewIndex < rateMaps.count && viewIndex < stereoEyeCount + if loggedFoveationState != available { + loggedFoveationState = available + if available { + let rateMap = rateMaps[viewIndex] + let screen = rateMap.screenSize + let physical = rateMap.physicalSize(layer: 0) + print("โœ“ XR foveation: drawing through the compositor's rasterization rate maps, screen \(screen.width)ร—\(screen.height), textures \(physical.width)ร—\(physical.height) per eye; view texture map viewport \(viewport.originX),\(viewport.originY) \(viewport.width)ร—\(viewport.height)") + } else if rateMaps.isEmpty { + print("XR foveation off: the layer provides no rasterization rate maps (isFoveationEnabled is false, or the simulator); drawing uniformly") + } else { + print("XR foveation off by setting (.foveatedRendering(.disabled)); the layer's rate maps are ignored and the picture will be warped") + } + } + guard available else { return nil } + + let rateMap = rateMaps[viewIndex] + let sizeAndAlign = rateMap.parameterDataSizeAndAlign + let align = max(1, sizeAndAlign.align) + let length = max(16, (sizeAndAlign.size + align - 1) / align * align) + if rateMapParameterBuffers[slot][viewIndex].map({ $0.length < length }) ?? true { + let buffer = renderInfo.device.makeBuffer(length: length, options: .storageModeShared) + buffer?.label = "XR Rate Map Parameters eye \(viewIndex) slot \(slot)" + rateMapParameterBuffers[slot][viewIndex] = buffer + } + guard let buffer = rateMapParameterBuffers[slot][viewIndex], + let data = XRRasterizationRateMapData(rateMap: rateMap, buffer: buffer) + else { + Logger.logWarning(message: "XR foveation: could not copy the rate map parameters for eye \(viewIndex); drawing this eye uniformly") + return nil + } + if let texture = drawable.colorTextures.indices.contains(viewIndex) ? drawable.colorTextures[viewIndex] : nil, + Int(data.physicalSize.x) != texture.width || Int(data.physicalSize.y) != texture.height + { + Logger.logWarning(message: "XR foveation: eye \(viewIndex) rate map physical size \(data.physicalSize) differs from its texture \(texture.width)ร—\(texture.height)") + } + // The eye's viewport in the map's screen space. In the dedicated layout the view + // is the whole texture, so it is the map's screen size at the origin; the view's own + // texture map viewport is logged above for the day another layout needs it. + let screenViewport = MTLViewport( + originX: 0, originY: 0, + width: Double(data.screenSize.x), height: Double(data.screenSize.y), + znear: 0, zfar: 1 + ) + return XRFoveationFrame(rateMap: rateMap, viewport: screenViewport, rateMapData: data) + } + + func executeXRSystemPass(frame _: LayerRenderer.Frame, drawable: LayerRenderer.Drawable, loading: Bool, primary: Bool = true) { + TemporalAntiAliasing.shared.historyUpdatesEnabled = primary + defer { TemporalAntiAliasing.shared.historyUpdatesEnabled = true } + if !loggedDepthSetup, let depthTexture = drawable.depthTextures.first { + loggedDepthSetup = true + // What the depth test can tell apart: the layer's depth format and the + // drawable's depth range (far, near) the projection is built from. + let format: String + switch depthTexture.pixelFormat { + case .depth16Unorm: format = "depth16Unorm" + case .depth32Float: format = "depth32Float" + case .depth32Float_stencil8: format = "depth32Float_stencil8" + default: format = "pixel format \(depthTexture.pixelFormat.rawValue)" + } + depthSetupDescription = String(format: "%@, range far %.3f near %.3f", format, drawable.depthRange.x, drawable.depthRange.y) + print("XR depth: \(depthSetupDescription)") + } // Wait for available command buffer slot to prevent unbounded memory growth let semaphoreWaitStart = CACurrentMediaTime() let firstWaitResult = commandBufferSemaphore.wait(timeout: .now() + .milliseconds(100)) @@ -777,12 +1192,17 @@ commandBufferSemaphore.signal() return } + GPUPassTiming.shared.beginFrame(device: renderInfo.device) #if ENGINE_STATS_ENABLED let renderTotalStart = CACurrentMediaTime() #endif renderInfo.currentInFlightFrameSlot = acquireUniformFrameSlot() + let rateMapSlot = rateMapFrameCounter % rateMapParameterBuffers.count + rateMapFrameCounter += 1 - // Update viewport to match actual drawable size (per-eye texture dimensions) + // Update viewport to match actual drawable size (per-eye texture dimensions). With + // foveation on, that is the rate map's physical size: every texture of the frame is + // allocated at it, while viewports and clip space live in the map's screen size. if let firstColorTexture = drawable.colorTextures.first { let actualViewPort = simd_float2(Float(firstColorTexture.width), Float(firstColorTexture.height)) if renderInfo.viewPort != actualViewPort { @@ -867,6 +1287,15 @@ } renderInfo.currentEye = viewIndex + // The eye's rasterization rate map (foveation), for the passes that draw the + // scene; the final pass into the compositor's textures copies physical pixels + // one to one and keeps passDescriptor without it. + renderInfo.xrFoveation = makeFoveationFrame( + drawable: drawable, + viewport: view.textureMap.viewport, + viewIndex: viewIndex, + slot: rateMapSlot + ) #if ENGINE_STATS_ENABLED let encodeStart = CACurrentMediaTime() @@ -884,22 +1313,35 @@ // Build the HZB depth pyramid once after both eyes are rendered. // In stereo mode the mono pyramid (from the last eye rendered) is used - // for both per-eye VP occlusion tests in executeFrustumCulling. - buildHZBDepthPyramid(commandBuffer) + // for both per-eye VP occlusion tests in executeFrustumCulling. A capture + // drawable's depth is seen from its own pose and does not replace it. + if primary { + buildHZBDepthPyramid(commandBuffer) + } drawable.encodePresent(commandBuffer: commandBuffer) EngineProfiler.shared.attach(to: commandBuffer, label: "XRFrame") + // The instant the GPU must have finished by for the compositor to show this frame at + // its presentation time, in the base of the command buffer's own timestamps. + let deadline = Self.mediaTime(of: drawable.frameTiming.renderingDeadline) + let presentation = Self.mediaTime(of: drawable.frameTiming.presentationTime) + let countsDeadline = primary // Add completion handler to signal semaphore when GPU work is done - commandBuffer.addCompletedHandler { cb in + commandBuffer.addCompletedHandler { [weak self] cb in #if ENGINE_STATS_ENABLED - let gpuExecutionMs = (cb.gpuEndTime - cb.gpuStartTime) * 1000.0 - EngineStatsMonitor.shared.recordGPUCompletion(executionMs: gpuExecutionMs) + // The frame's GPU time beyond the previous frame's: the two overlap on the + // GPU, and the command buffer's own span would count the overlap twice. + EngineStatsMonitor.shared.recordGPUCompletion(gpuStartTime: cb.gpuStartTime, gpuEndTime: cb.gpuEndTime) #endif + if countsDeadline { + self?.noteGPUCompletion(at: cb.gpuEndTime, start: cb.gpuStartTime, deadline: deadline, presentation: presentation) + } commandBufferSemaphore.signal() } + GPUPassTiming.shared.endFrame(commandBuffer) #if ENGINE_STATS_ENABLED let submitStart = CACurrentMediaTime() #endif @@ -1035,20 +1477,161 @@ #endif } - @inline(__always) - private func compositorInstantToCATime(_ instant: LayerRenderer.Clock.Instant) -> TimeInterval { - let compositorClock = LayerRenderer.Clock() - let nowInstant = compositorClock.now - let nowCA = CACurrentMediaTime() + /// An instant of the compositor's clock as seconds in the `CACurrentMediaTime()` base: + /// both count mach absolute time from the boot, the compositor clock's epoch. + private static func mediaTime(of instant: LayerRenderer.Clock.Instant) -> TimeInterval { + seconds(LayerRenderer.Clock.Instant.epoch.duration(to: instant)) + } - // Compute duration between now and the target instant - let duration: Duration = nowInstant.duration(to: instant) + private static func seconds(_ duration: Duration) -> TimeInterval { + let parts = duration.components + return Double(parts.seconds) + Double(parts.attoseconds) / 1e18 + } - // Convert Duration โ†’ seconds - let components = duration.components - let seconds = Double(components.seconds) + Double(components.attoseconds) / 1e18 + /// Prints once how far the compositor clock's epoch conversion is from the media time + /// read at the same moment: the gap between the two reads, and nothing more, if the + /// clocks share their base as the engine assumes. + private func logClockBasesOnce() { + guard !loggedClockBases else { return } + loggedClockBases = true + let epochBased = Self.mediaTime(of: LayerRenderer.Clock().now) + let mediaTime = CACurrentMediaTime() + print(String(format: "XR clock: the compositor epoch's time differs from the media time by %.3f ms", (epochBased - mediaTime) * 1000.0)) + } - return nowCA + seconds + /// Counts a frame whose GPU work finished after the compositor's rendering deadline, keeps + /// the longest overrun, and prints once a second how many of that second's frames were + /// late and by how much. Runs on the GPU's completion thread. + private func noteGPUCompletion(at gpuEnd: CFTimeInterval, start gpuStart: CFTimeInterval, deadline: TimeInterval, presentation: TimeInterval) { + let overrunMs = (gpuEnd - deadline) * 1000.0 + let usableMarginMs = XRHalfRatePolicy.usableMarginMs(deadlineMarginMs: (presentation - deadline) * 1000.0) + let now = CACurrentMediaTime() + var report: String? + deadlineLock.lock() + secondFrames += 1 + secondLeastSlackMs = min(secondLeastSlackMs, -overrunMs) + let gpuMs = (gpuEnd - gpuStart) * 1000.0 + secondMaxGPUMs = max(secondMaxGPUMs, gpuMs) + secondGPUSumMs += gpuMs + secondOwnGPUSumMs += max(0, (gpuEnd - max(gpuStart, lastGPUEndTime)) * 1000.0) + lastGPUEndTime = max(lastGPUEndTime, gpuEnd) + if overrunMs > 0 { + deadlineMissCount += 1 + secondLate += 1 + secondOverrunSumMs += overrunMs + secondWorstMs = max(secondWorstMs, overrunMs) + worstOverrunMs = max(worstOverrunMs, overrunMs) + if overrunMs > usableMarginMs { + secondMissed += 1 + } + } + if lastDeadlineLogTime == 0 { + lastDeadlineLogTime = now + } else if now - lastDeadlineLogTime >= 1.0 { + if secondLate > 0 || secondSkipped > 0 { + let mode: String + switch xrOptimalInputWaitMode { + case .always: mode = "always" + case .never: mode = "never" + case .automatic: mode = "automatic" + } + let meanOverrunMs = secondLate > 0 ? secondOverrunSumMs / Double(secondLate) : 0 + report = String(format: "XR frame timing: the GPU finished after the rendering deadline on %d of %d frames in the last second (worst +%.2f ms, mean +%.2f ms; %d late frames so far; input-time wait %@, %@%@)", secondLate, secondFrames, secondWorstMs, meanOverrunMs, deadlineMissCount, mode, lastFrameWaited ? "waited" : "not waited", pacingActiveNow ? String(format: "; half rate, every %d refreshes, landing +%d", halfRate.cadenceRefreshes, halfRate.landingRefreshes) : "") + report = (report ?? "") + String(format: "; GPU %.1f ms a frame (%.1f ms its own after the frame before), longest %.1f%@", secondGPUSumMs / Double(secondFrames), secondOwnGPUSumMs / Double(secondFrames), secondMaxGPUMs, secondPacedWaits > 0 ? String(format: ", paced wait %.1f ms", secondPacedWaitSumMs / Double(secondPacedWaits)) : "") + report = (report ?? "") + String(format: "; %d past the deadline's margin, %d refreshes skipped", secondMissed, secondSkipped) + if secondTimedFrames > 0 { + let timed = Double(secondTimedFrames) + report = (report ?? "") + String(format: "; the query blocked %.1f ms, the loop took %.1f ms between frames, the presentation was predicted %.1f ms after the query, the deadline %.1f ms before it", secondQueryBlockSumMs / timed, secondLoopSumMs / timed, secondPredictedAheadSumMs / timed, secondDeadlineMarginSumMs / timed) + } + if secondSpacings > 0 { + report = (report ?? "") + String(format: "; presentations spaced %.1f ms (%.1f to %.1f)", secondSpacingSumMs / Double(secondSpacings), secondSpacingMinMs, secondSpacingMaxMs) + } + // With the GPU pass timing on, every pass of the last resolved frame, heaviest + // first, with their sum: what a second of frames well past the deadline spent + // its time on, and how much of the frame's span lies between passes. + if meanOverrunMs > 2.0, GPUPassTiming.shared.isEnabled { + let passes = GPUPassTiming.shared.lastFrame().passes.sorted { $0.totalMs > $1.totalMs } + if !passes.isEmpty { + let sum = passes.reduce(0.0) { $0 + $1.totalMs } + let list = passes.map { String(format: "%@ %.2f", $0.label, $0.totalMs) }.joined(separator: ", ") + report = (report ?? "") + String(format: "\nXR frame timing: the last frame's passes, %.1f ms in all (a label sums both eyes): ", sum) + list + } + } + } + // The automatic mode: stop waiting once a second had late frames (one in + // twenty or more); wait again once a second's least slack would have covered + // the longest wait skipped, with a margin. + if xrOptimalInputWaitMode == .automatic, secondFrames > 0 { + if automaticWaits, secondLate * 20 >= secondFrames { + automaticWaits = false + let line = String(format: "XR frame timing: %d of %d frames late: encoding without the wait for the optimal input time", secondLate, secondFrames) + report = report.map { $0 + "\n" + line } ?? line + } else if !automaticWaits, secondLeastSlackMs.isFinite, secondLeastSlackMs > secondSkippedWaitMaxMs + 1.5 { + automaticWaits = true + let line = String(format: "XR frame timing: the least slack of %.2f ms covers the skipped wait of %.2f ms: waiting for the optimal input time again", secondLeastSlackMs, secondSkippedWaitMaxMs) + report = report.map { $0 + "\n" + line } ?? line + } + } + // The half-rate pacing, by its policy when the setting leaves it to the frames. + if xrFramePacing == .halfRateWhenLate { + let waitStillToSkip = xrOptimalInputWaitMode == .automatic && automaticWaits + let meanLateMs = secondLate > 0 ? secondOverrunSumMs / Double(secondLate) : 0 + let periodMs = currentFrameTime.period * 1000.0 + let marginMs = secondTimedFrames > 0 ? secondDeadlineMarginSumMs / Double(secondTimedFrames) : 0 + switch halfRate.secondEnded(frames: secondFrames, missed: secondMissed, skipped: secondSkipped, meanOverrunMs: meanLateMs, meanGPUDurationMs: secondGPUSumMs / Double(secondFrames), periodMs: periodMs, deadlineMarginMs: marginMs, waitStillToSkip: waitStillToSkip) { + case .entered: + let line = String(format: "XR frame timing: %d of %d frames past the deadline's margin, %d refreshes skipped, no wait left to skip: rendering every other refresh", secondMissed, secondFrames, secondSkipped) + report = report.map { $0 + "\n" + line } ?? line + case .left: + let line = String(format: "XR frame timing: the frame's GPU work of %.2f ms fits the refresh of %.2f ms: rendering every refresh again", secondGPUSumMs / Double(secondFrames), periodMs) + report = report.map { $0 + "\n" + line } ?? line + case .none: + break + } + } else { + halfRate.reset() + } + secondFrames = 0 + secondLate = 0 + secondWorstMs = 0 + secondOverrunSumMs = 0 + secondLeastSlackMs = .infinity + secondSkippedWaitMaxMs = 0 + secondMaxGPUMs = 0 + secondGPUSumMs = 0 + secondOwnGPUSumMs = 0 + secondPacedWaitSumMs = 0 + secondPacedWaits = 0 + secondMissed = 0 + secondSkipped = 0 + secondQueryBlockSumMs = 0 + secondLoopSumMs = 0 + secondPredictedAheadSumMs = 0 + secondDeadlineMarginSumMs = 0 + secondTimedFrames = 0 + secondSpacingSumMs = 0 + secondSpacingMinMs = .infinity + secondSpacingMaxMs = 0 + secondSpacings = 0 + lastDeadlineLogTime = now + } + deadlineLock.unlock() + if let report { + print(report) + } + } + + /// Counts the frames whose final presentation time (the drawable's) differs from the + /// one predicted at the update, which the game posed the frame for. + private func notePresentationRevision(predicted: TimeInterval, final: TimeInterval) { + let difference = final - predicted + guard abs(difference) > 0.0005 else { return } + presentationTimeRevisions += 1 + let now = CACurrentMediaTime() + if presentationTimeRevisions == 1 || now - lastPresentationRevisionLogTime >= 1.0 { + lastPresentationRevisionLogTime = now + print(String(format: "XR frame timing: the drawable's presentation time differs from the predicted one by %.2f ms (%d frames so far)", difference * 1000.0, presentationTimeRevisions)) + } } public func setImmersionMode(xrImmersionMode: UntoldImmersionMode) { diff --git a/Sources/UntoldEngineXR/XREnvironmentLightingSystem.swift b/Sources/UntoldEngineXR/XREnvironmentLightingSystem.swift index 8d76f980b..4f33502de 100644 --- a/Sources/UntoldEngineXR/XREnvironmentLightingSystem.swift +++ b/Sources/UntoldEngineXR/XREnvironmentLightingSystem.swift @@ -32,6 +32,15 @@ public var acceptedProbeUpdateCount: Int public var skippedProbeUpdateCount: Int public var fallbackReason: String? + /// The probe texture as ARKit last handed it over: pixel format, face size, mip count. + public var latestProbeDescription: String = "" + /// The intensity scale as published to the renderer (smoothed), and how many times a + /// prefiltered probe has been published. + public var publishedIntensityScale: Float? + public var publishCount: Int = 0 + /// The mean luminance of each face of the last probe read back (+x, โˆ’x, +y, โˆ’y, +z, โˆ’z), + /// before the intensity scale: where the room's light is in the probe's frame. + public var latestProbeFaceLuminance: [Float] = [] } public final class XREnvironmentLightingSystem: @unchecked Sendable { @@ -48,15 +57,55 @@ private var acceptedProbeUpdateCountValue = 0 private var skippedProbeUpdateCountValue = 0 private var fallbackReasonValue: String? + private var latestProbeDescriptionValue = "" + private var publishCountValue = 0 + private var latestProbeFaceLuminanceValue: [Float] = [] private var lastAcceptedProbeUpdateTime: CFTimeInterval = 0 + /// The newest probe not yet prefiltered: one that arrived while another was being + /// prefiltered, or inside the minimum interval. ARKit sends its estimates in a burst + /// when the provider starts and may then send nothing for a long while in a still + /// room, so the newest one must not be dropped. + private struct PendingProbe { + var texture: MTLTexture + var timestamp: CFTimeInterval + var intensityScale: Float + var retainedAnchor: Any + } + + private var pendingProbeValue: PendingProbe? + private var pendingProbeDrainTask: Task? + private var lastReadbackStartTime: CFTimeInterval? + private var readbackInFlightValue = false + /// Bumped when the system is disabled: a readback or a transition step from before + /// then has nothing to come back to. + private var sessionGeneration: UInt64 = 0 + /// The light's transition. The newest estimate read back is the target: its first + /// mip's texels (six faces, row by row, linear rgb), its scale and its tint. What is + /// shown is the current, which moves towards the target step by step; each step + /// uploads the current texels into the blend cube, prefilters it and publishes. + private var targetProbeTexels: [Float] = [] + private var targetProbeSize = 0 + private var targetIntensityScaleValue: Float = 1.0 + private var targetTintColorValue = simd_float3(1.0, 1.0, 1.0) + private var targetTimestampValue: CFTimeInterval = 0 + private var targetGeneration: UInt64 = 0 + private var currentProbeTexels: [Float] = [] + private var currentProbeSize = 0 + private var currentIntensityScaleValue: Float = 1.0 + private var currentTintColorValue = simd_float3(1.0, 1.0, 1.0) + private var blendedProbeCube: MTLTexture? + private var transitionRunning = false + private var lastTransitionStepTime: CFTimeInterval? + private var publishedIntensityScaleValue: Float? private var probeMonitorTask: Task? private var textureSets: [RuntimeEnvironmentLightingTextureSet] = [] private var currentReadTextureSetIndex = 0 private var prefilterGeneration: UInt64 = 0 - private let cameraScaleReferenceWhitePoint: Float = 5000.0 - private var smoothedIntensityScaleValue: Float? - private var smoothedTintColorValue = simd_float3(1.0, 1.0, 1.0) - private var lastSmoothedLightingTimestamp: CFTimeInterval? + /// The probe's camera scale reference at which the room's light is applied as it is. + /// Calibrated against RealityKit's lighting of the same object in the same room (a + /// dartboard's white segments matched the passthrough's whites at a reference of + /// 19016 with a scale of about 19): the reference over a thousand. + private let cameraScaleReferenceWhitePoint: Float = 1000.0 #if canImport(ARKit) /// ARKit data providers are one-shot: an instance that has been passed to @@ -65,15 +114,20 @@ private var environmentLightEstimationProvider: EnvironmentLightEstimationProvider? #endif + /// The least time between two prefilters: a probe that arrives inside it waits, and the + /// newest one waiting is the one prefiltered. public var minimumProbeUpdateInterval: CFTimeInterval = 0.5 - /// Exponential-smoothing time constant (seconds) applied to the probe-driven - /// intensity scale and tint before publishing. The raw values track headset - /// camera auto-exposure (`cameraScaleReference`) and can step sharply when the - /// user looks toward or away from bright real-world light sources; without - /// smoothing every real-world-tinted light portal steps in brightness with - /// head movement. Set to 0 to publish raw values immediately. - public var lightingSmoothingTimeConstant: CFTimeInterval = 1.0 + /// The time constant (seconds) of the light's transition from one estimate to the + /// next. A new estimate does not replace the light at once: the probe's texels, its + /// scale and its tint move towards it exponentially, re-prefiltered every + /// `transitionStepInterval`, so a step in the camera's exposure or in what the probe + /// sees when the wearer turns towards a window becomes a fade, not a cut. 0 publishes + /// each estimate as it arrives. + public var lightingSmoothingTimeConstant: CFTimeInterval = 3.0 + + /// How often the transition re-prefilters the blended probe while it is under way. + public var transitionStepInterval: CFTimeInterval = 0.1 public init() { #if canImport(ARKit) @@ -146,10 +200,18 @@ prefilterInFlightValue = false lastPrefilterDurationMsValue = nil lastAcceptedProbeUpdateTime = 0 + pendingProbeValue = nil + pendingProbeDrainTask?.cancel() + pendingProbeDrainTask = nil + lastReadbackStartTime = nil + readbackInFlightValue = false prefilterGeneration &+= 1 - smoothedIntensityScaleValue = nil - smoothedTintColorValue = simd_float3(1.0, 1.0, 1.0) - lastSmoothedLightingTimestamp = nil + sessionGeneration &+= 1 + targetProbeTexels = [] + currentProbeTexels = [] + transitionRunning = false + lastTransitionStepTime = nil + publishedIntensityScaleValue = nil RuntimeEnvironmentLightingStore.shared.publishXRLighting(nil) } lock.unlock() @@ -177,11 +239,6 @@ return false } - guard timestamp - lastAcceptedProbeUpdateTime >= minimumProbeUpdateInterval else { - skippedProbeUpdateCountValue += 1 - return false - } - lastAcceptedProbeUpdateTime = timestamp latestProbeTimestampValue = timestamp acceptedProbeUpdateCountValue += 1 @@ -235,18 +292,24 @@ guard shouldAcceptProbeUpdate(timestamp: anchor.timestamp) else { return } let hasTexture = anchor.environmentTexture != nil + if let texture = anchor.environmentTexture { + let description = "\(Self.pixelFormatName(texture.pixelFormat)) \(texture.width)ร—\(texture.height) ร—\(texture.mipmapLevelCount) mips" + lock.lock() + latestProbeDescriptionValue = description + lock.unlock() + } let intensityScale = updateProbeCameraScaleReference( anchor.cameraScaleReference, hasTexture: hasTexture ) if let environmentTexture = anchor.environmentTexture { - scheduleProbePrefilter( - environmentTexture: environmentTexture, + enqueueProbe(PendingProbe( + texture: environmentTexture, timestamp: anchor.timestamp, intensityScale: intensityScale, retainedAnchor: anchor - ) + )) } case .removed: @@ -303,6 +366,9 @@ return (writeIndex, textureSet, prefilterGeneration) } + /// Publishes the prefiltered set with the scale and tint it is given; false when it + /// could not (disabled, superseded, or the prefilter failed, which publishes nothing). + @discardableResult private func finishPrefilter( succeeded: Bool, textureSetIndex: Int, @@ -311,12 +377,12 @@ intensityScale: Float, tintColor: simd_float3, durationMs: Double - ) { + ) -> Bool { lock.lock() guard enabledValue, generation == prefilterGeneration else { lock.unlock() - return + return false } prefilterInFlightValue = false @@ -326,31 +392,15 @@ fallbackReasonValue = "XR probe prefilter failed" lock.unlock() RuntimeEnvironmentLightingStore.shared.publishXRLighting(nil) - return + return false } currentReadTextureSetIndex = textureSetIndex let textureSet = textureSets[textureSetIndex] latestTintColorValue = tintColor + publishedIntensityScaleValue = intensityScale fallbackReasonValue = nil - - // Smooth the published intensity/tint so auto-exposure steps in the raw - // probe estimate don't pop every real-world-tinted light in the scene. - // Diagnostics keep the raw values; only the published lighting is smoothed. - var publishedIntensityScale = intensityScale - var publishedTintColor = tintColor - if let previousScale = smoothedIntensityScaleValue, - let previousTimestamp = lastSmoothedLightingTimestamp, - lightingSmoothingTimeConstant > 0.0 - { - let deltaTime = min(max(timestamp - previousTimestamp, 0.0), 10.0) - let alpha = Float(1.0 - exp(-deltaTime / lightingSmoothingTimeConstant)) - publishedIntensityScale = previousScale + (intensityScale - previousScale) * alpha - publishedTintColor = smoothedTintColorValue + (tintColor - smoothedTintColorValue) * alpha - } - smoothedIntensityScaleValue = publishedIntensityScale - smoothedTintColorValue = publishedTintColor - lastSmoothedLightingTimestamp = timestamp + publishCountValue += 1 lock.unlock() RuntimeEnvironmentLightingStore.shared.publishXRLighting( @@ -358,12 +408,13 @@ irradianceMap: textureSet.irradianceMap, specularMap: textureSet.specularMap, brdfMap: textureSet.brdfMap, - intensityScale: publishedIntensityScale, - tintColor: publishedTintColor, + intensityScale: intensityScale, + tintColor: tintColor, timestamp: timestamp, isValid: true ) ) + return true } private func updateProbeCameraScaleReference(_ cameraScaleReference: Float?, hasTexture: Bool) -> Float { @@ -387,7 +438,7 @@ return latestIntensityScaleValue } - let intensityScale = min(max(cameraScaleReference / cameraScaleReferenceWhitePoint, 0.0), 8.0) + let intensityScale = min(max(cameraScaleReference / cameraScaleReferenceWhitePoint, 0.0), 64.0) latestIntensityScaleValue = intensityScale return intensityScale } @@ -402,7 +453,62 @@ var faceCount: Int } - private func scheduleProbePrefilter( + /// Takes a probe in: the newest one waits for the readback in flight and for the + /// minimum interval, then is read back and becomes the light's target; an older one + /// still waiting is superseded. + private func enqueueProbe(_ probe: PendingProbe) { + lock.lock() + pendingProbeValue = probe + lock.unlock() + drainPendingProbe() + } + + /// Reads the waiting probe back once no readback is in flight and the interval has + /// passed; until then the readback's completion, or a timer for the rest of the + /// interval, comes back here. + private func drainPendingProbe() { + lock.lock() + guard enabledValue, let probe = pendingProbeValue, !readbackInFlightValue else { + lock.unlock() + return + } + let now = ProcessInfo.processInfo.systemUptime + if let lastStart = lastReadbackStartTime { + let remaining = minimumProbeUpdateInterval - (now - lastStart) + if remaining > 0 { + if pendingProbeDrainTask == nil { + pendingProbeDrainTask = Task { [weak self] in + try? await Task.sleep(for: .seconds(remaining)) + guard !Task.isCancelled else { return } + self?.drainTimerFired() + } + } + lock.unlock() + return + } + } + pendingProbeValue = nil + lastReadbackStartTime = now + lock.unlock() + scheduleProbeReadback( + environmentTexture: probe.texture, + timestamp: probe.timestamp, + intensityScale: probe.intensityScale, + retainedAnchor: probe.retainedAnchor + ) + } + + /// The rest of the interval has passed: the waiting probe may go now. + private func drainTimerFired() { + lock.lock() + pendingProbeDrainTask = nil + lock.unlock() + drainPendingProbe() + } + + /// Reads the probe back: its first mip, the texels the light's transition blends, and + /// its last mip, the per-face light the diagnostics report and the tint comes from. + private func scheduleProbeReadback( environmentTexture: MTLTexture, timestamp: CFTimeInterval, intensityScale: Float, @@ -413,55 +519,268 @@ return } - guard let prefilter = beginPrefilterIfPossible() else { return } + lock.lock() + guard enabledValue, !readbackInFlightValue else { + lock.unlock() + return + } + readbackInFlightValue = true + let session = sessionGeneration + lock.unlock() + + guard let commandBuffer = renderInfo.commandQueue.makeCommandBuffer(), + let texels = encodeEnvironmentTintReadback(commandBuffer: commandBuffer, environmentTexture: environmentTexture, mipLevel: 0), + let faces = encodeEnvironmentTintReadback(commandBuffer: commandBuffer, environmentTexture: environmentTexture) + else { + lock.lock() + readbackInFlightValue = false + lock.unlock() + publishUnavailableProbe(reason: "XR probe readback could not be encoded") + return + } + + commandBuffer.label = "XR Environment Probe Readback" + commandBuffer.addCompletedHandler { [weak self, retainedAnchor] commandBuffer in + _ = retainedAnchor + guard let self else { return } + let succeeded = commandBuffer.status == .completed + let tintColor = succeeded ? environmentTint(from: faces) : simd_float3(1.0, 1.0, 1.0) + let rgb = succeeded ? probeTexels(from: texels) : [] + probeEstimateArrived( + texels: rgb, + size: texels.width, + intensityScale: intensityScale, + tintColor: tintColor, + timestamp: timestamp, + session: session, + succeeded: succeeded + ) + } + commandBuffer.commit() + } + + /// A readback's texels, face by face and row by row, as linear rgb. + private func probeTexels(from readback: EnvironmentTintReadback) -> [Float] { + let contents = readback.buffer.contents() + let bytesPerPixel = readback.bytesPerRow / max(readback.width, 1) + var texels: [Float] = [] + texels.reserveCapacity(readback.faceCount * readback.width * readback.height * 3) + for face in 0 ..< readback.faceCount { + for y in 0 ..< readback.height { + for x in 0 ..< readback.width { + let offset = face * readback.faceBytes + y * readback.bytesPerRow + x * bytesPerPixel + let c = environmentTintPixel(contents: contents, offset: offset, pixelFormat: readback.pixelFormat) + texels.append(c.x) + texels.append(c.y) + texels.append(c.z) + } + } + } + return texels + } + + /// An estimate has been read back: it is the target the shown light moves to. The + /// first estimate, or one of another size, is shown as it is. + private func probeEstimateArrived( + texels: [Float], + size: Int, + intensityScale: Float, + tintColor: simd_float3, + timestamp: CFTimeInterval, + session: UInt64, + succeeded: Bool + ) { + lock.lock() + readbackInFlightValue = false + guard enabledValue, session == sessionGeneration else { + lock.unlock() + return + } + guard succeeded, !texels.isEmpty else { + lock.unlock() + publishUnavailableProbe(reason: "XR probe readback failed") + drainPendingProbe() + return + } + targetProbeTexels = texels + targetProbeSize = size + targetIntensityScaleValue = intensityScale + targetTintColorValue = tintColor + targetTimestampValue = timestamp + targetGeneration &+= 1 + if currentProbeTexels.count != texels.count { + currentProbeTexels = texels + currentProbeSize = size + currentIntensityScaleValue = intensityScale + currentTintColorValue = tintColor + } + let start = !transitionRunning + if start { + transitionRunning = true + lastTransitionStepTime = ProcessInfo.processInfo.systemUptime + } + lock.unlock() + if start { + transitionStep() + } + drainPendingProbe() + } + + /// One step of the light's transition: the shown texels, scale and tint move towards + /// the target by the time constant's share of the time since the last step, the blend + /// cube is prefiltered and published. The step that reaches the target is the last + /// until a new estimate arrives. + private func transitionStep() { + lock.lock() + guard enabledValue, !currentProbeTexels.isEmpty, targetProbeTexels.count == currentProbeTexels.count else { + transitionRunning = false + lock.unlock() + return + } + let now = ProcessInfo.processInfo.systemUptime + let deltaTime = lastTransitionStepTime.map { min(max(now - $0, 0.0), 10.0) } ?? 0.0 + let timeConstant = lightingSmoothingTimeConstant + var alpha: Float = timeConstant > 0.0 ? Float(1.0 - exp(-deltaTime / timeConstant)) : 1.0 + var maxDifference: Float = 0.0 + var maxTarget: Float = 0.0 + for index in currentProbeTexels.indices { + maxDifference = max(maxDifference, abs(targetProbeTexels[index] - currentProbeTexels[index])) + maxTarget = max(maxTarget, targetProbeTexels[index]) + } + let scaleDifference = abs(targetIntensityScaleValue - currentIntensityScaleValue) + // Close enough counts as there: the step snaps to the target and is the last. + let final = alpha >= 1.0 + || (maxDifference <= 0.002 * max(maxTarget, 1.0) && scaleDifference <= 0.005 * max(targetIntensityScaleValue, 0.01)) + if final { + alpha = 1.0 + } + for index in currentProbeTexels.indices { + currentProbeTexels[index] += (targetProbeTexels[index] - currentProbeTexels[index]) * alpha + } + currentIntensityScaleValue += (targetIntensityScaleValue - currentIntensityScaleValue) * alpha + currentTintColorValue += (targetTintColorValue - currentTintColorValue) * alpha + lastTransitionStepTime = now + let shownGeneration = targetGeneration + let session = sessionGeneration + let texels = currentProbeTexels + let size = currentProbeSize + let intensityScale = currentIntensityScaleValue + let tintColor = currentTintColorValue + let timestamp = targetTimestampValue + lock.unlock() + + guard let cube = uploadBlendedProbe(texels: texels, size: size) else { + lock.lock() + transitionRunning = false + lock.unlock() + publishUnavailableProbe(reason: "XR probe blend texture unavailable") + return + } + prefilterBlendedProbe(cube, intensityScale: intensityScale, tintColor: tintColor, timestamp: timestamp) { [weak self] published in + self?.continueTransition(after: shownGeneration, session: session, final: final || !published) + } + } + + /// After a step: the next one after the interval, or none when the target is reached + /// and no newer estimate has arrived in the meantime. + private func continueTransition(after shownGeneration: UInt64, session: UInt64, final: Bool) { + lock.lock() + guard enabledValue, session == sessionGeneration else { + transitionRunning = false + lock.unlock() + return + } + if final, targetGeneration == shownGeneration { + transitionRunning = false + lock.unlock() + return + } + let interval = final ? 0.0 : transitionStepInterval + lock.unlock() + Task { [weak self] in + if interval > 0.0 { + try? await Task.sleep(for: .seconds(interval)) + } + self?.transitionStep() + } + } + + /// The shown texels, uploaded into the blend cube the prefilter reads. + private func uploadBlendedProbe(texels: [Float], size: Int) -> MTLTexture? { + lock.lock() + if blendedProbeCube == nil || blendedProbeCube?.width != size { + let descriptor = MTLTextureDescriptor.textureCubeDescriptor(pixelFormat: .rgba16Float, size: size, mipmapped: true) + descriptor.usage = [.shaderRead, .shaderWrite] + descriptor.storageMode = .shared + blendedProbeCube = renderInfo.device.makeTexture(descriptor: descriptor) + blendedProbeCube?.label = "XR Environment Probe Blend" + } + let cube = blendedProbeCube + lock.unlock() + guard let cube, texels.count == 6 * size * size * 3 else { return nil } + + let faceTexels = size * size + var half = [UInt16](repeating: 0, count: faceTexels * 4) + for face in 0 ..< 6 { + for index in 0 ..< faceTexels { + let base = (face * faceTexels + index) * 3 + half[index * 4] = Float16(texels[base]).bitPattern + half[index * 4 + 1] = Float16(texels[base + 1]).bitPattern + half[index * 4 + 2] = Float16(texels[base + 2]).bitPattern + half[index * 4 + 3] = Float16(1.0).bitPattern + } + half.withUnsafeBytes { bytes in + cube.replace( + region: MTLRegionMake2D(0, 0, size, size), + mipmapLevel: 0, + slice: face, + withBytes: bytes.baseAddress!, + bytesPerRow: size * 8, + bytesPerImage: faceTexels * 8 + ) + } + } + return cube + } + + /// Prefilters the blend cube into the free texture set and publishes it; the completion + /// says whether it was published. + private func prefilterBlendedProbe( + _ cube: MTLTexture, + intensityScale: Float, + tintColor: simd_float3, + timestamp: CFTimeInterval, + completion: @escaping (Bool) -> Void + ) { + guard let prefilter = beginPrefilterIfPossible() else { + completion(false) + return + } let textureSetIndex = prefilter.textureSetIndex let textureSet = prefilter.textureSet let generation = prefilter.generation - guard let commandBuffer = renderInfo.commandQueue.makeCommandBuffer() else { - finishPrefilter( - succeeded: false, - textureSetIndex: textureSetIndex, - generation: generation, - timestamp: timestamp, - intensityScale: intensityScale, - tintColor: simd_float3(1.0, 1.0, 1.0), - durationMs: 0 - ) + guard let commandBuffer = renderInfo.commandQueue.makeCommandBuffer(), + let blit = commandBuffer.makeBlitCommandEncoder() + else { + finishPrefilter(succeeded: false, textureSetIndex: textureSetIndex, generation: generation, timestamp: timestamp, intensityScale: intensityScale, tintColor: tintColor, durationMs: 0) + completion(false) return } commandBuffer.label = "XR Environment Probe IBL Prefilter" + blit.generateMipmaps(for: cube) + blit.endEncoding() let startTime = Date().timeIntervalSinceReferenceDate - let tintReadback = encodeEnvironmentTintReadback( - commandBuffer: commandBuffer, - environmentTexture: environmentTexture - ) - let encoded = executeXRIBLCubePreFilterPass( - commandBuffer: commandBuffer, - environmentCubeTexture: environmentTexture, - target: textureSet - ) - - guard encoded else { - finishPrefilter( - succeeded: false, - textureSetIndex: textureSetIndex, - generation: generation, - timestamp: timestamp, - intensityScale: intensityScale, - tintColor: simd_float3(1.0, 1.0, 1.0), - durationMs: 0 - ) + guard executeXRIBLCubePreFilterPass(commandBuffer: commandBuffer, environmentCubeTexture: cube, target: textureSet) else { + finishPrefilter(succeeded: false, textureSetIndex: textureSetIndex, generation: generation, timestamp: timestamp, intensityScale: intensityScale, tintColor: tintColor, durationMs: 0) + completion(false) return } - commandBuffer.addCompletedHandler { [weak self, retainedAnchor] commandBuffer in - _ = retainedAnchor + commandBuffer.addCompletedHandler { [weak self] commandBuffer in let durationMs = (Date().timeIntervalSinceReferenceDate - startTime) * 1000.0 - let tintColor = tintReadback.map { self?.environmentTint(from: $0) ?? simd_float3(1.0, 1.0, 1.0) } - ?? simd_float3(1.0, 1.0, 1.0) - self?.finishPrefilter( + let published = self?.finishPrefilter( succeeded: commandBuffer.status == .completed, textureSetIndex: textureSetIndex, generation: generation, @@ -469,20 +788,22 @@ intensityScale: intensityScale, tintColor: tintColor, durationMs: durationMs - ) + ) ?? false + completion(published) } commandBuffer.commit() } private func encodeEnvironmentTintReadback( commandBuffer: MTLCommandBuffer, - environmentTexture: MTLTexture + environmentTexture: MTLTexture, + mipLevel: Int? = nil ) -> EnvironmentTintReadback? { guard let bytesPerPixel = environmentTintBytesPerPixel(for: environmentTexture.pixelFormat), let blit = commandBuffer.makeBlitCommandEncoder() else { return nil } - let mipLevel = max(environmentTexture.mipmapLevelCount - 1, 0) + let mipLevel = min(max(mipLevel ?? (environmentTexture.mipmapLevelCount - 1), 0), max(environmentTexture.mipmapLevelCount - 1, 0)) let width = max(environmentTexture.width >> mipLevel, 1) let height = max(environmentTexture.height >> mipLevel, 1) let bytesPerRow = width * bytesPerPixel @@ -522,6 +843,23 @@ ) } + /// A readable name for the probe's pixel format (Metal prints only the raw value). + private static func pixelFormatName(_ pixelFormat: MTLPixelFormat) -> String { + switch pixelFormat { + case .rgba16Float: return "rgba16Float" + case .rgba32Float: return "rgba32Float" + case .rgba8Unorm: return "rgba8Unorm" + case .rgba8Unorm_srgb: return "rgba8Unorm_srgb" + case .bgra8Unorm: return "bgra8Unorm" + case .bgra8Unorm_srgb: return "bgra8Unorm_srgb" + case .rg11b10Float: return "rg11b10Float" + case .rgb9e5Float: return "rgb9e5Float" + case .bgr10a2Unorm: return "bgr10a2Unorm" + case .rgb10a2Unorm: return "rgb10a2Unorm" + default: return "pixelFormat \(pixelFormat.rawValue)" + } + } + private func environmentTintBytesPerPixel(for pixelFormat: MTLPixelFormat) -> Int? { switch pixelFormat { case .rgba16Float: @@ -540,17 +878,25 @@ var total = simd_float3(0.0, 0.0, 0.0) var sampleCount: Float = 0.0 + var faceLuminance: [Float] = [] for face in 0 ..< readback.faceCount { let faceBase = face * readback.faceBytes + var faceTotal = simd_float3(0.0, 0.0, 0.0) for y in 0 ..< readback.height { let rowBase = faceBase + y * readback.bytesPerRow for x in 0 ..< readback.width { let offset = rowBase + x * environmentTintBytesPerPixel(for: readback.pixelFormat)! - total += environmentTintPixel(contents: contents, offset: offset, pixelFormat: readback.pixelFormat) + faceTotal += environmentTintPixel(contents: contents, offset: offset, pixelFormat: readback.pixelFormat) sampleCount += 1.0 } } + total += faceTotal + let faceSamples = Float(max(readback.width * readback.height, 1)) + faceLuminance.append(simd_dot(faceTotal / faceSamples, simd_float3(0.2126, 0.7152, 0.0722))) } + lock.lock() + latestProbeFaceLuminanceValue = faceLuminance + lock.unlock() guard sampleCount > 0.0 else { return simd_float3(1.0, 1.0, 1.0) } let average = total / sampleCount @@ -605,7 +951,11 @@ realWorldLightingContribution: RuntimeEnvironmentLightingStore.shared.realWorldLightingContribution, acceptedProbeUpdateCount: acceptedProbeUpdateCountValue, skippedProbeUpdateCount: skippedProbeUpdateCountValue, - fallbackReason: fallbackReasonValue + fallbackReason: fallbackReasonValue, + latestProbeDescription: latestProbeDescriptionValue, + publishedIntensityScale: publishedIntensityScaleValue, + publishCount: publishCountValue, + latestProbeFaceLuminance: latestProbeFaceLuminanceValue ) lock.unlock() return diagnostics diff --git a/Tests/UntoldEngineRenderTests/BaseRenderSetup.swift b/Tests/UntoldEngineRenderTests/BaseRenderSetup.swift index 1d8c0e223..4c3d05512 100644 --- a/Tests/UntoldEngineRenderTests/BaseRenderSetup.swift +++ b/Tests/UntoldEngineRenderTests/BaseRenderSetup.swift @@ -78,6 +78,7 @@ class BaseRenderSetup: XCTestCase { globalEntityCounter = 0 timeSinceLastUpdatePreviousTime = nil timeSinceLastUpdate = nil + currentFrameTime = .zero firstUpdateCall = false frameCount = 0 timePassedSinceLastFrame = 0.0 diff --git a/Tests/UntoldEngineRenderTests/ColorPrimariesRenderTests.swift b/Tests/UntoldEngineRenderTests/ColorPrimariesRenderTests.swift new file mode 100644 index 000000000..082683b29 --- /dev/null +++ b/Tests/UntoldEngineRenderTests/ColorPrimariesRenderTests.swift @@ -0,0 +1,93 @@ +// +// ColorPrimariesRenderTests.swift +// UntoldEngine +// +// The look pass shows the scene with the display's P3 primaries when asked: the colour +// taken as Display P3 coordinates and converted to sRGB, which widens a saturated colour +// and clips what leaves the gamut, as RealityKit shows a mixed-immersion scene. +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import simd +@testable import UntoldEngine +import XCTest + +@MainActor +final class ColorPrimariesRenderTests: BaseRenderSetup { + private var savedMode: AntiAliasingMode = .fxaa + private var savedOperator: TonemapOperator = .aces + private var savedPrimaries: ColorPrimaries = .sRGB + private var savedGrading = false + + override func setUp() async throws { + try await super.setUp() + savedMode = antiAliasingMode + savedOperator = TonemapParams.shared.operator + savedPrimaries = TonemapParams.shared.primaries + savedGrading = ColorGradingParams.shared.enabled + antiAliasingMode = .none + setPostFX(.tonemapOperator(.none)) + // The look must show the emissive as it is: no grade, whatever a test before left on. + ColorGradingParams.shared.enabled = false + } + + override func tearDown() async throws { + antiAliasingMode = savedMode + setPostFX(.tonemapOperator(savedOperator)) + setPostFX(.colorPrimaries(savedPrimaries)) + ColorGradingParams.shared.enabled = savedGrading + destroyAllEntities() + try await super.tearDown() + } + + /// The board's red as a picker gives it, in linear: an emissive black metal shows it as is. + private let red = simd_float3(0.4793, 0.0152, 0.0116) + + override func initializeAssets() { + let camera = createEntity() + createGameCamera(entityId: camera) + CameraSystem.shared.activeCamera = camera + cameraLookAt(entityId: camera, eye: simd_float3(0, 0, 2), target: .zero, up: simd_float3(0, 1, 0)) + let swatch = createEntity() + setEntityMeshDirect(entityId: swatch, meshes: BasicPrimitives.createCube(extent: 1.0), assetName: "Swatch") + updateMaterialColor(entityId: swatch, color: .black) + updateMaterialMetallic(entityId: swatch, metallic: 1) + updateMaterialRoughness(entityId: swatch, roughness: 1) + updateMaterialEmmisive(entityId: swatch, emmissive: red) + } + + /// The look pass's output at the centre of the frame. + private func lookCentre() throws -> simd_float3 { + for _ in 0 ..< 2 { + renderer.draw(in: renderer.metalView) + renderInfo.lastCommandBuffer?.waitUntilCompleted() + } + let texture = try XCTUnwrap(textureResources.lookTexture) + var halves = [UInt16](repeating: 0, count: 4) + halves.withUnsafeMutableBytes { bytes in + texture.getBytes(bytes.baseAddress!, bytesPerRow: 8, from: MTLRegionMake2D(texture.width / 2, texture.height / 2, 1, 1), mipmapLevel: 0) + } + return simd_float3(Float(Float16(bitPattern: halves[0])), Float(Float16(bitPattern: halves[1])), Float(Float16(bitPattern: halves[2]))) + } + + func testSRGBShowsTheColourAsItIs() throws { + setPostFX(.colorPrimaries(.sRGB)) + let shown = try lookCentre() + XCTAssertEqual(shown.x, red.x, accuracy: 0.01) + XCTAssertEqual(shown.y, red.y, accuracy: 0.005) + XCTAssertEqual(shown.z, red.z, accuracy: 0.005) + } + + func testDisplayP3WidensTheColourAndClipsWhatLeavesTheGamut() throws { + setPostFX(.colorPrimaries(.displayP3)) + let shown = try lookCentre() + // The P3โ†’sRGB matrix of the red: more red, the green below zero and clipped, a trace of blue. + XCTAssertEqual(shown.x, 0.5833, accuracy: 0.01) + XCTAssertEqual(shown.y, 0.0, accuracy: 0.002) + XCTAssertEqual(shown.z, 0.0021, accuracy: 0.003) + } +} diff --git a/Tests/UntoldEngineRenderTests/CullingSlotTests.swift b/Tests/UntoldEngineRenderTests/CullingSlotTests.swift new file mode 100644 index 000000000..6f7d63a9e --- /dev/null +++ b/Tests/UntoldEngineRenderTests/CullingSlotTests.swift @@ -0,0 +1,101 @@ +// +// CullingSlotTests.swift +// UntoldEngine +// +// The culling's rings: a frame's GPU work reads the list and the frustum the frame wrote, +// not the frame before's. +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import simd +@testable import UntoldEngine +import XCTest + +final class CullingSlotTests: BaseRenderSetup { + private var camera: EntityID = 0 + private var savedStep: Float = 0.5 + private var savedPixels: Float = 1 + + override func setUp() async throws { + try await super.setUp() + savedStep = HZBOcclusionCulling.maxCameraStep + savedPixels = SmallObjectCulling.minimumPixels + // The frustum alone decides: no occlusion test, no small-object threshold. + HZBOcclusionCulling.maxCameraStep = 0 + SmallObjectCulling.minimumPixels = 0 + } + + override func tearDown() async throws { + HZBOcclusionCulling.maxCameraStep = savedStep + SmallObjectCulling.minimumPixels = savedPixels + destroyAllEntities() + try await super.tearDown() + } + + override func initializeAssets() { + camera = createEntity() + createGameCamera(entityId: camera) + CameraSystem.shared.activeCamera = camera + look(towards: simd_float3(0, 0, -1)) + } + + private func look(towards direction: simd_float3) { + let eye = simd_float3(0, 1, 0) + cameraLookAt(entityId: camera, eye: eye, target: eye + direction, up: simd_float3(0, 1, 0)) + } + + private func makeBox(at position: simd_float3) -> EntityID { + let entity = createEntity() + setEntityMeshDirect(entityId: entity, meshes: BasicPrimitives.createCube(extent: 1.0), assetName: "Box") + translateTo(entityId: entity, position: position) + return entity + } + + /// One frame, through to its command buffer's completion: the frame's cull has published + /// its list by then. + private func drawFrame() { + renderer.draw(in: renderer.metalView) + renderInfo.lastCommandBuffer?.waitUntilCompleted() + } + + /// The list the last frame's cull published. + private var published: [EntityID] { + tripleVisibleEntities.snapshotForRead(frame: cullFrameIndex) + } + + func testTheEntitiesAfterOneThatLeftTheListAreStillTested() { + // Two boxes in view, the first ahead of the second in the list. + let first = makeBox(at: simd_float3(-1.5, 1, -5)) + let second = makeBox(at: simd_float3(1.5, 1, -5)) + for _ in 0 ..< 4 { + drawFrame() + } + XCTAssertTrue(published.contains(first)) + XCTAssertTrue(published.contains(second)) + + // The first leaves the list, and the second takes its place in it. The frame's cull + // must test the list as it is: the frame before's list under this frame's count tested + // the first again and never reached the second. + scene.get(component: RenderComponent.self, for: first)?.isVisible = false + drawFrame() + XCTAssertFalse(published.contains(first), "hidden") + XCTAssertTrue(published.contains(second), "tested in the frame its place in the list changed") + } + + func testTheFrameTheCameraTurnsInSeesWhatItTurnedTo() { + let box = makeBox(at: simd_float3(0, 1, -5)) + look(towards: simd_float3(0, 0, 1)) + for _ in 0 ..< 4 { + drawFrame() + } + XCTAssertFalse(published.contains(box), "behind the camera") + + look(towards: simd_float3(0, 0, -1)) + drawFrame() + XCTAssertTrue(published.contains(box), "in the frustum of the frame that turned, not the frame before's") + } +} diff --git a/Tests/UntoldEngineRenderTests/DefaultEntitiesTests.swift b/Tests/UntoldEngineRenderTests/DefaultEntitiesTests.swift new file mode 100644 index 000000000..aa08fe1ba --- /dev/null +++ b/Tests/UntoldEngineRenderTests/DefaultEntitiesTests.swift @@ -0,0 +1,43 @@ +// +// DefaultEntitiesTests.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import simd +@testable import UntoldEngine +import XCTest + +/// What a scene has before the app adds anything: a camera on every path, since the headset +/// writes the head pose into it, and a sun everywhere but the headset, where the passthrough +/// room is lit by its probe and the immersive environment by its own light. +@MainActor +final class DefaultEntitiesTests: BaseRenderSetup { + override func tearDown() async throws { + destroyAllEntities() + try await super.tearDown() + } + + func testTheHeadsetGetsACameraAndNoSun() { + destroyAllEntities() + LightingSystem.shared.activeDirectionalLight = nil + let made = renderer.createDefaultEntities(sun: false) + XCTAssertNil(made.sun) + XCTAssertEqual(CameraSystem.shared.activeCamera, made.camera) + XCTAssertNil(LightingSystem.shared.activeDirectionalLight) + XCTAssertEqual(getDirectionalLightParameters().intensity, 0.0, "no sun lights the scene") + } + + func testTheOtherPathsGetASunToo() { + destroyAllEntities() + LightingSystem.shared.activeDirectionalLight = nil + let made = renderer.createDefaultEntities(sun: true) + XCTAssertEqual(CameraSystem.shared.activeCamera, made.camera) + XCTAssertEqual(LightingSystem.shared.activeDirectionalLight, made.sun) + XCTAssertEqual(getDirectionalLightParameters().intensity, 1.0) + } +} diff --git a/Tests/UntoldEngineRenderTests/EngineStatsMonitorTests.swift b/Tests/UntoldEngineRenderTests/EngineStatsMonitorTests.swift index 88781030e..8ba16bd8c 100644 --- a/Tests/UntoldEngineRenderTests/EngineStatsMonitorTests.swift +++ b/Tests/UntoldEngineRenderTests/EngineStatsMonitorTests.swift @@ -8,6 +8,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +import QuartzCore @testable import UntoldEngine import XCTest @@ -177,6 +178,30 @@ final class EngineStatsMonitorTests: XCTestCase { XCTAssertEqual(snapshot.timing.gpuFrameCadenceMs, 5.0, accuracy: 0.001) } + // MARK: - GPU time beyond the previous frame's + + func testGPUTimeExcludingOverlap_countsOnlyTheTimeAfterThePreviousFrame() { + // Back to back: the frame began before the previous one ended. + XCTAssertEqual(gpuTimeExcludingOverlap(gpuStartTime: 0.008, gpuEndTime: 0.018, previousGPUEndTime: 0.010), 0.008, accuracy: 1e-9) + // Apart: the whole span. + XCTAssertEqual(gpuTimeExcludingOverlap(gpuStartTime: 0.020, gpuEndTime: 0.025, previousGPUEndTime: 0.018), 0.005, accuracy: 1e-9) + // Finished before the previous one: nothing. + XCTAssertEqual(gpuTimeExcludingOverlap(gpuStartTime: 0.001, gpuEndTime: 0.004, previousGPUEndTime: 0.010), 0, accuracy: 1e-9) + } + + func testRecordGPUCompletion_timestamps_excludeThePreviousFramesTail() { + let base = CACurrentMediaTime() + 1000 // past anything an earlier test recorded + EngineStatsMonitor.shared.beginFrame(timestampSeconds: 1.0) + EngineStatsMonitor.shared.recordGPUCompletion(gpuStartTime: base, gpuEndTime: base + 0.020) + EngineStatsMonitor.shared.completeFrame() + XCTAssertEqual(getEngineStatsSnapshot().timing.gpuExecutionMs, 20.0, accuracy: 0.01) + + EngineStatsMonitor.shared.beginFrame(timestampSeconds: 1.02) + EngineStatsMonitor.shared.recordGPUCompletion(gpuStartTime: base + 0.015, gpuEndTime: base + 0.035) + EngineStatsMonitor.shared.completeFrame() + XCTAssertEqual(getEngineStatsSnapshot().timing.gpuExecutionMs, 15.0, accuracy: 0.01, "the 5 ms the two frames shared on the GPU count once") + } + // MARK: - Formatter bottleneck label func testFormatEngineStats_cpuBoundLabel() { diff --git a/Tests/UntoldEngineRenderTests/FrameTimeLoopTests.swift b/Tests/UntoldEngineRenderTests/FrameTimeLoopTests.swift new file mode 100644 index 000000000..0064151db --- /dev/null +++ b/Tests/UntoldEngineRenderTests/FrameTimeLoopTests.swift @@ -0,0 +1,116 @@ +// +// FrameTimeLoopTests.swift +// UntoldEngineRenderTests +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import simd +@testable import UntoldEngine +import XCTest + +/// The frame's time reaches the game, the update event and the extensions from the loop, and +/// a pose the game samples at it is the pose for the frame's presentation instant, not for +/// the wall clock the update happens to run at. +final class FrameTimeLoopTests: BaseRenderSetup { + private final class TimeRecordingExtension: EngineExtension, @unchecked Sendable { + let id = "test.frameTime.recording" + var frames: [FrameTime] = [] + + func update(deltaTime _: Float, context: EngineExtensionUpdateContext) { + frames.append(context.frameTime) + } + } + + override func setUp() async throws { + try await super.setUp() + if CameraSystem.shared.activeCamera == nil { + let camera = createEntity() + createGameCamera(entityId: camera) + CameraSystem.shared.activeCamera = camera + } + gameMode = true + } + + override func tearDown() async throws { + EngineExtensionRegistry.shared.removeAll() + try await super.tearDown() + } + + func testAPoseSampledByTheGameMatchesTheFramesPresentationTime() throws { + // A timeline nowhere near the wall clock: a pose taken from CACurrentMediaTime() or + // from the time between two of these calls would be nowhere near the expected one. + let start: TimeInterval = 1000.0 + let period: TimeInterval = 0.010 + let metrePerSecond = TransformTrack(keys: [ + TransformKey(time: 0, position: simd_float3(0, 0, 0)), + TransformKey(time: 1, position: simd_float3(1, 0, 0)), + ]) + let entity = createEntity() + registerTransformComponent(entityId: entity) + + var frames: [FrameTime] = [] + renderer.setupCallbacks( + frameUpdate: { frame in + frames.append(frame) + applyTransformTrack(entityId: entity, track: metrePerSecond, startedAt: start, frame: frame) + }, + handleInput: {} + ) + var events: [FrameTime] = [] + let token = renderer.onUpdate { events.append($0.frameTime) } + defer { token.cancel() } + let recorder = TimeRecordingExtension() + EngineExtensionRegistry.shared.register(recorder) + + for index in 0 ..< 3 { + XCTAssertTrue(renderer.updateXR(presentationTime: start + Double(index) * period)) + XCTAssertEqual(getLocalPosition(entityId: entity).x, Float(index) * Float(period), accuracy: 1e-5) + } + + let presentations = [start, start + period, start + 2 * period] + XCTAssertEqual(frames.map(\.presentationTime), presentations) + XCTAssertEqual(frames.count, 3) + for (frame, delta) in zip(frames, [0, period, period]) { + XCTAssertEqual(frame.deltaTime, delta, accuracy: 1e-9) + } + XCTAssertEqual(frames.map(\.skippedFrames), [0, 0, 0]) + // The engine's delta is the presentation delta, not the microseconds between the calls. + XCTAssertEqual(timeSinceLastUpdate, Float(period), accuracy: 1e-6) + XCTAssertEqual(events.map(\.presentationTime), presentations) + XCTAssertEqual(recorder.frames.map(\.presentationTime), presentations) + XCTAssertEqual(renderer.frameTime, try XCTUnwrap(frames.last)) + XCTAssertEqual(currentFrameTime, try XCTUnwrap(frames.last)) + } + + func testTheGameUpdateDeltaIsThePresentationDeltaAndASkippedFrameIsReported() { + var deltas: [Float] = [] + renderer.setupCallbacks(gameUpdate: { deltas.append($0) }, handleInput: {}) + let start: TimeInterval = 2000.0 + let period: TimeInterval = 1.0 / 90.0 + XCTAssertTrue(renderer.updateXR(presentationTime: start)) + XCTAssertTrue(renderer.updateXR(presentationTime: start + period)) + // The display showed the second frame twice: the third is two periods on. + XCTAssertTrue(renderer.updateXR(presentationTime: start + 3 * period)) + + XCTAssertEqual(deltas.count, 3) + XCTAssertEqual(deltas[0], 0) + XCTAssertEqual(deltas[1], Float(period), accuracy: 1e-6) + XCTAssertEqual(deltas[2], Float(2 * period), accuracy: 1e-6) + XCTAssertEqual(renderer.frameTime.skippedFrames, 1) + XCTAssertEqual(renderer.frameTime.period, period, accuracy: 1e-9) + } + + func testAFrameWithoutAPredictionIsPosedOnePeriodAheadOfNow() { + let before = CACurrentMediaTime() + XCTAssertTrue(renderer.updateXR()) + let after = CACurrentMediaTime() + let period = UntoldRenderer.defaultFramePeriod + XCTAssertGreaterThanOrEqual(renderer.frameTime.presentationTime, before + period) + XCTAssertLessThanOrEqual(renderer.frameTime.presentationTime, after + period) + XCTAssertEqual(renderer.frameTime.period, period, accuracy: 1e-9) + } +} diff --git a/Tests/UntoldEngineRenderTests/GPUPassTimingTests.swift b/Tests/UntoldEngineRenderTests/GPUPassTimingTests.swift new file mode 100644 index 000000000..56c3908b6 --- /dev/null +++ b/Tests/UntoldEngineRenderTests/GPUPassTimingTests.swift @@ -0,0 +1,74 @@ +// +// GPUPassTimingTests.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Metal +import simd +@testable import UntoldEngine +import XCTest + +/// The GPU times its own passes: with the timing on, a frame's passes come back with their +/// vertex and fragment milliseconds, which add up to no more than the frame; off, nothing. +@MainActor +final class GPUPassTimingTests: BaseRenderSetup { + override func tearDown() async throws { + setRendering(.gpuPassTiming(.disabled)) + destroyAllEntities() + try await super.tearDown() + } + + override func initializeAssets() { + let camera = createEntity() + createGameCamera(entityId: camera) + CameraSystem.shared.activeCamera = camera + cameraLookAt(entityId: camera, eye: simd_float3(0, 1, 4), target: .zero, up: simd_float3(0, 1, 0)) + let box = createEntity() + setEntityMeshDirect(entityId: box, meshes: BasicPrimitives.createCube(extent: 1.0), assetName: "Box") + translateTo(entityId: box, position: simd_float3(0, 0.5, 0)) + let sun = createEntity() + createDirLight(entityId: sun) + rotateTo(entityId: sun, pitch: -40, yaw: 30, roll: 0) + applyIBL = true + } + + private func drawFrames(_ count: Int) -> Double { + var span = 0.0 + for _ in 0 ..< count { + renderer.draw(in: renderer.metalView) + guard let cb = renderInfo.lastCommandBuffer else { continue } + cb.waitUntilCompleted() + span = (cb.gpuEndTime - cb.gpuStartTime) * 1000 + } + return span + } + + func testPassesAreTimedWhenOnAndNotWhenOff() throws { + try XCTSkipUnless(renderInfo.device.supportsCounterSampling(.atStageBoundary), "no stage-boundary sampling here") + _ = drawFrames(3) + XCTAssertTrue(GPUPassTiming.shared.averages().isEmpty, "off by default") + + setRendering(.gpuPassTiming(.enabled)) + let frameMs = drawFrames(30) + let passes = GPUPassTiming.shared.averages() + XCTAssertFalse(passes.isEmpty, "the frame's passes are timed") + // The combined pass: the G-buffer's when the light pass lights, the forward one when the + // geometry lights itself (the default). + let gBuffer = try XCTUnwrap(passes.first { $0.label == "G-Buffer" || $0.label == "Forward" }, "the combined pass is timed: \(passes.map(\.label))") + XCTAssertGreaterThan(gBuffer.fragmentMs, 0, "the combined pass shades") + XCTAssertGreaterThanOrEqual(gBuffer.vertexMs, 0) + let total = passes.reduce(0) { $0 + $1.totalMs } + XCTAssertGreaterThan(total, 0) + XCTAssertLessThan(total, frameMs * 1.5 + 0.5, "the passes add up to no more than the frame (\(frameMs) ms): \(passes)") + print("GPU pass timing (Mac): " + passes.map { String(format: "%@ v%.3f f%.3f", $0.label, $0.vertexMs, $0.fragmentMs) }.joined(separator: ", ") + String(format: " | frame %.3f", frameMs)) + + setRendering(.gpuPassTiming(.disabled)) + _ = drawFrames(3) + XCTAssertTrue(GPUPassTiming.shared.averages().isEmpty, "off again") + } +} diff --git a/Tests/UntoldEngineRenderTests/GaussianChunkCullTest.swift b/Tests/UntoldEngineRenderTests/GaussianChunkCullTest.swift index 481e6b99c..82bf2b0c3 100644 --- a/Tests/UntoldEngineRenderTests/GaussianChunkCullTest.swift +++ b/Tests/UntoldEngineRenderTests/GaussianChunkCullTest.swift @@ -1095,7 +1095,7 @@ final class GaussianChunkCullTest: BaseRenderSetup { XCTAssertEqual(constants.hzbOcclusionBias, gaussianCullHZBOcclusionBias) XCTAssertEqual(constants.uniformQuotas, 0, "the weighted quotas are the default") XCTAssertEqual(gaussianChunkCullConstants(chunkTable: table, modelMatrix: world.space, viewMatrix: matrix_identity_float4x4, hzbValid: false, uniformQuotas: true).uniformQuotas, 1) - XCTAssertEqual(MemoryLayout.stride, 192) + XCTAssertEqual(MemoryLayout.stride, 208) XCTAssertEqual(MemoryLayout.stride, 16) XCTAssertEqual(MemoryLayout.stride, 48) XCTAssertEqual(MemoryLayout.stride, 48) diff --git a/Tests/UntoldEngineRenderTests/GaussianChunkLevelTest.swift b/Tests/UntoldEngineRenderTests/GaussianChunkLevelTest.swift index f80387fc7..ae60e1d9e 100644 --- a/Tests/UntoldEngineRenderTests/GaussianChunkLevelTest.swift +++ b/Tests/UntoldEngineRenderTests/GaussianChunkLevelTest.swift @@ -1888,7 +1888,7 @@ final class GaussianChunkLevelTest: BaseRenderSetup { XCTAssertEqual(MemoryLayout.stride, 16) XCTAssertEqual(MemoryLayout.offset(of: \.coarseAvailable), 12) XCTAssertEqual(MemoryLayout.stride, 32) - XCTAssertEqual(MemoryLayout.stride, 192) + XCTAssertEqual(MemoryLayout.stride, 208) XCTAssertEqual(MemoryLayout.stride, 48) XCTAssertEqual(MemoryLayout.stride, 64) XCTAssertEqual(MemoryLayout.stride, 48) diff --git a/Tests/UntoldEngineRenderTests/GaussianPagingTest.swift b/Tests/UntoldEngineRenderTests/GaussianPagingTest.swift index 158a67565..d26e253ac 100644 --- a/Tests/UntoldEngineRenderTests/GaussianPagingTest.swift +++ b/Tests/UntoldEngineRenderTests/GaussianPagingTest.swift @@ -1784,9 +1784,10 @@ final class GaussianPagingTest: BaseRenderSetup { XCTAssertEqual(MemoryLayout.offset(of: \.frameIndex), 8) XCTAssertEqual(MemoryLayout.offset(of: \.fadeFrames), 12) XCTAssertEqual(MemoryLayout.offset(of: \.debugMode), 16) - XCTAssertEqual(MemoryLayout.stride, 192) + XCTAssertEqual(MemoryLayout.stride, 208) XCTAssertEqual(MemoryLayout.offset(of: \.paged), 172) XCTAssertEqual(MemoryLayout.offset(of: \.rangeCount), 176) + XCTAssertEqual(MemoryLayout.offset(of: \.hzbRateMapSizes), 192) XCTAssertEqual(gaussianChunkCullResidencyIndex.rawValue, 8) XCTAssertEqual(gaussianChunkCullDemandIndex.rawValue, 9) XCTAssertEqual(gaussianChunkPreprocessResidencyIndex.rawValue, 13) diff --git a/Tests/UntoldEngineRenderTests/GaussianPassSkipTests.swift b/Tests/UntoldEngineRenderTests/GaussianPassSkipTests.swift new file mode 100644 index 000000000..fcfca4449 --- /dev/null +++ b/Tests/UntoldEngineRenderTests/GaussianPassSkipTests.swift @@ -0,0 +1,132 @@ +// +// GaussianPassSkipTests.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Metal +import simd +@testable import UntoldEngine +import XCTest + +/// A frame without a splat entity skips the Gaussian pass whole (the depth snapshot, the tile +/// pass, the clear of its layer), and the composite ignores whatever the layer holds from an +/// earlier frame. +@MainActor +final class GaussianPassSkipTests: BaseRenderSetup { + override func tearDown() async throws { + destroyAllEntities() + try await super.tearDown() + } + + override func initializeAssets() { + let camera = createEntity() + createGameCamera(entityId: camera) + CameraSystem.shared.activeCamera = camera + cameraLookAt(entityId: camera, eye: simd_float3(0, 1, 4), target: .zero, up: simd_float3(0, 1, 0)) + let box = createEntity() + setEntityMeshDirect(entityId: box, meshes: BasicPrimitives.createCube(extent: 1.0), assetName: "Box") + translateTo(entityId: box, position: simd_float3(0, 0.5, 0)) + } + + private func drawFrame() { + renderer.draw(in: renderer.metalView) + renderInfo.lastCommandBuffer?.waitUntilCompleted() + } + + /// Fills the Gaussian pass's layer, as a frame that drew splats would have left it. + private func fillSplatLayer(_ color: MTLClearColor) throws { + let layer = try XCTUnwrap(renderInfo.gaussianRenderPassDescriptor.colorAttachments[0].texture) + let descriptor = MTLRenderPassDescriptor() + descriptor.colorAttachments[0].texture = layer + descriptor.colorAttachments[0].loadAction = .clear + descriptor.colorAttachments[0].storeAction = .store + descriptor.colorAttachments[0].clearColor = color + let commandBuffer = try XCTUnwrap(renderInfo.commandQueue.makeCommandBuffer()) + try XCTUnwrap(commandBuffer.makeRenderCommandEncoder(descriptor: descriptor)).endEncoding() + commandBuffer.commit() + commandBuffer.waitUntilCompleted() + } + + private func readFloats(_ texture: MTLTexture) -> [Float] { + var halves = [UInt16](repeating: 0, count: texture.width * texture.height * 4) + halves.withUnsafeMutableBytes { bytes in + texture.getBytes(bytes.baseAddress!, bytesPerRow: texture.width * 8, from: MTLRegionMake2D(0, 0, texture.width, texture.height), mipmapLevel: 0) + } + return halves.map { Float(Float16(bitPattern: $0)) } + } + + func testFrameWithoutSplatEntitiesSkipsThePass() { + drawFrame() + XCTAssertFalse(renderInfo.gaussianCoverageWritten, "no splat entity: the pass is skipped and its layer is not read") + let slot = min(renderInfo.currentInFlightFrameSlot, maxInFlightCommandBuffers - 1) + XCTAssertTrue(GaussianSharedWorkingSet.shared.entityOrder(slot: slot).isEmpty, "the preprocess leaves the slot's order empty") + } + + func testCompositeIgnoresTheLayerOfAnEarlierFrame() throws { + // Two frames settle the frame-to-frame state (shadows, the depth pyramid) so that the + // two frames compared below differ in the layer alone. + drawFrame() + drawFrame() + + // Opaque red where a frame that drew splats left its layer: a skipped pass does not clear it. + try fillSplatLayer(MTLClearColor(red: 1, green: 0, blue: 0, alpha: 1)) + drawFrame() + XCTAssertFalse(renderInfo.gaussianCoverageWritten) + let withStaleLayer = try readFloats(XCTUnwrap(textureResources.sceneCompositeTexture)) + + try fillSplatLayer(MTLClearColor(red: 0, green: 0, blue: 0, alpha: 0)) + drawFrame() + let withEmptyLayer = try readFloats(XCTUnwrap(textureResources.sceneCompositeTexture)) + + XCTAssertGreaterThan(withEmptyLayer.max() ?? 0, 0.01, "the scene draws something") + var maxDifference: Float = 0 + for i in 0 ..< min(withStaleLayer.count, withEmptyLayer.count) { + maxDifference = max(maxDifference, abs(withStaleLayer[i] - withEmptyLayer[i])) + } + XCTAssertEqual(maxDifference, 0, accuracy: 1e-3, "the composite reads the layer only on a frame the pass drew it") + } +} + +/// A scene that had splats and loses them stops the pass. +@MainActor +final class GaussianPassSkipAfterRemovalTests: BaseRenderSetup { + override func tearDown() async throws { + destroyAllEntities() + try await super.tearDown() + } + + override func initializeAssets() { + let gaussian = createEntity() + setEntityGaussian(entityId: gaussian, filename: "test_gaussians", withExtension: "ply") + _ = placeGaussianTestCamera(eye: simd_float3(0, 0, 3)) + } + + private func drawFrame() { + renderer.draw(in: renderer.metalView) + renderInfo.lastCommandBuffer?.waitUntilCompleted() + } + + func testRemovingTheLastSplatEntityStopsThePass() { + for _ in 0 ..< maxInFlightCommandBuffers { + drawFrame() + } + XCTAssertTrue(renderInfo.gaussianCoverageWritten, "with a splat entity the pass draws its layer") + + let gaussianId = getComponentId(for: GaussianComponent.self) + for entity in queryEntitiesWithComponentIds([gaussianId], in: scene) { + destroyEntity(entityId: entity) + } + for _ in 0 ..< maxInFlightCommandBuffers { + drawFrame() + } + XCTAssertFalse(renderInfo.gaussianCoverageWritten, "without one the pass is skipped") + for slot in 0 ..< maxInFlightCommandBuffers { + XCTAssertTrue(GaussianSharedWorkingSet.shared.entityOrder(slot: slot).isEmpty, "slot \(slot): the preprocess cleared the order its records were stamped with") + } + } +} diff --git a/Tests/UntoldEngineRenderTests/GaussianScreenWeightedQuotaTest.swift b/Tests/UntoldEngineRenderTests/GaussianScreenWeightedQuotaTest.swift index 953c6758c..3e61dce6f 100644 --- a/Tests/UntoldEngineRenderTests/GaussianScreenWeightedQuotaTest.swift +++ b/Tests/UntoldEngineRenderTests/GaussianScreenWeightedQuotaTest.swift @@ -320,7 +320,8 @@ final class GaussianScreenWeightedQuotaTest: BaseRenderSetup { /// The shared layouts the kernels and the readbacks depend on. func testLayoutsArePinned() { - XCTAssertEqual(MemoryLayout.stride, 192) + XCTAssertEqual(MemoryLayout.stride, 208) + XCTAssertEqual(MemoryLayout.offset(of: \.hzbRateMapSizes), 192) XCTAssertEqual(MemoryLayout.stride, 16) XCTAssertEqual(MemoryLayout.stride, 48) XCTAssertEqual(MemoryLayout.stride, 48) diff --git a/Tests/UntoldEngineRenderTests/GlassShadingTests.swift b/Tests/UntoldEngineRenderTests/GlassShadingTests.swift index b4f054257..8b3bd6882 100644 --- a/Tests/UntoldEngineRenderTests/GlassShadingTests.swift +++ b/Tests/UntoldEngineRenderTests/GlassShadingTests.swift @@ -105,6 +105,91 @@ final class GlassShadingTests: MaterialShadingTestCase { /// near one: what is behind it crosses both. private static let throughBothFaces: Float = (1.0 - reflectedHeadOn) * (1.0 - reflectedHeadOn) + /// A solid, glowing block whose near face lies in the same plane as the shape's near + /// face, over the middle of the frame: the way a pane is modelled into its frame. + @discardableResult + private func putASolidBlockInThePlaneOfTheNearFace(glowing color: simd_float3) throws -> EntityID { + let block = createEntity() + var meshes = BasicPrimitives.createCube(extent: 1.0) + let glow = Material( + runtimeMaterial: RuntimeMaterialSource( + baseColorFactor: simd_float4(0, 0, 0, 1), + emissiveFactor: color, + metallicFactor: 0.0, + roughnessFactor: 1.0 + ), + device: renderInfo.device + ) + for meshIndex in meshes.indices { + for submeshIndex in meshes[meshIndex].submeshes.indices { + meshes[meshIndex].submeshes[submeshIndex].material = glow + } + } + let renderComponent = try XCTUnwrap(scene.assign(to: block, component: RenderComponent.self)) + renderComponent.mesh = meshes + renderComponent.assetURL = URL(fileURLWithPath: "/dev/null/glass-shading-block.untold") + if let local = scene.get(component: LocalTransformComponent.self, for: block) { + local.boundingBox = Mesh.computeMeshBoundingBox(for: meshes) + } + // The shape is a cube of extent 3 at the origin: its near face is at z = 1.5. The + // block, of extent 1, is centred at z = 1 so that its near face is at z = 1.5 too. + translateTo(entityId: block, position: simd_float3(0, 0, 1)) + setVisibleEntities() + return block + } + + // MARK: - A pane in its frame + + /// A pane modelled into its frame shares a plane with the frame. The two used to + /// fight for depth there, pixel by pixel and frame by frame, and the pane's edge + /// flickered at every move of the camera. Glass now draws a hair behind where it is, + /// so where the two coincide the frame wins, everywhere and every time. + func testAPaneInThePlaneOfASolidSurfaceShowsTheSolidSurface() throws { + try buildScene(.cube, towardsLight: nil) + let glow = simd_float3(0.2, 0.6, 0.2) + try putASolidBlockInThePlaneOfTheNearFace(glowing: glow) + try putAWallBehind(glowing: simd_float3(1, 1, 1)) + + let frame = try shadeFrame(material(transmission: 1.0)) + let window = 24 + let middle = Self.meanColor(of: frame, aroundX: frame.width / 2, y: frame.height / 2, window: window) + XCTAssertEqual(middle.x, glow.x, accuracy: 0.02, "the block, not the pane over it: red") + XCTAssertEqual(middle.y, glow.y, accuracy: 0.02, "the block, not the pane over it: green") + XCTAssertEqual(middle.z, glow.z, accuracy: 0.02, "the block, not the pane over it: blue") + + // Not one pixel of the window shows the pane instead of the block. + var furthest: Float = 0 + for row in (frame.height / 2 - window / 2) ..< (frame.height / 2 + window / 2) { + for column in (frame.width / 2 - window / 2) ..< (frame.width / 2 + window / 2) { + let pixel = frame.pixels[row * frame.width + column] + furthest = max(furthest, simd_length(pixel - glow)) + } + } + XCTAssertLessThan(furthest, 0.02, "every pixel of the block's face is the block's own") + } + + // MARK: - Depth in a headset + + /// A headset's compositor reprojects every pixel by the depth it finds. A pane that + /// mostly shows itself writes its own depth, so its tint and reflections stay on the + /// pane; a clear pane leaves the depth of what is seen through it, which is what the + /// eye follows. The line is drawn at half the light crossing the pane. + func testAPaneWritesItsDepthInAHeadsetWhenItMostlyShowsItself() { + func showsItself(_ source: RuntimeMaterialSource) -> Bool { + Material(runtimeMaterial: source, device: renderInfo.device).paneShowsItself + } + // Clear window glass: nearly all the light crosses it. + XCTAssertFalse(showsItself(material(color: simd_float3(0.91, 0.97, 0.96), transmission: 1.0, roughness: 0.035))) + // Smoked car glass: a dark base colour lets little through. + XCTAssertTrue(showsItself(material(color: simd_float3(0.03, 0.05, 0.06), transmission: 1.0, roughness: 0.09, metallic: 0.5))) + // Frosted glass: too rough to see through, it glows instead. + XCTAssertTrue(showsItself(material(color: simd_float3(0.9, 0.9, 0.9), transmission: 1.0, roughness: 0.6))) + // A pane half there by its alpha lets at least half through whatever its tint. + XCTAssertFalse(showsItself(material(color: simd_float3(0.03, 0.05, 0.06), transmission: 1.0, roughness: 0.09, alpha: 0.5))) + // Light grey glass with a transmission of a half: half of what crosses is kept. + XCTAssertTrue(showsItself(material(color: simd_float3(0.5, 0.5, 0.5), transmission: 0.5, roughness: 0.0))) + } + // MARK: - What shows through /// Glass was drawn as a blended surface 10 % opaque. A transmissive material now diff --git a/Tests/UntoldEngineRenderTests/HZBOcclusionReprojectionTests.swift b/Tests/UntoldEngineRenderTests/HZBOcclusionReprojectionTests.swift index 64146fd51..27ee8a478 100644 --- a/Tests/UntoldEngineRenderTests/HZBOcclusionReprojectionTests.swift +++ b/Tests/UntoldEngineRenderTests/HZBOcclusionReprojectionTests.swift @@ -151,8 +151,14 @@ final class HZBOcclusionReprojectionTests: BaseRenderSetup { } func testThePyramidRecordsTheCameraItWasBuiltFrom() { + // The pyramid is built only for a frame whose depth could hide something: two mesh + // entities in the culling set, counted by the frame's cull before the pyramid is built. + // The scene is this case's own; another case's count must not stand in for it. + _ = makeBox(size: simd_float3(1, 1, 1), at: simd_float3(0, 2, -4)) + _ = makeBox(size: simd_float3(1, 1, 1), at: simd_float3(2, 2, -6)) look(from: simd_float3(1, 2, 3)) drawFrame() + drawFrame() let frame = renderInfo.hzbFrame XCTAssertNotNil(frame) XCTAssertEqual(frame?.cameraPosition.x ?? 0, 1, accuracy: 1e-4) diff --git a/Tests/UntoldEngineRenderTests/HZBPyramidGateTests.swift b/Tests/UntoldEngineRenderTests/HZBPyramidGateTests.swift new file mode 100644 index 000000000..24bd9fff7 --- /dev/null +++ b/Tests/UntoldEngineRenderTests/HZBPyramidGateTests.swift @@ -0,0 +1,126 @@ +// +// HZBPyramidGateTests.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Metal +import simd +@testable import UntoldEngine +import XCTest + +/// The depth pyramid, and the depth copy that feeds it, are built only for a frame whose depth +/// could hide something next frame: the occlusion test on, and enough objects (entities with +/// their children, or any splats) in the scene. The GPU's own pass timing says which passes +/// ran. +@MainActor +final class HZBPyramidGateTests: BaseRenderSetup { + private var savedMinimum = 2 + private var savedStep: Float = 0.5 + + override func setUp() async throws { + try await super.setUp() + savedMinimum = getOcclusionCullingMinimumEntities() + savedStep = getOcclusionCullingMaxCameraStep() + setRendering(.gpuPassTiming(.enabled)) + } + + override func tearDown() async throws { + setRendering(.gpuPassTiming(.disabled)) + setRendering(.occlusionCullingMinimumEntities(savedMinimum)) + setRendering(.occlusionCullingMaxCameraStep(savedStep)) + destroyAllEntities() + try await super.tearDown() + } + + override func initializeAssets() { + let camera = createEntity() + createGameCamera(entityId: camera) + CameraSystem.shared.activeCamera = camera + cameraLookAt(entityId: camera, eye: simd_float3(0, 1, 4), target: .zero, up: simd_float3(0, 1, 0)) + addBox(at: simd_float3(0, 0.5, 0)) + let sun = createEntity() + createDirLight(entityId: sun) + rotateTo(entityId: sun, pitch: -40, yaw: 30, roll: 0) + } + + private func addBox(at position: simd_float3) { + let box = createEntity() + setEntityMeshDirect(entityId: box, meshes: BasicPrimitives.createCube(extent: 0.6), assetName: "Box") + translateTo(entityId: box, position: position) + } + + /// The labels of the passes the last frame ran, after a few frames. + private func passesRun() -> Set { + for _ in 0 ..< 4 { + renderer.draw(in: renderer.metalView) + renderInfo.lastCommandBuffer?.waitUntilCompleted() + } + return Set(GPUPassTiming.shared.lastFrame().passes.map(\.label)) + } + + func testOneEntityGetsNoPyramid() throws { + try XCTSkipUnless(renderInfo.device.supportsCounterSampling(.atStageBoundary), "no pass timing here") + setRendering(.occlusionCullingMinimumEntities(2)) + setRendering(.occlusionCullingMaxCameraStep(0.5)) + let passes = passesRun() + XCTAssertEqual(renderInfo.hzbCandidateCount, 1) + XCTAssertFalse(passes.contains("HZB"), "no pyramid for one entity: \(passes.sorted())") + XCTAssertFalse(passes.contains("DepthCopy"), "no depth copy without a pyramid") + XCTAssertFalse(renderInfo.hzbIsValid) + } + + func testTwoEntitiesGetThePyramidUnlessTheTestIsOff() throws { + try XCTSkipUnless(renderInfo.device.supportsCounterSampling(.atStageBoundary), "no pass timing here") + addBox(at: simd_float3(1.2, 0.5, -1)) + setVisibleEntities() + setRendering(.occlusionCullingMinimumEntities(2)) + setRendering(.occlusionCullingMaxCameraStep(0.5)) + var passes = passesRun() + XCTAssertEqual(renderInfo.hzbCandidateCount, 2) + XCTAssertTrue(passes.contains("HZB"), "two entities: the pyramid is built: \(passes.sorted())") + XCTAssertTrue(passes.contains("DepthCopy")) + XCTAssertTrue(renderInfo.hzbIsValid) + + setRendering(.occlusionCullingMinimumEntities(5)) + passes = passesRun() + XCTAssertFalse(passes.contains("HZB"), "below the minimum: no pyramid") + XCTAssertFalse(renderInfo.hzbIsValid) + + setRendering(.occlusionCullingMinimumEntities(2)) + setRendering(.occlusionCullingMaxCameraStep(0)) + passes = passesRun() + XCTAssertFalse(passes.contains("HZB"), "the test off: no pyramid") + XCTAssertFalse(passes.contains("DepthCopy")) + } + + func testTheModelsPartsCountAsOneObject() throws { + try XCTSkipUnless(renderInfo.device.supportsCounterSampling(.atStageBoundary), "no pass timing here") + // A model loaded from a file is one entity per node under the entity it was set on: + // its parts hide nothing but themselves, so they count as one object, like the box. + let model = createEntity() + for part in 0 ..< 3 { + let entity = createEntity() + setEntityMeshDirect(entityId: entity, meshes: BasicPrimitives.createCube(extent: 0.3), assetName: "Part\(part)") + setParent(childId: entity, parentId: model, offset: simd_float3(0.8 + Float(part) * 0.4, 0.5, -1)) + } + setVisibleEntities() + setRendering(.occlusionCullingMinimumEntities(2)) + setRendering(.occlusionCullingMaxCameraStep(0.5)) + var passes = passesRun() + XCTAssertEqual(renderInfo.hzbCandidateCount, 2, "the box and the model, not the box and three parts") + XCTAssertTrue(passes.contains("HZB"), "two objects: the pyramid is built: \(passes.sorted())") + + // Three parts and a box are four entities but two objects: below a minimum of three. + setRendering(.occlusionCullingMinimumEntities(3)) + passes = passesRun() + XCTAssertEqual(renderInfo.hzbCandidateCount, 2) + XCTAssertFalse(passes.contains("HZB"), "two objects below a minimum of three: no pyramid: \(passes.sorted())") + XCTAssertFalse(passes.contains("DepthCopy")) + XCTAssertFalse(renderInfo.hzbIsValid) + } +} diff --git a/Tests/UntoldEngineRenderTests/HeightMaterialAPITests.swift b/Tests/UntoldEngineRenderTests/HeightMaterialAPITests.swift index aa3f5344e..945c3d642 100644 --- a/Tests/UntoldEngineRenderTests/HeightMaterialAPITests.swift +++ b/Tests/UntoldEngineRenderTests/HeightMaterialAPITests.swift @@ -38,7 +38,7 @@ final class HeightMaterialAPITests: BaseRenderSetup { XCTAssertEqual(getMaterialHeightMidlevel(entityId: entity), 0.5, accuracy: 0.0001) XCTAssertEqual(getMaterialHeightRemapMin(entityId: entity), 0.0, accuracy: 0.0001) XCTAssertEqual(getMaterialHeightRemapMax(entityId: entity), 1.0, accuracy: 0.0001) - XCTAssertTrue(getMaterialHeightEnabled(entityId: entity), "POM should be enabled by default") + XCTAssertFalse(getMaterialHeightEnabled(entityId: entity), "POM should be opt-in and disabled by default (issue #1360)") } // MARK: - Round trips diff --git a/Tests/UntoldEngineRenderTests/HybridRenderPathTests.swift b/Tests/UntoldEngineRenderTests/HybridRenderPathTests.swift new file mode 100644 index 000000000..dc8eb9247 --- /dev/null +++ b/Tests/UntoldEngineRenderTests/HybridRenderPathTests.swift @@ -0,0 +1,165 @@ +// +// HybridRenderPathTests.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Metal +import simd +@testable import UntoldEngine +import XCTest + +/// The hybrid path, drawn: the geometry carries the image-based light and the sun's shadow in +/// the G-buffer and the light quad adds the direct lights once per sample. Its picture is the +/// deferred path's and the forward path's; the quad still draws, the tile light stage stands +/// aside, and a point light is counted once โ€” by the quad, not by the geometry as well. +@MainActor +final class HybridRenderPathTests: BaseRenderSetup { + private var savedMode: AntiAliasingMode = .fxaa + private var savedPath: RenderPath = .deferred + private var pointLight: EntityID = .max + + override func setUp() async throws { + try await super.setUp() + savedMode = antiAliasingMode + savedPath = getRenderPath() + antiAliasingMode = .none + } + + override func tearDown() async throws { + setRendering(.renderPath(savedPath)) + setRendering(.multisampleCount(4)) + antiAliasingMode = savedMode + destroyAllEntities() + try await super.tearDown() + } + + override func initializeAssets() { + let camera = createEntity() + createGameCamera(entityId: camera) + CameraSystem.shared.activeCamera = camera + cameraLookAt(entityId: camera, eye: simd_float3(0, 2, 5), target: simd_float3(0, 0.3, 0), up: simd_float3(0, 1, 0)) + // A box over a slab under the sun and a point light: the sun's shadow falls on the slab, + // the point light reaches the box's near side, and the indirect light is everywhere. + let box = createEntity() + setEntityMeshDirect(entityId: box, meshes: BasicPrimitives.createCube(extent: 1.0), assetName: "Box") + translateTo(entityId: box, position: simd_float3(0, 0.5, 0)) + let slab = createEntity() + setEntityMeshDirect(entityId: slab, meshes: BasicPrimitives.createCube(extent: 1.0), assetName: "Slab") + scaleTo(entityId: slab, scale: simd_float3(6, 0.1, 6)) + translateTo(entityId: slab, position: simd_float3(0, -0.05, 0)) + let sun = createEntity() + createDirLight(entityId: sun) + rotateTo(entityId: sun, pitch: -55, yaw: 30, roll: 0) + pointLight = createEntity() + createPointLight(entityId: pointLight) + translateTo(entityId: pointLight, position: simd_float3(1.2, 1.2, 1.5)) + applyIBL = true + } + + private func drawFrames(_ count: Int = 3) { + for _ in 0 ..< count { + renderer.draw(in: renderer.metalView) + renderInfo.lastCommandBuffer?.waitUntilCompleted() + } + } + + /// The lit scene colour of a frame on the path, the pipelines rebuilt for it. + private func frame(on path: RenderPath) throws -> [Float] { + setRendering(.renderPath(path)) + drawFrames() + XCTAssertEqual(getRenderPath(), path) + XCTAssertEqual(renderInfo.perFragmentIndirectLightingBuilt, path != .deferred, "the pipelines follow the path") + XCTAssertEqual(renderInfo.perFragmentLightingBuilt, path == .forward, "the pipelines follow the path") + let texture = try XCTUnwrap(textureResources.deferredColorMap) + var halves = [UInt16](repeating: 0, count: texture.width * texture.height * 4) + halves.withUnsafeMutableBytes { bytes in + texture.getBytes(bytes.baseAddress!, bytesPerRow: texture.width * 8, from: MTLRegionMake2D(0, 0, texture.width, texture.height), mipmapLevel: 0) + } + return halves.map { Float(Float16(bitPattern: $0)) } + } + + private struct Difference { + let mean: Float + let max: Float + let lit: Int + } + + private func compare(_ a: [Float], _ b: [Float]) -> Difference { + var sum: Float = 0, maxDifference: Float = 0, lit = 0 + for i in stride(from: 0, to: a.count, by: 4) { + for c in 0 ..< 3 { + let d = abs(a[i + c] - b[i + c]) + sum += d + maxDifference = max(maxDifference, d) + } + if a[i] + a[i + 1] + a[i + 2] > 0.02 { lit += 1 } + } + return Difference(mean: sum / Float(a.count / 4 * 3), max: maxDifference, lit: lit) + } + + func testTheSamePictureAsDeferredAtOneSample() throws { + let deferred = try frame(on: .deferred) + let hybrid = try frame(on: .hybrid) + let result = compare(deferred, hybrid) + XCTAssertGreaterThan(result.lit, 1000, "the scene is lit") + XCTAssertLessThan(result.mean, 0.002, "mean difference per channel") + XCTAssertLessThan(result.max, 0.06, "the material's 8-bit roughness and metallic in the G-buffer, which the deferred light pass read the indirect light from, bound the per-pixel difference") + print(String(format: "Hybrid against deferred at 1 sample: mean difference %.5f, max %.4f over %d lit pixels", result.mean, result.max, result.lit)) + } + + func testTheSamePictureAsDeferredUnderMSAA() throws { + try XCTSkipUnless(renderInfo.device.supportsTextureSampleCount(4), "no 4ร— MSAA here") + antiAliasingMode = .msaa + setRendering(.multisampleCount(4)) + let deferred = try frame(on: .deferred) + XCTAssertEqual(renderInfo.opaqueSampleCount, 4) + let hybrid = try frame(on: .hybrid) + let result = compare(deferred, hybrid) + XCTAssertGreaterThan(result.lit, 1000, "the scene is lit") + XCTAssertLessThan(result.mean, 0.003, "mean difference per channel: only edge pixels differ, the indirect light evaluated once per pixel instead of once per sample") + print(String(format: "Hybrid against deferred at 4 samples: mean difference %.5f, max %.4f over %d lit pixels", result.mean, result.max, result.lit)) + } + + func testTheSamePictureAsForward() throws { + let hybrid = try frame(on: .hybrid) + let forward = try frame(on: .forward) + let result = compare(hybrid, forward) + XCTAssertGreaterThan(result.lit, 1000, "the scene is lit") + XCTAssertLessThan(result.mean, 0.0005, "mean difference per channel") + XCTAssertLessThan(result.max, 0.02, "the G-buffer's half-precision records, which the quad lit the direct part from, bound the per-pixel difference") + print(String(format: "Hybrid against forward at 1 sample: mean difference %.5f, max %.4f over %d lit pixels", result.mean, result.max, result.lit)) + } + + func testTheQuadDrawsAndTheTileStageStandsAside() throws { + _ = try frame(on: .forward) + let forwardDraws = RenderStatsCollector.shared.snapshot().drawCallsTotal + _ = try frame(on: .hybrid) + let hybridDraws = RenderStatsCollector.shared.snapshot().drawCallsTotal + XCTAssertEqual(hybridDraws, forwardDraws + 1, "the light quad's draw is back: the direct lights are its") + XCTAssertFalse(renderInfo.tileLightStageUsed, "the tile light stage would light the indirect part a second time") + } + + func testThePointLightIsCountedOnce() throws { + // The point light's own contribution on each path: the frame with it less the frame + // without. The quad adds the light once per sample on both; were the geometry adding it + // as well on the hybrid path, the contribution would double. + let hybridWith = try frame(on: .hybrid) + let deferredWith = try frame(on: .deferred) + destroyEntity(entityId: pointLight) + let deferredWithout = try frame(on: .deferred) + let hybridWithout = try frame(on: .hybrid) + let hybridLight = zip(hybridWith, hybridWithout).map { $0 - $1 } + let deferredLight = zip(deferredWith, deferredWithout).map { $0 - $1 } + let present = compare(hybridLight, [Float](repeating: 0, count: hybridLight.count)) + XCTAssertGreaterThan(present.max, 0.05, "the point light lit the box's near side (max contribution \(present.max))") + let once = compare(hybridLight, deferredLight) + XCTAssertLessThan(once.max, 0.03, "the light's contribution is the deferred path's, not twice it (max difference \(once.max) against a contribution of \(present.max))") + XCTAssertLessThan(once.mean, 0.0005, "mean difference per channel") + print(String(format: "Point light on hybrid against deferred: contribution max %.4f, difference mean %.5f, max %.4f", present.max, once.mean, once.max)) + } +} diff --git a/Tests/UntoldEngineRenderTests/LookPassDepthTests.swift b/Tests/UntoldEngineRenderTests/LookPassDepthTests.swift new file mode 100644 index 000000000..c8070bcab --- /dev/null +++ b/Tests/UntoldEngineRenderTests/LookPassDepthTests.swift @@ -0,0 +1,62 @@ +// +// LookPassDepthTests.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Metal +import simd +@testable import UntoldEngine +import XCTest + +/// The look pass draws into its own descriptor, colour alone: it never tests or writes +/// depth, and the frame's depth loaded and stored through it was a full-size read and write +/// per eye for nothing. The other post-process passes keep the frame's depth. +@MainActor +final class LookPassDepthTests: BaseRenderSetup { + override func tearDown() async throws { + destroyAllEntities() + try await super.tearDown() + } + + override func initializeAssets() { + let camera = createEntity() + createGameCamera(entityId: camera) + CameraSystem.shared.activeCamera = camera + cameraLookAt(entityId: camera, eye: simd_float3(0, 1, 4), target: .zero, up: simd_float3(0, 1, 0)) + let box = createEntity() + setEntityMeshDirect(entityId: box, meshes: BasicPrimitives.createCube(extent: 1.0), assetName: "Box") + translateTo(entityId: box, position: simd_float3(0, 0.5, 0)) + let sun = createEntity() + createDirLight(entityId: sun) + rotateTo(entityId: sun, pitch: -40, yaw: 30, roll: 0) + } + + private func readFloats(_ texture: MTLTexture) -> [Float] { + var halves = [UInt16](repeating: 0, count: texture.width * texture.height * 4) + halves.withUnsafeMutableBytes { bytes in + texture.getBytes(bytes.baseAddress!, bytesPerRow: texture.width * 8, from: MTLRegionMake2D(0, 0, texture.width, texture.height), mipmapLevel: 0) + } + return halves.map { Float(Float16(bitPattern: $0)) } + } + + func testTheLookPassCarriesNoDepthAttachment() throws { + let look = try XCTUnwrap(renderInfo.lookRenderPassDescriptor) + XCTAssertNil(look.depthAttachment.texture, "the look pass has no depth to load or store") + let post = try XCTUnwrap(renderInfo.postProcessRenderPassDescriptor) + XCTAssertNotNil(post.depthAttachment.texture, "the other post-process passes keep the frame's depth") + + for _ in 0 ..< 3 { + renderer.draw(in: renderer.metalView) + renderInfo.lastCommandBuffer?.waitUntilCompleted() + } + let output = try XCTUnwrap(textureResources.lookTexture) + XCTAssertTrue(output.usage.contains(.renderTarget)) + XCTAssertFalse(output.usage.contains(.shaderWrite), "no shader write: the GPU may compress it") + XCTAssertTrue(readFloats(output).contains { $0 > 0.01 }, "the look pass still wrote the lit frame") + } +} diff --git a/Tests/UntoldEngineRenderTests/LookPresentFoldTests.swift b/Tests/UntoldEngineRenderTests/LookPresentFoldTests.swift new file mode 100644 index 000000000..fd8226df1 --- /dev/null +++ b/Tests/UntoldEngineRenderTests/LookPresentFoldTests.swift @@ -0,0 +1,124 @@ +// +// LookPresentFoldTests.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Metal +import simd +@testable import UntoldEngine +import XCTest + +/// When nothing sits between the temporal resolve and the output in the frame's graph, the +/// resolve presents the frame itself โ€” the display's colour, encoded, and the opaque depth โ€” +/// and the output transform pass stands down. The display gets what the output pass would +/// have written from the resolve. +@MainActor +final class LookPresentFoldTests: BaseRenderSetup { + private var savedMode: AntiAliasingMode = .fxaa + + override func setUp() async throws { + try await super.setUp() + savedMode = antiAliasingMode + // The view's drawable is the display here; readable, to check what was presented. + renderer.metalView.framebufferOnly = false + setRendering(.gpuPassTiming(.enabled)) + } + + override func tearDown() async throws { + setRendering(.gpuPassTiming(.disabled)) + antiAliasingMode = savedMode + destroyAllEntities() + try await super.tearDown() + } + + override func initializeAssets() { + let camera = createEntity() + createGameCamera(entityId: camera) + CameraSystem.shared.activeCamera = camera + cameraLookAt(entityId: camera, eye: simd_float3(0, 1, 4), target: .zero, up: simd_float3(0, 1, 0)) + let box = createEntity() + setEntityMeshDirect(entityId: box, meshes: BasicPrimitives.createCube(extent: 1.0), assetName: "Box") + translateTo(entityId: box, position: simd_float3(0, 0.5, 0)) + let sun = createEntity() + createDirLight(entityId: sun) + rotateTo(entityId: sun, pitch: -40, yaw: 30, roll: 0) + applyIBL = true + } + + private func drawFrames(_ count: Int) { + for _ in 0 ..< count { + renderer.draw(in: renderer.metalView) + renderInfo.lastCommandBuffer?.waitUntilCompleted() + } + } + + private func linearToSRGB(_ x: Float) -> Float { + x <= 0.0031308 ? x * 12.92 : 1.055 * powf(max(x, 0), 1 / 2.4) - 0.055 + } + + func testTheLookPassPresentsAndTheOutputPassStandsDown() throws { + try XCTSkipUnless(renderInfo.device.supportsCounterSampling(.atStageBoundary), "no pass timing here") + antiAliasingMode = .msaa + drawFrames(4) + let presented = try XCTUnwrap(renderInfo.renderPassDescriptor?.colorAttachments[0].texture, "the view has a drawable") + try XCTSkipUnless(presented.pixelFormat == .bgra8Unorm_srgb || presented.pixelFormat == .bgra8Unorm, "unexpected display format \(presented.pixelFormat.rawValue)") + XCTAssertTrue(renderInfo.lookMayPresent, "nothing between the look pass and the output") + XCTAssertTrue(renderInfo.lookPresentedThisFrame, "the look pass presented") + let passes = Set(GPUPassTiming.shared.lastFrame().passes.map(\.label)) + XCTAssertTrue(passes.contains("Post")) + XCTAssertFalse(passes.contains("Output"), "the output transform pass stood down: \(passes.sorted())") + + // The display holds the look output, encoded as the output pass encodes it. + let resolved = try XCTUnwrap(textureResources.lookTexture) + let shared = try XCTUnwrap(renderInfo.device.makeTexture(descriptor: { + let d = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: resolved.pixelFormat, width: resolved.width, height: resolved.height, mipmapped: false) + d.storageMode = .shared + d.usage = [.shaderRead] + return d + }())) + let copy = try XCTUnwrap(renderInfo.commandQueue.makeCommandBuffer()) + let blit = try XCTUnwrap(copy.makeBlitCommandEncoder()) + blit.copy(from: resolved, to: shared) + blit.endEncoding() + copy.commit() + copy.waitUntilCompleted() + var halves = [UInt16](repeating: 0, count: shared.width * shared.height * 4) + halves.withUnsafeMutableBytes { bytes in + shared.getBytes(bytes.baseAddress!, bytesPerRow: shared.width * 8, from: MTLRegionMake2D(0, 0, shared.width, shared.height), mipmapLevel: 0) + } + var bytes = [UInt8](repeating: 0, count: presented.width * presented.height * 4) + bytes.withUnsafeMutableBytes { buffer in + presented.getBytes(buffer.baseAddress!, bytesPerRow: presented.width * 4, from: MTLRegionMake2D(0, 0, presented.width, presented.height), mipmapLevel: 0) + } + // The pass encodes when the pipeline says so; an sRGB display format encodes in hardware. + let encodes = renderInfo.colorPipeline.present.encodingMode.rawValue == 1 || presented.pixelFormat == .bgra8Unorm_srgb + var checked = 0 + for (x, y) in [(presented.width / 2, presented.height / 2), (presented.width / 3, presented.height * 2 / 3), (presented.width * 2 / 3, presented.height / 3)] { + let i = y * presented.width + x + for (channel, byteIndex) in [(0, 2), (1, 1), (2, 0)] { // bgra + var value = Float(Float16(bitPattern: halves[i * 4 + channel])) + value = max(value, 0) + if encodes { value = linearToSRGB(value) } + let expected = min(max(value, 0), 1) * 255 + XCTAssertEqual(Float(bytes[i * 4 + byteIndex]), expected, accuracy: 2.5, "pixel (\(x), \(y)) channel \(channel)") + checked += 1 + } + } + XCTAssertEqual(checked, 9) + } + + func testNoFoldWithAPassAfterTheLook() throws { + try XCTSkipUnless(renderInfo.device.supportsCounterSampling(.atStageBoundary), "no pass timing here") + antiAliasingMode = .fxaa + drawFrames(3) + XCTAssertFalse(renderInfo.lookMayPresent) + XCTAssertFalse(renderInfo.lookPresentedThisFrame) + let passes = Set(GPUPassTiming.shared.lastFrame().passes.map(\.label)) + XCTAssertTrue(passes.contains("Output"), "the output transform pass presents: \(passes.sorted())") + } +} diff --git a/Tests/UntoldEngineRenderTests/MSAARenderingTests.swift b/Tests/UntoldEngineRenderTests/MSAARenderingTests.swift index d56392ee7..9952478f9 100644 --- a/Tests/UntoldEngineRenderTests/MSAARenderingTests.swift +++ b/Tests/UntoldEngineRenderTests/MSAARenderingTests.swift @@ -37,6 +37,9 @@ final class MSAARenderingTests: BaseRenderSetup { XCTAssertEqual(modelPipeline.rasterSampleCount, expectedSampleCount, "Model pipeline sample count mismatch", file: file, line: line) XCTAssertEqual(lightPipeline.rasterSampleCount, expectedSampleCount, "Light pipeline sample count mismatch", file: file, line: line) + if let tilePipeline = PipelineManager.shared.renderPipelinesByType[.lightTile] { + XCTAssertEqual(tilePipeline.rasterSampleCount, expectedSampleCount, "Light tile pipeline sample count mismatch", file: file, line: line) + } } private func assertOpaqueRenderTargetsUseSampleCount( diff --git a/Tests/UntoldEngineRenderTests/OpaqueHiddenSurfaceRemovalTests.swift b/Tests/UntoldEngineRenderTests/OpaqueHiddenSurfaceRemovalTests.swift new file mode 100644 index 000000000..23f5410cf --- /dev/null +++ b/Tests/UntoldEngineRenderTests/OpaqueHiddenSurfaceRemovalTests.swift @@ -0,0 +1,156 @@ +// +// OpaqueHiddenSurfaceRemovalTests.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import CShaderTypes +import Metal +import simd +@testable import UntoldEngine +import XCTest + +/// Opaque draws that never discard a fragment go through the model pipeline compiled without +/// discards, so the GPU resolves visibility before shading them; a masked material or a fade +/// keeps the general pipeline. Either way the picture is the same. +@MainActor +final class OpaqueHiddenSurfaceRemovalTests: BaseRenderSetup { + private var box: EntityID = 0 + private var savedMode: AntiAliasingMode = .fxaa + + override func setUp() async throws { + try await super.setUp() + savedMode = antiAliasingMode + } + + override func tearDown() async throws { + setRendering(.opaqueHiddenSurfaceRemoval(.enabled)) + antiAliasingMode = savedMode + destroyAllEntities() + try await super.tearDown() + } + + override func initializeAssets() { + let camera = createEntity() + createGameCamera(entityId: camera) + CameraSystem.shared.activeCamera = camera + cameraLookAt(entityId: camera, eye: simd_float3(0, 1, 4), target: .zero, up: simd_float3(0, 1, 0)) + box = createEntity() + setEntityMeshDirect(entityId: box, meshes: BasicPrimitives.createCube(extent: 1.0), assetName: "Box") + translateTo(entityId: box, position: simd_float3(0, 0.5, 0)) + // A second surface in front of the box: fragments of the box behind it are hidden. + let slab = createEntity() + setEntityMeshDirect(entityId: slab, meshes: BasicPrimitives.createCube(extent: 1.0), assetName: "Slab") + scaleTo(entityId: slab, scale: simd_float3(1.5, 0.4, 0.05)) + translateTo(entityId: slab, position: simd_float3(0, 0.5, 1.0)) + let sun = createEntity() + createDirLight(entityId: sun) + rotateTo(entityId: sun, pitch: -40, yaw: 30, roll: 0) + applyIBL = true + } + + private func drawFrame() { + renderer.draw(in: renderer.metalView) + renderInfo.lastCommandBuffer?.waitUntilCompleted() + } + + private func readFloats(_ texture: MTLTexture) -> [Float] { + var halves = [UInt16](repeating: 0, count: texture.width * texture.height * 4) + halves.withUnsafeMutableBytes { bytes in + texture.getBytes(bytes.baseAddress!, bytesPerRow: texture.width * 8, from: MTLRegionMake2D(0, 0, texture.width, texture.height), mipmapLevel: 0) + } + return halves.map { Float(Float16(bitPattern: $0)) } + } + + private func litScene(hiddenSurfaceRemoval: Bool) throws -> [Float] { + setRendering(.opaqueHiddenSurfaceRemoval(hiddenSurfaceRemoval ? .enabled : .disabled)) + for _ in 0 ..< 3 { + drawFrame() + } + return try readFloats(XCTUnwrap(textureResources.deferredColorMap)) + } + + private func assertSameLitScene(file: StaticString = #filePath, line: UInt = #line) throws { + let general = try litScene(hiddenSurfaceRemoval: false) + XCTAssertEqual(renderInfo.opaqueDrawsPlain, 0, "disabled, every draw takes the general pipeline", file: file, line: line) + let plain = try litScene(hiddenSurfaceRemoval: true) + XCTAssertGreaterThan(general.max() ?? 0, 0.01, "the scene draws something", file: file, line: line) + var maxDifference: Float = 0 + for i in 0 ..< min(general.count, plain.count) { + maxDifference = max(maxDifference, abs(general[i] - plain[i])) + } + XCTAssertLessThanOrEqual(maxDifference, 1e-3, "the plain pipeline draws the same picture (max difference \(maxDifference))", file: file, line: line) + } + + func testBothModelPipelinesExistAndFollowTheSampleCount() throws { + let general = try XCTUnwrap(PipelineManager.shared.renderPipelinesByType[.model]) + let plain = try XCTUnwrap(PipelineManager.shared.renderPipelinesByType[.modelOpaque]) + XCTAssertTrue(plain.success) + XCTAssertEqual(plain.rasterSampleCount, general.rasterSampleCount) + guard renderInfo.device.supportsTextureSampleCount(4) else { return } + antiAliasingMode = .msaa + drawFrame() + XCTAssertEqual(renderInfo.opaqueSampleCount, 4) + let plainMSAA = try XCTUnwrap(PipelineManager.shared.renderPipelinesByType[.modelOpaque]) + XCTAssertEqual(plainMSAA.rasterSampleCount, 4, "the plain pipeline is rebuilt with the pass's sample count") + } + + func testOpaqueDrawsTakeThePlainPipelineAndDrawTheSamePicture() throws { + antiAliasingMode = .none + try assertSameLitScene() + XCTAssertEqual(renderInfo.opaqueDrawsDiscarding, 0, "nothing in this scene discards") + XCTAssertEqual(renderInfo.opaqueDrawsPlain, 2, "the box and the slab") + } + + func testOpaqueDrawsTakeThePlainPipelineUnderMSAA() throws { + try XCTSkipUnless(renderInfo.device.supportsTextureSampleCount(4), "no 4ร— MSAA here") + antiAliasingMode = .msaa + try assertSameLitScene() + XCTAssertEqual(renderInfo.opaqueDrawsPlain, 2) + } + + func testAMaskedMaterialKeepsTheGeneralPipelineAndItsHoles() throws { + antiAliasingMode = .none + // The box's base colour: a 2ร—2 texture whose alpha is 0 on one diagonal. Masked at 0.5, + // half of every face is cut away; only the general pipeline can cut it. + let render = try XCTUnwrap(scene.get(component: RenderComponent.self, for: box)) + let descriptor = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: .rgba8Unorm, width: 2, height: 2, mipmapped: false) + descriptor.usage = [.shaderRead] + let texture = try XCTUnwrap(renderInfo.device.makeTexture(descriptor: descriptor)) + let texels: [UInt8] = [200, 200, 200, 255, 200, 200, 200, 0, 200, 200, 200, 0, 200, 200, 200, 255] + texels.withUnsafeBytes { texture.replace(region: MTLRegionMake2D(0, 0, 2, 2), mipmapLevel: 0, withBytes: $0.baseAddress!, bytesPerRow: 8) } + for meshIndex in render.mesh.indices { + for subIndex in render.mesh[meshIndex].submeshes.indices { + render.mesh[meshIndex].submeshes[subIndex].material?.baseColor.texture = texture + render.mesh[meshIndex].submeshes[subIndex].material?.alphaMode = .mask + render.mesh[meshIndex].submeshes[subIndex].material?.alphaCutoff = 0.5 + } + } + + let general = try litScene(hiddenSurfaceRemoval: false) + let withRemoval = try litScene(hiddenSurfaceRemoval: true) + XCTAssertEqual(renderInfo.opaqueDrawsDiscarding, 1, "the masked box keeps the general pipeline") + XCTAssertEqual(renderInfo.opaqueDrawsPlain, 1, "the slab takes the plain one") + var maxDifference: Float = 0 + for i in 0 ..< min(general.count, withRemoval.count) { + maxDifference = max(maxDifference, abs(general[i] - withRemoval[i])) + } + XCTAssertLessThanOrEqual(maxDifference, 1e-3, "the same picture with and without the plain pipeline") + } + + func testAFadingEntityKeepsTheGeneralPipeline() throws { + antiAliasingMode = .none + let fade = try XCTUnwrap(scene.assign(to: box, component: MeshFadeComponent.self)) + fade.direction = .fadeOut + fade.progress = 0.5 + setRendering(.opaqueHiddenSurfaceRemoval(.enabled)) + drawFrame() + XCTAssertEqual(renderInfo.opaqueDrawsDiscarding, 1, "the fading box dithers, so it keeps the general pipeline") + XCTAssertEqual(renderInfo.opaqueDrawsPlain, 1, "the slab takes the plain one") + scene.remove(component: MeshFadeComponent.self, from: box) + } +} diff --git a/Tests/UntoldEngineRenderTests/PerFragmentIndirectLightingTests.swift b/Tests/UntoldEngineRenderTests/PerFragmentIndirectLightingTests.swift new file mode 100644 index 000000000..81d071ba9 --- /dev/null +++ b/Tests/UntoldEngineRenderTests/PerFragmentIndirectLightingTests.swift @@ -0,0 +1,136 @@ +// +// PerFragmentIndirectLightingTests.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Metal +import simd +@testable import UntoldEngine +import XCTest + +/// The geometry evaluates the image-based light and the sun's cascade shadow once per fragment +/// and carries them in the G-buffer; the light pass adds them from there: the hybrid path. The +/// picture is the one the light pass makes computing them once per sample (the deferred path), +/// and the pipelines follow the path on the next frame. `HybridRenderPathTests` has the rest of +/// the hybrid path: the quad's draw, the tile stage, the direct lights counted once. +@MainActor +final class PerFragmentIndirectLightingTests: BaseRenderSetup { + private var savedMode: AntiAliasingMode = .fxaa + private var savedPath: RenderPath = .deferred + + override func setUp() async throws { + try await super.setUp() + savedMode = antiAliasingMode + savedPath = getRenderPath() + } + + override func tearDown() async throws { + setRendering(.renderPath(savedPath)) + antiAliasingMode = savedMode + destroyAllEntities() + try await super.tearDown() + } + + override func initializeAssets() { + let camera = createEntity() + createGameCamera(entityId: camera) + CameraSystem.shared.activeCamera = camera + cameraLookAt(entityId: camera, eye: simd_float3(0, 2, 5), target: simd_float3(0, 0.3, 0), up: simd_float3(0, 1, 0)) + // A box over a slab: the box's shadow falls on the slab, so both the indirect light and + // the sun's shadow show in the picture. + let box = createEntity() + setEntityMeshDirect(entityId: box, meshes: BasicPrimitives.createCube(extent: 1.0), assetName: "Box") + translateTo(entityId: box, position: simd_float3(0, 0.5, 0)) + let slab = createEntity() + setEntityMeshDirect(entityId: slab, meshes: BasicPrimitives.createCube(extent: 1.0), assetName: "Slab") + scaleTo(entityId: slab, scale: simd_float3(6, 0.1, 6)) + translateTo(entityId: slab, position: simd_float3(0, -0.05, 0)) + let sun = createEntity() + createDirLight(entityId: sun) + rotateTo(entityId: sun, pitch: -55, yaw: 30, roll: 0) + applyIBL = true + } + + private func drawFrames(_ count: Int = 3) { + for _ in 0 ..< count { + renderer.draw(in: renderer.metalView) + renderInfo.lastCommandBuffer?.waitUntilCompleted() + } + } + + private func litFrame(on path: RenderPath) throws -> [Float] { + setRendering(.renderPath(path)) + drawFrames() + XCTAssertEqual(getRenderPath(), path) + XCTAssertEqual(renderInfo.perFragmentIndirectLightingBuilt, path == .hybrid, "the pipelines follow the path") + XCTAssertFalse(renderInfo.perFragmentLightingBuilt, "the quad lights on both paths") + let texture = try XCTUnwrap(textureResources.deferredColorMap) + var halves = [UInt16](repeating: 0, count: texture.width * texture.height * 4) + halves.withUnsafeMutableBytes { bytes in + texture.getBytes(bytes.baseAddress!, bytesPerRow: texture.width * 8, from: MTLRegionMake2D(0, 0, texture.width, texture.height), mipmapLevel: 0) + } + return halves.map { Float(Float16(bitPattern: $0)) } + } + + private func compare(_ a: [Float], _ b: [Float]) -> (mean: Float, max: Float, lit: Int) { + var sum: Float = 0, maxDifference: Float = 0, lit = 0 + for i in stride(from: 0, to: a.count, by: 4) { + for c in 0 ..< 3 { + let d = abs(a[i + c] - b[i + c]) + sum += d + maxDifference = max(maxDifference, d) + } + if a[i] + a[i + 1] + a[i + 2] > 0.02 { lit += 1 } + } + return (sum / Float(a.count / 4 * 3), maxDifference, lit) + } + + func testTheSamePictureAtOneSample() throws { + antiAliasingMode = .none + let perSample = try litFrame(on: .deferred) + let perFragment = try litFrame(on: .hybrid) + let result = compare(perSample, perFragment) + XCTAssertGreaterThan(result.lit, 1000, "the scene is lit") + XCTAssertLessThan(result.mean, 0.002, "mean difference per channel") + XCTAssertLessThan(result.max, 0.06, "the material's 8-bit roughness and metallic in the G-buffer bound the per-pixel difference") + print(String(format: "Per-fragment indirect at 1 sample: mean difference %.5f, max %.4f over %d lit pixels", result.mean, result.max, result.lit)) + } + + func testTheSamePictureUnderMSAA() throws { + try XCTSkipUnless(renderInfo.device.supportsTextureSampleCount(4), "no 4ร— MSAA here") + antiAliasingMode = .msaa + setRendering(.multisampleCount(4)) + let perSample = try litFrame(on: .deferred) + XCTAssertEqual(renderInfo.opaqueSampleCount, 4) + let perFragment = try litFrame(on: .hybrid) + let result = compare(perSample, perFragment) + XCTAssertGreaterThan(result.lit, 1000, "the scene is lit") + XCTAssertLessThan(result.mean, 0.003, "mean difference per channel: only edge pixels differ") + print(String(format: "Per-fragment indirect at 4 samples: mean difference %.5f, max %.4f over %d lit pixels", result.mean, result.max, result.lit)) + } + + func testTheShadowIsCarried() throws { + antiAliasingMode = .none + let perFragment = try litFrame(on: .hybrid) + let texture = try XCTUnwrap(textureResources.deferredColorMap) + // The slab fills the lower part of the frame; the box's shadow is its darkest region, + // the open slab its brightest. Both modes made the same picture (above), so this is + // the shadow being there at all. + var luminance: [Float] = [] + for y in Int(Double(texture.height) * 0.6) ..< Int(Double(texture.height) * 0.95) { + for x in 0 ..< texture.width { + let i = (y * texture.width + x) * 4 + luminance.append(perFragment[i] * 0.2126 + perFragment[i + 1] * 0.7152 + perFragment[i + 2] * 0.0722) + } + } + luminance.sort() + let dark = luminance[luminance.count / 50], bright = luminance[luminance.count * 49 / 50] + XCTAssertGreaterThan(bright, 0.05, "the slab is lit") + XCTAssertLessThan(dark, bright * 0.7, "the slab has the box's shadow (\(dark) against \(bright))") + } +} diff --git a/Tests/UntoldEngineRenderTests/PerFragmentLightingTests.swift b/Tests/UntoldEngineRenderTests/PerFragmentLightingTests.swift new file mode 100644 index 000000000..98ca07e48 --- /dev/null +++ b/Tests/UntoldEngineRenderTests/PerFragmentLightingTests.swift @@ -0,0 +1,139 @@ +// +// PerFragmentLightingTests.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Metal +import simd +@testable import UntoldEngine +import XCTest + +/// The geometry lights itself whole once per fragment โ€” the sun, a point light, the indirect +/// light, the shadows โ€” and writes the lit colour, so the light quad stands down: the forward +/// path. The picture is the one the quad makes lighting once per sample over the indirect light +/// the geometry carried (the hybrid path), the quad's draw is gone from the frame, and the +/// pipelines follow the path on the next frame. +@MainActor +final class PerFragmentLightingTests: BaseRenderSetup { + private var savedMode: AntiAliasingMode = .fxaa + private var savedPath: RenderPath = .deferred + private var pointLight: EntityID = .max + + override func setUp() async throws { + try await super.setUp() + savedMode = antiAliasingMode + savedPath = getRenderPath() + } + + override func tearDown() async throws { + setRendering(.renderPath(savedPath)) + antiAliasingMode = savedMode + destroyAllEntities() + try await super.tearDown() + } + + override func initializeAssets() { + let camera = createEntity() + createGameCamera(entityId: camera) + CameraSystem.shared.activeCamera = camera + cameraLookAt(entityId: camera, eye: simd_float3(0, 2, 5), target: simd_float3(0, 0.3, 0), up: simd_float3(0, 1, 0)) + // A box over a slab under the sun and a point light: the sun's shadow falls on the slab, + // the point light reaches the box's near side, and the indirect light is everywhere. + let box = createEntity() + setEntityMeshDirect(entityId: box, meshes: BasicPrimitives.createCube(extent: 1.0), assetName: "Box") + translateTo(entityId: box, position: simd_float3(0, 0.5, 0)) + let slab = createEntity() + setEntityMeshDirect(entityId: slab, meshes: BasicPrimitives.createCube(extent: 1.0), assetName: "Slab") + scaleTo(entityId: slab, scale: simd_float3(6, 0.1, 6)) + translateTo(entityId: slab, position: simd_float3(0, -0.05, 0)) + let sun = createEntity() + createDirLight(entityId: sun) + rotateTo(entityId: sun, pitch: -55, yaw: 30, roll: 0) + pointLight = createEntity() + createPointLight(entityId: pointLight) + translateTo(entityId: pointLight, position: simd_float3(1.2, 1.2, 1.5)) + applyIBL = true + } + + private func drawFrames(_ count: Int = 3) { + for _ in 0 ..< count { + renderer.draw(in: renderer.metalView) + renderInfo.lastCommandBuffer?.waitUntilCompleted() + } + } + + private func litFrame(on path: RenderPath) throws -> [Float] { + setRendering(.renderPath(path)) + drawFrames() + XCTAssertEqual(getRenderPath(), path) + XCTAssertEqual(renderInfo.perFragmentLightingBuilt, path == .forward, "the pipelines follow the path") + XCTAssertTrue(renderInfo.perFragmentIndirectLightingBuilt, "the indirect light stays per fragment either way") + let texture = try XCTUnwrap(textureResources.deferredColorMap) + var halves = [UInt16](repeating: 0, count: texture.width * texture.height * 4) + halves.withUnsafeMutableBytes { bytes in + texture.getBytes(bytes.baseAddress!, bytesPerRow: texture.width * 8, from: MTLRegionMake2D(0, 0, texture.width, texture.height), mipmapLevel: 0) + } + return halves.map { Float(Float16(bitPattern: $0)) } + } + + private func compare(_ a: [Float], _ b: [Float]) -> (mean: Float, max: Float, lit: Int) { + var sum: Float = 0, maxDifference: Float = 0, lit = 0 + for i in stride(from: 0, to: a.count, by: 4) { + for c in 0 ..< 3 { + let d = abs(a[i + c] - b[i + c]) + sum += d + maxDifference = max(maxDifference, d) + } + if a[i] + a[i + 1] + a[i + 2] > 0.02 { lit += 1 } + } + return (sum / Float(a.count / 4 * 3), maxDifference, lit) + } + + func testTheSamePictureAtOneSample() throws { + antiAliasingMode = .none + let quad = try litFrame(on: .hybrid) + let perFragment = try litFrame(on: .forward) + let result = compare(quad, perFragment) + XCTAssertGreaterThan(result.lit, 1000, "the scene is lit") + XCTAssertLessThan(result.mean, 0.0005, "mean difference per channel") + XCTAssertLessThan(result.max, 0.02, "the G-buffer's half-precision records, which the quad lit from, bound the per-pixel difference (measured 0.001)") + print(String(format: "Per-fragment lighting at 1 sample: mean difference %.5f, max %.4f over %d lit pixels", result.mean, result.max, result.lit)) + } + + func testTheSamePictureUnderMSAA() throws { + try XCTSkipUnless(renderInfo.device.supportsTextureSampleCount(4), "no 4ร— MSAA here") + antiAliasingMode = .msaa + setRendering(.multisampleCount(4)) + let quad = try litFrame(on: .hybrid) + XCTAssertEqual(renderInfo.opaqueSampleCount, 4) + let perFragment = try litFrame(on: .forward) + let result = compare(quad, perFragment) + XCTAssertGreaterThan(result.lit, 1000, "the scene is lit") + XCTAssertLessThan(result.mean, 0.001, "mean difference per channel: only edge pixels differ, lit once instead of once per sample (measured 0.00003)") + print(String(format: "Per-fragment lighting at 4 samples: mean difference %.5f, max %.4f over %d lit pixels", result.mean, result.max, result.lit)) + } + + func testTheQuadStandsDown() throws { + antiAliasingMode = .none + _ = try litFrame(on: .hybrid) + let withQuad = RenderStatsCollector.shared.snapshot().drawCallsTotal + _ = try litFrame(on: .forward) + let without = RenderStatsCollector.shared.snapshot().drawCallsTotal + XCTAssertFalse(renderInfo.tileLightStageUsed) + XCTAssertEqual(without, withQuad - 1, "the frame has the same draws but the light quad's") + } + + func testThePointLightIsThere() throws { + antiAliasingMode = .none + let withLight = try litFrame(on: .forward) + destroyEntity(entityId: pointLight) + let without = try litFrame(on: .forward) + let result = compare(withLight, without) + XCTAssertGreaterThan(result.max, 0.05, "the point light lit the box's near side (max difference \(result.max))") + } +} diff --git a/Tests/UntoldEngineRenderTests/PreCompositeBypassTests.swift b/Tests/UntoldEngineRenderTests/PreCompositeBypassTests.swift new file mode 100644 index 000000000..a3a272222 --- /dev/null +++ b/Tests/UntoldEngineRenderTests/PreCompositeBypassTests.swift @@ -0,0 +1,123 @@ +// +// PreCompositeBypassTests.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Metal +import simd +@testable import UntoldEngine +import XCTest + +/// In XR passthrough the pre-composite pass copies the lit scene whenever no layer joins it; +/// such a frame skips the pass and the look pass reads the lit scene directly. +@MainActor +final class PreCompositeBypassTests: BaseRenderSetup { + private var savedImmersionStyle: UntoldImmersionMode = .none + private var savedGameMode = false + private var savedSSAO = false + private var savedAntiAliasing: AntiAliasingMode = .fxaa + + override func setUp() async throws { + try await super.setUp() + // No splat entity here; an order a Gaussian test left in the shared set would say otherwise. + for slot in 0 ..< maxInFlightCommandBuffers { + GaussianSharedWorkingSet.shared.setEntityOrder([], slot: slot) + } + savedImmersionStyle = renderInfo.immersionStyle + savedGameMode = gameMode + savedSSAO = SSAOParams.shared.enabled + savedAntiAliasing = antiAliasingMode + } + + override func tearDown() async throws { + renderInfo.immersionStyle = savedImmersionStyle + gameMode = savedGameMode + SSAOParams.shared.enabled = savedSSAO + antiAliasingMode = savedAntiAliasing + setRendering(.preCompositeBypass(.enabled)) + renderInfo.preCompositeBypassSource = nil + destroyAllEntities() + try await super.tearDown() + } + + override func initializeAssets() { + let camera = createEntity() + createGameCamera(entityId: camera) + CameraSystem.shared.activeCamera = camera + cameraLookAt(entityId: camera, eye: simd_float3(0, 1, 4), target: .zero, up: simd_float3(0, 1, 0)) + let box = createEntity() + setEntityMeshDirect(entityId: box, meshes: BasicPrimitives.createCube(extent: 1.0), assetName: "Box") + translateTo(entityId: box, position: simd_float3(0, 0.5, 0)) + } + + private func drawFrame() { + renderer.draw(in: renderer.metalView) + renderInfo.lastCommandBuffer?.waitUntilCompleted() + } + + private func readFloats(_ texture: MTLTexture) -> [Float] { + var halves = [UInt16](repeating: 0, count: texture.width * texture.height * 4) + halves.withUnsafeMutableBytes { bytes in + texture.getBytes(bytes.baseAddress!, bytesPerRow: texture.width * 8, from: MTLRegionMake2D(0, 0, texture.width, texture.height), mipmapLevel: 0) + } + return halves.map { Float(Float16(bitPattern: $0)) } + } + + func testPassthroughFrameWithNoLayerToJoinSkipsThePass() throws { + renderInfo.immersionStyle = .mixed + gameMode = true + SSAOParams.shared.enabled = false + antiAliasingMode = .none + + // A few frames settle the frame-to-frame state (the GPU culling's in-flight results, the + // shadows, the depth pyramid) so that the two frames compared differ in the pass alone. + for _ in 0 ..< 3 { + drawFrame() + } + drawFrame() + XCTAssertNotNil(renderInfo.preCompositeBypassSource, "the pass is skipped: the look pass reads the lit scene") + let bypassed = try readFloats(XCTUnwrap(textureResources.lookTexture)) + + setRendering(.preCompositeBypass(.disabled)) + drawFrame() + XCTAssertNil(renderInfo.preCompositeBypassSource, "disabled, the pass runs on every frame") + let composited = try readFloats(XCTUnwrap(textureResources.lookTexture)) + + XCTAssertGreaterThan(composited.max() ?? 0, 0.01, "the scene draws something") + var maxDifference: Float = 0 + for i in 0 ..< min(bypassed.count, composited.count) { + maxDifference = max(maxDifference, abs(bypassed[i] - composited[i])) + } + XCTAssertEqual(maxDifference, 0, accuracy: 1e-3, "the look pass sees the same pixels whether the composite pass copied them or not") + } + + func testThePassRunsWhereverALayerJoinsTheScene() { + gameMode = true + SSAOParams.shared.enabled = false + antiAliasingMode = .none + + // The environment lies behind the scene outside passthrough. + renderInfo.immersionStyle = .none + drawFrame() + XCTAssertNil(renderInfo.preCompositeBypassSource, "the environment joins the scene") + + renderInfo.immersionStyle = .mixed + SSAOParams.shared.enabled = true + drawFrame() + XCTAssertNil(renderInfo.preCompositeBypassSource, "ambient occlusion joins the scene") + SSAOParams.shared.enabled = false + + gameMode = false + drawFrame() + XCTAssertNil(renderInfo.preCompositeBypassSource, "the editor's gizmo layer joins the scene") + gameMode = true + + drawFrame() + XCTAssertNotNil(renderInfo.preCompositeBypassSource, "nothing joins the lit scene: the pass is skipped") + } +} diff --git a/Tests/UntoldEngineRenderTests/RenderGraphBuilderTest.swift b/Tests/UntoldEngineRenderTests/RenderGraphBuilderTest.swift index 315dcf741..382ba9cd7 100644 --- a/Tests/UntoldEngineRenderTests/RenderGraphBuilderTest.swift +++ b/Tests/UntoldEngineRenderTests/RenderGraphBuilderTest.swift @@ -1530,7 +1530,7 @@ final class RenderGraphBuilderTest: BaseRenderSetup { for environmentEnabled in [false, true] { renderEnvironment = environmentEnabled - for mode in [AntiAliasingMode.none, .fxaa, .smaa, .msaa] { + for mode in [AntiAliasingMode.none, .fxaa, .smaa, .msaa, .taa, .msaaTaa] { antiAliasingMode = mode try captureGraphPassIDs() } diff --git a/Tests/UntoldEngineRenderTests/ShadowCascadeCountTests.swift b/Tests/UntoldEngineRenderTests/ShadowCascadeCountTests.swift new file mode 100644 index 000000000..1fc79f31c --- /dev/null +++ b/Tests/UntoldEngineRenderTests/ShadowCascadeCountTests.swift @@ -0,0 +1,120 @@ +// +// ShadowCascadeCountTests.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Metal +import simd +@testable import UntoldEngine +import XCTest + +/// The sun's shadow is drawn in as many cascades as the setting asks for, one on the headset +/// by default and every cascade elsewhere: one cascade reaches the whole shadow distance, the +/// first of two stops short of it, and the shader is told how many to read. +@MainActor +final class ShadowCascadeCountTests: BaseRenderSetup { + override func tearDown() async throws { + ShadowSystem.cascadesWanted = nil + destroyAllEntities() + try await super.tearDown() + } + + override func initializeAssets() { + let camera = createEntity() + createGameCamera(entityId: camera) + CameraSystem.shared.activeCamera = camera + cameraLookAt(entityId: camera, eye: simd_float3(0, 1, 4), target: .zero, up: simd_float3(0, 1, 0)) + let box = createEntity() + setEntityMeshDirect(entityId: box, meshes: BasicPrimitives.createCube(extent: 1.0), assetName: "Box") + translateTo(entityId: box, position: simd_float3(0, 0.5, 0)) + // The base scene's own directional light is the active sun. + } + + private func drawFrames() { + for _ in 0 ..< 2 { + renderer.draw(in: renderer.metalView) + renderInfo.lastCommandBuffer?.waitUntilCompleted() + } + } + + func testTheSettingChoosesTheCascades() { + setRendering(.shadowCascades(1)) + drawFrames() + XCTAssertEqual(getShadowCascades(), 1) + XCTAssertEqual(shadowSystem.makeUniforms().cascadeCount, 1, "the shader reads one") + XCTAssertTrue(shadowSystem.isActive) + let single = shadowSystem.cascadeSplitDistances[0] + + setRendering(.shadowCascades(2)) + drawFrames() + XCTAssertEqual(getShadowCascades(), 2) + XCTAssertEqual(shadowSystem.makeUniforms().cascadeCount, 2) + XCTAssertLessThan(shadowSystem.cascadeSplitDistances[0], single, "the first of two stops short of the shadow distance") + XCTAssertEqual(shadowSystem.cascadeSplitDistances[1], single, accuracy: 1e-3, "the last split is the shadow distance either way") + + setRendering(.shadowCascades(5)) + XCTAssertEqual(getShadowCascades(), csmCascadeCount, "never more than the maps hold") + setRendering(.shadowCascades(0)) + XCTAssertEqual(getShadowCascades(), 1, "never none: a sun casts") + } + + private func readFloats(_ texture: MTLTexture) -> [Float] { + var halves = [UInt16](repeating: 0, count: texture.width * texture.height * 4) + halves.withUnsafeMutableBytes { bytes in + texture.getBytes(bytes.baseAddress!, bytesPerRow: texture.width * 8, from: MTLRegionMake2D(0, 0, texture.width, texture.height), mipmapLevel: 0) + } + return halves.map { Float(Float16(bitPattern: $0)) } + } + + /// The lit frame's pixels that the sun's shadow darkens by a third or more: the floor + /// under and beside the box. + private func shadowedPixels(cascades: Int) throws -> Int { + let sun = try XCTUnwrap(LightingSystem.shared.activeDirectionalLight) + setRendering(.shadowCascades(cascades)) + setLight(entityId: sun, .directional(.castsShadow(true))) + drawFrames() + let shadowed = try readFloats(XCTUnwrap(textureResources.deferredColorMap)) + setLight(entityId: sun, .directional(.castsShadow(false))) + drawFrames() + let unshadowed = try readFloats(XCTUnwrap(textureResources.deferredColorMap)) + setLight(entityId: sun, .directional(.castsShadow(true))) + var count = 0 + for i in stride(from: 0, to: min(shadowed.count, unshadowed.count), by: 4) { + let lit = unshadowed[i] + unshadowed[i + 1] + unshadowed[i + 2] + let dark = shadowed[i] + shadowed[i + 1] + shadowed[i + 2] + if lit > 0.05, dark < 0.67 * lit { count += 1 } + } + return count + } + + func testOneCascadeStillCastsAShadow() throws { + // A floor for the box to cast onto, and a sun that reaches across it. + let floor = createEntity() + setEntityMeshDirect(entityId: floor, meshes: BasicPrimitives.createCube(extent: 6.0), assetName: "Floor") + translateTo(entityId: floor, position: simd_float3(0, -3.0, 0)) + let sun = try XCTUnwrap(LightingSystem.shared.activeDirectionalLight) + rotateTo(entityId: sun, pitch: -55, yaw: 30, roll: 0) + setVisibleEntities() + + let withTwo = try shadowedPixels(cascades: 2) + let withOne = try shadowedPixels(cascades: 1) + XCTAssertGreaterThan(withTwo, 0, "two cascades shadow the floor under the box") + XCTAssertGreaterThan(withOne, 0, "one cascade shadows it too") + XCTAssertGreaterThan(withOne, withTwo / 2, "one cascade over the whole distance shadows about the same pixels") + } + + func testTheHeadsetDefaultsToOneCascadeAndTheDesktopToAll() { + ShadowSystem.cascadesWanted = nil + let saved = renderInfo.isXRStereoMode + defer { renderInfo.isXRStereoMode = saved } + renderInfo.isXRStereoMode = true + XCTAssertEqual(shadowSystem.cascadeCount, 1) + renderInfo.isXRStereoMode = false + XCTAssertEqual(shadowSystem.cascadeCount, csmCascadeCount) + } +} diff --git a/Tests/UntoldEngineRenderTests/ShadowPassGateTests.swift b/Tests/UntoldEngineRenderTests/ShadowPassGateTests.swift new file mode 100644 index 000000000..6f0f6fb69 --- /dev/null +++ b/Tests/UntoldEngineRenderTests/ShadowPassGateTests.swift @@ -0,0 +1,69 @@ +// +// ShadowPassGateTests.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Metal +import simd +@testable import UntoldEngine +import XCTest + +/// The sun's shadow cascades are drawn only while the sun lights something: the shading +/// multiplies the shadow by the sun's intensity, so a sun at zero would have its two depth +/// maps drawn every frame for nothing. The GPU's own pass timing says which passes ran. +@MainActor +final class ShadowPassGateTests: BaseRenderSetup { + override func setUp() async throws { + try await super.setUp() + setRendering(.gpuPassTiming(.enabled)) + } + + override func tearDown() async throws { + setRendering(.gpuPassTiming(.disabled)) + destroyAllEntities() + try await super.tearDown() + } + + override func initializeAssets() { + let camera = createEntity() + createGameCamera(entityId: camera) + CameraSystem.shared.activeCamera = camera + cameraLookAt(entityId: camera, eye: simd_float3(0, 1, 4), target: .zero, up: simd_float3(0, 1, 0)) + let box = createEntity() + setEntityMeshDirect(entityId: box, meshes: BasicPrimitives.createCube(extent: 1.0), assetName: "Box") + translateTo(entityId: box, position: simd_float3(0, 0.5, 0)) + // The base scene's own directional light is the active sun; a second one would not be. + } + + /// The labels of the passes the last frame ran, after a few frames. + private func passesRun() -> Set { + for _ in 0 ..< 4 { + renderer.draw(in: renderer.metalView) + renderInfo.lastCommandBuffer?.waitUntilCompleted() + } + return Set(GPUPassTiming.shared.lastFrame().passes.map(\.label)) + } + + func testASunAtZeroIntensityDrawsNoCascades() throws { + try XCTSkipUnless(renderInfo.device.supportsCounterSampling(.atStageBoundary), "no pass timing here") + let sun = try XCTUnwrap(LightingSystem.shared.activeDirectionalLight, "the scene's sun") + updateLightIntensity(entityId: sun, intensity: 2.0) + var passes = passesRun() + XCTAssertTrue(passes.contains("Shadows"), "a lit sun casts: \(passes.sorted())") + XCTAssertTrue(shadowSystem.isActive) + + updateLightIntensity(entityId: sun, intensity: 0.0) + passes = passesRun() + XCTAssertFalse(passes.contains("Shadows"), "a sun at zero lights nothing: no cascades: \(passes.sorted())") + XCTAssertFalse(shadowSystem.isActive) + + updateLightIntensity(entityId: sun, intensity: 1.0) + passes = passesRun() + XCTAssertTrue(passes.contains("Shadows"), "lit again: the cascades are back: \(passes.sorted())") + } +} diff --git a/Tests/UntoldEngineRenderTests/StaticBatchingTest.swift b/Tests/UntoldEngineRenderTests/StaticBatchingTest.swift index 9ec2b170c..d0586fe96 100644 --- a/Tests/UntoldEngineRenderTests/StaticBatchingTest.swift +++ b/Tests/UntoldEngineRenderTests/StaticBatchingTest.swift @@ -41,6 +41,10 @@ final class StaticBatchingTest: BaseRenderSetup { // Clean up batches after tests clearSceneBatches() enableBatching(false) + // The channel render modes some tests set are global: hidden context geometry left + // behind hides every entity of every test after this class. + setSceneChannel(.contextGeometry, .renderMode(.normal)) + setSceneChannel(.ghostGeometry, .renderMode(.normal)) BatchingSystem.shared.setBatchCellSize(32.0) BatchingSystem.shared.setMaxDirtyCellsPerTick(8) BatchingSystem.shared.setMaxRebuildVerticesPerTick(1_000_000_000) diff --git a/Tests/UntoldEngineRenderTests/TAAMovingObjectTests.swift b/Tests/UntoldEngineRenderTests/TAAMovingObjectTests.swift new file mode 100644 index 000000000..25629f07e --- /dev/null +++ b/Tests/UntoldEngineRenderTests/TAAMovingObjectTests.swift @@ -0,0 +1,176 @@ +// +// TAAMovingObjectTests.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Metal +import simd +@testable import UntoldEngine +import XCTest + +/// A surface that moves on its own under a still camera. On the forward path the opaque pass +/// writes its motion per pixel and the resolve finds its history where it was, so the resolve +/// of a moving box differs from the box's own frame about as little as a resolve held still +/// does, instead of trailing a ghost of where the box has been. The deferred and hybrid paths +/// write no motion: there the neighbourhood clip is what bounds the ghost. Measured (M4 Max, +/// a plain box over a plain background): the excess over a resolve held still is 0.1โ€“0.3 % of +/// the box's motion with the motion written, 0.2โ€“0.4 % without, at either stride below. +@MainActor +final class TAAMovingObjectTests: BaseRenderSetup { + private var camera: EntityID = 0 + private var box: EntityID = 0 + private var savedMode: AntiAliasingMode = .fxaa + private var savedPath: RenderPath = .deferred + + override func setUp() async throws { + try await super.setUp() + savedMode = antiAliasingMode + savedPath = getRenderPath() + } + + override func tearDown() async throws { + antiAliasingMode = savedMode + setRendering(.renderPath(savedPath)) + TemporalAntiAliasing.shared.invalidateHistory() + destroyAllEntities() + try await super.tearDown() + } + + override func initializeAssets() { + camera = createEntity() + createGameCamera(entityId: camera) + CameraSystem.shared.activeCamera = camera + cameraLookAt(entityId: camera, eye: simd_float3(0, 1, 4), target: .zero, up: simd_float3(0, 1, 0)) + box = createEntity() + setEntityMeshDirect(entityId: box, meshes: BasicPrimitives.createCube(extent: 1.0), assetName: "Box") + translateTo(entityId: box, position: simd_float3(-0.6, 0.5, 0)) + let sun = createEntity() + createDirLight(entityId: sun) + rotateTo(entityId: sun, pitch: -40, yaw: 30, roll: 0) + applyIBL = true + } + + private func drawFrame() { + renderer.draw(in: renderer.metalView) + renderInfo.lastCommandBuffer?.waitUntilCompleted() + } + + private func readFloats(_ texture: MTLTexture) throws -> [Float] { + let shared = try XCTUnwrap(renderInfo.device.makeTexture(descriptor: { + let d = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: texture.pixelFormat, width: texture.width, height: texture.height, mipmapped: false) + d.storageMode = .shared + d.usage = [.shaderRead] + return d + }())) + let copy = try XCTUnwrap(renderInfo.commandQueue.makeCommandBuffer()) + let blit = try XCTUnwrap(copy.makeBlitCommandEncoder()) + blit.copy(from: texture, to: shared) + blit.endEncoding() + copy.commit() + copy.waitUntilCompleted() + var halves = [UInt16](repeating: 0, count: shared.width * shared.height * 4) + halves.withUnsafeMutableBytes { bytes in + shared.getBytes(bytes.baseAddress!, bytesPerRow: shared.width * 8, from: MTLRegionMake2D(0, 0, shared.width, shared.height), mipmapLevel: 0) + } + return halves.map { Float(Float16(bitPattern: $0)) } + } + + /// The frame as the output pass would present it: the resolve in a temporal mode, the look output otherwise. + private func readOutput() throws -> [Float] { + try readFloats(XCTUnwrap(antiAliasingMode.isTemporal ? TemporalAntiAliasing.shared.resolvedTexture : textureResources.lookTexture)) + } + + private func meanAbsoluteDifference(_ a: [Float], _ b: [Float]) -> Float { + var sum: Float = 0 + for i in stride(from: 0, to: min(a.count, b.count), by: 4) { + sum += abs(a[i] - b[i]) + abs(a[i + 1] - b[i + 1]) + abs(a[i + 2] - b[i + 2]) + } + return sum / Float(min(a.count, b.count) / 4 * 3) + } + + /// How far the resolve of the moving box ends from the box's own frame (`shadow`), against + /// how far a resolve held still on that frame ends from it (`floor`): the excess is the + /// ghost. `motion` is how far apart the box's first and last frames are. + private struct Ghost { + let motion: Float + let shadow: Float + let floor: Float + } + + /// A stride of 0.16 a frame crosses the box's width in six frames: a pixel the box leaves + /// has only the background around it, so the clip alone rejects the box's history there. + private static let fastStride: Float = 0.16 + /// A stride of 0.02 a frame, a few pixels, is the ghosting case: the pixels the trailing + /// edge leaves still have the box in their neighbourhood, so the clip keeps part of the + /// stale history, while the motion vectors find the surface's own history instead. + private static let slowStride: Float = 0.02 + + /// The box walks eight poses, a stride apart, under the temporal resolve on the given path, + /// then the resolve is held on the last pose for as long. + private func ghost(on path: RenderPath, mode: AntiAliasingMode, stride: Float) throws -> Ghost { + func pose(_ frame: Int) -> simd_float3 { + simd_float3(-0.6 + stride * Float(frame), 0.5, 0) + } + let frameCount = 8 + + setRendering(.renderPath(path)) + antiAliasingMode = .none + translateTo(entityId: box, position: pose(0)) + drawFrame() + drawFrame() + let first = try readOutput() + translateTo(entityId: box, position: pose(frameCount - 1)) + drawFrame() + drawFrame() + let last = try readOutput() + XCTAssertEqual(getRenderPath(), path) + XCTAssertEqual(renderInfo.perFragmentLightingBuilt, path == .forward, "the pipelines follow the path") + + antiAliasingMode = mode + TemporalAntiAliasing.shared.invalidateHistory() + for frame in 0 ..< frameCount { + translateTo(entityId: box, position: pose(frame)) + drawFrame() + } + let moved = try readOutput() + TemporalAntiAliasing.shared.invalidateHistory() + for _ in 0 ..< frameCount { + drawFrame() + } + let held = try readOutput() + let measured = Ghost(motion: meanAbsoluteDifference(first, last), shadow: meanAbsoluteDifference(moved, last), floor: meanAbsoluteDifference(held, last)) + print("TAA moving-box \(path) \(mode) stride \(stride): resolve differs from the box's own frame by \(measured.shadow), held still by \(measured.floor), the first pose by \(measured.motion)") + XCTAssertGreaterThan(measured.motion, 0.0005, "the box's path moves it in the frame") + return measured + } + + func testTheResolveFollowsAMovingBoxWithoutAGhost() throws { + for mode in [AntiAliasingMode.taa, .msaaTaa] { + for stride in [Self.fastStride, Self.slowStride] { + let measured = try ghost(on: .forward, mode: mode, stride: stride) + XCTAssertLessThan(measured.shadow, measured.floor + 0.1 * measured.motion, "\(mode), stride \(stride): the resolve follows the box: it differs from the box's own frame by \(measured.shadow), held still by \(measured.floor), the first pose differs by \(measured.motion)") + } + } + } + + /// Without motion vectors the resolve looks for the box's history where the camera alone + /// puts it โ€” where the box was โ€” and the clip to the neighbourhood is what keeps that + /// history from trailing. The ghost it leaves is bounded as the forward path's is, at a + /// tenth of the motion (a clip that let the history through would leave most of it), and + /// is about twice the forward path's in the measurement above. + func testWithoutMotionVectorsTheClipBoundsTheGhost() throws { + for path in [RenderPath.deferred, .hybrid] { + for mode in [AntiAliasingMode.taa, .msaaTaa] { + for stride in [Self.fastStride, Self.slowStride] { + let measured = try ghost(on: path, mode: mode, stride: stride) + XCTAssertLessThan(measured.shadow, measured.floor + 0.1 * measured.motion, "\(path) \(mode), stride \(stride): the clip bounds the ghost: the resolve differs from the box's own frame by \(measured.shadow), held still by \(measured.floor), the first pose differs by \(measured.motion)") + } + } + } + } +} diff --git a/Tests/UntoldEngineRenderTests/TAAPresentFoldTests.swift b/Tests/UntoldEngineRenderTests/TAAPresentFoldTests.swift new file mode 100644 index 000000000..9f01b12c5 --- /dev/null +++ b/Tests/UntoldEngineRenderTests/TAAPresentFoldTests.swift @@ -0,0 +1,124 @@ +// +// TAAPresentFoldTests.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Metal +import simd +@testable import UntoldEngine +import XCTest + +/// When nothing sits between the temporal resolve and the output in the frame's graph, the +/// resolve presents the frame itself โ€” the display's colour, encoded, and the opaque depth โ€” +/// and the output transform pass stands down. The display gets what the output pass would +/// have written from the resolve. +@MainActor +final class TAAPresentFoldTests: BaseRenderSetup { + private var savedMode: AntiAliasingMode = .fxaa + + override func setUp() async throws { + try await super.setUp() + savedMode = antiAliasingMode + // The view's drawable is the display here; readable, to check what was presented. + renderer.metalView.framebufferOnly = false + setRendering(.gpuPassTiming(.enabled)) + } + + override func tearDown() async throws { + setRendering(.gpuPassTiming(.disabled)) + antiAliasingMode = savedMode + destroyAllEntities() + try await super.tearDown() + } + + override func initializeAssets() { + let camera = createEntity() + createGameCamera(entityId: camera) + CameraSystem.shared.activeCamera = camera + cameraLookAt(entityId: camera, eye: simd_float3(0, 1, 4), target: .zero, up: simd_float3(0, 1, 0)) + let box = createEntity() + setEntityMeshDirect(entityId: box, meshes: BasicPrimitives.createCube(extent: 1.0), assetName: "Box") + translateTo(entityId: box, position: simd_float3(0, 0.5, 0)) + let sun = createEntity() + createDirLight(entityId: sun) + rotateTo(entityId: sun, pitch: -40, yaw: 30, roll: 0) + applyIBL = true + } + + private func drawFrames(_ count: Int) { + for _ in 0 ..< count { + renderer.draw(in: renderer.metalView) + renderInfo.lastCommandBuffer?.waitUntilCompleted() + } + } + + private func linearToSRGB(_ x: Float) -> Float { + x <= 0.0031308 ? x * 12.92 : 1.055 * powf(max(x, 0), 1 / 2.4) - 0.055 + } + + func testTheResolvePresentsAndTheOutputPassStandsDown() throws { + try XCTSkipUnless(renderInfo.device.supportsCounterSampling(.atStageBoundary), "no pass timing here") + antiAliasingMode = .taa + drawFrames(4) + let presented = try XCTUnwrap(renderInfo.renderPassDescriptor?.colorAttachments[0].texture, "the view has a drawable") + try XCTSkipUnless(presented.pixelFormat == .bgra8Unorm_srgb || presented.pixelFormat == .bgra8Unorm, "unexpected display format \(presented.pixelFormat.rawValue)") + XCTAssertTrue(renderInfo.temporalResolveMayPresent, "nothing registered between the resolve and the output") + XCTAssertTrue(TemporalAntiAliasing.shared.presentedThisFrame, "the resolve presented") + let passes = Set(GPUPassTiming.shared.lastFrame().passes.map(\.label)) + XCTAssertTrue(passes.contains("TAA")) + XCTAssertFalse(passes.contains("Output"), "the output transform pass stood down: \(passes.sorted())") + + // The display holds the resolve, encoded as the output pass encodes it. + let resolved = try XCTUnwrap(TemporalAntiAliasing.shared.resolvedTexture) + let shared = try XCTUnwrap(renderInfo.device.makeTexture(descriptor: { + let d = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: resolved.pixelFormat, width: resolved.width, height: resolved.height, mipmapped: false) + d.storageMode = .shared + d.usage = [.shaderRead] + return d + }())) + let copy = try XCTUnwrap(renderInfo.commandQueue.makeCommandBuffer()) + let blit = try XCTUnwrap(copy.makeBlitCommandEncoder()) + blit.copy(from: resolved, to: shared) + blit.endEncoding() + copy.commit() + copy.waitUntilCompleted() + var halves = [UInt16](repeating: 0, count: shared.width * shared.height * 4) + halves.withUnsafeMutableBytes { bytes in + shared.getBytes(bytes.baseAddress!, bytesPerRow: shared.width * 8, from: MTLRegionMake2D(0, 0, shared.width, shared.height), mipmapLevel: 0) + } + var bytes = [UInt8](repeating: 0, count: presented.width * presented.height * 4) + bytes.withUnsafeMutableBytes { buffer in + presented.getBytes(buffer.baseAddress!, bytesPerRow: presented.width * 4, from: MTLRegionMake2D(0, 0, presented.width, presented.height), mipmapLevel: 0) + } + // The pass encodes when the pipeline says so; an sRGB display format encodes in hardware. + let encodes = renderInfo.colorPipeline.present.encodingMode.rawValue == 1 || presented.pixelFormat == .bgra8Unorm_srgb + var checked = 0 + for (x, y) in [(presented.width / 2, presented.height / 2), (presented.width / 3, presented.height * 2 / 3), (presented.width * 2 / 3, presented.height / 3)] { + let i = y * presented.width + x + for (channel, byteIndex) in [(0, 2), (1, 1), (2, 0)] { // bgra + var value = Float(Float16(bitPattern: halves[i * 4 + channel])) + value = max(value, 0) + if encodes { value = linearToSRGB(value) } + let expected = min(max(value, 0), 1) * 255 + XCTAssertEqual(Float(bytes[i * 4 + byteIndex]), expected, accuracy: 2.5, "pixel (\(x), \(y)) channel \(channel)") + checked += 1 + } + } + XCTAssertEqual(checked, 9) + } + + func testNoFoldWithoutTheTemporalResolve() throws { + try XCTSkipUnless(renderInfo.device.supportsCounterSampling(.atStageBoundary), "no pass timing here") + antiAliasingMode = .fxaa + drawFrames(3) + XCTAssertFalse(renderInfo.temporalResolveMayPresent) + XCTAssertFalse(TemporalAntiAliasing.shared.presentedThisFrame) + let passes = Set(GPUPassTiming.shared.lastFrame().passes.map(\.label)) + XCTAssertTrue(passes.contains("Output"), "the output transform pass presents: \(passes.sorted())") + } +} diff --git a/Tests/UntoldEngineRenderTests/TAATransparentHistoryTests.swift b/Tests/UntoldEngineRenderTests/TAATransparentHistoryTests.swift new file mode 100644 index 000000000..7e39fbf22 --- /dev/null +++ b/Tests/UntoldEngineRenderTests/TAATransparentHistoryTests.swift @@ -0,0 +1,127 @@ +// +// TAATransparentHistoryTests.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Metal +import simd +@testable import UntoldEngine +import XCTest + +/// Over a transparent background (a headset's passthrough) a surface that moves across the +/// frame covers pixels whose history holds no surface at all. The resolve gives such a history +/// no weight: the surface's leading edge stays as opaque and as bright as its middle, instead +/// of fading toward the room behind it for the twenty frames the history takes to catch up. +@MainActor +final class TAATransparentHistoryTests: BaseRenderSetup { + private var savedMode: AntiAliasingMode = .fxaa + private var savedImmersion: UntoldImmersionMode = .none + private var box: EntityID = .max + + override func setUp() async throws { + try await super.setUp() + savedMode = antiAliasingMode + savedImmersion = renderInfo.immersionStyle + // The passthrough's immersion style: the composite keeps the scene's coverage, so the + // background stays transparent, as on a headset. + renderInfo.immersionStyle = .mixed + } + + override func tearDown() async throws { + antiAliasingMode = savedMode + renderInfo.immersionStyle = savedImmersion + setRendering(.environment(.visible(true))) + destroyAllEntities() + try await super.tearDown() + } + + override func initializeAssets() { + let camera = createEntity() + createGameCamera(entityId: camera) + CameraSystem.shared.activeCamera = camera + cameraLookAt(entityId: camera, eye: simd_float3(0, 0, 4), target: .zero, up: simd_float3(0, 1, 0)) + box = createEntity() + setEntityMeshDirect(entityId: box, meshes: BasicPrimitives.createCube(extent: 1.0), assetName: "Box") + let sun = createEntity() + createDirLight(entityId: sun) + rotateTo(entityId: sun, pitch: -40, yaw: 30, roll: 0) + applyIBL = true + // Nothing behind the box: the background stays the clear colour, transparent. + setRendering(.environment(.visible(false))) + } + + private func drawFrame() { + renderer.draw(in: renderer.metalView) + renderInfo.lastCommandBuffer?.waitUntilCompleted() + } + + /// The look output's pixels (the resolved frame in the temporal modes), as floats. + private func frame() throws -> (pixels: [Float], width: Int, height: Int) { + let texture = try XCTUnwrap(antiAliasingMode.isTemporal ? TemporalAntiAliasing.shared.resolvedTexture : textureResources.lookTexture) + let shared = try XCTUnwrap(renderInfo.device.makeTexture(descriptor: { + let d = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: texture.pixelFormat, width: texture.width, height: texture.height, mipmapped: false) + d.storageMode = .shared + d.usage = [.shaderRead] + return d + }())) + let copy = try XCTUnwrap(renderInfo.commandQueue.makeCommandBuffer()) + let blit = try XCTUnwrap(copy.makeBlitCommandEncoder()) + blit.copy(from: texture, to: shared) + blit.endEncoding() + copy.commit() + copy.waitUntilCompleted() + var halves = [UInt16](repeating: 0, count: shared.width * shared.height * 4) + halves.withUnsafeMutableBytes { bytes in + shared.getBytes(bytes.baseAddress!, bytesPerRow: shared.width * 8, from: MTLRegionMake2D(0, 0, shared.width, shared.height), mipmapLevel: 0) + } + return (halves.map { Float(Float16(bitPattern: $0)) }, shared.width, shared.height) + } + + func testASurfaceMovingOverTheTransparentBackgroundKeepsItsLeadingEdge() throws { + antiAliasingMode = .taa + translateTo(entityId: box, position: simd_float3(1.2, 0, 0)) + for _ in 0 ..< 6 { + drawFrame() + } + let still = try frame() + let row = still.height / 2 + try XCTSkipUnless(still.pixels[(row * still.width + 8) * 4 + 3] < 0.01, "the background is not transparent here: alpha \(still.pixels[(row * still.width + 8) * 4 + 3])") + // The box moves left, about forty pixels a frame; the pixels it covers each frame had + // only the background in the history. + var x = Float(1.2) + for _ in 0 ..< 5 { + x -= 0.16 + translateTo(entityId: box, position: simd_float3(x, 0, 0)) + drawFrame() + } + let moving = try frame() + // The leading edge: the leftmost pixel of the box's middle row with some coverage. + var leading = -1 + for px in 0 ..< moving.width where moving.pixels[(row * moving.width + px) * 4 + 3] > 0.5 { + leading = px + break + } + XCTAssertGreaterThan(leading, 0, "the box is in the frame") + // Ten pixels inside the edge the box is opaque and as bright as its middle. + var centre = -1 + for px in leading ..< moving.width where moving.pixels[(row * moving.width + px) * 4 + 3] < 0.5 { + centre = (leading + px) / 2 + break + } + XCTAssertGreaterThan(centre, leading + 40, "the box spans more than eighty pixels") + func luminance(_ px: Int) -> Float { + let i = (row * moving.width + px) * 4 + return moving.pixels[i] * 0.2126 + moving.pixels[i + 1] * 0.7152 + moving.pixels[i + 2] * 0.0722 + } + let edgeAlpha = moving.pixels[(row * moving.width + leading + 10) * 4 + 3] + XCTAssertGreaterThan(edgeAlpha, 0.95, "the leading edge is opaque (alpha \(edgeAlpha))") + let edge = luminance(leading + 10), middle = luminance(centre) + XCTAssertGreaterThan(middle, 0.02, "the box is lit") + XCTAssertGreaterThan(edge, middle * 0.8, "the leading edge is as bright as the middle (\(edge) against \(middle))") + } +} diff --git a/Tests/UntoldEngineRenderTests/TemporalAntiAliasingTests.swift b/Tests/UntoldEngineRenderTests/TemporalAntiAliasingTests.swift new file mode 100644 index 000000000..fe532e673 --- /dev/null +++ b/Tests/UntoldEngineRenderTests/TemporalAntiAliasingTests.swift @@ -0,0 +1,385 @@ +// +// TemporalAntiAliasingTests.swift +// UntoldEngine +// +// The temporal resolve (TemporalAntiAliasing.swift, TAAShader.metal): the jitter sequence, +// the jitter going into and out of the projection, and the resolve of a still scene settling +// to a stable image that stays close to the un-anti-aliased frame. +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import simd +@testable import UntoldEngine +import XCTest + +@MainActor +final class TemporalAntiAliasingTests: BaseRenderSetup { + private var camera: EntityID = 0 + private var savedMode: AntiAliasingMode = .fxaa + + override func setUp() async throws { + try await super.setUp() + savedMode = antiAliasingMode + } + + override func tearDown() async throws { + antiAliasingMode = savedMode + TemporalAntiAliasing.shared.applyJitter() + TemporalAntiAliasing.shared.invalidateHistory() + destroyAllEntities() + try await super.tearDown() + } + + override func initializeAssets() { + camera = createEntity() + createGameCamera(entityId: camera) + CameraSystem.shared.activeCamera = camera + cameraLookAt(entityId: camera, eye: simd_float3(0, 1, 4), target: .zero, up: simd_float3(0, 1, 0)) + let box = createEntity() + setEntityMeshDirect(entityId: box, meshes: BasicPrimitives.createCube(extent: 1.0), assetName: "Box") + translateTo(entityId: box, position: simd_float3(0, 0.5, 0)) + let slab = createEntity() + setEntityMeshDirect(entityId: slab, meshes: BasicPrimitives.createCube(extent: 1.0), assetName: "Slab") + scaleTo(entityId: slab, scale: simd_float3(4, 0.05, 0.05)) + translateTo(entityId: slab, position: simd_float3(0, 1.6, -1)) + } + + // MARK: - Helpers + + private func drawFrame() { + renderer.draw(in: renderer.metalView) + renderInfo.lastCommandBuffer?.waitUntilCompleted() + } + + /// The resolve's output as floats, RGBA per pixel. + private func readAntiAliasingOutput() throws -> [Float] { + let texture = try XCTUnwrap(TemporalAntiAliasing.shared.resolvedTexture) + XCTAssertEqual(texture.pixelFormat, .rgba16Float) + return readFloats(texture) + } + + private func meanAbsoluteDifference(_ a: [Float], _ b: [Float]) -> Float { + var total: Float = 0 + for i in 0 ..< min(a.count, b.count) { + total += abs(a[i] - b[i]) + } + return total / Float(max(1, min(a.count, b.count))) + } + + // MARK: - Tests + + func testJitterSequenceIsSubPixelAndCentred() { + XCTAssertEqual(taaJitterSequence.count, 8) + var mean = simd_float2.zero + for offset in taaJitterSequence { + XCTAssertLessThanOrEqual(abs(offset.x), 0.5) + XCTAssertLessThanOrEqual(abs(offset.y), 0.5) + mean += offset + } + mean /= Float(taaJitterSequence.count) + XCTAssertLessThan(abs(mean.x), 0.1, "the offsets average near the pixel centre") + XCTAssertLessThan(abs(mean.y), 0.1) + XCTAssertEqual(Set(taaJitterSequence.map { "\($0.x),\($0.y)" }).count, 8, "eight distinct offsets") + } + + func testJitterGoesIntoAndOutOfTheProjection() { + antiAliasingMode = .fxaa + TemporalAntiAliasing.shared.applyJitter() + let base = renderInfo.perspectiveSpace + + antiAliasingMode = .taa + TemporalAntiAliasing.shared.applyJitter() + let jittered = renderInfo.perspectiveSpace + let pixels = TemporalAntiAliasing.shared.currentJitterPixels + XCTAssertNotEqual(pixels, .zero) + XCTAssertEqual(jittered.columns.2.x - base.columns.2.x, 2 * pixels.x / renderInfo.viewPort.x, accuracy: 1e-6) + XCTAssertEqual(jittered.columns.2.y - base.columns.2.y, 2 * pixels.y / renderInfo.viewPort.y, accuracy: 1e-6) + XCTAssertEqual(jittered.columns.0, base.columns.0) + XCTAssertEqual(jittered.columns.1, base.columns.1) + XCTAssertEqual(jittered.columns.3, base.columns.3) + XCTAssertEqual(TemporalAntiAliasing.shared.unjitteredProjection, base, "the unjittered projection is the base one") + + TemporalAntiAliasing.shared.applyJitter() + XCTAssertNotEqual(renderInfo.perspectiveSpace.columns.2.x, jittered.columns.2.x, "the next frame takes the next offset, not a sum") + + antiAliasingMode = .fxaa + TemporalAntiAliasing.shared.applyJitter() + XCTAssertEqual(renderInfo.perspectiveSpace, base, "leaving the temporal mode leaves the projection as it was") + } + + /// A rate map over the window whose dense column sits at `denseColumn` of four; the + /// three other columns are drawn at a quarter of the rate. Every column layout has the + /// same physical size, as the compositor's gaze-following maps do. + private func makeRateMap(denseColumn: Int) throws -> MTLRasterizationRateMap { + let device = try XCTUnwrap(renderInfo.device) + guard device.supportsRasterizationRateMap(layerCount: 1) else { + throw XCTSkip("this GPU has no rasterization rate maps") + } + let descriptor = MTLRasterizationRateMapDescriptor(screenSize: MTLSize(width: windowWidth, height: windowHeight, depth: 1)) + // Dense in one column and one row (a headset's map is dense around the gaze and coarse + // both ways in the periphery), a quarter of the rate elsewhere. + let layer = MTLRasterizationRateLayerDescriptor(sampleCount: MTLSize(width: 4, height: 4, depth: 1)) + for index in 0 ..< 4 { + layer.horizontal[index] = index == denseColumn ? 1.0 : 0.25 + layer.vertical[index] = index == denseColumn ? 1.0 : 0.25 + } + descriptor.setLayer(layer, at: 0) + return try XCTUnwrap(device.makeRasterizationRateMap(descriptor: descriptor)) + } + + private func makeFoveationFrame(_ rateMap: MTLRasterizationRateMap) throws -> XRFoveationFrame { + let device = try XCTUnwrap(renderInfo.device) + let buffer = try XCTUnwrap(device.makeBuffer(length: rateMap.parameterDataSizeAndAlign.size, options: .storageModeShared)) + let data = try XCTUnwrap(XRRasterizationRateMapData(rateMap: rateMap, buffer: buffer)) + let viewport = MTLViewport(originX: 0, originY: 0, width: Double(windowWidth), height: Double(windowHeight), znear: 0, zfar: 1) + return XRFoveationFrame(rateMap: rateMap, viewport: viewport, rateMapData: data) + } + + /// Reads a physical-size texture of the look format as floats. The history textures are + /// private to the GPU, so those are copied into a shared texture first. + private func readFloats(_ texture: MTLTexture) -> [Float] { + var readable = texture + if texture.storageMode == .private, + let device = renderInfo.device, + let commandBuffer = renderInfo.commandQueue.makeCommandBuffer(), + let blit = commandBuffer.makeBlitCommandEncoder() + { + let descriptor = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: texture.pixelFormat, width: texture.width, height: texture.height, mipmapped: false) + descriptor.storageMode = .shared + descriptor.usage = [.shaderRead] + guard let copy = device.makeTexture(descriptor: descriptor) else { + XCTFail("no staging texture") + return [] + } + blit.copy(from: texture, to: copy) + blit.endEncoding() + commandBuffer.commit() + commandBuffer.waitUntilCompleted() + readable = copy + } + var halves = [UInt16](repeating: 0, count: readable.width * readable.height * 4) + halves.withUnsafeMutableBytes { bytes in + readable.getBytes(bytes.baseAddress!, bytesPerRow: readable.width * 8, from: MTLRegionMake2D(0, 0, readable.width, readable.height), mipmapLevel: 0) + } + return halves.map { Float(Float16(bitPattern: $0)) } + } + + /// The headset's situation: a still scene drawn through a rate map whose dense zone moves + /// every frame, as a gaze-following map does. The history of each frame is laid out in the + /// previous map's physical space; reprojected through both maps it must land where this + /// frame's map puts the same point, or the resolve shows the scene with a displaced copy. + func testResolveStaysStableWhileTheRateMapsDenseZoneMoves() throws { + let maps = try [makeRateMap(denseColumn: 0), makeRateMap(denseColumn: 3)] + let physical = maps[0].physicalSize(layer: 0) + guard physical.width == maps[1].physicalSize(layer: 0).width, physical.height == maps[1].physicalSize(layer: 0).height else { + throw XCTSkip("the two layouts got different physical sizes on this GPU") + } + XCTAssertLessThan(physical.width, windowWidth, "three quarter-rate columns narrow the texture") + let frames = try maps.map { try makeFoveationFrame($0) } + + // Draw into textures of the physical size, through the maps, like a foveated eye. + let savedViewPort = renderInfo.viewPort + defer { + renderInfo.xrFoveation = nil + renderInfo.viewPort = savedViewPort + renderer.initSizeableResources() + TemporalAntiAliasing.shared.invalidateHistory() + } + renderInfo.viewPort = simd_float2(Float(physical.width), Float(physical.height)) + renderer.initSizeableResources() + + // The single-sample frame in the first map's layout. + antiAliasingMode = .none + renderInfo.xrFoveation = frames[0] + for _ in 0 ..< 2 { + drawFrame() + } + let single = try readFloats(XCTUnwrap(textureResources.lookTexture)) + XCTAssertGreaterThan(single.max() ?? 0, 0.01, "the scene draws something through the map") + + // The same frame in the second layout differs a lot from the first: the test can tell + // the layouts apart, so a history mapped through the wrong one would be caught. + renderInfo.xrFoveation = frames[1] + drawFrame() + let singleOther = try readFloats(XCTUnwrap(textureResources.lookTexture)) + let layoutDifference = meanAbsoluteDifference(single, singleOther) + XCTAssertGreaterThan(layoutDifference, 0.002, "the two layouts place the scene differently in the texture") + + // Temporal, with the dense zone jumping every frame, ending on the first layout. The + // jitter is off here so the frames are deterministic: what this checks is the history + // mapped through the maps, not the jitter's edge averaging. + antiAliasingMode = .taa + TemporalAntiAliasing.shared.invalidateHistory() + let savedJitterScale = TAAParams.shared.jitterScale + TAAParams.shared.jitterScale = 0 + defer { TAAParams.shared.jitterScale = savedJitterScale } + for frame in 0 ..< 12 { + let layout = (frame + 1) % 2 + renderInfo.xrFoveation = frames[layout] + drawFrame() + if frame >= 6 { + // Lit pixels that lost most of their colour against the single-sample frame of + // this layout: the thin bar's edge rows, where a history coarser than the frame + // on one axis overrode the frame until the rate ratio was taken per axis. A mean + // over the frame would not show it. + let frameResolved = try readFloats(XCTUnwrap(TemporalAntiAliasing.shared.resolvedTexture)) + let reference = layout == 0 ? single : singleOther + var lost = 0 + for i in stride(from: 0, to: min(frameResolved.count, reference.count), by: 4) where reference[i] + reference[i + 1] + reference[i + 2] > 0.6 { + let sum = frameResolved[i] + frameResolved[i + 1] + frameResolved[i + 2] + if sum < 0.4 * (reference[i] + reference[i + 1] + reference[i + 2]) { + lost += 1 + } + } + XCTAssertEqual(lost, 0, "frame \(frame) in layout \(layout): no lit pixel loses most of its colour to a history drawn through the other map") + } + } + let resolved = try readFloats(XCTUnwrap(TemporalAntiAliasing.shared.resolvedTexture)) + + // The floor: the resolve of the first layout held still differs from the single-sample + // frame by the jitter's averaging and the history filter alone. + TemporalAntiAliasing.shared.invalidateHistory() + renderInfo.xrFoveation = frames[0] + for _ in 0 ..< 12 { + drawFrame() + } + let still = try readFloats(XCTUnwrap(TemporalAntiAliasing.shared.resolvedTexture)) + let floor = meanAbsoluteDifference(still, single) + + XCTAssertFalse(resolved.contains { $0.isNaN || $0.isInfinite }) + let resolveDifference = meanAbsoluteDifference(resolved, single) + print("TAA moving-map test: resolve differs from the single-sample frame by \(resolveDifference), held still by \(floor), the other layout by \(layoutDifference)") + XCTAssertLessThan(resolveDifference, floor + 0.1 * layoutDifference, "the history reprojected through the moving maps lands where this frame's map puts the scene: resolve differs by \(resolveDifference), held still by \(floor), the other layout by \(layoutDifference)") + } + + /// A camera that moves a little every frame, as a head does: the history is reprojected + /// through the depth buffer and the two view-projections, and the resolve at the final + /// pose must match the single-sample frame at that pose rather than drag the earlier poses + /// along as a shadow. + func testResolveFollowsAMovingCameraWithoutAShadow() throws { + func pose(_ frame: Int) -> simd_float3 { + simd_float3(0.03 * Float(frame), 1 + 0.01 * Float(frame), 4 - 0.02 * Float(frame)) + } + let frameCount = 12 + + antiAliasingMode = .none + cameraLookAt(entityId: camera, eye: pose(0), target: .zero, up: simd_float3(0, 1, 0)) + drawFrame() + drawFrame() + let first = try readFloats(XCTUnwrap(textureResources.lookTexture)) + cameraLookAt(entityId: camera, eye: pose(frameCount - 1), target: .zero, up: simd_float3(0, 1, 0)) + drawFrame() + drawFrame() + let last = try readFloats(XCTUnwrap(textureResources.lookTexture)) + let motion = meanAbsoluteDifference(first, last) + XCTAssertGreaterThan(motion, 0.002, "the camera's path moves the scene in the frame") + + for mode in [AntiAliasingMode.taa, .msaaTaa] { + antiAliasingMode = mode + TemporalAntiAliasing.shared.invalidateHistory() + for frame in 0 ..< frameCount { + cameraLookAt(entityId: camera, eye: pose(frame), target: .zero, up: simd_float3(0, 1, 0)) + drawFrame() + } + let moved = try readAntiAliasingOutput() + TemporalAntiAliasing.shared.invalidateHistory() + for _ in 0 ..< frameCount { + drawFrame() + } + let held = try readAntiAliasingOutput() + print("TAA moving-camera \(mode): resolve differs from the final pose's frame by \(meanAbsoluteDifference(moved, last)), held still by \(meanAbsoluteDifference(held, last)), the first pose by \(motion)") + } + + antiAliasingMode = .taa + TemporalAntiAliasing.shared.invalidateHistory() + for frame in 0 ..< frameCount { + cameraLookAt(entityId: camera, eye: pose(frame), target: .zero, up: simd_float3(0, 1, 0)) + drawFrame() + } + let resolved = try readAntiAliasingOutput() + let shadow = meanAbsoluteDifference(resolved, last) + + // The floor: the resolve held still at the final pose. + TemporalAntiAliasing.shared.invalidateHistory() + for _ in 0 ..< frameCount { + drawFrame() + } + let stillResolved = try readAntiAliasingOutput() + let floor = meanAbsoluteDifference(stillResolved, last) + print("TAA moving-camera test: resolve differs from the final pose's frame by \(shadow), held still by \(floor), the first pose by \(motion)") + XCTAssertLessThan(shadow, floor + 0.1 * motion, "the resolve follows the camera: it differs from the final pose's frame by \(shadow), held still by \(floor), the first pose differs by \(motion)") + } + + func testResolveSettlesOnAStillSceneCloseToTheSingleSampleFrame() throws { + antiAliasingMode = .none + for _ in 0 ..< 2 { + drawFrame() + } + // The look output is what the output pass shows without a post-look pass. + let lookTexture = try XCTUnwrap(textureResources.lookTexture) + var reference = [UInt16](repeating: 0, count: lookTexture.width * lookTexture.height * 4) + reference.withUnsafeMutableBytes { bytes in + lookTexture.getBytes(bytes.baseAddress!, bytesPerRow: lookTexture.width * 8, from: MTLRegionMake2D(0, 0, lookTexture.width, lookTexture.height), mipmapLevel: 0) + } + let single = reference.map { Float(Float16(bitPattern: $0)) } + XCTAssertGreaterThan(single.max() ?? 0, 0.01, "the scene draws something") + + antiAliasingMode = .taa + TemporalAntiAliasing.shared.invalidateHistory() + for _ in 0 ..< 10 { + drawFrame() + } + let settled = try readAntiAliasingOutput() + drawFrame() + let next = try readAntiAliasingOutput() + + XCTAssertFalse(settled.contains { $0.isNaN || $0.isInfinite }, "no NaN or infinity in the resolve") + print("TAA still test: frame to frame \(meanAbsoluteDifference(settled, next)), resolve vs single-sample \(meanAbsoluteDifference(settled, single))") + // The frame-to-frame residue of a settled resolve under jitter: 0.0030 on an M-series + // Mac, 0.0042 on the continuous-integration runner's GPU; a drift or a smear is an + // order of magnitude more. + XCTAssertLessThan(meanAbsoluteDifference(settled, next), 0.005, "a still scene no longer changes between frames") + XCTAssertLessThan(meanAbsoluteDifference(settled, single), 0.02, "the resolve stays close to the single-sample frame: no drift, no smear") + XCTAssertGreaterThan(meanAbsoluteDifference(settled, single), 0, "and is not the single-sample frame") + } + + func testResolveWritesTheHistoryOnceAndPresentsIt() throws { + antiAliasingMode = .taa + TemporalAntiAliasing.shared.invalidateHistory() + + // The anti-aliasing texture is FXAA's and SMAA's output; the temporal resolve writes its + // frame once, as the eye's new history, and leaves that texture alone. + let antiAliasing = try XCTUnwrap(textureResources.antiAliasingTexture) + let marker = MTLClearColor(red: 0.25, green: 0.5, blue: 0.75, alpha: 1) + let descriptor = MTLRenderPassDescriptor() + descriptor.colorAttachments[0].texture = antiAliasing + descriptor.colorAttachments[0].loadAction = .clear + descriptor.colorAttachments[0].storeAction = .store + descriptor.colorAttachments[0].clearColor = marker + let fill = try XCTUnwrap(renderInfo.commandQueue.makeCommandBuffer()) + try XCTUnwrap(fill.makeRenderCommandEncoder(descriptor: descriptor)).endEncoding() + fill.commit() + fill.waitUntilCompleted() + + drawFrame() + drawFrame() + + let resolved = try XCTUnwrap(TemporalAntiAliasing.shared.resolvedTexture, "the resolve publishes what it wrote") + XCTAssertTrue(resolved.label?.hasPrefix("TAA History") ?? false, "the resolve's output is the eye's new history: \(resolved.label ?? "")") + XCTAssertFalse(resolved === antiAliasing) + let output = readFloats(resolved) + XCTAssertFalse(output.contains { $0.isNaN || $0.isInfinite }) + XCTAssertGreaterThan(output.max() ?? 0, 0.01, "the resolve holds the frame") + + let untouched = readFloats(antiAliasing) + for i in stride(from: 0, to: min(untouched.count, 400), by: 4) { + XCTAssertEqual(untouched[i], 0.25, accuracy: 1e-3) + XCTAssertEqual(untouched[i + 1], 0.5, accuracy: 1e-3) + XCTAssertEqual(untouched[i + 2], 0.75, accuracy: 1e-3) + } + } +} diff --git a/Tests/UntoldEngineRenderTests/TileLightStageTests.swift b/Tests/UntoldEngineRenderTests/TileLightStageTests.swift new file mode 100644 index 000000000..49349ecde --- /dev/null +++ b/Tests/UntoldEngineRenderTests/TileLightStageTests.swift @@ -0,0 +1,172 @@ +// +// TileLightStageTests.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Metal +import simd +@testable import UntoldEngine +import XCTest + +/// The deferred light pass as a tile stage lights each pixel once per colour it holds; the +/// quad lights every sample. Both must produce the same lit scene, with and without MSAA. +@MainActor +final class TileLightStageTests: BaseRenderSetup { + private var savedMode: AntiAliasingMode = .fxaa + private var savedPath: RenderPath = .deferred + + override func setUp() async throws { + try await super.setUp() + // The stage lights the indirect part itself: the geometry must not carry it. + savedPath = getRenderPath() + setRendering(.renderPath(.deferred)) + savedMode = antiAliasingMode + } + + override func tearDown() async throws { + setRendering(.tileLightStage(.enabled)) + setRendering(.renderPath(savedPath)) + setRendering(.multisampleCount(4)) + antiAliasingMode = savedMode + destroyAllEntities() + try await super.tearDown() + } + + override func initializeAssets() { + let camera = createEntity() + createGameCamera(entityId: camera) + CameraSystem.shared.activeCamera = camera + cameraLookAt(entityId: camera, eye: simd_float3(0, 1, 4), target: .zero, up: simd_float3(0, 1, 0)) + let box = createEntity() + setEntityMeshDirect(entityId: box, meshes: BasicPrimitives.createCube(extent: 1.0), assetName: "Box") + translateTo(entityId: box, position: simd_float3(0, 0.5, 0)) + // A thin bar at an angle: many edge pixels, where the tile stage lights per colour. + let bar = createEntity() + setEntityMeshDirect(entityId: bar, meshes: BasicPrimitives.createCube(extent: 1.0), assetName: "Bar") + scaleTo(entityId: bar, scale: simd_float3(4, 0.05, 0.05)) + rotateTo(entityId: bar, angle: 20, axis: simd_float3(0, 0, 1)) + translateTo(entityId: bar, position: simd_float3(0, 1.6, -1)) + let sun = createEntity() + createDirLight(entityId: sun) + rotateTo(entityId: sun, pitch: -40, yaw: 30, roll: 0) + applyIBL = true + } + + private var tileShadingAvailable: Bool { + PipelineManager.shared.renderPipelinesByType[.lightTile]?.success == true + } + + private func drawFrame() { + renderer.draw(in: renderer.metalView) + renderInfo.lastCommandBuffer?.waitUntilCompleted() + } + + private func readFloats(_ texture: MTLTexture) -> [Float] { + var halves = [UInt16](repeating: 0, count: texture.width * texture.height * 4) + halves.withUnsafeMutableBytes { bytes in + texture.getBytes(bytes.baseAddress!, bytesPerRow: texture.width * 8, from: MTLRegionMake2D(0, 0, texture.width, texture.height), mipmapLevel: 0) + } + return halves.map { Float(Float16(bitPattern: $0)) } + } + + /// Fills the lit scene with a colour no lighting produces, so that a pass that failed to + /// run (an encoder Metal refused) cannot pass as the previous frame's result. + private func markLitScene() throws { + let texture = try XCTUnwrap(textureResources.deferredColorMap) + let descriptor = MTLRenderPassDescriptor() + descriptor.colorAttachments[0].texture = texture + descriptor.colorAttachments[0].loadAction = .clear + descriptor.colorAttachments[0].storeAction = .store + descriptor.colorAttachments[0].clearColor = MTLClearColor(red: 1, green: 0, blue: 1, alpha: 1) + let commandBuffer = try XCTUnwrap(renderInfo.commandQueue.makeCommandBuffer()) + try XCTUnwrap(commandBuffer.makeRenderCommandEncoder(descriptor: descriptor)).endEncoding() + commandBuffer.commit() + commandBuffer.waitUntilCompleted() + } + + /// The lit scene after a few frames, through the tile stage or the quad. + private func litScene(tileStage: Bool, requireStage: Bool = true) throws -> [Float] { + setRendering(.tileLightStage(tileStage ? .enabled : .disabled)) + for _ in 0 ..< 3 { + try markLitScene() + drawFrame() + } + if requireStage || !tileStage { + XCTAssertEqual(renderInfo.tileLightStageUsed, tileStage, "the frame lit through the \(tileStage ? "tile stage" : "quad")") + } + let lit = try readFloats(XCTUnwrap(textureResources.deferredColorMap)) + var marked = 0 + for i in stride(from: 0, to: lit.count, by: 4) where lit[i] > 0.99 && lit[i + 1] < 0.01 && lit[i + 2] > 0.99 { + marked += 1 + } + XCTAssertEqual(marked, 0, "the pass ran and overwrote the marker") + return lit + } + + private func assertSameLitScene(requireStage: Bool = true, file: StaticString = #filePath, line: UInt = #line) throws { + let quad = try litScene(tileStage: false) + let tile = try litScene(tileStage: true, requireStage: requireStage) + XCTAssertGreaterThan(quad.max() ?? 0, 0.01, "the scene draws something", file: file, line: line) + var maxDifference: Float = 0 + var covered = 0 + for i in stride(from: 0, to: min(quad.count, tile.count), by: 4) where quad[i + 3] > 0 || tile[i + 3] > 0 { + covered += 1 + for c in 0 ..< 4 { + maxDifference = max(maxDifference, abs(quad[i + c] - tile[i + c])) + } + } + XCTAssertGreaterThan(covered, 1000, "the scene covers pixels", file: file, line: line) + XCTAssertLessThanOrEqual(maxDifference, 2e-3, "the tile stage and the quad light the same scene, edges included (max difference \(maxDifference))", file: file, line: line) + } + + func testTileStageLightsTheSameSceneAsTheQuad() throws { + try XCTSkipUnless(tileShadingAvailable, "tile shading is unavailable here") + antiAliasingMode = .none + try assertSameLitScene() + } + + func testTileStageLightsTheSameMultisampledSceneAsTheQuad() throws { + try XCTSkipUnless(tileShadingAvailable, "tile shading is unavailable here") + try XCTSkipUnless(renderInfo.device.supportsTextureSampleCount(2), "no 2ร— MSAA here") + // Two samples: the G-buffer's imageblock fits the tile memory of every Apple GPU. + setRendering(.multisampleCount(2)) + antiAliasingMode = .msaa + drawFrame() + XCTAssertEqual(renderInfo.opaqueSampleCount, 2) + let pipeline = try XCTUnwrap(PipelineManager.shared.renderPipelinesByType[.lightTile]) + XCTAssertEqual(pipeline.rasterSampleCount, 2, "the tile pipeline follows the pass's sample count") + try assertSameLitScene() + } + + /// Four samples of the G-buffer do not fit a tile on every GPU; where they do not, the pass + /// refuses the stage up front (asking Metal for it would assert under API validation) and + /// the quad lights the frame instead. + func testFourSamplesLightThroughTheStageOrFallBackToTheQuad() throws { + try XCTSkipUnless(tileShadingAvailable, "tile shading is unavailable here") + try XCTSkipUnless(renderInfo.device.supportsTextureSampleCount(4), "no 4ร— MSAA here") + antiAliasingMode = .msaa + drawFrame() + XCTAssertEqual(renderInfo.opaqueSampleCount, 4) + try assertSameLitScene(requireStage: false) + print("Tile light stage at 4 samples on this GPU: \(renderInfo.tileLightStageUsed ? "runs" : "refused, the quad lit the frame")") + } + + func testTileStageFollowsTheSetting() throws { + try XCTSkipUnless(tileShadingAvailable, "tile shading is unavailable here") + antiAliasingMode = .none + setRendering(.tileLightStage(.disabled)) + drawFrame() + XCTAssertFalse(renderInfo.tileLightStageUsed) + XCTAssertEqual(renderInfo.offscreenRenderPassDescriptor.imageblockSampleLength, 0, "the quad needs no imageblock") + setRendering(.tileLightStage(.enabled)) + drawFrame() + XCTAssertTrue(renderInfo.tileLightStageUsed) + XCTAssertEqual(renderInfo.offscreenRenderPassDescriptor.tileWidth, 0, "the tile size is left to Metal") + XCTAssertGreaterThan(renderInfo.offscreenRenderPassDescriptor.imageblockSampleLength, 0, "the stage tells the pass its imageblock") + } +} diff --git a/Tests/UntoldEngineRenderTests/TransparencyPassGateTests.swift b/Tests/UntoldEngineRenderTests/TransparencyPassGateTests.swift new file mode 100644 index 000000000..0e09cf4b5 --- /dev/null +++ b/Tests/UntoldEngineRenderTests/TransparencyPassGateTests.swift @@ -0,0 +1,72 @@ +// +// TransparencyPassGateTests.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Metal +import simd +@testable import UntoldEngine +import XCTest + +/// The transparency pass runs only on a frame with a transparent surface to draw: its +/// encoder loads and stores the lit colour and the depth of the whole frame, which a frame +/// with nothing transparent would pay to draw nothing. The GPU's own pass timing says which +/// passes ran. +@MainActor +final class TransparencyPassGateTests: BaseRenderSetup { + private var box: EntityID = .invalid + + override func setUp() async throws { + try await super.setUp() + setRendering(.gpuPassTiming(.enabled)) + } + + override func tearDown() async throws { + setRendering(.gpuPassTiming(.disabled)) + destroyAllEntities() + try await super.tearDown() + } + + override func initializeAssets() { + let camera = createEntity() + createGameCamera(entityId: camera) + CameraSystem.shared.activeCamera = camera + cameraLookAt(entityId: camera, eye: simd_float3(0, 1, 4), target: .zero, up: simd_float3(0, 1, 0)) + box = createEntity() + setEntityMeshDirect(entityId: box, meshes: BasicPrimitives.createCube(extent: 1.0), assetName: "Box") + translateTo(entityId: box, position: simd_float3(0, 0.5, 0)) + let sun = createEntity() + createDirLight(entityId: sun) + rotateTo(entityId: sun, pitch: -40, yaw: 30, roll: 0) + } + + /// The labels of the passes the last frame ran, after a few frames. + private func passesRun() -> Set { + for _ in 0 ..< 4 { + renderer.draw(in: renderer.metalView) + renderInfo.lastCommandBuffer?.waitUntilCompleted() + } + return Set(GPUPassTiming.shared.lastFrame().passes.map(\.label)) + } + + func testNothingTransparentNoPass() throws { + try XCTSkipUnless(renderInfo.device.supportsCounterSampling(.atStageBoundary), "no pass timing here") + var passes = passesRun() + XCTAssertFalse(passes.contains("Transparency"), "nothing transparent: no pass: \(passes.sorted())") + + // A blended material on the box: the pass has a surface to draw. + let render = try XCTUnwrap(scene.get(component: RenderComponent.self, for: box)) + for mesh in render.mesh.indices { + for submesh in render.mesh[mesh].submeshes.indices { + render.mesh[mesh].submeshes[submesh].material?.alphaMode = .blend + } + } + passes = passesRun() + XCTAssertTrue(passes.contains("Transparency"), "a blended surface: the pass runs: \(passes.sorted())") + } +} diff --git a/Tests/UntoldEngineRenderTests/XRFoveationTests.swift b/Tests/UntoldEngineRenderTests/XRFoveationTests.swift new file mode 100644 index 000000000..208f709be --- /dev/null +++ b/Tests/UntoldEngineRenderTests/XRFoveationTests.swift @@ -0,0 +1,204 @@ +// +// XRFoveationTests.swift +// UntoldEngine +// +// Drawing through a rasterization rate map (visionOS foveation, XRFoveation.swift): the map's +// parameter data the shaders decode, which pass descriptors carry the map and which do not, +// and the occlusion cull sampling a pyramid drawn through a map where the map put the rect, +// not where the rect's screen UV falls in the texture. +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import CShaderTypes +import Metal +import simd +@testable import UntoldEngine +import XCTest + +@MainActor +final class XRFoveationTests: BaseRenderSetup { + private let screen = 256 + + override func initializeAssets() {} + + override func tearDown() async throws { + renderInfo.xrFoveation = nil + applyXRFoveationToSceneRenderPassDescriptors() + try await super.tearDown() + } + + // MARK: - Helpers + + /// A map over a 256-pixel-square screen whose right half is drawn at a quarter of the rate + /// of its left half, so the physical texture is narrower than the screen and a screen + /// position in the right half lands at a different texel than its UV says. + private func makeRateMap() throws -> MTLRasterizationRateMap { + let device = try XCTUnwrap(renderInfo.device) + guard device.supportsRasterizationRateMap(layerCount: 1) else { + throw XCTSkip("this GPU has no rasterization rate maps") + } + let descriptor = MTLRasterizationRateMapDescriptor(screenSize: MTLSize(width: screen, height: screen, depth: 1)) + let layer = MTLRasterizationRateLayerDescriptor(sampleCount: MTLSize(width: 2, height: 1, depth: 1)) + layer.horizontal[0] = 1.0 + layer.horizontal[1] = 0.25 + layer.vertical[0] = 1.0 + descriptor.setLayer(layer, at: 0) + descriptor.label = "XRFoveationTests" + return try XCTUnwrap(device.makeRasterizationRateMap(descriptor: descriptor)) + } + + private func makeRateMapData(_ rateMap: MTLRasterizationRateMap) throws -> XRRasterizationRateMapData { + let device = try XCTUnwrap(renderInfo.device) + let buffer = try XCTUnwrap(device.makeBuffer(length: rateMap.parameterDataSizeAndAlign.size, options: .storageModeShared)) + return try XCTUnwrap(XRRasterizationRateMapData(rateMap: rateMap, buffer: buffer)) + } + + // MARK: - Tests + + func testRateMapDataCarriesTheSizesAndParameters() throws { + let rateMap = try makeRateMap() + let data = try makeRateMapData(rateMap) + + let physical = rateMap.physicalSize(layer: 0) + XCTAssertEqual(data.screenSize, simd_float2(Float(screen), Float(screen))) + XCTAssertEqual(data.physicalSize, simd_float2(Float(physical.width), Float(physical.height))) + XCTAssertLessThan(physical.width, screen, "a quarter-rate half narrows the physical texture") + XCTAssertEqual(data.sizes, simd_float4(Float(screen), Float(screen), Float(physical.width), Float(physical.height))) + XCTAssertEqual(rateMapSizes(nil), .zero, "the word the shaders read as 'no map'") + + let bytes = UnsafeRawBufferPointer(start: data.buffer.contents(), count: rateMap.parameterDataSizeAndAlign.size) + XCTAssertTrue(bytes.contains { $0 != 0 }, "the map's parameters were copied") + + let short = try XCTUnwrap(renderInfo.device.makeBuffer(length: 4, options: .storageModeShared)) + XCTAssertNil(XRRasterizationRateMapData(rateMap: rateMap, buffer: short), "a buffer too short for the parameters is refused") + } + + func testScenePassesCarryTheRateMapAndScreenPassesDoNot() throws { + let rateMap = try makeRateMap() + let data = try makeRateMapData(rateMap) + let viewport = MTLViewport(originX: 0, originY: 0, width: Double(screen), height: Double(screen), znear: 0, zfar: 1) + + renderInfo.xrFoveation = XRFoveationFrame(rateMap: rateMap, viewport: viewport, rateMapData: data) + applyXRFoveationToSceneRenderPassDescriptors() + + XCTAssertTrue(renderInfo.offscreenRenderPassDescriptor.rasterizationRateMap === rateMap, "the G-buffer + light pass draws through the map") + XCTAssertTrue(renderInfo.deferredRenderPassDescriptor.rasterizationRateMap === rateMap, "transparency, wireframe and debug draw through the map") + XCTAssertTrue(renderInfo.environmentRenderPassDescriptor.rasterizationRateMap === rateMap, "the background draws through the map") + XCTAssertTrue(renderInfo.gaussianRenderPassDescriptor.rasterizationRateMap === rateMap, "the splats draw through the map") + XCTAssertNil(renderInfo.sceneCompositeRenderPassDescriptor.rasterizationRateMap, "the composite copies physical pixels one to one") + XCTAssertNil(renderInfo.ssaoRenderPassDescriptor.rasterizationRateMap, "SSAO reads the physical depth one to one") + XCTAssertNil(renderInfo.postProcessRenderPassDescriptor?.rasterizationRateMap, "post-processing copies physical pixels one to one") + // A render target size is read in screen pixels under a map: the physical size would + // clip the scene to the top-left corner of the screen, so the size is left to the + // attachments while the map is attached. + XCTAssertEqual(renderInfo.offscreenRenderPassDescriptor.renderTargetWidth, 0) + XCTAssertEqual(renderInfo.deferredRenderPassDescriptor.renderTargetHeight, 0) + XCTAssertEqual(renderInfo.sceneCompositeRenderPassDescriptor.renderTargetWidth, Int(renderInfo.viewPort.x), "a screen pass keeps its texture-size target") + + renderInfo.xrFoveation = nil + applyXRFoveationToSceneRenderPassDescriptors() + XCTAssertNil(renderInfo.offscreenRenderPassDescriptor.rasterizationRateMap, "a uniform frame detaches the map") + XCTAssertEqual(renderInfo.offscreenRenderPassDescriptor.renderTargetWidth, Int(renderInfo.viewPort.x), "and puts the texture-size target back") + XCTAssertEqual(renderInfo.deferredRenderPassDescriptor.renderTargetHeight, Int(renderInfo.viewPort.y)) + XCTAssertNil(renderInfo.deferredRenderPassDescriptor.rasterizationRateMap) + XCTAssertNil(renderInfo.environmentRenderPassDescriptor.rasterizationRateMap) + XCTAssertNil(renderInfo.gaussianRenderPassDescriptor.rasterizationRateMap) + } + + /// A box in the right half of the screen, in front of a pyramid that holds an occluder only + /// where the map put that half. Read where the box's screen UV falls in the texture, the + /// pyramid shows background and the box is kept; moved through the map, the rect lies on + /// the occluder and the box is dropped. + func testOcclusionCullSamplesThePyramidWhereTheRateMapPutTheRect() throws { + let rateMap = try makeRateMap() + let data = try makeRateMapData(rateMap) + let device = try XCTUnwrap(renderInfo.device) + let physical = rateMap.physicalSize(layer: 0) + + // The box: NDC x in 0.5โ€ฆ0.9 โ€” screen pixels 192โ€ฆ243 of 256 โ€” a sliver of y, at depth 0.5. + let box = VisibleEntity(center: simd_float4(0.7, 0, 0.5, 1), halfExtent: simd_float4(0.2, 0.05, 0, 0), index: 1, version: 1) + // Where the rect's left edge lands in the texture by Metal's own mapping, against where + // its screen UV (0.75) would be read without the map. + let mappedMinX = rateMap.physicalCoordinates(screenCoordinates: MTLCoordinate2D(x: 192, y: 128), layer: 0).x + let unmappedMinX = 0.75 * Float(physical.width) + XCTAssertGreaterThan(mappedMinX, unmappedMinX + 2, "the quarter-rate half moves the rect to the right in the texture") + + // One mip: an occluder (0.9, near in reverse-Z) from just before the mapped rect to the + // right edge, background (0) everywhere else. + let occluderStart = max(0, Int(mappedMinX.rounded(.down)) - 1) + let pyramidDescriptor = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: .r32Float, width: physical.width, height: physical.height, mipmapped: false) + pyramidDescriptor.usage = [.shaderRead] + pyramidDescriptor.storageMode = .managed + let pyramid = try XCTUnwrap(device.makeTexture(descriptor: pyramidDescriptor)) + var row = [Float](repeating: 0, count: physical.width) + for x in occluderStart ..< physical.width { + row[x] = 0.9 + } + var texels = [Float]() + texels.reserveCapacity(physical.width * physical.height) + for _ in 0 ..< physical.height { + texels.append(contentsOf: row) + } + texels.withUnsafeBytes { bytes in + pyramid.replace( + region: MTLRegionMake2D(0, 0, physical.width, physical.height), + mipmapLevel: 0, + withBytes: bytes.baseAddress!, + bytesPerRow: physical.width * MemoryLayout.stride + ) + } + + // The frame state the cull reads, restored afterwards. + let savedViewPort = renderInfo.viewPort + let savedValid = renderInfo.hzbIsValid + let savedFrame = renderInfo.hzbFrame + let savedMipCount = renderInfo.hzbMipCount + let savedReverseZ = renderInfo.reverseZEnabled + let savedPyramid = textureResources.hzbDepthPyramid + defer { + renderInfo.viewPort = savedViewPort + renderInfo.hzbIsValid = savedValid + renderInfo.hzbFrame = savedFrame + renderInfo.hzbMipCount = savedMipCount + renderInfo.reverseZEnabled = savedReverseZ + textureResources.hzbDepthPyramid = savedPyramid + } + renderInfo.viewPort = simd_float2(Float(physical.width), Float(physical.height)) + renderInfo.hzbIsValid = true + renderInfo.hzbMipCount = 1 + renderInfo.reverseZEnabled = true + textureResources.hzbDepthPyramid = pyramid + + func survivors(rateMapData: XRRasterizationRateMapData?) throws -> Int { + renderInfo.hzbFrame = HZBPyramidFrame(viewProjection: matrix_identity_float4x4, cameraPosition: .zero, rateMapData: rateMapData) + var candidate = box + var candidateCount: UInt32 = 1 + let inputVisibility = try XCTUnwrap(device.makeBuffer(bytes: &candidate, length: MemoryLayout.stride, options: .storageModeShared)) + let inputCount = try XCTUnwrap(device.makeBuffer(bytes: &candidateCount, length: MemoryLayout.stride, options: .storageModeShared)) + let outputVisibility = try XCTUnwrap(device.makeBuffer(length: MemoryLayout.stride, options: .storageModeShared)) + let outputCount = try XCTUnwrap(device.makeBuffer(length: MemoryLayout.stride, options: .storageModeShared)) + + let commandBuffer = try XCTUnwrap(renderInfo.commandQueue.makeCommandBuffer()) + let ran = executeHZBOcclusionCulling( + commandBuffer, + cameraPosition: .zero, + dispatchCount: 1, + inputVisibilityBuffer: inputVisibility, + inputVisibleCountBuffer: inputCount, + outputVisibilityBuffer: outputVisibility, + outputVisibleCountBuffer: outputCount + ) + XCTAssertTrue(ran, "the pyramid is valid and the camera has not moved") + commandBuffer.commit() + commandBuffer.waitUntilCompleted() + return Int(outputCount.contents().load(as: UInt32.self)) + } + + XCTAssertEqual(try survivors(rateMapData: nil), 1, "read at its screen UV, the rect sees background and the box is kept") + XCTAssertEqual(try survivors(rateMapData: data), 0, "moved through the map, the rect lies on the occluder and the box is dropped") + } +} diff --git a/Tests/UntoldEngineRenderTests/XRImmersionContractTest.swift b/Tests/UntoldEngineRenderTests/XRImmersionContractTest.swift index 795d96722..fb6ab8e3f 100644 --- a/Tests/UntoldEngineRenderTests/XRImmersionContractTest.swift +++ b/Tests/UntoldEngineRenderTests/XRImmersionContractTest.swift @@ -111,6 +111,8 @@ final class XRImmersionContractTest: BaseRenderSetup { encoder.setFragmentBytes(&passthrough, length: MemoryLayout.stride, index: Int(prePassPassthroughBufferIndex.rawValue)) var ssaoEnabled = false encoder.setFragmentBytes(&ssaoEnabled, length: MemoryLayout.stride, index: Int(prePassSSAOEnabledIndex.rawValue)) + var gaussianLayerPresent = true // the (transparent) splat layer takes part in the composite + encoder.setFragmentBytes(&gaussianLayerPresent, length: MemoryLayout.stride, index: Int(prePassGaussianLayerIndex.rawValue)) encoder.drawIndexedPrimitives( type: .triangle, diff --git a/Tests/UntoldEngineRenderTests/XRProbeOrientationTests.swift b/Tests/UntoldEngineRenderTests/XRProbeOrientationTests.swift new file mode 100644 index 000000000..68a47fa3d --- /dev/null +++ b/Tests/UntoldEngineRenderTests/XRProbeOrientationTests.swift @@ -0,0 +1,143 @@ +// +// XRProbeOrientationTests.swift +// UntoldEngine +// +// The room probe's faces light the sides of the world they stand for: its +y face, the +// ceiling, lights what faces up; its +z face, behind the wearer, lights what faces them. +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Metal +import simd +@testable import UntoldEngine +import XCTest + +@MainActor +final class XRProbeOrientationTests: BaseRenderSetup { + private var savedMode: AntiAliasingMode = .fxaa + private var savedPath: RenderPath = .deferred + private var sphere: EntityID = 0 + + override func setUp() async throws { + try await super.setUp() + savedMode = antiAliasingMode + antiAliasingMode = .none + applyIBL = true + savedPath = getRenderPath() + setRendering(.renderPath(.forward)) + // The probe alone: the harness's scene brings a sun of its own. + LightingSystem.shared.activeDirectionalLight = nil + } + + override func tearDown() async throws { + RuntimeEnvironmentLightingStore.shared.publishXRLighting(nil) + RuntimeEnvironmentLightingStore.shared.setMode(.staticIBL) + setRendering(.renderPath(savedPath)) + antiAliasingMode = savedMode + destroyAllEntities() + try await super.tearDown() + } + + override func initializeAssets() { + let camera = createEntity() + createGameCamera(entityId: camera) + CameraSystem.shared.activeCamera = camera + cameraLookAt(entityId: camera, eye: .zero, target: simd_float3(0, 0, -1), up: simd_float3(0, 1, 0)) + // A white rough sphere 2 m ahead: what each side of it gets is the light from that side. + sphere = createEntity() + setEntityMeshDirect(entityId: sphere, meshes: BasicPrimitives.createSphere(extent: 0.5), assetName: "Sphere") + translateTo(entityId: sphere, position: simd_float3(0, 0, -2)) + updateMaterialColor(entityId: sphere, color: .white) + updateMaterialRoughness(entityId: sphere, roughness: 1) + updateMaterialMetallic(entityId: sphere, metallic: 0) + } + + /// A probe of one bright face, pre-filtered as the headset's room probe is, published as + /// the scene's light. + private func light(fromFace bright: Int) throws { + let device = try XCTUnwrap(renderInfo.device) + let descriptor = MTLTextureDescriptor.textureCubeDescriptor(pixelFormat: .rgba16Float, size: 8, mipmapped: false) + descriptor.usage = [.shaderRead] + descriptor.storageMode = .shared + let cube = try XCTUnwrap(device.makeTexture(descriptor: descriptor)) + for face in 0 ..< 6 { + let value = Float16(face == bright ? 1.0 : 0.02) + var texels = [UInt16](repeating: 0, count: 8 * 8 * 4) + for i in 0 ..< 64 { + texels[i * 4] = value.bitPattern + texels[i * 4 + 1] = value.bitPattern + texels[i * 4 + 2] = value.bitPattern + texels[i * 4 + 3] = Float16(1.0).bitPattern + } + texels.withUnsafeBytes { bytes in + cube.replace(region: MTLRegionMake2D(0, 0, 8, 8), mipmapLevel: 0, slice: face, withBytes: bytes.baseAddress!, bytesPerRow: 8 * 8, bytesPerImage: 8 * 8 * 8) + } + } + let set = try XCTUnwrap(makeRuntimeEnvironmentLightingTextureSet(labelPrefix: "Orientation test")) + let commandBuffer = try XCTUnwrap(renderInfo.commandQueue.makeCommandBuffer()) + XCTAssertTrue(executeXRIBLCubePreFilterPass(commandBuffer: commandBuffer, environmentCubeTexture: cube, target: set)) + commandBuffer.commit() + commandBuffer.waitUntilCompleted() + RuntimeEnvironmentLightingStore.shared.publishXRLighting(RuntimeEnvironmentLighting(irradianceMap: set.irradianceMap, specularMap: set.specularMap, brdfMap: set.brdfMap, intensityScale: 1, isValid: true)) + RuntimeEnvironmentLightingStore.shared.setMode(.realWorldEstimate) + for _ in 0 ..< 2 { + renderer.draw(in: renderer.metalView) + renderInfo.lastCommandBuffer?.waitUntilCompleted() + } + } + + /// The lit colour at a point of the frame. + private func lit(_ x: Int, _ y: Int) throws -> Float { + let texture = try XCTUnwrap(textureResources.deferredColorMap) + var halves = [UInt16](repeating: 0, count: 4) + halves.withUnsafeMutableBytes { bytes in + texture.getBytes(bytes.baseAddress!, bytesPerRow: 8, from: MTLRegionMake2D(x, y, 1, 1), mipmapLevel: 0) + } + return Float(Float16(bitPattern: halves[0])) + } + + /// The sphere's centre (its side facing the camera, +z), and its top, bottom, left and + /// right at three quarters of its radius on screen. + private func sides() throws -> (centre: Float, top: Float, bottom: Float, left: Float, right: Float) { + let texture = try XCTUnwrap(textureResources.deferredColorMap) + let width = texture.width, height = texture.height + // The sphere's radius on screen from the projection in use: its angular radius + // (0.25 at 2 m) times the projection's vertical scale, in half-heights. + let verticalScale = renderInfo.perspectiveSpace.columns.1.y + let r = Int(0.25 / 2 * verticalScale * Float(height) / 2 * 0.6) + let cx = width / 2, cy = height / 2 + return try (lit(cx, cy), lit(cx, cy - r), lit(cx, cy + r), lit(cx - r, cy), lit(cx + r, cy)) + } + + // A side facing the bright face gets its light whole, a side at right angles to it a + // share (the irradiance integrates the hemisphere), and the side facing away next to none. + + func testTheCeilingFaceLightsWhatFacesUp() throws { + try light(fromFace: 2) // +y + let s = try sides() + XCTAssertGreaterThan(s.top, s.bottom * 3, "lit from above") + XCTAssertGreaterThan(s.top, s.left * 2) + XCTAssertGreaterThan(s.top, s.right * 2) + } + + func testTheFaceBehindTheWearerLightsWhatFacesThem() throws { + try light(fromFace: 4) // +z: behind the camera, which looks down โˆ’z + let s = try sides() + XCTAssertGreaterThan(s.centre, s.top * 1.1, "lit from the camera's side") + XCTAssertGreaterThan(s.centre, s.bottom * 1.1) + XCTAssertGreaterThan(s.centre, s.left * 1.1) + XCTAssertGreaterThan(s.centre, s.right * 1.1) + } + + func testTheRightFaceLightsWhatFacesRight() throws { + try light(fromFace: 0) // +x + let s = try sides() + XCTAssertGreaterThan(s.right, s.left * 3, "lit from the right") + XCTAssertGreaterThan(s.right, s.top * 2) + XCTAssertGreaterThan(s.right, s.bottom * 2) + } +} diff --git a/Tests/UntoldEngineTests/BuildSystemTests.swift b/Tests/UntoldEngineTests/BuildSystemTests.swift index a96ea6c00..4800dfd56 100644 --- a/Tests/UntoldEngineTests/BuildSystemTests.swift +++ b/Tests/UntoldEngineTests/BuildSystemTests.swift @@ -11,6 +11,9 @@ import Foundation @testable import UntoldEngine import XCTest +#if canImport(Darwin) + import Darwin +#endif @MainActor final class BuildSystemTests: XCTestCase { @@ -426,9 +429,11 @@ final class BuildSystemTests: XCTestCase { XCTAssertTrue(yamlContent.contains("PRODUCT_BUNDLE_IDENTIFIER: \(bundleID)"), "XcodeGen spec should include bundle identifier") - // Verify GameData is also included as resources - XCTAssertTrue(yamlContent.contains("path: Sources/\(projectName)/GameData"), + // Verify GameData is also included as resources, at the project root + XCTAssertTrue(yamlContent.contains("path: GameData"), "XcodeGen spec should include GameData path") + XCTAssertFalse(yamlContent.contains("path: Sources/\(projectName)/GameData"), + "GameData should no longer be nested under Sources") // Verify project structure XCTAssertTrue(yamlContent.contains("name: \(projectName)"), @@ -1212,11 +1217,16 @@ final class BuildSystemTests: XCTestCase { "Should include visionOS source path") // All targets should share the same Sources folder - // Each target references it twice: once for the source path and once for GameData let sourcesPattern = "path: Sources/MyMultiPlatformGame" let occurrences = yamlContent.components(separatedBy: sourcesPattern).count - 1 - XCTAssertEqual(occurrences, 8, - "All 4 targets should reference shared Sources folder (2 refs per target)") + XCTAssertEqual(occurrences, 4, + "All 4 targets should reference shared Sources folder") + + // GameData lives at the project root, a sibling of Sources, referenced once per target + let gameDataPattern = "- path: GameData" + let gameDataOccurrences = yamlContent.components(separatedBy: gameDataPattern).count - 1 + XCTAssertEqual(gameDataOccurrences, 4, + "All 4 targets should reference the root-level GameData folder") } func testMultiPlatformWithTeamID() throws { @@ -1482,4 +1492,216 @@ final class BuildSystemTests: XCTestCase { "Single-platform should not have multi-platform folder structure") } } + + // MARK: - Project Layout Conversion + + func testConvertingToRootLevelGameDataSinglePlatform() throws { + let projectName = "MyGame" + let settings = BuildSettings( + projectName: projectName, bundleIdentifier: "com.test.game", outputPath: tempDirectory, + target: .macOS(deployment: .v15) + ) + let currentYAML = try XcodeGenProjectSpec.generateYAML(settings: settings) + // What the pre-root-level generator produced: GameData nested under + // Sources//, with the plain "Sources" entry excluding it. + let legacyYAML = currentYAML.replacingOccurrences( + of: " - path: Sources\n - path: GameData\n type: folder\n buildPhase: resources", + with: " - path: Sources\n excludes:\n - \"\(projectName)/GameData\"\n - path: Sources/\(projectName)/GameData\n type: folder\n buildPhase: resources" + ) + XCTAssertNotEqual(legacyYAML, currentYAML, "the legacy fixture should actually differ from current output") + + let converted = XcodeGenProjectSpec.convertingToRootLevelGameData(inYAML: legacyYAML, projectName: projectName) + XCTAssertEqual(converted, currentYAML, "converting the legacy spec should match what a fresh project gets today") + } + + func testConvertingToRootLevelGameDataMultiPlatform() throws { + let projectName = "MyMultiGame" + let settings = BuildSettings( + projectName: projectName, bundleIdentifier: "com.test.game", outputPath: tempDirectory, + target: .multi(macOS: .v15, iOS: .v17, visionOS: .v2) + ) + let currentYAML = try XcodeGenProjectSpec.generateYAML(settings: settings) + + let legacyYAML = currentYAML.replacingOccurrences( + of: "- path: GameData\n type: folder\n buildPhase: resources", + with: "- path: Sources/\(projectName)/GameData\n type: folder\n buildPhase: resources" + ) + XCTAssertNotEqual(legacyYAML, currentYAML, "the legacy fixture should actually differ from current output") + + let converted = XcodeGenProjectSpec.convertingToRootLevelGameData(inYAML: legacyYAML, projectName: projectName) + XCTAssertEqual(converted, currentYAML, "converting the legacy spec should match what a fresh project gets today") + } + + func testConvertingToRootLevelGameDataIsANoOpWhenAlreadyConverted() throws { + let settings = BuildSettings( + projectName: "MyGame", bundleIdentifier: "com.test.game", outputPath: tempDirectory, + target: .macOS(deployment: .v15) + ) + let yaml = try XcodeGenProjectSpec.generateYAML(settings: settings) + XCTAssertEqual(XcodeGenProjectSpec.convertingToRootLevelGameData(inYAML: yaml, projectName: "MyGame"), yaml) + } + + /// Builds a legacy-layout project on disk: GameData nested under + /// Sources//GameData with one scene file in it, and a legacy-shaped + /// project.yml (reconstructed the same way the YAML-only tests above do). + private func makeLegacyProject(named projectName: String) throws -> URL { + let projectDir = tempDirectory.appendingPathComponent(projectName) + let gameDataDir = projectDir.appendingPathComponent("Sources").appendingPathComponent(projectName).appendingPathComponent("GameData") + let scenesDir = gameDataDir.appendingPathComponent("Scenes") + try FileManager.default.createDirectory(at: scenesDir, withIntermediateDirectories: true) + try Data("fake scene".utf8).write(to: scenesDir.appendingPathComponent("Test.untoldscene")) + + let settings = BuildSettings( + projectName: projectName, bundleIdentifier: "com.test.game", outputPath: tempDirectory, + target: .macOS(deployment: .v15) + ) + let currentYAML = try XcodeGenProjectSpec.generateYAML(settings: settings) + let legacyYAML = currentYAML.replacingOccurrences( + of: " - path: Sources\n - path: GameData\n type: folder\n buildPhase: resources", + with: " - path: Sources\n excludes:\n - \"\(projectName)/GameData\"\n - path: Sources/\(projectName)/GameData\n type: folder\n buildPhase: resources" + ) + try legacyYAML.write(to: projectDir.appendingPathComponent("project.yml"), atomically: true, encoding: .utf8) + + return projectDir + } + + func testConvertProjectLayoutMovesGameDataAndCreatesResourcesAndDocs() throws { + let projectDir = try makeLegacyProject(named: "LegacyGame") + + let result = try buildSystem.convertProjectLayout(projectRoot: projectDir, projectName: "LegacyGame") + + let fm = FileManager.default + XCTAssertFalse(fm.fileExists(atPath: projectDir.appendingPathComponent("Sources/LegacyGame/GameData").path), + "the legacy GameData location should no longer exist") + let rootGameData = projectDir.appendingPathComponent("GameData") + XCTAssertTrue(fm.fileExists(atPath: rootGameData.appendingPathComponent("Scenes/Test.untoldscene").path), + "the moved GameData should keep its contents") + XCTAssertTrue(fm.fileExists(atPath: projectDir.appendingPathComponent("Resources").path)) + XCTAssertTrue(fm.fileExists(atPath: projectDir.appendingPathComponent("Docs").path)) + XCTAssertTrue(fm.fileExists(atPath: projectDir.appendingPathComponent(".gitignore").path)) + XCTAssertEqual(result.projectRoot, projectDir) + XCTAssertFalse(result.log.isEmpty) + } + + func testConvertProjectLayoutRewritesProjectYML() throws { + let projectDir = try makeLegacyProject(named: "LegacyGame2") + + try buildSystem.convertProjectLayout(projectRoot: projectDir, projectName: "LegacyGame2") + + let rewritten = try String(contentsOf: projectDir.appendingPathComponent("project.yml"), encoding: .utf8) + XCTAssertTrue(rewritten.contains("- path: GameData")) + XCTAssertFalse(rewritten.contains("Sources/LegacyGame2/GameData")) + } + + func testConvertProjectLayoutThrowsWhenAlreadyOnRootLevelLayout() throws { + let projectDir = tempDirectory.appendingPathComponent("AlreadyConverted") + try FileManager.default.createDirectory(at: projectDir.appendingPathComponent("GameData"), withIntermediateDirectories: true) + + XCTAssertThrowsError(try buildSystem.convertProjectLayout(projectRoot: projectDir, projectName: "AlreadyConverted")) { error in + guard case let BuildError.invalidProjectStructure(message) = error else { + return XCTFail("expected invalidProjectStructure, got \(error)") + } + XCTAssertTrue(message.contains("already")) + } + } + + func testConvertProjectLayoutThrowsWhenLegacyGameDataIsMissing() throws { + let projectDir = tempDirectory.appendingPathComponent("NoGameDataYet") + try FileManager.default.createDirectory(at: projectDir, withIntermediateDirectories: true) + + XCTAssertThrowsError(try buildSystem.convertProjectLayout(projectRoot: projectDir, projectName: "NoGameDataYet")) { error in + guard case BuildError.invalidProjectStructure = error else { + return XCTFail("expected invalidProjectStructure, got \(error)") + } + } + } + + func testConvertProjectLayoutThrowsWhenProjectDoesNotExist() throws { + let projectDir = tempDirectory.appendingPathComponent("NeverCreated") + + XCTAssertThrowsError(try buildSystem.convertProjectLayout(projectRoot: projectDir, projectName: "NeverCreated")) { error in + guard case BuildError.projectNotFound = error else { + return XCTFail("expected projectNotFound, got \(error)") + } + } + } + + func testConvertProjectLayoutVerifiesPackSourceAssetStillResolves() throws { + let projectDir = try makeLegacyProject(named: "LegacyGameWithPack") + let modelDir = projectDir + .appendingPathComponent("Sources").appendingPathComponent("LegacyGameWithPack") + .appendingPathComponent("GameData").appendingPathComponent("Models").appendingPathComponent("Tower") + try FileManager.default.createDirectory(at: modelDir, withIntermediateDirectories: true) + // Today's convention: the source sits beside the pack, referenced as a bare filename. + try Data("fake blend".utf8).write(to: modelDir.appendingPathComponent("Tower.blend")) + let packJSON = """ + {"formatVersion": 1, "sourceAsset": "Tower.blend", "models": [ + {"displayName": "Tower", "path": "Tower.untold", "transform": [[1,0,0,0],[0,1,0,0],[0,0,1,0],[0,0,0,1]]} + ]} + """ + try Data(packJSON.utf8).write(to: modelDir.appendingPathComponent("Tower.untoldpack")) + + let result = try buildSystem.convertProjectLayout(projectRoot: projectDir, projectName: "LegacyGameWithPack") + + // The pack and its source moved together with the rest of GameData, so + // sourceAsset -- relative to the pack's own folder -- still resolves with no + // rewrite needed. + let movedPack = projectDir + .appendingPathComponent("GameData").appendingPathComponent("Models").appendingPathComponent("Tower") + .appendingPathComponent("Tower.untoldpack") + XCTAssertTrue(FileManager.default.fileExists(atPath: movedPack.path)) + XCTAssertTrue(result.log.contains { $0.contains("1 pack manifest(s) checked") }) + } + + func testConvertProjectLayoutWarnsInsteadOfClaimingSuccessOnAHandEditedProjectYML() throws { + let projectDir = try makeLegacyProject(named: "LegacyGameHandEdited") + + // A project.yml that still points at the legacy GameData path, but not in the exact + // shape the generator produces (e.g. single quotes instead of double) -- the textual + // rewrite won't recognize this, and should say so rather than reporting the project + // as already on the root-level layout. + let projectYMLPath = projectDir.appendingPathComponent("project.yml") + let handEditedYAML = """ + name: LegacyGameHandEdited + targets: + LegacyGameHandEdited: + sources: + - path: 'Sources/LegacyGameHandEdited/GameData' + type: folder + """ + try handEditedYAML.write(to: projectYMLPath, atomically: true, encoding: .utf8) + + let result = try buildSystem.convertProjectLayout(projectRoot: projectDir, projectName: "LegacyGameHandEdited") + + XCTAssertFalse(result.log.contains { $0.contains("already matched the root-level layout") }, + "should not claim success when project.yml still references the legacy path") + XCTAssertTrue(result.log.contains { $0.contains("hand-edited project.yml?") }, + "should warn that project.yml needs manual attention") + } + + func testConvertProjectLayoutRollsBackGameDataMoveOnFailure() throws { + let projectDir = try makeLegacyProject(named: "LegacyGameRollback") + + // Make project.yml immutable so the rewrite step fails after GameData has already + // been moved to the root. Without a rollback, this would leave the project + // half-converted: GameData at the root, but project.yml still pointing at the legacy + // Sources//GameData path. + let projectYMLPath = projectDir.appendingPathComponent("project.yml") + #if canImport(Darwin) + chflags(projectYMLPath.path, UInt32(UF_IMMUTABLE)) + defer { chflags(projectYMLPath.path, 0) } + #endif + + XCTAssertThrowsError(try buildSystem.convertProjectLayout(projectRoot: projectDir, projectName: "LegacyGameRollback")) + + let fm = FileManager.default + XCTAssertTrue( + fm.fileExists(atPath: projectDir.appendingPathComponent("Sources/LegacyGameRollback/GameData/Scenes/Test.untoldscene").path), + "GameData should move back to the legacy location after the failure" + ) + XCTAssertFalse( + fm.fileExists(atPath: projectDir.appendingPathComponent("GameData").path), + "GameData should not be left at the root-level location after a rolled-back conversion" + ) + } } diff --git a/Tests/UntoldEngineTests/ColorPrimariesTests.swift b/Tests/UntoldEngineTests/ColorPrimariesTests.swift new file mode 100644 index 000000000..96139b4a4 --- /dev/null +++ b/Tests/UntoldEngineTests/ColorPrimariesTests.swift @@ -0,0 +1,32 @@ +// +// ColorPrimariesTests.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +@testable import UntoldEngine +import XCTest + +final class ColorPrimariesTests: XCTestCase { + func testEachChoiceSelectsItsShaderBranch() { + // The shader's ColorPrimariesID: sRGB 0, Display P3 1. + XCTAssertEqual(ColorPrimaries.sRGB.shaderValue, 0) + XCTAssertEqual(ColorPrimaries.displayP3.shaderValue, 1) + } + + func testTheSettingReachesTheLookPassAndDefaultsToSRGB() { + let saved = TonemapParams.shared.primaries + defer { setPostFX(.colorPrimaries(saved)) } + setPostFX(.colorPrimaries(.displayP3)) + XCTAssertEqual(TonemapParams.shared.primaries, .displayP3) + setPostFX(.colorPrimaries(.sRGB)) + XCTAssertEqual(TonemapParams.shared.primaries, .sRGB) + TonemapParams.shared.primaries = .displayP3 + TonemapParams.shared.resetToDefaults() + XCTAssertEqual(TonemapParams.shared.primaries, .sRGB, "the default") + } +} diff --git a/Tests/UntoldEngineTests/FrameClockTests.swift b/Tests/UntoldEngineTests/FrameClockTests.swift new file mode 100644 index 000000000..261302ddf --- /dev/null +++ b/Tests/UntoldEngineTests/FrameClockTests.swift @@ -0,0 +1,121 @@ +// +// FrameClockTests.swift +// UntoldEngineTests +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +@testable import UntoldEngine +import XCTest + +/// The frame clock turns the presentation times a loop predicts into each frame's time: the +/// delta between presentations, the period it learns and the refreshes skipped between frames. +final class FrameClockTests: XCTestCase { + private let period: TimeInterval = 1.0 / 100.0 + + func testTheFirstFrameHasNoDeltaAndNoPeriodYet() { + var clock = FrameClock() + let frame = clock.advance(toPresentation: 500.0) + XCTAssertEqual(frame.presentationTime, 500.0) + XCTAssertEqual(frame.deltaTime, 0) + XCTAssertEqual(frame.period, 0) + XCTAssertEqual(frame.skippedFrames, 0) + XCTAssertEqual(frame.frameIndex, 1) + XCTAssertFalse(frame.wasFrameSkipped) + XCTAssertEqual(clock.current, frame) + } + + func testASteadyCadenceStepsByOnePeriodAndSkipsNothing() { + var clock = FrameClock() + var frames: [FrameTime] = [] + for index in 0 ..< 5 { + frames.append(clock.advance(toPresentation: 500.0 + Double(index) * period)) + } + XCTAssertEqual(frames.map(\.frameIndex), [1, 2, 3, 4, 5]) + for frame in frames.dropFirst() { + XCTAssertEqual(frame.deltaTime, period, accuracy: 1e-9) + XCTAssertEqual(frame.period, period, accuracy: 1e-9) + XCTAssertEqual(frame.skippedFrames, 0) + } + } + + func testASkippedRefreshIsCountedAndTheNextFrameStepsByOnePeriodAgain() { + var clock = FrameClock() + // Slots 0, 1, 2, [3 missed], 4, 5. + let frames = [0, 1, 2, 4, 5].map { clock.advance(toPresentation: 500.0 + Double($0) * period) } + XCTAssertEqual(frames.map(\.skippedFrames), [0, 0, 0, 1, 0]) + XCTAssertTrue(frames[3].wasFrameSkipped) + XCTAssertEqual(frames[3].deltaTime, 2 * period, accuracy: 1e-9) + XCTAssertEqual(frames[4].deltaTime, period, accuracy: 1e-9) + // The skip did not teach the clock a longer period. + XCTAssertEqual(frames[4].period, period, accuracy: 1e-9) + } + + func testAJitteringCadenceIsNotASkip() { + var clock = FrameClock() + var time = 500.0 + var skipped = 0 + for index in 0 ..< 20 { + // The compositor's predictions wander by a tenth of a millisecond. + time += period + (index % 2 == 0 ? 0.0001 : -0.0001) + skipped += clock.advance(toPresentation: time).skippedFrames + } + XCTAssertEqual(skipped, 0) + XCTAssertEqual(clock.period, period - 0.0001, accuracy: 1e-9) + } + + func testAStallCountsEverySkippedRefresh() { + var clock = FrameClock() + clock.advance(toPresentation: 500.0) + clock.advance(toPresentation: 500.0 + period) + let frame = clock.advance(toPresentation: 500.0 + period + 0.5) + XCTAssertEqual(frame.deltaTime, 0.5, accuracy: 1e-9) + XCTAssertEqual(frame.skippedFrames, 49) + // A stall is not a period either. + XCTAssertEqual(clock.period, period, accuracy: 1e-9) + } + + func testANominalPeriodIsUsedInsteadOfLearningOne() { + var clock = FrameClock(nominalPeriod: 1.0 / 60.0) + let first = clock.advance(toPresentation: 500.0) + XCTAssertEqual(first.period, 1.0 / 60.0, accuracy: 1e-9) + // Two draws a third of a frame apart do not shorten the period; two frames apart is a skip. + let close = clock.advance(toPresentation: 500.0 + 1.0 / 180.0) + XCTAssertEqual(close.period, 1.0 / 60.0, accuracy: 1e-9) + XCTAssertEqual(close.skippedFrames, 0) + let late = clock.advance(toPresentation: 500.0 + 1.0 / 180.0 + 2.0 / 60.0) + XCTAssertEqual(late.skippedFrames, 1) + } + + func testTimeGoingBackwardsPassesNoTime() { + var clock = FrameClock() + clock.advance(toPresentation: 500.0) + clock.advance(toPresentation: 500.0 + period) + let frame = clock.advance(toPresentation: 500.0) + XCTAssertEqual(frame.deltaTime, 0) + XCTAssertEqual(frame.skippedFrames, 0) + } + + func testAnEstimateIsOneLeadAheadOfNow() { + var clock = FrameClock(nominalPeriod: 1.0 / 60.0) + let frame = clock.advance(now: 500.0, lead: 1.0 / 60.0) + XCTAssertEqual(frame.presentationTime, 500.0 + 1.0 / 60.0, accuracy: 1e-9) + let next = clock.advance(now: 500.0 + 1.0 / 60.0, lead: 1.0 / 60.0) + XCTAssertEqual(next.deltaTime, 1.0 / 60.0, accuracy: 1e-9) + } + + func testResetMakesTheNextFrameAFirstFrame() { + var clock = FrameClock() + clock.advance(toPresentation: 500.0) + clock.advance(toPresentation: 500.0 + period) + clock.reset() + XCTAssertNil(clock.current) + XCTAssertEqual(clock.period, 0) + let frame = clock.advance(toPresentation: 900.0) + XCTAssertEqual(frame.deltaTime, 0) + XCTAssertEqual(frame.frameIndex, 1) + } +} diff --git a/Tests/UntoldEngineTests/ProjectLayoutTests.swift b/Tests/UntoldEngineTests/ProjectLayoutTests.swift new file mode 100644 index 000000000..f071f82f1 --- /dev/null +++ b/Tests/UntoldEngineTests/ProjectLayoutTests.swift @@ -0,0 +1,128 @@ +// +// ProjectLayoutTests.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Foundation +@testable import UntoldEngine +import XCTest + +final class ProjectLayoutTests: XCTestCase { + var tempDirectory: URL! + + override func setUp() async throws { + tempDirectory = FileManager.default.temporaryDirectory + .appendingPathComponent("ProjectLayoutTests_\(UUID().uuidString)") + try? FileManager.default.createDirectory(at: tempDirectory, withIntermediateDirectories: true) + } + + override func tearDown() async throws { + if let tempDirectory { + try? FileManager.default.removeItem(at: tempDirectory) + } + } + + func testLegacyGameDataDirectory() { + let layout = ProjectLayout(kind: .legacy, projectRoot: tempDirectory, projectName: "MyGame") + + XCTAssertEqual( + layout.gameDataDirectory, + tempDirectory.appendingPathComponent("Sources").appendingPathComponent("MyGame").appendingPathComponent("GameData") + ) + XCTAssertNil(layout.resourcesDirectory, "legacy projects have no Resources folder") + XCTAssertNil(layout.docsDirectory, "legacy projects have no Docs folder") + } + + func testRootLevelGameDataDirectory() { + let layout = ProjectLayout(kind: .rootLevel, projectRoot: tempDirectory, projectName: "MyGame") + + XCTAssertEqual(layout.gameDataDirectory, tempDirectory.appendingPathComponent("GameData")) + XCTAssertEqual(layout.resourcesDirectory, tempDirectory.appendingPathComponent("Resources")) + XCTAssertEqual(layout.docsDirectory, tempDirectory.appendingPathComponent("Docs")) + } + + func testSourcesDirectoryIsTheSameShapeForBothLayouts() { + let legacy = ProjectLayout(kind: .legacy, projectRoot: tempDirectory, projectName: "MyGame") + let rootLevel = ProjectLayout(kind: .rootLevel, projectRoot: tempDirectory, projectName: "MyGame") + + XCTAssertEqual(legacy.sourcesDirectory, rootLevel.sourcesDirectory) + XCTAssertEqual(legacy.sourcesDirectory, tempDirectory.appendingPathComponent("Sources")) + } + + func testDetectDefaultsToLegacyWhenNeitherLayoutExistsYet() { + let layout = ProjectLayout.detect(projectRoot: tempDirectory, projectName: "MyGame") + XCTAssertEqual(layout.kind, .legacy) + } + + func testDetectFindsLegacyLayoutOnDisk() throws { + let gameData = tempDirectory + .appendingPathComponent("Sources") + .appendingPathComponent("MyGame") + .appendingPathComponent("GameData") + try FileManager.default.createDirectory(at: gameData, withIntermediateDirectories: true) + + let layout = ProjectLayout.detect(projectRoot: tempDirectory, projectName: "MyGame") + XCTAssertEqual(layout.kind, .legacy) + XCTAssertEqual(layout.gameDataDirectory.path, gameData.path) + } + + func testDetectFindsRootLevelLayoutOnDisk() throws { + let gameData = tempDirectory.appendingPathComponent("GameData") + try FileManager.default.createDirectory(at: gameData, withIntermediateDirectories: true) + + let layout = ProjectLayout.detect(projectRoot: tempDirectory, projectName: "MyGame") + XCTAssertEqual(layout.kind, .rootLevel) + XCTAssertEqual(layout.gameDataDirectory.path, gameData.path) + } + + func testDetectPrefersRootLevelWhenBothExist() throws { + // A project mid-conversion (or one where both happen to exist) should resolve + // to the new layout rather than silently using the stale legacy one. + let legacyGameData = tempDirectory + .appendingPathComponent("Sources") + .appendingPathComponent("MyGame") + .appendingPathComponent("GameData") + let rootGameData = tempDirectory.appendingPathComponent("GameData") + try FileManager.default.createDirectory(at: legacyGameData, withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: rootGameData, withIntermediateDirectories: true) + + let layout = ProjectLayout.detect(projectRoot: tempDirectory, projectName: "MyGame") + XCTAssertEqual(layout.kind, .rootLevel) + } + + func testDetectFromGameDataDirectoryHandlesLegacyLayout() { + let gameData = tempDirectory + .appendingPathComponent("Sources") + .appendingPathComponent("MyGame") + .appendingPathComponent("GameData") + + let layout = ProjectLayout.detect(gameDataDirectory: gameData) + XCTAssertEqual(layout.kind, .legacy) + XCTAssertEqual(layout.projectName, "MyGame") + XCTAssertEqual(layout.projectRoot.path, tempDirectory.path) + XCTAssertEqual(layout.gameDataDirectory.path, gameData.path, "round-tripping through gameDataDirectory should land back on the same path") + } + + func testDetectFromGameDataDirectoryHandlesRootLevelLayout() { + let projectRoot = tempDirectory.appendingPathComponent("MyGame") + let gameData = projectRoot.appendingPathComponent("GameData") + + let layout = ProjectLayout.detect(gameDataDirectory: gameData) + XCTAssertEqual(layout.kind, .rootLevel) + XCTAssertEqual(layout.projectName, "MyGame") + XCTAssertEqual(layout.projectRoot.path, projectRoot.path) + XCTAssertEqual(layout.gameDataDirectory.path, gameData.path, "round-tripping through gameDataDirectory should land back on the same path") + } + + func testGameDataCategoriesMatchesWhatBuildSystemCreates() { + XCTAssertEqual( + ProjectLayout.gameDataCategories, + ["Scenes", "Scripts", "Models", "StreamModels", "Animations", "Gaussians", "Textures", "Shaders", "LUT"] + ) + } +} diff --git a/Tests/UntoldEngineTests/RenderPathTests.swift b/Tests/UntoldEngineTests/RenderPathTests.swift new file mode 100644 index 000000000..fd2687e67 --- /dev/null +++ b/Tests/UntoldEngineTests/RenderPathTests.swift @@ -0,0 +1,100 @@ +// +// RenderPathTests.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +@testable import UntoldEngine +import XCTest + +/// The render path names what the two per-fragment settings compose. +final class RenderPathTests: XCTestCase { + private var savedIndirect = true + private var savedLit = true + private var savedChosen = false + + override func setUp() { + super.setUp() + savedIndirect = renderInfo.perFragmentIndirectLighting + savedLit = renderInfo.perFragmentLighting + savedChosen = renderInfo.renderPathChosen + } + + override func tearDown() { + renderInfo.perFragmentIndirectLighting = savedIndirect + renderInfo.perFragmentLighting = savedLit + renderInfo.renderPathChosen = savedChosen + super.tearDown() + } + + func testEachPathSetsTheTwoSettings() { + setRendering(.renderPath(.deferred)) + XCTAssertFalse(renderInfo.perFragmentIndirectLighting) + XCTAssertFalse(renderInfo.perFragmentLighting) + XCTAssertEqual(getRenderPath(), .deferred) + + setRendering(.renderPath(.hybrid)) + XCTAssertTrue(renderInfo.perFragmentIndirectLighting) + XCTAssertFalse(renderInfo.perFragmentLighting) + XCTAssertEqual(getRenderPath(), .hybrid) + + setRendering(.renderPath(.forward)) + XCTAssertTrue(renderInfo.perFragmentIndirectLighting) + XCTAssertTrue(renderInfo.perFragmentLighting) + XCTAssertEqual(getRenderPath(), .forward) + } + + func testTheSettingsSetOneByOneReadBackAsAPath() { + setRendering(.perFragmentIndirectLighting(.disabled)) + setRendering(.perFragmentLighting(.enabled)) + XCTAssertEqual(getRenderPath(), .deferred, "the direct light per fragment needs the indirect: without it the quad lights") + setRendering(.perFragmentIndirectLighting(.enabled)) + XCTAssertEqual(getRenderPath(), .forward) + setRendering(.perFragmentLighting(.disabled)) + XCTAssertEqual(getRenderPath(), .hybrid) + } + + func testTheDefaultPathIsThePlatforms() { + // The two settings' defaults (RenderResources) compose the platform's path: forward on + // visionOS, the headset's measured match for RealityKit's cost; deferred everywhere + // else, as the engine was built. + let defaults = RenderInfo() + #if os(visionOS) + XCTAssertEqual(RenderInfo.defaultRenderPath, .forward) + XCTAssertTrue(defaults.perFragmentIndirectLighting) + XCTAssertTrue(defaults.perFragmentLighting) + #else + XCTAssertEqual(RenderInfo.defaultRenderPath, .deferred) + XCTAssertFalse(defaults.perFragmentIndirectLighting) + XCTAssertFalse(defaults.perFragmentLighting) + #endif + XCTAssertFalse(defaults.renderPathChosen) + } + + func testTheXRLoopAdoptsForwardUnlessThePathWasChosen() { + // A Mac rendering a remote immersive space starts deferred like any Mac app and takes + // the headset's path when its XR loop starts; the adoption is not the app's choice. + renderInfo.renderPathChosen = false + renderInfo.perFragmentIndirectLighting = false + renderInfo.perFragmentLighting = false + adoptXRRenderPathUnlessChosen() + XCTAssertEqual(getRenderPath(), .forward) + XCTAssertFalse(renderInfo.renderPathChosen) + + // A path the app chose stays, whichever way it was set. + setRendering(.renderPath(.deferred)) + XCTAssertTrue(renderInfo.renderPathChosen) + adoptXRRenderPathUnlessChosen() + XCTAssertEqual(getRenderPath(), .deferred) + + renderInfo.renderPathChosen = false + setRendering(.perFragmentIndirectLighting(.enabled)) + XCTAssertTrue(renderInfo.renderPathChosen) + adoptXRRenderPathUnlessChosen() + XCTAssertEqual(getRenderPath(), .hybrid) + } +} diff --git a/Tests/UntoldEngineTests/ShadowCascadeTests.swift b/Tests/UntoldEngineTests/ShadowCascadeTests.swift index 21f1b17d9..463ec90a5 100644 --- a/Tests/UntoldEngineTests/ShadowCascadeTests.swift +++ b/Tests/UntoldEngineTests/ShadowCascadeTests.swift @@ -124,6 +124,8 @@ final class ShadowSystemMakeUniformsTests: XCTestCase { func testUsedSlotsCarryAssignedMatrices() { var sys = ShadowSystem() + // The matrices of a sun that casts: an updated system. + sys.isActive = true let m0 = sentinel(1.0) let m1 = sentinel(2.0) sys.cascadeLightSpaceMatrices[0] = m0 @@ -140,6 +142,21 @@ final class ShadowSystemMakeUniformsTests: XCTestCase { } } + /// A sun that casts no shadow (none active, or castsShadow off) draws no cascades and + /// the maps hold whatever was drawn last: its matrices land every point outside the maps, + /// which the shader reads as lit, so no stale shadow shows. + func testASunThatDoesNotCastShadowsNothing() { + var sys = ShadowSystem() + sys.cascadeLightSpaceMatrices[0] = sentinel(1.0) + XCTAssertFalse(sys.isActive) + let (r0, r1, _) = sys.makeUniforms().lightSpaceMatrices + XCTAssertEqual(r0.columns.3.x, 10.0, accuracy: 1e-6, "outside the map's [0, 1]") + XCTAssertEqual(r0.columns.3.w, 1.0, accuracy: 1e-6) + if csmCascadeCount > 1 { + XCTAssertEqual(r1.columns.3.x, 10.0, accuracy: 1e-6) + } + } + func testUnusedMatrixSlotIsIdentity() { // When csmCascadeCount < 3 the third matrix slot must default to identity // so the shader does not read garbage when it checks cascadeCount first. @@ -238,6 +255,30 @@ final class ShadowSystemMakeUniformsTests: XCTestCase { XCTAssertEqual(u.shadowSoftnessNear, 3.0, accuracy: 1e-6) XCTAssertEqual(u.shadowSoftnessFar, 7.5, accuracy: 1e-6) } + + func testTheFilterTakesTheWholeDiskByDefaultAndHalfOfItOnAHeadset() { + let originalXRMode = renderInfo.isXRStereoMode + defer { renderInfo.isXRStereoMode = originalXRMode } + + renderInfo.isXRStereoMode = false + XCTAssertEqual(ShadowSystem().makeUniforms().shadowFilterTaps, 16, accuracy: 1e-6) + renderInfo.isXRStereoMode = true + XCTAssertEqual(ShadowSystem().makeUniforms().shadowFilterTaps, 8, accuracy: 1e-6) + } + + func testTheTapCountsAreTheDiskOrEverySecondOrFourthPointOfIt() { + let originalXRMode = renderInfo.isXRStereoMode + defer { renderInfo.isXRStereoMode = originalXRMode } + + var sys = ShadowSystem() + for (asked, packed) in [(16, 16), (12, 8), (8, 8), (5, 4), (4, 4), (1, 4), (0, 4)] { + sys.setSoftness(ShadowSoftnessSettings(filterTaps: asked, xrFilterTaps: asked)) + renderInfo.isXRStereoMode = false + XCTAssertEqual(sys.makeUniforms().shadowFilterTaps, Float(packed), accuracy: 1e-6, "\(asked) taps asked") + renderInfo.isXRStereoMode = true + XCTAssertEqual(sys.makeUniforms().shadowFilterTaps, Float(packed), accuracy: 1e-6, "\(asked) taps asked, headset") + } + } } // MARK: - ShadowSystem.cascadeBlendStart diff --git a/Tests/UntoldEngineTests/ShadowSoftnessBeforeSystemTests.swift b/Tests/UntoldEngineTests/ShadowSoftnessBeforeSystemTests.swift new file mode 100644 index 000000000..bdc162df1 --- /dev/null +++ b/Tests/UntoldEngineTests/ShadowSoftnessBeforeSystemTests.swift @@ -0,0 +1,61 @@ +// +// ShadowSoftnessBeforeSystemTests.swift +// UntoldEngineTests +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +@testable import UntoldEngine +import XCTest + +/// A shadow softness set before the shadow system exists (before the renderer, or between two +/// sessions) is kept and given to the system when it is created, instead of crashing. +@MainActor +final class ShadowSoftnessBeforeSystemTests: XCTestCase { + private var previousSystem: ShadowSystem? + + override func setUp() { + previousSystem = shadowSystem + shadowSystem = nil + shadowSoftnessBeforeSystem = nil + } + + override func tearDown() { + shadowSystem = previousSystem + shadowSoftnessBeforeSystem = nil + } + + func testASettingBeforeTheSystemIsKeptAndGivenToItAtItsCreation() { + let wanted = ShadowSoftnessSettings(filterTaps: 4, xrFilterTaps: 4) + setShadowSoftness(wanted) + var reference = ShadowSystem() + reference.setSoftness(wanted) + XCTAssertEqual(getShadowSoftness(), reference.softnessSettings) + + shadowSystem = ShadowSystem() + applyShadowSoftnessSetBeforeSystem() + XCTAssertEqual(shadowSystem.softnessSettings, reference.softnessSettings) + XCTAssertNil(shadowSoftnessBeforeSystem) + + // With the system there, a setting goes to it directly. + let later = ShadowSoftnessSettings(filterTaps: 16, xrFilterTaps: 16) + setShadowSoftness(later) + var laterReference = ShadowSystem() + laterReference.setSoftness(later) + XCTAssertEqual(getShadowSoftness(), laterReference.softnessSettings) + XCTAssertNil(shadowSoftnessBeforeSystem) + } + + func testWithoutAnySettingTheDefaultIsReported() { + XCTAssertEqual(getShadowSoftness(), ShadowSystem().softnessSettings) + } + + func testASystemCreatedWithNothingKeptKeepsItsDefault() { + shadowSystem = ShadowSystem() + applyShadowSoftnessSetBeforeSystem() + XCTAssertEqual(shadowSystem.softnessSettings, ShadowSystem().softnessSettings) + } +} diff --git a/Tests/UntoldEngineTests/TestEngineReset.swift b/Tests/UntoldEngineTests/TestEngineReset.swift index ff8079a70..111b8b4bf 100644 --- a/Tests/UntoldEngineTests/TestEngineReset.swift +++ b/Tests/UntoldEngineTests/TestEngineReset.swift @@ -24,6 +24,7 @@ entityNameMap.removeAll() reverseEntityNameMap.removeAll() globalEntityCounter = 0 + currentFrameTime = .zero needsFinalizeDestroys = false hasPendingDestroys = false EngineExtensionRegistry.shared.removeAll() diff --git a/Tests/UntoldEngineTests/TonemapOperatorTests.swift b/Tests/UntoldEngineTests/TonemapOperatorTests.swift new file mode 100644 index 000000000..5341d4b75 --- /dev/null +++ b/Tests/UntoldEngineTests/TonemapOperatorTests.swift @@ -0,0 +1,30 @@ +// +// TonemapOperatorTests.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +@testable import UntoldEngine +import XCTest + +final class TonemapOperatorTests: XCTestCase { + func testEachOperatorSelectsItsShaderBranch() { + // The shader's TonemapOperatorID: ACES 0, AgX 1, none 2. + XCTAssertEqual(TonemapOperator.aces.shaderValue, 0) + XCTAssertEqual(TonemapOperator.agx.shaderValue, 1) + XCTAssertEqual(TonemapOperator.none.shaderValue, 2) + } + + func testTheSettingReachesTheLookPass() { + let saved = TonemapParams.shared.operator + defer { setPostFX(.tonemapOperator(saved)) } + setPostFX(.tonemapOperator(.none)) + XCTAssertEqual(TonemapParams.shared.operator, .none) + setPostFX(.tonemapOperator(.aces)) + XCTAssertEqual(TonemapParams.shared.operator, .aces) + } +} diff --git a/Tests/UntoldEngineTests/TransformTrackTests.swift b/Tests/UntoldEngineTests/TransformTrackTests.swift new file mode 100644 index 000000000..91d128fbe --- /dev/null +++ b/Tests/UntoldEngineTests/TransformTrackTests.swift @@ -0,0 +1,135 @@ +// +// TransformTrackTests.swift +// UntoldEngineTests +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import simd +@testable import UntoldEngine +import XCTest + +/// A transform track sampled at an absolute time: the pose for a frame depends on the frame's +/// presentation time alone. +final class TransformTrackTests: XCTestCase { + /// One metre per second along x for one second. + private let metrePerSecond = TransformTrack(keys: [ + TransformKey(time: 0, position: simd_float3(0, 0, 0)), + TransformKey(time: 1, position: simd_float3(1, 0, 0)), + ]) + + private func x(_ pose: TransformPose?) -> Float { + pose?.position.x ?? .nan + } + + func testPoseAtTheKeysAndBetweenThem() { + let turn = simd_quatf(angle: .pi / 2, axis: simd_float3(0, 1, 0)) + let track = TransformTrack(keys: [ + TransformKey(time: 2, position: simd_float3(2, 0, 0), rotation: turn, scale: simd_float3(repeating: 2)), + TransformKey(time: 0, position: simd_float3(0, 0, 0)), + ]) + // Keys are kept in time order whatever order they were given in. + XCTAssertEqual(track.keys.map(\.time), [0, 2]) + XCTAssertEqual(track.duration, 2) + XCTAssertEqual(x(track.pose(at: 0)), 0) + XCTAssertEqual(x(track.pose(at: 2)), 2) + let middle = track.pose(at: 1) + XCTAssertEqual(x(middle), 1, accuracy: 1e-6) + XCTAssertEqual(middle?.scale.x ?? .nan, 1.5, accuracy: 1e-6) + XCTAssertEqual(middle?.rotation.angle ?? .nan, .pi / 4, accuracy: 1e-5) + } + + func testThePoseHoldsOutsideTheKeysAndWrapsWhenLooping() { + XCTAssertEqual(x(metrePerSecond.pose(at: -1)), 0) + XCTAssertEqual(x(metrePerSecond.pose(at: 5)), 1) + let looping = TransformTrack(keys: metrePerSecond.keys, loops: true) + XCTAssertEqual(x(looping.pose(at: 1.25)), 0.25, accuracy: 1e-6) + XCTAssertEqual(x(looping.pose(at: -0.25)), 0.75, accuracy: 1e-6) + XCTAssertEqual(x(looping.pose(at: 1)), 0, accuracy: 1e-6) + XCTAssertNil(TransformTrack(keys: []).pose(at: 0)) + } + + func testEasingKeepsTheEndpointsAndShapesTheMiddle() { + for easing in [TrackEasing.linear, .easeIn, .easeOut, .easeInOut] { + XCTAssertEqual(easing.progress(0), 0) + XCTAssertEqual(easing.progress(1), 1) + XCTAssertEqual(easing.progress(-3), 0) + XCTAssertEqual(easing.progress(3), 1) + } + XCTAssertEqual(TrackEasing.linear.progress(0.25), 0.25) + XCTAssertEqual(TrackEasing.easeIn.progress(0.5), 0.25) + XCTAssertEqual(TrackEasing.easeOut.progress(0.5), 0.75) + XCTAssertEqual(TrackEasing.easeInOut.progress(0.5), 0.5) + XCTAssertLessThan(TrackEasing.easeInOut.progress(0.25), 0.25) + let eased = TransformTrack(keys: [ + TransformKey(time: 0, position: simd_float3(0, 0, 0), easing: .easeIn), + TransformKey(time: 1, position: simd_float3(1, 0, 0)), + ]) + XCTAssertEqual(x(eased.pose(at: 0.5)), 0.25, accuracy: 1e-6) + } + + func testThePoseForAFrameIsTakenAtItsPresentationTime() { + let frame = FrameTime(presentationTime: 1000.25, deltaTime: 0.01, period: 0.01, skippedFrames: 0, frameIndex: 7) + XCTAssertEqual(x(metrePerSecond.pose(for: frame, startedAt: 1000.0)), 0.25, accuracy: 1e-6) + } + + /// The display showed one frame twice (the engine missed a slot), so the next frame's + /// presentation is two periods on. A pose sampled at the frame's presentation time lands + /// where the timeline says, and the frame after it steps by one period again. An object + /// moved by the time between two updates โ€” the wall clock the update happens to run at, + /// a varying lag before each presentation โ€” steps unevenly every frame and is off the + /// timeline after the skip. + func testARepeatedFrameDoesNotDoubleStepAnAbsoluteTimeAnimation() { + let period: TimeInterval = 0.010 + let start: TimeInterval = 100.0 + // Presentation slots 0, 1, 2, [3 missed], 4, 5. + let slots = [0, 1, 2, 4, 5] + let presentations = slots.map { start + Double($0) * period } + // How long before its presentation each frame's update ran. + let lags: [TimeInterval] = [0.0063, 0.0071, 0.0058, 0.0094, 0.0066] + + var clock = FrameClock() + var frames: [FrameTime] = [] + var sampled: [Float] = [] + var stepped: [Float] = [] + var steppedPosition: Float = 0 + var previousUpdate: TimeInterval? + for (presentation, lag) in zip(presentations, lags) { + let frame = clock.advance(toPresentation: presentation) + frames.append(frame) + sampled.append(x(metrePerSecond.pose(for: frame, startedAt: start))) + + let updateTime = presentation - lag + if let previousUpdate { + steppedPosition += Float(updateTime - previousUpdate) // one metre per second + } + previousUpdate = updateTime + stepped.append(steppedPosition) + } + + XCTAssertEqual(frames.map(\.skippedFrames), [0, 0, 0, 1, 0]) + + // Sampled at presentation time: exactly the timeline's positions. + let expected = slots.map { Float($0) * Float(period) } + for (position, wanted) in zip(sampled, expected) { + XCTAssertEqual(position, wanted, accuracy: 1e-6) + } + // One period per frame, two across the skipped slot, one again after it. + let sampledSteps = zip(sampled.dropFirst(), sampled).map { $0 - $1 } + for (step, frame) in zip(sampledSteps, frames.dropFirst()) { + XCTAssertEqual(step, Float(period) * Float(1 + frame.skippedFrames), accuracy: 1e-6) + } + + // Stepped by the time between updates: no step is a period, and after the skip the + // position is off the timeline by the change in lag. + let steppedSteps = zip(stepped.dropFirst(), stepped).map { $0 - $1 } + for step in steppedSteps { + XCTAssertNotEqual(step, Float(period), accuracy: 0.0005) + XCTAssertNotEqual(step, Float(2 * period), accuracy: 0.0005) + } + XCTAssertNotEqual(stepped[3], expected[3], accuracy: 0.002) + } +} diff --git a/Tests/UntoldEngineTests/UntoldPackTests.swift b/Tests/UntoldEngineTests/UntoldPackTests.swift index 3c7b63f1d..2fc049672 100644 --- a/Tests/UntoldEngineTests/UntoldPackTests.swift +++ b/Tests/UntoldEngineTests/UntoldPackTests.swift @@ -83,6 +83,64 @@ final class UntoldPackTests: XCTestCase { XCTAssertEqual(pack.models[1].transform.columns.3.x, 2.5, accuracy: 0.0001) } + func testResolvedSourceAssetURLHandlesABareFilenameBesideThePack() throws { + // The exporter's original convention, still written for a pack whose source + // sits in the same folder: a bare filename is a relative path with zero + // intermediate directories, so it resolves the same way as any other. + let packURL = try writePack(named: "Robot", models: [ + (displayName: "Body", path: "Body/Body.untold", translationX: 0.0), + ]) + let pack = try XCTUnwrap(loadUntoldPack(url: packURL)) + + let resolved = try XCTUnwrap(pack.resolvedSourceAssetURL(packURL: packURL)) + XCTAssertEqual(resolved, tempRoot.appendingPathComponent("Robot.blend").standardizedFileURL) + } + + func testResolvedSourceAssetURLHandlesARelativePathIntoASeparateResourcesTree() throws { + // A pack at GameData/Models/Tower/Tower.untoldpack whose source lives at + // Resources/Models/Tower/Tower.blend under the same project root. + let packDir = tempRoot + .appendingPathComponent("GameData").appendingPathComponent("Models").appendingPathComponent("Tower") + try FileManager.default.createDirectory(at: packDir, withIntermediateDirectories: true) + let json = """ + { + "formatVersion": 1, + "sourceAsset": "../../../Resources/Models/Tower/Tower.blend", + "models": [ + { + "displayName": "Tower", + "path": "Tower.untold", + "transform": [[1,0,0,0],[0,1,0,0],[0,0,1,0],[0,0,0,1]] + } + ] + } + """ + let packURL = packDir.appendingPathComponent("Tower.untoldpack") + try Data(json.utf8).write(to: packURL) + + let pack = try XCTUnwrap(loadUntoldPack(url: packURL)) + let resolved = try XCTUnwrap(pack.resolvedSourceAssetURL(packURL: packURL)) + XCTAssertEqual( + resolved, + tempRoot.appendingPathComponent("Resources").appendingPathComponent("Models") + .appendingPathComponent("Tower").appendingPathComponent("Tower.blend").standardizedFileURL + ) + } + + func testResolvedSourceAssetURLIsNilWhenSourceAssetIsAbsent() throws { + let json = """ + { + "formatVersion": 1, + "models": [] + } + """ + let packURL = tempRoot.appendingPathComponent("NoSource.untoldpack") + try Data(json.utf8).write(to: packURL) + + let pack = try XCTUnwrap(loadUntoldPack(url: packURL)) + XCTAssertNil(pack.resolvedSourceAssetURL(packURL: packURL)) + } + func testCreateUntoldSceneWritesOneEntityPerModelWithResolvedTransform() throws { let packURL = try writePack(named: "Robot", models: [ (displayName: "Body", path: "Body/Body.untold", translationX: 0.0), diff --git a/Tests/UntoldEngineTests/XRHalfRatePolicyTests.swift b/Tests/UntoldEngineTests/XRHalfRatePolicyTests.swift new file mode 100644 index 000000000..c0a8a3588 --- /dev/null +++ b/Tests/UntoldEngineTests/XRHalfRatePolicyTests.swift @@ -0,0 +1,168 @@ +// +// XRHalfRatePolicyTests.swift +// UntoldEngineTests +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +@testable import UntoldEngine +import XCTest + +/// The half-rate pacing enters after two seconds of skipped refreshes or missed frames with no +/// wait left to skip, spaces the frames by the refreshes their GPU work with its encode needs, +/// aims the pose where the frames land past the deadline's margin, and leaves after two seconds +/// in which the frames' GPU work fitted most of a refresh. +final class XRHalfRatePolicyTests: XCTestCase { + private let period = 10.0 + private let margin = 15.0 + + private func lateSecond(_ policy: inout XRHalfRatePolicy, waitStillToSkip: Bool = false, meanGPU: Double = 12.0) -> XRHalfRatePolicy.Change { + policy.secondEnded(frames: 100, missed: 30, skipped: 5, meanOverrunMs: 14.0, meanGPUDurationMs: meanGPU, periodMs: period, deadlineMarginMs: margin, waitStillToSkip: waitStillToSkip) + } + + private func goodSecond(_ policy: inout XRHalfRatePolicy, meanGPU: Double = 7.0) -> XRHalfRatePolicy.Change { + policy.secondEnded(frames: 100, missed: 0, skipped: 0, meanOverrunMs: 0.2, meanGPUDurationMs: meanGPU, periodMs: period, deadlineMarginMs: margin, waitStillToSkip: false) + } + + private func pacedSecond(_ policy: inout XRHalfRatePolicy, overrun: Double, meanGPU: Double) -> XRHalfRatePolicy.Change { + policy.secondEnded(frames: 40, missed: 0, skipped: 0, meanOverrunMs: overrun, meanGPUDurationMs: meanGPU, periodMs: period, deadlineMarginMs: margin, waitStillToSkip: false) + } + + func testEntersAfterTwoLateSecondsNotOne() { + var policy = XRHalfRatePolicy() + XCTAssertEqual(lateSecond(&policy), .none) + XCTAssertFalse(policy.active) + XCTAssertEqual(lateSecond(&policy), .entered) + XCTAssertTrue(policy.active) + } + + func testAGoodSecondBetweenTwoLateOnesStartsTheCountAgain() { + var policy = XRHalfRatePolicy() + XCTAssertEqual(lateSecond(&policy), .none) + XCTAssertEqual(goodSecond(&policy), .none) + XCTAssertEqual(lateSecond(&policy), .none) + XCTAssertFalse(policy.active) + } + + func testDoesNotEnterWhileAWaitCouldStillBeSkipped() { + var policy = XRHalfRatePolicy() + XCTAssertEqual(lateSecond(&policy, waitStillToSkip: true), .none) + XCTAssertEqual(lateSecond(&policy, waitStillToSkip: true), .none) + XCTAssertFalse(policy.active) + } + + func testDoesNotEnterForFramesWithinTheDeadlinesMargin() { + var policy = XRHalfRatePolicy() + // Past the deadline but within its margin, nothing skipped: shown on time. + for _ in 0 ..< 3 { + XCTAssertEqual(policy.secondEnded(frames: 100, missed: 0, skipped: 0, meanOverrunMs: 4.0, meanGPUDurationMs: 10.2, periodMs: period, deadlineMarginMs: margin, waitStillToSkip: false), .none) + } + XCTAssertFalse(policy.active) + // Fewer than one frame in ten missed or a refresh skipped: still below the bar. + for _ in 0 ..< 3 { + XCTAssertEqual(policy.secondEnded(frames: 100, missed: 9, skipped: 9, meanOverrunMs: 4.0, meanGPUDurationMs: 10.2, periodMs: period, deadlineMarginMs: margin, waitStillToSkip: false), .none) + } + XCTAssertFalse(policy.active) + } + + func testEntersOnSkippedRefreshesAlone() { + var policy = XRHalfRatePolicy() + XCTAssertEqual(policy.secondEnded(frames: 100, missed: 0, skipped: 10, meanOverrunMs: 2.0, meanGPUDurationMs: 10.5, periodMs: period, deadlineMarginMs: margin, waitStillToSkip: false), .none) + XCTAssertEqual(policy.secondEnded(frames: 100, missed: 0, skipped: 10, meanOverrunMs: 2.0, meanGPUDurationMs: 10.5, periodMs: period, deadlineMarginMs: margin, waitStillToSkip: false), .entered) + XCTAssertTrue(policy.active) + } + + func testWhileActiveTheCadenceFollowsTheTypicalGPUCostWithItsEncode() { + var policy = XRHalfRatePolicy() + _ = lateSecond(&policy) + _ = lateSecond(&policy) + XCTAssertTrue(policy.active) + XCTAssertEqual(policy.cadenceRefreshes, 2) + // A mean of 14 ms, a fifth more and 3 ms of encode: 19.8 ms, two refreshes of 10 ms. + XCTAssertEqual(pacedSecond(&policy, overrun: 8.0, meanGPU: 14.0), .none) + XCTAssertEqual(policy.cadenceRefreshes, 2) + // A mean of 14.5 ms: 20.4 ms, within the deadband past two refreshes, still two. + XCTAssertEqual(pacedSecond(&policy, overrun: 8.0, meanGPU: 14.5), .none) + XCTAssertEqual(policy.cadenceRefreshes, 2) + // A mean of 16 ms: 22.2 ms, three. + XCTAssertEqual(pacedSecond(&policy, overrun: 8.0, meanGPU: 16.0), .none) + XCTAssertEqual(policy.cadenceRefreshes, 3) + // Back to 14.5 ms, 20.4 ms, just under three refreshes but within the deadband of two: stays three. + XCTAssertEqual(pacedSecond(&policy, overrun: 8.0, meanGPU: 14.5), .none) + XCTAssertEqual(pacedSecond(&policy, overrun: 8.0, meanGPU: 14.5), .none) + XCTAssertEqual(policy.cadenceRefreshes, 3) + // Never more than four. + XCTAssertEqual(pacedSecond(&policy, overrun: 8.0, meanGPU: 60.0), .none) + XCTAssertEqual(policy.cadenceRefreshes, 4) + // A shorter cadence is taken only after two seconds running that fit it clearly. + XCTAssertEqual(pacedSecond(&policy, overrun: 8.0, meanGPU: 9.0), .none) + XCTAssertEqual(policy.cadenceRefreshes, 4) + XCTAssertEqual(pacedSecond(&policy, overrun: 8.0, meanGPU: 9.0), .none) + XCTAssertEqual(policy.cadenceRefreshes, 2) + // A longer one at once, and a single fitting second between two needing it does not shorten it. + XCTAssertEqual(pacedSecond(&policy, overrun: 8.0, meanGPU: 16.0), .none) + XCTAssertEqual(policy.cadenceRefreshes, 3) + XCTAssertEqual(pacedSecond(&policy, overrun: 8.0, meanGPU: 12.0), .none) + XCTAssertEqual(pacedSecond(&policy, overrun: 8.0, meanGPU: 16.0), .none) + XCTAssertEqual(policy.cadenceRefreshes, 3) + XCTAssertTrue(policy.active) + } + + func testThePoseAimsAtTheRefreshTheOverrunReachesBeyondTheMargin() { + var policy = XRHalfRatePolicy() + XCTAssertEqual(policy.landingRefreshes, 0) + _ = lateSecond(&policy) + _ = lateSecond(&policy) + // Finished 9 ms past the deadline, within its 15 ms margin less the safety: shown on time. + XCTAssertEqual(pacedSecond(&policy, overrun: 9.0, meanGPU: 14.0), .none) + XCTAssertEqual(policy.landingRefreshes, 0) + // 14 ms past, a millisecond beyond the usable margin: one refresh after the prediction. + XCTAssertEqual(pacedSecond(&policy, overrun: 14.0, meanGPU: 14.0), .none) + XCTAssertEqual(policy.landingRefreshes, 1) + // 28 ms past: two. + XCTAssertEqual(pacedSecond(&policy, overrun: 28.0, meanGPU: 14.0), .none) + XCTAssertEqual(policy.landingRefreshes, 2) + // Never beyond three. + XCTAssertEqual(pacedSecond(&policy, overrun: 90.0, meanGPU: 14.0), .none) + XCTAssertEqual(policy.landingRefreshes, 3) + XCTAssertEqual(XRHalfRatePolicy.usableMarginMs(deadlineMarginMs: 15.0), 13.0) + XCTAssertEqual(XRHalfRatePolicy.usableMarginMs(deadlineMarginMs: 1.0), 0.0) + XCTAssertEqual(goodSecond(&policy), .none) + XCTAssertEqual(goodSecond(&policy), .left) + XCTAssertEqual(policy.landingRefreshes, 0) + } + + func testLeavesAfterTwoSecondsThatFitMostOfARefresh() { + var policy = XRHalfRatePolicy() + _ = lateSecond(&policy) + _ = lateSecond(&policy) + XCTAssertEqual(goodSecond(&policy, meanGPU: 7.9), .none) + XCTAssertTrue(policy.active) + XCTAssertEqual(goodSecond(&policy, meanGPU: 7.9), .left) + XCTAssertFalse(policy.active) + XCTAssertEqual(policy.cadenceRefreshes, 2) + } + + func testAHeavySecondWhileActiveStartsTheFitCountAgain() { + var policy = XRHalfRatePolicy() + _ = lateSecond(&policy) + _ = lateSecond(&policy) + XCTAssertEqual(goodSecond(&policy, meanGPU: 7.0), .none) + XCTAssertEqual(goodSecond(&policy, meanGPU: 9.0), .none) + XCTAssertEqual(goodSecond(&policy, meanGPU: 7.0), .none) + XCTAssertTrue(policy.active) + XCTAssertEqual(goodSecond(&policy, meanGPU: 7.0), .left) + } + + func testAnEmptySecondChangesNothingAndResetForgetsEverything() { + var policy = XRHalfRatePolicy() + XCTAssertEqual(policy.secondEnded(frames: 0, missed: 0, skipped: 0, meanOverrunMs: 0, meanGPUDurationMs: 0, periodMs: period, deadlineMarginMs: margin, waitStillToSkip: false), .none) + _ = lateSecond(&policy) + _ = lateSecond(&policy) + policy.reset() + XCTAssertEqual(policy, XRHalfRatePolicy()) + } +} diff --git a/Tools/UntoldEngineCLI/Sources/UntoldEngineCLI/AssetInstaller.swift b/Tools/UntoldEngineCLI/Sources/UntoldEngineCLI/AssetInstaller.swift index 61d59d6aa..ad70274cd 100644 --- a/Tools/UntoldEngineCLI/Sources/UntoldEngineCLI/AssetInstaller.swift +++ b/Tools/UntoldEngineCLI/Sources/UntoldEngineCLI/AssetInstaller.swift @@ -182,13 +182,16 @@ enum AssetError: LocalizedError { return """ Could not find a GameData folder in the current directory. Run this command from your project root, or specify the path: - untoldengine assets install --output ~/Projects/MyGame/Sources/MyGame/GameData + untoldengine assets install --output ~/Projects/MyGame/GameData """ case let .ambiguousGameData(paths): let list = paths.map { " \($0.path)" }.joined(separator: "\n") return """ Found multiple GameData folders โ€” specify which one to use with --output: \(list) + A project mid-conversion (see `untoldengine convert`) can briefly have both the \ + root-level GameData and the old Sources//GameData; finish the conversion \ + or delete the stale copy before installing a pack. """ } } diff --git a/Tools/UntoldEngineCLI/Sources/UntoldEngineCLI/AssetsCommand.swift b/Tools/UntoldEngineCLI/Sources/UntoldEngineCLI/AssetsCommand.swift index 8716d1224..63febf4a7 100644 --- a/Tools/UntoldEngineCLI/Sources/UntoldEngineCLI/AssetsCommand.swift +++ b/Tools/UntoldEngineCLI/Sources/UntoldEngineCLI/AssetsCommand.swift @@ -62,7 +62,7 @@ struct AssetsCommand: AsyncParsableCommand { untoldengine assets install starter # Specify GameData path explicitly - untoldengine assets install soccer --output ~/Projects/MyGame/Sources/MyGame/GameData + untoldengine assets install soccer --output ~/Projects/MyGame/GameData # Overwrite existing files without prompting untoldengine assets install starter --force diff --git a/Tools/UntoldEngineCLI/Sources/UntoldEngineCLI/CLIHelpers.swift b/Tools/UntoldEngineCLI/Sources/UntoldEngineCLI/CLIHelpers.swift index e6f2b581c..95f116bb3 100644 --- a/Tools/UntoldEngineCLI/Sources/UntoldEngineCLI/CLIHelpers.swift +++ b/Tools/UntoldEngineCLI/Sources/UntoldEngineCLI/CLIHelpers.swift @@ -308,25 +308,32 @@ func generateBundleIdentifier(projectName: String) -> String { // MARK: - GameData Discovery -/// Searches `startingAt` and up to `depth` levels of subdirectories for a `GameData` folder. +/// Searches `startingAt` and up to `depth` levels of subdirectories for `GameData` folders. /// Skips hidden directories and `.build` to avoid slow traversal of tooling artifacts. +/// +/// Collects every match rather than stopping at the first one: a project mid-conversion +/// (see `untoldengine convert`) can briefly have both a root-level `GameData` and the old +/// `Sources//GameData`, and callers (e.g. `assets install`) need to see both to +/// raise a clear ambiguity error instead of silently picking one. func findGameDataDirectory(startingAt url: URL, depth: Int = 3) -> [URL] { let fm = FileManager.default + var results: [URL] = [] let candidate = url.appendingPathComponent("GameData") var isDir: ObjCBool = false if fm.fileExists(atPath: candidate.path, isDirectory: &isDir), isDir.boolValue { - return [candidate] + results.append(candidate) } guard depth > 0, let contents = try? fm.contentsOfDirectory(at: url, includingPropertiesForKeys: [.isDirectoryKey]) - else { return [] } + else { return results } - var results: [URL] = [] for item in contents { let name = item.lastPathComponent - guard !name.hasPrefix("."), name != "build" else { continue } + // Skip recursing into a matched GameData itself -- its own subfolders (Scenes, + // Scripts, ...) are never going to contain another GameData worth reporting. + guard !name.hasPrefix("."), name != "build", name != "GameData" else { continue } var itemIsDir: ObjCBool = false fm.fileExists(atPath: item.path, isDirectory: &itemIsDir) if itemIsDir.boolValue { diff --git a/Tools/UntoldEngineCLI/Sources/UntoldEngineCLI/ConvertCommand.swift b/Tools/UntoldEngineCLI/Sources/UntoldEngineCLI/ConvertCommand.swift new file mode 100644 index 000000000..d8898292c --- /dev/null +++ b/Tools/UntoldEngineCLI/Sources/UntoldEngineCLI/ConvertCommand.swift @@ -0,0 +1,112 @@ +// +// ConvertCommand.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +// +// ConvertCommand.swift +// UntoldEngineCLI +// +// Moves an existing project from the legacy layout (GameData nested under +// Sources//GameData) to the root-level one (GameData, Resources and +// Docs at the project root). +// + +import ArgumentParser +import Foundation +import UntoldEngine + +struct ConvertCommand: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "convert", + abstract: "Move a project from the legacy GameData layout to the root-level one", + discussion: """ + Moves GameData from Sources//GameData to the project root, creates empty + Resources/ and Docs/ folders there, writes a .gitignore if the project doesn't have + one yet, and rewrites project.yml (regenerating the Xcode project with xcodegen). + + Refuses to run if the project is already on the root-level layout, or if GameData + isn't where the legacy layout expects it. + + Examples: + # Convert the project in the current directory + untoldengine convert MyGame + + # Convert a project at a specific path + untoldengine convert ~/Projects/MyGame + """ + ) + + // MARK: - Arguments + + @Argument(help: "Project name or path to the project directory") + var project: String + + // MARK: - Execution + + func run() async throws { + printInfo("๐Ÿ” UntoldEngine Project Layout Converter") + print("") + + let projectURL = resolveProjectPath() + + guard FileManager.default.fileExists(atPath: projectURL.path) else { + throw ConvertError.projectNotFound(projectURL.path) + } + + printInfo("Project: \(projectURL.path)") + print("") + + do { + let result = try BuildSystem.shared.convertProjectLayout( + projectRoot: projectURL, + projectName: projectURL.lastPathComponent + ) { message in + printProgress(message) + } + + print("") + printSuccess("Converted \(result.projectRoot.lastPathComponent) to the root-level layout") + } catch { + print("") + throw ConvertError.conversionFailed(error.localizedDescription) + } + } + + // MARK: - Helper Methods + + private func resolveProjectPath() -> URL { + let resolved = resolvePath(project) + + // If the path doesn't end with the project name (user provided just name), + // assume it's in the default output directory, same convention as `update`. + if !resolved.path.contains("/") { + let homeDir = FileManager.default.homeDirectoryForCurrentUser + let defaultOutput = homeDir.appendingPathComponent("UntoldEngineBuilds") + return defaultOutput.appendingPathComponent(project) + } + + return resolved + } +} + +// MARK: - Convert Errors + +enum ConvertError: LocalizedError { + case projectNotFound(String) + case conversionFailed(String) + + var errorDescription: String? { + switch self { + case let .projectNotFound(path): + return "Project not found: \(path)" + case let .conversionFailed(message): + return "Conversion failed: \(message)" + } + } +} diff --git a/Tools/UntoldEngineCLI/Sources/UntoldEngineCLI/CreateCommand.swift b/Tools/UntoldEngineCLI/Sources/UntoldEngineCLI/CreateCommand.swift index e545b56e1..1795a6f1d 100644 --- a/Tools/UntoldEngineCLI/Sources/UntoldEngineCLI/CreateCommand.swift +++ b/Tools/UntoldEngineCLI/Sources/UntoldEngineCLI/CreateCommand.swift @@ -96,10 +96,8 @@ struct CreateCommand: AsyncParsableCommand { ) let projectDir = outputURL.appendingPathComponent(projectName) - let gameDataPath = projectDir - .appendingPathComponent("Sources") - .appendingPathComponent(projectName) - .appendingPathComponent("GameData") + let layout = ProjectLayout(kind: .rootLevel, projectRoot: projectDir, projectName: projectName) + let gameDataPath = layout.gameDataDirectory assetBasePath = gameDataPath diff --git a/Tools/UntoldEngineCLI/Sources/UntoldEngineCLI/Resources/manifest.json b/Tools/UntoldEngineCLI/Sources/UntoldEngineCLI/Resources/manifest.json index 46965868e..8066587f7 100644 --- a/Tools/UntoldEngineCLI/Sources/UntoldEngineCLI/Resources/manifest.json +++ b/Tools/UntoldEngineCLI/Sources/UntoldEngineCLI/Resources/manifest.json @@ -6,32 +6,32 @@ "name": "Starter Assets", "description": "Soccer field, goals, ball, and a sample scene.", "version": "1.0.0", - "downloadURL": "https://d8pyi1c08k1w.cloudfront.net/StarterPack.zip", - "size": "108 MB" + "downloadURL": "https://github.com/untoldengine/UntoldEngine/releases/download/assets-v2/StarterPack.zip", + "size": "3.9 MB" }, { "id": "starter-archviz", "name": "ArchViz Starter Assets", "description": "Interior/exterior architectural props and a sample scene.", "version": "1.0.0", - "downloadURL": "https://d8pyi1c08k1w.cloudfront.net/ArchvizPack.zip", - "size": "253.4 MB" + "downloadURL": "https://github.com/untoldengine/UntoldEngine/releases/download/assets-v2/ArchvizPack.zip", + "size": "419.0 MB" }, { "id": "starter-streamed-city", "name": "Streamed City Starter Assets", "description": "A low-poly city scene for the tile-based Geometry Streaming System.", "version": "1.0.0", - "downloadURL": "https://d8pyi1c08k1w.cloudfront.net/LowPolyCity.zip", - "size": "58.9 MB" + "downloadURL": "https://github.com/untoldengine/UntoldEngine/releases/download/assets-v2/LowPolyCity.zip", + "size": "57.4 MB" }, { "id": "starter-digital-twin", "name": "Digital Twin Starter Assets", "description": "Bedroom digital-twin scene with light portal and selectable devices.", "version": "1.0.0", - "downloadURL": "https://d8pyi1c08k1w.cloudfront.net/DigitalTwinPack.zip", - "size": "424.3 MB" + "downloadURL": "https://github.com/untoldengine/UntoldEngine/releases/download/assets-v2/DigitalTwinPack.zip", + "size": "416.5 MB" } ] } diff --git a/Tools/UntoldEngineCLI/Sources/UntoldEngineCLI/UntoldEngineCLI.swift b/Tools/UntoldEngineCLI/Sources/UntoldEngineCLI/UntoldEngineCLI.swift index 0f52a0b97..c64338565 100644 --- a/Tools/UntoldEngineCLI/Sources/UntoldEngineCLI/UntoldEngineCLI.swift +++ b/Tools/UntoldEngineCLI/Sources/UntoldEngineCLI/UntoldEngineCLI.swift @@ -16,6 +16,6 @@ struct UntoldEngineCLI: AsyncParsableCommand { commandName: "untoldengine", abstract: "UntoldEngine command-line tools", version: "0.1.0", - subcommands: [CreateCommand.self, UpdateCommand.self, AssetsCommand.self, ExportCommand.self, ExportTilesCommand.self, GaussianLinkCommand.self, TexbakeCommand.self, BakeLODsCommand.self, BakeMLDeformerCommand.self, BootstrapCommand.self, BlenderAddonCommand.self, StudioCommand.self] + subcommands: [CreateCommand.self, UpdateCommand.self, ConvertCommand.self, AssetsCommand.self, ExportCommand.self, ExportTilesCommand.self, GaussianLinkCommand.self, TexbakeCommand.self, BakeLODsCommand.self, BakeMLDeformerCommand.self, BootstrapCommand.self, BlenderAddonCommand.self, StudioCommand.self] ) } diff --git a/Tools/UntoldEngineCLI/Tests/UntoldEngineCLITests/FindGameDataDirectoryTests.swift b/Tools/UntoldEngineCLI/Tests/UntoldEngineCLITests/FindGameDataDirectoryTests.swift new file mode 100644 index 000000000..b51efd057 --- /dev/null +++ b/Tools/UntoldEngineCLI/Tests/UntoldEngineCLITests/FindGameDataDirectoryTests.swift @@ -0,0 +1,89 @@ +// +// FindGameDataDirectoryTests.swift +// UntoldEngineCLI +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Foundation +@testable import UntoldEngineCLI +import XCTest + +final class FindGameDataDirectoryTests: XCTestCase { + private var tempDirectory: URL! + + override func setUp() { + tempDirectory = FileManager.default.temporaryDirectory + .appendingPathComponent("FindGameDataDirectoryTests_\(UUID().uuidString)") + try? FileManager.default.createDirectory(at: tempDirectory, withIntermediateDirectories: true) + } + + /// `/var` is a symlink to `/private/var` on macOS, and `FileManager.contentsOfDirectory` + /// resolves it inconsistently depending on recursion depth. Normalize before comparing. + private func resolvedPath(_ url: URL) -> String { + url.resolvingSymlinksInPath().path + } + + override func tearDown() { + if let tempDirectory { + try? FileManager.default.removeItem(at: tempDirectory) + } + } + + func testFindsRootLevelGameData() throws { + let gameData = tempDirectory.appendingPathComponent("GameData") + try FileManager.default.createDirectory(at: gameData, withIntermediateDirectories: true) + + let found = findGameDataDirectory(startingAt: tempDirectory) + XCTAssertEqual(found.map(resolvedPath), [resolvedPath(gameData)]) + } + + func testFindsLegacyNestedGameData() throws { + let gameData = tempDirectory + .appendingPathComponent("Sources") + .appendingPathComponent("MyGame") + .appendingPathComponent("GameData") + try FileManager.default.createDirectory(at: gameData, withIntermediateDirectories: true) + + let found = findGameDataDirectory(startingAt: tempDirectory) + XCTAssertEqual(found.map(resolvedPath), [resolvedPath(gameData)]) + } + + func testReturnsEmptyWhenNoGameDataExists() throws { + try FileManager.default.createDirectory( + at: tempDirectory.appendingPathComponent("Sources"), withIntermediateDirectories: true + ) + + XCTAssertTrue(findGameDataDirectory(startingAt: tempDirectory).isEmpty) + } + + func testFindsBothRootLevelAndLegacyGameDataWhenMidConversion() throws { + // A project between `untoldengine convert` moving the root GameData into place and + // deleting the stale legacy one should surface both, not silently pick one. + let rootGameData = tempDirectory.appendingPathComponent("GameData") + let legacyGameData = tempDirectory + .appendingPathComponent("Sources") + .appendingPathComponent("MyGame") + .appendingPathComponent("GameData") + try FileManager.default.createDirectory(at: rootGameData, withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: legacyGameData, withIntermediateDirectories: true) + + let found = Set(findGameDataDirectory(startingAt: tempDirectory).map(resolvedPath)) + XCTAssertEqual(found, [resolvedPath(rootGameData), resolvedPath(legacyGameData)]) + } + + func testDoesNotRecurseIntoAMatchedGameDataFolder() throws { + // A category folder literally named "GameData" inside GameData itself would be + // bizarre, but confirms the recursion-skip doesn't miss real matches either. + let gameData = tempDirectory.appendingPathComponent("GameData") + try FileManager.default.createDirectory( + at: gameData.appendingPathComponent("Scenes"), withIntermediateDirectories: true + ) + + let found = findGameDataDirectory(startingAt: tempDirectory) + XCTAssertEqual(found.map(resolvedPath), [resolvedPath(gameData)]) + } +} diff --git a/buildkernels.sh b/buildkernels.sh index 0c2209bee..dbf97e4d0 100755 --- a/buildkernels.sh +++ b/buildkernels.sh @@ -61,3 +61,12 @@ xcrun -sdk xros metallib "$airDir/UntoldEngineKernels-xros.air" -o UntoldEngineK xcrun -sdk xrsimulator metal -mtargetos=xros$XROS_MIN-simulator UntoldEngineKernels.metal -c -o "$airDir/UntoldEngineKernels-xrossim.air" xcrun -sdk xrsimulator metallib "$airDir/UntoldEngineKernels-xrossim.air" -o UntoldEngineKernels-xrossim.metallib + +# Stamp the source hash MetalKernelsStaleCheck compares against, so `swift build` can tell +# these metallibs are current. Covers UntoldEngineKernels.metal plus every file it #includes +# from ../Shaders (sorted, to match the plugin's directory listing order) since editing any +# of those changes the compiled kernel too. Commit this file along with the rebuilt metallibs. +{ + cat UntoldEngineKernels.metal + find ../Shaders -maxdepth 1 -name "*.metal" -print0 | LC_ALL=C sort -z | xargs -0 cat +} | shasum -a 256 | awk '{print $1}' > .metalsource.sha256 diff --git a/docs/API/GettingStarted.md b/docs/API/GettingStarted.md index 000683098..82ce5027c 100644 --- a/docs/API/GettingStarted.md +++ b/docs/API/GettingStarted.md @@ -27,7 +27,7 @@ Clone the repository and launch the demo: ```bash git clone https://github.com/untoldengine/UntoldEngine.git cd UntoldEngine -git checkout v0.22.0 +git checkout v0.23.0 swift run ShowcaseDemo ``` @@ -324,7 +324,7 @@ You can convert assets with either the Untold Engine Blender addon or the CLI. To convert a USDZ file into the `.untold` format using the add-on, follow the directions in [Using Blender Addon](UsingBlenderAddon.md). -After the model has been converted to `.untold` format, copy it into your Xcode project under `Sources//GameData/Models//`. +After the model has been converted to `.untold` format, copy it into your Xcode project under `GameData/Models//`. ### Option 2: CLI diff --git a/docs/API/UsingAssetPaths.md b/docs/API/UsingAssetPaths.md index 9b4314c35..bc8e9aad5 100644 --- a/docs/API/UsingAssetPaths.md +++ b/docs/API/UsingAssetPaths.md @@ -6,21 +6,28 @@ Every mesh, animation, texture, and scene a game loads by name โ€” `setEntityMes ## The `GameData` Folder -A project created with the CLI (`untoldengine create`) or Xcode template ships with a `GameData` folder alongside your source: +A project created with the CLI (`untoldengine create`) or Xcode template ships with a `GameData` folder at its root, alongside `Resources/`, `Docs/`, and `Sources/`: ``` -GameData/ -โ”œโ”€โ”€ Scenes/ -โ”œโ”€โ”€ Scripts/ -โ”œโ”€โ”€ Models/ -โ”œโ”€โ”€ StreamModels/ -โ”œโ”€โ”€ Animations/ -โ”œโ”€โ”€ Gaussians/ -โ”œโ”€โ”€ Textures/ -โ””โ”€โ”€ Shaders/ +MyGame/ +โ”œโ”€โ”€ GameData/ <- what the engine reads; bundled into the shipped app +โ”‚ โ”œโ”€โ”€ Scenes/ +โ”‚ โ”œโ”€โ”€ Scripts/ +โ”‚ โ”œโ”€โ”€ Models/ +โ”‚ โ”œโ”€โ”€ StreamModels/ +โ”‚ โ”œโ”€โ”€ Animations/ +โ”‚ โ”œโ”€โ”€ Gaussians/ +โ”‚ โ”œโ”€โ”€ Textures/ +โ”‚ โ”œโ”€โ”€ Shaders/ +โ”‚ โ””โ”€โ”€ LUT/ +โ”œโ”€โ”€ Resources/ <- artist sources (.blend, .usd, references); never bundled +โ”œโ”€โ”€ Docs/ <- project documentation +โ””โ”€โ”€ Sources/ <- your app's code ``` -These are the **canonical asset directories** the engine's resource resolver knows about. The project generator creates them for you, and Xcode adds `GameData` as a folder reference โ€” meaning it's copied into the built app **verbatim**, subfolders and all, on macOS, iOS, and visionOS. You don't need to worry about your directory structure surviving into a shipped build. +`GameData` is the only one of these four the engine's resource resolver (and this page) cares about โ€” `Resources/` and `Docs/` are for everything else your project needs a home for, and are never bundled into the app. A project created before this layout landed may still have `GameData` nested under `Sources//GameData` instead; run `untoldengine convert` to move it to the root (see [Using the Untold Engine CLI](UsingUntoldEngineCLI.md#converting-an-existing-project)). + +Inside `GameData`, Xcode adds it as a folder reference โ€” meaning it's copied into the built app **verbatim**, subfolders and all, on macOS, iOS, and visionOS. You don't need to worry about your directory structure surviving into a shipped build. Where each asset type belongs: @@ -34,6 +41,7 @@ Where each asset type belongs: | `Gaussians/` | Gaussian splat assets | | `Textures/` | standalone textures loaded by name, outside of a material | | `Shaders/` | custom shader source | +| `LUT/` | standalone `.cube` color-grade LUTs (see [Using Color Management](UsingColorManagement.md)) | Materials are the one exception: they resolve under `Materials//...`, keyed by the material name passed alongside the resource name, rather than a fixed top-level folder. diff --git a/docs/API/UsingCodeComponents.md b/docs/API/UsingCodeComponents.md index 8d3bb66e9..f06a26b32 100644 --- a/docs/API/UsingCodeComponents.md +++ b/docs/API/UsingCodeComponents.md @@ -366,3 +366,19 @@ ScenePluginSystem.shared.finishReload() // new instances, saved v ``` A type that no longer exists leaves its slot unbound with its values intact. + +## Common Failures + +These were found during development, in a packaged app or a generated game rather than in unit +tests, and are already fixed in the engine โ€” kept here because the fixes explain constraints +mentioned above (why `entity`, `isAttached` and `hasStarted` are `final`, for instance) and +because the symptoms are useful if you hit something similar on a custom build or packaging step. + +| Symptom | Cause | +| --- | --- | +| A component loads in a debug build from Xcode but fails to load in a packaged release build | A subclass in a loaded library copies its base class's dispatch table, internal members included, and release builds hide symbols that aren't exported. This is why `ScenePlugin`'s non-overridable members are `final` โ€” an internal member that isn't would break the same way. `scripts/verify-component-sdk.sh` checks this at packaging time against the bundled SDK. | +| `discoverInApp()` finds no plugin types at all | Xcode builds an app's code into `.debug.dylib` and leaves the executable itself as a stub, so scanning only the main executable finds nothing. Discovery must cover every image in the app bundle, which is what `discoverInApp()` does. | +| A library that was just `dlopen`ed isn't recognized by discovery | Path spelling can differ between how a library was opened and how the loader records it (`/private/var/...` vs `/var/...`). Discovery matches against the list of loaded images, not the literal path string. | +| An entity pending destruction never receives `onDetach()` | `scene.mask(for:)` hides entities already marked for destruction, so a lookup through the mask can't reach their instances to detach them. The cleanup path has to look the entity up by its component bit instead. | +| A packaged app crashes at launch when run outside the build tree | A packaging script that hardcodes a build path, or skips copying a resource bundle the editor needs, will work in the build tree and fail anywhere else. Worth checking if you maintain a custom packaging step. | +| Viewport icons or editor-representation markers look washed out | Marker textures read as linear when the pixel data is sRGB. Marker textures should be `rgba8Unorm_srgb`. | diff --git a/docs/API/UsingColorManagement.md b/docs/API/UsingColorManagement.md index 9a2052a4b..7ff714665 100644 --- a/docs/API/UsingColorManagement.md +++ b/docs/API/UsingColorManagement.md @@ -26,6 +26,37 @@ Neither operator is a substitute for a custom OCIO config, a non-stock View Transform, or a Look โ€” for those, use a creative grade LUT below to get closer, or accept the approximation. +## Mixed-Immersion Spaces: No Tone Map, The Display's Primaries + +A mixed-immersion space on the headset shows the scene next to the passthrough, +whose response the compositor applies to both, so a filmic curve there compresses +an object's whites and lifts its blacks against the room. `.none` hands the +compositor the scene's linear colour as it is: + +```swift +setPostFX(.tonemapOperator(.none)) +``` + +RealityKit shows such a scene with the display's P3 primaries: measured on a +panel of swatches it was given, its colours are the Display P3 โ†’ sRGB matrix of +their sRGB values (a red of (0.72, 0.13, 0.11) shows as (192, 0, 8) of 255), +its greys unchanged. That is the system's colour pipeline, which works in +Display P3 and expects content in it (WWDC24 session 10092, "Render Metal with +passthrough in visionOS"); a layer in an sRGB format is converted, so its colour +stays as specified. The look pass does the same as RealityKit when asked, after +the tone map: every saturated colour widens, and a channel the matrix takes +negative clips to black, as it does there. The default keeps the sRGB colour as +it is. + +In bright sunlight RealityKit adapts further: its content turns part +transparent, a curve goes on it and its lit surfaces darken. That is the +system's doing, and not something to reproduce; the match above holds in rooms. + +```swift +setPostFX(.colorPrimaries(.displayP3)) // as RealityKit shows a mixed-immersion scene +setPostFX(.colorPrimaries(.sRGB)) // default +``` + ## Using An Externally-Authored Grade LUT (.cube) If you already have (or want to hand-author, or export from DaVinci diff --git a/docs/API/UsingEngineAPI.md b/docs/API/UsingEngineAPI.md index 7889c439f..a57fe68bb 100644 --- a/docs/API/UsingEngineAPI.md +++ b/docs/API/UsingEngineAPI.md @@ -18,7 +18,11 @@ Existing direct APIs such as `LODConfig.shared`, `SSAOParams.shared`, `antiAlias setRendering(.antiAliasing(.fxaa)) setRendering(.antiAliasing(.smaa)) setRendering(.antiAliasing(.msaa)) +setRendering(.antiAliasing(.taa)) // temporal: jittered frames blended with their reprojected history +setRendering(.antiAliasing(.msaaTaa)) // 4ร— MSAA on the G-buffer plus the temporal resolve setRendering(.antiAliasing(.none)) +TAAParams.shared.historyWeight = 0.9 // share of the history in each resolved pixel (0โ€ฆ0.98) +TAAParams.shared.clipGamma = 1.25 // clip box = 3ร—3 mean ยฑ ฮณฯƒ: lower rejects ghosts sooner, higher keeps pixel-thin features setRendering(.debugView(.lit)) setRendering(.debugView(.depth)) @@ -82,6 +86,17 @@ setRendering(.occlusionCullingMaxCameraStep(0.5)) getOcclusionCullingMaxCameraStep() ``` +On visionOS, a layer configured with `isFoveationEnabled` hands the engine a rasterization +rate map per eye, and the engine draws through it by default: dense where the eyes look, +coarse in the periphery, and the only configuration in which the compositor accepts a +`maxRenderQuality` above the platform default. Disabling it in such a layer is a diagnostic +(the frame is drawn uniformly into textures the compositor still unwarps): + +```swift +setRendering(.foveatedRendering(.disabled)) +getFoveatedRenderingEnabled() +``` + ## PostFX Use `setPostFX` for individual post-processing and SSAO settings: diff --git a/docs/API/UsingMaterials.md b/docs/API/UsingMaterials.md index ba0f168de..c281004b7 100644 --- a/docs/API/UsingMaterials.md +++ b/docs/API/UsingMaterials.md @@ -296,11 +296,15 @@ updateMaterialHeightScale(entityId: entity, heightScale: 0.08) updateMaterialHeightMidlevel(entityId: entity, heightMidlevel: 0.5) ``` -POM only runs when a material actually has a height texture, and can be disabled without -discarding the texture assignment: +POM only runs when a material actually has a height texture, and is opt-in: `heightEnabled` +defaults to `false` for a material assembled at runtime (e.g. assigning a height texture +through the editor), even once a height texture is assigned. Load paths that bring in an +asset already authored with a baked height texture (model import, `.untold`/`.untoldpack` +load) set `heightEnabled = true` automatically so existing authored content keeps rendering +unchanged. Toggle it explicitly either way: ```swift -updateMaterialHeightEnabled(entityId: entity, heightEnabled: false) +updateMaterialHeightEnabled(entityId: entity, heightEnabled: true) ``` ### Height Remap diff --git a/docs/API/UsingTheExporter.md b/docs/API/UsingTheExporter.md index 7b67f4f67..f57206d6e 100644 --- a/docs/API/UsingTheExporter.md +++ b/docs/API/UsingTheExporter.md @@ -65,8 +65,8 @@ generated game project: cd /path/to/MyGame untoldengine export \ - --input Sources/MyGame/GameData/Models/robot/robot.usdz \ - --output Sources/MyGame/GameData/Models/robot/robot.untold \ + --input GameData/Models/robot/robot.usdz \ + --output GameData/Models/robot/robot.untold \ --convert-orientation ``` diff --git a/docs/API/UsingUntoldEngineCLI.md b/docs/API/UsingUntoldEngineCLI.md index 5cf5e2953..83df71bad 100644 --- a/docs/API/UsingUntoldEngineCLI.md +++ b/docs/API/UsingUntoldEngineCLI.md @@ -111,6 +111,21 @@ untoldengine update ~/Projects/MyGame --asset-path ~/GameAssets --- +## Converting an Existing Project + +Projects created before `GameData` moved to the project root still have it nested under `Sources//GameData`. `convert` moves it to the root, creates empty `Resources/` and `Docs/` folders there, writes a `.gitignore` if the project doesn't have one yet, and rewrites `project.yml` (regenerating the Xcode project): + +```bash +untoldengine convert MyGame + +# Or point to an absolute project path +untoldengine convert ~/Projects/MyGame +``` + +It refuses to run if the project is already on the root-level layout, or if `GameData` isn't where the legacy layout expects it. Safe to run more than once โ€” the second run is a no-op error, not a silent overwrite. + +--- + ## Bootstrapping Dependencies Some optimizations (ASTC texture compression) rely on external tools and @@ -488,7 +503,7 @@ untoldengine assets list untoldengine assets install starter # Install into a specific GameData path, overwriting existing files -untoldengine assets install soccer --output ~/Projects/MyGame/Sources/MyGame/GameData --force +untoldengine assets install soccer --output ~/Projects/MyGame/GameData --force ``` `assets install` auto-detects the `GameData` folder from the current @@ -535,8 +550,21 @@ engine-repo maintenance tool, not something a game project needs to run. ``` MyGame/ +โ”œโ”€โ”€ .gitignore โ”œโ”€โ”€ Package.swift โ”œโ”€โ”€ README.md +โ”œโ”€โ”€ GameData/ +โ”‚ โ”œโ”€โ”€ Scenes/ +โ”‚ โ”œโ”€โ”€ Scripts/ +โ”‚ โ”œโ”€โ”€ Models/ +โ”‚ โ”œโ”€โ”€ StreamModels/ +โ”‚ โ”œโ”€โ”€ Animations/ +โ”‚ โ”œโ”€โ”€ Gaussians/ +โ”‚ โ”œโ”€โ”€ Textures/ +โ”‚ โ”œโ”€โ”€ Shaders/ +โ”‚ โ””โ”€โ”€ LUT/ +โ”œโ”€โ”€ Resources/ +โ”œโ”€โ”€ Docs/ โ””โ”€โ”€ Sources/ โ””โ”€โ”€ MyGame/ โ”œโ”€โ”€ AppDelegate.swift @@ -544,15 +572,11 @@ MyGame/ โ”œโ”€โ”€ GameViewController.swift โ”œโ”€โ”€ Base.lproj/ โ”‚ โ””โ”€โ”€ Main.storyboard - โ”œโ”€โ”€ Info.plist - โ””โ”€โ”€ GameData/ - โ”œโ”€โ”€ Scenes/ - โ”œโ”€โ”€ Scripts/ - โ”œโ”€โ”€ Models/ - โ”œโ”€โ”€ Textures/ - โ””โ”€โ”€ Shaders/ + โ””โ”€โ”€ Info.plist ``` +`GameData` is what the engine reads and is bundled into the shipped app. `Resources/` holds artist sources (`.blend`, `.usd`, reference images) and `Docs/` holds project documentation โ€” neither is ever bundled. + The starter `GameScene.swift` shows how to load `.untold` runtime assets, use `setEntityStreamScene(...)` for streamed scenes, and enable static batching. > **Note:** Runtime examples expect `.untold` assets. Convert USD/USDZ authoring files with the exporter before placing them in `GameData/Models/`. diff --git a/docs/API/UsingXRImmersionMode.md b/docs/API/UsingXRImmersionMode.md index 982289351..c6a0ba23a 100644 --- a/docs/API/UsingXRImmersionMode.md +++ b/docs/API/UsingXRImmersionMode.md @@ -2,7 +2,9 @@ This guide shows how to run your Vision Pro game in **mixed** (passthrough) or **full** immersion mode, how to enable the environment (skybox), and how to -load a custom IBL/HDR environment. +load a custom IBL/HDR environment. The settings a mixed-immersion frame needs to +look like RealityKit's, and what the headset hands the engine, are measured in +[The Vision Pro Rendering Profile](VisionProRenderingProfile.md). --- @@ -257,11 +259,9 @@ Place the HDR inside your project's `GameData/HDR` folder: ``` MyGame/ -โ””โ”€โ”€ Sources/ - โ””โ”€โ”€ MyGame/ - โ””โ”€โ”€ GameData/ - โ””โ”€โ”€ HDR/ - โ””โ”€โ”€ suburban_garden_2k.hdr +โ””โ”€โ”€ GameData/ + โ””โ”€โ”€ HDR/ + โ””โ”€โ”€ suburban_garden_2k.hdr ``` The name is resolved through the engine's standard asset search diff --git a/docs/API/VisionProRenderingProfile.md b/docs/API/VisionProRenderingProfile.md new file mode 100644 index 000000000..26c9653c3 --- /dev/null +++ b/docs/API/VisionProRenderingProfile.md @@ -0,0 +1,145 @@ +# The Vision Pro Rendering Profile, As Measured + +What a mixed-immersion frame on Apple Vision Pro needs to look like RealityKit's, and +what the headset actually hands the engine. Everything here was measured on the device in +October 2026 (visionOS 26, display cadence 90, 96 or 100 Hz) with a dartboard and a +calibration panel shown by both renderers in the same room, and is kept so the numbers do +not have to be found twice. The viewer app that produced them is UntoldArcade's viewer. + +## The profile + +The settings a mixed-immersion space wants, with the calls that set them: + +| Setting | Value | Why | +| --- | --- | --- | +| Render path | `setRendering(.renderPath(.forward))` | The geometry lights itself once per fragment; matches RealityKit's cost on the headset, and is visionOS's default (`RenderInfo.defaultRenderPath`; macOS and iOS start deferred). Deferred and hybrid stay for many lights, projected decals and the G-buffer effects. | +| Anti-aliasing | `.msaaTaa`, 4 samples | Multisampled geometry with the temporal resolve on top; the foveation makes the periphery coarse, and the resolve hides it. | +| Foveation | on (the compositor's rasterization rate maps) | The frame draws at the display's own resolution only where the eyes look. | +| Tone map | `setPostFX(.tonemapOperator(.none))` | RealityKit applies no curve below white in a mixed space; a filmic curve compresses whites and lifts blacks against the room. | +| Primaries | `setPostFX(.colorPrimaries(.displayP3))` | RealityKit shows sRGB-specified colour with the display's P3 primaries; the look pass applies the P3โ†’sRGB matrix after the grade to do the same. | +| Lighting | real-world estimate, contribution 1 | ARKit's environment probe, prefiltered on the device, lights the scene as the room does. | +| Light transition | `lightingSmoothingTimeConstant` 3 s | A new estimate fades in instead of switching the light's direction and colour in one frame. | +| Sun | off | The room's light is the probe's; a sun over it doubles the key light. | +| Occlusion culling | on, camera step 0.5 | The depth pyramid is built only for a frame whose depth could hide something. | +| Texture budget | 1536 MB | Under the default 200 MB the streaming shed tiers of a 4K object under the wearer's eyes. | + +A full-immersion space is different: no passthrough to match, so a tone map (ACES or AgX) +is right again, the room light makes no sense, and the primaries are a taste. + +## Rendered on a Mac, shown on the headset + +A remote immersive space, where the Mac renders through Compositor Services and the headset +shows the frames, splits the table in two: + +- **Presentation follows the display.** The frame is still composited on the headset next + to the same passthrough, so the no tone map, the P3 primaries, the premultiplied alpha and + the room light from the probe stay exactly the headset's values, whichever machine renders. +- **Budget follows the renderer.** The render path starts forward there as well โ€” the XR + loop adopts the headset's path wherever it renders, unless the game chose one โ€” and the + anti-aliasing and its samples, the shadow taps, the texture budget, the LOD distances and + the tile light stage are the Mac's to raise: full shadow taps, more samples, deferred with + many lights if the game wants it. + Foveation depends on whether the remote layer hands the Mac the headset's rate maps; if it + does not, render uniformly at the quality the Mac affords. +- **What the Mac does not buy back.** The cadence is still the headset's, so every frame + must still be delivered on time or the compositor repeats one; and the link adds latency, + which makes posing animation for the frame's presentation time matter more, not less. + +Whether the environment probe and the rate maps reach a remote renderer has not been tested. +In a profile system this is a third entry: the mixed-immersion presentation with the Mac's +budget. + +## How RealityKit shows a mixed-immersion scene + +Measured on a panel of swatches that both renderers built from the same numbers (two grey +rows and two colour rows, one row through RealityKit's post-process tone map and one with +`applyPostProcessToneMap: false`, and a white, a black and a chrome sphere): + +- **No curve below white.** RealityKit's tone-mapped row and its exact row read the same + in a room: greys 0, 23, 49, 73, 111, 147, 196, 246 of 255 for inputs 0, 0.1, 0.2, 0.3, + 0.45, 0.6, 0.8 and 1. The engine with `.none` reads the same within two counts. +- **P3 primaries.** Its colours are the Display P3 โ†’ sRGB matrix of their sRGB values: a + board red of (0.72, 0.13, 0.11) shows as (192, 0, 8), the board's cream (0.93, 0.85, 0.62) + as (231, 207, 145). With `.displayP3` the engine reads the same values to the count. The + system's colour pipeline is Display P3 (WWDC24 session 10092); an sRGB-format layer is + converted and keeps its colour, which is why the matrix is needed to match. +- **Sunlight is RealityKit's own adaptation.** In a sunlit train its panel went part + transparent (dark greys lifted by the passthrough), its exact row fell back to plain sRGB, + its tone-mapped row gained a curve and its white sphere read half as bright. None of that + is a target; the match above holds in rooms, which is where a game is played. + +## The room probe + +- ARKit's `EnvironmentProbeAnchor` hands over an `rgba16Float` cube of **8ร—8 texels a face + with 4 mips**, and a `cameraScaleReference`. The engine's intensity scale is the reference + over 1000, calibrated against RealityKit lighting the same object: 2350 in a dim room, + 8200 to 11400 in a restaurant, 14000 to 22000 in a sunlit train. The engine's ambient + intensity (0.4 by default) multiplies it. +- **Estimates arrive in a burst** of two or three when the provider starts and then not at + all in a still room; in a moving train, seven in thirty seconds. The engine keeps the + newest estimate pending until the prefilter in flight finishes (the second of the burst is + the better one) and fades towards it over the transition time constant, re-prefiltering a + blended cube every 0.1 s, about a millisecond of GPU a step. +- **The maps are convolved, not sampled.** Importance sampling an eight-texel probe with + 128 samples a texel left the irradiance map with the structure of the sample set: with a + window's light in one texel, a white sphere showed ridges and spirals. The irradiance now + comes from the probe's first nine spherical harmonics over a fixed quadrature of 1024 + directions, the rougher specular mips (roughness 0.25 and up) from the same directions + weighted by the GGX lobe; the sharper mips keep importance sampling. +- **The prefilter's frame.** The maps are read with a y-up equirect lookup and were written + in a z-pole frame; the two compose to a fixed turn of the axes, applied to the cube's + sample direction. Without it the ceiling lit the left of a sphere and what was behind the + wearer lit its underside. `XRProbeOrientationTests` holds it. +- **What RealityKit does that we do not get.** Its chrome reflects a sharp, legible room; + ours reflects the eight-texel probe. The source of its reflections is not exposed. + +## Measuring + +- **Captures.** The headset's direct captures come through as sRGB-tagged images whose + greys read 0.96 of the value shown (246 for white); the saturated colours are converted + correctly. A screenshot taken inside the headset of two captures shown in Photos is not + comparable: it de-widens the saturated colours by exactly the matrix. Compare renderers + within one capture, made the same way, and read patch means, not single pixels. +- **The calibration panel** is the instrument: greys at 0, 0.1, 0.2, 0.3, 0.45, 0.6, 0.8, 1; + pure red, green and blue, the board's cream (0.93, 0.85, 0.62), red (0.72, 0.13, 0.11) and + green (0.16, 0.50, 0.28), a yellow (1, 0.85, 0.2) and a salmon (0.9, 0.6, 0.5); a white + rough sphere, a black one (0.03) and a chrome one; a backing of 0.35. On the engine side + every swatch is an emissive black metal (no diffuse, no reflection, the colour as given). +- **A pass** logs the frame's GPU time with and without the object, the frame rate against + the cadence, the frame's CPU phases, the GPU time per pass, the probe's scale and reference, + publishes and per-face light, dark frames, and the look state (render path, primaries, tone + map, encode) when the space opens. + +## What the device does + +The dartboard (744 draws, 741k triangles, 4K textures) on the forward path with `.msaaTaa` +and foveation: 10.3 to 10.7 ms of GPU a frame, of which the empty frame is 4.0 to 4.7 and +the object 6.3 to 7.5; 94 of 100 Hz, 90 of 90. The frame's CPU: loop 10.5 ms, the game +update 1.0, the wait for the compositor's optimal input time 6.2 to 6.4, the encode 3.1 to +3.6. Passes: forward 7.0 (vertex 2.0, fragment 5.0), temporal resolve 2.1, depth pyramid +1.6, shadows 0.6, post 0.5. Dark frames: none to one in a thirty-second pass. A debug build +measures the compiler, not the engine: the headset numbers only mean something with +`-O` and the engine stats enabled. + +## Motion + +A smooth path judders on the headset for two reasons that a keyframe system would not +cure: the game update runs before the loop's wait for the compositor, so an object posed +from the wall clock belongs to an instant a varying 6 to 7 ms behind the frame's +presentation time (the head is posed for the presentation time; objects should be too), and +a dropped frame makes the compositor repeat an image, after which an animation driven by +accumulated deltas takes a double step. The fix is to hand the frame's predicted +presentation time to the game and sample every animation at it. + +## Pitfalls met on the way + +- The culling's compute read the entity list and the frustum of a frame two slots old: an + object leaving the list made the next one vanish for a frame. +- A test that hides a scene channel's render mode and does not restore it leaves every later + render test drawing nothing; the mode is global. +- `thread` is a Metal keyword; a kernel parameter cannot be called that. +- The pinch ray of a spatial event follows the gaze, not the hand: an object that rides it + does not move sideways with the hand. RealityKit's drag is the hand's displacement, one to + one on every axis. +- The viewer's Controls window placed the object thirty degrees to the right to stay out of + its own way, which read as a placement bug; a measurement pass placed it straight ahead. diff --git a/docs/Architecture/renderingSystem.md b/docs/Architecture/renderingSystem.md index 9aac362fc..1f76ac905 100644 --- a/docs/Architecture/renderingSystem.md +++ b/docs/Architecture/renderingSystem.md @@ -226,6 +226,14 @@ The graph still contains a `batchedModel` node and a `lightPass` node for depend > **Why TBDR?** Keeping the G-Buffer in tile memory avoids storing and reloading several full-resolution render targets every frame. This is especially important on Apple GPUs, where framebuffer fetch and memoryless attachments let the renderer shade from G-Buffer data while it is still resident on the tile. +### Render Paths + +`setRendering(.renderPath(_:))` chooses where the opaque geometry is shaded (`RenderPath`; `getRenderPath()` reads it back). The model and light pipelines follow the choice on the next frame. + +- **`.deferred`** โ€” the pipeline above: the geometry writes the G-Buffer and the light sub-pass lights it, once per sample from the quad or once per pixel through the tile light stage. The default on macOS and iOS (`RenderInfo.defaultRenderPath`), and the path for a scene of many lights and for the effects that read the G-Buffer (SSAO, the debug views). +- **`.hybrid`** โ€” deferred, with the image-based light and the sun's cascade shadow evaluated by the geometry once per fragment and carried in the emissive attachment (the shadow factor in its alpha); the light sub-pass adds the direct lights from there, once per sample. The same picture with less lighting work under MSAA; the tile light stage stands aside. +- **`.forward`** โ€” the geometry lights itself whole once per fragment and writes the lit colour to attachment 5 and its motion to attachment 6 (the velocity map, for the temporal resolve); the light quad stands down and the pass ends after the geometry. The default on visionOS, where it is the measured match for RealityKit's cost under MSAA and the rasterization rate maps. The effects that read the G-Buffer stand down with it. + ### Transparency Pass ```swift @@ -332,10 +340,13 @@ After the look pass, the graph inserts an anti-aliasing pass whose topology depe |---|---|---| | `.fxaa` | `fxaa` | `look โ†’ fxaa โ†’ outputTransform` | | `.smaa` | `smaaEdges`, `smaaBlendWeights`, `smaaNeighborhood` | `look โ†’ smaaEdges โ†’ smaaBlendWeights โ†’ smaaNeighborhood โ†’ outputTransform` | -| `.none` | *(none)* | `look โ†’ outputTransform` | +| `.taa`, `.msaaTaa` | `taa` | `look โ†’ taa โ†’ outputTransform` (`.msaaTaa` also draws the G-buffer at 4 samples, like `.msaa`) | +| `.none`, `.msaa` | *(none)* | `look โ†’ outputTransform` | **FXAA** is a single-pass screen-space filter that attenuates aliased edges using local luma contrast. +**TAA** (temporal anti-aliasing, `TemporalAntiAliasing.swift`, `TAAShader.metal`) jitters the projection by a sub-pixel Halton offset before each frame's graph is built (`applyJitter`, in the projection's third column, so the offset is a constant NDC shift; the previous frame's offset is taken out first) and, after the look pass, blends the frame with the eye's history: each pixel is reprojected through the depth buffer and the unjittered view-projections of this frame and the history's, the history colour is clipped to the YCoCg box of the pixel's 3ร—3 neighbourhood (so what moved or was uncovered does not ghost), and the two are mixed with `TAAParams.shared.historyWeight` (0.9). The resolve writes the anti-aliasing texture and the eye's next history texture in one pass (two colour targets); the history is per eye, ping-ponged, and invalidated by a resize or a mode change. Under a foveated XR frame the textures are in physical space and each frame's rate map differs, so the reprojection goes physical โ†’ screen โ†’ world โ†’ previous screen โ†’ previous physical through both maps, and the jitter is measured in screen pixels. Camera motion is handled on every path through the depth buffer. On the forward path the opaque pass also writes each pixel's own motion (the velocity map, the forward descriptor's attachment 6), so an object moving on its own finds its history where it was; on the deferred and hybrid paths there are no motion vectors and such an object relies on the clip. + **SMAA** (Subpixel Morphological Anti-Aliasing) is a three-pass chain: 1. **Edge detection** (`smaaEdges`) โ€” identifies aliased edges from the look texture using luma and chroma gradients. Also detects diagonal patterns. 2. **Blend-weight calculation** (`smaaBlendWeights`) โ€” computes per-pixel blend weights from the SMAA area and search look-up textures, accounting for corner patterns. diff --git a/docs/Architecture/xrRenderingSystem.md b/docs/Architecture/xrRenderingSystem.md index b4c3059ae..726bc66b8 100644 --- a/docs/Architecture/xrRenderingSystem.md +++ b/docs/Architecture/xrRenderingSystem.md @@ -63,7 +63,7 @@ let frame = layerRenderer.queryNextFrame() let timing = frame.predictTiming() ``` -`queryNextFrame()` dequeues the next compositor frame. `predictTiming()` returns `optimalInputTime` โ€” the deadline by which you must finish reading input and preparing CPU-side data for the frame. These are not suggestions; missing them causes judder. +`queryNextFrame()` dequeues the next compositor frame. `predictTiming()` returns `optimalInputTime` โ€” the deadline by which you must finish reading input and preparing CPU-side data for the frame. These are not suggestions; missing them causes judder. It also returns `presentationTime`, the instant the frame will be shown; the engine converts it to the media-time base (through the compositor clock's epoch, which is the boot, like `CACurrentMediaTime()`) and poses the whole update for it. ### 2b. Update Phase @@ -79,7 +79,7 @@ Everything between `startUpdate` and `endUpdate` is CPU-side frame preparation: - **Spatial input processing**: `updateSpatialInputState()` drains the queued `XRSpatialInputSnapshot` events and updates the `InputSystem`. If assets are loading or the scene isn't ready, input is cleared instead to avoid acting on stale state. -- **Game update**: `renderer.updateXR()` calls the user's `gameUpdate` and `handleInput` callbacks. This is where game logic runs โ€” entity movement, animation state machines, physics steps. It is skipped entirely while `AssetLoadingGate.shared.isLoadingAny` is true. +- **Game update**: `renderer.updateXR(presentationTime:)` calls the user's `gameUpdate` (or `frameUpdate`) and `handleInput` callbacks. This is where game logic runs โ€” entity movement, animation state machines, physics steps. The frame is posed for the instant the compositor predicts it will be shown, the same instant the head pose is queried for at submission: the engine's delta is the time between two presentations, and the game receives the instant itself as a `FrameTime`, so a moving object steps by the display's period from frame to frame instead of by the varying time between two updates, which is what reads as judder against the head-tracked world (see [Presentation-time animation](../proposals/PresentationTimeAnimation.md)). It is skipped entirely while `AssetLoadingGate.shared.isLoadingAny` is true. - **XR lighting probe processing**: When real-world lighting mode is active, accepted environment-light probe updates are prefiltered into runtime IBL textures. The renderer uses the latest valid probe plus `realWorldLightingContribution`; if no valid probe is available, it falls back to the configured static environment path. @@ -100,6 +100,8 @@ defer { frame.endSubmission() } The `defer` is important: `endSubmission()` **must** be called even if rendering fails partway through. If it isn't called, the compositor stalls. The `defer` guarantees this regardless of how the function exits. +**Drawables.** On visionOS 26 the frame is queried with `frame.queryDrawables()`, which returns one drawable per target: the built-in display, plus a capture target while the view is being recorded or streamed. A layer configured with foveation or a render quality aborts the app ("BUG IN CLIENT: called cp_frame_query_drawable() which does not support setting render quality or fovea") if the older single-drawable `queryDrawable()` is used, which the engine still calls on visionOS 2. An empty array means the compositor cancelled the frame: it is left without `endSubmission()`. Every drawable is presented; the scene is drawn into the built-in one and into a capture drawable only while its textures are the size the frame's resources are allocated at โ€” otherwise it is presented cleared, since every sizeable resource is sized to one viewport and a second size would reallocate them twice a frame. + ### 2e. Device Anchor Acquisition ```swift @@ -109,7 +111,7 @@ drawable.deviceAnchor = anchor The device anchor is the head pose โ€” a 4ร—4 transform from world space to the device. The compositor requires it to be attached to the drawable before presenting; without it, the system can't reproject the frame correctly for the user's eyes. -The engine queries the anchor at **presentation time** (the future moment when the frame will appear on screen), not at "now". This is predictive โ€” it compensates for the latency between encoding and display by predicting where the head will be. +The engine queries the anchor at **presentation time** (the future moment when the frame will appear on screen), not at "now". This is predictive โ€” it compensates for the latency between encoding and display by predicting where the head will be. The game's objects were posed for the same instant in the update phase, so head and objects agree; `presentationTimeRevisions` counts the frames whose drawable's presentation time differs from the prediction the update used. **Resilience strategy**: ARKit can occasionally return `nil` for the anchor (e.g., tracking hiccup, recovery). A three-level fallback prevents dropped frames: 1. Query at predicted presentation time โ†’ use it if valid @@ -167,6 +169,15 @@ The compositor provides the exact asymmetric projection for each eye. This accou - `renderInfo.currentEye = viewIndex` โ€” tells uniform uploads which eye's matrices to use - The base pass mode: `.mixed` immersion omits the base pass (camera passthrough is the background), `.full` immersion renders the skybox +**Foveation (rasterization rate maps).** With `isFoveationEnabled` in the app's `CompositorLayerConfiguration`, the compositor hands each eye a `MTLRasterizationRateMap` and colour/depth textures of the map's *physical* size, smaller than the *screen* the eye sees; the GPU draws through the map (dense where the eyes look, coarse in the periphery) and the compositor unwarps the texture for the display. It is also the only configuration in which the compositor accepts a `maxRenderQuality` above the platform default. The engine supports it as follows (`XRFoveation.swift`, on by default, `setRendering(.foveatedRendering(.disabled))` to ignore the maps for diagnosis): + +- `executeXRSystemPass` makes an `XRFoveationFrame` per eye (`renderInfo.xrFoveation`): the map, the eye's viewport in the map's screen space (`view.textureMap.viewport`), and the map's parameter data copied into a ring of shared buffers for the shaders. `renderInfo.viewPort` stays the texture size, the physical size, so every sizeable resource is allocated at it. +- After the graph is built, `applyXRFoveationToSceneRenderPassDescriptors` attaches the map to the descriptors of the passes that rasterize the scene at drawable size (G-buffer + light, the deferred colour that transparency/wireframe/debug draw into, environment/sky/grid, splats, gizmos), and each of those encoders sets the eye's viewport (`applyXRFoveationViewport`) in the map's screen space. Under a map, Metal reads a descriptor's `renderTargetWidth/Height` and the viewport in *screen* pixels (measured on a Mac GPU, and seen on the headset as a screen-fixed rectangle outside which nothing was drawn), so the attached descriptors leave the target size at 0 โ€” the attachments' own size โ€” and get it back when the map is detached. The full-screen passes (SSAO, post-processing, anti-aliasing, the output transform into the compositor's texture) keep plain descriptors: they copy physical pixels one to one and the layout is already the map's. +- Two kinds of shader decode the map (`rasterization_rate_map_decoder`): SSAO, which rebuilds view positions from a depth texel's UV and must use the texel's *screen* position; and the occlusion culls (`hzbCullVisibleEntities`, `gaussianChunkCull`, the per-splat test), which project boxes and splats to screen UV and sample a pyramid laid out in physical space โ€” the rect or point goes through the map first (`HZBOcclusion.h`). The pyramid carries the map it was drawn through into the next frame's culls (`HZBPyramidFrame.rateMapData`). +- The Gaussian fragment takes its offset from the splat centre from a varying rather than from `[[position]]`, which the rate map places in the physical texture. Fragment position is physical everywhere else too, which is what every one-to-one texture read wants; `SmallObjectCulling` measures its pixel threshold against the screen height. + +Without foveation (the layer configured without it, the simulator), `drawable.rasterizationRateMaps` is empty and the frame is drawn uniformly exactly as before. + ### 3c. HZB Pyramid (built once after both eyes) ```swift @@ -220,7 +231,7 @@ Selection rays update from interaction events, not continuously as the user look โ”œโ”€ frame.endUpdate() โ”œโ”€ wait(until: optimalInputTime) (sleep until compositor deadline) โ”œโ”€ frame.startSubmission() - โ”œโ”€ queryDrawable() (get per-eye textures) + โ”œโ”€ queryDrawables() (one drawable per target; the old single query aborts a foveated layer) โ”œโ”€ queryDeviceAnchor() โ†’ fallback chain โ†’ drawable.deviceAnchor = anchor โ”‚ โ””โ”€ executeXRSystemPass() @@ -256,6 +267,7 @@ Selection rays update from interaction events, not continuously as the user look | Frame timing | MTKView drives at display rate | CompositorServices dictates via `optimalInputTime` | | Eyes | 1 | 2 (per-eye loop over `drawable.views`) | | Compute passes | Once per frame | Once per frame (shared across both eyes) | +| Default scene | A camera and a directional light, both active | A camera, the entity the head pose is written into, and no directional light unless the configuration asks for one (`UntoldRendererConfig.defaultSun`): the passthrough room is lit by its probe and the immersive environment by its own light, so a sun is the app's to add (`DirectionalLightNode`, or `createDirLight` and `setDirectionalLight(.active(_:))`), and it casts in one cascade by default (`setRendering(.shadowCascades(_:))`) | | View matrix | Camera entity transform | `(originFromDevice ร— deviceFromView)โปยน` from ARKit anchor | | Projection | Camera component FOV | `drawable.computeProjection()` โ€” asymmetric per-eye | | Present call | `commandBuffer.present(drawable)` | `drawable.encodePresent(commandBuffer:)` โ€” encoded as GPU command | diff --git a/docs/LearningPaths/ArchvizToVisionPro.md b/docs/LearningPaths/ArchvizToVisionPro.md index b6a2c03d8..8e435f341 100644 --- a/docs/LearningPaths/ArchvizToVisionPro.md +++ b/docs/LearningPaths/ArchvizToVisionPro.md @@ -81,15 +81,17 @@ The project also includes a bundled `GameData` folder. Runtime assets should liv ```text ArchvizViewer/ + GameData/ + Models/ + StreamModels/ + Textures/ + Scenes/ + Scripts/ + Resources/ + Docs/ Sources/ ArchvizViewer/ GameScene.swift - GameData/ - Models/ - StreamModels/ - Textures/ - Scenes/ - Scripts/ ``` ## Install The Starter Archviz Asset diff --git a/docs/LearningPaths/BedroomDigitalTwin.md b/docs/LearningPaths/BedroomDigitalTwin.md index 111b334bf..53bc394b5 100644 --- a/docs/LearningPaths/BedroomDigitalTwin.md +++ b/docs/LearningPaths/BedroomDigitalTwin.md @@ -65,7 +65,7 @@ The CLI finds the project's `GameData` folder and merges the asset files into it The important file for this path is the `DigitalTwin.untold` model, installed under: ```text -Sources/BedroomTwin/GameData/Models/DigitalTwin/ +GameData/Models/DigitalTwin/ DigitalTwin.untold Textures/ ... diff --git a/docs/LearningPaths/CityStreamingOnVisionPro.md b/docs/LearningPaths/CityStreamingOnVisionPro.md index affb68359..96ba38188 100644 --- a/docs/LearningPaths/CityStreamingOnVisionPro.md +++ b/docs/LearningPaths/CityStreamingOnVisionPro.md @@ -60,7 +60,7 @@ The CLI finds the project's `GameData` folder and merges the asset files into it The important files for this path are the `LowPolyCity.json` manifest and its `LowPolyCity/` tile folder, installed under: ```text -Sources/CityStreaming/GameData/StreamModels/ +GameData/StreamModels/ LowPolyCity.json LowPolyCity/ ``` diff --git a/docs/Tutorials/CLIExporterTutorial.md b/docs/Tutorials/CLIExporterTutorial.md index 4a589e6e4..626d082a0 100644 --- a/docs/Tutorials/CLIExporterTutorial.md +++ b/docs/Tutorials/CLIExporterTutorial.md @@ -11,7 +11,7 @@ when you want a visual Blender export workflow. Generated projects bundle runtime assets from: ```text -Sources//GameData/ +GameData/ Models/ StreamModels/ Animations/ @@ -62,7 +62,7 @@ From any directory: ```bash untoldengine export \ --input /path/to/robot.usdz \ - --output ~/Projects/MyGame/Sources/MyGame/GameData/Models/robot/robot.untold \ + --output ~/Projects/MyGame/GameData/Models/robot/robot.untold \ --convert-orientation ``` @@ -71,7 +71,7 @@ untoldengine export \ ```bash untoldengine export \ --input /path/to/robot.blend \ - --output ~/Projects/MyGame/Sources/MyGame/GameData/Models/robot/robot.untold \ + --output ~/Projects/MyGame/GameData/Models/robot/robot.untold \ --convert-orientation ``` @@ -99,7 +99,7 @@ Use `--validate` while building an asset pipeline: ```bash untoldengine export \ --input /path/to/robot.usdz \ - --output ~/Projects/MyGame/Sources/MyGame/GameData/Models/robot/robot.untold \ + --output ~/Projects/MyGame/GameData/Models/robot/robot.untold \ --convert-orientation \ --validate ``` @@ -121,7 +121,7 @@ Then export with optimization: ```bash untoldengine export \ --input /path/to/robot.usdz \ - --output ~/Projects/MyGame/Sources/MyGame/GameData/Models/robot/robot.untold \ + --output ~/Projects/MyGame/GameData/Models/robot/robot.untold \ --convert-orientation \ --optimize ``` @@ -142,7 +142,7 @@ Export animation-only `.untoldanim` files with `--animation`: ```bash untoldengine export \ --input /path/to/running.usdz \ - --output ~/Projects/MyGame/Sources/MyGame/GameData/Animations/running/running.untoldanim \ + --output ~/Projects/MyGame/GameData/Animations/running/running.untoldanim \ --convert-orientation \ --animation ``` @@ -166,7 +166,7 @@ operator, stage an externally-authored `.cube` LUT with the export: ```bash untoldengine export \ --input /path/to/office.blend \ - --output ~/Projects/MyGame/Sources/MyGame/GameData/Models/office/office.untold \ + --output ~/Projects/MyGame/GameData/Models/office/office.untold \ --convert-orientation \ --color-grade-lut GameData/LUTs/warm_grade.cube ``` @@ -191,7 +191,7 @@ For large scenes, export a manifest and tile payloads into `GameData/StreamModel ```bash untoldengine export-tiles \ --input /path/to/city.usdz \ - --output-dir ~/Projects/MyGame/Sources/MyGame/GameData/StreamModels/City/tile_exports \ + --output-dir ~/Projects/MyGame/GameData/StreamModels/City/tile_exports \ --tile-size-x 25 \ --tile-size-z 25 \ --generate-hlod \ @@ -238,7 +238,7 @@ Use a uniform grid for simple outdoor scenes: ```bash untoldengine export-tiles \ --input /path/to/site.usdz \ - --output-dir ~/Projects/MyGame/Sources/MyGame/GameData/StreamModels/Site/tile_exports \ + --output-dir ~/Projects/MyGame/GameData/StreamModels/Site/tile_exports \ --tile-size-x 25 \ --tile-size-z 25 ``` @@ -248,7 +248,7 @@ Use quadtree for multi-floor buildings: ```bash untoldengine export-tiles \ --input /path/to/building.usdz \ - --output-dir ~/Projects/MyGame/Sources/MyGame/GameData/StreamModels/Building/tile_exports \ + --output-dir ~/Projects/MyGame/GameData/StreamModels/Building/tile_exports \ --quadtree \ --scene-profile indoor ``` @@ -258,7 +258,7 @@ Use KD-tree when geometry is clustered unevenly: ```bash untoldengine export-tiles \ --input /path/to/building.usdz \ - --output-dir ~/Projects/MyGame/Sources/MyGame/GameData/StreamModels/Building/tile_exports \ + --output-dir ~/Projects/MyGame/GameData/StreamModels/Building/tile_exports \ --kdtree \ --scene-profile indoor ``` @@ -267,7 +267,7 @@ untoldengine export-tiles \ | Problem | Fix | | --- | --- | -| Asset does not load | Confirm the `.untold` file is under `Sources//GameData/Models//`. | +| Asset does not load | Confirm the `.untold` file is under `GameData/Models//`. | | Animation does not play | Confirm the clip was exported with `--animation` and registered with `setEntityAnimations(...)`. | | Blender look does not match | Try `setPostFX(.tonemapOperator(.agx))`, or export with `--color-grade-lut` and call `loadSceneAuthored(...)`. | | Tiled scene does not stream | Confirm the manifest is under `GameData/StreamModels//` and tile paths are relative to it. | diff --git a/docs/Tutorials/CreateXcodeProjectTutorial.md b/docs/Tutorials/CreateXcodeProjectTutorial.md index e3fd2d5b9..770d4ab00 100644 --- a/docs/Tutorials/CreateXcodeProjectTutorial.md +++ b/docs/Tutorials/CreateXcodeProjectTutorial.md @@ -83,21 +83,25 @@ A generated project includes source files and a bundled `GameData` folder: ```text MyGame/ + .gitignore Package.swift README.md + GameData/ + Scenes/ + Scripts/ + Models/ + StreamModels/ + Animations/ + Gaussians/ + Textures/ + Shaders/ + LUT/ + Resources/ + Docs/ Sources/ MyGame/ AppDelegate.swift GameScene.swift - GameData/ - Scenes/ - Scripts/ - Models/ - StreamModels/ - Animations/ - Gaussians/ - Textures/ - Shaders/ ``` `GameScene.swift` is where your engine scene setup usually begins. `GameData` is @@ -126,7 +130,7 @@ Install into a specific `GameData` path: ```bash untoldengine assets install starter-archviz \ - --output ~/Projects/MyGame/Sources/MyGame/GameData + --output ~/Projects/MyGame/GameData ``` ## Load A Single Asset @@ -134,7 +138,7 @@ untoldengine assets install starter-archviz \ Place always-resident models under: ```text -Sources/MyGame/GameData/Models//.untold +GameData/Models//.untold ``` Then load the asset from `GameScene.swift`: @@ -182,7 +186,7 @@ separate step. Place streamed scene manifests and tile payloads under: ```text -Sources/MyGame/GameData/StreamModels// +GameData/StreamModels// .json tile_exports/ ``` diff --git a/docs/Tutorials/StarterDemo.md b/docs/Tutorials/StarterDemo.md index af03d5172..a99bac893 100644 --- a/docs/Tutorials/StarterDemo.md +++ b/docs/Tutorials/StarterDemo.md @@ -76,6 +76,22 @@ renderer.setupCallbacks( Use `gameUpdate` for simulation, animation, and other time-based scene changes. Use `handleInput` for keyboard, mouse, touch, gamepad, or XR input state. +`deltaTime` is the time from the previous frame's presentation to this one's. To pose +a moving object for the instant the frame will be shown (on a headset the head pose is +predicted for that same instant), take the frame's time instead: + +```swift +renderer.setupCallbacks( + frameUpdate: { [weak self] frame in + // frame.presentationTime, frame.deltaTime, frame.skippedFrames + self?.gameScene.update(frame: frame) + }, + handleInput: { [weak self] in + self?.gameScene.handleInput() + } +) +``` + The important API shape is: - `gameUpdate` receives `deltaTime`, so movement and animation can be frame-rate independent. diff --git a/docs/images/coolsaber-thumbnail.png b/docs/images/coolsaber-thumbnail.png deleted file mode 100644 index 0354beeaa..000000000 Binary files a/docs/images/coolsaber-thumbnail.png and /dev/null differ diff --git a/docs/images/state-of-the-engine-2026-thumbnail.png b/docs/images/state-of-the-engine-2026-thumbnail.png new file mode 100644 index 000000000..bf45320e7 Binary files /dev/null and b/docs/images/state-of-the-engine-2026-thumbnail.png differ diff --git a/docs/images/untoldengine-logo.png b/docs/images/untoldengine-logo.png new file mode 100644 index 000000000..a5f17da6e Binary files /dev/null and b/docs/images/untoldengine-logo.png differ diff --git a/docs/index.md b/docs/index.md index 06ac51b6d..75476a3c6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -24,7 +24,7 @@ and input, so you can build your own interactive spatial and XR worlds. Creator & Lead Developer: [Harold Serrano](http://www.haroldserrano.com) -[![CoolSaber demo](images/coolsaber-thumbnail.png)](https://youtu.be/4JWZf_NFddY) +[![State of the Engine 2026](images/state-of-the-engine-2026-thumbnail.png)](https://youtu.be/AnN2NBRKl8E) --- @@ -49,6 +49,21 @@ workflow, product experience, and customer-specific behavior. --- +## Try the Untold Editor + +The Untold Editor is a companion app for building scenes visually โ€” placing +entities, setting up lighting and cameras, and authoring `.untoldscene` files +that load straight into your app with `loadUntoldScene(named:)`. It's +optional โ€” nothing in Untold Engine requires it โ€” but it's the fastest way to +get a scene on screen without hand-writing setup code. + +The Editor is now stable enough for regular use. Grab the latest build from +the [Releases page](https://github.com/untoldengine/UntoldEditor/releases). + +![untold-editor](images/editor-main-highlight.png) + +--- + ## Watch It in Action โ€” Apple Vision Pro Demos | Demo | Description | @@ -78,7 +93,7 @@ Clone the repository and launch the Starter Demo: ```bash git clone https://github.com/untoldengine/UntoldEngine.git cd UntoldEngine -git checkout v0.22.0 +git checkout v0.23.0 swift run StarterDemo ``` @@ -100,40 +115,28 @@ Engine as a package dependency. --- -## Try the Untold Editor - -The Untold Editor is a companion app for building scenes visually โ€” placing -entities, setting up lighting and cameras, and authoring `.untoldscene` files -that load straight into your app with `loadUntoldScene(named:)`. It's -optional โ€” nothing in Untold Engine requires it โ€” but it's the fastest way to -get a scene on screen without hand-writing setup code. - -The Editor is now stable enough for regular use. Grab the latest build from -the [Releases page](https://github.com/untoldengine/UntoldEditor/releases). - -![untold-editor](images/editor-main-highlight.png) - ---- - ## Getting Started To create your own XR, 3D, or spatial visualization app using Untold Engine, see [Getting Started](API/GettingStarted.md). -## Core Direction +## Our Direction + +### Vision โ€” Why We Exist + +> An engine that empowers creativity and never stands in its way. -**Vision:** An engine that empowers creativity and never stands in its way. +### Mission โ€” What We Commit To -**Mission:** Build a reliable engine that lets developers focus on creating, -not troubleshooting the tools they depend on. +> Build a reliable engine that lets developers focus on creating, not troubleshooting the tools they depend on. -Untold Engine is built around three focused goals: +### Engineering Principles โ€” How We Operate -- **Spatial Engine First** โ€” Designed for spatial computing applications. LOD, geometry streaming, and static batching exist to support large, real-world-scale environments where presence and performance both matter. +**Reliability ยท Testing ยท Simplicity ยท Compatibility ยท Developer Experience ยท Performance** -- **XR / visionOS Support** โ€” Spatial input, AR workflows, and Vision Pro support are functional today and expanding with each release. +These principles guide how we design, build, test, and maintain Untold Engine. -- **Metal-First Architecture** โ€” The rendering layer stays close to Metal to maintain performance, shader control, and pipeline flexibility without abstraction layers getting in the way. +Read our [Engineering Principles](Contributor/EngineeringPrinciples.md) for the standards that guide our development decisions. --- @@ -200,6 +203,7 @@ See [COMMERCIAL.md](https://github.com/untoldengine/UntoldEngine/blob/main/COMME - [Async Loading](API/UsingAsyncLoading.md) - [Spatial Input](API/UsingSpatialInput.md) - [XR Immersion Modes](API/UsingXRImmersionMode.md) +- [The Vision Pro Rendering Profile, As Measured](API/VisionProRenderingProfile.md) - [Light Portals](API/UsingLightPortals.md) - [Rendering Extensions](API/UsingRenderingExtensions.md) - [Rendering Extension Examples](https://github.com/untoldengine/UntoldEngine/blob/main/Examples/RenderingExtensions/README.md) diff --git a/docs/proposals/EditorCodeComponents.md b/docs/proposals/EditorCodeComponents.md index 5aacd970c..795ab2a27 100644 --- a/docs/proposals/EditorCodeComponents.md +++ b/docs/proposals/EditorCodeComponents.md @@ -22,6 +22,15 @@ Revision 5.2: control points can be dragged. `.handles(properties:tint:)` names ## Implementation status (2026-09-17) +**Update (2026-10-13):** the `UntoldEngine` side of this work (`feature/component_kit`) is merged +to `develop` โ€” `Sources/UntoldComponentKit`, `ComponentPlugin`/`EntityPlugin`/`EditorMenuPlugin`, +entity-kind and draggable-handle support, and the sibling-removal-during-`onAttach` fix are all +present, with the last of those commits landing 2026-09-18. The branch/PR table and stage +breakdown below are left as the historical record of how it got there; treat "no pull request +opened" as stale for the engine repo specifically. The `UntoldEditor`-side branch and the +`UntoldGaussianTwins` adoption (stage 5) were not re-checked as part of this update โ€” verify +those separately before assuming the full loop (editor + twins) is also merged. + Branches, all pushed to the miolabs fork, no pull request opened: | Repo | Branch | Holds | diff --git a/docs/proposals/PresentationTimeAnimation.md b/docs/proposals/PresentationTimeAnimation.md new file mode 100644 index 000000000..3628aef68 --- /dev/null +++ b/docs/proposals/PresentationTimeAnimation.md @@ -0,0 +1,210 @@ +# Posing the Frame for Its Presentation Time โ€” Why a Moving Object Judders on Vision Pro, and the Fix + +Status: proposal with a prototype on this branch (engine side done and tested; the viewer app's change applied in the viewer's pull request, and kept as `viewer-frame-time.patch` beside this document; the headset check still to do). The branch stacks on `feature/xr_headset_rendering` (untoldengine/UntoldEngine#1359), which rewrote the try branch's history. Written 2026-10-14 from a headset session of the same day with the viewer app (`UntoldArcade-viewer`), the dartboard on its eased "tour" path. + +## TL;DR + +1. **The engine poses the head for the instant the frame will be shown, and every object for "now at update time".** The XR loop queries the device anchor at `drawable.frameTiming.presentationTime`, but the game ran earlier on a wall-clock delta between two updates. The gap between the two instants is the wait for the compositor's optimal input time plus the encode, and it varies with the previous frame's CPU work: the object's position steps unevenly from frame to frame while the head's does not. That reads as judder at full frame rate. +2. **The viewer makes it worse by reading the system clock inside the update.** Its tour pose comes from `ProcessInfo.processInfo.systemUptime` in the game update, so it carries the same varying lag; the engine never told it when the frame would be shown. +3. **A skipped frame doubles the step either way, but delta-driven motion drifts after it.** The measured 94 fps against a 100 Hz cadence and 85 fps against 90 Hz mean about one refresh in sixteen shows the previous image again. Motion that accumulates the time between updates is then off the display's timeline by the change in lag; motion sampled at the presentation time lands where the timeline says and steps by one period again on the next frame. +4. **The fix is one value handed down the loop: the frame's predicted presentation time**, with the delta from the previous presentation and the refreshes skipped between them (`FrameTime`). The compositor predicts it (`frame.predictTiming().presentationTime`) before the update phase, so the game update stays where it is, before the wait, and poses the frame for that instant. The engine's own delta (`timeSinceLastUpdate`) becomes the presentation delta, so every delta-driven path (skeletal clips, the camera path, scripts, extensions, physics accumulation) moves in step with the display without changing. +5. **Nothing existing breaks.** `gameUpdate(deltaTime:)` keeps its parameter (now the presentation delta); `setupCallbacks(frameUpdate:handleInput:)` hands the full `FrameTime`; `UpdateEvent`, `EngineExtensionUpdateContext`, `RenderGraphBuildContext` and `RenderPassContext` gain a `frameTime` field with a default. The macOS, iOS and AR loops build the same `FrameTime` from an estimate one period ahead of the draw, or from the host's display-link target when it gives one. + +## 1. What was seen + +On the headset, the dartboard moving along the tour shows small random steps every frame and larger occasional jumps. RealityKit moving the same object along the same path judders less. The engine's stats reported 94 fps on the 100 Hz display mode and 85 fps on 90 Hz: about one refresh in sixteen repeated the previous frame. + +The loop's budget at 100 Hz, from the engine stats (`EngineTimingStats`): loop 10.5 ms, update 1 ms, wait for the optimal input time 6.4 ms, encode 3 ms. + +## 2. Where the loop samples time today (verified in the code) + +The XR frame, `renderNewFrame()` in `Sources/UntoldEngineXR/UntoldEngineXR.swift`: + +1. `layerRenderer.queryNextFrame()`, then `frame.predictTiming()`: the compositor's predicted `optimalInputTime`, `renderingDeadline` and `presentationTime` for this frame (`frame_timing.h`: "the Mach absolute time at which the layer presents the frame onscreen"). +2. `frame.startUpdate()`; spatial input; `renderer.updateXR(useExternalStatsLifecycle: true)`, which is `runFrame` in `Sources/UntoldEngine/Renderer/UntoldEngine.swift`: `calculateDeltaTime()` set `timeSinceLastUpdate` from `CACurrentMediaTime()` minus the previous update's `CACurrentMediaTime()`; then animation, scripts, physics and `gameUpdateCallback?(timeSinceLastUpdate)`. +3. `frame.endUpdate()`; `LayerRenderer.Clock().wait(until: timing.optimalInputTime)` โ€” the 6.4 ms, which is whatever the period leaves after the previous frame's work. +4. `frame.startSubmission()`; for each drawable, the head pose is queried at `drawable.frameTiming.presentationTime` (`queryDeviceAnchor(atTimestamp:)`), and the eyes are drawn with it. + +So the head is posed for the presentation instant and the objects for the update's wall-clock instant, 9 to 11 ms earlier and varying. The viewer app (`Viewer/App/EngineSession.swift`, `update()`) computes `tourTime` from `ProcessInfo.processInfo.systemUptime` inside that update, and its RealityKit path (`Viewer/App/RealityKitSpace.swift`, `apply()`) does the same in `SceneEvents.Update`. The engine's `gameUpdate` callback only ever received a `Float` delta; nothing told the game when the frame would be shown. + +The three clocks involved share one base. `CACurrentMediaTime()`, `ProcessInfo.processInfo.systemUptime` and `mach_absolute_time()` agreed to within the time between the calls when read back to back on this machine (78 ยตs apart, nothing more), the engine already mixes them (ARKit anchor timestamps against `systemUptime` in the lighting system, `CACurrentMediaTime()` for the anchor query), and the compositor's `LayerRenderer.Clock` counts mach absolute time from its `epoch`. The engine used to convert a compositor instant through "now" on both clocks; the prototype converts through the epoch, as Apple's Metal immersive sample does, and prints once at start how far the two conversions are apart (`XR clock: ...`), which should be the microseconds between two reads. + +There is no transform keyframe or tween system for entities. The keyframes in `Sources/UntoldEngine/Mesh/Skeleton.swift` are skeletal clips sampled by `AnimationSystem`, and the camera path in `CameraSystem.swift` moves cameras. Neither is the cause; both are delta-driven and covered by the delta change below. + +## 3. The fix: the frame's time, handed down the loop + +### 3.1 The data + +`Sources/UntoldEngine/Renderer/FrameTime.swift` (new): + +```swift +public struct FrameTime: Sendable, Equatable { + /// Seconds at which the frame is predicted to be shown (CACurrentMediaTime base). + public let presentationTime: TimeInterval + /// Seconds from the previous frame's presentation to this one's; 0 on the first frame. + public let deltaTime: TimeInterval + /// The display's period as the loop knows it (nominal, or the shortest spacing seen). + public let period: TimeInterval + /// Refreshes between the previous frame and this one that showed the previous frame again. + public let skippedFrames: Int + /// The engine's count of frames prepared, from 1. + public let frameIndex: UInt64 + public var wasFrameSkipped: Bool { skippedFrames > 0 } +} +``` + +`FrameClock` (same file) makes one per frame from the presentation times the host predicts: the delta between consecutive presentations; the period, which the host declares (an MTKView's `1 / preferredFramesPerSecond`) or the clock learns as the shortest spacing between 4 ms and 42 ms seen so far (a skip never teaches it a longer period, a stall is not a period either); and `skippedFrames = round(delta / period) โˆ’ 1`, clamped at 0. A presentation before the previous one (a clock going backwards) passes no time. A jitter of a tenth of a millisecond on the compositor's predictions is not a skip. + +### 3.2 How it reaches the game without breaking existing callers + +- `UntoldRenderer.setupCallbacks(gameUpdate:handleInput:)` is unchanged in shape. Its `deltaTime` is now the presentation delta (`timeSinceLastUpdate`); the frame's time is `renderer.frameTime` or the `currentFrameTime` global while the callback runs. The global follows the engine's `timeSinceLastUpdate` style so code components and custom systems can read it too. +- `UntoldRenderer.setupCallbacks(frameUpdate:handleInput:)` (new, also on `UntoldEngineXR` and `UntoldEngineAR`) hands the `FrameTime` itself. A distinct label, not an overload of `gameUpdate:`, because an existing `gameUpdate: { _ in ... }` would be ambiguous between a `Float` and a `FrameTime` closure and stop compiling. Each form replaces the other. +- `UpdateEvent.frameTime` for `renderer.onUpdate { event in ... }` subscribers (`Sources/UntoldEngine/Systems/FrameEvents.swift`); the internal initializer keeps `UpdateEvent(deltaTime:renderer:)` working. +- `EngineExtensionUpdateContext.frameTime` (`Sources/UntoldEngine/ECS/EngineExtensions.swift`) for `EngineExtension.update(deltaTime:context:)` and `RenderExtension`s, built by `makeEngineExtensionUpdateContext()`. +- `RenderGraphBuildContext.frameTime` and `RenderPassContext.frameTime` (`Sources/UntoldEngine/Systems/GraphBuilder.swift`), next to their `deltaTime: Float`, built by `makeRenderGraphBuildContext()` and `makeRenderPassContext(...)`. All three contexts take `frameTime: FrameTime = .zero` as the last initializer parameter, so every existing call site compiles. +- `EngineTimingStats.presentationDeltaMs` and `.skippedFrames` (`Sources/UntoldEngine/Profiling/EngineStatsSnapshot.swift`) so the viewer's diagnostics can count skipped refreshes directly instead of inferring them from fps. +- `UntoldRenderer.updateXR(presentationTime:useExternalStatsLifecycle:)`: the host loop passes the predicted presentation time; `updateXR()` and `updateXR(useExternalStatsLifecycle:)` still compile and estimate one period ahead of now. + +`calculateDeltaTime(presentationTime:nominalPeriod:)` in `UntoldEngine.swift` is the only place the engine's delta is made. It advances the clock and sets `currentFrameTime`, `timeSinceLastUpdate` (the presentation delta) and `timeSinceLastUpdatePreviousTime` (now the presentation time). `firstUpdateCall == false` resets the clock, as the tests' state reset expects. + +### 3.3 The XR loop + +`renderNewFrame()` converts `timing.presentationTime` once, right after `predictTiming()`, and passes it to `updateXR(presentationTime:)`; the anchor query at submission converts `drawable.frameTiming.presentationTime` the same way. The two should be the same instant. `notePresentationRevision` counts the frames where the drawable's differs from the prediction by more than half a millisecond (`presentationTimeRevisions`, printed at most once a second): on such a frame the object was posed for a slightly different instant than the head, by exactly that difference. The count is the headset check that the prediction is stable (section 8). + +### 3.4 The viewer's change (`viewer-frame-time.patch`, beside this document) + +`Viewer/App/EngineSession.swift`: `xr.setupCallbacks(frameUpdate: { [weak self] frame in self?.update(frame: frame) }, handleInput: ...)`, `update(frame:)`, and the tour's time from the frame: `tourTime = (frame.presentationTime - object.placed).truncatingRemainder(dividingBy: Catalog.Tour.period)`. `object.placed` is already in the same clock base (`systemUptime`). For the headset check, the session sums `frame.skippedFrames` over all frames and reads `xr.presentationTimeRevisions`; both go into `EngineDiagnostics` (`skippedFrames`, `presentationRevisions`), the run summary (`MeasurementRun.swift`: ", N skipped refreshes, N presentation revisions") and the controls' engine line ("frames N (M skipped)"). The RealityKit path keeps `SceneEvents.Update`, which gives no presentation time; RealityKit poses its own frames for the compositor. + +The viewer's project points at the `xr_foveation` worktree (`project.yml`, `packages.UntoldEngine.path`), which builds `feature/xr_headset_rendering`; the viewer with this change compiles once this branch is merged into that one. The change is applied in the viewer's pull request and kept as a patch here (`patch -p1 < viewer-frame-time.patch` from the `UntoldViewer` directory; three files, taken against the viewer's tree of 2026-10-14). It was compiled against this branch through a throwaway XcodeGen project outside the viewer's tree. + +## 4. Where the game update runs: before the wait, posed for the predicted presentation + +Two places are possible for the update that poses the objects: + +**A. Before the wait (today's place), sampling the predicted presentation time.** The prediction is available from `predictTiming()` at the start of the update phase, so nothing moves. The CPU budget is untouched: at 100 Hz the update's 1 ms sits inside the 6.4 ms the loop spends waiting anyway; a slow update (a batching rebuild, a streaming tick) shortens the wait instead of threatening the deadline. The object is posed for the instant the compositor predicts, the same instant the head is posed for 6.4 ms later; whatever the prediction's error, head and object share it, which is what removes the judder. + +**B. After the wait, in the submission phase.** The update would run 6.4 ms closer to presentation, so game logic that reads input (a pinch dragging the object) would act on input 6.4 ms fresher. The cost: the CPU work after `optimalInputTime` grows from the 3 ms encode to 4 ms, a third more, on every frame, and the update's variance lands directly on the rendering deadline, where it drops frames instead of shortening a wait. With the loop at 10.5 ms for a 10 ms period there is no margin for that; at 90 Hz (11.1 ms) barely. + +Recommendation: A. It fixes the judder with no budget change. If fresher input matters later, the cheap form is a small "late input" hook after the wait that only reads input poses (hands at `timing.trackableAnchorTime`, the head) and writes transforms, not a full update after the wait. + +## 5. Engine paths that sample time, and what each needs + +| Path | File / function | Today | Change | +| --- | --- | --- | --- | +| The engine's delta | `UntoldEngine.swift` `calculateDeltaTime` | wall clock between updates | presentation delta from `FrameClock` (done) | +| Skeletal clips | `Systems/AnimationSystem.swift` `updateAnimationSystem(deltaTime:)`, `AnimationComponent.currentTime += deltaTime * playbackSpeed`; `currentGlobalTime += deltaTime` | delta-driven | none: the delta is now the presentation delta, so a clip's time stays in phase with the display; sampling a clip at an absolute time is possible but not needed | +| Camera path | `Systems/CameraSystem.swift` `updateCameraPath(deltaTime:)`, `segmentT += deltaTime / segmentDuration` | delta-driven, called by the game | none; the game passes the delta it receives. An `updateCameraPath(frame:)` sampling `segmentT` from an absolute time would make a skipped frame exact rather than in phase, a small follow-up | +| Tweening | none existed | โ€” | `Animation/TransformTrack.swift` (new): keys with easing sampled at an absolute time, `pose(for: frame, startedAt:)` and `applyTransformTrack(entityId:track:startedAt:frame:)` | +| Scripts | `Scripting/USCScripting.swift` `USCSystem.shared.update(timeSinceLastUpdate)` | delta-driven | none | +| Physics | `UntoldEngine.swift` `physicsAccumulator += timeSinceLastUpdate`, fixed step | accumulated | none; the accumulator now follows presentations | +| Particles | none in the engine; the muscle simulation's mass points (`Systems/Muscle/MuscleSimulation.swift`) step by a fixed substep | โ€” | none; a particle system added later takes `FrameTime` like everything else | +| Code components, custom systems | `onUpdate(deltaTime:)`, `updateCustomSystems(deltaTime:)` | delta-driven | none; `currentFrameTime` is readable inside them | +| Engine and render extensions | `ECS/EngineExtensions.swift` `update(deltaTime:context:)`; `Systems/GraphBuilder.swift` `makeRenderGraphBuildContext`, `makeRenderPassContext` | `deltaTime: Float` | `context.frameTime` added (done) | +| Update event | `Systems/FrameEvents.swift` `UpdateEvent` | `deltaTime` | `frameTime` added (done) | +| Head pose | `UntoldEngineXR.swift` `renderNewFrame`, anchor query | `drawable.frameTiming.presentationTime` through "now" on both clocks | through the compositor clock's epoch, the same conversion the game's time uses (done) | +| Room light fade | `UntoldEngineXR/XREnvironmentLightingSystem.swift` (`ProcessInfo.systemUptime`) | wall clock | optional: a slow fade, not judder; could read `currentFrameTime.presentationTime` for consistency | + +## 6. The platform side: the same `FrameTime` everywhere + +- **visionOS** (`UntoldEngineXR`): the compositor's prediction, as above. The period is learnt (the compositor does not state the display's rate; the loop measures 10.0 ms at 100 Hz, 11.1 ms at 90 Hz). +- **macOS / iOS `MTKView`** (`UntoldRenderer.draw(in:)`): MTKView's delegate gets no timestamp from the view's own display link, so the frame is posed one period (`1 / view.preferredFramesPerSecond`) ahead of the draw and the period is the nominal one. The delta between frames is then the spacing of the draws, as before. A host that drives the view from its own display link (`CADisplayLink` on iOS, `NSView.displayLink(target:selector:)` on macOS 14) pauses the view's loop (`isPaused = true`, `enableSetNeedsDisplay = false`), sets `renderer.predictedPresentationTime = link.targetTimestamp` and calls `view.draw()` from the link; the frame is then posed for the link's target. A follow-up can phase-lock the estimate from `CAMetalDrawable.addPresentedHandler`'s `presentedTime` without a host display link. +- **iOS AR** (`UntoldEngineAR.draw(in:)`): one of the view's periods ahead of now, passed as `updateXR(presentationTime:)`. The camera image is already in the past; the objects are posed for when the composite is shown. + +## 7. Tests (all passing) + +- `Tests/UntoldEngineTests/FrameClockTests.swift` (9): first frame, steady cadence, a skipped refresh counted once and the next frame back to one period, jitter is not a skip, a stall counts every refresh, a nominal period, time going backwards, the estimate ahead of now, reset. +- `Tests/UntoldEngineTests/TransformTrackTests.swift` (5), including `testARepeatedFrameDoesNotDoubleStepAnAbsoluteTimeAnimation`: slots 0, 1, 2, [3 missed], 4, 5 with updates that run a varying lag before each presentation. Sampled at presentation time, the positions are exactly the timeline's, one period per frame, two across the skipped slot and one again after it; stepped by the time between updates, no step is a period and the position after the skip is off the timeline. +- `Tests/UntoldEngineRenderTests/FrameTimeLoopTests.swift` (3, with a real renderer): the loop drives `updateXR(presentationTime:)` on a timeline a thousand seconds from the wall clock; the pose the game samples in `frameUpdate` is the pose for the frame's presentation time, the engine's delta is the presentation delta and not the microseconds between the calls, the update event and an extension receive the same `FrameTime`, the legacy `gameUpdate` delta is the presentation delta and a two-period gap reports one skipped frame; a frame without a prediction is posed one period ahead of now. + +The compositor part cannot run in `swift test`; section 8 is its check. + +## 8. Headset check + +1. The start-up line `XR clock: the compositor epoch's time differs from the media time by N ms`: N should be the microseconds between two reads. A large constant N means the epoch assumption is wrong and the loop must go back to converting through "now" on both clocks. +2. `presentationTimeRevisions` (and its log line) over a minute of the tour: near zero means the predicted and final presentation times agree and the object is posed for the head's instant; a steady count means the prediction is revised by the compositor and the game should be posed for the drawable's time instead (option B's place), or the revision applied as a per-frame correction. +3. The viewer's run summary: "N skipped refreshes" over the run against the fps shortfall (6 in 100, 5 in 90): they should match. "N presentation revisions" is check 2. +4. The tour, with the viewer's change of section 3.4: the small per-frame steps should be gone; the occasional larger jump remains for every skipped refresh and should now be one clean double step with no drift after it. + +### Results, 2026-10-15, the viewer's dartboard tour at 90 Hz + +1. `XR clock: the compositor epoch's time differs from the media time by -0.000 ms`: the clock bases agree. +2. Presentation revisions: none. The drawable's presentation time is the one predicted at the update. +3. Skipped refreshes: none. The loop never saw a presentation slot go by without a frame. +4. The small per-frame steps are gone. A hop remained on the tour's lateral sweep only (0.63 m/s peak, 0.15ยฐ per frame at 2.37 m, where a frame shown a refresh late is a 12 mm sideways hop), with every counter above at zero. + +The hop is the GPU finishing after the compositor's rendering deadline, which no counter of the loop's pacing sees; `deadlineMisses` and its per-second line (`XR frame timing: the GPU finished after the rendering deadline on N of M frames in the last second`) found it: + +| Run | Late frames per second | Mean overrun | Worst | +| --- | --- | --- | --- | +| Wait for the optimal input time (default), board close | 80 to 91 of 90 | 0.5 to 1.3 ms | 2.6 ms | +| The same, board far (fewer pixels) | 15 to 44 of 90 | 0.2 to 0.7 ms | 3.0 ms | +| `setRendering(.optimalInputWait(.disabled))`: encode right after the update | 56 of 92 | 0.42 ms | 2.2 ms | +| A room probe's prefilter, either run, for about six seconds | every frame, at 71 to 78 fps | 5 to 15 ms | 21 ms | + +The compositor keeps some slack past its stated deadline (the loop still gets 90 frames a second with nearly every frame a millisecond late), and the frames beyond that slack are the hops. Encoding right after the update starts the GPU about 6 ms earlier and brought the hops down to a few; it does not make the frames finish before the deadline, so the frame's GPU work is close to the whole window. The remedies, in order of what they keep: late-latching the head pose (encode before the wait, write the eye matrices after it: the early GPU start and the fresh head pose), cutting the frame's GPU cost (the temporal resolve, the render quality), and spreading a probe's prefilter over frames so a new probe does not cost six seconds at 74 fps. + +### The adaptive wait (done) + +`setRendering(.optimalInputWait(.always | .never | .automatic))`, automatic by default: the loop waits for the optimal input time while the frames meet the deadline, stops once a second had late frames (one in twenty or more), and waits again once a second's least slack before the deadline would have covered the longest wait skipped, with a 1.5 ms margin. The fresh head pose whenever the budget allows; the compositor's reprojection of an older prediction only when it does not. `optimalInputWaitActive` says what the last frame did; the viewer's switch maps to automatic or never and its summary notes "input-time wait skipped (automatic)". + +### What the passes cost (2026-10-15, the viewer at 100 Hz, render quality at the platform default) + +With the GPU pass timing on, a second of late frames names its heaviest passes. The board close (the tour's first phase, mistaken at first for a probe's prefilter): Forward 8.4 to 9.9 ms, TAA 1.4 to 3.5 ms, the depth pyramid 1 to 2.5 ms, a frame of 12.5 to 13 ms at 72 to 80 fps. The board far: Forward 4.2 to 6 ms, a frame of about 10 ms, half its frames a millisecond late. One switch at a time: TAA only (no MSAA) saves 1.5 to 2 ms, the one real saving; occlusion cull off removes the pyramid, built needlessly for one object's many meshes, for half a millisecond of frame rate at most; shadow taps 4 saves nothing measurable. The far phase can meet 100 Hz with TAA only and occlusion cull off; the close phase cannot by settings: the Forward pass is the board's own shading, scaling with the pixels it covers. + +### Half-rate pacing (done) + +`setRendering(.framePacing(.fullRate | .halfRateWhenLate | .halfRate))`, half rate when late by default. `XRHalfRatePolicy` decides from each second's measurements. It enters when, with no wait for the optimal input time left to skip, the display skipped a refresh for one frame in ten or more (from the spacing of the compositor's consecutive predicted presentations) or one frame in ten or more finished past the deadline by more than the deadline's margin, for two seconds running: the deadline sits 15 ms before the presentation on the headset, so a frame a few milliseconds past it is still shown on time (the full-rate runs with nearly every frame a millisecond late dropped nothing), and fewer hops than one in ten are fewer than the steps the half rate adds. While active the loop waits, before it asks the compositor for the next frame, until the next slot of a fixed grid: the instant the pacing started from plus a whole number of cadences (two refreshes, or three or four when the frame's typical GPU work, its mean span plus a fifth and 3 ms of encode, needs them; the cadence rises at once and falls after two fitting seconds, a millisecond's deadband at the boundary). The query itself never blocks, the compositor predicts the presentation three refreshes after it, so the grid, not the previous frame's presentation, sets the spacing, and the predicted presentations come exactly a cadence apart (22.2 ms flat at 90 Hz every second of the heavy run). Each frame is posed for the compositor's prediction plus the refreshes the frames land past it: none while they finish within the deadline's margin less a 2 ms safety, else as many as the rest reaches. Nothing waits for the GPU: the next frame's work queues behind the previous one's. (Lessons from the device: a first version let every other frame go unsubmitted, and the compositor never releases a frame taken and not submitted, three of them aborting the app with "We reached more than 3 frames in flight", so the pacing is in the taking; a second waited for each frame's GPU work before taking the next, and the idle gaps let the GPU clock down, the Forward pass going from 9 to 12 ms and the cadence from two to three, 30 fps, so the GPU is kept queued; a third snapped the query to the grid of the last frame's presentation instant, which the compositor predicts three refreshes ahead, so every frame waited a refresh too many, 30 fps again; a fourth counted the cadence from the instant the last frame was handed out, which drifted with the query's own lateness and gave an uneven 16.6 ms.) It leaves when the rendered frames' GPU work has fitted 80% of the refresh for two seconds running. `halfRatePacingActive` says the state; the per-second line adds "half rate, every N refreshes, landing +N", the frames past the margin and the refreshes skipped, and the spacing of the presentations; the viewer's switch maps to half rate when late or full rate, with "half-rate pacing active" in its summary. By eye on the device (2026-10-15, MSAA 4ร— with TAA and the occlusion cull at 90 Hz, the GPU at 13 to 15 ms a frame): the even 45 fps shows fewer jumps than the hopping full rate did, and still a visible stepping: a sweep at half the refresh rate is not smooth, it is evenly unsmooth, and the frame rate itself is the limit there. The cure for that configuration is the frame's GPU cost (the Forward pass at 6 to 9 ms, the 4ร— resolve, the HZB built for one object), not the timing. + +### The frame's cost on the headset (`feature/xr_frame_cost`, in part) + +The heavy configuration (MSAA 4ร— with TAA, the occlusion cull on, foveated 2048ร—1984 per eye, render quality at the platform default) measured on 2026-10-15 at 90 Hz with the sun switch off: at full rate the GPU took 16.8 to 17.6 ms a frame (Forward 8.7 to 9.2, TAA 2.2 to 3.4, Shadows 1.7), and paced to every other refresh 14 to 16 ms (the Forward pass 10 to 14 at the lower clock an idle GPU falls to). The viewer's per-pass line, both eyes summed, with the board close: Forward 10.40 ms (3.46 vertex, 6.94 fragment), TAA 2.50, Shadows 1.17 (0.93 of it vertex), Post 0.76, Culling 0.14, Transparency 0.04. The budget is 11.1 ms. Five readers mapped the passes, the forward shading, the pyramid, the temporal resolve and the timing labels, and each candidate cut was refuted against the code before it was taken; the ones that held: + +- **The depth pyramid's gate counts objects.** A model loaded from a file is one entity per node under the entity it was set on, so one dartboard was many culling entities and built the depth copy, the pyramid and the occlusion test every frame (1 to 2.5 ms) for an object that hides nothing but itself. The gate counts an entity with its children as one. Confirmed on the device: HZB left the pass list. +- **No shadow cascades for a sun at zero intensity.** The viewer's sun switch leaves the light at intensity 0 with `castsShadow` on, and the cascades were drawn regardless, two 2048ยฒ depth maps of every part of the board: 0.5 to 2.9 ms a frame with no sun. The shading multiplies the shadow by the intensity, so the cascade update refuses a sun at zero. +- **The look pass without a depth attachment.** It drew through the post-process descriptor, which loads and stores the frame's depth for the passes that read it; the look pass never touches depth. It has its own colour-only descriptor and pipeline: a full-size depth read and write per eye gone. +- **The frame's colour textures compressible.** The lit colour, the composite, the look and anti-aliasing outputs and the environment were created with `shaderWrite` usage, which no kernel uses and which forbids the GPU's lossless compression of a texture; every resolve, read and write of them moved uncompressed pixels. +- **Fence waits before the fragment stage.** Every scene pass waited on the shared fence before its vertex stage, so one pass's tiler could not run while the previous pass shaded; a pass's vertex stage reads nothing a previous pass's fragments wrote, and every resource is hazard-tracked, so the wait moves to the fragment stage at 23 sites, as the temporal resolve already had it. The Forward pass's 3.5 ms of binning can now overlap the shadows' and culling's fragments. +- **No transparency pass with nothing transparent.** Its encoder loaded and stored both full-size attachments to draw nothing; the GPU spends 0.04 ms on an empty pass, so this one is tidiness. +- **The per-second line lists every pass with their sum**, and the frame's own GPU time after the frame before, so the device's numbers read against the span. + +What did not hold, and why: the half-size pyramid, its single encoder and the first eye's copy skipped matter only to multi-object scenes; the transparency pass's cost was a guess an order of magnitude too high; moving the grade into the temporal resolve changes the history's space; float to half across the forward shader is not safe without a device check. + +The device run of those cuts (the sun off, 90 Hz): the frame held the full rate, 8.4 to 9.0 ms a frame, no refresh skipped after the placement, the pacing never engaged; Forward 10.40 to 8.02 ms, TAA 2.50 to 2.06, Post 0.76 to 0.45, the shadows and the transparency pass gone, the cull 0.14 to 0.01. + +What remained was the Forward pass's fragment work (5.5 of its 8 ms) and the temporal resolve, whose levers are in the shaders. Every `.metal` file is included by `UntoldEngineKernels.metal` and only the prebuilt metallibs ship, so those cuts rebuild the kernel libraries; they are on `feature/xr_shader_cost`, stacked on the cost branch, after twelve reviewers (Metal semantics, picture equivalence, performance, three per cut) tried to refute each: + +- **The sun by its intensity.** The cascade shadow taps and the sun's Cook-Torrance BRDF are evaluated only when the sun's intensity is not zero, in the forward shader, the two light passes and the transparency shader, on a per-frame constant, so the branch is uniform; any other intensity takes the old path in the old order of products, bit for bit. The CPU's cascade gate says the same. Reviewers' estimate: 1 to 1.5 ms with the sun off. +- **Each material map sampled only when the material has it**, the normal map's decode and tangent transform included; the flags are per-draw constants, so the implicit-derivative samples stay in uniform control flow, as the height-map branch already was. The reviewers put this at nothing measurable (the removed samples were on nil textures); it stays as a cleanup. +- **Once per pixel instead of twice**: the view vector, the environment's rotation angle in radians, and the BRDF lookup table's pair, from which both reflectances derive (`brdfIntegration`); the rotation of the lookup direction is skipped at angle zero, the headset's room probe's case, with the normalize kept, since the simulator's light pass hands in a G-buffer normal as stored and the equirectangular latitude's `asin` is not scale-invariant (the reviewers' one blocker); that `asin` is clamped now, which removes a pre-existing chance of a NaN pixel at the poles. +- **The temporal resolve leaves early** for a pixel with no history or at the clear depth, before the rate-map decodes, the reprojection and the velocity fetch, which is most of a passthrough frame; the foveation rates are read as texels per screen pixel over the same screen interval from both maps, a second decode of each, instead of derivatives across the quad (which forbade any early return): the ratio is exactly 1 wherever the two maps agree, zone boundaries included, and where they differ a band at most a pixel wide before a boundary weighs a little differently than the quad's span did. Reviewers' estimate: 0.2 to 0.4 ms. + +The picture tests (the material shading, per-fragment lighting, glass, probe, post-effects and temporal tests, 148 in the targeted run) pass on the rebuilt library. On the viewer's side, MSAA 2ร— under TAA costs 1.5 to 2 ms less than 4ร— for edges the temporal accumulation already resolves. + +The device run of the shader cuts, the sun off, the headset at 100 Hz this time: Forward 8.02 to 6.56 ms (the fragment part 5.51 to 4.15), TAA 2.06 to 1.92 (on the empty frame 1.81 to 1.47), the frame 6.8 ms against a 10 ms refresh, no late frame after the placement. With the sun on, the frame no longer fitted (10 to 12 ms at full rate, then paced to an even 51 frames a second), and the shadow-tap count made no difference (Forward 9.48 with eight taps, 9.75 with four): the Forward pass's vertex stage had risen too, 2.41 to 3.35 ms on the same geometry, which is the device's GPU clock falling as it heats through a long session, so the sun's real cost is about 1.5 ms, 0.85 of it the two cascades and the rest the taps' setup and the sun's BRDF, as its arithmetic says. + +**The cheaper sun.** The headset starts without a sun (`UntoldRendererConfig.defaultSun`), and a project that switches one on gets it in one cascade: `setRendering(.shadowCascades(1 or 2))`, the headset's default one and the desktop's two. A room or a tabletop never shows the distance the second cascade covers, and with `maxShadowCastingDistance` set to the scene's size one cascade is sharp on it; the second cascade's draw of every caster and the cross-fade's double taps go. The cascade count flows from the setting through the splits, the fitting, the passes and the shader's uniforms; the depth array keeps its slices. + +### The true late latch (not done: a refactor, for a decision) + +The head pose is consumed on the CPU while the frame is encoded: the view and projection matrices are read at 41 places in `RenderPasses.swift` (each pass copies them into its own uniforms: the batches, the grid, the sky, the environment, the shadow cascades, the SSAO's projection scale), 7 in `CullingSystem.swift` (the frustum planes and the view-projection for the occlusion test), 5 in `TemporalAntiAliasing.swift` (the jitter and the reprojection), 3 in `GraphBuilder.swift`, and a few in the extension surface and the muscle debug pass. Encoding before the wait and writing the pose after it therefore needs one per-frame, per-eye camera buffer (view, projection, their inverses, the view-projection, the eye position, the jitter) bound at a fixed index for every pass and read by every shader in place of the copies; the frustum planes built on the GPU from it (a small kernel before the culls, or the culls reading the matrices directly); the temporal resolve taking the previous frame's buffer as its reprojection; the shadow cascades fitted from a provisional pose a few milliseconds older, which is harmless; and the drawable, with its foveation rate maps, taken before the wait, earlier than Apple's sample does, to be verified on the device first. It touches every pass and most shaders, a few days of work, and it should follow #1359 rather than ride on it. It buys about 3 ms of GPU window (the encode after the wait) with today's head pose; the adaptive wait buys about 9 ms with an older one when needed. + +## 9. Size, and the risk to the three draft pull requests + +Size: about 330 lines in `Sources` (`FrameTime.swift` 130, `TransformTrack.swift` 150, the loop and context wiring 50), 330 lines of tests, this document. The viewer's change is a few lines. + +The branch was cut from `try/xr_foveation_specular_aa` and then rebased onto `feature/xr_headset_rendering` (untoldengine/UntoldEngine#1359, which supersedes the three drafts below), with no conflicts; it sits on top of foveation (`feature/xr_foveation`, untoldengine/UntoldEngine#1342), temporal anti-aliasing (`feature/taa`, untoldengine/UntoldEngine#1343) and the back-face lighting fix (`bugfix/backface_lighting`, untoldengine/UntoldEngine#1344). Landing order: #1359, then this change on top of it. + +- **Foveation (#1342)**: textual overlap in `UntoldEngineXR.swift`. The PR rewrote the drawables loop (`for drawable in drawables`, the rate maps); this change adds two lines inside it (the once-per-frame revision check) and five properties next to the foveation ones. Applied after the PR: clean. Applied on `develop` without it: the loop does not exist in that form, so the hunk must be redone by hand (two lines). `GraphBuilder.swift` is touched by both, in different functions (`applyXRFoveationViewport` against the three context builders), ~15 lines apart: at worst an adjacent-hunk conflict. +- **Temporal anti-aliasing (#1343)**: one certain conflict, three lines: `updateXR` in `UntoldEngine.swift`, where the PR added `ObjectMotion.shared.beginFrame()` and this change adds the `presentationTime` parameter; the resolution keeps both. Semantically the change helps TAA: `ObjectMotion` records each entity's previous transform per frame, and an object posed for the presentation instant moves by whole periods, so its motion vectors are steady instead of carrying the update's lag jitter. +- **Back-face lighting (#1344)**: shaders only, no overlap. + +Behaviour changes to note in review: `timeSinceLastUpdate` is the presentation delta (exactly a period or a multiple on the headset; the draw spacing on a Mac as before); `timeSinceLastUpdatePreviousTime` holds the presentation time; `EngineTimingStats.frameTotalMs` on the headset is the presentation delta. + +## 10. Follow-ups + +- The headset check (section 8), with the viewer's change of section 3.4. +- A `TransformTrackComponent` with a system that samples it at `currentFrameTime.presentationTime` every frame, so a track plays with no game code; `TransformTrack` is the value type it would hold. +- `updateCameraPath(frame:)` sampling from an absolute time. +- A presented-time feedback for the `MTKView` estimate (section 6). +- A late-input hook after the wait (section 4), or its stronger form from the headset results: late-latching the head pose, so the frame is encoded before the wait and only the eye matrices are written after it. +- The room probe's prefilter spread over frames (section 8). diff --git a/docs/proposals/viewer-frame-time.patch b/docs/proposals/viewer-frame-time.patch new file mode 100644 index 000000000..00f116cec --- /dev/null +++ b/docs/proposals/viewer-frame-time.patch @@ -0,0 +1,106 @@ +diff -ruN a/Viewer/App/EngineSession.swift b/Viewer/App/EngineSession.swift +--- a/Viewer/App/EngineSession.swift 2026-10-14 23:22:53 ++++ b/Viewer/App/EngineSession.swift 2026-10-14 23:22:52 +@@ -103,6 +103,12 @@ + var textureBudgetMB = 0 + /// Seconds into the debug tour's loop, or nil when the object stands still. + var tourTime: Double? ++ /// Refreshes at which the display showed the previous frame again because the engine's ++ /// next was not ready, over all frames: the fps shortfall. And the frames whose final ++ /// presentation time differed from the one predicted at the update, which the tour's ++ /// pose was taken at. ++ var skippedFrames = 0 ++ var presentationRevisions = 0 + /// The frame's draw calls and triangles, from the engine's own counters. + var drawCalls = 0 + var triangles = 0 +@@ -168,6 +174,8 @@ + private let lock = NSLock() + private var diagnosticsStorage = EngineDiagnostics() + private var frames: UInt64 = 0 ++ /// Refreshes the display skipped over those frames (the previous frame shown again). ++ private var skippedFrames = 0 + private var objectName = "" + private var phase: OSSignpostIntervalState? + /// Exponential moving averages of the GPU time: all frames, and the frames before the object. +@@ -227,13 +235,16 @@ + .rotateBy(angle: -50, axis: [.x]) + .rotateBy(angle: 30, axis: [.y]) + frames = 0 ++ skippedFrames = 0 + phase = MeasurementRun.signposter.beginInterval("Engine", id: MeasurementRun.signposter.makeSignpostID(), "empty") + // A pinch-drag moves the object (closer, farther, up, down): the engine + // reports the pinching hand's position from the layer's spatial events. + InputSystem.shared.registerXREvents() + setInput(.xr(.sceneReady(true))) ++ // The frame's time: the instant the compositor predicts the frame will be shown, which ++ // the head pose is predicted for too, so the tour's pose for it keeps step with the head. + xr.setupCallbacks( +- gameUpdate: { [weak self] _ in self?.update() }, ++ frameUpdate: { [weak self] frame in self?.update(frame: frame) }, + handleInput: { [weak self] in self?.handleInput() } + ) + let thread = Thread { [self] in +@@ -326,9 +337,13 @@ + Self.log.log("engine: \(placement.object.id, privacy: .public) centred at (\(placement.centre.x, format: .fixed(precision: 2)), \(placement.centre.y, format: .fixed(precision: 2)), \(placement.centre.z, format: .fixed(precision: 2))), \(head == nil ? "room frame" : "in front of the wearer", privacy: .public)") + } + +- /// Once per frame on the render thread. +- private func update() { +- lock.withLock { frames &+= 1 } ++ /// Once per frame on the render thread, with the frame's time: the instant the compositor ++ /// predicts the frame will be shown. ++ private func update(frame: FrameTime) { ++ lock.withLock { ++ frames &+= 1 ++ skippedFrames += frame.skippedFrames ++ } + // About a second's window at 90 Hz; the first frames after a mode change are noisy. + let timing = EngineStatsMonitor.shared.snapshot().timing + let gpu = timing.gpuExecutionMs +@@ -374,7 +389,11 @@ + var tourTime: Double? + if let object = lock.withLock({ object }) { + if Catalog.motionIsTour { +- tourTime = (ProcessInfo.processInfo.systemUptime - object.placed).truncatingRemainder(dividingBy: Catalog.Tour.period) ++ // The tour's time at the instant the frame will be shown, not at this update: ++ // the pose then steps by the display's period from one frame to the next, and by ++ // two after a skipped one, instead of by the varying lag of the update behind ++ // the presentation. `placed` is in the same clock base (the system uptime). ++ tourTime = (frame.presentationTime - object.placed).truncatingRemainder(dividingBy: Catalog.Tour.period) + } + let pose = object.placement.pose(tourTime: tourTime) + translateTo(entityId: object.entity, position: pose.position) +@@ -422,6 +441,8 @@ + textureMemoryUsedMB: MemoryBudgetManager.shared.getStats().textureMemoryUsed / (1024 * 1024), + textureBudgetMB: MemoryBudgetManager.shared.textureBudget / (1024 * 1024), + tourTime: tourTime, ++ skippedFrames: lock.withLock { skippedFrames }, ++ presentationRevisions: xr?.presentationTimeRevisions ?? 0, + drawCalls: stats.render.drawCallsTotal, + triangles: stats.render.trianglesTotal, + opaqueDrawsLeast: opaqueDrawsLeast == Int.max ? 0 : opaqueDrawsLeast, +diff -ruN a/Viewer/App/MeasurementRun.swift b/Viewer/App/MeasurementRun.swift +--- a/Viewer/App/MeasurementRun.swift 2026-10-14 23:22:53 ++++ b/Viewer/App/MeasurementRun.swift 2026-10-14 23:22:52 +@@ -213,7 +213,7 @@ + object, d.gpuMilliseconds, d.emptyGpuMilliseconds, d.objectGpuMilliseconds ?? 0, fps, emptyFps, + d.antiAliasing, d.opaqueSampleCount, d.viewport.x, d.viewport.y, + d.screenSize.x > 0 ? String(format: " fov %dร—%d", d.screenSize.x, d.screenSize.y) : "", +- d.drawCalls, d.triangles, (d.compositeSkipped ? ", composite skipped" : "") + ", " + d.renderPath + (d.tileLightStage ? " (tile light)" : "") + String(format: ", draws %d plain / %d discarding", d.plainDraws, d.discardingDraws) + (d.foveation ? "" : ", no foveation") + String(format: ", cull set %d", d.cullCandidates) + (d.shadowTaps == 8 ? "" : String(format: ", shadow taps %d", d.shadowTaps)) + (d.opaqueDrawsMost > 0 ? String(format: ", opaque draws %dโ€“%d%@", d.opaqueDrawsLeast, d.opaqueDrawsMost, d.batchedDrawsMost > 0 ? String(format: " + %d batched", d.batchedDrawsMost) : "") : "") + (d.depthFormat.isEmpty ? "" : String(format: ", depth %@ (engine near %.3f)", d.depthFormat, d.nearPlane)) + (d.toneMap == "none" ? "" : ", tone map " + d.toneMap) + String(format: ", textures %d up / %d down, %d at full, %d in flight, %d of %d MB", d.textureUpgrades, d.textureDowngrades, d.textureEntitiesAtFull, d.textureOpsInFlight, d.textureMemoryUsedMB, d.textureBudgetMB) + (d.sun ? "" : ", no sun") + (d.roomLightScale > 0 ? String(format: ", room light scale %.2f (camera ref %.0f, published %.2f) ร— %.0f", d.roomLightScale, d.roomLightCameraReference, d.roomLightPublished, d.roomLightFactor) : "") + (d.probe.isEmpty ? "" : String(format: ", probe %@, %d publishes (%d updates accepted, %d skipped, last %.0f s ago)", d.probe, d.probePublishes, d.probeAccepted, d.probeSkipped, d.probeAge)) + (d.probeFaces.count == 6 ? String(format: ", probe faces +x %.3f โˆ’x %.3f +y %.3f โˆ’y %.3f +z %.3f โˆ’z %.3f", d.probeFaces[0], d.probeFaces[1], d.probeFaces[2], d.probeFaces[3], d.probeFaces[4], d.probeFaces[5]) : "") + (d.wideGamut ? "" : ", sRGB primaries") + (d.occlusionCull ? "" : ", occlusion off") + (d.taaJitter ? "" : ", jitter off") + (d.textureStreaming ? "" : ", streaming off") + (d.lighting == "room" ? "" : ", studio light") + String(format: ", %d dark frames%@", d.darkFrames, d.darkFrameTimes.isEmpty ? "" : " (tour " + d.darkFrameTimes.map { String(format: "%.1f", $0) }.joined(separator: " ") + " s)") + (d.engineOptimized ? "" : ", UNOPTIMIZED ENGINE") + (d.drawCalls == 0 && d.gpuMilliseconds == 0 ? ", NO ENGINE STATS (build with -DENGINE_STATS_ENABLED)" : "")) ++ d.drawCalls, d.triangles, (d.compositeSkipped ? ", composite skipped" : "") + ", " + d.renderPath + (d.tileLightStage ? " (tile light)" : "") + String(format: ", draws %d plain / %d discarding", d.plainDraws, d.discardingDraws) + (d.foveation ? "" : ", no foveation") + String(format: ", cull set %d", d.cullCandidates) + (d.shadowTaps == 8 ? "" : String(format: ", shadow taps %d", d.shadowTaps)) + (d.opaqueDrawsMost > 0 ? String(format: ", opaque draws %dโ€“%d%@", d.opaqueDrawsLeast, d.opaqueDrawsMost, d.batchedDrawsMost > 0 ? String(format: " + %d batched", d.batchedDrawsMost) : "") : "") + (d.depthFormat.isEmpty ? "" : String(format: ", depth %@ (engine near %.3f)", d.depthFormat, d.nearPlane)) + (d.toneMap == "none" ? "" : ", tone map " + d.toneMap) + String(format: ", textures %d up / %d down, %d at full, %d in flight, %d of %d MB", d.textureUpgrades, d.textureDowngrades, d.textureEntitiesAtFull, d.textureOpsInFlight, d.textureMemoryUsedMB, d.textureBudgetMB) + (d.sun ? "" : ", no sun") + (d.roomLightScale > 0 ? String(format: ", room light scale %.2f (camera ref %.0f, published %.2f) ร— %.0f", d.roomLightScale, d.roomLightCameraReference, d.roomLightPublished, d.roomLightFactor) : "") + (d.probe.isEmpty ? "" : String(format: ", probe %@, %d publishes (%d updates accepted, %d skipped, last %.0f s ago)", d.probe, d.probePublishes, d.probeAccepted, d.probeSkipped, d.probeAge)) + (d.probeFaces.count == 6 ? String(format: ", probe faces +x %.3f โˆ’x %.3f +y %.3f โˆ’y %.3f +z %.3f โˆ’z %.3f", d.probeFaces[0], d.probeFaces[1], d.probeFaces[2], d.probeFaces[3], d.probeFaces[4], d.probeFaces[5]) : "") + (d.wideGamut ? "" : ", sRGB primaries") + (d.occlusionCull ? "" : ", occlusion off") + (d.taaJitter ? "" : ", jitter off") + (d.textureStreaming ? "" : ", streaming off") + (d.lighting == "room" ? "" : ", studio light") + String(format: ", %d dark frames%@", d.darkFrames, d.darkFrameTimes.isEmpty ? "" : " (tour " + d.darkFrameTimes.map { String(format: "%.1f", $0) }.joined(separator: " ") + " s)") + String(format: ", %d skipped refreshes, %d presentation revisions", d.skippedFrames, d.presentationRevisions) + (d.engineOptimized ? "" : ", UNOPTIMIZED ENGINE") + (d.drawCalls == 0 && d.gpuMilliseconds == 0 ? ", NO ENGINE STATS (build with -DENGINE_STATS_ENABLED)" : "")) + summary.append(line) + consoleLine("measure: " + line) + Self.log.log("measure: \(line, privacy: .public)") +diff -ruN a/Viewer/UI/ControlsView.swift b/Viewer/UI/ControlsView.swift +--- a/Viewer/UI/ControlsView.swift 2026-10-14 23:22:53 ++++ b/Viewer/UI/ControlsView.swift 2026-10-14 23:22:52 +@@ -383,7 +383,7 @@ + .lineLimit(2) + } + Text(engineOpen +- ? "engine ยท \(d.object.isEmpty ? "placingโ€ฆ" : d.object)\(d.tourTime.map { String(format: " ยท tour %.1f s", $0) } ?? "") ยท frames \(d.frames) ยท AA \(d.antiAliasing) ร—\(d.opaqueSampleCount) ยท \(d.viewport.x)ร—\(d.viewport.y)\(d.screenSize.x > 0 ? " fov \(d.screenSize.x)ร—\(d.screenSize.y)" : "") ยท \(gpu)\(fps)\(d.compositeSkipped ? " ยท composite skipped" : "") ยท \(d.renderPath)\(d.tileLightStage ? " (tile light)" : "")" ++ ? "engine ยท \(d.object.isEmpty ? "placingโ€ฆ" : d.object)\(d.tourTime.map { String(format: " ยท tour %.1f s", $0) } ?? "") ยท frames \(d.frames)\(d.skippedFrames > 0 ? " (\(d.skippedFrames) skipped)" : "") ยท AA \(d.antiAliasing) ร—\(d.opaqueSampleCount) ยท \(d.viewport.x)ร—\(d.viewport.y)\(d.screenSize.x > 0 ? " fov \(d.screenSize.x)ร—\(d.screenSize.y)" : "") ยท \(gpu)\(fps)\(d.compositeSkipped ? " ยท composite skipped" : "") ยท \(d.renderPath)\(d.tileLightStage ? " (tile light)" : "")" + : (rkOpen ? "realitykit ยท \(Catalog.selectedID) ยท updates \(String(format: "%.0f", rkRate)) Hz ยท gpu: out of process โ€” Instruments โ–ธ RealityKit Trace, with and without the object" : "no space open ยท \(EngineSession.shared.lastOpenResult)")) + .font(.system(size: 11, design: .monospaced)) + .foregroundStyle(.secondary) diff --git a/scripts/install-untoldengine-create.sh b/scripts/install-untoldengine-create.sh index 7cbff0674..850458541 100755 --- a/scripts/install-untoldengine-create.sh +++ b/scripts/install-untoldengine-create.sh @@ -88,6 +88,25 @@ else sudo chmod +x "$INSTALL_DIR/$CLI_NAME" fi +# SwiftPM generates a resource bundle (manifest.json, etc.) as a sibling of the +# binary under .build//. Bundle.main's resource lookup expects it next +# to the installed binary too, so copy it alongside $CLI_NAME in $INSTALL_DIR. +RESOURCE_BUNDLE="UntoldEngineCLI_UntoldEngineCLI.bundle" +BUNDLE_PATH=".build/$BUILD_CONFIG/$RESOURCE_BUNDLE" +if [ -d "$BUNDLE_PATH" ]; then + echo -e "${CYAN}๐Ÿ“ฆ Installing resource bundle to $INSTALL_DIR/$RESOURCE_BUNDLE...${NC}" + if [ -w "$INSTALL_DIR" ]; then + rm -rf "$INSTALL_DIR/$RESOURCE_BUNDLE" + cp -R "$BUNDLE_PATH" "$INSTALL_DIR/$RESOURCE_BUNDLE" + else + sudo rm -rf "$INSTALL_DIR/$RESOURCE_BUNDLE" + sudo cp -R "$BUNDLE_PATH" "$INSTALL_DIR/$RESOURCE_BUNDLE" + fi +else + echo -e "${RED}โŒ Error: Resource bundle not found at $BUNDLE_PATH${NC}" + exit 1 +fi + # Install Blender exporter support files used by `untoldengine export`. echo -e "${CYAN}๐Ÿ“ฆ Installing exporter support files to $SUPPORT_DIR...${NC}" if [ ! -d "$SUPPORT_DIR" ]; then diff --git a/scripts/tests/test_addon_bridge.py b/scripts/tests/test_addon_bridge.py index 03ffa9916..915c36034 100644 --- a/scripts/tests/test_addon_bridge.py +++ b/scripts/tests/test_addon_bridge.py @@ -144,7 +144,7 @@ def export_objects_to_untold_or_pack(export_objects, *, output_path, file_type_n validate=validate, progress_callback=progress_callback, ) return u.write_untold_pack_from_groups( - models, source_asset_name="scene.blend", output_path=output_path, file_type_name=file_type_name, + models, source_asset_path=output_path.parent / "scene.blend", output_path=output_path, file_type_name=file_type_name, compress_geometry=compress_geometry, validate=validate, progress_callback=progress_callback, ) diff --git a/scripts/tests/test_untoldexplorer.py b/scripts/tests/test_untoldexplorer.py index fbf9f333a..cb2d86d93 100644 --- a/scripts/tests/test_untoldexplorer.py +++ b/scripts/tests/test_untoldexplorer.py @@ -2650,7 +2650,7 @@ def node(name: str, root_x: float) -> "u.ExportedNode": with tempfile.TemporaryDirectory() as tmpdir: output = Path(tmpdir) / "Park.untold" result = u.write_untold_pack_from_groups( - groups, source_asset_name="Park.blend", output_path=output, file_type_name="tile", + groups, source_asset_path=output.parent / "Park.blend", output_path=output, file_type_name="tile", compress_geometry=False, validate=False, progress_callback=None, ) self.assertEqual((result["written_model_count"], result["shared_model_count"]), (1, 2)) @@ -2678,7 +2678,7 @@ def test_a_pack_writes_each_model_once_and_places_every_copy(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: output = Path(tmpdir) / "Models" / "Park.untold" result = u.write_untold_pack_from_groups( - groups, source_asset_name="Park.blend", output_path=output, file_type_name="tile", + groups, source_asset_path=output.parent / "Park.blend", output_path=output, file_type_name="tile", compress_geometry=False, validate=False, progress_callback=None, assets_dir=Path(tmpdir) / "Models" / "Park", ) manifest = json.loads(output.with_suffix(".untoldpack").read_text()) @@ -2711,7 +2711,7 @@ def empty_node(name: str) -> "u.ExportedNode": with contextlib.redirect_stdout(io.StringIO()): return u.write_untold_pack_from_groups( {"Floor": [empty_node("Floor")], "Wall": [empty_node("Wall")]}, - source_asset_name="scene.blend", + source_asset_path=self.output_dir / "scene.blend", output_path=self.output_dir / output_name, file_type_name="tile", compress_geometry=False, diff --git a/scripts/untoldexplorer.py b/scripts/untoldexplorer.py index 918ff6609..931a21f52 100644 --- a/scripts/untoldexplorer.py +++ b/scripts/untoldexplorer.py @@ -9192,7 +9192,7 @@ def digest(data: bytes) -> str: def write_untold_pack_from_groups( model_groups: dict[str, list[ExportedNode]], *, - source_asset_name: str, + source_asset_path: Path, output_path: Path, file_type_name: str, compress_geometry: bool, @@ -9311,6 +9311,11 @@ def write_untold_pack_from_groups( if signature is not None: written_by_signature[signature] = model_output_path + # Relative to the manifest's own folder, like each model's path -- resolves the same + # way whether the source sits beside the pack (today's convention: a bare filename, + # no "/" needed) or in a separate Resources/ tree the pack's GameData/ doesn't share + # (then e.g. "../../../Resources/Models/Tower/Tower.blend"). + source_asset_name = relative_asset_uri(source_asset_path, pack_path.parent) write_untoldpack_manifest(pack_path, source_asset_name, manifest_models) # A previous export at this same --output stem may have been a single @@ -9423,7 +9428,7 @@ def export_objects_to_untold_or_pack( result = write_untold_pack_from_groups( model_groups, - source_asset_name=source_asset_path.name, + source_asset_path=source_asset_path, output_path=output_path, file_type_name=file_type_name, compress_geometry=compress_geometry, @@ -9660,7 +9665,7 @@ def main(argv: list[str]) -> int: result = write_untold_pack_from_groups( model_groups, - source_asset_name=input_path.name, + source_asset_path=input_path, output_path=output_path, file_type_name=args.file_type, compress_geometry=args.compress_geometry,