diff --git a/Sources/CShaderTypes/ShaderTypes.h b/Sources/CShaderTypes/ShaderTypes.h index 599d73072..7b138ce6b 100644 --- a/Sources/CShaderTypes/ShaderTypes.h +++ b/Sources/CShaderTypes/ShaderTypes.h @@ -559,6 +559,8 @@ typedef enum{ ssaoPassFrustumIndex, ssaoPassReverseZIndex, ssaoPassProjScaleIndex, + ssaoPassRateMapDataIndex, // rasterization_rate_map_data of the eye's rate map (XR foveation; a never-read placeholder otherwise) + ssaoPassRateMapSizesIndex, // float4: the map's screen size in xy, the depth texture's size in zw; x <= 0 when there is no map }SSAOBufferIndices; typedef enum{ @@ -767,7 +769,9 @@ typedef enum{ hzbCullPassViewportIndex, hzbCullPassMipCountIndex, hzbCullPassReverseZIndex, - hzbCullPassOcclusionBiasIndex + hzbCullPassOcclusionBiasIndex, + hzbCullPassRateMapDataIndex, // rasterization_rate_map_data the pyramid was drawn through (XR foveation; a never-read placeholder otherwise) + hzbCullPassRateMapSizesIndex // float4: the map's screen size in xy, the pyramid's base size in zw; x <= 0 when there is no map }HZBOcclusionCullingBufferIndices; typedef enum{ @@ -800,6 +804,8 @@ typedef enum{ gaussianCullHZBReverseZIndex, gaussianCullHZBOcclusionBiasIndex, gaussianCullHZBValidIndex, + gaussianCullHZBRateMapDataIndex, // rasterization_rate_map_data the pyramid was drawn through (XR foveation; a never-read placeholder otherwise) + gaussianCullHZBRateMapSizesIndex, // float4: the map's screen size in xy, the pyramid's base size in zw; x <= 0 when there is no map }GaussianDepthBufferIndices; typedef enum{ @@ -1034,7 +1040,8 @@ typedef struct{ uint32_t uniformQuotas; // GaussianDebugOptions.disableScreenWeightedQuotas: screenArea = splatCount, the uniform quota rule uint32_t paged; // 0: every record resident; 1: a paged entity (the cull writes the demand table and lists resident ranks only, the fused pass reads the page pool); 2: demand only (a warming tier: write the demand word and return) uint32_t rangeCount; // entries of ranges[] (gaussianChunkCullRangesIndex, GaussianChunkTreeCull); a thread whose chunkIndex falls in none of them skips the per-chunk test. 0 (GaussianDebugOptions.disableTreeSkip, or a file without a tree) skips the check entirely, as before the tree was wired in -}GaussianChunkCullConstants; // 192 bytes (176 + rangeCount, padded to the struct's 16-byte alignment) + simd_float4 hzbRateMapSizes; // the rasterization rate map the HZB was drawn through (XR foveation): its screen size in xy, the pyramid's base size in zw; x <= 0 when there is none and the rate map buffer is a never-read placeholder +}GaussianChunkCullConstants; // 208 bytes (180 + hzbRateMapSizes at 192, padded to the struct's 16-byte alignment) /// One span of GaussianChunkTreeCull.visibleChunkRanges (Swift): chunk indices [firstChunk, /// firstChunk + chunkCount) the tree walk could not rule out this frame. Every chunk still gets @@ -1062,6 +1069,7 @@ typedef enum{ gaussianChunkCullLevelStateIndex = 11, // GaussianChunkLevelState[chunkCount], persistent, written by gaussianComputeChunkQuotas (a never-read stand-in without coarse levels) gaussianChunkCullLevelConstantsIndex = 12, // GaussianChunkLevelConstants (setBytes); hasCoarse = 0 keeps every path as it was gaussianChunkCullRangesIndex = 13, // GaussianChunkCullRange[rangeCount] (setBytes; a never-read stand-in when rangeCount == 0) + gaussianChunkCullRateMapDataIndex = 14, // rasterization_rate_map_data the HZB was drawn through (XR foveation; read only when hzbRateMapSizes.x > 0, a placeholder otherwise) }GaussianChunkCullBufferIndices; typedef enum{ @@ -1299,6 +1307,7 @@ typedef enum{ gaussianChunkPreprocessCoarseTableIndex, // 17: GaussianChunkDecodeConstants[levelCount × chunkCount], the coarse rows, level-major gaussianChunkPreprocessLevelStateIndex, // 18: GaussianChunkLevelState[chunkCount] as gaussianComputeChunkQuotas left it this frame gaussianChunkPreprocessLevelConstantsIndex, // 19: GaussianChunkLevelConstants (setBytes) + gaussianChunkPreprocessRateMapDataIndex, // 20: rasterization_rate_map_data the HZB was drawn through (XR foveation; read only when the cull constants' hzbRateMapSizes.x > 0, a placeholder otherwise) }GaussianChunkPreprocessBufferIndices; typedef enum{ @@ -1354,6 +1363,38 @@ typedef enum{ fxaaPassSplatCoverageTextureIndex = 1 }FXAATextureIndices; +// MARK: - Temporal anti-aliasing (TAAShader.metal) + +/// The resolve's inputs: the eye's unjittered matrices for this frame and the frame its +/// history holds, and the rasterization rate maps of both when the frames were foveated +/// (XR): the history is laid out in its own frame's physical space, so a pixel goes +/// physical → screen → world → previous screen → previous physical before the history is read. +typedef struct { + matrix_float4x4 invViewProjection; // this frame, unjittered, scene root included + matrix_float4x4 prevViewProjection; // the history's frame, unjittered, scene root included + simd_float4 rateMapSizes; // this frame's map: screen size xy, physical size zw; x <= 0 when drawn uniformly + simd_float4 prevRateMapSizes; // the history frame's map, same layout + simd_float2 physicalSize; // the colour, depth and history textures' size in pixels + float historyWeight; // share of the clipped history in the output (0.9) + uint32_t historyValid; // 0 on the first frame, after a resize or a mode change: the output is the current frame + uint32_t reverseZ; + float clipGamma; // the clip box is the 3×3 mean ± clipGamma × standard deviation (1.25) + uint32_t pad0; + uint32_t pad1; +} TAAConstants; // 176 bytes + +typedef enum{ + taaPassConstantsIndex = 0, // TAAConstants + taaPassRateMapDataIndex, // rasterization_rate_map_data of this frame (read only when rateMapSizes.x > 0; a placeholder otherwise) + taaPassPrevRateMapDataIndex // rasterization_rate_map_data of the history's frame (read only when prevRateMapSizes.x > 0) +}TAABufferIndices; + +typedef enum{ + taaPassColorTextureIndex = 0, // this frame's look output + taaPassHistoryTextureIndex, // the previous resolve of this eye + taaPassDepthTextureIndex // this frame's opaque depth +}TAATextureIndices; + typedef enum{ smaaPassTexelSizeIndex, smaaPassEdgeThresholdIndex, diff --git a/Sources/UntoldEngine/Renderer/Pipelines/RenderPipeLines.swift b/Sources/UntoldEngine/Renderer/Pipelines/RenderPipeLines.swift index 7fc744d59..2a7b78b46 100644 --- a/Sources/UntoldEngine/Renderer/Pipelines/RenderPipeLines.swift +++ b/Sources/UntoldEngine/Renderer/Pipelines/RenderPipeLines.swift @@ -963,6 +963,20 @@ public func InitFXAAPipeline() -> RenderPipeline? { ) } +/// The temporal resolve: the look output and the eye's history into the anti-aliasing +/// texture and the eye's next history, two colour targets of the look format. +public func InitTAAPipeline() -> RenderPipeline? { + CreatePipeline( + vertexShader: "vertexTAAShader", + fragmentShader: "fragmentTAAShader", + vertexDescriptor: createPostProcessVertexDescriptor(), + colorFormats: [renderInfo.colorPipeline.working.lookOutput, renderInfo.colorPipeline.working.lookOutput], + depthFormat: .invalid, + depthEnabled: false, + name: "TAA Pipeline" + ) +} + public func InitFXAAEdgeDebugPipeline() -> RenderPipeline? { CreatePipeline( vertexShader: "vertexFXAAShader", @@ -1176,6 +1190,7 @@ public func DefaultPipeLines() -> [(RenderPipelineType, RenderPipelineInitBlock) (.look, InitLookPipeline), (.fxaa, InitFXAAPipeline), (.fxaaEdgeDebug, InitFXAAEdgeDebugPipeline), + (.taa, InitTAAPipeline), (.smaaEdges, InitSMAAEdgesPipeline), (.smaaBlendWeights, InitSMAABlendWeightsPipeline), (.smaaNeighborhood, InitSMAANeighborhoodPipeline), diff --git a/Sources/UntoldEngine/Renderer/Pipelines/RenderPipelineType.swift b/Sources/UntoldEngine/Renderer/Pipelines/RenderPipelineType.swift index 0d935727f..809870e47 100644 --- a/Sources/UntoldEngine/Renderer/Pipelines/RenderPipelineType.swift +++ b/Sources/UntoldEngine/Renderer/Pipelines/RenderPipelineType.swift @@ -57,6 +57,7 @@ public extension RenderPipelineType { static let outputTransform: RenderPipelineType = "outputTransform" static let fxaa: RenderPipelineType = "fxaa" static let fxaaEdgeDebug: RenderPipelineType = "fxaaEdgeDebug" + static let taa: RenderPipelineType = "taa" static let smaaEdges: RenderPipelineType = "smaaEdges" static let smaaBlendWeights: RenderPipelineType = "smaaBlendWeights" static let smaaNeighborhood: RenderPipelineType = "smaaNeighborhood" diff --git a/Sources/UntoldEngine/Renderer/RenderExtensionModelSurface.swift b/Sources/UntoldEngine/Renderer/RenderExtensionModelSurface.swift index 83f2deaba..e80f6d21e 100644 --- a/Sources/UntoldEngine/Renderer/RenderExtensionModelSurface.swift +++ b/Sources/UntoldEngine/Renderer/RenderExtensionModelSurface.swift @@ -258,6 +258,7 @@ public extension RenderPassContext { renderEncoder.label = label renderEncoder.pushDebugGroup(label) + applyXRFoveationViewport(renderEncoder) renderEncoder.setRenderPipelineState(pipelineState) renderEncoder.setDepthStencilState(pipeline.depthState) diff --git a/Sources/UntoldEngine/Renderer/RenderInitializer.swift b/Sources/UntoldEngine/Renderer/RenderInitializer.swift index b7e38243d..e831ee3e0 100644 --- a/Sources/UntoldEngine/Renderer/RenderInitializer.swift +++ b/Sources/UntoldEngine/Renderer/RenderInitializer.swift @@ -118,11 +118,11 @@ func requestedOpaqueSampleCount() -> Int { return 1 #else switch antiAliasingMode { - case .msaa: + case .msaa, .msaaTaa: let preferredSampleCount = 4 guard let device = renderInfo.device else { return 1 } return device.supportsTextureSampleCount(preferredSampleCount) ? preferredSampleCount : 1 - case .none, .fxaa, .smaa: + case .none, .fxaa, .smaa, .taa: return 1 } #endif diff --git a/Sources/UntoldEngine/Renderer/RenderPasses+MuscleDebug.swift b/Sources/UntoldEngine/Renderer/RenderPasses+MuscleDebug.swift index 3cdeef819..544c16366 100644 --- a/Sources/UntoldEngine/Renderer/RenderPasses+MuscleDebug.swift +++ b/Sources/UntoldEngine/Renderer/RenderPasses+MuscleDebug.swift @@ -155,6 +155,7 @@ public extension RenderPasses { } renderEncoder.label = "Muscle Debug Pass" renderEncoder.pushDebugGroup("Muscle Debug Pass") + applyXRFoveationViewport(renderEncoder) renderEncoder.setRenderPipelineState(pipelineState) // The cages live under the skin, so never depth-test them. In visionOS // mixed immersion the compositor drops pixels that wrote no depth, so diff --git a/Sources/UntoldEngine/Renderer/RenderPasses.swift b/Sources/UntoldEngine/Renderer/RenderPasses.swift index a6810cc66..c12b69184 100644 --- a/Sources/UntoldEngine/Renderer/RenderPasses.swift +++ b/Sources/UntoldEngine/Renderer/RenderPasses.swift @@ -1076,6 +1076,7 @@ public enum RenderPasses { } renderEncoder.label = "Grid Pass" + applyXRFoveationViewport(renderEncoder) renderEncoder.pushDebugGroup("Grid Pass") @@ -1167,6 +1168,7 @@ public enum RenderPasses { } renderEncoder.label = "Sky Pass" + applyXRFoveationViewport(renderEncoder) renderEncoder.pushDebugGroup("Sky Pass") @@ -1237,6 +1239,7 @@ public enum RenderPasses { } renderEncoder.label = "Environment Pass" + applyXRFoveationViewport(renderEncoder) renderEncoder.pushDebugGroup("Environment Pass") @@ -1835,6 +1838,7 @@ public enum RenderPasses { } renderEncoder.label = "Model Pass" + applyXRFoveationViewport(renderEncoder) renderEncoder.pushDebugGroup("Model Pass") @@ -2102,6 +2106,7 @@ public enum RenderPasses { } renderEncoder.label = "Batched Model Pass" + applyXRFoveationViewport(renderEncoder) renderEncoder.pushDebugGroup("Batched Model Pass") renderEncoder.setRenderPipelineState(modelPipeline.pipelineState!) @@ -2342,6 +2347,10 @@ public enum RenderPasses { renderEncoder.label = "G-Buffer + Light Pass (TBDR)" renderEncoder.pushDebugGroup("G-Buffer + Light Pass (TBDR)") + // Through the eye's rasterization rate map when there is one (XR foveation): the + // geometry in the map's screen space, the light quad over the whole of it, the + // G-buffer read back from the tile under each fragment either way. + applyXRFoveationViewport(renderEncoder) // What every draw of the pass shares, read once. let pass = OpaquePassState(cameraComponent: cameraComponent) @@ -2800,6 +2809,14 @@ public enum RenderPasses { var projScale = simd_float2(renderInfo.perspectiveSpace.columns.0.x, renderInfo.perspectiveSpace.columns.1.y) renderEncoder.setFragmentBytes(&projScale, length: MemoryLayout.stride, index: Int(ssaoPassProjScaleIndex.rawValue)) + // The eye's rasterization rate map (XR foveation): the depth was drawn through it, so + // the view positions are rebuilt from where its texels sit on screen, not in the texture. + bindRateMap( + renderInfo.xrFoveation?.rateMapData, + toFragmentsOf: renderEncoder, + dataIndex: Int(ssaoPassRateMapDataIndex.rawValue), + sizesIndex: Int(ssaoPassRateMapSizesIndex.rawValue) + ) // set the draw command renderEncoder.drawIndexedPrimitivesTracked( @@ -3000,6 +3017,14 @@ public enum RenderPasses { var projScale = simd_float2(renderInfo.perspectiveSpace.columns.0.x, renderInfo.perspectiveSpace.columns.1.y) renderEncoder.setFragmentBytes(&projScale, length: MemoryLayout.stride, index: Int(ssaoPassProjScaleIndex.rawValue)) + // The eye's rasterization rate map (XR foveation): the depth was drawn through it, so + // the view positions are rebuilt from where its texels sit on screen, not in the texture. + bindRateMap( + renderInfo.xrFoveation?.rateMapData, + toFragmentsOf: renderEncoder, + dataIndex: Int(ssaoPassRateMapDataIndex.rawValue), + sizesIndex: Int(ssaoPassRateMapSizesIndex.rawValue) + ) // SSAO properties renderEncoder.setFragmentBytes(&SSAOParams.shared.radius, length: MemoryLayout.stride, index: Int(ssaoPassRadiusIndex.rawValue)) @@ -3619,6 +3644,7 @@ public enum RenderPasses { } renderEncoder.label = "Transparency Pass" + applyXRFoveationViewport(renderEncoder) renderEncoder.pushDebugGroup("Transparency Pass") renderEncoder.setRenderPipelineState(transparencyPipeline.pipelineState!) @@ -4034,6 +4060,9 @@ public enum RenderPasses { descriptor.depthAttachment.texture = depthTexture descriptor.depthAttachment.loadAction = .load descriptor.depthAttachment.storeAction = .store + // The depth was drawn through the eye's rasterization rate map when there is one + // (XR foveation); so is what this pass adds to it. + attachXRFoveation(renderInfo.xrFoveation?.rateMap, to: descriptor) guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: descriptor) else { handleError(.renderPassCreationFailed, "Mesh Occluder Shell Pass") @@ -4047,6 +4076,7 @@ public enum RenderPasses { } renderEncoder.label = "Mesh Occluder Shell Pass" + applyXRFoveationViewport(renderEncoder) renderEncoder.pushDebugGroup("Mesh Occluder Shell Pass") renderEncoder.setRenderPipelineState(pipelineState) renderEncoder.setDepthStencilState(pipeline.depthState) @@ -4172,6 +4202,9 @@ public enum RenderPasses { descriptor.depthAttachment.texture = depthTexture descriptor.depthAttachment.loadAction = .load descriptor.depthAttachment.storeAction = .store + // The depth was drawn through the eye's rasterization rate map when there is one + // (XR foveation); so is what this pass adds to it. + attachXRFoveation(renderInfo.xrFoveation?.rateMap, to: descriptor) guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: descriptor) else { handleError(.renderPassCreationFailed, "Wireframe Occlusion Depth Pass") @@ -4185,6 +4218,7 @@ public enum RenderPasses { } renderEncoder.label = "Wireframe Occlusion Depth Pass" + applyXRFoveationViewport(renderEncoder) renderEncoder.pushDebugGroup("Wireframe Occlusion Depth Pass") renderEncoder.setRenderPipelineState(pipelineState) renderEncoder.setDepthStencilState(pipeline.depthState) @@ -4369,6 +4403,7 @@ public enum RenderPasses { } renderEncoder.label = "Wireframe Pass" + applyXRFoveationViewport(renderEncoder) renderEncoder.pushDebugGroup("Wireframe Pass") renderEncoder.setRenderPipelineState(wireframePipelineState) @@ -4749,6 +4784,7 @@ public enum RenderPasses { } renderEncoder.label = "Spatial Debug Bounds Pass" + applyXRFoveationViewport(renderEncoder) renderEncoder.pushDebugGroup("Spatial Debug Bounds Pass") renderEncoder.setRenderPipelineState(spatialDebugPipeline.pipelineState!) @@ -4897,6 +4933,7 @@ public enum RenderPasses { } renderEncoder.label = "Gaussian Pass" + applyXRFoveationViewport(renderEncoder) renderInfo.gaussianCoverageWritten = true renderEncoder.pushDebugGroup("Gaussian Pass") diff --git a/Sources/UntoldEngine/Renderer/RenderResources.swift b/Sources/UntoldEngine/Renderer/RenderResources.swift index d30c80743..b75ead173 100644 --- a/Sources/UntoldEngine/Renderer/RenderResources.swift +++ b/Sources/UntoldEngine/Renderer/RenderResources.swift @@ -87,6 +87,10 @@ public struct RenderInfo { /// shadingCameraPosition so each eye gets its own view vector (specular highlights, /// Fresnel), while streaming, LOD and culling keep the head-centre camera position. public var xrEyeCameraPosition: simd_float3? + /// The rasterization rate map of the eye being drawn (visionOS foveation), with the + /// viewport and the decoder data that go with it; nil when the frame is drawn uniformly. + /// `viewPort` stays the size of the textures, the map's physical size. See XRFoveation.swift. + public var xrFoveation: XRFoveationFrame? } /// Where an eye is, from its view matrix (the inverse of the eye's camera transform). @@ -176,6 +180,9 @@ public struct BufferResources { var reduceScanCapacity: Int = 0 /// Whether a frame with more bounding boxes than the scan takes was reported. var reportedReduceScanOverflow: Bool = false + + /// Bound where a shader declares a rasterization rate map it will not read (XRFoveation.swift). + var rateMapPlaceholder: MTLBuffer? } public struct TripleBufferResources { @@ -328,9 +335,14 @@ public struct HZBPyramidFrame: Sendable { public var viewProjection: simd_float4x4 /// Where that frame's camera stood, in world space. public var cameraPosition: simd_float3 + /// The rasterization rate map that frame was drawn through (XR foveation), nil when it was + /// drawn uniformly: the pyramid is laid out in the map's physical space, so a screen rect or + /// point the test projects must go through the map before the pyramid is sampled there. + public var rateMapData: XRRasterizationRateMapData? - public init(viewProjection: simd_float4x4, cameraPosition: simd_float3) { + public init(viewProjection: simd_float4x4, cameraPosition: simd_float3, rateMapData: XRRasterizationRateMapData? = nil) { self.viewProjection = viewProjection self.cameraPosition = cameraPosition + self.rateMapData = rateMapData } } diff --git a/Sources/UntoldEngine/Renderer/TemporalAntiAliasing.swift b/Sources/UntoldEngine/Renderer/TemporalAntiAliasing.swift new file mode 100644 index 000000000..67c73969b --- /dev/null +++ b/Sources/UntoldEngine/Renderer/TemporalAntiAliasing.swift @@ -0,0 +1,291 @@ +// +// TemporalAntiAliasing.swift +// UntoldEngine +// +// Temporal anti-aliasing: the projection is jittered by a sub-pixel amount each frame, and +// the resolve pass after the look pass blends the frame with the eye's reprojected, clipped +// history (TAAShader.metal). Sub-pixel detail a single sample flickers on — thin wires, fine +// normal maps, the periphery of a foveated frame where a pixel spans several screen pixels — +// averages out over frames. The reprojection follows the camera through the depth buffer; +// what moves on its own is covered by the neighbourhood clip, not by motion vectors. +// +// State is per eye (XR renders the two eyes through the same passes): a history texture +// ping-pong, the unjittered view-projection of the frame the history holds, and that frame's +// rasterization rate map when it was foveated — the history is laid out in that map's +// physical space, so the resolve maps through both frames' maps. +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import CShaderTypes +import Metal +import simd + +/// Tunables of the temporal resolve. +public final class TAAParams: @unchecked Sendable { + public static let shared = TAAParams() + private let lock = NSLock() + private var historyWeightValue: Float = 0.9 + private var clipGammaValue: Float = 1.25 + private var jitterScaleValue: Float = 1 + + /// Scales the sub-pixel jitter: 1 is the full half-pixel sequence, 0 draws every frame + /// from the pixel centre (the resolve still runs, as a diagnostic of what the jitter + /// itself changes). Clamped to [0, 1]. + public var jitterScale: Float { + get { lock.withLock { jitterScaleValue } } + set { lock.withLock { jitterScaleValue = min(max(newValue, 0), 1) } } + } + + /// The share of the (clipped) history in each resolved pixel: higher converges further + /// and reacts slower. Clamped to [0, 0.98]; the default is 0.9. + public var historyWeight: Float { + get { lock.withLock { historyWeightValue } } + set { lock.withLock { historyWeightValue = min(max(newValue, 0), 0.98) } } + } + + /// How far the history may stray from a pixel's 3×3 neighbourhood: the clip box is the + /// neighbourhood's mean ± this many standard deviations. Lower rejects ghosting sooner, + /// higher keeps pixel-thin features (wires, text) through the frames that miss them. + /// Clamped to [0.5, 3]; the default is 1.25. + public var clipGamma: Float { + get { lock.withLock { clipGammaValue } } + set { lock.withLock { clipGammaValue = min(max(newValue, 0.5), 3) } } + } +} + +/// Sub-pixel offsets, in pixels, of the Halton (2, 3) sequence centred on the pixel: eight +/// of them, so the average over a short window sits at the pixel centre. +let taaJitterSequence: [simd_float2] = (1 ... 8).map { index in + func halton(_ index: Int, _ base: Int) -> Float { + var result: Float = 0 + var fraction: Float = 1 + var i = index + while i > 0 { + fraction /= Float(base) + result += fraction * Float(i % base) + i /= base + } + return result + } + return simd_float2(halton(index, 2) - 0.5, halton(index, 3) - 0.5) +} + +/// Whether a mode runs the temporal resolve. +extension AntiAliasingMode { + var isTemporal: Bool { + switch self { + case .taa, .msaaTaa: + return true + case .none, .fxaa, .smaa, .msaa: + return false + } + } +} + +public final class TemporalAntiAliasing: @unchecked Sendable { + public static let shared = TemporalAntiAliasing() + + /// What one eye carries from frame to frame. + struct EyeHistory { + var textures: [MTLTexture] = [] + var readIndex = 0 + var viewProjection = matrix_identity_float4x4 + var rateMapData: XRRasterizationRateMapData? + var valid = false + } + + private(set) var eyes: [EyeHistory] = [EyeHistory(), EyeHistory()] + private var jitterIndex = 0 + /// The jitter currently added to `renderInfo.perspectiveSpace`, in NDC, so it can be + /// taken out before the next one goes in (the macOS projection persists across frames). + private(set) var appliedJitter = simd_float2.zero + /// The jitter of this frame in pixels of the screen, for diagnostics and tests. + private(set) var currentJitterPixels = simd_float2.zero + private var loggedResolve = false + /// False while a secondary target is drawn (XR: a capture drawable of a recording or a + /// stream, seen from its own pose): the resolve outputs the frame as is and leaves the + /// eye's history, matrices and map untouched for the built-in display's next frame. + public var historyUpdatesEnabled = true + + private var eyeIndex: Int { + renderInfo.isXRStereoMode ? min(max(renderInfo.currentEye, 0), 1) : 0 + } + + /// The projection was set afresh (a resize, a new eye): nothing of the old jitter is in it. + func projectionDidChange() { + appliedJitter = .zero + } + + /// Before a frame's graph is built: takes last frame's jitter out of the projection and + /// puts this frame's in — or just takes it out when the mode is not temporal. The offset + /// lives in the projection's third column: for a perspective projection, whose clip w is + /// the view-space depth, it shifts every point by a constant amount in NDC. + func applyJitter() { + var projection = renderInfo.perspectiveSpace + projection.columns.2.x -= appliedJitter.x + projection.columns.2.y -= appliedJitter.y + appliedJitter = .zero + currentJitterPixels = .zero + + if antiAliasingMode.isTemporal, let screen = screenSize(), screen.x > 0, screen.y > 0 { + let pixels = taaJitterSequence[jitterIndex % taaJitterSequence.count] * TAAParams.shared.jitterScale + jitterIndex += 1 + let ndc = simd_float2(2 * pixels.x / screen.x, 2 * pixels.y / screen.y) + projection.columns.2.x += ndc.x + projection.columns.2.y += ndc.y + appliedJitter = ndc + currentJitterPixels = pixels + } + renderInfo.perspectiveSpace = projection + } + + /// The size of the screen the projection maps to: the rate map's under foveation, the + /// textures' otherwise. + private func screenSize() -> simd_float2? { + if let foveation = renderInfo.xrFoveation { + return foveation.screenSize + } + return renderInfo.viewPort + } + + /// This frame's projection without its jitter. + var unjitteredProjection: simd_float4x4 { + var projection = renderInfo.perspectiveSpace + projection.columns.2.x -= appliedJitter.x + projection.columns.2.y -= appliedJitter.y + return projection + } + + /// Forgets every eye's history: the next resolve of each eye outputs its frame as is. + func invalidateHistory() { + for index in eyes.indices { + eyes[index].valid = false + } + } + + /// The eye's history textures at the current viewport, made or remade when needed (a + /// remake invalidates the history). + private func historyTextures(for eye: Int) -> [MTLTexture]? { + let width = max(1, Int(renderInfo.viewPort.x)) + let height = max(1, Int(renderInfo.viewPort.y)) + if eyes[eye].textures.count == 2, eyes[eye].textures[0].width == width, eyes[eye].textures[0].height == height { + return eyes[eye].textures + } + guard let device = renderInfo.device else { return nil } + var textures: [MTLTexture] = [] + for slot in 0 ..< 2 { + guard let texture = createTexture( + device: device, + label: "TAA History Eye \(eye) \(slot)", + pixelFormat: renderInfo.colorPipeline.working.lookOutput, + width: width, + height: height, + usage: [.shaderRead, .renderTarget], + storageMode: .private + ) else { return nil } + textures.append(texture) + } + eyes[eye].textures = textures + eyes[eye].readIndex = 0 + eyes[eye].valid = false + return textures + } + + /// The resolve: the look output and the eye's history into the anti-aliasing texture and + /// the eye's other history texture, then the frame's matrices and map become the history's. + func encodeResolve(_ commandBuffer: MTLCommandBuffer) { + guard let source = textureResources.lookTexture, + let destination = textureResources.antiAliasingTexture, + let depth = textureResources.depthMap, + let pipeline = PipelineManager.shared.renderPipelinesByType[.taa], + pipeline.success, + let pipelineState = pipeline.pipelineState, + let camera = CameraSystem.shared.activeCamera, + let cameraComponent = scene.get(component: CameraComponent.self, for: camera) + else { + handleError(.renderPassCreationFailed, "TAA Pass: missing texture, pipeline or camera") + return + } + let eye = eyeIndex + guard let textures = historyTextures(for: eye) else { + handleError(.renderPassCreationFailed, "TAA Pass: history textures") + return + } + if !loggedResolve { + loggedResolve = true + let foveated = renderInfo.xrFoveation != nil ? "rate-map reprojection, rate-weighted history" : "uniform reprojection" + print("✓ TAA resolve: \(source.width)×\(source.height), history weight \(TAAParams.shared.historyWeight), clip gamma \(TAAParams.shared.clipGamma) (variance clip), \(foveated)") + } + let readTexture = textures[eyes[eye].readIndex] + let writeTexture = textures[1 - eyes[eye].readIndex] + + let view = SceneRootTransform.shared.effectiveViewMatrix(cameraComponent.viewSpace) + let viewProjection = simd_mul(unjitteredProjection, view) + let currentRateMap = renderInfo.xrFoveation?.rateMapData + var constants = TAAConstants() + constants.invViewProjection = simd_inverse(viewProjection) + constants.prevViewProjection = eyes[eye].viewProjection + constants.rateMapSizes = rateMapSizes(currentRateMap) + constants.prevRateMapSizes = rateMapSizes(eyes[eye].rateMapData) + constants.physicalSize = simd_float2(Float(source.width), Float(source.height)) + constants.historyWeight = TAAParams.shared.historyWeight + constants.clipGamma = TAAParams.shared.clipGamma + constants.historyValid = eyes[eye].valid && historyUpdatesEnabled ? 1 : 0 + constants.reverseZ = renderInfo.reverseZEnabled ? 1 : 0 + + let descriptor = MTLRenderPassDescriptor() + descriptor.colorAttachments[0].texture = destination + descriptor.colorAttachments[0].loadAction = .dontCare + descriptor.colorAttachments[0].storeAction = .store + descriptor.colorAttachments[1].texture = writeTexture + descriptor.colorAttachments[1].loadAction = .dontCare + descriptor.colorAttachments[1].storeAction = .store + + guard let encoder = commandBuffer.makeRenderCommandEncoder(descriptor: descriptor) else { + handleError(.renderPassCreationFailed, "TAA Pass") + return + } + encoder.label = "TAA Pass" + encoder.pushDebugGroup("TAA Pass") + encoder.setRenderPipelineState(pipelineState) + encoder.waitForFence(renderInfo.fence, before: .fragment) + encoder.setVertexBuffer(bufferResources.quadVerticesBuffer, offset: 0, index: 0) + encoder.setVertexBuffer(bufferResources.quadTexCoordsBuffer, offset: 0, index: 1) + encoder.setFragmentTexture(source, index: Int(taaPassColorTextureIndex.rawValue)) + encoder.setFragmentTexture(readTexture, index: Int(taaPassHistoryTextureIndex.rawValue)) + encoder.setFragmentTexture(depth, index: Int(taaPassDepthTextureIndex.rawValue)) + encoder.setFragmentBytes(&constants, length: MemoryLayout.stride, index: Int(taaPassConstantsIndex.rawValue)) + // The maps' sizes are in the constants; the shader reads a map's parameters only when + // its sizes say there is one, so a placeholder stands in for a uniform frame. + let placeholder = rateMapPlaceholderBuffer() + encoder.setFragmentBuffer(currentRateMap?.buffer ?? placeholder, offset: 0, index: Int(taaPassRateMapDataIndex.rawValue)) + encoder.setFragmentBuffer(eyes[eye].rateMapData?.buffer ?? placeholder, offset: 0, index: Int(taaPassPrevRateMapDataIndex.rawValue)) + encoder.drawIndexedPrimitivesTracked( + type: .triangle, + indexCount: 6, + indexType: .uint16, + indexBuffer: bufferResources.quadIndexBuffer!, + indexBufferOffset: 0 + ) + encoder.updateFence(renderInfo.fence, after: .fragment) + encoder.popDebugGroup() + encoder.endEncoding() + + // A secondary target wrote the spare history texture, which the next primary resolve + // overwrites anyway; the one it reads, and the frame's matrices and map, are untouched. + guard historyUpdatesEnabled else { return } + eyes[eye].readIndex = 1 - eyes[eye].readIndex + eyes[eye].viewProjection = viewProjection + eyes[eye].rateMapData = currentRateMap + eyes[eye].valid = true + } +} + +/// The graph's temporal resolve pass, in the slot FXAA or SMAA otherwise take. +let taaRenderPass: RenderPasses.RenderPassExecution = { commandBuffer in + TemporalAntiAliasing.shared.encodeResolve(commandBuffer) +} diff --git a/Sources/UntoldEngine/Renderer/UntoldEngine.swift b/Sources/UntoldEngine/Renderer/UntoldEngine.swift index df278b6fa..6028b5519 100644 --- a/Sources/UntoldEngine/Renderer/UntoldEngine.swift +++ b/Sources/UntoldEngine/Renderer/UntoldEngine.swift @@ -727,6 +727,7 @@ public class UntoldRenderer: NSObject, MTKViewDelegate { : matrixPerspectiveRightHand(fovyRadians: degreesToRadians(degrees: fov), aspectRatio: aspect, nearZ: near, farZ: far) renderInfo.perspectiveSpace = projectionMatrix + TemporalAntiAliasing.shared.projectionDidChange() let viewPortSize: simd_float2 = simd_make_float2(Float(size.width), Float(size.height)) renderInfo.viewPort = viewPortSize @@ -830,6 +831,7 @@ public class UntoldRenderer: NSObject, MTKViewDelegate { eyeIndex: Int ) { renderInfo.perspectiveSpace = projectionMatrix + TemporalAntiAliasing.shared.projectionDidChange() guard let camera = CameraSystem.shared.activeCamera, let cameraComponent = scene.get(component: CameraComponent.self, for: camera) else { handleError(.noActiveCamera) diff --git a/Sources/UntoldEngine/Renderer/XRFoveation.swift b/Sources/UntoldEngine/Renderer/XRFoveation.swift new file mode 100644 index 000000000..1ce4ebde4 --- /dev/null +++ b/Sources/UntoldEngine/Renderer/XRFoveation.swift @@ -0,0 +1,182 @@ +// +// XRFoveation.swift +// UntoldEngine +// +// Variable-rate rasterization in XR. With foveation enabled in its layer configuration, the +// visionOS compositor hands the engine, per eye and per frame, a rasterization rate map and +// colour and depth textures of the map's physical size — smaller than the screen the eye +// sees. The GPU draws through the map, dense where the eyes look and coarse in the periphery, +// and the compositor unwarps the texture on the way to the display; a layer configured this +// way also accepts a render quality above the platform default, which a uniform layer does not. +// +// What this asks of the renderer: every pass that rasterizes scene geometry at drawable size +// attaches the eye's map to its descriptor and sets its viewport in the map's screen space +// (`applyXRFoveationToSceneRenderPassDescriptors`, `applyXRFoveationViewport`). The full-screen +// passes that follow — SSAO, post-processing, anti-aliasing, the output transform into the +// compositor's texture — read and write the physical textures one pixel to one and never see +// the map; it is already baked into the layout of what they copy. Two kinds of shader do have +// to know: one that turns a texel's position back into a direction in space (SSAO rebuilds view +// positions from depth) and the occlusion culls, which project boxes and splats to the screen +// and sample a depth pyramid that is laid out in the map's physical space. Both decode the +// map's parameters (`rasterization_rate_map_decoder`, bound by `bindRateMap`), and the pyramid +// carries the map it was drawn through into the next frame's culls (`HZBPyramidFrame`). +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Metal +import simd + +/// A rasterization rate map's parameter data in a buffer the shaders decode, with the two sizes +/// a decode needs. Immutable once made, so it can travel with the depth pyramid built through +/// it (`HZBPyramidFrame.rateMapData`) into the culls of the frame after. +public final class XRRasterizationRateMapData: @unchecked Sendable { + /// The map's parameters, as `copyParameterData(buffer:offset:)` wrote them at offset 0. + public let buffer: MTLBuffer + /// The map's screen size in pixels: the space viewports and clip-space positions live in. + public let screenSize: simd_float2 + /// The size in pixels of the textures drawn through the map (layer 0). + public let physicalSize: simd_float2 + + public init(buffer: MTLBuffer, screenSize: simd_float2, physicalSize: simd_float2) { + self.buffer = buffer + self.screenSize = screenSize + self.physicalSize = physicalSize + } + + /// Copies `rateMap`'s parameters into `buffer`, which must be shared and at least + /// `rateMap.parameterDataSizeAndAlign.size` bytes long; nil when it is shorter. + public convenience init?(rateMap: MTLRasterizationRateMap, buffer: MTLBuffer) { + let size = rateMap.parameterDataSizeAndAlign.size + guard size > 0, buffer.length >= size else { return nil } + rateMap.copyParameterData(buffer: buffer, offset: 0) + let screen = rateMap.screenSize + let physical = rateMap.physicalSize(layer: 0) + self.init( + buffer: buffer, + screenSize: simd_float2(Float(screen.width), Float(screen.height)), + physicalSize: simd_float2(Float(physical.width), Float(physical.height)) + ) + } + + /// What the shaders bind next to the buffer: the screen size in xy, the physical size in + /// zw. A zero x is the "no map" word (`rateMapSizes(nil)`). + public var sizes: simd_float4 { + simd_float4(screenSize.x, screenSize.y, physicalSize.x, physicalSize.y) + } +} + +/// The rate map of the eye being drawn, as `UntoldEngineXR` hands it to the renderer for each +/// eye of each frame (`renderInfo.xrFoveation`); nil when the frame is drawn uniformly — the +/// layer configured without foveation, the simulator, or `.foveatedRendering(.disabled)`. +public struct XRFoveationFrame { + public let rateMap: MTLRasterizationRateMap + /// The eye's viewport in the map's screen space (`view.textureMap.viewport`), which every + /// encoder drawing through the map sets explicitly: the default is the physical texture's. + public let viewport: MTLViewport + public let rateMapData: XRRasterizationRateMapData + + public init(rateMap: MTLRasterizationRateMap, viewport: MTLViewport, rateMapData: XRRasterizationRateMapData) { + self.rateMap = rateMap + self.viewport = viewport + self.rateMapData = rateMapData + } + + public var screenSize: simd_float2 { + rateMapData.screenSize + } + + public var physicalSize: simd_float2 { + rateMapData.physicalSize + } +} + +/// Attaches the current eye's rate map to the descriptors of the passes that rasterize the scene +/// at drawable size — the G-buffer + light pass, the deferred colour the transparency, wireframe +/// and debug passes draw into, the environment, sky or grid background, the splats and the +/// gizmos — or detaches it when the frame has none. The full-screen passes keep their +/// descriptors untouched: they copy physical pixels one to one. Called for every eye once its +/// graph is built, since a rebuild may have recreated the descriptors. +func applyXRFoveationToSceneRenderPassDescriptors() { + let rateMap = renderInfo.xrFoveation?.rateMap + for descriptor in [ + renderInfo.offscreenRenderPassDescriptor, + renderInfo.deferredRenderPassDescriptor, + renderInfo.environmentRenderPassDescriptor, + renderInfo.gaussianRenderPassDescriptor, + renderInfo.gizmoRenderPassDescriptor, + ] { + guard let descriptor else { continue } + attachXRFoveation(rateMap, to: descriptor) + } +} + +/// Attaches a rate map to a descriptor whose attachments are physical-size textures, or +/// detaches it. A descriptor's render target size is read in *screen* pixels while a map is +/// attached (measured: a target size equal to the physical size clips the scene to the +/// top-left physical-size corner of the screen), so the size is left at 0 — Metal then takes +/// the attachments' own size — and put back to the texture size without a map, which is what +/// `createRenderPassDescriptor` sets and the memoryless attachments rely on. +func attachXRFoveation(_ rateMap: MTLRasterizationRateMap?, to descriptor: MTLRenderPassDescriptor) { + descriptor.rasterizationRateMap = rateMap + if rateMap != nil { + descriptor.renderTargetWidth = 0 + descriptor.renderTargetHeight = 0 + } else if let viewPort = renderInfo.viewPort, descriptor.renderTargetWidth == 0 { + descriptor.renderTargetWidth = Int(viewPort.x) + descriptor.renderTargetHeight = Int(viewPort.y) + } +} + +/// Sets the eye's screen-space viewport on an encoder that draws through the rate map; nothing +/// outside foveated XR, where the default viewport, the texture's own size, is the right one. +/// (Measured: a viewport of the physical size under a map clips the scene the same way a +/// physical-size render target does; the map's screen size covers the whole texture.) +@inline(__always) +func applyXRFoveationViewport(_ encoder: MTLRenderCommandEncoder) { + guard let foveation = renderInfo.xrFoveation else { return } + encoder.setViewport(foveation.viewport) +} + +/// The sizes word of a map, or the "no map" word the shaders test for. +@inline(__always) +func rateMapSizes(_ data: XRRasterizationRateMapData?) -> simd_float4 { + data?.sizes ?? .zero +} + +/// The rate map the current depth pyramid was drawn through, for the culls that sample it. +@inline(__always) +func hzbPyramidRateMapData() -> XRRasterizationRateMapData? { + renderInfo.hzbFrame?.rateMapData +} + +/// Binds a map's parameters and sizes to a kernel, or a placeholder and the "no map" word when +/// there is none: Metal wants every declared buffer bound, and the kernels read the parameters +/// only when `sizes.x > 0`. +func bindRateMap(_ data: XRRasterizationRateMapData?, to encoder: MTLComputeCommandEncoder, dataIndex: Int, sizesIndex: Int) { + var sizes = rateMapSizes(data) + encoder.setBuffer(data?.buffer ?? rateMapPlaceholderBuffer(), offset: 0, index: dataIndex) + encoder.setBytes(&sizes, length: MemoryLayout.stride, index: sizesIndex) +} + +/// The fragment-stage form of `bindRateMap(_:to:dataIndex:sizesIndex:)`. +func bindRateMap(_ data: XRRasterizationRateMapData?, toFragmentsOf encoder: MTLRenderCommandEncoder, dataIndex: Int, sizesIndex: Int) { + var sizes = rateMapSizes(data) + encoder.setFragmentBuffer(data?.buffer ?? rateMapPlaceholderBuffer(), offset: 0, index: dataIndex) + encoder.setFragmentBytes(&sizes, length: MemoryLayout.stride, index: sizesIndex) +} + +/// A small shared buffer bound wherever a shader declares a rate map it will not read. +func rateMapPlaceholderBuffer() -> MTLBuffer? { + if let buffer = bufferResources.rateMapPlaceholder { + return buffer + } + guard let device = renderInfo.device else { return nil } + let buffer = device.makeBuffer(length: 256, options: .storageModeShared) + buffer?.label = "Rate Map Placeholder" + bufferResources.rateMapPlaceholder = buffer + return buffer +} diff --git a/Sources/UntoldEngine/Scenes/SceneSerializer.swift b/Sources/UntoldEngine/Scenes/SceneSerializer.swift index 53e026d6a..a64765261 100644 --- a/Sources/UntoldEngine/Scenes/SceneSerializer.swift +++ b/Sources/UntoldEngine/Scenes/SceneSerializer.swift @@ -117,6 +117,8 @@ enum AntiAliasingModeData: String, Codable { case fxaa case smaa case msaa + case taa + case msaaTaa } struct FXAAData: Codable { @@ -1384,6 +1386,10 @@ public func serializeScene() -> SceneData { antiAliasingModeData = .smaa case .msaa: antiAliasingModeData = .msaa + case .taa: + antiAliasingModeData = .taa + case .msaaTaa: + antiAliasingModeData = .msaaTaa } sceneData.antiAliasing = AntiAliasingData( mode: antiAliasingModeData, @@ -1696,6 +1702,10 @@ public func deserializeScene( antiAliasingMode = .smaa case .msaa: antiAliasingMode = .msaa + case .taa: + antiAliasingMode = .taa + case .msaaTaa: + antiAliasingMode = .msaaTaa } FXAAParams.shared.subpixelQuality = antiAliasing.fxaa.subpixelQuality diff --git a/Sources/UntoldEngine/Shaders/BitonicSort.metal b/Sources/UntoldEngine/Shaders/BitonicSort.metal index 7b04e7c53..babdf2d4e 100644 --- a/Sources/UntoldEngine/Shaders/BitonicSort.metal +++ b/Sources/UntoldEngine/Shaders/BitonicSort.metal @@ -19,6 +19,7 @@ #include "../../CShaderTypes/ShaderTypes.h" #include "ShaderStructs.h" #include "ShadersUtils.h" +#include "HZBOcclusion.h" using namespace metal; @@ -73,14 +74,18 @@ kernel void gaussianFinalizeVisibleSet( // (one thread per resident splat, head-centre view) and gaussianChunkDecodePreprocess // (GaussianChunkPreprocess.metal, one threadgroup per visible chunk, either eye) so the two // paths keep exactly the same splats: centre inside the guard-banded clip volume, then -// optionally not behind the previous frame's HZB. +// optionally not behind the previous frame's HZB. `hzbRateMap` and `hzbRateMapSizes` are the +// rasterization rate map the pyramid was drawn through, if any (HZBOcclusion.h): the centre's +// screen UV is moved into the pyramid's physical layout before the sample. inline bool gaussianClipCentrePassesCull( float4 centerClip, float clipGuardBand, uint hzbReverseZ, float hzbOcclusionBias, uint hzbValid, - texture2d hzbDepthPyramid) + texture2d hzbDepthPyramid, + constant rasterization_rate_map_data *hzbRateMap, + float4 hzbRateMapSizes) { if (centerClip.w <= 0.0f) return false; @@ -100,6 +105,9 @@ inline bool gaussianClipCentrePassesCull( // source of truth for partial occlusion. if (hzbValid != 0u) { float2 uv = float2(ndc.x * 0.5f + 0.5f, 1.0f - (ndc.y * 0.5f + 0.5f)); + if (hzbRateMapActive(hzbRateMapSizes)) { + uv = hzbScreenUVToPhysicalUV(hzbRateMap, hzbRateMapSizes, uv); + } float splatDepth = clamp(centerClip.z / centerClip.w, 0.0f, 1.0f); constexpr sampler pointSampler(coord::normalized, address::clamp_to_edge, filter::nearest); float hzbDepth = hzbDepthPyramid.sample(pointSampler, uv, level(0)).x; @@ -120,12 +128,14 @@ inline bool gaussianSplatPassesCull( uint hzbReverseZ, float hzbOcclusionBias, uint hzbValid, - texture2d hzbDepthPyramid) + texture2d hzbDepthPyramid, + constant rasterization_rate_map_data *hzbRateMap, + float4 hzbRateMapSizes) { float4 centerClip = uniforms.projectionMatrix * uniforms.modelViewMatrix * float4(position, 1.0f); - return gaussianClipCentrePassesCull(centerClip, clipGuardBand, hzbReverseZ, hzbOcclusionBias, hzbValid, hzbDepthPyramid); + return gaussianClipCentrePassesCull(centerClip, clipGuardBand, hzbReverseZ, hzbOcclusionBias, hzbValid, hzbDepthPyramid, hzbRateMap, hzbRateMapSizes); } // One thread per resident splat: the whole-buffer cull for entities without a chunk table @@ -141,11 +151,13 @@ kernel void gaussianFrustumCull( constant uint &hzbReverseZ [[buffer(gaussianCullHZBReverseZIndex)]], constant float &hzbOcclusionBias [[buffer(gaussianCullHZBOcclusionBiasIndex)]], constant uint &hzbValid [[buffer(gaussianCullHZBValidIndex)]], + constant rasterization_rate_map_data *hzbRateMap [[buffer(gaussianCullHZBRateMapDataIndex)]], + constant float4 &hzbRateMapSizes [[buffer(gaussianCullHZBRateMapSizesIndex)]], texture2d hzbDepthPyramid [[texture(gaussianCullHZBDepthPyramidTextureIndex)]], uint index [[thread_position_in_grid]]) { if (index >= numOfSplats) return; - if (!gaussianSplatPassesCull(splats[index].position, uniforms, clipGuardBand, hzbReverseZ, hzbOcclusionBias, hzbValid, hzbDepthPyramid)) return; + if (!gaussianSplatPassesCull(splats[index].position, uniforms, clipGuardBand, hzbReverseZ, hzbOcclusionBias, hzbValid, hzbDepthPyramid, hzbRateMap, hzbRateMapSizes)) return; uint writeIndex = atomic_fetch_add_explicit(visibleCount, 1u, memory_order_relaxed); visibleIndices[writeIndex] = index; diff --git a/Sources/UntoldEngine/Shaders/GaussianChunkCull.metal b/Sources/UntoldEngine/Shaders/GaussianChunkCull.metal index 49e4e3a2a..0d1602f75 100644 --- a/Sources/UntoldEngine/Shaders/GaussianChunkCull.metal +++ b/Sources/UntoldEngine/Shaders/GaussianChunkCull.metal @@ -93,6 +93,8 @@ static inline bool gaussianBoxPassesClipPlanes( // the HZB. The occlusion test is the mesh cull's (HZBOcclusion.h): the box's screen rect at the // mip that covers it, 5x5 samples, nearest box depth against the farthest sample plus the bias. // `area` is the box's clipped screen area in this view, 0 when the view rejects or occludes it. +// `hzbRateMap` is the rasterization rate map the pyramid was drawn through when it was +// (params.hzbRateMapSizes, HZBOcclusion.h): the rect is moved into the pyramid's layout. static inline bool gaussianChunkVisibleInView( const float3 boxMin, const float3 boxMax, @@ -100,6 +102,7 @@ static inline bool gaussianChunkVisibleInView( constant GaussianChunkCullConstants ¶ms, const uint hzbValid, texture2d hzbDepthPyramid, + constant rasterization_rate_map_data *hzbRateMap, thread float &area) { if (!gaussianBoxPassesClipPlanes(boxMin, boxMax, viewProjection, params.clipGuardBand, area)) { @@ -116,6 +119,7 @@ static inline bool gaussianChunkVisibleInView( if (!projectAABBToScreenRect(0.5f * (boxMin + boxMax), 0.5f * (boxMax - boxMin), viewProjection, reverseZ, uvMin, uvMax, nearDepth)) { return true; } + hzbRemapRectToPhysical(hzbRateMap, params.hzbRateMapSizes, uvMin, uvMax); if (hzbRectIsOccluded(hzbDepthPyramid, uvMin, uvMax, nearDepth, params.viewport, params.hzbMipCount, reverseZ, params.hzbOcclusionBias)) { area = 0.0f; return false; @@ -189,6 +193,7 @@ kernel void gaussianChunkCull( const device GaussianChunkLevelState *levelState [[buffer(gaussianChunkCullLevelStateIndex)]], constant GaussianChunkLevelConstants &lvl [[buffer(gaussianChunkCullLevelConstantsIndex)]], const device GaussianChunkCullRange *ranges [[buffer(gaussianChunkCullRangesIndex)]], + constant rasterization_rate_map_data *hzbRateMap [[buffer(gaussianChunkCullRateMapDataIndex)]], texture2d hzbDepthPyramid [[texture(gaussianChunkCullHZBDepthPyramidTextureIndex)]], uint chunkIndex [[thread_position_in_grid]]) { @@ -227,9 +232,9 @@ kernel void gaussianChunkCull( const uint hzbValidForView0 = params.viewCount > 1u ? 0u : params.hzbValid; float area0 = 0.0f; float area1 = 0.0f; - const bool keep0 = gaussianChunkVisibleInView(boxMin, boxMax, params.viewProjection0, params, hzbValidForView0, hzbDepthPyramid, area0); + const bool keep0 = gaussianChunkVisibleInView(boxMin, boxMax, params.viewProjection0, params, hzbValidForView0, hzbDepthPyramid, hzbRateMap, area0); const bool keep1 = params.viewCount > 1u - ? gaussianChunkVisibleInView(boxMin, boxMax, params.viewProjection1, params, params.hzbValid, hzbDepthPyramid, area1) + ? gaussianChunkVisibleInView(boxMin, boxMax, params.viewProjection1, params, params.hzbValid, hzbDepthPyramid, hzbRateMap, area1) : false; const bool seen = keep0 || keep1; const float area = clamp(max(keep0 ? area0 : 0.0f, keep1 ? area1 : 0.0f), kGaussianScreenAreaMin, limit * limit); diff --git a/Sources/UntoldEngine/Shaders/GaussianChunkPreprocess.metal b/Sources/UntoldEngine/Shaders/GaussianChunkPreprocess.metal index c55a82bad..790b4435a 100644 --- a/Sources/UntoldEngine/Shaders/GaussianChunkPreprocess.metal +++ b/Sources/UntoldEngine/Shaders/GaussianChunkPreprocess.metal @@ -60,16 +60,17 @@ inline bool gaussianChunkSplatPassesCull( float3 position, constant Uniforms &uniforms, constant GaussianChunkCullConstants ¶ms, - texture2d hzbDepthPyramid) + texture2d hzbDepthPyramid, + constant rasterization_rate_map_data *hzbRateMap) { if (params.viewCount <= 1u) { - return gaussianSplatPassesCull(position, uniforms, params.clipGuardBand, params.hzbReverseZ, params.hzbOcclusionBias, params.hzbValid, hzbDepthPyramid); + return gaussianSplatPassesCull(position, uniforms, params.clipGuardBand, params.hzbReverseZ, params.hzbOcclusionBias, params.hzbValid, hzbDepthPyramid, hzbRateMap, params.hzbRateMapSizes); } const float4 local = float4(position, 1.0f); - if (gaussianClipCentrePassesCull(params.viewProjection0 * local, params.clipGuardBand, params.hzbReverseZ, params.hzbOcclusionBias, 0u, hzbDepthPyramid)) { + if (gaussianClipCentrePassesCull(params.viewProjection0 * local, params.clipGuardBand, params.hzbReverseZ, params.hzbOcclusionBias, 0u, hzbDepthPyramid, hzbRateMap, params.hzbRateMapSizes)) { return true; } - return gaussianClipCentrePassesCull(params.viewProjection1 * local, params.clipGuardBand, params.hzbReverseZ, params.hzbOcclusionBias, params.hzbValid, hzbDepthPyramid); + return gaussianClipCentrePassesCull(params.viewProjection1 * local, params.clipGuardBand, params.hzbReverseZ, params.hzbOcclusionBias, params.hzbValid, hzbDepthPyramid, hzbRateMap, params.hzbRateMapSizes); } kernel void gaussianChunkDecodePreprocess( @@ -93,6 +94,7 @@ kernel void gaussianChunkDecodePreprocess( const device GaussianChunkDecodeConstants *coarseTable [[buffer(gaussianChunkPreprocessCoarseTableIndex)]], const device GaussianChunkLevelState *levelState [[buffer(gaussianChunkPreprocessLevelStateIndex)]], constant GaussianChunkLevelConstants &lvl [[buffer(gaussianChunkPreprocessLevelConstantsIndex)]], + constant rasterization_rate_map_data *hzbRateMap [[buffer(gaussianChunkPreprocessRateMapDataIndex)]], texture2d hzbDepthPyramid [[texture(gaussianChunkPreprocessHZBDepthPyramidTextureIndex)]], uint chunkSlot [[threadgroup_position_in_grid]], uint localIndex [[thread_position_in_threadgroup]], @@ -161,7 +163,7 @@ kernel void gaussianChunkDecodePreprocess( // bit-identical to what the whole-buffer path loaded, and the covariance and colour // pass through half precision the way EncodedGaussianSplat stores them. const float3 centerLocal = mix(aabbMin, aabbMax, gaussianUnpack11_10_11(record.x)); - if (!gaussianChunkSplatPassesCull(centerLocal, uniforms, cull, hzbDepthPyramid)) { + if (!gaussianChunkSplatPassesCull(centerLocal, uniforms, cull, hzbDepthPyramid, hzbRateMap)) { continue; } diff --git a/Sources/UntoldEngine/Shaders/Gaussians.metal b/Sources/UntoldEngine/Shaders/Gaussians.metal index 9f5d06d78..e97b846c2 100644 --- a/Sources/UntoldEngine/Shaders/Gaussians.metal +++ b/Sources/UntoldEngine/Shaders/Gaussians.metal @@ -621,15 +621,6 @@ inline float calcPowerFromConic(float3 conic, float2 d) 2.0f * conic.y * d.x * d.y); } -inline float2 calcScreenSpaceDelta(float2 pixelPos, - float2 centerPixel, - float projYSign) -{ - float2 d = pixelPos - centerPixel; - d.y *= projYSign; - return d; -} - typedef struct { half4 color [[raster_order_group(0)]]; @@ -693,20 +684,20 @@ vertex GaussianOutData vertexGaussianTBDRShader( return out; } - const float projYSign = -1.0f; - float2 centerNDC = centerClip.xy / centerClip.w; - float2 centerUV = centerNDC * float2(0.5f, 0.5f * projYSign) + 0.5f; - out.coordxy = centerUV * viewport; - out.conic = record.conicAndOpacity.xyz; // Tight, rotated quad along the ellipse's true principal axes (see // computeInverseCovarianceConic) instead of an axis-aligned bounding box. The axes live - // in the pixel frame of the conic (y down, as coordxy and the fragment's position), while - // NDC y points up: the y offset flips sign on the way, or a tilted splat's quad is the - // mirror image of its ellipse and the fragment falloff gets clipped to their overlap. + // in the pixel frame of the conic (y down), while NDC y points up: the y offset flips + // sign on the way, or a tilted splat's quad is the mirror image of its ellipse and the + // fragment falloff gets clipped to their overlap. The offset itself goes to the fragment + // as a varying: interpolated across the quad it is the fragment's own offset from the + // centre in the conic's frame, whatever the rasterizer did to the quad's pixels — under + // a rasterization rate map (XR foveation) the fragment's position is in the physical + // texture, not on the screen the centre was projected to. float2 pixelOffset = quad.x * record.axes.xy + quad.y * record.axes.zw; float2 ndcOffset = pixelOffset * float2(2.0f, -2.0f) / viewport; + out.pixelDelta = pixelOffset; out.position = centerClip; out.position.xy += ndcOffset * centerClip.w; out.color = record.color.xyz; @@ -745,8 +736,10 @@ fragment GaussianTBDRFragmentStore fragmentGaussianTBDRShader( // (gaussianAdaptiveSigma sigma out) — has negligible alpha. Rejecting those tail fragments // here means they never pay for the depth-texture read at all, on top of never reaching // the blend math below. - const float projYSign = 1.0f; - float2 d = calcScreenSpaceDelta(in.position.xy, in.coordxy, projYSign); + // The fragment's offset from the splat's centre in the conic's pixel frame, from the + // quad's corners (see vertexGaussianTBDRShader) rather than from in.position, which a + // rasterization rate map (XR foveation) places in the physical texture, not on screen. + float2 d = in.pixelDelta; float power = calcPowerFromConic(in.conic, d); float falloff = exp(power); diff --git a/Sources/UntoldEngine/Shaders/HZBCompute.metal b/Sources/UntoldEngine/Shaders/HZBCompute.metal index 9cfea93a4..0ad63f72d 100644 --- a/Sources/UntoldEngine/Shaders/HZBCompute.metal +++ b/Sources/UntoldEngine/Shaders/HZBCompute.metal @@ -80,6 +80,8 @@ kernel void hzbCullVisibleEntities( constant uint &mipCount [[buffer(hzbCullPassMipCountIndex)]], constant uint &reverseZ [[buffer(hzbCullPassReverseZIndex)]], constant float &occlusionBias [[buffer(hzbCullPassOcclusionBiasIndex)]], + constant rasterization_rate_map_data *rateMap [[buffer(hzbCullPassRateMapDataIndex)]], + constant float4 &rateMapSizes [[buffer(hzbCullPassRateMapSizesIndex)]], texture2d hzbDepthPyramid [[texture(hzbCullPassDepthPyramidTextureIndex)]], uint tid [[thread_position_in_grid]] ) { @@ -99,7 +101,9 @@ kernel void hzbCullVisibleEntities( return; } - // Shared with the Gaussian chunk cull — see HZBOcclusion.h. + // Shared with the Gaussian chunk cull — see HZBOcclusion.h. A pyramid drawn through a + // rasterization rate map (XR foveation) holds the rect elsewhere than its screen UV. + hzbRemapRectToPhysical(rateMap, rateMapSizes, uvMin, uvMax); bool isOccluded = hzbRectIsOccluded(hzbDepthPyramid, uvMin, uvMax, nearDepth, viewport, mipCount, reverseZ != 0u, occlusionBias); if (!isOccluded) { uint dst = atomic_fetch_add_explicit(outVisibleCount, 1u, memory_order_relaxed); diff --git a/Sources/UntoldEngine/Shaders/HZBOcclusion.h b/Sources/UntoldEngine/Shaders/HZBOcclusion.h index 0e7c2dcb1..550428736 100644 --- a/Sources/UntoldEngine/Shaders/HZBOcclusion.h +++ b/Sources/UntoldEngine/Shaders/HZBOcclusion.h @@ -80,6 +80,41 @@ static inline bool projectAABBToScreenRect( return true; } +/// The rasterization rate map a depth pyramid was drawn through, when its frame was (visionOS +/// foveation): `rateMapSizes.xy` is the map's screen size and `.zw` the pyramid's base size, in +/// pixels, and `rateMapSizes.x <= 0` says there is none, in which case `rateMap` is a placeholder +/// that is never read. Such a pyramid is not laid out over the screen the boxes above project +/// to — dense where the eyes look, sparse in the periphery — so a screen UV must be moved to +/// where the map put it before the pyramid is sampled there. +static inline bool hzbRateMapActive(const float4 rateMapSizes) { + return rateMapSizes.x > 0.0f && rateMapSizes.z > 0.0f; +} + +static inline float2 hzbScreenUVToPhysicalUV( + constant rasterization_rate_map_data *rateMap, + const float4 rateMapSizes, + const float2 uv +) { + rasterization_rate_map_decoder decoder(*rateMap); + const float2 physical = decoder.map_screen_to_physical_coordinates(uv * rateMapSizes.xy); + return clamp(physical / rateMapSizes.zw, 0.0f, 1.0f); +} + +/// Moves a screen rect from `projectAABBToScreenRect` into the pyramid's physical layout; the +/// map is monotonic along each axis, so the two corners are enough. Nothing without a map. +static inline void hzbRemapRectToPhysical( + constant rasterization_rate_map_data *rateMap, + const float4 rateMapSizes, + thread float2 &uvMin, + thread float2 &uvMax +) { + if (!hzbRateMapActive(rateMapSizes)) { + return; + } + uvMin = hzbScreenUVToPhysicalUV(rateMap, rateMapSizes, uvMin); + uvMax = hzbScreenUVToPhysicalUV(rateMap, rateMapSizes, uvMax); +} + /// True when the rect `uvMin..uvMax` whose nearest depth is `nearDepth` lies behind the HZB /// everywhere it is sampled. The mip is the one whose texel spans the rect; a 5x5 grid across /// the rect keeps porous occluders (window frames, glass assemblies) from falsely culling what diff --git a/Sources/UntoldEngine/Shaders/SSAOShader.metal b/Sources/UntoldEngine/Shaders/SSAOShader.metal index d93b867ab..7df4249a8 100644 --- a/Sources/UntoldEngine/Shaders/SSAOShader.metal +++ b/Sources/UntoldEngine/Shaders/SSAOShader.metal @@ -34,6 +34,19 @@ static float3 reconstructViewPos(float2 uv, float linearDepth, float2 projScale) return float3(xy, linearDepth); } +// The screen UV a depth texel was projected to: its own UV, unless the eye was drawn through +// a rasterization rate map (XR foveation, rateMapSizes.x > 0: the map's screen size in xy, the +// depth texture's size in zw). Such a texture is dense where the eyes look and sparse in the +// periphery, so a view position rebuilt from the texel's own UV would bend the geometry there +// and shade false creases; the map's decoder says where the texel sits on screen. +static float2 screenUVOfTexel(float2 uv, constant rasterization_rate_map_data *rateMap, float4 rateMapSizes) { + if (rateMapSizes.x <= 0.0 || rateMapSizes.z <= 0.0) { + return uv; + } + rasterization_rate_map_decoder decoder(*rateMap); + return decoder.map_physical_to_screen_coordinates(uv * rateMapSizes.zw) / rateMapSizes.xy; +} + fragment float4 fragmentSSAOShader(VertexCompositeOutput vertexOut [[stage_in]], depth2d depthTexture [[texture(ssaoDepthTextureIndex)]], constant float &radius [[buffer(ssaoPassRadiusIndex)]], @@ -43,7 +56,9 @@ fragment float4 fragmentSSAOShader(VertexCompositeOutput vertexOut [[stage_in]], constant float2 &viewPort [[buffer(ssaoPassViewPortIndex)]], constant float2 &frustumPlanes [[buffer(ssaoPassFrustumIndex)]], constant bool &reverseZ [[buffer(ssaoPassReverseZIndex)]], - constant float2 &projScale [[buffer(ssaoPassProjScaleIndex)]] + constant float2 &projScale [[buffer(ssaoPassProjScaleIndex)]], + constant rasterization_rate_map_data *rateMap [[buffer(ssaoPassRateMapDataIndex)]], + constant float4 &rateMapSizes [[buffer(ssaoPassRateMapSizesIndex)]] ) { if (!enabled){ @@ -62,7 +77,7 @@ fragment float4 fragmentSSAOShader(VertexCompositeOutput vertexOut [[stage_in]], float2 texelSize = 1.0 / max(viewPort, float2(1.0)); float pixelRadius = clamp(radius * 220.0 / max(centerDepth, 0.001), 2.0, 48.0); - float3 centerPosition = reconstructViewPos(vertexOut.uvCoords, centerDepth, projScale); + float3 centerPosition = reconstructViewPos(screenUVOfTexel(vertexOut.uvCoords, rateMap, rateMapSizes), centerDepth, projScale); // No normal G-buffer is available here — normalMap/positionMap are memoryless // (TBDR tile-only) in normal rendering and can't be bound to a later, separate // pass. Derive the surface normal from the depth buffer itself via screen-space @@ -94,7 +109,7 @@ fragment float4 fragmentSSAOShader(VertexCompositeOutput vertexOut [[stage_in]], } float sampleLinearDepth = linearizeDepth(sampleRawDepth, frustumPlanes.x, frustumPlanes.y, reverseZ); - float3 samplePosition = reconstructViewPos(sampleUV, sampleLinearDepth, projScale); + float3 samplePosition = reconstructViewPos(screenUVOfTexel(sampleUV, rateMap, rateMapSizes), sampleLinearDepth, projScale); float3 toSample = samplePosition - centerPosition; float sampleDistance = length(toSample); if (sampleDistance < 1e-5) { diff --git a/Sources/UntoldEngine/Shaders/ShaderStructs.h b/Sources/UntoldEngine/Shaders/ShaderStructs.h index 8a5d939c1..db6a59375 100644 --- a/Sources/UntoldEngine/Shaders/ShaderStructs.h +++ b/Sources/UntoldEngine/Shaders/ShaderStructs.h @@ -187,10 +187,10 @@ typedef struct{ float3 cov2; float3 conic; float3 color; - float2 coordxy; + float2 pixelDelta; // the vertex's offset from the splat's centre in the conic's pixel frame (y down) float alpha; bool valid; - + }GaussianOutData; // Cascaded Shadow Maps: 3 cascades shared between both eyes in XR. diff --git a/Sources/UntoldEngine/Shaders/TAAShader.metal b/Sources/UntoldEngine/Shaders/TAAShader.metal new file mode 100644 index 000000000..74e81c8e2 --- /dev/null +++ b/Sources/UntoldEngine/Shaders/TAAShader.metal @@ -0,0 +1,221 @@ +// +// TAAShader.metal +// UntoldEngine +// +// Temporal anti-aliasing resolve. The frame was drawn with the projection jittered by a +// sub-pixel amount (TemporalAntiAliasing.swift); this pass blends the jittered frame with the +// eye's history, reprojected through the depth buffer and the two frames' unjittered +// view-projections, after clipping the history to the colour range of the pixel's 3×3 +// neighbourhood so that what moved or was uncovered does not ghost. Over frames the jitter +// averages sub-pixel detail — thin wires, fine normal maps — that a single sample flickers on. +// +// Under a rasterization rate map (XR foveation) the colour, depth and history textures are in +// physical space, not the screen the geometry was projected to, and each frame's map differs: +// a pixel is mapped physical → screen (this frame's map) → world → screen (previous frame, +// previous matrices) → physical (the history frame's map) before the history is sampled. +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +#include +#include "../../CShaderTypes/ShaderTypes.h" +#include "ShaderStructs.h" +using namespace metal; + +vertex VertexCompositeOutput vertexTAAShader(VertexCompositeIn in [[stage_in]]) { + VertexCompositeOutput out; + out.position = float4(float3(in.position), 1.0); + out.uvCoords = in.uvCoords; + return out; +} + +struct TAAFragmentOut { + float4 color [[color(0)]]; + float4 history [[color(1)]]; +}; + +// Luma-chroma space for the neighbourhood clip: a box in YCoCg hugs the local colours more +// tightly than one in RGB, so less ghosting gets through on moving edges. +static inline float3 taaRGBToYCoCg(float3 c) { + return float3(0.25 * c.r + 0.5 * c.g + 0.25 * c.b, + 0.5 * c.r - 0.5 * c.b, + -0.25 * c.r + 0.5 * c.g - 0.25 * c.b); +} + +static inline float3 taaYCoCgToRGB(float3 c) { + return float3(c.x + c.y - c.z, c.x + c.z, c.x - c.y - c.z); +} + +// Moves `history` towards `current` until it lies inside the box [lo, hi]: the direction the +// history came from is kept, only its distance is cut, which avoids the colour shifts a plain +// clamp introduces (Karis 2014). +static inline float3 taaClipToBox(float3 lo, float3 hi, float3 current, float3 history) { + const float3 center = 0.5 * (hi + lo); + const float3 extent = 0.5 * (hi - lo) + 1e-4; + const float3 offset = history - center; + const float3 unit = abs(offset / extent); + const float maxUnit = max(unit.x, max(unit.y, unit.z)); + return maxUnit > 1.0 ? center + offset / maxUnit : history; +} + +// Where a texel of this frame's textures sits on the screen, in pixels. +static inline float2 taaScreenFromPhysical(float2 physical, float4 sizes, constant rasterization_rate_map_data *rateMap) { + if (sizes.x <= 0.0) { + return physical; + } + rasterization_rate_map_decoder decoder(*rateMap); + return decoder.map_physical_to_screen_coordinates(physical); +} + +// Where a screen position of the history's frame sits in the history texture, in pixels. +static inline float2 taaPhysicalFromScreen(float2 screen, float4 sizes, constant rasterization_rate_map_data *rateMap) { + if (sizes.x <= 0.0) { + return screen; + } + rasterization_rate_map_decoder decoder(*rateMap); + return decoder.map_screen_to_physical_coordinates(screen); +} + +// The history at a fractional position, Catmull-Rom filtered with nine bilinear taps (Jimenez +// 2016): resampling the history every frame with a bilinear filter smears it a little more +// each time, which under a moving head or a moving rate map shows as a soft halo that settles +// only when everything stands still; the cubic keeps it sharp. +static inline float4 taaSampleHistory(texture2d history, sampler linearSampler, float2 uv, float2 size) { + const float2 texel = 1.0 / size; + const float2 position = uv * size; + const float2 center = floor(position - 0.5) + 0.5; + const float2 f = position - center; + const float2 w0 = f * (-0.5 + f * (1.0 - 0.5 * f)); + const float2 w1 = 1.0 + f * f * (-2.5 + 1.5 * f); + const float2 w2 = f * (0.5 + f * (2.0 - 1.5 * f)); + const float2 w3 = f * f * (-0.5 + 0.5 * f); + const float2 w12 = w1 + w2; + const float2 offset12 = w2 / w12; + const float2 uv0 = (center - 1.0) * texel; + const float2 uv3 = (center + 2.0) * texel; + const float2 uv12 = (center + offset12) * texel; + float4 result = float4(0.0); + result += history.sample(linearSampler, float2(uv0.x, uv0.y)) * w0.x * w0.y; + result += history.sample(linearSampler, float2(uv12.x, uv0.y)) * w12.x * w0.y; + result += history.sample(linearSampler, float2(uv3.x, uv0.y)) * w3.x * w0.y; + result += history.sample(linearSampler, float2(uv0.x, uv12.y)) * w0.x * w12.y; + result += history.sample(linearSampler, float2(uv12.x, uv12.y)) * w12.x * w12.y; + result += history.sample(linearSampler, float2(uv3.x, uv12.y)) * w3.x * w12.y; + result += history.sample(linearSampler, float2(uv0.x, uv3.y)) * w0.x * w3.y; + result += history.sample(linearSampler, float2(uv12.x, uv3.y)) * w12.x * w3.y; + result += history.sample(linearSampler, float2(uv3.x, uv3.y)) * w3.x * w3.y; + return max(result, 0.0); +} + +fragment TAAFragmentOut fragmentTAAShader( + VertexCompositeOutput in [[stage_in]], + texture2d colorTexture [[texture(taaPassColorTextureIndex)]], + texture2d historyTexture [[texture(taaPassHistoryTextureIndex)]], + depth2d depthTexture [[texture(taaPassDepthTextureIndex)]], + constant TAAConstants &taa [[buffer(taaPassConstantsIndex)]], + constant rasterization_rate_map_data *rateMap [[buffer(taaPassRateMapDataIndex)]], + constant rasterization_rate_map_data *prevRateMap [[buffer(taaPassPrevRateMapDataIndex)]] +) { + constexpr sampler linearSampler(min_filter::linear, mag_filter::linear, mip_filter::none, address::clamp_to_edge); + constexpr sampler pointSampler(min_filter::nearest, mag_filter::nearest, mip_filter::none, address::clamp_to_edge); + + const float2 uv = in.uvCoords; + const float2 texel = 1.0 / max(taa.physicalSize, float2(1.0)); + const float4 current = colorTexture.sample(pointSampler, uv); + + TAAFragmentOut out; + if (taa.historyValid == 0u) { + out.color = current; + out.history = current; + return out; + } + + // The colour box of the 3×3 neighbourhood, in YCoCg: the mean ± γ·σ (variance clipping, + // Salvi 2016) rather than the hard min/max. A feature a pixel wide — a wire, a line of + // text — is rasterized in some jittered frames and not others; a min/max box of a frame + // that missed it excludes its colour and the clip wipes it from the history, so it flickers. + // The statistical box still reaches towards it through the frames that miss it. + float3 m1 = float3(0.0); + float3 m2 = float3(0.0); + for (int y = -1; y <= 1; ++y) { + for (int x = -1; x <= 1; ++x) { + const float3 c = taaRGBToYCoCg(colorTexture.sample(pointSampler, uv + float2(x, y) * texel).rgb); + m1 += c; + m2 += c * c; + } + } + const float3 mean = m1 / 9.0; + const float3 sigma = sqrt(max(m2 / 9.0 - mean * mean, 0.0)); + const float3 lo = mean - taa.clipGamma * sigma; + const float3 hi = mean + taa.clipGamma * sigma; + + // This pixel's position in space, from its depth. A texel at the clear depth has none: + // nothing opaque was drawn there — the background, or a transparent quad over it — and + // reprojecting it from the far plane would fetch the wrong history and ghost or wipe it + // under a moving camera; such a pixel keeps its own frame. + const float depth = depthTexture.sample(pointSampler, uv); + const bool cleared = (taa.reverseZ != 0u) ? (depth <= 1e-6) : (depth >= 1.0 - 1e-6); + if (cleared) { + out.color = current; + out.history = current; + return out; + } + const float2 physical = in.position.xy; + const float2 screen = taaScreenFromPhysical(physical, taa.rateMapSizes, rateMap); + const float2 screenSize = taa.rateMapSizes.x > 0.0 ? taa.rateMapSizes.xy : taa.physicalSize; + const float2 ndc = float2(screen.x / screenSize.x * 2.0 - 1.0, 1.0 - screen.y / screenSize.y * 2.0); + float4 world = taa.invViewProjection * float4(ndc, depth, 1.0); + if (abs(world.w) < 1e-12) { + out.color = current; + out.history = current; + return out; + } + world /= world.w; + + // Where it was on the previous frame's screen, then in the history texture. + const float4 prevClip = taa.prevViewProjection * world; + if (prevClip.w <= 0.0) { + out.color = current; + out.history = current; + return out; + } + const float2 prevNdc = prevClip.xy / prevClip.w; + const float2 prevScreenSize = taa.prevRateMapSizes.x > 0.0 ? taa.prevRateMapSizes.xy : taa.physicalSize; + const float2 prevScreen = float2((prevNdc.x * 0.5 + 0.5) * prevScreenSize.x, (1.0 - (prevNdc.y * 0.5 + 0.5)) * prevScreenSize.y); + const float2 prevPhysical = taaPhysicalFromScreen(prevScreen, taa.prevRateMapSizes, prevRateMap); + const float2 historyUV = prevPhysical * texel; + if (any(historyUV < 0.0) || any(historyUV > 1.0)) { + out.color = current; + out.history = current; + return out; + } + + // Under foveation the history may be coarser here than this frame: the spot was in the + // periphery of the previous map (one texel spanning several screen pixels) and the gaze + // has moved onto it. Trusting such a history at full weight shows the sharp frame with a + // blurry shadow that fades over twenty frames; its weight is cut by how much coarser it + // was, measured from the two maps' local rates (texels per screen pixel). + float weight = taa.historyWeight; + if (taa.rateMapSizes.x > 0.0 || taa.prevRateMapSizes.x > 0.0) { + // Texels per screen pixel of each frame, per axis: a map is coarser in one axis than + // the other wherever its rows and columns differ, so the ratio is taken on the axis + // where the history is coarsest relative to this frame. + const float2 screenStep = abs(taaScreenFromPhysical(physical + float2(1.0, 1.0), taa.rateMapSizes, rateMap) - screen); + const float2 currentRate = 1.0 / max(screenStep, float2(1e-3)); + const float2 prevStep = abs(taaPhysicalFromScreen(prevScreen + float2(1.0, 1.0), taa.prevRateMapSizes, prevRateMap) - prevPhysical); + const float2 prevRate = max(prevStep, float2(1e-3)); + const float2 ratio = prevRate / currentRate; + weight *= saturate(min(ratio.x, ratio.y)); + } + + const float4 history = taaSampleHistory(historyTexture, linearSampler, historyUV, taa.physicalSize); + const float3 clipped = taaYCoCgToRGB(taaClipToBox(lo, hi, taaRGBToYCoCg(current.rgb), taaRGBToYCoCg(history.rgb))); + const float3 blended = mix(current.rgb, clipped, weight); + const float alpha = mix(current.a, history.a, weight); + out.color = float4(blended, alpha); + out.history = out.color; + return out; +} diff --git a/Sources/UntoldEngine/Systems/CullingSystem.swift b/Sources/UntoldEngine/Systems/CullingSystem.swift index 040abdb5d..4f62bbd92 100644 --- a/Sources/UntoldEngine/Systems/CullingSystem.swift +++ b/Sources/UntoldEngine/Systems/CullingSystem.swift @@ -548,6 +548,14 @@ func executeHZBOcclusionCulling( computeEncoder.setBytes(&mipCount, length: MemoryLayout.stride, index: Int(hzbCullPassMipCountIndex.rawValue)) computeEncoder.setBytes(&reverseZFlag, length: MemoryLayout.stride, index: Int(hzbCullPassReverseZIndex.rawValue)) computeEncoder.setBytes(&occlusionBias, length: MemoryLayout.stride, index: Int(hzbCullPassOcclusionBiasIndex.rawValue)) + // The rasterization rate map the pyramid's frame was drawn through (XR foveation): the + // rects are moved into the pyramid's physical layout before it is sampled. + bindRateMap( + pyramidFrame.rateMapData, + to: computeEncoder, + dataIndex: Int(hzbCullPassRateMapDataIndex.rawValue), + sizesIndex: Int(hzbCullPassRateMapSizesIndex.rawValue) + ) computeEncoder.setTexture(hzbDepthPyramid, index: Int(hzbCullPassDepthPyramidTextureIndex.rawValue)) let tew = pipelineState.threadExecutionWidth @@ -1257,9 +1265,11 @@ extension HZBPyramidFrame { let cameraComponent = scene.get(component: CameraComponent.self, for: camera) else { return nil } let view = SceneRootTransform.shared.effectiveViewMatrix(cameraComponent.viewSpace) + // In stereo the last eye drawn: its view, projection and rate map alike. return HZBPyramidFrame( viewProjection: simd_mul(renderInfo.perspectiveSpace, view), - cameraPosition: eyePosition(ofView: view) + cameraPosition: eyePosition(ofView: view), + rateMapData: renderInfo.xrFoveation?.rateMapData ) } } diff --git a/Sources/UntoldEngine/Systems/GaussianChunkCull.swift b/Sources/UntoldEngine/Systems/GaussianChunkCull.swift index 93c307986..bf425a541 100644 --- a/Sources/UntoldEngine/Systems/GaussianChunkCull.swift +++ b/Sources/UntoldEngine/Systems/GaussianChunkCull.swift @@ -1033,6 +1033,9 @@ func gaussianChunkCullConstants( constants.hzbValid = hzbValid ? 1 : 0 constants.hzbReverseZ = renderInfo.reverseZEnabled ? 1 : 0 constants.hzbMipCount = UInt32(max(0, renderInfo.hzbMipCount)) + // The rasterization rate map the pyramid was drawn through (XR foveation), only with a + // pyramid to sample: the "no map" word otherwise, and the kernels never touch the buffer. + constants.hzbRateMapSizes = hzbValid ? rateMapSizes(hzbPyramidRateMapData()) : .zero constants.forceAllVisible = forceAllVisible ? 1 : 0 constants.uniformQuotas = uniformQuotas ? 1 : 0 constants.paged = paged @@ -1255,6 +1258,12 @@ private func encodeGaussianChunkCullDispatch( } else { encoder.setBytes(&gpuRanges, length: MemoryLayout.stride * gpuRanges.count, index: Int(gaussianChunkCullRangesIndex.rawValue)) } + // The pyramid's rate map (XR foveation), read only when the constants' hzbRateMapSizes say so. + encoder.setBuffer( + hzbPyramidRateMapData()?.buffer ?? chunkTable.constantsBuffer, + offset: 0, + index: Int(gaussianChunkCullRateMapDataIndex.rawValue) + ) encoder.setTexture(hzbTexture, index: Int(gaussianChunkCullHZBDepthPyramidTextureIndex.rawValue)) let tew = pipelines.cull.threadExecutionWidth let block = max(min(256, pipelines.cull.maxTotalThreadsPerThreadgroup) / tew * tew, tew) @@ -1530,6 +1539,12 @@ func encodeGaussianChunkDecodePreprocess( encoder.setBuffer(inputs.levels?.coarseTable ?? inputs.chunkTable.constantsBuffer, offset: 0, index: Int(gaussianChunkPreprocessCoarseTableIndex.rawValue)) encoder.setBuffer(inputs.levels?.levelState ?? inputs.chunkTable.constantsBuffer, offset: 0, index: Int(gaussianChunkPreprocessLevelStateIndex.rawValue)) encoder.setBytes(&inputs.levelConstants, length: MemoryLayout.stride, index: Int(gaussianChunkPreprocessLevelConstantsIndex.rawValue)) + // The pyramid's rate map (XR foveation), read only when the cull constants' hzbRateMapSizes say so. + encoder.setBuffer( + hzbPyramidRateMapData()?.buffer ?? inputs.chunkTable.constantsBuffer, + offset: 0, + index: Int(gaussianChunkPreprocessRateMapDataIndex.rawValue) + ) encoder.setTexture(inputs.hzbTexture, index: Int(gaussianChunkPreprocessHZBDepthPyramidTextureIndex.rawValue)) let threads = threadsPerThreadgroup ?? gaussianChunkPreprocessThreadsPerThreadgroup(chunkTable: inputs.chunkTable, pipelineState: pipelineState) diff --git a/Sources/UntoldEngine/Systems/GaussianSystem.swift b/Sources/UntoldEngine/Systems/GaussianSystem.swift index f80679111..13e4d3882 100644 --- a/Sources/UntoldEngine/Systems/GaussianSystem.swift +++ b/Sources/UntoldEngine/Systems/GaussianSystem.swift @@ -128,6 +128,14 @@ private func bindGaussianSplatCullInputs( encoder.setBytes(&inputs.hzbReverseZ, length: MemoryLayout.stride, index: Int(gaussianCullHZBReverseZIndex.rawValue)) encoder.setBytes(&inputs.hzbOcclusionBias, length: MemoryLayout.stride, index: Int(gaussianCullHZBOcclusionBiasIndex.rawValue)) encoder.setBytes(&inputs.hzbValid, length: MemoryLayout.stride, index: Int(gaussianCullHZBValidIndex.rawValue)) + // The rasterization rate map the pyramid was drawn through (XR foveation), with a pyramid + // to sample; the "no map" word otherwise. + bindRateMap( + inputs.hzbValid != 0 ? hzbPyramidRateMapData() : nil, + to: encoder, + dataIndex: Int(gaussianCullHZBRateMapDataIndex.rawValue), + sizesIndex: Int(gaussianCullHZBRateMapSizesIndex.rawValue) + ) encoder.setTexture(hzbTexture, index: Int(gaussianCullHZBDepthPyramidTextureIndex.rawValue)) } diff --git a/Sources/UntoldEngine/Systems/GraphBuilder.swift b/Sources/UntoldEngine/Systems/GraphBuilder.swift index df194fc4a..925d60dea 100644 --- a/Sources/UntoldEngine/Systems/GraphBuilder.swift +++ b/Sources/UntoldEngine/Systems/GraphBuilder.swift @@ -949,6 +949,9 @@ func makeSceneRenderTargetAccess( return nil } encoder.label = label + // The copied descriptor carries the eye's rasterization rate map (XR foveation); the + // viewport that goes with it is the eye's, not the texture's. + applyXRFoveationViewport(encoder) return encoder } } diff --git a/Sources/UntoldEngine/Systems/RenderingSystem.swift b/Sources/UntoldEngine/Systems/RenderingSystem.swift index 42da9e94d..b0a3d6d0a 100644 --- a/Sources/UntoldEngine/Systems/RenderingSystem.swift +++ b/Sources/UntoldEngine/Systems/RenderingSystem.swift @@ -94,7 +94,12 @@ func UpdateRenderingSystem(in view: MTKView) { commandBuffer.label = "Rendering Command Buffer" do { + // This frame's sub-pixel jitter (temporal anti-aliasing), before anything reads the projection. + TemporalAntiAliasing.shared.applyJitter() let graph = try buildExecutableGameModeGraph() + // A rasterization rate map set on renderInfo.xrFoveation (the foveation tests + // draw through one on the Mac) goes on the scene passes here as in XR. + applyXRFoveationToSceneRenderPassDescriptors() #if ENGINE_STATS_ENABLED let encodeStart = CACurrentMediaTime() @@ -168,8 +173,14 @@ func UpdateXRRenderingSystem(commandBuffer: MTLCommandBuffer, passDescriptor: MT commandBuffer.label = "XR Rendering Command Buffer" do { + // This eye's sub-pixel jitter (temporal anti-aliasing), before anything reads the projection. + TemporalAntiAliasing.shared.applyJitter() let graph = try buildExecutableGameModeGraph() + // The eye's rasterization rate map (visionOS foveation) on the passes that draw the + // scene, after the build: a rebuild may have recreated their descriptors. + applyXRFoveationToSceneRenderPassDescriptors() + #if ENGINE_STATS_ENABLED let encodeStart = shouldRecordStatsInThisCallback ? CACurrentMediaTime() : 0.0 #endif @@ -256,6 +267,7 @@ let gameModeReservedPassIDs: Set = [ "look", "fxaa", "fxaaEdgeDebug", + "taa", "smaaEdges", "smaaBlendWeights", "smaaNeighborhood", @@ -526,6 +538,10 @@ private func buildGameModeGraphWithCompilation() throws -> CompiledRenderGraphRe let fxaaPass = RenderPass(id: "fxaa", dependencies: [lookPass.id], execute: fxaaRenderPass) builder.addPass(fxaaPass) outputDependency = fxaaPass.id + case .taa, .msaaTaa: + let taaPass = RenderPass(id: "taa", dependencies: [lookPass.id], execute: taaRenderPass) + builder.addPass(taaPass) + outputDependency = taaPass.id case .smaa: let smaaEdgesPass = RenderPass(id: "smaaEdges", dependencies: [lookPass.id], execute: smaaEdgesRenderPass) builder.addPass(smaaEdgesPass) diff --git a/Sources/UntoldEngine/Systems/SmallObjectCulling.swift b/Sources/UntoldEngine/Systems/SmallObjectCulling.swift index c37c72b0e..2f8241454 100644 --- a/Sources/UntoldEngine/Systems/SmallObjectCulling.swift +++ b/Sources/UntoldEngine/Systems/SmallObjectCulling.swift @@ -57,10 +57,13 @@ struct SmallObjectCulling { guard let viewPort, viewPort.y > 0, projection.columns.3.w == 0, projection.columns.1.y > 0 else { return nil } + // Pixels of the screen the eye sees: under a rasterization rate map (XR foveation) the + // textures are smaller than that screen, and a size in their pixels would cull early. + let screenHeight = renderInfo.xrFoveation?.screenSize.y ?? viewPort.y return SmallObjectCulling( cameraPosition: SceneRootTransform.shared.effectiveCameraPosition(cameraComponent.localPosition), minimumPixels: pixels, - viewportHeight: viewPort.y, + viewportHeight: screenHeight, tanHalfFovY: 1 / projection.columns.1.y ) } diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-ios.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-ios.metallib index c6a09d922..e079f3c35 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-ios.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-ios.metallib differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-iossim.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-iossim.metallib index 2460ea36a..34d4ef911 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-iossim.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-iossim.metallib differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvos.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvos.metallib index f00658282..b073a9dc7 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvos.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvos.metallib differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvossim.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvossim.metallib index 00249458f..39e744740 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvossim.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvossim.metallib differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xros.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xros.metallib index d81cb37be..4156f7c1b 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xros.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xros.metallib differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xrossim.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xrossim.metallib index bcd5eba84..8ac6f3672 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xrossim.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xrossim.metallib differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels.metal b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels.metal index 23a16dff4..1d4773887 100644 --- a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels.metal +++ b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels.metal @@ -50,6 +50,7 @@ using namespace metal; #include "../Shaders/spatialDebugShader.metal" #include "../Shaders/FXAAShader.metal" #include "../Shaders/SMAAShader.metal" +#include "../Shaders/TAAShader.metal" // Gaussian kernels #include "../Shaders/BitonicSort.metal" #include "../Shaders/DeviceRadixSort.metal" diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels.metallib index 37cee1e05..30dd8e82b 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels.metallib differ diff --git a/Sources/UntoldEngine/Utils/EngineSettingsAPI.swift b/Sources/UntoldEngine/Utils/EngineSettingsAPI.swift index dea53a774..67a25658c 100644 --- a/Sources/UntoldEngine/Utils/EngineSettingsAPI.swift +++ b/Sources/UntoldEngine/Utils/EngineSettingsAPI.swift @@ -86,6 +86,16 @@ public enum RenderingProperty: Sendable { /// frustum keeps. In scene units; the default is 0.5. See /// `getOcclusionCullingMaxCameraStep()` to read the current value. case occlusionCullingMaxCameraStep(Float) + /// Whether the XR frame is drawn through the rasterization rate maps the visionOS + /// compositor provides when the layer is configured with foveation enabled + /// (`LayerRenderer.Configuration.isFoveationEnabled`): dense where the eyes look, coarse + /// in the periphery, and the only configuration in which the compositor accepts a render + /// quality above the platform default. Enabled by default; a layer configured without + /// foveation provides no maps and draws uniformly whatever this says. Disabling it in a + /// foveated layer is a diagnostic: the frame is then drawn uniformly into textures the + /// compositor still unwarps, so the picture is distorted. See + /// `getFoveatedRenderingEnabled()` to read the current value. + case foveatedRendering(RenderingToggle) } public enum RenderingToggle: Sendable { @@ -148,9 +158,19 @@ public func setRendering(_ property: RenderingProperty) { SmallObjectCulling.minimumPixels = pixels.isFinite ? max(pixels, 0) : 0 case let .occlusionCullingMaxCameraStep(step): HZBOcclusionCulling.maxCameraStep = step.isFinite ? max(step, 0) : 0 + case .foveatedRendering(.enabled): + xrFoveatedRenderingEnabled = true + case .foveatedRendering(.disabled): + xrFoveatedRenderingEnabled = false } } +/// Whether the XR frame draws through the compositor's rasterization rate maps when the layer +/// provides them. See `RenderingProperty.foveatedRendering`. +public func getFoveatedRenderingEnabled() -> Bool { + xrFoveatedRenderingEnabled +} + public enum RenderResourceQuery { case texture(String) } diff --git a/Sources/UntoldEngine/Utils/Globals.swift b/Sources/UntoldEngine/Utils/Globals.swift index 1a92d1f3a..19fbe4bb7 100644 --- a/Sources/UntoldEngine/Utils/Globals.swift +++ b/Sources/UntoldEngine/Utils/Globals.swift @@ -1103,6 +1103,7 @@ private final class RuntimeGlobalsStore: @unchecked Sendable { private var activeEntityValue: EntityID = .invalid private var enableEngineMetricsValue: Bool = false private var bypassPostProcessingValue: Bool = false + private var xrFoveatedRenderingEnabledValue: Bool = true private var antiAliasingModeValue: AntiAliasingMode = .fxaa private var renderDebugViewModeValue: RenderDebugViewMode = .lit private var pomQualitySettingsValue: POMQualitySettings = .platformDefault @@ -1594,6 +1595,20 @@ private final class RuntimeGlobalsStore: @unchecked Sendable { } } + var xrFoveatedRenderingEnabled: Bool { + get { + lock.lock() + let value = xrFoveatedRenderingEnabledValue + lock.unlock() + return value + } + set { + lock.lock() + xrFoveatedRenderingEnabledValue = newValue + lock.unlock() + } + } + var antiAliasingMode: AntiAliasingMode { get { lock.lock() @@ -2093,16 +2108,30 @@ public var bypassPostProcessing: Bool { set { RuntimeGlobalsStore.shared.bypassPostProcessing = newValue } } +/// Whether the XR frame draws through the rasterization rate maps the compositor provides +/// (visionOS foveation), the default. See `RenderingProperty.foveatedRendering`. +public var xrFoveatedRenderingEnabled: Bool { + get { RuntimeGlobalsStore.shared.xrFoveatedRenderingEnabled } + set { RuntimeGlobalsStore.shared.xrFoveatedRenderingEnabled = newValue } +} + /// Selects the active anti-aliasing pass inserted after the look pass. public enum AntiAliasingMode: Sendable { case none case fxaa case smaa case msaa + /// Temporal: the projection jittered by a sub-pixel amount each frame and the frame + /// blended with its reprojected history after the look pass (TemporalAntiAliasing.swift). + /// Averages the sub-pixel detail a single sample flickers on; moving objects rely on the + /// neighbourhood clip, not on motion vectors. + case taa + /// 4× MSAA on the G-buffer and the temporal resolve on top. + case msaaTaa var usesPostLookPass: Bool { switch self { - case .fxaa, .smaa: + case .fxaa, .smaa, .taa, .msaaTaa: return true case .none, .msaa: return false diff --git a/Sources/UntoldEngineXR/UntoldEngineXR.swift b/Sources/UntoldEngineXR/UntoldEngineXR.swift index b32d7c122..3e98fb7ae 100644 --- a/Sources/UntoldEngineXR/UntoldEngineXR.swift +++ b/Sources/UntoldEngineXR/UntoldEngineXR.swift @@ -69,6 +69,17 @@ // Reuse render pass descriptors to avoid allocation churn (2 eyes × 90 FPS = 180 allocs/sec) private let passDescriptorLeft = MTLRenderPassDescriptor() private let passDescriptorRight = MTLRenderPassDescriptor() + + // Foveation: the parameter data of each eye's rasterization rate map, copied every frame + // (the maps follow the gaze) into a ring one slot deeper than the frames in flight — + // the depth pyramid built on a frame carries its last eye's data into the culls of the + // frame after (HZBPyramidFrame.rateMapData), which may still run when this frame's + // slot comes round again with only the usual in-flight depth. + private var rateMapParameterBuffers: [[MTLBuffer?]] = Array(repeating: [nil, nil], count: maxInFlightCommandBuffers + 1) + private var rateMapFrameCounter = 0 + private var loggedFoveationState: Bool? + private var loggedCaptureTargetSkip = false + private var loggedDrawableCount = 0 private let spatialGestureRecognizer = XRSpatialGestureRecognizer() private let xrEnvironmentLightingSystem = XREnvironmentLightingSystem() private var runtimeLightingModeObserverId: UUID? @@ -616,64 +627,96 @@ } } - // 9. Encode any drawing commands that depend on the device position or orientation - guard let drawable = frame.queryDrawable() else { + // 9. Encode any drawing commands that depend on the device position or orientation. + // visionOS 26 hands a frame one drawable per target (the built-in display, and a + // capture target while the view is recorded or streamed); a layer with foveation + // or a render quality set aborts the app if the single-drawable query is used. + let drawables: [LayerRenderer.Drawable] + if #available(visionOS 26.0, *) { + drawables = frame.queryDrawables() + } else { + drawables = frame.queryDrawable().map { [$0] } ?? [] + } + guard !drawables.isEmpty else { #if ENGINE_STATS_ENABLED renderer.finalizeXRStatsAndMonitors(frameStartTime: xrFrameStartTime) #else renderer.finalizeXRStatsAndMonitors(frameStartTime: 0.0) #endif - // queryDrawable() can fail during shutdown/teardown; in that case the + // No drawables (shutdown/teardown, or the compositor cancelled the frame): the // frame is already invalid and endSubmission() must not be called. shouldEndSubmission = false return } - - // 10. Fetch the predicted device anchor from ARKit using the frameTiming information, and - // apply the anchor to your frame - let presentationInstant = drawable.frameTiming.presentationTime - let presentationTimeCA: TimeInterval = compositorInstantToCATime(presentationInstant) - var deviceAnchor = queryDeviceAnchorIfTrackingRunning(atTimestamp: presentationTimeCA) - - // If predicted-time query misses, retry at "now" to reduce one-frame anchor gaps. - if deviceAnchor == nil { - let nowTimestamp = CACurrentMediaTime() - if let retryAnchor = queryDeviceAnchorIfTrackingRunning(atTimestamp: nowTimestamp) { - deviceAnchor = retryAnchor - } - } - - // Use current anchor if valid, otherwise fall back to last valid anchor. - // This prevents "Presenting a drawable without a device anchor" while maintaining - // stable rendering. We must always present the drawable once we have it. - if let anchor = deviceAnchor { - if missingAnchorFrameCount > 0 { - print("✓ XR device anchor recovered after \(missingAnchorFrameCount) missing frame(s)") - } - missingAnchorFrameCount = 0 - lastValidDeviceAnchor = anchor - drawable.deviceAnchor = anchor - } else if let cachedAnchor = lastValidDeviceAnchor { - missingAnchorFrameCount += 1 - if shouldLogAnchorDiagnostics() { - printXRAnchorDiagnostics(message: "XR device anchor missing; using cached anchor") - } - drawable.deviceAnchor = cachedAnchor - } else { - missingAnchorFrameCount += 1 - if shouldLogAnchorDiagnostics() { - printXRAnchorDiagnostics(message: "XR device anchor missing; no cached anchor available") + if drawables.count != loggedDrawableCount { + loggedDrawableCount = drawables.count + let targets = drawables.map { drawable -> String in + if #available(visionOS 26.0, *) { + return drawable.target == .builtIn ? "built-in" : "capture" + } + return "built-in" } + print("XR: \(drawables.count) drawable(s) per frame (\(targets.joined(separator: ", ")))") } - // Note: If we have no cached anchor either, drawable.deviceAnchor remains nil - // and the render loop will skip rendering but still present (required by compositor) // Re-check loading state right before render pass to catch any mutations that started // after our initial snapshot. This provides a second layer of protection against races. let loadingNow = AssetLoadingGate.shared.isLoadingAny let effectiveLoading = loading || loadingNow - executeXRSystemPass(frame: frame, drawable: drawable, loading: effectiveLoading) + for drawable in drawables { + // 10. Fetch the predicted device anchor from ARKit using the frameTiming information, and + // apply the anchor to your frame + let presentationInstant = drawable.frameTiming.presentationTime + let presentationTimeCA: TimeInterval = compositorInstantToCATime(presentationInstant) + var deviceAnchor = queryDeviceAnchorIfTrackingRunning(atTimestamp: presentationTimeCA) + + // If predicted-time query misses, retry at "now" to reduce one-frame anchor gaps. + if deviceAnchor == nil { + let nowTimestamp = CACurrentMediaTime() + if let retryAnchor = queryDeviceAnchorIfTrackingRunning(atTimestamp: nowTimestamp) { + deviceAnchor = retryAnchor + } + } + + // Use current anchor if valid, otherwise fall back to last valid anchor. + // This prevents "Presenting a drawable without a device anchor" while maintaining + // stable rendering. We must always present the drawable once we have it. + if let anchor = deviceAnchor { + if missingAnchorFrameCount > 0 { + print("✓ XR device anchor recovered after \(missingAnchorFrameCount) missing frame(s)") + } + missingAnchorFrameCount = 0 + lastValidDeviceAnchor = anchor + drawable.deviceAnchor = anchor + } else if let cachedAnchor = lastValidDeviceAnchor { + missingAnchorFrameCount += 1 + if shouldLogAnchorDiagnostics() { + printXRAnchorDiagnostics(message: "XR device anchor missing; using cached anchor") + } + drawable.deviceAnchor = cachedAnchor + } else { + missingAnchorFrameCount += 1 + if shouldLogAnchorDiagnostics() { + printXRAnchorDiagnostics(message: "XR device anchor missing; no cached anchor available") + } + } + // Note: If we have no cached anchor either, drawable.deviceAnchor remains nil + // and the render loop will skip rendering but still present (required by compositor) + + if drawsScene(into: drawable) { + // A capture drawable (a recording or a stream of the view) is drawn from + // its own pose, so it must not advance the frame-to-frame state the + // built-in display relies on: the temporal history and the depth pyramid. + var primary = true + if #available(visionOS 26.0, *) { + primary = drawable.target == .builtIn + } + executeXRSystemPass(frame: frame, drawable: drawable, loading: effectiveLoading, primary: primary) + } else { + presentCleared(drawable) + } + } #if ENGINE_STATS_ENABLED renderer.finalizeXRStatsAndMonitors(frameStartTime: xrFrameStartTime) #else @@ -681,6 +724,51 @@ #endif } + /// Whether the scene is drawn into a drawable. The built-in display always; another + /// target (a capture of the view for recording or streaming) only while its textures are + /// the size the frame's resources are allocated at — every sizeable resource is sized + /// to one viewport, and a second size would reallocate them twice a frame. + private func drawsScene(into drawable: LayerRenderer.Drawable) -> Bool { + if #available(visionOS 26.0, *), drawable.target != .builtIn, + let texture = drawable.colorTextures.first, + renderInfo.viewPort != simd_float2(Float(texture.width), Float(texture.height)) + { + if !loggedCaptureTargetSkip { + loggedCaptureTargetSkip = true + print("XR: a \(texture.width)×\(texture.height) capture drawable differs from the \(Int(renderInfo.viewPort.x))×\(Int(renderInfo.viewPort.y)) frame resources; presenting it cleared") + } + return false + } + return true + } + + /// Presents a drawable with its colour cleared (the immersion mode's alpha) and its depth + /// cleared, without drawing the scene: the compositor wants every drawable it handed out + /// presented. + private func presentCleared(_ drawable: LayerRenderer.Drawable) { + guard let commandBuffer = renderInfo.commandQueue.makeCommandBuffer() else { return } + commandBuffer.label = "XR Cleared Drawable" + for viewIndex in drawable.views.indices + where viewIndex < drawable.colorTextures.count && viewIndex < drawable.depthTextures.count + { + let descriptor = MTLRenderPassDescriptor() + descriptor.colorAttachments[0].texture = drawable.colorTextures[viewIndex] + descriptor.colorAttachments[0].loadAction = .clear + descriptor.colorAttachments[0].storeAction = .store + descriptor.colorAttachments[0].clearColor = MTLClearColor(red: 0, green: 0, blue: 0, alpha: Double(getAlphaForImmersionMode())) + descriptor.depthAttachment.texture = drawable.depthTextures[viewIndex] + descriptor.depthAttachment.loadAction = .clear + descriptor.depthAttachment.storeAction = .store + descriptor.depthAttachment.clearDepth = 0.0 + if viewIndex < drawable.rasterizationRateMaps.count { + descriptor.rasterizationRateMap = drawable.rasterizationRateMaps[viewIndex] + } + commandBuffer.makeRenderCommandEncoder(descriptor: descriptor)?.endEncoding() + } + drawable.encodePresent(commandBuffer: commandBuffer) + commandBuffer.commit() + } + private func updateSpatialInputState() { spatialGestureRecognizer.updateSpatialInputState() } @@ -756,7 +844,68 @@ spatialGestureRecognizer.resetAllSpatialInteractionTracking() } - func executeXRSystemPass(frame _: LayerRenderer.Frame, drawable: LayerRenderer.Drawable, loading: Bool) { + /// The rasterization rate map the compositor made for a view, as the renderer draws + /// through it, or nil when the frame is drawn uniformly: the layer configured without + /// foveation (no maps; also the simulator) or `setRendering(.foveatedRendering(.disabled))`. + /// The map's parameters go into this frame's slot of the ring for the shaders that decode + /// them; the viewport is the view's in the map's screen space. + private func makeFoveationFrame( + drawable: LayerRenderer.Drawable, + viewport: MTLViewport, + viewIndex: Int, + slot: Int + ) -> XRFoveationFrame? { + let rateMaps = drawable.rasterizationRateMaps + let available = xrFoveatedRenderingEnabled && viewIndex < rateMaps.count && viewIndex < stereoEyeCount + if loggedFoveationState != available { + loggedFoveationState = available + if available { + let rateMap = rateMaps[viewIndex] + let screen = rateMap.screenSize + let physical = rateMap.physicalSize(layer: 0) + print("✓ XR foveation: drawing through the compositor's rasterization rate maps, screen \(screen.width)×\(screen.height), textures \(physical.width)×\(physical.height) per eye; view texture map viewport \(viewport.originX),\(viewport.originY) \(viewport.width)×\(viewport.height)") + } else if rateMaps.isEmpty { + print("XR foveation off: the layer provides no rasterization rate maps (isFoveationEnabled is false, or the simulator); drawing uniformly") + } else { + print("XR foveation off by setting (.foveatedRendering(.disabled)); the layer's rate maps are ignored and the picture will be warped") + } + } + guard available else { return nil } + + let rateMap = rateMaps[viewIndex] + let sizeAndAlign = rateMap.parameterDataSizeAndAlign + let align = max(1, sizeAndAlign.align) + let length = max(16, (sizeAndAlign.size + align - 1) / align * align) + if rateMapParameterBuffers[slot][viewIndex].map({ $0.length < length }) ?? true { + let buffer = renderInfo.device.makeBuffer(length: length, options: .storageModeShared) + buffer?.label = "XR Rate Map Parameters eye \(viewIndex) slot \(slot)" + rateMapParameterBuffers[slot][viewIndex] = buffer + } + guard let buffer = rateMapParameterBuffers[slot][viewIndex], + let data = XRRasterizationRateMapData(rateMap: rateMap, buffer: buffer) + else { + Logger.logWarning(message: "XR foveation: could not copy the rate map parameters for eye \(viewIndex); drawing this eye uniformly") + return nil + } + if let texture = drawable.colorTextures.indices.contains(viewIndex) ? drawable.colorTextures[viewIndex] : nil, + Int(data.physicalSize.x) != texture.width || Int(data.physicalSize.y) != texture.height + { + Logger.logWarning(message: "XR foveation: eye \(viewIndex) rate map physical size \(data.physicalSize) differs from its texture \(texture.width)×\(texture.height)") + } + // The eye's viewport in the map's screen space. In the dedicated layout the view + // is the whole texture, so it is the map's screen size at the origin; the view's own + // texture map viewport is logged above for the day another layout needs it. + let screenViewport = MTLViewport( + originX: 0, originY: 0, + width: Double(data.screenSize.x), height: Double(data.screenSize.y), + znear: 0, zfar: 1 + ) + return XRFoveationFrame(rateMap: rateMap, viewport: screenViewport, rateMapData: data) + } + + func executeXRSystemPass(frame _: LayerRenderer.Frame, drawable: LayerRenderer.Drawable, loading: Bool, primary: Bool = true) { + TemporalAntiAliasing.shared.historyUpdatesEnabled = primary + defer { TemporalAntiAliasing.shared.historyUpdatesEnabled = true } // Wait for available command buffer slot to prevent unbounded memory growth let semaphoreWaitStart = CACurrentMediaTime() let firstWaitResult = commandBufferSemaphore.wait(timeout: .now() + .milliseconds(100)) @@ -781,8 +930,12 @@ let renderTotalStart = CACurrentMediaTime() #endif renderInfo.currentInFlightFrameSlot = acquireUniformFrameSlot() + let rateMapSlot = rateMapFrameCounter % rateMapParameterBuffers.count + rateMapFrameCounter += 1 - // Update viewport to match actual drawable size (per-eye texture dimensions) + // Update viewport to match actual drawable size (per-eye texture dimensions). With + // foveation on, that is the rate map's physical size: every texture of the frame is + // allocated at it, while viewports and clip space live in the map's screen size. if let firstColorTexture = drawable.colorTextures.first { let actualViewPort = simd_float2(Float(firstColorTexture.width), Float(firstColorTexture.height)) if renderInfo.viewPort != actualViewPort { @@ -867,6 +1020,15 @@ } renderInfo.currentEye = viewIndex + // The eye's rasterization rate map (foveation), for the passes that draw the + // scene; the final pass into the compositor's textures copies physical pixels + // one to one and keeps passDescriptor without it. + renderInfo.xrFoveation = makeFoveationFrame( + drawable: drawable, + viewport: view.textureMap.viewport, + viewIndex: viewIndex, + slot: rateMapSlot + ) #if ENGINE_STATS_ENABLED let encodeStart = CACurrentMediaTime() @@ -884,8 +1046,11 @@ // Build the HZB depth pyramid once after both eyes are rendered. // In stereo mode the mono pyramid (from the last eye rendered) is used - // for both per-eye VP occlusion tests in executeFrustumCulling. - buildHZBDepthPyramid(commandBuffer) + // for both per-eye VP occlusion tests in executeFrustumCulling. A capture + // drawable's depth is seen from its own pose and does not replace it. + if primary { + buildHZBDepthPyramid(commandBuffer) + } drawable.encodePresent(commandBuffer: commandBuffer) diff --git a/Tests/UntoldEngineRenderTests/GaussianChunkCullTest.swift b/Tests/UntoldEngineRenderTests/GaussianChunkCullTest.swift index 481e6b99c..82bf2b0c3 100644 --- a/Tests/UntoldEngineRenderTests/GaussianChunkCullTest.swift +++ b/Tests/UntoldEngineRenderTests/GaussianChunkCullTest.swift @@ -1095,7 +1095,7 @@ final class GaussianChunkCullTest: BaseRenderSetup { XCTAssertEqual(constants.hzbOcclusionBias, gaussianCullHZBOcclusionBias) XCTAssertEqual(constants.uniformQuotas, 0, "the weighted quotas are the default") XCTAssertEqual(gaussianChunkCullConstants(chunkTable: table, modelMatrix: world.space, viewMatrix: matrix_identity_float4x4, hzbValid: false, uniformQuotas: true).uniformQuotas, 1) - XCTAssertEqual(MemoryLayout.stride, 192) + XCTAssertEqual(MemoryLayout.stride, 208) XCTAssertEqual(MemoryLayout.stride, 16) XCTAssertEqual(MemoryLayout.stride, 48) XCTAssertEqual(MemoryLayout.stride, 48) diff --git a/Tests/UntoldEngineRenderTests/GaussianChunkLevelTest.swift b/Tests/UntoldEngineRenderTests/GaussianChunkLevelTest.swift index f80387fc7..ae60e1d9e 100644 --- a/Tests/UntoldEngineRenderTests/GaussianChunkLevelTest.swift +++ b/Tests/UntoldEngineRenderTests/GaussianChunkLevelTest.swift @@ -1888,7 +1888,7 @@ final class GaussianChunkLevelTest: BaseRenderSetup { XCTAssertEqual(MemoryLayout.stride, 16) XCTAssertEqual(MemoryLayout.offset(of: \.coarseAvailable), 12) XCTAssertEqual(MemoryLayout.stride, 32) - XCTAssertEqual(MemoryLayout.stride, 192) + XCTAssertEqual(MemoryLayout.stride, 208) XCTAssertEqual(MemoryLayout.stride, 48) XCTAssertEqual(MemoryLayout.stride, 64) XCTAssertEqual(MemoryLayout.stride, 48) diff --git a/Tests/UntoldEngineRenderTests/GaussianPagingTest.swift b/Tests/UntoldEngineRenderTests/GaussianPagingTest.swift index 158a67565..d26e253ac 100644 --- a/Tests/UntoldEngineRenderTests/GaussianPagingTest.swift +++ b/Tests/UntoldEngineRenderTests/GaussianPagingTest.swift @@ -1784,9 +1784,10 @@ final class GaussianPagingTest: BaseRenderSetup { XCTAssertEqual(MemoryLayout.offset(of: \.frameIndex), 8) XCTAssertEqual(MemoryLayout.offset(of: \.fadeFrames), 12) XCTAssertEqual(MemoryLayout.offset(of: \.debugMode), 16) - XCTAssertEqual(MemoryLayout.stride, 192) + XCTAssertEqual(MemoryLayout.stride, 208) XCTAssertEqual(MemoryLayout.offset(of: \.paged), 172) XCTAssertEqual(MemoryLayout.offset(of: \.rangeCount), 176) + XCTAssertEqual(MemoryLayout.offset(of: \.hzbRateMapSizes), 192) XCTAssertEqual(gaussianChunkCullResidencyIndex.rawValue, 8) XCTAssertEqual(gaussianChunkCullDemandIndex.rawValue, 9) XCTAssertEqual(gaussianChunkPreprocessResidencyIndex.rawValue, 13) diff --git a/Tests/UntoldEngineRenderTests/GaussianScreenWeightedQuotaTest.swift b/Tests/UntoldEngineRenderTests/GaussianScreenWeightedQuotaTest.swift index 953c6758c..3e61dce6f 100644 --- a/Tests/UntoldEngineRenderTests/GaussianScreenWeightedQuotaTest.swift +++ b/Tests/UntoldEngineRenderTests/GaussianScreenWeightedQuotaTest.swift @@ -320,7 +320,8 @@ final class GaussianScreenWeightedQuotaTest: BaseRenderSetup { /// The shared layouts the kernels and the readbacks depend on. func testLayoutsArePinned() { - XCTAssertEqual(MemoryLayout.stride, 192) + XCTAssertEqual(MemoryLayout.stride, 208) + XCTAssertEqual(MemoryLayout.offset(of: \.hzbRateMapSizes), 192) XCTAssertEqual(MemoryLayout.stride, 16) XCTAssertEqual(MemoryLayout.stride, 48) XCTAssertEqual(MemoryLayout.stride, 48) diff --git a/Tests/UntoldEngineRenderTests/RenderGraphBuilderTest.swift b/Tests/UntoldEngineRenderTests/RenderGraphBuilderTest.swift index 315dcf741..382ba9cd7 100644 --- a/Tests/UntoldEngineRenderTests/RenderGraphBuilderTest.swift +++ b/Tests/UntoldEngineRenderTests/RenderGraphBuilderTest.swift @@ -1530,7 +1530,7 @@ final class RenderGraphBuilderTest: BaseRenderSetup { for environmentEnabled in [false, true] { renderEnvironment = environmentEnabled - for mode in [AntiAliasingMode.none, .fxaa, .smaa, .msaa] { + for mode in [AntiAliasingMode.none, .fxaa, .smaa, .msaa, .taa, .msaaTaa] { antiAliasingMode = mode try captureGraphPassIDs() } diff --git a/Tests/UntoldEngineRenderTests/TemporalAntiAliasingTests.swift b/Tests/UntoldEngineRenderTests/TemporalAntiAliasingTests.swift new file mode 100644 index 000000000..b7cf0a7fc --- /dev/null +++ b/Tests/UntoldEngineRenderTests/TemporalAntiAliasingTests.swift @@ -0,0 +1,330 @@ +// +// TemporalAntiAliasingTests.swift +// UntoldEngine +// +// The temporal resolve (TemporalAntiAliasing.swift, TAAShader.metal): the jitter sequence, +// the jitter going into and out of the projection, and the resolve of a still scene settling +// to a stable image that stays close to the un-anti-aliased frame. +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import simd +@testable import UntoldEngine +import XCTest + +@MainActor +final class TemporalAntiAliasingTests: BaseRenderSetup { + private var camera: EntityID = 0 + private var savedMode: AntiAliasingMode = .fxaa + + override func setUp() async throws { + try await super.setUp() + savedMode = antiAliasingMode + } + + override func tearDown() async throws { + antiAliasingMode = savedMode + TemporalAntiAliasing.shared.applyJitter() + TemporalAntiAliasing.shared.invalidateHistory() + destroyAllEntities() + try await super.tearDown() + } + + override func initializeAssets() { + camera = createEntity() + createGameCamera(entityId: camera) + CameraSystem.shared.activeCamera = camera + cameraLookAt(entityId: camera, eye: simd_float3(0, 1, 4), target: .zero, up: simd_float3(0, 1, 0)) + let box = createEntity() + setEntityMeshDirect(entityId: box, meshes: BasicPrimitives.createCube(extent: 1.0), assetName: "Box") + translateTo(entityId: box, position: simd_float3(0, 0.5, 0)) + let slab = createEntity() + setEntityMeshDirect(entityId: slab, meshes: BasicPrimitives.createCube(extent: 1.0), assetName: "Slab") + scaleTo(entityId: slab, scale: simd_float3(4, 0.05, 0.05)) + translateTo(entityId: slab, position: simd_float3(0, 1.6, -1)) + } + + // MARK: - Helpers + + private func drawFrame() { + renderer.draw(in: renderer.metalView) + renderInfo.lastCommandBuffer?.waitUntilCompleted() + } + + /// The anti-aliasing output as floats, RGBA per pixel. + private func readAntiAliasingOutput() throws -> [Float] { + let texture = try XCTUnwrap(textureResources.antiAliasingTexture) + XCTAssertEqual(texture.pixelFormat, .rgba16Float) + var halves = [UInt16](repeating: 0, count: texture.width * texture.height * 4) + halves.withUnsafeMutableBytes { bytes in + texture.getBytes(bytes.baseAddress!, bytesPerRow: texture.width * 8, from: MTLRegionMake2D(0, 0, texture.width, texture.height), mipmapLevel: 0) + } + return halves.map { Float(Float16(bitPattern: $0)) } + } + + private func meanAbsoluteDifference(_ a: [Float], _ b: [Float]) -> Float { + var total: Float = 0 + for i in 0 ..< min(a.count, b.count) { + total += abs(a[i] - b[i]) + } + return total / Float(max(1, min(a.count, b.count))) + } + + // MARK: - Tests + + func testJitterSequenceIsSubPixelAndCentred() { + XCTAssertEqual(taaJitterSequence.count, 8) + var mean = simd_float2.zero + for offset in taaJitterSequence { + XCTAssertLessThanOrEqual(abs(offset.x), 0.5) + XCTAssertLessThanOrEqual(abs(offset.y), 0.5) + mean += offset + } + mean /= Float(taaJitterSequence.count) + XCTAssertLessThan(abs(mean.x), 0.1, "the offsets average near the pixel centre") + XCTAssertLessThan(abs(mean.y), 0.1) + XCTAssertEqual(Set(taaJitterSequence.map { "\($0.x),\($0.y)" }).count, 8, "eight distinct offsets") + } + + func testJitterGoesIntoAndOutOfTheProjection() { + antiAliasingMode = .fxaa + TemporalAntiAliasing.shared.applyJitter() + let base = renderInfo.perspectiveSpace + + antiAliasingMode = .taa + TemporalAntiAliasing.shared.applyJitter() + let jittered = renderInfo.perspectiveSpace + let pixels = TemporalAntiAliasing.shared.currentJitterPixels + XCTAssertNotEqual(pixels, .zero) + XCTAssertEqual(jittered.columns.2.x - base.columns.2.x, 2 * pixels.x / renderInfo.viewPort.x, accuracy: 1e-6) + XCTAssertEqual(jittered.columns.2.y - base.columns.2.y, 2 * pixels.y / renderInfo.viewPort.y, accuracy: 1e-6) + XCTAssertEqual(jittered.columns.0, base.columns.0) + XCTAssertEqual(jittered.columns.1, base.columns.1) + XCTAssertEqual(jittered.columns.3, base.columns.3) + XCTAssertEqual(TemporalAntiAliasing.shared.unjitteredProjection, base, "the unjittered projection is the base one") + + TemporalAntiAliasing.shared.applyJitter() + XCTAssertNotEqual(renderInfo.perspectiveSpace.columns.2.x, jittered.columns.2.x, "the next frame takes the next offset, not a sum") + + antiAliasingMode = .fxaa + TemporalAntiAliasing.shared.applyJitter() + XCTAssertEqual(renderInfo.perspectiveSpace, base, "leaving the temporal mode leaves the projection as it was") + } + + /// A rate map over the window whose dense column sits at `denseColumn` of four; the + /// three other columns are drawn at a quarter of the rate. Every column layout has the + /// same physical size, as the compositor's gaze-following maps do. + private func makeRateMap(denseColumn: Int) throws -> MTLRasterizationRateMap { + let device = try XCTUnwrap(renderInfo.device) + guard device.supportsRasterizationRateMap(layerCount: 1) else { + throw XCTSkip("this GPU has no rasterization rate maps") + } + let descriptor = MTLRasterizationRateMapDescriptor(screenSize: MTLSize(width: windowWidth, height: windowHeight, depth: 1)) + // Dense in one column and one row (a headset's map is dense around the gaze and coarse + // both ways in the periphery), a quarter of the rate elsewhere. + let layer = MTLRasterizationRateLayerDescriptor(sampleCount: MTLSize(width: 4, height: 4, depth: 1)) + for index in 0 ..< 4 { + layer.horizontal[index] = index == denseColumn ? 1.0 : 0.25 + layer.vertical[index] = index == denseColumn ? 1.0 : 0.25 + } + descriptor.setLayer(layer, at: 0) + return try XCTUnwrap(device.makeRasterizationRateMap(descriptor: descriptor)) + } + + private func makeFoveationFrame(_ rateMap: MTLRasterizationRateMap) throws -> XRFoveationFrame { + let device = try XCTUnwrap(renderInfo.device) + let buffer = try XCTUnwrap(device.makeBuffer(length: rateMap.parameterDataSizeAndAlign.size, options: .storageModeShared)) + let data = try XCTUnwrap(XRRasterizationRateMapData(rateMap: rateMap, buffer: buffer)) + let viewport = MTLViewport(originX: 0, originY: 0, width: Double(windowWidth), height: Double(windowHeight), znear: 0, zfar: 1) + return XRFoveationFrame(rateMap: rateMap, viewport: viewport, rateMapData: data) + } + + /// Reads a physical-size texture of the look format as floats. + private func readFloats(_ texture: MTLTexture) -> [Float] { + var halves = [UInt16](repeating: 0, count: texture.width * texture.height * 4) + halves.withUnsafeMutableBytes { bytes in + texture.getBytes(bytes.baseAddress!, bytesPerRow: texture.width * 8, from: MTLRegionMake2D(0, 0, texture.width, texture.height), mipmapLevel: 0) + } + return halves.map { Float(Float16(bitPattern: $0)) } + } + + /// The headset's situation: a still scene drawn through a rate map whose dense zone moves + /// every frame, as a gaze-following map does. The history of each frame is laid out in the + /// previous map's physical space; reprojected through both maps it must land where this + /// frame's map puts the same point, or the resolve shows the scene with a displaced copy. + func testResolveStaysStableWhileTheRateMapsDenseZoneMoves() throws { + let maps = try [makeRateMap(denseColumn: 0), makeRateMap(denseColumn: 3)] + let physical = maps[0].physicalSize(layer: 0) + guard physical.width == maps[1].physicalSize(layer: 0).width, physical.height == maps[1].physicalSize(layer: 0).height else { + throw XCTSkip("the two layouts got different physical sizes on this GPU") + } + XCTAssertLessThan(physical.width, windowWidth, "three quarter-rate columns narrow the texture") + let frames = try maps.map { try makeFoveationFrame($0) } + + // Draw into textures of the physical size, through the maps, like a foveated eye. + let savedViewPort = renderInfo.viewPort + defer { + renderInfo.xrFoveation = nil + renderInfo.viewPort = savedViewPort + renderer.initSizeableResources() + TemporalAntiAliasing.shared.invalidateHistory() + } + renderInfo.viewPort = simd_float2(Float(physical.width), Float(physical.height)) + renderer.initSizeableResources() + + // The single-sample frame in the first map's layout. + antiAliasingMode = .none + renderInfo.xrFoveation = frames[0] + for _ in 0 ..< 2 { + drawFrame() + } + let single = try readFloats(XCTUnwrap(textureResources.lookTexture)) + XCTAssertGreaterThan(single.max() ?? 0, 0.01, "the scene draws something through the map") + + // The same frame in the second layout differs a lot from the first: the test can tell + // the layouts apart, so a history mapped through the wrong one would be caught. + renderInfo.xrFoveation = frames[1] + drawFrame() + let singleOther = try readFloats(XCTUnwrap(textureResources.lookTexture)) + let layoutDifference = meanAbsoluteDifference(single, singleOther) + XCTAssertGreaterThan(layoutDifference, 0.002, "the two layouts place the scene differently in the texture") + + // Temporal, with the dense zone jumping every frame, ending on the first layout. The + // jitter is off here so the frames are deterministic: what this checks is the history + // mapped through the maps, not the jitter's edge averaging. + antiAliasingMode = .taa + TemporalAntiAliasing.shared.invalidateHistory() + let savedJitterScale = TAAParams.shared.jitterScale + TAAParams.shared.jitterScale = 0 + defer { TAAParams.shared.jitterScale = savedJitterScale } + for frame in 0 ..< 12 { + let layout = (frame + 1) % 2 + renderInfo.xrFoveation = frames[layout] + drawFrame() + if frame >= 6 { + // Lit pixels that lost most of their colour against the single-sample frame of + // this layout: the thin bar's edge rows, where a history coarser than the frame + // on one axis overrode the frame until the rate ratio was taken per axis. A mean + // over the frame would not show it. + let frameResolved = try readFloats(XCTUnwrap(textureResources.antiAliasingTexture)) + let reference = layout == 0 ? single : singleOther + var lost = 0 + for i in stride(from: 0, to: min(frameResolved.count, reference.count), by: 4) where reference[i] + reference[i + 1] + reference[i + 2] > 0.6 { + let sum = frameResolved[i] + frameResolved[i + 1] + frameResolved[i + 2] + if sum < 0.4 * (reference[i] + reference[i + 1] + reference[i + 2]) { + lost += 1 + } + } + XCTAssertEqual(lost, 0, "frame \(frame) in layout \(layout): no lit pixel loses most of its colour to a history drawn through the other map") + } + } + let resolved = try readFloats(XCTUnwrap(textureResources.antiAliasingTexture)) + + // The floor: the resolve of the first layout held still differs from the single-sample + // frame by the jitter's averaging and the history filter alone. + TemporalAntiAliasing.shared.invalidateHistory() + renderInfo.xrFoveation = frames[0] + for _ in 0 ..< 12 { + drawFrame() + } + let still = try readFloats(XCTUnwrap(textureResources.antiAliasingTexture)) + let floor = meanAbsoluteDifference(still, single) + + XCTAssertFalse(resolved.contains { $0.isNaN || $0.isInfinite }) + let resolveDifference = meanAbsoluteDifference(resolved, single) + print("TAA moving-map test: resolve differs from the single-sample frame by \(resolveDifference), held still by \(floor), the other layout by \(layoutDifference)") + XCTAssertLessThan(resolveDifference, floor + 0.1 * layoutDifference, "the history reprojected through the moving maps lands where this frame's map puts the scene: resolve differs by \(resolveDifference), held still by \(floor), the other layout by \(layoutDifference)") + } + + /// A camera that moves a little every frame, as a head does: the history is reprojected + /// through the depth buffer and the two view-projections, and the resolve at the final + /// pose must match the single-sample frame at that pose rather than drag the earlier poses + /// along as a shadow. + func testResolveFollowsAMovingCameraWithoutAShadow() throws { + func pose(_ frame: Int) -> simd_float3 { + simd_float3(0.03 * Float(frame), 1 + 0.01 * Float(frame), 4 - 0.02 * Float(frame)) + } + let frameCount = 12 + + antiAliasingMode = .none + cameraLookAt(entityId: camera, eye: pose(0), target: .zero, up: simd_float3(0, 1, 0)) + drawFrame() + drawFrame() + let first = try readFloats(XCTUnwrap(textureResources.lookTexture)) + cameraLookAt(entityId: camera, eye: pose(frameCount - 1), target: .zero, up: simd_float3(0, 1, 0)) + drawFrame() + drawFrame() + let last = try readFloats(XCTUnwrap(textureResources.lookTexture)) + let motion = meanAbsoluteDifference(first, last) + XCTAssertGreaterThan(motion, 0.002, "the camera's path moves the scene in the frame") + + for mode in [AntiAliasingMode.taa, .msaaTaa] { + antiAliasingMode = mode + TemporalAntiAliasing.shared.invalidateHistory() + for frame in 0 ..< frameCount { + cameraLookAt(entityId: camera, eye: pose(frame), target: .zero, up: simd_float3(0, 1, 0)) + drawFrame() + } + let moved = try readAntiAliasingOutput() + TemporalAntiAliasing.shared.invalidateHistory() + for _ in 0 ..< frameCount { + drawFrame() + } + let held = try readAntiAliasingOutput() + print("TAA moving-camera \(mode): resolve differs from the final pose's frame by \(meanAbsoluteDifference(moved, last)), held still by \(meanAbsoluteDifference(held, last)), the first pose by \(motion)") + } + + antiAliasingMode = .taa + TemporalAntiAliasing.shared.invalidateHistory() + for frame in 0 ..< frameCount { + cameraLookAt(entityId: camera, eye: pose(frame), target: .zero, up: simd_float3(0, 1, 0)) + drawFrame() + } + let resolved = try readAntiAliasingOutput() + let shadow = meanAbsoluteDifference(resolved, last) + + // The floor: the resolve held still at the final pose. + TemporalAntiAliasing.shared.invalidateHistory() + for _ in 0 ..< frameCount { + drawFrame() + } + let stillResolved = try readAntiAliasingOutput() + let floor = meanAbsoluteDifference(stillResolved, last) + print("TAA moving-camera test: resolve differs from the final pose's frame by \(shadow), held still by \(floor), the first pose by \(motion)") + XCTAssertLessThan(shadow, floor + 0.1 * motion, "the resolve follows the camera: it differs from the final pose's frame by \(shadow), held still by \(floor), the first pose differs by \(motion)") + } + + func testResolveSettlesOnAStillSceneCloseToTheSingleSampleFrame() throws { + antiAliasingMode = .none + for _ in 0 ..< 2 { + drawFrame() + } + // The look output is what the output pass shows without a post-look pass. + let lookTexture = try XCTUnwrap(textureResources.lookTexture) + var reference = [UInt16](repeating: 0, count: lookTexture.width * lookTexture.height * 4) + reference.withUnsafeMutableBytes { bytes in + lookTexture.getBytes(bytes.baseAddress!, bytesPerRow: lookTexture.width * 8, from: MTLRegionMake2D(0, 0, lookTexture.width, lookTexture.height), mipmapLevel: 0) + } + let single = reference.map { Float(Float16(bitPattern: $0)) } + XCTAssertGreaterThan(single.max() ?? 0, 0.01, "the scene draws something") + + antiAliasingMode = .taa + TemporalAntiAliasing.shared.invalidateHistory() + for _ in 0 ..< 10 { + drawFrame() + } + let settled = try readAntiAliasingOutput() + drawFrame() + let next = try readAntiAliasingOutput() + + XCTAssertFalse(settled.contains { $0.isNaN || $0.isInfinite }, "no NaN or infinity in the resolve") + print("TAA still test: frame to frame \(meanAbsoluteDifference(settled, next)), resolve vs single-sample \(meanAbsoluteDifference(settled, single))") + XCTAssertLessThan(meanAbsoluteDifference(settled, next), 0.004, "a still scene no longer changes between frames") + XCTAssertLessThan(meanAbsoluteDifference(settled, single), 0.02, "the resolve stays close to the single-sample frame: no drift, no smear") + XCTAssertGreaterThan(meanAbsoluteDifference(settled, single), 0, "and is not the single-sample frame") + } +} diff --git a/Tests/UntoldEngineRenderTests/XRFoveationTests.swift b/Tests/UntoldEngineRenderTests/XRFoveationTests.swift new file mode 100644 index 000000000..208f709be --- /dev/null +++ b/Tests/UntoldEngineRenderTests/XRFoveationTests.swift @@ -0,0 +1,204 @@ +// +// XRFoveationTests.swift +// UntoldEngine +// +// Drawing through a rasterization rate map (visionOS foveation, XRFoveation.swift): the map's +// parameter data the shaders decode, which pass descriptors carry the map and which do not, +// and the occlusion cull sampling a pyramid drawn through a map where the map put the rect, +// not where the rect's screen UV falls in the texture. +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import CShaderTypes +import Metal +import simd +@testable import UntoldEngine +import XCTest + +@MainActor +final class XRFoveationTests: BaseRenderSetup { + private let screen = 256 + + override func initializeAssets() {} + + override func tearDown() async throws { + renderInfo.xrFoveation = nil + applyXRFoveationToSceneRenderPassDescriptors() + try await super.tearDown() + } + + // MARK: - Helpers + + /// A map over a 256-pixel-square screen whose right half is drawn at a quarter of the rate + /// of its left half, so the physical texture is narrower than the screen and a screen + /// position in the right half lands at a different texel than its UV says. + private func makeRateMap() throws -> MTLRasterizationRateMap { + let device = try XCTUnwrap(renderInfo.device) + guard device.supportsRasterizationRateMap(layerCount: 1) else { + throw XCTSkip("this GPU has no rasterization rate maps") + } + let descriptor = MTLRasterizationRateMapDescriptor(screenSize: MTLSize(width: screen, height: screen, depth: 1)) + let layer = MTLRasterizationRateLayerDescriptor(sampleCount: MTLSize(width: 2, height: 1, depth: 1)) + layer.horizontal[0] = 1.0 + layer.horizontal[1] = 0.25 + layer.vertical[0] = 1.0 + descriptor.setLayer(layer, at: 0) + descriptor.label = "XRFoveationTests" + return try XCTUnwrap(device.makeRasterizationRateMap(descriptor: descriptor)) + } + + private func makeRateMapData(_ rateMap: MTLRasterizationRateMap) throws -> XRRasterizationRateMapData { + let device = try XCTUnwrap(renderInfo.device) + let buffer = try XCTUnwrap(device.makeBuffer(length: rateMap.parameterDataSizeAndAlign.size, options: .storageModeShared)) + return try XCTUnwrap(XRRasterizationRateMapData(rateMap: rateMap, buffer: buffer)) + } + + // MARK: - Tests + + func testRateMapDataCarriesTheSizesAndParameters() throws { + let rateMap = try makeRateMap() + let data = try makeRateMapData(rateMap) + + let physical = rateMap.physicalSize(layer: 0) + XCTAssertEqual(data.screenSize, simd_float2(Float(screen), Float(screen))) + XCTAssertEqual(data.physicalSize, simd_float2(Float(physical.width), Float(physical.height))) + XCTAssertLessThan(physical.width, screen, "a quarter-rate half narrows the physical texture") + XCTAssertEqual(data.sizes, simd_float4(Float(screen), Float(screen), Float(physical.width), Float(physical.height))) + XCTAssertEqual(rateMapSizes(nil), .zero, "the word the shaders read as 'no map'") + + let bytes = UnsafeRawBufferPointer(start: data.buffer.contents(), count: rateMap.parameterDataSizeAndAlign.size) + XCTAssertTrue(bytes.contains { $0 != 0 }, "the map's parameters were copied") + + let short = try XCTUnwrap(renderInfo.device.makeBuffer(length: 4, options: .storageModeShared)) + XCTAssertNil(XRRasterizationRateMapData(rateMap: rateMap, buffer: short), "a buffer too short for the parameters is refused") + } + + func testScenePassesCarryTheRateMapAndScreenPassesDoNot() throws { + let rateMap = try makeRateMap() + let data = try makeRateMapData(rateMap) + let viewport = MTLViewport(originX: 0, originY: 0, width: Double(screen), height: Double(screen), znear: 0, zfar: 1) + + renderInfo.xrFoveation = XRFoveationFrame(rateMap: rateMap, viewport: viewport, rateMapData: data) + applyXRFoveationToSceneRenderPassDescriptors() + + XCTAssertTrue(renderInfo.offscreenRenderPassDescriptor.rasterizationRateMap === rateMap, "the G-buffer + light pass draws through the map") + XCTAssertTrue(renderInfo.deferredRenderPassDescriptor.rasterizationRateMap === rateMap, "transparency, wireframe and debug draw through the map") + XCTAssertTrue(renderInfo.environmentRenderPassDescriptor.rasterizationRateMap === rateMap, "the background draws through the map") + XCTAssertTrue(renderInfo.gaussianRenderPassDescriptor.rasterizationRateMap === rateMap, "the splats draw through the map") + XCTAssertNil(renderInfo.sceneCompositeRenderPassDescriptor.rasterizationRateMap, "the composite copies physical pixels one to one") + XCTAssertNil(renderInfo.ssaoRenderPassDescriptor.rasterizationRateMap, "SSAO reads the physical depth one to one") + XCTAssertNil(renderInfo.postProcessRenderPassDescriptor?.rasterizationRateMap, "post-processing copies physical pixels one to one") + // A render target size is read in screen pixels under a map: the physical size would + // clip the scene to the top-left corner of the screen, so the size is left to the + // attachments while the map is attached. + XCTAssertEqual(renderInfo.offscreenRenderPassDescriptor.renderTargetWidth, 0) + XCTAssertEqual(renderInfo.deferredRenderPassDescriptor.renderTargetHeight, 0) + XCTAssertEqual(renderInfo.sceneCompositeRenderPassDescriptor.renderTargetWidth, Int(renderInfo.viewPort.x), "a screen pass keeps its texture-size target") + + renderInfo.xrFoveation = nil + applyXRFoveationToSceneRenderPassDescriptors() + XCTAssertNil(renderInfo.offscreenRenderPassDescriptor.rasterizationRateMap, "a uniform frame detaches the map") + XCTAssertEqual(renderInfo.offscreenRenderPassDescriptor.renderTargetWidth, Int(renderInfo.viewPort.x), "and puts the texture-size target back") + XCTAssertEqual(renderInfo.deferredRenderPassDescriptor.renderTargetHeight, Int(renderInfo.viewPort.y)) + XCTAssertNil(renderInfo.deferredRenderPassDescriptor.rasterizationRateMap) + XCTAssertNil(renderInfo.environmentRenderPassDescriptor.rasterizationRateMap) + XCTAssertNil(renderInfo.gaussianRenderPassDescriptor.rasterizationRateMap) + } + + /// A box in the right half of the screen, in front of a pyramid that holds an occluder only + /// where the map put that half. Read where the box's screen UV falls in the texture, the + /// pyramid shows background and the box is kept; moved through the map, the rect lies on + /// the occluder and the box is dropped. + func testOcclusionCullSamplesThePyramidWhereTheRateMapPutTheRect() throws { + let rateMap = try makeRateMap() + let data = try makeRateMapData(rateMap) + let device = try XCTUnwrap(renderInfo.device) + let physical = rateMap.physicalSize(layer: 0) + + // The box: NDC x in 0.5…0.9 — screen pixels 192…243 of 256 — a sliver of y, at depth 0.5. + let box = VisibleEntity(center: simd_float4(0.7, 0, 0.5, 1), halfExtent: simd_float4(0.2, 0.05, 0, 0), index: 1, version: 1) + // Where the rect's left edge lands in the texture by Metal's own mapping, against where + // its screen UV (0.75) would be read without the map. + let mappedMinX = rateMap.physicalCoordinates(screenCoordinates: MTLCoordinate2D(x: 192, y: 128), layer: 0).x + let unmappedMinX = 0.75 * Float(physical.width) + XCTAssertGreaterThan(mappedMinX, unmappedMinX + 2, "the quarter-rate half moves the rect to the right in the texture") + + // One mip: an occluder (0.9, near in reverse-Z) from just before the mapped rect to the + // right edge, background (0) everywhere else. + let occluderStart = max(0, Int(mappedMinX.rounded(.down)) - 1) + let pyramidDescriptor = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: .r32Float, width: physical.width, height: physical.height, mipmapped: false) + pyramidDescriptor.usage = [.shaderRead] + pyramidDescriptor.storageMode = .managed + let pyramid = try XCTUnwrap(device.makeTexture(descriptor: pyramidDescriptor)) + var row = [Float](repeating: 0, count: physical.width) + for x in occluderStart ..< physical.width { + row[x] = 0.9 + } + var texels = [Float]() + texels.reserveCapacity(physical.width * physical.height) + for _ in 0 ..< physical.height { + texels.append(contentsOf: row) + } + texels.withUnsafeBytes { bytes in + pyramid.replace( + region: MTLRegionMake2D(0, 0, physical.width, physical.height), + mipmapLevel: 0, + withBytes: bytes.baseAddress!, + bytesPerRow: physical.width * MemoryLayout.stride + ) + } + + // The frame state the cull reads, restored afterwards. + let savedViewPort = renderInfo.viewPort + let savedValid = renderInfo.hzbIsValid + let savedFrame = renderInfo.hzbFrame + let savedMipCount = renderInfo.hzbMipCount + let savedReverseZ = renderInfo.reverseZEnabled + let savedPyramid = textureResources.hzbDepthPyramid + defer { + renderInfo.viewPort = savedViewPort + renderInfo.hzbIsValid = savedValid + renderInfo.hzbFrame = savedFrame + renderInfo.hzbMipCount = savedMipCount + renderInfo.reverseZEnabled = savedReverseZ + textureResources.hzbDepthPyramid = savedPyramid + } + renderInfo.viewPort = simd_float2(Float(physical.width), Float(physical.height)) + renderInfo.hzbIsValid = true + renderInfo.hzbMipCount = 1 + renderInfo.reverseZEnabled = true + textureResources.hzbDepthPyramid = pyramid + + func survivors(rateMapData: XRRasterizationRateMapData?) throws -> Int { + renderInfo.hzbFrame = HZBPyramidFrame(viewProjection: matrix_identity_float4x4, cameraPosition: .zero, rateMapData: rateMapData) + var candidate = box + var candidateCount: UInt32 = 1 + let inputVisibility = try XCTUnwrap(device.makeBuffer(bytes: &candidate, length: MemoryLayout.stride, options: .storageModeShared)) + let inputCount = try XCTUnwrap(device.makeBuffer(bytes: &candidateCount, length: MemoryLayout.stride, options: .storageModeShared)) + let outputVisibility = try XCTUnwrap(device.makeBuffer(length: MemoryLayout.stride, options: .storageModeShared)) + let outputCount = try XCTUnwrap(device.makeBuffer(length: MemoryLayout.stride, options: .storageModeShared)) + + let commandBuffer = try XCTUnwrap(renderInfo.commandQueue.makeCommandBuffer()) + let ran = executeHZBOcclusionCulling( + commandBuffer, + cameraPosition: .zero, + dispatchCount: 1, + inputVisibilityBuffer: inputVisibility, + inputVisibleCountBuffer: inputCount, + outputVisibilityBuffer: outputVisibility, + outputVisibleCountBuffer: outputCount + ) + XCTAssertTrue(ran, "the pyramid is valid and the camera has not moved") + commandBuffer.commit() + commandBuffer.waitUntilCompleted() + return Int(outputCount.contents().load(as: UInt32.self)) + } + + XCTAssertEqual(try survivors(rateMapData: nil), 1, "read at its screen UV, the rect sees background and the box is kept") + XCTAssertEqual(try survivors(rateMapData: data), 0, "moved through the map, the rect lies on the occluder and the box is dropped") + } +} diff --git a/docs/API/UsingEngineAPI.md b/docs/API/UsingEngineAPI.md index 7889c439f..a57fe68bb 100644 --- a/docs/API/UsingEngineAPI.md +++ b/docs/API/UsingEngineAPI.md @@ -18,7 +18,11 @@ Existing direct APIs such as `LODConfig.shared`, `SSAOParams.shared`, `antiAlias setRendering(.antiAliasing(.fxaa)) setRendering(.antiAliasing(.smaa)) setRendering(.antiAliasing(.msaa)) +setRendering(.antiAliasing(.taa)) // temporal: jittered frames blended with their reprojected history +setRendering(.antiAliasing(.msaaTaa)) // 4× MSAA on the G-buffer plus the temporal resolve setRendering(.antiAliasing(.none)) +TAAParams.shared.historyWeight = 0.9 // share of the history in each resolved pixel (0…0.98) +TAAParams.shared.clipGamma = 1.25 // clip box = 3×3 mean ± γσ: lower rejects ghosts sooner, higher keeps pixel-thin features setRendering(.debugView(.lit)) setRendering(.debugView(.depth)) @@ -82,6 +86,17 @@ setRendering(.occlusionCullingMaxCameraStep(0.5)) getOcclusionCullingMaxCameraStep() ``` +On visionOS, a layer configured with `isFoveationEnabled` hands the engine a rasterization +rate map per eye, and the engine draws through it by default: dense where the eyes look, +coarse in the periphery, and the only configuration in which the compositor accepts a +`maxRenderQuality` above the platform default. Disabling it in such a layer is a diagnostic +(the frame is drawn uniformly into textures the compositor still unwarps): + +```swift +setRendering(.foveatedRendering(.disabled)) +getFoveatedRenderingEnabled() +``` + ## PostFX Use `setPostFX` for individual post-processing and SSAO settings: diff --git a/docs/Architecture/renderingSystem.md b/docs/Architecture/renderingSystem.md index 9aac362fc..0bd58eaf7 100644 --- a/docs/Architecture/renderingSystem.md +++ b/docs/Architecture/renderingSystem.md @@ -332,10 +332,13 @@ After the look pass, the graph inserts an anti-aliasing pass whose topology depe |---|---|---| | `.fxaa` | `fxaa` | `look → fxaa → outputTransform` | | `.smaa` | `smaaEdges`, `smaaBlendWeights`, `smaaNeighborhood` | `look → smaaEdges → smaaBlendWeights → smaaNeighborhood → outputTransform` | -| `.none` | *(none)* | `look → outputTransform` | +| `.taa`, `.msaaTaa` | `taa` | `look → taa → outputTransform` (`.msaaTaa` also draws the G-buffer at 4 samples, like `.msaa`) | +| `.none`, `.msaa` | *(none)* | `look → outputTransform` | **FXAA** is a single-pass screen-space filter that attenuates aliased edges using local luma contrast. +**TAA** (temporal anti-aliasing, `TemporalAntiAliasing.swift`, `TAAShader.metal`) jitters the projection by a sub-pixel Halton offset before each frame's graph is built (`applyJitter`, in the projection's third column, so the offset is a constant NDC shift; the previous frame's offset is taken out first) and, after the look pass, blends the frame with the eye's history: each pixel is reprojected through the depth buffer and the unjittered view-projections of this frame and the history's, the history colour is clipped to the YCoCg box of the pixel's 3×3 neighbourhood (so what moved or was uncovered does not ghost), and the two are mixed with `TAAParams.shared.historyWeight` (0.9). The resolve writes the anti-aliasing texture and the eye's next history texture in one pass (two colour targets); the history is per eye, ping-ponged, and invalidated by a resize or a mode change. Under a foveated XR frame the textures are in physical space and each frame's rate map differs, so the reprojection goes physical → screen → world → previous screen → previous physical through both maps, and the jitter is measured in screen pixels. Camera motion is handled; objects moving on their own rely on the clip, since there are no motion vectors yet. + **SMAA** (Subpixel Morphological Anti-Aliasing) is a three-pass chain: 1. **Edge detection** (`smaaEdges`) — identifies aliased edges from the look texture using luma and chroma gradients. Also detects diagonal patterns. 2. **Blend-weight calculation** (`smaaBlendWeights`) — computes per-pixel blend weights from the SMAA area and search look-up textures, accounting for corner patterns. diff --git a/docs/Architecture/xrRenderingSystem.md b/docs/Architecture/xrRenderingSystem.md index b4c3059ae..19ffdc1e0 100644 --- a/docs/Architecture/xrRenderingSystem.md +++ b/docs/Architecture/xrRenderingSystem.md @@ -100,6 +100,8 @@ defer { frame.endSubmission() } The `defer` is important: `endSubmission()` **must** be called even if rendering fails partway through. If it isn't called, the compositor stalls. The `defer` guarantees this regardless of how the function exits. +**Drawables.** On visionOS 26 the frame is queried with `frame.queryDrawables()`, which returns one drawable per target: the built-in display, plus a capture target while the view is being recorded or streamed. A layer configured with foveation or a render quality aborts the app ("BUG IN CLIENT: called cp_frame_query_drawable() which does not support setting render quality or fovea") if the older single-drawable `queryDrawable()` is used, which the engine still calls on visionOS 2. An empty array means the compositor cancelled the frame: it is left without `endSubmission()`. Every drawable is presented; the scene is drawn into the built-in one and into a capture drawable only while its textures are the size the frame's resources are allocated at — otherwise it is presented cleared, since every sizeable resource is sized to one viewport and a second size would reallocate them twice a frame. + ### 2e. Device Anchor Acquisition ```swift @@ -167,6 +169,15 @@ The compositor provides the exact asymmetric projection for each eye. This accou - `renderInfo.currentEye = viewIndex` — tells uniform uploads which eye's matrices to use - The base pass mode: `.mixed` immersion omits the base pass (camera passthrough is the background), `.full` immersion renders the skybox +**Foveation (rasterization rate maps).** With `isFoveationEnabled` in the app's `CompositorLayerConfiguration`, the compositor hands each eye a `MTLRasterizationRateMap` and colour/depth textures of the map's *physical* size, smaller than the *screen* the eye sees; the GPU draws through the map (dense where the eyes look, coarse in the periphery) and the compositor unwarps the texture for the display. It is also the only configuration in which the compositor accepts a `maxRenderQuality` above the platform default. The engine supports it as follows (`XRFoveation.swift`, on by default, `setRendering(.foveatedRendering(.disabled))` to ignore the maps for diagnosis): + +- `executeXRSystemPass` makes an `XRFoveationFrame` per eye (`renderInfo.xrFoveation`): the map, the eye's viewport in the map's screen space (`view.textureMap.viewport`), and the map's parameter data copied into a ring of shared buffers for the shaders. `renderInfo.viewPort` stays the texture size, the physical size, so every sizeable resource is allocated at it. +- After the graph is built, `applyXRFoveationToSceneRenderPassDescriptors` attaches the map to the descriptors of the passes that rasterize the scene at drawable size (G-buffer + light, the deferred colour that transparency/wireframe/debug draw into, environment/sky/grid, splats, gizmos), and each of those encoders sets the eye's viewport (`applyXRFoveationViewport`) in the map's screen space. Under a map, Metal reads a descriptor's `renderTargetWidth/Height` and the viewport in *screen* pixels (measured on a Mac GPU, and seen on the headset as a screen-fixed rectangle outside which nothing was drawn), so the attached descriptors leave the target size at 0 — the attachments' own size — and get it back when the map is detached. The full-screen passes (SSAO, post-processing, anti-aliasing, the output transform into the compositor's texture) keep plain descriptors: they copy physical pixels one to one and the layout is already the map's. +- Two kinds of shader decode the map (`rasterization_rate_map_decoder`): SSAO, which rebuilds view positions from a depth texel's UV and must use the texel's *screen* position; and the occlusion culls (`hzbCullVisibleEntities`, `gaussianChunkCull`, the per-splat test), which project boxes and splats to screen UV and sample a pyramid laid out in physical space — the rect or point goes through the map first (`HZBOcclusion.h`). The pyramid carries the map it was drawn through into the next frame's culls (`HZBPyramidFrame.rateMapData`). +- The Gaussian fragment takes its offset from the splat centre from a varying rather than from `[[position]]`, which the rate map places in the physical texture. Fragment position is physical everywhere else too, which is what every one-to-one texture read wants; `SmallObjectCulling` measures its pixel threshold against the screen height. + +Without foveation (the layer configured without it, the simulator), `drawable.rasterizationRateMaps` is empty and the frame is drawn uniformly exactly as before. + ### 3c. HZB Pyramid (built once after both eyes) ```swift @@ -220,7 +231,7 @@ Selection rays update from interaction events, not continuously as the user look ├─ frame.endUpdate() ├─ wait(until: optimalInputTime) (sleep until compositor deadline) ├─ frame.startSubmission() - ├─ queryDrawable() (get per-eye textures) + ├─ queryDrawables() (one drawable per target; the old single query aborts a foveated layer) ├─ queryDeviceAnchor() → fallback chain → drawable.deviceAnchor = anchor │ └─ executeXRSystemPass()