diff --git a/Sources/CShaderTypes/ShaderTypes.h b/Sources/CShaderTypes/ShaderTypes.h index 599d73072..40bcd4b90 100644 --- a/Sources/CShaderTypes/ShaderTypes.h +++ b/Sources/CShaderTypes/ShaderTypes.h @@ -559,6 +559,8 @@ typedef enum{ ssaoPassFrustumIndex, ssaoPassReverseZIndex, ssaoPassProjScaleIndex, + ssaoPassRateMapDataIndex, // rasterization_rate_map_data of the eye's rate map (XR foveation; a never-read placeholder otherwise) + ssaoPassRateMapSizesIndex, // float4: the map's screen size in xy, the depth texture's size in zw; x <= 0 when there is no map }SSAOBufferIndices; typedef enum{ @@ -767,7 +769,9 @@ typedef enum{ hzbCullPassViewportIndex, hzbCullPassMipCountIndex, hzbCullPassReverseZIndex, - hzbCullPassOcclusionBiasIndex + hzbCullPassOcclusionBiasIndex, + hzbCullPassRateMapDataIndex, // rasterization_rate_map_data the pyramid was drawn through (XR foveation; a never-read placeholder otherwise) + hzbCullPassRateMapSizesIndex // float4: the map's screen size in xy, the pyramid's base size in zw; x <= 0 when there is no map }HZBOcclusionCullingBufferIndices; typedef enum{ @@ -800,6 +804,8 @@ typedef enum{ gaussianCullHZBReverseZIndex, gaussianCullHZBOcclusionBiasIndex, gaussianCullHZBValidIndex, + gaussianCullHZBRateMapDataIndex, // rasterization_rate_map_data the pyramid was drawn through (XR foveation; a never-read placeholder otherwise) + gaussianCullHZBRateMapSizesIndex, // float4: the map's screen size in xy, the pyramid's base size in zw; x <= 0 when there is no map }GaussianDepthBufferIndices; typedef enum{ @@ -1034,7 +1040,8 @@ typedef struct{ uint32_t uniformQuotas; // GaussianDebugOptions.disableScreenWeightedQuotas: screenArea = splatCount, the uniform quota rule uint32_t paged; // 0: every record resident; 1: a paged entity (the cull writes the demand table and lists resident ranks only, the fused pass reads the page pool); 2: demand only (a warming tier: write the demand word and return) uint32_t rangeCount; // entries of ranges[] (gaussianChunkCullRangesIndex, GaussianChunkTreeCull); a thread whose chunkIndex falls in none of them skips the per-chunk test. 0 (GaussianDebugOptions.disableTreeSkip, or a file without a tree) skips the check entirely, as before the tree was wired in -}GaussianChunkCullConstants; // 192 bytes (176 + rangeCount, padded to the struct's 16-byte alignment) + simd_float4 hzbRateMapSizes; // the rasterization rate map the HZB was drawn through (XR foveation): its screen size in xy, the pyramid's base size in zw; x <= 0 when there is none and the rate map buffer is a never-read placeholder +}GaussianChunkCullConstants; // 208 bytes (180 + hzbRateMapSizes at 192, padded to the struct's 16-byte alignment) /// One span of GaussianChunkTreeCull.visibleChunkRanges (Swift): chunk indices [firstChunk, /// firstChunk + chunkCount) the tree walk could not rule out this frame. Every chunk still gets @@ -1062,6 +1069,7 @@ typedef enum{ gaussianChunkCullLevelStateIndex = 11, // GaussianChunkLevelState[chunkCount], persistent, written by gaussianComputeChunkQuotas (a never-read stand-in without coarse levels) gaussianChunkCullLevelConstantsIndex = 12, // GaussianChunkLevelConstants (setBytes); hasCoarse = 0 keeps every path as it was gaussianChunkCullRangesIndex = 13, // GaussianChunkCullRange[rangeCount] (setBytes; a never-read stand-in when rangeCount == 0) + gaussianChunkCullRateMapDataIndex = 14, // rasterization_rate_map_data the HZB was drawn through (XR foveation; read only when hzbRateMapSizes.x > 0, a placeholder otherwise) }GaussianChunkCullBufferIndices; typedef enum{ @@ -1299,6 +1307,7 @@ typedef enum{ gaussianChunkPreprocessCoarseTableIndex, // 17: GaussianChunkDecodeConstants[levelCount × chunkCount], the coarse rows, level-major gaussianChunkPreprocessLevelStateIndex, // 18: GaussianChunkLevelState[chunkCount] as gaussianComputeChunkQuotas left it this frame gaussianChunkPreprocessLevelConstantsIndex, // 19: GaussianChunkLevelConstants (setBytes) + gaussianChunkPreprocessRateMapDataIndex, // 20: rasterization_rate_map_data the HZB was drawn through (XR foveation; read only when the cull constants' hzbRateMapSizes.x > 0, a placeholder otherwise) }GaussianChunkPreprocessBufferIndices; typedef enum{ diff --git a/Sources/UntoldEngine/Renderer/RenderExtensionModelSurface.swift b/Sources/UntoldEngine/Renderer/RenderExtensionModelSurface.swift index 83f2deaba..e80f6d21e 100644 --- a/Sources/UntoldEngine/Renderer/RenderExtensionModelSurface.swift +++ b/Sources/UntoldEngine/Renderer/RenderExtensionModelSurface.swift @@ -258,6 +258,7 @@ public extension RenderPassContext { renderEncoder.label = label renderEncoder.pushDebugGroup(label) + applyXRFoveationViewport(renderEncoder) renderEncoder.setRenderPipelineState(pipelineState) renderEncoder.setDepthStencilState(pipeline.depthState) diff --git a/Sources/UntoldEngine/Renderer/RenderPasses+MuscleDebug.swift b/Sources/UntoldEngine/Renderer/RenderPasses+MuscleDebug.swift index 3cdeef819..544c16366 100644 --- a/Sources/UntoldEngine/Renderer/RenderPasses+MuscleDebug.swift +++ b/Sources/UntoldEngine/Renderer/RenderPasses+MuscleDebug.swift @@ -155,6 +155,7 @@ public extension RenderPasses { } renderEncoder.label = "Muscle Debug Pass" renderEncoder.pushDebugGroup("Muscle Debug Pass") + applyXRFoveationViewport(renderEncoder) renderEncoder.setRenderPipelineState(pipelineState) // The cages live under the skin, so never depth-test them. In visionOS // mixed immersion the compositor drops pixels that wrote no depth, so diff --git a/Sources/UntoldEngine/Renderer/RenderPasses.swift b/Sources/UntoldEngine/Renderer/RenderPasses.swift index a6810cc66..c12b69184 100644 --- a/Sources/UntoldEngine/Renderer/RenderPasses.swift +++ b/Sources/UntoldEngine/Renderer/RenderPasses.swift @@ -1076,6 +1076,7 @@ public enum RenderPasses { } renderEncoder.label = "Grid Pass" + applyXRFoveationViewport(renderEncoder) renderEncoder.pushDebugGroup("Grid Pass") @@ -1167,6 +1168,7 @@ public enum RenderPasses { } renderEncoder.label = "Sky Pass" + applyXRFoveationViewport(renderEncoder) renderEncoder.pushDebugGroup("Sky Pass") @@ -1237,6 +1239,7 @@ public enum RenderPasses { } renderEncoder.label = "Environment Pass" + applyXRFoveationViewport(renderEncoder) renderEncoder.pushDebugGroup("Environment Pass") @@ -1835,6 +1838,7 @@ public enum RenderPasses { } renderEncoder.label = "Model Pass" + applyXRFoveationViewport(renderEncoder) renderEncoder.pushDebugGroup("Model Pass") @@ -2102,6 +2106,7 @@ public enum RenderPasses { } renderEncoder.label = "Batched Model Pass" + applyXRFoveationViewport(renderEncoder) renderEncoder.pushDebugGroup("Batched Model Pass") renderEncoder.setRenderPipelineState(modelPipeline.pipelineState!) @@ -2342,6 +2347,10 @@ public enum RenderPasses { renderEncoder.label = "G-Buffer + Light Pass (TBDR)" renderEncoder.pushDebugGroup("G-Buffer + Light Pass (TBDR)") + // Through the eye's rasterization rate map when there is one (XR foveation): the + // geometry in the map's screen space, the light quad over the whole of it, the + // G-buffer read back from the tile under each fragment either way. + applyXRFoveationViewport(renderEncoder) // What every draw of the pass shares, read once. let pass = OpaquePassState(cameraComponent: cameraComponent) @@ -2800,6 +2809,14 @@ public enum RenderPasses { var projScale = simd_float2(renderInfo.perspectiveSpace.columns.0.x, renderInfo.perspectiveSpace.columns.1.y) renderEncoder.setFragmentBytes(&projScale, length: MemoryLayout.stride, index: Int(ssaoPassProjScaleIndex.rawValue)) + // The eye's rasterization rate map (XR foveation): the depth was drawn through it, so + // the view positions are rebuilt from where its texels sit on screen, not in the texture. + bindRateMap( + renderInfo.xrFoveation?.rateMapData, + toFragmentsOf: renderEncoder, + dataIndex: Int(ssaoPassRateMapDataIndex.rawValue), + sizesIndex: Int(ssaoPassRateMapSizesIndex.rawValue) + ) // set the draw command renderEncoder.drawIndexedPrimitivesTracked( @@ -3000,6 +3017,14 @@ public enum RenderPasses { var projScale = simd_float2(renderInfo.perspectiveSpace.columns.0.x, renderInfo.perspectiveSpace.columns.1.y) renderEncoder.setFragmentBytes(&projScale, length: MemoryLayout.stride, index: Int(ssaoPassProjScaleIndex.rawValue)) + // The eye's rasterization rate map (XR foveation): the depth was drawn through it, so + // the view positions are rebuilt from where its texels sit on screen, not in the texture. + bindRateMap( + renderInfo.xrFoveation?.rateMapData, + toFragmentsOf: renderEncoder, + dataIndex: Int(ssaoPassRateMapDataIndex.rawValue), + sizesIndex: Int(ssaoPassRateMapSizesIndex.rawValue) + ) // SSAO properties renderEncoder.setFragmentBytes(&SSAOParams.shared.radius, length: MemoryLayout.stride, index: Int(ssaoPassRadiusIndex.rawValue)) @@ -3619,6 +3644,7 @@ public enum RenderPasses { } renderEncoder.label = "Transparency Pass" + applyXRFoveationViewport(renderEncoder) renderEncoder.pushDebugGroup("Transparency Pass") renderEncoder.setRenderPipelineState(transparencyPipeline.pipelineState!) @@ -4034,6 +4060,9 @@ public enum RenderPasses { descriptor.depthAttachment.texture = depthTexture descriptor.depthAttachment.loadAction = .load descriptor.depthAttachment.storeAction = .store + // The depth was drawn through the eye's rasterization rate map when there is one + // (XR foveation); so is what this pass adds to it. + attachXRFoveation(renderInfo.xrFoveation?.rateMap, to: descriptor) guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: descriptor) else { handleError(.renderPassCreationFailed, "Mesh Occluder Shell Pass") @@ -4047,6 +4076,7 @@ public enum RenderPasses { } renderEncoder.label = "Mesh Occluder Shell Pass" + applyXRFoveationViewport(renderEncoder) renderEncoder.pushDebugGroup("Mesh Occluder Shell Pass") renderEncoder.setRenderPipelineState(pipelineState) renderEncoder.setDepthStencilState(pipeline.depthState) @@ -4172,6 +4202,9 @@ public enum RenderPasses { descriptor.depthAttachment.texture = depthTexture descriptor.depthAttachment.loadAction = .load descriptor.depthAttachment.storeAction = .store + // The depth was drawn through the eye's rasterization rate map when there is one + // (XR foveation); so is what this pass adds to it. + attachXRFoveation(renderInfo.xrFoveation?.rateMap, to: descriptor) guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: descriptor) else { handleError(.renderPassCreationFailed, "Wireframe Occlusion Depth Pass") @@ -4185,6 +4218,7 @@ public enum RenderPasses { } renderEncoder.label = "Wireframe Occlusion Depth Pass" + applyXRFoveationViewport(renderEncoder) renderEncoder.pushDebugGroup("Wireframe Occlusion Depth Pass") renderEncoder.setRenderPipelineState(pipelineState) renderEncoder.setDepthStencilState(pipeline.depthState) @@ -4369,6 +4403,7 @@ public enum RenderPasses { } renderEncoder.label = "Wireframe Pass" + applyXRFoveationViewport(renderEncoder) renderEncoder.pushDebugGroup("Wireframe Pass") renderEncoder.setRenderPipelineState(wireframePipelineState) @@ -4749,6 +4784,7 @@ public enum RenderPasses { } renderEncoder.label = "Spatial Debug Bounds Pass" + applyXRFoveationViewport(renderEncoder) renderEncoder.pushDebugGroup("Spatial Debug Bounds Pass") renderEncoder.setRenderPipelineState(spatialDebugPipeline.pipelineState!) @@ -4897,6 +4933,7 @@ public enum RenderPasses { } renderEncoder.label = "Gaussian Pass" + applyXRFoveationViewport(renderEncoder) renderInfo.gaussianCoverageWritten = true renderEncoder.pushDebugGroup("Gaussian Pass") diff --git a/Sources/UntoldEngine/Renderer/RenderResources.swift b/Sources/UntoldEngine/Renderer/RenderResources.swift index d30c80743..b75ead173 100644 --- a/Sources/UntoldEngine/Renderer/RenderResources.swift +++ b/Sources/UntoldEngine/Renderer/RenderResources.swift @@ -87,6 +87,10 @@ public struct RenderInfo { /// shadingCameraPosition so each eye gets its own view vector (specular highlights, /// Fresnel), while streaming, LOD and culling keep the head-centre camera position. public var xrEyeCameraPosition: simd_float3? + /// The rasterization rate map of the eye being drawn (visionOS foveation), with the + /// viewport and the decoder data that go with it; nil when the frame is drawn uniformly. + /// `viewPort` stays the size of the textures, the map's physical size. See XRFoveation.swift. + public var xrFoveation: XRFoveationFrame? } /// Where an eye is, from its view matrix (the inverse of the eye's camera transform). @@ -176,6 +180,9 @@ public struct BufferResources { var reduceScanCapacity: Int = 0 /// Whether a frame with more bounding boxes than the scan takes was reported. var reportedReduceScanOverflow: Bool = false + + /// Bound where a shader declares a rasterization rate map it will not read (XRFoveation.swift). + var rateMapPlaceholder: MTLBuffer? } public struct TripleBufferResources { @@ -328,9 +335,14 @@ public struct HZBPyramidFrame: Sendable { public var viewProjection: simd_float4x4 /// Where that frame's camera stood, in world space. public var cameraPosition: simd_float3 + /// The rasterization rate map that frame was drawn through (XR foveation), nil when it was + /// drawn uniformly: the pyramid is laid out in the map's physical space, so a screen rect or + /// point the test projects must go through the map before the pyramid is sampled there. + public var rateMapData: XRRasterizationRateMapData? - public init(viewProjection: simd_float4x4, cameraPosition: simd_float3) { + public init(viewProjection: simd_float4x4, cameraPosition: simd_float3, rateMapData: XRRasterizationRateMapData? = nil) { self.viewProjection = viewProjection self.cameraPosition = cameraPosition + self.rateMapData = rateMapData } } diff --git a/Sources/UntoldEngine/Renderer/XRFoveation.swift b/Sources/UntoldEngine/Renderer/XRFoveation.swift new file mode 100644 index 000000000..ee345707d --- /dev/null +++ b/Sources/UntoldEngine/Renderer/XRFoveation.swift @@ -0,0 +1,182 @@ +// +// XRFoveation.swift +// UntoldEngine +// +// Variable-rate rasterization in XR. With foveation enabled in its layer configuration, the +// visionOS compositor hands the engine, per eye and per frame, a rasterization rate map and +// colour and depth textures of the map's physical size — smaller than the screen the eye +// sees. The GPU draws through the map, dense where the eyes look and coarse in the periphery, +// and the compositor unwarps the texture on the way to the display; a layer configured this +// way also accepts a render quality above the platform default, which a uniform layer does not. +// +// What this asks of the renderer: every pass that rasterizes scene geometry at drawable size +// attaches the eye's map to its descriptor and sets its viewport in the map's screen space +// (`applyXRFoveationToSceneRenderPassDescriptors`, `applyXRFoveationViewport`). The full-screen +// passes that follow — SSAO, post-processing, anti-aliasing, the output transform into the +// compositor's texture — read and write the physical textures one pixel to one and never see +// the map; it is already baked into the layout of what they copy. Two kinds of shader do have +// to know: one that turns a texel's position back into a direction in space (SSAO rebuilds view +// positions from depth) and the occlusion culls, which project boxes and splats to the screen +// and sample a depth pyramid that is laid out in the map's physical space. Both decode the +// map's parameters (`rasterization_rate_map_decoder`, bound by `bindRateMap`), and the pyramid +// carries the map it was drawn through into the next frame's culls (`HZBPyramidFrame`). +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Metal +import simd + +/// A rasterization rate map's parameter data in a buffer the shaders decode, with the two sizes +/// a decode needs. Immutable once made, so it can travel with the depth pyramid built through +/// it (`HZBPyramidFrame.rateMapData`) into the culls of the frame after. +public final class XRRasterizationRateMapData: @unchecked Sendable { + /// The map's parameters, as `copyParameterData(buffer:offset:)` wrote them at offset 0. + public let buffer: MTLBuffer + /// The map's screen size in pixels: the space viewports and clip-space positions live in. + public let screenSize: simd_float2 + /// The size in pixels of the textures drawn through the map (layer 0). + public let physicalSize: simd_float2 + + public init(buffer: MTLBuffer, screenSize: simd_float2, physicalSize: simd_float2) { + self.buffer = buffer + self.screenSize = screenSize + self.physicalSize = physicalSize + } + + /// Copies `rateMap`'s parameters into `buffer`, which must be shared and at least + /// `rateMap.parameterDataSizeAndAlign.size` bytes long; nil when it is shorter. + public convenience init?(rateMap: MTLRasterizationRateMap, buffer: MTLBuffer) { + let size = rateMap.parameterDataSizeAndAlign.size + guard size > 0, buffer.length >= size else { return nil } + rateMap.copyParameterData(buffer: buffer, offset: 0) + let screen = rateMap.screenSize + let physical = rateMap.physicalSize(layer: 0) + self.init( + buffer: buffer, + screenSize: simd_float2(Float(screen.width), Float(screen.height)), + physicalSize: simd_float2(Float(physical.width), Float(physical.height)) + ) + } + + /// What the shaders bind next to the buffer: the screen size in xy, the physical size in + /// zw. A zero x is the "no map" word (`rateMapSizes(nil)`). + public var sizes: simd_float4 { + simd_float4(screenSize.x, screenSize.y, physicalSize.x, physicalSize.y) + } +} + +/// The rate map of the eye being drawn, as `UntoldEngineXR` hands it to the renderer for each +/// eye of each frame (`renderInfo.xrFoveation`); nil when the frame is drawn uniformly — the +/// layer configured without foveation, the simulator, or `.foveatedRendering(.disabled)`. +public struct XRFoveationFrame { + public let rateMap: MTLRasterizationRateMap + /// The eye's viewport in the map's screen space (`view.textureMap.viewport`), which every + /// encoder drawing through the map sets explicitly: the default is the physical texture's. + public let viewport: MTLViewport + public let rateMapData: XRRasterizationRateMapData + + public init(rateMap: MTLRasterizationRateMap, viewport: MTLViewport, rateMapData: XRRasterizationRateMapData) { + self.rateMap = rateMap + self.viewport = viewport + self.rateMapData = rateMapData + } + + public var screenSize: simd_float2 { + rateMapData.screenSize + } + + public var physicalSize: simd_float2 { + rateMapData.physicalSize + } +} + +/// Attaches the current eye's rate map to the descriptors of the passes that rasterize the scene +/// at drawable size — the G-buffer + light pass, the deferred colour the transparency, wireframe +/// and debug passes draw into, the environment, sky or grid background, the splats and the +/// gizmos — or detaches it when the frame has none. The full-screen passes keep their +/// descriptors untouched: they copy physical pixels one to one. Called for every eye once its +/// graph is built, since a rebuild may have recreated the descriptors. +func applyXRFoveationToSceneRenderPassDescriptors() { + let rateMap = renderInfo.xrFoveation?.rateMap + for descriptor in [ + renderInfo.offscreenRenderPassDescriptor, + renderInfo.deferredRenderPassDescriptor, + renderInfo.environmentRenderPassDescriptor, + renderInfo.gaussianRenderPassDescriptor, + renderInfo.gizmoRenderPassDescriptor, + ] { + guard let descriptor else { continue } + attachXRFoveation(rateMap, to: descriptor) + } +} + +/// Attaches a rate map to a descriptor whose attachments are physical-size textures, or +/// detaches it. A descriptor's render target size is read in *screen* pixels while a map is +/// attached (measured: a target size equal to the physical size clips the scene to the +/// top-left physical-size corner of the screen), so the size is left at 0 — Metal then takes +/// the attachments' own size — and put back to the texture size without a map, which is what +/// `createRenderPassDescriptor` sets and the memoryless attachments rely on. +func attachXRFoveation(_ rateMap: MTLRasterizationRateMap?, to descriptor: MTLRenderPassDescriptor) { + descriptor.rasterizationRateMap = rateMap + if rateMap != nil { + descriptor.renderTargetWidth = 0 + descriptor.renderTargetHeight = 0 + } else if let viewPort = renderInfo.viewPort, descriptor.renderTargetWidth == 0 { + descriptor.renderTargetWidth = Int(viewPort.x) + descriptor.renderTargetHeight = Int(viewPort.y) + } +} + +/// Sets the eye's screen-space viewport on an encoder that draws through the rate map; nothing +/// outside foveated XR, where the default viewport, the texture's own size, is the right one. +/// (Measured: a viewport of the physical size under a map clips the scene the same way a +/// physical-size render target does; the map's screen size covers the whole texture.) +@inline(__always) +func applyXRFoveationViewport(_ encoder: MTLRenderCommandEncoder) { + guard let foveation = renderInfo.xrFoveation else { return } + encoder.setViewport(foveation.viewport) +} + +/// The sizes word of a map, or the "no map" word the shaders test for. +@inline(__always) +func rateMapSizes(_ data: XRRasterizationRateMapData?) -> simd_float4 { + data?.sizes ?? .zero +} + +/// The rate map the current depth pyramid was drawn through, for the culls that sample it. +@inline(__always) +func hzbPyramidRateMapData() -> XRRasterizationRateMapData? { + renderInfo.hzbFrame?.rateMapData +} + +/// Binds a map's parameters and sizes to a kernel, or a placeholder and the "no map" word when +/// there is none: Metal wants every declared buffer bound, and the kernels read the parameters +/// only when `sizes.x > 0`. +func bindRateMap(_ data: XRRasterizationRateMapData?, to encoder: MTLComputeCommandEncoder, dataIndex: Int, sizesIndex: Int) { + var sizes = rateMapSizes(data) + encoder.setBuffer(data?.buffer ?? rateMapPlaceholderBuffer(), offset: 0, index: dataIndex) + encoder.setBytes(&sizes, length: MemoryLayout.stride, index: sizesIndex) +} + +/// The fragment-stage form of `bindRateMap(_:to:dataIndex:sizesIndex:)`. +func bindRateMap(_ data: XRRasterizationRateMapData?, toFragmentsOf encoder: MTLRenderCommandEncoder, dataIndex: Int, sizesIndex: Int) { + var sizes = rateMapSizes(data) + encoder.setFragmentBuffer(data?.buffer ?? rateMapPlaceholderBuffer(), offset: 0, index: dataIndex) + encoder.setFragmentBytes(&sizes, length: MemoryLayout.stride, index: sizesIndex) +} + +/// A small shared buffer bound wherever a shader declares a rate map it will not read. +private func rateMapPlaceholderBuffer() -> MTLBuffer? { + if let buffer = bufferResources.rateMapPlaceholder { + return buffer + } + guard let device = renderInfo.device else { return nil } + let buffer = device.makeBuffer(length: 256, options: .storageModeShared) + buffer?.label = "Rate Map Placeholder" + bufferResources.rateMapPlaceholder = buffer + return buffer +} diff --git a/Sources/UntoldEngine/Shaders/BitonicSort.metal b/Sources/UntoldEngine/Shaders/BitonicSort.metal index 7b04e7c53..babdf2d4e 100644 --- a/Sources/UntoldEngine/Shaders/BitonicSort.metal +++ b/Sources/UntoldEngine/Shaders/BitonicSort.metal @@ -19,6 +19,7 @@ #include "../../CShaderTypes/ShaderTypes.h" #include "ShaderStructs.h" #include "ShadersUtils.h" +#include "HZBOcclusion.h" using namespace metal; @@ -73,14 +74,18 @@ kernel void gaussianFinalizeVisibleSet( // (one thread per resident splat, head-centre view) and gaussianChunkDecodePreprocess // (GaussianChunkPreprocess.metal, one threadgroup per visible chunk, either eye) so the two // paths keep exactly the same splats: centre inside the guard-banded clip volume, then -// optionally not behind the previous frame's HZB. +// optionally not behind the previous frame's HZB. `hzbRateMap` and `hzbRateMapSizes` are the +// rasterization rate map the pyramid was drawn through, if any (HZBOcclusion.h): the centre's +// screen UV is moved into the pyramid's physical layout before the sample. inline bool gaussianClipCentrePassesCull( float4 centerClip, float clipGuardBand, uint hzbReverseZ, float hzbOcclusionBias, uint hzbValid, - texture2d hzbDepthPyramid) + texture2d hzbDepthPyramid, + constant rasterization_rate_map_data *hzbRateMap, + float4 hzbRateMapSizes) { if (centerClip.w <= 0.0f) return false; @@ -100,6 +105,9 @@ inline bool gaussianClipCentrePassesCull( // source of truth for partial occlusion. if (hzbValid != 0u) { float2 uv = float2(ndc.x * 0.5f + 0.5f, 1.0f - (ndc.y * 0.5f + 0.5f)); + if (hzbRateMapActive(hzbRateMapSizes)) { + uv = hzbScreenUVToPhysicalUV(hzbRateMap, hzbRateMapSizes, uv); + } float splatDepth = clamp(centerClip.z / centerClip.w, 0.0f, 1.0f); constexpr sampler pointSampler(coord::normalized, address::clamp_to_edge, filter::nearest); float hzbDepth = hzbDepthPyramid.sample(pointSampler, uv, level(0)).x; @@ -120,12 +128,14 @@ inline bool gaussianSplatPassesCull( uint hzbReverseZ, float hzbOcclusionBias, uint hzbValid, - texture2d hzbDepthPyramid) + texture2d hzbDepthPyramid, + constant rasterization_rate_map_data *hzbRateMap, + float4 hzbRateMapSizes) { float4 centerClip = uniforms.projectionMatrix * uniforms.modelViewMatrix * float4(position, 1.0f); - return gaussianClipCentrePassesCull(centerClip, clipGuardBand, hzbReverseZ, hzbOcclusionBias, hzbValid, hzbDepthPyramid); + return gaussianClipCentrePassesCull(centerClip, clipGuardBand, hzbReverseZ, hzbOcclusionBias, hzbValid, hzbDepthPyramid, hzbRateMap, hzbRateMapSizes); } // One thread per resident splat: the whole-buffer cull for entities without a chunk table @@ -141,11 +151,13 @@ kernel void gaussianFrustumCull( constant uint &hzbReverseZ [[buffer(gaussianCullHZBReverseZIndex)]], constant float &hzbOcclusionBias [[buffer(gaussianCullHZBOcclusionBiasIndex)]], constant uint &hzbValid [[buffer(gaussianCullHZBValidIndex)]], + constant rasterization_rate_map_data *hzbRateMap [[buffer(gaussianCullHZBRateMapDataIndex)]], + constant float4 &hzbRateMapSizes [[buffer(gaussianCullHZBRateMapSizesIndex)]], texture2d hzbDepthPyramid [[texture(gaussianCullHZBDepthPyramidTextureIndex)]], uint index [[thread_position_in_grid]]) { if (index >= numOfSplats) return; - if (!gaussianSplatPassesCull(splats[index].position, uniforms, clipGuardBand, hzbReverseZ, hzbOcclusionBias, hzbValid, hzbDepthPyramid)) return; + if (!gaussianSplatPassesCull(splats[index].position, uniforms, clipGuardBand, hzbReverseZ, hzbOcclusionBias, hzbValid, hzbDepthPyramid, hzbRateMap, hzbRateMapSizes)) return; uint writeIndex = atomic_fetch_add_explicit(visibleCount, 1u, memory_order_relaxed); visibleIndices[writeIndex] = index; diff --git a/Sources/UntoldEngine/Shaders/GaussianChunkCull.metal b/Sources/UntoldEngine/Shaders/GaussianChunkCull.metal index 49e4e3a2a..0d1602f75 100644 --- a/Sources/UntoldEngine/Shaders/GaussianChunkCull.metal +++ b/Sources/UntoldEngine/Shaders/GaussianChunkCull.metal @@ -93,6 +93,8 @@ static inline bool gaussianBoxPassesClipPlanes( // the HZB. The occlusion test is the mesh cull's (HZBOcclusion.h): the box's screen rect at the // mip that covers it, 5x5 samples, nearest box depth against the farthest sample plus the bias. // `area` is the box's clipped screen area in this view, 0 when the view rejects or occludes it. +// `hzbRateMap` is the rasterization rate map the pyramid was drawn through when it was +// (params.hzbRateMapSizes, HZBOcclusion.h): the rect is moved into the pyramid's layout. static inline bool gaussianChunkVisibleInView( const float3 boxMin, const float3 boxMax, @@ -100,6 +102,7 @@ static inline bool gaussianChunkVisibleInView( constant GaussianChunkCullConstants ¶ms, const uint hzbValid, texture2d hzbDepthPyramid, + constant rasterization_rate_map_data *hzbRateMap, thread float &area) { if (!gaussianBoxPassesClipPlanes(boxMin, boxMax, viewProjection, params.clipGuardBand, area)) { @@ -116,6 +119,7 @@ static inline bool gaussianChunkVisibleInView( if (!projectAABBToScreenRect(0.5f * (boxMin + boxMax), 0.5f * (boxMax - boxMin), viewProjection, reverseZ, uvMin, uvMax, nearDepth)) { return true; } + hzbRemapRectToPhysical(hzbRateMap, params.hzbRateMapSizes, uvMin, uvMax); if (hzbRectIsOccluded(hzbDepthPyramid, uvMin, uvMax, nearDepth, params.viewport, params.hzbMipCount, reverseZ, params.hzbOcclusionBias)) { area = 0.0f; return false; @@ -189,6 +193,7 @@ kernel void gaussianChunkCull( const device GaussianChunkLevelState *levelState [[buffer(gaussianChunkCullLevelStateIndex)]], constant GaussianChunkLevelConstants &lvl [[buffer(gaussianChunkCullLevelConstantsIndex)]], const device GaussianChunkCullRange *ranges [[buffer(gaussianChunkCullRangesIndex)]], + constant rasterization_rate_map_data *hzbRateMap [[buffer(gaussianChunkCullRateMapDataIndex)]], texture2d hzbDepthPyramid [[texture(gaussianChunkCullHZBDepthPyramidTextureIndex)]], uint chunkIndex [[thread_position_in_grid]]) { @@ -227,9 +232,9 @@ kernel void gaussianChunkCull( const uint hzbValidForView0 = params.viewCount > 1u ? 0u : params.hzbValid; float area0 = 0.0f; float area1 = 0.0f; - const bool keep0 = gaussianChunkVisibleInView(boxMin, boxMax, params.viewProjection0, params, hzbValidForView0, hzbDepthPyramid, area0); + const bool keep0 = gaussianChunkVisibleInView(boxMin, boxMax, params.viewProjection0, params, hzbValidForView0, hzbDepthPyramid, hzbRateMap, area0); const bool keep1 = params.viewCount > 1u - ? gaussianChunkVisibleInView(boxMin, boxMax, params.viewProjection1, params, params.hzbValid, hzbDepthPyramid, area1) + ? gaussianChunkVisibleInView(boxMin, boxMax, params.viewProjection1, params, params.hzbValid, hzbDepthPyramid, hzbRateMap, area1) : false; const bool seen = keep0 || keep1; const float area = clamp(max(keep0 ? area0 : 0.0f, keep1 ? area1 : 0.0f), kGaussianScreenAreaMin, limit * limit); diff --git a/Sources/UntoldEngine/Shaders/GaussianChunkPreprocess.metal b/Sources/UntoldEngine/Shaders/GaussianChunkPreprocess.metal index c55a82bad..790b4435a 100644 --- a/Sources/UntoldEngine/Shaders/GaussianChunkPreprocess.metal +++ b/Sources/UntoldEngine/Shaders/GaussianChunkPreprocess.metal @@ -60,16 +60,17 @@ inline bool gaussianChunkSplatPassesCull( float3 position, constant Uniforms &uniforms, constant GaussianChunkCullConstants ¶ms, - texture2d hzbDepthPyramid) + texture2d hzbDepthPyramid, + constant rasterization_rate_map_data *hzbRateMap) { if (params.viewCount <= 1u) { - return gaussianSplatPassesCull(position, uniforms, params.clipGuardBand, params.hzbReverseZ, params.hzbOcclusionBias, params.hzbValid, hzbDepthPyramid); + return gaussianSplatPassesCull(position, uniforms, params.clipGuardBand, params.hzbReverseZ, params.hzbOcclusionBias, params.hzbValid, hzbDepthPyramid, hzbRateMap, params.hzbRateMapSizes); } const float4 local = float4(position, 1.0f); - if (gaussianClipCentrePassesCull(params.viewProjection0 * local, params.clipGuardBand, params.hzbReverseZ, params.hzbOcclusionBias, 0u, hzbDepthPyramid)) { + if (gaussianClipCentrePassesCull(params.viewProjection0 * local, params.clipGuardBand, params.hzbReverseZ, params.hzbOcclusionBias, 0u, hzbDepthPyramid, hzbRateMap, params.hzbRateMapSizes)) { return true; } - return gaussianClipCentrePassesCull(params.viewProjection1 * local, params.clipGuardBand, params.hzbReverseZ, params.hzbOcclusionBias, params.hzbValid, hzbDepthPyramid); + return gaussianClipCentrePassesCull(params.viewProjection1 * local, params.clipGuardBand, params.hzbReverseZ, params.hzbOcclusionBias, params.hzbValid, hzbDepthPyramid, hzbRateMap, params.hzbRateMapSizes); } kernel void gaussianChunkDecodePreprocess( @@ -93,6 +94,7 @@ kernel void gaussianChunkDecodePreprocess( const device GaussianChunkDecodeConstants *coarseTable [[buffer(gaussianChunkPreprocessCoarseTableIndex)]], const device GaussianChunkLevelState *levelState [[buffer(gaussianChunkPreprocessLevelStateIndex)]], constant GaussianChunkLevelConstants &lvl [[buffer(gaussianChunkPreprocessLevelConstantsIndex)]], + constant rasterization_rate_map_data *hzbRateMap [[buffer(gaussianChunkPreprocessRateMapDataIndex)]], texture2d hzbDepthPyramid [[texture(gaussianChunkPreprocessHZBDepthPyramidTextureIndex)]], uint chunkSlot [[threadgroup_position_in_grid]], uint localIndex [[thread_position_in_threadgroup]], @@ -161,7 +163,7 @@ kernel void gaussianChunkDecodePreprocess( // bit-identical to what the whole-buffer path loaded, and the covariance and colour // pass through half precision the way EncodedGaussianSplat stores them. const float3 centerLocal = mix(aabbMin, aabbMax, gaussianUnpack11_10_11(record.x)); - if (!gaussianChunkSplatPassesCull(centerLocal, uniforms, cull, hzbDepthPyramid)) { + if (!gaussianChunkSplatPassesCull(centerLocal, uniforms, cull, hzbDepthPyramid, hzbRateMap)) { continue; } diff --git a/Sources/UntoldEngine/Shaders/Gaussians.metal b/Sources/UntoldEngine/Shaders/Gaussians.metal index 9f5d06d78..e97b846c2 100644 --- a/Sources/UntoldEngine/Shaders/Gaussians.metal +++ b/Sources/UntoldEngine/Shaders/Gaussians.metal @@ -621,15 +621,6 @@ inline float calcPowerFromConic(float3 conic, float2 d) 2.0f * conic.y * d.x * d.y); } -inline float2 calcScreenSpaceDelta(float2 pixelPos, - float2 centerPixel, - float projYSign) -{ - float2 d = pixelPos - centerPixel; - d.y *= projYSign; - return d; -} - typedef struct { half4 color [[raster_order_group(0)]]; @@ -693,20 +684,20 @@ vertex GaussianOutData vertexGaussianTBDRShader( return out; } - const float projYSign = -1.0f; - float2 centerNDC = centerClip.xy / centerClip.w; - float2 centerUV = centerNDC * float2(0.5f, 0.5f * projYSign) + 0.5f; - out.coordxy = centerUV * viewport; - out.conic = record.conicAndOpacity.xyz; // Tight, rotated quad along the ellipse's true principal axes (see // computeInverseCovarianceConic) instead of an axis-aligned bounding box. The axes live - // in the pixel frame of the conic (y down, as coordxy and the fragment's position), while - // NDC y points up: the y offset flips sign on the way, or a tilted splat's quad is the - // mirror image of its ellipse and the fragment falloff gets clipped to their overlap. + // in the pixel frame of the conic (y down), while NDC y points up: the y offset flips + // sign on the way, or a tilted splat's quad is the mirror image of its ellipse and the + // fragment falloff gets clipped to their overlap. The offset itself goes to the fragment + // as a varying: interpolated across the quad it is the fragment's own offset from the + // centre in the conic's frame, whatever the rasterizer did to the quad's pixels — under + // a rasterization rate map (XR foveation) the fragment's position is in the physical + // texture, not on the screen the centre was projected to. float2 pixelOffset = quad.x * record.axes.xy + quad.y * record.axes.zw; float2 ndcOffset = pixelOffset * float2(2.0f, -2.0f) / viewport; + out.pixelDelta = pixelOffset; out.position = centerClip; out.position.xy += ndcOffset * centerClip.w; out.color = record.color.xyz; @@ -745,8 +736,10 @@ fragment GaussianTBDRFragmentStore fragmentGaussianTBDRShader( // (gaussianAdaptiveSigma sigma out) — has negligible alpha. Rejecting those tail fragments // here means they never pay for the depth-texture read at all, on top of never reaching // the blend math below. - const float projYSign = 1.0f; - float2 d = calcScreenSpaceDelta(in.position.xy, in.coordxy, projYSign); + // The fragment's offset from the splat's centre in the conic's pixel frame, from the + // quad's corners (see vertexGaussianTBDRShader) rather than from in.position, which a + // rasterization rate map (XR foveation) places in the physical texture, not on screen. + float2 d = in.pixelDelta; float power = calcPowerFromConic(in.conic, d); float falloff = exp(power); diff --git a/Sources/UntoldEngine/Shaders/HZBCompute.metal b/Sources/UntoldEngine/Shaders/HZBCompute.metal index 9cfea93a4..0ad63f72d 100644 --- a/Sources/UntoldEngine/Shaders/HZBCompute.metal +++ b/Sources/UntoldEngine/Shaders/HZBCompute.metal @@ -80,6 +80,8 @@ kernel void hzbCullVisibleEntities( constant uint &mipCount [[buffer(hzbCullPassMipCountIndex)]], constant uint &reverseZ [[buffer(hzbCullPassReverseZIndex)]], constant float &occlusionBias [[buffer(hzbCullPassOcclusionBiasIndex)]], + constant rasterization_rate_map_data *rateMap [[buffer(hzbCullPassRateMapDataIndex)]], + constant float4 &rateMapSizes [[buffer(hzbCullPassRateMapSizesIndex)]], texture2d hzbDepthPyramid [[texture(hzbCullPassDepthPyramidTextureIndex)]], uint tid [[thread_position_in_grid]] ) { @@ -99,7 +101,9 @@ kernel void hzbCullVisibleEntities( return; } - // Shared with the Gaussian chunk cull — see HZBOcclusion.h. + // Shared with the Gaussian chunk cull — see HZBOcclusion.h. A pyramid drawn through a + // rasterization rate map (XR foveation) holds the rect elsewhere than its screen UV. + hzbRemapRectToPhysical(rateMap, rateMapSizes, uvMin, uvMax); bool isOccluded = hzbRectIsOccluded(hzbDepthPyramid, uvMin, uvMax, nearDepth, viewport, mipCount, reverseZ != 0u, occlusionBias); if (!isOccluded) { uint dst = atomic_fetch_add_explicit(outVisibleCount, 1u, memory_order_relaxed); diff --git a/Sources/UntoldEngine/Shaders/HZBOcclusion.h b/Sources/UntoldEngine/Shaders/HZBOcclusion.h index 0e7c2dcb1..550428736 100644 --- a/Sources/UntoldEngine/Shaders/HZBOcclusion.h +++ b/Sources/UntoldEngine/Shaders/HZBOcclusion.h @@ -80,6 +80,41 @@ static inline bool projectAABBToScreenRect( return true; } +/// The rasterization rate map a depth pyramid was drawn through, when its frame was (visionOS +/// foveation): `rateMapSizes.xy` is the map's screen size and `.zw` the pyramid's base size, in +/// pixels, and `rateMapSizes.x <= 0` says there is none, in which case `rateMap` is a placeholder +/// that is never read. Such a pyramid is not laid out over the screen the boxes above project +/// to — dense where the eyes look, sparse in the periphery — so a screen UV must be moved to +/// where the map put it before the pyramid is sampled there. +static inline bool hzbRateMapActive(const float4 rateMapSizes) { + return rateMapSizes.x > 0.0f && rateMapSizes.z > 0.0f; +} + +static inline float2 hzbScreenUVToPhysicalUV( + constant rasterization_rate_map_data *rateMap, + const float4 rateMapSizes, + const float2 uv +) { + rasterization_rate_map_decoder decoder(*rateMap); + const float2 physical = decoder.map_screen_to_physical_coordinates(uv * rateMapSizes.xy); + return clamp(physical / rateMapSizes.zw, 0.0f, 1.0f); +} + +/// Moves a screen rect from `projectAABBToScreenRect` into the pyramid's physical layout; the +/// map is monotonic along each axis, so the two corners are enough. Nothing without a map. +static inline void hzbRemapRectToPhysical( + constant rasterization_rate_map_data *rateMap, + const float4 rateMapSizes, + thread float2 &uvMin, + thread float2 &uvMax +) { + if (!hzbRateMapActive(rateMapSizes)) { + return; + } + uvMin = hzbScreenUVToPhysicalUV(rateMap, rateMapSizes, uvMin); + uvMax = hzbScreenUVToPhysicalUV(rateMap, rateMapSizes, uvMax); +} + /// True when the rect `uvMin..uvMax` whose nearest depth is `nearDepth` lies behind the HZB /// everywhere it is sampled. The mip is the one whose texel spans the rect; a 5x5 grid across /// the rect keeps porous occluders (window frames, glass assemblies) from falsely culling what diff --git a/Sources/UntoldEngine/Shaders/SSAOShader.metal b/Sources/UntoldEngine/Shaders/SSAOShader.metal index d93b867ab..7df4249a8 100644 --- a/Sources/UntoldEngine/Shaders/SSAOShader.metal +++ b/Sources/UntoldEngine/Shaders/SSAOShader.metal @@ -34,6 +34,19 @@ static float3 reconstructViewPos(float2 uv, float linearDepth, float2 projScale) return float3(xy, linearDepth); } +// The screen UV a depth texel was projected to: its own UV, unless the eye was drawn through +// a rasterization rate map (XR foveation, rateMapSizes.x > 0: the map's screen size in xy, the +// depth texture's size in zw). Such a texture is dense where the eyes look and sparse in the +// periphery, so a view position rebuilt from the texel's own UV would bend the geometry there +// and shade false creases; the map's decoder says where the texel sits on screen. +static float2 screenUVOfTexel(float2 uv, constant rasterization_rate_map_data *rateMap, float4 rateMapSizes) { + if (rateMapSizes.x <= 0.0 || rateMapSizes.z <= 0.0) { + return uv; + } + rasterization_rate_map_decoder decoder(*rateMap); + return decoder.map_physical_to_screen_coordinates(uv * rateMapSizes.zw) / rateMapSizes.xy; +} + fragment float4 fragmentSSAOShader(VertexCompositeOutput vertexOut [[stage_in]], depth2d depthTexture [[texture(ssaoDepthTextureIndex)]], constant float &radius [[buffer(ssaoPassRadiusIndex)]], @@ -43,7 +56,9 @@ fragment float4 fragmentSSAOShader(VertexCompositeOutput vertexOut [[stage_in]], constant float2 &viewPort [[buffer(ssaoPassViewPortIndex)]], constant float2 &frustumPlanes [[buffer(ssaoPassFrustumIndex)]], constant bool &reverseZ [[buffer(ssaoPassReverseZIndex)]], - constant float2 &projScale [[buffer(ssaoPassProjScaleIndex)]] + constant float2 &projScale [[buffer(ssaoPassProjScaleIndex)]], + constant rasterization_rate_map_data *rateMap [[buffer(ssaoPassRateMapDataIndex)]], + constant float4 &rateMapSizes [[buffer(ssaoPassRateMapSizesIndex)]] ) { if (!enabled){ @@ -62,7 +77,7 @@ fragment float4 fragmentSSAOShader(VertexCompositeOutput vertexOut [[stage_in]], float2 texelSize = 1.0 / max(viewPort, float2(1.0)); float pixelRadius = clamp(radius * 220.0 / max(centerDepth, 0.001), 2.0, 48.0); - float3 centerPosition = reconstructViewPos(vertexOut.uvCoords, centerDepth, projScale); + float3 centerPosition = reconstructViewPos(screenUVOfTexel(vertexOut.uvCoords, rateMap, rateMapSizes), centerDepth, projScale); // No normal G-buffer is available here — normalMap/positionMap are memoryless // (TBDR tile-only) in normal rendering and can't be bound to a later, separate // pass. Derive the surface normal from the depth buffer itself via screen-space @@ -94,7 +109,7 @@ fragment float4 fragmentSSAOShader(VertexCompositeOutput vertexOut [[stage_in]], } float sampleLinearDepth = linearizeDepth(sampleRawDepth, frustumPlanes.x, frustumPlanes.y, reverseZ); - float3 samplePosition = reconstructViewPos(sampleUV, sampleLinearDepth, projScale); + float3 samplePosition = reconstructViewPos(screenUVOfTexel(sampleUV, rateMap, rateMapSizes), sampleLinearDepth, projScale); float3 toSample = samplePosition - centerPosition; float sampleDistance = length(toSample); if (sampleDistance < 1e-5) { diff --git a/Sources/UntoldEngine/Shaders/ShaderStructs.h b/Sources/UntoldEngine/Shaders/ShaderStructs.h index 8a5d939c1..db6a59375 100644 --- a/Sources/UntoldEngine/Shaders/ShaderStructs.h +++ b/Sources/UntoldEngine/Shaders/ShaderStructs.h @@ -187,10 +187,10 @@ typedef struct{ float3 cov2; float3 conic; float3 color; - float2 coordxy; + float2 pixelDelta; // the vertex's offset from the splat's centre in the conic's pixel frame (y down) float alpha; bool valid; - + }GaussianOutData; // Cascaded Shadow Maps: 3 cascades shared between both eyes in XR. diff --git a/Sources/UntoldEngine/Systems/CullingSystem.swift b/Sources/UntoldEngine/Systems/CullingSystem.swift index 040abdb5d..4f62bbd92 100644 --- a/Sources/UntoldEngine/Systems/CullingSystem.swift +++ b/Sources/UntoldEngine/Systems/CullingSystem.swift @@ -548,6 +548,14 @@ func executeHZBOcclusionCulling( computeEncoder.setBytes(&mipCount, length: MemoryLayout.stride, index: Int(hzbCullPassMipCountIndex.rawValue)) computeEncoder.setBytes(&reverseZFlag, length: MemoryLayout.stride, index: Int(hzbCullPassReverseZIndex.rawValue)) computeEncoder.setBytes(&occlusionBias, length: MemoryLayout.stride, index: Int(hzbCullPassOcclusionBiasIndex.rawValue)) + // The rasterization rate map the pyramid's frame was drawn through (XR foveation): the + // rects are moved into the pyramid's physical layout before it is sampled. + bindRateMap( + pyramidFrame.rateMapData, + to: computeEncoder, + dataIndex: Int(hzbCullPassRateMapDataIndex.rawValue), + sizesIndex: Int(hzbCullPassRateMapSizesIndex.rawValue) + ) computeEncoder.setTexture(hzbDepthPyramid, index: Int(hzbCullPassDepthPyramidTextureIndex.rawValue)) let tew = pipelineState.threadExecutionWidth @@ -1257,9 +1265,11 @@ extension HZBPyramidFrame { let cameraComponent = scene.get(component: CameraComponent.self, for: camera) else { return nil } let view = SceneRootTransform.shared.effectiveViewMatrix(cameraComponent.viewSpace) + // In stereo the last eye drawn: its view, projection and rate map alike. return HZBPyramidFrame( viewProjection: simd_mul(renderInfo.perspectiveSpace, view), - cameraPosition: eyePosition(ofView: view) + cameraPosition: eyePosition(ofView: view), + rateMapData: renderInfo.xrFoveation?.rateMapData ) } } diff --git a/Sources/UntoldEngine/Systems/GaussianChunkCull.swift b/Sources/UntoldEngine/Systems/GaussianChunkCull.swift index 93c307986..bf425a541 100644 --- a/Sources/UntoldEngine/Systems/GaussianChunkCull.swift +++ b/Sources/UntoldEngine/Systems/GaussianChunkCull.swift @@ -1033,6 +1033,9 @@ func gaussianChunkCullConstants( constants.hzbValid = hzbValid ? 1 : 0 constants.hzbReverseZ = renderInfo.reverseZEnabled ? 1 : 0 constants.hzbMipCount = UInt32(max(0, renderInfo.hzbMipCount)) + // The rasterization rate map the pyramid was drawn through (XR foveation), only with a + // pyramid to sample: the "no map" word otherwise, and the kernels never touch the buffer. + constants.hzbRateMapSizes = hzbValid ? rateMapSizes(hzbPyramidRateMapData()) : .zero constants.forceAllVisible = forceAllVisible ? 1 : 0 constants.uniformQuotas = uniformQuotas ? 1 : 0 constants.paged = paged @@ -1255,6 +1258,12 @@ private func encodeGaussianChunkCullDispatch( } else { encoder.setBytes(&gpuRanges, length: MemoryLayout.stride * gpuRanges.count, index: Int(gaussianChunkCullRangesIndex.rawValue)) } + // The pyramid's rate map (XR foveation), read only when the constants' hzbRateMapSizes say so. + encoder.setBuffer( + hzbPyramidRateMapData()?.buffer ?? chunkTable.constantsBuffer, + offset: 0, + index: Int(gaussianChunkCullRateMapDataIndex.rawValue) + ) encoder.setTexture(hzbTexture, index: Int(gaussianChunkCullHZBDepthPyramidTextureIndex.rawValue)) let tew = pipelines.cull.threadExecutionWidth let block = max(min(256, pipelines.cull.maxTotalThreadsPerThreadgroup) / tew * tew, tew) @@ -1530,6 +1539,12 @@ func encodeGaussianChunkDecodePreprocess( encoder.setBuffer(inputs.levels?.coarseTable ?? inputs.chunkTable.constantsBuffer, offset: 0, index: Int(gaussianChunkPreprocessCoarseTableIndex.rawValue)) encoder.setBuffer(inputs.levels?.levelState ?? inputs.chunkTable.constantsBuffer, offset: 0, index: Int(gaussianChunkPreprocessLevelStateIndex.rawValue)) encoder.setBytes(&inputs.levelConstants, length: MemoryLayout.stride, index: Int(gaussianChunkPreprocessLevelConstantsIndex.rawValue)) + // The pyramid's rate map (XR foveation), read only when the cull constants' hzbRateMapSizes say so. + encoder.setBuffer( + hzbPyramidRateMapData()?.buffer ?? inputs.chunkTable.constantsBuffer, + offset: 0, + index: Int(gaussianChunkPreprocessRateMapDataIndex.rawValue) + ) encoder.setTexture(inputs.hzbTexture, index: Int(gaussianChunkPreprocessHZBDepthPyramidTextureIndex.rawValue)) let threads = threadsPerThreadgroup ?? gaussianChunkPreprocessThreadsPerThreadgroup(chunkTable: inputs.chunkTable, pipelineState: pipelineState) diff --git a/Sources/UntoldEngine/Systems/GaussianSystem.swift b/Sources/UntoldEngine/Systems/GaussianSystem.swift index f80679111..13e4d3882 100644 --- a/Sources/UntoldEngine/Systems/GaussianSystem.swift +++ b/Sources/UntoldEngine/Systems/GaussianSystem.swift @@ -128,6 +128,14 @@ private func bindGaussianSplatCullInputs( encoder.setBytes(&inputs.hzbReverseZ, length: MemoryLayout.stride, index: Int(gaussianCullHZBReverseZIndex.rawValue)) encoder.setBytes(&inputs.hzbOcclusionBias, length: MemoryLayout.stride, index: Int(gaussianCullHZBOcclusionBiasIndex.rawValue)) encoder.setBytes(&inputs.hzbValid, length: MemoryLayout.stride, index: Int(gaussianCullHZBValidIndex.rawValue)) + // The rasterization rate map the pyramid was drawn through (XR foveation), with a pyramid + // to sample; the "no map" word otherwise. + bindRateMap( + inputs.hzbValid != 0 ? hzbPyramidRateMapData() : nil, + to: encoder, + dataIndex: Int(gaussianCullHZBRateMapDataIndex.rawValue), + sizesIndex: Int(gaussianCullHZBRateMapSizesIndex.rawValue) + ) encoder.setTexture(hzbTexture, index: Int(gaussianCullHZBDepthPyramidTextureIndex.rawValue)) } diff --git a/Sources/UntoldEngine/Systems/GraphBuilder.swift b/Sources/UntoldEngine/Systems/GraphBuilder.swift index df194fc4a..925d60dea 100644 --- a/Sources/UntoldEngine/Systems/GraphBuilder.swift +++ b/Sources/UntoldEngine/Systems/GraphBuilder.swift @@ -949,6 +949,9 @@ func makeSceneRenderTargetAccess( return nil } encoder.label = label + // The copied descriptor carries the eye's rasterization rate map (XR foveation); the + // viewport that goes with it is the eye's, not the texture's. + applyXRFoveationViewport(encoder) return encoder } } diff --git a/Sources/UntoldEngine/Systems/RenderingSystem.swift b/Sources/UntoldEngine/Systems/RenderingSystem.swift index 42da9e94d..8d121dfc6 100644 --- a/Sources/UntoldEngine/Systems/RenderingSystem.swift +++ b/Sources/UntoldEngine/Systems/RenderingSystem.swift @@ -170,6 +170,10 @@ func UpdateXRRenderingSystem(commandBuffer: MTLCommandBuffer, passDescriptor: MT do { let graph = try buildExecutableGameModeGraph() + // The eye's rasterization rate map (visionOS foveation) on the passes that draw the + // scene, after the build: a rebuild may have recreated their descriptors. + applyXRFoveationToSceneRenderPassDescriptors() + #if ENGINE_STATS_ENABLED let encodeStart = shouldRecordStatsInThisCallback ? CACurrentMediaTime() : 0.0 #endif diff --git a/Sources/UntoldEngine/Systems/SmallObjectCulling.swift b/Sources/UntoldEngine/Systems/SmallObjectCulling.swift index c37c72b0e..2f8241454 100644 --- a/Sources/UntoldEngine/Systems/SmallObjectCulling.swift +++ b/Sources/UntoldEngine/Systems/SmallObjectCulling.swift @@ -57,10 +57,13 @@ struct SmallObjectCulling { guard let viewPort, viewPort.y > 0, projection.columns.3.w == 0, projection.columns.1.y > 0 else { return nil } + // Pixels of the screen the eye sees: under a rasterization rate map (XR foveation) the + // textures are smaller than that screen, and a size in their pixels would cull early. + let screenHeight = renderInfo.xrFoveation?.screenSize.y ?? viewPort.y return SmallObjectCulling( cameraPosition: SceneRootTransform.shared.effectiveCameraPosition(cameraComponent.localPosition), minimumPixels: pixels, - viewportHeight: viewPort.y, + viewportHeight: screenHeight, tanHalfFovY: 1 / projection.columns.1.y ) } diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-ios.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-ios.metallib index c6a09d922..a8f903cb3 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-ios.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-ios.metallib differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-iossim.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-iossim.metallib index 2460ea36a..45a8b9c15 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-iossim.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-iossim.metallib differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvos.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvos.metallib index f00658282..fb5072e2b 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvos.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvos.metallib differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvossim.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvossim.metallib index 00249458f..c6d6cd139 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvossim.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvossim.metallib differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xros.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xros.metallib index d81cb37be..fc1714a18 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xros.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xros.metallib differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xrossim.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xrossim.metallib index bcd5eba84..86b58685b 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xrossim.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xrossim.metallib differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels.metallib index 37cee1e05..b4bc92f89 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels.metallib differ diff --git a/Sources/UntoldEngine/Utils/EngineSettingsAPI.swift b/Sources/UntoldEngine/Utils/EngineSettingsAPI.swift index dea53a774..67a25658c 100644 --- a/Sources/UntoldEngine/Utils/EngineSettingsAPI.swift +++ b/Sources/UntoldEngine/Utils/EngineSettingsAPI.swift @@ -86,6 +86,16 @@ public enum RenderingProperty: Sendable { /// frustum keeps. In scene units; the default is 0.5. See /// `getOcclusionCullingMaxCameraStep()` to read the current value. case occlusionCullingMaxCameraStep(Float) + /// Whether the XR frame is drawn through the rasterization rate maps the visionOS + /// compositor provides when the layer is configured with foveation enabled + /// (`LayerRenderer.Configuration.isFoveationEnabled`): dense where the eyes look, coarse + /// in the periphery, and the only configuration in which the compositor accepts a render + /// quality above the platform default. Enabled by default; a layer configured without + /// foveation provides no maps and draws uniformly whatever this says. Disabling it in a + /// foveated layer is a diagnostic: the frame is then drawn uniformly into textures the + /// compositor still unwarps, so the picture is distorted. See + /// `getFoveatedRenderingEnabled()` to read the current value. + case foveatedRendering(RenderingToggle) } public enum RenderingToggle: Sendable { @@ -148,9 +158,19 @@ public func setRendering(_ property: RenderingProperty) { SmallObjectCulling.minimumPixels = pixels.isFinite ? max(pixels, 0) : 0 case let .occlusionCullingMaxCameraStep(step): HZBOcclusionCulling.maxCameraStep = step.isFinite ? max(step, 0) : 0 + case .foveatedRendering(.enabled): + xrFoveatedRenderingEnabled = true + case .foveatedRendering(.disabled): + xrFoveatedRenderingEnabled = false } } +/// Whether the XR frame draws through the compositor's rasterization rate maps when the layer +/// provides them. See `RenderingProperty.foveatedRendering`. +public func getFoveatedRenderingEnabled() -> Bool { + xrFoveatedRenderingEnabled +} + public enum RenderResourceQuery { case texture(String) } diff --git a/Sources/UntoldEngine/Utils/Globals.swift b/Sources/UntoldEngine/Utils/Globals.swift index 1a92d1f3a..1e5b0c5cf 100644 --- a/Sources/UntoldEngine/Utils/Globals.swift +++ b/Sources/UntoldEngine/Utils/Globals.swift @@ -1103,6 +1103,7 @@ private final class RuntimeGlobalsStore: @unchecked Sendable { private var activeEntityValue: EntityID = .invalid private var enableEngineMetricsValue: Bool = false private var bypassPostProcessingValue: Bool = false + private var xrFoveatedRenderingEnabledValue: Bool = true private var antiAliasingModeValue: AntiAliasingMode = .fxaa private var renderDebugViewModeValue: RenderDebugViewMode = .lit private var pomQualitySettingsValue: POMQualitySettings = .platformDefault @@ -1594,6 +1595,20 @@ private final class RuntimeGlobalsStore: @unchecked Sendable { } } + var xrFoveatedRenderingEnabled: Bool { + get { + lock.lock() + let value = xrFoveatedRenderingEnabledValue + lock.unlock() + return value + } + set { + lock.lock() + xrFoveatedRenderingEnabledValue = newValue + lock.unlock() + } + } + var antiAliasingMode: AntiAliasingMode { get { lock.lock() @@ -2093,6 +2108,13 @@ public var bypassPostProcessing: Bool { set { RuntimeGlobalsStore.shared.bypassPostProcessing = newValue } } +/// Whether the XR frame draws through the rasterization rate maps the compositor provides +/// (visionOS foveation), the default. See `RenderingProperty.foveatedRendering`. +public var xrFoveatedRenderingEnabled: Bool { + get { RuntimeGlobalsStore.shared.xrFoveatedRenderingEnabled } + set { RuntimeGlobalsStore.shared.xrFoveatedRenderingEnabled = newValue } +} + /// Selects the active anti-aliasing pass inserted after the look pass. public enum AntiAliasingMode: Sendable { case none diff --git a/Sources/UntoldEngineXR/UntoldEngineXR.swift b/Sources/UntoldEngineXR/UntoldEngineXR.swift index b32d7c122..5cec2ad71 100644 --- a/Sources/UntoldEngineXR/UntoldEngineXR.swift +++ b/Sources/UntoldEngineXR/UntoldEngineXR.swift @@ -69,6 +69,16 @@ // Reuse render pass descriptors to avoid allocation churn (2 eyes × 90 FPS = 180 allocs/sec) private let passDescriptorLeft = MTLRenderPassDescriptor() private let passDescriptorRight = MTLRenderPassDescriptor() + + // Foveation: the parameter data of each eye's rasterization rate map, copied every frame + // (the maps follow the gaze) into a ring one slot deeper than the frames in flight — + // the depth pyramid built on a frame carries its last eye's data into the culls of the + // frame after (HZBPyramidFrame.rateMapData), which may still run when this frame's + // slot comes round again with only the usual in-flight depth. + private var rateMapParameterBuffers: [[MTLBuffer?]] = Array(repeating: [nil, nil], count: maxInFlightCommandBuffers + 1) + private var rateMapFrameCounter = 0 + private var loggedFoveationState: Bool? + private var loggedCaptureTargetSkip = false private let spatialGestureRecognizer = XRSpatialGestureRecognizer() private let xrEnvironmentLightingSystem = XREnvironmentLightingSystem() private var runtimeLightingModeObserverId: UUID? @@ -616,64 +626,79 @@ } } - // 9. Encode any drawing commands that depend on the device position or orientation - guard let drawable = frame.queryDrawable() else { + // 9. Encode any drawing commands that depend on the device position or orientation. + // visionOS 26 hands a frame one drawable per target (the built-in display, and a + // capture target while the view is recorded or streamed); a layer with foveation + // or a render quality set aborts the app if the single-drawable query is used. + let drawables: [LayerRenderer.Drawable] + if #available(visionOS 26.0, *) { + drawables = frame.queryDrawables() + } else { + drawables = frame.queryDrawable().map { [$0] } ?? [] + } + guard !drawables.isEmpty else { #if ENGINE_STATS_ENABLED renderer.finalizeXRStatsAndMonitors(frameStartTime: xrFrameStartTime) #else renderer.finalizeXRStatsAndMonitors(frameStartTime: 0.0) #endif - // queryDrawable() can fail during shutdown/teardown; in that case the + // No drawables (shutdown/teardown, or the compositor cancelled the frame): the // frame is already invalid and endSubmission() must not be called. shouldEndSubmission = false return } - // 10. Fetch the predicted device anchor from ARKit using the frameTiming information, and - // apply the anchor to your frame - let presentationInstant = drawable.frameTiming.presentationTime - let presentationTimeCA: TimeInterval = compositorInstantToCATime(presentationInstant) - var deviceAnchor = queryDeviceAnchorIfTrackingRunning(atTimestamp: presentationTimeCA) - - // If predicted-time query misses, retry at "now" to reduce one-frame anchor gaps. - if deviceAnchor == nil { - let nowTimestamp = CACurrentMediaTime() - if let retryAnchor = queryDeviceAnchorIfTrackingRunning(atTimestamp: nowTimestamp) { - deviceAnchor = retryAnchor - } - } - - // Use current anchor if valid, otherwise fall back to last valid anchor. - // This prevents "Presenting a drawable without a device anchor" while maintaining - // stable rendering. We must always present the drawable once we have it. - if let anchor = deviceAnchor { - if missingAnchorFrameCount > 0 { - print("✓ XR device anchor recovered after \(missingAnchorFrameCount) missing frame(s)") - } - missingAnchorFrameCount = 0 - lastValidDeviceAnchor = anchor - drawable.deviceAnchor = anchor - } else if let cachedAnchor = lastValidDeviceAnchor { - missingAnchorFrameCount += 1 - if shouldLogAnchorDiagnostics() { - printXRAnchorDiagnostics(message: "XR device anchor missing; using cached anchor") - } - drawable.deviceAnchor = cachedAnchor - } else { - missingAnchorFrameCount += 1 - if shouldLogAnchorDiagnostics() { - printXRAnchorDiagnostics(message: "XR device anchor missing; no cached anchor available") - } - } - // Note: If we have no cached anchor either, drawable.deviceAnchor remains nil - // and the render loop will skip rendering but still present (required by compositor) - // Re-check loading state right before render pass to catch any mutations that started // after our initial snapshot. This provides a second layer of protection against races. let loadingNow = AssetLoadingGate.shared.isLoadingAny let effectiveLoading = loading || loadingNow - executeXRSystemPass(frame: frame, drawable: drawable, loading: effectiveLoading) + for drawable in drawables { + // 10. Fetch the predicted device anchor from ARKit using the frameTiming information, and + // apply the anchor to your frame + let presentationInstant = drawable.frameTiming.presentationTime + let presentationTimeCA: TimeInterval = compositorInstantToCATime(presentationInstant) + var deviceAnchor = queryDeviceAnchorIfTrackingRunning(atTimestamp: presentationTimeCA) + + // If predicted-time query misses, retry at "now" to reduce one-frame anchor gaps. + if deviceAnchor == nil { + let nowTimestamp = CACurrentMediaTime() + if let retryAnchor = queryDeviceAnchorIfTrackingRunning(atTimestamp: nowTimestamp) { + deviceAnchor = retryAnchor + } + } + + // Use current anchor if valid, otherwise fall back to last valid anchor. + // This prevents "Presenting a drawable without a device anchor" while maintaining + // stable rendering. We must always present the drawable once we have it. + if let anchor = deviceAnchor { + if missingAnchorFrameCount > 0 { + print("✓ XR device anchor recovered after \(missingAnchorFrameCount) missing frame(s)") + } + missingAnchorFrameCount = 0 + lastValidDeviceAnchor = anchor + drawable.deviceAnchor = anchor + } else if let cachedAnchor = lastValidDeviceAnchor { + missingAnchorFrameCount += 1 + if shouldLogAnchorDiagnostics() { + printXRAnchorDiagnostics(message: "XR device anchor missing; using cached anchor") + } + drawable.deviceAnchor = cachedAnchor + } else { + missingAnchorFrameCount += 1 + if shouldLogAnchorDiagnostics() { + printXRAnchorDiagnostics(message: "XR device anchor missing; no cached anchor available") + } + } + // Note: If we have no cached anchor either, drawable.deviceAnchor remains nil + // and the render loop will skip rendering but still present (required by compositor) + + if drawsScene(into: drawable) { + executeXRSystemPass(frame: frame, drawable: drawable, loading: effectiveLoading) + } else { + presentCleared(drawable) + } + } #if ENGINE_STATS_ENABLED renderer.finalizeXRStatsAndMonitors(frameStartTime: xrFrameStartTime) #else @@ -681,6 +706,51 @@ #endif } + /// Whether the scene is drawn into a drawable. The built-in display always; another + /// target (a capture of the view for recording or streaming) only while its textures are + /// the size the frame's resources are allocated at — every sizeable resource is sized + /// to one viewport, and a second size would reallocate them twice a frame. + private func drawsScene(into drawable: LayerRenderer.Drawable) -> Bool { + if #available(visionOS 26.0, *), drawable.target != .builtIn, + let texture = drawable.colorTextures.first, + renderInfo.viewPort != simd_float2(Float(texture.width), Float(texture.height)) + { + if !loggedCaptureTargetSkip { + loggedCaptureTargetSkip = true + print("XR: a \(texture.width)×\(texture.height) capture drawable differs from the \(Int(renderInfo.viewPort.x))×\(Int(renderInfo.viewPort.y)) frame resources; presenting it cleared") + } + return false + } + return true + } + + /// Presents a drawable with its colour cleared (the immersion mode's alpha) and its depth + /// cleared, without drawing the scene: the compositor wants every drawable it handed out + /// presented. + private func presentCleared(_ drawable: LayerRenderer.Drawable) { + guard let commandBuffer = renderInfo.commandQueue.makeCommandBuffer() else { return } + commandBuffer.label = "XR Cleared Drawable" + for viewIndex in drawable.views.indices + where viewIndex < drawable.colorTextures.count && viewIndex < drawable.depthTextures.count + { + let descriptor = MTLRenderPassDescriptor() + descriptor.colorAttachments[0].texture = drawable.colorTextures[viewIndex] + descriptor.colorAttachments[0].loadAction = .clear + descriptor.colorAttachments[0].storeAction = .store + descriptor.colorAttachments[0].clearColor = MTLClearColor(red: 0, green: 0, blue: 0, alpha: Double(getAlphaForImmersionMode())) + descriptor.depthAttachment.texture = drawable.depthTextures[viewIndex] + descriptor.depthAttachment.loadAction = .clear + descriptor.depthAttachment.storeAction = .store + descriptor.depthAttachment.clearDepth = 0.0 + if viewIndex < drawable.rasterizationRateMaps.count { + descriptor.rasterizationRateMap = drawable.rasterizationRateMaps[viewIndex] + } + commandBuffer.makeRenderCommandEncoder(descriptor: descriptor)?.endEncoding() + } + drawable.encodePresent(commandBuffer: commandBuffer) + commandBuffer.commit() + } + private func updateSpatialInputState() { spatialGestureRecognizer.updateSpatialInputState() } @@ -756,6 +826,65 @@ spatialGestureRecognizer.resetAllSpatialInteractionTracking() } + /// The rasterization rate map the compositor made for a view, as the renderer draws + /// through it, or nil when the frame is drawn uniformly: the layer configured without + /// foveation (no maps; also the simulator) or `setRendering(.foveatedRendering(.disabled))`. + /// The map's parameters go into this frame's slot of the ring for the shaders that decode + /// them; the viewport is the view's in the map's screen space. + private func makeFoveationFrame( + drawable: LayerRenderer.Drawable, + viewport: MTLViewport, + viewIndex: Int, + slot: Int + ) -> XRFoveationFrame? { + let rateMaps = drawable.rasterizationRateMaps + let available = xrFoveatedRenderingEnabled && viewIndex < rateMaps.count && viewIndex < stereoEyeCount + if loggedFoveationState != available { + loggedFoveationState = available + if available { + let rateMap = rateMaps[viewIndex] + let screen = rateMap.screenSize + let physical = rateMap.physicalSize(layer: 0) + print("✓ XR foveation: drawing through the compositor's rasterization rate maps, screen \(screen.width)×\(screen.height), textures \(physical.width)×\(physical.height) per eye; view texture map viewport \(viewport.originX),\(viewport.originY) \(viewport.width)×\(viewport.height)") + } else if rateMaps.isEmpty { + print("XR foveation off: the layer provides no rasterization rate maps (isFoveationEnabled is false, or the simulator); drawing uniformly") + } else { + print("XR foveation off by setting (.foveatedRendering(.disabled)); the layer's rate maps are ignored and the picture will be warped") + } + } + guard available else { return nil } + + let rateMap = rateMaps[viewIndex] + let sizeAndAlign = rateMap.parameterDataSizeAndAlign + let align = max(1, sizeAndAlign.align) + let length = max(16, (sizeAndAlign.size + align - 1) / align * align) + if rateMapParameterBuffers[slot][viewIndex].map({ $0.length < length }) ?? true { + let buffer = renderInfo.device.makeBuffer(length: length, options: .storageModeShared) + buffer?.label = "XR Rate Map Parameters eye \(viewIndex) slot \(slot)" + rateMapParameterBuffers[slot][viewIndex] = buffer + } + guard let buffer = rateMapParameterBuffers[slot][viewIndex], + let data = XRRasterizationRateMapData(rateMap: rateMap, buffer: buffer) + else { + Logger.logWarning(message: "XR foveation: could not copy the rate map parameters for eye \(viewIndex); drawing this eye uniformly") + return nil + } + if let texture = drawable.colorTextures.indices.contains(viewIndex) ? drawable.colorTextures[viewIndex] : nil, + Int(data.physicalSize.x) != texture.width || Int(data.physicalSize.y) != texture.height + { + Logger.logWarning(message: "XR foveation: eye \(viewIndex) rate map physical size \(data.physicalSize) differs from its texture \(texture.width)×\(texture.height)") + } + // The eye's viewport in the map's screen space. In the dedicated layout the view + // is the whole texture, so it is the map's screen size at the origin; the view's own + // texture map viewport is logged above for the day another layout needs it. + let screenViewport = MTLViewport( + originX: 0, originY: 0, + width: Double(data.screenSize.x), height: Double(data.screenSize.y), + znear: 0, zfar: 1 + ) + return XRFoveationFrame(rateMap: rateMap, viewport: screenViewport, rateMapData: data) + } + func executeXRSystemPass(frame _: LayerRenderer.Frame, drawable: LayerRenderer.Drawable, loading: Bool) { // Wait for available command buffer slot to prevent unbounded memory growth let semaphoreWaitStart = CACurrentMediaTime() @@ -781,8 +910,12 @@ let renderTotalStart = CACurrentMediaTime() #endif renderInfo.currentInFlightFrameSlot = acquireUniformFrameSlot() + let rateMapSlot = rateMapFrameCounter % rateMapParameterBuffers.count + rateMapFrameCounter += 1 - // Update viewport to match actual drawable size (per-eye texture dimensions) + // Update viewport to match actual drawable size (per-eye texture dimensions). With + // foveation on, that is the rate map's physical size: every texture of the frame is + // allocated at it, while viewports and clip space live in the map's screen size. if let firstColorTexture = drawable.colorTextures.first { let actualViewPort = simd_float2(Float(firstColorTexture.width), Float(firstColorTexture.height)) if renderInfo.viewPort != actualViewPort { @@ -867,6 +1000,15 @@ } renderInfo.currentEye = viewIndex + // The eye's rasterization rate map (foveation), for the passes that draw the + // scene; the final pass into the compositor's textures copies physical pixels + // one to one and keeps passDescriptor without it. + renderInfo.xrFoveation = makeFoveationFrame( + drawable: drawable, + viewport: view.textureMap.viewport, + viewIndex: viewIndex, + slot: rateMapSlot + ) #if ENGINE_STATS_ENABLED let encodeStart = CACurrentMediaTime() diff --git a/Tests/UntoldEngineRenderTests/GaussianChunkCullTest.swift b/Tests/UntoldEngineRenderTests/GaussianChunkCullTest.swift index 481e6b99c..82bf2b0c3 100644 --- a/Tests/UntoldEngineRenderTests/GaussianChunkCullTest.swift +++ b/Tests/UntoldEngineRenderTests/GaussianChunkCullTest.swift @@ -1095,7 +1095,7 @@ final class GaussianChunkCullTest: BaseRenderSetup { XCTAssertEqual(constants.hzbOcclusionBias, gaussianCullHZBOcclusionBias) XCTAssertEqual(constants.uniformQuotas, 0, "the weighted quotas are the default") XCTAssertEqual(gaussianChunkCullConstants(chunkTable: table, modelMatrix: world.space, viewMatrix: matrix_identity_float4x4, hzbValid: false, uniformQuotas: true).uniformQuotas, 1) - XCTAssertEqual(MemoryLayout.stride, 192) + XCTAssertEqual(MemoryLayout.stride, 208) XCTAssertEqual(MemoryLayout.stride, 16) XCTAssertEqual(MemoryLayout.stride, 48) XCTAssertEqual(MemoryLayout.stride, 48) diff --git a/Tests/UntoldEngineRenderTests/GaussianChunkLevelTest.swift b/Tests/UntoldEngineRenderTests/GaussianChunkLevelTest.swift index f80387fc7..ae60e1d9e 100644 --- a/Tests/UntoldEngineRenderTests/GaussianChunkLevelTest.swift +++ b/Tests/UntoldEngineRenderTests/GaussianChunkLevelTest.swift @@ -1888,7 +1888,7 @@ final class GaussianChunkLevelTest: BaseRenderSetup { XCTAssertEqual(MemoryLayout.stride, 16) XCTAssertEqual(MemoryLayout.offset(of: \.coarseAvailable), 12) XCTAssertEqual(MemoryLayout.stride, 32) - XCTAssertEqual(MemoryLayout.stride, 192) + XCTAssertEqual(MemoryLayout.stride, 208) XCTAssertEqual(MemoryLayout.stride, 48) XCTAssertEqual(MemoryLayout.stride, 64) XCTAssertEqual(MemoryLayout.stride, 48) diff --git a/Tests/UntoldEngineRenderTests/GaussianPagingTest.swift b/Tests/UntoldEngineRenderTests/GaussianPagingTest.swift index 158a67565..d26e253ac 100644 --- a/Tests/UntoldEngineRenderTests/GaussianPagingTest.swift +++ b/Tests/UntoldEngineRenderTests/GaussianPagingTest.swift @@ -1784,9 +1784,10 @@ final class GaussianPagingTest: BaseRenderSetup { XCTAssertEqual(MemoryLayout.offset(of: \.frameIndex), 8) XCTAssertEqual(MemoryLayout.offset(of: \.fadeFrames), 12) XCTAssertEqual(MemoryLayout.offset(of: \.debugMode), 16) - XCTAssertEqual(MemoryLayout.stride, 192) + XCTAssertEqual(MemoryLayout.stride, 208) XCTAssertEqual(MemoryLayout.offset(of: \.paged), 172) XCTAssertEqual(MemoryLayout.offset(of: \.rangeCount), 176) + XCTAssertEqual(MemoryLayout.offset(of: \.hzbRateMapSizes), 192) XCTAssertEqual(gaussianChunkCullResidencyIndex.rawValue, 8) XCTAssertEqual(gaussianChunkCullDemandIndex.rawValue, 9) XCTAssertEqual(gaussianChunkPreprocessResidencyIndex.rawValue, 13) diff --git a/Tests/UntoldEngineRenderTests/GaussianScreenWeightedQuotaTest.swift b/Tests/UntoldEngineRenderTests/GaussianScreenWeightedQuotaTest.swift index 953c6758c..3e61dce6f 100644 --- a/Tests/UntoldEngineRenderTests/GaussianScreenWeightedQuotaTest.swift +++ b/Tests/UntoldEngineRenderTests/GaussianScreenWeightedQuotaTest.swift @@ -320,7 +320,8 @@ final class GaussianScreenWeightedQuotaTest: BaseRenderSetup { /// The shared layouts the kernels and the readbacks depend on. func testLayoutsArePinned() { - XCTAssertEqual(MemoryLayout.stride, 192) + XCTAssertEqual(MemoryLayout.stride, 208) + XCTAssertEqual(MemoryLayout.offset(of: \.hzbRateMapSizes), 192) XCTAssertEqual(MemoryLayout.stride, 16) XCTAssertEqual(MemoryLayout.stride, 48) XCTAssertEqual(MemoryLayout.stride, 48) diff --git a/Tests/UntoldEngineRenderTests/XRFoveationTests.swift b/Tests/UntoldEngineRenderTests/XRFoveationTests.swift new file mode 100644 index 000000000..208f709be --- /dev/null +++ b/Tests/UntoldEngineRenderTests/XRFoveationTests.swift @@ -0,0 +1,204 @@ +// +// XRFoveationTests.swift +// UntoldEngine +// +// Drawing through a rasterization rate map (visionOS foveation, XRFoveation.swift): the map's +// parameter data the shaders decode, which pass descriptors carry the map and which do not, +// and the occlusion cull sampling a pyramid drawn through a map where the map put the rect, +// not where the rect's screen UV falls in the texture. +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import CShaderTypes +import Metal +import simd +@testable import UntoldEngine +import XCTest + +@MainActor +final class XRFoveationTests: BaseRenderSetup { + private let screen = 256 + + override func initializeAssets() {} + + override func tearDown() async throws { + renderInfo.xrFoveation = nil + applyXRFoveationToSceneRenderPassDescriptors() + try await super.tearDown() + } + + // MARK: - Helpers + + /// A map over a 256-pixel-square screen whose right half is drawn at a quarter of the rate + /// of its left half, so the physical texture is narrower than the screen and a screen + /// position in the right half lands at a different texel than its UV says. + private func makeRateMap() throws -> MTLRasterizationRateMap { + let device = try XCTUnwrap(renderInfo.device) + guard device.supportsRasterizationRateMap(layerCount: 1) else { + throw XCTSkip("this GPU has no rasterization rate maps") + } + let descriptor = MTLRasterizationRateMapDescriptor(screenSize: MTLSize(width: screen, height: screen, depth: 1)) + let layer = MTLRasterizationRateLayerDescriptor(sampleCount: MTLSize(width: 2, height: 1, depth: 1)) + layer.horizontal[0] = 1.0 + layer.horizontal[1] = 0.25 + layer.vertical[0] = 1.0 + descriptor.setLayer(layer, at: 0) + descriptor.label = "XRFoveationTests" + return try XCTUnwrap(device.makeRasterizationRateMap(descriptor: descriptor)) + } + + private func makeRateMapData(_ rateMap: MTLRasterizationRateMap) throws -> XRRasterizationRateMapData { + let device = try XCTUnwrap(renderInfo.device) + let buffer = try XCTUnwrap(device.makeBuffer(length: rateMap.parameterDataSizeAndAlign.size, options: .storageModeShared)) + return try XCTUnwrap(XRRasterizationRateMapData(rateMap: rateMap, buffer: buffer)) + } + + // MARK: - Tests + + func testRateMapDataCarriesTheSizesAndParameters() throws { + let rateMap = try makeRateMap() + let data = try makeRateMapData(rateMap) + + let physical = rateMap.physicalSize(layer: 0) + XCTAssertEqual(data.screenSize, simd_float2(Float(screen), Float(screen))) + XCTAssertEqual(data.physicalSize, simd_float2(Float(physical.width), Float(physical.height))) + XCTAssertLessThan(physical.width, screen, "a quarter-rate half narrows the physical texture") + XCTAssertEqual(data.sizes, simd_float4(Float(screen), Float(screen), Float(physical.width), Float(physical.height))) + XCTAssertEqual(rateMapSizes(nil), .zero, "the word the shaders read as 'no map'") + + let bytes = UnsafeRawBufferPointer(start: data.buffer.contents(), count: rateMap.parameterDataSizeAndAlign.size) + XCTAssertTrue(bytes.contains { $0 != 0 }, "the map's parameters were copied") + + let short = try XCTUnwrap(renderInfo.device.makeBuffer(length: 4, options: .storageModeShared)) + XCTAssertNil(XRRasterizationRateMapData(rateMap: rateMap, buffer: short), "a buffer too short for the parameters is refused") + } + + func testScenePassesCarryTheRateMapAndScreenPassesDoNot() throws { + let rateMap = try makeRateMap() + let data = try makeRateMapData(rateMap) + let viewport = MTLViewport(originX: 0, originY: 0, width: Double(screen), height: Double(screen), znear: 0, zfar: 1) + + renderInfo.xrFoveation = XRFoveationFrame(rateMap: rateMap, viewport: viewport, rateMapData: data) + applyXRFoveationToSceneRenderPassDescriptors() + + XCTAssertTrue(renderInfo.offscreenRenderPassDescriptor.rasterizationRateMap === rateMap, "the G-buffer + light pass draws through the map") + XCTAssertTrue(renderInfo.deferredRenderPassDescriptor.rasterizationRateMap === rateMap, "transparency, wireframe and debug draw through the map") + XCTAssertTrue(renderInfo.environmentRenderPassDescriptor.rasterizationRateMap === rateMap, "the background draws through the map") + XCTAssertTrue(renderInfo.gaussianRenderPassDescriptor.rasterizationRateMap === rateMap, "the splats draw through the map") + XCTAssertNil(renderInfo.sceneCompositeRenderPassDescriptor.rasterizationRateMap, "the composite copies physical pixels one to one") + XCTAssertNil(renderInfo.ssaoRenderPassDescriptor.rasterizationRateMap, "SSAO reads the physical depth one to one") + XCTAssertNil(renderInfo.postProcessRenderPassDescriptor?.rasterizationRateMap, "post-processing copies physical pixels one to one") + // A render target size is read in screen pixels under a map: the physical size would + // clip the scene to the top-left corner of the screen, so the size is left to the + // attachments while the map is attached. + XCTAssertEqual(renderInfo.offscreenRenderPassDescriptor.renderTargetWidth, 0) + XCTAssertEqual(renderInfo.deferredRenderPassDescriptor.renderTargetHeight, 0) + XCTAssertEqual(renderInfo.sceneCompositeRenderPassDescriptor.renderTargetWidth, Int(renderInfo.viewPort.x), "a screen pass keeps its texture-size target") + + renderInfo.xrFoveation = nil + applyXRFoveationToSceneRenderPassDescriptors() + XCTAssertNil(renderInfo.offscreenRenderPassDescriptor.rasterizationRateMap, "a uniform frame detaches the map") + XCTAssertEqual(renderInfo.offscreenRenderPassDescriptor.renderTargetWidth, Int(renderInfo.viewPort.x), "and puts the texture-size target back") + XCTAssertEqual(renderInfo.deferredRenderPassDescriptor.renderTargetHeight, Int(renderInfo.viewPort.y)) + XCTAssertNil(renderInfo.deferredRenderPassDescriptor.rasterizationRateMap) + XCTAssertNil(renderInfo.environmentRenderPassDescriptor.rasterizationRateMap) + XCTAssertNil(renderInfo.gaussianRenderPassDescriptor.rasterizationRateMap) + } + + /// A box in the right half of the screen, in front of a pyramid that holds an occluder only + /// where the map put that half. Read where the box's screen UV falls in the texture, the + /// pyramid shows background and the box is kept; moved through the map, the rect lies on + /// the occluder and the box is dropped. + func testOcclusionCullSamplesThePyramidWhereTheRateMapPutTheRect() throws { + let rateMap = try makeRateMap() + let data = try makeRateMapData(rateMap) + let device = try XCTUnwrap(renderInfo.device) + let physical = rateMap.physicalSize(layer: 0) + + // The box: NDC x in 0.5…0.9 — screen pixels 192…243 of 256 — a sliver of y, at depth 0.5. + let box = VisibleEntity(center: simd_float4(0.7, 0, 0.5, 1), halfExtent: simd_float4(0.2, 0.05, 0, 0), index: 1, version: 1) + // Where the rect's left edge lands in the texture by Metal's own mapping, against where + // its screen UV (0.75) would be read without the map. + let mappedMinX = rateMap.physicalCoordinates(screenCoordinates: MTLCoordinate2D(x: 192, y: 128), layer: 0).x + let unmappedMinX = 0.75 * Float(physical.width) + XCTAssertGreaterThan(mappedMinX, unmappedMinX + 2, "the quarter-rate half moves the rect to the right in the texture") + + // One mip: an occluder (0.9, near in reverse-Z) from just before the mapped rect to the + // right edge, background (0) everywhere else. + let occluderStart = max(0, Int(mappedMinX.rounded(.down)) - 1) + let pyramidDescriptor = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: .r32Float, width: physical.width, height: physical.height, mipmapped: false) + pyramidDescriptor.usage = [.shaderRead] + pyramidDescriptor.storageMode = .managed + let pyramid = try XCTUnwrap(device.makeTexture(descriptor: pyramidDescriptor)) + var row = [Float](repeating: 0, count: physical.width) + for x in occluderStart ..< physical.width { + row[x] = 0.9 + } + var texels = [Float]() + texels.reserveCapacity(physical.width * physical.height) + for _ in 0 ..< physical.height { + texels.append(contentsOf: row) + } + texels.withUnsafeBytes { bytes in + pyramid.replace( + region: MTLRegionMake2D(0, 0, physical.width, physical.height), + mipmapLevel: 0, + withBytes: bytes.baseAddress!, + bytesPerRow: physical.width * MemoryLayout.stride + ) + } + + // The frame state the cull reads, restored afterwards. + let savedViewPort = renderInfo.viewPort + let savedValid = renderInfo.hzbIsValid + let savedFrame = renderInfo.hzbFrame + let savedMipCount = renderInfo.hzbMipCount + let savedReverseZ = renderInfo.reverseZEnabled + let savedPyramid = textureResources.hzbDepthPyramid + defer { + renderInfo.viewPort = savedViewPort + renderInfo.hzbIsValid = savedValid + renderInfo.hzbFrame = savedFrame + renderInfo.hzbMipCount = savedMipCount + renderInfo.reverseZEnabled = savedReverseZ + textureResources.hzbDepthPyramid = savedPyramid + } + renderInfo.viewPort = simd_float2(Float(physical.width), Float(physical.height)) + renderInfo.hzbIsValid = true + renderInfo.hzbMipCount = 1 + renderInfo.reverseZEnabled = true + textureResources.hzbDepthPyramid = pyramid + + func survivors(rateMapData: XRRasterizationRateMapData?) throws -> Int { + renderInfo.hzbFrame = HZBPyramidFrame(viewProjection: matrix_identity_float4x4, cameraPosition: .zero, rateMapData: rateMapData) + var candidate = box + var candidateCount: UInt32 = 1 + let inputVisibility = try XCTUnwrap(device.makeBuffer(bytes: &candidate, length: MemoryLayout.stride, options: .storageModeShared)) + let inputCount = try XCTUnwrap(device.makeBuffer(bytes: &candidateCount, length: MemoryLayout.stride, options: .storageModeShared)) + let outputVisibility = try XCTUnwrap(device.makeBuffer(length: MemoryLayout.stride, options: .storageModeShared)) + let outputCount = try XCTUnwrap(device.makeBuffer(length: MemoryLayout.stride, options: .storageModeShared)) + + let commandBuffer = try XCTUnwrap(renderInfo.commandQueue.makeCommandBuffer()) + let ran = executeHZBOcclusionCulling( + commandBuffer, + cameraPosition: .zero, + dispatchCount: 1, + inputVisibilityBuffer: inputVisibility, + inputVisibleCountBuffer: inputCount, + outputVisibilityBuffer: outputVisibility, + outputVisibleCountBuffer: outputCount + ) + XCTAssertTrue(ran, "the pyramid is valid and the camera has not moved") + commandBuffer.commit() + commandBuffer.waitUntilCompleted() + return Int(outputCount.contents().load(as: UInt32.self)) + } + + XCTAssertEqual(try survivors(rateMapData: nil), 1, "read at its screen UV, the rect sees background and the box is kept") + XCTAssertEqual(try survivors(rateMapData: data), 0, "moved through the map, the rect lies on the occluder and the box is dropped") + } +} diff --git a/docs/API/UsingEngineAPI.md b/docs/API/UsingEngineAPI.md index 7889c439f..19b17b9c1 100644 --- a/docs/API/UsingEngineAPI.md +++ b/docs/API/UsingEngineAPI.md @@ -82,6 +82,17 @@ setRendering(.occlusionCullingMaxCameraStep(0.5)) getOcclusionCullingMaxCameraStep() ``` +On visionOS, a layer configured with `isFoveationEnabled` hands the engine a rasterization +rate map per eye, and the engine draws through it by default: dense where the eyes look, +coarse in the periphery, and the only configuration in which the compositor accepts a +`maxRenderQuality` above the platform default. Disabling it in such a layer is a diagnostic +(the frame is drawn uniformly into textures the compositor still unwarps): + +```swift +setRendering(.foveatedRendering(.disabled)) +getFoveatedRenderingEnabled() +``` + ## PostFX Use `setPostFX` for individual post-processing and SSAO settings: diff --git a/docs/Architecture/xrRenderingSystem.md b/docs/Architecture/xrRenderingSystem.md index b4c3059ae..19ffdc1e0 100644 --- a/docs/Architecture/xrRenderingSystem.md +++ b/docs/Architecture/xrRenderingSystem.md @@ -100,6 +100,8 @@ defer { frame.endSubmission() } The `defer` is important: `endSubmission()` **must** be called even if rendering fails partway through. If it isn't called, the compositor stalls. The `defer` guarantees this regardless of how the function exits. +**Drawables.** On visionOS 26 the frame is queried with `frame.queryDrawables()`, which returns one drawable per target: the built-in display, plus a capture target while the view is being recorded or streamed. A layer configured with foveation or a render quality aborts the app ("BUG IN CLIENT: called cp_frame_query_drawable() which does not support setting render quality or fovea") if the older single-drawable `queryDrawable()` is used, which the engine still calls on visionOS 2. An empty array means the compositor cancelled the frame: it is left without `endSubmission()`. Every drawable is presented; the scene is drawn into the built-in one and into a capture drawable only while its textures are the size the frame's resources are allocated at — otherwise it is presented cleared, since every sizeable resource is sized to one viewport and a second size would reallocate them twice a frame. + ### 2e. Device Anchor Acquisition ```swift @@ -167,6 +169,15 @@ The compositor provides the exact asymmetric projection for each eye. This accou - `renderInfo.currentEye = viewIndex` — tells uniform uploads which eye's matrices to use - The base pass mode: `.mixed` immersion omits the base pass (camera passthrough is the background), `.full` immersion renders the skybox +**Foveation (rasterization rate maps).** With `isFoveationEnabled` in the app's `CompositorLayerConfiguration`, the compositor hands each eye a `MTLRasterizationRateMap` and colour/depth textures of the map's *physical* size, smaller than the *screen* the eye sees; the GPU draws through the map (dense where the eyes look, coarse in the periphery) and the compositor unwarps the texture for the display. It is also the only configuration in which the compositor accepts a `maxRenderQuality` above the platform default. The engine supports it as follows (`XRFoveation.swift`, on by default, `setRendering(.foveatedRendering(.disabled))` to ignore the maps for diagnosis): + +- `executeXRSystemPass` makes an `XRFoveationFrame` per eye (`renderInfo.xrFoveation`): the map, the eye's viewport in the map's screen space (`view.textureMap.viewport`), and the map's parameter data copied into a ring of shared buffers for the shaders. `renderInfo.viewPort` stays the texture size, the physical size, so every sizeable resource is allocated at it. +- After the graph is built, `applyXRFoveationToSceneRenderPassDescriptors` attaches the map to the descriptors of the passes that rasterize the scene at drawable size (G-buffer + light, the deferred colour that transparency/wireframe/debug draw into, environment/sky/grid, splats, gizmos), and each of those encoders sets the eye's viewport (`applyXRFoveationViewport`) in the map's screen space. Under a map, Metal reads a descriptor's `renderTargetWidth/Height` and the viewport in *screen* pixels (measured on a Mac GPU, and seen on the headset as a screen-fixed rectangle outside which nothing was drawn), so the attached descriptors leave the target size at 0 — the attachments' own size — and get it back when the map is detached. The full-screen passes (SSAO, post-processing, anti-aliasing, the output transform into the compositor's texture) keep plain descriptors: they copy physical pixels one to one and the layout is already the map's. +- Two kinds of shader decode the map (`rasterization_rate_map_decoder`): SSAO, which rebuilds view positions from a depth texel's UV and must use the texel's *screen* position; and the occlusion culls (`hzbCullVisibleEntities`, `gaussianChunkCull`, the per-splat test), which project boxes and splats to screen UV and sample a pyramid laid out in physical space — the rect or point goes through the map first (`HZBOcclusion.h`). The pyramid carries the map it was drawn through into the next frame's culls (`HZBPyramidFrame.rateMapData`). +- The Gaussian fragment takes its offset from the splat centre from a varying rather than from `[[position]]`, which the rate map places in the physical texture. Fragment position is physical everywhere else too, which is what every one-to-one texture read wants; `SmallObjectCulling` measures its pixel threshold against the screen height. + +Without foveation (the layer configured without it, the simulator), `drawable.rasterizationRateMaps` is empty and the frame is drawn uniformly exactly as before. + ### 3c. HZB Pyramid (built once after both eyes) ```swift @@ -220,7 +231,7 @@ Selection rays update from interaction events, not continuously as the user look ├─ frame.endUpdate() ├─ wait(until: optimalInputTime) (sleep until compositor deadline) ├─ frame.startSubmission() - ├─ queryDrawable() (get per-eye textures) + ├─ queryDrawables() (one drawable per target; the old single query aborts a foveated layer) ├─ queryDeviceAnchor() → fallback chain → drawable.deviceAnchor = anchor │ └─ executeXRSystemPass()