From ecb831f8703d56a1b3764adc9f701257e021d7f6 Mon Sep 17 00:00:00 2001 From: Javier Segura Date: Wed, 14 Oct 2026 21:09:44 +0200 Subject: [PATCH 01/22] [Feature] The frame is posed for its predicted presentation time, and the game receives it On the headset the head pose is predicted for the instant the frame will be shown, but every object was posed for "now at update time", a varying lag behind: a moving object stepped unevenly from frame to frame and read as judder at full frame rate. The XR loop now converts the compositor's predicted presentation time (through the compositor clock's epoch, the boot, like CACurrentMediaTime) and hands it to updateXR(presentationTime:); the engine's delta is the time between two presentations, not between two updates. FrameTime (presentation time, delta, period, skipped refreshes, frame index) and FrameClock, which learns the period and counts the refreshes a frame skipped, make the frame's time. It reaches the game through setupCallbacks(frameUpdate:handleInput:), renderer.frameTime and the currentFrameTime global; gameUpdate(deltaTime:) keeps its parameter. UpdateEvent, EngineExtensionUpdateContext, RenderGraphBuildContext and RenderPassContext gain a frameTime field with a default. The MTKView and AR loops pose the frame one of the view's periods ahead of the draw, or for the host's display-link target set in predictedPresentationTime. The engine stats report the presentation delta and the skipped refreshes; the XR loop counts the frames whose drawable's presentation time differs from the prediction. --- .../UntoldEngine/ECS/EngineExtensions.swift | 7 +- .../Profiling/EngineStatsSnapshot.swift | 12 +- Sources/UntoldEngine/Renderer/FrameTime.swift | 127 ++++++++++++++++++ .../UntoldEngine/Renderer/UntoldEngine.swift | 116 +++++++++++----- .../UntoldEngine/Systems/FrameEvents.swift | 10 +- .../UntoldEngine/Systems/GraphBuilder.swift | 23 +++- Sources/UntoldEngine/Utils/Globals.swift | 22 +++ Sources/UntoldEngineAR/UntoldEngineAR.swift | 13 +- Sources/UntoldEngineXR/UntoldEngineXR.swift | 75 +++++++++-- .../BaseRenderSetup.swift | 1 + Tests/UntoldEngineTests/FrameClockTests.swift | 121 +++++++++++++++++ Tests/UntoldEngineTests/TestEngineReset.swift | 1 + 12 files changed, 474 insertions(+), 54 deletions(-) create mode 100644 Sources/UntoldEngine/Renderer/FrameTime.swift create mode 100644 Tests/UntoldEngineTests/FrameClockTests.swift diff --git a/Sources/UntoldEngine/ECS/EngineExtensions.swift b/Sources/UntoldEngine/ECS/EngineExtensions.swift index 019948701..e333213d9 100644 --- a/Sources/UntoldEngine/ECS/EngineExtensions.swift +++ b/Sources/UntoldEngine/ECS/EngineExtensions.swift @@ -17,19 +17,24 @@ public struct EngineExtensionUpdateContext: Sendable { public let frameIndex: UInt64 public let currentEye: Int public let isPrimaryEye: Bool + /// The frame's predicted presentation time, its delta from the frame before and the + /// refreshes skipped between them; `.zero` for a context built outside a frame. + public let frameTime: FrameTime public init( viewport: SIMD2, immersionStyle: UntoldImmersionMode, frameIndex: UInt64, currentEye: Int, - isPrimaryEye: Bool + isPrimaryEye: Bool, + frameTime: FrameTime = .zero ) { self.viewport = viewport self.immersionStyle = immersionStyle self.frameIndex = frameIndex self.currentEye = currentEye self.isPrimaryEye = isPrimaryEye + self.frameTime = frameTime } } diff --git a/Sources/UntoldEngine/Profiling/EngineStatsSnapshot.swift b/Sources/UntoldEngine/Profiling/EngineStatsSnapshot.swift index 4b503eac3..0ceb1821a 100644 --- a/Sources/UntoldEngine/Profiling/EngineStatsSnapshot.swift +++ b/Sources/UntoldEngine/Profiling/EngineStatsSnapshot.swift @@ -33,6 +33,12 @@ public struct EngineTimingStats { /// `encodeMs` is then the submission phase: the drawables' command buffers encoded and /// committed. The CPU's work in a frame is `updateMs` + `encodeMs`. public var xrWaitMs: Double = 0.0 + /// Milliseconds from the previous frame's predicted presentation to this one's: the + /// display's period on a steady cadence, two periods after a skipped frame. + public var presentationDeltaMs: Double = 0.0 + /// Refreshes between the previous frame and this one that showed the previous frame again + /// (`FrameTime.skippedFrames`). + public var skippedFrames: Int = 0 public var streamingRegionMs: Double = 0.0 public var geometryStreamingMs: Double = 0.0 public var batchingTickMs: Double = 0.0 @@ -53,7 +59,9 @@ public struct EngineTimingStats { streamingRegionMs: Double = 0.0, geometryStreamingMs: Double = 0.0, batchingTickMs: Double = 0.0, - batchingRebuildMs: Double = 0.0 + batchingRebuildMs: Double = 0.0, + presentationDeltaMs: Double = 0.0, + skippedFrames: Int = 0 ) { self.frameTotalMs = frameTotalMs self.smoothedFrameMs = smoothedFrameMs @@ -70,6 +78,8 @@ public struct EngineTimingStats { self.geometryStreamingMs = geometryStreamingMs self.batchingTickMs = batchingTickMs self.batchingRebuildMs = batchingRebuildMs + self.presentationDeltaMs = presentationDeltaMs + self.skippedFrames = skippedFrames } } diff --git a/Sources/UntoldEngine/Renderer/FrameTime.swift b/Sources/UntoldEngine/Renderer/FrameTime.swift new file mode 100644 index 000000000..d747fedcd --- /dev/null +++ b/Sources/UntoldEngine/Renderer/FrameTime.swift @@ -0,0 +1,127 @@ +// +// FrameTime.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Foundation + +/// When the frame being prepared is predicted to reach the display, and how far that is from +/// the frame before. +/// +/// A headset frame is drawn with the head pose predicted for its presentation instant, so an +/// object posed for `presentationTime` keeps its place against the world while the head moves, +/// and one moving along a path steps by the display's period from one frame to the next +/// instead of by the varying time between two game updates. A frame the display skipped lands +/// where the timeline says, and the frames after it step by one period again. +/// +/// `presentationTime` is in the `CACurrentMediaTime()` base, which `ProcessInfo.systemUptime` +/// and ARKit's anchor timestamps share, so a game may keep its own timelines in that base. +public struct FrameTime: Sendable, Equatable { + /// Seconds at which the frame is predicted to be shown. + public let presentationTime: TimeInterval + /// Seconds from the previous frame's presentation to this one's: one period on a steady + /// cadence, more after a skipped frame, 0 on the first frame. + public let deltaTime: TimeInterval + /// The display's period as the loop knows it: the host's nominal frame interval, or the + /// shortest spacing between presentations seen so far (0 until one has been seen). + public let period: TimeInterval + /// Refreshes between the previous frame and this one at which the display showed the + /// previous frame again because no new one was ready. 0 on a steady cadence. + public let skippedFrames: Int + /// The engine's count of frames prepared, from 1. + public let frameIndex: UInt64 + + public init( + presentationTime: TimeInterval, + deltaTime: TimeInterval, + period: TimeInterval, + skippedFrames: Int, + frameIndex: UInt64 + ) { + self.presentationTime = presentationTime + self.deltaTime = deltaTime + self.period = period + self.skippedFrames = skippedFrames + self.frameIndex = frameIndex + } + + /// Whether the display repeated the previous frame before this one. + public var wasFrameSkipped: Bool { skippedFrames > 0 } + + /// No frame yet: every field zero. + public static let zero = FrameTime(presentationTime: 0, deltaTime: 0, period: 0, skippedFrames: 0, frameIndex: 0) +} + +/// Makes each frame's `FrameTime` from the presentation times a host loop predicts: the delta +/// between consecutive presentations, the period (the host's nominal one, or the shortest +/// spacing seen) and the refreshes skipped between two frames. +public struct FrameClock: Sendable, Equatable { + /// The most recent frame, or nil before the first. + public private(set) var current: FrameTime? + /// The period the host declares (1 / the view's preferred frames per second), or nil to + /// learn it from the presentation times. + public var nominalPeriod: TimeInterval? + private var learnedPeriod: TimeInterval? + private var frameIndex: UInt64 = 0 + + /// A spacing shorter than this is two frames for one refresh, not a period. + static let shortestPeriod: TimeInterval = 1.0 / 250.0 + /// A spacing longer than this is a stall, not a period. + static let longestPeriod: TimeInterval = 1.0 / 24.0 + + public init(nominalPeriod: TimeInterval? = nil) { + self.nominalPeriod = nominalPeriod + } + + /// The display's period as the clock knows it, 0 before it knows one. + public var period: TimeInterval { nominalPeriod ?? learnedPeriod ?? 0 } + + /// Advances to a frame predicted to be shown at `presentationTime` and returns its time. + @discardableResult + public mutating func advance(toPresentation presentationTime: TimeInterval) -> FrameTime { + frameIndex &+= 1 + guard let previous = current else { + let frame = FrameTime(presentationTime: presentationTime, deltaTime: 0, period: period, skippedFrames: 0, frameIndex: frameIndex) + current = frame + return frame + } + + // A presentation before the previous one is a clock going backwards: no time passes. + let delta = max(0, presentationTime - previous.presentationTime) + if nominalPeriod == nil, delta >= Self.shortestPeriod, delta <= Self.longestPeriod { + learnedPeriod = min(learnedPeriod ?? delta, delta) + } + + let period = period + var skipped = 0 + if period > 0, delta > 0 { + // The refreshes the delta spans, less the one every frame takes. + let refreshes = min(delta / period, 1_000_000).rounded() + skipped = max(0, Int(refreshes) - 1) + } + + let frame = FrameTime(presentationTime: presentationTime, deltaTime: delta, period: period, skippedFrames: skipped, frameIndex: frameIndex) + current = frame + return frame + } + + /// Advances to a frame the loop estimates `lead` ahead of `now`, for a host with no + /// predicted presentation time (an MTKView's draw): the delta is then the spacing of the + /// draws on the wall clock. + @discardableResult + public mutating func advance(now: TimeInterval, lead: TimeInterval) -> FrameTime { + advance(toPresentation: now + lead) + } + + /// Forgets every frame: the next one is a first frame again. + public mutating func reset() { + current = nil + learnedPeriod = nil + frameIndex = 0 + } +} diff --git a/Sources/UntoldEngine/Renderer/UntoldEngine.swift b/Sources/UntoldEngine/Renderer/UntoldEngine.swift index 88f511b35..53258cc4f 100644 --- a/Sources/UntoldEngine/Renderer/UntoldEngine.swift +++ b/Sources/UntoldEngine/Renderer/UntoldEngine.swift @@ -23,8 +23,21 @@ public class UntoldRenderer: NSObject, MTKViewDelegate { public let metalView: MTKView var gameUpdateCallback: ((_ deltaTime: Float) -> Void)? + var frameUpdateCallback: ((FrameTime) -> Void)? var handleInputCallback: (() -> Void)? + /// Makes each frame's `FrameTime` from the presentation time the host loop predicts. + var frameClock = FrameClock() + + /// A host that drives `draw(in:)` from its own display link sets the link's target + /// timestamp here before each draw; the frame is then posed for that instant instead of + /// one period ahead of the draw. The frame consumes it. + public var predictedPresentationTime: TimeInterval? + + /// The frame being prepared or drawn: its predicted presentation time, the delta from the + /// frame before and whether the display skipped a frame between them. + public var frameTime: FrameTime { currentFrameTime } + let frameEvents = FrameEventDispatcher() private var configuration: UntoldRendererConfig @@ -137,11 +150,27 @@ public class UntoldRenderer: NSObject, MTKViewDelegate { return renderer } + /// Installs the game's per-frame callbacks. `gameUpdate` receives the seconds from the + /// previous frame's presentation to this one's; the frame's own time is `frameTime` (or + /// the `currentFrameTime` global) while it runs. Replaces a `frameUpdate` callback. public func setupCallbacks( gameUpdate: @escaping (_ deltaTime: Float) -> Void, handleInput: @escaping () -> Void ) { gameUpdateCallback = gameUpdate + frameUpdateCallback = nil + handleInputCallback = handleInput + } + + /// Installs the game's per-frame callbacks with the frame's time: the predicted + /// presentation time to pose a moving object for, the delta from the frame before and + /// the refreshes skipped between them. Replaces a `gameUpdate` callback. + public func setupCallbacks( + frameUpdate: @escaping (_ frame: FrameTime) -> Void, + handleInput: @escaping () -> Void + ) { + frameUpdateCallback = frameUpdate + gameUpdateCallback = nil handleInputCallback = handleInput } @@ -221,38 +250,43 @@ public class UntoldRenderer: NSObject, MTKViewDelegate { reinitSSAOTextures() } - func calculateDeltaTime() { + /// The period assumed for a loop that neither predicts a presentation time nor names its + /// display's rate. + static let defaultFramePeriod: TimeInterval = 1.0 / 60.0 + + /// Advances the frame clock to the frame's predicted presentation time — the host's, or + /// one period ahead of now for a loop that predicts none — and derives the frame's delta + /// from it: the time between two presentations, not between two updates, so a path + /// stepped by the delta moves by the display's period from one frame to the next. + /// `nominalPeriod` is the host display's frame interval; nil lets the clock learn it from + /// the presentation times. + func calculateDeltaTime(presentationTime: TimeInterval?, nominalPeriod: TimeInterval?) { if !firstUpdateCall { - // init the time properties for the update - - timeSinceLastUpdate = 0.0 - - timeSinceLastUpdatePreviousTime = CACurrentMediaTime() - + // The first frame after a reset: no time has passed. + frameClock.reset() firstUpdateCall = true - - // init fps time properties frameCount = 0 timePassedSinceLastFrame = 0.0 + } + let frame: FrameTime + if let presentationTime { + frameClock.nominalPeriod = nominalPeriod + frame = frameClock.advance(toPresentation: presentationTime) } else { - // figure out the time since we last we drew - let currentTime: TimeInterval = CACurrentMediaTime() - - timeSinceLastUpdate = Float(currentTime - timeSinceLastUpdatePreviousTime) - - // keep track of the time interval between draws - timeSinceLastUpdatePreviousTime = currentTime - - // get fps - timePassedSinceLastFrame += Float(timeSinceLastUpdate) - - if timePassedSinceLastFrame > 0.1 { - // let fps:Float=Float(frameCount)/timePassedSinceLastFrame + let period = nominalPeriod ?? Self.defaultFramePeriod + frameClock.nominalPeriod = period + frame = frameClock.advance(now: CACurrentMediaTime(), lead: period) + } + currentFrameTime = frame + timeSinceLastUpdate = Float(frame.deltaTime) + timeSinceLastUpdatePreviousTime = frame.presentationTime - frameCount = 0 - timePassedSinceLastFrame = 0.0 - } + // fps bookkeeping + timePassedSinceLastFrame += Float(frame.deltaTime) + if timePassedSinceLastFrame > 0.1 { + frameCount = 0 + timePassedSinceLastFrame = 0.0 } } @@ -294,9 +328,12 @@ public class UntoldRenderer: NSObject, MTKViewDelegate { let drawStats = RenderStatsCollector.shared.snapshot() let tileRenderCosts = auditVisibleTileRenderCosts() let memStats = MemoryBudgetManager.shared.getStats() + let frame = currentFrameTime EngineStatsMonitor.shared.update { snapshot in snapshot.timing.frameTotalMs = frameTotalMs + snapshot.timing.presentationDeltaMs = frame.deltaTime * 1000.0 + snapshot.timing.skippedFrames = frame.skippedFrames snapshot.render.drawCallsTotal = drawStats.drawCallsTotal snapshot.render.drawCallsOpaque = drawStats.drawCallsOpaque @@ -463,8 +500,13 @@ public class UntoldRenderer: NSObject, MTKViewDelegate { #endif } + /// Runs one frame: the simulation posed for `presentationTime` — the instant the host + /// loop predicts the frame will be shown, or one period ahead of now when nil — then the + /// render hooks. `nominalPeriod` is the host display's frame interval, if it knows one. @discardableResult - private func runFrame(beforeRender: (() -> Void)? = nil, + private func runFrame(presentationTime: TimeInterval? = nil, + nominalPeriod: TimeInterval? = nil, + beforeRender: (() -> Void)? = nil, render: (() -> Void)? = nil, afterRender: (() -> Void)? = nil, statsLifecycle: StatsLifecycleMode = .internalManaged) -> Bool @@ -588,7 +630,7 @@ public class UntoldRenderer: NSObject, MTKViewDelegate { let updateStart = CACurrentMediaTime() #endif EngineProfiler.shared.beginScope(.update) - calculateDeltaTime() + calculateDeltaTime(presentationTime: presentationTime, nominalPeriod: nominalPeriod) traverseSceneGraph() handleInputCallback?() @@ -657,12 +699,13 @@ public class UntoldRenderer: NSObject, MTKViewDelegate { // user game update gameUpdateCallback?(timeSinceLastUpdate) + frameUpdateCallback?(currentFrameTime) } // Per-frame view update event — fires every frame regardless of gameMode // (RealityKit SceneEvents.Update semantics) if frameEvents.hasSubscribers { - frameEvents.dispatch(UpdateEvent(deltaTime: TimeInterval(timeSinceLastUpdate), renderer: self)) + frameEvents.dispatch(UpdateEvent(deltaTime: TimeInterval(timeSinceLastUpdate), renderer: self, frameTime: currentFrameTime)) } EngineProfiler.shared.endScope(.update) #if ENGINE_STATS_ENABLED @@ -704,7 +747,14 @@ public class UntoldRenderer: NSObject, MTKViewDelegate { // In XR, UntoldEngineXR.renderNewFrame() dispatches this to the main thread // separately, since its runLoop() runs on the compositor render thread. ProgressiveAssetLoader.shared.tick() + // The host's display link target, if it gave one; otherwise the frame is posed one + // of the view's periods ahead of now. + let presentationTime = predictedPresentationTime + predictedPresentationTime = nil + let period = 1.0 / Double(max(view.preferredFramesPerSecond, 1)) _ = runFrame( + presentationTime: presentationTime, + nominalPeriod: period, beforeRender: { [weak self] in self?.delegate?.willDraw(in: view) }, render: { [weak self] in guard let self else { return } @@ -779,15 +829,17 @@ public class UntoldRenderer: NSObject, MTKViewDelegate { return renderer } + /// Runs the frame's simulation posed for `presentationTime`, the instant the host loop + /// predicts the frame will be shown (nil: one period ahead of now). @discardableResult - public func updateXR(useExternalStatsLifecycle: Bool = false) -> Bool { + public func updateXR(presentationTime: TimeInterval? = nil, useExternalStatsLifecycle: Bool = false) -> Bool { ObjectMotion.shared.beginFrame() - return runFrame(statsLifecycle: useExternalStatsLifecycle ? .externalManaged : .internalManaged) + return runFrame(presentationTime: presentationTime, statsLifecycle: useExternalStatsLifecycle ? .externalManaged : .internalManaged) } @discardableResult - public func updateXR(render: @escaping () -> Void, useExternalStatsLifecycle: Bool = false) -> Bool { - runFrame(render: render, statsLifecycle: useExternalStatsLifecycle ? .externalManaged : .internalManaged) + public func updateXR(presentationTime: TimeInterval? = nil, render: @escaping () -> Void, useExternalStatsLifecycle: Bool = false) -> Bool { + runFrame(presentationTime: presentationTime, render: render, statsLifecycle: useExternalStatsLifecycle ? .externalManaged : .internalManaged) } /// Syncs the physical headset world position into the ECS camera components used by diff --git a/Sources/UntoldEngine/Systems/FrameEvents.swift b/Sources/UntoldEngine/Systems/FrameEvents.swift index cefe60ac2..2196feacd 100644 --- a/Sources/UntoldEngine/Systems/FrameEvents.swift +++ b/Sources/UntoldEngine/Systems/FrameEvents.swift @@ -13,10 +13,18 @@ import Foundation /// Per-frame update event, delivered once per rendered frame regardless of `gameMode`. /// Mirrors RealityKit's `SceneEvents.Update`. public struct UpdateEvent { - /// Seconds elapsed since the previous frame. + /// Seconds elapsed since the previous frame: from its presentation to this one's. public let deltaTime: TimeInterval /// The renderer that produced this frame. public let renderer: UntoldRenderer + /// The frame's predicted presentation time, its delta and the refreshes skipped before it. + public let frameTime: FrameTime + + init(deltaTime: TimeInterval, renderer: UntoldRenderer, frameTime: FrameTime = .zero) { + self.deltaTime = deltaTime + self.renderer = renderer + self.frameTime = frameTime + } } /// Token returned by update subscriptions. Call `cancel()` to stop receiving events. diff --git a/Sources/UntoldEngine/Systems/GraphBuilder.swift b/Sources/UntoldEngine/Systems/GraphBuilder.swift index 925d60dea..76774a7aa 100644 --- a/Sources/UntoldEngine/Systems/GraphBuilder.swift +++ b/Sources/UntoldEngine/Systems/GraphBuilder.swift @@ -217,6 +217,9 @@ public struct RenderGraphBuildContext: Sendable { public let frameIndex: UInt64 public let currentEye: Int public let isPrimaryEye: Bool + /// The frame's predicted presentation time, its delta from the frame before and the + /// refreshes skipped between them (`deltaTime` is its delta as a Float). + public let frameTime: FrameTime public init( viewport: SIMD2, @@ -224,7 +227,8 @@ public struct RenderGraphBuildContext: Sendable { deltaTime: Float = 0, frameIndex: UInt64 = 0, currentEye: Int, - isPrimaryEye: Bool = true + isPrimaryEye: Bool = true, + frameTime: FrameTime = .zero ) { self.viewport = viewport self.immersionStyle = immersionStyle @@ -232,6 +236,7 @@ public struct RenderGraphBuildContext: Sendable { self.frameIndex = frameIndex self.currentEye = currentEye self.isPrimaryEye = isPrimaryEye + self.frameTime = frameTime } } @@ -252,6 +257,9 @@ public struct RenderPassContext { public let computePipelines: ComputePipelineAccess public let renderPipelines: RenderPipelineAccess public let sceneRenderTargets: SceneRenderTargetAccess + /// The frame's predicted presentation time, its delta from the frame before and the + /// refreshes skipped between them (`deltaTime` is its delta as a Float). + public let frameTime: FrameTime public init( commandBuffer: MTLCommandBuffer, @@ -269,7 +277,8 @@ public struct RenderPassContext { resources: RenderResourceAccess = RenderResourceAccess(), computePipelines: ComputePipelineAccess = ComputePipelineAccess(), renderPipelines: RenderPipelineAccess = RenderPipelineAccess(), - sceneRenderTargets: SceneRenderTargetAccess = SceneRenderTargetAccess() + sceneRenderTargets: SceneRenderTargetAccess = SceneRenderTargetAccess(), + frameTime: FrameTime = .zero ) { self.commandBuffer = commandBuffer self.device = device @@ -287,6 +296,7 @@ public struct RenderPassContext { self.computePipelines = computePipelines self.renderPipelines = renderPipelines self.sceneRenderTargets = sceneRenderTargets + self.frameTime = frameTime } } @@ -861,7 +871,8 @@ func makeRenderGraphBuildContext() -> RenderGraphBuildContext { deltaTime: timeSinceLastUpdate ?? 0, frameIndex: renderInfo.frameIndex, currentEye: renderInfo.currentEye, - isPrimaryEye: isPrimaryRenderEye() + isPrimaryEye: isPrimaryRenderEye(), + frameTime: currentFrameTime ) } @@ -909,7 +920,8 @@ func makeRenderPassContext( resources: resources, computePipelines: ComputePipelineAccess(), renderPipelines: RenderPipelineAccess(), - sceneRenderTargets: sceneRenderTargets + sceneRenderTargets: sceneRenderTargets, + frameTime: currentFrameTime ) } @@ -922,7 +934,8 @@ func makeEngineExtensionUpdateContext() -> EngineExtensionUpdateContext { immersionStyle: renderInfo.immersionStyle, frameIndex: renderInfo.frameIndex, currentEye: renderInfo.currentEye, - isPrimaryEye: isPrimaryRenderEye() + isPrimaryEye: isPrimaryRenderEye(), + frameTime: currentFrameTime ) } diff --git a/Sources/UntoldEngine/Utils/Globals.swift b/Sources/UntoldEngine/Utils/Globals.swift index 63685657f..6b5bf58dd 100644 --- a/Sources/UntoldEngine/Utils/Globals.swift +++ b/Sources/UntoldEngine/Utils/Globals.swift @@ -305,6 +305,13 @@ public var timeSinceLastUpdate: Float! { set { RuntimeGlobalsStore.shared.timeSinceLastUpdate = newValue } } +/// The frame being prepared: its predicted presentation time, the delta from the frame +/// before and whether the display skipped a frame between them. `.zero` before the first. +public internal(set) var currentFrameTime: FrameTime { + get { RuntimeGlobalsStore.shared.currentFrameTime } + set { RuntimeGlobalsStore.shared.currentFrameTime = newValue } +} + var firstUpdateCall: Bool { get { RuntimeGlobalsStore.shared.firstUpdateCall } set { RuntimeGlobalsStore.shared.firstUpdateCall = newValue } @@ -1096,6 +1103,7 @@ private final class RuntimeGlobalsStore: @unchecked Sendable { private var globalEntityCounterValue: UInt32 = 0 private var timeSinceLastUpdatePreviousTimeValue: TimeInterval? private var timeSinceLastUpdateValue: Float? + private var currentFrameTimeValue: FrameTime = .zero private var firstUpdateCallValue: Bool = false private var frameCountValue: Int = 0 private var timePassedSinceLastFrameValue: Float = 0.0 @@ -1195,6 +1203,20 @@ private final class RuntimeGlobalsStore: @unchecked Sendable { } } + var currentFrameTime: FrameTime { + get { + lock.lock() + let value = currentFrameTimeValue + lock.unlock() + return value + } + set { + lock.lock() + currentFrameTimeValue = newValue + lock.unlock() + } + } + var firstUpdateCall: Bool { get { lock.lock() diff --git a/Sources/UntoldEngineAR/UntoldEngineAR.swift b/Sources/UntoldEngineAR/UntoldEngineAR.swift index 3fad071bf..d73f97873 100644 --- a/Sources/UntoldEngineAR/UntoldEngineAR.swift +++ b/Sources/UntoldEngineAR/UntoldEngineAR.swift @@ -85,6 +85,14 @@ renderer?.setupCallbacks(gameUpdate: gameUpdate, handleInput: handleInput) } + /// Installs the game's per-frame callbacks with the frame's time (see `FrameTime`). + public func setupCallbacks( + frameUpdate: @escaping (_ frame: FrameTime) -> Void, + handleInput: @escaping () -> Void + ) { + renderer?.setupCallbacks(frameUpdate: frameUpdate, handleInput: handleInput) + } + public func drawRectResized(size: CGSize) { renderInfo.viewPort = simd_float2(Float(size.width), Float(size.height)) viewportSizeDidChange = true @@ -131,7 +139,10 @@ updateARStates(currentFrame: currentFrame) EngineProfiler.shared.endScope(.renderPrep) - renderer.updateXR() + // No presentation time is predicted for an MTKView: the frame is posed one of the + // view's periods ahead of now. + let period = 1.0 / Double(max(view.preferredFramesPerSecond, 1)) + renderer.updateXR(presentationTime: CACurrentMediaTime() + period) if let renderPassDescriptor = view.currentRenderPassDescriptor { renderInfo.renderPassDescriptor = renderPassDescriptor diff --git a/Sources/UntoldEngineXR/UntoldEngineXR.swift b/Sources/UntoldEngineXR/UntoldEngineXR.swift index 50f60abd3..237fec9be 100644 --- a/Sources/UntoldEngineXR/UntoldEngineXR.swift +++ b/Sources/UntoldEngineXR/UntoldEngineXR.swift @@ -82,6 +82,12 @@ private var loggedDrawableCount = 0 /// Whether the compositor's frame timing budget has been printed (once per space). private var loggedFrameTimingBudget = false + /// Whether the compositor clock's base has been checked against the media time (once). + private var loggedClockBases = false + /// Frames whose drawable's presentation time differed from the one predicted at the + /// update: the game posed those frames for a slightly different instant than the head. + public private(set) var presentationTimeRevisions = 0 + private var lastPresentationRevisionLogTime: CFTimeInterval = 0 /// Whether the layer's depth format and the drawable's depth range have been printed. private var loggedDepthSetup = false /// The layer's depth format and the drawable's depth range (far, near), once a drawable @@ -171,6 +177,16 @@ renderer?.setupCallbacks(gameUpdate: gameUpdate, handleInput: handleInput) } + /// Installs the game's per-frame callbacks with the frame's time: the instant the + /// compositor predicts the frame will be shown, which the head pose is predicted for + /// too, so an object posed for it keeps step with the head. + public func setupCallbacks( + frameUpdate: @escaping (_ frame: FrameTime) -> Void, + handleInput: @escaping () -> Void + ) { + renderer?.setupCallbacks(frameUpdate: frameUpdate, handleInput: handleInput) + } + public func enqueueSpatialInputSnapshot(_ snapshot: XRSpatialInputSnapshot) { InputSystem.shared.enqueueXRSpatialSnapshot(snapshot) } @@ -576,6 +592,13 @@ // 2. Call predictTiming to get the predicted render deadlines for code guard let timing = frame.predictTiming() else { return } + // The instant the compositor predicts the frame will be shown, in the media-time + // base: the game poses the frame for it in the update phase, and the head is posed + // for the same instant at submission (the drawable's frameTiming, normally this + // very value; notePresentationRevision counts the frames where it is not). + let predictedPresentationTime = Self.mediaTime(of: timing.presentationTime) + logClockBasesOnce() + // 3. Call startUpdate() to mark the start of the update phase frame.startUpdate() #if ENGINE_STATS_ENABLED @@ -617,7 +640,7 @@ syncStreamingCameraPosition() if !loading { - renderer.updateXR(useExternalStatsLifecycle: true) + renderer.updateXR(presentationTime: predictedPresentationTime, useExternalStatsLifecycle: true) } // 6. Call endupdate() to mark the end of the update phase @@ -694,11 +717,18 @@ let loadingNow = AssetLoadingGate.shared.isLoadingAny let effectiveLoading = loading || loadingNow + // The frame's presentation time is checked on its first drawable (the built-in + // display); a capture drawable shares the timing. + var presentationChecked = false for drawable in drawables { // 10. Fetch the predicted device anchor from ARKit using the frameTiming information, and // apply the anchor to your frame let presentationInstant = drawable.frameTiming.presentationTime - let presentationTimeCA: TimeInterval = compositorInstantToCATime(presentationInstant) + let presentationTimeCA: TimeInterval = Self.mediaTime(of: presentationInstant) + if !presentationChecked { + presentationChecked = true + notePresentationRevision(predicted: predictedPresentationTime, final: presentationTimeCA) + } var deviceAnchor = queryDeviceAnchorIfTrackingRunning(atTimestamp: presentationTimeCA) // If predicted-time query misses, retry at "now" to reduce one-frame anchor gaps. @@ -1251,20 +1281,39 @@ #endif } - @inline(__always) - private func compositorInstantToCATime(_ instant: LayerRenderer.Clock.Instant) -> TimeInterval { - let compositorClock = LayerRenderer.Clock() - let nowInstant = compositorClock.now - let nowCA = CACurrentMediaTime() + /// An instant of the compositor's clock as seconds in the `CACurrentMediaTime()` base: + /// both count mach absolute time from the boot, the compositor clock's epoch. + private static func mediaTime(of instant: LayerRenderer.Clock.Instant) -> TimeInterval { + seconds(LayerRenderer.Clock.Instant.epoch.duration(to: instant)) + } - // Compute duration between now and the target instant - let duration: Duration = nowInstant.duration(to: instant) + private static func seconds(_ duration: Duration) -> TimeInterval { + let parts = duration.components + return Double(parts.seconds) + Double(parts.attoseconds) / 1e18 + } - // Convert Duration → seconds - let components = duration.components - let seconds = Double(components.seconds) + Double(components.attoseconds) / 1e18 + /// Prints once how far the compositor clock's epoch conversion is from the media time + /// read at the same moment: the gap between the two reads, and nothing more, if the + /// clocks share their base as the engine assumes. + private func logClockBasesOnce() { + guard !loggedClockBases else { return } + loggedClockBases = true + let epochBased = Self.mediaTime(of: LayerRenderer.Clock().now) + let mediaTime = CACurrentMediaTime() + print(String(format: "XR clock: the compositor epoch's time differs from the media time by %.3f ms", (epochBased - mediaTime) * 1000.0)) + } - return nowCA + seconds + /// Counts the frames whose final presentation time (the drawable's) differs from the + /// one predicted at the update, which the game posed the frame for. + private func notePresentationRevision(predicted: TimeInterval, final: TimeInterval) { + let difference = final - predicted + guard abs(difference) > 0.0005 else { return } + presentationTimeRevisions += 1 + let now = CACurrentMediaTime() + if presentationTimeRevisions == 1 || now - lastPresentationRevisionLogTime >= 1.0 { + lastPresentationRevisionLogTime = now + print(String(format: "XR frame timing: the drawable's presentation time differs from the predicted one by %.2f ms (%d frames so far)", difference * 1000.0, presentationTimeRevisions)) + } } public func setImmersionMode(xrImmersionMode: UntoldImmersionMode) { diff --git a/Tests/UntoldEngineRenderTests/BaseRenderSetup.swift b/Tests/UntoldEngineRenderTests/BaseRenderSetup.swift index 1d8c0e223..4c3d05512 100644 --- a/Tests/UntoldEngineRenderTests/BaseRenderSetup.swift +++ b/Tests/UntoldEngineRenderTests/BaseRenderSetup.swift @@ -78,6 +78,7 @@ class BaseRenderSetup: XCTestCase { globalEntityCounter = 0 timeSinceLastUpdatePreviousTime = nil timeSinceLastUpdate = nil + currentFrameTime = .zero firstUpdateCall = false frameCount = 0 timePassedSinceLastFrame = 0.0 diff --git a/Tests/UntoldEngineTests/FrameClockTests.swift b/Tests/UntoldEngineTests/FrameClockTests.swift new file mode 100644 index 000000000..261302ddf --- /dev/null +++ b/Tests/UntoldEngineTests/FrameClockTests.swift @@ -0,0 +1,121 @@ +// +// FrameClockTests.swift +// UntoldEngineTests +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +@testable import UntoldEngine +import XCTest + +/// The frame clock turns the presentation times a loop predicts into each frame's time: the +/// delta between presentations, the period it learns and the refreshes skipped between frames. +final class FrameClockTests: XCTestCase { + private let period: TimeInterval = 1.0 / 100.0 + + func testTheFirstFrameHasNoDeltaAndNoPeriodYet() { + var clock = FrameClock() + let frame = clock.advance(toPresentation: 500.0) + XCTAssertEqual(frame.presentationTime, 500.0) + XCTAssertEqual(frame.deltaTime, 0) + XCTAssertEqual(frame.period, 0) + XCTAssertEqual(frame.skippedFrames, 0) + XCTAssertEqual(frame.frameIndex, 1) + XCTAssertFalse(frame.wasFrameSkipped) + XCTAssertEqual(clock.current, frame) + } + + func testASteadyCadenceStepsByOnePeriodAndSkipsNothing() { + var clock = FrameClock() + var frames: [FrameTime] = [] + for index in 0 ..< 5 { + frames.append(clock.advance(toPresentation: 500.0 + Double(index) * period)) + } + XCTAssertEqual(frames.map(\.frameIndex), [1, 2, 3, 4, 5]) + for frame in frames.dropFirst() { + XCTAssertEqual(frame.deltaTime, period, accuracy: 1e-9) + XCTAssertEqual(frame.period, period, accuracy: 1e-9) + XCTAssertEqual(frame.skippedFrames, 0) + } + } + + func testASkippedRefreshIsCountedAndTheNextFrameStepsByOnePeriodAgain() { + var clock = FrameClock() + // Slots 0, 1, 2, [3 missed], 4, 5. + let frames = [0, 1, 2, 4, 5].map { clock.advance(toPresentation: 500.0 + Double($0) * period) } + XCTAssertEqual(frames.map(\.skippedFrames), [0, 0, 0, 1, 0]) + XCTAssertTrue(frames[3].wasFrameSkipped) + XCTAssertEqual(frames[3].deltaTime, 2 * period, accuracy: 1e-9) + XCTAssertEqual(frames[4].deltaTime, period, accuracy: 1e-9) + // The skip did not teach the clock a longer period. + XCTAssertEqual(frames[4].period, period, accuracy: 1e-9) + } + + func testAJitteringCadenceIsNotASkip() { + var clock = FrameClock() + var time = 500.0 + var skipped = 0 + for index in 0 ..< 20 { + // The compositor's predictions wander by a tenth of a millisecond. + time += period + (index % 2 == 0 ? 0.0001 : -0.0001) + skipped += clock.advance(toPresentation: time).skippedFrames + } + XCTAssertEqual(skipped, 0) + XCTAssertEqual(clock.period, period - 0.0001, accuracy: 1e-9) + } + + func testAStallCountsEverySkippedRefresh() { + var clock = FrameClock() + clock.advance(toPresentation: 500.0) + clock.advance(toPresentation: 500.0 + period) + let frame = clock.advance(toPresentation: 500.0 + period + 0.5) + XCTAssertEqual(frame.deltaTime, 0.5, accuracy: 1e-9) + XCTAssertEqual(frame.skippedFrames, 49) + // A stall is not a period either. + XCTAssertEqual(clock.period, period, accuracy: 1e-9) + } + + func testANominalPeriodIsUsedInsteadOfLearningOne() { + var clock = FrameClock(nominalPeriod: 1.0 / 60.0) + let first = clock.advance(toPresentation: 500.0) + XCTAssertEqual(first.period, 1.0 / 60.0, accuracy: 1e-9) + // Two draws a third of a frame apart do not shorten the period; two frames apart is a skip. + let close = clock.advance(toPresentation: 500.0 + 1.0 / 180.0) + XCTAssertEqual(close.period, 1.0 / 60.0, accuracy: 1e-9) + XCTAssertEqual(close.skippedFrames, 0) + let late = clock.advance(toPresentation: 500.0 + 1.0 / 180.0 + 2.0 / 60.0) + XCTAssertEqual(late.skippedFrames, 1) + } + + func testTimeGoingBackwardsPassesNoTime() { + var clock = FrameClock() + clock.advance(toPresentation: 500.0) + clock.advance(toPresentation: 500.0 + period) + let frame = clock.advance(toPresentation: 500.0) + XCTAssertEqual(frame.deltaTime, 0) + XCTAssertEqual(frame.skippedFrames, 0) + } + + func testAnEstimateIsOneLeadAheadOfNow() { + var clock = FrameClock(nominalPeriod: 1.0 / 60.0) + let frame = clock.advance(now: 500.0, lead: 1.0 / 60.0) + XCTAssertEqual(frame.presentationTime, 500.0 + 1.0 / 60.0, accuracy: 1e-9) + let next = clock.advance(now: 500.0 + 1.0 / 60.0, lead: 1.0 / 60.0) + XCTAssertEqual(next.deltaTime, 1.0 / 60.0, accuracy: 1e-9) + } + + func testResetMakesTheNextFrameAFirstFrame() { + var clock = FrameClock() + clock.advance(toPresentation: 500.0) + clock.advance(toPresentation: 500.0 + period) + clock.reset() + XCTAssertNil(clock.current) + XCTAssertEqual(clock.period, 0) + let frame = clock.advance(toPresentation: 900.0) + XCTAssertEqual(frame.deltaTime, 0) + XCTAssertEqual(frame.frameIndex, 1) + } +} diff --git a/Tests/UntoldEngineTests/TestEngineReset.swift b/Tests/UntoldEngineTests/TestEngineReset.swift index ff8079a70..111b8b4bf 100644 --- a/Tests/UntoldEngineTests/TestEngineReset.swift +++ b/Tests/UntoldEngineTests/TestEngineReset.swift @@ -24,6 +24,7 @@ entityNameMap.removeAll() reverseEntityNameMap.removeAll() globalEntityCounter = 0 + currentFrameTime = .zero needsFinalizeDestroys = false hasPendingDestroys = false EngineExtensionRegistry.shared.removeAll() From 5b216225f224ff284316d24180ef1b47d8803370 Mon Sep 17 00:00:00 2001 From: Javier Segura Date: Wed, 14 Oct 2026 21:09:44 +0200 Subject: [PATCH 02/22] [Feature] A transform track sampled at the frame's presentation time Transform keys with easing, sampled at an absolute time: a frame's pose is a function of the frame's presentation time alone, so a frame the display skipped lands where the timeline says and the frames after it step by one period again. applyTransformTrack poses an entity from a track for a frame. The tests show a repeated frame does not double-step an absolute-time animation while delta-stepped motion drifts, and, with a real renderer, that a pose the game samples in frameUpdate matches the frame's presentation time rather than the wall clock, that the update event and the extensions receive the same frame time, and that a two-period gap reports one skipped frame. --- .../Animation/TransformTrack.swift | 144 ++++++++++++++++++ .../FrameTimeLoopTests.swift | 116 ++++++++++++++ .../TransformTrackTests.swift | 135 ++++++++++++++++ 3 files changed, 395 insertions(+) create mode 100644 Sources/UntoldEngine/Animation/TransformTrack.swift create mode 100644 Tests/UntoldEngineRenderTests/FrameTimeLoopTests.swift create mode 100644 Tests/UntoldEngineTests/TransformTrackTests.swift diff --git a/Sources/UntoldEngine/Animation/TransformTrack.swift b/Sources/UntoldEngine/Animation/TransformTrack.swift new file mode 100644 index 000000000..4ffca86a1 --- /dev/null +++ b/Sources/UntoldEngine/Animation/TransformTrack.swift @@ -0,0 +1,144 @@ +// +// TransformTrack.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Foundation +import simd + +/// How a track moves from one key to the next. +public enum TrackEasing: Sendable, Equatable { + /// At a constant rate. + case linear + /// Slowly away from the key, faster towards the next. + case easeIn + /// Fast away from the key, slowing into the next. + case easeOut + /// Slowly away and slowly in (a smoothstep). + case easeInOut + + /// The progress through a segment, 0 to 1, for a fraction of its duration. + public func progress(_ fraction: Float) -> Float { + let t = min(max(fraction, 0), 1) + switch self { + case .linear: + return t + case .easeIn: + return t * t + case .easeOut: + return 1 - (1 - t) * (1 - t) + case .easeInOut: + return t * t * (3 - 2 * t) + } + } +} + +/// A transform at a time on a track, and how the track leaves it for the next key. +public struct TransformKey: Sendable, Equatable { + /// Seconds from the track's start. + public var time: TimeInterval + public var position: simd_float3 + public var rotation: simd_quatf + public var scale: simd_float3 + /// How the track moves from this key to the next. + public var easing: TrackEasing + + public init( + time: TimeInterval, + position: simd_float3, + rotation: simd_quatf = simd_quatf(angle: 0, axis: simd_float3(0, 1, 0)), + scale: simd_float3 = simd_float3(repeating: 1), + easing: TrackEasing = .linear + ) { + self.time = time + self.position = position + self.rotation = rotation + self.scale = scale + self.easing = easing + } +} + +/// A position, rotation and scale sampled from a track. +public struct TransformPose: Sendable, Equatable { + public var position: simd_float3 + public var rotation: simd_quatf + public var scale: simd_float3 + + public init(position: simd_float3, rotation: simd_quatf, scale: simd_float3) { + self.position = position + self.rotation = rotation + self.scale = scale + } +} + +/// Transform keys sampled at an absolute time: a frame's pose is a function of the frame's +/// presentation time alone, with no state carried over from the frame before, so a frame the +/// display skipped lands where the timeline says and the frames after it step by one period +/// again. Sample it at `FrameTime.presentationTime` less the time the track started. +public struct TransformTrack: Sendable, Equatable { + /// The keys in time order. + public let keys: [TransformKey] + /// Whether the track starts again after its last key, or holds the last key. + public let loops: Bool + + public init(keys: [TransformKey], loops: Bool = false) { + self.keys = keys.sorted { $0.time < $1.time } + self.loops = loops + } + + /// The time of the last key: the length of one pass. + public var duration: TimeInterval { keys.last?.time ?? 0 } + + /// The pose `time` seconds into the track: before the first key the first key's, after the + /// last the last's (or, looping, the pose that far into the next pass). Nil without keys. + public func pose(at time: TimeInterval) -> TransformPose? { + guard let first = keys.first, let last = keys.last else { return nil } + var t = time + if loops, duration > 0 { + t = t.truncatingRemainder(dividingBy: duration) + if t < 0 { t += duration } + } + if t <= first.time { return pose(of: first) } + if t >= last.time { return pose(of: last) } + + // The segment the time falls in: the last key at or before it, and the one after. + var index = 0 + while index + 1 < keys.count, keys[index + 1].time <= t { + index += 1 + } + let from = keys[index] + let to = keys[index + 1] + let span = to.time - from.time + let fraction = span > 0 ? Float((t - from.time) / span) : 1 + let s = from.easing.progress(fraction) + return TransformPose( + position: simd_mix(from.position, to.position, simd_float3(repeating: s)), + rotation: simd_slerp(from.rotation, to.rotation, s), + scale: simd_mix(from.scale, to.scale, simd_float3(repeating: s)) + ) + } + + /// The pose for a frame: the track's time is the frame's presentation time less + /// `startedAt`, the presentation time of the frame the track started in. + public func pose(for frame: FrameTime, startedAt: TimeInterval) -> TransformPose? { + pose(at: frame.presentationTime - startedAt) + } + + private func pose(of key: TransformKey) -> TransformPose { + TransformPose(position: key.position, rotation: key.rotation, scale: key.scale) + } +} + +/// Poses an entity from a track at a frame's presentation time: its position, rotation and +/// scale. `startedAt` is the presentation time of the frame the track started in. +public func applyTransformTrack(entityId: EntityID, track: TransformTrack, startedAt: TimeInterval, frame: FrameTime) { + guard let pose = track.pose(for: frame, startedAt: startedAt) else { return } + translateTo(entityId: entityId, position: pose.position) + rotateTo(entityId: entityId, rotation: pose.rotation) + scaleTo(entityId: entityId, scale: pose.scale) +} diff --git a/Tests/UntoldEngineRenderTests/FrameTimeLoopTests.swift b/Tests/UntoldEngineRenderTests/FrameTimeLoopTests.swift new file mode 100644 index 000000000..0064151db --- /dev/null +++ b/Tests/UntoldEngineRenderTests/FrameTimeLoopTests.swift @@ -0,0 +1,116 @@ +// +// FrameTimeLoopTests.swift +// UntoldEngineRenderTests +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import simd +@testable import UntoldEngine +import XCTest + +/// The frame's time reaches the game, the update event and the extensions from the loop, and +/// a pose the game samples at it is the pose for the frame's presentation instant, not for +/// the wall clock the update happens to run at. +final class FrameTimeLoopTests: BaseRenderSetup { + private final class TimeRecordingExtension: EngineExtension, @unchecked Sendable { + let id = "test.frameTime.recording" + var frames: [FrameTime] = [] + + func update(deltaTime _: Float, context: EngineExtensionUpdateContext) { + frames.append(context.frameTime) + } + } + + override func setUp() async throws { + try await super.setUp() + if CameraSystem.shared.activeCamera == nil { + let camera = createEntity() + createGameCamera(entityId: camera) + CameraSystem.shared.activeCamera = camera + } + gameMode = true + } + + override func tearDown() async throws { + EngineExtensionRegistry.shared.removeAll() + try await super.tearDown() + } + + func testAPoseSampledByTheGameMatchesTheFramesPresentationTime() throws { + // A timeline nowhere near the wall clock: a pose taken from CACurrentMediaTime() or + // from the time between two of these calls would be nowhere near the expected one. + let start: TimeInterval = 1000.0 + let period: TimeInterval = 0.010 + let metrePerSecond = TransformTrack(keys: [ + TransformKey(time: 0, position: simd_float3(0, 0, 0)), + TransformKey(time: 1, position: simd_float3(1, 0, 0)), + ]) + let entity = createEntity() + registerTransformComponent(entityId: entity) + + var frames: [FrameTime] = [] + renderer.setupCallbacks( + frameUpdate: { frame in + frames.append(frame) + applyTransformTrack(entityId: entity, track: metrePerSecond, startedAt: start, frame: frame) + }, + handleInput: {} + ) + var events: [FrameTime] = [] + let token = renderer.onUpdate { events.append($0.frameTime) } + defer { token.cancel() } + let recorder = TimeRecordingExtension() + EngineExtensionRegistry.shared.register(recorder) + + for index in 0 ..< 3 { + XCTAssertTrue(renderer.updateXR(presentationTime: start + Double(index) * period)) + XCTAssertEqual(getLocalPosition(entityId: entity).x, Float(index) * Float(period), accuracy: 1e-5) + } + + let presentations = [start, start + period, start + 2 * period] + XCTAssertEqual(frames.map(\.presentationTime), presentations) + XCTAssertEqual(frames.count, 3) + for (frame, delta) in zip(frames, [0, period, period]) { + XCTAssertEqual(frame.deltaTime, delta, accuracy: 1e-9) + } + XCTAssertEqual(frames.map(\.skippedFrames), [0, 0, 0]) + // The engine's delta is the presentation delta, not the microseconds between the calls. + XCTAssertEqual(timeSinceLastUpdate, Float(period), accuracy: 1e-6) + XCTAssertEqual(events.map(\.presentationTime), presentations) + XCTAssertEqual(recorder.frames.map(\.presentationTime), presentations) + XCTAssertEqual(renderer.frameTime, try XCTUnwrap(frames.last)) + XCTAssertEqual(currentFrameTime, try XCTUnwrap(frames.last)) + } + + func testTheGameUpdateDeltaIsThePresentationDeltaAndASkippedFrameIsReported() { + var deltas: [Float] = [] + renderer.setupCallbacks(gameUpdate: { deltas.append($0) }, handleInput: {}) + let start: TimeInterval = 2000.0 + let period: TimeInterval = 1.0 / 90.0 + XCTAssertTrue(renderer.updateXR(presentationTime: start)) + XCTAssertTrue(renderer.updateXR(presentationTime: start + period)) + // The display showed the second frame twice: the third is two periods on. + XCTAssertTrue(renderer.updateXR(presentationTime: start + 3 * period)) + + XCTAssertEqual(deltas.count, 3) + XCTAssertEqual(deltas[0], 0) + XCTAssertEqual(deltas[1], Float(period), accuracy: 1e-6) + XCTAssertEqual(deltas[2], Float(2 * period), accuracy: 1e-6) + XCTAssertEqual(renderer.frameTime.skippedFrames, 1) + XCTAssertEqual(renderer.frameTime.period, period, accuracy: 1e-9) + } + + func testAFrameWithoutAPredictionIsPosedOnePeriodAheadOfNow() { + let before = CACurrentMediaTime() + XCTAssertTrue(renderer.updateXR()) + let after = CACurrentMediaTime() + let period = UntoldRenderer.defaultFramePeriod + XCTAssertGreaterThanOrEqual(renderer.frameTime.presentationTime, before + period) + XCTAssertLessThanOrEqual(renderer.frameTime.presentationTime, after + period) + XCTAssertEqual(renderer.frameTime.period, period, accuracy: 1e-9) + } +} diff --git a/Tests/UntoldEngineTests/TransformTrackTests.swift b/Tests/UntoldEngineTests/TransformTrackTests.swift new file mode 100644 index 000000000..91d128fbe --- /dev/null +++ b/Tests/UntoldEngineTests/TransformTrackTests.swift @@ -0,0 +1,135 @@ +// +// TransformTrackTests.swift +// UntoldEngineTests +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import simd +@testable import UntoldEngine +import XCTest + +/// A transform track sampled at an absolute time: the pose for a frame depends on the frame's +/// presentation time alone. +final class TransformTrackTests: XCTestCase { + /// One metre per second along x for one second. + private let metrePerSecond = TransformTrack(keys: [ + TransformKey(time: 0, position: simd_float3(0, 0, 0)), + TransformKey(time: 1, position: simd_float3(1, 0, 0)), + ]) + + private func x(_ pose: TransformPose?) -> Float { + pose?.position.x ?? .nan + } + + func testPoseAtTheKeysAndBetweenThem() { + let turn = simd_quatf(angle: .pi / 2, axis: simd_float3(0, 1, 0)) + let track = TransformTrack(keys: [ + TransformKey(time: 2, position: simd_float3(2, 0, 0), rotation: turn, scale: simd_float3(repeating: 2)), + TransformKey(time: 0, position: simd_float3(0, 0, 0)), + ]) + // Keys are kept in time order whatever order they were given in. + XCTAssertEqual(track.keys.map(\.time), [0, 2]) + XCTAssertEqual(track.duration, 2) + XCTAssertEqual(x(track.pose(at: 0)), 0) + XCTAssertEqual(x(track.pose(at: 2)), 2) + let middle = track.pose(at: 1) + XCTAssertEqual(x(middle), 1, accuracy: 1e-6) + XCTAssertEqual(middle?.scale.x ?? .nan, 1.5, accuracy: 1e-6) + XCTAssertEqual(middle?.rotation.angle ?? .nan, .pi / 4, accuracy: 1e-5) + } + + func testThePoseHoldsOutsideTheKeysAndWrapsWhenLooping() { + XCTAssertEqual(x(metrePerSecond.pose(at: -1)), 0) + XCTAssertEqual(x(metrePerSecond.pose(at: 5)), 1) + let looping = TransformTrack(keys: metrePerSecond.keys, loops: true) + XCTAssertEqual(x(looping.pose(at: 1.25)), 0.25, accuracy: 1e-6) + XCTAssertEqual(x(looping.pose(at: -0.25)), 0.75, accuracy: 1e-6) + XCTAssertEqual(x(looping.pose(at: 1)), 0, accuracy: 1e-6) + XCTAssertNil(TransformTrack(keys: []).pose(at: 0)) + } + + func testEasingKeepsTheEndpointsAndShapesTheMiddle() { + for easing in [TrackEasing.linear, .easeIn, .easeOut, .easeInOut] { + XCTAssertEqual(easing.progress(0), 0) + XCTAssertEqual(easing.progress(1), 1) + XCTAssertEqual(easing.progress(-3), 0) + XCTAssertEqual(easing.progress(3), 1) + } + XCTAssertEqual(TrackEasing.linear.progress(0.25), 0.25) + XCTAssertEqual(TrackEasing.easeIn.progress(0.5), 0.25) + XCTAssertEqual(TrackEasing.easeOut.progress(0.5), 0.75) + XCTAssertEqual(TrackEasing.easeInOut.progress(0.5), 0.5) + XCTAssertLessThan(TrackEasing.easeInOut.progress(0.25), 0.25) + let eased = TransformTrack(keys: [ + TransformKey(time: 0, position: simd_float3(0, 0, 0), easing: .easeIn), + TransformKey(time: 1, position: simd_float3(1, 0, 0)), + ]) + XCTAssertEqual(x(eased.pose(at: 0.5)), 0.25, accuracy: 1e-6) + } + + func testThePoseForAFrameIsTakenAtItsPresentationTime() { + let frame = FrameTime(presentationTime: 1000.25, deltaTime: 0.01, period: 0.01, skippedFrames: 0, frameIndex: 7) + XCTAssertEqual(x(metrePerSecond.pose(for: frame, startedAt: 1000.0)), 0.25, accuracy: 1e-6) + } + + /// The display showed one frame twice (the engine missed a slot), so the next frame's + /// presentation is two periods on. A pose sampled at the frame's presentation time lands + /// where the timeline says, and the frame after it steps by one period again. An object + /// moved by the time between two updates — the wall clock the update happens to run at, + /// a varying lag before each presentation — steps unevenly every frame and is off the + /// timeline after the skip. + func testARepeatedFrameDoesNotDoubleStepAnAbsoluteTimeAnimation() { + let period: TimeInterval = 0.010 + let start: TimeInterval = 100.0 + // Presentation slots 0, 1, 2, [3 missed], 4, 5. + let slots = [0, 1, 2, 4, 5] + let presentations = slots.map { start + Double($0) * period } + // How long before its presentation each frame's update ran. + let lags: [TimeInterval] = [0.0063, 0.0071, 0.0058, 0.0094, 0.0066] + + var clock = FrameClock() + var frames: [FrameTime] = [] + var sampled: [Float] = [] + var stepped: [Float] = [] + var steppedPosition: Float = 0 + var previousUpdate: TimeInterval? + for (presentation, lag) in zip(presentations, lags) { + let frame = clock.advance(toPresentation: presentation) + frames.append(frame) + sampled.append(x(metrePerSecond.pose(for: frame, startedAt: start))) + + let updateTime = presentation - lag + if let previousUpdate { + steppedPosition += Float(updateTime - previousUpdate) // one metre per second + } + previousUpdate = updateTime + stepped.append(steppedPosition) + } + + XCTAssertEqual(frames.map(\.skippedFrames), [0, 0, 0, 1, 0]) + + // Sampled at presentation time: exactly the timeline's positions. + let expected = slots.map { Float($0) * Float(period) } + for (position, wanted) in zip(sampled, expected) { + XCTAssertEqual(position, wanted, accuracy: 1e-6) + } + // One period per frame, two across the skipped slot, one again after it. + let sampledSteps = zip(sampled.dropFirst(), sampled).map { $0 - $1 } + for (step, frame) in zip(sampledSteps, frames.dropFirst()) { + XCTAssertEqual(step, Float(period) * Float(1 + frame.skippedFrames), accuracy: 1e-6) + } + + // Stepped by the time between updates: no step is a period, and after the skip the + // position is off the timeline by the change in lag. + let steppedSteps = zip(stepped.dropFirst(), stepped).map { $0 - $1 } + for step in steppedSteps { + XCTAssertNotEqual(step, Float(period), accuracy: 0.0005) + XCTAssertNotEqual(step, Float(2 * period), accuracy: 0.0005) + } + XCTAssertNotEqual(stepped[3], expected[3], accuracy: 0.002) + } +} From d6d86d2cbf417eae7c202f3e81a5df69a7d016fa Mon Sep 17 00:00:00 2001 From: Javier Segura Date: Wed, 14 Oct 2026 21:09:44 +0200 Subject: [PATCH 03/22] [Docs] Why a moving object judders on Vision Pro, and the frame-time fix The proposal records what was seen on the headset, where the loop sampled time, the FrameTime data and how it reaches the game without breaking callers, why the game update stays before the wait, every engine path that samples time, the platform side, the tests, the headset check still to do and the change's size and risk to the foveation, temporal anti-aliasing and back-face lighting pull requests. The XR architecture doc and the starter tutorial mention the frame's time. --- docs/Architecture/xrRenderingSystem.md | 6 +- docs/Tutorials/StarterDemo.md | 16 +++ docs/proposals/PresentationTimeAnimation.md | 144 ++++++++++++++++++++ 3 files changed, 163 insertions(+), 3 deletions(-) create mode 100644 docs/proposals/PresentationTimeAnimation.md diff --git a/docs/Architecture/xrRenderingSystem.md b/docs/Architecture/xrRenderingSystem.md index 19ffdc1e0..8e2b0cd42 100644 --- a/docs/Architecture/xrRenderingSystem.md +++ b/docs/Architecture/xrRenderingSystem.md @@ -63,7 +63,7 @@ let frame = layerRenderer.queryNextFrame() let timing = frame.predictTiming() ``` -`queryNextFrame()` dequeues the next compositor frame. `predictTiming()` returns `optimalInputTime` — the deadline by which you must finish reading input and preparing CPU-side data for the frame. These are not suggestions; missing them causes judder. +`queryNextFrame()` dequeues the next compositor frame. `predictTiming()` returns `optimalInputTime` — the deadline by which you must finish reading input and preparing CPU-side data for the frame. These are not suggestions; missing them causes judder. It also returns `presentationTime`, the instant the frame will be shown; the engine converts it to the media-time base (through the compositor clock's epoch, which is the boot, like `CACurrentMediaTime()`) and poses the whole update for it. ### 2b. Update Phase @@ -79,7 +79,7 @@ Everything between `startUpdate` and `endUpdate` is CPU-side frame preparation: - **Spatial input processing**: `updateSpatialInputState()` drains the queued `XRSpatialInputSnapshot` events and updates the `InputSystem`. If assets are loading or the scene isn't ready, input is cleared instead to avoid acting on stale state. -- **Game update**: `renderer.updateXR()` calls the user's `gameUpdate` and `handleInput` callbacks. This is where game logic runs — entity movement, animation state machines, physics steps. It is skipped entirely while `AssetLoadingGate.shared.isLoadingAny` is true. +- **Game update**: `renderer.updateXR(presentationTime:)` calls the user's `gameUpdate` (or `frameUpdate`) and `handleInput` callbacks. This is where game logic runs — entity movement, animation state machines, physics steps. The frame is posed for the instant the compositor predicts it will be shown, the same instant the head pose is queried for at submission: the engine's delta is the time between two presentations, and the game receives the instant itself as a `FrameTime`, so a moving object steps by the display's period from frame to frame instead of by the varying time between two updates, which is what reads as judder against the head-tracked world (see [Presentation-time animation](../proposals/PresentationTimeAnimation.md)). It is skipped entirely while `AssetLoadingGate.shared.isLoadingAny` is true. - **XR lighting probe processing**: When real-world lighting mode is active, accepted environment-light probe updates are prefiltered into runtime IBL textures. The renderer uses the latest valid probe plus `realWorldLightingContribution`; if no valid probe is available, it falls back to the configured static environment path. @@ -111,7 +111,7 @@ drawable.deviceAnchor = anchor The device anchor is the head pose — a 4×4 transform from world space to the device. The compositor requires it to be attached to the drawable before presenting; without it, the system can't reproject the frame correctly for the user's eyes. -The engine queries the anchor at **presentation time** (the future moment when the frame will appear on screen), not at "now". This is predictive — it compensates for the latency between encoding and display by predicting where the head will be. +The engine queries the anchor at **presentation time** (the future moment when the frame will appear on screen), not at "now". This is predictive — it compensates for the latency between encoding and display by predicting where the head will be. The game's objects were posed for the same instant in the update phase, so head and objects agree; `presentationTimeRevisions` counts the frames whose drawable's presentation time differs from the prediction the update used. **Resilience strategy**: ARKit can occasionally return `nil` for the anchor (e.g., tracking hiccup, recovery). A three-level fallback prevents dropped frames: 1. Query at predicted presentation time → use it if valid diff --git a/docs/Tutorials/StarterDemo.md b/docs/Tutorials/StarterDemo.md index af03d5172..a99bac893 100644 --- a/docs/Tutorials/StarterDemo.md +++ b/docs/Tutorials/StarterDemo.md @@ -76,6 +76,22 @@ renderer.setupCallbacks( Use `gameUpdate` for simulation, animation, and other time-based scene changes. Use `handleInput` for keyboard, mouse, touch, gamepad, or XR input state. +`deltaTime` is the time from the previous frame's presentation to this one's. To pose +a moving object for the instant the frame will be shown (on a headset the head pose is +predicted for that same instant), take the frame's time instead: + +```swift +renderer.setupCallbacks( + frameUpdate: { [weak self] frame in + // frame.presentationTime, frame.deltaTime, frame.skippedFrames + self?.gameScene.update(frame: frame) + }, + handleInput: { [weak self] in + self?.gameScene.handleInput() + } +) +``` + The important API shape is: - `gameUpdate` receives `deltaTime`, so movement and animation can be frame-rate independent. diff --git a/docs/proposals/PresentationTimeAnimation.md b/docs/proposals/PresentationTimeAnimation.md new file mode 100644 index 000000000..4ec7b6316 --- /dev/null +++ b/docs/proposals/PresentationTimeAnimation.md @@ -0,0 +1,144 @@ +# Posing the Frame for Its Presentation Time — Why a Moving Object Judders on Vision Pro, and the Fix + +Status: proposal with a prototype on this branch (engine side done and tested; the viewer app's one-line change and the headset check still to do). Written 2026-10-14 from a headset session of the same day with the viewer app (`UntoldArcade-viewer`), the dartboard on its eased "tour" path. + +## TL;DR + +1. **The engine poses the head for the instant the frame will be shown, and every object for "now at update time".** The XR loop queries the device anchor at `drawable.frameTiming.presentationTime`, but the game ran earlier on a wall-clock delta between two updates. The gap between the two instants is the wait for the compositor's optimal input time plus the encode, and it varies with the previous frame's CPU work: the object's position steps unevenly from frame to frame while the head's does not. That reads as judder at full frame rate. +2. **The viewer makes it worse by reading the system clock inside the update.** Its tour pose comes from `ProcessInfo.processInfo.systemUptime` in the game update, so it carries the same varying lag; the engine never told it when the frame would be shown. +3. **A skipped frame doubles the step either way, but delta-driven motion drifts after it.** The measured 94 fps against a 100 Hz cadence and 85 fps against 90 Hz mean about one refresh in sixteen shows the previous image again. Motion that accumulates the time between updates is then off the display's timeline by the change in lag; motion sampled at the presentation time lands where the timeline says and steps by one period again on the next frame. +4. **The fix is one value handed down the loop: the frame's predicted presentation time**, with the delta from the previous presentation and the refreshes skipped between them (`FrameTime`). The compositor predicts it (`frame.predictTiming().presentationTime`) before the update phase, so the game update stays where it is, before the wait, and poses the frame for that instant. The engine's own delta (`timeSinceLastUpdate`) becomes the presentation delta, so every delta-driven path (skeletal clips, the camera path, scripts, extensions, physics accumulation) moves in step with the display without changing. +5. **Nothing existing breaks.** `gameUpdate(deltaTime:)` keeps its parameter (now the presentation delta); `setupCallbacks(frameUpdate:handleInput:)` hands the full `FrameTime`; `UpdateEvent`, `EngineExtensionUpdateContext`, `RenderGraphBuildContext` and `RenderPassContext` gain a `frameTime` field with a default. The macOS, iOS and AR loops build the same `FrameTime` from an estimate one period ahead of the draw, or from the host's display-link target when it gives one. + +## 1. What was seen + +On the headset, the dartboard moving along the tour shows small random steps every frame and larger occasional jumps. RealityKit moving the same object along the same path judders less. The engine's stats reported 94 fps on the 100 Hz display mode and 85 fps on 90 Hz: about one refresh in sixteen repeated the previous frame. + +The loop's budget at 100 Hz, from the engine stats (`EngineTimingStats`): loop 10.5 ms, update 1 ms, wait for the optimal input time 6.4 ms, encode 3 ms. + +## 2. Where the loop samples time today (verified in the code) + +The XR frame, `renderNewFrame()` in `Sources/UntoldEngineXR/UntoldEngineXR.swift`: + +1. `layerRenderer.queryNextFrame()`, then `frame.predictTiming()`: the compositor's predicted `optimalInputTime`, `renderingDeadline` and `presentationTime` for this frame (`frame_timing.h`: "the Mach absolute time at which the layer presents the frame onscreen"). +2. `frame.startUpdate()`; spatial input; `renderer.updateXR(useExternalStatsLifecycle: true)`, which is `runFrame` in `Sources/UntoldEngine/Renderer/UntoldEngine.swift`: `calculateDeltaTime()` set `timeSinceLastUpdate` from `CACurrentMediaTime()` minus the previous update's `CACurrentMediaTime()`; then animation, scripts, physics and `gameUpdateCallback?(timeSinceLastUpdate)`. +3. `frame.endUpdate()`; `LayerRenderer.Clock().wait(until: timing.optimalInputTime)` — the 6.4 ms, which is whatever the period leaves after the previous frame's work. +4. `frame.startSubmission()`; for each drawable, the head pose is queried at `drawable.frameTiming.presentationTime` (`queryDeviceAnchor(atTimestamp:)`), and the eyes are drawn with it. + +So the head is posed for the presentation instant and the objects for the update's wall-clock instant, 9 to 11 ms earlier and varying. The viewer app (`Viewer/App/EngineSession.swift`, `update()`) computes `tourTime` from `ProcessInfo.processInfo.systemUptime` inside that update, and its RealityKit path (`Viewer/App/RealityKitSpace.swift`, `apply()`) does the same in `SceneEvents.Update`. The engine's `gameUpdate` callback only ever received a `Float` delta; nothing told the game when the frame would be shown. + +The three clocks involved share one base. `CACurrentMediaTime()`, `ProcessInfo.processInfo.systemUptime` and `mach_absolute_time()` agreed to within the time between the calls when read back to back on this machine (78 µs apart, nothing more), the engine already mixes them (ARKit anchor timestamps against `systemUptime` in the lighting system, `CACurrentMediaTime()` for the anchor query), and the compositor's `LayerRenderer.Clock` counts mach absolute time from its `epoch`. The engine used to convert a compositor instant through "now" on both clocks; the prototype converts through the epoch, as Apple's Metal immersive sample does, and prints once at start how far the two conversions are apart (`XR clock: ...`), which should be the microseconds between two reads. + +There is no transform keyframe or tween system for entities. The keyframes in `Sources/UntoldEngine/Mesh/Skeleton.swift` are skeletal clips sampled by `AnimationSystem`, and the camera path in `CameraSystem.swift` moves cameras. Neither is the cause; both are delta-driven and covered by the delta change below. + +## 3. The fix: the frame's time, handed down the loop + +### 3.1 The data + +`Sources/UntoldEngine/Renderer/FrameTime.swift` (new): + +```swift +public struct FrameTime: Sendable, Equatable { + /// Seconds at which the frame is predicted to be shown (CACurrentMediaTime base). + public let presentationTime: TimeInterval + /// Seconds from the previous frame's presentation to this one's; 0 on the first frame. + public let deltaTime: TimeInterval + /// The display's period as the loop knows it (nominal, or the shortest spacing seen). + public let period: TimeInterval + /// Refreshes between the previous frame and this one that showed the previous frame again. + public let skippedFrames: Int + /// The engine's count of frames prepared, from 1. + public let frameIndex: UInt64 + public var wasFrameSkipped: Bool { skippedFrames > 0 } +} +``` + +`FrameClock` (same file) makes one per frame from the presentation times the host predicts: the delta between consecutive presentations; the period, which the host declares (an MTKView's `1 / preferredFramesPerSecond`) or the clock learns as the shortest spacing between 4 ms and 42 ms seen so far (a skip never teaches it a longer period, a stall is not a period either); and `skippedFrames = round(delta / period) − 1`, clamped at 0. A presentation before the previous one (a clock going backwards) passes no time. A jitter of a tenth of a millisecond on the compositor's predictions is not a skip. + +### 3.2 How it reaches the game without breaking existing callers + +- `UntoldRenderer.setupCallbacks(gameUpdate:handleInput:)` is unchanged in shape. Its `deltaTime` is now the presentation delta (`timeSinceLastUpdate`); the frame's time is `renderer.frameTime` or the `currentFrameTime` global while the callback runs. The global follows the engine's `timeSinceLastUpdate` style so code components and custom systems can read it too. +- `UntoldRenderer.setupCallbacks(frameUpdate:handleInput:)` (new, also on `UntoldEngineXR` and `UntoldEngineAR`) hands the `FrameTime` itself. A distinct label, not an overload of `gameUpdate:`, because an existing `gameUpdate: { _ in ... }` would be ambiguous between a `Float` and a `FrameTime` closure and stop compiling. Each form replaces the other. +- `UpdateEvent.frameTime` for `renderer.onUpdate { event in ... }` subscribers (`Sources/UntoldEngine/Systems/FrameEvents.swift`); the internal initializer keeps `UpdateEvent(deltaTime:renderer:)` working. +- `EngineExtensionUpdateContext.frameTime` (`Sources/UntoldEngine/ECS/EngineExtensions.swift`) for `EngineExtension.update(deltaTime:context:)` and `RenderExtension`s, built by `makeEngineExtensionUpdateContext()`. +- `RenderGraphBuildContext.frameTime` and `RenderPassContext.frameTime` (`Sources/UntoldEngine/Systems/GraphBuilder.swift`), next to their `deltaTime: Float`, built by `makeRenderGraphBuildContext()` and `makeRenderPassContext(...)`. All three contexts take `frameTime: FrameTime = .zero` as the last initializer parameter, so every existing call site compiles. +- `EngineTimingStats.presentationDeltaMs` and `.skippedFrames` (`Sources/UntoldEngine/Profiling/EngineStatsSnapshot.swift`) so the viewer's diagnostics can count skipped refreshes directly instead of inferring them from fps. +- `UntoldRenderer.updateXR(presentationTime:useExternalStatsLifecycle:)`: the host loop passes the predicted presentation time; `updateXR()` and `updateXR(useExternalStatsLifecycle:)` still compile and estimate one period ahead of now. + +`calculateDeltaTime(presentationTime:nominalPeriod:)` in `UntoldEngine.swift` is the only place the engine's delta is made. It advances the clock and sets `currentFrameTime`, `timeSinceLastUpdate` (the presentation delta) and `timeSinceLastUpdatePreviousTime` (now the presentation time). `firstUpdateCall == false` resets the clock, as the tests' state reset expects. + +### 3.3 The XR loop + +`renderNewFrame()` converts `timing.presentationTime` once, right after `predictTiming()`, and passes it to `updateXR(presentationTime:)`; the anchor query at submission converts `drawable.frameTiming.presentationTime` the same way. The two should be the same instant. `notePresentationRevision` counts the frames where the drawable's differs from the prediction by more than half a millisecond (`presentationTimeRevisions`, printed at most once a second): on such a frame the object was posed for a slightly different instant than the head, by exactly that difference. The count is the headset check that the prediction is stable (section 8). + +### 3.4 The viewer's change (not made here; the viewer is read-only for this branch) + +`EngineSession.swift`: `xr.setupCallbacks(frameUpdate: { [weak self] frame in self?.update(frame) }, handleInput: ...)` and, in `update`, `tourTime = (frame.presentationTime - object.placed).truncatingRemainder(dividingBy: Catalog.Tour.period)`. `object.placed` is already in the same clock base (`systemUptime`); recording it from `currentFrameTime.presentationTime` at placement makes the first frame exact too. The RealityKit path keeps `SceneEvents.Update`, which gives no presentation time; RealityKit poses its own frames for the compositor. + +## 4. Where the game update runs: before the wait, posed for the predicted presentation + +Two places are possible for the update that poses the objects: + +**A. Before the wait (today's place), sampling the predicted presentation time.** The prediction is available from `predictTiming()` at the start of the update phase, so nothing moves. The CPU budget is untouched: at 100 Hz the update's 1 ms sits inside the 6.4 ms the loop spends waiting anyway; a slow update (a batching rebuild, a streaming tick) shortens the wait instead of threatening the deadline. The object is posed for the instant the compositor predicts, the same instant the head is posed for 6.4 ms later; whatever the prediction's error, head and object share it, which is what removes the judder. + +**B. After the wait, in the submission phase.** The update would run 6.4 ms closer to presentation, so game logic that reads input (a pinch dragging the object) would act on input 6.4 ms fresher. The cost: the CPU work after `optimalInputTime` grows from the 3 ms encode to 4 ms, a third more, on every frame, and the update's variance lands directly on the rendering deadline, where it drops frames instead of shortening a wait. With the loop at 10.5 ms for a 10 ms period there is no margin for that; at 90 Hz (11.1 ms) barely. + +Recommendation: A. It fixes the judder with no budget change. If fresher input matters later, the cheap form is a small "late input" hook after the wait that only reads input poses (hands at `timing.trackableAnchorTime`, the head) and writes transforms, not a full update after the wait. + +## 5. Engine paths that sample time, and what each needs + +| Path | File / function | Today | Change | +| --- | --- | --- | --- | +| The engine's delta | `UntoldEngine.swift` `calculateDeltaTime` | wall clock between updates | presentation delta from `FrameClock` (done) | +| Skeletal clips | `Systems/AnimationSystem.swift` `updateAnimationSystem(deltaTime:)`, `AnimationComponent.currentTime += deltaTime * playbackSpeed`; `currentGlobalTime += deltaTime` | delta-driven | none: the delta is now the presentation delta, so a clip's time stays in phase with the display; sampling a clip at an absolute time is possible but not needed | +| Camera path | `Systems/CameraSystem.swift` `updateCameraPath(deltaTime:)`, `segmentT += deltaTime / segmentDuration` | delta-driven, called by the game | none; the game passes the delta it receives. An `updateCameraPath(frame:)` sampling `segmentT` from an absolute time would make a skipped frame exact rather than in phase, a small follow-up | +| Tweening | none existed | — | `Animation/TransformTrack.swift` (new): keys with easing sampled at an absolute time, `pose(for: frame, startedAt:)` and `applyTransformTrack(entityId:track:startedAt:frame:)` | +| Scripts | `Scripting/USCScripting.swift` `USCSystem.shared.update(timeSinceLastUpdate)` | delta-driven | none | +| Physics | `UntoldEngine.swift` `physicsAccumulator += timeSinceLastUpdate`, fixed step | accumulated | none; the accumulator now follows presentations | +| Particles | none in the engine; the muscle simulation's mass points (`Systems/Muscle/MuscleSimulation.swift`) step by a fixed substep | — | none; a particle system added later takes `FrameTime` like everything else | +| Code components, custom systems | `onUpdate(deltaTime:)`, `updateCustomSystems(deltaTime:)` | delta-driven | none; `currentFrameTime` is readable inside them | +| Engine and render extensions | `ECS/EngineExtensions.swift` `update(deltaTime:context:)`; `Systems/GraphBuilder.swift` `makeRenderGraphBuildContext`, `makeRenderPassContext` | `deltaTime: Float` | `context.frameTime` added (done) | +| Update event | `Systems/FrameEvents.swift` `UpdateEvent` | `deltaTime` | `frameTime` added (done) | +| Head pose | `UntoldEngineXR.swift` `renderNewFrame`, anchor query | `drawable.frameTiming.presentationTime` through "now" on both clocks | through the compositor clock's epoch, the same conversion the game's time uses (done) | +| Room light fade | `UntoldEngineXR/XREnvironmentLightingSystem.swift` (`ProcessInfo.systemUptime`) | wall clock | optional: a slow fade, not judder; could read `currentFrameTime.presentationTime` for consistency | + +## 6. The platform side: the same `FrameTime` everywhere + +- **visionOS** (`UntoldEngineXR`): the compositor's prediction, as above. The period is learnt (the compositor does not state the display's rate; the loop measures 10.0 ms at 100 Hz, 11.1 ms at 90 Hz). +- **macOS / iOS `MTKView`** (`UntoldRenderer.draw(in:)`): MTKView's delegate gets no timestamp from the view's own display link, so the frame is posed one period (`1 / view.preferredFramesPerSecond`) ahead of the draw and the period is the nominal one. The delta between frames is then the spacing of the draws, as before. A host that drives the view from its own display link (`CADisplayLink` on iOS, `NSView.displayLink(target:selector:)` on macOS 14) pauses the view's loop (`isPaused = true`, `enableSetNeedsDisplay = false`), sets `renderer.predictedPresentationTime = link.targetTimestamp` and calls `view.draw()` from the link; the frame is then posed for the link's target. A follow-up can phase-lock the estimate from `CAMetalDrawable.addPresentedHandler`'s `presentedTime` without a host display link. +- **iOS AR** (`UntoldEngineAR.draw(in:)`): one of the view's periods ahead of now, passed as `updateXR(presentationTime:)`. The camera image is already in the past; the objects are posed for when the composite is shown. + +## 7. Tests (all passing) + +- `Tests/UntoldEngineTests/FrameClockTests.swift` (9): first frame, steady cadence, a skipped refresh counted once and the next frame back to one period, jitter is not a skip, a stall counts every refresh, a nominal period, time going backwards, the estimate ahead of now, reset. +- `Tests/UntoldEngineTests/TransformTrackTests.swift` (5), including `testARepeatedFrameDoesNotDoubleStepAnAbsoluteTimeAnimation`: slots 0, 1, 2, [3 missed], 4, 5 with updates that run a varying lag before each presentation. Sampled at presentation time, the positions are exactly the timeline's, one period per frame, two across the skipped slot and one again after it; stepped by the time between updates, no step is a period and the position after the skip is off the timeline. +- `Tests/UntoldEngineRenderTests/FrameTimeLoopTests.swift` (3, with a real renderer): the loop drives `updateXR(presentationTime:)` on a timeline a thousand seconds from the wall clock; the pose the game samples in `frameUpdate` is the pose for the frame's presentation time, the engine's delta is the presentation delta and not the microseconds between the calls, the update event and an extension receive the same `FrameTime`, the legacy `gameUpdate` delta is the presentation delta and a two-period gap reports one skipped frame; a frame without a prediction is posed one period ahead of now. + +The compositor part cannot run in `swift test`; section 8 is its check. + +## 8. Headset check (to do) + +1. The start-up line `XR clock: the compositor epoch's time differs from the media time by N ms`: N should be the microseconds between two reads. A large constant N means the epoch assumption is wrong and the loop must go back to converting through "now" on both clocks. +2. `presentationTimeRevisions` (and its log line) over a minute of the tour: near zero means the predicted and final presentation times agree and the object is posed for the head's instant; a steady count means the prediction is revised by the compositor and the game should be posed for the drawable's time instead (option B's place), or the revision applied as a per-frame correction. +3. `EngineTimingStats.skippedFrames` summed over a minute against the fps shortfall (6 in 100, 5 in 90): they should match. +4. The tour, with the viewer's change of section 3.4: the small per-frame steps should be gone; the occasional larger jump remains for every skipped refresh and should now be one clean double step with no drift after it. + +## 9. Size, and the risk to the three draft pull requests + +Size: about 330 lines in `Sources` (`FrameTime.swift` 130, `TransformTrack.swift` 150, the loop and context wiring 50), 330 lines of tests, this document. The viewer's change is a few lines. + +The branch is cut from `try/xr_foveation_specular_aa`, so it already sits on top of foveation (`feature/xr_foveation`, untoldengine/UntoldEngine#1342), temporal anti-aliasing (`feature/taa`, untoldengine/UntoldEngine#1343) and the back-face lighting fix (`bugfix/backface_lighting`, untoldengine/UntoldEngine#1344). Landing order: the three PRs, then this change rebased onto `develop`. + +- **Foveation (#1342)**: textual overlap in `UntoldEngineXR.swift`. The PR rewrote the drawables loop (`for drawable in drawables`, the rate maps); this change adds two lines inside it (the once-per-frame revision check) and five properties next to the foveation ones. Applied after the PR: clean. Applied on `develop` without it: the loop does not exist in that form, so the hunk must be redone by hand (two lines). `GraphBuilder.swift` is touched by both, in different functions (`applyXRFoveationViewport` against the three context builders), ~15 lines apart: at worst an adjacent-hunk conflict. +- **Temporal anti-aliasing (#1343)**: one certain conflict, three lines: `updateXR` in `UntoldEngine.swift`, where the PR added `ObjectMotion.shared.beginFrame()` and this change adds the `presentationTime` parameter; the resolution keeps both. Semantically the change helps TAA: `ObjectMotion` records each entity's previous transform per frame, and an object posed for the presentation instant moves by whole periods, so its motion vectors are steady instead of carrying the update's lag jitter. +- **Back-face lighting (#1344)**: shaders only, no overlap. + +Behaviour changes to note in review: `timeSinceLastUpdate` is the presentation delta (exactly a period or a multiple on the headset; the draw spacing on a Mac as before); `timeSinceLastUpdatePreviousTime` holds the presentation time; `EngineTimingStats.frameTotalMs` on the headset is the presentation delta. + +## 10. Follow-ups + +- The viewer's change (section 3.4) and the headset check (section 8). +- A `TransformTrackComponent` with a system that samples it at `currentFrameTime.presentationTime` every frame, so a track plays with no game code; `TransformTrack` is the value type it would hold. +- `updateCameraPath(frame:)` sampling from an absolute time. +- A presented-time feedback for the `MTKView` estimate (section 6). +- A late-input hook after the wait (section 4). From d23e8589df182dca9b5b3d58b64983c7d4b00585 Mon Sep 17 00:00:00 2001 From: Javier Segura Date: Wed, 14 Oct 2026 22:07:43 +0200 Subject: [PATCH 04/22] [Docs] The viewer's frame-time change is drafted, and its summary counts the skipped refreshes The proposal records the viewer's drafted change (uncommitted in its working tree): the tour's pose taken at the frame's presentation time through the frameUpdate callback, and the skipped refreshes and presentation revisions in its diagnostics and run summary, which the headset check reads. It notes that the viewer's project points at the xr_foveation worktree, which must be on this branch to build. --- docs/proposals/PresentationTimeAnimation.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/proposals/PresentationTimeAnimation.md b/docs/proposals/PresentationTimeAnimation.md index 4ec7b6316..c3a18f9b0 100644 --- a/docs/proposals/PresentationTimeAnimation.md +++ b/docs/proposals/PresentationTimeAnimation.md @@ -1,6 +1,6 @@ # Posing the Frame for Its Presentation Time — Why a Moving Object Judders on Vision Pro, and the Fix -Status: proposal with a prototype on this branch (engine side done and tested; the viewer app's one-line change and the headset check still to do). Written 2026-10-14 from a headset session of the same day with the viewer app (`UntoldArcade-viewer`), the dartboard on its eased "tour" path. +Status: proposal with a prototype on this branch (engine side done and tested; the viewer app's change drafted in its working tree, uncommitted; the headset check still to do). Written 2026-10-14 from a headset session of the same day with the viewer app (`UntoldArcade-viewer`), the dartboard on its eased "tour" path. ## TL;DR @@ -71,9 +71,11 @@ public struct FrameTime: Sendable, Equatable { `renderNewFrame()` converts `timing.presentationTime` once, right after `predictTiming()`, and passes it to `updateXR(presentationTime:)`; the anchor query at submission converts `drawable.frameTiming.presentationTime` the same way. The two should be the same instant. `notePresentationRevision` counts the frames where the drawable's differs from the prediction by more than half a millisecond (`presentationTimeRevisions`, printed at most once a second): on such a frame the object was posed for a slightly different instant than the head, by exactly that difference. The count is the headset check that the prediction is stable (section 8). -### 3.4 The viewer's change (not made here; the viewer is read-only for this branch) +### 3.4 The viewer's change (drafted in the viewer's working tree, not committed there) -`EngineSession.swift`: `xr.setupCallbacks(frameUpdate: { [weak self] frame in self?.update(frame) }, handleInput: ...)` and, in `update`, `tourTime = (frame.presentationTime - object.placed).truncatingRemainder(dividingBy: Catalog.Tour.period)`. `object.placed` is already in the same clock base (`systemUptime`); recording it from `currentFrameTime.presentationTime` at placement makes the first frame exact too. The RealityKit path keeps `SceneEvents.Update`, which gives no presentation time; RealityKit poses its own frames for the compositor. +`Viewer/App/EngineSession.swift`: `xr.setupCallbacks(frameUpdate: { [weak self] frame in self?.update(frame: frame) }, handleInput: ...)`, `update(frame:)`, and the tour's time from the frame: `tourTime = (frame.presentationTime - object.placed).truncatingRemainder(dividingBy: Catalog.Tour.period)`. `object.placed` is already in the same clock base (`systemUptime`). For the headset check, the session sums `frame.skippedFrames` over all frames and reads `xr.presentationTimeRevisions`; both go into `EngineDiagnostics` (`skippedFrames`, `presentationRevisions`), the run summary (`MeasurementRun.swift`: ", N skipped refreshes, N presentation revisions") and the controls' engine line ("frames N (M skipped)"). The RealityKit path keeps `SceneEvents.Update`, which gives no presentation time; RealityKit poses its own frames for the compositor. + +The viewer's project points at the `xr_foveation` worktree (`project.yml`, `packages.UntoldEngine.path`), which does not have this branch; building it needs that worktree on this branch (or the path changed and `xcodegen generate` run). The draft was compiled against this branch through a throwaway XcodeGen project outside the viewer's tree. ## 4. Where the game update runs: before the wait, posed for the predicted presentation @@ -120,7 +122,7 @@ The compositor part cannot run in `swift test`; section 8 is its check. 1. The start-up line `XR clock: the compositor epoch's time differs from the media time by N ms`: N should be the microseconds between two reads. A large constant N means the epoch assumption is wrong and the loop must go back to converting through "now" on both clocks. 2. `presentationTimeRevisions` (and its log line) over a minute of the tour: near zero means the predicted and final presentation times agree and the object is posed for the head's instant; a steady count means the prediction is revised by the compositor and the game should be posed for the drawable's time instead (option B's place), or the revision applied as a per-frame correction. -3. `EngineTimingStats.skippedFrames` summed over a minute against the fps shortfall (6 in 100, 5 in 90): they should match. +3. The viewer's run summary: "N skipped refreshes" over the run against the fps shortfall (6 in 100, 5 in 90): they should match. "N presentation revisions" is check 2. 4. The tour, with the viewer's change of section 3.4: the small per-frame steps should be gone; the occasional larger jump remains for every skipped refresh and should now be one clean double step with no drift after it. ## 9. Size, and the risk to the three draft pull requests @@ -137,7 +139,7 @@ Behaviour changes to note in review: `timeSinceLastUpdate` is the presentation d ## 10. Follow-ups -- The viewer's change (section 3.4) and the headset check (section 8). +- The headset check (section 8), with the viewer's change of section 3.4. - A `TransformTrackComponent` with a system that samples it at `currentFrameTime.presentationTime` every frame, so a track plays with no game code; `TransformTrack` is the value type it would hold. - `updateCameraPath(frame:)` sampling from an absolute time. - A presented-time feedback for the `MTKView` estimate (section 6). From 0e674032f61de9351ec78d3a42bbb22c084dc5e1 Mon Sep 17 00:00:00 2001 From: Javier Segura Date: Wed, 14 Oct 2026 23:23:51 +0200 Subject: [PATCH 05/22] [Docs] The viewer's frame-time change as a patch, until this branch reaches the try branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The viewer must build against try/xr_foveation_specular_aa until this branch is merged into it, so its change — the tour's pose taken at the frame's presentation time through the frameUpdate callback, and the skipped refreshes and presentation revisions in its summary — is kept as a unified patch beside the proposal, which applies with patch -p1 from the viewer's directory, instead of in the viewer's tree. --- docs/proposals/PresentationTimeAnimation.md | 6 +- docs/proposals/viewer-frame-time.patch | 106 ++++++++++++++++++++ 2 files changed, 109 insertions(+), 3 deletions(-) create mode 100644 docs/proposals/viewer-frame-time.patch diff --git a/docs/proposals/PresentationTimeAnimation.md b/docs/proposals/PresentationTimeAnimation.md index c3a18f9b0..37525d469 100644 --- a/docs/proposals/PresentationTimeAnimation.md +++ b/docs/proposals/PresentationTimeAnimation.md @@ -1,6 +1,6 @@ # Posing the Frame for Its Presentation Time — Why a Moving Object Judders on Vision Pro, and the Fix -Status: proposal with a prototype on this branch (engine side done and tested; the viewer app's change drafted in its working tree, uncommitted; the headset check still to do). Written 2026-10-14 from a headset session of the same day with the viewer app (`UntoldArcade-viewer`), the dartboard on its eased "tour" path. +Status: proposal with a prototype on this branch (engine side done and tested; the viewer app's change saved as `viewer-frame-time.patch` beside this document; the headset check still to do). Written 2026-10-14 from a headset session of the same day with the viewer app (`UntoldArcade-viewer`), the dartboard on its eased "tour" path. ## TL;DR @@ -71,11 +71,11 @@ public struct FrameTime: Sendable, Equatable { `renderNewFrame()` converts `timing.presentationTime` once, right after `predictTiming()`, and passes it to `updateXR(presentationTime:)`; the anchor query at submission converts `drawable.frameTiming.presentationTime` the same way. The two should be the same instant. `notePresentationRevision` counts the frames where the drawable's differs from the prediction by more than half a millisecond (`presentationTimeRevisions`, printed at most once a second): on such a frame the object was posed for a slightly different instant than the head, by exactly that difference. The count is the headset check that the prediction is stable (section 8). -### 3.4 The viewer's change (drafted in the viewer's working tree, not committed there) +### 3.4 The viewer's change (`viewer-frame-time.patch`, beside this document) `Viewer/App/EngineSession.swift`: `xr.setupCallbacks(frameUpdate: { [weak self] frame in self?.update(frame: frame) }, handleInput: ...)`, `update(frame:)`, and the tour's time from the frame: `tourTime = (frame.presentationTime - object.placed).truncatingRemainder(dividingBy: Catalog.Tour.period)`. `object.placed` is already in the same clock base (`systemUptime`). For the headset check, the session sums `frame.skippedFrames` over all frames and reads `xr.presentationTimeRevisions`; both go into `EngineDiagnostics` (`skippedFrames`, `presentationRevisions`), the run summary (`MeasurementRun.swift`: ", N skipped refreshes, N presentation revisions") and the controls' engine line ("frames N (M skipped)"). The RealityKit path keeps `SceneEvents.Update`, which gives no presentation time; RealityKit poses its own frames for the compositor. -The viewer's project points at the `xr_foveation` worktree (`project.yml`, `packages.UntoldEngine.path`), which does not have this branch; building it needs that worktree on this branch (or the path changed and `xcodegen generate` run). The draft was compiled against this branch through a throwaway XcodeGen project outside the viewer's tree. +The viewer's project points at the `xr_foveation` worktree (`project.yml`, `packages.UntoldEngine.path`), which does not have this branch, and the viewer must keep building against the try branch until this branch is merged into it. So the draft lives as a patch: `patch -p1 < viewer-frame-time.patch` from the `UntoldViewer` directory applies it (three files; it was taken against the viewer's tree of 2026-10-14). It was compiled against this branch through a throwaway XcodeGen project outside the viewer's tree. ## 4. Where the game update runs: before the wait, posed for the predicted presentation diff --git a/docs/proposals/viewer-frame-time.patch b/docs/proposals/viewer-frame-time.patch new file mode 100644 index 000000000..00f116cec --- /dev/null +++ b/docs/proposals/viewer-frame-time.patch @@ -0,0 +1,106 @@ +diff -ruN a/Viewer/App/EngineSession.swift b/Viewer/App/EngineSession.swift +--- a/Viewer/App/EngineSession.swift 2026-10-14 23:22:53 ++++ b/Viewer/App/EngineSession.swift 2026-10-14 23:22:52 +@@ -103,6 +103,12 @@ + var textureBudgetMB = 0 + /// Seconds into the debug tour's loop, or nil when the object stands still. + var tourTime: Double? ++ /// Refreshes at which the display showed the previous frame again because the engine's ++ /// next was not ready, over all frames: the fps shortfall. And the frames whose final ++ /// presentation time differed from the one predicted at the update, which the tour's ++ /// pose was taken at. ++ var skippedFrames = 0 ++ var presentationRevisions = 0 + /// The frame's draw calls and triangles, from the engine's own counters. + var drawCalls = 0 + var triangles = 0 +@@ -168,6 +174,8 @@ + private let lock = NSLock() + private var diagnosticsStorage = EngineDiagnostics() + private var frames: UInt64 = 0 ++ /// Refreshes the display skipped over those frames (the previous frame shown again). ++ private var skippedFrames = 0 + private var objectName = "" + private var phase: OSSignpostIntervalState? + /// Exponential moving averages of the GPU time: all frames, and the frames before the object. +@@ -227,13 +235,16 @@ + .rotateBy(angle: -50, axis: [.x]) + .rotateBy(angle: 30, axis: [.y]) + frames = 0 ++ skippedFrames = 0 + phase = MeasurementRun.signposter.beginInterval("Engine", id: MeasurementRun.signposter.makeSignpostID(), "empty") + // A pinch-drag moves the object (closer, farther, up, down): the engine + // reports the pinching hand's position from the layer's spatial events. + InputSystem.shared.registerXREvents() + setInput(.xr(.sceneReady(true))) ++ // The frame's time: the instant the compositor predicts the frame will be shown, which ++ // the head pose is predicted for too, so the tour's pose for it keeps step with the head. + xr.setupCallbacks( +- gameUpdate: { [weak self] _ in self?.update() }, ++ frameUpdate: { [weak self] frame in self?.update(frame: frame) }, + handleInput: { [weak self] in self?.handleInput() } + ) + let thread = Thread { [self] in +@@ -326,9 +337,13 @@ + Self.log.log("engine: \(placement.object.id, privacy: .public) centred at (\(placement.centre.x, format: .fixed(precision: 2)), \(placement.centre.y, format: .fixed(precision: 2)), \(placement.centre.z, format: .fixed(precision: 2))), \(head == nil ? "room frame" : "in front of the wearer", privacy: .public)") + } + +- /// Once per frame on the render thread. +- private func update() { +- lock.withLock { frames &+= 1 } ++ /// Once per frame on the render thread, with the frame's time: the instant the compositor ++ /// predicts the frame will be shown. ++ private func update(frame: FrameTime) { ++ lock.withLock { ++ frames &+= 1 ++ skippedFrames += frame.skippedFrames ++ } + // About a second's window at 90 Hz; the first frames after a mode change are noisy. + let timing = EngineStatsMonitor.shared.snapshot().timing + let gpu = timing.gpuExecutionMs +@@ -374,7 +389,11 @@ + var tourTime: Double? + if let object = lock.withLock({ object }) { + if Catalog.motionIsTour { +- tourTime = (ProcessInfo.processInfo.systemUptime - object.placed).truncatingRemainder(dividingBy: Catalog.Tour.period) ++ // The tour's time at the instant the frame will be shown, not at this update: ++ // the pose then steps by the display's period from one frame to the next, and by ++ // two after a skipped one, instead of by the varying lag of the update behind ++ // the presentation. `placed` is in the same clock base (the system uptime). ++ tourTime = (frame.presentationTime - object.placed).truncatingRemainder(dividingBy: Catalog.Tour.period) + } + let pose = object.placement.pose(tourTime: tourTime) + translateTo(entityId: object.entity, position: pose.position) +@@ -422,6 +441,8 @@ + textureMemoryUsedMB: MemoryBudgetManager.shared.getStats().textureMemoryUsed / (1024 * 1024), + textureBudgetMB: MemoryBudgetManager.shared.textureBudget / (1024 * 1024), + tourTime: tourTime, ++ skippedFrames: lock.withLock { skippedFrames }, ++ presentationRevisions: xr?.presentationTimeRevisions ?? 0, + drawCalls: stats.render.drawCallsTotal, + triangles: stats.render.trianglesTotal, + opaqueDrawsLeast: opaqueDrawsLeast == Int.max ? 0 : opaqueDrawsLeast, +diff -ruN a/Viewer/App/MeasurementRun.swift b/Viewer/App/MeasurementRun.swift +--- a/Viewer/App/MeasurementRun.swift 2026-10-14 23:22:53 ++++ b/Viewer/App/MeasurementRun.swift 2026-10-14 23:22:52 +@@ -213,7 +213,7 @@ + object, d.gpuMilliseconds, d.emptyGpuMilliseconds, d.objectGpuMilliseconds ?? 0, fps, emptyFps, + d.antiAliasing, d.opaqueSampleCount, d.viewport.x, d.viewport.y, + d.screenSize.x > 0 ? String(format: " fov %d×%d", d.screenSize.x, d.screenSize.y) : "", +- d.drawCalls, d.triangles, (d.compositeSkipped ? ", composite skipped" : "") + ", " + d.renderPath + (d.tileLightStage ? " (tile light)" : "") + String(format: ", draws %d plain / %d discarding", d.plainDraws, d.discardingDraws) + (d.foveation ? "" : ", no foveation") + String(format: ", cull set %d", d.cullCandidates) + (d.shadowTaps == 8 ? "" : String(format: ", shadow taps %d", d.shadowTaps)) + (d.opaqueDrawsMost > 0 ? String(format: ", opaque draws %d–%d%@", d.opaqueDrawsLeast, d.opaqueDrawsMost, d.batchedDrawsMost > 0 ? String(format: " + %d batched", d.batchedDrawsMost) : "") : "") + (d.depthFormat.isEmpty ? "" : String(format: ", depth %@ (engine near %.3f)", d.depthFormat, d.nearPlane)) + (d.toneMap == "none" ? "" : ", tone map " + d.toneMap) + String(format: ", textures %d up / %d down, %d at full, %d in flight, %d of %d MB", d.textureUpgrades, d.textureDowngrades, d.textureEntitiesAtFull, d.textureOpsInFlight, d.textureMemoryUsedMB, d.textureBudgetMB) + (d.sun ? "" : ", no sun") + (d.roomLightScale > 0 ? String(format: ", room light scale %.2f (camera ref %.0f, published %.2f) × %.0f", d.roomLightScale, d.roomLightCameraReference, d.roomLightPublished, d.roomLightFactor) : "") + (d.probe.isEmpty ? "" : String(format: ", probe %@, %d publishes (%d updates accepted, %d skipped, last %.0f s ago)", d.probe, d.probePublishes, d.probeAccepted, d.probeSkipped, d.probeAge)) + (d.probeFaces.count == 6 ? String(format: ", probe faces +x %.3f −x %.3f +y %.3f −y %.3f +z %.3f −z %.3f", d.probeFaces[0], d.probeFaces[1], d.probeFaces[2], d.probeFaces[3], d.probeFaces[4], d.probeFaces[5]) : "") + (d.wideGamut ? "" : ", sRGB primaries") + (d.occlusionCull ? "" : ", occlusion off") + (d.taaJitter ? "" : ", jitter off") + (d.textureStreaming ? "" : ", streaming off") + (d.lighting == "room" ? "" : ", studio light") + String(format: ", %d dark frames%@", d.darkFrames, d.darkFrameTimes.isEmpty ? "" : " (tour " + d.darkFrameTimes.map { String(format: "%.1f", $0) }.joined(separator: " ") + " s)") + (d.engineOptimized ? "" : ", UNOPTIMIZED ENGINE") + (d.drawCalls == 0 && d.gpuMilliseconds == 0 ? ", NO ENGINE STATS (build with -DENGINE_STATS_ENABLED)" : "")) ++ d.drawCalls, d.triangles, (d.compositeSkipped ? ", composite skipped" : "") + ", " + d.renderPath + (d.tileLightStage ? " (tile light)" : "") + String(format: ", draws %d plain / %d discarding", d.plainDraws, d.discardingDraws) + (d.foveation ? "" : ", no foveation") + String(format: ", cull set %d", d.cullCandidates) + (d.shadowTaps == 8 ? "" : String(format: ", shadow taps %d", d.shadowTaps)) + (d.opaqueDrawsMost > 0 ? String(format: ", opaque draws %d–%d%@", d.opaqueDrawsLeast, d.opaqueDrawsMost, d.batchedDrawsMost > 0 ? String(format: " + %d batched", d.batchedDrawsMost) : "") : "") + (d.depthFormat.isEmpty ? "" : String(format: ", depth %@ (engine near %.3f)", d.depthFormat, d.nearPlane)) + (d.toneMap == "none" ? "" : ", tone map " + d.toneMap) + String(format: ", textures %d up / %d down, %d at full, %d in flight, %d of %d MB", d.textureUpgrades, d.textureDowngrades, d.textureEntitiesAtFull, d.textureOpsInFlight, d.textureMemoryUsedMB, d.textureBudgetMB) + (d.sun ? "" : ", no sun") + (d.roomLightScale > 0 ? String(format: ", room light scale %.2f (camera ref %.0f, published %.2f) × %.0f", d.roomLightScale, d.roomLightCameraReference, d.roomLightPublished, d.roomLightFactor) : "") + (d.probe.isEmpty ? "" : String(format: ", probe %@, %d publishes (%d updates accepted, %d skipped, last %.0f s ago)", d.probe, d.probePublishes, d.probeAccepted, d.probeSkipped, d.probeAge)) + (d.probeFaces.count == 6 ? String(format: ", probe faces +x %.3f −x %.3f +y %.3f −y %.3f +z %.3f −z %.3f", d.probeFaces[0], d.probeFaces[1], d.probeFaces[2], d.probeFaces[3], d.probeFaces[4], d.probeFaces[5]) : "") + (d.wideGamut ? "" : ", sRGB primaries") + (d.occlusionCull ? "" : ", occlusion off") + (d.taaJitter ? "" : ", jitter off") + (d.textureStreaming ? "" : ", streaming off") + (d.lighting == "room" ? "" : ", studio light") + String(format: ", %d dark frames%@", d.darkFrames, d.darkFrameTimes.isEmpty ? "" : " (tour " + d.darkFrameTimes.map { String(format: "%.1f", $0) }.joined(separator: " ") + " s)") + String(format: ", %d skipped refreshes, %d presentation revisions", d.skippedFrames, d.presentationRevisions) + (d.engineOptimized ? "" : ", UNOPTIMIZED ENGINE") + (d.drawCalls == 0 && d.gpuMilliseconds == 0 ? ", NO ENGINE STATS (build with -DENGINE_STATS_ENABLED)" : "")) + summary.append(line) + consoleLine("measure: " + line) + Self.log.log("measure: \(line, privacy: .public)") +diff -ruN a/Viewer/UI/ControlsView.swift b/Viewer/UI/ControlsView.swift +--- a/Viewer/UI/ControlsView.swift 2026-10-14 23:22:53 ++++ b/Viewer/UI/ControlsView.swift 2026-10-14 23:22:52 +@@ -383,7 +383,7 @@ + .lineLimit(2) + } + Text(engineOpen +- ? "engine · \(d.object.isEmpty ? "placing…" : d.object)\(d.tourTime.map { String(format: " · tour %.1f s", $0) } ?? "") · frames \(d.frames) · AA \(d.antiAliasing) ×\(d.opaqueSampleCount) · \(d.viewport.x)×\(d.viewport.y)\(d.screenSize.x > 0 ? " fov \(d.screenSize.x)×\(d.screenSize.y)" : "") · \(gpu)\(fps)\(d.compositeSkipped ? " · composite skipped" : "") · \(d.renderPath)\(d.tileLightStage ? " (tile light)" : "")" ++ ? "engine · \(d.object.isEmpty ? "placing…" : d.object)\(d.tourTime.map { String(format: " · tour %.1f s", $0) } ?? "") · frames \(d.frames)\(d.skippedFrames > 0 ? " (\(d.skippedFrames) skipped)" : "") · AA \(d.antiAliasing) ×\(d.opaqueSampleCount) · \(d.viewport.x)×\(d.viewport.y)\(d.screenSize.x > 0 ? " fov \(d.screenSize.x)×\(d.screenSize.y)" : "") · \(gpu)\(fps)\(d.compositeSkipped ? " · composite skipped" : "") · \(d.renderPath)\(d.tileLightStage ? " (tile light)" : "")" + : (rkOpen ? "realitykit · \(Catalog.selectedID) · updates \(String(format: "%.0f", rkRate)) Hz · gpu: out of process — Instruments ▸ RealityKit Trace, with and without the object" : "no space open · \(EngineSession.shared.lastOpenResult)")) + .font(.system(size: 11, design: .monospaced)) + .foregroundStyle(.secondary) From a22c6492456af60a674811cc5cb12597dd663b50 Mon Sep 17 00:00:00 2001 From: Javier Segura Date: Thu, 15 Oct 2026 00:19:46 +0200 Subject: [PATCH 06/22] [Docs] The frame-time branch stacks on the headset-rendering pull request, and the viewer carries its change The proposal's status and its landing note follow the rebase onto feature/xr_headset_rendering (untoldengine/UntoldEngine#1359), which rewrote the try branch's history, and the viewer's change now applied in the viewer's own pull request. --- docs/proposals/PresentationTimeAnimation.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/proposals/PresentationTimeAnimation.md b/docs/proposals/PresentationTimeAnimation.md index 37525d469..306f64c32 100644 --- a/docs/proposals/PresentationTimeAnimation.md +++ b/docs/proposals/PresentationTimeAnimation.md @@ -1,6 +1,6 @@ # Posing the Frame for Its Presentation Time — Why a Moving Object Judders on Vision Pro, and the Fix -Status: proposal with a prototype on this branch (engine side done and tested; the viewer app's change saved as `viewer-frame-time.patch` beside this document; the headset check still to do). Written 2026-10-14 from a headset session of the same day with the viewer app (`UntoldArcade-viewer`), the dartboard on its eased "tour" path. +Status: proposal with a prototype on this branch (engine side done and tested; the viewer app's change applied in the viewer's pull request, and kept as `viewer-frame-time.patch` beside this document; the headset check still to do). The branch stacks on `feature/xr_headset_rendering` (untoldengine/UntoldEngine#1359), which rewrote the try branch's history. Written 2026-10-14 from a headset session of the same day with the viewer app (`UntoldArcade-viewer`), the dartboard on its eased "tour" path. ## TL;DR @@ -75,7 +75,7 @@ public struct FrameTime: Sendable, Equatable { `Viewer/App/EngineSession.swift`: `xr.setupCallbacks(frameUpdate: { [weak self] frame in self?.update(frame: frame) }, handleInput: ...)`, `update(frame:)`, and the tour's time from the frame: `tourTime = (frame.presentationTime - object.placed).truncatingRemainder(dividingBy: Catalog.Tour.period)`. `object.placed` is already in the same clock base (`systemUptime`). For the headset check, the session sums `frame.skippedFrames` over all frames and reads `xr.presentationTimeRevisions`; both go into `EngineDiagnostics` (`skippedFrames`, `presentationRevisions`), the run summary (`MeasurementRun.swift`: ", N skipped refreshes, N presentation revisions") and the controls' engine line ("frames N (M skipped)"). The RealityKit path keeps `SceneEvents.Update`, which gives no presentation time; RealityKit poses its own frames for the compositor. -The viewer's project points at the `xr_foveation` worktree (`project.yml`, `packages.UntoldEngine.path`), which does not have this branch, and the viewer must keep building against the try branch until this branch is merged into it. So the draft lives as a patch: `patch -p1 < viewer-frame-time.patch` from the `UntoldViewer` directory applies it (three files; it was taken against the viewer's tree of 2026-10-14). It was compiled against this branch through a throwaway XcodeGen project outside the viewer's tree. +The viewer's project points at the `xr_foveation` worktree (`project.yml`, `packages.UntoldEngine.path`), which builds `feature/xr_headset_rendering`; the viewer with this change compiles once this branch is merged into that one. The change is applied in the viewer's pull request and kept as a patch here (`patch -p1 < viewer-frame-time.patch` from the `UntoldViewer` directory; three files, taken against the viewer's tree of 2026-10-14). It was compiled against this branch through a throwaway XcodeGen project outside the viewer's tree. ## 4. Where the game update runs: before the wait, posed for the predicted presentation @@ -129,7 +129,7 @@ The compositor part cannot run in `swift test`; section 8 is its check. Size: about 330 lines in `Sources` (`FrameTime.swift` 130, `TransformTrack.swift` 150, the loop and context wiring 50), 330 lines of tests, this document. The viewer's change is a few lines. -The branch is cut from `try/xr_foveation_specular_aa`, so it already sits on top of foveation (`feature/xr_foveation`, untoldengine/UntoldEngine#1342), temporal anti-aliasing (`feature/taa`, untoldengine/UntoldEngine#1343) and the back-face lighting fix (`bugfix/backface_lighting`, untoldengine/UntoldEngine#1344). Landing order: the three PRs, then this change rebased onto `develop`. +The branch was cut from `try/xr_foveation_specular_aa` and then rebased onto `feature/xr_headset_rendering` (untoldengine/UntoldEngine#1359, which supersedes the three drafts below), with no conflicts; it sits on top of foveation (`feature/xr_foveation`, untoldengine/UntoldEngine#1342), temporal anti-aliasing (`feature/taa`, untoldengine/UntoldEngine#1343) and the back-face lighting fix (`bugfix/backface_lighting`, untoldengine/UntoldEngine#1344). Landing order: #1359, then this change on top of it. - **Foveation (#1342)**: textual overlap in `UntoldEngineXR.swift`. The PR rewrote the drawables loop (`for drawable in drawables`, the rate maps); this change adds two lines inside it (the once-per-frame revision check) and five properties next to the foveation ones. Applied after the PR: clean. Applied on `develop` without it: the loop does not exist in that form, so the hunk must be redone by hand (two lines). `GraphBuilder.swift` is touched by both, in different functions (`applyXRFoveationViewport` against the three context builders), ~15 lines apart: at worst an adjacent-hunk conflict. - **Temporal anti-aliasing (#1343)**: one certain conflict, three lines: `updateXR` in `UntoldEngine.swift`, where the PR added `ObjectMotion.shared.beginFrame()` and this change adds the `presentationTime` parameter; the resolution keeps both. Semantically the change helps TAA: `ObjectMotion` records each entity's previous transform per frame, and an object posed for the presentation instant moves by whole periods, so its motion vectors are steady instead of carrying the update's lag jitter. From 7df6a33a6cc128212011095fb4cbd97ed7d08222 Mon Sep 17 00:00:00 2001 From: Javier Segura Date: Thu, 15 Oct 2026 08:11:08 +0200 Subject: [PATCH 07/22] [Chores] The XR loop counts the frames whose GPU work finished after the rendering deadline A frame submitted on time whose GPU work ends after the compositor's rendering deadline is shown a refresh late, posed for an instant already past: a hop in a moving object that neither the loop's skipped-refresh count nor the presentation revisions can see. The primary drawable's command buffer now compares its GPU end time with the frame's deadline; the misses and the longest overrun are readable (deadlineMisses, worstDeadlineOverrunMs) and logged at most once a second. --- Sources/UntoldEngineXR/UntoldEngineXR.swift | 37 ++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/Sources/UntoldEngineXR/UntoldEngineXR.swift b/Sources/UntoldEngineXR/UntoldEngineXR.swift index 237fec9be..903fafbff 100644 --- a/Sources/UntoldEngineXR/UntoldEngineXR.swift +++ b/Sources/UntoldEngineXR/UntoldEngineXR.swift @@ -88,6 +88,16 @@ /// update: the game posed those frames for a slightly different instant than the head. public private(set) var presentationTimeRevisions = 0 private var lastPresentationRevisionLogTime: CFTimeInterval = 0 + /// Frames whose GPU work finished after the compositor's rendering deadline: the display + /// showed the previous frame again and this one a refresh late, posed for an instant + /// already past — a hop in a moving object that no counter of the loop's pacing sees. + public var deadlineMisses: Int { deadlineLock.withLock { deadlineMissCount } } + /// The longest a frame's GPU work finished after its deadline, in milliseconds. + public var worstDeadlineOverrunMs: Double { deadlineLock.withLock { worstOverrunMs } } + private let deadlineLock = NSLock() + private var deadlineMissCount = 0 + private var worstOverrunMs = 0.0 + private var lastDeadlineLogTime: CFTimeInterval = 0 /// Whether the layer's depth format and the drawable's depth range have been printed. private var loggedDepthSetup = false /// The layer's depth format and the drawable's depth range (far, near), once a drawable @@ -1135,13 +1145,20 @@ EngineProfiler.shared.attach(to: commandBuffer, label: "XRFrame") + // The instant the GPU must have finished by for the compositor to show this frame at + // its presentation time, in the base of the command buffer's own timestamps. + let deadline = Self.mediaTime(of: drawable.frameTiming.renderingDeadline) + let countsDeadline = primary // Add completion handler to signal semaphore when GPU work is done - commandBuffer.addCompletedHandler { cb in + commandBuffer.addCompletedHandler { [weak self] cb in #if ENGINE_STATS_ENABLED // The frame's GPU time beyond the previous frame's: the two overlap on the // GPU, and the command buffer's own span would count the overlap twice. EngineStatsMonitor.shared.recordGPUCompletion(gpuStartTime: cb.gpuStartTime, gpuEndTime: cb.gpuEndTime) #endif + if countsDeadline { + self?.noteGPUCompletion(at: cb.gpuEndTime, deadline: deadline) + } commandBufferSemaphore.signal() } @@ -1303,6 +1320,24 @@ print(String(format: "XR clock: the compositor epoch's time differs from the media time by %.3f ms", (epochBased - mediaTime) * 1000.0)) } + /// Counts a frame whose GPU work finished after the compositor's rendering deadline, and + /// keeps the longest overrun. Runs on the GPU's completion thread. + private func noteGPUCompletion(at gpuEnd: CFTimeInterval, deadline: TimeInterval) { + let overrunMs = (gpuEnd - deadline) * 1000.0 + guard overrunMs > 0 else { return } + deadlineLock.lock() + deadlineMissCount += 1 + worstOverrunMs = max(worstOverrunMs, overrunMs) + let count = deadlineMissCount + let now = CACurrentMediaTime() + let logs = count == 1 || now - lastDeadlineLogTime >= 1.0 + if logs { lastDeadlineLogTime = now } + deadlineLock.unlock() + if logs { + print(String(format: "XR frame timing: the GPU finished %.2f ms after the rendering deadline (%d late frames so far)", overrunMs, count)) + } + } + /// Counts the frames whose final presentation time (the drawable's) differs from the /// one predicted at the update, which the game posed the frame for. private func notePresentationRevision(predicted: TimeInterval, final: TimeInterval) { From 6b542913ffbd926c787e25403457f334194cc81e Mon Sep 17 00:00:00 2001 From: Javier Segura Date: Thu, 15 Oct 2026 08:22:52 +0200 Subject: [PATCH 08/22] [Feature] A setting to encode the XR frame without waiting for the optimal input time, and a per-second late-frame summary On the headset nearly every frame's GPU work finished a fraction of a millisecond after the compositor's rendering deadline, and the ones that missed by more were the hop a moving object still showed on the tour's sweep. setRendering(.optimalInputWait(.disabled)) is the diagnostic for the cheapest remedy: the frame is encoded as soon as the update is done, so the GPU starts about the wait's length earlier, and the head pose is predicted from earlier for the compositor's reprojection to cover. The deadline counter now reports once a second how many of that second's frames were late, with the worst and mean overrun. --- .../Utils/EngineSettingsAPI.swift | 17 +++++++ Sources/UntoldEngine/Utils/Globals.swift | 24 +++++++++ Sources/UntoldEngineXR/UntoldEngineXR.swift | 50 ++++++++++++++----- 3 files changed, 79 insertions(+), 12 deletions(-) diff --git a/Sources/UntoldEngine/Utils/EngineSettingsAPI.swift b/Sources/UntoldEngine/Utils/EngineSettingsAPI.swift index e36bd8526..f3aea125a 100644 --- a/Sources/UntoldEngine/Utils/EngineSettingsAPI.swift +++ b/Sources/UntoldEngine/Utils/EngineSettingsAPI.swift @@ -100,6 +100,13 @@ public enum RenderingProperty: Sendable { /// compositor still unwarps, so the picture is distorted. See /// `getFoveatedRenderingEnabled()` to read the current value. case foveatedRendering(RenderingToggle) + /// Whether the XR loop waits for the compositor's optimal input time before it queries the + /// head pose and encodes the frame. Enabled by default: the head pose is as fresh as the + /// frame allows. Disabled, a diagnostic, the frame is encoded as soon as the update is + /// done, so the GPU starts about the wait's length earlier (a frame that finished after + /// the rendering deadline may then make it) and the head pose is predicted from earlier, + /// which the compositor's reprojection must cover. See `getOptimalInputWaitEnabled()`. + case optimalInputWait(RenderingToggle) /// Whether the pre-composite pass is skipped on a frame it would only copy the lit scene /// (XR passthrough with no environment, splat, ambient-occlusion or gizmo layer to join; /// the lit debug view; no render extension). Enabled by default; disabling it runs the @@ -227,6 +234,10 @@ public func setRendering(_ property: RenderingProperty) { xrFoveatedRenderingEnabled = true case .foveatedRendering(.disabled): xrFoveatedRenderingEnabled = false + case .optimalInputWait(.enabled): + xrOptimalInputWaitEnabled = true + case .optimalInputWait(.disabled): + xrOptimalInputWaitEnabled = false case .preCompositeBypass(.enabled): renderInfo.preCompositeBypassEnabled = true case .preCompositeBypass(.disabled): @@ -265,6 +276,12 @@ public func getFoveatedRenderingEnabled() -> Bool { xrFoveatedRenderingEnabled } +/// Whether the XR loop waits for the compositor's optimal input time before it encodes the +/// frame. See `RenderingProperty.optimalInputWait`. +public func getOptimalInputWaitEnabled() -> Bool { + xrOptimalInputWaitEnabled +} + public enum RenderResourceQuery { case texture(String) } diff --git a/Sources/UntoldEngine/Utils/Globals.swift b/Sources/UntoldEngine/Utils/Globals.swift index 6b5bf58dd..97ee61f5c 100644 --- a/Sources/UntoldEngine/Utils/Globals.swift +++ b/Sources/UntoldEngine/Utils/Globals.swift @@ -1134,6 +1134,7 @@ private final class RuntimeGlobalsStore: @unchecked Sendable { private var enableEngineMetricsValue: Bool = false private var bypassPostProcessingValue: Bool = false private var xrFoveatedRenderingEnabledValue: Bool = true + private var xrOptimalInputWaitEnabledValue: Bool = true private var antiAliasingModeValue: AntiAliasingMode = .fxaa private var renderDebugViewModeValue: RenderDebugViewMode = .lit private var pomQualitySettingsValue: POMQualitySettings = .platformDefault @@ -1653,6 +1654,20 @@ private final class RuntimeGlobalsStore: @unchecked Sendable { } } + var xrOptimalInputWaitEnabled: Bool { + get { + lock.lock() + let value = xrOptimalInputWaitEnabledValue + lock.unlock() + return value + } + set { + lock.lock() + xrOptimalInputWaitEnabledValue = newValue + lock.unlock() + } + } + var antiAliasingMode: AntiAliasingMode { get { lock.lock() @@ -2159,6 +2174,15 @@ public var xrFoveatedRenderingEnabled: Bool { set { RuntimeGlobalsStore.shared.xrFoveatedRenderingEnabled = newValue } } +/// Whether the XR loop waits for the compositor's optimal input time before it queries the +/// head pose and encodes the frame, the default. Off, it encodes as soon as the update is +/// done: the GPU starts about the wait's length earlier, and the head pose is predicted from +/// earlier. See `RenderingProperty.optimalInputWait`. +public var xrOptimalInputWaitEnabled: Bool { + get { RuntimeGlobalsStore.shared.xrOptimalInputWaitEnabled } + set { RuntimeGlobalsStore.shared.xrOptimalInputWaitEnabled = newValue } +} + /// Selects the active anti-aliasing pass inserted after the look pass. public enum AntiAliasingMode: Sendable { case none diff --git a/Sources/UntoldEngineXR/UntoldEngineXR.swift b/Sources/UntoldEngineXR/UntoldEngineXR.swift index 903fafbff..145ae6415 100644 --- a/Sources/UntoldEngineXR/UntoldEngineXR.swift +++ b/Sources/UntoldEngineXR/UntoldEngineXR.swift @@ -98,6 +98,11 @@ private var deadlineMissCount = 0 private var worstOverrunMs = 0.0 private var lastDeadlineLogTime: CFTimeInterval = 0 + /// The last second's frames: how many, how many late, their worst and summed overruns. + private var secondFrames = 0 + private var secondLate = 0 + private var secondWorstMs = 0.0 + private var secondOverrunSumMs = 0.0 /// Whether the layer's depth format and the drawable's depth range have been printed. private var loggedDepthSetup = false /// The layer's depth format and the drawable's depth range (far, near), once a drawable @@ -660,7 +665,12 @@ #if ENGINE_STATS_ENABLED let xrWaitStart = CACurrentMediaTime() #endif - LayerRenderer.Clock().wait(until: timing.optimalInputTime, tolerance: .zero) + // The wait for the compositor's optimal input time: the head pose is queried as late + // as the frame allows. Off (a diagnostic), the frame is encoded now, so the GPU + // starts about the wait's length earlier and the head is predicted from earlier. + if xrOptimalInputWaitEnabled { + LayerRenderer.Clock().wait(until: timing.optimalInputTime, tolerance: .zero) + } #if ENGINE_STATS_ENABLED let xrWaitMs = (CACurrentMediaTime() - xrWaitStart) * 1000.0 EngineStatsMonitor.shared.update { snapshot in @@ -1320,21 +1330,37 @@ print(String(format: "XR clock: the compositor epoch's time differs from the media time by %.3f ms", (epochBased - mediaTime) * 1000.0)) } - /// Counts a frame whose GPU work finished after the compositor's rendering deadline, and - /// keeps the longest overrun. Runs on the GPU's completion thread. + /// Counts a frame whose GPU work finished after the compositor's rendering deadline, keeps + /// the longest overrun, and prints once a second how many of that second's frames were + /// late and by how much. Runs on the GPU's completion thread. private func noteGPUCompletion(at gpuEnd: CFTimeInterval, deadline: TimeInterval) { let overrunMs = (gpuEnd - deadline) * 1000.0 - guard overrunMs > 0 else { return } - deadlineLock.lock() - deadlineMissCount += 1 - worstOverrunMs = max(worstOverrunMs, overrunMs) - let count = deadlineMissCount let now = CACurrentMediaTime() - let logs = count == 1 || now - lastDeadlineLogTime >= 1.0 - if logs { lastDeadlineLogTime = now } + var report: String? + deadlineLock.lock() + secondFrames += 1 + if overrunMs > 0 { + deadlineMissCount += 1 + secondLate += 1 + secondOverrunSumMs += overrunMs + secondWorstMs = max(secondWorstMs, overrunMs) + worstOverrunMs = max(worstOverrunMs, overrunMs) + } + if lastDeadlineLogTime == 0 { + lastDeadlineLogTime = now + } else if now - lastDeadlineLogTime >= 1.0 { + if secondLate > 0 { + report = String(format: "XR frame timing: the GPU finished after the rendering deadline on %d of %d frames in the last second (worst +%.2f ms, mean +%.2f ms; %d late frames so far)", secondLate, secondFrames, secondWorstMs, secondOverrunSumMs / Double(secondLate), deadlineMissCount) + } + secondFrames = 0 + secondLate = 0 + secondWorstMs = 0 + secondOverrunSumMs = 0 + lastDeadlineLogTime = now + } deadlineLock.unlock() - if logs { - print(String(format: "XR frame timing: the GPU finished %.2f ms after the rendering deadline (%d late frames so far)", overrunMs, count)) + if let report { + print(report) } } From c8d4be167f09e48718d54f4c5760f59faaead5f4 Mon Sep 17 00:00:00 2001 From: Javier Segura Date: Thu, 15 Oct 2026 08:36:00 +0200 Subject: [PATCH 09/22] [Docs] The headset check's results: the hop is the GPU finishing after the compositor's deadline, and encoding before the wait halves it --- docs/proposals/PresentationTimeAnimation.md | 23 +++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/docs/proposals/PresentationTimeAnimation.md b/docs/proposals/PresentationTimeAnimation.md index 306f64c32..f40d5bd25 100644 --- a/docs/proposals/PresentationTimeAnimation.md +++ b/docs/proposals/PresentationTimeAnimation.md @@ -118,13 +118,31 @@ Recommendation: A. It fixes the judder with no budget change. If fresher input m The compositor part cannot run in `swift test`; section 8 is its check. -## 8. Headset check (to do) +## 8. Headset check 1. The start-up line `XR clock: the compositor epoch's time differs from the media time by N ms`: N should be the microseconds between two reads. A large constant N means the epoch assumption is wrong and the loop must go back to converting through "now" on both clocks. 2. `presentationTimeRevisions` (and its log line) over a minute of the tour: near zero means the predicted and final presentation times agree and the object is posed for the head's instant; a steady count means the prediction is revised by the compositor and the game should be posed for the drawable's time instead (option B's place), or the revision applied as a per-frame correction. 3. The viewer's run summary: "N skipped refreshes" over the run against the fps shortfall (6 in 100, 5 in 90): they should match. "N presentation revisions" is check 2. 4. The tour, with the viewer's change of section 3.4: the small per-frame steps should be gone; the occasional larger jump remains for every skipped refresh and should now be one clean double step with no drift after it. +### Results, 2026-10-15, the viewer's dartboard tour at 90 Hz + +1. `XR clock: the compositor epoch's time differs from the media time by -0.000 ms`: the clock bases agree. +2. Presentation revisions: none. The drawable's presentation time is the one predicted at the update. +3. Skipped refreshes: none. The loop never saw a presentation slot go by without a frame. +4. The small per-frame steps are gone. A hop remained on the tour's lateral sweep only (0.63 m/s peak, 0.15° per frame at 2.37 m, where a frame shown a refresh late is a 12 mm sideways hop), with every counter above at zero. + +The hop is the GPU finishing after the compositor's rendering deadline, which no counter of the loop's pacing sees; `deadlineMisses` and its per-second line (`XR frame timing: the GPU finished after the rendering deadline on N of M frames in the last second`) found it: + +| Run | Late frames per second | Mean overrun | Worst | +| --- | --- | --- | --- | +| Wait for the optimal input time (default), board close | 80 to 91 of 90 | 0.5 to 1.3 ms | 2.6 ms | +| The same, board far (fewer pixels) | 15 to 44 of 90 | 0.2 to 0.7 ms | 3.0 ms | +| `setRendering(.optimalInputWait(.disabled))`: encode right after the update | 56 of 92 | 0.42 ms | 2.2 ms | +| A room probe's prefilter, either run, for about six seconds | every frame, at 71 to 78 fps | 5 to 15 ms | 21 ms | + +The compositor keeps some slack past its stated deadline (the loop still gets 90 frames a second with nearly every frame a millisecond late), and the frames beyond that slack are the hops. Encoding right after the update starts the GPU about 6 ms earlier and brought the hops down to a few; it does not make the frames finish before the deadline, so the frame's GPU work is close to the whole window. The remedies, in order of what they keep: late-latching the head pose (encode before the wait, write the eye matrices after it: the early GPU start and the fresh head pose), cutting the frame's GPU cost (the temporal resolve, the render quality), and spreading a probe's prefilter over frames so a new probe does not cost six seconds at 74 fps. + ## 9. Size, and the risk to the three draft pull requests Size: about 330 lines in `Sources` (`FrameTime.swift` 130, `TransformTrack.swift` 150, the loop and context wiring 50), 330 lines of tests, this document. The viewer's change is a few lines. @@ -143,4 +161,5 @@ Behaviour changes to note in review: `timeSinceLastUpdate` is the presentation d - A `TransformTrackComponent` with a system that samples it at `currentFrameTime.presentationTime` every frame, so a track plays with no game code; `TransformTrack` is the value type it would hold. - `updateCameraPath(frame:)` sampling from an absolute time. - A presented-time feedback for the `MTKView` estimate (section 6). -- A late-input hook after the wait (section 4). +- A late-input hook after the wait (section 4), or its stronger form from the headset results: late-latching the head pose, so the frame is encoded before the wait and only the eye matrices are written after it. +- The room probe's prefilter spread over frames (section 8). From 3838dfb5717daf286ee07bf169aa36d1b14b987b Mon Sep 17 00:00:00 2001 From: Javier Segura Date: Thu, 15 Oct 2026 09:15:41 +0200 Subject: [PATCH 10/22] [Feature] The XR loop waits for the optimal input time while the frames meet the deadline, and not while they do not setRendering(.optimalInputWait(.always | .never | .automatic)), automatic by default: the loop keeps the wait, and the freshest head pose, while the GPU finishes before the compositor's rendering deadline; once a second had late frames (one in twenty or more) it encodes right after the update, so the GPU starts about the wait's length earlier and the compositor's reprojection covers the older prediction; it waits again once a second's least slack would have covered the longest wait skipped, with a margin. optimalInputWaitActive says what the last frame did. The proposal records the adaptive wait and the design and estimate of the true late latch (one camera buffer per frame and eye, written after the wait, read by every pass), which is a refactor of every pass for after the headset-rendering pull request. --- .../Utils/EngineSettingsAPI.swift | 36 ++++++++----- Sources/UntoldEngine/Utils/Globals.swift | 21 ++++---- Sources/UntoldEngineXR/UntoldEngineXR.swift | 53 +++++++++++++++++-- docs/proposals/PresentationTimeAnimation.md | 8 +++ 4 files changed, 90 insertions(+), 28 deletions(-) diff --git a/Sources/UntoldEngine/Utils/EngineSettingsAPI.swift b/Sources/UntoldEngine/Utils/EngineSettingsAPI.swift index f3aea125a..6193a2763 100644 --- a/Sources/UntoldEngine/Utils/EngineSettingsAPI.swift +++ b/Sources/UntoldEngine/Utils/EngineSettingsAPI.swift @@ -101,12 +101,13 @@ public enum RenderingProperty: Sendable { /// `getFoveatedRenderingEnabled()` to read the current value. case foveatedRendering(RenderingToggle) /// Whether the XR loop waits for the compositor's optimal input time before it queries the - /// head pose and encodes the frame. Enabled by default: the head pose is as fresh as the - /// frame allows. Disabled, a diagnostic, the frame is encoded as soon as the update is - /// done, so the GPU starts about the wait's length earlier (a frame that finished after - /// the rendering deadline may then make it) and the head pose is predicted from earlier, - /// which the compositor's reprojection must cover. See `getOptimalInputWaitEnabled()`. - case optimalInputWait(RenderingToggle) + /// head pose and encodes the frame. Waiting gives the freshest head pose; not waiting + /// starts the GPU about the wait's length earlier, so a frame whose GPU work would end + /// after the rendering deadline may make it, with the head pose predicted from earlier for + /// the compositor's reprojection to cover. `.automatic`, the default, waits while the + /// frames meet the deadline, stops when a second had late frames, and resumes once the + /// measured slack would cover the wait again. See `getOptimalInputWaitMode()`. + case optimalInputWait(XRInputWaitMode) /// Whether the pre-composite pass is skipped on a frame it would only copy the lit scene /// (XR passthrough with no environment, splat, ambient-occlusion or gizmo layer to join; /// the lit debug view; no render extension). Enabled by default; disabling it runs the @@ -173,6 +174,17 @@ public enum RenderingToggle: Sendable { case disabled } +/// When the XR loop waits for the compositor's optimal input time before encoding the frame. +/// See `RenderingProperty.optimalInputWait`. +public enum XRInputWaitMode: Sendable, Equatable { + /// Every frame: the freshest head pose, the latest GPU start. + case always + /// Never: the frame is encoded as soon as the update is done. + case never + /// By the measured deadline: waiting while the frames make it, not while they do not. + case automatic +} + public enum RenderExtensionProperty: Sendable { case register(any RenderExtension) case unregister(String) @@ -234,10 +246,8 @@ public func setRendering(_ property: RenderingProperty) { xrFoveatedRenderingEnabled = true case .foveatedRendering(.disabled): xrFoveatedRenderingEnabled = false - case .optimalInputWait(.enabled): - xrOptimalInputWaitEnabled = true - case .optimalInputWait(.disabled): - xrOptimalInputWaitEnabled = false + case let .optimalInputWait(mode): + xrOptimalInputWaitMode = mode case .preCompositeBypass(.enabled): renderInfo.preCompositeBypassEnabled = true case .preCompositeBypass(.disabled): @@ -276,10 +286,10 @@ public func getFoveatedRenderingEnabled() -> Bool { xrFoveatedRenderingEnabled } -/// Whether the XR loop waits for the compositor's optimal input time before it encodes the +/// When the XR loop waits for the compositor's optimal input time before it encodes the /// frame. See `RenderingProperty.optimalInputWait`. -public func getOptimalInputWaitEnabled() -> Bool { - xrOptimalInputWaitEnabled +public func getOptimalInputWaitMode() -> XRInputWaitMode { + xrOptimalInputWaitMode } public enum RenderResourceQuery { diff --git a/Sources/UntoldEngine/Utils/Globals.swift b/Sources/UntoldEngine/Utils/Globals.swift index 97ee61f5c..9ba6ae5e5 100644 --- a/Sources/UntoldEngine/Utils/Globals.swift +++ b/Sources/UntoldEngine/Utils/Globals.swift @@ -1134,7 +1134,7 @@ private final class RuntimeGlobalsStore: @unchecked Sendable { private var enableEngineMetricsValue: Bool = false private var bypassPostProcessingValue: Bool = false private var xrFoveatedRenderingEnabledValue: Bool = true - private var xrOptimalInputWaitEnabledValue: Bool = true + private var xrOptimalInputWaitModeValue: XRInputWaitMode = .automatic private var antiAliasingModeValue: AntiAliasingMode = .fxaa private var renderDebugViewModeValue: RenderDebugViewMode = .lit private var pomQualitySettingsValue: POMQualitySettings = .platformDefault @@ -1654,16 +1654,16 @@ private final class RuntimeGlobalsStore: @unchecked Sendable { } } - var xrOptimalInputWaitEnabled: Bool { + var xrOptimalInputWaitMode: XRInputWaitMode { get { lock.lock() - let value = xrOptimalInputWaitEnabledValue + let value = xrOptimalInputWaitModeValue lock.unlock() return value } set { lock.lock() - xrOptimalInputWaitEnabledValue = newValue + xrOptimalInputWaitModeValue = newValue lock.unlock() } } @@ -2174,13 +2174,12 @@ public var xrFoveatedRenderingEnabled: Bool { set { RuntimeGlobalsStore.shared.xrFoveatedRenderingEnabled = newValue } } -/// Whether the XR loop waits for the compositor's optimal input time before it queries the -/// head pose and encodes the frame, the default. Off, it encodes as soon as the update is -/// done: the GPU starts about the wait's length earlier, and the head pose is predicted from -/// earlier. See `RenderingProperty.optimalInputWait`. -public var xrOptimalInputWaitEnabled: Bool { - get { RuntimeGlobalsStore.shared.xrOptimalInputWaitEnabled } - set { RuntimeGlobalsStore.shared.xrOptimalInputWaitEnabled = newValue } +/// When the XR loop waits for the compositor's optimal input time before it queries the head +/// pose and encodes the frame: always, never, or by the measured deadline (the default). See +/// `RenderingProperty.optimalInputWait`. +public var xrOptimalInputWaitMode: XRInputWaitMode { + get { RuntimeGlobalsStore.shared.xrOptimalInputWaitMode } + set { RuntimeGlobalsStore.shared.xrOptimalInputWaitMode = newValue } } /// Selects the active anti-aliasing pass inserted after the look pass. diff --git a/Sources/UntoldEngineXR/UntoldEngineXR.swift b/Sources/UntoldEngineXR/UntoldEngineXR.swift index 145ae6415..e515eb276 100644 --- a/Sources/UntoldEngineXR/UntoldEngineXR.swift +++ b/Sources/UntoldEngineXR/UntoldEngineXR.swift @@ -98,11 +98,39 @@ private var deadlineMissCount = 0 private var worstOverrunMs = 0.0 private var lastDeadlineLogTime: CFTimeInterval = 0 - /// The last second's frames: how many, how many late, their worst and summed overruns. + /// The last second's frames: how many, how many late, their worst and summed overruns, + /// the least slack any had before the deadline, and the longest wait skipped. private var secondFrames = 0 private var secondLate = 0 private var secondWorstMs = 0.0 private var secondOverrunSumMs = 0.0 + private var secondLeastSlackMs = Double.infinity + private var secondSkippedWaitMaxMs = 0.0 + /// The automatic mode's current choice: waiting for the optimal input time, or not. + private var automaticWaits = true + /// Whether the last frame waited for the compositor's optimal input time: the setting, + /// or in the automatic mode what the measured deadline allowed. + public var optimalInputWaitActive: Bool { deadlineLock.withLock { lastFrameWaited } } + private var lastFrameWaited = true + + /// Whether this frame waits for the optimal input time, by the setting and, in the + /// automatic mode, by what the last seconds' deadlines allowed. `skippedWaitMs` is how + /// long the wait would be, kept for the automatic mode to know what resuming costs. + private func waitsForOptimalInput(skippedWaitMs: Double) -> Bool { + let waits: Bool + deadlineLock.lock() + switch xrOptimalInputWaitMode { + case .always: waits = true + case .never: waits = false + case .automatic: waits = automaticWaits + } + if !waits { + secondSkippedWaitMaxMs = max(secondSkippedWaitMaxMs, skippedWaitMs) + } + lastFrameWaited = waits + deadlineLock.unlock() + return waits + } /// Whether the layer's depth format and the drawable's depth range have been printed. private var loggedDepthSetup = false /// The layer's depth format and the drawable's depth range (far, near), once a drawable @@ -666,9 +694,11 @@ let xrWaitStart = CACurrentMediaTime() #endif // The wait for the compositor's optimal input time: the head pose is queried as late - // as the frame allows. Off (a diagnostic), the frame is encoded now, so the GPU - // starts about the wait's length earlier and the head is predicted from earlier. - if xrOptimalInputWaitEnabled { + // as the frame allows. Without it the frame is encoded now, so the GPU starts about + // the wait's length earlier and the head is predicted from earlier; the automatic + // mode waits while the frames meet the deadline and not while they do not. + let waitLengthMs = max(0, Self.milliseconds(LayerRenderer.Clock().now.duration(to: timing.optimalInputTime))) + if waitsForOptimalInput(skippedWaitMs: waitLengthMs) { LayerRenderer.Clock().wait(until: timing.optimalInputTime, tolerance: .zero) } #if ENGINE_STATS_ENABLED @@ -1339,6 +1369,7 @@ var report: String? deadlineLock.lock() secondFrames += 1 + secondLeastSlackMs = min(secondLeastSlackMs, -overrunMs) if overrunMs > 0 { deadlineMissCount += 1 secondLate += 1 @@ -1352,10 +1383,24 @@ if secondLate > 0 { report = String(format: "XR frame timing: the GPU finished after the rendering deadline on %d of %d frames in the last second (worst +%.2f ms, mean +%.2f ms; %d late frames so far)", secondLate, secondFrames, secondWorstMs, secondOverrunSumMs / Double(secondLate), deadlineMissCount) } + // The automatic mode: stop waiting once a second had late frames (one in + // twenty or more); wait again once a second's least slack would have covered + // the longest wait skipped, with a margin. + if xrOptimalInputWaitMode == .automatic, secondFrames > 0 { + if automaticWaits, secondLate * 20 >= secondFrames { + automaticWaits = false + report = (report ?? "") + String(format: "\nXR frame timing: %d of %d frames late: encoding without the wait for the optimal input time", secondLate, secondFrames) + } else if !automaticWaits, secondLeastSlackMs.isFinite, secondLeastSlackMs > secondSkippedWaitMaxMs + 1.5 { + automaticWaits = true + report = (report ?? "") + String(format: "\nXR frame timing: the least slack of %.2f ms covers the skipped wait of %.2f ms: waiting for the optimal input time again", secondLeastSlackMs, secondSkippedWaitMaxMs) + } + } secondFrames = 0 secondLate = 0 secondWorstMs = 0 secondOverrunSumMs = 0 + secondLeastSlackMs = .infinity + secondSkippedWaitMaxMs = 0 lastDeadlineLogTime = now } deadlineLock.unlock() diff --git a/docs/proposals/PresentationTimeAnimation.md b/docs/proposals/PresentationTimeAnimation.md index f40d5bd25..8b92700ee 100644 --- a/docs/proposals/PresentationTimeAnimation.md +++ b/docs/proposals/PresentationTimeAnimation.md @@ -143,6 +143,14 @@ The hop is the GPU finishing after the compositor's rendering deadline, which no The compositor keeps some slack past its stated deadline (the loop still gets 90 frames a second with nearly every frame a millisecond late), and the frames beyond that slack are the hops. Encoding right after the update starts the GPU about 6 ms earlier and brought the hops down to a few; it does not make the frames finish before the deadline, so the frame's GPU work is close to the whole window. The remedies, in order of what they keep: late-latching the head pose (encode before the wait, write the eye matrices after it: the early GPU start and the fresh head pose), cutting the frame's GPU cost (the temporal resolve, the render quality), and spreading a probe's prefilter over frames so a new probe does not cost six seconds at 74 fps. +### The adaptive wait (done) + +`setRendering(.optimalInputWait(.always | .never | .automatic))`, automatic by default: the loop waits for the optimal input time while the frames meet the deadline, stops once a second had late frames (one in twenty or more), and waits again once a second's least slack before the deadline would have covered the longest wait skipped, with a 1.5 ms margin. The fresh head pose whenever the budget allows; the compositor's reprojection of an older prediction only when it does not. `optimalInputWaitActive` says what the last frame did; the viewer's switch maps to automatic or never and its summary notes "input-time wait skipped (automatic)". + +### The true late latch (not done: a refactor, for a decision) + +The head pose is consumed on the CPU while the frame is encoded: the view and projection matrices are read at 41 places in `RenderPasses.swift` (each pass copies them into its own uniforms: the batches, the grid, the sky, the environment, the shadow cascades, the SSAO's projection scale), 7 in `CullingSystem.swift` (the frustum planes and the view-projection for the occlusion test), 5 in `TemporalAntiAliasing.swift` (the jitter and the reprojection), 3 in `GraphBuilder.swift`, and a few in the extension surface and the muscle debug pass. Encoding before the wait and writing the pose after it therefore needs one per-frame, per-eye camera buffer (view, projection, their inverses, the view-projection, the eye position, the jitter) bound at a fixed index for every pass and read by every shader in place of the copies; the frustum planes built on the GPU from it (a small kernel before the culls, or the culls reading the matrices directly); the temporal resolve taking the previous frame's buffer as its reprojection; the shadow cascades fitted from a provisional pose a few milliseconds older, which is harmless; and the drawable, with its foveation rate maps, taken before the wait, earlier than Apple's sample does, to be verified on the device first. It touches every pass and most shaders, a few days of work, and it should follow #1359 rather than ride on it. It buys about 3 ms of GPU window (the encode after the wait) with today's head pose; the adaptive wait buys about 9 ms with an older one when needed. + ## 9. Size, and the risk to the three draft pull requests Size: about 330 lines in `Sources` (`FrameTime.swift` 130, `TransformTrack.swift` 150, the loop and context wiring 50), 330 lines of tests, this document. The viewer's change is a few lines. From 785ecae440b78f81f2a402ce871d47bc389ef3b9 Mon Sep 17 00:00:00 2001 From: Javier Segura Date: Thu, 15 Oct 2026 09:25:33 +0200 Subject: [PATCH 11/22] [Chores] The per-second late-frame line says the input-time wait's mode and whether the frame waited --- Sources/UntoldEngineXR/UntoldEngineXR.swift | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Sources/UntoldEngineXR/UntoldEngineXR.swift b/Sources/UntoldEngineXR/UntoldEngineXR.swift index e515eb276..497912409 100644 --- a/Sources/UntoldEngineXR/UntoldEngineXR.swift +++ b/Sources/UntoldEngineXR/UntoldEngineXR.swift @@ -1381,7 +1381,13 @@ lastDeadlineLogTime = now } else if now - lastDeadlineLogTime >= 1.0 { if secondLate > 0 { - report = String(format: "XR frame timing: the GPU finished after the rendering deadline on %d of %d frames in the last second (worst +%.2f ms, mean +%.2f ms; %d late frames so far)", secondLate, secondFrames, secondWorstMs, secondOverrunSumMs / Double(secondLate), deadlineMissCount) + let mode: String + switch xrOptimalInputWaitMode { + case .always: mode = "always" + case .never: mode = "never" + case .automatic: mode = "automatic" + } + report = String(format: "XR frame timing: the GPU finished after the rendering deadline on %d of %d frames in the last second (worst +%.2f ms, mean +%.2f ms; %d late frames so far; input-time wait %@, %@)", secondLate, secondFrames, secondWorstMs, secondOverrunSumMs / Double(secondLate), deadlineMissCount, mode, lastFrameWaited ? "waited" : "not waited") } // The automatic mode: stop waiting once a second had late frames (one in // twenty or more); wait again once a second's least slack would have covered From 1b595ca14336236d97b1ca8dcc47b28dc374b967 Mon Sep 17 00:00:00 2001 From: Javier Segura Date: Thu, 15 Oct 2026 09:31:29 +0200 Subject: [PATCH 12/22] [Chores] A second of frames well past the deadline names the heaviest passes of the last frame --- Sources/UntoldEngineXR/UntoldEngineXR.swift | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Sources/UntoldEngineXR/UntoldEngineXR.swift b/Sources/UntoldEngineXR/UntoldEngineXR.swift index 497912409..155ef6740 100644 --- a/Sources/UntoldEngineXR/UntoldEngineXR.swift +++ b/Sources/UntoldEngineXR/UntoldEngineXR.swift @@ -1387,7 +1387,17 @@ case .never: mode = "never" case .automatic: mode = "automatic" } - report = String(format: "XR frame timing: the GPU finished after the rendering deadline on %d of %d frames in the last second (worst +%.2f ms, mean +%.2f ms; %d late frames so far; input-time wait %@, %@)", secondLate, secondFrames, secondWorstMs, secondOverrunSumMs / Double(secondLate), deadlineMissCount, mode, lastFrameWaited ? "waited" : "not waited") + let meanOverrunMs = secondOverrunSumMs / Double(secondLate) + report = String(format: "XR frame timing: the GPU finished after the rendering deadline on %d of %d frames in the last second (worst +%.2f ms, mean +%.2f ms; %d late frames so far; input-time wait %@, %@)", secondLate, secondFrames, secondWorstMs, meanOverrunMs, deadlineMissCount, mode, lastFrameWaited ? "waited" : "not waited") + // With the GPU pass timing on, the heaviest passes of the last resolved frame + // name what a second of frames well past the deadline spent its time on. + if meanOverrunMs > 2.0, GPUPassTiming.shared.isEnabled { + let passes = GPUPassTiming.shared.lastFrame().passes.sorted { $0.totalMs > $1.totalMs }.prefix(3) + if !passes.isEmpty { + let heaviest = passes.map { String(format: "%@ %.2f ms", $0.label, $0.totalMs) }.joined(separator: ", ") + report = (report ?? "") + "\nXR frame timing: the heaviest passes of the last frame: " + heaviest + } + } } // The automatic mode: stop waiting once a second had late frames (one in // twenty or more); wait again once a second's least slack would have covered From d029790b7a971588d38b19ee870cf1a70a9919e5 Mon Sep 17 00:00:00 2001 From: Javier Segura Date: Thu, 15 Oct 2026 10:27:32 +0200 Subject: [PATCH 13/22] [Bugfix] A shadow softness set before the shadow system exists is kept for it instead of crashing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit setShadowSoftness dereferenced the shadow system, which the renderer creates, so a setting made with no space open — the viewer re-applies its switches, shadow taps among them, on any change — was a crash. The setting is now kept while no system exists, reported by getShadowSoftness as the system will hold it, and given to the system when the renderer creates it. --- .../UntoldEngine/Renderer/UntoldEngine.swift | 1 + .../UntoldEngine/Systems/ShadowSystem.swift | 26 +++++++- Sources/UntoldEngine/Utils/Globals.swift | 22 +++++++ .../ShadowSoftnessBeforeSystemTests.swift | 61 +++++++++++++++++++ 4 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 Tests/UntoldEngineTests/ShadowSoftnessBeforeSystemTests.swift diff --git a/Sources/UntoldEngine/Renderer/UntoldEngine.swift b/Sources/UntoldEngine/Renderer/UntoldEngine.swift index 53258cc4f..057b09461 100644 --- a/Sources/UntoldEngine/Renderer/UntoldEngine.swift +++ b/Sources/UntoldEngine/Renderer/UntoldEngine.swift @@ -193,6 +193,7 @@ public class UntoldRenderer: NSObject, MTKViewDelegate { initSizeableResources() // TODO: Find a better name function shadowSystem = ShadowSystem() + applyShadowSoftnessSetBeforeSystem() pointShadowState = PointShadowState() spotShadowState = SpotShadowState() diff --git a/Sources/UntoldEngine/Systems/ShadowSystem.swift b/Sources/UntoldEngine/Systems/ShadowSystem.swift index 0302c9b7d..952ca4b79 100644 --- a/Sources/UntoldEngine/Systems/ShadowSystem.swift +++ b/Sources/UntoldEngine/Systems/ShadowSystem.swift @@ -364,7 +364,7 @@ struct ShadowSystem { softnessSettings = Self.sanitizedSoftnessSettings(settings) } - private static func sanitizedSoftnessSettings(_ settings: ShadowSoftnessSettings) -> ShadowSoftnessSettings { + static func sanitizedSoftnessSettings(_ settings: ShadowSoftnessSettings) -> ShadowSoftnessSettings { let nearRadius = simd_clamp(settings.nearRadiusTexels, 0.25, 8.0) let farRadius = simd_clamp(settings.farRadiusTexels, nearRadius, 12.0) let depthScale = simd_clamp(settings.depthScale, 0.0, 2.0) @@ -705,12 +705,34 @@ struct ShadowSystem { } } +/// Sets the sun shadow's softness. Before the shadow system exists (before the renderer is +/// created, or between two sessions) the setting is kept and given to the system when the +/// renderer creates it, so a setting made with no space open is neither lost nor a crash. public func setShadowSoftness(_ settings: ShadowSoftnessSettings) { + guard shadowSystem != nil else { + shadowSoftnessBeforeSystem = settings + return + } shadowSystem.setSoftness(settings) } +/// The sun shadow's softness: the system's, or, before it exists, the setting kept for it, +/// as the system will hold it. public func getShadowSoftness() -> ShadowSoftnessSettings { - shadowSystem.softnessSettings + if let system = shadowSystem { + return system.softnessSettings + } + if let kept = shadowSoftnessBeforeSystem { + return ShadowSystem.sanitizedSoftnessSettings(kept) + } + return ShadowSystem().softnessSettings +} + +/// Gives a shadow system just created the softness set before it existed, if any. +func applyShadowSoftnessSetBeforeSystem() { + guard let settings = shadowSoftnessBeforeSystem else { return } + shadowSoftnessBeforeSystem = nil + shadowSystem?.setSoftness(settings) } /// Current value set via `setRendering(.maxShadowCastingDistance(_:))`. diff --git a/Sources/UntoldEngine/Utils/Globals.swift b/Sources/UntoldEngine/Utils/Globals.swift index 9ba6ae5e5..80daff8d3 100644 --- a/Sources/UntoldEngine/Utils/Globals.swift +++ b/Sources/UntoldEngine/Utils/Globals.swift @@ -1135,6 +1135,7 @@ private final class RuntimeGlobalsStore: @unchecked Sendable { private var bypassPostProcessingValue: Bool = false private var xrFoveatedRenderingEnabledValue: Bool = true private var xrOptimalInputWaitModeValue: XRInputWaitMode = .automatic + private var shadowSoftnessBeforeSystemValue: ShadowSoftnessSettings? private var antiAliasingModeValue: AntiAliasingMode = .fxaa private var renderDebugViewModeValue: RenderDebugViewMode = .lit private var pomQualitySettingsValue: POMQualitySettings = .platformDefault @@ -1668,6 +1669,20 @@ private final class RuntimeGlobalsStore: @unchecked Sendable { } } + var shadowSoftnessBeforeSystem: ShadowSoftnessSettings? { + get { + lock.lock() + let value = shadowSoftnessBeforeSystemValue + lock.unlock() + return value + } + set { + lock.lock() + shadowSoftnessBeforeSystemValue = newValue + lock.unlock() + } + } + var antiAliasingMode: AntiAliasingMode { get { lock.lock() @@ -2182,6 +2197,13 @@ public var xrOptimalInputWaitMode: XRInputWaitMode { set { RuntimeGlobalsStore.shared.xrOptimalInputWaitMode = newValue } } +/// The sun shadow's softness set while no shadow system existed, for the system the renderer +/// creates next. See `setShadowSoftness`. +var shadowSoftnessBeforeSystem: ShadowSoftnessSettings? { + get { RuntimeGlobalsStore.shared.shadowSoftnessBeforeSystem } + set { RuntimeGlobalsStore.shared.shadowSoftnessBeforeSystem = newValue } +} + /// Selects the active anti-aliasing pass inserted after the look pass. public enum AntiAliasingMode: Sendable { case none diff --git a/Tests/UntoldEngineTests/ShadowSoftnessBeforeSystemTests.swift b/Tests/UntoldEngineTests/ShadowSoftnessBeforeSystemTests.swift new file mode 100644 index 000000000..bdc162df1 --- /dev/null +++ b/Tests/UntoldEngineTests/ShadowSoftnessBeforeSystemTests.swift @@ -0,0 +1,61 @@ +// +// ShadowSoftnessBeforeSystemTests.swift +// UntoldEngineTests +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +@testable import UntoldEngine +import XCTest + +/// A shadow softness set before the shadow system exists (before the renderer, or between two +/// sessions) is kept and given to the system when it is created, instead of crashing. +@MainActor +final class ShadowSoftnessBeforeSystemTests: XCTestCase { + private var previousSystem: ShadowSystem? + + override func setUp() { + previousSystem = shadowSystem + shadowSystem = nil + shadowSoftnessBeforeSystem = nil + } + + override func tearDown() { + shadowSystem = previousSystem + shadowSoftnessBeforeSystem = nil + } + + func testASettingBeforeTheSystemIsKeptAndGivenToItAtItsCreation() { + let wanted = ShadowSoftnessSettings(filterTaps: 4, xrFilterTaps: 4) + setShadowSoftness(wanted) + var reference = ShadowSystem() + reference.setSoftness(wanted) + XCTAssertEqual(getShadowSoftness(), reference.softnessSettings) + + shadowSystem = ShadowSystem() + applyShadowSoftnessSetBeforeSystem() + XCTAssertEqual(shadowSystem.softnessSettings, reference.softnessSettings) + XCTAssertNil(shadowSoftnessBeforeSystem) + + // With the system there, a setting goes to it directly. + let later = ShadowSoftnessSettings(filterTaps: 16, xrFilterTaps: 16) + setShadowSoftness(later) + var laterReference = ShadowSystem() + laterReference.setSoftness(later) + XCTAssertEqual(getShadowSoftness(), laterReference.softnessSettings) + XCTAssertNil(shadowSoftnessBeforeSystem) + } + + func testWithoutAnySettingTheDefaultIsReported() { + XCTAssertEqual(getShadowSoftness(), ShadowSystem().softnessSettings) + } + + func testASystemCreatedWithNothingKeptKeepsItsDefault() { + shadowSystem = ShadowSystem() + applyShadowSoftnessSetBeforeSystem() + XCTAssertEqual(shadowSystem.softnessSettings, ShadowSystem().softnessSettings) + } +} From 9e978d2cd678f58aa57caffd8d1fed9f832791c9 Mon Sep 17 00:00:00 2001 From: Javier Segura Date: Thu, 15 Oct 2026 11:02:22 +0200 Subject: [PATCH 14/22] [Feature] The XR loop renders every other refresh, evenly, while a frame's GPU work does not fit one A frame whose GPU work does not fit one refresh is shown a refresh late now and then, which hops; shown two refreshes apart, evenly, and posed for that instant, it moves smoothly at half the rate while the compositor reprojects the head in between. setRendering(.framePacing(.fullRate | .halfRateWhenLate | .halfRate)), half rate when late by default: XRHalfRatePolicy enters when, with no wait for the optimal input time left to skip, half the frames or more were late by over a millisecond on average for two seconds running, and leaves when the rendered frames' GPU work has fitted 80% of the refresh for two seconds running. While active the loop lets every other compositor frame go, its update phase marked and nothing submitted, and poses the rendered frames one refresh later than the compositor's prediction while they land a refresh late. halfRatePacingActive says the state; the per-second line adds "half rate". Eight tests cover the policy; the proposal records the pass costs the pacing answers. --- .../UntoldEngine/Renderer/XRFramePacing.swift | 89 +++++++++++++++ .../Utils/EngineSettingsAPI.swift | 14 +++ Sources/UntoldEngine/Utils/Globals.swift | 23 ++++ Sources/UntoldEngineXR/UntoldEngineXR.swift | 82 ++++++++++++-- .../XRHalfRatePolicyTests.swift | 102 ++++++++++++++++++ docs/proposals/PresentationTimeAnimation.md | 8 ++ 6 files changed, 311 insertions(+), 7 deletions(-) create mode 100644 Sources/UntoldEngine/Renderer/XRFramePacing.swift create mode 100644 Tests/UntoldEngineTests/XRHalfRatePolicyTests.swift diff --git a/Sources/UntoldEngine/Renderer/XRFramePacing.swift b/Sources/UntoldEngine/Renderer/XRFramePacing.swift new file mode 100644 index 000000000..267ef2bef --- /dev/null +++ b/Sources/UntoldEngine/Renderer/XRFramePacing.swift @@ -0,0 +1,89 @@ +// +// XRFramePacing.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Foundation + +/// How the XR loop paces its frames against the display's refreshes. A frame whose GPU work +/// does not fit one refresh is shown a refresh late, sometimes, which hops; rendered every +/// other refresh it is shown two refreshes apart, evenly, posed for that instant, with the +/// compositor reprojecting the head in between. See `RenderingProperty.framePacing`. +public enum XRFramePacing: Sendable, Equatable { + /// Every refresh, whatever the frames' cost. + case fullRate + /// Every refresh while the frames meet the deadline; every other one while they do not. + case halfRateWhenLate + /// Every other refresh, always. + case halfRate +} + +/// The half-rate decision from each second's deadline measurements. Enter when, with no wait +/// for the optimal input time left to skip, half the frames or more were late by over a +/// millisecond on average for two seconds running; leave when the rendered frames' GPU work +/// has fitted 80% of the refresh for two seconds running. +public struct XRHalfRatePolicy: Equatable, Sendable { + /// Whether every other refresh is being skipped. + public private(set) var active = false + /// Whether the last second's rendered frames mostly landed a refresh late: the pose then + /// aims one refresh later, where the display shows them. + public private(set) var framesLandLate = false + private(set) var lateSeconds = 0 + private(set) var fitSeconds = 0 + + public init() {} + + /// What a second's measurements changed. + public enum Change: Equatable, Sendable { + case none + case entered + case left + } + + /// The share of a refresh the rendered frames' GPU work must fit for the pacing to end. + public static let fitShare = 0.8 + + /// A second of frames ended: `frames` rendered, `late` of them past the deadline by + /// `meanOverrunMs` on average, the longest GPU span among them `maxGPUDurationMs`, the + /// display's refresh `periodMs`, and whether the loop still has a wait it could skip first. + public mutating func secondEnded(frames: Int, late: Int, meanOverrunMs: Double, maxGPUDurationMs: Double, periodMs: Double, waitStillToSkip: Bool) -> Change { + guard frames > 0 else { return .none } + let mostlyLate = late * 2 >= frames + if !active { + framesLandLate = false + if !waitStillToSkip, mostlyLate, meanOverrunMs > 1.0 { + lateSeconds += 1 + } else { + lateSeconds = 0 + } + if lateSeconds >= 2 { + active = true + lateSeconds = 0 + return .entered + } + return .none + } + framesLandLate = mostlyLate + if periodMs > 0, maxGPUDurationMs < Self.fitShare * periodMs { + fitSeconds += 1 + } else { + fitSeconds = 0 + } + if fitSeconds >= 2 { + active = false + fitSeconds = 0 + framesLandLate = false + return .left + } + return .none + } + + public mutating func reset() { + self = XRHalfRatePolicy() + } +} diff --git a/Sources/UntoldEngine/Utils/EngineSettingsAPI.swift b/Sources/UntoldEngine/Utils/EngineSettingsAPI.swift index 6193a2763..2b8f4741a 100644 --- a/Sources/UntoldEngine/Utils/EngineSettingsAPI.swift +++ b/Sources/UntoldEngine/Utils/EngineSettingsAPI.swift @@ -108,6 +108,12 @@ public enum RenderingProperty: Sendable { /// frames meet the deadline, stops when a second had late frames, and resumes once the /// measured slack would cover the wait again. See `getOptimalInputWaitMode()`. case optimalInputWait(XRInputWaitMode) + /// How the XR loop paces its frames against the display's refreshes: every refresh; every + /// other refresh while the frames run late (the default), so a frame whose GPU work does + /// not fit one refresh is shown two refreshes apart, evenly, and posed for that instant, + /// instead of landing a refresh late now and then, which hops; or every other refresh + /// always. The compositor reprojects the head between the frames. See `getFramePacing()`. + case framePacing(XRFramePacing) /// Whether the pre-composite pass is skipped on a frame it would only copy the lit scene /// (XR passthrough with no environment, splat, ambient-occlusion or gizmo layer to join; /// the lit debug view; no render extension). Enabled by default; disabling it runs the @@ -248,6 +254,8 @@ public func setRendering(_ property: RenderingProperty) { xrFoveatedRenderingEnabled = false case let .optimalInputWait(mode): xrOptimalInputWaitMode = mode + case let .framePacing(pacing): + xrFramePacing = pacing case .preCompositeBypass(.enabled): renderInfo.preCompositeBypassEnabled = true case .preCompositeBypass(.disabled): @@ -292,6 +300,12 @@ public func getOptimalInputWaitMode() -> XRInputWaitMode { xrOptimalInputWaitMode } +/// How the XR loop paces its frames against the display's refreshes. See +/// `RenderingProperty.framePacing`. +public func getFramePacing() -> XRFramePacing { + xrFramePacing +} + public enum RenderResourceQuery { case texture(String) } diff --git a/Sources/UntoldEngine/Utils/Globals.swift b/Sources/UntoldEngine/Utils/Globals.swift index 80daff8d3..d0ad1c6e5 100644 --- a/Sources/UntoldEngine/Utils/Globals.swift +++ b/Sources/UntoldEngine/Utils/Globals.swift @@ -1135,6 +1135,7 @@ private final class RuntimeGlobalsStore: @unchecked Sendable { private var bypassPostProcessingValue: Bool = false private var xrFoveatedRenderingEnabledValue: Bool = true private var xrOptimalInputWaitModeValue: XRInputWaitMode = .automatic + private var xrFramePacingValue: XRFramePacing = .halfRateWhenLate private var shadowSoftnessBeforeSystemValue: ShadowSoftnessSettings? private var antiAliasingModeValue: AntiAliasingMode = .fxaa private var renderDebugViewModeValue: RenderDebugViewMode = .lit @@ -1669,6 +1670,20 @@ private final class RuntimeGlobalsStore: @unchecked Sendable { } } + var xrFramePacing: XRFramePacing { + get { + lock.lock() + let value = xrFramePacingValue + lock.unlock() + return value + } + set { + lock.lock() + xrFramePacingValue = newValue + lock.unlock() + } + } + var shadowSoftnessBeforeSystem: ShadowSoftnessSettings? { get { lock.lock() @@ -2197,6 +2212,14 @@ public var xrOptimalInputWaitMode: XRInputWaitMode { set { RuntimeGlobalsStore.shared.xrOptimalInputWaitMode = newValue } } +/// How the XR loop paces its frames against the display's refreshes: every one, every other +/// one while the frames run late (the default), or every other one always. See +/// `RenderingProperty.framePacing`. +public var xrFramePacing: XRFramePacing { + get { RuntimeGlobalsStore.shared.xrFramePacing } + set { RuntimeGlobalsStore.shared.xrFramePacing = newValue } +} + /// The sun shadow's softness set while no shadow system existed, for the system the renderer /// creates next. See `setShadowSoftness`. var shadowSoftnessBeforeSystem: ShadowSoftnessSettings? { diff --git a/Sources/UntoldEngineXR/UntoldEngineXR.swift b/Sources/UntoldEngineXR/UntoldEngineXR.swift index 155ef6740..8998d77dc 100644 --- a/Sources/UntoldEngineXR/UntoldEngineXR.swift +++ b/Sources/UntoldEngineXR/UntoldEngineXR.swift @@ -108,6 +108,15 @@ private var secondSkippedWaitMaxMs = 0.0 /// The automatic mode's current choice: waiting for the optimal input time, or not. private var automaticWaits = true + /// The half-rate pacing: its policy, the alternator that lets every other compositor + /// frame go while it is active, whether this frame is paced, and the last second's + /// longest GPU span. + private var halfRate = XRHalfRatePolicy() + private var halfRateSkipNext = true + private var pacingActiveNow = false + private var secondMaxGPUMs = 0.0 + /// Whether the loop is rendering every other refresh at the moment. + public var halfRatePacingActive: Bool { deadlineLock.withLock { pacingActiveNow } } /// Whether the last frame waited for the compositor's optimal input time: the setting, /// or in the automatic mode what the measured deadline allowed. public var optimalInputWaitActive: Bool { deadlineLock.withLock { lastFrameWaited } } @@ -131,6 +140,33 @@ deadlineLock.unlock() return waits } + + /// Whether this compositor frame is let go for the half-rate pacing: every other one + /// while the pacing is active, by the setting or by the policy when the frames run late. + private func skipsFrameForPacing() -> Bool { + let active: Bool + deadlineLock.lock() + switch xrFramePacing { + case .fullRate: active = false + case .halfRate: active = true + case .halfRateWhenLate: active = halfRate.active + } + pacingActiveNow = active + deadlineLock.unlock() + guard active else { + halfRateSkipNext = true + return false + } + halfRateSkipNext.toggle() + return halfRateSkipNext + } + + /// How far past the compositor's predicted presentation the frame is posed for: one + /// refresh while the half-rate pacing's rendered frames land a refresh late, where the + /// display then shows them; nothing otherwise. + private func presentationLeadSeconds() -> TimeInterval { + deadlineLock.withLock { pacingActiveNow && halfRate.framesLandLate ? currentFrameTime.period : 0 } + } /// Whether the layer's depth format and the drawable's depth range have been printed. private var loggedDepthSetup = false /// The layer's depth format and the drawable's depth range (far, near), once a drawable @@ -632,6 +668,15 @@ guard let frame = layerRenderer.queryNextFrame() else { return } + // The half-rate pacing: every other compositor frame is let go, so the frames + // rendered are shown two refreshes apart, evenly, when one refresh is not enough + // for a frame's GPU work; the display shows the previous frame again meanwhile. + if skipsFrameForPacing() { + frame.startUpdate() + frame.endUpdate() + return + } + // 2. Call predictTiming to get the predicted render deadlines for code guard let timing = frame.predictTiming() else { return } @@ -639,7 +684,8 @@ // base: the game poses the frame for it in the update phase, and the head is posed // for the same instant at submission (the drawable's frameTiming, normally this // very value; notePresentationRevision counts the frames where it is not). - let predictedPresentationTime = Self.mediaTime(of: timing.presentationTime) + let compositorPresentationTime = Self.mediaTime(of: timing.presentationTime) + let predictedPresentationTime = compositorPresentationTime + presentationLeadSeconds() logClockBasesOnce() // 3. Call startUpdate() to mark the start of the update phase @@ -777,7 +823,7 @@ let presentationTimeCA: TimeInterval = Self.mediaTime(of: presentationInstant) if !presentationChecked { presentationChecked = true - notePresentationRevision(predicted: predictedPresentationTime, final: presentationTimeCA) + notePresentationRevision(predicted: compositorPresentationTime, final: presentationTimeCA) } var deviceAnchor = queryDeviceAnchorIfTrackingRunning(atTimestamp: presentationTimeCA) @@ -1197,7 +1243,7 @@ EngineStatsMonitor.shared.recordGPUCompletion(gpuStartTime: cb.gpuStartTime, gpuEndTime: cb.gpuEndTime) #endif if countsDeadline { - self?.noteGPUCompletion(at: cb.gpuEndTime, deadline: deadline) + self?.noteGPUCompletion(at: cb.gpuEndTime, start: cb.gpuStartTime, deadline: deadline) } commandBufferSemaphore.signal() } @@ -1363,13 +1409,14 @@ /// Counts a frame whose GPU work finished after the compositor's rendering deadline, keeps /// the longest overrun, and prints once a second how many of that second's frames were /// late and by how much. Runs on the GPU's completion thread. - private func noteGPUCompletion(at gpuEnd: CFTimeInterval, deadline: TimeInterval) { + private func noteGPUCompletion(at gpuEnd: CFTimeInterval, start gpuStart: CFTimeInterval, deadline: TimeInterval) { let overrunMs = (gpuEnd - deadline) * 1000.0 let now = CACurrentMediaTime() var report: String? deadlineLock.lock() secondFrames += 1 secondLeastSlackMs = min(secondLeastSlackMs, -overrunMs) + secondMaxGPUMs = max(secondMaxGPUMs, (gpuEnd - gpuStart) * 1000.0) if overrunMs > 0 { deadlineMissCount += 1 secondLate += 1 @@ -1388,7 +1435,7 @@ case .automatic: mode = "automatic" } let meanOverrunMs = secondOverrunSumMs / Double(secondLate) - report = String(format: "XR frame timing: the GPU finished after the rendering deadline on %d of %d frames in the last second (worst +%.2f ms, mean +%.2f ms; %d late frames so far; input-time wait %@, %@)", secondLate, secondFrames, secondWorstMs, meanOverrunMs, deadlineMissCount, mode, lastFrameWaited ? "waited" : "not waited") + report = String(format: "XR frame timing: the GPU finished after the rendering deadline on %d of %d frames in the last second (worst +%.2f ms, mean +%.2f ms; %d late frames so far; input-time wait %@, %@%@)", secondLate, secondFrames, secondWorstMs, meanOverrunMs, deadlineMissCount, mode, lastFrameWaited ? "waited" : "not waited", pacingActiveNow ? "; half rate" : "") // With the GPU pass timing on, the heaviest passes of the last resolved frame // name what a second of frames well past the deadline spent its time on. if meanOverrunMs > 2.0, GPUPassTiming.shared.isEnabled { @@ -1405,18 +1452,39 @@ if xrOptimalInputWaitMode == .automatic, secondFrames > 0 { if automaticWaits, secondLate * 20 >= secondFrames { automaticWaits = false - report = (report ?? "") + String(format: "\nXR frame timing: %d of %d frames late: encoding without the wait for the optimal input time", secondLate, secondFrames) + let line = String(format: "XR frame timing: %d of %d frames late: encoding without the wait for the optimal input time", secondLate, secondFrames) + report = report.map { $0 + "\n" + line } ?? line } else if !automaticWaits, secondLeastSlackMs.isFinite, secondLeastSlackMs > secondSkippedWaitMaxMs + 1.5 { automaticWaits = true - report = (report ?? "") + String(format: "\nXR frame timing: the least slack of %.2f ms covers the skipped wait of %.2f ms: waiting for the optimal input time again", secondLeastSlackMs, secondSkippedWaitMaxMs) + let line = String(format: "XR frame timing: the least slack of %.2f ms covers the skipped wait of %.2f ms: waiting for the optimal input time again", secondLeastSlackMs, secondSkippedWaitMaxMs) + report = report.map { $0 + "\n" + line } ?? line } } + // The half-rate pacing, by its policy when the setting leaves it to the frames. + if xrFramePacing == .halfRateWhenLate { + let waitStillToSkip = xrOptimalInputWaitMode == .automatic && automaticWaits + let meanLateMs = secondLate > 0 ? secondOverrunSumMs / Double(secondLate) : 0 + let periodMs = currentFrameTime.period * 1000.0 + switch halfRate.secondEnded(frames: secondFrames, late: secondLate, meanOverrunMs: meanLateMs, maxGPUDurationMs: secondMaxGPUMs, periodMs: periodMs, waitStillToSkip: waitStillToSkip) { + case .entered: + let line = String(format: "XR frame timing: %d of %d frames late with no wait left to skip: rendering every other refresh", secondLate, secondFrames) + report = report.map { $0 + "\n" + line } ?? line + case .left: + let line = String(format: "XR frame timing: the frame's GPU work of %.2f ms fits the refresh of %.2f ms: rendering every refresh again", secondMaxGPUMs, periodMs) + report = report.map { $0 + "\n" + line } ?? line + case .none: + break + } + } else { + halfRate.reset() + } secondFrames = 0 secondLate = 0 secondWorstMs = 0 secondOverrunSumMs = 0 secondLeastSlackMs = .infinity secondSkippedWaitMaxMs = 0 + secondMaxGPUMs = 0 lastDeadlineLogTime = now } deadlineLock.unlock() diff --git a/Tests/UntoldEngineTests/XRHalfRatePolicyTests.swift b/Tests/UntoldEngineTests/XRHalfRatePolicyTests.swift new file mode 100644 index 000000000..55a906e7d --- /dev/null +++ b/Tests/UntoldEngineTests/XRHalfRatePolicyTests.swift @@ -0,0 +1,102 @@ +// +// XRHalfRatePolicyTests.swift +// UntoldEngineTests +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +@testable import UntoldEngine +import XCTest + +/// The half-rate pacing enters after two seconds of mostly late frames with no wait left to +/// skip, aims the pose a refresh later while the rendered frames land late, and leaves after +/// two seconds in which the frames' GPU work fitted most of a refresh. +final class XRHalfRatePolicyTests: XCTestCase { + private let period = 10.0 + + private func lateSecond(_ policy: inout XRHalfRatePolicy, waitStillToSkip: Bool = false, maxGPU: Double = 12.0) -> XRHalfRatePolicy.Change { + policy.secondEnded(frames: 100, late: 80, meanOverrunMs: 3.0, maxGPUDurationMs: maxGPU, periodMs: period, waitStillToSkip: waitStillToSkip) + } + + private func goodSecond(_ policy: inout XRHalfRatePolicy, maxGPU: Double = 7.0) -> XRHalfRatePolicy.Change { + policy.secondEnded(frames: 100, late: 2, meanOverrunMs: 0.2, maxGPUDurationMs: maxGPU, periodMs: period, waitStillToSkip: false) + } + + func testEntersAfterTwoLateSecondsNotOne() { + var policy = XRHalfRatePolicy() + XCTAssertEqual(lateSecond(&policy), .none) + XCTAssertFalse(policy.active) + XCTAssertEqual(lateSecond(&policy), .entered) + XCTAssertTrue(policy.active) + } + + func testAGoodSecondBetweenTwoLateOnesStartsTheCountAgain() { + var policy = XRHalfRatePolicy() + XCTAssertEqual(lateSecond(&policy), .none) + XCTAssertEqual(goodSecond(&policy), .none) + XCTAssertEqual(lateSecond(&policy), .none) + XCTAssertFalse(policy.active) + } + + func testDoesNotEnterWhileAWaitCouldStillBeSkipped() { + var policy = XRHalfRatePolicy() + XCTAssertEqual(lateSecond(&policy, waitStillToSkip: true), .none) + XCTAssertEqual(lateSecond(&policy, waitStillToSkip: true), .none) + XCTAssertFalse(policy.active) + } + + func testDoesNotEnterForSlightlyLateFrames() { + var policy = XRHalfRatePolicy() + for _ in 0 ..< 3 { + XCTAssertEqual(policy.secondEnded(frames: 100, late: 60, meanOverrunMs: 0.4, maxGPUDurationMs: 10.2, periodMs: period, waitStillToSkip: false), .none) + } + XCTAssertFalse(policy.active) + } + + func testWhileActiveThePoseAimsLaterOnlyWhileTheFramesLandLate() { + var policy = XRHalfRatePolicy() + _ = lateSecond(&policy) + _ = lateSecond(&policy) + XCTAssertTrue(policy.active) + XCTAssertFalse(policy.framesLandLate) + XCTAssertEqual(policy.secondEnded(frames: 50, late: 40, meanOverrunMs: 0.5, maxGPUDurationMs: 12.0, periodMs: period, waitStillToSkip: false), .none) + XCTAssertTrue(policy.framesLandLate) + XCTAssertEqual(policy.secondEnded(frames: 50, late: 5, meanOverrunMs: 0.1, maxGPUDurationMs: 12.0, periodMs: period, waitStillToSkip: false), .none) + XCTAssertFalse(policy.framesLandLate) + XCTAssertTrue(policy.active) + } + + func testLeavesAfterTwoSecondsThatFitMostOfARefresh() { + var policy = XRHalfRatePolicy() + _ = lateSecond(&policy) + _ = lateSecond(&policy) + XCTAssertEqual(goodSecond(&policy, maxGPU: 7.9), .none) + XCTAssertTrue(policy.active) + XCTAssertEqual(goodSecond(&policy, maxGPU: 7.9), .left) + XCTAssertFalse(policy.active) + XCTAssertFalse(policy.framesLandLate) + } + + func testAHeavySecondWhileActiveStartsTheFitCountAgain() { + var policy = XRHalfRatePolicy() + _ = lateSecond(&policy) + _ = lateSecond(&policy) + XCTAssertEqual(goodSecond(&policy, maxGPU: 7.0), .none) + XCTAssertEqual(goodSecond(&policy, maxGPU: 9.0), .none) + XCTAssertEqual(goodSecond(&policy, maxGPU: 7.0), .none) + XCTAssertTrue(policy.active) + XCTAssertEqual(goodSecond(&policy, maxGPU: 7.0), .left) + } + + func testAnEmptySecondChangesNothingAndResetForgetsEverything() { + var policy = XRHalfRatePolicy() + XCTAssertEqual(policy.secondEnded(frames: 0, late: 0, meanOverrunMs: 0, maxGPUDurationMs: 0, periodMs: period, waitStillToSkip: false), .none) + _ = lateSecond(&policy) + _ = lateSecond(&policy) + policy.reset() + XCTAssertEqual(policy, XRHalfRatePolicy()) + } +} diff --git a/docs/proposals/PresentationTimeAnimation.md b/docs/proposals/PresentationTimeAnimation.md index 8b92700ee..d6a4c761f 100644 --- a/docs/proposals/PresentationTimeAnimation.md +++ b/docs/proposals/PresentationTimeAnimation.md @@ -147,6 +147,14 @@ The compositor keeps some slack past its stated deadline (the loop still gets 90 `setRendering(.optimalInputWait(.always | .never | .automatic))`, automatic by default: the loop waits for the optimal input time while the frames meet the deadline, stops once a second had late frames (one in twenty or more), and waits again once a second's least slack before the deadline would have covered the longest wait skipped, with a 1.5 ms margin. The fresh head pose whenever the budget allows; the compositor's reprojection of an older prediction only when it does not. `optimalInputWaitActive` says what the last frame did; the viewer's switch maps to automatic or never and its summary notes "input-time wait skipped (automatic)". +### What the passes cost (2026-10-15, the viewer at 100 Hz, render quality at the platform default) + +With the GPU pass timing on, a second of late frames names its heaviest passes. The board close (the tour's first phase, mistaken at first for a probe's prefilter): Forward 8.4 to 9.9 ms, TAA 1.4 to 3.5 ms, the depth pyramid 1 to 2.5 ms, a frame of 12.5 to 13 ms at 72 to 80 fps. The board far: Forward 4.2 to 6 ms, a frame of about 10 ms, half its frames a millisecond late. One switch at a time: TAA only (no MSAA) saves 1.5 to 2 ms, the one real saving; occlusion cull off removes the pyramid, built needlessly for one object's many meshes, for half a millisecond of frame rate at most; shadow taps 4 saves nothing measurable. The far phase can meet 100 Hz with TAA only and occlusion cull off; the close phase cannot by settings: the Forward pass is the board's own shading, scaling with the pixels it covers. + +### Half-rate pacing (done) + +`setRendering(.framePacing(.fullRate | .halfRateWhenLate | .halfRate))`, half rate when late by default. `XRHalfRatePolicy` decides from each second's deadline measurements: it enters when, with no wait for the optimal input time left to skip, half the frames or more were late by over a millisecond on average for two seconds running; while active the loop lets every other compositor frame go (its update phase marked, nothing submitted), so the frames rendered are shown two refreshes apart, evenly, and it poses them one refresh later than the compositor's prediction while they land a refresh late, where the display then shows them; it leaves when the rendered frames' GPU work has fitted 80% of the refresh for two seconds running. `halfRatePacingActive` says the state; the per-second line adds "half rate", and the viewer's switch maps to half rate when late or full rate, with "half-rate pacing active" in its summary. An even cadence at 50 fps, with the compositor reprojecting the head between frames, against a hop between 75 and 100: the trade the other headset runtimes make, to be judged by eye on the device. + ### The true late latch (not done: a refactor, for a decision) The head pose is consumed on the CPU while the frame is encoded: the view and projection matrices are read at 41 places in `RenderPasses.swift` (each pass copies them into its own uniforms: the batches, the grid, the sky, the environment, the shadow cascades, the SSAO's projection scale), 7 in `CullingSystem.swift` (the frustum planes and the view-projection for the occlusion test), 5 in `TemporalAntiAliasing.swift` (the jitter and the reprojection), 3 in `GraphBuilder.swift`, and a few in the extension surface and the muscle debug pass. Encoding before the wait and writing the pose after it therefore needs one per-frame, per-eye camera buffer (view, projection, their inverses, the view-projection, the eye position, the jitter) bound at a fixed index for every pass and read by every shader in place of the copies; the frustum planes built on the GPU from it (a small kernel before the culls, or the culls reading the matrices directly); the temporal resolve taking the previous frame's buffer as its reprojection; the shadow cascades fitted from a provisional pose a few milliseconds older, which is harmless; and the drawable, with its foveation rate maps, taken before the wait, earlier than Apple's sample does, to be verified on the device first. It touches every pass and most shaders, a few days of work, and it should follow #1359 rather than ride on it. It buys about 3 ms of GPU window (the encode after the wait) with today's head pose; the adaptive wait buys about 9 ms with an older one when needed. From 0811a779665aad7291040dfa6f1ba7b101d3d6c2 Mon Sep 17 00:00:00 2001 From: Javier Segura Date: Thu, 15 Oct 2026 11:23:03 +0200 Subject: [PATCH 15/22] [Bugfix] The half-rate pacing waits before taking a frame instead of letting a taken frame go A compositor frame taken with queryNextFrame and not submitted is never released, and three of them abort the app ("We reached more than 3 frames in flight"): the first pacing, which let every other frame go, crashed as soon as it engaged. The pacing is now in the taking: with it active, the loop waits for the last rendered frame's GPU work to finish and then for the next refresh after that, counted from that frame's presentation instant, before it takes the next frame. Every frame taken is submitted, nothing queues on the GPU, and the frames land on every other refresh while one refresh is not enough for a frame's work. --- .../UntoldEngine/Renderer/XRFramePacing.swift | 2 +- Sources/UntoldEngineXR/UntoldEngineXR.swift | 53 +++++++++++-------- docs/proposals/PresentationTimeAnimation.md | 2 +- 3 files changed, 33 insertions(+), 24 deletions(-) diff --git a/Sources/UntoldEngine/Renderer/XRFramePacing.swift b/Sources/UntoldEngine/Renderer/XRFramePacing.swift index 267ef2bef..7a6617611 100644 --- a/Sources/UntoldEngine/Renderer/XRFramePacing.swift +++ b/Sources/UntoldEngine/Renderer/XRFramePacing.swift @@ -28,7 +28,7 @@ public enum XRFramePacing: Sendable, Equatable { /// millisecond on average for two seconds running; leave when the rendered frames' GPU work /// has fitted 80% of the refresh for two seconds running. public struct XRHalfRatePolicy: Equatable, Sendable { - /// Whether every other refresh is being skipped. + /// Whether the frames are being paced to every other refresh. public private(set) var active = false /// Whether the last second's rendered frames mostly landed a refresh late: the pose then /// aims one refresh later, where the display shows them. diff --git a/Sources/UntoldEngineXR/UntoldEngineXR.swift b/Sources/UntoldEngineXR/UntoldEngineXR.swift index 8998d77dc..2bfe99489 100644 --- a/Sources/UntoldEngineXR/UntoldEngineXR.swift +++ b/Sources/UntoldEngineXR/UntoldEngineXR.swift @@ -108,13 +108,15 @@ private var secondSkippedWaitMaxMs = 0.0 /// The automatic mode's current choice: waiting for the optimal input time, or not. private var automaticWaits = true - /// The half-rate pacing: its policy, the alternator that lets every other compositor - /// frame go while it is active, whether this frame is paced, and the last second's - /// longest GPU span. + /// The half-rate pacing: its policy, whether this frame is paced, the last second's + /// longest GPU span, and its anchors: the last rendered frame's command buffer, waited + /// for before the next frame is taken, and that frame's presentation instant, the + /// refresh grid the next frame is taken on. private var halfRate = XRHalfRatePolicy() - private var halfRateSkipNext = true private var pacingActiveNow = false private var secondMaxGPUMs = 0.0 + private var lastRenderedCommandBuffer: MTLCommandBuffer? + private var lastRenderedPresentation: LayerRenderer.Clock.Instant? /// Whether the loop is rendering every other refresh at the moment. public var halfRatePacingActive: Bool { deadlineLock.withLock { pacingActiveNow } } /// Whether the last frame waited for the compositor's optimal input time: the setting, @@ -141,9 +143,14 @@ return waits } - /// Whether this compositor frame is let go for the half-rate pacing: every other one - /// while the pacing is active, by the setting or by the policy when the frames run late. - private func skipsFrameForPacing() -> Bool { + /// The half-rate pacing, before the next compositor frame is taken. With the pacing + /// active, by the setting or by the policy when the frames run late, the loop first + /// waits for the last rendered frame's GPU work to finish and then for the next refresh + /// after that, counted from that frame's presentation instant. Every frame taken is + /// submitted (a frame taken and let go is never released by the compositor, and three + /// of them abort the app), nothing queues on the GPU, and the frames land on every + /// other refresh while one refresh is not enough for a frame's work. + private func paceForHalfRate() { let active: Bool deadlineLock.lock() switch xrFramePacing { @@ -153,12 +160,14 @@ } pacingActiveNow = active deadlineLock.unlock() - guard active else { - halfRateSkipNext = true - return false - } - halfRateSkipNext.toggle() - return halfRateSkipNext + guard active, let anchor = lastRenderedPresentation else { return } + lastRenderedCommandBuffer?.waitUntilCompleted() + let period = currentFrameTime.period + guard period > 0 else { return } + let clock = LayerRenderer.Clock() + let behind = Self.seconds(anchor.duration(to: clock.now)) + let refreshes = behind > 0 ? (behind / period).rounded(.up) : 0 + clock.wait(until: anchor.advanced(by: .seconds(refreshes * period)), tolerance: .zero) } /// How far past the compositor's predicted presentation the frame is posed for: one @@ -666,16 +675,12 @@ guard let layerRenderer else { return } guard isRunning() else { return } - guard let frame = layerRenderer.queryNextFrame() else { return } + // The half-rate pacing waits here, before a frame is taken, so that the frames + // rendered land on every other refresh, evenly, when one refresh is not enough for + // a frame's GPU work; the display shows the previous frame again meanwhile. + paceForHalfRate() - // The half-rate pacing: every other compositor frame is let go, so the frames - // rendered are shown two refreshes apart, evenly, when one refresh is not enough - // for a frame's GPU work; the display shows the previous frame again meanwhile. - if skipsFrameForPacing() { - frame.startUpdate() - frame.endUpdate() - return - } + guard let frame = layerRenderer.queryNextFrame() else { return } // 2. Call predictTiming to get the predicted render deadlines for code guard let timing = frame.predictTiming() else { return } @@ -882,6 +887,7 @@ #else renderer.finalizeXRStatsAndMonitors(frameStartTime: 0.0) #endif + lastRenderedPresentation = timing.presentationTime } /// Whether the scene is drawn into a drawable. The built-in display always; another @@ -1253,6 +1259,9 @@ let submitStart = CACurrentMediaTime() #endif commandBuffer.commit() + if primary { + lastRenderedCommandBuffer = commandBuffer + } #if ENGINE_STATS_ENABLED let submitMs = (CACurrentMediaTime() - submitStart) * 1000.0 let renderTotalMs = (CACurrentMediaTime() - renderTotalStart) * 1000.0 diff --git a/docs/proposals/PresentationTimeAnimation.md b/docs/proposals/PresentationTimeAnimation.md index d6a4c761f..e5db609e4 100644 --- a/docs/proposals/PresentationTimeAnimation.md +++ b/docs/proposals/PresentationTimeAnimation.md @@ -153,7 +153,7 @@ With the GPU pass timing on, a second of late frames names its heaviest passes. ### Half-rate pacing (done) -`setRendering(.framePacing(.fullRate | .halfRateWhenLate | .halfRate))`, half rate when late by default. `XRHalfRatePolicy` decides from each second's deadline measurements: it enters when, with no wait for the optimal input time left to skip, half the frames or more were late by over a millisecond on average for two seconds running; while active the loop lets every other compositor frame go (its update phase marked, nothing submitted), so the frames rendered are shown two refreshes apart, evenly, and it poses them one refresh later than the compositor's prediction while they land a refresh late, where the display then shows them; it leaves when the rendered frames' GPU work has fitted 80% of the refresh for two seconds running. `halfRatePacingActive` says the state; the per-second line adds "half rate", and the viewer's switch maps to half rate when late or full rate, with "half-rate pacing active" in its summary. An even cadence at 50 fps, with the compositor reprojecting the head between frames, against a hop between 75 and 100: the trade the other headset runtimes make, to be judged by eye on the device. +`setRendering(.framePacing(.fullRate | .halfRateWhenLate | .halfRate))`, half rate when late by default. `XRHalfRatePolicy` decides from each second's deadline measurements: it enters when, with no wait for the optimal input time left to skip, half the frames or more were late by over a millisecond on average for two seconds running; while active the loop, before taking the next compositor frame, waits for the last rendered frame's GPU work to finish and then for the next refresh after that, counted from that frame's presentation instant, so the frames rendered land on every other refresh, evenly, with nothing queued on the GPU, and it poses them one refresh later than the compositor's prediction while they land a refresh late, where the display then shows them. (A first version let every other frame go unsubmitted: the compositor never releases a frame taken and not submitted, and three of them abort the app, "We reached more than 3 frames in flight". A frame taken is a frame submitted; the pacing is in the taking.); it leaves when the rendered frames' GPU work has fitted 80% of the refresh for two seconds running. `halfRatePacingActive` says the state; the per-second line adds "half rate", and the viewer's switch maps to half rate when late or full rate, with "half-rate pacing active" in its summary. An even cadence at 50 fps, with the compositor reprojecting the head between frames, against a hop between 75 and 100: the trade the other headset runtimes make, to be judged by eye on the device. ### The true late latch (not done: a refactor, for a decision) From 63e43da9aaed94e3d5c23f4b7f9b99cba9d5d90b Mon Sep 17 00:00:00 2001 From: Javier Segura Date: Thu, 15 Oct 2026 11:34:07 +0200 Subject: [PATCH 16/22] [Patch] The half-rate pacing takes its frames on the refresh grid and keeps the GPU queued Waiting for each frame's GPU work to finish before taking the next frame idled the GPU between frames, and it clocked down: the Forward pass went from 9 to 12 ms, the frame no longer fitted two refreshes, and the pacing settled at three, 30 fps. The frames are now taken on the refresh grid (the last rendered frame's presentation instant and its multiples of the period) every two refreshes, or every three or four when the frame's GPU work with 3 ms of encode needs them, with the next frame's work queued behind the previous one's; the policy sets that cadence from the second's longest GPU span, and each frame is posed for the refresh the cadence lands on. The per-second line names the cadence. --- .../UntoldEngine/Renderer/XRFramePacing.swift | 25 ++++++--- Sources/UntoldEngineXR/UntoldEngineXR.swift | 56 +++++++++++-------- .../XRHalfRatePolicyTests.swift | 26 ++++++--- docs/proposals/PresentationTimeAnimation.md | 2 +- 4 files changed, 69 insertions(+), 40 deletions(-) diff --git a/Sources/UntoldEngine/Renderer/XRFramePacing.swift b/Sources/UntoldEngine/Renderer/XRFramePacing.swift index 7a6617611..3d98e94d3 100644 --- a/Sources/UntoldEngine/Renderer/XRFramePacing.swift +++ b/Sources/UntoldEngine/Renderer/XRFramePacing.swift @@ -26,13 +26,15 @@ public enum XRFramePacing: Sendable, Equatable { /// The half-rate decision from each second's deadline measurements. Enter when, with no wait /// for the optimal input time left to skip, half the frames or more were late by over a /// millisecond on average for two seconds running; leave when the rendered frames' GPU work -/// has fitted 80% of the refresh for two seconds running. +/// has fitted 80% of the refresh for two seconds running. While active, the refreshes between +/// two frames follow the measured GPU cost: two, or more when a frame's work with its encode +/// does not fit two. public struct XRHalfRatePolicy: Equatable, Sendable { - /// Whether the frames are being paced to every other refresh. + /// Whether the frames are being paced to every other refresh (or more). public private(set) var active = false - /// Whether the last second's rendered frames mostly landed a refresh late: the pose then - /// aims one refresh later, where the display shows them. - public private(set) var framesLandLate = false + /// Refreshes between two frames while active: the fewest the frame's GPU work with its + /// encode fits, two at least, four at most. + public private(set) var cadenceRefreshes = 2 private(set) var lateSeconds = 0 private(set) var fitSeconds = 0 @@ -47,6 +49,10 @@ public struct XRHalfRatePolicy: Equatable, Sendable { /// The share of a refresh the rendered frames' GPU work must fit for the pacing to end. public static let fitShare = 0.8 + /// The CPU encode allowed for between the frame's query and its GPU work, in milliseconds, + /// when the cadence is chosen. + public static let encodeAllowanceMs = 3.0 + public static let mostRefreshes = 4 /// A second of frames ended: `frames` rendered, `late` of them past the deadline by /// `meanOverrunMs` on average, the longest GPU span among them `maxGPUDurationMs`, the @@ -55,7 +61,7 @@ public struct XRHalfRatePolicy: Equatable, Sendable { guard frames > 0 else { return .none } let mostlyLate = late * 2 >= frames if !active { - framesLandLate = false + cadenceRefreshes = 2 if !waitStillToSkip, mostlyLate, meanOverrunMs > 1.0 { lateSeconds += 1 } else { @@ -68,7 +74,10 @@ public struct XRHalfRatePolicy: Equatable, Sendable { } return .none } - framesLandLate = mostlyLate + if periodMs > 0 { + let needed = Int(((maxGPUDurationMs + Self.encodeAllowanceMs) / periodMs).rounded(.up)) + cadenceRefreshes = min(Self.mostRefreshes, max(2, needed)) + } if periodMs > 0, maxGPUDurationMs < Self.fitShare * periodMs { fitSeconds += 1 } else { @@ -77,7 +86,7 @@ public struct XRHalfRatePolicy: Equatable, Sendable { if fitSeconds >= 2 { active = false fitSeconds = 0 - framesLandLate = false + cadenceRefreshes = 2 return .left } return .none diff --git a/Sources/UntoldEngineXR/UntoldEngineXR.swift b/Sources/UntoldEngineXR/UntoldEngineXR.swift index 2bfe99489..45bbf1cb2 100644 --- a/Sources/UntoldEngineXR/UntoldEngineXR.swift +++ b/Sources/UntoldEngineXR/UntoldEngineXR.swift @@ -109,14 +109,14 @@ /// The automatic mode's current choice: waiting for the optimal input time, or not. private var automaticWaits = true /// The half-rate pacing: its policy, whether this frame is paced, the last second's - /// longest GPU span, and its anchors: the last rendered frame's command buffer, waited - /// for before the next frame is taken, and that frame's presentation instant, the - /// refresh grid the next frame is taken on. + /// longest GPU span, and its anchors: the last rendered frame's presentation instant, + /// the refresh grid the frames are taken on, and the instant the last paced frame was + /// taken at. private var halfRate = XRHalfRatePolicy() private var pacingActiveNow = false private var secondMaxGPUMs = 0.0 - private var lastRenderedCommandBuffer: MTLCommandBuffer? private var lastRenderedPresentation: LayerRenderer.Clock.Instant? + private var lastPacedQuery: LayerRenderer.Clock.Instant? /// Whether the loop is rendering every other refresh at the moment. public var halfRatePacingActive: Bool { deadlineLock.withLock { pacingActiveNow } } /// Whether the last frame waited for the compositor's optimal input time: the setting, @@ -144,37 +144,52 @@ } /// The half-rate pacing, before the next compositor frame is taken. With the pacing - /// active, by the setting or by the policy when the frames run late, the loop first - /// waits for the last rendered frame's GPU work to finish and then for the next refresh - /// after that, counted from that frame's presentation instant. Every frame taken is - /// submitted (a frame taken and let go is never released by the compositor, and three - /// of them abort the app), nothing queues on the GPU, and the frames land on every - /// other refresh while one refresh is not enough for a frame's work. + /// active, by the setting or by the policy when the frames run late, the frames are + /// taken on the refresh grid (the last rendered frame's presentation instant and its + /// multiples of the period) every `cadenceRefreshes` refreshes: two, or as many as the + /// frame's GPU work with its encode needs. Nothing waits for the GPU: the next frame's + /// work queues behind the previous one's, so the GPU stays busy and keeps its clock + /// (idling it between frames made every frame a third longer). Every frame taken is + /// submitted: a frame taken and let go is never released by the compositor, and three + /// of them abort the app. private func paceForHalfRate() { let active: Bool + let cadence: Int deadlineLock.lock() switch xrFramePacing { case .fullRate: active = false case .halfRate: active = true case .halfRateWhenLate: active = halfRate.active } + cadence = halfRate.cadenceRefreshes pacingActiveNow = active deadlineLock.unlock() - guard active, let anchor = lastRenderedPresentation else { return } - lastRenderedCommandBuffer?.waitUntilCompleted() + guard active, let grid = lastRenderedPresentation else { + lastPacedQuery = nil + return + } let period = currentFrameTime.period guard period > 0 else { return } let clock = LayerRenderer.Clock() - let behind = Self.seconds(anchor.duration(to: clock.now)) + // The earliest the next frame may be taken: a cadence after the last paced one, and + // not before now; snapped up to the refresh grid. + var earliest = clock.now + if let last = lastPacedQuery { + let due = last.advanced(by: .seconds(Double(cadence) * period)) + if due > earliest { earliest = due } + } + let behind = Self.seconds(grid.duration(to: earliest)) let refreshes = behind > 0 ? (behind / period).rounded(.up) : 0 - clock.wait(until: anchor.advanced(by: .seconds(refreshes * period)), tolerance: .zero) + let target = grid.advanced(by: .seconds(refreshes * period)) + clock.wait(until: target, tolerance: .zero) + lastPacedQuery = target } - /// How far past the compositor's predicted presentation the frame is posed for: one - /// refresh while the half-rate pacing's rendered frames land a refresh late, where the - /// display then shows them; nothing otherwise. + /// How far past the compositor's predicted presentation (one refresh after the frame is + /// taken) the frame is posed for: the refreshes of the pacing's cadence beyond that + /// first one, where the display shows a paced frame; nothing without the pacing. private func presentationLeadSeconds() -> TimeInterval { - deadlineLock.withLock { pacingActiveNow && halfRate.framesLandLate ? currentFrameTime.period : 0 } + deadlineLock.withLock { pacingActiveNow ? Double(halfRate.cadenceRefreshes - 1) * currentFrameTime.period : 0 } } /// Whether the layer's depth format and the drawable's depth range have been printed. private var loggedDepthSetup = false @@ -1259,9 +1274,6 @@ let submitStart = CACurrentMediaTime() #endif commandBuffer.commit() - if primary { - lastRenderedCommandBuffer = commandBuffer - } #if ENGINE_STATS_ENABLED let submitMs = (CACurrentMediaTime() - submitStart) * 1000.0 let renderTotalMs = (CACurrentMediaTime() - renderTotalStart) * 1000.0 @@ -1444,7 +1456,7 @@ case .automatic: mode = "automatic" } let meanOverrunMs = secondOverrunSumMs / Double(secondLate) - report = String(format: "XR frame timing: the GPU finished after the rendering deadline on %d of %d frames in the last second (worst +%.2f ms, mean +%.2f ms; %d late frames so far; input-time wait %@, %@%@)", secondLate, secondFrames, secondWorstMs, meanOverrunMs, deadlineMissCount, mode, lastFrameWaited ? "waited" : "not waited", pacingActiveNow ? "; half rate" : "") + report = String(format: "XR frame timing: the GPU finished after the rendering deadline on %d of %d frames in the last second (worst +%.2f ms, mean +%.2f ms; %d late frames so far; input-time wait %@, %@%@)", secondLate, secondFrames, secondWorstMs, meanOverrunMs, deadlineMissCount, mode, lastFrameWaited ? "waited" : "not waited", pacingActiveNow ? String(format: "; half rate, every %d refreshes", halfRate.cadenceRefreshes) : "") // With the GPU pass timing on, the heaviest passes of the last resolved frame // name what a second of frames well past the deadline spent its time on. if meanOverrunMs > 2.0, GPUPassTiming.shared.isEnabled { diff --git a/Tests/UntoldEngineTests/XRHalfRatePolicyTests.swift b/Tests/UntoldEngineTests/XRHalfRatePolicyTests.swift index 55a906e7d..f7d23112f 100644 --- a/Tests/UntoldEngineTests/XRHalfRatePolicyTests.swift +++ b/Tests/UntoldEngineTests/XRHalfRatePolicyTests.swift @@ -12,8 +12,8 @@ import XCTest /// The half-rate pacing enters after two seconds of mostly late frames with no wait left to -/// skip, aims the pose a refresh later while the rendered frames land late, and leaves after -/// two seconds in which the frames' GPU work fitted most of a refresh. +/// skip, spaces the frames by the refreshes their GPU work with its encode needs, and leaves +/// after two seconds in which the frames' GPU work fitted most of a refresh. final class XRHalfRatePolicyTests: XCTestCase { private let period = 10.0 @@ -56,16 +56,24 @@ final class XRHalfRatePolicyTests: XCTestCase { XCTAssertFalse(policy.active) } - func testWhileActiveThePoseAimsLaterOnlyWhileTheFramesLandLate() { + func testWhileActiveTheCadenceFollowsTheGPUCostWithItsEncode() { var policy = XRHalfRatePolicy() _ = lateSecond(&policy) _ = lateSecond(&policy) XCTAssertTrue(policy.active) - XCTAssertFalse(policy.framesLandLate) - XCTAssertEqual(policy.secondEnded(frames: 50, late: 40, meanOverrunMs: 0.5, maxGPUDurationMs: 12.0, periodMs: period, waitStillToSkip: false), .none) - XCTAssertTrue(policy.framesLandLate) - XCTAssertEqual(policy.secondEnded(frames: 50, late: 5, meanOverrunMs: 0.1, maxGPUDurationMs: 12.0, periodMs: period, waitStillToSkip: false), .none) - XCTAssertFalse(policy.framesLandLate) + XCTAssertEqual(policy.cadenceRefreshes, 2) + // 12 ms of GPU work and 3 ms of encode fit two refreshes of 10 ms. + XCTAssertEqual(policy.secondEnded(frames: 50, late: 50, meanOverrunMs: 8.0, maxGPUDurationMs: 12.0, periodMs: period, waitStillToSkip: false), .none) + XCTAssertEqual(policy.cadenceRefreshes, 2) + // 18 ms and 3 ms need three. + XCTAssertEqual(policy.secondEnded(frames: 33, late: 33, meanOverrunMs: 8.0, maxGPUDurationMs: 18.0, periodMs: period, waitStillToSkip: false), .none) + XCTAssertEqual(policy.cadenceRefreshes, 3) + // Never more than four. + XCTAssertEqual(policy.secondEnded(frames: 20, late: 20, meanOverrunMs: 8.0, maxGPUDurationMs: 60.0, periodMs: period, waitStillToSkip: false), .none) + XCTAssertEqual(policy.cadenceRefreshes, 4) + // Never fewer than two while active. + XCTAssertEqual(policy.secondEnded(frames: 50, late: 50, meanOverrunMs: 8.0, maxGPUDurationMs: 9.0, periodMs: period, waitStillToSkip: false), .none) + XCTAssertEqual(policy.cadenceRefreshes, 2) XCTAssertTrue(policy.active) } @@ -77,7 +85,7 @@ final class XRHalfRatePolicyTests: XCTestCase { XCTAssertTrue(policy.active) XCTAssertEqual(goodSecond(&policy, maxGPU: 7.9), .left) XCTAssertFalse(policy.active) - XCTAssertFalse(policy.framesLandLate) + XCTAssertEqual(policy.cadenceRefreshes, 2) } func testAHeavySecondWhileActiveStartsTheFitCountAgain() { diff --git a/docs/proposals/PresentationTimeAnimation.md b/docs/proposals/PresentationTimeAnimation.md index e5db609e4..b5242b9c4 100644 --- a/docs/proposals/PresentationTimeAnimation.md +++ b/docs/proposals/PresentationTimeAnimation.md @@ -153,7 +153,7 @@ With the GPU pass timing on, a second of late frames names its heaviest passes. ### Half-rate pacing (done) -`setRendering(.framePacing(.fullRate | .halfRateWhenLate | .halfRate))`, half rate when late by default. `XRHalfRatePolicy` decides from each second's deadline measurements: it enters when, with no wait for the optimal input time left to skip, half the frames or more were late by over a millisecond on average for two seconds running; while active the loop, before taking the next compositor frame, waits for the last rendered frame's GPU work to finish and then for the next refresh after that, counted from that frame's presentation instant, so the frames rendered land on every other refresh, evenly, with nothing queued on the GPU, and it poses them one refresh later than the compositor's prediction while they land a refresh late, where the display then shows them. (A first version let every other frame go unsubmitted: the compositor never releases a frame taken and not submitted, and three of them abort the app, "We reached more than 3 frames in flight". A frame taken is a frame submitted; the pacing is in the taking.); it leaves when the rendered frames' GPU work has fitted 80% of the refresh for two seconds running. `halfRatePacingActive` says the state; the per-second line adds "half rate", and the viewer's switch maps to half rate when late or full rate, with "half-rate pacing active" in its summary. An even cadence at 50 fps, with the compositor reprojecting the head between frames, against a hop between 75 and 100: the trade the other headset runtimes make, to be judged by eye on the device. +`setRendering(.framePacing(.fullRate | .halfRateWhenLate | .halfRate))`, half rate when late by default. `XRHalfRatePolicy` decides from each second's deadline measurements: it enters when, with no wait for the optimal input time left to skip, half the frames or more were late by over a millisecond on average for two seconds running; while active the loop takes its frames on the refresh grid (the last rendered frame's presentation instant and its multiples of the period) every two refreshes, or every three or four when the frame's GPU work with 3 ms of encode needs them, and poses each frame for the refresh that cadence lands on, the compositor's prediction (one refresh after the frame is taken) plus the cadence's remaining refreshes. Nothing waits for the GPU: the next frame's work queues behind the previous one's. (Two lessons from the device: a first version let every other frame go unsubmitted, and the compositor never releases a frame taken and not submitted, three of them aborting the app with "We reached more than 3 frames in flight", so the pacing is in the taking; a second version waited for each frame's GPU work to finish before taking the next, and the idle gaps let the GPU clock down, the Forward pass going from 9 to 12 ms and the cadence from two refreshes to three, 30 fps, so the GPU is kept queued.); it leaves when the rendered frames' GPU work has fitted 80% of the refresh for two seconds running. `halfRatePacingActive` says the state; the per-second line adds "half rate", and the viewer's switch maps to half rate when late or full rate, with "half-rate pacing active" in its summary. An even cadence at 50 fps, with the compositor reprojecting the head between frames, against a hop between 75 and 100: the trade the other headset runtimes make, to be judged by eye on the device. ### The true late latch (not done: a refactor, for a decision) From 203e923a2f6c0e4d86dc1759cd268b0dd5f4a7f2 Mon Sep 17 00:00:00 2001 From: Javier Segura Date: Thu, 15 Oct 2026 11:42:39 +0200 Subject: [PATCH 17/22] [Patch] The pacing's cadence falls only after two seconds that fit it, and the late-frame line shows the GPU span and the pacing wait A frame whose GPU work with its encode sat at the two-refresh boundary had the cadence rounded up and down by turns, a second each way, and a cadence that changes every second is itself a hop. It now rises at once and falls only after two seconds running that fit the shorter one. The per-second line adds the GPU span's mean and longest and the pacing wait's mean, so a run shows where a frame's time goes. --- .../UntoldEngine/Renderer/XRFramePacing.swift | 19 +++++++++++++++++-- Sources/UntoldEngineXR/UntoldEngineXR.swift | 16 +++++++++++++++- .../XRHalfRatePolicyTests.swift | 10 +++++++++- 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/Sources/UntoldEngine/Renderer/XRFramePacing.swift b/Sources/UntoldEngine/Renderer/XRFramePacing.swift index 3d98e94d3..a41dc4a4a 100644 --- a/Sources/UntoldEngine/Renderer/XRFramePacing.swift +++ b/Sources/UntoldEngine/Renderer/XRFramePacing.swift @@ -37,6 +37,8 @@ public struct XRHalfRatePolicy: Equatable, Sendable { public private(set) var cadenceRefreshes = 2 private(set) var lateSeconds = 0 private(set) var fitSeconds = 0 + /// Seconds running in which a shorter cadence than the current one would have done. + private(set) var shorterSeconds = 0 public init() {} @@ -75,8 +77,20 @@ public struct XRHalfRatePolicy: Equatable, Sendable { return .none } if periodMs > 0 { - let needed = Int(((maxGPUDurationMs + Self.encodeAllowanceMs) / periodMs).rounded(.up)) - cadenceRefreshes = min(Self.mostRefreshes, max(2, needed)) + // The cadence rises at once when the frame's work needs more refreshes, and falls + // only after two seconds running that fit the shorter one: a cadence that changed + // every second would itself be a hop. + let needed = min(Self.mostRefreshes, max(2, Int(((maxGPUDurationMs + Self.encodeAllowanceMs) / periodMs).rounded(.up)))) + if needed >= cadenceRefreshes { + cadenceRefreshes = needed + shorterSeconds = 0 + } else { + shorterSeconds += 1 + if shorterSeconds >= 2 { + cadenceRefreshes = needed + shorterSeconds = 0 + } + } } if periodMs > 0, maxGPUDurationMs < Self.fitShare * periodMs { fitSeconds += 1 @@ -86,6 +100,7 @@ public struct XRHalfRatePolicy: Equatable, Sendable { if fitSeconds >= 2 { active = false fitSeconds = 0 + shorterSeconds = 0 cadenceRefreshes = 2 return .left } diff --git a/Sources/UntoldEngineXR/UntoldEngineXR.swift b/Sources/UntoldEngineXR/UntoldEngineXR.swift index 45bbf1cb2..3dd7d7042 100644 --- a/Sources/UntoldEngineXR/UntoldEngineXR.swift +++ b/Sources/UntoldEngineXR/UntoldEngineXR.swift @@ -106,6 +106,9 @@ private var secondOverrunSumMs = 0.0 private var secondLeastSlackMs = Double.infinity private var secondSkippedWaitMaxMs = 0.0 + private var secondGPUSumMs = 0.0 + private var secondPacedWaitSumMs = 0.0 + private var secondPacedWaits = 0 /// The automatic mode's current choice: waiting for the optimal input time, or not. private var automaticWaits = true /// The half-rate pacing: its policy, whether this frame is paced, the last second's @@ -181,8 +184,13 @@ let behind = Self.seconds(grid.duration(to: earliest)) let refreshes = behind > 0 ? (behind / period).rounded(.up) : 0 let target = grid.advanced(by: .seconds(refreshes * period)) + let waitMs = max(0, Self.milliseconds(clock.now.duration(to: target))) clock.wait(until: target, tolerance: .zero) lastPacedQuery = target + deadlineLock.lock() + secondPacedWaitSumMs += waitMs + secondPacedWaits += 1 + deadlineLock.unlock() } /// How far past the compositor's predicted presentation (one refresh after the frame is @@ -1437,7 +1445,9 @@ deadlineLock.lock() secondFrames += 1 secondLeastSlackMs = min(secondLeastSlackMs, -overrunMs) - secondMaxGPUMs = max(secondMaxGPUMs, (gpuEnd - gpuStart) * 1000.0) + let gpuMs = (gpuEnd - gpuStart) * 1000.0 + secondMaxGPUMs = max(secondMaxGPUMs, gpuMs) + secondGPUSumMs += gpuMs if overrunMs > 0 { deadlineMissCount += 1 secondLate += 1 @@ -1457,6 +1467,7 @@ } let meanOverrunMs = secondOverrunSumMs / Double(secondLate) report = String(format: "XR frame timing: the GPU finished after the rendering deadline on %d of %d frames in the last second (worst +%.2f ms, mean +%.2f ms; %d late frames so far; input-time wait %@, %@%@)", secondLate, secondFrames, secondWorstMs, meanOverrunMs, deadlineMissCount, mode, lastFrameWaited ? "waited" : "not waited", pacingActiveNow ? String(format: "; half rate, every %d refreshes", halfRate.cadenceRefreshes) : "") + report = (report ?? "") + String(format: "; GPU %.1f ms a frame, longest %.1f%@", secondGPUSumMs / Double(secondFrames), secondMaxGPUMs, secondPacedWaits > 0 ? String(format: ", paced wait %.1f ms", secondPacedWaitSumMs / Double(secondPacedWaits)) : "") // With the GPU pass timing on, the heaviest passes of the last resolved frame // name what a second of frames well past the deadline spent its time on. if meanOverrunMs > 2.0, GPUPassTiming.shared.isEnabled { @@ -1506,6 +1517,9 @@ secondLeastSlackMs = .infinity secondSkippedWaitMaxMs = 0 secondMaxGPUMs = 0 + secondGPUSumMs = 0 + secondPacedWaitSumMs = 0 + secondPacedWaits = 0 lastDeadlineLogTime = now } deadlineLock.unlock() diff --git a/Tests/UntoldEngineTests/XRHalfRatePolicyTests.swift b/Tests/UntoldEngineTests/XRHalfRatePolicyTests.swift index f7d23112f..498846952 100644 --- a/Tests/UntoldEngineTests/XRHalfRatePolicyTests.swift +++ b/Tests/UntoldEngineTests/XRHalfRatePolicyTests.swift @@ -71,9 +71,17 @@ final class XRHalfRatePolicyTests: XCTestCase { // Never more than four. XCTAssertEqual(policy.secondEnded(frames: 20, late: 20, meanOverrunMs: 8.0, maxGPUDurationMs: 60.0, periodMs: period, waitStillToSkip: false), .none) XCTAssertEqual(policy.cadenceRefreshes, 4) - // Never fewer than two while active. + // A shorter cadence is taken only after two seconds running that fit it. + XCTAssertEqual(policy.secondEnded(frames: 50, late: 50, meanOverrunMs: 8.0, maxGPUDurationMs: 9.0, periodMs: period, waitStillToSkip: false), .none) + XCTAssertEqual(policy.cadenceRefreshes, 4) XCTAssertEqual(policy.secondEnded(frames: 50, late: 50, meanOverrunMs: 8.0, maxGPUDurationMs: 9.0, periodMs: period, waitStillToSkip: false), .none) XCTAssertEqual(policy.cadenceRefreshes, 2) + // A longer one at once, and a single fitting second between two needing it does not shorten it. + XCTAssertEqual(policy.secondEnded(frames: 33, late: 33, meanOverrunMs: 8.0, maxGPUDurationMs: 18.0, periodMs: period, waitStillToSkip: false), .none) + XCTAssertEqual(policy.cadenceRefreshes, 3) + XCTAssertEqual(policy.secondEnded(frames: 33, late: 33, meanOverrunMs: 8.0, maxGPUDurationMs: 12.0, periodMs: period, waitStillToSkip: false), .none) + XCTAssertEqual(policy.secondEnded(frames: 33, late: 33, meanOverrunMs: 8.0, maxGPUDurationMs: 18.0, periodMs: period, waitStillToSkip: false), .none) + XCTAssertEqual(policy.cadenceRefreshes, 3) XCTAssertTrue(policy.active) } From de93a4d430884d1f0ccdc4ddbdf73bc109d6084c Mon Sep 17 00:00:00 2001 From: Javier Segura Date: Thu, 15 Oct 2026 11:56:03 +0200 Subject: [PATCH 18/22] [Patch] The pacing's cadence comes from the typical GPU span, and the pose aims where the frames are measured to land Sized from each second's longest GPU span, the cadence stayed at three refreshes with a typical frame of 14 ms: a GPU used on half the refreshes runs at a lower clock, and its longest spans, 19 to 21 ms with the compositor's own work in them, never earned back the shorter cadence that would raise the clock again. The cadence now follows the mean span plus a fifth with the encode, and the pacing ends on the mean too. And the pose no longer assumes a paced frame lands at the next cadence: the frames finished 9 ms past their deadline and were shown one refresh after the prediction while the pose aimed two refreshes after it; the overrun now says the landing refresh, which the per-second line names. --- .../UntoldEngine/Renderer/XRFramePacing.swift | 33 ++++++--- Sources/UntoldEngineXR/UntoldEngineXR.swift | 14 ++-- .../XRHalfRatePolicyTests.swift | 72 +++++++++++++------ 3 files changed, 79 insertions(+), 40 deletions(-) diff --git a/Sources/UntoldEngine/Renderer/XRFramePacing.swift b/Sources/UntoldEngine/Renderer/XRFramePacing.swift index a41dc4a4a..6f146ec7d 100644 --- a/Sources/UntoldEngine/Renderer/XRFramePacing.swift +++ b/Sources/UntoldEngine/Renderer/XRFramePacing.swift @@ -25,16 +25,21 @@ public enum XRFramePacing: Sendable, Equatable { /// The half-rate decision from each second's deadline measurements. Enter when, with no wait /// for the optimal input time left to skip, half the frames or more were late by over a -/// millisecond on average for two seconds running; leave when the rendered frames' GPU work -/// has fitted 80% of the refresh for two seconds running. While active, the refreshes between -/// two frames follow the measured GPU cost: two, or more when a frame's work with its encode -/// does not fit two. +/// millisecond on average for two seconds running; leave when the rendered frames' mean GPU +/// work has fitted 80% of the refresh for two seconds running. While active, the refreshes +/// between two frames follow the measured GPU cost: two, or more when a frame's typical work +/// (the mean span plus a fifth) with its encode does not fit two. The mean, not the longest +/// span: a GPU used on half the refreshes runs at a lower clock, and its longest spans would +/// never earn back the shorter cadence that would raise the clock again. public struct XRHalfRatePolicy: Equatable, Sendable { /// Whether the frames are being paced to every other refresh (or more). public private(set) var active = false - /// Refreshes between two frames while active: the fewest the frame's GPU work with its - /// encode fits, two at least, four at most. + /// Refreshes between two frames while active: the fewest the frame's typical GPU work + /// with its encode fits, two at least, four at most. public private(set) var cadenceRefreshes = 2 + /// Refreshes after the compositor's predicted presentation at which the paced frames are + /// shown, from how far past their deadline they finish: the pose aims there. + public private(set) var landingRefreshes = 0 private(set) var lateSeconds = 0 private(set) var fitSeconds = 0 /// Seconds running in which a shorter cadence than the current one would have done. @@ -54,16 +59,19 @@ public struct XRHalfRatePolicy: Equatable, Sendable { /// The CPU encode allowed for between the frame's query and its GPU work, in milliseconds, /// when the cadence is chosen. public static let encodeAllowanceMs = 3.0 + /// The share above the mean GPU span the cadence allows for the longer frames. + public static let spanMargin = 0.2 public static let mostRefreshes = 4 /// A second of frames ended: `frames` rendered, `late` of them past the deadline by - /// `meanOverrunMs` on average, the longest GPU span among them `maxGPUDurationMs`, the + /// `meanOverrunMs` on average, their GPU spans `meanGPUDurationMs` on average, the /// display's refresh `periodMs`, and whether the loop still has a wait it could skip first. - public mutating func secondEnded(frames: Int, late: Int, meanOverrunMs: Double, maxGPUDurationMs: Double, periodMs: Double, waitStillToSkip: Bool) -> Change { + public mutating func secondEnded(frames: Int, late: Int, meanOverrunMs: Double, meanGPUDurationMs: Double, periodMs: Double, waitStillToSkip: Bool) -> Change { guard frames > 0 else { return .none } let mostlyLate = late * 2 >= frames if !active { cadenceRefreshes = 2 + landingRefreshes = 0 if !waitStillToSkip, mostlyLate, meanOverrunMs > 1.0 { lateSeconds += 1 } else { @@ -77,10 +85,14 @@ public struct XRHalfRatePolicy: Equatable, Sendable { return .none } if periodMs > 0 { + // Where the frames land: a frame finished past its deadline is shown at the next + // refresh its overrun reaches; one within a millisecond makes its predicted one. + landingRefreshes = meanOverrunMs > 1.0 ? min(Self.mostRefreshes - 1, Int((meanOverrunMs / periodMs).rounded(.up))) : 0 // The cadence rises at once when the frame's work needs more refreshes, and falls // only after two seconds running that fit the shorter one: a cadence that changed // every second would itself be a hop. - let needed = min(Self.mostRefreshes, max(2, Int(((maxGPUDurationMs + Self.encodeAllowanceMs) / periodMs).rounded(.up)))) + let typical = meanGPUDurationMs * (1 + Self.spanMargin) + Self.encodeAllowanceMs + let needed = min(Self.mostRefreshes, max(2, Int((typical / periodMs).rounded(.up)))) if needed >= cadenceRefreshes { cadenceRefreshes = needed shorterSeconds = 0 @@ -92,7 +104,7 @@ public struct XRHalfRatePolicy: Equatable, Sendable { } } } - if periodMs > 0, maxGPUDurationMs < Self.fitShare * periodMs { + if periodMs > 0, meanGPUDurationMs < Self.fitShare * periodMs { fitSeconds += 1 } else { fitSeconds = 0 @@ -102,6 +114,7 @@ public struct XRHalfRatePolicy: Equatable, Sendable { fitSeconds = 0 shorterSeconds = 0 cadenceRefreshes = 2 + landingRefreshes = 0 return .left } return .none diff --git a/Sources/UntoldEngineXR/UntoldEngineXR.swift b/Sources/UntoldEngineXR/UntoldEngineXR.swift index 3dd7d7042..db60ac261 100644 --- a/Sources/UntoldEngineXR/UntoldEngineXR.swift +++ b/Sources/UntoldEngineXR/UntoldEngineXR.swift @@ -193,11 +193,11 @@ deadlineLock.unlock() } - /// How far past the compositor's predicted presentation (one refresh after the frame is - /// taken) the frame is posed for: the refreshes of the pacing's cadence beyond that - /// first one, where the display shows a paced frame; nothing without the pacing. + /// How far past the compositor's predicted presentation the frame is posed for: the + /// refreshes the paced frames were measured to land after it, from how far past their + /// deadline they finish; nothing without the pacing. private func presentationLeadSeconds() -> TimeInterval { - deadlineLock.withLock { pacingActiveNow ? Double(halfRate.cadenceRefreshes - 1) * currentFrameTime.period : 0 } + deadlineLock.withLock { pacingActiveNow ? Double(halfRate.landingRefreshes) * currentFrameTime.period : 0 } } /// Whether the layer's depth format and the drawable's depth range have been printed. private var loggedDepthSetup = false @@ -1466,7 +1466,7 @@ case .automatic: mode = "automatic" } let meanOverrunMs = secondOverrunSumMs / Double(secondLate) - report = String(format: "XR frame timing: the GPU finished after the rendering deadline on %d of %d frames in the last second (worst +%.2f ms, mean +%.2f ms; %d late frames so far; input-time wait %@, %@%@)", secondLate, secondFrames, secondWorstMs, meanOverrunMs, deadlineMissCount, mode, lastFrameWaited ? "waited" : "not waited", pacingActiveNow ? String(format: "; half rate, every %d refreshes", halfRate.cadenceRefreshes) : "") + report = String(format: "XR frame timing: the GPU finished after the rendering deadline on %d of %d frames in the last second (worst +%.2f ms, mean +%.2f ms; %d late frames so far; input-time wait %@, %@%@)", secondLate, secondFrames, secondWorstMs, meanOverrunMs, deadlineMissCount, mode, lastFrameWaited ? "waited" : "not waited", pacingActiveNow ? String(format: "; half rate, every %d refreshes, landing +%d", halfRate.cadenceRefreshes, halfRate.landingRefreshes) : "") report = (report ?? "") + String(format: "; GPU %.1f ms a frame, longest %.1f%@", secondGPUSumMs / Double(secondFrames), secondMaxGPUMs, secondPacedWaits > 0 ? String(format: ", paced wait %.1f ms", secondPacedWaitSumMs / Double(secondPacedWaits)) : "") // With the GPU pass timing on, the heaviest passes of the last resolved frame // name what a second of frames well past the deadline spent its time on. @@ -1497,12 +1497,12 @@ let waitStillToSkip = xrOptimalInputWaitMode == .automatic && automaticWaits let meanLateMs = secondLate > 0 ? secondOverrunSumMs / Double(secondLate) : 0 let periodMs = currentFrameTime.period * 1000.0 - switch halfRate.secondEnded(frames: secondFrames, late: secondLate, meanOverrunMs: meanLateMs, maxGPUDurationMs: secondMaxGPUMs, periodMs: periodMs, waitStillToSkip: waitStillToSkip) { + switch halfRate.secondEnded(frames: secondFrames, late: secondLate, meanOverrunMs: meanLateMs, meanGPUDurationMs: secondGPUSumMs / Double(secondFrames), periodMs: periodMs, waitStillToSkip: waitStillToSkip) { case .entered: let line = String(format: "XR frame timing: %d of %d frames late with no wait left to skip: rendering every other refresh", secondLate, secondFrames) report = report.map { $0 + "\n" + line } ?? line case .left: - let line = String(format: "XR frame timing: the frame's GPU work of %.2f ms fits the refresh of %.2f ms: rendering every refresh again", secondMaxGPUMs, periodMs) + let line = String(format: "XR frame timing: the frame's GPU work of %.2f ms fits the refresh of %.2f ms: rendering every refresh again", secondGPUSumMs / Double(secondFrames), periodMs) report = report.map { $0 + "\n" + line } ?? line case .none: break diff --git a/Tests/UntoldEngineTests/XRHalfRatePolicyTests.swift b/Tests/UntoldEngineTests/XRHalfRatePolicyTests.swift index 498846952..98a679572 100644 --- a/Tests/UntoldEngineTests/XRHalfRatePolicyTests.swift +++ b/Tests/UntoldEngineTests/XRHalfRatePolicyTests.swift @@ -17,12 +17,16 @@ import XCTest final class XRHalfRatePolicyTests: XCTestCase { private let period = 10.0 - private func lateSecond(_ policy: inout XRHalfRatePolicy, waitStillToSkip: Bool = false, maxGPU: Double = 12.0) -> XRHalfRatePolicy.Change { - policy.secondEnded(frames: 100, late: 80, meanOverrunMs: 3.0, maxGPUDurationMs: maxGPU, periodMs: period, waitStillToSkip: waitStillToSkip) + private func lateSecond(_ policy: inout XRHalfRatePolicy, waitStillToSkip: Bool = false, meanGPU: Double = 12.0) -> XRHalfRatePolicy.Change { + policy.secondEnded(frames: 100, late: 80, meanOverrunMs: 3.0, meanGPUDurationMs: meanGPU, periodMs: period, waitStillToSkip: waitStillToSkip) } - private func goodSecond(_ policy: inout XRHalfRatePolicy, maxGPU: Double = 7.0) -> XRHalfRatePolicy.Change { - policy.secondEnded(frames: 100, late: 2, meanOverrunMs: 0.2, maxGPUDurationMs: maxGPU, periodMs: period, waitStillToSkip: false) + private func goodSecond(_ policy: inout XRHalfRatePolicy, meanGPU: Double = 7.0) -> XRHalfRatePolicy.Change { + policy.secondEnded(frames: 100, late: 2, meanOverrunMs: 0.2, meanGPUDurationMs: meanGPU, periodMs: period, waitStillToSkip: false) + } + + private func pacedSecond(_ policy: inout XRHalfRatePolicy, overrun: Double, meanGPU: Double) -> XRHalfRatePolicy.Change { + policy.secondEnded(frames: 40, late: 40, meanOverrunMs: overrun, meanGPUDurationMs: meanGPU, periodMs: period, waitStillToSkip: false) } func testEntersAfterTwoLateSecondsNotOne() { @@ -51,47 +55,69 @@ final class XRHalfRatePolicyTests: XCTestCase { func testDoesNotEnterForSlightlyLateFrames() { var policy = XRHalfRatePolicy() for _ in 0 ..< 3 { - XCTAssertEqual(policy.secondEnded(frames: 100, late: 60, meanOverrunMs: 0.4, maxGPUDurationMs: 10.2, periodMs: period, waitStillToSkip: false), .none) + XCTAssertEqual(policy.secondEnded(frames: 100, late: 60, meanOverrunMs: 0.4, meanGPUDurationMs: 10.2, periodMs: period, waitStillToSkip: false), .none) } XCTAssertFalse(policy.active) } - func testWhileActiveTheCadenceFollowsTheGPUCostWithItsEncode() { + func testWhileActiveTheCadenceFollowsTheTypicalGPUCostWithItsEncode() { var policy = XRHalfRatePolicy() _ = lateSecond(&policy) _ = lateSecond(&policy) XCTAssertTrue(policy.active) XCTAssertEqual(policy.cadenceRefreshes, 2) - // 12 ms of GPU work and 3 ms of encode fit two refreshes of 10 ms. - XCTAssertEqual(policy.secondEnded(frames: 50, late: 50, meanOverrunMs: 8.0, maxGPUDurationMs: 12.0, periodMs: period, waitStillToSkip: false), .none) + // A mean of 14 ms, a fifth more and 3 ms of encode: 19.8 ms, two refreshes of 10 ms. + XCTAssertEqual(pacedSecond(&policy, overrun: 8.0, meanGPU: 14.0), .none) XCTAssertEqual(policy.cadenceRefreshes, 2) - // 18 ms and 3 ms need three. - XCTAssertEqual(policy.secondEnded(frames: 33, late: 33, meanOverrunMs: 8.0, maxGPUDurationMs: 18.0, periodMs: period, waitStillToSkip: false), .none) + // A mean of 16 ms: 22.2 ms, three. + XCTAssertEqual(pacedSecond(&policy, overrun: 8.0, meanGPU: 16.0), .none) XCTAssertEqual(policy.cadenceRefreshes, 3) // Never more than four. - XCTAssertEqual(policy.secondEnded(frames: 20, late: 20, meanOverrunMs: 8.0, maxGPUDurationMs: 60.0, periodMs: period, waitStillToSkip: false), .none) + XCTAssertEqual(pacedSecond(&policy, overrun: 8.0, meanGPU: 60.0), .none) XCTAssertEqual(policy.cadenceRefreshes, 4) // A shorter cadence is taken only after two seconds running that fit it. - XCTAssertEqual(policy.secondEnded(frames: 50, late: 50, meanOverrunMs: 8.0, maxGPUDurationMs: 9.0, periodMs: period, waitStillToSkip: false), .none) + XCTAssertEqual(pacedSecond(&policy, overrun: 8.0, meanGPU: 9.0), .none) XCTAssertEqual(policy.cadenceRefreshes, 4) - XCTAssertEqual(policy.secondEnded(frames: 50, late: 50, meanOverrunMs: 8.0, maxGPUDurationMs: 9.0, periodMs: period, waitStillToSkip: false), .none) + XCTAssertEqual(pacedSecond(&policy, overrun: 8.0, meanGPU: 9.0), .none) XCTAssertEqual(policy.cadenceRefreshes, 2) // A longer one at once, and a single fitting second between two needing it does not shorten it. - XCTAssertEqual(policy.secondEnded(frames: 33, late: 33, meanOverrunMs: 8.0, maxGPUDurationMs: 18.0, periodMs: period, waitStillToSkip: false), .none) + XCTAssertEqual(pacedSecond(&policy, overrun: 8.0, meanGPU: 16.0), .none) XCTAssertEqual(policy.cadenceRefreshes, 3) - XCTAssertEqual(policy.secondEnded(frames: 33, late: 33, meanOverrunMs: 8.0, maxGPUDurationMs: 12.0, periodMs: period, waitStillToSkip: false), .none) - XCTAssertEqual(policy.secondEnded(frames: 33, late: 33, meanOverrunMs: 8.0, maxGPUDurationMs: 18.0, periodMs: period, waitStillToSkip: false), .none) + XCTAssertEqual(pacedSecond(&policy, overrun: 8.0, meanGPU: 12.0), .none) + XCTAssertEqual(pacedSecond(&policy, overrun: 8.0, meanGPU: 16.0), .none) XCTAssertEqual(policy.cadenceRefreshes, 3) XCTAssertTrue(policy.active) } + func testThePoseAimsAtTheRefreshTheOverrunReaches() { + var policy = XRHalfRatePolicy() + XCTAssertEqual(policy.landingRefreshes, 0) + _ = lateSecond(&policy) + _ = lateSecond(&policy) + // Finished 9 ms past the deadline: shown one refresh after the prediction. + XCTAssertEqual(pacedSecond(&policy, overrun: 9.0, meanGPU: 14.0), .none) + XCTAssertEqual(policy.landingRefreshes, 1) + // 15 ms past: two. + XCTAssertEqual(pacedSecond(&policy, overrun: 15.0, meanGPU: 14.0), .none) + XCTAssertEqual(policy.landingRefreshes, 2) + // Within a millisecond: the predicted one. + XCTAssertEqual(pacedSecond(&policy, overrun: 0.5, meanGPU: 14.0), .none) + XCTAssertEqual(policy.landingRefreshes, 0) + // Never beyond three. + XCTAssertEqual(pacedSecond(&policy, overrun: 80.0, meanGPU: 14.0), .none) + XCTAssertEqual(policy.landingRefreshes, 3) + XCTAssertEqual(goodSecond(&policy), .none) + XCTAssertEqual(goodSecond(&policy), .left) + XCTAssertEqual(policy.landingRefreshes, 0) + } + func testLeavesAfterTwoSecondsThatFitMostOfARefresh() { var policy = XRHalfRatePolicy() _ = lateSecond(&policy) _ = lateSecond(&policy) - XCTAssertEqual(goodSecond(&policy, maxGPU: 7.9), .none) + XCTAssertEqual(goodSecond(&policy, meanGPU: 7.9), .none) XCTAssertTrue(policy.active) - XCTAssertEqual(goodSecond(&policy, maxGPU: 7.9), .left) + XCTAssertEqual(goodSecond(&policy, meanGPU: 7.9), .left) XCTAssertFalse(policy.active) XCTAssertEqual(policy.cadenceRefreshes, 2) } @@ -100,16 +126,16 @@ final class XRHalfRatePolicyTests: XCTestCase { var policy = XRHalfRatePolicy() _ = lateSecond(&policy) _ = lateSecond(&policy) - XCTAssertEqual(goodSecond(&policy, maxGPU: 7.0), .none) - XCTAssertEqual(goodSecond(&policy, maxGPU: 9.0), .none) - XCTAssertEqual(goodSecond(&policy, maxGPU: 7.0), .none) + XCTAssertEqual(goodSecond(&policy, meanGPU: 7.0), .none) + XCTAssertEqual(goodSecond(&policy, meanGPU: 9.0), .none) + XCTAssertEqual(goodSecond(&policy, meanGPU: 7.0), .none) XCTAssertTrue(policy.active) - XCTAssertEqual(goodSecond(&policy, maxGPU: 7.0), .left) + XCTAssertEqual(goodSecond(&policy, meanGPU: 7.0), .left) } func testAnEmptySecondChangesNothingAndResetForgetsEverything() { var policy = XRHalfRatePolicy() - XCTAssertEqual(policy.secondEnded(frames: 0, late: 0, meanOverrunMs: 0, maxGPUDurationMs: 0, periodMs: period, waitStillToSkip: false), .none) + XCTAssertEqual(policy.secondEnded(frames: 0, late: 0, meanOverrunMs: 0, meanGPUDurationMs: 0, periodMs: period, waitStillToSkip: false), .none) _ = lateSecond(&policy) _ = lateSecond(&policy) policy.reset() From 455c939aff9282185546369dce143a11edda9c9a Mon Sep 17 00:00:00 2001 From: Javier Segura Date: Thu, 15 Oct 2026 12:04:45 +0200 Subject: [PATCH 19/22] [Bugfix] The paced query snaps to the refresh grid with a tolerance, and the cadence changes only past a deadband Asked for two refreshes, the pacing waited three: the distance to the next grid instant was rounded up, and with the learned period a hair short of the compositor's, two refreshes computed as a fraction over two and became three, every frame, 30 frames a second at a cadence meant for 45. The snap now tolerates a quarter of a refresh. And with the typical frame right at a cadence's boundary, the cadence rose on a slightly heavy second and fell two seconds later, a hop every few seconds; it now changes only when the typical frame is a millisecond past the boundary, either way. --- .../UntoldEngine/Renderer/XRFramePacing.swift | 17 +++++++++++------ Sources/UntoldEngineXR/UntoldEngineXR.swift | 5 ++++- .../XRHalfRatePolicyTests.swift | 9 ++++++++- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/Sources/UntoldEngine/Renderer/XRFramePacing.swift b/Sources/UntoldEngine/Renderer/XRFramePacing.swift index 6f146ec7d..c2543350a 100644 --- a/Sources/UntoldEngine/Renderer/XRFramePacing.swift +++ b/Sources/UntoldEngine/Renderer/XRFramePacing.swift @@ -62,6 +62,9 @@ public struct XRHalfRatePolicy: Equatable, Sendable { /// The share above the mean GPU span the cadence allows for the longer frames. public static let spanMargin = 0.2 public static let mostRefreshes = 4 + /// How far past a cadence's boundary the typical frame must be for the cadence to change, + /// in milliseconds: a frame on the boundary would otherwise change it every few seconds. + public static let cadenceDeadbandMs = 1.0 /// A second of frames ended: `frames` rendered, `late` of them past the deadline by /// `meanOverrunMs` on average, their GPU spans `meanGPUDurationMs` on average, the @@ -90,18 +93,20 @@ public struct XRHalfRatePolicy: Equatable, Sendable { landingRefreshes = meanOverrunMs > 1.0 ? min(Self.mostRefreshes - 1, Int((meanOverrunMs / periodMs).rounded(.up))) : 0 // The cadence rises at once when the frame's work needs more refreshes, and falls // only after two seconds running that fit the shorter one: a cadence that changed - // every second would itself be a hop. + // every second would itself be a hop. Both ways with a deadband past the boundary. let typical = meanGPUDurationMs * (1 + Self.spanMargin) + Self.encodeAllowanceMs - let needed = min(Self.mostRefreshes, max(2, Int((typical / periodMs).rounded(.up)))) - if needed >= cadenceRefreshes { - cadenceRefreshes = needed + let current = Double(cadenceRefreshes) * periodMs + if typical > current + Self.cadenceDeadbandMs { + cadenceRefreshes = min(Self.mostRefreshes, Int(((typical - Self.cadenceDeadbandMs) / periodMs).rounded(.up))) shorterSeconds = 0 - } else { + } else if cadenceRefreshes > 2, typical < current - periodMs - Self.cadenceDeadbandMs { shorterSeconds += 1 if shorterSeconds >= 2 { - cadenceRefreshes = needed + cadenceRefreshes = max(2, Int(((typical + Self.cadenceDeadbandMs) / periodMs).rounded(.up))) shorterSeconds = 0 } + } else { + shorterSeconds = 0 } } if periodMs > 0, meanGPUDurationMs < Self.fitShare * periodMs { diff --git a/Sources/UntoldEngineXR/UntoldEngineXR.swift b/Sources/UntoldEngineXR/UntoldEngineXR.swift index db60ac261..6b842a6fe 100644 --- a/Sources/UntoldEngineXR/UntoldEngineXR.swift +++ b/Sources/UntoldEngineXR/UntoldEngineXR.swift @@ -181,8 +181,11 @@ let due = last.advanced(by: .seconds(Double(cadence) * period)) if due > earliest { earliest = due } } + // Snapped up to the grid with a quarter of a refresh of tolerance: the learned period + // is a hair short of the compositor's, and without it "two refreshes" computed as + // 2.0001 and rounded up to three, every time. let behind = Self.seconds(grid.duration(to: earliest)) - let refreshes = behind > 0 ? (behind / period).rounded(.up) : 0 + let refreshes = behind > 0 ? max(0, (behind / period - 0.25).rounded(.up)) : 0 let target = grid.advanced(by: .seconds(refreshes * period)) let waitMs = max(0, Self.milliseconds(clock.now.duration(to: target))) clock.wait(until: target, tolerance: .zero) diff --git a/Tests/UntoldEngineTests/XRHalfRatePolicyTests.swift b/Tests/UntoldEngineTests/XRHalfRatePolicyTests.swift index 98a679572..778c89fb7 100644 --- a/Tests/UntoldEngineTests/XRHalfRatePolicyTests.swift +++ b/Tests/UntoldEngineTests/XRHalfRatePolicyTests.swift @@ -69,13 +69,20 @@ final class XRHalfRatePolicyTests: XCTestCase { // A mean of 14 ms, a fifth more and 3 ms of encode: 19.8 ms, two refreshes of 10 ms. XCTAssertEqual(pacedSecond(&policy, overrun: 8.0, meanGPU: 14.0), .none) XCTAssertEqual(policy.cadenceRefreshes, 2) + // A mean of 14.5 ms: 20.4 ms, within the deadband past two refreshes, still two. + XCTAssertEqual(pacedSecond(&policy, overrun: 8.0, meanGPU: 14.5), .none) + XCTAssertEqual(policy.cadenceRefreshes, 2) // A mean of 16 ms: 22.2 ms, three. XCTAssertEqual(pacedSecond(&policy, overrun: 8.0, meanGPU: 16.0), .none) XCTAssertEqual(policy.cadenceRefreshes, 3) + // Back to 14.5 ms, 20.4 ms, just under three refreshes but within the deadband of two: stays three. + XCTAssertEqual(pacedSecond(&policy, overrun: 8.0, meanGPU: 14.5), .none) + XCTAssertEqual(pacedSecond(&policy, overrun: 8.0, meanGPU: 14.5), .none) + XCTAssertEqual(policy.cadenceRefreshes, 3) // Never more than four. XCTAssertEqual(pacedSecond(&policy, overrun: 8.0, meanGPU: 60.0), .none) XCTAssertEqual(policy.cadenceRefreshes, 4) - // A shorter cadence is taken only after two seconds running that fit it. + // A shorter cadence is taken only after two seconds running that fit it clearly. XCTAssertEqual(pacedSecond(&policy, overrun: 8.0, meanGPU: 9.0), .none) XCTAssertEqual(policy.cadenceRefreshes, 4) XCTAssertEqual(pacedSecond(&policy, overrun: 8.0, meanGPU: 9.0), .none) From 55131876f4d940ab0d1b79e3f301ee7326c7d137 Mon Sep 17 00:00:00 2001 From: Javier Segura Date: Thu, 15 Oct 2026 12:21:01 +0200 Subject: [PATCH 20/22] [Bugfix] The pacing waits from the instant the compositor handed the last frame, and the late-frame line shows the loop's split Asked for two refreshes, the pacing still took three: its wait was aimed at a presentation instant on the refresh grid, and a query made there landed just after the compositor's hand-out for that refresh and blocked until the next one, a refresh lost on every frame. The pacing now waits until the cadence's refreshes after the compositor handed it the last frame, less three milliseconds, and lets the query's own block align the frames to the refreshes. The per-second line adds how long the query blocked, how long the loop took between frames, how far after the query the presentation was predicted and how far before it the deadline sat, so a run shows where a paced frame's time goes. --- Sources/UntoldEngineXR/UntoldEngineXR.swift | 75 +++++++++++++-------- 1 file changed, 47 insertions(+), 28 deletions(-) diff --git a/Sources/UntoldEngineXR/UntoldEngineXR.swift b/Sources/UntoldEngineXR/UntoldEngineXR.swift index 6b842a6fe..538a2beaa 100644 --- a/Sources/UntoldEngineXR/UntoldEngineXR.swift +++ b/Sources/UntoldEngineXR/UntoldEngineXR.swift @@ -109,6 +109,9 @@ private var secondGPUSumMs = 0.0 private var secondPacedWaitSumMs = 0.0 private var secondPacedWaits = 0 + /// How much earlier than the cadence the paced query is made, in seconds: the query + /// blocks until the compositor hands out the frame, which aligns it to the refreshes. + private static let pacedQueryLead: TimeInterval = 0.003 /// The automatic mode's current choice: waiting for the optimal input time, or not. private var automaticWaits = true /// The half-rate pacing: its policy, whether this frame is paced, the last second's @@ -119,7 +122,16 @@ private var pacingActiveNow = false private var secondMaxGPUMs = 0.0 private var lastRenderedPresentation: LayerRenderer.Clock.Instant? - private var lastPacedQuery: LayerRenderer.Clock.Instant? + /// When the compositor handed the loop its last frame: the pacing's anchor. + private var lastFrameHandedAt: LayerRenderer.Clock.Instant? + /// The last second's loop: how long the query for a frame blocked, how long the loop + /// took from one handed frame to the next query, how far after the query the compositor + /// predicted the presentation, and how far before it the rendering deadline sat. + private var secondQueryBlockSumMs = 0.0 + private var secondLoopSumMs = 0.0 + private var secondPredictedAheadSumMs = 0.0 + private var secondDeadlineMarginSumMs = 0.0 + private var secondTimedFrames = 0 /// Whether the loop is rendering every other refresh at the moment. public var halfRatePacingActive: Bool { deadlineLock.withLock { pacingActiveNow } } /// Whether the last frame waited for the compositor's optimal input time: the setting, @@ -147,14 +159,15 @@ } /// The half-rate pacing, before the next compositor frame is taken. With the pacing - /// active, by the setting or by the policy when the frames run late, the frames are - /// taken on the refresh grid (the last rendered frame's presentation instant and its - /// multiples of the period) every `cadenceRefreshes` refreshes: two, or as many as the - /// frame's GPU work with its encode needs. Nothing waits for the GPU: the next frame's - /// work queues behind the previous one's, so the GPU stays busy and keeps its clock - /// (idling it between frames made every frame a third longer). Every frame taken is - /// submitted: a frame taken and let go is never released by the compositor, and three - /// of them abort the app. + /// active, by the setting or by the policy when the frames run late, the loop waits + /// until `cadenceRefreshes` refreshes after the compositor handed it the last frame, + /// less a few milliseconds, then asks for the next: the query blocks until the + /// compositor hands it out, which aligns the frames to the refreshes without the loop + /// guessing their grid (a wait aimed at a presentation instant landed just after the + /// hand-out and cost a refresh every frame). Nothing waits for the GPU: the next + /// frame's work queues behind the previous one's, so the GPU stays busy and keeps its + /// clock. Every frame taken is submitted: a frame taken and let go is never released by + /// the compositor, and three of them abort the app. private func paceForHalfRate() { let active: Bool let cadence: Int @@ -167,29 +180,15 @@ cadence = halfRate.cadenceRefreshes pacingActiveNow = active deadlineLock.unlock() - guard active, let grid = lastRenderedPresentation else { - lastPacedQuery = nil - return - } + guard active, let handed = lastFrameHandedAt else { return } let period = currentFrameTime.period guard period > 0 else { return } let clock = LayerRenderer.Clock() - // The earliest the next frame may be taken: a cadence after the last paced one, and - // not before now; snapped up to the refresh grid. - var earliest = clock.now - if let last = lastPacedQuery { - let due = last.advanced(by: .seconds(Double(cadence) * period)) - if due > earliest { earliest = due } - } - // Snapped up to the grid with a quarter of a refresh of tolerance: the learned period - // is a hair short of the compositor's, and without it "two refreshes" computed as - // 2.0001 and rounded up to three, every time. - let behind = Self.seconds(grid.duration(to: earliest)) - let refreshes = behind > 0 ? max(0, (behind / period - 0.25).rounded(.up)) : 0 - let target = grid.advanced(by: .seconds(refreshes * period)) + let target = handed.advanced(by: .seconds(Double(cadence) * period - Self.pacedQueryLead)) let waitMs = max(0, Self.milliseconds(clock.now.duration(to: target))) - clock.wait(until: target, tolerance: .zero) - lastPacedQuery = target + if waitMs > 0 { + clock.wait(until: target, tolerance: .zero) + } deadlineLock.lock() secondPacedWaitSumMs += waitMs secondPacedWaits += 1 @@ -706,7 +705,11 @@ // a frame's GPU work; the display shows the previous frame again meanwhile. paceForHalfRate() + let queryStart = LayerRenderer.Clock().now guard let frame = layerRenderer.queryNextFrame() else { return } + let handedAt = LayerRenderer.Clock().now + let loopMs = lastFrameHandedAt.map { Self.milliseconds($0.duration(to: queryStart)) } ?? 0 + lastFrameHandedAt = handedAt // 2. Call predictTiming to get the predicted render deadlines for code guard let timing = frame.predictTiming() else { return } @@ -718,6 +721,13 @@ let compositorPresentationTime = Self.mediaTime(of: timing.presentationTime) let predictedPresentationTime = compositorPresentationTime + presentationLeadSeconds() logClockBasesOnce() + deadlineLock.lock() + secondQueryBlockSumMs += Self.milliseconds(queryStart.duration(to: handedAt)) + secondLoopSumMs += loopMs + secondPredictedAheadSumMs += Self.milliseconds(handedAt.duration(to: timing.presentationTime)) + secondDeadlineMarginSumMs += Self.milliseconds(timing.renderingDeadline.duration(to: timing.presentationTime)) + secondTimedFrames += 1 + deadlineLock.unlock() // 3. Call startUpdate() to mark the start of the update phase frame.startUpdate() @@ -1471,6 +1481,10 @@ let meanOverrunMs = secondOverrunSumMs / Double(secondLate) report = String(format: "XR frame timing: the GPU finished after the rendering deadline on %d of %d frames in the last second (worst +%.2f ms, mean +%.2f ms; %d late frames so far; input-time wait %@, %@%@)", secondLate, secondFrames, secondWorstMs, meanOverrunMs, deadlineMissCount, mode, lastFrameWaited ? "waited" : "not waited", pacingActiveNow ? String(format: "; half rate, every %d refreshes, landing +%d", halfRate.cadenceRefreshes, halfRate.landingRefreshes) : "") report = (report ?? "") + String(format: "; GPU %.1f ms a frame, longest %.1f%@", secondGPUSumMs / Double(secondFrames), secondMaxGPUMs, secondPacedWaits > 0 ? String(format: ", paced wait %.1f ms", secondPacedWaitSumMs / Double(secondPacedWaits)) : "") + if secondTimedFrames > 0 { + let timed = Double(secondTimedFrames) + report = (report ?? "") + String(format: "; the query blocked %.1f ms, the loop took %.1f ms between frames, the presentation was predicted %.1f ms after the query, the deadline %.1f ms before it", secondQueryBlockSumMs / timed, secondLoopSumMs / timed, secondPredictedAheadSumMs / timed, secondDeadlineMarginSumMs / timed) + } // With the GPU pass timing on, the heaviest passes of the last resolved frame // name what a second of frames well past the deadline spent its time on. if meanOverrunMs > 2.0, GPUPassTiming.shared.isEnabled { @@ -1523,6 +1537,11 @@ secondGPUSumMs = 0 secondPacedWaitSumMs = 0 secondPacedWaits = 0 + secondQueryBlockSumMs = 0 + secondLoopSumMs = 0 + secondPredictedAheadSumMs = 0 + secondDeadlineMarginSumMs = 0 + secondTimedFrames = 0 lastDeadlineLogTime = now } deadlineLock.unlock() From 118e7b6a7a21c1501b10fab363ed7176bb9134ff Mon Sep 17 00:00:00 2001 From: Javier Segura Date: Thu, 15 Oct 2026 12:29:38 +0200 Subject: [PATCH 21/22] [Bugfix] The paced queries fall on a fixed grid, exactly a cadence apart, and the line reports the presentations' spacing The compositor hands out a frame on demand (the query blocked 0.0 ms) and predicts its presentation about three refreshes after the query. A wait aimed at the previous frame's presentation instant therefore made every cadence three refreshes; a wait of a cadence less a three-millisecond lead spaced the frames 16.6 ms apart at a cadence of 20, shown two refreshes apart, then one. The queries now fall on the pacing's own grid, the instant it started from, every cadence exactly, a query up to a quarter refresh late still taking its slot. The per-second line adds the mean, shortest and longest spacing between consecutive predicted presentations, so a run reads its evenness instead of assuming it. --- Sources/UntoldEngineXR/UntoldEngineXR.swift | 65 ++++++++++++++++----- 1 file changed, 50 insertions(+), 15 deletions(-) diff --git a/Sources/UntoldEngineXR/UntoldEngineXR.swift b/Sources/UntoldEngineXR/UntoldEngineXR.swift index 538a2beaa..767344d00 100644 --- a/Sources/UntoldEngineXR/UntoldEngineXR.swift +++ b/Sources/UntoldEngineXR/UntoldEngineXR.swift @@ -109,9 +109,6 @@ private var secondGPUSumMs = 0.0 private var secondPacedWaitSumMs = 0.0 private var secondPacedWaits = 0 - /// How much earlier than the cadence the paced query is made, in seconds: the query - /// blocks until the compositor hands out the frame, which aligns it to the refreshes. - private static let pacedQueryLead: TimeInterval = 0.003 /// The automatic mode's current choice: waiting for the optimal input time, or not. private var automaticWaits = true /// The half-rate pacing: its policy, whether this frame is paced, the last second's @@ -122,8 +119,17 @@ private var pacingActiveNow = false private var secondMaxGPUMs = 0.0 private var lastRenderedPresentation: LayerRenderer.Clock.Instant? - /// When the compositor handed the loop its last frame: the pacing's anchor. + /// When the compositor handed the loop its last frame, and the pacing's own grid: the + /// instant it started from, on which the paced queries fall every cadence. private var lastFrameHandedAt: LayerRenderer.Clock.Instant? + private var pacingOrigin: LayerRenderer.Clock.Instant? + /// The last frame's predicted presentation, and the last second's spacing between + /// consecutive ones: its sum, shortest and longest, the evenness of the frames shown. + private var lastPredictedPresentation: LayerRenderer.Clock.Instant? + private var secondSpacingSumMs = 0.0 + private var secondSpacingMinMs = Double.infinity + private var secondSpacingMaxMs = 0.0 + private var secondSpacings = 0 /// The last second's loop: how long the query for a frame blocked, how long the loop /// took from one handed frame to the next query, how far after the query the compositor /// predicted the presentation, and how far before it the rendering deadline sat. @@ -159,15 +165,17 @@ } /// The half-rate pacing, before the next compositor frame is taken. With the pacing - /// active, by the setting or by the policy when the frames run late, the loop waits - /// until `cadenceRefreshes` refreshes after the compositor handed it the last frame, - /// less a few milliseconds, then asks for the next: the query blocks until the - /// compositor hands it out, which aligns the frames to the refreshes without the loop - /// guessing their grid (a wait aimed at a presentation instant landed just after the - /// hand-out and cost a refresh every frame). Nothing waits for the GPU: the next - /// frame's work queues behind the previous one's, so the GPU stays busy and keeps its - /// clock. Every frame taken is submitted: a frame taken and let go is never released by - /// the compositor, and three of them abort the app. + /// active, by the setting or by the policy when the frames run late, the queries fall + /// on the pacing's own grid: the instant it started from, every `cadenceRefreshes` + /// refreshes, exactly. The compositor hands out a frame on demand and predicts its + /// presentation a fixed time after the query, so queries spaced an exact number of + /// refreshes apart are shown an exact number of refreshes apart. (A wait aimed at the + /// previous frame's presentation instant made every cadence three refreshes, the + /// compositor predicting three ahead; a wait of a cadence less a lead spaced the frames + /// unevenly.) A query up to a quarter refresh late still takes its slot. Nothing waits + /// for the GPU: the next frame's work queues behind the previous one's, so the GPU + /// stays busy and keeps its clock. Every frame taken is submitted: a frame taken and + /// let go is never released by the compositor, and three of them abort the app. private func paceForHalfRate() { let active: Bool let cadence: Int @@ -180,11 +188,21 @@ cadence = halfRate.cadenceRefreshes pacingActiveNow = active deadlineLock.unlock() - guard active, let handed = lastFrameHandedAt else { return } + guard active, let handed = lastFrameHandedAt else { + pacingOrigin = nil + return + } let period = currentFrameTime.period guard period > 0 else { return } let clock = LayerRenderer.Clock() - let target = handed.advanced(by: .seconds(Double(cadence) * period - Self.pacedQueryLead)) + guard let origin = pacingOrigin else { + pacingOrigin = handed + return + } + let step = Double(cadence) * period + let elapsed = Self.seconds(origin.duration(to: clock.now)) + let slots = max(1, ((elapsed - 0.25 * period) / step).rounded(.up)) + let target = origin.advanced(by: .seconds(slots * step)) let waitMs = max(0, Self.milliseconds(clock.now.duration(to: target))) if waitMs > 0 { clock.wait(until: target, tolerance: .zero) @@ -721,6 +739,16 @@ let compositorPresentationTime = Self.mediaTime(of: timing.presentationTime) let predictedPresentationTime = compositorPresentationTime + presentationLeadSeconds() logClockBasesOnce() + if let last = lastPredictedPresentation { + let spacingMs = Self.milliseconds(last.duration(to: timing.presentationTime)) + deadlineLock.lock() + secondSpacingSumMs += spacingMs + secondSpacingMinMs = min(secondSpacingMinMs, spacingMs) + secondSpacingMaxMs = max(secondSpacingMaxMs, spacingMs) + secondSpacings += 1 + deadlineLock.unlock() + } + lastPredictedPresentation = timing.presentationTime deadlineLock.lock() secondQueryBlockSumMs += Self.milliseconds(queryStart.duration(to: handedAt)) secondLoopSumMs += loopMs @@ -1485,6 +1513,9 @@ let timed = Double(secondTimedFrames) report = (report ?? "") + String(format: "; the query blocked %.1f ms, the loop took %.1f ms between frames, the presentation was predicted %.1f ms after the query, the deadline %.1f ms before it", secondQueryBlockSumMs / timed, secondLoopSumMs / timed, secondPredictedAheadSumMs / timed, secondDeadlineMarginSumMs / timed) } + if secondSpacings > 0 { + report = (report ?? "") + String(format: "; presentations spaced %.1f ms (%.1f to %.1f)", secondSpacingSumMs / Double(secondSpacings), secondSpacingMinMs, secondSpacingMaxMs) + } // With the GPU pass timing on, the heaviest passes of the last resolved frame // name what a second of frames well past the deadline spent its time on. if meanOverrunMs > 2.0, GPUPassTiming.shared.isEnabled { @@ -1542,6 +1573,10 @@ secondPredictedAheadSumMs = 0 secondDeadlineMarginSumMs = 0 secondTimedFrames = 0 + secondSpacingSumMs = 0 + secondSpacingMinMs = .infinity + secondSpacingMaxMs = 0 + secondSpacings = 0 lastDeadlineLogTime = now } deadlineLock.unlock() From 27e121e8da019c3ce4d27703d4eb9b39f8a55927 Mon Sep 17 00:00:00 2001 From: Javier Segura Date: Thu, 15 Oct 2026 12:53:02 +0200 Subject: [PATCH 22/22] [Feature] The half-rate pacing enters on skipped refreshes or frames past the deadline's margin, and the pose lands by that margin The compositor's rendering deadline sits 15 ms before the presentation on the headset, so a frame a few milliseconds past it is still shown on time: the full-rate runs with nearly every frame a millisecond late dropped nothing, and the heavy run's paced frames, 3 to 6 ms past the deadline, were posed a refresh ahead of where they were shown ("landing +1"). The policy now takes the deadline's margin into account, less a 2 ms safety (XRHalfRatePolicy.usableMarginMs): - The pose lands on the compositor's prediction while the frames finish within the usable margin, and as many refreshes later as the rest of the mean overrun reaches. - The pacing enters when, with no wait left to skip, the display skipped a refresh for one frame in ten or more (counted in the loop from the spacing of consecutive predicted presentations, beyond the pacing's own cadence) or one frame in ten or more finished past the usable margin, for two seconds running. Frames merely past the deadline no longer count: the heavy run's second before it paced had 90 frames a millisecond past it, shown on time, and fewer hops than one in ten are fewer than the steps the half rate adds. The loop counts both per second (secondMissed, secondSkipped) and prints them on the per-second line ("N past the deadline's margin, N refreshes skipped"), which now also appears for a second with skipped refreshes and no late frame; the GPU completion handler receives the drawable's presentation time for the margin. The proposal's pacing paragraph describes the final mechanism (the fixed grid, the landing, the four lessons from the device) and records the by-eye result of the heavy run: the even 45 fps shows fewer jumps than the hopping full rate, and a visible stepping remains, the frame rate itself; the cure for that configuration is the frame's GPU cost, not the timing. --- .../UntoldEngine/Renderer/XRFramePacing.swift | 44 +++++++++++----- Sources/UntoldEngineXR/UntoldEngineXR.swift | 37 +++++++++++--- .../XRHalfRatePolicyTests.swift | 51 ++++++++++++------- docs/proposals/PresentationTimeAnimation.md | 2 +- 4 files changed, 95 insertions(+), 39 deletions(-) diff --git a/Sources/UntoldEngine/Renderer/XRFramePacing.swift b/Sources/UntoldEngine/Renderer/XRFramePacing.swift index c2543350a..b7e0c3a09 100644 --- a/Sources/UntoldEngine/Renderer/XRFramePacing.swift +++ b/Sources/UntoldEngine/Renderer/XRFramePacing.swift @@ -24,9 +24,12 @@ public enum XRFramePacing: Sendable, Equatable { } /// The half-rate decision from each second's deadline measurements. Enter when, with no wait -/// for the optimal input time left to skip, half the frames or more were late by over a -/// millisecond on average for two seconds running; leave when the rendered frames' mean GPU -/// work has fitted 80% of the refresh for two seconds running. While active, the refreshes +/// for the optimal input time left to skip, the display skipped a refresh for one frame in +/// ten or more, or one frame in ten or more missed its presentation (finishing past the +/// compositor's deadline by more than the deadline's margin before the presentation, less a +/// safety), for two seconds running: fewer hops than that are fewer than the steps the half +/// rate would add. Leave when the rendered frames' mean GPU work has fitted 80% of the +/// refresh for two seconds running. While active, the refreshes /// between two frames follow the measured GPU cost: two, or more when a frame's typical work /// (the mean span plus a fifth) with its encode does not fit two. The mean, not the longest /// span: a GPU used on half the refreshes runs at a lower clock, and its longest spans would @@ -38,7 +41,8 @@ public struct XRHalfRatePolicy: Equatable, Sendable { /// with its encode fits, two at least, four at most. public private(set) var cadenceRefreshes = 2 /// Refreshes after the compositor's predicted presentation at which the paced frames are - /// shown, from how far past their deadline they finish: the pose aims there. + /// shown, from how far past their deadline they finish beyond the deadline's margin before + /// the presentation: the pose aims there. public private(set) var landingRefreshes = 0 private(set) var lateSeconds = 0 private(set) var fitSeconds = 0 @@ -65,17 +69,29 @@ public struct XRHalfRatePolicy: Equatable, Sendable { /// How far past a cadence's boundary the typical frame must be for the cadence to change, /// in milliseconds: a frame on the boundary would otherwise change it every few seconds. public static let cadenceDeadbandMs = 1.0 + /// The part of the deadline's margin before the presentation not counted on, in + /// milliseconds: a frame past the deadline by less than the margin less this is taken as + /// shown on time. + public static let marginSafetyMs = 2.0 - /// A second of frames ended: `frames` rendered, `late` of them past the deadline by - /// `meanOverrunMs` on average, their GPU spans `meanGPUDurationMs` on average, the - /// display's refresh `periodMs`, and whether the loop still has a wait it could skip first. - public mutating func secondEnded(frames: Int, late: Int, meanOverrunMs: Double, meanGPUDurationMs: Double, periodMs: Double, waitStillToSkip: Bool) -> Change { + /// How far past the deadline a frame may finish and still be taken as shown on time. + public static func usableMarginMs(deadlineMarginMs: Double) -> Double { + max(0, deadlineMarginMs - marginSafetyMs) + } + + /// A second of frames ended: `frames` rendered, `missed` of them past the deadline by more + /// than its usable margin, `skipped` refreshes the display showed no new frame at beyond + /// the cadence, the frames past the deadline by `meanOverrunMs` on average, their GPU + /// spans `meanGPUDurationMs` on average, the display's refresh `periodMs`, the deadline + /// `deadlineMarginMs` before the presentation, and whether the loop still has a wait it + /// could skip first. + public mutating func secondEnded(frames: Int, missed: Int, skipped: Int, meanOverrunMs: Double, meanGPUDurationMs: Double, periodMs: Double, deadlineMarginMs: Double, waitStillToSkip: Bool) -> Change { guard frames > 0 else { return .none } - let mostlyLate = late * 2 >= frames + let hopping = skipped * 10 >= frames || missed * 10 >= frames if !active { cadenceRefreshes = 2 landingRefreshes = 0 - if !waitStillToSkip, mostlyLate, meanOverrunMs > 1.0 { + if !waitStillToSkip, hopping { lateSeconds += 1 } else { lateSeconds = 0 @@ -88,9 +104,11 @@ public struct XRHalfRatePolicy: Equatable, Sendable { return .none } if periodMs > 0 { - // Where the frames land: a frame finished past its deadline is shown at the next - // refresh its overrun reaches; one within a millisecond makes its predicted one. - landingRefreshes = meanOverrunMs > 1.0 ? min(Self.mostRefreshes - 1, Int((meanOverrunMs / periodMs).rounded(.up))) : 0 + // Where the frames land: a frame finished past its deadline by more than the + // deadline's usable margin is shown at the next refresh the rest reaches; one + // within the margin makes its predicted one. + let beyond = meanOverrunMs - Self.usableMarginMs(deadlineMarginMs: deadlineMarginMs) + landingRefreshes = beyond > 0 ? min(Self.mostRefreshes - 1, Int((beyond / periodMs).rounded(.up))) : 0 // The cadence rises at once when the frame's work needs more refreshes, and falls // only after two seconds running that fit the shorter one: a cadence that changed // every second would itself be a hop. Both ways with a deadband past the boundary. diff --git a/Sources/UntoldEngineXR/UntoldEngineXR.swift b/Sources/UntoldEngineXR/UntoldEngineXR.swift index 767344d00..7ead4769a 100644 --- a/Sources/UntoldEngineXR/UntoldEngineXR.swift +++ b/Sources/UntoldEngineXR/UntoldEngineXR.swift @@ -88,8 +88,10 @@ /// update: the game posed those frames for a slightly different instant than the head. public private(set) var presentationTimeRevisions = 0 private var lastPresentationRevisionLogTime: CFTimeInterval = 0 - /// Frames whose GPU work finished after the compositor's rendering deadline: the display - /// showed the previous frame again and this one a refresh late, posed for an instant + /// Frames whose GPU work finished after the compositor's rendering deadline. The deadline + /// sits well before the presentation (15 ms on the headset), so a frame a few + /// milliseconds past it is still shown on time; one past that margin has the display + /// show the previous frame again and this one a refresh late, posed for an instant /// already past — a hop in a moving object that no counter of the loop's pacing sees. public var deadlineMisses: Int { deadlineLock.withLock { deadlineMissCount } } /// The longest a frame's GPU work finished after its deadline, in milliseconds. @@ -109,6 +111,11 @@ private var secondGPUSumMs = 0.0 private var secondPacedWaitSumMs = 0.0 private var secondPacedWaits = 0 + /// The last second's frames past the deadline by more than its usable margin before the + /// presentation (shown a refresh late), and the refreshes the display skipped beyond + /// the pacing's cadence, from the spacing of the predicted presentations. + private var secondMissed = 0 + private var secondSkipped = 0 /// The automatic mode's current choice: waiting for the optimal input time, or not. private var automaticWaits = true /// The half-rate pacing: its policy, whether this frame is paced, the last second's @@ -746,6 +753,11 @@ secondSpacingMinMs = min(secondSpacingMinMs, spacingMs) secondSpacingMaxMs = max(secondSpacingMaxMs, spacingMs) secondSpacings += 1 + let periodMs = currentFrameTime.period * 1000.0 + if periodMs > 0 { + let expected = pacingActiveNow ? halfRate.cadenceRefreshes : 1 + secondSkipped += max(0, Int((spacingMs / periodMs).rounded()) - expected) + } deadlineLock.unlock() } lastPredictedPresentation = timing.presentationTime @@ -1304,6 +1316,7 @@ // The instant the GPU must have finished by for the compositor to show this frame at // its presentation time, in the base of the command buffer's own timestamps. let deadline = Self.mediaTime(of: drawable.frameTiming.renderingDeadline) + let presentation = Self.mediaTime(of: drawable.frameTiming.presentationTime) let countsDeadline = primary // Add completion handler to signal semaphore when GPU work is done commandBuffer.addCompletedHandler { [weak self] cb in @@ -1313,7 +1326,7 @@ EngineStatsMonitor.shared.recordGPUCompletion(gpuStartTime: cb.gpuStartTime, gpuEndTime: cb.gpuEndTime) #endif if countsDeadline { - self?.noteGPUCompletion(at: cb.gpuEndTime, start: cb.gpuStartTime, deadline: deadline) + self?.noteGPUCompletion(at: cb.gpuEndTime, start: cb.gpuStartTime, deadline: deadline, presentation: presentation) } commandBufferSemaphore.signal() } @@ -1479,8 +1492,9 @@ /// Counts a frame whose GPU work finished after the compositor's rendering deadline, keeps /// the longest overrun, and prints once a second how many of that second's frames were /// late and by how much. Runs on the GPU's completion thread. - private func noteGPUCompletion(at gpuEnd: CFTimeInterval, start gpuStart: CFTimeInterval, deadline: TimeInterval) { + private func noteGPUCompletion(at gpuEnd: CFTimeInterval, start gpuStart: CFTimeInterval, deadline: TimeInterval, presentation: TimeInterval) { let overrunMs = (gpuEnd - deadline) * 1000.0 + let usableMarginMs = XRHalfRatePolicy.usableMarginMs(deadlineMarginMs: (presentation - deadline) * 1000.0) let now = CACurrentMediaTime() var report: String? deadlineLock.lock() @@ -1495,20 +1509,24 @@ secondOverrunSumMs += overrunMs secondWorstMs = max(secondWorstMs, overrunMs) worstOverrunMs = max(worstOverrunMs, overrunMs) + if overrunMs > usableMarginMs { + secondMissed += 1 + } } if lastDeadlineLogTime == 0 { lastDeadlineLogTime = now } else if now - lastDeadlineLogTime >= 1.0 { - if secondLate > 0 { + if secondLate > 0 || secondSkipped > 0 { let mode: String switch xrOptimalInputWaitMode { case .always: mode = "always" case .never: mode = "never" case .automatic: mode = "automatic" } - let meanOverrunMs = secondOverrunSumMs / Double(secondLate) + let meanOverrunMs = secondLate > 0 ? secondOverrunSumMs / Double(secondLate) : 0 report = String(format: "XR frame timing: the GPU finished after the rendering deadline on %d of %d frames in the last second (worst +%.2f ms, mean +%.2f ms; %d late frames so far; input-time wait %@, %@%@)", secondLate, secondFrames, secondWorstMs, meanOverrunMs, deadlineMissCount, mode, lastFrameWaited ? "waited" : "not waited", pacingActiveNow ? String(format: "; half rate, every %d refreshes, landing +%d", halfRate.cadenceRefreshes, halfRate.landingRefreshes) : "") report = (report ?? "") + String(format: "; GPU %.1f ms a frame, longest %.1f%@", secondGPUSumMs / Double(secondFrames), secondMaxGPUMs, secondPacedWaits > 0 ? String(format: ", paced wait %.1f ms", secondPacedWaitSumMs / Double(secondPacedWaits)) : "") + report = (report ?? "") + String(format: "; %d past the deadline's margin, %d refreshes skipped", secondMissed, secondSkipped) if secondTimedFrames > 0 { let timed = Double(secondTimedFrames) report = (report ?? "") + String(format: "; the query blocked %.1f ms, the loop took %.1f ms between frames, the presentation was predicted %.1f ms after the query, the deadline %.1f ms before it", secondQueryBlockSumMs / timed, secondLoopSumMs / timed, secondPredictedAheadSumMs / timed, secondDeadlineMarginSumMs / timed) @@ -1545,9 +1563,10 @@ let waitStillToSkip = xrOptimalInputWaitMode == .automatic && automaticWaits let meanLateMs = secondLate > 0 ? secondOverrunSumMs / Double(secondLate) : 0 let periodMs = currentFrameTime.period * 1000.0 - switch halfRate.secondEnded(frames: secondFrames, late: secondLate, meanOverrunMs: meanLateMs, meanGPUDurationMs: secondGPUSumMs / Double(secondFrames), periodMs: periodMs, waitStillToSkip: waitStillToSkip) { + let marginMs = secondTimedFrames > 0 ? secondDeadlineMarginSumMs / Double(secondTimedFrames) : 0 + switch halfRate.secondEnded(frames: secondFrames, missed: secondMissed, skipped: secondSkipped, meanOverrunMs: meanLateMs, meanGPUDurationMs: secondGPUSumMs / Double(secondFrames), periodMs: periodMs, deadlineMarginMs: marginMs, waitStillToSkip: waitStillToSkip) { case .entered: - let line = String(format: "XR frame timing: %d of %d frames late with no wait left to skip: rendering every other refresh", secondLate, secondFrames) + let line = String(format: "XR frame timing: %d of %d frames past the deadline's margin, %d refreshes skipped, no wait left to skip: rendering every other refresh", secondMissed, secondFrames, secondSkipped) report = report.map { $0 + "\n" + line } ?? line case .left: let line = String(format: "XR frame timing: the frame's GPU work of %.2f ms fits the refresh of %.2f ms: rendering every refresh again", secondGPUSumMs / Double(secondFrames), periodMs) @@ -1568,6 +1587,8 @@ secondGPUSumMs = 0 secondPacedWaitSumMs = 0 secondPacedWaits = 0 + secondMissed = 0 + secondSkipped = 0 secondQueryBlockSumMs = 0 secondLoopSumMs = 0 secondPredictedAheadSumMs = 0 diff --git a/Tests/UntoldEngineTests/XRHalfRatePolicyTests.swift b/Tests/UntoldEngineTests/XRHalfRatePolicyTests.swift index 778c89fb7..c0a8a3588 100644 --- a/Tests/UntoldEngineTests/XRHalfRatePolicyTests.swift +++ b/Tests/UntoldEngineTests/XRHalfRatePolicyTests.swift @@ -11,22 +11,24 @@ @testable import UntoldEngine import XCTest -/// The half-rate pacing enters after two seconds of mostly late frames with no wait left to -/// skip, spaces the frames by the refreshes their GPU work with its encode needs, and leaves -/// after two seconds in which the frames' GPU work fitted most of a refresh. +/// The half-rate pacing enters after two seconds of skipped refreshes or missed frames with no +/// wait left to skip, spaces the frames by the refreshes their GPU work with its encode needs, +/// aims the pose where the frames land past the deadline's margin, and leaves after two seconds +/// in which the frames' GPU work fitted most of a refresh. final class XRHalfRatePolicyTests: XCTestCase { private let period = 10.0 + private let margin = 15.0 private func lateSecond(_ policy: inout XRHalfRatePolicy, waitStillToSkip: Bool = false, meanGPU: Double = 12.0) -> XRHalfRatePolicy.Change { - policy.secondEnded(frames: 100, late: 80, meanOverrunMs: 3.0, meanGPUDurationMs: meanGPU, periodMs: period, waitStillToSkip: waitStillToSkip) + policy.secondEnded(frames: 100, missed: 30, skipped: 5, meanOverrunMs: 14.0, meanGPUDurationMs: meanGPU, periodMs: period, deadlineMarginMs: margin, waitStillToSkip: waitStillToSkip) } private func goodSecond(_ policy: inout XRHalfRatePolicy, meanGPU: Double = 7.0) -> XRHalfRatePolicy.Change { - policy.secondEnded(frames: 100, late: 2, meanOverrunMs: 0.2, meanGPUDurationMs: meanGPU, periodMs: period, waitStillToSkip: false) + policy.secondEnded(frames: 100, missed: 0, skipped: 0, meanOverrunMs: 0.2, meanGPUDurationMs: meanGPU, periodMs: period, deadlineMarginMs: margin, waitStillToSkip: false) } private func pacedSecond(_ policy: inout XRHalfRatePolicy, overrun: Double, meanGPU: Double) -> XRHalfRatePolicy.Change { - policy.secondEnded(frames: 40, late: 40, meanOverrunMs: overrun, meanGPUDurationMs: meanGPU, periodMs: period, waitStillToSkip: false) + policy.secondEnded(frames: 40, missed: 0, skipped: 0, meanOverrunMs: overrun, meanGPUDurationMs: meanGPU, periodMs: period, deadlineMarginMs: margin, waitStillToSkip: false) } func testEntersAfterTwoLateSecondsNotOne() { @@ -52,14 +54,27 @@ final class XRHalfRatePolicyTests: XCTestCase { XCTAssertFalse(policy.active) } - func testDoesNotEnterForSlightlyLateFrames() { + func testDoesNotEnterForFramesWithinTheDeadlinesMargin() { var policy = XRHalfRatePolicy() + // Past the deadline but within its margin, nothing skipped: shown on time. for _ in 0 ..< 3 { - XCTAssertEqual(policy.secondEnded(frames: 100, late: 60, meanOverrunMs: 0.4, meanGPUDurationMs: 10.2, periodMs: period, waitStillToSkip: false), .none) + XCTAssertEqual(policy.secondEnded(frames: 100, missed: 0, skipped: 0, meanOverrunMs: 4.0, meanGPUDurationMs: 10.2, periodMs: period, deadlineMarginMs: margin, waitStillToSkip: false), .none) + } + XCTAssertFalse(policy.active) + // Fewer than one frame in ten missed or a refresh skipped: still below the bar. + for _ in 0 ..< 3 { + XCTAssertEqual(policy.secondEnded(frames: 100, missed: 9, skipped: 9, meanOverrunMs: 4.0, meanGPUDurationMs: 10.2, periodMs: period, deadlineMarginMs: margin, waitStillToSkip: false), .none) } XCTAssertFalse(policy.active) } + func testEntersOnSkippedRefreshesAlone() { + var policy = XRHalfRatePolicy() + XCTAssertEqual(policy.secondEnded(frames: 100, missed: 0, skipped: 10, meanOverrunMs: 2.0, meanGPUDurationMs: 10.5, periodMs: period, deadlineMarginMs: margin, waitStillToSkip: false), .none) + XCTAssertEqual(policy.secondEnded(frames: 100, missed: 0, skipped: 10, meanOverrunMs: 2.0, meanGPUDurationMs: 10.5, periodMs: period, deadlineMarginMs: margin, waitStillToSkip: false), .entered) + XCTAssertTrue(policy.active) + } + func testWhileActiveTheCadenceFollowsTheTypicalGPUCostWithItsEncode() { var policy = XRHalfRatePolicy() _ = lateSecond(&policy) @@ -96,23 +111,25 @@ final class XRHalfRatePolicyTests: XCTestCase { XCTAssertTrue(policy.active) } - func testThePoseAimsAtTheRefreshTheOverrunReaches() { + func testThePoseAimsAtTheRefreshTheOverrunReachesBeyondTheMargin() { var policy = XRHalfRatePolicy() XCTAssertEqual(policy.landingRefreshes, 0) _ = lateSecond(&policy) _ = lateSecond(&policy) - // Finished 9 ms past the deadline: shown one refresh after the prediction. + // Finished 9 ms past the deadline, within its 15 ms margin less the safety: shown on time. XCTAssertEqual(pacedSecond(&policy, overrun: 9.0, meanGPU: 14.0), .none) + XCTAssertEqual(policy.landingRefreshes, 0) + // 14 ms past, a millisecond beyond the usable margin: one refresh after the prediction. + XCTAssertEqual(pacedSecond(&policy, overrun: 14.0, meanGPU: 14.0), .none) XCTAssertEqual(policy.landingRefreshes, 1) - // 15 ms past: two. - XCTAssertEqual(pacedSecond(&policy, overrun: 15.0, meanGPU: 14.0), .none) + // 28 ms past: two. + XCTAssertEqual(pacedSecond(&policy, overrun: 28.0, meanGPU: 14.0), .none) XCTAssertEqual(policy.landingRefreshes, 2) - // Within a millisecond: the predicted one. - XCTAssertEqual(pacedSecond(&policy, overrun: 0.5, meanGPU: 14.0), .none) - XCTAssertEqual(policy.landingRefreshes, 0) // Never beyond three. - XCTAssertEqual(pacedSecond(&policy, overrun: 80.0, meanGPU: 14.0), .none) + XCTAssertEqual(pacedSecond(&policy, overrun: 90.0, meanGPU: 14.0), .none) XCTAssertEqual(policy.landingRefreshes, 3) + XCTAssertEqual(XRHalfRatePolicy.usableMarginMs(deadlineMarginMs: 15.0), 13.0) + XCTAssertEqual(XRHalfRatePolicy.usableMarginMs(deadlineMarginMs: 1.0), 0.0) XCTAssertEqual(goodSecond(&policy), .none) XCTAssertEqual(goodSecond(&policy), .left) XCTAssertEqual(policy.landingRefreshes, 0) @@ -142,7 +159,7 @@ final class XRHalfRatePolicyTests: XCTestCase { func testAnEmptySecondChangesNothingAndResetForgetsEverything() { var policy = XRHalfRatePolicy() - XCTAssertEqual(policy.secondEnded(frames: 0, late: 0, meanOverrunMs: 0, meanGPUDurationMs: 0, periodMs: period, waitStillToSkip: false), .none) + XCTAssertEqual(policy.secondEnded(frames: 0, missed: 0, skipped: 0, meanOverrunMs: 0, meanGPUDurationMs: 0, periodMs: period, deadlineMarginMs: margin, waitStillToSkip: false), .none) _ = lateSecond(&policy) _ = lateSecond(&policy) policy.reset() diff --git a/docs/proposals/PresentationTimeAnimation.md b/docs/proposals/PresentationTimeAnimation.md index b5242b9c4..3ef6ca9d4 100644 --- a/docs/proposals/PresentationTimeAnimation.md +++ b/docs/proposals/PresentationTimeAnimation.md @@ -153,7 +153,7 @@ With the GPU pass timing on, a second of late frames names its heaviest passes. ### Half-rate pacing (done) -`setRendering(.framePacing(.fullRate | .halfRateWhenLate | .halfRate))`, half rate when late by default. `XRHalfRatePolicy` decides from each second's deadline measurements: it enters when, with no wait for the optimal input time left to skip, half the frames or more were late by over a millisecond on average for two seconds running; while active the loop takes its frames on the refresh grid (the last rendered frame's presentation instant and its multiples of the period) every two refreshes, or every three or four when the frame's GPU work with 3 ms of encode needs them, and poses each frame for the refresh that cadence lands on, the compositor's prediction (one refresh after the frame is taken) plus the cadence's remaining refreshes. Nothing waits for the GPU: the next frame's work queues behind the previous one's. (Two lessons from the device: a first version let every other frame go unsubmitted, and the compositor never releases a frame taken and not submitted, three of them aborting the app with "We reached more than 3 frames in flight", so the pacing is in the taking; a second version waited for each frame's GPU work to finish before taking the next, and the idle gaps let the GPU clock down, the Forward pass going from 9 to 12 ms and the cadence from two refreshes to three, 30 fps, so the GPU is kept queued.); it leaves when the rendered frames' GPU work has fitted 80% of the refresh for two seconds running. `halfRatePacingActive` says the state; the per-second line adds "half rate", and the viewer's switch maps to half rate when late or full rate, with "half-rate pacing active" in its summary. An even cadence at 50 fps, with the compositor reprojecting the head between frames, against a hop between 75 and 100: the trade the other headset runtimes make, to be judged by eye on the device. +`setRendering(.framePacing(.fullRate | .halfRateWhenLate | .halfRate))`, half rate when late by default. `XRHalfRatePolicy` decides from each second's measurements. It enters when, with no wait for the optimal input time left to skip, the display skipped a refresh for one frame in ten or more (from the spacing of the compositor's consecutive predicted presentations) or one frame in ten or more finished past the deadline by more than the deadline's margin, for two seconds running: the deadline sits 15 ms before the presentation on the headset, so a frame a few milliseconds past it is still shown on time (the full-rate runs with nearly every frame a millisecond late dropped nothing), and fewer hops than one in ten are fewer than the steps the half rate adds. While active the loop waits, before it asks the compositor for the next frame, until the next slot of a fixed grid: the instant the pacing started from plus a whole number of cadences (two refreshes, or three or four when the frame's typical GPU work, its mean span plus a fifth and 3 ms of encode, needs them; the cadence rises at once and falls after two fitting seconds, a millisecond's deadband at the boundary). The query itself never blocks, the compositor predicts the presentation three refreshes after it, so the grid, not the previous frame's presentation, sets the spacing, and the predicted presentations come exactly a cadence apart (22.2 ms flat at 90 Hz every second of the heavy run). Each frame is posed for the compositor's prediction plus the refreshes the frames land past it: none while they finish within the deadline's margin less a 2 ms safety, else as many as the rest reaches. Nothing waits for the GPU: the next frame's work queues behind the previous one's. (Lessons from the device: a first version let every other frame go unsubmitted, and the compositor never releases a frame taken and not submitted, three of them aborting the app with "We reached more than 3 frames in flight", so the pacing is in the taking; a second waited for each frame's GPU work before taking the next, and the idle gaps let the GPU clock down, the Forward pass going from 9 to 12 ms and the cadence from two to three, 30 fps, so the GPU is kept queued; a third snapped the query to the grid of the last frame's presentation instant, which the compositor predicts three refreshes ahead, so every frame waited a refresh too many, 30 fps again; a fourth counted the cadence from the instant the last frame was handed out, which drifted with the query's own lateness and gave an uneven 16.6 ms.) It leaves when the rendered frames' GPU work has fitted 80% of the refresh for two seconds running. `halfRatePacingActive` says the state; the per-second line adds "half rate, every N refreshes, landing +N", the frames past the margin and the refreshes skipped, and the spacing of the presentations; the viewer's switch maps to half rate when late or full rate, with "half-rate pacing active" in its summary. By eye on the device (2026-10-15, MSAA 4× with TAA and the occlusion cull at 90 Hz, the GPU at 13 to 15 ms a frame): the even 45 fps shows fewer jumps than the hopping full rate did, and still a visible stepping: a sweep at half the refresh rate is not smooth, it is evenly unsmooth, and the frame rate itself is the limit there. The cure for that configuration is the frame's GPU cost (the Forward pass at 6 to 9 ms, the 4× resolve, the HZB built for one object), not the timing. ### The true late latch (not done: a refactor, for a decision)