[Feature] The frame is posed for its predicted presentation time: no judder for a moving object on the headset

Sources/UntoldEngine/Animation/TransformTrack.swift

@@ -0,0 +1,144 @@
+//
+//  TransformTrack.swift
+//  UntoldEngine
+//
+// Copyright (C) Untold Engine Studios
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+import Foundation
+import simd
+
+/// How a track moves from one key to the next.
+public enum TrackEasing: Sendable, Equatable {
+    /// At a constant rate.
+    case linear
+    /// Slowly away from the key, faster towards the next.
+    case easeIn
+    /// Fast away from the key, slowing into the next.
+    case easeOut
+    /// Slowly away and slowly in (a smoothstep).
+    case easeInOut
+
+    /// The progress through a segment, 0 to 1, for a fraction of its duration.
+    public func progress(_ fraction: Float) -> Float {
+        let t = min(max(fraction, 0), 1)
+        switch self {
+        case .linear:
+            return t
+        case .easeIn:
+            return t * t
+        case .easeOut:
+            return 1 - (1 - t) * (1 - t)
+        case .easeInOut:
+            return t * t * (3 - 2 * t)
+        }
+    }
+}
+
+/// A transform at a time on a track, and how the track leaves it for the next key.
+public struct TransformKey: Sendable, Equatable {
+    /// Seconds from the track's start.
+    public var time: TimeInterval
+    public var position: simd_float3
+    public var rotation: simd_quatf
+    public var scale: simd_float3
+    /// How the track moves from this key to the next.
+    public var easing: TrackEasing
+
+    public init(
+        time: TimeInterval,
+        position: simd_float3,
+        rotation: simd_quatf = simd_quatf(angle: 0, axis: simd_float3(0, 1, 0)),
+        scale: simd_float3 = simd_float3(repeating: 1),
+        easing: TrackEasing = .linear
+    ) {
+        self.time = time
+        self.position = position
+        self.rotation = rotation
+        self.scale = scale
+        self.easing = easing
+    }
+}
+
+/// A position, rotation and scale sampled from a track.
+public struct TransformPose: Sendable, Equatable {
+    public var position: simd_float3
+    public var rotation: simd_quatf
+    public var scale: simd_float3
+
+    public init(position: simd_float3, rotation: simd_quatf, scale: simd_float3) {
+        self.position = position
+        self.rotation = rotation
+        self.scale = scale
+    }
+}
+
+/// Transform keys sampled at an absolute time: a frame's pose is a function of the frame's
+/// presentation time alone, with no state carried over from the frame before, so a frame the
+/// display skipped lands where the timeline says and the frames after it step by one period
+/// again. Sample it at `FrameTime.presentationTime` less the time the track started.
+public struct TransformTrack: Sendable, Equatable {
+    /// The keys in time order.
+    public let keys: [TransformKey]
+    /// Whether the track starts again after its last key, or holds the last key.
+    public let loops: Bool
+
+    public init(keys: [TransformKey], loops: Bool = false) {
+        self.keys = keys.sorted { $0.time < $1.time }
+        self.loops = loops
+    }
+
+    /// The time of the last key: the length of one pass.
+    public var duration: TimeInterval { keys.last?.time ?? 0 }
+
+    /// The pose `time` seconds into the track: before the first key the first key's, after the
+    /// last the last's (or, looping, the pose that far into the next pass). Nil without keys.
+    public func pose(at time: TimeInterval) -> TransformPose? {
+        guard let first = keys.first, let last = keys.last else { return nil }
+        var t = time
+        if loops, duration > 0 {
+            t = t.truncatingRemainder(dividingBy: duration)
+            if t < 0 { t += duration }
+        }
+        if t <= first.time { return pose(of: first) }
+        if t >= last.time { return pose(of: last) }
+
+        // The segment the time falls in: the last key at or before it, and the one after.
+        var index = 0
+        while index + 1 < keys.count, keys[index + 1].time <= t {
+            index += 1
+        }
+        let from = keys[index]
+        let to = keys[index + 1]
+        let span = to.time - from.time
+        let fraction = span > 0 ? Float((t - from.time) / span) : 1
+        let s = from.easing.progress(fraction)
+        return TransformPose(
+            position: simd_mix(from.position, to.position, simd_float3(repeating: s)),
+            rotation: simd_slerp(from.rotation, to.rotation, s),
+            scale: simd_mix(from.scale, to.scale, simd_float3(repeating: s))
+        )
+    }
+
+    /// The pose for a frame: the track's time is the frame's presentation time less
+    /// `startedAt`, the presentation time of the frame the track started in.
+    public func pose(for frame: FrameTime, startedAt: TimeInterval) -> TransformPose? {
+        pose(at: frame.presentationTime - startedAt)
+    }
+
+    private func pose(of key: TransformKey) -> TransformPose {
+        TransformPose(position: key.position, rotation: key.rotation, scale: key.scale)
+    }
+}
+
+/// Poses an entity from a track at a frame's presentation time: its position, rotation and
+/// scale. `startedAt` is the presentation time of the frame the track started in.
+public func applyTransformTrack(entityId: EntityID, track: TransformTrack, startedAt: TimeInterval, frame: FrameTime) {
+    guard let pose = track.pose(for: frame, startedAt: startedAt) else { return }
+    translateTo(entityId: entityId, position: pose.position)
+    rotateTo(entityId: entityId, rotation: pose.rotation)
+    scaleTo(entityId: entityId, scale: pose.scale)
+}

Sources/UntoldEngine/ECS/EngineExtensions.swift

@@ -17,19 +17,24 @@ public struct EngineExtensionUpdateContext: Sendable {
     public let frameIndex: UInt64
     public let currentEye: Int
     public let isPrimaryEye: Bool
+    /// The frame's predicted presentation time, its delta from the frame before and the
+    /// refreshes skipped between them; `.zero` for a context built outside a frame.
+    public let frameTime: FrameTime
 
     public init(
         viewport: SIMD2<Int>,
         immersionStyle: UntoldImmersionMode,
         frameIndex: UInt64,
         currentEye: Int,
-        isPrimaryEye: Bool
+        isPrimaryEye: Bool,
+        frameTime: FrameTime = .zero
     ) {
         self.viewport = viewport
         self.immersionStyle = immersionStyle
         self.frameIndex = frameIndex
         self.currentEye = currentEye
         self.isPrimaryEye = isPrimaryEye
+        self.frameTime = frameTime
     }
 }
 

Sources/UntoldEngine/Profiling/EngineStatsSnapshot.swift

@@ -33,6 +33,12 @@ public struct EngineTimingStats {
     /// `encodeMs` is then the submission phase: the drawables' command buffers encoded and
     /// committed. The CPU's work in a frame is `updateMs` + `encodeMs`.
     public var xrWaitMs: Double = 0.0
+    /// Milliseconds from the previous frame's predicted presentation to this one's: the
+    /// display's period on a steady cadence, two periods after a skipped frame.
+    public var presentationDeltaMs: Double = 0.0
+    /// Refreshes between the previous frame and this one that showed the previous frame again
+    /// (`FrameTime.skippedFrames`).
+    public var skippedFrames: Int = 0
     public var streamingRegionMs: Double = 0.0
     public var geometryStreamingMs: Double = 0.0
     public var batchingTickMs: Double = 0.0
@@ -53,7 +59,9 @@ public struct EngineTimingStats {
         streamingRegionMs: Double = 0.0,
         geometryStreamingMs: Double = 0.0,
         batchingTickMs: Double = 0.0,
-        batchingRebuildMs: Double = 0.0
+        batchingRebuildMs: Double = 0.0,
+        presentationDeltaMs: Double = 0.0,
+        skippedFrames: Int = 0
     ) {
         self.frameTotalMs = frameTotalMs
         self.smoothedFrameMs = smoothedFrameMs
@@ -70,6 +78,8 @@ public struct EngineTimingStats {
         self.geometryStreamingMs = geometryStreamingMs
         self.batchingTickMs = batchingTickMs
         self.batchingRebuildMs = batchingRebuildMs
+        self.presentationDeltaMs = presentationDeltaMs
+        self.skippedFrames = skippedFrames
     }
 }
 

Sources/UntoldEngine/Renderer/FrameTime.swift

@@ -0,0 +1,127 @@
+//
+//  FrameTime.swift
+//  UntoldEngine
+//
+// Copyright (C) Untold Engine Studios
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+import Foundation
+
+/// When the frame being prepared is predicted to reach the display, and how far that is from
+/// the frame before.
+///
+/// A headset frame is drawn with the head pose predicted for its presentation instant, so an
+/// object posed for `presentationTime` keeps its place against the world while the head moves,
+/// and one moving along a path steps by the display's period from one frame to the next
+/// instead of by the varying time between two game updates. A frame the display skipped lands
+/// where the timeline says, and the frames after it step by one period again.
+///
+/// `presentationTime` is in the `CACurrentMediaTime()` base, which `ProcessInfo.systemUptime`
+/// and ARKit's anchor timestamps share, so a game may keep its own timelines in that base.
+public struct FrameTime: Sendable, Equatable {
+    /// Seconds at which the frame is predicted to be shown.
+    public let presentationTime: TimeInterval
+    /// Seconds from the previous frame's presentation to this one's: one period on a steady
+    /// cadence, more after a skipped frame, 0 on the first frame.
+    public let deltaTime: TimeInterval
+    /// The display's period as the loop knows it: the host's nominal frame interval, or the
+    /// shortest spacing between presentations seen so far (0 until one has been seen).
+    public let period: TimeInterval
+    /// Refreshes between the previous frame and this one at which the display showed the
+    /// previous frame again because no new one was ready. 0 on a steady cadence.
+    public let skippedFrames: Int
+    /// The engine's count of frames prepared, from 1.
+    public let frameIndex: UInt64
+
+    public init(
+        presentationTime: TimeInterval,
+        deltaTime: TimeInterval,
+        period: TimeInterval,
+        skippedFrames: Int,
+        frameIndex: UInt64
+    ) {
+        self.presentationTime = presentationTime
+        self.deltaTime = deltaTime
+        self.period = period
+        self.skippedFrames = skippedFrames
+        self.frameIndex = frameIndex
+    }
+
+    /// Whether the display repeated the previous frame before this one.
+    public var wasFrameSkipped: Bool { skippedFrames > 0 }
+
+    /// No frame yet: every field zero.
+    public static let zero = FrameTime(presentationTime: 0, deltaTime: 0, period: 0, skippedFrames: 0, frameIndex: 0)
+}
+
+/// Makes each frame's `FrameTime` from the presentation times a host loop predicts: the delta
+/// between consecutive presentations, the period (the host's nominal one, or the shortest
+/// spacing seen) and the refreshes skipped between two frames.
+public struct FrameClock: Sendable, Equatable {
+    /// The most recent frame, or nil before the first.
+    public private(set) var current: FrameTime?
+    /// The period the host declares (1 / the view's preferred frames per second), or nil to
+    /// learn it from the presentation times.
+    public var nominalPeriod: TimeInterval?
+    private var learnedPeriod: TimeInterval?
+    private var frameIndex: UInt64 = 0
+
+    /// A spacing shorter than this is two frames for one refresh, not a period.
+    static let shortestPeriod: TimeInterval = 1.0 / 250.0
+    /// A spacing longer than this is a stall, not a period.
+    static let longestPeriod: TimeInterval = 1.0 / 24.0
+
+    public init(nominalPeriod: TimeInterval? = nil) {
+        self.nominalPeriod = nominalPeriod
+    }
+
+    /// The display's period as the clock knows it, 0 before it knows one.
+    public var period: TimeInterval { nominalPeriod ?? learnedPeriod ?? 0 }
+
+    /// Advances to a frame predicted to be shown at `presentationTime` and returns its time.
+    @discardableResult
+    public mutating func advance(toPresentation presentationTime: TimeInterval) -> FrameTime {
+        frameIndex &+= 1
+        guard let previous = current else {
+            let frame = FrameTime(presentationTime: presentationTime, deltaTime: 0, period: period, skippedFrames: 0, frameIndex: frameIndex)
+            current = frame
+            return frame
+        }
+
+        // A presentation before the previous one is a clock going backwards: no time passes.
+        let delta = max(0, presentationTime - previous.presentationTime)
+        if nominalPeriod == nil, delta >= Self.shortestPeriod, delta <= Self.longestPeriod {
+            learnedPeriod = min(learnedPeriod ?? delta, delta)
+        }
+
+        let period = period
+        var skipped = 0
+        if period > 0, delta > 0 {
+            // The refreshes the delta spans, less the one every frame takes.
+            let refreshes = min(delta / period, 1_000_000).rounded()
+            skipped = max(0, Int(refreshes) - 1)
+        }
+
+        let frame = FrameTime(presentationTime: presentationTime, deltaTime: delta, period: period, skippedFrames: skipped, frameIndex: frameIndex)
+        current = frame
+        return frame
+    }
+
+    /// Advances to a frame the loop estimates `lead` ahead of `now`, for a host with no
+    /// predicted presentation time (an MTKView's draw): the delta is then the spacing of the
+    /// draws on the wall clock.
+    @discardableResult
+    public mutating func advance(now: TimeInterval, lead: TimeInterval) -> FrameTime {
+        advance(toPresentation: now + lead)
+    }
+
+    /// Forgets every frame: the next one is a first frame again.
+    public mutating func reset() {
+        current = nil
+        learnedPeriod = nil
+        frameIndex = 0
+    }
+}