diff --git a/electron/native/AudioSampleQueue.test.ts b/electron/native/AudioSampleQueue.test.ts new file mode 100644 index 000000000..420608afb --- /dev/null +++ b/electron/native/AudioSampleQueue.test.ts @@ -0,0 +1,192 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; + +// Compile production queue logic and use real AAC writers. No capture permissions or devices. +describe.skipIf(process.platform !== "darwin")("native audio continuity", () => { + let directory: string; + let results: Record; + let pressure: { + dropped: number; + peakBytes: number; + peakSeconds: number; + failedFinish: boolean; + terminalFailure: boolean; + }; + beforeAll(() => { + directory = mkdtempSync(join(tmpdir(), "recordly-audio-continuity-")); + const source = readFileSync( + new URL("./ScreenCaptureKitRecorder.swift", import.meta.url), + "utf8", + ); + const queue = source.slice( + source.indexOf("final class AudioSampleQueue"), + source.indexOf("final class ScreenCaptureRecorder"), + ); + writeFileSync( + join(directory, "main.swift"), + String.raw` +import Foundation +import AVFoundation +import CoreMedia +${queue} +func at(_ seconds: Double) -> CMTime { CMTime(seconds: seconds, preferredTimescale: 48_000) } +func tone(index: Int, rate: Double = 48_000, channels: AVAudioChannelCount = 2, interleaved: Bool = false) -> CMSampleBuffer { + let frames = Int(rate / 100) + let format = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: rate, channels: channels, interleaved: interleaved)! + let pcm = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(frames))! + pcm.frameLength = AVAudioFrameCount(frames) + let frequency: Float = index >= 70 ? 1500 : 500 + for channel in 0.., offset: Int, count: Int, frequency: Double) -> Double { + var real = 0.0, imaginary = 0.0 + for index in 0.. String { + let url = URL(fileURLWithPath: CommandLine.arguments[1]).appendingPathComponent(name + ".m4a") + let writer = try! AVAssetWriter(url: url, fileType: .m4a) + let input = AVAssetWriterInput(mediaType: .audio, outputSettings: [AVFormatIDKey:kAudioFormatMPEG4AAC, AVSampleRateKey:48_000, AVNumberOfChannelsKey:2, AVEncoderBitRateKey:160_000]) + input.expectsMediaDataInRealTime = true + writer.add(input) + precondition(writer.startWriting()) + writer.startSession(atSourceTime: .zero) + var blocked = false + let samples = AudioSampleQueue(isReady: { !blocked && input.isReadyForMoreMediaData }, canWrite: { writer.status == .writing }, append: { input.append($0) }) + for index in 0..<(name == "overflow" ? 400 : 100) { + if name == "gap" && index >= 40 && index < 60 { continue } + if name == "tail" && index >= 80 { continue } + if name == "leading" && index < 20 { continue } + blocked = (name == "stall" && index >= 40 && index < 60) || (name == "overflow" && index < 300) + if name == "overflow" && index >= 300 { try? await Task.sleep(nanoseconds: 1_000_000) } + let toneIndex = name == "overflow" ? (index >= 370 ? 70 : 0) : index + samples.enqueue(tone(index: toneIndex, rate: name == "mono" ? 24_000 : 48_000, channels: name == "mono" ? 1 : 2, interleaved: name == "interleaved"), at: at(Double(index) / 100 - ((name == "partialOverlap" || name == "interleaved") && index == 50 ? 0.005 : 0))) + if name == "overlap" && index == 50 { samples.enqueue(tone(index: index), at: at(0.50)) } + } + blocked = false + let error = await samples.finish(at: at(name == "overflow" ? 4 : 1)) + precondition(error == nil, name + ": " + String(describing: error)) + input.markAsFinished() + await writer.finishWriting() + precondition(writer.status == .completed, String(describing: writer.error)) + let file = try! AVAudioFile(forReading: url, commonFormat: .pcmFormatFloat32, interleaved: false) + let pcm = AVAudioPCMBuffer(pcmFormat: file.processingFormat, frameCapacity: AVAudioFrameCount(file.length))! + try! file.read(into: pcm) + let data = pcm.floatChannelData![0] + var marker = -1.0 + for offset in stride(from: 0, to: Int(pcm.frameLength) - 480, by: 480) { + if energy(data, offset: offset, count: 480, frequency: 1500) > 4 * energy(data, offset: offset, count: 480, frequency: 500) { + marker = Double(offset) / 48_000 + break + } + } + var gapPower = 0.0 + for index in 22_080..<26_400 { gapPower += Double(data[index] * data[index]) } + return "\"\(name)\":{\"frames\":\(pcm.frameLength),\"marker\":\(marker),\"silentGap\":\(gapPower / 4320 < 0.001)}" +} +let done = DispatchSemaphore(value: 0) +Task { + var entries: [String] = [] + for name in ["contiguous", "gap", "stall", "tail", "leading", "overlap", "partialOverlap", "interleaved", "mono", "overflow"] { entries.append(await run(name)) } + var blocked = true + var writable = true + let pressure = AudioSampleQueue(isReady: { !blocked }, canWrite: { writable }, append: { _ in true }) + for index in 0..<1000 { pressure.enqueue(tone(index: index), at: at(Double(index) / 100)) } + let failedFinish = await pressure.finish(at: at(10), timeout: 0) != nil + let terminal = AudioSampleQueue(isReady: { false }, canWrite: { writable }, append: { _ in true }) + terminal.enqueue(tone(index: 0), at: .zero) + writable = false + let terminalFailure = await terminal.finish(at: at(0.01), timeout: 0) != nil + var writes = 0 + let longGap = AudioSampleQueue(isReady: { true }, canWrite: { true }, append: { _ in writes += 1; return true }) + longGap.enqueue(tone(index: 0), at: at(10)) + precondition(writes <= 16) + let longGapError = await longGap.finish(at: at(10), timeout: 0) + precondition(longGapError == nil, "Progressing gap restoration must not time out") + precondition(longGap.endTime == at(10)) + var clipReady = false + var clippedFrames = 0 + let clipped = AudioSampleQueue(isReady: { clipReady }, canWrite: { true }, append: { clippedFrames += $0.numSamples; return true }) + for index in 0..<10 { clipped.enqueue(tone(index: index), at: at(Double(index) / 100)) } + clipReady = true + let clipError = await clipped.finish(at: at(0.055), timeout: 0) + precondition(clipError == nil && clippedFrames == 2640, "Queued audio must stop at the final media time") + print("{\"tracks\":{" + entries.joined(separator: ",") + "},\"pressure\":{\"dropped\":\(pressure.droppedBuffers),\"peakBytes\":\(pressure.peakQueuedBytes),\"peakSeconds\":\(pressure.peakQueuedDuration),\"failedFinish\":\(failedFinish),\"terminalFailure\":\(terminalFailure)}}") + done.signal() +} +done.wait() +`, + ); + const executable = join(directory, "probe"); + const build = spawnSync( + "swiftc", + [ + "-module-cache-path", + join(directory, "cache"), + join(directory, "main.swift"), + "-o", + executable, + ], + { encoding: "utf8", timeout: 60_000 }, + ); + expect(build.status, build.stderr).toBe(0); + const run = spawnSync(executable, [directory], { encoding: "utf8", timeout: 30_000 }); + expect(run.status, run.stderr).toBe(0); + ({ tracks: results, pressure } = JSON.parse(run.stdout)); + }, 90_000); + afterAll(() => { + if (directory) rmSync(directory, { recursive: true, force: true }); + }); + + it.each([ + "contiguous", + "gap", + "stall", + "tail", + "leading", + "overlap", + "partialOverlap", + "interleaved", + "mono", + ])("keeps %s audio one second long and its later tone aligned", (name) => { + expect(results[name].frames / 48_000).toBeCloseTo(1, 1); + expect(results[name].marker).toBeCloseTo(0.7, 1); + expect(results[name].marker - results.contiguous.marker).toBeCloseTo(0, 2); + }); + it("keeps later speech aligned after sustained overload exceeds the queue limit", () => { + expect(results.overflow.frames / 48_000).toBeCloseTo(4, 1); + expect(results.overflow.marker).toBeCloseTo(3.7, 1); + expect(results.overflow.marker - results.contiguous.marker).toBeCloseTo(3, 2); + }); + it("preserves an actual missing interval as silence", () => { + expect(results.gap.silentGap).toBe(true); + }); + it("retains speech across temporary backpressure", () => { + expect(results.stall.silentGap).toBe(false); + }); + it("bounds retained audio under sustained backpressure", () => { + expect(pressure.dropped).toBeGreaterThan(0); + expect(pressure.peakBytes).toBeLessThanOrEqual(4 * 1024 * 1024); + expect(pressure.peakSeconds).toBeLessThanOrEqual(2); + }); + it("fails finalization when the encoder remains blocked or stops", () => { + expect(pressure.failedFinish).toBe(true); + expect(pressure.terminalFailure).toBe(true); + }); +}); diff --git a/electron/native/ScreenCaptureKitRecorder.swift b/electron/native/ScreenCaptureKitRecorder.swift index a9da8f91b..52f9b20ac 100644 --- a/electron/native/ScreenCaptureKitRecorder.swift +++ b/electron/native/ScreenCaptureKitRecorder.swift @@ -29,6 +29,206 @@ struct CaptureConfig: Codable { let targetCaptureFPS = 60 let maxInlineAudioTailExtension = CMTime(seconds: 2.0, preferredTimescale: 600) + +/// Keeps short encoder stalls from losing speech. If the bounded queue fills, later +/// timestamps restore the missing interval as silence rather than compacting AAC audio. +/// Accessed only on the recorder queue, or after capture has stopped during finish. +final class AudioSampleQueue { + private struct Pending { + let buffer: CMSampleBuffer + let time: CMTime + let duration: Double + let bytes: Int + } + private let isReady: () -> Bool + private let canWrite: () -> Bool + private let appendSample: (CMSampleBuffer) -> Bool + private var pending: [Pending] = [] + private var queuedDuration = 0.0 + private var queuedBytes = 0 + private var lastFormat: CMFormatDescription? + private var failure: Error? + private(set) var endTime = CMTime.zero + private(set) var droppedBuffers = 0 + private(set) var silenceFrames: Int64 = 0 + private(set) var peakQueuedBytes = 0 + private(set) var peakQueuedDuration = 0.0 + + init(isReady: @escaping () -> Bool, canWrite: @escaping () -> Bool, append: @escaping (CMSampleBuffer) -> Bool) { + self.isReady = isReady + self.canWrite = canWrite + self.appendSample = append + } + + func enqueue(_ buffer: CMSampleBuffer, at time: CMTime) { + drain() + guard failure == nil, + let format = buffer.formatDescription, + let asbd = format.audioStreamBasicDescription, + asbd.mFormatID == kAudioFormatLinearPCM, + asbd.mSampleRate.isFinite, asbd.mSampleRate > 0, asbd.mSampleRate <= Double(Int32.max), + time.isValid, time.seconds.isFinite else { + failure = failure ?? audioError("Invalid PCM audio buffer") + return + } + lastFormat = format + let duration = Double(buffer.numSamples) / asbd.mSampleRate + let bytes = buffer.dataBuffer?.dataLength ?? buffer.totalSampleSize + // Bound both media time and retained memory, even with unusual input formats. + guard pending.count < 256, queuedDuration + duration <= 2.0, + bytes <= 4 * 1024 * 1024 - queuedBytes else { + droppedBuffers += 1 + return + } + pending.append(Pending(buffer: buffer, time: time, duration: duration, bytes: bytes)) + queuedDuration += duration + queuedBytes += bytes + peakQueuedBytes = max(peakQueuedBytes, queuedBytes) + peakQueuedDuration = max(peakQueuedDuration, queuedDuration) + drain() + } + + /// Pads the end too: a final dropped buffer has no later timestamp to reveal its gap. + func finish(at targetTime: CMTime, timeout: TimeInterval = 5.0) async -> Error? { + var deadline = ProcessInfo.processInfo.systemUptime + timeout + while failure == nil { + if Task.isCancelled { failure = CancellationError(); break } + let previousEnd = endTime + drain(until: targetTime) + if pending.isEmpty { + if let format = lastFormat { + fillSilence(to: targetTime, format: format) + } + if failure != nil || lastFormat == nil || endTime >= targetTime { + break + } + } + if endTime > previousEnd { + // Timeout means stalled, not successfully restoring a long quiet interval. + deadline = ProcessInfo.processInfo.systemUptime + timeout + await Task.yield() + continue + } + if ProcessInfo.processInfo.systemUptime >= deadline { + failure = audioError("Audio encoder did not drain before finalization") + break + } + do { try await Task.sleep(nanoseconds: 10_000_000) } + catch { failure = error } + } + pending.removeAll() + queuedDuration = 0 + queuedBytes = 0 + return failure + } + + private func drain(until limit: CMTime? = nil) { + guard failure == nil else { return } + guard canWrite() else { + failure = audioError("Audio writer stopped accepting data") + pending.removeAll() + queuedDuration = 0 + queuedBytes = 0 + return + } + while let next = pending.first, failure == nil { + if let limit, next.time >= limit || endTime >= limit { + removeFirst() + continue + } + guard isReady() else { return } + guard let format = next.buffer.formatDescription, + let asbd = format.audioStreamBasicDescription else { + failure = audioError("Missing audio format") + return + } + let rate = CMTimeScale(asbd.mSampleRate) + guard fillSilence(to: next.time, format: format) else { return } + guard failure == nil, isReady() else { return } + // Trim overlapping or repeated samples instead of writing time twice. + let overlap = Int(min(Double(next.buffer.numSamples), max(0, ((endTime - next.time).seconds * asbd.mSampleRate).rounded()))) + let skip = overlap <= 1 ? 0 : overlap // CMTime(seconds:) can round adjacent timestamps by one frame. + let remainingFrames = next.buffer.numSamples - skip + let count = limit.map { Int(min(Double(remainingFrames), max(0, (($0 - endTime).seconds * asbd.mSampleRate).rounded()))) } ?? remainingFrames + if count > 0 { + do { + let buffer: CMSampleBuffer + if skip > 0 || count < next.buffer.numSamples { + let pcmFormat = AVAudioFormat(cmAudioFormatDescription: format) + guard let pcm = AVAudioPCMBuffer(pcmFormat: pcmFormat, frameCapacity: AVAudioFrameCount(count)) else { + throw audioError("Cannot allocate trimmed audio") + } + pcm.frameLength = AVAudioFrameCount(count) + guard CMSampleBufferCopyPCMDataIntoAudioBufferList(next.buffer, at: Int32(skip), frameCount: Int32(count), into: pcm.mutableAudioBufferList) == noErr, + let slice = makeSample(pcm, format: format, at: endTime) else { throw audioError("Cannot trim overlapping audio") } + buffer = slice + } else { buffer = next.buffer } + let timing = CMSampleTimingInfo(duration: CMTime(value: 1, timescale: rate), presentationTimeStamp: endTime, decodeTimeStamp: .invalid) + let retimed = try CMSampleBuffer(copying: buffer, withNewTiming: [timing]) + guard appendSample(retimed) else { throw audioError("Cannot append audio") } + endTime = endTime + CMTime(value: Int64(buffer.numSamples), timescale: rate) + } catch { failure = error; return } + } + removeFirst() + } + } + + private func removeFirst() { + let next = pending.removeFirst() + queuedDuration = max(0, queuedDuration - next.duration) + queuedBytes -= next.bytes + } + + @discardableResult + private func fillSilence(to target: CMTime, format: CMFormatDescription) -> Bool { + guard let asbd = format.audioStreamBasicDescription else { return false } + let rate = CMTimeScale(asbd.mSampleRate) + // Bound work as well as memory. Long gaps resume on the next callback or finish poll. + var chunksRemaining = 16 + while isReady(), failure == nil, chunksRemaining > 0 { + chunksRemaining -= 1 + let frames = Int64(min(4096, max(0, ((target - endTime).seconds * asbd.mSampleRate).rounded()))) + guard frames > 0 else { + // Avoid sub-sample rounding leaving finish in a retry loop. + if target > endTime { endTime = target } + return true + } + let pcmFormat = AVAudioFormat(cmAudioFormatDescription: format) + guard let pcm = AVAudioPCMBuffer(pcmFormat: pcmFormat, frameCapacity: AVAudioFrameCount(frames)) else { + failure = audioError("Cannot allocate audio silence") + return false + } + pcm.frameLength = AVAudioFrameCount(frames) + for buffer in UnsafeMutableAudioBufferListPointer(pcm.mutableAudioBufferList) { + buffer.mData?.initializeMemory(as: UInt8.self, repeating: 0, count: Int(buffer.mDataByteSize)) + } + guard let silence = makeSample(pcm, format: format, at: endTime), appendSample(silence) else { + failure = audioError("Cannot write audio silence") + return false + } + endTime = endTime + CMTime(value: frames, timescale: rate) + silenceFrames += frames + } + return endTime >= target + } + + private func makeSample(_ pcm: AVAudioPCMBuffer, format: CMFormatDescription, at time: CMTime) -> CMSampleBuffer? { + var timing = CMSampleTimingInfo(duration: CMTime(value: 1, timescale: CMTimeScale(pcm.format.sampleRate)), presentationTimeStamp: time, decodeTimeStamp: .invalid) + var sample: CMSampleBuffer? + guard CMSampleBufferCreate(allocator: kCFAllocatorDefault, dataBuffer: nil, + dataReady: false, makeDataReadyCallback: nil, refcon: nil, formatDescription: format, + sampleCount: Int(pcm.frameLength), sampleTimingEntryCount: 1, sampleTimingArray: &timing, + sampleSizeEntryCount: 0, sampleSizeArray: nil, sampleBufferOut: &sample) == noErr, + let sample, + CMSampleBufferSetDataBufferFromAudioBufferList(sample, blockBufferAllocator: nil, + blockBufferMemoryAllocator: nil, flags: 0, bufferList: pcm.audioBufferList) == noErr else { return nil } + return sample + } + + private func audioError(_ message: String) -> Error { + NSError(domain: "RecordlyCapture", code: 17, userInfo: [NSLocalizedDescriptionKey: message]) + } +} /// How long finalization waits for a backed-up encoder queue before giving up on /// the optional tail frame: 100 polls x 10 ms = 1 s. let writerReadinessPollAttempts = 100 @@ -80,6 +280,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { private var finalizationWaiters: [CheckedContinuation] = [] private var inlineAudioInput: AVAssetWriterInput? private var firstInlineAudioSampleTime: CMTime? + private var audioSampleQueues: [ObjectIdentifier: AudioSampleQueue] = [:] private var capturesSystemAudio = false private var capturesMicrophone = false private var writesSystemAudioToSeparateTrack = false @@ -476,7 +677,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { guard let systemAudioInput else { return } appendAudioSampleBuffer(sampleBuffer, to: systemAudioInput, of: systemAudioWriter, firstSampleTime: &firstSystemAudioSampleTime, lastPresentationTime: &lastSystemAudioPresentationTime, presentationTime: presentationTime) // Also write system audio to the inline video track - if let inlineAudioInput, inlineAudioInput.isReadyForMoreMediaData { + if let inlineAudioInput { appendAudioSampleBuffer(sampleBuffer, to: inlineAudioInput, of: assetWriter, firstSampleTime: &firstInlineAudioSampleTime, lastPresentationTime: &lastInlineAudioPresentationTime, presentationTime: presentationTime) } return @@ -487,7 +688,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { appendAudioSampleBuffer(sampleBuffer, to: microphoneOnlyInput, of: microphoneOnlyWriter, firstSampleTime: &firstMicrophoneSampleTime, lastPresentationTime: &lastMicrophonePresentationTime, presentationTime: presentationTime) } // Write mic to inline video track only if there's no system audio (avoids double-writing) - if !capturesSystemAudio, let inlineAudioInput, inlineAudioInput.isReadyForMoreMediaData { + if !capturesSystemAudio, let inlineAudioInput { appendAudioSampleBuffer(sampleBuffer, to: inlineAudioInput, of: assetWriter, firstSampleTime: &firstInlineAudioSampleTime, lastPresentationTime: &lastInlineAudioPresentationTime, presentationTime: presentationTime) } return @@ -637,6 +838,15 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { // example a full disk), which would abort the helper the same way. let videoEndTime = lastVideoPresentationTime + (lastSampleBuffer.map { frameDuration(for: $0) } ?? .zero) let endTime = resolvedCaptureEndTime(videoEndTime: videoEndTime) + var audioFailure: Error? + for samples in audioSampleQueues.values { + if let error = await samples.finish(at: endTime) { audioFailure = audioFailure ?? error } + if samples.droppedBuffers > 0 || samples.silenceFrames > 0 { + fputs("AUDIO_GAPS: droppedBuffers=\(samples.droppedBuffers) silenceFramesInserted=\(samples.silenceFrames)\n", stderr) + fflush(stderr) + } + } + audioSampleQueues.removeAll() if let assetWriter, assetWriter.status == .writing { assetWriter.endSession(atSourceTime: endTime) videoInput?.markAsFinished() @@ -661,7 +871,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { ? nil : (writer.error ?? unfinalizedWriterError(status: writer.status)) } - .first + .first ?? audioFailure let path = outputURL?.path ?? "" assetWriter = nil videoInput = nil @@ -822,27 +1032,27 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { } private func appendAudioSampleBuffer(_ sampleBuffer: CMSampleBuffer, to input: AVAssetWriterInput, of writer: AVAssetWriter?, firstSampleTime: inout CMTime?, lastPresentationTime: inout CMTime, presentationTime: CMTime) { - // A writer that failed mid-capture (a full disk, say) raises on every - // further append, which would abort the helper and lose the whole file. - guard writer?.status == .writing, input.isReadyForMoreMediaData else { return } + guard let writer, writer.status == .writing else { return } guard !lastPresentationTime.isValid || CMTimeCompare(presentationTime, lastPresentationTime) > 0 else { return } - - if firstSampleTime == nil { - firstSampleTime = presentationTime - } - - // presentationTime is already relative to the video's first frame - // (computed by adjustedPresentationTime), so use it directly. - let timing = CMSampleTimingInfo(duration: sampleBuffer.duration, presentationTimeStamp: presentationTime, decodeTimeStamp: sampleBuffer.decodeTimeStamp) - if let retimedSampleBuffer = try? CMSampleBuffer(copying: sampleBuffer, withNewTiming: [timing]) { - let appended = input.append(retimedSampleBuffer) - if appended { - lastPresentationTime = presentationTime - if input === inlineAudioInput { - lastInlineAudioDuration = sampleBuffer.duration - } - } + let key = ObjectIdentifier(input) + let samples: AudioSampleQueue + if let existing = audioSampleQueues[key] { + samples = existing + } else { + samples = AudioSampleQueue( + isReady: { [weak writer] in writer?.status == .writing && input.isReadyForMoreMediaData }, + canWrite: { [weak writer] in writer?.status == .writing }, + append: { [weak writer] sample in + guard writer?.status == .writing, input.isReadyForMoreMediaData else { return false } + return input.append(sample) + }) + audioSampleQueues[key] = samples } + samples.enqueue(sampleBuffer, at: presentationTime) + if firstSampleTime == nil { firstSampleTime = presentationTime } + // Track offered time, including queued samples, when calculating the MP4 tail. + lastPresentationTime = presentationTime + if input === inlineAudioInput { lastInlineAudioDuration = sampleBuffer.duration } } private static func audioOutputSettings(bitRate: Int) -> [String: Any] {