diff --git a/android/src/main/java/com/oney/WebRTCModule/FanoutCapturerObserver.java b/android/src/main/java/com/oney/WebRTCModule/FanoutCapturerObserver.java new file mode 100644 index 000000000..c73f415b7 --- /dev/null +++ b/android/src/main/java/com/oney/WebRTCModule/FanoutCapturerObserver.java @@ -0,0 +1,85 @@ +package com.oney.WebRTCModule; + +import androidx.annotation.Nullable; + +import org.webrtc.CapturerObserver; +import org.webrtc.VideoFrame; +import org.webrtc.VideoSink; + +/** + * A {@link CapturerObserver} that fans captured frames out to two consumers: + * + *
    + *
  1. a {@link VideoSink} renderer — the lobby camera preview, and
    2. + *
  2. an optional downstream {@link CapturerObserver} — the per-call {@code VideoSource}'s + * observer, attached at join.
    3. + *
+ * + *

Android's {@code VideoCapturer.initialize(...)} binds the capturer to a single observer for the + * capturer's lifetime. Installing this fan-out as that observer lets one running camera session be + * reused across the lobby -> join hand-off: the preview renders from the start, and the per-call + * {@code VideoSource}'s observer is attached as {@code downstream} at join by flipping the pointer. + * The camera never closes; frames simply start flowing to the WebRTC track in addition to the + * preview. + * + *

All mutable fields are {@code volatile}: {@link #onFrameCaptured} runs on the capturer's frame + * thread, while {@link #setDownstream}/{@link #setRenderer} are called from the getUserMedia worker + * and the UI thread respectively. + */ +class FanoutCapturerObserver implements CapturerObserver { + @Nullable + private volatile VideoSink renderer; + @Nullable + private volatile CapturerObserver downstream; + private volatile boolean started; + + FanoutCapturerObserver(@Nullable VideoSink renderer) { + this.renderer = renderer; + } + + void setRenderer(@Nullable VideoSink renderer) { + this.renderer = renderer; + } + + /** + * Attaches (or clears) the downstream observer — the per-call {@code VideoSource}'s observer. If + * the capturer is already running, the downstream missed the original {@code onCapturerStarted}, + * so replay it before frames begin flowing. + */ + void setDownstream(@Nullable CapturerObserver downstream) { + if (downstream != null && started) { + downstream.onCapturerStarted(true); + } + this.downstream = downstream; + } + + @Override + public void onCapturerStarted(boolean success) { + started = success; + CapturerObserver d = downstream; + if (d != null) { + d.onCapturerStarted(success); + } + } + + @Override + public void onCapturerStopped() { + started = false; + CapturerObserver d = downstream; + if (d != null) { + d.onCapturerStopped(); + } + } + + @Override + public void onFrameCaptured(VideoFrame frame) { + VideoSink r = renderer; + if (r != null) { + r.onFrame(frame); + } + CapturerObserver d = downstream; + if (d != null) { + d.onFrameCaptured(frame); + } + } +} diff --git a/android/src/main/java/com/oney/WebRTCModule/GetUserMediaImpl.java b/android/src/main/java/com/oney/WebRTCModule/GetUserMediaImpl.java index b9c3e5fe3..b6985f7d8 100644 --- a/android/src/main/java/com/oney/WebRTCModule/GetUserMediaImpl.java +++ b/android/src/main/java/com/oney/WebRTCModule/GetUserMediaImpl.java @@ -64,6 +64,9 @@ public class GetUserMediaImpl { private final WebRTCModule webRTCModule; private Promise displayMediaPromise; + // The factory the in-flight getDisplayMedia call resolved to; used later by the async + // createScreenStream() (invoked from the MediaProjection service connection). + private PeerConnectionFactoryProvider displayMediaFactory; private Intent mediaProjectionPermissionResultData; private boolean createConfigForDefaultDisplay = false; private float resolutionScale = 1.0f; @@ -103,6 +106,7 @@ public void onActivityResult(Activity activity, int requestCode, int resultCode, if (resultCode != Activity.RESULT_OK) { displayMediaPromise.reject("DOMException", "NotAllowedError"); displayMediaPromise = null; + displayMediaFactory = null; return; } @@ -114,13 +118,13 @@ public void onActivityResult(Activity activity, int requestCode, int resultCode, }); } - private AudioTrack createAudioTrack(ReadableMap constraints) { + private AudioTrack createAudioTrack(ReadableMap constraints, PeerConnectionFactoryProvider factoryProvider) { ReadableMap audioConstraintsMap = constraints.getMap("audio"); Log.d(TAG, "getUserMedia(audio): " + audioConstraintsMap); String id = UUID.randomUUID().toString(); - PeerConnectionFactory pcFactory = webRTCModule.mFactory; + PeerConnectionFactory pcFactory = factoryProvider.factory; MediaConstraints peerConstraints = webRTCModule.constraintsForOptions(audioConstraintsMap); // Convert given constraints into the internal webrtc media constraints. @@ -150,6 +154,7 @@ private AudioTrack createAudioTrack(ReadableMap constraints) { // surfaceTextureHelper is initialized for videoTrack only, so its null here. tracks.put(id, new TrackPrivate(track, audioSource, /* videoCapturer */ null, /* surfaceTextureHelper */ null)); + factoryProvider.ownedTrackIds.add(id); return track; } @@ -229,12 +234,16 @@ MediaStreamTrack getTrack(String id) { * if audio permission was not granted, there will be no "audio" key in * the constraints map. */ - void getUserMedia(final ReadableMap constraints, final Callback successCallback, final Callback errorCallback) { + void getUserMedia( + final ReadableMap constraints, + final Callback successCallback, + final Callback errorCallback) { + final PeerConnectionFactoryProvider factoryProvider = webRTCModule.factoryRegistry.getOrCreateDefault(); AudioTrack audioTrack = null; VideoTrack videoTrack = null; if (constraints.hasKey("audio")) { - audioTrack = createAudioTrack(constraints); + audioTrack = createAudioTrack(constraints, factoryProvider); } if (constraints.hasKey("video")) { @@ -251,10 +260,20 @@ void getUserMedia(final ReadableMap constraints, final Callback successCallback, return; } - CameraCaptureController cameraCaptureController = - new CameraCaptureController(currentActivity, getCameraEnumerator(), videoConstraintsMap); + // If a lobby camera preview is already running, adopt its camera session by routing its + // frames into this track's source; the camera keeps running. Falls back to creating a + // fresh capturer when there's no preview to adopt. + RTCCameraPreviewView preview = webRTCModule.getActiveCameraPreview(); + if (preview != null) { + videoTrack = createVideoTrackFromPreview(preview, videoConstraintsMap, factoryProvider); + } + + if (videoTrack == null) { + CameraCaptureController cameraCaptureController = + new CameraCaptureController(currentActivity, getCameraEnumerator(), videoConstraintsMap); - videoTrack = createVideoTrack(cameraCaptureController); + videoTrack = createVideoTrack(cameraCaptureController, factoryProvider); + } } if (audioTrack == null && videoTrack == null) { @@ -264,7 +283,7 @@ void getUserMedia(final ReadableMap constraints, final Callback successCallback, return; } - createStream(new MediaStreamTrack[] {audioTrack, videoTrack}, (streamId, tracksInfo) -> { + createStream(new MediaStreamTrack[]{audioTrack, videoTrack}, factoryProvider, (streamId, tracksInfo) -> { WritableArray tracksInfoWritableArray = Arguments.createArray(); for (WritableMap trackInfo : tracksInfo) { @@ -292,6 +311,7 @@ void disposeAllTracks() { for (Map.Entry entry : tracks.entrySet()) { try { entry.getValue().dispose(); + webRTCModule.factoryRegistry.forgetTrack(entry.getKey()); } catch (Exception e) { Log.w(TAG, "disposeAllTracks: error disposing " + entry.getKey(), e); } @@ -303,6 +323,7 @@ void disposeTrack(String id) { TrackPrivate track = tracks.remove(id); if (track != null) { track.dispose(); + webRTCModule.factoryRegistry.forgetTrack(id); } } @@ -375,6 +396,7 @@ void getDisplayMedia(final ReadableMap constraints, Promise promise) { this.initializeConstraints(constraints); this.displayMediaPromise = promise; + this.displayMediaFactory = webRTCModule.factoryRegistry.getOrCreateDefault(); MediaProjectionManager mediaProjectionManager = (MediaProjectionManager) currentActivity.getApplication().getSystemService( @@ -401,24 +423,27 @@ public void run() { } else { promise.reject(new RuntimeException("MediaProjectionManager is null.")); + displayMediaPromise = null; + displayMediaFactory = null; } } private void createScreenStream() { - // Guards against onServiceConnected firing after invalidate() has disposed and nulled mFactory. - if (webRTCModule.mFactory == null) { + final PeerConnectionFactoryProvider factoryProvider = displayMediaFactory; + if (factoryProvider == null || factoryProvider.isDisposed()) { if (displayMediaPromise != null) { displayMediaPromise.reject("ERR_MODULE_DISPOSED", "WebRTCModule disposed during getDisplayMedia"); displayMediaPromise = null; } + displayMediaFactory = null; return; } - VideoTrack track = createScreenTrack(); + VideoTrack track = createScreenTrack(factoryProvider); if (track == null) { displayMediaPromise.reject(new RuntimeException("ScreenTrack is null.")); } else { - createStream(new MediaStreamTrack[] {track}, (streamId, tracksInfo) -> { + createStream(new MediaStreamTrack[]{track}, factoryProvider, (streamId, tracksInfo) -> { WritableMap data = Arguments.createMap(); data.putString("streamId", streamId); @@ -437,11 +462,15 @@ private void createScreenStream() { // It is retained so it can be reused to create a MediaProjection for // screen share audio capture (AudioPlaybackCaptureConfiguration). displayMediaPromise = null; + displayMediaFactory = null; } - void createStream(MediaStreamTrack[] tracks, BiConsumer> successCallback) { + void createStream( + MediaStreamTrack[] tracks, + PeerConnectionFactoryProvider factoryProvider, + BiConsumer> successCallback) { String streamId = UUID.randomUUID().toString(); - MediaStream mediaStream = webRTCModule.mFactory.createLocalMediaStream(streamId); + MediaStream mediaStream = factoryProvider.factory.createLocalMediaStream(streamId); ArrayList tracksInfo = new ArrayList<>(); @@ -487,16 +516,18 @@ void createStream(MediaStreamTrack[] tracks, BiConsumerThe video encoder/decoder factories are created once by {@link WebRTCModule} and shared across + * all factories (passed in via {@link BuildOptions}). The ADM's sample-delivery callbacks + * (screen-audio mixing, speech-activity detection, playback fan-out) are also supplied by the module + * so that wiring stays in one place; this class only owns the factory + ADM lifecycle and the + * profile-driven ADM build settings. + */ +class PeerConnectionFactoryProvider { + private static final String TAG = "PeerConnectionFactoryProvider"; + + static final class BuildOptions { + @NonNull + Context context; + @NonNull + VideoEncoderFactory videoEncoderFactory; + @NonNull + VideoDecoderFactory videoDecoderFactory; + @Nullable + AudioProcessingFactory audioProcessingFactory; + /** A pre-built ADM injected via {@link WebRTCModuleOptions#audioDeviceModule}. */ + @Nullable + AudioDeviceModule injectedAudioDeviceModule; + /** The music / high-quality audio profile: disables HW AEC/NS, uses the raw mic source. */ + boolean bypassVoiceProcessing; + /** Stereo microphone capture. Only takes effect under {@link #bypassVoiceProcessing}. */ + boolean stereoInputEnabled; + /** Receives speaking-while-muted events; the module backs this with the RN event emitter. */ + @Nullable + SpeechActivityDetector.Listener speechActivityListener; + } + + @NonNull + final String id; + @NonNull + final PeerConnectionFactory factory; + @NonNull + final AudioDeviceModule adm; + final boolean bypassVoiceProcessing; + + final Set ownedPcIds = ConcurrentHashMap.newKeySet(); + final Set ownedTrackIds = ConcurrentHashMap.newKeySet(); + + private volatile boolean disposed = false; + + private PeerConnectionFactoryProvider(@NonNull String id, @NonNull PeerConnectionFactory factory, + @NonNull AudioDeviceModule adm, boolean bypassVoiceProcessing) { + this.id = id; + this.factory = factory; + this.adm = adm; + this.bypassVoiceProcessing = bypassVoiceProcessing; + } + + @NonNull + static PeerConnectionFactoryProvider build(@NonNull String id, @NonNull BuildOptions options) { + Log.d(TAG, "build() id=" + id + " bypassVoiceProcessing=" + options.bypassVoiceProcessing + + " stereoInputEnabled=" + options.stereoInputEnabled); + + AudioDeviceModule adm = options.injectedAudioDeviceModule != null + ? options.injectedAudioDeviceModule + : buildAudioDeviceModule(options); + + PeerConnectionFactory.Builder pcFactoryBuilder = PeerConnectionFactory.builder() + .setAudioDeviceModule(adm) + .setVideoEncoderFactory(options.videoEncoderFactory) + .setVideoDecoderFactory(options.videoDecoderFactory); + + if (options.audioProcessingFactory != null) { + pcFactoryBuilder.setAudioProcessingFactory(options.audioProcessingFactory); + } + + PeerConnectionFactory factory = pcFactoryBuilder.createPeerConnectionFactory(); + + return new PeerConnectionFactoryProvider(id, factory, adm, options.bypassVoiceProcessing); + } + + @NonNull + private static JavaAudioDeviceModule buildAudioDeviceModule(@NonNull BuildOptions options) { + JavaAudioDeviceModule.Builder builder = JavaAudioDeviceModule.builder(options.context); + + if (options.bypassVoiceProcessing) { + // Music / high-quality profile: bypass the platform voice pipeline so the raw, + // native-rate signal reaches WebRTC unmodified. Stereo input is opt-in (it requires the + // bypassed pipeline, since the platform voice path is mono). + builder.setUseHardwareAcousticEchoCanceler(false) + .setUseHardwareNoiseSuppressor(false) + .setUseStereoInput(options.stereoInputEnabled) + .setUseStereoOutput(true) + .setAudioSource(MediaRecorder.AudioSource.MIC) + .setOutputSampleRate(nativeOutputSampleRate(options.context)); + } else { + // Default voice profile: hardware AEC/NS where the platform supports it. + builder.setUseHardwareAcousticEchoCanceler(Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) + .setUseHardwareNoiseSuppressor(Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) + .setUseStereoOutput(true); + } + + final SpeechActivityDetector speechDetector = options.speechActivityListener != null + ? new SpeechActivityDetector(options.speechActivityListener) + : null; + + // Speaking-while-muted detection + screen-audio mixing, run on every captured mic buffer. + builder.setAudioBufferCallback( + (audioBuffer, audioFormat, channelCount, sampleRate, bytesRead, captureTimeNs) -> { + // 1. Speech activity detection on raw mic data, BEFORE any mutation. + if (speechDetector != null) { + speechDetector.processBuffer(audioBuffer, bytesRead); + } + // 2. Screen-audio mixing — mutates audioBuffer in place. + if (bytesRead > 0) { + WebRTCModuleOptions.ScreenAudioBytesProvider provider = + WebRTCModuleOptions.getInstance().screenAudioBytesProvider; + if (provider != null) { + ByteBuffer screenBuffer = provider.getScreenAudioBytes(bytesRead); + if (screenBuffer != null && screenBuffer.remaining() > 0) { + mixScreenAudioIntoBuffer(audioBuffer, screenBuffer, bytesRead); + } + } + } + return captureTimeNs; + }); + + if (speechDetector != null) { + builder.setAudioRecordStateCallback(new JavaAudioDeviceModule.AudioRecordStateCallback() { + @Override + public void onWebRtcAudioRecordStart() { + speechDetector.reset(); + } + + @Override + public void onWebRtcAudioRecordStop() { + speechDetector.onRecordStop(); + } + }); + } + + builder.setPlaybackSamplesReadyCallback(samples -> { + // Fan-out to every registered consumer. The list is a CopyOnWriteArrayList so iteration is + // safe even if a consumer registers/unregisters mid-call. + for (JavaAudioDeviceModule.PlaybackSamplesReadyCallback obs : + WebRTCModuleOptions.getInstance().getPlaybackSamplesObservers()) { + try { + obs.onWebRtcAudioTrackSamplesReady(samples); + } catch (Throwable t) { + // Audio device module thread must not throw. + Log.w(TAG, "playback samples observer threw", t); + } + } + }); + + return builder.createAudioDeviceModule(); + } + + /** + * Mixes screen audio into the microphone buffer using PCM 16-bit additive mixing with clamping. + * Handles different buffer sizes safely: each buffer is read only within its own bounds. When one + * buffer is shorter, the other's samples pass through unmodified (mic samples stay as-is, or + * screen-only samples are written). + */ + private static void mixScreenAudioIntoBuffer(ByteBuffer micBuffer, ByteBuffer screenBuffer, int bytesRead) { + micBuffer.position(0); + screenBuffer.position(0); + + micBuffer.order(ByteOrder.LITTLE_ENDIAN); + screenBuffer.order(ByteOrder.LITTLE_ENDIAN); + + ShortBuffer micShorts = micBuffer.asShortBuffer(); + ShortBuffer screenShorts = screenBuffer.asShortBuffer(); + + int micSamples = Math.min(bytesRead / 2, micShorts.remaining()); + int screenSamples = screenShorts.remaining(); + int totalSamples = Math.max(micSamples, screenSamples); + + for (int i = 0; i < totalSamples; i++) { + int sum; + if (i >= micSamples) { + // Screen-only: mic buffer is shorter — write screen sample directly + sum = screenShorts.get(i); + } else if (i >= screenSamples) { + // Mic-only: screen buffer is shorter — keep mic sample as-is + break; + } else { + // Both buffers have data — add samples + sum = micShorts.get(i) + screenShorts.get(i); + } + if (sum > Short.MAX_VALUE) sum = Short.MAX_VALUE; + if (sum < Short.MIN_VALUE) sum = Short.MIN_VALUE; + micShorts.put(i, (short) sum); + } + } + + private static int nativeOutputSampleRate(@NonNull Context context) { + AudioManager am = (AudioManager) context.getSystemService(Context.AUDIO_SERVICE); + if (am != null) { + String rate = am.getProperty(AudioManager.PROPERTY_OUTPUT_SAMPLE_RATE); + if (rate != null) { + try { + return Integer.parseInt(rate); + } catch (NumberFormatException e) { + Log.w(TAG, "failed to parse native output sample rate, using 48000: " + e.getMessage()); + } + } + } + return 48000; + } + + void dispose() { + if (disposed) { + return; + } + disposed = true; + + try { + factory.dispose(); + } catch (Throwable t) { + Log.w(TAG, "dispose(): factory.dispose() failed", t); + } + + try { + adm.release(); + } catch (Throwable t) { + Log.w(TAG, "dispose(): adm.release() failed", t); + } + } + + boolean isDisposed() { + return disposed; + } +} diff --git a/android/src/main/java/com/oney/WebRTCModule/PeerConnectionFactoryRegistry.java b/android/src/main/java/com/oney/WebRTCModule/PeerConnectionFactoryRegistry.java new file mode 100644 index 000000000..e72fbd924 --- /dev/null +++ b/android/src/main/java/com/oney/WebRTCModule/PeerConnectionFactoryRegistry.java @@ -0,0 +1,199 @@ +package com.oney.WebRTCModule; + +import android.util.Log; + +import androidx.annotation.Nullable; + +import org.webrtc.PeerConnectionFactory; + +import java.util.UUID; + +/** + * Holds the single live {@link PeerConnectionFactoryProvider}. + * + *

Only ONE factory — hence one ADM — may be live at a time. The SDK builds the call factory at + * join ({@link #create}) and disposes it at leave ({@link #dispose}). A lazily-built default covers + * bare-fork / non-Stream globals usage when no call factory exists. + * + *

{@link WebRTCModule} supplies a {@link FactoryBuilder} that carries the native build inputs + * (shared codec factories, ADM configuration, speech-activity listener); the registry only decides + * when a factory is created, resolved, and disposed. + */ +class PeerConnectionFactoryRegistry { + private static final String TAG = "PCFactoryRegistry"; + + interface FactoryBuilder { + PeerConnectionFactoryProvider build(String id, boolean bypassVoiceProcessing, boolean stereoInputEnabled); + } + + private final FactoryBuilder builder; + + @Nullable + private PeerConnectionFactoryProvider currentFactory; + + private boolean currentIsBareForkDefault = false; + + private boolean disposed = false; + + PeerConnectionFactoryRegistry(FactoryBuilder builder) { + this.builder = builder; + } + + /** + * Returns the live factory, building a bare-fork default on first use. Returns {@code null} + * only after {@link #disposeAll()}. + * + *

A live per-call factory is returned as-is so ADM consumers act on the call's single ADM + * rather than a phantom default. A real default is built only when no factory exists at all. + */ + synchronized PeerConnectionFactoryProvider getOrCreateDefault() { + if (currentFactory != null && !currentFactory.isDisposed()) { + if (currentIsBareForkDefault) { + // An op resolved to a lingering bare-fork default (no call factory ever took its + // place). Normal in bare-fork use; in the Stream SDK it means an op fired outside + // the join↔leave window. The build that created it already dumped a stack trace. + Log.w(TAG, "⚠️ op resolved to the bare-fork DEFAULT factory " + currentFactory.id + + " (outside call window)"); + } + return currentFactory; + } + if (disposed) { + return null; // torn down; do not rebuild + } + return buildAndSetCurrent(false, false, true); + } + + @Nullable + PeerConnectionFactory defaultPeerConnectionFactory() { + PeerConnectionFactoryProvider factory = getOrCreateDefault(); + return factory == null ? null : factory.factory; + } + + /** + * Returns the live factory if one exists, else null — NEVER builds a default. For ADM consumers + * (in-call manager) that must follow the call's factory but must not trigger a default build + * when no call is active (e.g. pre-join / post-leave). + */ + @Nullable + synchronized PeerConnectionFactoryProvider resolveCurrentOrNil() { + if (currentFactory != null && !currentFactory.isDisposed()) { + return currentFactory; + } + return null; + } + + synchronized PeerConnectionFactoryProvider create(boolean bypassVoiceProcessing, boolean stereoInputEnabled) { + if (disposed) { + throw new IllegalStateException("PeerConnectionFactoryRegistry is disposed"); + } + // One factory at a time. A bare-fork default built pre-join is replaced cleanly. A live call + // factory means a second concurrent join, which the single-ADM constraint cannot support: + // keep it so the in-progress call's peer connections stay valid, and return it unchanged + // rather than tearing the live call down. + if (currentFactory != null && !currentFactory.isDisposed()) { + PeerConnectionFactoryProvider existing = currentFactory; + if (currentIsBareForkDefault) { + Log.d(TAG, "disposed stale default before creating call factory"); + try { + existing.dispose(); + } catch (Exception e) { + Log.w(TAG, "create(): error disposing stale default " + existing.id, e); + } + } else { + Log.w(TAG, "⚠️ call factory " + existing.id + + " already live; refusing to create a second — concurrent calls are unsupported"); + return existing; + } + } + return buildAndSetCurrent(bypassVoiceProcessing, stereoInputEnabled, false); + } + + private PeerConnectionFactoryProvider buildAndSetCurrent( + boolean bypassVoiceProcessing, boolean stereoInputEnabled, boolean isDefault) { + String id = UUID.randomUUID().toString(); + PeerConnectionFactoryProvider factory = builder.build(id, bypassVoiceProcessing, stereoInputEnabled); + currentFactory = factory; + currentIsBareForkDefault = isDefault; + String kind = isDefault ? "DEFAULT (bare-fork)" : "per-call"; + Log.d(TAG, "🏭 CREATED " + kind + " factory " + id + " (bypassVoiceProcessing=" + bypassVoiceProcessing + + ", stereoInputEnabled=" + stereoInputEnabled + ")"); + if (isDefault) { + // Should not happen during normal Stream SDK operation: every consumer resolves through + // the live call factory built at join. A default build means something reached the + // registry with no call factory present — dump the stack so the caller is identifiable. + Log.w(TAG, "⚠️ default factory built with no call factory present. Trigger:", + new Throwable("default factory creation trace")); + } + return factory; + } + + /** + * Snapshot of the PeerConnection ids the live factory owns, or an empty list if none is live. + * Returned as a copy so callers can dispose PCs (which mutates the underlying set via + * {@link #unbindPeerConnection}) while iterating. + */ + java.util.List currentOwnedPcIds() { + if (currentFactory == null) { + return java.util.Collections.emptyList(); + } + return new java.util.ArrayList<>(currentFactory.ownedPcIds); + } + + /** + * Snapshot of the track ids the live factory owns, or an empty list if none is live. Returned as + * a copy so callers can dispose tracks (which mutates the underlying set via {@link #forgetTrack}) + * while iterating. + */ + java.util.List currentOwnedTrackIds() { + if (currentFactory == null) { + return java.util.Collections.emptyList(); + } + return new java.util.ArrayList<>(currentFactory.ownedTrackIds); + } + + void bindPeerConnection(int pcId, PeerConnectionFactoryProvider factory) { + factory.ownedPcIds.add(pcId); + Log.d(TAG, "bound pc " + pcId + " -> factory " + factory.id); + } + + void unbindPeerConnection(int pcId) { + if (currentFactory != null) { + currentFactory.ownedPcIds.remove(pcId); + } + } + + void forgetTrack(String trackId) { + if (currentFactory != null) { + currentFactory.ownedTrackIds.remove(trackId); + } + } + + synchronized boolean disposeCurrent() { + if (currentFactory == null) { + Log.w(TAG, "disposeCurrent(): no live factory (already disposed?)"); + return false; + } + boolean wasDefault = currentIsBareForkDefault; + String factoryId = currentFactory.id; + currentFactory.dispose(); + currentFactory = null; + currentIsBareForkDefault = false; + Log.d(TAG, "🗑️ DISPOSED " + (wasDefault ? "DEFAULT (bare-fork)" : "per-call") + " factory " + factoryId); + return true; + } + + synchronized void disposeAll() { + disposed = true; + if (currentFactory != null) { + String id = currentFactory.id; + try { + currentFactory.dispose(); + Log.d(TAG, "🗑️ DISPOSED factory " + id + " (module teardown)"); + } catch (Exception e) { + Log.w(TAG, "disposeAll(): error disposing factory " + id, e); + } + } + currentFactory = null; + currentIsBareForkDefault = false; + } +} diff --git a/android/src/main/java/com/oney/WebRTCModule/RTCCameraPreviewView.java b/android/src/main/java/com/oney/WebRTCModule/RTCCameraPreviewView.java new file mode 100644 index 000000000..be144100c --- /dev/null +++ b/android/src/main/java/com/oney/WebRTCModule/RTCCameraPreviewView.java @@ -0,0 +1,320 @@ +package com.oney.WebRTCModule; + +import android.content.Context; +import android.util.Log; +import android.widget.FrameLayout; + +import androidx.annotation.Nullable; + +import com.facebook.react.bridge.JavaOnlyMap; +import com.facebook.react.bridge.ReactContext; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.webrtc.Camera1Enumerator; +import org.webrtc.Camera2Enumerator; +import org.webrtc.CameraEnumerator; +import org.webrtc.EglBase; +import org.webrtc.RendererCommon.ScalingType; +import org.webrtc.SurfaceTextureHelper; +import org.webrtc.SurfaceViewRenderer; +import org.webrtc.VideoCapturer; + +/** + *

Unlike {@link WebRTCView}, this view does NOT render a WebRTC track. It drives a + * {@link CameraCaptureController} directly and forwards the captured frames to an embedded + * {@link SurfaceViewRenderer} via a {@link FanoutCapturerObserver} — without creating a + * {@code VideoSource}, a {@code VideoTrack}, or touching the {@code PeerConnectionFactory}. + * + *

It is intended for the call lobby, before any peer connection factory exists. At join the + * running camera session is adopted by the WebRTC track (see {@link #yieldForAdoption()}): the + * per-call {@code VideoSource} is attached as the downstream of the {@link FanoutCapturerObserver}, + * so the camera keeps running and is never stopped or restarted. The preview keeps rendering until + * this view unmounts. + */ +public class RTCCameraPreviewView extends FrameLayout { + private static final String TAG = "RTCCameraPreviewView"; + private static final int DEFAULT_WIDTH = 1280; + private static final int DEFAULT_HEIGHT = 720; + private static final int DEFAULT_FPS = 30; + + private final SurfaceViewRenderer surfaceViewRenderer; + + private boolean rendererInitialized = false; + + // Desired props (applied as a batch via commitProps()). + private String facing = "front"; + @Nullable + private String deviceId; + private boolean isActive = false; + private int captureWidth = DEFAULT_WIDTH; + private int captureHeight = DEFAULT_HEIGHT; + + // Serial executor for the blocking capture start/stop calls (CameraCaptureController.startCapture + // and VideoCapturer.stopCapture block until the camera session changes state). These must NOT run + // on the RN UI thread (commitProps/dispose are invoked there) or the UI freezes for the whole + // camera start/stop. All fields below are owned by this executor's thread. + private final ExecutorService captureExecutor = Executors.newSingleThreadExecutor(); + + @Nullable + private CameraCaptureController captureController; + @Nullable + private SurfaceTextureHelper surfaceTextureHelper; + private boolean running = false; + @Nullable + private String runningFacing; + @Nullable + private String runningDeviceId; + + @Nullable + private FanoutCapturerObserver fanoutObserver; + + private volatile boolean handedOff = false; + + public RTCCameraPreviewView(Context context) { + super(context); + + surfaceViewRenderer = new SurfaceViewRenderer(context); + addView(surfaceViewRenderer, new FrameLayout.LayoutParams( + FrameLayout.LayoutParams.MATCH_PARENT, FrameLayout.LayoutParams.MATCH_PARENT)); + + EglBase.Context sharedContext = EglUtils.getRootEglBaseContext(); + if (sharedContext != null) { + surfaceViewRenderer.init(sharedContext, null); + rendererInitialized = true; + } else { + Log.e(TAG, "Unable to obtain root EglBase context; preview will not render"); + } + surfaceViewRenderer.setMirror(true); + surfaceViewRenderer.setScalingType(ScalingType.SCALE_ASPECT_FILL); + } + + void setFacing(@Nullable String facing) { + this.facing = (facing == null) ? "front" : facing; + } + + void setDeviceId(@Nullable String deviceId) { + this.deviceId = deviceId; + } + + void setIsActive(boolean isActive) { + this.isActive = isActive; + } + + void setMirror(boolean mirror) { + surfaceViewRenderer.setMirror(mirror); + } + + void setObjectFit(@Nullable String objectFit) { + ScalingType type = "contain".equals(objectFit) ? ScalingType.SCALE_ASPECT_FIT : ScalingType.SCALE_ASPECT_FILL; + surfaceViewRenderer.setScalingType(type); + } + + void setCaptureWidth(int width) { + if (width > 0) { + this.captureWidth = width; + } + } + + void setCaptureHeight(int height) { + if (height > 0) { + this.captureHeight = height; + } + } + + void commitProps() { + if (handedOff) { + return; + } + + final boolean active = isActive; + final String reqFacing = facing; + final String reqDeviceId = deviceId; + WebRTCModule module = getModule(); + if (module != null) { + if (active) { + module.setActiveCameraPreview(this); + } else { + module.clearActiveCameraPreview(this); + } + } + + captureExecutor.execute(() -> reconcile(active, reqFacing, reqDeviceId)); + } + + private void reconcile(boolean active, String reqFacing, String reqDeviceId) { + if (handedOff) { + return; + } + + if (active) { + boolean configChanged = running + && (!stringsEqual(runningFacing, reqFacing) || !stringsEqual(runningDeviceId, reqDeviceId)); + if (running && !configChanged) { + return; + } + + if (running) { + stopCaptureInternal(); + } + startCaptureInternal(reqFacing, reqDeviceId); + } else if (running) { + stopCaptureInternal(); + } + } + + private void startCaptureInternal(String reqFacing, @Nullable String reqDeviceId) { + if (running || !rendererInitialized || handedOff) { + return; + } + + JavaOnlyMap constraints = new JavaOnlyMap(); + constraints.putInt("width", captureWidth); + constraints.putInt("height", captureHeight); + constraints.putInt("frameRate", DEFAULT_FPS); + constraints.putString("facingMode", "back".equals(reqFacing) ? "environment" : "user"); + if (reqDeviceId != null) { + constraints.putString("deviceId", reqDeviceId); + } + + CameraEnumerator enumerator = Camera2Enumerator.isSupported(getContext()) + ? new Camera2Enumerator(getContext()) + : new Camera1Enumerator(false); + + captureController = new CameraCaptureController(getContext(), enumerator, constraints); + captureController.initializeVideoCapturer(); + + VideoCapturer videoCapturer = captureController.getVideoCapturer(); + if (videoCapturer == null) { + Log.e(TAG, "Unable to create a camera capturer for preview"); + captureController = null; + return; + } + + EglBase.Context eglContext = EglUtils.getRootEglBaseContext(); + surfaceTextureHelper = SurfaceTextureHelper.create("PreviewCaptureThread", eglContext); + + if (surfaceTextureHelper == null) { + Log.e(TAG, "Unable to create SurfaceTextureHelper for preview"); + captureController.dispose(); + captureController = null; + return; + } + + fanoutObserver = new FanoutCapturerObserver(surfaceViewRenderer); + videoCapturer.initialize(surfaceTextureHelper, getContext(), fanoutObserver); + captureController.startCapture(); + + running = true; + runningFacing = reqFacing; + runningDeviceId = reqDeviceId; + } + + private void stopCaptureInternal() { + if (captureController != null) { + captureController.stopCapture(); + captureController.dispose(); + captureController = null; + } + if (surfaceTextureHelper != null) { + surfaceTextureHelper.dispose(); + surfaceTextureHelper = null; + } + running = false; + runningFacing = null; + runningDeviceId = null; + } + + /** + * Yields the running camera session to the WebRTC track: the caller attaches the per-call + * {@code VideoSource}'s observer as the fan-out's downstream (see {@link FanoutCapturerObserver}) + * and takes ownership of the returned controller / surface texture helper. The camera is not + * stopped — frames keep flowing. Returns {@code null} if there is no running capture to adopt + * (caller should then create a fresh capturer). Invoked off the UI thread (getUserMedia worker). + * + *

This view keeps rendering the preview (it retains the fan-out and clears its renderer on + * {@link #dispose()}); ownership of the capturer/controller transfers to the track. + */ + @Nullable + PreviewHandoff yieldForAdoption() { + handedOff = true; + clearActivePreview(); + final PreviewHandoff[] out = new PreviewHandoff[1]; + try { + Future done = captureExecutor.submit(() -> { + if (running && captureController != null && surfaceTextureHelper != null && fanoutObserver != null) { + out[0] = new PreviewHandoff(captureController, surfaceTextureHelper, fanoutObserver); + // Release our ownership WITHOUT stopping: the track now owns the capturer (via the + // controller it retains) and the camera keeps running. We deliberately keep + // `fanoutObserver` so the preview keeps rendering until unmount. + captureController = null; + surfaceTextureHelper = null; + running = false; + } + }); + done.get(); + } catch (Exception e) { + Log.e(TAG, "preview adoption yield failed", e); + } + return out[0]; + } + + void dispose() { + handedOff = true; + clearActivePreview(); + + final boolean releaseRenderer = rendererInitialized; + final FanoutCapturerObserver fanout = fanoutObserver; + rendererInitialized = false; + + captureExecutor.execute(() -> { + // Detach the preview renderer first so no frame reaches a released SurfaceViewRenderer + // (the capturer may still be running if it was adopted by a track). + if (fanout != null) { + fanout.setRenderer(null); + } + stopCaptureInternal(); // no-op if the capturer was adopted (controller already null) + if (releaseRenderer) { + surfaceViewRenderer.release(); + } + }); + + captureExecutor.shutdown(); + } + + /** Bundle of the running capture state handed to a WebRTC track when it adopts the preview. */ + static class PreviewHandoff { + final AbstractVideoCaptureController controller; + final SurfaceTextureHelper surfaceTextureHelper; + final FanoutCapturerObserver fanout; + + PreviewHandoff(AbstractVideoCaptureController controller, SurfaceTextureHelper surfaceTextureHelper, + FanoutCapturerObserver fanout) { + this.controller = controller; + this.surfaceTextureHelper = surfaceTextureHelper; + this.fanout = fanout; + } + } + + private void clearActivePreview() { + WebRTCModule module = getModule(); + if (module != null) { + module.clearActiveCameraPreview(this); + } + } + + @Nullable + private WebRTCModule getModule() { + Context context = getContext(); + if (context instanceof ReactContext) { + return ((ReactContext) context).getNativeModule(WebRTCModule.class); + } + return null; + } + + private static boolean stringsEqual(@Nullable String a, @Nullable String b) { + return a == null ? b == null : a.equals(b); + } +} diff --git a/android/src/main/java/com/oney/WebRTCModule/RTCCameraPreviewViewManager.java b/android/src/main/java/com/oney/WebRTCModule/RTCCameraPreviewViewManager.java new file mode 100644 index 000000000..5fd66b263 --- /dev/null +++ b/android/src/main/java/com/oney/WebRTCModule/RTCCameraPreviewViewManager.java @@ -0,0 +1,68 @@ +package com.oney.WebRTCModule; + +import androidx.annotation.Nullable; + +import com.facebook.react.uimanager.SimpleViewManager; +import com.facebook.react.uimanager.ThemedReactContext; +import com.facebook.react.uimanager.annotations.ReactProp; + +public class RTCCameraPreviewViewManager extends SimpleViewManager { + private static final String REACT_CLASS = "RTCCameraPreviewView"; + + @Override + public String getName() { + return REACT_CLASS; + } + + @Override + public RTCCameraPreviewView createViewInstance(ThemedReactContext context) { + return new RTCCameraPreviewView(context); + } + + @ReactProp(name = "facing") + public void setFacing(RTCCameraPreviewView view, @Nullable String facing) { + view.setFacing(facing); + } + + @ReactProp(name = "deviceId") + public void setDeviceId(RTCCameraPreviewView view, @Nullable String deviceId) { + view.setDeviceId(deviceId); + } + + @ReactProp(name = "isActive", defaultBoolean = false) + public void setIsActive(RTCCameraPreviewView view, boolean isActive) { + view.setIsActive(isActive); + } + + @ReactProp(name = "mirror", defaultBoolean = true) + public void setMirror(RTCCameraPreviewView view, boolean mirror) { + view.setMirror(mirror); + } + + @ReactProp(name = "objectFit") + public void setObjectFit(RTCCameraPreviewView view, @Nullable String objectFit) { + view.setObjectFit(objectFit); + } + + @ReactProp(name = "captureWidth", defaultInt = 1280) + public void setCaptureWidth(RTCCameraPreviewView view, int width) { + view.setCaptureWidth(width); + } + + @ReactProp(name = "captureHeight", defaultInt = 720) + public void setCaptureHeight(RTCCameraPreviewView view, int height) { + view.setCaptureHeight(height); + } + + @Override + protected void onAfterUpdateTransaction(RTCCameraPreviewView view) { + super.onAfterUpdateTransaction(view); + view.commitProps(); + } + + @Override + public void onDropViewInstance(RTCCameraPreviewView view) { + view.dispose(); + super.onDropViewInstance(view); + } +} diff --git a/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java b/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java index 545ef3523..d89f97464 100644 --- a/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java +++ b/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java @@ -1,6 +1,5 @@ package com.oney.WebRTCModule; -import android.os.Build; import android.util.Log; import android.util.Pair; import android.util.SparseArray; @@ -26,13 +25,7 @@ import org.webrtc.*; import org.webrtc.audio.AudioDeviceModule; -import org.webrtc.audio.JavaAudioDeviceModule; -import java.io.ByteArrayInputStream; -import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; -import java.security.cert.CertificateFactory; -import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -46,10 +39,18 @@ public class WebRTCModule extends ReactContextBaseJavaModule { static final String TAG = WebRTCModule.class.getCanonicalName(); - PeerConnectionFactory mFactory; + // Shared across every per-call PeerConnectionFactory and also used to read codec capabilities. VideoEncoderFactory mVideoEncoderFactory; VideoDecoderFactory mVideoDecoderFactory; - AudioDeviceModule mAudioDeviceModule; + + // Owns the per-call factories + the lazy default, and routes PCs/tracks/streams to their factory. + final PeerConnectionFactoryRegistry factoryRegistry; + + // Build inputs captured at module init and reused for every factory built later. + @Nullable + private AudioProcessingFactory audioProcessingFactory; + @Nullable + private AudioDeviceModule injectedAudioDeviceModule; // Need to expose the peer connection codec factories here to get capabilities private final SparseArray mPeerConnectionObservers; @@ -59,7 +60,9 @@ public class WebRTCModule extends ReactContextBaseJavaModule { private static final Map mCertificates = new HashMap<>(); private final GetUserMediaImpl getUserMediaImpl; - private SpeechActivityDetector speechActivityDetector; + + @Nullable + private RTCCameraPreviewView activeCameraPreview; public WebRTCModule(ReactApplicationContext reactContext) { super(reactContext); @@ -69,7 +72,6 @@ public WebRTCModule(ReactApplicationContext reactContext) { WebRTCModuleOptions options = WebRTCModuleOptions.getInstance(); - AudioDeviceModule adm = options.audioDeviceModule; VideoEncoderFactory encoderFactory = options.videoEncoderFactory; VideoDecoderFactory decoderFactory = options.videoDecoderFactory; Loggable injectableLogger = options.injectableLogger; @@ -78,12 +80,11 @@ public WebRTCModule(ReactApplicationContext reactContext) { String fieldTrials = options.fieldTrials; PeerConnectionFactory.initialize(PeerConnectionFactory.InitializationOptions - .builder(reactContext) - - .setFieldTrials(fieldTrials) - .setNativeLibraryLoader(new LibraryLoader()) - .setInjectableLogger(injectableLogger, loggingSeverity) - .createInitializationOptions()); + .builder(reactContext) + .setFieldTrials(fieldTrials) + .setNativeLibraryLoader(new LibraryLoader()) + .setInjectableLogger(injectableLogger, loggingSeverity) + .createInitializationOptions()); if (injectableLogger == null && loggingSeverity != null) { Logging.enableLogToDebugOutput(loggingSeverity); @@ -97,11 +98,6 @@ public WebRTCModule(ReactApplicationContext reactContext) { decoderFactory = new SelectiveVideoDecoderFactory(eglContext, false, Arrays.asList("VP9", "AV1")); } - if (adm == null) { - adm = createAudioDeviceModule(reactContext); - } - - AudioProcessingFactory audioProcessingFactory = null; try { if (options.audioProcessingFactoryProvider != null) { audioProcessingFactory = options.audioProcessingFactoryProvider.getFactory(); @@ -115,26 +111,79 @@ public WebRTCModule(ReactApplicationContext reactContext) { Log.d(TAG, "Using video encoder factory: " + encoderFactory.getClass().getCanonicalName()); Log.d(TAG, "Using video decoder factory: " + decoderFactory.getClass().getCanonicalName()); - PeerConnectionFactory.Builder pcFactoryBuilder = PeerConnectionFactory.builder() - .setAudioDeviceModule(adm) - .setVideoEncoderFactory(encoderFactory) - .setVideoDecoderFactory(decoderFactory); - - if (audioProcessingFactory != null) { - pcFactoryBuilder.setAudioProcessingFactory(audioProcessingFactory); - } + mVideoEncoderFactory = encoderFactory; + mVideoDecoderFactory = decoderFactory; + injectedAudioDeviceModule = options.audioDeviceModule; + + factoryRegistry = new PeerConnectionFactoryRegistry((id, bypassVoiceProcessing, stereoInputEnabled) -> { + PeerConnectionFactoryProvider.BuildOptions buildOptions = new PeerConnectionFactoryProvider.BuildOptions(); + buildOptions.context = getReactApplicationContext(); + buildOptions.videoEncoderFactory = mVideoEncoderFactory; + buildOptions.videoDecoderFactory = mVideoDecoderFactory; + buildOptions.audioProcessingFactory = audioProcessingFactory; + buildOptions.injectedAudioDeviceModule = injectedAudioDeviceModule; + buildOptions.bypassVoiceProcessing = bypassVoiceProcessing; + buildOptions.stereoInputEnabled = stereoInputEnabled; + buildOptions.speechActivityListener = createSpeechActivityListener(); + return PeerConnectionFactoryProvider.build(id, buildOptions); + }); + getUserMediaImpl = new GetUserMediaImpl(this, reactContext); + } - mFactory = pcFactoryBuilder.createPeerConnectionFactory(); + @ReactMethod + public void createCallFactory(ReadableMap options, Promise promise) { + ThreadUtils.runOnExecutor(() -> { + try { + boolean bypassVoiceProcessing = options != null && options.hasKey("bypassVoiceProcessing") + && options.getBoolean("bypassVoiceProcessing"); + boolean stereoInputEnabled = options != null && options.hasKey("stereoInputEnabled") + && options.getBoolean("stereoInputEnabled"); + factoryRegistry.create(bypassVoiceProcessing, stereoInputEnabled); + promise.resolve(null); + } catch (Exception e) { + Log.e(TAG, "createCallFactory() failed", e); + promise.reject("E_FACTORY_CREATE", e); + } + }); + } - // PeerConnectionFactory now owns the adm native pointer, and we don't need it anymore. - adm.release(); + @ReactMethod + public void disposeCallFactory(Promise promise) { + ThreadUtils.runOnExecutor(() -> { + // Tear down in strict order: PeerConnections -> tracks -> factory. + // A libwebrtc PeerConnectionFactory must NOT be disposed while PeerConnections or + // tracks created from it are still alive — that is a use-after-free that can wedge the + // native worker threads, and since this runs on a single-threaded executor a wedge + // blocks every later call. Disposing the factory's own PCs and tracks here first (and + // idempotently) guarantees that ordering regardless of what else may still be in flight. + + // 1. Dispose the factory's PeerConnections first. + for (int pcId : factoryRegistry.currentOwnedPcIds()) { + try { + PeerConnectionObserver pco = mPeerConnectionObservers.get(pcId); + if (pco != null && pco.getPeerConnection() != null) { + pco.dispose(); + mPeerConnectionObservers.remove(pcId); + } + factoryRegistry.unbindPeerConnection(pcId); + } catch (Exception e) { + Log.w(TAG, "disposeCallFactory(): error disposing pc " + pcId, e); + } + } - // Saving the encoder and decoder factories to get codec info later when needed. - mVideoEncoderFactory = encoderFactory; - mVideoDecoderFactory = decoderFactory; - mAudioDeviceModule = adm; + // 2. Stop + dispose owned tracks (e.g. a camera capturer adopted from the lobby + // preview) so the camera2 session is fully closed before the VideoSources are freed. + for (String trackId : factoryRegistry.currentOwnedTrackIds()) { + try { + getUserMediaImpl.disposeTrack(trackId); + } catch (Exception e) { + Log.w(TAG, "disposeCallFactory(): error disposing track " + trackId, e); + } + } - getUserMediaImpl = new GetUserMediaImpl(this, reactContext); + // 3. Now it is safe to dispose the factory + its ADM. + promise.resolve(factoryRegistry.disposeCurrent()); + }); } @Override @@ -170,11 +219,8 @@ public void invalidate() { // 3. Stop capturers + dispose tracks (prevents use-after-free on factory threads) getUserMediaImpl.disposeAllTracks(); - // 4. Dispose factory (frees C++ factory + 3 threads) - if (mFactory != null) { - mFactory.dispose(); - mFactory = null; - } + // 4. Dispose all factories (frees each C++ factory + its ADM + 3 threads) + factoryRegistry.disposeAll(); return null; }) @@ -186,8 +232,8 @@ public void invalidate() { super.invalidate(); } - private JavaAudioDeviceModule createAudioDeviceModule(ReactApplicationContext reactContext) { - speechActivityDetector = new SpeechActivityDetector(new SpeechActivityDetector.Listener() { + private SpeechActivityDetector.Listener createSpeechActivityListener() { + return new SpeechActivityDetector.Listener() { @Override public void onSpeechStarted() { WritableMap params = Arguments.createMap(); @@ -201,95 +247,7 @@ public void onSpeechEnded() { params.putString("event", "ended"); sendEvent("audioDeviceModuleSpeechActivity", params); } - }); - - return JavaAudioDeviceModule.builder(reactContext) - .setUseHardwareAcousticEchoCanceler(Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) - .setUseHardwareNoiseSuppressor(Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) - .setUseStereoOutput(true) - .setAudioBufferCallback( - (audioBuffer, audioFormat, channelCount, sampleRate, bytesRead, captureTimeNs) -> { - // 1. Speech activity detection on raw mic data, BEFORE any mutation. - speechActivityDetector.processBuffer(audioBuffer, bytesRead); - - // 2. Existing screen-audio mixing — mutates audioBuffer in place. - if (bytesRead > 0) { - WebRTCModuleOptions.ScreenAudioBytesProvider provider = - WebRTCModuleOptions.getInstance().screenAudioBytesProvider; - if (provider != null) { - java.nio.ByteBuffer screenBuffer = provider.getScreenAudioBytes(bytesRead); - if (screenBuffer != null && screenBuffer.remaining() > 0) { - mixScreenAudioIntoBuffer(audioBuffer, screenBuffer, bytesRead); - } - } - } - return captureTimeNs; - }) - .setAudioRecordStateCallback(new JavaAudioDeviceModule.AudioRecordStateCallback() { - @Override - public void onWebRtcAudioRecordStart() { - speechActivityDetector.reset(); - } - - @Override - public void onWebRtcAudioRecordStop() { - speechActivityDetector.onRecordStop(); - } - }) - .setPlaybackSamplesReadyCallback(samples -> { - // Fan-out to every registered consumer. The list is - // a CopyOnWriteArrayList so iteration is safe even - // if a consumer registers/unregisters mid-call. - for (JavaAudioDeviceModule.PlaybackSamplesReadyCallback obs : - WebRTCModuleOptions.getInstance().getPlaybackSamplesObservers()) { - try { - obs.onWebRtcAudioTrackSamplesReady(samples); - } catch (Throwable t) { - // Audio device module thread must not throw. - android.util.Log.w(TAG, "playback samples observer threw", t); - } - } - }) - .createAudioDeviceModule(); - } - - /** - * Mixes screen audio into the microphone buffer using PCM 16-bit additive mixing - * with clamping. Handles different buffer sizes safely: each buffer is read only - * within its own bounds. When one buffer is shorter, the other's samples pass - * through unmodified (mic samples stay as-is, or screen-only samples are written). - */ - private static void mixScreenAudioIntoBuffer( - java.nio.ByteBuffer micBuffer, java.nio.ByteBuffer screenBuffer, int bytesRead) { - micBuffer.position(0); - screenBuffer.position(0); - - micBuffer.order(java.nio.ByteOrder.LITTLE_ENDIAN); - screenBuffer.order(java.nio.ByteOrder.LITTLE_ENDIAN); - - java.nio.ShortBuffer micShorts = micBuffer.asShortBuffer(); - java.nio.ShortBuffer screenShorts = screenBuffer.asShortBuffer(); - - int micSamples = Math.min(bytesRead / 2, micShorts.remaining()); - int screenSamples = screenShorts.remaining(); - int totalSamples = Math.max(micSamples, screenSamples); - - for (int i = 0; i < totalSamples; i++) { - int sum; - if (i >= micSamples) { - // Screen-only: mic buffer is shorter — write screen sample directly - sum = screenShorts.get(i); - } else if (i >= screenSamples) { - // Mic-only: screen buffer is shorter — keep mic sample as-is - break; - } else { - // Both buffers have data — add samples - sum = micShorts.get(i) + screenShorts.get(i); - } - if (sum > Short.MAX_VALUE) sum = Short.MAX_VALUE; - if (sum < Short.MIN_VALUE) sum = Short.MIN_VALUE; - micShorts.put(i, (short) sum); - } + }; } @NonNull @@ -298,14 +256,37 @@ public String getName() { return "WebRTCModule"; } + @Nullable public AudioDeviceModule getAudioDeviceModule() { - return mAudioDeviceModule; + PeerConnectionFactoryProvider factory = factoryRegistry.getOrCreateDefault(); + return factory == null ? null : factory.adm; + } + + @Nullable + public AudioDeviceModule currentAudioDeviceModuleOrNil() { + PeerConnectionFactoryProvider factory = factoryRegistry.resolveCurrentOrNil(); + return factory == null ? null : factory.adm; } public GetUserMediaImpl getUserMediaImpl() { return getUserMediaImpl; } + void setActiveCameraPreview(RTCCameraPreviewView preview) { + this.activeCameraPreview = preview; + } + + void clearActiveCameraPreview(RTCCameraPreviewView preview) { + if (this.activeCameraPreview == preview) { + this.activeCameraPreview = null; + } + } + + @Nullable + RTCCameraPreviewView getActiveCameraPreview() { + return this.activeCameraPreview; + } + public PeerConnectionObserver getPeerConnectionObserver(int id) { return mPeerConnectionObservers.get(id); } @@ -589,13 +570,15 @@ public boolean peerConnectionInit(ReadableMap configuration, int id) { try { return (boolean) ThreadUtils .submitToExecutor(() -> { + PeerConnectionFactoryProvider spcf = factoryRegistry.getOrCreateDefault(); PeerConnectionObserver observer = new PeerConnectionObserver(this, id); - PeerConnection peerConnection = mFactory.createPeerConnection(rtcConfiguration, observer); + PeerConnection peerConnection = spcf.factory.createPeerConnection(rtcConfiguration, observer); if (peerConnection == null) { return false; } observer.setPeerConnection(peerConnection); mPeerConnectionObservers.put(id, observer); + factoryRegistry.bindPeerConnection(id, spcf); return true; }) .get(); @@ -668,7 +651,7 @@ public MediaStreamTrack getTrackById(String trackId) { } public VideoTrack createVideoTrack(AbstractVideoCaptureController videoCaptureController) { - return getUserMediaImpl.createVideoTrack(videoCaptureController); + return getUserMediaImpl.createVideoTrack(videoCaptureController, factoryRegistry.getOrCreateDefault()); } public void registerTrack(AudioTrack track, AudioSource source) { @@ -682,7 +665,7 @@ public void registerTrack(VideoTrack track, VideoSource source, AbstractVideoCap public void createStream( MediaStreamTrack[] tracks, GetUserMediaImpl.BiConsumer> successCallback) { - getUserMediaImpl.createStream(tracks, successCallback); + getUserMediaImpl.createStream(tracks, factoryRegistry.getOrCreateDefault(), successCallback); } /** @@ -976,13 +959,14 @@ public boolean transceiverSetCodecPreferences(int id, String senderId, ReadableA return; } - // Convert JSON codec capabilities to the actual objects. + // Codec capabilities come from the live call factory. + PeerConnectionFactory factory = factoryRegistry.getOrCreateDefault().factory; RtpTransceiver.RtpTransceiverDirection direction = transceiver.getDirection(); List, RtpCapabilities.CodecCapability>> availableCodecs = new ArrayList<>(); if (direction.equals(RtpTransceiver.RtpTransceiverDirection.SEND_RECV) || direction.equals(RtpTransceiver.RtpTransceiverDirection.SEND_ONLY)) { - RtpCapabilities capabilities = mFactory.getRtpSenderCapabilities(transceiver.getMediaType()); + RtpCapabilities capabilities = factory.getRtpSenderCapabilities(transceiver.getMediaType()); for (RtpCapabilities.CodecCapability codec : capabilities.codecs) { Map codecDict = SerializeUtils.serializeRtpCapabilitiesCodec(codec).toHashMap(); availableCodecs.add(new Pair<>(codecDict, codec)); @@ -991,7 +975,7 @@ public boolean transceiverSetCodecPreferences(int id, String senderId, ReadableA if (direction.equals(RtpTransceiver.RtpTransceiverDirection.SEND_RECV) || direction.equals(RtpTransceiver.RtpTransceiverDirection.RECV_ONLY)) { - RtpCapabilities capabilities = mFactory.getRtpReceiverCapabilities(transceiver.getMediaType()); + RtpCapabilities capabilities = factory.getRtpReceiverCapabilities(transceiver.getMediaType()); for (RtpCapabilities.CodecCapability codec : capabilities.codecs) { Map codecDict = SerializeUtils.serializeRtpCapabilitiesCodec(codec).toHashMap(); availableCodecs.add(new Pair<>(codecDict, codec)); @@ -1027,7 +1011,8 @@ public void getDisplayMedia(ReadableMap constraints, Promise promise) { @ReactMethod public void getUserMedia(ReadableMap constraints, Callback successCallback, Callback errorCallback) { - ThreadUtils.runOnExecutor(() -> getUserMediaImpl.getUserMedia(constraints, successCallback, errorCallback)); + ThreadUtils.runOnExecutor(() -> getUserMediaImpl.getUserMedia( + constraints, successCallback, errorCallback)); } @ReactMethod @@ -1038,7 +1023,8 @@ public void enumerateDevices(Callback callback) { @ReactMethod public void mediaStreamCreate(String id) { ThreadUtils.runOnExecutor(() -> { - MediaStream mediaStream = mFactory.createLocalMediaStream(id); + PeerConnectionFactoryProvider spcf = factoryRegistry.getOrCreateDefault(); + MediaStream mediaStream = spcf.factory.createLocalMediaStream(id); localStreams.put(id, mediaStream); }); } @@ -1475,7 +1461,8 @@ public WritableMap receiverGetCapabilities(String kind) { return Arguments.createMap(); } - RtpCapabilities capabilities = mFactory.getRtpReceiverCapabilities(mediaType); + RtpCapabilities capabilities = + factoryRegistry.getOrCreateDefault().factory.getRtpReceiverCapabilities(mediaType); return SerializeUtils.serializeRtpCapabilities(capabilities); }) .get(); @@ -1499,7 +1486,8 @@ public WritableMap senderGetCapabilities(String kind) { return Arguments.createMap(); } - RtpCapabilities capabilities = mFactory.getRtpSenderCapabilities(mediaType); + RtpCapabilities capabilities = + factoryRegistry.getOrCreateDefault().factory.getRtpSenderCapabilities(mediaType); return SerializeUtils.serializeRtpCapabilities(capabilities); }) .get(); @@ -1607,11 +1595,16 @@ public void peerConnectionClose(int id) { public void peerConnectionDispose(int id) { ThreadUtils.runOnExecutor(() -> { PeerConnectionObserver pco = mPeerConnectionObservers.get(id); - if (pco == null || pco.getPeerConnection() == null) { - Log.d(TAG, "peerConnectionDispose() peerConnection is null"); + // Null-safe: the PC may already have been disposed (e.g. by + // disposeCallFactory, which tears down the factory's owned PCs first). Skip the + // dispose in that case instead of NPEing, but always clear the factory binding. + if (pco == null) { + Log.d(TAG, "peerConnectionDispose() peerConnection observer is null"); + } else { + pco.dispose(); + mPeerConnectionObservers.remove(id); } - pco.dispose(); - mPeerConnectionObservers.remove(id); + factoryRegistry.unbindPeerConnection(id); }); } diff --git a/android/src/main/java/com/oney/WebRTCModule/WebRTCModulePackage.java b/android/src/main/java/com/oney/WebRTCModule/WebRTCModulePackage.java index d2b3752d8..ba65536b4 100644 --- a/android/src/main/java/com/oney/WebRTCModule/WebRTCModulePackage.java +++ b/android/src/main/java/com/oney/WebRTCModule/WebRTCModulePackage.java @@ -16,6 +16,6 @@ public List createNativeModules(ReactApplicationContext reactConte @Override public List createViewManagers(ReactApplicationContext reactContext) { - return Arrays.asList(new RTCVideoViewManager()); + return Arrays.asList(new RTCVideoViewManager(), new RTCCameraPreviewViewManager()); } } diff --git a/ios/RCTWebRTC/RTCCameraPreviewViewManager.h b/ios/RCTWebRTC/RTCCameraPreviewViewManager.h new file mode 100644 index 000000000..4cf8efe5d --- /dev/null +++ b/ios/RCTWebRTC/RTCCameraPreviewViewManager.h @@ -0,0 +1,23 @@ +#import +#import + +@class CaptureController; +@class RTCVideoSource; + +/** + * Implemented by the lobby camera preview so its running capturer can be adopted by a WebRTC track + * at join: the capturer's delegate is re-pointed at the track's video source, so the camera keeps + * running and is never stopped or reopened. + */ +@protocol RTCCameraPreviewControl +/** + * Re-points the running preview capturer's delegate to {@code source} and yields the capture + * controller so the WebRTC track can take ownership — WITHOUT stopping the camera. Returns nil if + * there is no running preview capturer to adopt. + */ +- (CaptureController *)adoptCaptureForSource:(RTCVideoSource *)source; +@end + +@interface RTCCameraPreviewViewManager : RCTViewManager + +@end diff --git a/ios/RCTWebRTC/RTCCameraPreviewViewManager.m b/ios/RCTWebRTC/RTCCameraPreviewViewManager.m new file mode 100644 index 000000000..b35550136 --- /dev/null +++ b/ios/RCTWebRTC/RTCCameraPreviewViewManager.m @@ -0,0 +1,327 @@ +#if !TARGET_OS_TV + +#import + +#import +#import + +#import +#import +#import +#if TARGET_OS_OSX +#import +#else +#import +#endif + +#import "RTCCameraPreviewViewManager.h" +#import "VideoCaptureController.h" +#import "WebRTCModule.h" + +typedef NS_ENUM(NSInteger, RTCCameraPreviewObjectFit) { + /** + * The contain value defined by https://www.w3.org/TR/css3-images/#object-fit: + * + * The replaced content is sized to maintain its aspect ratio while fitting + * within the element's content box. + */ + RTCCameraPreviewObjectFitContain = 1, + /** + * The cover value defined by https://www.w3.org/TR/css3-images/#object-fit: + * + * The replaced content is sized to maintain its aspect ratio while filling + * the element's entire content box. + */ + RTCCameraPreviewObjectFitCover +}; + +/** + * Unlike RTCVideoView, this view does NOT render a WebRTC track. It drives an + * RTCCameraVideoCapturer directly (via VideoCaptureController for device/format/fps + * selection) and forwards the captured frames straight to its renderer — without ever + * creating an RTCVideoSource, an RTCVideoTrack, or touching the RTCPeerConnectionFactory. + * + * It is intended for the call lobby, before any peer connection factory exists. At join the running + * capturer is reused by the published WebRTC track: its delegate is re-pointed at the track's video + * source while the camera keeps running, so the preview and the published track share one session. + */ +@interface StreamCameraPreviewView : RCTView + +@property(nonatomic) BOOL mirror; + +@property(nonatomic) RTCCameraPreviewObjectFit objectFit; + +#if TARGET_OS_OSX +@property(nonatomic, readonly) RTCMTLNSVideoView *videoView; +#else +@property(nonatomic, readonly) RTCVideoRenderingView *videoView; +#endif + +/** + * Reference to the main WebRTC RN module. + */ +@property(nonatomic, weak) WebRTCModule *module; + +@property(nonatomic) BOOL isActive; + +@property(nonatomic, copy) NSString *facing; +@property(nonatomic, copy) NSString *deviceId; + +@property(nonatomic) NSInteger captureWidth; +@property(nonatomic) NSInteger captureHeight; + +@end + +@implementation StreamCameraPreviewView { + VideoCaptureController *_captureController; + BOOL _capturing; + // Once the camera has been handed off to the WebRTC capturer at join, never re-acquire it + // (a stray prop transaction must not restart capture and contend with the published track). + BOOL _handedOff; + // Serial queue for the blocking capture start/stop calls (VideoCaptureController waits on a + // semaphore for the AVCaptureSession). Must not run on the main thread, or the UI blocks while + // the camera session starts/stops. + dispatch_queue_t _captureQueue; +} + +@synthesize videoView = _videoView; + +- (instancetype)initWithFrame:(CGRect)frame { + if (self = [super initWithFrame:frame]) { + _facing = @"front"; + _mirror = YES; + _objectFit = RTCCameraPreviewObjectFitCover; + _captureWidth = 1280; + _captureHeight = 720; + _capturing = NO; + _captureQueue = dispatch_queue_create("io.getstream.camerapreview", DISPATCH_QUEUE_SERIAL); + +#if TARGET_OS_OSX + RTCMTLNSVideoView *subview = [[RTCMTLNSVideoView alloc] initWithFrame:CGRectZero]; + subview.wantsLayer = true; + _videoView = subview; +#else + RTCVideoRenderingView *subview = [[RTCVideoRenderingView alloc] initWithFrame:CGRectZero]; + subview.renderingBackend = RTCVideoRenderingBackendSharedMetal; + _videoView = subview; +#endif + [self addSubview:self.videoView]; + + // Apply the initial visual state directly: the property setters short-circuit when the + // incoming value equals the current one, so they would no-op for these defaults. + self.videoView.transform = _mirror ? CGAffineTransformMakeScale(-1.0, 1.0) : CGAffineTransformIdentity; +#if !TARGET_OS_OSX + if (_objectFit == RTCCameraPreviewObjectFitCover) { + self.videoView.videoContentMode = UIViewContentModeScaleAspectFill; + } else { + self.videoView.videoContentMode = UIViewContentModeScaleAspectFit; + } +#endif + } + + return self; +} + +- (void)dealloc { + [self stopCapture]; +} + +#if TARGET_OS_OSX +- (void)layout { + [super layout]; + self.videoView.frame = self.bounds; +} +#else +- (void)layoutSubviews { + [super layoutSubviews]; + // Size + position via bounds/center instead of `frame`: setting `frame` while a non-identity + // `transform` (the mirror) is applied is undefined per UIKit and can clobber the mirroring. + self.videoView.bounds = CGRectMake(0, 0, CGRectGetWidth(self.bounds), CGRectGetHeight(self.bounds)); + self.videoView.center = CGPointMake(CGRectGetMidX(self.bounds), CGRectGetMidY(self.bounds)); +} +#endif + +- (void)setMirror:(BOOL)mirror { + if (_mirror != mirror) { + _mirror = mirror; + + self.videoView.transform = mirror ? CGAffineTransformMakeScale(-1.0, 1.0) : CGAffineTransformIdentity; + } +} + +- (void)setObjectFit:(RTCCameraPreviewObjectFit)fit { + if (_objectFit != fit) { + _objectFit = fit; + +#if !TARGET_OS_OSX + if (fit == RTCCameraPreviewObjectFitCover) { + self.videoView.videoContentMode = UIViewContentModeScaleAspectFill; + } else { + self.videoView.videoContentMode = UIViewContentModeScaleAspectFit; + } +#endif + } +} + +- (NSDictionary *)currentConstraints { + NSMutableDictionary *constraints = [@{ + @"width" : @(self.captureWidth), + @"height" : @(self.captureHeight), + @"frameRate" : @30, + @"facingMode" : [@"back" isEqualToString:self.facing] ? @"environment" : @"user" + } mutableCopy]; + + if (self.deviceId.length > 0) { + constraints[@"deviceId"] = self.deviceId; + } + return constraints; +} + +/** + * Called by React Native once per prop transaction, after all props have been set. + */ +- (void)didSetProps:(NSArray *)changedProps { + [self commitProps]; +} + +- (void)commitProps { + if (_handedOff) { + // Camera already handed to the WebRTC capturer; do not re-acquire. + return; + } + + NSDictionary *constraints = [self currentConstraints]; + + if (self.isActive && !_capturing) { + // Mark intent synchronously so a follow-up transaction doesn't double-start; do the + // blocking create + startCapture off the main thread. + _capturing = YES; + self.module.activeCameraPreview = self; + dispatch_async(_captureQueue, ^{ + if (self->_handedOff) { + return; + } + if (!self->_captureController) { + RTCCameraVideoCapturer *capturer = [[RTCCameraVideoCapturer alloc] initWithDelegate:self]; + self->_captureController = [[VideoCaptureController alloc] initWithCapturer:capturer + andConstraints:constraints]; + } else { + [self->_captureController applyConstraints:constraints error:nil]; + } + [self->_captureController startCapture]; + }); + } else if (!self.isActive && _capturing) { + _capturing = NO; + + [self clearActivePreview]; + + VideoCaptureController *controller = _captureController; + dispatch_async(_captureQueue, ^{ + [controller stopCapture]; + }); + } else if (self.isActive && _capturing) { + // Possible config change (e.g. facing flip) while running; applyConstraints is a no-op if + // nothing changed and restarts the session otherwise. + VideoCaptureController *controller = _captureController; + dispatch_async(_captureQueue, ^{ + [controller applyConstraints:constraints error:nil]; + }); + } +} + +/** + * Releases the camera. The (potentially slow) session teardown runs off the main thread so it never + * blocks the UI. Used by dealloc and the camera-disable path; the join hand-off does NOT stop the + * camera — it adopts the running capturer (see {@link adoptCaptureForSource:}). + */ +- (void)stopCapture { + _capturing = NO; + + [self clearActivePreview]; + + VideoCaptureController *controller = _captureController; + _captureController = nil; + if (!controller) { + return; + } + dispatch_async(_captureQueue, ^{ + [controller stopCapture]; + }); +} + +- (void)clearActivePreview { + if (self.module.activeCameraPreview == self) { + self.module.activeCameraPreview = nil; + } +} + +#pragma mark - RTCCameraPreviewControl + +- (CaptureController *)adoptCaptureForSource:(RTCVideoSource *)source { + // Re-point the running capturer's delegate from this preview to the track's video source and + // yield the controller. The camera is not stopped or restarted; frames flow into the track. + _handedOff = YES; + [self clearActivePreview]; + + __block VideoCaptureController *controller = nil; + // Serialize against any in-flight start on the capture queue, then take ownership atomically. + dispatch_sync(_captureQueue, ^{ + controller = self->_captureController; + // Re-point the running capturer's delegate to the track's source. + controller.capturer.delegate = source; + // Release our reference without stopping; the track now owns the capturer (via the + // controller it retains) and the capturer keeps running. + self->_captureController = nil; + }); + return controller; +} + +#pragma mark - RTCVideoCapturerDelegate + +- (void)capturer:(RTCVideoCapturer *)capturer didCaptureVideoFrame:(RTCVideoFrame *)frame { + // static int frameCount = 0; + // if (frameCount++ % 60 == 0) { + // RCTLogInfo(@"[CameraPreview] frame %d -> renderer %dx%d", frameCount, (int)frame.width, (int)frame.height); + // } + [self.videoView renderFrame:frame]; +} + +@end + +@implementation RTCCameraPreviewViewManager + +RCT_EXPORT_MODULE() + +- (RCTView *)view { + StreamCameraPreviewView *v = [[StreamCameraPreviewView alloc] init]; + v.module = [self.bridge moduleForName:@"WebRTCModule"]; + v.clipsToBounds = YES; + return v; +} + +- (dispatch_queue_t)methodQueue { + return dispatch_get_main_queue(); +} + +#pragma mark - View properties + +RCT_EXPORT_VIEW_PROPERTY(mirror, BOOL) +RCT_EXPORT_VIEW_PROPERTY(facing, NSString *) +RCT_EXPORT_VIEW_PROPERTY(deviceId, NSString *) +RCT_EXPORT_VIEW_PROPERTY(isActive, BOOL) +RCT_EXPORT_VIEW_PROPERTY(captureWidth, NSInteger) +RCT_EXPORT_VIEW_PROPERTY(captureHeight, NSInteger) + +RCT_CUSTOM_VIEW_PROPERTY(objectFit, NSString *, StreamCameraPreviewView) { + NSString *fitStr = json; + view.objectFit = (fitStr && [fitStr isEqualToString:@"contain"]) ? RTCCameraPreviewObjectFitContain + : RTCCameraPreviewObjectFitCover; +} + ++ (BOOL)requiresMainQueueSetup { + return NO; +} + +@end + +#endif diff --git a/ios/RCTWebRTC/Utils/PeerConnectionFactory/PeerConnectionFactoryProvider.swift b/ios/RCTWebRTC/Utils/PeerConnectionFactory/PeerConnectionFactoryProvider.swift new file mode 100644 index 000000000..924ca62f7 --- /dev/null +++ b/ios/RCTWebRTC/Utils/PeerConnectionFactory/PeerConnectionFactoryProvider.swift @@ -0,0 +1,95 @@ +// +// Copyright © 2026 Stream.io Inc. All rights reserved. +// + +import Foundation +import WebRTC + +/// Owns one `RTCPeerConnectionFactory` and the AudioDeviceModule it is built with, identified by a +/// stable `factoryId`. The ADM type (`.audioEngine`) and the `bypassVoiceProcessing` flag are fixed +/// at construction, so each call gets its own factory built with the audio profile it needs. +/// +/// The encoder/decoder factories and the audio-processing module are created once by `WebRTCModule` +/// and passed in; this class only owns the factory lifecycle and the profile-driven build settings. +@objc public final class PeerConnectionFactoryProvider: NSObject { + + @objc public let factoryId: String + @objc public private(set) var factory: RTCPeerConnectionFactory? + @objc public private(set) var audioDeviceModule: AudioDeviceModule? + @objc public let bypassVoiceProcessing: Bool + + private var disposed = false + + private init( + factoryId: String, + factory: RTCPeerConnectionFactory, + audioDeviceModule: AudioDeviceModule?, + bypassVoiceProcessing: Bool + ) { + self.factoryId = factoryId + self.factory = factory + self.audioDeviceModule = audioDeviceModule + self.bypassVoiceProcessing = bypassVoiceProcessing + super.init() + } + + @objc public static func build( + withId factoryId: String, + bypassVoiceProcessing: Bool, + encoderFactory: RTCVideoEncoderFactory, + decoderFactory: RTCVideoDecoderFactory, + audioProcessingModule: RTCAudioProcessingModule?, + audioDevice: RTCAudioDevice?, + audioDeviceModuleObserver: RTCAudioDeviceModuleDelegate + ) -> PeerConnectionFactoryProvider { + let factory: RTCPeerConnectionFactory + if let audioProcessingModule = audioProcessingModule { + factory = RTCPeerConnectionFactory( + audioDeviceModuleType: .audioEngine, + bypassVoiceProcessing: bypassVoiceProcessing, + encoderFactory: encoderFactory, + decoderFactory: decoderFactory, + audioProcessingModule: audioProcessingModule + ) + } else if let audioDevice = audioDevice { + factory = RTCPeerConnectionFactory( + encoderFactory: encoderFactory, + decoderFactory: decoderFactory, + audioDevice: audioDevice + ) + } else { + factory = RTCPeerConnectionFactory( + audioDeviceModuleType: .audioEngine, + bypassVoiceProcessing: bypassVoiceProcessing, + encoderFactory: encoderFactory, + decoderFactory: decoderFactory, + audioProcessingModule: nil + ) + } + factory.frameBufferPolicy = .copyToNV12 + + let audioDeviceModule = AudioDeviceModule( + source: factory.audioDeviceModule, + delegateObserver: audioDeviceModuleObserver + ) + + return PeerConnectionFactoryProvider( + factoryId: factoryId, + factory: factory, + audioDeviceModule: audioDeviceModule, + bypassVoiceProcessing: bypassVoiceProcessing + ) + } + + // MARK: - Lifecycle + + @objc public func isDisposed() -> Bool { disposed } + + @objc public func dispose() { + guard !disposed else { return } + disposed = true + // Releasing the factory releases its raw ADM; drop the wrapper too (ARC handles teardown). + audioDeviceModule = nil + factory = nil + } +} diff --git a/ios/RCTWebRTC/Utils/PeerConnectionFactory/PeerConnectionFactoryRegistry.swift b/ios/RCTWebRTC/Utils/PeerConnectionFactory/PeerConnectionFactoryRegistry.swift new file mode 100644 index 000000000..af9bf683a --- /dev/null +++ b/ios/RCTWebRTC/Utils/PeerConnectionFactory/PeerConnectionFactoryRegistry.swift @@ -0,0 +1,135 @@ +// +// Copyright © 2026 Stream.io Inc. All rights reserved. +// + +import Foundation +import WebRTC + +public typealias PeerConnectionFactoryBuilder = (_ factoryId: String, _ bypassVoiceProcessing: Bool) + -> PeerConnectionFactoryProvider + +/// Holds the single live `PeerConnectionFactoryProvider`. +/// +/// Only ONE factory — hence one ADM, one `AVAudioEngine` on the shared `AVAudioSession` — may be +/// live at a time (a second engine crashes `IsFormatSampleRateAndChannelCountValid`). The SDK builds +/// the call factory at join (`create`) and disposes it at leave (`dispose`). A lazily-built default +/// covers bare-fork / non-Stream globals usage when no call factory exists. +/// +/// `WebRTCModule` supplies the builder closure carrying the native build inputs (shared codec +/// factories, audio-processing module, audio device). +@objc public final class PeerConnectionFactoryRegistry: NSObject { + + private let builder: PeerConnectionFactoryBuilder + private var currentFactory: PeerConnectionFactoryProvider? + private var currentIsBareForkDefault = false + private var disposed = false + // Recursive so the reentrant resolve() -> getOrCreateDefault() path doesn't deadlock. + private let lock = NSRecursiveLock() + + @objc public init(builder: @escaping PeerConnectionFactoryBuilder) { + self.builder = builder + super.init() + } + + /// Returns the live factory, building a bare-fork default on first use. Nil only after + /// `disposeAll()`. A live per-call factory is returned as-is so ADM consumers act on the call's + /// single ADM. A real default is built only when no factory exists at all. + @objc public func getOrCreateDefault() -> PeerConnectionFactoryProvider? { + lock.lock() + defer { lock.unlock() } + if let currentFactory = currentFactory, !currentFactory.isDisposed() { + if currentIsBareForkDefault { + // An op resolved to a lingering bare-fork default (no call factory ever took its + // place). Normal in bare-fork use; in the Stream SDK it means an op fired outside + // the join↔leave window. The build that created it already dumped a stack trace. + NSLog("[PCFactoryRegistry] ⚠️ op resolved to the bare-fork DEFAULT factory %@ (outside call window)", + currentFactory.factoryId) + } + return currentFactory + } + if disposed { + return nil + } + return buildAndSetCurrent(bypassVoiceProcessing: false, isDefault: true) + } + + /// Returns the live factory if one exists, else nil — NEVER builds a default. For ADM consumers + /// (callingx / in-call manager) that must follow the call's factory but must not trigger a + /// default build when no call is active (e.g. pre-join / post-leave). + @objc public func resolveCurrentOrNil() -> PeerConnectionFactoryProvider? { + lock.lock() + defer { lock.unlock() } + guard let currentFactory = currentFactory, !currentFactory.isDisposed() else { return nil } + return currentFactory + } + + @objc public func create(_ bypassVoiceProcessing: Bool) -> PeerConnectionFactoryProvider? { + lock.lock() + defer { lock.unlock() } + if disposed { + return nil + } + // One factory at a time. A bare-fork default built pre-join is replaced cleanly. A live call + // factory means a second concurrent join, which the single-ADM/AVAudioEngine constraint + // cannot support: keep it so the in-progress call's peer connections stay valid, and return + // it unchanged rather than tearing the live call down. + if let existing = currentFactory, !existing.isDisposed() { + if currentIsBareForkDefault { + NSLog("[PCFactoryRegistry] disposed stale default before creating call factory") + existing.dispose() + } else { + NSLog("[PCFactoryRegistry] ⚠️ call factory %@ already live; refusing to create a second — concurrent calls are unsupported", + existing.factoryId) + return existing + } + } + return buildAndSetCurrent(bypassVoiceProcessing: bypassVoiceProcessing, isDefault: false) + } + + private func buildAndSetCurrent(bypassVoiceProcessing: Bool, isDefault: Bool) -> PeerConnectionFactoryProvider { + let factoryId = UUID().uuidString + let factory = builder(factoryId, bypassVoiceProcessing) + currentFactory = factory + currentIsBareForkDefault = isDefault + let kind = isDefault ? "DEFAULT (bare-fork)" : "per-call" + NSLog("[PCFactoryRegistry] 🏭 CREATED %@ factory %@ (bypassVoiceProcessing=%@)", + kind, factoryId, bypassVoiceProcessing ? "true" : "false") + if isDefault { + // Should not happen during normal Stream SDK operation: every consumer resolves through + // the live call factory built at join. A default build means something reached the + // registry with no call factory present — dump the call stack so the caller is identifiable. + NSLog("[PCFactoryRegistry] ⚠️ default factory built with no call factory present. Trigger:\n%@", + Thread.callStackSymbols.joined(separator: "\n")) + } + return factory + } + + /// Disposes the live call factory and clears it. Returns false when nothing is live. + @objc public func disposeCurrent() -> Bool { + lock.lock() + defer { lock.unlock() } + guard let factory = currentFactory else { + NSLog("[PCFactoryRegistry] disposeCurrent(): no live factory (already disposed?)") + return false + } + let wasDefault = currentIsBareForkDefault + let factoryId = factory.factoryId + factory.dispose() + currentFactory = nil + currentIsBareForkDefault = false + NSLog("[PCFactoryRegistry] 🗑️ DISPOSED %@ factory %@", wasDefault ? "DEFAULT (bare-fork)" : "per-call", factoryId) + return true + } + + @objc public func disposeAll() { + lock.lock() + defer { lock.unlock() } + disposed = true + if let factory = currentFactory { + factory.dispose() + NSLog("[PCFactoryRegistry] 🗑️ DISPOSED factory %@ (module teardown)", factory.factoryId) + } + currentFactory = nil + currentIsBareForkDefault = false + } +} diff --git a/ios/RCTWebRTC/WebRTCModule+RTCMediaStream.m b/ios/RCTWebRTC/WebRTCModule+RTCMediaStream.m index 772657307..918da0df6 100644 --- a/ios/RCTWebRTC/WebRTCModule+RTCMediaStream.m +++ b/ios/RCTWebRTC/WebRTCModule+RTCMediaStream.m @@ -192,13 +192,23 @@ - (RTCVideoTrack *)createVideoTrack:(NSDictionary *)constraints { #endif if (hasRuntimeVideoDevice) { - RTCCameraVideoCapturer *videoCapturer = [[RTCCameraVideoCapturer alloc] initWithDelegate:videoSource]; - VideoCaptureController *videoCaptureController = - [[VideoCaptureController alloc] initWithCapturer:videoCapturer andConstraints:constraints[@"video"]]; - videoCaptureController.enableMultitaskingCameraAccess = - [WebRTCModuleOptions sharedInstance].enableMultitaskingCameraAccess; - videoTrack.captureController = videoCaptureController; - [videoCaptureController startCapture]; + // If a lobby camera preview is already running, adopt its capturer by re-pointing it at + // this track's video source. Falls back to creating a fresh capturer when there's no preview to adopt. + CaptureController *captureController = [self adoptActiveCameraPreviewForSource:videoSource]; + if ([captureController isKindOfClass:[VideoCaptureController class]]) { + VideoCaptureController *videoCaptureController = (VideoCaptureController *)captureController; + videoTrack.captureController = videoCaptureController; + // Already capturing — do NOT startCapture again. Reconcile to the requested constraints. + [videoCaptureController applyConstraints:constraints[@"video"] error:nil]; + } else { + RTCCameraVideoCapturer *videoCapturer = [[RTCCameraVideoCapturer alloc] initWithDelegate:videoSource]; + VideoCaptureController *videoCaptureController = + [[VideoCaptureController alloc] initWithCapturer:videoCapturer andConstraints:constraints[@"video"]]; + videoCaptureController.enableMultitaskingCameraAccess = + [WebRTCModuleOptions sharedInstance].enableMultitaskingCameraAccess; + videoTrack.captureController = videoCaptureController; + [videoCaptureController startCapture]; + } } // Add dimension detection for local video tracks immediately diff --git a/ios/RCTWebRTC/WebRTCModule.h b/ios/RCTWebRTC/WebRTCModule.h index 4d3258e36..2dc567d11 100644 --- a/ios/RCTWebRTC/WebRTCModule.h +++ b/ios/RCTWebRTC/WebRTCModule.h @@ -34,11 +34,16 @@ static NSString *const kEventAudioDeviceModuleAudioProcessingStateUpdated = @"audioDeviceModuleAudioProcessingStateUpdated"; @class AudioDeviceModule; +@class CaptureController; +@class PeerConnectionFactoryRegistry; + +@protocol RTCCameraPreviewControl; @interface WebRTCModule : RCTEventEmitter @property(nonatomic, strong) dispatch_queue_t workerQueue; +@property(nonatomic, strong) PeerConnectionFactoryRegistry *factoryRegistry; @property(nonatomic, strong) RTCPeerConnectionFactory *peerConnectionFactory; @property(nonatomic, strong) id decoderFactory; @property(nonatomic, strong) id encoderFactory; @@ -48,8 +53,14 @@ static NSString *const kEventAudioDeviceModuleAudioProcessingStateUpdated = @property(nonatomic, strong) NSMutableDictionary *localStreams; @property(nonatomic, strong) NSMutableDictionary *localTracks; +@property(nonatomic, weak) id activeCameraPreview; + +- (CaptureController *)adoptActiveCameraPreviewForSource:(RTCVideoSource *)source; + - (RTCMediaStream *)streamForReactTag:(NSString *)reactTag; - (nullable RTCMediaStreamTrack *)trackForId:(NSString *)trackId; +- (nullable AudioDeviceModule *)currentAudioDeviceModuleOrNil; + @end diff --git a/ios/RCTWebRTC/WebRTCModule.m b/ios/RCTWebRTC/WebRTCModule.m index 18b1184d3..c90e72358 100644 --- a/ios/RCTWebRTC/WebRTCModule.m +++ b/ios/RCTWebRTC/WebRTCModule.m @@ -8,6 +8,7 @@ #import #import "AudioDeviceModuleObserver.h" +#import "RTCCameraPreviewViewManager.h" #import "WebRTCModule+RTCPeerConnection.h" #import "WebRTCModule.h" #import "WebRTCModuleOptions.h" @@ -46,8 +47,7 @@ - (void)dealloc { [peerConnection close]; } [_peerConnections removeAllObjects]; - - _peerConnectionFactory = nil; + [_factoryRegistry disposeAll]; } - (instancetype)init { @@ -100,38 +100,26 @@ - (instancetype)init { RCTLogInfo(@"Created default audio processing module for screen share audio mixing"); } - if (audioProcessingModule != nil) { - if (audioDevice != nil) { - NSLog(@"Both audioProcessingModule and audioDevice are provided, but only one can be used. Ignoring " - @"audioDevice."); - } - RCTLogInfo(@"Using audio processing module: %@", NSStringFromClass([audioProcessingModule class])); - - _peerConnectionFactory = - [[RTCPeerConnectionFactory alloc] initWithAudioDeviceModuleType:RTCAudioDeviceModuleTypeAudioEngine - bypassVoiceProcessing:NO - encoderFactory:encoderFactory - decoderFactory:decoderFactory - audioProcessingModule:audioProcessingModule]; - } else if (audioDevice != nil) { - RCTLogInfo(@"Using custom audio device: %@", NSStringFromClass([audioDevice class])); - _peerConnectionFactory = [[RTCPeerConnectionFactory alloc] initWithEncoderFactory:encoderFactory - decoderFactory:decoderFactory - audioDevice:audioDevice]; - } else { - _peerConnectionFactory = - [[RTCPeerConnectionFactory alloc] initWithAudioDeviceModuleType:RTCAudioDeviceModuleTypeAudioEngine - bypassVoiceProcessing:NO - encoderFactory:encoderFactory - decoderFactory:decoderFactory - audioProcessingModule:nil]; + if (audioProcessingModule != nil && audioDevice != nil) { + NSLog(@"Both audioProcessingModule and audioDevice are provided, but only one can be used. Ignoring " + @"audioDevice."); } - - _peerConnectionFactory.frameBufferPolicy = RTCFrameBufferPolicyCopyToNV12; _rtcAudioDeviceModuleObserver = [[AudioDeviceModuleObserver alloc] initWithWebRTCModule:self]; - _audioDeviceModule = [[AudioDeviceModule alloc] initWithSource:_peerConnectionFactory.audioDeviceModule - delegateObserver:_rtcAudioDeviceModuleObserver]; + + // Capture the observer (not self) so the builder block doesn't retain the module. + AudioDeviceModuleObserver *audioDeviceModuleObserver = _rtcAudioDeviceModuleObserver; + + self.factoryRegistry = [[PeerConnectionFactoryRegistry alloc] + initWithBuilder:^PeerConnectionFactoryProvider *(NSString *factoryId, BOOL bypassVoiceProcessing) { + return [PeerConnectionFactoryProvider buildWithId:factoryId + bypassVoiceProcessing:bypassVoiceProcessing + encoderFactory:encoderFactory + decoderFactory:decoderFactory + audioProcessingModule:options.audioProcessingModule + audioDevice:options.audioDevice + audioDeviceModuleObserver:audioDeviceModuleObserver]; + }]; _peerConnections = [NSMutableDictionary new]; _localStreams = [NSMutableDictionary new]; @@ -145,6 +133,26 @@ - (instancetype)init { return self; } +- (RTCPeerConnectionFactory *)peerConnectionFactory { + return [self.factoryRegistry getOrCreateDefault].factory; +} + +- (AudioDeviceModule *)audioDeviceModule { + return [self.factoryRegistry getOrCreateDefault].audioDeviceModule; +} + +- (nullable AudioDeviceModule *)currentAudioDeviceModuleOrNil { + return [self.factoryRegistry resolveCurrentOrNil].audioDeviceModule; +} + +- (CaptureController *)adoptActiveCameraPreviewForSource:(RTCVideoSource *)source { + id preview = self.activeCameraPreview; + if (preview) { + return [preview adoptCaptureForSource:source]; + } + return nil; +} + - (RTCMediaStream *)streamForReactTag:(NSString *)reactTag { RTCMediaStream *stream = _localStreams[reactTag]; if (!stream) { @@ -191,6 +199,25 @@ - (dispatch_queue_t)methodQueue { return _workerQueue; } +RCT_EXPORT_METHOD(createCallFactory + : (NSDictionary *)options resolver + : (RCTPromiseResolveBlock)resolve rejecter + : (RCTPromiseRejectBlock)reject) { + BOOL bypassVoiceProcessing = [options[@"bypassVoiceProcessing"] boolValue]; + PeerConnectionFactoryProvider *factory = [self.factoryRegistry create:bypassVoiceProcessing]; + if (factory == nil) { + reject(@"E_FACTORY_CREATE", @"Failed to create call factory: registry is disposed", nil); + return; + } + resolve(nil); +} + +RCT_EXPORT_METHOD(disposeCallFactory + : (RCTPromiseResolveBlock)resolve rejecter + : (RCTPromiseRejectBlock)reject) { + resolve(@([self.factoryRegistry disposeCurrent])); +} + - (NSArray *)supportedEvents { return @[ kEventPeerConnectionSignalingStateChanged, diff --git a/src/CallFactory.ts b/src/CallFactory.ts new file mode 100644 index 000000000..315f947fa --- /dev/null +++ b/src/CallFactory.ts @@ -0,0 +1,40 @@ +import { NativeModules } from 'react-native'; + +const { WebRTCModule } = NativeModules; + +export interface CallFactoryOptions { + /** + * Music / high-quality audio profile: builds the AudioDeviceModule with hardware AEC/NS disabled + * and the raw mic source so the unprocessed signal reaches WebRTC. Baked in at construction — + * immutable for the life of the factory. + */ + bypassVoiceProcessing?: boolean; + /** + * Builds the AudioDeviceModule for stereo microphone capture. Android only; ignored on iOS. + * Baked in at construction — immutable for the life of the factory. + */ + stereoInputEnabled?: boolean; +} + +/** + * Handle to the native call PeerConnectionFactory (and its AudioDeviceModule). Creating one builds a + * fresh native factory and makes it the single live factory, so the standard globals + * (`mediaDevices.getUserMedia` / `new RTCPeerConnection`) resolve to it for the life of the call. + * Create one per call and {@link dispose} it when the call ends. + */ +export default class CallFactory { + /** Builds a fresh native factory with the given audio profile and makes it the live factory. */ + static async create(options: CallFactoryOptions = {}): Promise { + await WebRTCModule.createCallFactory({ + bypassVoiceProcessing: options.bypassVoiceProcessing ?? false, + stereoInputEnabled: options.stereoInputEnabled ?? false + }); + + return new CallFactory(); + } + + /** Disposes the live call factory and its ADM. Resolves to true if a factory was disposed. */ + dispose(): Promise { + return WebRTCModule.disposeCallFactory(); + } +} diff --git a/src/RTCCameraPreviewView.ts b/src/RTCCameraPreviewView.ts new file mode 100644 index 000000000..8350ee5fd --- /dev/null +++ b/src/RTCCameraPreviewView.ts @@ -0,0 +1,79 @@ +import { requireNativeComponent, ViewProps } from 'react-native'; + +/** + * A WebRTC-free camera preview. + * + * Unlike {@link RTCView}, this component does NOT render a WebRTC track. It + * drives the platform camera capturer directly and renders the captured frames + * to a local view — without creating an `RTCVideoSource`, a track, or a + * `PeerConnectionFactory`. It is intended for the call lobby, before any peer + * connection factory exists. + * + * At join the running capturer is reused by the published WebRTC track: its + * frames are routed into the track's video source while the camera keeps + * running, so the preview and the published track share one camera session. + * + * Native prop validation was removed from RN in: + * https://github.com/facebook/react-native/commit/8dc3ba0444c94d9bbb66295b5af885bff9b9cd34 + * So we list the props here for documentation purposes. + */ +interface RTCCameraPreviewViewProps extends ViewProps { + /** + * Which camera to preview. Ignored when {@link #deviceId} is set. + * + * facing: 'front' | 'back' + */ + facing?: 'front' | 'back'; + + /** + * Explicit camera device id to preview. Takes precedence over + * {@link #facing}. + * + * deviceId: string + */ + deviceId?: string; + + /** + * Whether the preview capture is running. Set to `false` to stop capturing + * and release the camera (e.g. when leaving the lobby, or just before the + * WebRTC capturer takes over at join). + * + * isActive: boolean + */ + isActive?: boolean; + + /** + * Whether the preview should be mirrored during rendering. Commonly enabled + * for the user-facing (front) camera. + * + * mirror: boolean + */ + mirror?: boolean; + + /** + * Resembles the CSS style object-fit. + * + * objectFit: 'contain' | 'cover' + */ + objectFit?: 'contain' | 'cover'; + + /** + * Capture width in pixels (landscape). Defaults to 1280. Set this to the + * call's target resolution so the running capturer can be adopted by the + * WebRTC track at join without reconfiguring. + * + * captureWidth: number + */ + captureWidth?: number; + + /** + * Capture height in pixels (landscape). Defaults to 720. + * + * captureHeight: number + */ + captureHeight?: number; +} + +export default requireNativeComponent( + 'RTCCameraPreviewView', +); diff --git a/src/index.ts b/src/index.ts index c96ab58e2..6d48b9620 100644 --- a/src/index.ts +++ b/src/index.ts @@ -10,6 +10,9 @@ if (WebRTCModule === null) { import { AudioDeviceModule, AudioEngineMuteMode } from './AudioDeviceModule'; import { audioDeviceModuleEvents } from './AudioDeviceModuleEvents'; +import CallFactory, { + type CallFactoryOptions, +} from './CallFactory'; import { setupNativeEvents } from './EventEmitter'; import Logger from './Logger'; import mediaDevices from './MediaDevices'; @@ -18,6 +21,7 @@ import MediaStreamTrack, { type MediaTrackSettings } from './MediaStreamTrack'; import MediaStreamTrackEvent from './MediaStreamTrackEvent'; import permissions from './Permissions'; import RTCAudioSession from './RTCAudioSession'; +import RTCCameraPreviewView from './RTCCameraPreviewView'; import RTCCertificate from './RTCCertificate'; import RTCErrorEvent from './RTCErrorEvent'; import RTCIceCandidate from './RTCIceCandidate'; @@ -45,7 +49,10 @@ export { RTCSessionDescription, RTCCertificate, RTCView, + RTCCameraPreviewView, ScreenCapturePickerView, + CallFactory, + type CallFactoryOptions, RTCRtpEncodingParameters, RTCRtpTransceiver, RTCRtpReceiver,