diff --git a/docs/help/help.json b/docs/help/help.json index 751a85f7..dd5b6166 100644 --- a/docs/help/help.json +++ b/docs/help/help.json @@ -3733,13 +3733,16 @@ "controls": { "source": { "line": "Where the picture comes from.", - "more": "The test pattern is a test card in the layout of Philips' PM5544, drawn at any size." + "more": "The test pattern is a test card in the layout of Philips' PM5544, drawn at any size. **usb capture** decodes MJPEG from an HDMI grabber, scaled to fit." + }, + "fps": { + "line": "Frames a second asked of a capture device, which streams the nearest rate it offers." }, "width": { - "line": "The picture's width in pixels." + "line": "The picture's width in pixels, and the most a capture scales its frames to." }, "height": { - "line": "The picture's height in pixels." + "line": "The picture's height in pixels, under the same ceiling as the width." } }, "role": "service", diff --git a/docs/reference/metrics/code.md b/docs/reference/metrics/code.md index d736f352..99e845c3 100644 --- a/docs/reference/metrics/code.md +++ b/docs/reference/metrics/code.md @@ -4,7 +4,7 @@ Generated by [`moondeck/check/check_code.py`](../../../moondeck/check/check_code Every function, block and file over a limit the [coding standards](../../contributing/coding-standards.md#static-checks) set. Current state only: the trend is this file's git history. The list only shrinks, per rule and in total, and a touched file clears its own rows. -**1610 finding(s)** across 320 file(s). 2.57% of all lines are duplicated, 3586 lines. +**1611 finding(s)** across 322 file(s). 2.40% of all lines are duplicated, 3345 lines. ## By rule @@ -19,7 +19,7 @@ Every function, block and file over a limit the [coding standards](../../contrib | long parameter list | 35 | | large stack frame | 33 | | raw allocation in light | 28 | -| large file | 25 | +| large file | 26 | | name spread | 22 | ## How a finding is solved @@ -44,10 +44,10 @@ Each rule is a code smell with a named move from the refactoring catalog, the ta | Area | Findings | |---|---:| -| `test` | 558 | +| `test` | 557 | | `light` | 402 | | `core` | 364 | -| `platform` | 174 | +| `platform` | 176 | | `ui` | 81 | | `moondeck` | 22 | | `src` | 7 | @@ -157,7 +157,6 @@ The next function worth simplifying is at the top of each file's list. | `test/unit/light/unit_FireEffect.cpp` | 5 | `lines 29-42, also in test/unit/light/unit_RainbowEffect.cpp:69`, duplicated block 14 | | `test/unit/light/unit_FreqSawsEffect.cpp` | 5 | `lines 19-29, also in test/unit/light/unit_RainbowEffect.cpp:69`, duplicated block 11 | | `test/unit/light/unit_GEQ3DEffect.cpp` | 5 | `lines 58-75, also in test/unit/light/unit_PaintBrushEffect.cpp:44`, duplicated block 18 | -| `test/unit/light/unit_MoonLiveScripts.cpp` | 5 | `lines 292-300, also in test/unit/light/unit_MoonLiveScripts.cpp:264`, duplicated block 9 | | `test/unit/light/unit_RandomEffect.cpp` | 5 | `lines 10-22, also in test/unit/light/unit_RandomMapModifier.cpp:115`, duplicated block 13 | | `src/core/module/Scheduler.cpp` | 4 | `Scheduler::getControlWide`, complex function 18 | | `src/core/moonlive/MoonLive.cpp` | 4 | `moonlive::MoonLive::ensureArena`, complex function 21 | @@ -183,6 +182,7 @@ The next function worth simplifying is at the top of each file's list. | `test/unit/light/unit_FixedRectangleEffect.cpp` | 4 | `lines 90-103, also in test/unit/light/unit_SolidEffect.cpp:9`, duplicated block 14 | | `test/unit/light/unit_Layer_extrude.cpp` | 4 | `lines 155-168, also in test/unit/light/unit_Layer_extrude.cpp:124`, duplicated block 14 | | `test/unit/light/unit_Layer_live_modifier.cpp` | 4 | `lines 96-105, also in test/unit/light/unit_RainbowEffect.cpp:69`, duplicated block 10 | +| `test/unit/light/unit_MoonLiveScripts.cpp` | 4 | `lines 292-300, also in test/unit/light/unit_MoonLiveScripts.cpp:264`, duplicated block 9 | | `test/unit/light/unit_NetworkSendDriver_no_alloc_in_loop.cpp` | 4 | `lines 49-59, also in test/unit/light/unit_NetworkSendDriver_no_alloc_in_loop.cpp:18`, duplicated block 11 | | `test/unit/light/unit_NetworkSendDriver_packet.cpp` | 4 | `lines 346-354, also in test/unit/light/unit_NetworkSendDriver_packet.cpp:292`, duplicated block 9 | | `test/unit/light/unit_ParallelSlots.cpp` | 4 | `lines 522-532, also in test/unit/light/unit_ParallelSlots.cpp:493`, duplicated block 11 | @@ -204,10 +204,10 @@ The next function worth simplifying is at the top of each file's list. | `src/module_types.cpp` | 3 | `registerModuleTypes`, long function 153 | | `src/platform/desktop/moonlive_asm_arm64.cpp` | 3 | `lines 34-39, also in src/platform/esp32/moonlive_asm_riscv.cpp:33`, duplicated block 6 | | `src/platform/esp32/moonlive_asm_xtensa.cpp` | 3 | `lines 482-488, also in src/platform/esp32/moonlive_asm_riscv.cpp:296`, duplicated block 7 | +| `src/platform/teensy/platform_teensy.cpp` | 3 | `platform::resolveHostStep`, complex function 16 | | `src/platform/teensy/platform_teensy_ota.cpp` | 3 | `platform::otaWriteStream`, complex function 16 | | `test/unit/core/unit_NetworkModule_ethernet.cpp` | 3 | `lines 276-285, also in test/unit/core/unit_NetworkModule_ethernet.cpp:193`, duplicated block 10 | | `test/unit/light/unit_AuroraEffect.cpp` | 3 | `lines 40-48, also in test/unit/light/unit_PolarLut_equivalence.cpp:40`, duplicated block 9 | -| `test/unit/light/unit_MoonLiveLayout.cpp` | 3 | `lines 685-935, also in test/unit/light/unit_MoonLiveLayout.cpp:403`, duplicated block 251 | | `test/unit/light/unit_PolarLut_equivalence.cpp` | 3 | `lines 40-55, also in test/unit/light/unit_NoiseEffect.cpp:33`, duplicated block 16 | | `test/unit/light/unit_PongEffect.cpp` | 3 | `lines 80-91, also in test/unit/light/unit_PongEffect.cpp:10`, duplicated block 12 | | `test/unit/light/unit_RmtLedDriver_lifecycle.cpp` | 3 | `lines 19-30, also in test/unit/light/unit_RmtLedDriver_pins.cpp:17`, duplicated block 12 | @@ -254,7 +254,6 @@ The next function worth simplifying is at the top of each file's list. | `src/platform/esp32/moonlive_asm_riscv.h` | 2 | `lines 70-87, also in src/platform/esp32/moonlive_asm_xtensa.h:74`, duplicated block 18 | | `src/platform/esp32/moonlive_emit_riscv.cpp` | 2 | `lines 51-56, also in src/platform/esp32/moonlive_emit_riscv.cpp:24`, duplicated block 6 | | `src/platform/teensy/moonlive_asm_thumb2.cpp` | 2 | `lines 66-71, also in src/platform/esp32/moonlive_asm_xtensa.cpp:189`, duplicated block 6 | -| `src/platform/teensy/platform_teensy.cpp` | 2 | `platform::resolveHostStep`, complex function 16 | | `test/js/ui-midi-bridge.test.mjs` | 2 | `lines 4-16, also in test/js/ui-gamepad-bridge.test.mjs:4`, duplicated block 13 | | `test/unit/core/unit_AudioService_sync.cpp` | 2 | `lines 313-322, also in test/unit/core/unit_AudioService_sync.cpp:269`, duplicated block 10 | | `test/unit/core/unit_DevicesModule_hue.cpp` | 2 | `lines 63-69, also in test/unit/core/unit_DevicesModule_hue.cpp:42`, duplicated block 7 | @@ -274,6 +273,7 @@ The next function worth simplifying is at the top of each file's list. | `test/unit/light/unit_MoonLiveScriptResolve.cpp` | 2 | `lines 90-97, also in test/unit/light/unit_MoonLiveScriptResolve.cpp:78`, duplicated block 8 | | `test/unit/light/unit_NetworkReceiveEffect_protocols.cpp` | 2 | `lines 23-44, also in test/unit/light/unit_NetworkReceiveEffect.cpp:23`, duplicated block 22 | | `test/unit/light/unit_NoiseEffect.cpp` | 2 | `lines 33-48, also in test/unit/light/unit_PolarLut_equivalence.cpp:40`, duplicated block 16 | +| `test/unit/light/unit_PictureEffect.cpp` | 2 | `lines 218-226, also in test/unit/light/unit_PictureEffect.cpp:205`, duplicated block 9 | | `test/unit/light/unit_Scroll.cpp` | 2 | `lines 6-20, also in test/unit/light/unit_Splat.cpp:6`, duplicated block 15 | | `test/unit/light/unit_SineEffect.cpp` | 2 | `lines 22-30, also in test/unit/light/unit_DistortionWavesEffect.cpp:26`, duplicated block 9 | | `test/unit/light/unit_SingleColumnLayout.cpp` | 2 | `lines 12-25, also in test/unit/light/unit_GridLayout.cpp:8`, duplicated block 14 | @@ -346,7 +346,8 @@ The next function worth simplifying is at the top of each file's list. | `src/platform/esp32/platform_esp32_sdmmc.cpp` | 1 | `platform::mountCard`, complex function 13 | | `src/platform/esp32/platform_esp32_tasks.cpp` | 1 | `platform::taskSnapshot`, complex function 12 | | `src/platform/esp32/platform_esp32_usb_midi.cpp` | 1 | `lines 253-259, also in src/platform/teensy/platform_teensy.cpp:767`, duplicated block 7 | -| `src/platform/platform.h` | 1 | `platform.h`, large file 1448 | +| `src/platform/esp32/platform_esp32_usbvideo.cpp` | 1 | `lines 10-17, also in src/platform/teensy/platform_teensy.cpp:774`, duplicated block 8 | +| `src/platform/platform.h` | 1 | `platform.h`, large file 1492 | | `src/platform/teensy/platform_teensy_gpio_dma.cpp` | 1 | `lines 310-315, also in src/platform/esp32/platform_esp32_parlio.cpp:286`, duplicated block 6 | | `src/platform/teensy/platform_teensy_improv.cpp` | 1 | `platform::improvProvisioningInit`, long parameter list 9 | | `src/ui/install-picker.js` | 1 | `install-picker.js`, large file 1048 | @@ -372,6 +373,7 @@ The next function worth simplifying is at the top of each file's list. | `test/unit/light/unit_GridLayout.cpp` | 1 | `lines 8-21, also in test/unit/light/unit_SingleColumnLayout.cpp:12`, duplicated block 14 | | `test/unit/light/unit_Layer_persistence.cpp` | 1 | `lines 45-51, also in test/unit/light/unit_TextEffect.cpp:18`, duplicated block 7 | | `test/unit/light/unit_Layer_phase_animation.cpp` | 1 | `lines 77-85, also in test/unit/light/unit_RandomMapModifier.cpp:115`, duplicated block 9 | +| `test/unit/light/unit_MoonLiveLayout.cpp` | 1 | `unit_MoonLiveLayout.cpp`, large file 1113 | | `test/unit/light/unit_NetworkReceiveEffect.cpp` | 1 | `lines 23-44, also in test/unit/light/unit_NetworkReceiveEffect_protocols.cpp:23`, duplicated block 22 | | `test/unit/light/unit_ParallelLedDriver_doublebuffer.cpp` | 1 | `lines 93-104, also in test/unit/light/unit_ParallelLedDriver_pinexpander.cpp:94`, duplicated block 12 | | `test/unit/light/unit_Rings241Layout.cpp` | 1 | `lines 18-30, also in test/unit/light/unit_SingleColumnLayout.cpp:19`, duplicated block 13 | diff --git a/esp32/main/CMakeLists.txt b/esp32/main/CMakeLists.txt index 3eb1d04b..4c2ae852 100644 --- a/esp32/main/CMakeLists.txt +++ b/esp32/main/CMakeLists.txt @@ -44,6 +44,7 @@ idf_component_register( "../../src/platform/esp32/platform_esp32_i2c.cpp" "../../src/platform/esp32/platform_esp32_ir.cpp" "../../src/platform/esp32/platform_esp32_usb_midi.cpp" + "../../src/platform/esp32/platform_esp32_usbvideo.cpp" "../../src/platform/esp32/platform_esp32_es8311.cpp" INCLUDE_DIRS "../../src" diff --git a/src/core/services/PictureService.h b/src/core/services/PictureService.h index f9c84e22..d0e0a6ab 100644 --- a/src/core/services/PictureService.h +++ b/src/core/services/PictureService.h @@ -3,11 +3,13 @@ #include "core/module/MoonModule.h" #include "core/util/ActiveInstance.h" // the one-active-source election #include "core/util/color.h" // RGB +#include "core/util/format.h" // formatTo: the status line built off snprintf #include "core/util/PictureFrame.h" #include "core/util/ScratchBuffer.h" #include "core/util/TryLock.h" // the swap a reader holds off for its gather #include "platform/platform.h" // maxAllocBlock, freeHeap: whether the second picture fits +#include #include #include @@ -22,20 +24,26 @@ class PictureService : public MoonModule { /// A service, the source half of the picture pipeline. It swaps a front and a back picture, or locks the one picture when there is no room for two, so frames skip but never tear. ModuleRole role() const MM_NONBLOCKING override { return ModuleRole::Service; } + /// Stop any decode first: a task holding this service must never outlive it, even unreleased. + ~PictureService() override { stopCapture(); } + /// The sources `source` picks from, in option order. - enum Source : uint8_t { kTestPattern, kSourceCount }; + enum Source : uint8_t { kTestPattern, kUsbCapture, kSourceCount }; /// Where the picture comes from. - /// @guide The test pattern is a test card in the layout of Philips' PM5544, drawn at any size. + /// @guide The test pattern is a test card in the layout of Philips' PM5544, drawn at any size. **usb capture** decodes MJPEG from an HDMI grabber, scaled to fit. uint8_t source = kTestPattern; - /// The picture's width in pixels. + /// Frames a second asked of a capture device, which streams the nearest rate it offers. + uint8_t fps = 30; + /// The picture's width in pixels, and the most a capture scales its frames to. uint16_t width = 64; - /// The picture's height in pixels. + /// The picture's height in pixels, under the same ceiling as the width. uint16_t height = 48; /// Publish the source and the picture's size. void defineControls() override { - static constexpr const char* kSources[] = {"test pattern"}; + static constexpr const char* kSources[] = {"test pattern", "usb capture"}; controls_.addSelect("source", source, kSources, kSourceCount); + if (platform::hasUsbVideo) controls_.addControl("fps", fps, 1, 120); controls_.addControl("width", width, 1, kMaxSide); controls_.addControl("height", height, 1, kMaxSide); } @@ -46,7 +54,9 @@ class PictureService : public MoonModule { /// Size the pixels for the source and draw a still one into them; a failed allocation publishes no picture. void prepare() override { seat_.claim(); + stopCapture(); // before the pictures move under a decode still writing one if (!prepareBuffers(kRunsInTask[source])) return; + if (source == kUsbCapture) { startCapture(); return; } drawTestPattern(front_, width, height); // the frame prepareBuffers published, filled before any reader ticks } @@ -61,6 +71,8 @@ class PictureService : public MoonModule { return false; } front_ = a_.data(); + bufW_ = width; + bufH_ = height; back_ = writerTask && secondFits(bytes) && b_.resize(bytes) ? b_.data() : nullptr; if (!back_) b_.resize(0); if (writerTask && !back_) setStatus("one picture buffer: not enough memory, a capture may skip frames", Severity::Warning); @@ -78,27 +90,48 @@ class PictureService : public MoonModule { while (!lock_.tryAcquire()) platform::yield(); // readers hold it for one gather at most return front_; } - /// Publish what beginWrite handed out: swap the two pictures, or unlock the one, and bump the generation. - void endWrite() { + /// Publish what beginWrite handed out at the size written, for a source that cannot hit the one it asked for; a size past the picture's is clamped to it. + void endWrite(uint16_t w, uint16_t h) { + if (w > bufW_) w = bufW_; + if (h > bufH_) h = bufH_; if (back_) { while (!lock_.tryAcquire()) platform::yield(); uint8_t* filled = back_; back_ = front_; front_ = filled; } - frame_ = PictureFrame{front_, width, height, frame_.generation + 1}; + frame_ = PictureFrame{front_, w, h, frame_.generation + 1}; lock_.release(); } + /// Publish what beginWrite handed out: swap the two pictures, or unlock the one, and bump the generation. + void endWrite() { endWrite(bufW_, bufH_); } + /// Give back what beginWrite handed out without publishing it; with one picture, what a failed decode already wrote stays in it. + void abortWrite() { if (!back_) lock_.release(); } /// Re-claim an empty seat, so removing the active service lets a survivor take over. void tick() MM_NONBLOCKING override { seat_.claim(); } /// Vacate the seat for any survivor. void release() override { + stopCapture(); seat_.vacate(); MoonModule::release(); } + /// Name the capture's health once a second: the size it publishes, the frames that landed and those lost. + void tick1s() MM_NONBLOCKING override { + if (source != kUsbCapture || !task_.impl) return; + if (!lock_.tryAcquire()) return; // the decode task is publishing: the next second reads it + const unsigned w = frame_.width, h = frame_.height; + const bool one = !back_; // the writer swaps back_ under the lock + lock_.release(); + const platform::VideoCaptureStats st = platform::videoCaptureStats(); + const uint32_t lost = st.dropped + st.failed; + mm::formatTo(status_, sizeof(status_), "%u x %u, %u frames, %u lost%s", w, h, + static_cast(st.decoded), static_cast(lost), one ? ", one picture" : ""); + setStatus(status_, lost || one ? Severity::Warning : Severity::Status); + } + /// What a consumer reads with no source: a class constant, since a function-local static needs a guard on first use. static constexpr PictureFrame kNone{}; @@ -208,7 +241,49 @@ class PictureService : public MoonModule { static constexpr uint16_t kMaxSide = 512; ///< a side above this exceeds what a board holds for one picture /// Which sources write from a task of their own, in `source` order; the test pattern is drawn at prepare. - static constexpr bool kRunsInTask[kSourceCount] = {false}; + static constexpr bool kRunsInTask[kSourceCount] = {false, true}; + /// How long the task waits for a frame before checking for a stop, which bounds how long a control change waits on it. + static constexpr uint32_t kFrameWaitMs = 50; + + /// Open the grabber and decode on a task of its own, so the render loop never waits on the device. + void startCapture() { + if (!platform::videoCaptureInit(capture_, width, height, fps)) { + // The format list outlives a failed open, so a device that enumerated but refused is a different fault. + setStatus(platform::videoCaptureFormatGeneration() ? "the grabber offers no mode that fits" : "no grabber attached", Severity::Warning); + return; + } + if (back_) clearStatus(); // before the task swaps back_; with one picture, prepareBuffers' warning stays until the first status line + stop_.store(false, std::memory_order_release); + // Core 1, beside the USB host's own tasks, so a decode never lands on the render core. + if (!platform::spawnPinnedTask(task_, "mmPicCap", &captureTrampoline, this, 4096, 3, 1)) { + platform::videoCaptureDeinit(capture_); + setStatus("no task for the capture", Severity::Error); + return; + } + } + + /// Stop the decode and release the device, in that order, so nothing writes a picture that has moved. + void stopCapture() { + if (task_.impl) { + stop_.store(true, std::memory_order_release); + platform::stopPinnedTask(task_); + } + if (capture_.impl) platform::videoCaptureDeinit(capture_); + } + + static void captureTrampoline(void* self) { static_cast(self)->runCapture(); } + + /// Wait for a frame, then decode it straight into the back picture and publish it, until prepare or release stops it. + void runCapture() { + while (!stop_.load(std::memory_order_acquire)) { + // The wait comes before beginWrite, so with one picture the lock covers the decode alone and readers sample between frames. + if (!platform::videoCaptureAwaitFrame(capture_, kFrameWaitMs)) continue; + uint16_t w = 0, h = 0; + uint8_t* px = beginWrite(); + if (platform::videoCaptureDecodeInto(capture_, px, bufW_, bufH_, w, h)) endWrite(w, h); // the size the pictures hold, never a control a write changed since + else abortWrite(); // a refused frame: the lights keep the one they have rather than a new generation of nothing + } + } /// Whether an optional picture of `bytes` fits one free block with the heap's reserve intact, the rule the mapping uses; 0 from the platform means no limit. static bool secondFits(size_t bytes) { @@ -222,7 +297,12 @@ class PictureService : public MoonModule { uint8_t* front_ = nullptr; ///< what readers see uint8_t* back_ = nullptr; ///< what a writer fills, or null with one picture PictureFrame frame_; ///< the front picture, as a Reader hands it out + uint16_t bufW_ = 0, bufH_ = 0; ///< the size the pictures were allocated at, which a writer fills; the controls may already say otherwise TryLock lock_; ///< held by a reader for its gather, by the writer to swap or to fill the only picture + platform::VideoCaptureHandle capture_; ///< the open grabber, or an empty handle + platform::WorkerTask task_; ///< the decode task, or an empty handle + std::atomic stop_{false}; ///< tells the decode task to return + char status_[64] = {}; ///< the capture's status line, rebuilt once a second ActiveInstance seat_{*this}; }; diff --git a/src/platform/desktop/platform_config.h b/src/platform/desktop/platform_config.h index 49223ec1..16e6c867 100644 --- a/src/platform/desktop/platform_config.h +++ b/src/platform/desktop/platform_config.h @@ -88,6 +88,8 @@ constexpr bool hasHls = true; constexpr bool hasRtsp = true; /// False: a desk on the computer reaches the MIDI service through the browser. constexpr bool hasUsbMidiHost = false; +/// False: no grabber on a computer, so only the tests drive the stand-in, as USB MIDI does. +constexpr bool hasUsbVideo = false; /// True: ffmpeg offers several encoders, so the pick is the user's. constexpr bool hasEncoderChoice = true; /// True: ffmpeg writes the playlist to disk, so the server serves segments as files. diff --git a/src/platform/desktop/platform_desktop.cpp b/src/platform/desktop/platform_desktop.cpp index ca3822f0..66426ace 100644 --- a/src/platform/desktop/platform_desktop.cpp +++ b/src/platform/desktop/platform_desktop.cpp @@ -2863,6 +2863,100 @@ size_t takeTestUsbMidiSent(uint8_t (*packets)[4], size_t max) { return n; } +// No grabber on the computer, so USB video here is a stand-in that the capture tests drive. The test thread and the capture task share it, so what both touch is atomic. +namespace { +struct TestGrabber { + std::vector formats; ///< written by a test while no capture runs + std::atomic generation{0}; + uint16_t openW = 0, openH = 0; ///< the mode videoCaptureInit settled on, before the task starts + uint8_t openFps = 0; ///< and the rate it delivers at + std::atomic r{0}, g{0}, b{0}; ///< what a decode writes into every pixel + std::atomic decodeFails{false}; + std::atomic open{false}; + std::atomic decoded{0}, failed{0}; +} g_grabber; +// How far `have` is from `want`, so the nearest advertised mode can be picked without . +long apart(long have, long want) { return have > want ? have - want : want - have; } +// Halve `w` x `h` until it fits `maxW` x `maxH`, which is the scaling the decode hardware offers; false when even one pixel does not fit. +bool halveToFit(uint16_t& w, uint16_t& h, uint16_t maxW, uint16_t maxH) { + while ((w > maxW || h > maxH) && w > 1 && h > 1) { w = uint16_t(w / 2); h = uint16_t(h / 2); } + return w <= maxW && h <= maxH; +} +} // namespace + +uint32_t videoCaptureFormatGeneration() MM_NONBLOCKING { return g_grabber.generation.load(); } + +bool videoCaptureInit(VideoCaptureHandle& h, uint16_t width, uint16_t height, uint8_t fps) { + if (g_grabber.formats.empty() || g_grabber.open) return false; // one owner at a time, as a device is claimed once + // The nearest advertised mode by area then by rate, as a grabber settles a request it cannot meet. + const VideoCaptureFormat* best = nullptr; + long bestCost = 0; + for (const VideoCaptureFormat& f : g_grabber.formats) { + const long cost = apart(long(f.width) * f.height, long(width) * height) + apart(f.fps, fps); + if (!best || cost < bestCost) { best = &f; bestCost = cost; } + } + g_grabber.openW = best->width; + g_grabber.openH = best->height; + g_grabber.openFps = best->fps; + g_grabber.decoded = 0; + g_grabber.failed = 0; + g_grabber.open = true; + h.impl = &g_grabber; + return true; +} + +bool videoCaptureAwaitFrame(VideoCaptureHandle& h, uint32_t timeoutMs) { + if (!h.impl || !g_grabber.open) return false; + // A frame every period of the mode's rate; a period longer than the wait answers no frame yet. + const uint32_t period = g_grabber.openFps ? 1000u / g_grabber.openFps : 33u; + std::this_thread::sleep_for(std::chrono::milliseconds(period < timeoutMs ? period : timeoutMs)); + return period <= timeoutMs; +} + +bool videoCaptureDecodeInto(VideoCaptureHandle& h, uint8_t* dst, uint16_t maxWidth, uint16_t maxHeight, + uint16_t& outWidth, uint16_t& outHeight) { + if (!h.impl || !g_grabber.open) return false; + if (g_grabber.decodeFails) { g_grabber.failed++; return false; } + uint16_t w = g_grabber.openW, hh = g_grabber.openH; + if (!halveToFit(w, hh, maxWidth, maxHeight)) { g_grabber.failed++; return false; } + const uint8_t r = g_grabber.r, g = g_grabber.g, b = g_grabber.b; + for (size_t i = 0, px = size_t(w) * hh; i < px; i++) { + dst[i * 3 + 0] = r; + dst[i * 3 + 1] = g; + dst[i * 3 + 2] = b; + } + outWidth = w; + outHeight = hh; + g_grabber.decoded++; + return true; +} + +void videoCaptureDeinit(VideoCaptureHandle& h) { + if (h.impl == &g_grabber) g_grabber.open = false; // only the owning handle closes it + h.impl = nullptr; +} + +VideoCaptureStats videoCaptureStats() MM_NONBLOCKING { + VideoCaptureStats s; + s.decoded = g_grabber.decoded.load(); + s.failed = g_grabber.failed.load(); + return s; +} + +void setTestVideoCaptureFormats(const VideoCaptureFormat* formats, size_t count) { + g_grabber.formats.assign(formats, formats + count); + g_grabber.generation = count ? g_grabber.generation + 1 : 0; + if (!count) g_grabber.open = false; +} + +void setTestVideoCaptureColor(uint8_t r, uint8_t g, uint8_t b) { + g_grabber.r = r; + g_grabber.g = g; + g_grabber.b = b; +} + +void setTestVideoCaptureDecodeFails(bool fail) { g_grabber.decodeFails = fail; } + // Video output, resolved on demand and never linked: @xref{the-video-runtimes-structures-are-transcribed-not-included|why the declarations below must not be tidied}. namespace { @@ -2965,7 +3059,7 @@ bool ndiLoad() { } // NDIlib_initialize returns false when the CPU is unsupported, a real "cannot use it" that must not read as "installed and working". if (!ndiInit_()) { ndiSendVideo_ = nullptr; return false; } - // NDIlib_destroy pairs with NDIlib_initialize at process exit, since a sender may reopen later in the same run. Only with no sender open: a quit closes it first, while a reboot exits with the render thread still sending, where the process ending is the teardown. + // NDIlib_destroy pairs with NDIlib_initialize at process exit, since a sender may reopen later in the same run. Only with no sender open: a reboot exits with the render thread still sending, and the process ending is the teardown. std::atexit([]() { if (ndiDestroy_ && !ndiSender_) ndiDestroy_(); }); return true; } diff --git a/src/platform/esp32/platform_config.h b/src/platform/esp32/platform_config.h index 599ea23f..f8739faa 100644 --- a/src/platform/esp32/platform_config.h +++ b/src/platform/esp32/platform_config.h @@ -209,6 +209,13 @@ constexpr bool hasUsbMidiHost = true; #else constexpr bool hasUsbMidiHost = false; #endif + +// The UVC host and the JPEG decoder cost flash, so only the images that set the symbol carry them. +#if defined(CONFIG_MM_USB_VIDEO) +constexpr bool hasUsbVideo = true; +#else +constexpr bool hasUsbVideo = false; +#endif /// False: one hardware encoder, so there is nothing to choose and the control stays hidden. constexpr bool hasEncoderChoice = false; /// False: segments live in a PSRAM ring, flash wear buying nothing for a file stale within seconds. diff --git a/src/platform/esp32/platform_esp32_usbvideo.cpp b/src/platform/esp32/platform_esp32_usbvideo.cpp new file mode 100644 index 00000000..2f60a31b --- /dev/null +++ b/src/platform/esp32/platform_esp32_usbvideo.cpp @@ -0,0 +1,19 @@ +/// USB video capture on the ESP32: the stubs every image without `CONFIG_MM_USB_VIDEO` links, which report no device. + +#include "platform/platform.h" + +namespace mm::platform { + +#if !defined(CONFIG_MM_USB_VIDEO) + +// An image built without the capture reports no device, which the Picture service shows as a status. +uint32_t videoCaptureFormatGeneration() MM_NONBLOCKING { return 0; } +bool videoCaptureInit(VideoCaptureHandle&, uint16_t, uint16_t, uint8_t) { return false; } +bool videoCaptureAwaitFrame(VideoCaptureHandle&, uint32_t) { return false; } +bool videoCaptureDecodeInto(VideoCaptureHandle&, uint8_t*, uint16_t, uint16_t, uint16_t&, uint16_t&) { return false; } +void videoCaptureDeinit(VideoCaptureHandle&) {} +VideoCaptureStats videoCaptureStats() MM_NONBLOCKING { return VideoCaptureStats{}; } + +#endif + +} // namespace mm::platform diff --git a/src/platform/platform.h b/src/platform/platform.h index ba4d90d5..59accf3a 100644 --- a/src/platform/platform.h +++ b/src/platform/platform.h @@ -1445,4 +1445,48 @@ void injectTestUsbMidi(const uint8_t (*packets)[4], size_t count); /// Take up to `max` of the packets sent to the test desk, oldest first; the count taken. size_t takeTestUsbMidiSent(uint8_t (*packets)[4], size_t max); +// --- USB video capture, gated by `hasUsbVideo`: an HDMI grabber presenting itself as a webcam ---- + +/// Opaque per-device state, owned by the platform between init and deinit. +struct VideoCaptureHandle { void* impl = nullptr; }; + +/// One mode a grabber advertises, so a source offers what the device has rather than guesses. +struct VideoCaptureFormat { + uint16_t width = 0; ///< frame width in pixels + uint16_t height = 0; ///< frame height in pixels + uint8_t fps = 0; ///< frames a second offered at that size +}; + +/// Bumped whenever that list is rewritten, which is when a device enumerates; 0 until then. +uint32_t videoCaptureFormatGeneration() MM_NONBLOCKING; + +/// Claim the first grabber and stream MJPEG near this mode; false when none is attached or none fits. +bool videoCaptureInit(VideoCaptureHandle& h, uint16_t width, uint16_t height, uint8_t fps); + +/// Wait up to `timeoutMs` for the next frame; true once one is ready to decode. +bool videoCaptureAwaitFrame(VideoCaptureHandle& h, uint32_t timeoutMs); + +/// Decode the frame videoCaptureAwaitFrame announced into `dst`, scaled down to fit `maxWidth` x `maxHeight` x 3 bytes, top row first. +bool videoCaptureDecodeInto(VideoCaptureHandle& h, uint8_t* dst, uint16_t maxWidth, uint16_t maxHeight, + uint16_t& outWidth, uint16_t& outHeight); + +/// Release the device and free its buffers. +void videoCaptureDeinit(VideoCaptureHandle& h); + +/// Why frames did not reach the lights, cumulative since the device was opened. +struct VideoCaptureStats { + uint32_t decoded = 0; ///< frames decoded into a picture + uint32_t dropped = 0; ///< frames that arrived while the one before was still being decoded + uint32_t failed = 0; ///< frames the decoder refused, a mis-read payload stride among them +}; +/// The counters above, read atomically enough for a once-a-second status line. +VideoCaptureStats videoCaptureStats() MM_NONBLOCKING; + +/// Attach a stand-in grabber advertising these modes, so a capture source is host-testable; 0 detaches. +void setTestVideoCaptureFormats(const VideoCaptureFormat* formats, size_t count); +/// Make the stand-in decode every pixel this color, so a test tells its picture from the test card. +void setTestVideoCaptureColor(uint8_t r, uint8_t g, uint8_t b); +/// Make the stand-in refuse to decode, as a bad bitstream does; false lets it decode again. +void setTestVideoCaptureDecodeFails(bool fail); + } // namespace mm::platform diff --git a/src/platform/teensy/platform_config.h b/src/platform/teensy/platform_config.h index c93b4881..8b90f3a0 100644 --- a/src/platform/teensy/platform_config.h +++ b/src/platform/teensy/platform_config.h @@ -72,6 +72,8 @@ constexpr bool hasHls = false; constexpr bool hasRtsp = false; /// No USB MIDI host. constexpr bool hasUsbMidiHost = false; +/// No USB host, so no video grabber. +constexpr bool hasUsbVideo = false; /// One encoder at most. constexpr bool hasEncoderChoice = false; /// No segment files. diff --git a/src/platform/teensy/platform_teensy.cpp b/src/platform/teensy/platform_teensy.cpp index e991c7b7..3d033cc7 100644 --- a/src/platform/teensy/platform_teensy.cpp +++ b/src/platform/teensy/platform_teensy.cpp @@ -770,4 +770,12 @@ bool usbMidiConnected() { return false; } size_t usbMidiRead(uint8_t (*)[4], size_t) { return 0; } bool usbMidiWrite(const uint8_t (*)[4], size_t) { return false; } +// No USB host on this chip, so a capture source finds no device and says so in its status. +uint32_t videoCaptureFormatGeneration() MM_NONBLOCKING { return 0; } +bool videoCaptureInit(VideoCaptureHandle&, uint16_t, uint16_t, uint8_t) { return false; } +bool videoCaptureAwaitFrame(VideoCaptureHandle&, uint32_t) { return false; } +bool videoCaptureDecodeInto(VideoCaptureHandle&, uint8_t*, uint16_t, uint16_t, uint16_t&, uint16_t&) { return false; } +void videoCaptureDeinit(VideoCaptureHandle&) {} +VideoCaptureStats videoCaptureStats() MM_NONBLOCKING { return VideoCaptureStats{}; } + } // namespace mm::platform diff --git a/test/unit/core/unit_PictureService.cpp b/test/unit/core/unit_PictureService.cpp index dbfefe96..f3e826e1 100644 --- a/test/unit/core/unit_PictureService.cpp +++ b/test/unit/core/unit_PictureService.cpp @@ -6,6 +6,7 @@ #include "core/services/PictureService.h" #include +#include #include #include #include @@ -124,3 +125,157 @@ TEST_CASE("without room for a second picture the service degrades to one, locked CHECK(std::string(p.svc.status()) == "one picture buffer: not enough memory, a capture may skip frames"); writeWhileReading(p, 300); } + +namespace { +// A stand-in grabber advertising one mode, detached again when the test ends. +struct Grabber { + Grabber(uint16_t w, uint16_t h, uint8_t fps, uint8_t r, uint8_t g, uint8_t b) { + const mm::platform::VideoCaptureFormat f{w, h, fps}; + mm::platform::setTestVideoCaptureFormats(&f, 1); + mm::platform::setTestVideoCaptureColor(r, g, b); + } + ~Grabber() { + mm::platform::setTestVideoCaptureFormats(nullptr, 0); + mm::platform::setTestVideoCaptureDecodeFails(false); + } +}; +// The published generation, or `was` while the writer holds the swap, so a poll never fails on timing. +uint32_t generationNow(uint32_t was) { + const mm::PictureService::Reader r; + return r ? r.frame().generation : was; +} +// Whether a picture past `was` arrives within two seconds, which a decode on its own task has ample time for. +bool publishedPast(uint32_t was) { + for (int i = 0; i < 400; i++) { + if (generationNow(was) != was) return true; + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + return false; +} +// Whether the grabber has answered `n` decodes, passed or refused, within two seconds: the task's own count rather than a guess at its pace. +bool decodedAtLeast(uint32_t n) { + for (int i = 0; i < 400; i++) { + const mm::platform::VideoCaptureStats st = mm::platform::videoCaptureStats(); + if (st.decoded + st.failed >= n) return true; + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + return false; +} +// The status line once tick1s finds the lock free, which a publishing task holds only for a swap. +std::string captureStatus(mm::PictureService& svc) { + for (int i = 0; i < 400; i++) { + svc.tick1s(); + if (svc.status() && std::strchr(svc.status(), 'x')) return svc.status(); + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + return {}; +} +// The capture source, started on a service sized `w` x `h`; false when no picture arrived. +bool startCapture(Picture& p, uint32_t& was) { + p.svc.source = mm::PictureService::kUsbCapture; + p.svc.prepare(); + was = current().generation; // prepareBuffers published the resized picture; a frame comes after it + return publishedPast(was); +} +} // namespace + +// A 640 x 480 grabber on a 256 x 256 picture: halved to 160 x 120, every pixel the color it decoded. +TEST_CASE("the usb capture source publishes the grabber's frames, scaled to fit the picture") { + Grabber g(640, 480, 30, 10, 200, 90); + Picture p(256, 256); + uint32_t was = 0; + REQUIRE(startCapture(p, was)); + const mm::PictureFrame f = current(); + CHECK(f.width == 160); + CHECK(f.height == 120); + const uint8_t* px = pixel(f, 80, 60); + CHECK(px[0] == 10); + CHECK(px[1] == 200); + CHECK(px[2] == 90); + p.svc.release(); +} + +// A board whose USB port has nothing on it says so, rather than failing or waiting on a device. +TEST_CASE("the usb capture source without a grabber reports it") { + mm::platform::setTestVideoCaptureFormats(nullptr, 0); + Picture p(64, 48); + p.svc.source = mm::PictureService::kUsbCapture; + p.svc.prepare(); + REQUIRE(p.svc.status() != nullptr); + CHECK(std::string(p.svc.status()) == "no grabber attached"); + p.svc.release(); +} + +// A refused bitstream leaves the lights the picture they have, rather than a new generation of nothing. +TEST_CASE("a frame the decoder refuses publishes nothing, and recovery publishes again") { + Grabber g(320, 240, 30, 1, 2, 3); + mm::platform::setTestVideoCaptureDecodeFails(true); + Picture p(64, 48); + p.svc.source = mm::PictureService::kUsbCapture; + p.svc.prepare(); + const uint32_t settled = current().generation; + REQUIRE(decodedAtLeast(3)); // three refusals answered + CHECK(generationNow(settled) == settled); + mm::platform::setTestVideoCaptureDecodeFails(false); + CHECK(publishedPast(settled)); + p.svc.release(); +} + +// The status names the size it publishes and what it lost, so a climbing loss is visible without a log. +TEST_CASE("the capture status counts the frames that landed and those lost") { + Grabber g(320, 240, 30, 5, 5, 5); + Picture p(64, 48); + uint32_t was = 0; + REQUIRE(startCapture(p, was)); + const std::string line = captureStatus(p.svc); + CHECK(line.rfind("40 x 30, ", 0) == 0); // 320 x 240 halved three times to fit 64 x 48 + CHECK(line.find(" frames, 0 lost") != std::string::npos); + CHECK(line.find(" 0 frames") == std::string::npos); // the picture that arrived is counted + CHECK(line.find("one picture") == std::string::npos); + CHECK(p.svc.severity() == mm::MoonModule::Severity::Status); + p.svc.release(); +} + +// Short of memory for a second picture, the capture still publishes and readers still read. The task holds the one picture only while it decodes, never while it waits for a frame. +TEST_CASE("the usb capture with one picture publishes, and the status says it runs on one") { + Grabber g(320, 240, 30, 7, 7, 7); + Picture p(64, 48); + mm::platform::setTestMaxAllocBlock(64u * 48u * 3u - 1); // the optional picture finds no block + uint32_t was = 0; + const bool published = startCapture(p, was); + mm::platform::setTestMaxAllocBlock(0); + REQUIRE(published); + CHECK_FALSE(p.svc.twoBuffers()); + int reads = 0; + const uint32_t target = mm::platform::videoCaptureStats().decoded + 3; + while (mm::platform::videoCaptureStats().decoded < target) { + const mm::PictureService::Reader r; + if (r) reads++; + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + CHECK(reads > 0); + CHECK(captureStatus(p.svc).find(", one picture") != std::string::npos); + CHECK(p.svc.severity() == mm::MoonModule::Severity::Warning); + p.svc.release(); +} + +// Switching back stops the decode and draws the card, so the grabber is released rather than still writing. +TEST_CASE("switching away from the capture stops its task") { + Grabber g(320, 240, 30, 0, 0, 0); + Picture p(160, 120); + uint32_t was = 0; + REQUIRE(startCapture(p, was)); + p.svc.source = mm::PictureService::kTestPattern; + p.svc.prepare(); + mm::platform::VideoCaptureHandle again; + CHECK(mm::platform::videoCaptureInit(again, 320, 240, 30)); // the grabber was let go, so it opens anew + mm::platform::videoCaptureDeinit(again); + const mm::PictureFrame f = current(); + CHECK(f.width == 160); + CHECK(f.height == 120); + const uint8_t* px = pixel(f, 5, 2); // the card's grid gray, not the black the grabber decoded + CHECK(px[0] == 128); + CHECK(px[1] == 128); + CHECK(px[2] == 128); + p.svc.release(); +}