feature: the hardware-free half of USB video capture

src/core/util/AreaSample.h

@@ -0,0 +1,47 @@
+#pragma once
+
+/// Sampling a picture's rectangle onto one output cell: the proportional span and its mean in linear light.
+
+#include "core/util/color.h"    // RGB
+#include "core/util/math16.h"   // isqrt: the root the linear-light mean undoes its squares with
+
+#include <cstddef>
+#include <cstdint>
+
+namespace mm {
+
+/// The source pixels one output cell covers, `hi` one past the last.
+struct Span {
+    uint16_t lo = 0;   ///< the first source pixel the cell covers
+    uint16_t hi = 0;   ///< one past the last, so hi - lo is the count
+};
+
+/// Taps an axis of one cell, so a cell's sum stays well inside 32 bits whatever the source size.
+constexpr uint32_t kAreaTapsPerAxis = 4;
+
+/// The source span output cell `i` of `out` covers, the proportional split every sampler shares.
+inline Span cellSpan(uint32_t i, uint32_t out, uint32_t src) {
+    uint32_t lo = i * src / out, hi = (i + 1u) * src / out;
+    if (hi <= lo) hi = lo + 1;   // a source smaller than the output: neighboring cells share a pixel
+    return Span{static_cast<uint16_t>(lo), static_cast<uint16_t>(hi)};
+}
+
+/// The rectangle's mean in linear light (square, average, root: gamma 2 for the display curve), from at most four reads per axis.
+inline RGB areaMean(const uint8_t* rgb, size_t stride, Span cx, Span ry) {
+    const uint32_t sx = (cx.hi - cx.lo + kAreaTapsPerAxis - 1u) / kAreaTapsPerAxis;
+    const uint32_t sy = (ry.hi - ry.lo + kAreaTapsPerAxis - 1u) / kAreaTapsPerAxis;
+    uint32_t sum[3] = {0, 0, 0}, taps = 0;
+    for (uint32_t y = ry.lo; y < ry.hi; y += sy) {
+        const uint8_t* row = rgb + static_cast<size_t>(y) * stride;
+        for (uint32_t x = cx.lo; x < cx.hi; x += sx) {
+            const uint8_t* p = row + static_cast<size_t>(x) * 3u;
+            for (int c = 0; c < 3; c++) sum[c] += static_cast<uint32_t>(p[c]) * p[c];
+            taps++;
+        }
+    }
+    if (!taps) return RGB{0, 0, 0};
+    return RGB{static_cast<uint8_t>(isqrt(sum[0] / taps)), static_cast<uint8_t>(isqrt(sum[1] / taps)),
+               static_cast<uint8_t>(isqrt(sum[2] / taps))};
+}
+
+}   // namespace mm

src/light/effects/PictureEffect.h

@@ -2,6 +2,7 @@
 
 #include "light/effects/EffectBase.h"
 #include "core/services/PictureService.h"   // the picture it samples, as audio effects include AudioService
+#include "core/util/AreaSample.h"            // cellSpan, areaMean: the shrink every sampler shares
 
 namespace mm {
 
@@ -105,7 +106,7 @@ class PictureEffect : public EffectBase {
 
 private:
     /// One axis of a cell's source rectangle, [lo, hi) in picture pixels; empty outside a fitted picture.
-    struct Range { uint16_t lo, hi; };
+    using Range = Span;
 
     /// An edge cell's range at `deep` picture lines in: the first reaches inward from 0, the last back from the end.
     static void reachEdge(int32_t k, int32_t used, uint32_t pic, uint32_t deep, uint32_t& lo, uint32_t& hi) {
@@ -126,9 +127,10 @@ class PictureEffect : public EffectBase {
         for (uint32_t i = 0; i < n; i++) {
             const int32_t k = static_cast<int32_t>(i) - offset;
             if (k < 0 || k >= used) { out[i] = {0, 0}; continue; }
-            uint32_t lo = static_cast<uint32_t>(k) * pic / static_cast<uint32_t>(used), hi = (static_cast<uint32_t>(k) + 1) * pic / static_cast<uint32_t>(used);
+            const Span base = cellSpan(static_cast<uint32_t>(k), static_cast<uint32_t>(used), pic);
+            uint32_t lo = base.lo, hi = base.hi;
             if (deep) reachEdge(k, used, pic, deep, lo, hi);
-            if (hi <= lo) hi = lo + 1;   // a picture smaller than the grid: neighboring cells share a pixel
+            if (hi <= lo) hi = lo + 1;   // an edge reaching inward can land on its own start
             out[i] = {static_cast<uint16_t>(lo), static_cast<uint16_t>(hi)};
         }
     }
@@ -201,27 +203,7 @@ class PictureEffect : public EffectBase {
 
     /// The rectangle's mean in linear light (square, average, root: gamma 2 for the display curve), from at most four reads per axis.
     static RGB average(const PictureFrame& f, Range c, Range r) {
-        const uint32_t sx = (c.hi - c.lo + 3u) / 4u, sy = (r.hi - r.lo + 3u) / 4u;
-        uint32_t sum[3] = {0, 0, 0}, n = 0;
-        for (uint32_t y = r.lo; y < r.hi; y += sy)
-            for (uint32_t x = c.lo; x < c.hi; x += sx) {
-                const RGB p = at(f, x, y);
-                sum[0] += static_cast<uint32_t>(p.r) * p.r;
-                sum[1] += static_cast<uint32_t>(p.g) * p.g;
-                sum[2] += static_cast<uint32_t>(p.b) * p.b;
-                n++;
-            }
-        return RGB{isqrt(sum[0] / n), isqrt(sum[1] / n), isqrt(sum[2] / n)};
-    }
-
-    /// The integer square root of a value below 65,536, the bitwise method.
-    static uint8_t isqrt(uint32_t v) {
-        uint32_t root = 0;
-        for (uint32_t bit = 1u << 14; bit; bit >>= 2) {
-            if (v >= root + bit) { v -= root + bit; root = (root >> 1) + bit; }
-            else root >>= 1;
-        }
-        return static_cast<uint8_t>(root);
+        return areaMean(f.rgb, static_cast<size_t>(f.width) * 3u, c, r);
     }
 
     ScratchBuffer<uint16_t> state_{*this};   ///< 8.8 per channel per cell, only while smoothing

src/platform/desktop/platform_desktop.cpp

@@ -133,6 +133,7 @@
 /// A host has no second card, so a subfolder of the one real root stands in. `sdMounted_` gates it as a missing card would, and format is a wipe-and-recreate.
 
 #include "platform/platform.h"
+#include "platform/usbvideo_frames.h"   // areaAverage: the shrink the device backend also uses
 #include "core/util/FirmwareImage.h"  // identify/moonBaseRejection: shared image vetting
 #include "core/util/H264Bitstream.h"  // where the encoder's frames begin and end
 
@@ -2879,14 +2880,11 @@ struct TestGrabber {
     std::atomic<bool> decodeFails{false};
     std::atomic<bool> open{false};
     std::atomic<uint32_t> decoded{0}, failed{0};
+    std::vector<uint8_t> frame;   ///< the synthesized source, rebuilt only when its size or color moves
+    uint8_t frameRgb[3] = {};
 } g_grabber;
 // How far `have` is from `want`, so the nearest advertised mode can be picked without <cstdlib>.
 long apart(long have, long want) { return have > want ? have - want : want - have; }
-// Halve `w` x `h` until it fits `maxW` x `maxH`, which is the scaling the decode hardware offers; false when even one pixel does not fit.
-bool halveToFit(uint16_t& w, uint16_t& h, uint16_t maxW, uint16_t maxH) {
-    while ((w > maxW || h > maxH) && w > 1 && h > 1) { w = uint16_t(w / 2); h = uint16_t(h / 2); }
-    return w <= maxW && h <= maxH;
-}
 }  // namespace
 
 uint32_t videoCaptureFormatGeneration() MM_NONBLOCKING { return g_grabber.generation.load(); }
@@ -2922,16 +2920,19 @@ bool videoCaptureDecodeInto(VideoCaptureHandle& h, uint8_t* dst, uint16_t maxWid
                             uint16_t& outWidth, uint16_t& outHeight) {
     if (!h.impl || !g_grabber.open) return false;
     if (g_grabber.decodeFails) { g_grabber.failed++; return false; }
-    uint16_t w = g_grabber.openW, hh = g_grabber.openH;
-    if (!halveToFit(w, hh, maxWidth, maxHeight)) { g_grabber.failed++; return false; }
-    const uint8_t r = g_grabber.r, g = g_grabber.g, b = g_grabber.b;
-    for (size_t i = 0, px = size_t(w) * hh; i < px; i++) {
-        dst[i * 3 + 0] = r;
-        dst[i * 3 + 1] = g;
-        dst[i * 3 + 2] = b;
-    }
-    outWidth = w;
-    outHeight = hh;
+    if (!maxWidth || !maxHeight) { g_grabber.failed++; return false; }
+    // A frame at the mode the device opened, averaged onto the whole picture. It runs the same code the device backend does, so the host proves that path rather than a shortcut past it.
+    const uint16_t w = g_grabber.openW, hh = g_grabber.openH;
+    const uint8_t rgb[3] = {g_grabber.r, g_grabber.g, g_grabber.b};
+    const size_t need = static_cast<size_t>(w) * hh * 3;
+    if (g_grabber.frame.size() != need || std::memcmp(g_grabber.frameRgb, rgb, 3) != 0) {
+        g_grabber.frame.resize(need);
+        for (size_t i = 0; i < need; i += 3) std::memcpy(g_grabber.frame.data() + i, rgb, 3);
+        std::memcpy(g_grabber.frameRgb, rgb, 3);
+    }
+    usbvideo::areaAverage(g_grabber.frame.data(), static_cast<size_t>(w) * 3, w, hh, maxWidth, maxHeight, dst);
+    outWidth = maxWidth;
+    outHeight = maxHeight;
     g_grabber.decoded++;
     return true;
 }

src/platform/usbvideo_frames.h

@@ -0,0 +1,172 @@
+#pragma once
+
+/// The hardware-free half of USB video capture: the UVC payload headers, the mode choice and the shrink onto a picture.
+
+#include "core/util/AreaSample.h"   // cellSpan, areaMean: the shrink every sampler shares
+
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+
+namespace mm::usbvideo {
+
+/// One mode a grabber advertises, as its frame list reports it.
+struct Mode {
+    uint16_t width = 0;    ///< frame width in pixels
+    uint16_t height = 0;   ///< frame height in pixels
+    uint8_t fps = 0;       ///< frames a second at that size
+};
+
+// --- UVC bulk payload headers (UVC 1.5 2.4.3.3) ---------------------------------------------------
+
+constexpr uint8_t kPts = 0x04, kScr = 0x08, kEoh = 0x80, kErr = 0x40, kReserved = 0x10;
+constexpr size_t kBulkMps = 512;        ///< high-speed bulk packet size, the stride headers sit on
+constexpr size_t kConfirmStrides = 3;   ///< a short header carries few bits, so one match proves nothing
+
+/// A header's own length from its flags: 2 bytes, plus 4 for a PTS and 6 for an SCR.
+inline size_t headerLenFor(uint8_t flags) {
+    return 2u + ((flags & kPts) ? 4u : 0u) + ((flags & kScr) ? 6u : 0u);
+}
+
+/// Whether the flags are a header's: end of header set, no error, no reserved bit.
+inline bool headerFlagsOk(uint8_t flags) { return (flags & kEoh) && !(flags & (kErr | kReserved)); }
+
+/// Whether the SCR's reserved top 5 bits are zero, the strongest single check a header carries.
+inline bool headerScrOk(const uint8_t* h, uint8_t flags, uint8_t len) {
+    return !(flags & kScr) || !(h[len - 1] & 0xF8);
+}
+
+/// Whether the PTS matches the frame's first header, which is constant across one frame's payloads.
+inline bool headerPtsOk(const uint8_t* h, uint8_t flags, const uint8_t* pts) {
+    return !pts || !(flags & kPts) || std::memcmp(h + 2, pts, 4) == 0;
+}
+
+/// Whether `h` is a payload header of exactly `want` bytes; `pts` pins it to the frame's first one.
+inline bool isPayloadHeader(const uint8_t* h, size_t avail, size_t want, const uint8_t* pts) {
+    if (avail < 2) return false;
+    const uint8_t len = h[0], flags = h[1];
+    if (len != want || len != headerLenFor(flags) || avail < len) return false;
+    return headerFlagsOk(flags) && headerScrOk(h, flags, len) && headerPtsOk(h, flags, pts);
+}
+
+/// Whether headers of length `hl` recur after `first`, which is what rules out a chance match.
+inline bool confirmedRun(const uint8_t* d, size_t len, size_t first, size_t hl) {
+    const uint8_t* pts = (d[first + 1] & kPts) ? d + first + 2 : nullptr;
+    const size_t stride = first + hl;
+    size_t seen = 0;
+    for (size_t at = first + stride; at + hl <= len && seen < kConfirmStrides; at += stride, seen++)
+        if (!isPayloadHeader(d + at, len - at, hl, pts)) return false;
+    return seen > 0 || first + stride + hl > len;   // a frame too short to confirm is taken as it is
+}
+
+/// The offset of the first confirmed header of length `hl`, or 0 when the frame carries none.
+inline size_t findHeader(const uint8_t* d, size_t len, size_t hl) {
+    for (size_t i = kBulkMps - hl; i + hl <= len; i += kBulkMps)
+        if (isPayloadHeader(d + i, len - i, hl, nullptr) && confirmedRun(d, len, i, hl)) return i;
+    return 0;
+}
+
+/// Close the gaps the headers from `hdr` on leave, answering the compacted length.
+inline size_t compactFrom(uint8_t* d, size_t len, size_t hdr, size_t hl) {
+    const size_t payload = hdr + hl;
+    uint8_t pts[4] = {};   // copied: the compaction overwrites the header it came from
+    const bool hasPts = (d[hdr + 1] & kPts) != 0;
+    if (hasPts) std::memcpy(pts, d + hdr + 2, sizeof pts);
+    size_t w = hdr, r = hdr;
+    while (isPayloadHeader(d + r, len - r, hl, hasPts ? pts : nullptr)) {
+        r += hl;
+        const size_t run = payload - hl < len - r ? payload - hl : len - r;
+        std::memmove(d + w, d + r, run);
+        w += run;
+        r += run;
+    }
+    std::memmove(d + w, d + r, len - r);   // whatever follows a missing header is left as delivered
+    return w + (len - r);
+}
+
+/// Drops the payload headers a bulk frame still carries, answering the compacted length.
+inline size_t stripPayloadHeaders(uint8_t* d, size_t len) {
+    static constexpr size_t kLens[] = {12, 8, 6, 2};   // longest first: the more fields, the surer the match
+    for (size_t hl : kLens) {
+        const size_t hdr = findHeader(d, len, hl);
+        if (hdr) return compactFrom(d, len, hdr, hl);
+    }
+    return len;
+}
+
+// --- Choosing a mode ----------------------------------------------------------------------------
+
+/// UVC states a rate as a period in 100 ns ticks. Rounded, so 59.94 fps reads as 60 rather than 59.
+inline uint8_t fpsFrom(uint32_t interval) {
+    constexpr uint32_t kTicksPerSecond = 10000000;
+    return interval ? static_cast<uint8_t>((kTicksPerSecond + interval / 2) / interval) : 0;
+}
+
+/// Of two advertised rates, the one to ask for: the fastest at or below `wanted`, else the slowest.
+inline uint8_t betterFps(uint8_t have, uint8_t row, uint8_t wanted) {
+    if (!have) return row;
+    const bool haveOk = have <= wanted, rowOk = row <= wanted;
+    if (haveOk != rowOk) return haveOk ? have : row;
+    if (haveOk) return row > have ? row : have;
+    return row < have ? row : have;
+}
+
+/// Whether a mode covers the ceiling and is no bigger to decode than the best so far.
+inline bool modeFits(const Mode& m, uint16_t width, uint16_t height, uint32_t area, uint32_t bestArea) {
+    if (m.width < width || m.height < height) return false;   // below the ceiling it would be upscaled
+    return !bestArea || area <= bestArea;                     // a bigger picture to decode every frame
+}
+
+/// The smallest mode at or above the ceiling, at the rate asked for, so decode and shrink stay cheap.
+inline bool pickMode(const Mode* modes, size_t count, uint16_t& width, uint16_t& height, uint8_t& fps) {
+    uint32_t bestArea = 0;
+    uint16_t bw = 0, bh = 0;
+    uint8_t bestFps = 0;
+    for (size_t i = 0; i < count; i++) {
+        const Mode& m = modes[i];
+        const uint32_t area = static_cast<uint32_t>(m.width) * m.height;
+        if (!modeFits(m, width, height, area, bestArea)) continue;
+        // The rates accumulate per resolution, not per area: two shapes can share an area.
+        if (m.width != bw || m.height != bh) bestFps = 0;
+        bestArea = area;
+        bw = m.width;
+        bh = m.height;
+        bestFps = betterFps(bestFps, m.fps, fps);
+    }
+    if (!bestArea) return false;   // nothing listed, or nothing big enough: ask for what was wanted
+    width = bw;
+    height = bh;
+    if (bestFps) fps = bestFps;
+    return true;
+}
+
+// --- The decoded frame --------------------------------------------------------------------------
+
+/// `v` rounded up to a multiple of `to`, the MCU boundary a decoder pads a frame's axis to.
+inline uint32_t alignUp(uint32_t v, uint8_t to) { return (v + to - 1u) / to * to; }
+
+/// The row stride in bytes the decoder wrote, or 0 when `outSize` matches no layout its MCU allows.
+inline size_t strideFor(uint16_t w, uint16_t h, uint8_t mcuW, uint8_t mcuH, uint32_t outSize) {
+    if (!w || !h || !mcuW || !mcuH) return 0;
+    if (outSize == static_cast<uint32_t>(w) * h * 3u) return static_cast<size_t>(w) * 3u;
+    const uint32_t padW = alignUp(w, mcuW), padH = alignUp(h, mcuH);
+    if (outSize == padW * padH * 3u) return static_cast<size_t>(padW) * 3u;
+    return 0;
+}
+
+/// Each output pixel the mean of its rectangle, through the shared sampler so one fix reaches both paths.
+inline void areaAverage(const uint8_t* src, size_t stride, uint16_t w, uint16_t h, uint16_t ow,
+                        uint16_t oh, uint8_t* dst) {
+    for (uint16_t y = 0; y < oh; y++) {
+        const Span ry = cellSpan(y, oh, h);
+        for (uint16_t x = 0; x < ow; x++) {
+            const RGB mean = areaMean(src, stride, cellSpan(x, ow, w), ry);
+            uint8_t* out = dst + (static_cast<size_t>(y) * ow + x) * 3;
+            out[0] = mean.r;
+            out[1] = mean.g;
+            out[2] = mean.b;
+        }
+    }
+}
+
+}   // namespace mm::usbvideo

test/CMakeLists.txt

@@ -17,6 +17,8 @@ add_executable(mm_tests
     unit/core/unit_Color.cpp
     unit/core/unit_math8.cpp
     unit/core/unit_math16.cpp
+    unit/core/unit_AreaSample.cpp
+    unit/core/unit_usbvideo_frames.cpp
     unit/core/unit_noise.cpp
     unit/core/unit_fields.cpp
     unit/core/unit_crc.cpp