diff --git a/CHANGELOG.md b/CHANGELOG.md index d6f80e27..6f52e29a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,31 @@ because it turns other people's test suites red. ### Added +- **Films: the `webm` format.** A WebM video with an AV1 picture and no + sound, at the exact size you ask for, like every other format. Its length + is a setting of its own, independent of the size: an hour at thirty frames + a second fits in less than a megabyte (977 128 B at the smallest), because + a film that shows one picture costs a few bytes for every frame after the + first. Set `duration`, + `frame_rate` (whole rates from 1 to 60), `keyframe_interval` - how far + apart the frames a player can start from are - `width`, `height` and + `quality`. The picture is the gradient with the self describing label the + image formats draw, and it stays the same for the whole film. It can be up + to 4096 pixels wide and at most 4096x2304 pixels in all, the largest the + built in encoder writes correctly - a larger one is refused with a pair + that fits. The manifest + says what a test can check: `duration_ms`, `frame_count`, `frame_rate`, + `keyframe_count`, `width`, `height`, `compression: av1` and `audio: false`. + A length that does not end on a frame is refused with the two nearest that + do - at 30 frames a second lengths go in steps of 100ms. Below the + smallest film the settings allow, the refusal says how small it can be and + which settings make it smaller. AV1 plays in current browsers, and an older + player or a pipeline that expects H.264 may refuse it - which is a test + worth having. The window lists it under Video. +- **A length of time as a setting.** `duration` and `keyframe_interval` take + `10s`, `1m30s`, `1h`, `500ms` or `59.9s`, to the millisecond. A bare number + is refused rather than read as seconds, and so is a length that does not + land on a whole millisecond. - **The website in twenty-two languages.** Beside English and Polish it now reads in Simplified and Traditional Chinese, Japanese, German, French, Spanish, Brazilian Portuguese, Italian, Indonesian, Russian, Turkish, Czech, @@ -103,6 +128,12 @@ because it turns other people's test suites red. ### Changed +- **`empty-and-minimal` now includes a film.** The preset builds the smallest + file of every format this build has, so with `formats` left at `all` it + gains a `webm` target, and the recipe `tfg preset eject empty-and-minimal` + prints is a target longer. A manifest from a run of that preset records the + hash of that recipe, so its `recipe_hash` differs from one written before. + No generated file changes, and a run that names its `formats` is untouched. - **An empty text box in the window reads "not set"** instead of "worked out from the size" when leaving it empty means going without - the password of an archive, the author of a PDF. What that means for the file is in the diff --git a/README.md b/README.md index 6fbf7112..0d88429c 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ **Testing Files Generator** is a tool for QA engineers and developers who need real files to test against - an upload form, a parser, anything that takes a file and -has an opinion about it. You pick one of its 26 formats and the size you want, +has an opinion about it. You pick one of its 27 formats and the size you want, and you get **exactly that**: ask for a 10 MB PDF and you get a PDF that a reader will open, at 10 MB to the byte. Every run also leaves a manifest saying **what your system should do with each file**, which is the part other generators leave @@ -23,7 +23,7 @@ needs it finds out it exists. - **Hit an exact size, to the byte** - ask for 10485761 bytes and get exactly that, never a silently rounded file. -- **Write 26 real formats** - a generated PNG opens in an image viewer, a DOCX +- **Write 27 real formats** - a generated PNG opens in an image viewer, a DOCX opens in Word, a ZIP extracts. Not padded zeros with an extension. - **Say what should happen to each file** - the manifest carries an expected outcome, so your test reads the assertion instead of you writing it out. @@ -156,8 +156,8 @@ logs matches ./logs/manifest.json: 10000 files checked ## 📁 Formats it generates -Twenty six, and every one is a **real file of that format** - it opens in the -software that owns it, at the exact size you asked for: +Every one is a **real file of that format** - it opens in the software that +owns it, at the exact size you asked for: | group | formats | |---|---| @@ -167,6 +167,7 @@ software that owns it, at the exact size you asked for: | ⚙️ **Configuration** | `yaml`, `toml` | | 🗜️ **Archives** | `zip`, `targz` (`.tar.gz`) | | 🔊 **Audio** | `wav` | +| 🎞️ **Video** | `webm` (AV1, no sound) | Coming next: `7z`, `mp3`, `mp4`. @@ -582,7 +583,7 @@ recipe cannot build on another file. ## 📁 Formats in detail -The twenty four formats are listed near the top of this file. Each is produced at an +The formats are listed near the top of this file. Each is produced at an exact size and checked against independent readers before it ships - a PNG is opened and its pixels compared, a DOCX is read back by three separate libraries, an archive is extracted. @@ -605,6 +606,7 @@ recipe. `tfg formats ` prints the allowed range or list for each: | `avif`, `jpg`, `jxl` | `width`, `height`, `quality` | | `ico` | `width`, `height`, `embed` | | `wav` | `sample_rate`, `bit_depth`, `channels`, `content` | +| `webm` | `width`, `height`, `duration`, `frame_rate`, `keyframe_interval`, `quality` | | `zip` | `entries`, `entry_format`, `entry_size`, `compression`, `depth`, `directory_entries`, `password`, `encryption` | | `targz` | `entries`, `entry_format`, `entry_size`, `compression`, `depth`, `directory_entries`, `entry_mode`, `entry_owner` | | `docx` | `paragraphs` | @@ -833,7 +835,7 @@ a valid one of its format. Honest scope, because a tool that oversells itself wastes your afternoon. -**Working end to end:** twenty four formats, recipes, presets, the desktop window, +**Working end to end:** every format above, recipes, presets, the desktop window, `generate`, `validate`, `verify`, `cleanup`, boundary sets, archive contents, size ranges, per format settings, manifests and every exit code above. diff --git a/internal/core/duration.go b/internal/core/duration.go new file mode 100644 index 00000000..d86716ab --- /dev/null +++ b/internal/core/duration.go @@ -0,0 +1,215 @@ +package core + +import ( + "math" + "strconv" + "strings" +) + +// ParseDuration turns a written length of time into an exact number of +// milliseconds. +// +// 10s -> 10 000 +// 1m30s -> 90 000 +// 1h -> 3 600 000 +// 1.5s -> 1 500 +// 500ms -> 500 +// 1h 30m -> 5 400 000 +// +// The units are h, m, s and ms, and case does not matter. A length written in +// several parts goes from the largest unit down, each unit once, so 1h30m and +// 1m30s read the way a clock does. A space between the parts is allowed, +// because that is how people write it by hand. +// +// A number with no unit is refused rather than read as seconds. Ten could be +// ten seconds or ten milliseconds, and the tool picking one is exactly the +// guess this type exists to remove - the same reason a size refuses 1e5 and +// 0x10. Sizes can take a bare number because bytes are the only thing a size +// counts, and a length of time has no such single unit. +// +// A length that does not land on a whole millisecond is refused rather than +// rounded, as a size that does not land on a whole byte is. A video or a sound +// built on a rounded length would carry a duration nobody asked for. +// +// The arithmetic is done on the digits rather than through a float, so 0.1s is +// a hundred milliseconds exactly and not whatever 0.1 times a thousand happens +// to come to. +func ParseDuration(s string) (int64, error) { + raw := strings.ToLower(strings.TrimSpace(s)) + if raw == "" { + return 0, Refuse(Says("core.DurationEmpty", "an empty length of time: write one such as 10s, 1m30s or 500ms")) + } + + var total int64 + previous := -1 + rest := raw + for rest != "" { + rest = strings.TrimLeft(rest, " ") + number, afterNumber := leadingDecimal(rest) + rest = strings.TrimLeft(afterNumber, " ") + unit, afterUnit := leadingLetters(rest) + rest = afterUnit + + // A number followed by something that is neither a unit nor the end - + // 1,5s or -5s - is a spelling, not a missing unit, and is called one. + if number == "" || (unit == "" && rest != "") { + return 0, Refuse(Says("core.DurationNotALength", + "%q is not a length of time: write one such as 10s, 1m30s or 500ms. A sign, an exponent or a comma is refused rather than guessed at", A("Value", s))) + } + if unit == "" { + return 0, Refuse(Says("core.DurationNoUnit", + "%q has no unit: write 10s for seconds or 10ms for milliseconds, because a bare number could be either", A("Value", s))) + } + index, per, ok := durationUnit(unit) + if !ok { + return 0, Refuse(Says("core.DurationUnknownUnit", + "%q uses an unknown unit %q: use h, m, s or ms", A("Value", s), A("Unit", unit))) + } + if index <= previous { + return 0, Refuse(Says("core.DurationOutOfOrder", + "%q writes a unit twice or out of order: write the larger units first, each once, as 1h30m or 1m30s", A("Value", s))) + } + previous = index + + part, err := wholeMilliseconds(s, number, per) + if err != nil { + return 0, err + } + if total > math.MaxInt64-part { + return 0, durationTooLong(s) + } + total += part + } + return total, nil +} + +// FormatDuration writes a number of milliseconds the way ParseDuration reads +// it, in the shortest form a person would write: 10s, 1m30s, 1h, 500ms, +// 1.033s. ParseDuration of the result is the number again, which a guard holds +// for every value it tries. +func FormatDuration(ms int64) string { + if ms <= 0 { + return "0s" + } + hours := ms / 3_600_000 + minutes := ms % 3_600_000 / 60_000 + rest := ms % 60_000 + + var b strings.Builder + if hours > 0 { + b.WriteString(strconv.FormatInt(hours, 10) + "h") + } + if minutes > 0 { + b.WriteString(strconv.FormatInt(minutes, 10) + "m") + } + switch { + case rest == 0: + case rest%1000 == 0: + b.WriteString(strconv.FormatInt(rest/1000, 10) + "s") + case rest < 1000 && hours == 0 && minutes == 0: + b.WriteString(strconv.FormatInt(rest, 10) + "ms") + default: + fraction := strings.TrimRight(strconv.FormatInt(1000+rest%1000, 10)[1:], "0") + b.WriteString(strconv.FormatInt(rest/1000, 10) + "." + fraction + "s") + } + return b.String() +} + +// durationUnit is where a unit stands among the four, largest first, and how +// many milliseconds one of it is. +func durationUnit(unit string) (index int, per int64, ok bool) { + switch unit { + case "h": + return 0, 3_600_000, true + case "m": + return 1, 60_000, true + case "s": + return 2, 1_000, true + case "ms": + return 3, 1, true + } + return 0, 0, false +} + +// leadingDecimal splits off digits with at most one decimal point. Anything +// else - a sign, an exponent, a comma - ends the number, and the caller then +// meets it where a unit should be. +func leadingDecimal(s string) (number, rest string) { + i, dot := 0, false + for i < len(s) { + c := s[i] + if c >= '0' && c <= '9' { + i++ + continue + } + if c == '.' && !dot { + dot = true + i++ + continue + } + break + } + number = s[:i] + if strings.Trim(number, ".") == "" { + // A lone point is not a number, and saying so here keeps the message + // the one about numbers rather than one about units. + return "", s + } + return number, s[i:] +} + +func leadingLetters(s string) (letters, rest string) { + i := 0 + for i < len(s) && s[i] >= 'a' && s[i] <= 'z' { + i++ + } + return s[:i], s[i:] +} + +// wholeMilliseconds is one part of a length, worked out on the digits. +func wholeMilliseconds(s, number string, per int64) (int64, error) { + whole, fraction, _ := strings.Cut(number, ".") + fraction = strings.TrimRight(fraction, "0") + + var n int64 + if whole != "" { + w, err := strconv.ParseInt(whole, 10, 64) + if err != nil || w > math.MaxInt64/per { + return 0, durationTooLong(s) + } + n = w * per + } + if fraction == "" { + return n, nil + } + + // Nine digits after the point is past anything a millisecond can come + // out of: the largest unit is an hour, 36 times ten to the fifth + // milliseconds, so a fraction whose last digit is not zero lands on a + // whole millisecond only within seven places. + notWhole := Refuse(Says("core.DurationNotWhole", + "%q does not land on a whole millisecond: write it to the millisecond, as 1.5s or 1500ms", A("Value", s))) + if len(fraction) > 9 { + return 0, notWhole + } + f, err := strconv.ParseInt(fraction, 10, 64) + if err != nil { + return 0, notWhole + } + scale := int64(1) + for range len(fraction) { + scale *= 10 + } + if f*per%scale != 0 { + return 0, notWhole + } + part := f * per / scale + if n > math.MaxInt64-part { + return 0, durationTooLong(s) + } + return n + part, nil +} + +func durationTooLong(s string) error { + return Refuse(Says("core.DurationTooLong", "%q is too long to count in milliseconds", A("Value", s))) +} diff --git a/internal/format/all/all.go b/internal/format/all/all.go index 67ab8fcf..f65b553d 100644 --- a/internal/format/all/all.go +++ b/internal/format/all/all.go @@ -29,6 +29,7 @@ import ( _ "github.com/donislawdev/TestingFilesGenerator/internal/format/tomlfile" _ "github.com/donislawdev/TestingFilesGenerator/internal/format/txt" _ "github.com/donislawdev/TestingFilesGenerator/internal/format/wav" + _ "github.com/donislawdev/TestingFilesGenerator/internal/format/webm" _ "github.com/donislawdev/TestingFilesGenerator/internal/format/webp" _ "github.com/donislawdev/TestingFilesGenerator/internal/format/xlsx" _ "github.com/donislawdev/TestingFilesGenerator/internal/format/xmlfile" diff --git a/internal/format/format.go b/internal/format/format.go index ac4dd495..68181f0c 100644 --- a/internal/format/format.go +++ b/internal/format/format.go @@ -118,6 +118,13 @@ const ( // it differently and because the same syntax failing here while working // for --size is the kind of difference nobody would predict. PropertySize PropertyKind = "size" + // PropertyDuration is a length of time written the way core.ParseDuration + // reads it, so 10s, 1m30s or 500ms, and counted in whole milliseconds. Its + // own kind for the reason size is one: a window draws it as a short box, + // and a duration typed as text would be read differently by every format + // that took one. The first to declare it is a video's length + // (docs/WIDEO-2026-10-06.md section 12), and a sound's is meant to follow. + PropertyDuration PropertyKind = "duration" // PropertyText is free text the format interprets itself. PropertyText PropertyKind = "text" ) @@ -139,7 +146,8 @@ type Property struct { Name string Kind PropertyKind - // Min and Max bound an int. Both zero means unbounded. + // Min and Max bound an int, and a duration in milliseconds. Both zero means + // unbounded. Min, Max int64 // Unit is what an int counts, for the message and for a window's field. // Empty when the number counts itself, as a page count does. @@ -383,15 +391,7 @@ func Containers() []string { func (p Property) Allows(raw string) (bad core.Said) { switch p.Kind { case PropertyInt: - n, err := strconv.ParseInt(raw, 10, 64) - if err != nil { - return p.takesWholeNumber() - } - if p.Min != 0 || p.Max != 0 { - if n < p.Min || n > p.Max { - return p.takesRange() - } - } + return p.allowsInt(raw) case PropertyChoice: // Spelled the way it is declared, not merely close to it. This used to // fold with EqualFold, and the folding was invisible here and decisive @@ -423,6 +423,8 @@ func (p Property) Allows(raw string) (bad core.Said) { if _, err := core.ParseSize(raw); err != nil { return core.Says("format.TakesSize", "it takes a size such as 2mb, or a plain byte count") } + case PropertyDuration: + return p.allowsDuration(raw) case PropertyText: // Named rather than left out. Free text has no shape to check, so // anything is allowed - and a kind added later reddens the linter @@ -448,6 +450,51 @@ func (p Property) takesRange() core.Said { core.A("Unit", core.UnitTerm(p.Unit)), core.A("Least", p.Min), core.A("Most", p.Max)) } +// allowsInt is Allows for a whole number. Its own function since the length of +// time arrived, because Allows with every kind inline reached the band the +// branching guard watches. +func (p Property) allowsInt(raw string) core.Said { + n, err := strconv.ParseInt(raw, 10, 64) + if err != nil { + return p.takesWholeNumber() + } + if p.Min != 0 || p.Max != 0 { + if n < p.Min || n > p.Max { + return p.takesRange() + } + } + return core.Said{} +} + +// allowsDuration is Allows for a length of time. +// +// One sentence for a spelling it cannot read and for a length outside the +// range, and it is the sentence Allowed puts under an empty field. The range +// and the examples together are what somebody needs to type the next value, +// whichever of the two was wrong. +func (p Property) allowsDuration(raw string) core.Said { + ms, err := core.ParseDuration(raw) + if err != nil { + return p.takesDuration() + } + if p.Min == 0 && p.Max == 0 { + return core.Said{} + } + if ms < p.Min || ms > p.Max { + return p.takesDuration() + } + return core.Said{} +} + +// takesDuration is what a duration setting takes, as a refusal. +func (p Property) takesDuration() core.Said { + if p.Min == 0 && p.Max == 0 { + return core.Says("format.TakesDuration", "it takes a length of time such as 10s, 1m30s or 500ms") + } + return core.Says("format.TakesDurationRange", "it takes a length of time from %s to %s, such as 10s, 1m30s or 500ms", + core.A("Least", core.FormatDuration(p.Min)), core.A("Most", core.FormatDuration(p.Max))) +} + // sizePhrase is what a size setting takes, written once. // // Allows and Allowed both have to say this, and until 2026-08-26 they said it @@ -497,6 +544,10 @@ func (p Property) Allowed() string { what = "true or false" case PropertySize: what = sizePhrase + case PropertyDuration: + // The refusal's own words without its first two, so an empty field + // and a refused one describe the setting identically. + what = strings.TrimPrefix(p.takesDuration().String(), "it takes ") default: // A text setting describes itself with Shape or not at all. Saying // "text" under a field is a word where a description should be. diff --git a/internal/format/video/av1.go b/internal/format/video/av1.go new file mode 100644 index 00000000..dab164f8 --- /dev/null +++ b/internal/format/video/av1.go @@ -0,0 +1,163 @@ +package video + +// OBU types this package writes, AV1 specification section 6.2.2. +const ( + obuSequenceHeader = 1 + obuTemporalDelimiter = 2 + obuFrameHeader = 3 + obuFrame = 6 +) + +// The colour every picture here is in, said in the sequence header so that a +// player does not have to guess: BT.709 primaries, transfer and matrix, studio +// range. That is what web video is, and an AV1 stream with no description is +// read as BT.709 by some players and as BT.601 by others - the same file in two +// colours. The numbers are the ones ISO/IEC 23091-4 gives, which the AV1 +// specification uses as they are. +const ( + colourPrimariesBT709 = 1 + transferBT709 = 1 + matrixBT709 = 1 +) + +// copySlot is the reference slot the hidden copy is kept in. A shown key frame +// fills all eight slots and leaves none of them showable, so the copy can take +// any of them - the number only has to be the same in the frame that fills it +// and in every frame that shows it. +const copySlot = 1 + +// obu wraps a payload with the header both containers take: no extension, and +// the size carried in the header. +func obu(typ int, payload []byte) []byte { + return obuOf(typ, payload, nil) +} + +// obuOf is an OBU whose payload is a header followed by data, written into +// one allocation of the right size - the data is a picture's tile, and copying +// it twice to join it to its header is the cost this avoids. +func obuOf(typ int, header, data []byte) []byte { + size := leb128(len(header) + len(data)) + out := make([]byte, 0, 1+len(size)+len(header)+len(data)) + out = append(out, byte(typ<<3)|0x02) + out = append(out, size...) + out = append(out, header...) + return append(out, data...) +} + +// sequenceHeader is the payload of a full sequence header, not the reduced +// still picture one gav1d writes, with the same coding tools gav1d's tile data +// was coded under: 64 by 64 superblocks, no filter intra, intra edge filter on, +// no superres, no CDEF, no loop restoration, 8 bit 4:2:0. Every inter tool is +// off and there is no order hint, because no frame here is predicted from +// another - repeating a picture is done by showing it again. +// +// The field order is the specification's, section 5.5, and the reason for +// each zero is written beside it. +func sequenceHeader(width, height, level int) []byte { + w := bitWriter{buf: make([]byte, 0, 16)} + w.bits(0, 3) // seq_profile: Main + w.bit(0) // still_picture + w.bit(0) // reduced_still_picture_header + w.bit(0) // timing_info_present_flag: the container keeps the time + w.bit(0) // initial_display_delay_present_flag + w.bits(0, 5) // operating_points_cnt_minus_1: one operating point + w.bits(0, 12) + w.bits(uint32(level), 5) + if level > 7 { + w.bit(0) // seq_tier: Main + } + wn, hn := bitsFor(width-1), bitsFor(height-1) + w.bits(uint32(wn-1), 4) + w.bits(uint32(hn-1), 4) + w.bits(uint32(width-1), wn) + w.bits(uint32(height-1), hn) + w.bit(0) // frame_id_numbers_present_flag + w.bit(0) // use_128x128_superblock + w.bit(0) // enable_filter_intra + w.bit(1) // enable_intra_edge_filter + w.bit(0) // enable_interintra_compound + w.bit(0) // enable_masked_compound + w.bit(0) // enable_warped_motion + w.bit(0) // enable_dual_filter + w.bit(0) // enable_order_hint + w.bit(0) // seq_choose_screen_content_tools + w.bit(0) // seq_force_screen_content_tools: off, so frames do not say it + w.bit(0) // enable_superres + w.bit(0) // enable_cdef + w.bit(0) // enable_restoration + // color_config + w.bit(0) // high_bitdepth + w.bit(0) // mono_chrome + w.bit(1) // color_description_present_flag + w.bits(colourPrimariesBT709, 8) + w.bits(transferBT709, 8) + w.bits(matrixBT709, 8) + w.bit(0) // color_range: studio swing + w.bits(0, 2) // chroma_sample_position: unknown, as gav1d declares it + w.bit(0) // separate_uv_delta_q + w.bit(0) // film_grain_params_present + w.trailing() + return w.bytes() +} + +func bitsFor(v int) int { + n := 1 + for v>>uint(n) != 0 { + n++ + } + return n +} + +// codecConfig is the AV1CodecConfigurationRecord both containers carry (the +// av1C box in MP4, CodecPrivate in Matroska), followed by the sequence header +// OBU it describes. Byte two is seq_tier 0, 8 bit, not 12 bit, not +// monochrome, subsampled in both directions, sample position unknown. +func codecConfig(level int, sequence []byte) []byte { + return append([]byte{0x81, byte(level & 0x1f), 0x0C, 0x00}, sequence...) +} + +// keyFrame is a shown KEY_FRAME around a coded picture. Shown, it is a place +// to start playing, and the specification makes it impossible to show again. +func keyFrame(c Coded) []byte { + w := bitWriter{buf: make([]byte, 0, 8)} + w.bit(0) // show_existing_frame + w.bits(0, 2) // frame_type: KEY_FRAME + w.bit(1) // show_frame + // error_resilient_mode is 1 by rule for a shown key frame, not written + w.bit(0) // disable_cdf_update + w.bit(0) // frame_size_override_flag + // refresh_frame_flags is every slot by rule, not written + w.bit(0) // render_and_frame_size_different + w.bit(1) // disable_frame_end_update_cdf + c.writeSuffix(&w) + w.align() + return obuOf(obuFrame, w.bytes(), c.tile) +} + +// hiddenCopy is the same picture as an INTRA_ONLY frame that is not shown now +// and is showable later, kept in copySlot. +func hiddenCopy(c Coded) []byte { + w := bitWriter{buf: make([]byte, 0, 9)} + w.bit(0) // show_existing_frame + w.bits(2, 2) // frame_type: INTRA_ONLY_FRAME + w.bit(0) // show_frame + w.bit(1) // showable_frame + w.bit(0) // error_resilient_mode + w.bit(0) // disable_cdf_update + w.bit(0) // frame_size_override_flag + w.bits(1<> (w.n % 8) + } + w.n++ +} + +func (w *bitWriter) bits(v uint32, n int) { + for i := n - 1; i >= 0; i-- { + w.bit(v >> uint(i)) + } +} + +// align is byte_alignment: zero bits to the next byte. +func (w *bitWriter) align() { + for w.n%8 != 0 { + w.bit(0) + } +} + +// trailing is trailing_bits: a one, then zeros to the next byte. +func (w *bitWriter) trailing() { + w.bit(1) + w.align() +} + +func (w *bitWriter) bytes() []byte { return w.buf } + +// bitReader reads the same way, and only over bytes it was given - reading +// past the end is a defect in the caller, reported rather than panicking. +type bitReader struct { + b []byte + pos uint + overrun bool +} + +func (r *bitReader) bit() uint32 { + if r.pos/8 >= uint(len(r.b)) { + r.overrun = true + return 0 + } + v := uint32(r.b[r.pos/8]>>(7-r.pos%8)) & 1 + r.pos++ + return v +} + +func (r *bitReader) bits(n int) uint32 { + var v uint32 + for range n { + v = v<<1 | r.bit() + } + return v +} + +// leb128 is the length an OBU header carries. +func leb128(v int) []byte { + var out []byte + for { + b := byte(v & 0x7f) + v >>= 7 + if v == 0 { + return append(out, b) + } + out = append(out, b|0x80) + } +} + +// readLeb128 is the value and how many bytes it took, or 0 bytes when the +// input ends inside it. +func readLeb128(b []byte) (value, used int) { + for i := 0; i < 8 && i < len(b); i++ { + value |= int(b[i]&0x7f) << (7 * i) + if b[i]&0x80 == 0 { + return value, i + 1 + } + } + return 0, 0 +} diff --git a/internal/format/video/choose.go b/internal/format/video/choose.go new file mode 100644 index 00000000..b24a5c0d --- /dev/null +++ b/internal/format/video/choose.go @@ -0,0 +1,213 @@ +package video + +import ( + "fmt" + + "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/format/imagedim" +) + +// Rung is a picture size planning can choose without coding the picture, with +// the largest tile it has ever been seen to code to. +type Rung struct { + Width, Height int + Ceiling int +} + +// Ladder is tried from the largest down when the request names no picture +// size, so a small file gets a small picture instead of being refused. +// +// Sixteen by nine, because that is the shape of a film, and it stops at +// 640x360 for the reason AVIF stops at 640x480: a larger picture costs every +// file of a run its encoding time, and somebody who wants Full HD says so. +// +// Ceiling is a tenth above the largest tile the rung's picture coded to in two +// passes - every seed offset there is with every length the label can have, +// either format's name in it, and the label absent, then 512 seeds far from +// zero, because the label's text moves with the whole seed and not only with +// the offset. Measured by tools/probes/videoladder at the default quality on +// 2026-10-06 (docs/WIDEO-2026-10-06.md section 13). +// +// The tenth is not caution for its own sake. On four rungs of eleven the far +// seeds coded larger than the largest of the first pass, by up to 3.7 percent +// (256x144: 4185 B, then 4338 B), so the first pass alone would have been a +// ceiling a seed nobody tried could pass. A tenth is close to three times the +// worst of that. A ceiling too high costs a smaller picture than there was +// room for and nothing else. One too low would let planning promise a size +// writing cannot keep, so writing checks and says so rather than trusting +// this table. +var Ladder = []Rung{ + {640, 360, 14644}, {426, 240, 8500}, {320, 180, 6192}, {256, 144, 4772}, + {160, 90, 2953}, {80, 45, 385}, {40, 23, 218}, {16, 9, 83}, + {4, 3, 38}, {2, 2, 21}, {1, 1, 6}, +} + +// Choice is the picture a film shows. +type Choice struct { + Width, Height int + Seed uint64 + Label string + QIndex int + // Coded is the picture when planning had to code it - a size named by hand, + // or a quality the ceilings were not measured at. Nil means the picture is + // a ladder rung and writing codes it. + Coded *Coded + // Ceiling is the largest tile the plan allowed for, which writing holds + // the coded picture to. + Ceiling int + // Named is whether the request gave the picture's size. A picture chosen + // to fit is the largest that does, so only a named one can be asked to be + // smaller. + Named bool +} + +// Labelled is whether this picture carries its label. +func (c Choice) Labelled() bool { return Labelled(c.Width, c.Label) } + +// Code is the picture's coded form, coding it when planning did not. +func (c Choice) Code() (Coded, error) { + if c.Coded != nil { + return *c.Coded, nil + } + coded, err := Encode(Picture(c.Width, c.Height, c.Seed, c.Label), c.QIndex) + if err != nil { + return Coded{}, err + } + if coded.Size() > c.Ceiling { + return Coded{}, core.Defect(fmt.Errorf("video: a %dx%d picture coded to a %d B tile and its rung allows %d B, so the file planned around it cannot be kept", + c.Width, c.Height, coded.Size(), c.Ceiling)) + } + return coded, nil +} + +// Choose settles the picture for a request, and the stream planning sizes the +// file by: exact when the picture was coded, the bound of its rung otherwise. +// +// fits says whether a stream leaves room in the requested size. When nothing +// fits, the smallest candidate comes back so the format can name the minimum. +func Choose(formatID string, r format.Request, s Settings, label string, fits func(Stream) bool) (Choice, Stream, error) { + base := Choice{Seed: r.Seed, Label: label, QIndex: s.QIndex} + _, wSet := r.Properties[imagedim.SettingWidth] + _, hSet := r.Properties[imagedim.SettingHeight] + if wSet || hSet { + return named(formatID, r, s, base) + } + var last Choice + var lastStream Stream + for _, rung := range Ladder { + c, st, err := onRung(base, rung, s) + if err != nil { + return Choice{}, Stream{}, err + } + if fits(st) { + return c, st, nil + } + last, lastStream = c, st + } + return last, lastStream, nil +} + +// onRung is the picture of one rung and the stream planning sizes it by: the +// rung's bound, or - when the request named a quality the ceilings were not +// measured at - the picture itself, coded. That is the slow road AVIF takes +// for the same reason, and only runs that asked for it pay. +func onRung(base Choice, rung Rung, s Settings) (Choice, Stream, error) { + c := base + c.Width, c.Height, c.Ceiling = rung.Width, rung.Height, rung.Ceiling + if !s.QualityNamed { + return c, Bound(s.Timeline, rung.Ceiling, rung.Width, rung.Height), nil + } + coded, err := Encode(Picture(c.Width, c.Height, c.Seed, c.Label), c.QIndex) + if err != nil { + return Choice{}, Stream{}, err + } + c.Coded, c.Ceiling = &coded, coded.Size() + return c, NewStream(s.Timeline, coded, c.Width, c.Height), nil +} + +// named codes a picture whose size the request gave, at planning, because +// nothing but coding it says how big it is. +func named(formatID string, r format.Request, s Settings, c Choice) (Choice, Stream, error) { + w, err := imagedim.Value(formatID, imagedim.SettingWidth, r.Properties, maxWidth, Ladder[0].Width) + if err != nil { + return Choice{}, Stream{}, err + } + h, err := imagedim.Value(formatID, imagedim.SettingHeight, r.Properties, maxHeight, Ladder[0].Height) + if err != nil { + return Choice{}, Stream{}, err + } + if err := checkJointLimits(formatID, w, h); err != nil { + return Choice{}, Stream{}, err + } + if err := checkOneTile(formatID, w, h); err != nil { + return Choice{}, Stream{}, err + } + coded, err := Encode(Picture(w, h, c.Seed, c.Label), c.QIndex) + if err != nil { + return Choice{}, Stream{}, err + } + c.Width, c.Height, c.Coded, c.Ceiling, c.Named = w, h, &coded, coded.Size(), true + return c, NewStream(s.Timeline, coded, w, h), nil +} + +// checkJointLimits asks the registry's own declaration, so the refusal, the +// sentence tfg formats prints and the field a window draws come from one line. +func checkJointLimits(formatID string, w, h int) error { + d, err := format.Get(formatID) + if err != nil { + return err + } + for _, j := range d.JointLimits { + if bad := j.Allows(formatID, int64(w), int64(h)); !bad.IsZero() { + return &format.PropertyValueError{ + Format: formatID, Key: j.Of + " and " + j.By, Subject: j.Subject(), + Value: fmt.Sprintf("%dx%d", w, h), + Reason: core.Says("format.JointSmallerPair", "%s. Ask for a smaller pair", core.A("Why", bad)), + } + } + } + return nil +} + +// checkOneTile refuses a picture that would need a second AV1 tile, before +// the encoder makes a frame nobody can decode out of it. +// +// The declared limit counts pixels and this counts blocks of 64 by 64, each +// side rounded up, which is what the encoder goes by - 4000x2359 is under the +// pixels and one block too many. Said as a setting the request can change, +// not as a fault of the program. +func checkOneTile(formatID string, w, h int) error { + blocks := ((w + 63) / 64) * ((h + 63) / 64) + if blocks <= tileBlocks { + return nil + } + return &format.PropertyValueError{ + Format: formatID, Key: imagedim.SettingWidth + " and " + imagedim.SettingHeight, + Subject: core.Says("format.TwoSettings", "%s and %s", core.A("Of", core.LabelTerm(imagedim.SettingWidth)), core.A("By", core.LabelTerm(imagedim.SettingHeight))), + Value: fmt.Sprintf("%dx%d", w, h), + Reason: core.Says("video.MoreThanOneTile", + "a %dx%d picture is %d blocks of 64 by 64 pixels once its sides are rounded up to whole blocks, and the encoder codes one AV1 tile, which holds %d. Ask for a smaller pair, such as 4096x2304 or 3840x2160", + core.A("Width", w), core.A("Height", h), core.A("Blocks", blocks), core.A("Most", tileBlocks)), + } +} + +// Facts is what a film's plan tells the manifest. The keys an image and a +// sound already use mean the same here - width, height, frame_count, +// duration_ms, compression - because they are public names a test asserts +// on (untouchable rule 10). +func Facts(c Choice, s Settings) map[string]any { + return map[string]any{ + "width": c.Width, + "height": c.Height, + "duration_ms": s.DurationMs, + "frame_count": s.Frames, + "frame_rate": s.FPS, + "keyframe_count": s.Keys(), + "keyframe_interval_ms": s.KeyEvery * 1000 / int64(s.FPS), + "quality": s.Quality, + "compression": "av1", + "audio": false, + format.PropertyLabelEmbedded: c.Labelled(), + } +} diff --git a/internal/format/video/doc.go b/internal/format/video/doc.go new file mode 100644 index 00000000..6f5b296e --- /dev/null +++ b/internal/format/video/doc.go @@ -0,0 +1,29 @@ +// Package video is what the video formats share: the timeline, the picture, +// and the AV1 stream they carry. WebM and MP4 are thin containers around it, +// the way docx, xlsx and pptx are thin around opc. +// +// The coding is gav1d's, and gav1d codes still pictures only - its own header +// says "one all-intra key frame" and declares a reduced still picture header. +// So this package keeps gav1d's tile data and writes everything around it +// itself: a full sequence header, the frame headers, and the frames that show +// a picture again without coding it again. The probe that settled this and +// every measurement under it is docs/WIDEO-2026-10-06.md sections 9 to 11. +// +// The stream has one shape, and the shape is the point: +// +// a key frame, shown - where a player can start +// the same picture again, - an intra only frame, not shown, kept in a +// hidden and showable slot a later frame can show +// show_existing_frame - every other frame, three bytes each +// +// A shown key frame cannot be shown a second time - the specification sets its +// showable_frame to 0 and libaom refuses the stream that tries ("Buffer does +// not contain a showable frame"). gav1d's own decoder and Chromium both play +// that stream without a word, which is why neither of them is a witness here. +// The hidden copy costs the picture's bytes once more and no second encoding, +// because it carries the very same tile. +// +// That shape is what makes a length independent of the size: an hour at thirty +// frames a second is about a megabyte, because 108 000 of its frames are three +// bytes each. +package video diff --git a/internal/format/video/level.go b/internal/format/video/level.go new file mode 100644 index 00000000..ac22c682 --- /dev/null +++ b/internal/format/video/level.go @@ -0,0 +1,113 @@ +package video + +import "math/big" + +// levelMaximumParameters is seq_level_idx 31: "no level-based constraints on +// the bitstream", which the specification reserves for streams that conform to +// no other level. +const levelMaximumParameters = 31 + +// level is one row of the AV1 level tables, Annex A.3, copied from the +// specification's own source (annex.a.levels.md in AOMediaCodec/av1-spec, +// read 2026-10-06), only the columns a stream from this package can reach. +// MainMbps is kept in kilobits so every comparison below is a whole number. +type level struct { + idx int + maxPicSize int64 + maxHSize int64 + maxVSize int64 + maxDisplayRate int64 + mainKbps int64 + mainCR int64 +} + +// levels runs from the lowest up, so the first that admits a stream is the one +// to declare. 2.2, 2.3, 3.2, 3.3, 4.2, 4.3 and 7.x are not defined, and the +// specification says so in the note under its table. +var levels = []level{ + {0, 147456, 2048, 1152, 4_423_680, 1_500, 2}, + {1, 278784, 2816, 1584, 8_363_520, 3_000, 2}, + {4, 665856, 4352, 2448, 19_975_680, 6_000, 2}, + {5, 1065024, 5504, 3096, 31_950_720, 10_000, 2}, + {8, 2359296, 6144, 3456, 70_778_880, 12_000, 4}, + {9, 2359296, 6144, 3456, 141_557_760, 20_000, 4}, + {12, 8912896, 8192, 4352, 267_386_880, 30_000, 6}, + {13, 8912896, 8192, 4352, 534_773_760, 40_000, 8}, + {14, 8912896, 8192, 4352, 1_069_547_520, 60_000, 8}, + {15, 8912896, 8192, 4352, 1_069_547_520, 60_000, 8}, + {16, 35651584, 16384, 8704, 1_069_547_520, 60_000, 8}, + {17, 35651584, 16384, 8704, 2_139_095_040, 100_000, 8}, + {18, 35651584, 16384, 8704, 4_278_190_080, 160_000, 8}, + {19, 35651584, 16384, 8704, 4_278_190_080, 160_000, 8}, +} + +// chooseLevel is the lowest level a stream of this package's shape conforms +// to, or the maximum parameters level when none does. +// +// The shape matters to three of the constraints. Every frame is shown at the +// frame rate, so the display rate is width times height times fps. A coded +// picture travels twice within one frame time - the key frame and its hidden +// copy - so the one second buffer has to hold both. And each of those two +// frames has to meet the compression ratio, which is why the level is chosen +// after the picture is coded rather than from its size alone: a picture at +// quality 100 can be too big for the level its size would suggest. +// +// A frame narrower or shorter than 16 conforms to no level - the specification +// asks FrameWidth and FrameHeight to be at least 16 - so a small picture +// always declares the maximum parameters level. +// +// Whole numbers throughout, because the answer is written into the sequence +// header and so into the bytes (D11), and a float compared at a boundary is a +// place for two machines to disagree. +func chooseLevel(width, height, fps, frameBytes int) int { + if width < 16 || height < 16 { + return levelMaximumParameters + } + pic := int64(width) * int64(height) + display := pic * int64(fps) + for _, l := range levels { + if pic > l.maxPicSize || int64(width) > l.maxHSize || int64(height) > l.maxVSize { + continue + } + if display > l.maxDisplayRate { + continue + } + if 2*8*int64(frameBytes) > l.mainKbps*1000 { + continue + } + if !compressedEnough(pic, display, int64(frameBytes), l) { + continue + } + return l.idx + } + return levelMaximumParameters +} + +// LevelFor is chooseLevel, exported for the guard that holds it to the +// specification's own examples. +func LevelFor(width, height, fps, frameBytes int) int { + return chooseLevel(width, height, fps, frameBytes) +} + +// compressedEnough is CompressedRatio >= MinPicCompressRatio, Annex A.3: +// +// UnCompressedSize / CompressedSize >= max(0.8, MainCR * display / MaxDisplayRate) +// +// with UnCompressedSize = pic * 15 / 8 for profile 0 and CompressedSize the +// frame's bytes less 128, cross multiplied so nothing is divided. +func compressedEnough(pic, display, frameBytes int64, l level) bool { + compressed := frameBytes - 128 + if compressed <= 0 { + return true + } + uncompressed := pic * 15 >> 3 + // uncompressed * 10 * MaxDisplayRate >= compressed * max(8 * MaxDisplayRate, 10 * MainCR * display) + floor := new(big.Int).Mul(big.NewInt(8), big.NewInt(l.maxDisplayRate)) + scaled := new(big.Int).Mul(big.NewInt(10*l.mainCR), big.NewInt(display)) + if scaled.Cmp(floor) > 0 { + floor = scaled + } + right := new(big.Int).Mul(big.NewInt(compressed), floor) + left := new(big.Int).Mul(big.NewInt(uncompressed*10), big.NewInt(l.maxDisplayRate)) + return left.Cmp(right) >= 0 +} diff --git a/internal/format/video/picture.go b/internal/format/video/picture.go new file mode 100644 index 00000000..96685c43 --- /dev/null +++ b/internal/format/video/picture.go @@ -0,0 +1,109 @@ +package video + +import ( + "image" + "image/color" + + "github.com/donislawdev/TestingFilesGenerator/internal/format/imagelabel" +) + +// Planes is a picture in the form the encoder takes: 8 bit luma, and two +// chroma planes at half the resolution each way, rounded up. +type Planes struct { + Width, Height int + Y, U, V []uint8 +} + +// Picture draws the one picture a film from this tool shows: the gradient +// every image format here draws, moved by the seed, with the label burned into +// the top of it when there is room. +// +// One picture for the whole film, by the owner's decision of 2026-10-06 +// (docs/WIDEO-2026-10-06.md section 12): it is coded once, so planning stays +// arithmetic and a run of ten thousand films costs ten thousand encodings +// rather than ten thousand times the number of key frames. +func Picture(width, height int, seed uint64, label string) Planes { + off := int(seed % 256) + img := image.NewRGBA(image.Rect(0, 0, width, height)) + for y := range height { + for x := range width { + img.SetRGBA(x, y, color.RGBA{ + R: uint8((x + off) % 256), + G: uint8((y + off) % 256), + B: uint8((x + y + off) % 256), + A: 255, + }) + } + } + if Labelled(width, label) { + imagelabel.Draw(img, label) + } + return toPlanes(img) +} + +// Labelled says whether a picture this wide carries a readable label - asked +// while planning, to report a film without one, and while drawing, so the +// manifest cannot claim a label the picture lacks. +func Labelled(width int, label string) bool { + return label != "" && imagelabel.Fits(width, len(label)) +} + +// toPlanes converts to BT.709 studio range in whole numbers. +// +// Whole numbers rather than the floats gav1d converts AVIF pictures with, and +// that is about the byte stability contract (D11): this project has measured +// arm64 rounding a float differently from amd64 (O120), and a film whose bytes +// depend on the machine that made it breaks the one promise a seed makes. The +// coefficients are BT.709's scaled by 256 and rounded, with each chroma row +// adjusted to sum to zero so a grey stays exactly grey: +// +// Y = 16 + ( 47 R + 157 G + 16 B) / 256 +// Cb = 128 + (-26 R - 86 G + 112 B) / 256 +// Cr = 128 + (112 R - 102 G - 10 B) / 256 +// +// Chroma is the rounded mean of the two by two block it covers, so an odd +// width or height averages the pixels that are there. +func toPlanes(img *image.RGBA) Planes { + w, h := img.Rect.Dx(), img.Rect.Dy() + cw, ch := (w+1)/2, (h+1)/2 + p := Planes{Width: w, Height: h, Y: make([]uint8, w*h), U: make([]uint8, cw*ch), V: make([]uint8, cw*ch)} + + for y := range h { + row := img.Pix[y*img.Stride:] + for x := range w { + r, g, b := int(row[4*x]), int(row[4*x+1]), int(row[4*x+2]) + p.Y[y*w+x] = uint8(16 + (47*r+157*g+16*b+128)>>8) + } + } + for cy := range ch { + for cx := range cw { + p.U[cy*cw+cx], p.V[cy*cw+cx] = chroma(img, 2*cx, 2*cy) + } + } + return p +} + +// chroma is the Cb and Cr of the two by two block whose top left is x0, y0, +// over the pixels of it that are inside the picture. +func chroma(img *image.RGBA, x0, y0 int) (cb, cr uint8) { + sumB, sumR, n := 0, 0, 0 + for y := y0; y < min(y0+2, img.Rect.Dy()); y++ { + row := img.Pix[y*img.Stride:] + for x := x0; x < min(x0+2, img.Rect.Dx()); x++ { + r, g, b := int(row[4*x]), int(row[4*x+1]), int(row[4*x+2]) + sumB += -26*r - 86*g + 112*b + sumR += 112*r - 102*g - 10*b + n++ + } + } + return uint8(128 + roundedMean(sumB, n*256)), uint8(128 + roundedMean(sumR, n*256)) +} + +// roundedMean is sum divided by n, rounded half away from zero, so a block +// and its mirror image come out the same distance from 128. +func roundedMean(sum, n int) int { + if sum < 0 { + return -((-sum + n/2) / n) + } + return (sum + n/2) / n +} diff --git a/internal/format/video/settings.go b/internal/format/video/settings.go new file mode 100644 index 00000000..aba62e7f --- /dev/null +++ b/internal/format/video/settings.go @@ -0,0 +1,196 @@ +package video + +import ( + "fmt" + "strconv" + + "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/format/imagedim" +) + +// Setting names. Public names (untouchable rule 10), so they are spelled once +// and every video format declares them from here. +const ( + SettingDuration = "duration" + SettingFrameRate = "frame_rate" + SettingKeyframeInterval = "keyframe_interval" + SettingQuality = "quality" +) + +// The defaults are the owner's of 2026-10-06 (docs/WIDEO-2026-10-06.md +// section 12): ten seconds, and a key frame a minute, which leaves a short film +// with one key frame and an hour at about a megabyte and a half. +const ( + defaultDurationMs = 10_000 + defaultKeyIntervalMs = 60_000 + defaultFPS = 30 + + // maxDurationMs is a day. A day at sixty frames a second is five million + // frames, which a container writes in seconds and holds in no memory - + // longer than that is not a test file anybody has asked for, and the bound + // keeps the arithmetic far inside what the containers' fields carry. + maxDurationMs = 86_400_000 + + // The quality scale AVIF uses, and the same mapping onto the quantizer, so + // one number means one thing in both. + minQuality = 1 + maxQuality = 100 + defaultQuality = 60 + + // The picture has to fit one AV1 tile, because gav1d codes one: a frame + // wider than 4096 or larger than 2304 blocks of 64 by 64 pixels comes out + // of its encoder with a header that announces several tiles over the data + // of one, and both libaom and gav1d's own decoder refuse it. Measured on + // 2026-10-06 (docs/REVIEW-165-2026-10-06.md): 4096x2304 decodes, 4096x2305 + // and 4097x64 do not. That is a limit of the encoder this tool carries, not + // of AV1, so it is the number to raise if a later gav1d codes several tiles. + // + // maxWidth is the widest tile. maxHeight is gav1d's longest side - a tall + // narrow picture is one tile as long as it stays within the area. + maxWidth = 4096 + maxHeight = 16384 + // tileBlocks is how many 64 by 64 blocks one tile holds, and + // maxTilePixels the same area in pixels, which is what the registry can + // declare: a product of the two sides. The blocks are what decides, because + // a side is rounded up to a whole block, so a pair just under the pixels + // can still be one block too many - checked before coding, see named. + tileBlocks = 2304 + maxTilePixels = 4096 * 2304 +) + +// frameRates are whole rates only. 23.976, 29.97 and 59.94 need a time scale +// that is not a whole number of milliseconds, which WebM does not have - left +// for later, section 11.5. +var frameRates = []string{"1", "5", "10", "12", "15", "24", "25", "30", "48", "50", "60"} + +// Properties is what every video format declares, in the order both surfaces +// show it. +func Properties() []format.Property { + return []format.Property{ + imagedim.Width(imagedim.Side{Largest: maxWidth, + Detail: "How wide the picture is. Left out, a size is chosen that fits the bytes you asked for."}), + imagedim.Height(imagedim.Side{Largest: maxHeight, + Detail: "How tall the picture is. Left out, a size is chosen that fits the bytes you asked for."}), + { + Name: SettingDuration, Kind: format.PropertyDuration, + Min: 1, Max: maxDurationMs, Default: core.FormatDuration(defaultDurationMs), + Detail: "How long the film plays. It has to end on a frame, so at 30 frames a second it goes in steps of 100ms.", + }, + { + Name: SettingFrameRate, Kind: format.PropertyChoice, + Choices: frameRates, Default: strconv.Itoa(defaultFPS), + Detail: "How many frames the film shows each second.", + }, + { + Name: SettingKeyframeInterval, Kind: format.PropertyDuration, + Min: 1, Max: maxDurationMs, Default: core.FormatDuration(defaultKeyIntervalMs), + Detail: "How far apart the frames are that a player can start from. Shorter makes skipping through a long film faster and the file bigger, because each of them carries the whole picture twice.", + }, + { + Name: SettingQuality, Kind: format.PropertyInt, + Min: minQuality, Max: maxQuality, Default: strconv.Itoa(defaultQuality), + Detail: "How much detail the picture keeps. Higher looks better and takes more of the file, leaving less room for padding.", + }, + } +} + +// JointLimits is the bound on the picture, declared once for both formats: +// the area of one AV1 tile. In pixels rather than megapixels, because 9 437 184 +// read as "9 megapixels" would be a limit nobody can aim at. +// +// The memory bound AVIF declares is not here, and not by omission: one tile is +// far below it, so it could never be the limit a request meets. +func JointLimits() []format.JointLimit { + return []format.JointLimit{{ + Of: imagedim.SettingWidth, By: imagedim.SettingHeight, Max: maxTilePixels, + Unit: "pixels", Base: "pixels", + Why: "the encoder codes a picture as one AV1 tile, which holds 4096 by 2304 pixels", + }} +} + +// Settings is what a request asked for, read and settled. +type Settings struct { + Timeline + Quality int + // QIndex is the quantizer the quality maps to, the mapping gav1d's AVIF + // encoder uses: quality 100 is 0, which is lossless. + QIndex int + // QualityNamed is whether the request named a quality, because the + // ladder's ceilings were measured at the default and speak for no other. + QualityNamed bool +} + +// Read settles a request's video settings. The registry has already checked +// each value against its declaration, so a failure to read one here is a +// defect rather than a refusal - except where a length does not end on a +// frame, which no declaration can say on its own. +func Read(formatID string, props map[string]string) (Settings, error) { + duration, err := durationOr(props, SettingDuration, defaultDurationMs) + if err != nil { + return Settings{}, err + } + interval, err := durationOr(props, SettingKeyframeInterval, defaultKeyIntervalMs) + if err != nil { + return Settings{}, err + } + fps := defaultFPS + if raw := props[SettingFrameRate]; raw != "" { + if fps, err = strconv.Atoi(raw); err != nil { + return Settings{}, core.Defect(fmt.Errorf("video: frame_rate %q passed the registry and is not a number", raw)) + } + } + quality := defaultQuality + raw, named := props[SettingQuality] + if named && raw != "" { + if quality, err = strconv.Atoi(raw); err != nil { + return Settings{}, core.Defect(fmt.Errorf("video: quality %q passed the registry and is not a number", raw)) + } + } + t, err := NewTimeline(formatID, duration, interval, fps) + if err != nil { + return Settings{}, err + } + if t.Keys() > MaxKeyFrames { + return Settings{}, tooManyKeyFrames(formatID, t, interval) + } + return Settings{ + Timeline: t, Quality: quality, QIndex: (100 - quality) * 255 / 100, + QualityNamed: named && raw != "" && quality != defaultQuality, + }, nil +} + +// MaxKeyFrames bounds how many key frames one film has. +// +// A container lays a film out per key frame - a cluster and a cue point each - +// so the work of planning one grows with them, and every key frame carries the +// picture twice. A key frame on every frame of a short clip is a real test +// (every frame a place to start) and stays possible. The bound is what stops +// the same setting on a day long film from asking for five million of them: +// a hundred thousand is a key frame a second for more than a day. +const MaxKeyFrames = 100_000 + +func tooManyKeyFrames(formatID string, t Timeline, interval int64) error { + step := stepMs(t.FPS) + shortest := (t.DurationMs + MaxKeyFrames - 1) / MaxKeyFrames + shortest = (shortest + step - 1) / step * step + return &format.PropertyValueError{ + Format: formatID, Key: SettingKeyframeInterval, Value: core.FormatDuration(interval), + Reason: core.Says("video.TooManyKeyFrames", + "a film of %s with a key frame every %s has %d of them, and a film can have at most %d. Ask for %s or longer, or a shorter film", + core.A("Duration", core.FormatDuration(t.DurationMs)), core.A("Interval", core.FormatDuration(interval)), + core.A("Keys", t.Keys()), core.A("Most", MaxKeyFrames), core.A("Shortest", core.FormatDuration(shortest))), + } +} + +func durationOr(props map[string]string, key string, fallback int64) (int64, error) { + raw := props[key] + if raw == "" { + return fallback, nil + } + ms, err := core.ParseDuration(raw) + if err != nil { + return 0, core.Defect(fmt.Errorf("video: %s %q passed the registry and does not read: %w", key, raw, err)) + } + return ms, nil +} diff --git a/internal/format/video/still.go b/internal/format/video/still.go new file mode 100644 index 00000000..61d66567 --- /dev/null +++ b/internal/format/video/still.go @@ -0,0 +1,212 @@ +package video + +import ( + "fmt" + + "github.com/gen2brain/gav1d/av1" + + "github.com/donislawdev/TestingFilesGenerator/internal/core" +) + +// encodeSpeed is gav1d's least searching setting, the one AVIF uses and for +// the same reason: the picture is a gradient with a label, and a run of ten +// thousand files should not spend its time on it. +const encodeSpeed = 10 + +// Coded is one picture through gav1d, taken apart into what this package +// wraps in headers of its own: the frame header bits from tile_info to the end +// of the header, and the tile data after them. +type Coded struct { + suffix []byte + suffixBits uint + tile []byte +} + +// Size is how many bytes the picture's tile data takes, which is what every +// frame that carries the picture costs above its few bytes of header. +func (c Coded) Size() int { return len(c.tile) } + +func (c Coded) writeSuffix(w *bitWriter) { + r := bitReader{b: c.suffix} + for range c.suffixBits { + w.bit(r.bit()) + } +} + +// Encode codes a picture. +// +// gav1d's output is read here by the specification rather than by what +// gav1d's writer happens to do, and anything other than the one shape this +// package knows how to re-wrap is refused as a defect. That refusal is the +// point: a gav1d raised underneath us that changed its header would otherwise +// go on producing files, and every one of them would carry a header this +// package wrote for a different encoder. +func Encode(p Planes, qindex int) (Coded, error) { + out := av1.Encode(av1.EncodeConfig{ + Width: p.Width, Height: p.Height, BitDepth: 8, QIndex: qindex, Speed: encodeSpeed, + Src: p.Y, SrcStride: p.Width, SrcU: p.U, SrcV: p.V, SrcUVStride: (p.Width + 1) / 2, + }) + if out == nil { + return Coded{}, core.Defect(fmt.Errorf("video: the encoder refused a %dx%d picture at quantizer %d", p.Width, p.Height, qindex)) + } + units, err := splitUnits(out) + if err != nil { + return Coded{}, core.Defect(fmt.Errorf("video: the encoder's output is not a sequence of OBUs: %w", err)) + } + if len(units) != 3 || units[0].typ != obuTemporalDelimiter || units[1].typ != obuSequenceHeader || units[2].typ != obuFrame { + return Coded{}, core.Defect(fmt.Errorf("video: the encoder wrote %s where a temporal delimiter, a sequence header and one frame were expected", describeUnits(units))) + } + c, err := splitFrame(units[2].payload, p.Width, p.Height) + if err != nil { + return Coded{}, core.Defect(fmt.Errorf("video: the encoder's frame header is not the one this package re-wraps: %w", err)) + } + return c, nil +} + +type unit struct { + typ int + payload []byte +} + +func splitUnits(b []byte) ([]unit, error) { + var out []unit + for len(b) > 0 { + h := b[0] + if h&0x80 != 0 || h&0x04 != 0 || h&0x02 == 0 { + return nil, core.Defect(fmt.Errorf("an OBU header %#02x with the forbidden bit, an extension or no size", h)) + } + size, used := readLeb128(b[1:]) + if used == 0 || 1+used+size > len(b) { + return nil, core.Defect(fmt.Errorf("an OBU whose size runs past the end")) + } + out = append(out, unit{typ: int(h>>3) & 0xf, payload: b[1+used : 1+used+size]}) + b = b[1+used+size:] + } + return out, nil +} + +func describeUnits(units []unit) string { + s := fmt.Sprintf("%d OBUs of types", len(units)) + for _, u := range units { + s += fmt.Sprintf(" %d", u.typ) + } + return s +} + +// splitFrame reads a still picture's frame header - reduced still picture +// header, key frame, shown - and keeps everything from tile_info on. +// +// The three bits before tile_info are disable_cdf_update, +// allow_screen_content_tools and render_and_frame_size_different, and all +// three have to be zero, because the headers this package writes say exactly +// that. From tile_info to reduced_tx_set the fields are the ones every intra +// frame carries, so they are copied as they are. Anything this reader does not +// expect - more than one tile, a quantizer matrix, segmentation, a delta +// quantizer, loop filter deltas - is refused, because none of it is something +// the copied bits would still mean once they sit under another header. +func splitFrame(frame []byte, width, height int) (Coded, error) { + r := bitReader{b: frame} + if r.bits(3) != 0 { + return Coded{}, core.Defect(fmt.Errorf("disable_cdf_update, allow_screen_content_tools or render_and_frame_size_different is set")) + } + start := r.pos + if err := readTileInfo(&r, width, height); err != nil { + return Coded{}, err + } + if err := readQuantizerToTxSet(&r); err != nil { + return Coded{}, err + } + end := r.pos + for r.pos%8 != 0 { + if r.bit() != 0 { + return Coded{}, core.Defect(fmt.Errorf("byte_alignment after the header is not zero")) + } + } + if r.overrun || r.pos/8 >= uint(len(frame)) { + return Coded{}, core.Defect(fmt.Errorf("the header runs to the end of the frame and leaves no tile")) + } + + var w bitWriter + copyBits := bitReader{b: frame, pos: start} + for copyBits.pos < end { + w.bit(copyBits.bit()) + } + return Coded{suffix: w.bytes(), suffixBits: end - start, tile: frame[r.pos/8:]}, nil +} + +// readTileInfo walks tile_info with uniform spacing, section 5.9.15, and +// accepts one tile only - the size of a frame this package codes stays within +// what one tile holds, and a frame that does not is one the copied header +// would describe wrongly. +func readTileInfo(r *bitReader, width, height int) error { + miCols := 2 * ((width + 7) >> 3) + miRows := 2 * ((height + 7) >> 3) + sbCols := (miCols + 15) >> 4 + sbRows := (miRows + 15) >> 4 + minLog2TileCols := tileLog2(4096>>6, sbCols) + maxLog2TileCols := tileLog2(1, min(sbCols, 64)) + maxLog2TileRows := tileLog2(1, min(sbRows, 64)) + minLog2Tiles := max(minLog2TileCols, tileLog2((4096*2304)>>12, sbRows*sbCols)) + + if r.bit() != 1 { + return core.Defect(fmt.Errorf("uniform_tile_spacing_flag is not set")) + } + cols := minLog2TileCols + for cols < maxLog2TileCols && r.bit() == 1 { + cols++ + } + rows := max(minLog2Tiles-cols, 0) + for rows < maxLog2TileRows && r.bit() == 1 { + rows++ + } + if cols > 0 || rows > 0 { + return core.Defect(fmt.Errorf("a %dx%d frame is coded as more than one tile", width, height)) + } + return nil +} + +func tileLog2(blk, target int) int { + k := 0 + for blk< 0 && r.bit() == 1 { + return core.Defect(fmt.Errorf("delta_q_present is set")) + } + if !lossless { + level0, level1 := r.bits(6), r.bits(6) + if level0 != 0 || level1 != 0 { + r.bits(12) // loop_filter_level[2] and [3] + } + r.bits(3) // loop_filter_sharpness + if r.bit() == 1 { + return core.Defect(fmt.Errorf("loop_filter_delta_enabled is set")) + } + r.bit() // tx_mode_select + } + r.bit() // reduced_tx_set + if r.overrun { + return core.Defect(fmt.Errorf("the frame ends inside its header")) + } + return nil +} diff --git a/internal/format/video/stream.go b/internal/format/video/stream.go new file mode 100644 index 00000000..ccab1796 --- /dev/null +++ b/internal/format/video/stream.go @@ -0,0 +1,86 @@ +package video + +// maxSuffixBits is the longest frame header splitFrame accepts from tile_info +// on: tile_info at most 3 bits for one tile, the quantizer 14, the loop filter +// 29 with the tx mode, and reduced_tx_set 1. A bound stream is built with it, +// so no picture the reader accepts can carry a longer header than planning +// counted. +const maxSuffixBits = 47 + +// Stream is a film's AV1 samples, one for each frame, as both containers carry +// them: no temporal delimiters, every OBU with its size. +type Stream struct { + Timeline + Width, Height int + // Level is the seq_level_idx the sequence header declares. + Level int + + config []byte + keyTU []byte + copyTU []byte + showTU []byte +} + +// NewStream wraps a coded picture in the stream's three kinds of sample. +// +// The sequence header travels in every key sample as well as in the +// container's codec configuration, so a player that starts at a key frame has +// it there - which is what the AV1 bindings of both containers ask of a sample +// a player can start from. +func NewStream(t Timeline, c Coded, width, height int) Stream { + key := keyFrame(c) + return newStream(t, c, width, height, chooseLevel(width, height, t.FPS, len(key))) +} + +// Bound is a stream no smaller than any picture of this size whose tile is at +// most tileBytes can make, for planning without coding the picture. +// +// The level is the maximum parameters one, because a level above seven costs +// the sequence header a tier bit and the bound must not be the short one. +func Bound(t Timeline, tileBytes, width, height int) Stream { + c := Coded{suffix: make([]byte, (maxSuffixBits+7)/8), suffixBits: maxSuffixBits, tile: make([]byte, tileBytes)} + return newStream(t, c, width, height, levelMaximumParameters) +} + +func newStream(t Timeline, c Coded, width, height, level int) Stream { + seq := obu(obuSequenceHeader, sequenceHeader(width, height, level)) + show := showCopy() + return Stream{ + Timeline: t, Width: width, Height: height, Level: level, + config: codecConfig(level, seq), + keyTU: joined(seq, keyFrame(c)), + copyTU: joined(hiddenCopy(c), show), + showTU: show, + } +} + +// joined is a followed by b in one allocation of the right size. +func joined(a, b []byte) []byte { + return append(append(make([]byte, 0, len(a)+len(b)), a...), b...) +} + +// Config is the AV1 codec configuration record with the sequence header OBU, +// what MP4 puts in av1C and Matroska in CodecPrivate. +func (s Stream) Config() []byte { return s.config } + +// Sample is frame i's bytes. The slices are shared between frames and must not +// be written to. +func (s Stream) Sample(i int64) []byte { + switch { + case s.IsKey(i): + return s.keyTU + case i%s.KeyEvery == 1: + return s.copyTU + default: + return s.showTU + } +} + +// SampleBytes is how long frame i's sample is. +func (s Stream) SampleBytes(i int64) int { return len(s.Sample(i)) } + +// KindBytes is the length of each of the three kinds of sample - what a +// container's arithmetic needs instead of walking every frame. +func (s Stream) KindBytes() (key, copied, shown int) { + return len(s.keyTU), len(s.copyTU), len(s.showTU) +} diff --git a/internal/format/video/timeline.go b/internal/format/video/timeline.go new file mode 100644 index 00000000..904e690b --- /dev/null +++ b/internal/format/video/timeline.go @@ -0,0 +1,99 @@ +package video + +import ( + "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/format" +) + +// Timeline is how many frames a film has, how far apart they are, and which +// of them are key frames. +type Timeline struct { + FPS int + Frames int64 + DurationMs int64 + // KeyEvery is how many frames apart key frames are. The first frame is + // always one. + KeyEvery int64 +} + +// NewTimeline settles a length and a key frame interval against a frame rate. +// +// A length has to end on a frame, and so does the interval, or the film would +// last a different time from the one asked for - refused with the two nearest +// lengths that do, by the owner's decision of 2026-10-06 +// (docs/WIDEO-2026-10-06.md section 12). Lengths are whole milliseconds, so at +// thirty frames a second they go in steps of 100ms, at twenty four in steps of +// 125ms: the step is the shortest whole number of milliseconds that is a whole +// number of frames. +func NewTimeline(formatID string, durationMs, keyIntervalMs int64, fps int) (Timeline, error) { + frames, err := onFrames(formatID, SettingDuration, durationMs, fps) + if err != nil { + return Timeline{}, err + } + keyEvery, err := onFrames(formatID, SettingKeyframeInterval, keyIntervalMs, fps) + if err != nil { + return Timeline{}, err + } + return Timeline{FPS: fps, Frames: frames, DurationMs: durationMs, KeyEvery: keyEvery}, nil +} + +// onFrames is how many frames a length is, or the refusal naming the two +// nearest lengths that are a whole number of frames. +func onFrames(formatID, key string, ms int64, fps int) (int64, error) { + if ms*int64(fps)%1000 == 0 && ms > 0 { + return ms * int64(fps) / 1000, nil + } + step := stepMs(fps) + below := ms / step * step + above := below + step + // The two lengths to ask for are part of the reason rather than a remedy + // beside it, because the command line prints the reason and not the + // remedy, and without them the refusal names a rule and no way out. + reason := core.Says("video.NotOnAFrame", + "at %d frames a second a length has to end on a frame, and lengths at that rate go in steps of %s. Ask for %s or %s", + core.A("FPS", fps), core.A("Step", core.FormatDuration(step)), + core.A("Below", core.FormatDuration(below)), core.A("Above", core.FormatDuration(above))) + if below == 0 { + reason = core.Says("video.NotOnAFrameShortest", + "at %d frames a second a length has to end on a frame, and lengths at that rate go in steps of %s. Ask for %s, the shortest film at this rate", + core.A("FPS", fps), core.A("Step", core.FormatDuration(step)), core.A("Above", core.FormatDuration(above))) + } + return 0, &format.PropertyValueError{Format: formatID, Key: key, Value: core.FormatDuration(ms), Reason: reason} +} + +// stepMs is the shortest whole number of milliseconds that is a whole number +// of frames at this rate: 1000 divided by the greatest common divisor of 1000 +// and the rate. +func stepMs(fps int) int64 { + a, b := int64(1000), int64(fps) + for b != 0 { + a, b = b, a%b + } + return 1000 / a +} + +// IsKey is whether frame i starts at a key frame. +func (t Timeline) IsKey(i int64) bool { return i%t.KeyEvery == 0 } + +// Keys is how many key frames the film has. +func (t Timeline) Keys() int64 { return (t.Frames + t.KeyEvery - 1) / t.KeyEvery } + +// Copies is how many hidden copies the film carries: one after each key frame +// that has a frame after it to show it in. A film whose every frame is a key +// frame needs none. +func (t Timeline) Copies() int64 { + if t.KeyEvery == 1 { + return 0 + } + n := t.Keys() + if (t.Keys()-1)*t.KeyEvery+1 >= t.Frames { + n-- // the last key frame is the film's last frame + } + return n +} + +// StartMs is when frame i is shown, in whole milliseconds rounded to the +// nearest, the way a container that counts in milliseconds stores it. +func (t Timeline) StartMs(i int64) int64 { + return (i*1000 + int64(t.FPS)/2) / int64(t.FPS) +} diff --git a/internal/format/webm/ebml.go b/internal/format/webm/ebml.go new file mode 100644 index 00000000..638dea2c --- /dev/null +++ b/internal/format/webm/ebml.go @@ -0,0 +1,166 @@ +package webm + +import ( + "encoding/binary" + "math" +) + +// Element IDs, RFC 9559 and the WebM subset of it. Written with their marker +// bits, the way they appear in the file. +const ( + idEBML = 0x1A45DFA3 + idEBMLVersion = 0x4286 + idEBMLReadVersion = 0x42F7 + idEBMLMaxIDLength = 0x42F2 + idEBMLMaxSizeLength = 0x42F3 + idDocType = 0x4282 + idDocTypeVersion = 0x4287 + idDocTypeReadVersion = 0x4285 + + idSegment = 0x18538067 + idSeekHead = 0x114D9B74 + idSeek = 0x4DBB + idSeekID = 0x53AB + idSeekPosition = 0x53AC + + idInfo = 0x1549A966 + idTimestampScale = 0x2AD7B1 + idDuration = 0x4489 + idMuxingApp = 0x4D80 + idWritingApp = 0x5741 + + idTracks = 0x1654AE6B + idTrackEntry = 0xAE + idTrackNumber = 0xD7 + idTrackUID = 0x73C5 + idTrackType = 0x83 + idFlagLacing = 0x9C + idCodecID = 0x86 + idCodecPrivate = 0x63A2 + idDefaultDuration = 0x23E383 + idVideo = 0xE0 + idPixelWidth = 0xB0 + idPixelHeight = 0xBA + idColour = 0x55B0 + idMatrix = 0x55B1 + idRange = 0x55B9 + idTransfer = 0x55BA + idPrimaries = 0x55BB + + idCluster = 0x1F43B675 + idTimestamp = 0xE7 + idSimpleBlock = 0xA3 + + idCues = 0x1C53BB6B + idCuePoint = 0xBB + idCueTime = 0xB3 + idCueTrackPositions = 0xB7 + idCueTrack = 0xF7 + idCueClusterPosition = 0xF1 + + idVoid = 0xEC +) + +// idBytes is an element ID as it is written: its own length, marker included. +func idBytes(id uint32) []byte { return appendBigEndian(nil, uint64(id), idLen(id)) } + +// sizeLen is how many bytes the shortest size field for n takes. The value +// whose bits are all ones in a length is reserved for "unknown size", which is +// why a length of k holds at most 2^(7k) - 2. +func sizeLen(n uint64) int { + for k := 1; k < 8; k++ { + if n < 1<<(7*uint(k))-1 { + return k + } + } + return 8 +} + +// sizeField writes n in exactly k bytes. +func sizeField(n uint64, k int) []byte { + out := make([]byte, k) + v := n | 1<<(7*uint(k)) + for i := k - 1; i >= 0; i-- { + out[i] = byte(v) + v >>= 8 + } + return out +} + +// appendHeader writes an element's ID and the shortest size field for its +// content onto dst, and allocates nothing when dst has the room - which is +// what lets the frames of a film be written without one allocation each. +func appendHeader(dst []byte, id uint32, content uint64) []byte { + dst = appendBigEndian(dst, uint64(id), idLen(id)) + k := sizeLen(content) + return appendBigEndian(dst, content|1<<(7*uint(k)), k) +} + +// appendUint is an unsigned integer element in the fewest bytes, at least one, +// written onto dst. +func appendUint(dst []byte, id uint32, v uint64) []byte { + n := uintLen(v) + return appendBigEndian(appendHeader(dst, id, uint64(n)), v, n) +} + +// appendBytes is an element holding b, written onto dst. +func appendBytes(dst []byte, id uint32, b []byte) []byte { + return append(appendHeader(dst, id, uint64(len(b))), b...) +} + +func appendString(dst []byte, id uint32, s string) []byte { + return append(appendHeader(dst, id, uint64(len(s))), s...) +} + +// appendFloat is a float element in eight bytes, which is how Matroska writes +// a duration. +func appendFloat(dst []byte, id uint32, f float64) []byte { + return binary.BigEndian.AppendUint64(appendHeader(dst, id, 8), math.Float64bits(f)) +} + +// appendFixedUint is an unsigned integer in eight bytes whatever its value. +func appendFixedUint(dst []byte, id uint32, v uint64) []byte { + return binary.BigEndian.AppendUint64(appendHeader(dst, id, 8), v) +} + +// appendBigEndian writes the low n bytes of v, most significant first. +func appendBigEndian(dst []byte, v uint64, n int) []byte { + for i := n - 1; i >= 0; i-- { + dst = append(dst, byte(v>>(8*uint(i)))) + } + return dst +} + +// idLen is how many bytes an element ID takes - its own length, marker +// included, which is where its highest byte is. +func idLen(id uint32) int { + switch { + case id >= 1<<24: + return 4 + case id >= 1<<16: + return 3 + case id >= 1<<8: + return 2 + } + return 1 +} + +// uintLen is how many bytes the shortest unsigned integer for v takes, at +// least one. +func uintLen(v uint64) int { + n := 1 + for n < 8 && v>>(8*uint(n)) != 0 { + n++ + } + return n +} + +// elementLen is how many bytes a whole element takes. +func elementLen(id uint32, content uint64) uint64 { + return uint64(idLen(id)+sizeLen(content)) + content +} + +// uintElementLen is uintElement's length without building it. +func uintElementLen(id uint32, v uint64) uint64 { + return elementLen(id, uint64(uintLen(v))) +} diff --git a/internal/format/webm/layout.go b/internal/format/webm/layout.go new file mode 100644 index 00000000..4a7b1708 --- /dev/null +++ b/internal/format/webm/layout.go @@ -0,0 +1,182 @@ +package webm + +import ( + "github.com/donislawdev/TestingFilesGenerator/internal/format/video" +) + +// application is what the file says made it, without a version, so a new +// release does not move the bytes of a file nobody changed (D11). The PDF +// producer says the same. +const application = "Testing Files Generator" + +// clusterSpanMs is the longest a cluster runs. A block's time is a 16 bit +// offset from its cluster's, so a cluster cannot pass 32 767 ms, and thirty +// seconds stays clear of that at every frame rate. +const clusterSpanMs = 30_000 + +// segmentSizeField is how long the segment's size is written, always. Fixed, +// so a file one byte longer is a Void one byte longer and nothing before it +// moves - measured with the probe of docs/WIDEO-2026-10-06.md section 9.6. +const segmentSizeField = 8 + +// minVoid is the smallest Void: its ID and a one byte size of nothing. Every +// file carries one, which is what makes every size above the minimum reachable +// a byte at a time. +const minVoid = 2 + +// cluster is one cluster's place in the file: the frames it holds, when it +// starts, where it is, and how big its content is. +type cluster struct { + first, last int64 // frames [first, last) + ts int64 + position uint64 // from the start of the segment's content + content uint64 +} + +// layout is the whole file worked out without writing it. Planning and writing +// both build one from a stream - planning from the bound of a rung, writing +// from the coded picture - so the two cannot come to disagree about where +// anything is. +// +// The work is per cluster, not per frame. A day at sixty frames a second is +// five million frames and at most ninety thousand clusters, because the key +// frame interval is at least a second. +type layout struct { + stream video.Stream + head []byte // the EBML header + seekHead []byte + info []byte + tracks []byte + clusters []cluster + cuesBody uint64 // what the Cues element holds + body uint64 // the segment's content without the Void +} + +func newLayout(s video.Stream) layout { + l := layout{stream: s, head: ebmlHeader(), info: info(s), tracks: tracks(s)} + l.seekHead = seekHead(0, 0, 0) + pos := uint64(len(l.seekHead) + len(l.info) + len(l.tracks)) + + var cues uint64 + for g := int64(0); g < s.Keys(); g++ { + pos, cues = l.addGroup(g, pos, cues) + } + l.cuesBody = cues + l.body = pos + elementLen(idCues, cues) + l.seekHead = seekHead(uint64(len(l.seekHead)), uint64(len(l.seekHead)+len(l.info)), pos) + return l +} + +// addGroup lays out the clusters of one group of pictures - the frames from +// one key frame to the next - from pos on, and gives back where the next +// group starts and the Cues so far. The first cluster of a group opens with +// its key frame and gets the group's cue point. +func (l *layout) addGroup(g int64, pos, cues uint64) (uint64, uint64) { + s := l.stream + key, copied, shown := s.KindBytes() + span := int64(s.FPS) * clusterSpanMs / 1000 + start, end := g*s.KeyEvery, min((g+1)*s.KeyEvery, s.Frames) + opening := len(l.clusters) + for first := start; first < end; first += span { + c := cluster{first: first, last: min(first+span, end), ts: s.StartMs(first), position: pos} + c.content = uintElementLen(idTimestamp, uint64(c.ts)) + blocks(c, s.KeyEvery, key, copied, shown) + pos += elementLen(idCluster, c.content) + l.clusters = append(l.clusters, c) + } + return pos, cues + cuePointLen(l.clusters[opening]) +} + +// blocks is the bytes a cluster's SimpleBlocks take. A cluster that starts a +// group of pictures opens with the key sample and the hidden copy, and the +// rest of every cluster is frames that show the copy again. +func blocks(c cluster, keyEvery int64, key, copied, shown int) uint64 { + n := c.last - c.first + var total uint64 + if c.first%keyEvery == 0 { + total += blockLen(key) + n-- + if n > 0 && keyEvery > 1 { + total += blockLen(copied) + n-- + } + } + return total + uint64(n)*blockLen(shown) +} + +// blockLen is a SimpleBlock element carrying a sample: track number, a two +// byte time offset and the flags, then the sample. +func blockLen(sample int) uint64 { return elementLen(idSimpleBlock, uint64(4+sample)) } + +func cuePointLen(c cluster) uint64 { + positions := uintElementLen(idCueTrack, 1) + uintElementLen(idCueClusterPosition, c.position) + return elementLen(idCuePoint, uintElementLen(idCueTime, uint64(c.ts))+elementLen(idCueTrackPositions, positions)) +} + +// fileBytes is the file without its Void. +func (l layout) fileBytes() int64 { + return int64(len(l.head)) + int64(idLen(idSegment)) + segmentSizeField + int64(l.body) +} + +func ebmlHeader() []byte { + b := make([]byte, 0, 48) + b = appendUint(b, idEBMLVersion, 1) + b = appendUint(b, idEBMLReadVersion, 1) + b = appendUint(b, idEBMLMaxIDLength, 4) + b = appendUint(b, idEBMLMaxSizeLength, 8) + b = appendString(b, idDocType, "webm") + b = appendUint(b, idDocTypeVersion, 4) + b = appendUint(b, idDocTypeReadVersion, 2) + return appendBytes(make([]byte, 0, len(b)+8), idEBML, b) +} + +// seekHead points at Info, Tracks and Cues. The positions are written in +// eight bytes each, so the SeekHead is the same length whatever they are and +// can be built before anything after it is known. +func seekHead(infoPos, tracksPos, cuesPos uint64) []byte { + b := make([]byte, 0, 96) + for _, s := range [3]struct { + id uint32 + pos uint64 + }{{idInfo, infoPos}, {idTracks, tracksPos}, {idCues, cuesPos}} { + b = appendHeader(b, idSeek, elementLen(idSeekID, uint64(idLen(s.id)))+elementLen(idSeekPosition, 8)) + b = appendBigEndian(appendHeader(b, idSeekID, uint64(idLen(s.id))), uint64(s.id), idLen(s.id)) + b = appendFixedUint(b, idSeekPosition, s.pos) + } + return appendBytes(make([]byte, 0, len(b)+8), idSeekHead, b) +} + +func info(s video.Stream) []byte { + b := make([]byte, 0, 80) + b = appendUint(b, idTimestampScale, 1_000_000) + b = appendFloat(b, idDuration, float64(s.DurationMs)) + b = appendString(b, idMuxingApp, application) + b = appendString(b, idWritingApp, application) + return appendBytes(make([]byte, 0, len(b)+8), idInfo, b) +} + +// tracks is the one video track. The colour is said again here, in the +// numbers the sequence header uses, for a player that reads the container's +// description rather than the stream's. +func tracks(s video.Stream) []byte { + v := make([]byte, 0, 48) + v = appendUint(v, idPixelWidth, uint64(s.Width)) + v = appendUint(v, idPixelHeight, uint64(s.Height)) + v = appendHeader(v, idColour, 4*uintElementLen(idMatrix, 1)) + v = appendUint(v, idMatrix, 1) + v = appendUint(v, idRange, 1) + v = appendUint(v, idTransfer, 1) + v = appendUint(v, idPrimaries, 1) + + e := make([]byte, 0, 64+len(s.Config())+len(v)) + e = appendUint(e, idTrackNumber, 1) + e = appendUint(e, idTrackUID, 1) + e = appendUint(e, idTrackType, 1) + e = appendUint(e, idFlagLacing, 0) + e = appendString(e, idCodecID, "V_AV1") + e = appendBytes(e, idCodecPrivate, s.Config()) + e = appendUint(e, idDefaultDuration, uint64(1_000_000_000/s.FPS)) + e = appendBytes(e, idVideo, v) + + t := appendBytes(make([]byte, 0, len(e)+16), idTrackEntry, e) + return appendBytes(make([]byte, 0, len(t)+8), idTracks, t) +} diff --git a/internal/format/webm/webm.go b/internal/format/webm/webm.go new file mode 100644 index 00000000..129b1849 --- /dev/null +++ b/internal/format/webm/webm.go @@ -0,0 +1,286 @@ +// Package webm generates WebM films: AV1 in the WebM subset of Matroska, with +// no sound. +// +// The picture and the stream are internal/format/video's, shared with MP4. +// What is here is the container: EBML written as a stream, a SeekHead, the one +// track, clusters, Cues, and a Void at the end that carries the padding. +// +// The padding channel is a Void element, the one Matroska defines for bytes +// that mean nothing, after the Cues and inside the segment. The segment's size +// is always written in eight bytes, so the Void can grow a byte at a time and +// nothing before it moves. Measured on 2026-10-06 (docs/WIDEO-2026-10-06.md +// section 9.6) at a Void of 2, 3, 128, 129, 16 385, 16 386 and 1 048 576 +// bytes and with none: every file came out at the size asked for, libaom +// decoded every frame of every one, and Chromium played and seeked them all. +// A Void is always written, at least its two bytes, which leaves no size above +// the minimum that cannot be reached. +package webm + +import ( + "context" + "fmt" + "io" + + "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/format/video" +) + +const ( + id = "webm" + generatorVersion = "1" +) + +func init() { + format.Register(format.Descriptor{ + ID: id, + Name: "WebM video", + Extension: ".webm", + Fidelity: format.FidelityFull, + Determinism: format.DeterminismByte, + + MinBytes: minimumBytes(), + + Padding: format.PaddingChannel{ + Name: "a Void element at the end of the file", + Where: format.PlacementEnd, + Capacity: 0, + }, + Label: format.LabelVisible, + Oracle: "libaom", + Properties: video.Properties(), + JointLimits: video.JointLimits(), + GeneratorVersion: generatorVersion, + Generator: generator{}, + }) +} + +// minimumBytes is the smallest film at the default settings - the smallest +// rung's bound, ten seconds at thirty frames, one key frame - with its Void. +// Worked out from the same layout planning uses rather than written down, so +// the two cannot disagree. +func minimumBytes() int64 { + t, err := video.NewTimeline(id, 10_000, 60_000, 30) + if err != nil { + panic(err) + } + smallest := video.Ladder[len(video.Ladder)-1] + return newLayout(video.Bound(t, smallest.Ceiling, smallest.Width, smallest.Height)).fileBytes() + minVoid +} + +type generator struct{} + +type memo struct { + choice video.Choice + settings video.Settings + total int64 +} + +func (generator) Plan(r format.Request) (format.Plan, error) { + label := "" + if r.Label { + label = core.Label(id, r.Bytes, r.Seed) + } + s, err := video.Read(id, r.Properties) + if err != nil { + return format.Plan{}, err + } + fits := func(st video.Stream) bool { return newLayout(st).fileBytes()+minVoid <= r.Bytes } + c, st, err := video.Choose(id, r, s, label, fits) + if err != nil { + return format.Plan{}, err + } + if need := newLayout(st).fileBytes() + minVoid; need > r.Bytes { + return format.Plan{}, belowMinimum(r.Bytes, need, c, s) + } + + p := format.Plan{ + Bytes: r.Bytes, + Exact: true, + Determinism: format.DeterminismByte, + Properties: video.Facts(c, s), + Memo: memo{choice: c, settings: s, total: r.Bytes}, + } + if r.Label && !c.Labelled() { + p.Notes = append(p.Notes, format.Note{ + Code: "label_omitted", + Detail: core.Says("format.ThePictureIsPxWideAnd", "The picture is %d px wide and the label needs more room, so this file carries no visible label. Its name and the manifest still identify it.", core.A("Width", c.Width)), + }) + } + return p, nil +} + +// belowMinimum names the four things D6 asks for, and the knobs that make a +// film smaller - every one of them is a setting, so each is named. +func belowMinimum(requested, need int64, c video.Choice, s video.Settings) error { + return &format.BelowMinimumError{ + Format: "WebM", + Requested: requested, + Minimum: need, + Reason: core.Says("webm.MinimumReason", + "a film of %s at %d frames a second with a key frame every %s and a %dx%d picture takes %d B, and the file always carries a Void element, which costs %d B even when it holds nothing", + core.A("Duration", core.FormatDuration(s.DurationMs)), core.A("FPS", s.FPS), + core.A("Interval", core.FormatDuration(s.KeyEvery*1000/int64(s.FPS))), + core.A("Width", c.Width), core.A("Height", c.Height), + core.A("Bytes", need-minVoid), core.A("Void", minVoid)), + Hint: hint(need, c), + } +} + +// hint offers a smaller picture only when the request named one. A picture +// chosen to fit is already the smallest that would, so offering a smaller +// one would send somebody to a setting that changes nothing. +func hint(need int64, c video.Choice) core.Said { + if c.Named { + return core.Says("webm.AskForBOrMore", + "Ask for %d B or more, or a shorter film, fewer frames a second, key frames further apart or a smaller picture", + core.A("Floor", need)) + } + return core.Says("webm.AskForBOrMoreFilm", + "Ask for %d B or more, or a shorter film, fewer frames a second or key frames further apart", + core.A("Floor", need)) +} + +func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { + m, ok := p.Memo.(memo) + if !ok { + return core.Defect(fmt.Errorf("webm: the plan was not produced by this generator")) + } + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + coded, err := m.choice.Code() + if err != nil { + return err + } + st := video.NewStream(m.settings.Timeline, coded, m.choice.Width, m.choice.Height) + l := newLayout(st) + void := m.total - l.fileBytes() + if void < minVoid { + return core.Defect(fmt.Errorf("webm: the film came to %d B and the file was to be %d B, which leaves no room for the Void every one of these carries", + l.fileBytes(), m.total)) + } + + out := &sticky{w: w} + out.write(l.head) + out.write(idBytes(idSegment)) + out.write(sizeField(l.body+uint64(void), segmentSizeField)) + out.write(l.seekHead) + out.write(l.info) + out.write(l.tracks) + if err := writeClusters(ctx, out, l); err != nil { + return err + } + writeCues(out, l) + if out.err != nil { + return out.err + } + return writeVoid(ctx, w, void, m.choice.Seed) +} + +// sticky keeps the first error, so a long run of small writes checks once, +// and carries the few bytes a header takes so writing one allocates nothing. +// A film's frames number in the millions, and a header built fresh for each +// was an allocation per frame - measured 2026-10-06, 853 objects for a ten +// second film where the guard on generators allows 128. +type sticky struct { + w io.Writer + err error + scratch [24]byte +} + +func (s *sticky) write(b []byte) { + if s.err == nil { + _, s.err = s.w.Write(b) + } +} + +func (s *sticky) header(id uint32, content uint64) { + s.write(appendHeader(s.scratch[:0], id, content)) +} + +func (s *sticky) uint(id uint32, v uint64) { + s.write(appendUint(s.scratch[:0], id, v)) +} + +func writeClusters(ctx context.Context, out *sticky, l layout) error { + for _, c := range l.clusters { + if err := ctx.Err(); err != nil { + return err + } + writeCluster(out, l.stream, c) + if out.err != nil { + return out.err + } + } + return nil +} + +// writeCluster writes one cluster: its time, then a SimpleBlock for each of +// its frames - track 1, the frame's time from the cluster's, and the key flag +// on a key frame. +func writeCluster(out *sticky, st video.Stream, c cluster) { + out.header(idCluster, c.content) + out.uint(idTimestamp, uint64(c.ts)) + var block [4]byte + block[0] = 0x81 + for i := c.first; i < c.last; i++ { + sample := st.Sample(i) + rel := st.StartMs(i) - c.ts + block[1], block[2], block[3] = byte(rel>>8), byte(rel), 0 + if st.IsKey(i) { + block[3] = 0x80 + } + out.header(idSimpleBlock, uint64(4+len(sample))) + out.write(block[:]) + out.write(sample) + } +} + +// writeCues points at every cluster that opens with a key frame, with the +// same arithmetic the layout counted them by. +func writeCues(out *sticky, l layout) { + out.header(idCues, l.cuesBody) + for _, c := range l.clusters { + if !l.stream.IsKey(c.first) { + continue + } + positions := uintElementLen(idCueTrack, 1) + uintElementLen(idCueClusterPosition, c.position) + out.header(idCuePoint, uintElementLen(idCueTime, uint64(c.ts))+elementLen(idCueTrackPositions, positions)) + out.uint(idCueTime, uint64(c.ts)) + out.header(idCueTrackPositions, positions) + out.uint(idCueTrack, 1) + out.uint(idCueClusterPosition, c.position) + } +} + +// writeVoid fills the rest of the file with one Void element of exactly total +// bytes, without ever holding its content. +func writeVoid(ctx context.Context, w io.Writer, total int64, seed uint64) error { + k := 1 + for ; k < 8; k++ { + if payload := uint64(total) - 1 - uint64(k); payload < 1<<(7*uint(k))-1 { + break + } + } + payload := total - 1 - int64(k) + if _, err := w.Write(append([]byte{idVoid}, sizeField(uint64(payload), k)...)); err != nil { + return err + } + rng := core.NewRand(seed) + buf := make([]byte, 32*1024) + for left := payload; left > 0; { + n := min(int64(len(buf)), left) + if err := ctx.Err(); err != nil { + return err + } + core.FillRandomLE(buf[:n], rng) + if _, err := w.Write(buf[:n]); err != nil { + return err + } + left -= n + } + return nil +} diff --git a/internal/guard/boxwidth_test.go b/internal/guard/boxwidth_test.go index 949bd19e..541a5f1d 100644 --- a/internal/guard/boxwidth_test.go +++ b/internal/guard/boxwidth_test.go @@ -33,7 +33,7 @@ func TestABoxForANumberIsNotAsWideAsTheForm(t *testing.T) { for _, d := range format.All() { choose.to(d.ID) for _, p := range d.Properties { - if p.Kind != format.PropertyInt && p.Kind != format.PropertySize { + if p.Kind != format.PropertyInt && p.Kind != format.PropertySize && p.Kind != format.PropertyDuration { continue } control := controlUnder(generate, settingLabelOf(p)) @@ -70,7 +70,7 @@ func TestABoxIsWideEnoughForItsOwnPlaceholder(t *testing.T) { for _, d := range format.All() { choose.to(d.ID) for _, p := range d.Properties { - if p.Kind != format.PropertyInt && p.Kind != format.PropertySize { + if p.Kind != format.PropertyInt && p.Kind != format.PropertySize && p.Kind != format.PropertyDuration { continue } control := controlUnder(generate, settingLabelOf(p)) @@ -235,7 +235,7 @@ func TestABoxForANumberIsNotAsWideAsTheFormOnTheBatchScreen(t *testing.T) { for _, d := range format.All() { narrow := false for _, p := range d.Properties { - if p.Kind == format.PropertyInt || p.Kind == format.PropertySize { + if p.Kind == format.PropertyInt || p.Kind == format.PropertySize || p.Kind == format.PropertyDuration { narrow = true } } @@ -251,7 +251,7 @@ func TestABoxForANumberIsNotAsWideAsTheFormOnTheBatchScreen(t *testing.T) { } layOut() for _, p := range d.Properties { - if p.Kind != format.PropertyInt && p.Kind != format.PropertySize { + if p.Kind != format.PropertyInt && p.Kind != format.PropertySize && p.Kind != format.PropertyDuration { continue } control := controlUnder(batches, settingLabelOf(p)) diff --git a/internal/guard/choiceseed_test.go b/internal/guard/choiceseed_test.go index 5ebadc79..8339fed2 100644 --- a/internal/guard/choiceseed_test.go +++ b/internal/guard/choiceseed_test.go @@ -87,7 +87,7 @@ func nonDefaultValues(p format.Property) []string { values = p.Choices case format.PropertyBool: values = []string{"true", "false"} - case format.PropertyInt, format.PropertySize, format.PropertyText: + case format.PropertyInt, format.PropertySize, format.PropertyDuration, format.PropertyText: // Open ranges and free text have no list to walk. A kind added // later reddens the linter here rather than being skipped unseen. } diff --git a/internal/guard/duration_test.go b/internal/guard/duration_test.go new file mode 100644 index 00000000..06688601 --- /dev/null +++ b/internal/guard/duration_test.go @@ -0,0 +1,194 @@ +package guard + +import ( + "strings" + "testing" + + "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/gui/parts" + "github.com/donislawdev/TestingFilesGenerator/internal/gui/text" +) + +// A length of time is written the way a person writes it and counted in whole +// milliseconds - 10s, 1m30s, 1h, 500ms. It is the value a video's duration is +// asked for in, and later a sound's (docs/WIDEO-2026-10-06.md section 12). +// +// The same promise as a size, for the same reason: exact or refused. A film +// asked for at 59.9 seconds that comes out at 60 is a test of the wrong limit, +// and nothing in a passing run would say so. + +func TestDurationsCountInWholeMilliseconds(t *testing.T) { + cases := []struct { + in string + want int64 + }{ + {"0s", 0}, + {"1ms", 1}, + {"500ms", 500}, + {"1s", 1_000}, + {"10s", 10_000}, + {"59.9s", 59_900}, + {"0.1s", 100}, + {".5s", 500}, + {"1m", 60_000}, + {"1m30s", 90_000}, + {"1h", 3_600_000}, + {"1.5h", 5_400_000}, + {"1h30m", 5_400_000}, + {"2h3m4s5ms", 7_384_005}, + {"1h0.033s", 3_600_033}, + + // Case does not matter, and a space between the parts is how people + // write it by hand. + {"10S", 10_000}, + {"1H 30M", 5_400_000}, + {" 1m 30s ", 90_000}, + {"1 h", 3_600_000}, + } + for _, c := range cases { + got, err := core.ParseDuration(c.in) + if err != nil { + t.Errorf("%q was refused: %v", c.in, err) + continue + } + if got != c.want { + t.Errorf("%q gave %d ms, expected %d ms", c.in, got, c.want) + } + } +} + +func TestEveryWayOfWritingOneLengthGivesTheSameMilliseconds(t *testing.T) { + // The arithmetic is done on the digits, not through a float. 0.025h is + // 90 000 ms exactly only if nothing on the way rounds. + for _, group := range [][]string{ + {"90s", "1m30s", "1.5m", "90000ms", "0.025h", "1m 30s"}, + {"1h30m", "90m", "5400s", "1.5h", "5400000ms"}, + {"0.1s", "100ms", "0.100s"}, + } { + first, err := core.ParseDuration(group[0]) + if err != nil { + t.Fatalf("%q was refused: %v", group[0], err) + } + for _, other := range group[1:] { + got, err := core.ParseDuration(other) + if err != nil { + t.Errorf("%q was refused: %v", other, err) + continue + } + if got != first { + t.Errorf("%q gave %d ms and %q gave %d ms - one length written two ways", group[0], first, other, got) + } + } + } +} + +func TestADurationThatIsNotAWholeMillisecondIsRefused(t *testing.T) { + for _, in := range []string{"0.0001s", "1.0005s", "0.5ms", "0.0000001h"} { + got, err := core.ParseDuration(in) + if err == nil { + t.Errorf("%q was accepted as %d ms instead of being refused", in, got) + continue + } + if !strings.Contains(err.Error(), "whole millisecond") { + t.Errorf("%q was refused without saying why: %v", in, err) + } + } +} + +func TestNonsenseDurationsAreRefusedWithTheReason(t *testing.T) { + // Each refusal names what is actually wrong. A bare number is the one that + // matters most: reading 10 as ten seconds would be a guess, and so would + // ten milliseconds. + cases := []struct{ in, says string }{ + {"", "empty"}, + {"10", "no unit"}, + {"-5s", "not a length of time"}, + {"1,5s", "not a length of time"}, + {"5min", "unknown unit"}, + {"30s1m", "out of order"}, + {"1m1m", "out of order"}, + {"99999999999999999999h", "too long"}, + } + for _, c := range cases { + got, err := core.ParseDuration(c.in) + if err == nil { + t.Errorf("%q was accepted as %d ms", c.in, got) + continue + } + if got != 0 { + t.Errorf("%q was refused and still came back as %d ms", c.in, got) + } + if !strings.Contains(err.Error(), c.says) { + t.Errorf("%q was refused as %q, which does not say %q", c.in, err.Error(), c.says) + } + } +} + +func TestAWrittenDurationReadsBackAsTheSameMilliseconds(t *testing.T) { + // FormatDuration is what a refusal and a window show, and a value shown + // to somebody has to be one they can type back in. + values := []int64{0, 1, 33, 999, 1_000, 1_033, 59_999, 60_000, 90_500, 3_599_999, 3_600_000, 3_600_033, 86_400_000, 1<<40 + 7} + for v := int64(0); v < 200_000; v += 7 { + values = append(values, v) + } + for _, v := range values { + written := core.FormatDuration(v) + got, err := core.ParseDuration(written) + if err != nil { + t.Errorf("%d ms was written as %q, which is refused: %v", v, written, err) + continue + } + if got != v { + t.Errorf("%d ms was written as %q, which reads back as %d ms", v, written, got) + } + } + for v, want := range map[int64]string{0: "0s", 500: "500ms", 10_000: "10s", 90_000: "1m30s", 3_600_000: "1h", 1_033: "1.033s"} { + if got := core.FormatDuration(v); got != want { + t.Errorf("%d ms was written as %q, expected the shortest form %q", v, got, want) + } + } +} + +// A setting of this kind takes a length inside its range, refuses one outside +// it or one it cannot read, and says what it takes in the same words empty or +// refused, on both surfaces. +// +// Asked of a declaration made up here, because on the day the kind arrived no +// format declared one yet - the registry-wide guards (the window against tfg +// formats, empty against refused) take over the moment one does, and this one +// keeps the kind covered if that format ever goes. +func TestADurationSettingTakesALengthInsideItsRangeAndSaysSo(t *testing.T) { + p := format.Property{Name: "duration", Kind: format.PropertyDuration, Min: 33, Max: 86_400_000, Default: "10s"} + + for _, ok := range []string{"10s", "33ms", "24h", "1m30s", "59.9s"} { + if bad := p.Allows(ok); !bad.IsZero() { + t.Errorf("%q is inside 33ms to 24h and was refused: %s", ok, bad) + } + } + const want = "a length of time from 33ms to 24h, such as 10s, 1m30s or 500ms" + if got := p.Allowed(); got != want+", default 10s" { + t.Errorf("an empty field says %q, expected %q", got, want+", default 10s") + } + for _, bad := range []string{"32ms", "24h1ms", "10", "banana", "-1s"} { + why := p.Allows(bad) + if why.IsZero() { + t.Errorf("%q is outside 33ms to 24h or not a length and was accepted", bad) + continue + } + if why.String() != "it takes "+want { + t.Errorf("%q was refused as %q, and an empty field says %q - one setting described two ways", bad, why.String(), want) + } + } + + if got := text.AllowedWholeNumber(); got != "whole number" { + t.Fatalf("this process speaks another language (%q), so nothing here is English to compare", got) + } + if got := parts.Allowed(p); got != p.Allowed() { + t.Errorf("the window says %q and tfg formats says %q", got, p.Allowed()) + } + unbounded := format.Property{Name: "duration", Kind: format.PropertyDuration} + if got := parts.Allowed(unbounded); got != unbounded.Allowed() || got != "a length of time such as 10s, 1m30s or 500ms" { + t.Errorf("without a range the window says %q and tfg formats says %q", got, unbounded.Allowed()) + } +} diff --git a/internal/guard/film_test.go b/internal/guard/film_test.go new file mode 100644 index 00000000..0621aaf1 --- /dev/null +++ b/internal/guard/film_test.go @@ -0,0 +1,542 @@ +package guard + +import ( + "context" + "encoding/binary" + "errors" + "fmt" + "math" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/donislawdev/TestingFilesGenerator/internal/engine" + "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/format/video" + "github.com/donislawdev/TestingFilesGenerator/internal/oracle" +) + +// A film read back from its bytes, not from the plan that made it - the lesson +// of animation_test.go, where a GIF the plan called animated came out still. +// +// The reader knows WebM's elements and AV1's headers as far as the questions +// below need, and no further. What it does not understand it refuses, so a +// change in the writer that this reader cannot follow fails loudly rather than +// being read as something it is not. + +type filmBlock struct { + ts int64 + key bool + data []byte +} + +type webmFilm struct { + durationMs float64 + width int + height int + codecPrivate []byte + blocks []filmBlock + cuePoints int + voidLast bool +} + +func filmOne(t *testing.T, target engine.Target) ([]byte, map[string]any) { + t.Helper() + dir := t.TempDir() + opt := engine.Options{OutDir: dir, Seed: goldenSeed, Command: "test"} + planned, err := engine.Plan([]engine.Target{target}, opt) + if err != nil { + t.Fatalf("planning %s: %v", target.Format, err) + } + if _, err := engine.Run(context.Background(), planned, opt); err != nil { + t.Fatalf("running %s: %v", target.Format, err) + } + b, err := os.ReadFile(filepath.Join(dir, planned[0].Name)) + if err != nil { + t.Fatalf("reading the film back: %v", err) + } + return b, planned[0].Plan.Properties +} + +// ebmlVint reads an element ID (keeping its marker) or a size (without it). +func ebmlVint(b []byte, keepMarker bool) (value uint64, length int, err error) { + if len(b) == 0 { + return 0, 0, fmt.Errorf("the file ends where an element should start") + } + length = 1 + for mask := byte(0x80); length <= 8 && b[0]&mask == 0; mask >>= 1 { + length++ + } + if length > 8 || length > len(b) { + return 0, 0, fmt.Errorf("a variable length number runs past the end") + } + value = uint64(b[0]) + if !keepMarker { + value &= uint64(0xFF >> length) + } + for _, c := range b[1:length] { + value = value<<8 | uint64(c) + } + return value, length, nil +} + +// ebmlElements walks a run of elements, each exactly covering the next, and +// refuses one whose size runs past the run. +func ebmlElements(b []byte, each func(id uint64, body []byte) error) error { + for pos := 0; pos < len(b); { + id, n, err := ebmlVint(b[pos:], true) + if err != nil { + return err + } + size, m, err := ebmlVint(b[pos+n:], false) + if err != nil { + return err + } + start := pos + n + m + if uint64(len(b)-start) < size { + return fmt.Errorf("element %#x at %d says %d B and %d remain", id, pos, size, len(b)-start) + } + if err := each(id, b[start:start+int(size)]); err != nil { + return err + } + pos = start + int(size) + } + return nil +} + +func beUint(b []byte) uint64 { + var v uint64 + for _, c := range b { + v = v<<8 | uint64(c) + } + return v +} + +func walkWebM(b []byte) (webmFilm, error) { + var f webmFilm + sawSegment := false + err := ebmlElements(b, func(id uint64, body []byte) error { + switch id { + case 0x1A45DFA3: + return nil + case 0x18538067: + sawSegment = true + return walkSegment(&f, body) + } + return fmt.Errorf("a top level element %#x that is neither the EBML header nor the segment", id) + }) + if err == nil && !sawSegment { + err = fmt.Errorf("no segment") + } + return f, err +} + +func walkSegment(f *webmFilm, body []byte) error { + return ebmlElements(body, func(id uint64, el []byte) error { + f.voidLast = false + switch id { + case 0x1549A966: // Info + return ebmlElements(el, func(id uint64, v []byte) error { + if id == 0x4489 { + f.durationMs = math.Float64frombits(binary.BigEndian.Uint64(v)) + } + return nil + }) + case 0x1654AE6B: // Tracks + return walkTracks(f, el) + case 0x1F43B675: // Cluster + return walkCluster(f, el) + case 0x1C53BB6B: // Cues + return ebmlElements(el, func(id uint64, _ []byte) error { + if id == 0xBB { + f.cuePoints++ + } + return nil + }) + case 0xEC: // Void + f.voidLast = true + } + return nil + }) +} + +func walkTracks(f *webmFilm, el []byte) error { + return ebmlElements(el, func(id uint64, entry []byte) error { + return ebmlElements(entry, func(id uint64, v []byte) error { + switch id { + case 0x63A2: + f.codecPrivate = v + case 0xE0: + return ebmlElements(v, func(id uint64, d []byte) error { + switch id { + case 0xB0: + f.width = int(beUint(d)) + case 0xBA: + f.height = int(beUint(d)) + } + return nil + }) + } + return nil + }) + }) +} + +func walkCluster(f *webmFilm, el []byte) error { + var clusterTs int64 + return ebmlElements(el, func(id uint64, v []byte) error { + switch id { + case 0xE7: + clusterTs = int64(beUint(v)) + case 0xA3: + if len(v) < 4 || v[0] != 0x81 { + return fmt.Errorf("a block that is not on track 1") + } + rel := int64(int16(binary.BigEndian.Uint16(v[1:3]))) + f.blocks = append(f.blocks, filmBlock{ts: clusterTs + rel, key: v[3]&0x80 != 0, data: v[4:]}) + default: + return fmt.Errorf("a cluster holds element %#x", id) + } + return nil + }) +} + +// av1Frame is what one OBU of a sample says about showing pictures. +type av1Frame struct { + showExisting bool + slot int + frameType int + shown bool + showable bool + refresh int +} + +// readFrames reads the frame headers of one sample, under the sequence header +// this tool writes: no frame ids, no order hint, screen content off. A sample +// holding anything else - a temporal delimiter, a tile group on its own - is +// refused, because the containers carry neither. +func readFrames(sample []byte) ([]av1Frame, error) { + var out []av1Frame + for len(sample) > 0 { + h := sample[0] + if h&0x80 != 0 || h&0x04 != 0 || h&0x02 == 0 { + return nil, fmt.Errorf("an OBU header %#02x", h) + } + typ := int(h>>3) & 0xf + size, used := 0, 0 + for i := 1; i < len(sample) && i < 9; i++ { + size |= int(sample[i]&0x7f) << (7 * (i - 1)) + if sample[i]&0x80 == 0 { + used = i + break + } + } + if used == 0 || 1+used+size > len(sample) { + return nil, fmt.Errorf("an OBU of type %d runs past its sample", typ) + } + payload := sample[1+used : 1+used+size] + sample = sample[1+used+size:] + switch typ { + case 1: // sequence header + case 3, 6: + out = append(out, readFrameHeader(payload)) + default: + return nil, fmt.Errorf("an OBU of type %d in a sample", typ) + } + } + return out, nil +} + +func readFrameHeader(p []byte) av1Frame { + pos := 0 + bit := func() int { + v := int(p[pos/8]>>(7-pos%8)) & 1 + pos++ + return v + } + bits := func(n int) int { + v := 0 + for range n { + v = v<<1 | bit() + } + return v + } + var f av1Frame + if bit() == 1 { + f.showExisting, f.slot = true, bits(3) + return f + } + f.frameType = bits(2) + f.shown = bit() == 1 + if f.shown { + f.showable = f.frameType != 0 + } else { + f.showable = bit() == 1 + } + if f.frameType == 0 && f.shown { + f.refresh = 0xFF + return f + } + bit() // error_resilient_mode + bit() // disable_cdf_update + bit() // frame_size_override_flag + f.refresh = bits(8) + return f +} + +// showRuleBroken walks a film's frames the way a decoder fills its slots and +// names the first frame that shows a slot holding nothing showable - which is +// what showing a shown key frame again is. +func showRuleBroken(blocks []filmBlock) (int, string) { + var showable [8]bool + for i, b := range blocks { + frames, err := readFrames(b.data) + if err != nil { + return i, err.Error() + } + for _, f := range frames { + switch { + case f.showExisting && !showable[f.slot]: + return i, fmt.Sprintf("shows slot %d, which holds nothing that may be shown again", f.slot) + case f.showExisting: + default: + for s := range 8 { + if f.refresh&(1<