From 8e1870e556067cb81ca91217fed7eb39874a7f1e Mon Sep 17 00:00:00 2001 From: TonyTonyCoder11 Date: Sat, 24 Oct 2026 04:47:08 +0200 Subject: [PATCH 1/7] Show a MessagePack body as indented text MessagePack describes itself, so a body whose media type says so needs no schema. Integers keep every digit up to 64 bits, bytes are hex, a map with keys that are not strings has them written as text, and a timestamp extension is an ISO 8601 date. A body that was cut, or that does not end where its last value does, is left to the hex dump. --- .../flutwire/lib/src/decode/decoded_text.dart | 119 ++++++++ .../flutwire/lib/src/decode/messagepack.dart | 275 ++++++++++++++++++ .../test/decode/decoded_text_test.dart | 110 +++++++ .../test/decode/messagepack_test.dart | 263 +++++++++++++++++ 4 files changed, 767 insertions(+) create mode 100644 packages/flutwire/lib/src/decode/decoded_text.dart create mode 100644 packages/flutwire/lib/src/decode/messagepack.dart create mode 100644 packages/flutwire/test/decode/decoded_text_test.dart create mode 100644 packages/flutwire/test/decode/messagepack_test.dart diff --git a/packages/flutwire/lib/src/decode/decoded_text.dart b/packages/flutwire/lib/src/decode/decoded_text.dart new file mode 100644 index 0000000..4f18f60 --- /dev/null +++ b/packages/flutwire/lib/src/decode/decoded_text.dart @@ -0,0 +1,119 @@ +import 'dart:convert'; +import 'dart:typed_data'; + +/// Binary content in a value a decoder read. It is written as hex, the way the +/// inspector writes bytes everywhere else. +final class DecodedBytes { + /// Wraps [bytes]. + const DecodedBytes(this.bytes); + + /// The bytes. + final Uint8List bytes; + + /// How many bytes are written out before the text says there are more. + static const int shown = 64; + + @override + String toString() { + final head = bytes.length <= shown ? bytes : bytes.sublist(0, shown); + final hex = [ + for (final byte in head) byte.toRadixString(16).padLeft(2, '0'), + ].join(' '); + return bytes.length > shown ? '$hex … (${bytes.length} bytes)' : hex; + } +} + +/// Writes a value a decoder read as indented text that reads like JSON. +/// +/// It takes `null`, `bool`, `int`, `BigInt`, `double`, `String`, `DateTime`, +/// [DecodedBytes], lists, and maps with string keys, and keeps the order of a +/// map. It is not JSON where JSON cannot say it: a number too big for a double +/// is written out in full, `NaN` and the infinities are strings, and bytes are +/// a string of hex. +String renderDecoded(Object? value) { + final out = StringBuffer(); + _write(out, value, 0); + return out.toString(); +} + +/// How deep a value is followed before the text gives up and writes `…`. +const int _maxDepth = 64; + +/// How long a list of plain values may be and still go on one line. +const int _inlineWidth = 72; + +void _write(StringBuffer out, Object? value, int depth) { + if (depth > _maxDepth) { + out.write('"…"'); + return; + } + switch (value) { + case null: + out.write('null'); + case final bool flag: + out.write(flag); + case final int number: + out.write(number); + case final BigInt number: + out.write(number); + case final double number: + out.write(number.isFinite ? number : jsonEncode(number.toString())); + case final String text: + out.write(jsonEncode(text)); + case final DateTime time: + out.write(jsonEncode(time.toUtc().toIso8601String())); + case final DecodedBytes bytes: + out.write(jsonEncode(bytes.toString())); + case final List list: + _writeList(out, list, depth); + case final Map map: + _writeMap(out, map, depth); + default: + out.write(jsonEncode('$value')); + } +} + +bool _isPlain(Object? value) => value is! List && value is! Map; + +void _writeList(StringBuffer out, List list, int depth) { + if (list.isEmpty) { + out.write('[]'); + return; + } + if (list.every(_isPlain)) { + final line = StringBuffer('['); + for (var i = 0; i < list.length; i++) { + if (i > 0) line.write(', '); + _write(line, list[i], depth + 1); + } + line.write(']'); + if (line.length <= _inlineWidth) { + out.write(line); + return; + } + } + final inner = ' ' * (depth + 1); + out.writeln('['); + for (var i = 0; i < list.length; i++) { + out.write(inner); + _write(out, list[i], depth + 1); + out.writeln(i == list.length - 1 ? '' : ','); + } + out.write('${' ' * depth}]'); +} + +void _writeMap(StringBuffer out, Map map, int depth) { + if (map.isEmpty) { + out.write('{}'); + return; + } + final inner = ' ' * (depth + 1); + out.writeln('{'); + var left = map.length; + for (final entry in map.entries) { + out.write('$inner${jsonEncode(entry.key)}: '); + _write(out, entry.value, depth + 1); + out.writeln(--left == 0 ? '' : ','); + } + out.write('${' ' * depth}}'); +} diff --git a/packages/flutwire/lib/src/decode/messagepack.dart b/packages/flutwire/lib/src/decode/messagepack.dart new file mode 100644 index 0000000..5b9231d --- /dev/null +++ b/packages/flutwire/lib/src/decode/messagepack.dart @@ -0,0 +1,275 @@ +import 'dart:convert'; +import 'dart:typed_data'; + +import '../config.dart'; +import '../model/body.dart'; +import '../model/call.dart'; +import 'decoded_text.dart'; + +/// Shows a MessagePack body as indented text that reads like JSON. +/// +/// MessagePack describes itself, so it needs no schema. The decoder answers for +/// bodies whose media type says MessagePack (`application/msgpack`, +/// `application/x-msgpack`, `application/vnd.msgpack` or a `+msgpack` suffix) +/// and passes on anything it cannot read to the end, or that was cut. Inspector +/// builds register it for you. +/// +/// What JSON cannot say is written as text: bytes are hex, a map with keys that +/// are not strings has them written as text, an extension is its type number +/// and its bytes, and a timestamp is an ISO 8601 date. +final class FlutwireMessagePackDecoder implements FlutwireBodyDecoder { + /// Creates the decoder. + const FlutwireMessagePackDecoder(); + + @override + String get name => 'MessagePack'; + + @override + String? decode( + FlutwireCall call, + FlutwireBody body, { + required bool isRequest, + }) { + if (!body.loaded || body.isEmpty || body.truncated) return null; + if (!_isMessagePack(body.mimeType)) return null; + try { + final reader = _Reader(body.bytes); + final values = []; + while (!reader.atEnd) { + values.add(reader.read(0)); + } + return renderDecoded(values.length == 1 ? values.single : values); + } on FormatException { + return null; + } + } + + static bool _isMessagePack(String? mime) => + mime == 'application/msgpack' || + mime == 'application/x-msgpack' || + mime == 'application/vnd.msgpack' || + (mime != null && mime.endsWith('+msgpack')); +} + +/// How deep values may nest before the reader gives up. +const int _maxDepth = 64; + +class _Reader { + _Reader(this._bytes) : _data = ByteData.sublistView(_bytes); + + final Uint8List _bytes; + final ByteData _data; + int _at = 0; + + bool get atEnd => _at >= _bytes.length; + + Object? read(int depth) { + if (depth > _maxDepth) throw const FormatException('nested too deep'); + final tag = _byte(); + if (tag <= 0x7f) return tag; + if (tag >= 0xe0) return tag - 256; + if (tag >= 0xa0 && tag <= 0xbf) return _string(tag & 0x1f); + if (tag >= 0x90 && tag <= 0x9f) return _array(tag & 0x0f, depth); + if (tag >= 0x80 && tag <= 0x8f) return _map(tag & 0x0f, depth); + switch (tag) { + case 0xc0: + return null; + case 0xc2: + return false; + case 0xc3: + return true; + case 0xc4: + return _bytesOf(_unsigned(1)); + case 0xc5: + return _bytesOf(_unsigned(2)); + case 0xc6: + return _bytesOf(_unsigned(4)); + case 0xc7: + return _extension(_unsigned(1)); + case 0xc8: + return _extension(_unsigned(2)); + case 0xc9: + return _extension(_unsigned(4)); + case 0xca: + return _float(4); + case 0xcb: + return _float(8); + case 0xcc: + return _unsigned(1); + case 0xcd: + return _unsigned(2); + case 0xce: + return _unsigned(4); + case 0xcf: + return _wide(signed: false); + case 0xd0: + return _signed(1); + case 0xd1: + return _signed(2); + case 0xd2: + return _signed(4); + case 0xd3: + return _wide(signed: true); + case 0xd4: + return _extension(1); + case 0xd5: + return _extension(2); + case 0xd6: + return _extension(4); + case 0xd7: + return _extension(8); + case 0xd8: + return _extension(16); + case 0xd9: + return _string(_unsigned(1)); + case 0xda: + return _string(_unsigned(2)); + case 0xdb: + return _string(_unsigned(4)); + case 0xdc: + return _array(_unsigned(2), depth); + case 0xdd: + return _array(_unsigned(4), depth); + case 0xde: + return _map(_unsigned(2), depth); + case 0xdf: + return _map(_unsigned(4), depth); + } + throw FormatException( + '0x${tag.toRadixString(16)} is not a MessagePack tag', + ); + } + + int _byte() { + if (_at >= _bytes.length) throw const FormatException('cut short'); + return _bytes[_at++]; + } + + void _need(int count) { + if (count < 0 || _at + count > _bytes.length) { + throw const FormatException('cut short'); + } + } + + int _unsigned(int size) { + _need(size); + final value = switch (size) { + 1 => _data.getUint8(_at), + 2 => _data.getUint16(_at), + _ => _data.getUint32(_at), + }; + _at += size; + return value; + } + + int _signed(int size) { + _need(size); + final value = switch (size) { + 1 => _data.getInt8(_at), + 2 => _data.getInt16(_at), + _ => _data.getInt32(_at), + }; + _at += size; + return value; + } + + /// A 64-bit integer: an `int` while it is safe to hold in one, a `BigInt` + /// from there on, so the number is never rounded. + Object _wide({required bool signed}) { + _need(8); + final high = signed ? _data.getInt32(_at) : _data.getUint32(_at); + final low = _data.getUint32(_at + 4); + _at += 8; + if (high >= -0x200000 && high < 0x200000) return high * 0x100000000 + low; + return (BigInt.from(high) << 32) + BigInt.from(low); + } + + double _float(int size) { + _need(size); + final value = size == 4 ? _data.getFloat32(_at) : _data.getFloat64(_at); + _at += size; + return value; + } + + String _string(int length) { + _need(length); + final text = utf8.decode( + Uint8List.sublistView(_bytes, _at, _at + length), + allowMalformed: true, + ); + _at += length; + return text; + } + + DecodedBytes _bytesOf(int length) { + _need(length); + final bytes = Uint8List.fromList(_bytes.sublist(_at, _at + length)); + _at += length; + return DecodedBytes(bytes); + } + + List _array(int count, int depth) { + // Every element takes a byte at least, so a count beyond what is left is a + // lie, and believing it would be a long loop. + if (count > _bytes.length - _at) throw const FormatException('cut short'); + return [for (var i = 0; i < count; i++) read(depth + 1)]; + } + + Map _map(int count, int depth) { + if (count * 2 > _bytes.length - _at) { + throw const FormatException('cut short'); + } + final map = {}; + for (var i = 0; i < count; i++) { + final key = read(depth + 1); + final value = read(depth + 1); + map[key is String ? key : renderDecoded(key)] = value; + } + return map; + } + + Object? _extension(int length) { + final type = _signed(1); + _need(length); + final data = Uint8List.fromList(_bytes.sublist(_at, _at + length)); + _at += length; + if (type == -1) { + final time = _timestamp(data); + if (time != null) return time; + } + return {'ext': type, 'data': DecodedBytes(data)}; + } + + /// The timestamp extension of the specification, in its three sizes, or null + /// when [data] is not one or names a moment `DateTime` cannot hold. + static DateTime? _timestamp(Uint8List data) { + final view = ByteData.sublistView(data); + final int seconds; + final int nanos; + switch (data.length) { + case 4: + seconds = view.getUint32(0); + nanos = 0; + case 8: + final packed = view.getUint32(0); + nanos = packed >> 2; + seconds = ((packed & 0x3) * 0x100000000) + view.getUint32(4); + case 12: + nanos = view.getUint32(0); + final high = view.getInt32(4); + if (high < -0x200000 || high >= 0x200000) return null; + seconds = high * 0x100000000 + view.getUint32(8); + default: + return null; + } + if (nanos > 999999999) return null; + try { + return DateTime.fromMicrosecondsSinceEpoch( + seconds * 1000000 + nanos ~/ 1000, + isUtc: true, + ); + } on Object { + return null; + } + } +} diff --git a/packages/flutwire/test/decode/decoded_text_test.dart b/packages/flutwire/test/decode/decoded_text_test.dart new file mode 100644 index 0000000..3bc04b1 --- /dev/null +++ b/packages/flutwire/test/decode/decoded_text_test.dart @@ -0,0 +1,110 @@ +import 'dart:convert'; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:flutwire/src/decode/decoded_text.dart'; + +void main() { + group('the text of a decoded value', () { + test('writes the plain values the way JSON does', () { + expect(renderDecoded(null), 'null'); + expect(renderDecoded(true), 'true'); + expect(renderDecoded(42), '42'); + expect(renderDecoded(-7), '-7'); + expect(renderDecoded(1.5), '1.5'); + expect(renderDecoded('a "quoted" line\n'), r'"a \"quoted\" line\n"'); + }); + + test('writes a number too big for a double in full', () { + expect( + renderDecoded(BigInt.parse('18446744073709551615')), + '18446744073709551615', + ); + }); + + test('writes the numbers JSON has no word for as strings', () { + expect(renderDecoded(double.nan), '"NaN"'); + expect(renderDecoded(double.infinity), '"Infinity"'); + expect(renderDecoded(double.negativeInfinity), '"-Infinity"'); + }); + + test('writes a moment as an ISO 8601 string in UTC', () { + expect( + renderDecoded(DateTime.utc(2023, 11, 14, 22, 13, 20, 500)), + '"2023-11-14T22:13:20.500Z"', + ); + }); + + test('writes bytes as hex', () { + expect( + renderDecoded(DecodedBytes(Uint8List.fromList([0, 15, 255]))), + '"00 0f ff"', + ); + }); + + test('says how many bytes there were when it does not write them all', () { + final text = DecodedBytes(Uint8List(100)).toString(); + + expect(text, endsWith('… (100 bytes)')); + expect(text.split(' ').where((p) => p == '00'), hasLength(64)); + }); + + test('keeps a short list of plain values on one line', () { + expect(renderDecoded([1, 'two', null, true]), '[1, "two", null, true]'); + expect(renderDecoded([]), '[]'); + }); + + test('puts a long list of plain values on a line each', () { + final text = renderDecoded([ + for (var i = 0; i < 30; i++) 'item number $i', + ]); + + expect(text.split('\n'), hasLength(32)); + expect(text, startsWith('[\n "item number 0",\n')); + expect(text, endsWith('"item number 29"\n]')); + }); + + test('puts a list that holds a list or a map on a line each', () { + expect( + renderDecoded([ + [1], + {'a': 1}, + ]), + '[\n [1],\n {\n "a": 1\n }\n]', + ); + }); + + test('indents a map and keeps the order of its keys', () { + expect( + renderDecoded({ + 'b': 1, + 'a': {'c': [], 'd': {}}, + }), + '{\n "b": 1,\n "a": {\n "c": [],\n "d": {}\n }\n}', + ); + }); + + test('reads back as JSON when it holds only what JSON holds', () { + final value = { + 'name': 'Ada', + 'tags': ['a', 'b'], + 'nested': {'n': 1.25, 'ok': false, 'nothing': null}, + }; + + expect(jsonDecode(renderDecoded(value)), value); + }); + + test('writes anything it does not know as a string', () { + expect(renderDecoded(const Duration(seconds: 3)), '"0:00:03.000000"'); + }); + + test('stops following a value that nests deeper than is sane', () { + Object? deep = 'bottom'; + for (var i = 0; i < 200; i++) { + deep = [deep]; + } + + expect(renderDecoded(deep), contains('"…"')); + }); + }); +} diff --git a/packages/flutwire/test/decode/messagepack_test.dart b/packages/flutwire/test/decode/messagepack_test.dart new file mode 100644 index 0000000..83e301e --- /dev/null +++ b/packages/flutwire/test/decode/messagepack_test.dart @@ -0,0 +1,263 @@ +import 'dart:convert'; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:flutwire/flutwire.dart'; + +const decoder = FlutwireMessagePackDecoder(); + +FlutwireCall call() => FlutwireCall( + id: 1, + method: 'GET', + uri: Uri.parse('https://api.test/items'), + source: FlutwireSource.io, + startedAt: DateTime.utc(2026, 10, 24), + request: FlutwireRequest( + headers: FlutwireHeaders.empty, + body: FlutwireBody.empty(), + ), +); + +/// What the decoder says about [bytes], or null when it passes. +String? decode( + List bytes, { + String type = 'application/msgpack', + int? total, +}) => decoder.decode( + call(), + FlutwireBody( + bytes: Uint8List.fromList(bytes), + totalBytes: total, + contentType: type, + ), + isRequest: false, +); + +List ascii(String text) => text.codeUnits; + +/// The value [bytes] hold, read back through the text. +Object? value(List bytes) => jsonDecode(decode(bytes)!); + +void main() { + group('the media types it answers for', () { + test('are the ones that say MessagePack', () { + for (final type in [ + 'application/msgpack', + 'application/x-msgpack', + 'application/vnd.msgpack', + 'application/vnd.api+msgpack', + 'application/msgpack; charset=binary', + ]) { + expect(decode([0x01], type: type), '1', reason: type); + } + }); + + test('are not anything else', () { + for (final type in [ + 'application/json', + 'application/octet-stream', + 'text/plain', + ]) { + expect(decode([0x01], type: type), isNull, reason: type); + } + }); + + test('are not missing', () { + final text = decoder.decode( + call(), + FlutwireBody(bytes: Uint8List.fromList([1])), + isRequest: false, + ); + + expect(text, isNull); + }); + + test('are named MessagePack', () { + expect(decoder.name, 'MessagePack'); + }); + }); + + group('the example of the specification', () { + test('is {"compact": true, "schema": 0}', () { + final bytes = [ + 0x82, + 0xa7, + ...ascii('compact'), + 0xc3, + 0xa6, + ...ascii('schema'), + 0x00, + ]; + + expect(decode(bytes), '{\n "compact": true,\n "schema": 0\n}'); + }); + }); + + group('integers', () { + test('fit in a byte, as a fixint', () { + expect(value([0x00]), 0); + expect(value([0x7f]), 127); + expect(value([0xff]), -1); + expect(value([0xe0]), -32); + }); + + test('are written in the width they need', () { + expect(value([0xcc, 0xff]), 255); + expect(value([0xcd, 0x01, 0x00]), 256); + expect(value([0xce, 0x00, 0x01, 0x00, 0x00]), 65536); + expect(value([0xcf, 0, 0, 0, 1, 0, 0, 0, 0]), 4294967296); + expect(value([0xd0, 0x80]), -128); + expect(value([0xd1, 0x80, 0x00]), -32768); + expect(value([0xd2, 0x80, 0, 0, 0]), -2147483648); + expect(value([0xd3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe]), -2); + }); + + test('keep every digit of a 64 bit number', () { + expect(decode([0xcf, ...List.filled(8, 0xff)]), '18446744073709551615'); + expect(decode([0xd3, 0x80, 0, 0, 0, 0, 0, 0, 0]), '-9223372036854775808'); + expect(decode([0xcf, 0x00, 0x20, 0, 0, 0, 0, 0, 0]), '9007199254740992'); + expect( + decode([0xcf, 0x00, 0x1f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]), + '9007199254740991', + ); + }); + }); + + group('the other plain values', () { + test('are nil and the booleans', () { + expect(decode([0xc0]), 'null'); + expect(decode([0xc2]), 'false'); + expect(decode([0xc3]), 'true'); + }); + + test('are floats of both sizes', () { + expect(decode([0xca, 0x3f, 0x80, 0x00, 0x00]), '1.0'); + expect(decode([0xcb, 0x3f, 0xf0, 0, 0, 0, 0, 0, 0]), '1.0'); + expect( + decode([0xcb, 0x40, 0x09, 0x21, 0xfb, 0x54, 0x44, 0x2d, 0x18]), + '3.141592653589793', + ); + }); + + test('are strings of every length class, in UTF-8', () { + expect(decode([0xa0]), '""'); + expect(decode([0xa3, ...ascii('abc')]), '"abc"'); + expect(decode([0xd9, 0x03, ...ascii('abc')]), '"abc"'); + expect(decode([0xda, 0x00, 0x03, ...ascii('abc')]), '"abc"'); + expect(decode([0xdb, 0x00, 0x00, 0x00, 0x03, ...ascii('abc')]), '"abc"'); + expect(decode([0xa2, 0xc3, 0xa9]), '"é"'); + }); + + test('are binary, written as hex', () { + expect(decode([0xc4, 0x03, 0x01, 0x02, 0x03]), '"01 02 03"'); + expect(decode([0xc5, 0x00, 0x01, 0xff]), '"ff"'); + }); + }); + + group('containers', () { + test('are arrays', () { + expect(value([0x93, 0x01, 0x02, 0x03]), [1, 2, 3]); + expect(value([0xdc, 0x00, 0x02, 0x01, 0x02]), [1, 2]); + expect(value([0x90]), []); + }); + + test('are maps, with keys that are not strings written as text', () { + expect(value([0x81, 0x01, 0xa1, 0x61]), {'1': 'a'}); + expect(value([0xde, 0x00, 0x01, 0xc3, 0x05]), {'true': 5}); + expect(value([0x80]), {}); + }); + + test('nest', () { + final bytes = [ + 0x81, + 0xa4, + ...ascii('rows'), + 0x92, + 0x81, + 0xa1, + 0x61, + 0x01, + 0x81, + 0xa1, + 0x61, + 0x02, + ]; + + expect(value(bytes), { + 'rows': [ + {'a': 1}, + {'a': 2}, + ], + }); + }); + + test('may follow one another, as the values of a stream', () { + expect(value([0x01, 0x02, 0xa1, 0x61]), [1, 2, 'a']); + }); + }); + + group('extensions', () { + test('are a type number and the bytes', () { + expect(value([0xd4, 0x05, 0xaa]), {'ext': 5, 'data': 'aa'}); + expect(value([0xc7, 0x03, 0x7f, 1, 2, 3]), { + 'ext': 127, + 'data': '01 02 03', + }); + }); + + test('of type -1 are timestamps, in all three sizes', () { + // 1700000000 seconds, as a 32 bit number. + expect( + decode([0xd6, 0xff, 0x65, 0x53, 0xf1, 0x00]), + '"2023-11-14T22:13:20.000Z"', + ); + // 500000000 nanoseconds above it, packed with the seconds in 64 bits. + final packed = (BigInt.from(500000000) << 34) | BigInt.from(1700000000); + final eight = [ + for (var shift = 56; shift >= 0; shift -= 8) + ((packed >> shift) & BigInt.from(0xff)).toInt(), + ]; + expect(decode([0xd7, 0xff, ...eight]), '"2023-11-14T22:13:20.500Z"'); + // And in 96 bits, with the seconds signed: one second before the epoch. + expect( + decode([0xc7, 0x0c, 0xff, 0, 0, 0, 0, ...List.filled(8, 0xff)]), + '"1969-12-31T23:59:59.000Z"', + ); + }); + + test('of type -1 that are not timestamps stay extensions', () { + expect(value([0xd5, 0xff, 0x01, 0x02]), {'ext': -1, 'data': '01 02'}); + }); + }); + + group('what it leaves alone', () { + test('is a body that was cut', () { + final bytes = [0x82, 0xa7, ...ascii('compact'), 0xc3]; + + expect(decode(bytes, total: 40), isNull); + }); + + test('is a body that ends in the middle of a value', () { + expect(decode([0x82, 0xa7, 0x63]), isNull); + expect(decode([0xcd, 0x01]), isNull); + expect(decode([0xd9, 0x05, 0x61]), isNull); + }); + + test('is a tag the specification never uses', () { + expect(decode([0xc1]), isNull); + }); + + test('is a count that promises more than there is', () { + expect(decode([0xdd, 0xff, 0xff, 0xff, 0xff]), isNull); + expect(decode([0xdf, 0xff, 0xff, 0xff, 0xff, 0x01]), isNull); + }); + + test('is a document nested far deeper than is sane', () { + expect(decode(List.filled(200, 0x91)), isNull); + }); + + test('is an empty body', () { + expect(decode([]), isNull); + }); + }); +} From ad2b6cf8e8cbdf8de3c7bd09faf86c4fed95f811 Mon Sep 17 00:00:00 2001 From: TonyTonyCoder11 Date: Sat, 24 Oct 2026 04:47:08 +0200 Subject: [PATCH 2/7] Read a Protobuf body with a schema or without one With the descriptor set protoc writes, fields come out by name and type: enums by name, maps as objects, packed and unpacked repeated numbers as lists, a duration and a timestamp as one string, and fields the schema does not know by their number. Without one they come out by field number, with a length delimited field taken for text, a message or bytes the way protoc --decode_raw takes it. A gRPC or gRPC-web call finds its message types from the method in its address, lists the messages of a stream and shows the trailers. --- .../lib/src/decode/protobuf/schema.dart | 307 ++++++++++ .../lib/src/decode/protobuf/values.dart | 323 ++++++++++ .../lib/src/decode/protobuf/wire.dart | 221 +++++++ .../lib/src/decode/protobuf_decoder.dart | 145 +++++ .../test/decode/protobuf_decoder_test.dart | 554 ++++++++++++++++++ .../test/decode/protobuf_schema_test.dart | 109 ++++ .../test/decode/protobuf_wire_test.dart | 210 +++++++ .../flutwire/test/support/proto_encoder.dart | 362 ++++++++++++ 8 files changed, 2231 insertions(+) create mode 100644 packages/flutwire/lib/src/decode/protobuf/schema.dart create mode 100644 packages/flutwire/lib/src/decode/protobuf/values.dart create mode 100644 packages/flutwire/lib/src/decode/protobuf/wire.dart create mode 100644 packages/flutwire/lib/src/decode/protobuf_decoder.dart create mode 100644 packages/flutwire/test/decode/protobuf_decoder_test.dart create mode 100644 packages/flutwire/test/decode/protobuf_schema_test.dart create mode 100644 packages/flutwire/test/decode/protobuf_wire_test.dart create mode 100644 packages/flutwire/test/support/proto_encoder.dart diff --git a/packages/flutwire/lib/src/decode/protobuf/schema.dart b/packages/flutwire/lib/src/decode/protobuf/schema.dart new file mode 100644 index 0000000..19aec25 --- /dev/null +++ b/packages/flutwire/lib/src/decode/protobuf/schema.dart @@ -0,0 +1,307 @@ +import 'dart:convert'; +import 'dart:typed_data'; + +import 'wire.dart'; + +/// The kinds of value a field can hold, numbered as `descriptor.proto` does. +enum ProtoType { + /// `double`. + double(1), + + /// `float`. + float(2), + + /// `int64`. + int64(3), + + /// `uint64`. + uint64(4), + + /// `int32`. + int32(5), + + /// `fixed64`. + fixed64(6), + + /// `fixed32`. + fixed32(7), + + /// `bool`. + bool(8), + + /// `string`. + string(9), + + /// A group. + group(10), + + /// A message. + message(11), + + /// `bytes`. + bytes(12), + + /// `uint32`. + uint32(13), + + /// An enum. + enumeration(14), + + /// `sfixed32`. + sfixed32(15), + + /// `sfixed64`. + sfixed64(16), + + /// `sint32`. + sint32(17), + + /// `sint64`. + sint64(18); + + const ProtoType(this.number); + + /// The number `descriptor.proto` gives it. + final int number; + + /// The type with the given [number], or null. + static ProtoType? of(int number) { + for (final type in values) { + if (type.number == number) return type; + } + return null; + } +} + +/// What a type allows. +extension ProtoTypeRules on ProtoType { + /// Whether a repeated field of this type may be packed: written as one + /// length delimited run of values. + bool get isPackable => switch (this) { + ProtoType.string || + ProtoType.group || + ProtoType.message || + ProtoType.bytes => false, + _ => true, + }; +} + +/// A field of a message. +final class ProtoField { + /// Creates a field description. + const ProtoField({ + required this.name, + required this.number, + required this.type, + required this.isRepeated, + this.typeName, + }); + + /// The name in the `.proto` file. + final String name; + + /// The field number. + final int number; + + /// What it holds. + final ProtoType type; + + /// Whether it is `repeated`. + final bool isRepeated; + + /// For a message, a group or an enum, the full name of the type, without the + /// dot a descriptor writes in front. + final String? typeName; +} + +/// A message type. +final class ProtoMessage { + /// Creates a message description. + const ProtoMessage({ + required this.fullName, + required this.fields, + required this.isMapEntry, + }); + + /// The name with its package and the messages it is nested in. + final String fullName; + + /// The fields by number. + final Map fields; + + /// Whether this is the entry type the compiler makes for a `map` field. + final bool isMapEntry; +} + +/// A method of a service. +final class ProtoMethod { + /// Creates a method description. + const ProtoMethod({required this.inputType, required this.outputType}); + + /// The full name of the message the method takes. + final String inputType; + + /// The full name of the message the method answers with. + final String outputType; +} + +/// What a descriptor set says about the messages, enums and services of an API. +/// +/// It is read from the bytes of a `FileDescriptorSet`, which `protoc` writes +/// with `--descriptor_set_out`, `--include_imports` included so that every type +/// a message uses is there. +final class ProtoSchema { + ProtoSchema._(this._messages, this._enums, this._methods); + + /// Reads a descriptor set. Throws a [FormatException] when [bytes] are not + /// one. + factory ProtoSchema.fromDescriptorSet(List bytes) { + final messages = {}; + final enums = >{}; + final methods = {}; + final set = readWireMessage(Uint8List.fromList(bytes)); + for (final file in set.where( + (f) => f.number == 1 && f.type == WireType.lengthDelimited, + )) { + _readFile( + readWireMessage(file.value as Uint8List), + messages, + enums, + methods, + ); + } + return ProtoSchema._(messages, enums, methods); + } + + final Map _messages; + final Map> _enums; + final Map _methods; + + /// The message with the given full name, such as `demo.Person`, or null. + ProtoMessage? message(String fullName) => _messages[fullName]; + + /// The names of the values of the enum with the given full name, by number, + /// or null. + Map? enumeration(String fullName) => _enums[fullName]; + + /// The method that a gRPC [path], such as `/demo.Directory/Find`, names, or + /// null. + ProtoMethod? methodAt(String path) { + final trimmed = path.startsWith('/') ? path.substring(1) : path; + return _methods[trimmed.replaceFirst('/', '.')] ?? _methods[trimmed]; + } + + static void _readFile( + List file, + Map messages, + Map> enums, + Map methods, + ) { + final package = _string(file, 2) ?? ''; + final prefix = package.isEmpty ? '' : '$package.'; + for (final message in _nested(file, 4)) { + _readMessage(message, prefix, messages, enums); + } + for (final enumeration in _nested(file, 5)) { + _readEnum(enumeration, prefix, enums); + } + for (final service in _nested(file, 6)) { + final name = '$prefix${_string(service, 1) ?? ''}'; + for (final method in _nested(service, 2)) { + final methodName = _string(method, 1); + final input = _typeName(method, 2); + final output = _typeName(method, 3); + if (methodName == null || input == null || output == null) continue; + methods['$name.$methodName'] = ProtoMethod( + inputType: input, + outputType: output, + ); + } + } + } + + static void _readMessage( + List message, + String prefix, + Map messages, + Map> enums, + ) { + final fullName = '$prefix${_string(message, 1) ?? ''}'; + final fields = {}; + for (final field in _nested(message, 2)) { + final number = _int(field, 3); + final type = ProtoType.of(_int(field, 5) ?? 0); + final name = _string(field, 1); + if (number == null || type == null || name == null) continue; + fields[number] = ProtoField( + name: name, + number: number, + type: type, + isRepeated: _int(field, 4) == 3, + typeName: _typeName(field, 6), + ); + } + final options = _nested(message, 7).firstOrNull; + messages[fullName] = ProtoMessage( + fullName: fullName, + fields: fields, + isMapEntry: options != null && _int(options, 7) == 1, + ); + for (final nested in _nested(message, 3)) { + _readMessage(nested, '$fullName.', messages, enums); + } + for (final enumeration in _nested(message, 4)) { + _readEnum(enumeration, '$fullName.', enums); + } + } + + static void _readEnum( + List enumeration, + String prefix, + Map> enums, + ) { + final values = {}; + for (final value in _nested(enumeration, 2)) { + final name = _string(value, 1); + final number = _int(value, 2); + if (name != null && number != null) { + values.putIfAbsent(number, () => name); + } + } + enums['$prefix${_string(enumeration, 1) ?? ''}'] = values; + } + + /// The nested messages written under field [number]. + static Iterable> _nested( + List fields, + int number, + ) => fields + .where((f) => f.number == number && f.type == WireType.lengthDelimited) + .map((f) => readWireMessage(f.value as Uint8List)); + + static String? _string(List fields, int number) { + for (final f in fields) { + if (f.number == number && f.type == WireType.lengthDelimited) { + return utf8.decode(f.value as Uint8List, allowMalformed: true); + } + } + return null; + } + + /// A type name as a descriptor writes it, `.demo.Person`, without the dot. + static String? _typeName(List fields, int number) { + final name = _string(fields, number); + if (name == null) return null; + return name.startsWith('.') ? name.substring(1) : name; + } + + static int? _int(List fields, int number) { + for (final f in fields) { + if (f.number == number && f.type == WireType.varint) { + final value = f.value; + return value is int ? value : null; + } + } + return null; + } +} diff --git a/packages/flutwire/lib/src/decode/protobuf/values.dart b/packages/flutwire/lib/src/decode/protobuf/values.dart new file mode 100644 index 0000000..2f34b18 --- /dev/null +++ b/packages/flutwire/lib/src/decode/protobuf/values.dart @@ -0,0 +1,323 @@ +import 'dart:convert'; +import 'dart:typed_data'; + +import '../decoded_text.dart'; +import 'schema.dart'; +import 'wire.dart'; + +/// Turns the bytes of a protobuf message into the values a renderer writes. +/// +/// When the schema knows the message its fields come out by name and typed: +/// enums by name, maps as objects, packed runs as lists. Without a schema the +/// fields come out by number, and what the wire cannot tell apart is guessed +/// the way `protoc --decode_raw` does: a length delimited field is text when +/// it reads as text, a message when it reads as one, and bytes otherwise. +final class ProtoValues { + /// Creates a converter that looks types up in [schema], if there is one. + const ProtoValues(this.schema); + + /// What the types are, or null to go by field number. + final ProtoSchema? schema; + + /// [bytes] as a message of [type], or by field number when [type] is null. + /// Throws a [FormatException] when [bytes] are not a message. + Map message( + Uint8List bytes, + ProtoMessage? type, [ + int depth = 0, + ]) { + if (depth > maxProtoDepth) throw const FormatException('nested too deep'); + return _fields(readWireMessage(bytes, depth: depth), type, depth); + } + + Map _fields( + List fields, + ProtoMessage? type, + int depth, + ) { + final out = {}; + for (final field in fields) { + final definition = type?.fields[field.number]; + if (definition == null) { + // A field the schema does not know, or no schema at all. + final key = type == null + ? '${field.number}' + : ''; + _put(out, key, _raw(field, depth)); + continue; + } + final entry = _mapEntry(definition); + if (entry != null && field.type == WireType.lengthDelimited) { + final map = out.putIfAbsent(definition.name, () => {}); + _putEntry( + map as Map, + entry, + field.value as Uint8List, + depth, + ); + continue; + } + final value = _field(definition, field, depth); + if (definition.isRepeated) { + final list = out.putIfAbsent(definition.name, () => []); + (list as List).addAll( + value is _Packed ? value.items : [value], + ); + } else { + out[definition.name] = value; + } + } + out.updateAll((_, value) => value is _Many ? value.items : value); + return out; + } + + /// Adds [value] under [key], keeping every value when the same field shows up + /// more than once. + static void _put(Map out, String key, Object? value) { + if (!out.containsKey(key)) { + out[key] = value; + return; + } + final before = out[key]; + if (before is _Many) { + before.items.add(value); + } else { + out[key] = _Many([before, value]); + } + } + + ProtoMessage? _mapEntry(ProtoField definition) { + if (!definition.isRepeated || definition.type != ProtoType.message) { + return null; + } + final type = schema?.message(definition.typeName ?? ''); + return type != null && type.isMapEntry ? type : null; + } + + void _putEntry( + Map map, + ProtoMessage entry, + Uint8List bytes, + int depth, + ) { + final fields = readWireMessage(bytes, depth: depth + 1); + final keyDefinition = entry.fields[1]; + final valueDefinition = entry.fields[2]; + if (keyDefinition == null || valueDefinition == null) return; + Object? key = ''; + Object? value; + for (final field in fields) { + if (field.number == 1) key = _field(keyDefinition, field, depth + 1); + if (field.number == 2) value = _field(valueDefinition, field, depth + 1); + } + map[key is String ? key : '$key'] = value; + } + + Object? _field(ProtoField definition, WireField field, int depth) { + final type = definition.type; + if (definition.isRepeated && + type.isPackable && + field.type == WireType.lengthDelimited) { + return _Packed(_unpack(definition, field.value as Uint8List, depth)); + } + final expected = switch (type) { + ProtoType.int32 || + ProtoType.int64 || + ProtoType.uint32 || + ProtoType.uint64 || + ProtoType.sint32 || + ProtoType.sint64 || + ProtoType.bool || + ProtoType.enumeration => WireType.varint, + ProtoType.float || + ProtoType.fixed32 || + ProtoType.sfixed32 => WireType.fixed32, + ProtoType.double || + ProtoType.fixed64 || + ProtoType.sfixed64 => WireType.fixed64, + ProtoType.string || + ProtoType.bytes || + ProtoType.message => WireType.lengthDelimited, + ProtoType.group => WireType.startGroup, + }; + // A field written with a wire type its declaration does not allow is shown + // as the wire has it, rather than guessed at. + if (field.type != expected) return _raw(field, depth); + return switch (field.value) { + final Uint8List bytes when expected == WireType.lengthDelimited => + _lengthDelimited(definition, bytes, depth), + final Uint8List bytes => _fixed(type, bytes), + final List group => _wellKnown( + definition.typeName, + _fields(group, schema?.message(definition.typeName ?? ''), depth + 1), + ), + final Object varint => _varint(definition, varint), + }; + } + + Object? _lengthDelimited(ProtoField definition, Uint8List bytes, int depth) { + switch (definition.type) { + case ProtoType.string: + return utf8.decode(bytes, allowMalformed: true); + case ProtoType.bytes: + return DecodedBytes(Uint8List.fromList(bytes)); + default: + final name = definition.typeName; + final nested = message(bytes, schema?.message(name ?? ''), depth + 1); + return _wellKnown(name, nested); + } + } + + List _unpack(ProtoField definition, Uint8List bytes, int depth) => + switch (definition.type) { + ProtoType.float || ProtoType.fixed32 || ProtoType.sfixed32 => [ + for (final item in readPackedFixed(bytes, 4)) + _fixed(definition.type, item), + ], + ProtoType.double || ProtoType.fixed64 || ProtoType.sfixed64 => [ + for (final item in readPackedFixed(bytes, 8)) + _fixed(definition.type, item), + ], + _ => [ + for (final item in readPackedVarints(bytes)) + _varint(definition, item), + ], + }; + + Object? _varint(ProtoField definition, Object value) { + switch (definition.type) { + case ProtoType.int64: + return signed64(value); + case ProtoType.int32: + return signed32(value); + case ProtoType.uint32: + return low32(value); + case ProtoType.sint32 || ProtoType.sint64: + return zigzag(value); + case ProtoType.bool: + return value is int ? value != 0 : value != BigInt.zero; + case ProtoType.enumeration: + final number = signed32(value); + return schema?.enumeration(definition.typeName ?? '')?[number] ?? + number; + default: + return value; + } + } + + static Object? _fixed(ProtoType type, Uint8List bytes) { + final view = ByteData.sublistView(bytes); + return switch (type) { + ProtoType.float => view.getFloat32(0, Endian.little), + ProtoType.double => view.getFloat64(0, Endian.little), + ProtoType.sfixed32 => view.getInt32(0, Endian.little), + ProtoType.sfixed64 => signedFixed64(bytes), + _ => unsignedFixed(bytes), + }; + } + + /// A field read with no help from a schema. + Object? _raw(WireField field, int depth) { + switch (field.type) { + case WireType.varint: + return field.value; + case WireType.fixed32 || WireType.fixed64: + final bytes = field.value as Uint8List; + final hex = [ + for (final byte in bytes.reversed) + byte.toRadixString(16).padLeft(2, '0'), + ].join(); + return '0x$hex'; + case WireType.startGroup: + return _fields(field.value as List, null, depth + 1); + default: + return _guess(field.value as Uint8List, depth); + } + } + + /// What a length delimited field with no declared type most likely is. + Object? _guess(Uint8List bytes, int depth) { + if (bytes.isEmpty) return ''; + final text = _text(bytes); + Map? nested; + if (depth < maxProtoDepth) { + try { + nested = message(bytes, null, depth + 1); + } on FormatException { + nested = null; + } + } + // Text that has a line break in it is as likely to be the bytes of a + // message, whose tags are often control characters, as a paragraph. + if (text != null && + !(nested != null && RegExp(r'[\t\n\r]').hasMatch(text))) { + return text; + } + return nested ?? DecodedBytes(Uint8List.fromList(bytes)); + } + + /// The text [bytes] hold, if they hold nothing but text. + static String? _text(Uint8List bytes) { + final String text; + try { + text = const Utf8Decoder().convert(bytes); + } on FormatException { + return null; + } + for (final unit in text.codeUnits) { + if (unit == 0x7f || + (unit < 0x20 && unit != 9 && unit != 10 && unit != 13)) { + return null; + } + } + return text; + } + + /// The two well known types that are better as a string than as the pair of + /// numbers they are written as. + static Object? _wellKnown(String? name, Map fields) { + if (name != 'google.protobuf.Timestamp' && + name != 'google.protobuf.Duration') { + return fields; + } + final seconds = fields['seconds']; + final nanos = fields['nanos']; + if ((seconds != null && seconds is! int) || + (nanos != null && nanos is! int)) { + return fields; + } + final s = (seconds as int?) ?? 0; + final n = (nanos as int?) ?? 0; + if (name == 'google.protobuf.Duration') { + final sign = s < 0 || n < 0 ? '-' : ''; + final fraction = n + .abs() + .toString() + .padLeft(9, '0') + .replaceFirst(RegExp(r'0+$'), ''); + return '$sign${s.abs()}${fraction.isEmpty ? '' : '.$fraction'}s'; + } + try { + return DateTime.fromMicrosecondsSinceEpoch( + s * 1000000 + n ~/ 1000, + isUtc: true, + ); + } on Object { + return fields; + } + } +} + +/// The values of a packed run, to be added to the list of the field. +final class _Packed { + const _Packed(this.items); + + final List items; +} + +/// A field that showed up more than once in a message with no schema. +final class _Many { + const _Many(this.items); + + final List items; +} diff --git a/packages/flutwire/lib/src/decode/protobuf/wire.dart b/packages/flutwire/lib/src/decode/protobuf/wire.dart new file mode 100644 index 0000000..f1ad57c --- /dev/null +++ b/packages/flutwire/lib/src/decode/protobuf/wire.dart @@ -0,0 +1,221 @@ +import 'dart:typed_data'; + +/// The wire types of the protobuf encoding. +abstract final class WireType { + /// A variable length integer. + static const int varint = 0; + + /// Eight bytes, little endian. + static const int fixed64 = 1; + + /// A length, then that many bytes. + static const int lengthDelimited = 2; + + /// The start of a group, which proto2 used for nested messages. + static const int startGroup = 3; + + /// The end of a group. + static const int endGroup = 4; + + /// Four bytes, little endian. + static const int fixed32 = 5; +} + +/// How deep messages and groups may nest before the reader gives up. +const int maxProtoDepth = 64; + +/// A field as it was written, before anything says what it means. +final class WireField { + /// Creates a field. + const WireField(this.number, this.type, this.value); + + /// The field number. + final int number; + + /// The wire type, one of [WireType]. + final int type; + + /// What the wire type holds: an `int`, or a `BigInt` once the number is too + /// big for one, for a varint; the eight or four bytes, as they were written, + /// for a fixed width field; the bytes for a length delimited one; and the + /// list of fields inside for a group. + final Object value; +} + +/// Reads the fields of a message from [bytes], in the order they were written. +/// Throws a [FormatException] when the bytes are not a protobuf message. +List readWireMessage(Uint8List bytes, {int depth = 0}) { + final reader = _Reader(bytes, depth); + final fields = []; + while (!reader.atEnd) { + fields.add(reader.field()); + } + return fields; +} + +/// The varints of a packed run of values, such as a `repeated int32` of proto3. +List readPackedVarints(Uint8List bytes) { + final reader = _Reader(bytes, 0); + final values = []; + while (!reader.atEnd) { + values.add(reader._varint()); + } + return values; +} + +/// The values of a packed run of fixed width values, each [size] bytes. +List readPackedFixed(Uint8List bytes, int size) { + if (bytes.length % size != 0) { + throw const FormatException('a packed run that does not divide'); + } + return [ + for (var at = 0; at < bytes.length; at += size) + Uint8List.sublistView(bytes, at, at + size), + ]; +} + +/// Joins the two halves of a 64 bit value: an `int` while it is safe to hold in +/// one, a `BigInt` from there on, so that the number is never rounded. +Object joinUnsigned(int high, int low) => high < 0x200000 + ? high * 0x100000000 + low + : (BigInt.from(high) << 32) | BigInt.from(low); + +/// [value], which was written as an unsigned 64 bit varint, as the signed +/// number it stands for. +Object signed64(Object value) { + if (value is int) return value; + final big = value as BigInt; + final result = big >= BigInt.two.pow(63) ? big - BigInt.two.pow(64) : big; + return _smallest(result); +} + +/// [value] with the zigzag encoding of `sint32` and `sint64` undone. +Object zigzag(Object value) { + if (value is int) { + return value.isEven ? value ~/ 2 : -((value + 1) ~/ 2); + } + final big = value as BigInt; + return _smallest( + big.isEven ? big ~/ BigInt.two : -((big + BigInt.one) ~/ BigInt.two), + ); +} + +/// The low 32 bits of [value], as an unsigned number. +int low32(Object value) => value is int + ? value & 0xffffffff + : (value as BigInt).toUnsigned(32).toInt(); + +/// The low 32 bits of [value], as a signed number. +int signed32(Object value) { + final low = low32(value); + return low >= 0x80000000 ? low - 0x100000000 : low; +} + +/// A fixed width field, read as unsigned. +Object unsignedFixed(Uint8List bytes) { + final view = ByteData.sublistView(bytes); + if (bytes.length == 4) return view.getUint32(0, Endian.little); + return joinUnsigned( + view.getUint32(4, Endian.little), + view.getUint32(0, Endian.little), + ); +} + +/// An eight byte field, read as signed. +Object signedFixed64(Uint8List bytes) { + final view = ByteData.sublistView(bytes); + final high = view.getInt32(4, Endian.little); + final low = view.getUint32(0, Endian.little); + if (high >= -0x200000 && high < 0x200000) return high * 0x100000000 + low; + return (BigInt.from(high) << 32) + BigInt.from(low); +} + +Object _smallest(BigInt value) => + value.abs() < BigInt.two.pow(53) ? value.toInt() : value; + +class _Reader { + _Reader(this._bytes, this._depth); + + final Uint8List _bytes; + final int _depth; + int _at = 0; + + bool get atEnd => _at >= _bytes.length; + + WireField field() { + final tag = _varint(); + if (tag is! int || tag > 0xffffffff) { + throw const FormatException('a tag too big to be one'); + } + final number = tag >> 3; + final type = tag & 0x7; + if (number == 0) throw const FormatException('field number 0'); + switch (type) { + case WireType.varint: + return WireField(number, type, _varint()); + case WireType.fixed64: + return WireField(number, type, _take(8)); + case WireType.lengthDelimited: + final length = _varint(); + if (length is! int) throw const FormatException('a length too big'); + return WireField(number, type, _take(length)); + case WireType.fixed32: + return WireField(number, type, _take(4)); + case WireType.startGroup: + return WireField(number, type, _group(number)); + } + throw FormatException('wire type $type'); + } + + List _group(int number) { + if (_depth + 1 > maxProtoDepth) { + throw const FormatException('nested too deep'); + } + final inner = _Reader(_bytes, _depth + 1).._at = _at; + final fields = []; + while (true) { + if (inner.atEnd) throw const FormatException('a group never ended'); + final mark = inner._at; + final tag = inner._varint(); + if (tag is int && (tag & 0x7) == WireType.endGroup) { + if (tag >> 3 != number) throw const FormatException('wrong group end'); + _at = inner._at; + return fields; + } + inner._at = mark; + fields.add(inner.field()); + } + } + + Uint8List _take(int length) { + if (length < 0 || _at + length > _bytes.length) { + throw const FormatException('cut short'); + } + final slice = Uint8List.sublistView(_bytes, _at, _at + length); + _at += length; + return slice; + } + + Object _varint() { + var low = 0; + var high = 0; + for (var i = 0; i < 10; i++) { + if (_at >= _bytes.length) throw const FormatException('cut short'); + final byte = _bytes[_at++]; + final bits = byte & 0x7f; + final shift = i * 7; + if (shift < 28) { + low |= bits << shift; + } else if (shift == 28) { + low |= (bits & 0x0f) << 28; + high |= bits >> 4; + } else { + high |= bits << (shift - 32); + } + if ((byte & 0x80) == 0) { + return joinUnsigned(high & 0xffffffff, low & 0xffffffff); + } + } + throw const FormatException('a varint of more than ten bytes'); + } +} diff --git a/packages/flutwire/lib/src/decode/protobuf_decoder.dart b/packages/flutwire/lib/src/decode/protobuf_decoder.dart new file mode 100644 index 0000000..5863982 --- /dev/null +++ b/packages/flutwire/lib/src/decode/protobuf_decoder.dart @@ -0,0 +1,145 @@ +import 'dart:convert'; +import 'dart:typed_data'; + +import '../config.dart'; +import '../model/body.dart'; +import '../model/call.dart'; +import 'decoded_text.dart'; +import 'protobuf/schema.dart'; +import 'protobuf/values.dart'; + +/// Shows a Protobuf body as indented text that reads like JSON. +/// +/// With a descriptor set the fields come out by name, typed, with enums by name, +/// maps as objects and packed runs as lists. Without one they come out by field +/// number, guessed the way `protoc --decode_raw` guesses. Inspector builds +/// register the schema-less one for you, after the decoders you list. +/// +/// The decoder answers for bodies whose media type says Protobuf +/// (`application/x-protobuf`, `application/protobuf` and +/// `application/vnd.google.protobuf`) and for gRPC and gRPC-web +/// (`application/grpc`, `application/grpc-web`, with or without `+proto`), +/// whose messages come in frames. It passes on anything it cannot read to the +/// end, or that was cut. +/// +/// The decoder runs when a body is shown, so the capture is not touched. +final class FlutwireProtobufDecoder implements FlutwireBodyDecoder { + /// Creates a decoder that reads messages with the types in [descriptorSet], + /// the bytes of a `FileDescriptorSet` that `protoc --descriptor_set_out` with + /// `--include_imports` writes. Throws a [FormatException] when they are not + /// one. + /// + /// A gRPC call finds its message types by itself, from the method in its + /// address. For any other Protobuf body [messageType] says which message it + /// is, by full name such as `demo.Person`, and a body it has no answer for is + /// read by field number. + FlutwireProtobufDecoder({required List descriptorSet, this.messageType}) + : _schema = ProtoSchema.fromDescriptorSet(descriptorSet); + + /// Creates a decoder with no schema, which shows fields by number. + const FlutwireProtobufDecoder.raw() : _schema = null, messageType = null; + + final ProtoSchema? _schema; + + /// Names the message a Protobuf body that is not gRPC holds, by full name, or + /// null when it does not know. + final String? Function(FlutwireCall call, {required bool isRequest})? + messageType; + + @override + String get name => _schema == null ? 'Protobuf (no schema)' : 'Protobuf'; + + @override + String? decode( + FlutwireCall call, + FlutwireBody body, { + required bool isRequest, + }) { + if (!body.loaded || body.isEmpty || body.truncated) return null; + final mime = body.mimeType; + if (mime == null) return null; + final grpc = _isGrpc(mime); + if (!grpc && !_isProtobuf(mime)) return null; + final values = ProtoValues(_schema); + final type = _typeFor(call, isRequest, grpc); + try { + return renderDecoded( + grpc + ? _frames(body.bytes, values, type) + : values.message(body.bytes, type), + ); + } on FormatException { + return null; + } + } + + ProtoMessage? _typeFor(FlutwireCall call, bool isRequest, bool grpc) { + final schema = _schema; + if (schema == null) return null; + if (grpc) { + final method = schema.methodAt(call.uri.path); + if (method == null) return null; + return schema.message(isRequest ? method.inputType : method.outputType); + } + final name = messageType?.call(call, isRequest: isRequest); + return name == null ? null : schema.message(name); + } + + /// The messages of a gRPC body, each behind a flag byte and a length, and the + /// trailers a gRPC-web response ends with. + static Object? _frames( + Uint8List bytes, + ProtoValues values, + ProtoMessage? type, + ) { + final frames = []; + var at = 0; + while (at < bytes.length) { + if (bytes.length - at < 5) throw const FormatException('cut short'); + final flag = bytes[at]; + final length = ByteData.sublistView(bytes, at + 1, at + 5).getUint32(0); + at += 5; + if (length > bytes.length - at) throw const FormatException('cut short'); + final payload = Uint8List.sublistView(bytes, at, at + length); + at += length; + if ((flag & 0x80) != 0) { + frames.add({'trailers': _trailers(payload)}); + } else if ((flag & 0x01) != 0) { + // The algorithm is named in a header, and the decoder has none to run. + frames.add({ + 'compressed': true, + 'bytes': DecodedBytes(Uint8List.fromList(payload)), + }); + } else { + frames.add(values.message(payload, type)); + } + } + return frames.length == 1 ? frames.single : frames; + } + + static Map _trailers(Uint8List payload) { + final trailers = {}; + for (final line in const LineSplitter().convert( + utf8.decode(payload, allowMalformed: true), + )) { + final colon = line.indexOf(':'); + if (colon <= 0) continue; + trailers[line.substring(0, colon).trim()] = line + .substring(colon + 1) + .trim(); + } + return trailers; + } + + static bool _isGrpc(String mime) => + mime == 'application/grpc' || + mime == 'application/grpc+proto' || + mime == 'application/grpc-web' || + mime == 'application/grpc-web+proto'; + + static bool _isProtobuf(String mime) => + mime == 'application/x-protobuf' || + mime == 'application/protobuf' || + mime == 'application/vnd.google.protobuf' || + mime == 'application/x-google-protobuf'; +} diff --git a/packages/flutwire/test/decode/protobuf_decoder_test.dart b/packages/flutwire/test/decode/protobuf_decoder_test.dart new file mode 100644 index 0000000..b42edcd --- /dev/null +++ b/packages/flutwire/test/decode/protobuf_decoder_test.dart @@ -0,0 +1,554 @@ +import 'dart:convert'; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:flutwire/flutwire.dart'; + +import '../support/proto_encoder.dart'; + +FlutwireCall call({String path = '/items'}) => FlutwireCall( + id: 1, + method: 'POST', + uri: Uri.parse('https://api.test$path'), + source: FlutwireSource.io, + startedAt: DateTime.utc(2026, 10, 24), + request: FlutwireRequest( + headers: FlutwireHeaders.empty, + body: FlutwireBody.empty(), + ), +); + +String? decode( + FlutwireBodyDecoder decoder, + List bytes, { + String? type = 'application/x-protobuf', + String path = '/items', + bool isRequest = false, + int? total, +}) => decoder.decode( + call(path: path), + FlutwireBody( + bytes: Uint8List.fromList(bytes), + totalBytes: total, + contentType: type, + ), + isRequest: isRequest, +); + +Map asMap(String? text) => + jsonDecode(text!) as Map; + +List grpcFrame(List payload, {int flag = 0}) => [ + flag, + (payload.length >> 24) & 0xff, + (payload.length >> 16) & 0xff, + (payload.length >> 8) & 0xff, + payload.length & 0xff, + ...payload, +]; + +void main() { + final descriptors = sampleDescriptorSet(); + final typed = FlutwireProtobufDecoder( + descriptorSet: descriptors, + messageType: (call, {required isRequest}) => + call.uri.path == '/people' ? 'demo.Person' : null, + ); + const raw = FlutwireProtobufDecoder.raw(); + + group('with a schema, a message', () { + test('comes out by name, typed, in the order it was written', () { + final text = decode(typed, samplePerson(), path: '/people'); + final person = asMap(text); + + expect(person.keys, [ + 'name', + 'id', + 'emails', + 'address', + 'scores', + 'kind', + 'lucky', + 'delta', + 'ratio', + 'avatar', + 'active', + 'code', + 'previous', + 'joined', + 'big', + 'negative', + 'phone', + ]); + expect(person['name'], 'Ada Lovelace'); + expect(person['id'], 42); + expect(person['emails'], ['ada@example.com', 'ada@analytical.engine']); + expect(person['address'], { + 'city': 'London', + 'street': "St James's Square", + }); + expect(person['scores'], {'math': 100, 'poetry': 97}); + expect(person['kind'], 'KIND_ADMIN'); + expect(person['lucky'], [3, 5, 8]); + expect(person['delta'], -17); + expect(person['ratio'], 0.5); + expect(person['avatar'], '01 02 ff'); + expect(person['active'], true); + expect(person['code'], 3735928559); + expect(person['previous'], [ + {'city': 'Bath'}, + {'city': 'Leeds'}, + ]); + expect(person['joined'], '2023-11-14T22:13:20.500Z'); + expect(person['negative'], -9007199254740993); + expect(person['phone'], '+44 20 7946 0000'); + }); + + test('keeps every digit of a 64 bit number', () { + final text = decode(typed, samplePerson(), path: '/people')!; + + expect(text, contains('"big": 18446744073709551615')); + expect(text, contains('"negative": -9007199254740993')); + }); + + test('reads a repeated number that was not packed as well', () { + final bytes = + (Wire() + ..varint(7, 3) + ..varint(7, 5) + ..packed(7, [8, 13])) + .build(); + + expect(asMap(decode(typed, bytes, path: '/people'))['lucky'], [ + 3, + 5, + 8, + 13, + ]); + }); + + test('takes the last value of a field that is not repeated', () { + final bytes = + (Wire() + ..varint(2, 1) + ..varint(2, 2)) + .build(); + + expect(asMap(decode(typed, bytes, path: '/people')), {'id': 2}); + }); + + test('shows an enum value the schema does not name as its number', () { + final bytes = (Wire()..varint(6, 9)).build(); + + expect(asMap(decode(typed, bytes, path: '/people')), {'kind': 9}); + }); + + test('reads a negative int32, which is written in ten bytes', () { + final bytes = (Wire()..varint(2, -5)).build(); + + expect(asMap(decode(typed, bytes, path: '/people')), {'id': -5}); + }); + + test('writes a map entry whose key was left out as the empty key', () { + final bytes = (Wire()..message(5, Wire()..varint(2, 5))).build(); + + expect(asMap(decode(typed, bytes, path: '/people')), { + 'scores': {'': 5}, + }); + }); + + test('shows a field the schema does not know by its number', () { + final lite = FlutwireProtobufDecoder( + descriptorSet: descriptors, + messageType: (call, {required isRequest}) => 'demo.PersonLite', + ); + + final person = asMap(decode(lite, samplePerson())); + + expect(person['name'], 'Ada Lovelace'); + expect(person['id'], 42); + expect(person[''], [ + 'ada@example.com', + 'ada@analytical.engine', + ]); + expect(person[''], {'1': 'London', '2': "St James's Square"}); + expect(person[''], 1); + }); + + test( + 'shows a field written in a wire type its declaration does not have', + () { + final bytes = + (Wire() + ..varint(1, 7) + ..string(2, 'text')) + .build(); + + expect(asMap(decode(typed, bytes, path: '/people')), { + 'name': 7, + 'id': 'text', + }); + }, + ); + + test('writes a duration and a timestamp as one string each', () { + final timing = FlutwireProtobufDecoder( + descriptorSet: descriptors, + messageType: (call, {required isRequest}) => 'demo.Timing', + ); + + String took(Wire duration) => + asMap(decode(timing, (Wire()..message(1, duration)).build()))['took']! + as String; + + expect( + took( + Wire() + ..varint(1, 90) + ..varint(2, 500000000), + ), + '90.5s', + ); + expect(took(Wire()..varint(1, 3)), '3s'); + expect( + took( + Wire() + ..varint(1, -1) + ..varint(2, -500000000), + ), + '-1.5s', + ); + expect(took(Wire()..varint(2, 1000)), '0.000001s'); + expect(asMap(decode(timing, (Wire()..message(2, Wire())).build())), { + 'at': '1970-01-01T00:00:00.000Z', + }); + }); + + test('names the last value of an enum field that was written twice', () { + final bytes = + (Wire() + ..varint(6, 1) + ..varint(6, 2)) + .build(); + + expect(asMap(decode(typed, bytes, path: '/people')), { + 'kind': 'KIND_GUEST', + }); + }); + }); + + group('with no schema, a message', () { + test('comes out by field number', () { + final bytes = + (Wire() + ..string(1, 'Ada') + ..varint(2, 42) + ..message(4, Wire()..string(1, 'London'))) + .build(); + + expect(asMap(decode(raw, bytes)), { + '1': 'Ada', + '2': 42, + '4': {'1': 'London'}, + }); + }); + + test('lists a field that shows up more than once', () { + final bytes = + (Wire() + ..varint(5, 1) + ..varint(5, 2) + ..varint(5, 3)) + .build(); + + expect(asMap(decode(raw, bytes)), { + '5': [1, 2, 3], + }); + }); + + test('shows a fixed width field as hex', () { + final bytes = + (Wire() + ..fixed32(6, 1078530011) + ..fixed64(7, BigInt.one)) + .build(); + + expect(asMap(decode(raw, bytes)), { + '6': '0x40490fdb', + '7': '0x0000000000000001', + }); + }); + + test('shows bytes that are neither text nor a message as hex', () { + final bytes = (Wire()..bytes(8, [0, 1, 255])).build(); + + expect(asMap(decode(raw, bytes)), {'8': '00 01 ff'}); + }); + + test('takes what reads as text for text, line breaks included', () { + final bytes = (Wire()..string(9, 'line one\nline two')).build(); + + expect(asMap(decode(raw, bytes)), {'9': 'line one\nline two'}); + }); + + test( + 'takes what reads as a message for one, though a tag is a control character', + () { + final bytes = + (Wire()..message( + 4, + Wire() + ..varint(1, 1) + ..varint(2, 2), + )) + .build(); + + expect(asMap(decode(raw, bytes)), { + '4': {'1': 1, '2': 2}, + }); + }, + ); + + test( + 'prefers a message to text that has a line break and parses as one', + () { + // 0a 0a is field 1, ten bytes long, and also a text of two line feeds. + final bytes = + (Wire()..bytes(3, [0x0a, 0x0a, ...utf8.encode('helloworld')])) + .build(); + + expect(asMap(decode(raw, bytes)), { + '3': {'1': 'helloworld'}, + }); + }, + ); + + test('prefers text to a message when it has no line break', () { + // "hi" is also field 13 holding 105. + final bytes = (Wire()..string(1, 'hi')).build(); + + expect(asMap(decode(raw, bytes)), {'1': 'hi'}); + }); + + test('shows an empty field as empty text', () { + final bytes = (Wire()..bytes(1, const [])).build(); + + expect(asMap(decode(raw, bytes)), {'1': ''}); + }); + + test('shows a group as the fields inside it', () { + final bytes = (Wire()..group(5, Wire()..varint(1, 7))).build(); + + expect(asMap(decode(raw, bytes)), { + '5': {'1': 7}, + }); + }); + }); + + group('the media types it answers for', () { + test('are the ones that say Protobuf', () { + final bytes = (Wire()..varint(1, 1)).build(); + for (final type in [ + 'application/x-protobuf', + 'application/protobuf', + 'application/vnd.google.protobuf', + 'application/x-google-protobuf', + 'application/x-protobuf; charset=binary', + ]) { + expect(decode(raw, bytes, type: type), isNotNull, reason: type); + } + }); + + test('are not anything else', () { + final bytes = (Wire()..varint(1, 1)).build(); + for (final type in [ + null, + 'application/json', + 'application/octet-stream', + 'application/grpc+json', + 'application/grpc-web-text', + ]) { + expect(decode(raw, bytes, type: type), isNull, reason: '$type'); + } + }); + + test('are named for whether there is a schema', () { + expect(raw.name, 'Protobuf (no schema)'); + expect(typed.name, 'Protobuf'); + }); + }); + + group('what it leaves alone', () { + test('is a body that was cut', () { + expect(decode(raw, samplePerson(), total: 5000), isNull); + }); + + test('is a body that is not a message', () { + expect(decode(raw, [0x08]), isNull); + expect(decode(raw, [0xff, 0xff]), isNull); + }); + + test('is an empty body', () { + expect(decode(raw, []), isNull); + }); + + test('is a descriptor set that is not one', () { + expect( + () => FlutwireProtobufDecoder(descriptorSet: [0x0a, 0xff, 0x01]), + throwsFormatException, + ); + }); + }); + + group('without a name for the message, a schema reads by number', () { + test('when the callback has none', () { + final bytes = (Wire()..string(1, 'Ada')).build(); + + expect(asMap(decode(typed, bytes, path: '/other')), {'1': 'Ada'}); + }); + + test('when there is no callback', () { + final plain = FlutwireProtobufDecoder(descriptorSet: descriptors); + final bytes = (Wire()..string(1, 'Ada')).build(); + + expect(asMap(decode(plain, bytes)), {'1': 'Ada'}); + }); + + test('when the callback names a message the schema does not have', () { + final wrong = FlutwireProtobufDecoder( + descriptorSet: descriptors, + messageType: (call, {required isRequest}) => 'demo.Missing', + ); + final bytes = (Wire()..string(1, 'Ada')).build(); + + expect(asMap(decode(wrong, bytes)), {'1': 'Ada'}); + }); + + test('and the callback is told which call and which side', () { + FlutwireCall? seenCall; + bool? seenSide; + final spy = FlutwireProtobufDecoder( + descriptorSet: descriptors, + messageType: (call, {required isRequest}) { + seenCall = call; + seenSide = isRequest; + return 'demo.FindRequest'; + }, + ); + + final text = decode( + spy, + (Wire()..string(1, 'ada')).build(), + path: '/find', + isRequest: true, + ); + + expect(seenCall!.uri.path, '/find'); + expect(seenSide, isTrue); + expect(asMap(text), {'query': 'ada'}); + }); + }); + + group('gRPC', () { + const find = '/demo.Directory/Find'; + final request = (Wire()..string(1, 'ada')).build(); + + test('finds the types of a method from its address', () { + final asked = decode( + typed, + grpcFrame(request), + type: 'application/grpc', + path: find, + isRequest: true, + ); + final answered = decode( + typed, + grpcFrame(samplePerson()), + type: 'application/grpc', + path: find, + ); + + expect(asMap(asked), {'query': 'ada'}); + expect(asMap(answered)['name'], 'Ada Lovelace'); + expect(asMap(answered)['kind'], 'KIND_ADMIN'); + }); + + test('lists the messages of a stream, and the trailers after them', () { + final trailers = utf8.encode('grpc-status: 0\r\ngrpc-message: OK\r\n'); + final body = [ + ...grpcFrame((Wire()..string(1, 'Ada')).build()), + ...grpcFrame((Wire()..string(1, 'Grace')).build()), + ...grpcFrame(trailers, flag: 0x80), + ]; + + final frames = + jsonDecode( + decode( + typed, + body, + type: 'application/grpc-web+proto', + path: find, + )!, + ) + as List; + + expect(frames, [ + {'name': 'Ada'}, + {'name': 'Grace'}, + { + 'trailers': {'grpc-status': '0', 'grpc-message': 'OK'}, + }, + ]); + }); + + test('says so about a message that was compressed', () { + final frames = asMap( + decode( + typed, + grpcFrame([1, 2, 3], flag: 1), + type: 'application/grpc', + path: find, + ), + ); + + expect(frames, {'compressed': true, 'bytes': '01 02 03'}); + }); + + test('reads by number when the method is not in the schema', () { + final text = decode( + typed, + grpcFrame(request), + type: 'application/grpc', + path: '/other.Service/Method', + isRequest: true, + ); + + expect(asMap(text), {'1': 'ada'}); + }); + + test('reads by number with no schema at all', () { + final text = decode( + raw, + grpcFrame(request), + type: 'application/grpc+proto', + path: find, + isRequest: true, + ); + + expect(asMap(text), {'1': 'ada'}); + }); + + test('leaves alone a body whose frames are cut', () { + expect( + decode(raw, grpcFrame(request).sublist(0, 6), type: 'application/grpc'), + isNull, + ); + expect(decode(raw, [0, 0, 0], type: 'application/grpc'), isNull); + expect(decode(raw, [0, 0, 0, 0, 9, 1], type: 'application/grpc'), isNull); + }); + + test('shows an empty message as an empty object', () { + expect(decode(raw, grpcFrame(const []), type: 'application/grpc'), '{}'); + }); + }); +} diff --git a/packages/flutwire/test/decode/protobuf_schema_test.dart b/packages/flutwire/test/decode/protobuf_schema_test.dart new file mode 100644 index 0000000..58cf006 --- /dev/null +++ b/packages/flutwire/test/decode/protobuf_schema_test.dart @@ -0,0 +1,109 @@ +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:flutwire/src/decode/protobuf/schema.dart'; + +import '../support/proto_encoder.dart'; + +void main() { + late ProtoSchema schema; + + setUp(() => schema = ProtoSchema.fromDescriptorSet(sampleDescriptorSet())); + + group('the messages of a descriptor set', () { + test('are found by their full name', () { + expect(schema.message('demo.Person'), isNotNull); + expect(schema.message('demo.Address'), isNotNull); + expect(schema.message('google.protobuf.Timestamp'), isNotNull); + expect(schema.message('demo.Missing'), isNull); + }); + + test('know their fields by number', () { + final person = schema.message('demo.Person')!; + + expect(person.fields[1]!.name, 'name'); + expect(person.fields[1]!.type, ProtoType.string); + expect(person.fields[3]!.isRepeated, isTrue); + expect(person.fields[4]!.type, ProtoType.message); + expect(person.fields[4]!.typeName, 'demo.Address'); + expect(person.fields[6]!.type, ProtoType.enumeration); + expect(person.fields[8]!.type, ProtoType.sint64); + expect(person.fields[99], isNull); + }); + + test('include the entry type of a map, marked as one', () { + final entry = schema.message('demo.Person.ScoresEntry')!; + + expect(entry.isMapEntry, isTrue); + expect(entry.fields[1]!.name, 'key'); + expect(entry.fields[2]!.type, ProtoType.int32); + expect(schema.message('demo.Person')!.isMapEntry, isFalse); + }); + }); + + group('the enums', () { + test('are found by full name, with their values by number', () { + expect(schema.enumeration('demo.Kind'), { + 0: 'KIND_UNKNOWN', + 1: 'KIND_ADMIN', + 2: 'KIND_GUEST', + }); + expect(schema.enumeration('demo.Nothing'), isNull); + }); + }); + + group('the services', () { + test('give a method its input and output types', () { + final method = schema.methodAt('/demo.Directory/Find')!; + + expect(method.inputType, 'demo.FindRequest'); + expect(method.outputType, 'demo.Person'); + }); + + test('are found with or without the leading slash', () { + expect(schema.methodAt('demo.Directory/Find'), isNotNull); + }); + + test('are not made up for a path they do not have', () { + expect(schema.methodAt('/demo.Directory/Lose'), isNull); + expect(schema.methodAt('/other.Service/Find'), isNull); + expect(schema.methodAt('/'), isNull); + }); + }); + + group('a descriptor set that is not one', () { + test('is rejected', () { + expect( + () => ProtoSchema.fromDescriptorSet([0x0a, 0xff, 0x01]), + throwsFormatException, + ); + }); + + test('is an empty one when it is empty', () { + final empty = ProtoSchema.fromDescriptorSet(Uint8List(0)); + + expect(empty.message('demo.Person'), isNull); + }); + + test('keeps the rest when a field has no name or number', () { + final broken = + (Wire()..message( + 1, + Wire() + ..string(1, 'x.proto') + ..string(2, 'x') + ..message( + 4, + Wire() + ..string(1, 'M') + ..message(2, Wire()..string(1, 'nameless')), + ), + )) + .build(); + + final partial = ProtoSchema.fromDescriptorSet(broken); + + expect(partial.message('x.M')!.fields, isEmpty); + }); + }); +} diff --git a/packages/flutwire/test/decode/protobuf_wire_test.dart b/packages/flutwire/test/decode/protobuf_wire_test.dart new file mode 100644 index 0000000..a705056 --- /dev/null +++ b/packages/flutwire/test/decode/protobuf_wire_test.dart @@ -0,0 +1,210 @@ +import 'dart:convert'; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:flutwire/src/decode/protobuf/wire.dart'; + +import '../support/proto_encoder.dart'; + +List read(List bytes) => + readWireMessage(Uint8List.fromList(bytes)); + +void main() { + group('the examples of the encoding guide', () { + test('field 1 holding 150 is 08 96 01', () { + final fields = read([0x08, 0x96, 0x01]); + + expect(fields.single.number, 1); + expect(fields.single.type, WireType.varint); + expect(fields.single.value, 150); + }); + + test('field 2 holding "testing" is 12 07 followed by the text', () { + final fields = read([0x12, 0x07, ...utf8.encode('testing')]); + + expect(fields.single.number, 2); + expect(fields.single.type, WireType.lengthDelimited); + expect(utf8.decode(fields.single.value as Uint8List), 'testing'); + }); + + test('a message in field 3 holding field 1 as 150 is 1a 03 08 96 01', () { + final fields = read([0x1a, 0x03, 0x08, 0x96, 0x01]); + final inner = readWireMessage(fields.single.value as Uint8List); + + expect(fields.single.number, 3); + expect(inner.single.value, 150); + }); + }); + + group('varints', () { + test('are read in every width up to ten bytes', () { + expect(read([0x08, 0x00]).single.value, 0); + expect(read([0x08, 0x7f]).single.value, 127); + expect(read([0x08, 0x80, 0x01]).single.value, 128); + expect( + read([0x08, 0xff, 0xff, 0xff, 0xff, 0x0f]).single.value, + 4294967295, + ); + expect( + read([0x08, ...List.filled(9, 0xff), 0x01]).single.value, + BigInt.parse('18446744073709551615'), + ); + }); + + test('hold a number up to 2^53 in an int and the rest in a BigInt', () { + final safe = Wire()..varint(1, 9007199254740991); + final unsafe = Wire()..varint(1, BigInt.parse('9007199254740993')); + + expect(readWireMessage(safe.build()).single.value, 9007199254740991); + expect( + readWireMessage(unsafe.build()).single.value, + BigInt.parse('9007199254740993'), + ); + }); + + test('are signed or zigzag decoded on request', () { + expect(signed64(150), 150); + expect(signed64(BigInt.parse('18446744073709551615')), -1); + expect(signed64(BigInt.parse('18446744073709551614')), -2); + expect( + signed64(BigInt.parse('9223372036854775808')), + BigInt.parse('-9223372036854775808'), + ); + expect(zigzag(0), 0); + expect(zigzag(1), -1); + expect(zigzag(2), 1); + expect(zigzag(4294967294), 2147483647); + expect(zigzag(4294967295), -2147483648); + expect( + zigzag(BigInt.parse('18446744073709551615')), + BigInt.parse('-9223372036854775808'), + ); + }); + + test('narrow to 32 bits the way int32 does, negative numbers included', () { + expect(signed32(BigInt.parse('18446744073709551615')), -1); + expect(signed32(5), 5); + expect(low32(BigInt.parse('18446744073709551615')), 4294967295); + expect(low32(4294967296 + 7), 7); + }); + + test('that run past ten bytes are not varints', () { + expect( + () => read([0x08, ...List.filled(11, 0xff)]), + throwsFormatException, + ); + }); + + test('that are cut short are not varints', () { + expect(() => read([0x08, 0x80]), throwsFormatException); + }); + }); + + group('fixed width fields', () { + test('are held as the bytes that were written', () { + final bytes = + (Wire() + ..fixed32(1, 0xdeadbeef) + ..fixed64(2, BigInt.parse('18446744073709551615'))) + .build(); + final fields = readWireMessage(bytes); + + expect(fields[0].type, WireType.fixed32); + expect(unsignedFixed(fields[0].value as Uint8List), 0xdeadbeef); + expect(fields[1].type, WireType.fixed64); + expect( + unsignedFixed(fields[1].value as Uint8List), + BigInt.parse('18446744073709551615'), + ); + expect(signedFixed64(fields[1].value as Uint8List), -1); + }); + + test('that are cut short are not fields', () { + expect(() => read([0x0d, 0x01, 0x02]), throwsFormatException); + expect(() => read([0x09, 0x01]), throwsFormatException); + }); + }); + + group('groups', () { + test('hold the fields between their start and their end', () { + final bytes = + (Wire()..group( + 5, + Wire() + ..varint(1, 7) + ..string(2, 'x'), + )) + .build(); + final fields = readWireMessage(bytes); + final inner = fields.single.value as List; + + expect(fields.single.type, WireType.startGroup); + expect(inner.map((f) => f.number), [1, 2]); + }); + + test('that never end are not groups', () { + expect(() => read([0x2b, 0x08, 0x01]), throwsFormatException); + }); + + test('that end with the wrong number are not groups', () { + expect(() => read([0x2b, 0x08, 0x01, 0x34]), throwsFormatException); + }); + + test('nested past the limit are not read', () { + final deep = Wire(); + var inner = Wire()..varint(1, 1); + for (var i = 0; i < 100; i++) { + inner = Wire()..group(1, inner); + } + deep.raw(inner.build()); + + expect(() => readWireMessage(deep.build()), throwsFormatException); + }); + }); + + group('packed runs', () { + test('of varints are read one after the other', () { + expect(readPackedVarints(Uint8List.fromList([3, 0x8e, 0x02, 5])), [ + 3, + 270, + 5, + ]); + }); + + test('of fixed width values are cut at the width', () { + final run = Uint8List.fromList([1, 0, 0, 0, 2, 0, 0, 0]); + + expect(readPackedFixed(run, 4).map(unsignedFixed), [1, 2]); + }); + + test('that do not divide into values are not runs', () { + expect(() => readPackedFixed(Uint8List(5), 4), throwsFormatException); + }); + }); + + group('what is not a message', () { + test('is a tag with field number 0', () { + expect(() => read([0x00, 0x01]), throwsFormatException); + }); + + test('is a wire type that does not exist', () { + expect(() => read([0x0e, 0x01]), throwsFormatException); + expect(() => read([0x0f, 0x01]), throwsFormatException); + }); + + test('is a lone group end', () { + expect(() => read([0x0c]), throwsFormatException); + }); + + test('is a length that runs past the end', () { + expect(() => read([0x0a, 0x05, 0x61]), throwsFormatException); + }); + + test('is a tag too big to be one', () { + expect( + () => read([0xff, 0xff, 0xff, 0xff, 0xff, 0x01]), + throwsFormatException, + ); + }); + }); +} diff --git a/packages/flutwire/test/support/proto_encoder.dart b/packages/flutwire/test/support/proto_encoder.dart new file mode 100644 index 0000000..4b8a2ec --- /dev/null +++ b/packages/flutwire/test/support/proto_encoder.dart @@ -0,0 +1,362 @@ +import 'dart:convert'; +import 'dart:typed_data'; + +/// Bytes in the protobuf wire format, written by hand for tests. It builds both +/// the messages and the descriptors that describe them. +final class Wire { + final BytesBuilder _out = BytesBuilder(); + + /// A varint field. + Wire varint(int field, Object value) { + _tag(field, 0); + _varint(value); + return this; + } + + /// A string field. + Wire string(int field, String value) => bytes(field, utf8.encode(value)); + + /// A length delimited field. + Wire bytes(int field, List value) { + _tag(field, 2); + _varint(value.length); + _out.add(value); + return this; + } + + /// A nested message. + Wire message(int field, Wire inner) => bytes(field, inner.build()); + + /// A packed run of varints. + Wire packed(int field, Iterable values) { + final inner = Wire(); + values.forEach(inner._varint); + return bytes(field, inner.build()); + } + + /// A four byte field holding [value] as an unsigned number. + Wire fixed32(int field, int value) { + _tag(field, 5); + final data = ByteData(4)..setUint32(0, value, Endian.little); + _out.add(data.buffer.asUint8List()); + return this; + } + + /// A four byte field holding a float. + Wire float32(int field, double value) { + _tag(field, 5); + final data = ByteData(4)..setFloat32(0, value, Endian.little); + _out.add(data.buffer.asUint8List()); + return this; + } + + /// An eight byte field holding a double. + Wire double64(int field, double value) { + _tag(field, 1); + final data = ByteData(8)..setFloat64(0, value, Endian.little); + _out.add(data.buffer.asUint8List()); + return this; + } + + /// An eight byte field holding [value] as an unsigned number. + Wire fixed64(int field, BigInt value) { + _tag(field, 1); + final data = ByteData(8) + ..setUint32(0, (value & BigInt.from(0xffffffff)).toInt(), Endian.little) + ..setUint32(4, (value >> 32).toInt(), Endian.little); + _out.add(data.buffer.asUint8List()); + return this; + } + + /// A group, as proto2 writes a nested message. + Wire group(int field, Wire inner) { + _tag(field, 3); + _out.add(inner.build()); + _tag(field, 4); + return this; + } + + /// Anything, written as it is. + Wire raw(List value) { + _out.add(value); + return this; + } + + /// The bytes so far. + Uint8List build() => _out.toBytes(); + + void _tag(int field, int type) => _varint((field << 3) | type); + + void _varint(Object value) { + var number = value is BigInt ? value : BigInt.from(value as int); + if (number.isNegative) number += BigInt.two.pow(64); + while (number >= BigInt.from(128)) { + _out.addByte(((number & BigInt.from(0x7f)) | BigInt.from(0x80)).toInt()); + number >>= 7; + } + _out.addByte(number.toInt()); + } +} + +/// The types of a field, numbered as `descriptor.proto` does. +abstract final class T { + static const int double_ = 1; + static const int float_ = 2; + static const int int64 = 3; + static const int uint64 = 4; + static const int int32 = 5; + static const int fixed64 = 6; + static const int fixed32 = 7; + static const int bool_ = 8; + static const int string = 9; + static const int group = 10; + static const int message = 11; + static const int bytes = 12; + static const int uint32 = 13; + static const int enum_ = 14; + static const int sfixed32 = 15; + static const int sfixed64 = 16; + static const int sint32 = 17; + static const int sint64 = 18; +} + +/// A `FieldDescriptorProto`. +Wire fieldOf( + String name, + int number, + int type, { + bool repeated = false, + String? typeName, +}) { + final field = Wire() + ..string(1, name) + ..varint(3, number) + ..varint(4, repeated ? 3 : 1) + ..varint(5, type); + if (typeName != null) field.string(6, typeName); + return field; +} + +/// The descriptor set of the API most tests decode, which is this file: +/// +/// ```proto +/// syntax = "proto3"; +/// package demo; +/// import "google/protobuf/timestamp.proto"; +/// +/// enum Kind { KIND_UNKNOWN = 0; KIND_ADMIN = 1; KIND_GUEST = 2; } +/// message Address { string city = 1; string street = 2; } +/// message Person { +/// string name = 1; int32 id = 2; +/// repeated string emails = 3; Address address = 4; +/// map scores = 5; Kind kind = 6; +/// repeated int32 lucky = 7; sint64 delta = 8; +/// double ratio = 9; bytes avatar = 10; +/// bool active = 11; fixed32 code = 12; +/// repeated Address previous = 13; google.protobuf.Timestamp joined = 14; +/// uint64 big = 15; int64 negative = 16; +/// oneof contact { string phone = 17; string fax = 18; } +/// } +/// message PersonLite { string name = 1; int32 id = 2; } +/// message FindRequest { string query = 1; } +/// message Timing { +/// google.protobuf.Duration took = 1; google.protobuf.Timestamp at = 2; +/// } +/// service Directory { rpc Find(FindRequest) returns (Person); } +/// ``` +Uint8List sampleDescriptorSet() { + final timestamp = Wire() + ..string(1, 'google/protobuf/timestamp.proto') + ..string(2, 'google.protobuf') + ..message( + 4, + Wire() + ..string(1, 'Timestamp') + ..message(2, fieldOf('seconds', 1, T.int64)) + ..message(2, fieldOf('nanos', 2, T.int32)), + ); + final duration = Wire() + ..string(1, 'google/protobuf/duration.proto') + ..string(2, 'google.protobuf') + ..message( + 4, + Wire() + ..string(1, 'Duration') + ..message(2, fieldOf('seconds', 1, T.int64)) + ..message(2, fieldOf('nanos', 2, T.int32)), + ); + final person = Wire() + ..string(1, 'Person') + ..message(2, fieldOf('name', 1, T.string)) + ..message(2, fieldOf('id', 2, T.int32)) + ..message(2, fieldOf('emails', 3, T.string, repeated: true)) + ..message(2, fieldOf('address', 4, T.message, typeName: '.demo.Address')) + ..message( + 2, + fieldOf( + 'scores', + 5, + T.message, + repeated: true, + typeName: '.demo.Person.ScoresEntry', + ), + ) + ..message(2, fieldOf('kind', 6, T.enum_, typeName: '.demo.Kind')) + ..message(2, fieldOf('lucky', 7, T.int32, repeated: true)) + ..message(2, fieldOf('delta', 8, T.sint64)) + ..message(2, fieldOf('ratio', 9, T.double_)) + ..message(2, fieldOf('avatar', 10, T.bytes)) + ..message(2, fieldOf('active', 11, T.bool_)) + ..message(2, fieldOf('code', 12, T.fixed32)) + ..message( + 2, + fieldOf( + 'previous', + 13, + T.message, + repeated: true, + typeName: '.demo.Address', + ), + ) + ..message( + 2, + fieldOf('joined', 14, T.message, typeName: '.google.protobuf.Timestamp'), + ) + ..message(2, fieldOf('big', 15, T.uint64)) + ..message(2, fieldOf('negative', 16, T.int64)) + ..message(2, fieldOf('phone', 17, T.string)) + ..message(2, fieldOf('fax', 18, T.string)) + ..message( + 3, + Wire() + ..string(1, 'ScoresEntry') + ..message(2, fieldOf('key', 1, T.string)) + ..message(2, fieldOf('value', 2, T.int32)) + ..message(7, Wire()..varint(7, 1)), + ); + final demo = Wire() + ..string(1, 'demo.proto') + ..string(2, 'demo') + ..string(3, 'google/protobuf/timestamp.proto') + ..string(3, 'google/protobuf/duration.proto') + ..message( + 4, + Wire() + ..string(1, 'Timing') + ..message( + 2, + fieldOf('took', 1, T.message, typeName: '.google.protobuf.Duration'), + ) + ..message( + 2, + fieldOf('at', 2, T.message, typeName: '.google.protobuf.Timestamp'), + ), + ) + ..message( + 4, + Wire() + ..string(1, 'Address') + ..message(2, fieldOf('city', 1, T.string)) + ..message(2, fieldOf('street', 2, T.string)), + ) + ..message(4, person) + ..message( + 4, + Wire() + ..string(1, 'PersonLite') + ..message(2, fieldOf('name', 1, T.string)) + ..message(2, fieldOf('id', 2, T.int32)), + ) + ..message( + 4, + Wire() + ..string(1, 'FindRequest') + ..message(2, fieldOf('query', 1, T.string)), + ) + ..message( + 5, + Wire() + ..string(1, 'Kind') + ..message( + 2, + Wire() + ..string(1, 'KIND_UNKNOWN') + ..varint(2, 0), + ) + ..message( + 2, + Wire() + ..string(1, 'KIND_ADMIN') + ..varint(2, 1), + ) + ..message( + 2, + Wire() + ..string(1, 'KIND_GUEST') + ..varint(2, 2), + ), + ) + ..message( + 6, + Wire() + ..string(1, 'Directory') + ..message( + 2, + Wire() + ..string(1, 'Find') + ..string(2, '.demo.FindRequest') + ..string(3, '.demo.Person'), + ), + ) + ..string(12, 'proto3'); + return (Wire() + ..message(1, timestamp) + ..message(1, duration) + ..message(1, demo)) + .build(); +} + +/// A `demo.Person` that uses every kind of field. +Uint8List samplePerson() => + (Wire() + ..string(1, 'Ada Lovelace') + ..varint(2, 42) + ..string(3, 'ada@example.com') + ..string(3, 'ada@analytical.engine') + ..message( + 4, + Wire() + ..string(1, 'London') + ..string(2, "St James's Square"), + ) + ..message( + 5, + Wire() + ..string(1, 'math') + ..varint(2, 100), + ) + ..message( + 5, + Wire() + ..string(1, 'poetry') + ..varint(2, 97), + ) + ..varint(6, 1) + ..packed(7, [3, 5, 8]) + ..varint(8, 33) + ..double64(9, 0.5) + ..bytes(10, [1, 2, 255]) + ..varint(11, 1) + ..fixed32(12, 3735928559) + ..message(13, Wire()..string(1, 'Bath')) + ..message(13, Wire()..string(1, 'Leeds')) + ..message( + 14, + Wire() + ..varint(1, 1700000000) + ..varint(2, 500000000), + ) + ..varint(15, BigInt.parse('18446744073709551615')) + ..varint(16, -9007199254740993) + ..string(17, '+44 20 7946 0000')) + .build(); From dca7a8a1168bd73b7ff4f9a316520d3994f63c17 Mon Sep 17 00:00:00 2001 From: TonyTonyCoder11 Date: Sat, 24 Oct 2026 04:47:08 +0200 Subject: [PATCH 3/7] Give every inspector the decoders that need no schema The viewer asks the decoders the app registered first, then MessagePack, then Protobuf by field number. Decoding happens when a body is opened, so the capture is untouched. --- packages/flutwire/lib/flutwire.dart | 2 + .../flutwire/lib/src/decode/built_in.dart | 11 ++ .../flutwire/lib/src/ui/detail/body_pane.dart | 3 +- .../test/ui/body/built_in_decoders_test.dart | 103 ++++++++++++++++++ 4 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 packages/flutwire/lib/src/decode/built_in.dart create mode 100644 packages/flutwire/test/ui/body/built_in_decoders_test.dart diff --git a/packages/flutwire/lib/flutwire.dart b/packages/flutwire/lib/flutwire.dart index 1b83303..0776c9a 100644 --- a/packages/flutwire/lib/flutwire.dart +++ b/packages/flutwire/lib/flutwire.dart @@ -8,6 +8,8 @@ export 'src/capture/recorder.dart' show FlutwireCallRecorder; export 'src/capture/redaction.dart'; export 'src/config.dart'; export 'src/coverage.dart' show FlutwireCoverageNote, FlutwireGap; +export 'src/decode/messagepack.dart' show FlutwireMessagePackDecoder; +export 'src/decode/protobuf_decoder.dart' show FlutwireProtobufDecoder; export 'src/engine.dart' show FlutwireStandDown; export 'src/export/export.dart' show FlutwireExportFormat, callToJson; export 'src/flutwire.dart' show Flutwire; diff --git a/packages/flutwire/lib/src/decode/built_in.dart b/packages/flutwire/lib/src/decode/built_in.dart new file mode 100644 index 0000000..b15d80f --- /dev/null +++ b/packages/flutwire/lib/src/decode/built_in.dart @@ -0,0 +1,11 @@ +import '../config.dart'; +import 'messagepack.dart'; +import 'protobuf_decoder.dart'; + +/// The decoders every inspector has, which the viewer asks after the ones the +/// app registered. Neither needs a schema: MessagePack describes itself, and a +/// Protobuf body with none is shown by field number. +const List builtInDecoders = [ + FlutwireMessagePackDecoder(), + FlutwireProtobufDecoder.raw(), +]; diff --git a/packages/flutwire/lib/src/ui/detail/body_pane.dart b/packages/flutwire/lib/src/ui/detail/body_pane.dart index 75fde71..4d33764 100644 --- a/packages/flutwire/lib/src/ui/detail/body_pane.dart +++ b/packages/flutwire/lib/src/ui/detail/body_pane.dart @@ -1,6 +1,7 @@ import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; +import '../../decode/built_in.dart'; import '../../model/body.dart'; import '../../model/call.dart'; import '../actions.dart'; @@ -45,7 +46,7 @@ class BodyPane extends StatelessWidget { call: call, body: body, isRequest: isRequest, - decoders: session.config.decoders, + decoders: [...session.config.decoders, ...builtInDecoders], actions: FlutwireBodyActions( onSaveBytes: (bytes, fileName, mimeType) async { final shared = await session.sharing.shareBytes( diff --git a/packages/flutwire/test/ui/body/built_in_decoders_test.dart b/packages/flutwire/test/ui/body/built_in_decoders_test.dart new file mode 100644 index 0000000..fe46426 --- /dev/null +++ b/packages/flutwire/test/ui/body/built_in_decoders_test.dart @@ -0,0 +1,103 @@ +import 'dart:typed_data'; + +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:flutwire/flutwire.dart'; +import 'package:flutwire/src/ui/detail/body_pane.dart'; + +import '../fake_session.dart'; +import '../harness.dart'; +import 'helpers.dart'; + +/// `{"compact": true, "schema": 0}`, in MessagePack. +final List msgpack = [ + 0x82, + 0xa7, + ...'compact'.codeUnits, + 0xc3, + 0xa6, + ...'schema'.codeUnits, + 0x00, +]; + +class _Custom implements FlutwireBodyDecoder { + @override + String get name => 'Custom'; + + @override + String? decode( + FlutwireCall call, + FlutwireBody body, { + required bool isRequest, + }) => 'the app read this one'; +} + +Future show( + WidgetTester tester, + FlutwireBody body, { + List decoders = const [], +}) async { + await pumpFlutwire( + tester, + BodyPane(call: testCall(), body: body, isRequest: false), + session: FakeSession( + config: FlutwireConfig( + enabled: true, + captureDartIo: false, + persist: false, + shortcut: false, + locale: const Locale('en'), + decoders: decoders, + notification: const FlutwireNotificationConfig(enabled: false), + ), + ), + size: const Size(420, 900), + ); + await tester.pump(const Duration(milliseconds: 300)); +} + +void main() { + group('the decoders every inspector has', () { + testWidgets('read a MessagePack body with no setup', (tester) async { + await show(tester, bytesBody(msgpack, type: 'application/msgpack')); + + expect(find.textContaining('MessagePack'), findsWidgets); + expect(find.textContaining('"compact": true'), findsOneWidget); + }); + + testWidgets('read a Protobuf body by field number with no setup', ( + tester, + ) async { + final proto = Uint8List.fromList([ + 0x0a, + 0x03, + ...'Ada'.codeUnits, + 0x10, + 0x2a, + ]); + + await show(tester, bytesBody(proto, type: 'application/x-protobuf')); + + expect(find.textContaining('Protobuf (no schema)'), findsWidgets); + expect(find.textContaining('"1": "Ada"'), findsOneWidget); + expect(find.textContaining('"2": 42'), findsOneWidget); + }); + + testWidgets('leave a body of another type as it was', (tester) async { + await show(tester, bytesBody(msgpack, type: 'application/octet-stream')); + + expect(find.textContaining('MessagePack'), findsNothing); + }); + + testWidgets('come after the ones the app registered', (tester) async { + await show( + tester, + bytesBody(msgpack, type: 'application/msgpack'), + decoders: [_Custom()], + ); + + expect(find.textContaining('the app read this one'), findsOneWidget); + expect(find.textContaining('"compact": true'), findsNothing); + }); + }); +} From 93c1b6101ba8d43dae481141d33cb6301320103c Mon Sep 17 00:00:00 2001 From: TonyTonyCoder11 Date: Sat, 24 Oct 2026 04:47:08 +0200 Subject: [PATCH 4/7] Describe the Protobuf and MessagePack decoders --- CHANGELOG.md | 1 + packages/flutwire/README.md | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77dd35d..34ab6da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ board can be cross-checked against what actually shipped. ### Added +- Protobuf and MessagePack bodies are readable. MessagePack needs nothing and is shown as indented text, and a Protobuf body with no schema is shown by field number, guessed the way `protoc --decode_raw` guesses. `FlutwireProtobufDecoder` takes the descriptor set that `protoc` writes and shows the fields by name and type, with enums by name, maps as objects and packed runs as lists. A gRPC or gRPC-web call finds its message types from the method in its address, lists the messages of a stream and shows the trailers. The decoders run when a body is opened, so the capture is untouched (M22) - WebSockets and server-sent event streams. A WebSocket is one call that stays open, and it lists its messages with the ping, pong and close frames, which way each went, how big it was and when it crossed, compressed messages included. An event stream lists its events as they complete, with their names and ids, and keeps the raw stream as the body of the response. Both get a Messages or Events tab, a badge in the list, a line in the overview, entries in the HAR and text exports, and their messages are kept on disk with the call. The latest 1000 messages are kept, with a megabyte of payload at most, and the tab says how many came before (M16) - Any call can be opened in an editor, changed and sent again. The editor starts from the method, address, header fields and body that were recorded. A value the redaction masked was never kept, so it is flagged and not sent unless it is typed again. What goes out is recorded as a new call that names the one it repeats, marked in the list and linked from its overview. It needs `dart:io`, so the web does not offer it (M15) - A line above the list that says when Flutwire may be missing calls, with the reason and what to change: a client or adapter that does not use `dart:io`, an adapter that ran in another isolate, or something that replaced the capture's `HttpOverrides`. It stays out of sight until Flutwire has seen one of them happen, and it names the client or the isolate. `Flutwire.coverage` lists the notes, and an adapter that stands down for the capture reports through `Flutwire.watchStandDown`. The example app has a probe for each case (M24) diff --git a/packages/flutwire/README.md b/packages/flutwire/README.md index f381a2d..ef572ba 100644 --- a/packages/flutwire/README.md +++ b/packages/flutwire/README.md @@ -55,6 +55,23 @@ A WebSocket opened through `dart:io` is one call that stays open. Its Messages t `dart:io` makes one client for all WebSockets the first time one is opened, so a WebSocket opened before `Flutwire.init` is not seen: call `init` first thing in `main`. On the web the browser opens WebSockets itself, and none is recorded there. +## Protobuf and MessagePack + +A MessagePack body is shown as indented text with no setup, and a Protobuf body is shown by field number. To see Protobuf with names and types, give the decoder the descriptor set that `protoc` writes with `--descriptor_set_out=api.desc --include_imports`: + +```dart +WidgetsFlutterBinding.ensureInitialized(); +final descriptors = (await rootBundle.load('assets/api.desc')).buffer.asUint8List(); + +await Flutwire.init( + config: FlutwireConfig( + decoders: [FlutwireProtobufDecoder(descriptorSet: descriptors)], + ), +); +``` + +A gRPC or gRPC-web call finds its message types from the method in its address. For any other Protobuf body, say which message it is with `messageType: (call, {required isRequest}) => 'demo.Person'`, and a body whose type is not known is shown by field number. The decoders run when a body is opened, so what the app sends and receives is not touched. + ## Sending a call again Open a call and tap the replay button in the top bar. The method, the address, the header fields and the body can all be changed, and Send makes the request. What goes out is recorded as a new call that names the one it repeats. @@ -76,7 +93,7 @@ await Flutwire.init( ); ``` -Authorization headers and the values of cookies are masked by default; the names and attributes of cookies stay readable. A masked value is replaced when the call is recorded, so it never reaches memory, disk, a notification or an export. The other options are documented on `FlutwireConfig`: how much of a body is kept, where the history is stored, what the notification lists, the language, the look, and decoders for formats the inspector does not read, such as Protobuf. +Authorization headers and the values of cookies are masked by default; the names and attributes of cookies stay readable. A masked value is replaced when the call is recorded, so it never reaches memory, disk, a notification or an export. The other options are documented on `FlutwireConfig`: how much of a body is kept, where the history is stored, what the notification lists, the language, the look, and decoders for formats the inspector does not read on its own. ## Clients that bypass `dart:io` From 558292e9711f0bc93346d25bd65f9bce418ff232 Mon Sep 17 00:00:00 2001 From: TonyTonyCoder11 Date: Sat, 24 Oct 2026 04:51:33 +0200 Subject: [PATCH 5/7] Add a MessagePack body and a Protobuf body to the example --- CHANGELOG.md | 2 +- packages/flutwire/example/lib/main.dart | 45 +++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34ab6da..f93259e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ board can be cross-checked against what actually shipped. ### Added -- Protobuf and MessagePack bodies are readable. MessagePack needs nothing and is shown as indented text, and a Protobuf body with no schema is shown by field number, guessed the way `protoc --decode_raw` guesses. `FlutwireProtobufDecoder` takes the descriptor set that `protoc` writes and shows the fields by name and type, with enums by name, maps as objects and packed runs as lists. A gRPC or gRPC-web call finds its message types from the method in its address, lists the messages of a stream and shows the trailers. The decoders run when a body is opened, so the capture is untouched (M22) +- Protobuf and MessagePack bodies are readable. MessagePack needs nothing and is shown as indented text, and a Protobuf body with no schema is shown by field number, guessed the way `protoc --decode_raw` guesses. `FlutwireProtobufDecoder` takes the descriptor set that `protoc` writes and shows the fields by name and type, with enums by name, maps as objects and packed runs as lists. A gRPC or gRPC-web call finds its message types from the method in its address, lists the messages of a stream and shows the trailers. The decoders run when a body is opened, so the capture is untouched. The example app has a body of each (M22) - WebSockets and server-sent event streams. A WebSocket is one call that stays open, and it lists its messages with the ping, pong and close frames, which way each went, how big it was and when it crossed, compressed messages included. An event stream lists its events as they complete, with their names and ids, and keeps the raw stream as the body of the response. Both get a Messages or Events tab, a badge in the list, a line in the overview, entries in the HAR and text exports, and their messages are kept on disk with the call. The latest 1000 messages are kept, with a megabyte of payload at most, and the tab says how many came before (M16) - Any call can be opened in an editor, changed and sent again. The editor starts from the method, address, header fields and body that were recorded. A value the redaction masked was never kept, so it is flagged and not sent unless it is typed again. What goes out is recorded as a new call that names the one it repeats, marked in the list and linked from its overview. It needs `dart:io`, so the web does not offer it (M15) - A line above the list that says when Flutwire may be missing calls, with the reason and what to change: a client or adapter that does not use `dart:io`, an adapter that ran in another isolate, or something that replaced the capture's `HttpOverrides`. It stays out of sight until Flutwire has seen one of them happen, and it names the client or the isolate. `Flutwire.coverage` lists the notes, and an adapter that stands down for the capture reports through `Flutwire.watchStandDown`. The example app has a probe for each case (M24) diff --git a/packages/flutwire/example/lib/main.dart b/packages/flutwire/example/lib/main.dart index ed9053f..120ae40 100644 --- a/packages/flutwire/example/lib/main.dart +++ b/packages/flutwire/example/lib/main.dart @@ -176,6 +176,8 @@ final List<(String, List)> groups = [ client.get(Uri.parse('$_bin/cookies/set?theme=dark&session=abc')), ), ), + Probe('MessagePack', () => _canned(_messagePack, 'application/msgpack')), + Probe('Protobuf', () => _canned(_protobuf, 'application/x-protobuf')), ], ), ( @@ -215,6 +217,33 @@ final List<(String, List)> groups = [ ), ]; +/// `{"club": "Orion", "members": 3, "tags": ["a", "b"], "founded": 1999.5}`. +const List _messagePack = [ + 0x84, // + 0xa4, 0x63, 0x6c, 0x75, 0x62, 0xa5, 0x4f, 0x72, 0x69, 0x6f, 0x6e, + 0xa7, 0x6d, 0x65, 0x6d, 0x62, 0x65, 0x72, 0x73, 0x03, + 0xa4, 0x74, 0x61, 0x67, 0x73, 0x92, 0xa1, 0x61, 0xa1, 0x62, + 0xa7, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x65, 0x64, + 0xcb, 0x40, 0x9f, 0x3e, 0x00, 0x00, 0x00, 0x00, 0x00, +]; + +/// A message of field 1 as "Orion", field 2 as 3 and field 3 as a message of +/// its own that holds "Vega" in field 1. +const List _protobuf = [ + 0x0a, 0x05, 0x4f, 0x72, 0x69, 0x6f, 0x6e, // + 0x10, 0x03, + 0x1a, 0x06, 0x0a, 0x04, 0x56, 0x65, 0x67, 0x61, +]; + +/// Answers with [body] and the media type [type], so that the viewer has +/// something to decode. The client records the call itself. +Future _canned(List body, String type) => _safe( + () => FlutwireClient( + _Canned(body, type), + force: true, + ).get(Uri.parse('$_todos/club')), +); + /// Opens a WebSocket to an echo server, sends a text and a binary message, and /// closes it, so the inspector has a conversation to list. `dart:io` makes one /// client for every WebSocket the first time it opens one, so this works @@ -294,6 +323,22 @@ Future _fromAnotherIsolate() async { } } +/// A client that answers every request with the same body. +class _Canned extends http.BaseClient { + _Canned(this.body, this.type); + + final List body; + final String type; + + @override + Future send(http.BaseRequest request) async => + http.StreamedResponse( + Stream.value(body), + 200, + headers: {'content-type': type}, + ); +} + /// A client with a transport of its own, as `cronet_http` has: it never /// reaches dart:io, so the capture cannot see what it sends. class _OffTheWire extends http.BaseClient { From 46846a47e802850faedad58cbbd701f2e9680ceb Mon Sep 17 00:00:00 2001 From: TonyTonyCoder11 Date: Sat, 24 Oct 2026 05:00:17 +0200 Subject: [PATCH 6/7] Compile the patterns once and copy a cut message once --- packages/flutwire/lib/src/capture/recorder.dart | 2 +- packages/flutwire/lib/src/decode/protobuf/values.dart | 8 +++++--- packages/flutwire/lib/src/model/message.dart | 4 +++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/flutwire/lib/src/capture/recorder.dart b/packages/flutwire/lib/src/capture/recorder.dart index d39cf04..3b45ec4 100644 --- a/packages/flutwire/lib/src/capture/recorder.dart +++ b/packages/flutwire/lib/src/capture/recorder.dart @@ -186,7 +186,7 @@ final class FlutwireCallRecorder { : FlutwireConversationKind.webSocket; final cap = _host.config.maxBodyBytes; var bytes = payload is Uint8List ? payload : Uint8List.fromList(payload); - if (bytes.length > cap) bytes = Uint8List.fromList(bytes.sublist(0, cap)); + if (bytes.length > cap) bytes = bytes.sublist(0, cap); if (kind == FlutwireMessageKind.text || kind == FlutwireMessageKind.event) { bytes = _host.config.redaction.applyToBody(bytes, contentType: null); } diff --git a/packages/flutwire/lib/src/decode/protobuf/values.dart b/packages/flutwire/lib/src/decode/protobuf/values.dart index 2f34b18..0c313d9 100644 --- a/packages/flutwire/lib/src/decode/protobuf/values.dart +++ b/packages/flutwire/lib/src/decode/protobuf/values.dart @@ -16,6 +16,9 @@ final class ProtoValues { /// Creates a converter that looks types up in [schema], if there is one. const ProtoValues(this.schema); + static final RegExp _lineBreak = RegExp(r'[\t\n\r]'); + static final RegExp _trailingZeros = RegExp(r'0+$'); + /// What the types are, or null to go by field number. final ProtoSchema? schema; @@ -249,8 +252,7 @@ final class ProtoValues { } // Text that has a line break in it is as likely to be the bytes of a // message, whose tags are often control characters, as a paragraph. - if (text != null && - !(nested != null && RegExp(r'[\t\n\r]').hasMatch(text))) { + if (text != null && !(nested != null && _lineBreak.hasMatch(text))) { return text; } return nested ?? DecodedBytes(Uint8List.fromList(bytes)); @@ -294,7 +296,7 @@ final class ProtoValues { .abs() .toString() .padLeft(9, '0') - .replaceFirst(RegExp(r'0+$'), ''); + .replaceFirst(_trailingZeros, ''); return '$sign${s.abs()}${fraction.isEmpty ? '' : '.$fraction'}s'; } try { diff --git a/packages/flutwire/lib/src/model/message.dart b/packages/flutwire/lib/src/model/message.dart index e7e96e4..da77b59 100644 --- a/packages/flutwire/lib/src/model/message.dart +++ b/packages/flutwire/lib/src/model/message.dart @@ -127,6 +127,8 @@ final class FlutwireMessage { ) : null; + static final RegExp _whitespace = RegExp(r'\s+'); + /// What the message holds in a line or two, for a list or a text export: the /// text of a text message or an event, the code and the reason of a close, /// and the first bytes, in hex, of anything else. @@ -134,7 +136,7 @@ final class FlutwireMessage { switch (kind) { case FlutwireMessageKind.text: case FlutwireMessageKind.event: - final flat = (text ?? '').replaceAll(RegExp(r'\s+'), ' ').trim(); + final flat = (text ?? '').replaceAll(_whitespace, ' ').trim(); return flat.length <= 240 ? flat : '${flat.substring(0, 240)}…'; case FlutwireMessageKind.close: final code = closeCode; From d64b1e3d13baa04b97d5ce6388275e58560c2f6d Mon Sep 17 00:00:00 2001 From: TonyTonyCoder11 Date: Sat, 24 Oct 2026 05:00:21 +0200 Subject: [PATCH 7/7] Name the files that need dart:io the way the rest of the package does --- .../{websocket_reader.dart => websocket_reader_io.dart} | 0 packages/flutwire/lib/src/capture/wire/wire_io.dart | 4 ++-- .../capture/wire/{wire_socket.dart => wire_socket_io.dart} | 2 +- .../lib/src/store/{disk_store.dart => disk_store_io.dart} | 0 packages/flutwire/lib/src/store/open_store_io.dart | 2 +- packages/flutwire/test/capture/websocket_reader_test.dart | 2 +- packages/flutwire/test/capture/wire_socket_test.dart | 2 +- packages/flutwire/test/store/disk_store_messages_test.dart | 2 +- packages/flutwire/test/store/disk_store_test.dart | 2 +- packages/flutwire/test/store/preferences_test.dart | 2 +- 10 files changed, 9 insertions(+), 9 deletions(-) rename packages/flutwire/lib/src/capture/messages/{websocket_reader.dart => websocket_reader_io.dart} (100%) rename packages/flutwire/lib/src/capture/wire/{wire_socket.dart => wire_socket_io.dart} (99%) rename packages/flutwire/lib/src/store/{disk_store.dart => disk_store_io.dart} (100%) diff --git a/packages/flutwire/lib/src/capture/messages/websocket_reader.dart b/packages/flutwire/lib/src/capture/messages/websocket_reader_io.dart similarity index 100% rename from packages/flutwire/lib/src/capture/messages/websocket_reader.dart rename to packages/flutwire/lib/src/capture/messages/websocket_reader_io.dart diff --git a/packages/flutwire/lib/src/capture/wire/wire_io.dart b/packages/flutwire/lib/src/capture/wire/wire_io.dart index e909a5e..3ef7fb7 100644 --- a/packages/flutwire/lib/src/capture/wire/wire_io.dart +++ b/packages/flutwire/lib/src/capture/wire/wire_io.dart @@ -5,9 +5,9 @@ import 'dart:io'; import '../../engine.dart'; import '../../model/call.dart'; import '../../model/headers.dart'; -import '../messages/websocket_reader.dart'; +import '../messages/websocket_reader_io.dart'; import '../recorder.dart'; -import 'wire_socket.dart'; +import 'wire_socket_io.dart'; /// Records every call made through `dart:io`, with no setup in the client. /// diff --git a/packages/flutwire/lib/src/capture/wire/wire_socket.dart b/packages/flutwire/lib/src/capture/wire/wire_socket_io.dart similarity index 99% rename from packages/flutwire/lib/src/capture/wire/wire_socket.dart rename to packages/flutwire/lib/src/capture/wire/wire_socket_io.dart index 37c2553..a2a6e3f 100644 --- a/packages/flutwire/lib/src/capture/wire/wire_socket.dart +++ b/packages/flutwire/lib/src/capture/wire/wire_socket_io.dart @@ -4,7 +4,7 @@ import 'dart:io'; import 'dart:typed_data'; import '../../model/message.dart'; -import '../messages/websocket_reader.dart'; +import '../messages/websocket_reader_io.dart'; import '../recorder.dart'; /// A socket that was an HTTP connection and became a WebSocket. It hands the diff --git a/packages/flutwire/lib/src/store/disk_store.dart b/packages/flutwire/lib/src/store/disk_store_io.dart similarity index 100% rename from packages/flutwire/lib/src/store/disk_store.dart rename to packages/flutwire/lib/src/store/disk_store_io.dart diff --git a/packages/flutwire/lib/src/store/open_store_io.dart b/packages/flutwire/lib/src/store/open_store_io.dart index d3935f7..03c371e 100644 --- a/packages/flutwire/lib/src/store/open_store_io.dart +++ b/packages/flutwire/lib/src/store/open_store_io.dart @@ -2,7 +2,7 @@ import 'dart:io'; import '../config.dart'; import '../diagnostics.dart'; -import 'disk_store.dart'; +import 'disk_store_io.dart'; import 'store.dart'; /// Opens the store the configuration asks for. diff --git a/packages/flutwire/test/capture/websocket_reader_test.dart b/packages/flutwire/test/capture/websocket_reader_test.dart index 91be8a4..3ac6855 100644 --- a/packages/flutwire/test/capture/websocket_reader_test.dart +++ b/packages/flutwire/test/capture/websocket_reader_test.dart @@ -4,7 +4,7 @@ import 'dart:typed_data'; import 'package:flutter_test/flutter_test.dart'; import 'package:flutwire/flutwire.dart'; -import 'package:flutwire/src/capture/messages/websocket_reader.dart'; +import 'package:flutwire/src/capture/messages/websocket_reader_io.dart'; import '../support/websocket_frames.dart'; diff --git a/packages/flutwire/test/capture/wire_socket_test.dart b/packages/flutwire/test/capture/wire_socket_test.dart index 67b4bc8..056a059 100644 --- a/packages/flutwire/test/capture/wire_socket_test.dart +++ b/packages/flutwire/test/capture/wire_socket_test.dart @@ -8,7 +8,7 @@ import 'dart:typed_data'; import 'package:flutter_test/flutter_test.dart'; import 'package:flutwire/flutwire.dart'; -import 'package:flutwire/src/capture/wire/wire_socket.dart'; +import 'package:flutwire/src/capture/wire/wire_socket_io.dart'; import 'package:flutwire/src/engine.dart'; import '../support/websocket_frames.dart'; diff --git a/packages/flutwire/test/store/disk_store_messages_test.dart b/packages/flutwire/test/store/disk_store_messages_test.dart index ec71c94..5b8f098 100644 --- a/packages/flutwire/test/store/disk_store_messages_test.dart +++ b/packages/flutwire/test/store/disk_store_messages_test.dart @@ -4,7 +4,7 @@ import 'dart:typed_data'; import 'package:flutter_test/flutter_test.dart'; import 'package:flutwire/flutwire.dart'; -import 'package:flutwire/src/store/disk_store.dart'; +import 'package:flutwire/src/store/disk_store_io.dart'; final epoch = DateTime.utc(2026, 10, 24, 12); diff --git a/packages/flutwire/test/store/disk_store_test.dart b/packages/flutwire/test/store/disk_store_test.dart index afd4460..dc2e535 100644 --- a/packages/flutwire/test/store/disk_store_test.dart +++ b/packages/flutwire/test/store/disk_store_test.dart @@ -7,7 +7,7 @@ import 'package:flutwire/src/config.dart'; import 'package:flutwire/src/model/body.dart'; import 'package:flutwire/src/model/call.dart'; import 'package:flutwire/src/model/headers.dart'; -import 'package:flutwire/src/store/disk_store.dart'; +import 'package:flutwire/src/store/disk_store_io.dart'; import 'package:flutwire/src/store/open_store.dart'; import 'package:flutwire/src/store/store.dart'; diff --git a/packages/flutwire/test/store/preferences_test.dart b/packages/flutwire/test/store/preferences_test.dart index ff35778..07a19d6 100644 --- a/packages/flutwire/test/store/preferences_test.dart +++ b/packages/flutwire/test/store/preferences_test.dart @@ -3,7 +3,7 @@ import 'dart:io'; import 'package:flutter_test/flutter_test.dart'; import 'package:flutwire/src/config.dart'; -import 'package:flutwire/src/store/disk_store.dart'; +import 'package:flutwire/src/store/disk_store_io.dart'; import 'package:flutwire/src/store/preferences.dart'; import 'package:flutwire/src/store/preferences_io.dart';