diff --git a/ARES.bt b/ARES.bt index 929f2ab7..2a88730e 100644 --- a/ARES.bt +++ b/ARES.bt @@ -108,6 +108,48 @@ struct CallInfo; struct ExecString; +/* Counts, sizes and indices are the unsigned LEB128 varints Luau bytecode + * uses, see write_uvarint() in ares.cpp. `v` is the decoded value. */ +typedef struct { + local uint64 v = 0; + local int shift = 0; + local uchar cur; + do { + cur = ReadUByte(FTell()); + uchar raw; + v |= (uint64)(cur & 0x7F) << shift; + shift += 7; + } while (cur & 0x80); +} UVarint ; + +/* An `int` field: the same encoding, holding the 32-bit two's complement. */ +typedef struct { + local uint64 v = 0; + local int shift = 0; + local uchar cur; + do { + cur = ReadUByte(FTell()); + uchar raw; + v |= (uint64)(cur & 0x7F) << shift; + shift += 7; + } while (cur & 0x80); +} IntVarint ; + +/* A genuinely signed field: 64-bit zigzag varint, see write_ares_sint() in + * ares.cpp. Integer payloads and the two signed CallInfo fields use it. */ +typedef struct { + local uint64 v = 0; + local int shift = 0; + local uchar cur; + do { + cur = ReadUByte(FTell()); + uchar raw; + v |= (uint64)(cur & 0x7F) << shift; + shift += 7; + } while (cur & 0x80); + local int64 s = (int64)(v >> 1) ^ -(int64)(v & 1); +} SVarint ; + // From LuauBytecode.bt typedef enum { // NOP: noop @@ -502,6 +544,10 @@ typedef uint32 Instruction ; local uint64 refNum = 0; // tracks where we saw each reference local uint64 refPositions[0xFFFF] = {0}; +// string table entries parsed so far, and where each one's bytes start +local uint64 strNum = 0; +local uint64 strPositions[0xFFFF] = {0}; +local uint64 strLengths[0xFFFF] = {0}; /* Closes a length-prefixed record: anything left before record_end is test * padding or fields a newer minor appended, both of which the reader skips. @@ -520,20 +566,22 @@ typedef struct { /* Mirrors ARES_FORMAT_MAJOR in VM/include/lua.h. Any minor under it * parses: fields are only ever appended inside records, and each record's * length word steps over what this template doesn't know. */ - Assert(major == 6); + Assert(major == 7); uint32_t record_len; local int64 record_end = FTell() + record_len; uint8_t sizeof_number; /* sizeof(lua_Number) to check type compatibility */ lua_Number test; /* -1.234567890 to check representation compatibility */ - uint8_t sizeof_int; /* sizeof(int) in persisted data */ - uint8_t sizeof_size_t; /* sizeof(size_t) in persisted data */ uint8_t vector_components; /* how many components are stored in vectors */ - /* Note that the last two fields determine the size of the int and size_t - * fields in the following definitions. We write each value in the native - * "size" and check for truncation when reading, if necessary. */ - /* Reserved when the header is written and patched in after the root - * object, see p_header_refcount() in ares.cpp. */ + /* The root is serialized to a scratch buffer before the header is + * written, so this is final. Strings do not count. */ uint32_t final_refcount; + /* Every feature bit the writer knows, i.e. every appended field whose + * bytes are in this stream's records. Mirrors kAresFeaturesSupported in + * ares.cpp. */ + uint64_t features_present; + /* The subset a reader has to understand: what populated fields asked for + * through require_feature(). An unknown bit here refuses the stream. */ + uint64_t features_required; ParseRecordEnd(record_end); } Header ; @@ -544,6 +592,10 @@ void ParseGCHeader() { } string ReadObject(Object &o) { + if (o.type == ARES_T_STRING) { + return Str("%s $%Lu \"%s\"", EnumToString(o.type), o.val.v, + ReadString(strPositions[o.val.v], strLengths[o.val.v])); + } // ref should be non-zero if present if (o.ourRef) { // ARES_T_REFERENCE refNums are references _to_ something @@ -571,6 +623,8 @@ typedef struct { case ARES_T_VECTOR: // storing a reference to a reference? no. case ARES_T_REFERENCE: + // strings are indexes into the string table, not references + case ARES_T_STRING: break; // Permanents take a number too: persist_keyed() allocates it before // the permanents table gets a look, and u_permanent() reserves it @@ -604,13 +658,13 @@ typedef struct { case ARES_T_NIL: break; case ARES_T_BOOLEAN: - int32_t val; break; + IntVarint val; break; case ARES_T_LIGHTUSERDATA: LightUserdata val; break; case ARES_T_NUMBER: Number val; break; case ARES_T_INTEGER: - int64_t val; break; + SVarint val; break; case ARES_T_VECTOR: float val[3]; break; /* Reserved but never emitted; p_vector refuses to write f64 vectors @@ -619,7 +673,8 @@ typedef struct { ParseGCHeader(); double val[3]; break; case ARES_T_STRING: - String val; break; + /* 1-based index into the string table ahead of the root */ + UVarint val ; break; case ARES_T_TABLE: Table val; break; case ARES_T_FUNCTION: @@ -643,8 +698,8 @@ typedef struct { /* Note that the types LUA_TNIL, LUA_TBOOLEAN, LUA_TNUMBER and * LUA_TLIGHTUSERDATA will never be "referenced", but always be written * directly. */ - uint32 reference ; /* The index the object was referenced with */ - ourRef = reference; + IntVarint reference ; /* The index the object was referenced with */ + ourRef = reference.v; break; case ARES_T_CLASS: case ARES_T_OBJECT: @@ -658,28 +713,45 @@ typedef struct { ParseRecordEnd(record_end); } Object; +/* A string table entry. Every string the root reaches, in first-sight order. */ typedef struct { ParseGCHeader(); - size_t length; /* The length of the string */ - char str[length]; /* The actual string (not always null terminated) */ + UVarint length; /* The length of the string */ + /* Where the bytes start, whether or not there are any */ + local int64 str_start = FTell(); + char str[length.v]; /* The actual string (not always null terminated) */ + strNum++; + strPositions[strNum] = str_start; + strLengths[strNum] = length.v; } String ; +typedef struct { + uint32_t record_len ; + local int64 record_end = FTell() + record_len; + UVarint count; + local int i; + for (i = 0; i < count.v; ++i) { + String entry; + } + ParseRecordEnd(record_end); +} StrTab ; + typedef struct { ParseGCHeader(); - size_t length; /* The length of the buffer */ - char data[length]; /* The actual buffer data */ + UVarint length; /* The length of the buffer */ + char data[length.v]; /* The actual buffer data */ } Buffer ; struct Table { ParseGCHeader(); uint8_t read_only; uint8_t safe_env; - int array_size; - int node_size; + IntVarint array_size; + IntVarint node_size; local int i; // This serialization format preserves `nil` holes, so we can't use `nil` as a // terminator. - for (i=0; i<(array_size+node_size); ++i) { + for (i=0; i<(array_size.v+node_size.v); ++i) { /* key/value pairs */ struct Pair { /* Both of these may legally be `nil` due to hole preservation */ @@ -704,7 +776,7 @@ typedef struct { } case UTAG_DETECTED_EVENT: { - int32_t index; + IntVarint index; uint8_t valid; uint8_t can_change_damage; break; @@ -723,16 +795,16 @@ typedef struct { } case UTAG_STRBUF: { - size_t capacity; - size_t used; - uchar data[used]; + UVarint capacity; + UVarint used; + uchar data[used.v]; break; } case UTAG_OPAQUE_BUFFER: default: { - size_t length; /* Size of the data */ - uchar data[length]; /* The actual data */ + UVarint length; /* Size of the data */ + uchar data[length.v]; /* The actual data */ } } Object metatable; /* The metatable (nil for none, otherwise LUA_TTABLE) */ @@ -740,7 +812,7 @@ typedef struct { typedef struct { uint8_t lutag; /* Userdata tag */ - size_t ptr; + uint64_t ptr; /* Pointer-sized, so fixed width rather than a varint */ } LightUserdata ; struct Closure { @@ -774,7 +846,7 @@ struct Proto { Object source; /* Textual source for the function, string or nil */ - int bytecode_id; + IntVarint bytecode_id; uint8_t maxstacksize; /* Size of stack reserved for the function */ uint8_t flags; /* Flags related to the function definition */ @@ -782,16 +854,16 @@ struct Proto { uint8_t nups; /* Number of upvalues */ uint8_t is_vararg; /* 1 if function accepts varargs, 0 otherwise */ - int sizecode; /* Number of instructions in code */ - Instruction code[sizecode]; /* The proto's code */ + IntVarint sizecode; /* Number of instructions in code */ + Instruction code[sizecode.v]; /* The proto's code */ - int sizek; /* Number of constants referenced */ - Object k[sizek]; /* Constants referenced */ + IntVarint sizek; /* Number of constants referenced */ + Object k[sizek.v]; /* Constants referenced */ - int sizep; /* Number of inner Protos referenced */ - Object p[sizep]; /* Inner Protos referenced */ + IntVarint sizep; /* Number of inner Protos referenced */ + Object p[sizep.v]; /* Inner Protos referenced */ - int linedefined; /* Start of line range */ + IntVarint linedefined; /* Start of line range */ Object debugname; /* Name of the function for debugging, string or nil */ // Neither of these are supported yet. @@ -817,53 +889,56 @@ struct Proto { Object upvalnames[sizeupvalues]; /* Upvalue names */ } - int sizeyieldpoints; - int yieldpoints[sizeyieldpoints]; + IntVarint sizeyieldpoints; + local int yp; + for (yp = 0; yp < sizeyieldpoints.v; ++yp) { + IntVarint yieldpoint; + } }; struct Thread { ParseGCHeader(); Object env; - uint32_t stacksize; /* Allocated stack slots, restored as-is */ - size_t top; /* top = L->top - L->stack; */ - Object stack[top]; /* All stack values, bottom up */ + UVarint stacksize; /* Allocated stack slots, restored as-is */ + UVarint top; /* top = L->top - L->stack; */ + Object stack[top.v]; /* All stack values, bottom up */ AresStatus status; /* current thread status (ok, yield) */ uint8_t activememcat ; Object namecall; /* The pending namecall string, nil for none */ - uint32_t size_ci; /* Allocated callinfo slots, never below BASIC_CI_SIZE */ - uint32_t num_cis; /* number of callinfo frames */ + UVarint size_ci; /* Allocated callinfo slots, never below BASIC_CI_SIZE */ + UVarint num_cis; /* number of callinfo frames */ /* The CallInfo stack, starting with base_ci. Each frame is its own record. */ struct CallInfo { uint32_t record_len ; local int64 record_end = FTell() + record_len; - size_t func; /* func = ci->func - thread->stack */ - size_t top; /* top = ci->top - thread->stack */ - size_t base; /* base = ci->base - thread-stack */ - int32_t nresults; /* expected number of results from this function */ + UVarint func; /* func = ci->func - thread->stack */ + UVarint top; /* top = ci->top - thread->stack */ + UVarint base; /* base = ci->base - thread-stack */ + SVarint nresults; /* expected number of results from this function, -1 for MULTRET */ uint8_t flags; /* What to do after completing this call, see lstate.h */ eris_CIKind ci_kind;/* What kind of CallInfo this is */ if (ci_kind == ERIS_CI_KIND_LUA) { - int yield_point; - int savedpc; /* savedpc = ci->u.l.savedpc - ci_func(ci)->p->code */ + SVarint yield_point; /* -1 on an errored thread */ + IntVarint savedpc; /* savedpc = ci->u.l.savedpc - ci_func(ci)->p->code */ } else if (ci_kind == ERIS_CI_KIND_C) { - int32_t errfunc; /* pcall's error handler, 1-based from ci->base, 0 for none */ + IntVarint errfunc; /* pcall's error handler, 1-based from ci->base, 0 for none */ Object function; } else { Assert(ci_kind == ERIS_CI_KIND_NONE); } ParseRecordEnd(record_end); - } ci[num_cis] ; + } ci[num_cis.v] ; while (TRUE) { struct OpenUpval { - size_t idx; /* stack index of the value + 1; 0 if end of list */ - if (idx) + UVarint idx; /* stack index of the value + 1; 0 if end of list */ + if (idx.v) Object upval; /* The upvalue */ } openupval ; - if (!openupval.idx) + if (!openupval.idx.v) break; } }; @@ -878,7 +953,9 @@ struct PermKey { typedef struct { refNum = 0; + strNum = 0; Header header; + StrTab strtab; /* Read in full before the root, see u_strtab() */ Object rootobj; /* The root object that was persisted. */ /* The same check the reader makes in u_finish() */ if (refNum != header.final_refcount) @@ -904,12 +981,19 @@ typedef struct { Assert(tag == "EXEC"); uint32_t major; uint32_t minor; - Assert(major == 2); + Assert(major == 3); + /* Feature masks for the core section: the features whose fields the + * writer emits, then the subset a reader has to understand. Both are + * zero until a feature exists; a nonzero required mask is refused. */ + uint64_t features_present; + uint64_t features_required; /* The concrete Script subclass's fingerprint (EXEC again for the base - * class), covering the extra section at the end. */ + * class), covering the extra section at the end, with its own masks. */ char class_tag[4] ; uint32_t class_major; uint32_t class_minor; + uint64_t class_features_present; + uint64_t class_features_required; struct CoreSection { uint32_t record_len ; diff --git a/Executor/include/Luau/Script.h b/Executor/include/Luau/Script.h index 8bcc574c..fb8be550 100644 --- a/Executor/include/Luau/Script.h +++ b/Executor/include/Luau/Script.h @@ -54,8 +54,10 @@ struct RunResult // Payload format written by Script::serializeState(). Leads every payload as // the family magic and versions the core section only; the concrete class's // fingerprint follows it and versions the extra section, so the two evolve -// independently. -constexpr StateFingerprint kScriptStateFingerprint{{'E', 'X', 'E', 'C'}, 2, 0}; +// independently. Each fingerprint is followed by a present and a required +// feature mask for its section, both zero until a feature exists; see +// serializeState(). +constexpr StateFingerprint kScriptStateFingerprint{{'E', 'X', 'E', 'C'}, 3, 0}; // We need to carry around the error messages in our state, but // the messages can be arbitrarily large. Cap them. diff --git a/Executor/src/Script.cpp b/Executor/src/Script.cpp index 8c9bd988..28d946ae 100644 --- a/Executor/src/Script.cpp +++ b/Executor/src/Script.cpp @@ -674,9 +674,17 @@ bool Script::serializeState(std::string& out) writer.writeBytes(kScriptStateFingerprint.tag, sizeof(kScriptStateFingerprint.tag)); writer.writeU32(kScriptStateFingerprint.major); writer.writeU32(kScriptStateFingerprint.minor); + // Each fingerprint is followed by two feature masks for its section: the + // features whose fields this writer emits, then the subset a reader has + // to understand to load this payload. No feature exists yet, so both are + // zero, and a reader refuses any nonzero required mask. + writer.writeU64(0); + writer.writeU64(0); writer.writeBytes(fingerprint.tag, sizeof(fingerprint.tag)); writer.writeU32(fingerprint.major); writer.writeU32(fingerprint.minor); + writer.writeU64(0); + writer.writeU64(0); size_t core = writer.beginSection(); writer.writeF32(mSleep); @@ -733,6 +741,8 @@ bool Script::restoreState(const char* data, size_t len) uint64_t sticky_handler = 0; uint64_t current_events = 0; uint64_t event_handlers = 0; + uint64_t present_features = 0; + uint64_t required_features = 0; ByteReader core{nullptr, 0}; ByteReader extra{nullptr, 0}; @@ -752,6 +762,14 @@ bool Script::restoreState(const char* data, size_t len) setFault(FaultKind::Runtime, "invalid script state"); return false; } + // Present features are skipped through their section lengths. + // We don't have any yet, so just bail if it's non-zero. + if (!reader.readU64(present_features) || !reader.readU64(required_features) || required_features != 0) + { + logWarn(logSource(), "Script state requires features 0x%llx this build doesn't know", (unsigned long long)required_features); + setFault(FaultKind::Runtime, "invalid script state"); + return false; + } // A payload meant for another class is refused here, before anything is forked if (!reader.readBytes(tag, sizeof(tag)) || memcmp(tag, fingerprint.tag, sizeof(tag)) != 0) { @@ -765,6 +783,12 @@ bool Script::restoreState(const char* data, size_t len) setFault(FaultKind::Runtime, "invalid script state"); return false; } + if (!reader.readU64(present_features) || !reader.readU64(required_features) || required_features != 0) + { + logWarn(logSource(), "Script extra state requires features 0x%llx this build doesn't know", (unsigned long long)required_features); + setFault(FaultKind::Runtime, "invalid script state"); + return false; + } // Every field fails the same way, keep each read to one line #define READ_OR_BAIL(read_expr) \ do \ diff --git a/VM/include/lua.h b/VM/include/lua.h index 20185054..ec5ceea3 100644 --- a/VM/include/lua.h +++ b/VM/include/lua.h @@ -785,14 +785,10 @@ static void populateperms(lua_State *L, bool forUnpersist) #undef eris_persist_base_cont #undef eris_persist_cont -// Ares stream format version. A reader accepts any minor under a major it -// supports: fields are only ever appended to length-prefixed records, so a -// newer minor's extra bytes are skipped and an older minor's missing ones -// default. The major moves only for a change that can't be expressed that way, -// and the reader keeps every major back to ARES_MIN_SUPPORTED_MAJOR readable. -#define ARES_FORMAT_MAJOR 6 +// Ares stream format versions +#define ARES_FORMAT_MAJOR 7 #define ARES_FORMAT_MINOR 0 -#define ARES_MIN_SUPPORTED_MAJOR 6 +#define ARES_MIN_SUPPORTED_MAJOR 7 LUA_API lua_State *eris_make_forkserver(lua_State *Lsrc); diff --git a/VM/src/ares.cpp b/VM/src/ares.cpp index f9c6369d..dcab72bc 100644 --- a/VM/src/ares.cpp +++ b/VM/src/ares.cpp @@ -63,6 +63,7 @@ THE SOFTWARE. #include "lstrbuf.h" #include "lljson.h" #include "Luau/Bytecode.h" +#include "Luau/BytecodeWire.h" LUAU_FASTFLAG(LuauCIProto) LUAU_FASTFLAG(LuauManagedDebugNames) @@ -192,8 +193,7 @@ typedef uint64_t ares_size_t; #define ERIS_ERR_TRUNC_INT "int value would get truncated" #define ERIS_ERR_TRUNC_SIZE "size_t value would get truncated" #define ERIS_ERR_TYPE_FLOAT "unsupported lua_Number type" -#define ERIS_ERR_TYPE_INT "unsupported int type" -#define ERIS_ERR_TYPE_SIZE "unsupported size_t type" +#define ERIS_ERR_VARINT "malformed data: unterminated varint" #define ERIS_ERR_TYPEP "trying to persist unknown type %d" #define ERIS_ERR_TYPEU "trying to unpersist unknown type %d" #define ERIS_ERR_UCFUNC "bad C closure (C function expected, got %s)" @@ -206,9 +206,12 @@ typedef uint64_t ares_size_t; #define ERIS_ERR_INVAL_PC "Tried to serialize thread yielded at invalid point" #define ERIS_ERR_RECORD "malformed data: record exceeds enclosing record" #define ERIS_ERR_REFCOUNT "malformed data: reference count mismatch (expected %u, got %u)" -#define ERIS_ERR_RAW_APPENDS "persisted a reference inside a record's raw-append section (refcount %d -> %d)" +#define ERIS_ERR_TRAILER "persisted a reference in a record's trailer without require_feature() (refcount %d -> %d)" #define ERIS_ERR_TRAILING "malformed data: trailing bytes after root object" #define ERIS_ERR_MAJOR "unsupported file format version %u.%u (want %u.x through %u.x)" +#define ERIS_ERR_FEATURE "file format version %u.%u uses features 0x%llx this reader doesn't know" +#define ERIS_ERR_FEATURE_SET "malformed data: required features 0x%llx are not among the present features 0x%llx" +#define ERIS_ERR_STRIDX "malformed data: string table index %llu out of range (table has %d)" /* ** ============================================================================ @@ -231,7 +234,9 @@ enum AresType : uint8_t ARES_T_VECTOR = 5, /* 6-15 reserved. */ - /* Collectable types, keyed into the reftable and carrying a memcat byte. */ + /* Collectable types, keyed into the reftable and carrying a memcat byte. + * Strings are the exception: the body is an index into the string table + * that precedes the root, which holds the memcat. */ ARES_T_STRING = 16, ARES_T_TABLE = 17, ARES_T_FUNCTION = 18, @@ -320,12 +325,17 @@ typedef struct PersistInfo { bool persistingCFunc; /* Junk bytes appended inside every record, for forward-compat tests. */ uint32_t testPadding; - /* info->refcount where the record being written reached BEGIN_RAW_APPENDS(), + /* info->refcount where the record being written reached BEGIN_TRAILER(), * -1 until its body gets there. */ - int rawAppendsRefcount; - /* Where the header reserved the final reference count, patched in once the - * root object has been written. A plain offset because this sits in a union. */ - std::streamoff refcountPos; + int trailerRefcount; + /* Whether require_feature() has been called since BEGIN_TRAILER(). A + * reference persisted in the trailer without it is an error, since a reader + * that skips the trailer would lose track of the numbering. */ + bool trailerRequired; + /* Every feature a populated field asked for through require_feature(). */ + uint64_t requiredFeatures; + /* Strings assigned a string table index so far. */ + int strcount; } PersistInfo; typedef uint8_t lu_byte; @@ -337,13 +347,15 @@ typedef struct UnpersistInfo { size_t pos; /* End of the innermost open record. Every read is bounded by it. */ size_t record_end; - size_t sizeof_int; - size_t sizeof_size_t; size_t vector_components; uint32_t major; uint32_t minor; /* The writer's final reference count, from the header. */ uint32_t expectedRefcount; + /* The features whose fields the writer emitted, from the header. */ + uint64_t presentFeatures; + /* Entries in the string table, once u_strtab has read it. */ + int strtabCount; /* When set, stamped as the threaddata of every thread of the unpersisted * tree; threads the tree spawns later inherit it through the userthread * callback. */ @@ -391,6 +403,21 @@ static char const kHeader[] = { 'A', 'R', 'E', 'S' }; /* Floating point number used to check compatibility of loaded data. */ static const lua_Number kHeaderNumber = (lua_Number)-1.234567890; +/* Feature bits, one per group of appended fields. Wire values: never + * renumbered or reused within a major; a major bump clears the set. A set bit + * in the header means "this build writes the field's bytes"; the field must be + * understood only when the writer also called require_feature() for it. To + * add one: define the bit, OR it into kAresFeaturesSupported, bump + * ARES_FORMAT_MINOR, and add a golden fixture. */ +enum AresFeature : uint64_t { + /* ARES_FEATURE_EXAMPLE = 1ull << 0, */ +}; + +/* The features this build implements. As a writer it emits the fields of all + * of them, so this is what the header's present mask says; as a reader it can + * be asked to require no more than this. */ +static const uint64_t kAresFeaturesSupported = 0; + /* Records that carry a length prefix: the VM-shaped ones whose layout tracks * the VM. Strings and buffers are self-delimiting, scalars never grow. Class * and object have no body yet but will be VM-shaped when they do. */ @@ -402,9 +429,10 @@ static inline bool type_is_framed(AresType type) { // The wire-tag equivalent of iscollectable(). ARES_T_PROTO and ARES_T_UPVAL are -// excluded, their stack stand-in needs a deref to get at the GCObject. +// excluded, their stack stand-in needs a deref to get at the GCObject. Strings +// carry theirs in the string table entry instead. static inline bool type_has_memcat(AresType type) { - return type == ARES_T_STRING || type == ARES_T_BUFFER || type == ARES_T_TABLE || + return type == ARES_T_BUFFER || type == ARES_T_TABLE || type == ARES_T_FUNCTION || type == ARES_T_USERDATA || type == ARES_T_THREAD || type == ARES_T_CLASS || type == ARES_T_OBJECT || type == ARES_T_VECTORD; } @@ -418,8 +446,10 @@ static inline bool ares_type_is_tvalue(AresType type) { /* Stack indices of some internal values/tables, to avoid magic numbers. */ #define PERMIDX 1 #define REFTIDX 2 -#define BUFFIDX 3 -#define PATHIDX 4 +/* Persisting: string -> index and index -> string. Unpersisting: index -> string. */ +#define STRTIDX 3 +#define BUFFIDX 4 +#define PATHIDX 5 /* Table indices for upvalue tables, keeping track of upvals to open. */ #define UVTOCL 1 @@ -748,6 +778,25 @@ set_setting(lua_State *L, void *key) { /* ... value */ lua_settable(L, LUA_REGISTRYINDEX); /* ... */ } +/* Marks the stream as needing a reader that knows this feature. Call it from + * the site that writes an appended field, under the same "is this the + * default" test the reader applies when the feature is absent, so a stream + * where the field was never populated still loads on older readers. Mandatory + * when the field takes a reference number, see RecordWriter::close. A call + * naming a bit this build doesn't know, such as one left over from before a + * major bump, fails the build. */ +#define require_feature(info, feature) do { \ + static_assert(((feature) & kAresFeaturesSupported) == (feature), "require_feature() names a feature this build doesn't support"); \ + static_assert((feature) != 0 && ((feature) & ((feature) - 1)) == 0, "require_feature() takes a single feature bit"); \ + (info)->u.pi.requiredFeatures |= (feature); \ + (info)->u.pi.trailerRequired = true; \ + } while(0) + +/* Whether the stream being read carries the field(s) behind this feature. The + * reader gates every appended field on this, never on the writer's minor or + * on bytes remaining in the record. */ +#define has_feature(info, feature) (((info)->u.upi.presentFeatures & (feature)) != 0) + /* Used as a callback for luaL_opt to check boolean setting values. */ static bool checkboolean(lua_State *L, int narg) { /* ... bool? ... */ @@ -848,16 +897,6 @@ write_uint64_t(Info *info, uint64_t value) { write_uint8_t(info, (uint8_t)(value >> 56)); } -static void -write_int32_t(Info *info, int32_t value) { - write_uint32_t(info, (uint32_t)value); -} - -static void -write_int64_t(Info *info, int64_t value) { - write_uint64_t(info, (uint64_t)value); -} - static void write_float32(Info *info, float value) { uint32_t rep; @@ -872,27 +911,36 @@ write_float64(Info *info, double value) { write_uint64_t(info, rep); } -/* Note regarding the following: any decent compiler should be able - * to reduce these to just the write call, since sizeof is constant. */ +/* Counts, sizes and indices are the same unsigned varints Luau bytecode uses, + * mirroring writeVarInt() in BytecodeBuilder.cpp. Nearly all fit in one byte. */ +static void +write_uvarint(Info *info, uint64_t value) { + do { + write_uint8_t(info, (uint8_t)((value & 127) | ((value > 127) << 7))); + value >>= 7; + } while (value); +} +/* Counts and offsets. Written as 32-bit two's complement, so a negative value + * costs five bytes; fields that are genuinely signed use ares_sint. */ static void write_int(Info *info, int value) { - if (sizeof(int) <= sizeof(int32_t)) { - write_int32_t(info, value); - } - else { - eris_error(info, ERIS_ERR_TYPE_INT); - } + static_assert(sizeof(int) == sizeof(int32_t), "int is assumed to be 32-bit on the wire"); + write_uvarint(info, (uint32_t)value); +} + +/* Zigzag varint for genuinely signed values: integer payloads, and the few + * fields where small negatives are normal, such as MULTRET in nresults. One + * 64-bit flavor covers them all; small values encode identically either way. */ +static void +write_ares_sint(Info *info, int64_t value) { + const uint64_t bits = (uint64_t)value; + write_uvarint(info, (bits << 1) ^ (uint64_t)(value >> 63)); } static void write_ares_size_t(Info *info, ares_size_t value) { - if (sizeof(size_t) <= sizeof(uint64_t)) { - write_uint64_t(info, (uint64_t)value); - } - else { - eris_error(info, ERIS_ERR_TYPE_SIZE); - } + write_uvarint(info, value); } static void @@ -931,13 +979,6 @@ read_uint8_t(Info *info) { return value; } -static uint16_t -read_uint16_t(Info *info) { - auto value = (uint16_t)read_uint8_t(info); - value |= (uint16_t)read_uint8_t(info) << 8; - return value; -} - static uint32_t read_uint32_t(Info *info) { auto value = (uint32_t)read_uint8_t(info); @@ -960,21 +1001,6 @@ read_uint64_t(Info *info) { return value; } -static int16_t -read_int16_t(Info *info) { - return (int16_t)read_uint16_t(info); -} - -static int32_t -read_int32_t(Info *info) { - return (int32_t)read_uint32_t(info); -} - -static int64_t -read_int64_t(Info *info) { - return (int64_t)read_uint64_t(info); -} - static float read_float32(Info *info) { float value; @@ -991,58 +1017,59 @@ read_float64(Info *info) { return value; } -/* Note regarding the following: unlike with writing the sizeof check will be - * impossible to optimize away, since it depends on the input; however, the - * truncation check may be optimized away in the case where the read data size - * equals the native one, so reading data written on the same machine should be - * reasonably quick. Doing a (rather rudimentary) benchmark this did not have - * any measurable impact on performance. */ +/* A 64-bit value is at most ten varint bytes. */ +static const size_t kMaxVarintBytes = 10; + +static uint64_t +read_uvarint(Info *info) { + /* Luau::readVarInt64 trusts its input, so find the terminating byte inside + * the record before handing it the buffer. */ + const char *data = info->u.upi.data; + size_t limit = RECORD_REMAINING(); + if (limit > kMaxVarintBytes) { + limit = kMaxVarintBytes; + } + size_t n = 0; + while (n < limit && ((uint8_t)data[info->u.upi.pos + n] & 0x80)) { + ++n; + } + if (n == limit) { + eris_error(info, ERIS_ERR_VARINT); + } + return Luau::readVarInt64(data, info->u.upi.pos); +} static int read_int(Info *info) { - int value; - if (info->u.upi.sizeof_int == sizeof(int16_t)) { - int16_t pvalue = read_int16_t(info); - value = (int)pvalue; - if ((int32_t)value != pvalue) { - eris_error(info, ERIS_ERR_TRUNC_INT); - } - } - else if (info->u.upi.sizeof_int == sizeof(int32_t)) { - int32_t pvalue = read_int32_t(info); - value = (int)pvalue; - if ((int32_t)value != pvalue) { - eris_error(info, ERIS_ERR_TRUNC_INT); - } - } - else if (info->u.upi.sizeof_int == sizeof(int64_t)) { - int64_t pvalue = read_int64_t(info); - value = (int)pvalue; - if ((int64_t)value != pvalue) { - eris_error(info, ERIS_ERR_TRUNC_INT); - } + const uint64_t value = read_uvarint(info); + if (value > UINT32_MAX) { + eris_error(info, ERIS_ERR_TRUNC_INT); } - else { - eris_error(info, ERIS_ERR_TYPE_INT); - value = 0; /* not reached */ + return (int)(uint32_t)value; +} + +static int64_t +read_ares_sint(Info *info) { + const uint64_t bits = read_uvarint(info); + return (int64_t)((bits >> 1) ^ (uint64_t)-(int64_t)(bits & 1)); +} + +/* An ares_sint that has to fit an int on this side. */ +static int +read_ares_sint32(Info *info) { + const int64_t value = read_ares_sint(info); + if (value < INT32_MIN || value > INT32_MAX) { + eris_error(info, ERIS_ERR_TRUNC_INT); } - return value; + return (int)value; } static ares_size_t read_ares_size_t(Info *info) { - ares_size_t value; - if (info->u.upi.sizeof_size_t <= sizeof(uint64_t)) { - value = read_uint64_t(info); - // Refuse to read sizes that would be truncated on 32-bit - if (value > (ares_size_t)SIZE_MAX) { - eris_error(info, "malformed data: size value out of range"); - } - return value; - } - else { - eris_error(info, ERIS_ERR_TYPE_SIZE); - value = 0; /* not reached */ + const uint64_t value = read_uvarint(info); + // Refuse to read sizes that would be truncated on 32-bit + if (value > (uint64_t)SIZE_MAX) { + eris_error(info, "malformed data: size value out of range"); } return value; } @@ -1067,18 +1094,25 @@ read_Instruction(Info *info) { /* * Length-prefixed records. A record is `u32 len` followed by `len` body bytes. - * Also keeps note of any reference types written so we can be sure we don't - * mess anything up. + * Everything after BEGIN_TRAILER() is the trailer: fields a later feature + * added, which an older reader steps over via the length word. A reference + * persisted there is only safe if require_feature() was called, so the readers + * that would have skipped it refuse the stream instead. */ -#define BEGIN_RAW_APPENDS() (info->u.pi.rawAppendsRefcount = info->refcount) +#define BEGIN_TRAILER() do { \ + info->u.pi.trailerRefcount = info->refcount; \ + info->u.pi.trailerRequired = false; \ + } while(0) // Write a length-prefixed binary section to struct RecordWriter { explicit RecordWriter(Info *info_) : info(info_), exceptions(std::uncaught_exceptions()), - outer_raw_appends_refcount(info_->u.pi.rawAppendsRefcount) { + outer_trailer_refcount(info_->u.pi.trailerRefcount), + outer_trailer_required(info_->u.pi.trailerRequired) { write_uint32_t(info, 0); - info->u.pi.rawAppendsRefcount = -1; + info->u.pi.trailerRefcount = -1; + info->u.pi.trailerRequired = false; body_start = info->u.pi.writer->tellp(); } @@ -1086,15 +1120,14 @@ struct RecordWriter { eris_assert(!closed); closed = true; - // A body that never called BEGIN_RAW_APPENDS() - eris_assert(info->u.pi.rawAppendsRefcount != -1); - // someone persist()ed something inside the raw trailer section. This is an error, - // and would definitely break compatibility. - if (info->u.pi.rawAppendsRefcount != info->refcount) { - eris_error(info, ERIS_ERR_RAW_APPENDS, info->u.pi.rawAppendsRefcount, info->refcount); + // A body that never called BEGIN_TRAILER() has no trailer to check + eris_assert(info->u.pi.trailerRefcount != -1); + if (info->u.pi.trailerRefcount != info->refcount && !info->u.pi.trailerRequired) { + eris_error(info, ERIS_ERR_TRAILER, info->u.pi.trailerRefcount, info->refcount); } - info->u.pi.rawAppendsRefcount = outer_raw_appends_refcount; + info->u.pi.trailerRefcount = outer_trailer_refcount; + info->u.pi.trailerRequired = outer_trailer_required; // write in some padding if we're in a test that wants malformed data for (uint32_t i = 0; i < info->u.pi.testPadding; ++i) { write_uint8_t(info, (uint8_t)(0xA5 ^ i) | 1); @@ -1121,7 +1154,8 @@ struct RecordWriter { Info *info; int exceptions; - int outer_raw_appends_refcount; + int outer_trailer_refcount; + bool outer_trailer_required; std::streampos body_start; bool closed = false; }; @@ -1165,14 +1199,14 @@ static void unpersist(Info*); static void p_boolean(Info *info) { /* ... bool */ // Have to use this rather than toboolean so that we keep the full int value - WRITE_VALUE(bvalue(info->L->top - 1), int32_t); + WRITE_VALUE(bvalue(info->L->top - 1), int); } static void u_boolean(Info *info) { /* ... */ eris_checkstack(info->L, 1); // Have to use this rather than pushboolean so that we keep the full int value - setbvalue(info->L->top, READ_VALUE(int32_t)); /* ... bool */ + setbvalue(info->L->top, READ_VALUE(int)); /* ... bool */ eris_incr_top(info->L); eris_checktype(info, -1, LUA_TBOOLEAN); @@ -1183,7 +1217,8 @@ u_boolean(Info *info) { /* ... */ static void p_pointer(Info *info) { /* ... ludata */ WRITE_VALUE((uint8_t)lua_lightuserdatatag(info->L, -1), uint8_t); - WRITE_VALUE((ares_size_t)lua_touserdata(info->L, -1), ares_size_t); + /* A pointer-sized value, not an index, so it stays fixed width. */ + WRITE_VALUE((uint64_t)(uintptr_t)lua_touserdata(info->L, -1), uint64_t); } static void @@ -1193,24 +1228,27 @@ u_pointer(Info *info) { /* ... */ if (tag >= LUTAG_ARES_START) { eris_error(info, "malformed data: invalid lightuserdata tag"); } - void *ptr = (void*)READ_VALUE(ares_size_t); + const uint64_t raw = READ_VALUE(uint64_t); + if (raw > (uint64_t)UINTPTR_MAX) { + eris_error(info, ERIS_ERR_TRUNC_SIZE); + } + void *ptr = (void*)(uintptr_t)raw; lua_pushlightuserdatatagged(info->L, ptr, tag); /* ... ludata */ eris_checktype(info, -1, LUA_TLIGHTUSERDATA); } /** ======================================================================== */ -// TODO: This could probably benefit from varint encoding. static void p_integer(Info *info) { /* ... int */ - WRITE_VALUE(lua_tointeger64(info->L, -1, NULL), int64_t); + WRITE_VALUE(lua_tointeger64(info->L, -1, NULL), ares_sint); } static void u_integer(Info *info) { /* ... */ eris_checkstack(info->L, 1); - lua_pushinteger64(info->L, READ_VALUE(int64_t)); /* ... int */ + lua_pushinteger64(info->L, READ_VALUE(ares_sint)); /* ... int */ eris_checktype(info, -1, LUA_TINTEGER); } @@ -1299,31 +1337,90 @@ u_vector_f64(Info *info) { /* ... */ /** ======================================================================== */ +/* Strings are written as an index into the string table, assigned on first + * sight. The table itself goes out ahead of the root, see p_strtab. */ static void p_string(Info *info) { /* ... str */ - size_t length; - const char *value = lua_tolstring(info->L, -1, &length); - WRITE_VALUE(length, ares_size_t); - WRITE_RAW(value, length); + eris_checkstack(info->L, 2); + lua_pushvalue(info->L, -1); /* ... str str */ + lua_rawget(info->L, STRTIDX); /* ... str idx? */ + int index; + if (lua_isnil(info->L, -1)) { /* ... str nil */ + lua_pop(info->L, 1); /* ... str */ + index = ++info->u.pi.strcount; + lua_pushvalue(info->L, -1); /* ... str str */ + lua_pushinteger(info->L, index); /* ... str str idx */ + lua_rawset(info->L, STRTIDX); /* ... str */ + lua_pushvalue(info->L, -1); /* ... str str */ + lua_rawseti(info->L, STRTIDX, index); /* ... str */ + } + else { /* ... str idx */ + index = lua_tointeger(info->L, -1); + lua_pop(info->L, 1); /* ... str */ + } + WRITE_VALUE((ares_size_t)index, ares_size_t); } static void u_string(Info *info) { /* ... */ - eris_checkstack(info->L, 2); - { - /* TODO Can we avoid this copy somehow? (Without it getting too nasty) */ - const size_t length = (size_t)READ_VALUE(ares_size_t); - VALIDATE_SIZE(length); - char *value = (char*)lua_newuserdata(info->L, length * sizeof(char)); /* ... tmp */ - READ_RAW(value, length); - lua_pushlstring(info->L, value, length); /* ... tmp str */ - lua_replace(info->L, -2); /* ... str */ + eris_checkstack(info->L, 1); + const ares_size_t index = READ_VALUE(ares_size_t); + if (index < 1 || index > (ares_size_t)info->u.upi.strtabCount) { + eris_error(info, ERIS_ERR_STRIDX, (unsigned long long)index, info->u.upi.strtabCount); } - registerobject(info); + lua_rawgeti(info->L, STRTIDX, (int)index); /* ... str */ eris_checktype(info, -1, LUA_TSTRING); } +/* The string table: every string the root reached, in first-sight order, each + * with the memcat it was allocated under. Written after the root has been + * serialized to a scratch buffer, placed before it in the stream. */ +static void +p_strtab(Info *info) { /* perms reftbl strtab ... */ + eris_checkstack(info->L, 1); + RecordWriter rec(info); + const int count = info->u.pi.strcount; + WRITE_VALUE((ares_size_t)count, ares_size_t); + for (int i = 1; i <= count; ++i) { + lua_rawgeti(info->L, STRTIDX, i); /* perms reftbl strtab ... str */ + const TValue *tv = luaA_toobject(info->L, -1); + eris_assert(ttisstring(tv)); + WRITE_VALUE(gcvalue(tv)->gch.memcat, uint8_t); + size_t length; + const char *value = lua_tolstring(info->L, -1, &length); + WRITE_VALUE(length, ares_size_t); + WRITE_RAW(value, length); + lua_pop(info->L, 1); /* perms reftbl strtab ... */ + } + BEGIN_TRAILER(); + rec.close(); +} + +static void +u_strtab(Info *info) { /* perms reftbl strtab ... */ + eris_checkstack(info->L, 2); + RecordReader rec(info); + const ares_size_t count = READ_VALUE(ares_size_t); + /* Every entry is at least its memcat byte. */ + if (count > RECORD_REMAINING()) { + eris_error(info, "malformed data: string table count exceeds stream data"); + } + for (ares_size_t i = 1; i <= count; ++i) { + const uint8_t memcat = READ_VALUE(uint8_t); + MemcatGuard guard(info->L, memcat); + const size_t length = (size_t)READ_VALUE(ares_size_t); + VALIDATE_SIZE(length); + /* The stream is already in memory, so the string is interned straight + * out of it. */ + lua_pushlstring(info->L, info->u.upi.data + info->u.upi.pos, length); + /* perms reftbl strtab ... str */ + info->u.upi.pos += length; + lua_rawseti(info->L, STRTIDX, (int)i); /* perms reftbl strtab ... */ + } + info->u.upi.strtabCount = (int)count; +} + /** ======================================================================== */ @@ -1469,7 +1566,7 @@ static void p_table(Info *info) { /* ... tbl */ } p_metatable(info); - BEGIN_RAW_APPENDS(); + BEGIN_TRAILER(); } static void u_table(Info *info) { /* ... */ @@ -1690,7 +1787,7 @@ static void p_userdata(Info *info) { /* ... udata case UTAG_DETECTED_EVENT: { const auto *detected_event = (lua_DetectedEvent*)value; - WRITE_VALUE(detected_event->index, int32_t); + WRITE_VALUE((int)detected_event->index, int); WRITE_VALUE(detected_event->valid, uint8_t); WRITE_VALUE(detected_event->can_change_damage, uint8_t); break; @@ -1735,7 +1832,7 @@ static void p_userdata(Info *info) { /* ... udata } p_metatable(info); /* ... udata */ eris_assert(top == lua_gettop(info->L)); - BEGIN_RAW_APPENDS(); + BEGIN_TRAILER(); } static void u_userdata(Info *info) { /* ... */ @@ -1770,10 +1867,6 @@ static void u_userdata(Info *info) { /* ... */ } case UTAG_UUID: { - // Because we have an inner, wrapped string reference we need to reserve - // the idx for the outer UUID first, since we saw it first. - int reference = allocate_ref_idx(info); - // Deserialize the wrapped string unpersist(info); /* ... str */ eris_checktype(info, -1, LUA_TSTRING); @@ -1781,10 +1874,7 @@ static void u_userdata(Info *info) { /* ... */ auto *str_val = luaL_checklstring(info->L, -1, &len); luaSL_pushuuidlstring(info->L, str_val, len); /* ... str uuid */ lua_replace(info->L, -2); /* ... uuid */ - - // Manually put the UUID in the references table at the correct reference index - lua_pushvalue(info->L, -1); /* perms reftbl ... obj obj */ - lua_rawseti(info->L, REFTIDX, reference); /* perms reftbl ... obj */ + registerobject(info); break; } case UTAG_DETECTED_EVENT: @@ -1796,7 +1886,7 @@ static void u_userdata(Info *info) { /* ... */ ); /* ... udata */ memset(detected_event, 0, sizeof(lua_DetectedEvent)); - detected_event->index = READ_VALUE(int32_t); + detected_event->index = READ_VALUE(int); detected_event->valid = (bool)READ_VALUE(uint8_t); detected_event->can_change_damage = (bool)READ_VALUE(uint8_t); registerobject(info); @@ -2022,12 +2112,12 @@ p_proto(Info *info) { /* ... proto */ // poppath(info); // } // poppath(info); - WRITE_VALUE(p->sizeyieldpoints, int32_t); + WRITE_VALUE(p->sizeyieldpoints, int); for (i=0; isizeyieldpoints; ++i) { - WRITE_VALUE(p->yieldpoints[i], int32_t); + WRITE_VALUE(p->yieldpoints[i], int); } - BEGIN_RAW_APPENDS(); + BEGIN_TRAILER(); } static void @@ -2179,7 +2269,7 @@ u_proto(Info *info) { /* ... proto */ SAFE_ALLOC_VECTOR(info->L, p->yieldpoints, 0, int, p->sizeyieldpoints, int); for (i=0; isizeyieldpoints; ++i) { - p->yieldpoints[i] = READ_VALUE(int32_t); + p->yieldpoints[i] = READ_VALUE(int); if (p->yieldpoints[i] < 0 || p->yieldpoints[i] >= p->sizecode) { eris_error(info, "malformed data: invalid yield point"); } @@ -2198,7 +2288,7 @@ u_proto(Info *info) { /* ... proto */ static void p_upval(Info *info) { /* ... obj */ persist(info); /* ... obj */ - BEGIN_RAW_APPENDS(); + BEGIN_TRAILER(); } static void @@ -2363,7 +2453,7 @@ p_closure(Info *info) { /* perms reftbl ... func */ } poppath(info); } - BEGIN_RAW_APPENDS(); + BEGIN_TRAILER(); } static void @@ -2610,7 +2700,7 @@ p_thread(Info *info) { /* ... thread */ eris_assert(lua_type(info->L, -1) == LUA_TTHREAD); /* Persist the stack. Save the total size and used space first. */ - WRITE_VALUE((uint32_t)thread->stacksize, uint32_t); + WRITE_VALUE((ares_size_t)thread->stacksize, ares_size_t); WRITE_VALUE(total, ares_size_t); /* The Lua stack looks like this: @@ -2679,8 +2769,8 @@ p_thread(Info *info) { /* ... thread */ // written above. The capacity has to survive the round trip on its own: the // VM assumes it never drops below BASIC_CI_SIZE, and lua_resetthread shrinks // to that rather than growing back up to it. - WRITE_VALUE((uint32_t)thread->size_ci, uint32_t); - WRITE_VALUE((uint32_t)num_cis, uint32_t); + WRITE_VALUE((ares_size_t)thread->size_ci, ares_size_t); + WRITE_VALUE((ares_size_t)num_cis, ares_size_t); for (int i=0; i < num_cis; ++i) { pushpath(info, "[%d]", level++); ci = thread->base_ci + i; @@ -2691,7 +2781,7 @@ p_thread(Info *info) { /* ... thread */ /* CallInfo.nresults and CallInfo.flags are only set for actual functions. * base_ci[0] is never set up by luau_precall, so reading them there would * put uninitialised bytes on the wire. */ - WRITE_VALUE(ttisfunction(ci->func) ? ci->nresults : 0, int); + WRITE_VALUE(ttisfunction(ci->func) ? ci->nresults : 0, ares_sint); WRITE_VALUE(ttisfunction(ci->func) ? ci->flags : 0, uint8_t); if (eris_isLua(ci)) { @@ -2736,7 +2826,7 @@ p_thread(Info *info) { /* ... thread */ } } - WRITE_VALUE(yield_point, int); + WRITE_VALUE(yield_point, ares_sint); WRITE_VALUE((int)pc_offset, int); } else if (ttisfunction(ci->func)) { @@ -2747,7 +2837,7 @@ p_thread(Info *info) { /* ... thread */ // Protected call's error function: 1-based index relative to ci->base, 0 for none. // C frames only, Lua frames use the same union member as savedpc. - WRITE_VALUE(ci->errfunc, int32_t); + WRITE_VALUE(ci->errfunc, int); eris_ifassert(const int pre_closure_top = lua_gettop(info->L)); // Copy the original closure from ci->func to info->L's stack for serialization. @@ -2763,7 +2853,7 @@ p_thread(Info *info) { /* ... thread */ WRITE_VALUE(ERIS_CI_KIND_NONE, uint8_t); eris_assert(ttisnil(ci->func)); } - BEGIN_RAW_APPENDS(); + BEGIN_TRAILER(); ci_rec.close(); poppath(info); } @@ -2796,7 +2886,7 @@ p_thread(Info *info) { /* ... thread */ lua_pop(info->L, 1); /* ... thread */ poppath(info); eris_assert(lua_type(info->L, -1) == LUA_TTHREAD); - BEGIN_RAW_APPENDS(); + BEGIN_TRAILER(); } /* Used in u_thread to validate read stack positions. */ @@ -2861,7 +2951,7 @@ u_thread(Info *info) { /* ... */ /* Unpersist the stack. Read size first and adjust accordingly. * stack_size is the full stacksize (including EXTRA_STACK) as written by p_thread. * luaD_reallocstack expects the usable size (without EXTRA_STACK) and adds it back. */ - uint32_t stack_size = READ_VALUE(uint32_t); + ares_size_t stack_size = READ_VALUE(ares_size_t); ares_size_t total = READ_VALUE(ares_size_t); if (stack_size < LUA_MINSTACK + EXTRA_STACK || stack_size > kMaxStackSize) { eris_error(info, "malformed data: invalid stack size"); @@ -2930,8 +3020,8 @@ u_thread(Info *info) { /* ... */ pushpath(info, ".callinfo"); UNLOCK(thread); - uint32_t size_ci = READ_VALUE(uint32_t); - uint32_t num_cis = READ_VALUE(uint32_t); + ares_size_t size_ci = READ_VALUE(ares_size_t); + ares_size_t num_cis = READ_VALUE(ares_size_t); if (num_cis < 1 || num_cis > LUAI_MAXCALLS) { eris_error(info, "malformed data: invalid call info count"); } @@ -2941,7 +3031,7 @@ u_thread(Info *info) { /* ... */ // bytes of its own in the stream. There's no floor: hardstacktests builds // shrink the array to exactly what's in use on every GC pass. if (size_ci < num_cis || - size_ci > (uint32_t)(LUAI_MAXCALLS + (LUAI_MAXCALLS >> 3))) { + size_ci > (ares_size_t)(LUAI_MAXCALLS + (LUAI_MAXCALLS >> 3))) { eris_error(info, "malformed data: invalid call info capacity"); } // conservative: each CI needs at least a few bytes @@ -2949,7 +3039,7 @@ u_thread(Info *info) { /* ... */ luaD_reallocCI(thread, (int)size_ci); thread->ci = thread->base_ci; level = 0; - for (uint32_t ci_idx=0; ci_idxci->func, thread->top - 1); u_stackidx(info, thread, &thread->ci->top, thread->stack_last); u_stackidx(info, thread, &thread->ci->base, thread->top); - thread->ci->nresults = READ_VALUE(int32_t); + thread->ci->nresults = READ_VALUE(ares_sint32); thread->ci->flags = READ_VALUE(uint8_t); // We'll set these later if relevant for the CI type. thread->ci->p = nullptr; @@ -2986,7 +3076,7 @@ u_thread(Info *info) { /* ... */ if (ci_kind == ERIS_CI_KIND_LUA) { Closure *lcl = eris_ci_func(thread->ci); - int yield_point = READ_VALUE(int); + int yield_point = READ_VALUE(ares_sint32); int real_pc = READ_VALUE(int); int pc_offset; @@ -3020,7 +3110,7 @@ u_thread(Info *info) { /* ... */ } } else if (ci_kind == ERIS_CI_KIND_C) { // resume_handle() resolves this as ci->base + (errfunc - 1). - int errfunc = READ_VALUE(int32_t); + int errfunc = READ_VALUE(int); if (errfunc != 0) { // Bounds-check the index before the pointer arithmetic can overflow if (errfunc < 1 || errfunc > (int)(thread->top - thread->ci->base) + 1) { @@ -3317,12 +3407,14 @@ persist(Info *info) { /* perms reftbl ... obj */ WRITE_VALUE(type, uint8_t); } /* Write simple values directly, because writing a "reference" would take up - * just as much space and we can save ourselves work this way. */ + * just as much space and we can save ourselves work this way. Strings have + * their own table and never take a reference number either. */ else if (type == ARES_T_BOOLEAN || type == ARES_T_LIGHTUSERDATA || type == ARES_T_NUMBER || type == ARES_T_INTEGER || - type == ARES_T_VECTOR) + type == ARES_T_VECTOR || + type == ARES_T_STRING) { persist_typed(info, type); /* perms reftbl ... obj */ } @@ -3470,6 +3562,8 @@ unpersist(Info *info) { /* perms reftbl ... */ ** ============================================================================ */ +/* Written after the root has gone to its scratch buffer, so the counts it + * carries are final. */ static void p_header(Info *info) { WRITE_RAW(kHeader, HEADER_LENGTH); @@ -3478,29 +3572,13 @@ p_header(Info *info) { RecordWriter rec(info); WRITE_VALUE(sizeof(lua_Number), uint8_t); WRITE_VALUE(kHeaderNumber, lua_Number); - WRITE_VALUE(sizeof(int), uint8_t); - WRITE_VALUE(sizeof(size_t), uint8_t); WRITE_VALUE(LUA_VECTOR_SIZE, uint8_t); - /* Final reference count, unknown until the root has been written. Reserved - * here and filled in by p_header_refcount. */ - info->u.pi.refcountPos = std::streamoff(info->u.pi.writer->tellp()); - WRITE_VALUE(0, uint32_t); - BEGIN_RAW_APPENDS(); - rec.close(); -} - -/* Patches the reference count the header reserved, so a reader that lost or - * gained a reference somewhere refuses the stream. */ -static void -p_header_refcount(Info *info) { - std::ostream *writer = info->u.pi.writer; - std::streampos end = writer->tellp(); - writer->seekp(info->u.pi.refcountPos); + /* So a reader that lost or gained a reference somewhere refuses the stream. */ WRITE_VALUE((uint32_t)info->refcount, uint32_t); - writer->seekp(end); - if (writer->fail()) { - eris_error(info, ERIS_ERR_WRITE); - } + WRITE_VALUE(kAresFeaturesSupported, uint64_t); + WRITE_VALUE(info->u.pi.requiredFeatures, uint64_t); + BEGIN_TRAILER(); + rec.close(); } static void @@ -3529,10 +3607,20 @@ u_header(Info *info) { if (READ_VALUE(lua_Number) != kHeaderNumber) { eris_error(info, "incompatible floating point representation"); } - info->u.upi.sizeof_int = READ_VALUE(uint8_t); - info->u.upi.sizeof_size_t = READ_VALUE(uint8_t); info->u.upi.vector_components = READ_VALUE(uint8_t); info->u.upi.expectedRefcount = READ_VALUE(uint32_t); + info->u.upi.presentFeatures = READ_VALUE(uint64_t); + const uint64_t required_features = READ_VALUE(uint64_t); + /* Present-but-unknown features are skipped through their record lengths; + * only required ones refuse the stream. */ + if (required_features & ~kAresFeaturesSupported) { + eris_error(info, ERIS_ERR_FEATURE, info->u.upi.major, info->u.upi.minor, + (unsigned long long)(required_features & ~kAresFeaturesSupported)); + } + if (required_features & ~info->u.upi.presentFeatures) { + eris_error(info, ERIS_ERR_FEATURE_SET, (unsigned long long)required_features, + (unsigned long long)info->u.upi.presentFeatures); + } } /* Once the root has been read, the stream must have assigned exactly the @@ -3899,9 +3987,12 @@ unchecked_persist(lua_State *L, std::ostream *writer) { info.u.pi.writeDebugInfo = kWriteDebugInformation; info.u.pi.persistingCFunc = false; info.u.pi.testPadding = 0; - info.u.pi.rawAppendsRefcount = -1; + info.u.pi.trailerRefcount = -1; + info.u.pi.trailerRequired = false; + info.u.pi.requiredFeatures = 0; + info.u.pi.strcount = 0; - eris_checkstack(L, 6); + eris_checkstack(L, 7); // Record lengths are back-patched if (writer->tellp() == std::streampos(-1)) { @@ -3931,9 +4022,11 @@ unchecked_persist(lua_State *L, std::ostream *writer) { lua_newtable(L); /* perms buff rootobj reftbl */ lua_insert(L, REFTIDX); /* perms reftbl buff rootobj */ + lua_newtable(L); /* perms reftbl buff rootobj strtab */ + lua_insert(L, STRTIDX); /* perms reftbl strtab buff rootobj */ if (info.generatePath) { - lua_newtable(L); /* perms reftbl buff rootobj path */ - lua_insert(L, PATHIDX); /* perms reftbl buff path rootobj */ + lua_newtable(L); /* perms reftbl strtab buff rootobj path */ + lua_insert(L, PATHIDX); /* perms reftbl strtab buff path rootobj */ pushpath(&info, "root"); } @@ -3942,23 +4035,37 @@ unchecked_persist(lua_State *L, std::ostream *writer) { lua_replace(L, PERMIDX); /* Populate perms table with Lua internals. */ - lua_pushvalue(L, PERMIDX); /* perms reftbl buff path? rootobj perms */ + lua_pushvalue(L, PERMIDX); /* perms reftbl strtab buff path? rootobj perms */ eris_populate_perms(L, false); - lua_pop(L, 1); /* perms reftbl buff path? rootobj */ + lua_pop(L, 1); /* perms reftbl strtab buff path? rootobj */ int pre_pad_top = lua_gettop(L); lua_hardenstack(L, 1); - p_header(&info); - persist(&info); /* perms reftbl buff path? rootobj */ - // go back and backpatch the header to include the refcount - p_header_refcount(&info); + /* The string table and the header's counts are only known once the root has + * been walked, and both precede it in the stream, so the root goes to a + * scratch buffer first. */ + { + std::ostringstream scratch; + info.u.pi.writer = &scratch; + persist(&info); /* perms reftbl strtab buff path? rootobj */ + info.u.pi.writer = writer; + + p_header(&info); + p_strtab(&info); + const std::string body = scratch.str(); + writer->write(body.data(), body.size()); + if (writer->fail()) { + eris_error(&info, ERIS_ERR_WRITE); + } + } lua_settop(L, pre_pad_top); - if (info.generatePath) { /* perms reftbl buff path rootobj */ - lua_remove(L, PATHIDX); /* perms reftbl buff rootobj */ - } /* perms reftbl buff rootobj */ + if (info.generatePath) { /* perms reftbl strtab buff path rootobj */ + lua_remove(L, PATHIDX); /* perms reftbl strtab buff rootobj */ + } /* perms reftbl strtab buff rootobj */ + lua_remove(L, STRTIDX); /* perms reftbl buff rootobj */ lua_remove(L, REFTIDX); /* perms buff rootobj */ eris_assert(lua_gettop(L) == old_top); } @@ -3981,8 +4088,9 @@ unchecked_unpersist(lua_State *L, const char *data, size_t size, void *threaddat info.u.upi.pos = 0; info.u.upi.record_end = size; info.u.upi.threaddata = threaddata; + info.u.upi.strtabCount = 0; - eris_checkstack(L, 6); + eris_checkstack(L, 7); if (get_setting(L, (void*)&kSettingMaxComplexity)) { /* perms buff rootobj value */ @@ -3997,13 +4105,15 @@ unchecked_unpersist(lua_State *L, const char *data, size_t size, void *threaddat lua_newtable(L); /* perms str? reftbl */ lua_insert(L, REFTIDX); /* perms reftbl str? */ + lua_newtable(L); /* perms reftbl str? strtab */ + lua_insert(L, STRTIDX); /* perms reftbl strtab str? */ if (info.generatePath) { - /* Make sure the path is always at index 4, so that it's the same for + /* Make sure the path is always at PATHIDX, so that it's the same for * persist and unpersist. */ - lua_pushnil(L); /* perms reftbl str? nil */ - lua_insert(L, BUFFIDX); /* perms reftbl nil str? */ - lua_newtable(L); /* perms reftbl nil str? path */ - lua_insert(L, PATHIDX); /* perms reftbl nil path str? */ + lua_pushnil(L); /* perms reftbl strtab str? nil */ + lua_insert(L, BUFFIDX); /* perms reftbl strtab nil str? */ + lua_newtable(L); /* perms reftbl strtab nil str? path */ + lua_insert(L, PATHIDX); /* perms reftbl strtab nil path str? */ pushpath(&info, "root"); } @@ -4012,15 +4122,16 @@ unchecked_unpersist(lua_State *L, const char *data, size_t size, void *threaddat lua_replace(L, PERMIDX); /* Populate perms table with Lua internals. */ - lua_pushvalue(L, PERMIDX); /* perms reftbl nil? path? str? perms */ + lua_pushvalue(L, PERMIDX); /* perms reftbl strtab nil? path? str? perms */ eris_populate_perms(L, true); - lua_pop(L, 1); /* perms reftbl nil? path? str? */ + lua_pop(L, 1); /* perms reftbl strtab nil? path? str? */ int pre_pad_top = lua_gettop(L); lua_hardenstack(L, 1); u_header(&info); - unpersist(&info); /* perms reftbl nil? path? str? rootobj */ + u_strtab(&info); + unpersist(&info); /* perms reftbl strtab nil? path? str? rootobj */ u_finish(&info); /* Get rid of any padding we might have added, leave just the result */ @@ -4029,10 +4140,11 @@ unchecked_unpersist(lua_State *L, const char *data, size_t size, void *threaddat lua_settop(L, pre_pad_top + 1); } - if (info.generatePath) { /* perms reftbl nil path str? rootobj */ - lua_remove(L, PATHIDX); /* perms reftbl nil str? rootobj */ - lua_remove(L, BUFFIDX); /* perms reftbl str? rootobj */ - } /* perms reftbl str? rootobj */ + if (info.generatePath) { /* perms reftbl strtab nil path str? rootobj */ + lua_remove(L, PATHIDX); /* perms reftbl strtab nil str? rootobj */ + lua_remove(L, BUFFIDX); /* perms reftbl strtab str? rootobj */ + } /* perms reftbl strtab str? rootobj */ + lua_remove(L, STRTIDX); /* perms reftbl str? rootobj */ lua_remove(L, REFTIDX); /* perms str? rootobj */ eris_assert(lua_gettop(L) == old_top + 1); } diff --git a/tests/SLExecutor.test.cpp b/tests/SLExecutor.test.cpp index 5cc0ab5e..3afbec8c 100644 --- a/tests/SLExecutor.test.cpp +++ b/tests/SLExecutor.test.cpp @@ -1710,7 +1710,7 @@ TEST_CASE_FIXTURE(SLuaFixture, "SLExecutor unserializable global is refused with // Size of the payload a `counter = 1` script serializes to. Update it when the // wire format moves; a change nobody meant to make is the thing worth catching. -constexpr size_t kExpectedDonorPayloadSize = 634; +constexpr size_t kExpectedDonorPayloadSize = 492; TEST_CASE_FIXTURE(SLuaFixture, "SLExecutor invalid restore") { @@ -1770,20 +1770,30 @@ TEST_CASE_FIXTURE(SLuaFixture, "SLExecutor invalid restore") bad_magic[0] = 'B'; CHECK_FALSE(second.exec.restoreState(bad_magic.data(), bad_magic.size())); - // The core section's major follows the magic, the class's own follows - // the class tag + // Each fingerprint is tag, major, minor, then a present and a required + // feature mask, so the class fingerprint starts at 28 std::string bad_core_version = payload; bad_core_version[4] = (char)(kScriptStateFingerprint.major + 99); CHECK_FALSE(second.exec.restoreState(bad_core_version.data(), bad_core_version.size())); std::string bad_class_tag = payload; - bad_class_tag[12] = 'B'; + bad_class_tag[28] = 'B'; CHECK_FALSE(second.exec.restoreState(bad_class_tag.data(), bad_class_tag.size())); std::string bad_class_version = payload; - bad_class_version[16] = (char)(kScriptStateFingerprint.major + 99); + bad_class_version[32] = (char)(kScriptStateFingerprint.major + 99); CHECK_FALSE(second.exec.restoreState(bad_class_version.data(), bad_class_version.size())); + // A required feature this build doesn't know is refused, for either + // section + std::string core_requires = payload; + core_requires[20] = 1; + CHECK_FALSE(second.exec.restoreState(core_requires.data(), core_requires.size())); + + std::string class_requires = payload; + class_requires[48] = 1; + CHECK_FALSE(second.exec.restoreState(class_requires.data(), class_requires.size())); + CHECK_FALSE(second.exec.restoreState("", 0)); for (size_t len : {size_t(4), size_t(8), size_t(16), payload.size() / 2, payload.size() - 1}) CHECK_FALSE(second.exec.restoreState(payload.data(), len)); @@ -1794,6 +1804,17 @@ TEST_CASE_FIXTURE(SLuaFixture, "SLExecutor invalid restore") CHECK(second.exec.getFaultKind() == FaultKind::None); } + SUBCASE("unknown present features are skipped") + { + // A feature that is present but not required belongs to a droppable + // field, and an older build loads the payload without it + std::string core_present = payload; + core_present[12] = 1; + core_present[40] = 1; + restore(second.exec, core_present); + CHECK(second.exec.getFaultKind() == FaultKind::None); + } + SUBCASE("restore directly after instantiate succeeds") { // Restoring persisted state is the common production start path, so diff --git a/tests/SLGoldenFixtures.test.cpp b/tests/SLGoldenFixtures.test.cpp index ce3fdfab..5c21e983 100644 --- a/tests/SLGoldenFixtures.test.cpp +++ b/tests/SLGoldenFixtures.test.cpp @@ -8,6 +8,14 @@ // asset and state under the current format versions. The load case restores // every committed state this build claims to read and runs its scenario's // verify against it, so a state written by an older build has to keep working. +// +// TODO: the other direction, an older reader loading states this build wrote, +// can only be checked by running the previous release's test binary against +// this tree's fixtures. That needs a runtime override for the fixture dir, +// unknown scenarios skipped rather than failed, and a way to mark a fixture as +// expected to be refused (a populated require_feature() field). +// Perhaps the mere presence of an unknown require_feature() field would be enough, +// given that we can parse those out ourselves. #include "SLExecutorFixture.h" #include "Luau/FileUtils.h" diff --git a/tests/conformance/ares.lua b/tests/conformance/ares.lua index 78f52226..cac32c49 100644 --- a/tests/conformance/ares.lua +++ b/tests/conformance/ares.lua @@ -227,6 +227,17 @@ assert(ares.unpersist(ares.persist(userdata)) ~= nil) assert_round_trips(vector(1, 2, 3)) +-- Strings live in a table ahead of the root, so a repeat costs an index, not a copy +local long_str = string.rep("abcdefgh", 64) +local once = #ares.persist({long_str}) +local four_times = #ares.persist({long_str, long_str, long_str, long_str}) +assert(four_times - once < #long_str) + +-- The same string as key and value resolves to the same table entry +local str_key_val = round_trip({[long_str] = long_str, other = "other"}) +assert(str_key_val[long_str] == long_str) +assert(str_key_val.other == "other") + -- Integer values survive round-trips at full 64-bit width. -- The integer library is only registered when the integer fflags are enabled, -- and this file must still parse without them, so no integer literals here. diff --git a/tests/conformance/ares_errors.lua b/tests/conformance/ares_errors.lua index 2cf60275..e6ca3d8a 100644 --- a/tests/conformance/ares_errors.lua +++ b/tests/conformance/ares_errors.lua @@ -11,8 +11,34 @@ local function assert_decode_fails(data) end +local function assert_decode_fails_with(data, pattern) + local success, err = pcall(function() ares.unpersist(uperms, data) end) + print(err) + assert(not success and string.find(err, pattern) ~= nil) +end + assert_decode_fails(tab_ser:sub(#tab_ser - 2)) assert_decode_fails(tab_ser:gsub("ARES", "ARTS")) +-- The header is "ARES", major, minor and the record length, then number size, +-- test number, vector size and the reference count ahead of the two feature +-- masks. Bit 63 will never be a feature this build knows. Luau numbers are +-- doubles, so the u64 is packed as two halves. +local features_present_pos = 4 + 4 + 4 + 4 + 1 + 8 + 1 + 4 +local features_required_pos = features_present_pos + 8 +local unknown_bit = string.pack("