diff --git a/JSTests/stress/get-by-val-buffered-identifier-lifetime.js b/JSTests/stress/get-by-val-buffered-identifier-lifetime.js new file mode 100644 index 000000000000..866ee46fe219 --- /dev/null +++ b/JSTests/stress/get-by-val-buffered-identifier-lifetime.js @@ -0,0 +1,81 @@ +//@ requireOptions("--useZombieMode=1", "--repatchBufferingCountdown=10") + +// A by-val inline cache buffers (structure, property name) pairs while it waits for enough cases to +// generate a stub, and every later miss on the site compares against all of them. The subscripts +// here are fresh strings that nothing else refers to once the call returns, so the buffered pairs +// have to keep working across collections that sweep those strings. + +function shouldBe(actual, expected) { + if (actual !== expected) + throw new Error("bad value: expected " + expected + " but got " + actual); +} + +var prefix = "ta"; +var suffix = "rgetx"; + +// A rope, which toPropertyKey resolves in place, so each call caches a different string cell. +function freshKey() { + return prefix + suffix.substring(0, 4); +} + +function makeSite() { + return new Function("o", "k", "return o[k];"); +} + +function makeBase(shape, value) { + var o = {}; + for (var i = 0; i <= shape; ++i) + o["pad" + shape + "_" + i] = i; + o.target = value; + o[0] = value + 1000; + return o; +} + +var siteCount = 16; +var shapeCount = 6; +var sites = []; +var bases = []; +for (var s = 0; s < siteCount; ++s) { + sites.push(makeSite()); + var group = []; + for (var shape = 0; shape < shapeCount; ++shape) + group.push(makeBase(shape + s * shapeCount, shape)); + bases.push(group); +} + +function churn() { + var last = ""; + for (var i = 0; i < 20000; ++i) + last = ("a" + i) + ("b" + i); + return last.length; +} + +for (var round = 0; round < 40; ++round) { + for (var s = 0; s < siteCount; ++s) { + var site = sites[s]; + var group = bases[s]; + for (var i = 0; i < 24; ++i) { + var shape = i % shapeCount; + shouldBe(site(group[shape], freshKey()), shape); + } + } + + churn(); + gc(); + churn(); + edenGC(); + + // Integer subscripts compare against every buffered pair on the site. + for (var s = 0; s < siteCount; ++s) { + var group = bases[s]; + for (var i = 0; i < shapeCount; ++i) + shouldBe(sites[s](group[i], 0), i + 1000); + } + + // String subscripts compare against them too, with keys that are fresh cells again. + for (var s = 0; s < siteCount; ++s) { + var group = bases[s]; + for (var i = 0; i < shapeCount; ++i) + shouldBe(sites[s](group[i], freshKey()), i); + } +} diff --git a/JSTests/stress/inline-cache-proxy-and-getter-spill-state.js b/JSTests/stress/inline-cache-proxy-and-getter-spill-state.js new file mode 100644 index 000000000000..d5cd581fb83a --- /dev/null +++ b/JSTests/stress/inline-cache-proxy-and-getter-spill-state.js @@ -0,0 +1,17 @@ +// A stub records one spill state shared by all of its access cases, so every case that calls out +// to JS has to spill the same registers. This inline cache sees both a Proxy and a JS getter. +const target = { p: 1 }; +const proxy = new Proxy(target, { get(t, k) { return t[k]; } }); +const withGetter = {}; +Object.defineProperty(withGetter, "p", { get: function () { return 2; } }); + +function f(o) { return o.p; } +noInline(f); + +globalThis.testLoopCount ??= 1e4; +for (let i = 0; i < testLoopCount; ++i) { + if (f(proxy) !== 1) + throw new Error("proxy load returned the wrong value"); + if (f(withGetter) !== 2) + throw new Error("getter load returned the wrong value"); +} diff --git a/JSTests/stress/instanceof-osr-exit-from-inlined-proxy-get.js b/JSTests/stress/instanceof-osr-exit-from-inlined-proxy-get.js new file mode 100644 index 000000000000..5cda2b97e4ba --- /dev/null +++ b/JSTests/stress/instanceof-osr-exit-from-inlined-proxy-get.js @@ -0,0 +1,36 @@ +//@ runDefault("--forceOSRExitToLLInt=true", "--thresholdForJITAfterWarmUp=10", "--thresholdForOptimizeAfterWarmUp=20") + +// The DFG inlines a proxy's get trap for the Symbol.hasInstance and prototype reads that instanceof +// performs. An OSR exit out of that inlined call returns into the LLInt at op_instanceof's return +// location, which has to finish the operation rather than fall through to the next opcode with the +// destination register still holding whatever was there before. + +function assert(b) { + if (!b) + throw new Error("Bad assertion"); +} + +function test(f) { + for (let i = 0; i < 1000; i++) + f(); +} + +// A pass-through proxy, so this instanceof is genuinely true and the site caches a hit first. +test(function() { + let proxy = new Proxy(function () { }, { }); + assert(new proxy instanceof proxy); +}); + +// The trap hands back a fresh object every time "prototype" is read, so the instance is never an +// instanceof the proxy. +test(function() { + let handler = { + get: function(target, prop) { + if (prop === "prototype") + return { }; + return target[prop]; + } + }; + let proxy = new Proxy(function () { }, handler); + assert(!(new proxy instanceof proxy)); +}); diff --git a/JSTests/stress/instanceof-osr-exit-hasInstance-getter.js b/JSTests/stress/instanceof-osr-exit-hasInstance-getter.js new file mode 100644 index 000000000000..ec2db5d52736 --- /dev/null +++ b/JSTests/stress/instanceof-osr-exit-hasInstance-getter.js @@ -0,0 +1,19 @@ +function F1() { + Object instanceof Proxy; +} +noInline(F1); + +let count = 0; +function f20() { + count++; + OSRExit(); + return () => { }; +} +Object.defineProperty(Proxy, Symbol.hasInstance, { get: f20 }); + +globalThis.testLoopCount ??= 1e4; +for (let i = 0; i < testLoopCount; i++) { + F1(); +} +if (count != testLoopCount) + throw new Error("bad!"); diff --git a/JSTests/stress/instanceof-osr-exit-prototype-getter.js b/JSTests/stress/instanceof-osr-exit-prototype-getter.js new file mode 100644 index 000000000000..40be3364c9c7 --- /dev/null +++ b/JSTests/stress/instanceof-osr-exit-prototype-getter.js @@ -0,0 +1,19 @@ +function F1() { + Object instanceof Proxy; +} +noInline(F1); + +let count = 0; +function f20() { + count++; + OSRExit(); + return f20; +} +Object.defineProperty(Proxy, "prototype", { get: f20 }); + +globalThis.testLoopCount ??= 1e4; +for (let i = 0; i < testLoopCount; i++) { + F1(); +} +if (count != testLoopCount) + throw new Error("bad!"); diff --git a/JSTests/stress/op-catch-restores-metadata-table-register.js b/JSTests/stress/op-catch-restores-metadata-table-register.js new file mode 100644 index 000000000000..9eaa3fcad8ef --- /dev/null +++ b/JSTests/stress/op-catch-restores-metadata-table-register.js @@ -0,0 +1,22 @@ +// A strict-mode indexed store whose Proxy set trap returns false throws out of the by-val inline +// cache. Nothing between the throw and the handler copies the handler frame's callee saves into the +// entry frame buffer, so op_catch restores a metadata table belonging to another CodeBlock and has to +// rematerialize metadataTableRegister. A named store (proxy.x) takes a different path and does not +// reach this, so keep the subscript numeric. + +globalThis.testLoopCount ??= 1e4; + +(function() { + "use strict"; + const proxy = new Proxy({}, { set: function() { return false; } }); + for (let i = 0; i < testLoopCount; ++i) { + let threw = false; + try { + proxy[42] = 40; + } catch (e) { + threw = e instanceof TypeError; + } + if (!threw) + throw new Error("strict indexed store through a rejecting proxy set trap should throw a TypeError"); + } +})(); diff --git a/JSTests/stress/proxy-ic-does-not-clobber-callee-saves.js b/JSTests/stress/proxy-ic-does-not-clobber-callee-saves.js new file mode 100644 index 000000000000..0212d68e74da --- /dev/null +++ b/JSTests/stress/proxy-ic-does-not-clobber-callee-saves.js @@ -0,0 +1,88 @@ +// A proxy inline cache emits a JS call, and values the enclosing JIT holds in callee save registers +// have to survive it. Each function below keeps more live integers than ARMv7 has allocatable GPRs, +// so some of them land in the registers that hold metadataTable and jitData there. + +function shouldBe(actual, expected) { + if (actual !== expected) + throw new Error(`bad value: ${actual}, expected ${expected}`); +} + +const handler = { + get(target, property) { + return target[property]; + }, + set(target, property, value) { + target[property] = value; + return true; + }, +}; + +const loadProxy = new Proxy({ field: 42 }, handler); +const indexedProxy = new Proxy({ 0: 7 }, handler); +const storeProxy = new Proxy({ field: 0 }, handler); + +function load(proxy, x) { + const v0 = (x + 1) | 0; + const v1 = (x + 2) | 0; + const v2 = (x + 3) | 0; + const v3 = (x + 4) | 0; + const v4 = (x + 5) | 0; + const v5 = (x + 6) | 0; + const v6 = (x + 7) | 0; + const v7 = (x + 8) | 0; + const v8 = (x + 9) | 0; + const v9 = (x + 10) | 0; + const v10 = (x + 11) | 0; + const v11 = (x + 12) | 0; + const got = proxy.field; + return (v0 + v1 + v2 + v3 + v4 + v5 + v6 + v7 + v8 + v9 + v10 + v11 + got) | 0; +} +noInline(load); + +function loadByVal(proxy, x, index) { + const v0 = (x + 1) | 0; + const v1 = (x + 2) | 0; + const v2 = (x + 3) | 0; + const v3 = (x + 4) | 0; + const v4 = (x + 5) | 0; + const v5 = (x + 6) | 0; + const v6 = (x + 7) | 0; + const v7 = (x + 8) | 0; + const v8 = (x + 9) | 0; + const v9 = (x + 10) | 0; + const v10 = (x + 11) | 0; + const v11 = (x + 12) | 0; + const got = proxy[index]; + return (v0 + v1 + v2 + v3 + v4 + v5 + v6 + v7 + v8 + v9 + v10 + v11 + got) | 0; +} +noInline(loadByVal); + +function store(proxy, x) { + const v0 = (x + 1) | 0; + const v1 = (x + 2) | 0; + const v2 = (x + 3) | 0; + const v3 = (x + 4) | 0; + const v4 = (x + 5) | 0; + const v5 = (x + 6) | 0; + const v6 = (x + 7) | 0; + const v7 = (x + 8) | 0; + const v8 = (x + 9) | 0; + const v9 = (x + 10) | 0; + const v10 = (x + 11) | 0; + const v11 = (x + 12) | 0; + proxy.field = x; + return (v0 + v1 + v2 + v3 + v4 + v5 + v6 + v7 + v8 + v9 + v10 + v11) | 0; +} +noInline(store); + +// The first call runs before any inline cache exists, so its result is the oracle for every tier. +const expectedLoad = load(loadProxy, 1); +const expectedLoadByVal = loadByVal(indexedProxy, 1, 0); +const expectedStore = store(storeProxy, 1); + +for (let i = 0; i < 5e4; ++i) { + shouldBe(load(loadProxy, 1), expectedLoad); + shouldBe(loadByVal(indexedProxy, 1, 0), expectedLoadByVal); + shouldBe(store(storeProxy, 1), expectedStore); + shouldBe(storeProxy.field, 1); +} diff --git a/JSTests/wasm/stress/armv7-fused-branch-compare-unsigned-ge-zero.js b/JSTests/wasm/stress/armv7-fused-branch-compare-unsigned-ge-zero.js new file mode 100644 index 000000000000..19db69dd731b --- /dev/null +++ b/JSTests/wasm/stress/armv7-fused-branch-compare-unsigned-ge-zero.js @@ -0,0 +1,74 @@ +//@ requireOptions("--useBBQJIT=1", "--useWasmLLInt=0", "--useOMGJIT=0") + +import { instantiate } from "../wabt-wrapper.js" +import * as assert from "../assert.js" + +// BBQ fuses a comparison into a following br_if/if and emits the inverted condition, so +// the jump it hands back is the "branch not taken" edge. i64.ge_u inverts to Below, and +// every unsigned value is >= 0, so these reach the never-taken case of +// MacroAssemblerARMv7::branch64(Below, hi, lo, TrustedImm64(0)). +let wat = ` +(module + (func (export "geUnsignedZeroBrIf") (param $x i64) (result i32) + (block $done + (br_if $done (i64.ge_u (local.get $x) (i64.const 0))) + (return (i32.const 0)) + ) + (i32.const 1) + ) + + ;; Constant on the left: i64.le_u inverts to Above, which emitBranchI64 commutes to Below. + (func (export "zeroLeUnsignedBrIf") (param $x i64) (result i32) + (block $done + (br_if $done (i64.le_u (i64.const 0) (local.get $x))) + (return (i32.const 0)) + ) + (i32.const 1) + ) + + ;; The if-fusion path, which stores the jump in ControlData::m_ifBranch. + (func (export "geUnsignedZeroIf") (param $x i64) (result i32) + (if (result i32) (i64.ge_u (local.get $x) (i64.const 0)) + (then (i32.const 1)) + (else (i32.const 0)) + ) + ) + + ;; i64.lt_u inverts to AboveOrEqual, the always-taken sibling of the case above. + (func (export "ltUnsignedZeroBrIf") (param $x i64) (result i32) + (block $done + (br_if $done (i64.lt_u (local.get $x) (i64.const 0))) + (return (i32.const 0)) + ) + (i32.const 1) + ) + + ;; A non-zero constant is genuinely conditional and misses the compare-with-zero paths. + (func (export "geUnsignedTenBrIf") (param $x i64) (result i32) + (block $done + (br_if $done (i64.ge_u (local.get $x) (i64.const 10))) + (return (i32.const 0)) + ) + (i32.const 1) + ) +) +` + +async function test() { + const instance = await instantiate(wat, {}, {}) + const { geUnsignedZeroBrIf, zeroLeUnsignedBrIf, geUnsignedZeroIf, ltUnsignedZeroBrIf, geUnsignedTenBrIf } = instance.exports + + // Cover values whose high word, low word, or neither is zero. + for (const x of [0n, 1n, 0xffffffffn, 0x100000000n, 0xffffffff00000000n, 0xffffffffffffffffn]) { + assert.eq(geUnsignedZeroBrIf(x), 1) + assert.eq(zeroLeUnsignedBrIf(x), 1) + assert.eq(geUnsignedZeroIf(x), 1) + assert.eq(ltUnsignedZeroBrIf(x), 0) + } + + assert.eq(geUnsignedTenBrIf(9n), 0) + assert.eq(geUnsignedTenBrIf(10n), 1) + assert.eq(geUnsignedTenBrIf(0xffffffffffffffffn), 1) +} + +await assert.asyncTest(test()) diff --git a/JSTests/wasm/stress/array-element-creation.js b/JSTests/wasm/stress/array-element-creation.js index 7f8b8da9a5b5..7387e29be357 100644 --- a/JSTests/wasm/stress/array-element-creation.js +++ b/JSTests/wasm/stress/array-element-creation.js @@ -1,3 +1,4 @@ +//@ skip if $memoryLimited //@ runDefault("--useConcurrentJIT=0") function main() { diff --git a/Source/JavaScriptCore/assembler/ARMv7Assembler.h b/Source/JavaScriptCore/assembler/ARMv7Assembler.h index e9028ce1075c..3b790e68203f 100644 --- a/Source/JavaScriptCore/assembler/ARMv7Assembler.h +++ b/Source/JavaScriptCore/assembler/ARMv7Assembler.h @@ -38,6 +38,26 @@ namespace JSC { +#if !defined(VALIDATE_ARMV7_ASSEMBLER) +#define VALIDATE_ARMV7_ASSEMBLER ASSERT_ENABLED +#endif + +static constexpr bool validateARMv7Assembler = VALIDATE_ARMV7_ASSEMBLER; + +#define ARMV7_ASM_ASSERT(assertion) do { \ + if constexpr (validateARMv7Assembler) { \ + if (UNLIKELY(!(assertion))) { \ + WTFReportAssertionFailure(__FILE__, __LINE__, WTF_PRETTY_FUNCTION, #assertion); \ + CRASH(); \ + } \ + } \ + } while (0) + +#define ARMV7_ASM_ASSERT_UNUSED(variable, assertion) do { \ + UNUSED_PARAM(variable); \ + ARMV7_ASM_ASSERT(assertion); \ + } while (0) + namespace RegisterNames { typedef enum : int8_t { @@ -80,19 +100,19 @@ namespace RegisterNames { inline FPSingleRegisterID asSingle(FPDoubleRegisterID reg) { - ASSERT(reg <= d15); + ARMV7_ASM_ASSERT(reg <= d15); return (FPSingleRegisterID)(reg << 1); } inline FPSingleRegisterID asSingleUpper(FPDoubleRegisterID reg) { - ASSERT(reg <= d15); + ARMV7_ASM_ASSERT(reg <= d15); return (FPSingleRegisterID)((reg << 1) + 1); } inline FPDoubleRegisterID asDouble(FPSingleRegisterID reg) { - ASSERT(!(reg & 1)); + ARMV7_ASM_ASSERT(!(reg & 1)); return (FPDoubleRegisterID)(reg >> 1); } @@ -177,7 +197,7 @@ class ARMThumbImmediate { // Make sure this constructor is only reached with type TypeUInt16; // this extra parameter makes the code a little clearer by making it // explicit at call sites which type is being constructed - ASSERT_UNUSED(type, type == TypeUInt16); + ARMV7_ASM_ASSERT_UNUSED(type, type == TypeUInt16); m_value.asInt = value; } @@ -197,7 +217,7 @@ class ARMThumbImmediate { int32_t leadingZeros = countLeadingZeros(value); // if there were 24 or more leading zeros, then we'd have hit the (value < 256) case. - ASSERT(leadingZeros < 24); + ARMV7_ASM_ASSERT(leadingZeros < 24); // Given a number with bit fields Z:B:C, where count(Z)+count(B)+count(C) == 32, // Z are the bits known zero, B is the 8-bit immediate, C are the bits to check for @@ -264,29 +284,28 @@ class ARMThumbImmediate { return m_type != TypeInvalid; } - uint16_t asUInt16() const { return m_value.asInt; } + uint16_t asUInt16() const { ARMV7_ASM_ASSERT(m_type != TypeInvalid); return m_value.asInt; } - // These methods rely on the format of encoded byte values. - bool isUInt3() { return !(m_value.asInt & 0xfff8); } - bool isUInt4() { return !(m_value.asInt & 0xfff0); } - bool isUInt5() { return !(m_value.asInt & 0xffe0); } - bool isUInt6() { return !(m_value.asInt & 0xffc0); } - bool isUInt7() { return !(m_value.asInt & 0xff80); } - bool isUInt8() { return !(m_value.asInt & 0xff00); } + bool isUInt3() { return (m_type != TypeInvalid) && !(m_value.asInt & 0xfff8); } + bool isUInt4() { return (m_type != TypeInvalid) && !(m_value.asInt & 0xfff0); } + bool isUInt5() { return (m_type != TypeInvalid) && !(m_value.asInt & 0xffe0); } + bool isUInt6() { return (m_type != TypeInvalid) && !(m_value.asInt & 0xffc0); } + bool isUInt7() { return (m_type != TypeInvalid) && !(m_value.asInt & 0xff80); } + bool isUInt8() { return (m_type != TypeInvalid) && !(m_value.asInt & 0xff00); } bool isUInt9() { return (m_type == TypeUInt16) && !(m_value.asInt & 0xfe00); } bool isUInt10() { return (m_type == TypeUInt16) && !(m_value.asInt & 0xfc00); } bool isUInt12() { return (m_type == TypeUInt16) && !(m_value.asInt & 0xf000); } bool isUInt16() { return m_type == TypeUInt16; } - uint8_t getUInt3() { ASSERT(isUInt3()); return m_value.asInt; } - uint8_t getUInt4() { ASSERT(isUInt4()); return m_value.asInt; } - uint8_t getUInt5() { ASSERT(isUInt5()); return m_value.asInt; } - uint8_t getUInt6() { ASSERT(isUInt6()); return m_value.asInt; } - uint8_t getUInt7() { ASSERT(isUInt7()); return m_value.asInt; } - uint8_t getUInt8() { ASSERT(isUInt8()); return m_value.asInt; } - uint16_t getUInt9() { ASSERT(isUInt9()); return m_value.asInt; } - uint16_t getUInt10() { ASSERT(isUInt10()); return m_value.asInt; } - uint16_t getUInt12() { ASSERT(isUInt12()); return m_value.asInt; } - uint16_t getUInt16() { ASSERT(isUInt16()); return m_value.asInt; } + uint8_t getUInt3() { ARMV7_ASM_ASSERT(isUInt3()); return m_value.asInt; } + uint8_t getUInt4() { ARMV7_ASM_ASSERT(isUInt4()); return m_value.asInt; } + uint8_t getUInt5() { ARMV7_ASM_ASSERT(isUInt5()); return m_value.asInt; } + uint8_t getUInt6() { ARMV7_ASM_ASSERT(isUInt6()); return m_value.asInt; } + uint8_t getUInt7() { ARMV7_ASM_ASSERT(isUInt7()); return m_value.asInt; } + uint8_t getUInt8() { ARMV7_ASM_ASSERT(isUInt8()); return m_value.asInt; } + uint16_t getUInt9() { ARMV7_ASM_ASSERT(isUInt9()); return m_value.asInt; } + uint16_t getUInt10() { ARMV7_ASM_ASSERT(isUInt10()); return m_value.asInt; } + uint16_t getUInt12() { ARMV7_ASM_ASSERT(isUInt12()); return m_value.asInt; } + uint16_t getUInt16() { ARMV7_ASM_ASSERT(isUInt16()); return m_value.asInt; } bool isEncodedImm() { return m_type == TypeEncoded; } @@ -316,6 +335,8 @@ class ShiftTypeAndAmount { ShiftTypeAndAmount(ARMShiftType type, unsigned amount) { + ARMV7_ASM_ASSERT(amount < 32); + ARMV7_ASM_ASSERT(type != SRType_ROR || amount); m_u.type = type; m_u.amount = amount & 31; } @@ -365,7 +386,7 @@ class ARMv7Assembler { static ASCIILiteral gprName(RegisterID id) { - ASSERT(id >= firstRegister() && id <= lastRegister()); + ARMV7_ASM_ASSERT(id >= firstRegister() && id <= lastRegister()); static constexpr ASCIILiteral nameForRegister[numberOfRegisters()] = { #define REGISTER_NAME(id, name, r, cs) name, FOR_EACH_GP_REGISTER(REGISTER_NAME) @@ -376,7 +397,7 @@ class ARMv7Assembler { static ASCIILiteral sprName(SPRegisterID id) { - ASSERT(id >= firstSPRegister() && id <= lastSPRegister()); + ARMV7_ASM_ASSERT(id >= firstSPRegister() && id <= lastSPRegister()); static constexpr ASCIILiteral nameForRegister[numberOfSPRegisters()] = { #define REGISTER_NAME(id, name) name, FOR_EACH_SP_REGISTER(REGISTER_NAME) @@ -387,7 +408,7 @@ class ARMv7Assembler { static ASCIILiteral fprName(FPRegisterID id) { - ASSERT(id >= firstFPRegister() && id <= lastFPRegister()); + ARMV7_ASM_ASSERT(id >= firstFPRegister() && id <= lastFPRegister()); static constexpr ASCIILiteral nameForRegister[numberOfFPRegisters()] = { #define REGISTER_NAME(id, name, r, cs) name, FOR_EACH_FP_DOUBLE_REGISTER(REGISTER_NAME) @@ -417,6 +438,11 @@ class ARMv7Assembler { ConditionInvalid } Condition; + static Condition invert(Condition cond) + { + return static_cast(cond ^ 1); + } + #define JUMP_ENUM_WITH_SIZE(index, value) (((value) << 3) | (index)) #define JUMP_ENUM_SIZE(jump) ((jump) >> 3) enum JumpType { JumpFixed = JUMP_ENUM_WITH_SIZE(0, 0), @@ -463,7 +489,7 @@ class ARMv7Assembler { intptr_t to(const ARMv7Assembler*) const { return data.realTypes.m_to; } JumpType type() const { return data.realTypes.m_type; } JumpLinkType linkType() const { return data.realTypes.m_linkType; } - void setLinkType(JumpLinkType linkType) { ASSERT(data.realTypes.m_linkType == LinkInvalid); data.realTypes.m_linkType = linkType; } + void setLinkType(JumpLinkType linkType) { ARMV7_ASM_ASSERT(data.realTypes.m_linkType == LinkInvalid); data.realTypes.m_linkType = linkType; } Condition condition() const { return data.realTypes.m_condition; } bool isThunk() const { return false; } private: @@ -498,7 +524,7 @@ class ARMv7Assembler { static bool isEven(AssemblerLabel &label) { return !(label.offset() & 1); } static void* makeEven(const void* ptr) { - ASSERT(!isEven(ptr)); + ARMV7_ASM_ASSERT(!isEven(ptr)); return reinterpret_cast(reinterpret_cast(ptr) & ~1); } @@ -617,6 +643,8 @@ class ARMv7Assembler { OP_VLDR = 0xED10, OP_VMOV_CtoS = 0xEE00, OP_VMOV_StoC = 0xEE10, + OP_VMLA_T2 = 0xEE00, + OP_VMLS_T2 = 0xEE10, OP_VMUL_T2 = 0xEE20, OP_VADD_T2 = 0xEE30, OP_VSUB_T2 = 0xEE30, @@ -711,6 +739,8 @@ class ARMv7Assembler { OP_VLDRb = 0x0A00, OP_VMOV_IMM_T2b = 0x0A00, OP_VMOV_T2b = 0x0A40, + OP_VMLA_T2b = 0x0A00, + OP_VMLS_T2b = 0x0A00, OP_VMUL_T2b = 0x0A00, OP_FSTSb = 0x0A00, OP_VSTRb = 0x0A00, @@ -772,31 +802,38 @@ class ARMv7Assembler { } static uint8_t ifThenElse(Condition condition, bool inst2if, bool inst3if, bool inst4if) { + ARMV7_ASM_ASSERT(condition < ConditionInvalid); + ARMV7_ASM_ASSERT(condition < ConditionInvalid); int mask = (ifThenElseConditionBit(condition, inst2if) << 3) | (ifThenElseConditionBit(condition, inst3if) << 2) | (ifThenElseConditionBit(condition, inst4if) << 1) | 1; - ASSERT((condition != ConditionAL) || !(mask & (mask - 1))); + ARMV7_ASM_ASSERT((condition != ConditionAL) || !(mask & (mask - 1))); return (condition << 4) | mask; } static uint8_t ifThenElse(Condition condition, bool inst2if, bool inst3if) { + ARMV7_ASM_ASSERT(condition < ConditionInvalid); + ARMV7_ASM_ASSERT(condition < ConditionInvalid); int mask = (ifThenElseConditionBit(condition, inst2if) << 3) | (ifThenElseConditionBit(condition, inst3if) << 2) | 2; - ASSERT((condition != ConditionAL) || !(mask & (mask - 1))); + ARMV7_ASM_ASSERT((condition != ConditionAL) || !(mask & (mask - 1))); return (condition << 4) | mask; } static uint8_t ifThenElse(Condition condition, bool inst2if) { + ARMV7_ASM_ASSERT(condition < ConditionInvalid); + ARMV7_ASM_ASSERT(condition < ConditionInvalid); int mask = (ifThenElseConditionBit(condition, inst2if) << 3) | 4; - ASSERT((condition != ConditionAL) || !(mask & (mask - 1))); + ARMV7_ASM_ASSERT((condition != ConditionAL) || !(mask & (mask - 1))); return (condition << 4) | mask; } static uint8_t ifThenElse(Condition condition) { + ARMV7_ASM_ASSERT(condition < ConditionInvalid); int mask = 8; return (condition << 4) | mask; } @@ -806,10 +843,10 @@ class ARMv7Assembler { void adc(RegisterID rd, RegisterID rn, ARMThumbImmediate imm) { // Rd can only be SP if Rn is also SP. - ASSERT((rd != ARMRegisters::sp) || (rn == ARMRegisters::sp)); - ASSERT(rd != ARMRegisters::pc); - ASSERT(rn != ARMRegisters::pc); - ASSERT(imm.isEncodedImm()); + ARMV7_ASM_ASSERT((rd != ARMRegisters::sp) || (rn == ARMRegisters::sp)); + ARMV7_ASM_ASSERT(rd != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(imm.isEncodedImm()); m_formatter.twoWordOp5i6Imm4Reg4EncodedImm(OP_ADC_imm, rn, rd, imm); } @@ -817,22 +854,21 @@ class ARMv7Assembler { void adc(RegisterID rd, RegisterID rn, RegisterID rm, ShiftTypeAndAmount shift = ShiftTypeAndAmount { }) { // Rd can only be SP if Rn is also SP. - ASSERT(!BadReg(rd)); - ASSERT(!BadReg(rn)); - ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rn)); + ARMV7_ASM_ASSERT(!BadReg(rm)); m_formatter.twoWordOp12Reg4FourFours(OP_ADC_reg_T2, rn, FourFours(shift.hi4(), rd, shift.lo4(), rm)); } void add(RegisterID rd, RegisterID rn, ARMThumbImmediate imm) { // Rd can only be SP if Rn is also SP. - ASSERT((rd != ARMRegisters::sp) || (rn == ARMRegisters::sp)); - ASSERT(rd != ARMRegisters::pc); - ASSERT(rn != ARMRegisters::pc); - ASSERT(imm.isValid()); + ARMV7_ASM_ASSERT((rd != ARMRegisters::sp) || (rn == ARMRegisters::sp)); + ARMV7_ASM_ASSERT(rd != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(imm.isValid()); - if (rn == ARMRegisters::sp && imm.isUInt16()) { - ASSERT(!(imm.getUInt16() & 3)); + if (rn == ARMRegisters::sp && imm.isUInt16() && !(imm.getUInt16() & 3)) { if (!(rd & 8) && imm.isUInt10()) { m_formatter.oneWordOp5Reg3Imm8(OP_ADD_SP_imm_T1, rd, static_cast(imm.getUInt10() >> 2)); return; @@ -853,24 +889,29 @@ class ARMv7Assembler { if (imm.isEncodedImm()) m_formatter.twoWordOp5i6Imm4Reg4EncodedImm(OP_ADD_imm_T3, rn, rd, imm); else { - ASSERT(imm.isUInt12()); + ARMV7_ASM_ASSERT(imm.isUInt12()); m_formatter.twoWordOp5i6Imm4Reg4EncodedImm(OP_ADD_imm_T4, rn, rd, imm); } } ALWAYS_INLINE void add(RegisterID rd, RegisterID rn, RegisterID rm, ShiftTypeAndAmount shift) { - ASSERT((rd != ARMRegisters::sp) || (rn == ARMRegisters::sp)); - ASSERT(rd != ARMRegisters::pc); - ASSERT(rn != ARMRegisters::pc); - ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT((rd != ARMRegisters::sp) || (rn == ARMRegisters::sp)); + ARMV7_ASM_ASSERT(rd != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(!BadReg(rm)); m_formatter.twoWordOp12Reg4FourFours(OP_ADD_reg_T3, rn, FourFours(shift.hi4(), rd, shift.lo4(), rm)); } // NOTE: In an IT block, add doesn't modify the flags register. ALWAYS_INLINE void add(RegisterID rd, RegisterID rn, RegisterID rm) { + ARMV7_ASM_ASSERT(rd != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rm != ARMRegisters::pc); + if (rd == ARMRegisters::sp && rd != rn) { + ARMV7_ASM_ASSERT(rm != rd); mov(rd, rn); rn = rd; } @@ -889,10 +930,10 @@ class ARMv7Assembler { ALWAYS_INLINE void add_S(RegisterID rd, RegisterID rn, ARMThumbImmediate imm) { // Rd can only be SP if Rn is also SP. - ASSERT((rd != ARMRegisters::sp) || (rn == ARMRegisters::sp)); - ASSERT(rd != ARMRegisters::pc); - ASSERT(rn != ARMRegisters::pc); - ASSERT(imm.isEncodedImm()); + ARMV7_ASM_ASSERT((rd != ARMRegisters::sp) || (rn == ARMRegisters::sp)); + ARMV7_ASM_ASSERT(rd != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(imm.isEncodedImm()); if (!((rd | rn) & 8)) { if (imm.isUInt3()) { @@ -910,10 +951,10 @@ class ARMv7Assembler { // Not allowed in an IT (if then) block? ALWAYS_INLINE void add_S(RegisterID rd, RegisterID rn, RegisterID rm, ShiftTypeAndAmount shift) { - ASSERT((rd != ARMRegisters::sp) || (rn == ARMRegisters::sp)); - ASSERT(rd != ARMRegisters::pc); - ASSERT(rn != ARMRegisters::pc); - ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT((rd != ARMRegisters::sp) || (rn == ARMRegisters::sp)); + ARMV7_ASM_ASSERT(rd != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(!BadReg(rm)); m_formatter.twoWordOp12Reg4FourFours(OP_ADD_S_reg_T3, rn, FourFours(shift.hi4(), rd, shift.lo4(), rm)); } @@ -928,17 +969,17 @@ class ARMv7Assembler { ALWAYS_INLINE void ARM_and(RegisterID rd, RegisterID rn, ARMThumbImmediate imm) { - ASSERT(!BadReg(rd)); - ASSERT(!BadReg(rn)); - ASSERT(imm.isEncodedImm()); + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rn)); + ARMV7_ASM_ASSERT(imm.isEncodedImm()); m_formatter.twoWordOp5i6Imm4Reg4EncodedImm(OP_AND_imm_T1, rn, rd, imm); } ALWAYS_INLINE void ARM_and(RegisterID rd, RegisterID rn, RegisterID rm, ShiftTypeAndAmount shift) { - ASSERT(!BadReg(rd)); - ASSERT(!BadReg(rn)); - ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rn)); + ARMV7_ASM_ASSERT(!BadReg(rm)); m_formatter.twoWordOp12Reg4FourFours(OP_AND_reg_T2, rn, FourFours(shift.hi4(), rd, shift.lo4(), rm)); } @@ -954,17 +995,18 @@ class ARMv7Assembler { ALWAYS_INLINE void asr(RegisterID rd, RegisterID rm, int32_t shiftAmount) { - ASSERT(!BadReg(rd)); - ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(shiftAmount >= 1 && shiftAmount <= 32); ShiftTypeAndAmount shift(SRType_ASR, shiftAmount); m_formatter.twoWordOp16FourFours(OP_ASR_imm_T1, FourFours(shift.hi4(), rd, shift.lo4(), rm)); } ALWAYS_INLINE void asr(RegisterID rd, RegisterID rn, RegisterID rm) { - ASSERT(!BadReg(rd)); - ASSERT(!BadReg(rn)); - ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rn)); + ARMV7_ASM_ASSERT(!BadReg(rm)); m_formatter.twoWordOp12Reg4FourFours(OP_ASR_reg_T2, rn, FourFours(0xf, rd, 0, rm)); } @@ -985,7 +1027,7 @@ class ARMv7Assembler { // Only allowed in IT (if then) block if last instruction. ALWAYS_INLINE AssemblerLabel blx(RegisterID rm) { - ASSERT(rm != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rm != ARMRegisters::pc); m_formatter.oneWordOp8RegReg143(OP_BLX, rm, (RegisterID)8); return m_formatter.label(); } @@ -999,9 +1041,9 @@ class ARMv7Assembler { ALWAYS_INLINE void bic(RegisterID rd, RegisterID rn, ARMThumbImmediate imm) { - ASSERT(!BadReg(rd)); - ASSERT(!BadReg(rn)); - ASSERT(imm.isEncodedImm()); + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rn)); + ARMV7_ASM_ASSERT(imm.isEncodedImm()); m_formatter.twoWordOp5i6Imm4Reg4EncodedImm(OP_BIC_imm_T1, rn, rd, imm); } @@ -1025,23 +1067,23 @@ class ARMv7Assembler { ALWAYS_INLINE void clz(RegisterID rd, RegisterID rm) { - ASSERT(!BadReg(rd)); - ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rm)); m_formatter.twoWordOp12Reg4FourFours(OP_CLZ, rm, FourFours(0xf, rd, 8, rm)); } ALWAYS_INLINE void cmn(RegisterID rn, ARMThumbImmediate imm) { - ASSERT(rn != ARMRegisters::pc); - ASSERT(imm.isEncodedImm()); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(imm.isEncodedImm()); m_formatter.twoWordOp5i6Imm4Reg4EncodedImm(OP_CMN_imm, rn, (RegisterID)0xf, imm); } ALWAYS_INLINE void cmp(RegisterID rn, ARMThumbImmediate imm) { - ASSERT(rn != ARMRegisters::pc); - ASSERT(imm.isEncodedImm()); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(imm.isEncodedImm()); if (!(rn & 8) && imm.isUInt8()) m_formatter.oneWordOp5Reg3Imm8(OP_CMP_imm_T1, rn, imm.getUInt8()); @@ -1051,8 +1093,8 @@ class ARMv7Assembler { ALWAYS_INLINE void cmp(RegisterID rn, RegisterID rm, ShiftTypeAndAmount shift) { - ASSERT(rn != ARMRegisters::pc); - ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(!BadReg(rm)); m_formatter.twoWordOp12Reg4FourFours(OP_CMP_reg_T2, rn, FourFours(shift.hi4(), 0xf, shift.lo4(), rm)); } @@ -1067,18 +1109,18 @@ class ARMv7Assembler { // xor is not spelled with an 'e'. :-( ALWAYS_INLINE void eor(RegisterID rd, RegisterID rn, ARMThumbImmediate imm) { - ASSERT(!BadReg(rd)); - ASSERT(!BadReg(rn)); - ASSERT(imm.isEncodedImm()); + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rn)); + ARMV7_ASM_ASSERT(imm.isEncodedImm()); m_formatter.twoWordOp5i6Imm4Reg4EncodedImm(OP_EOR_imm_T1, rn, rd, imm); } // xor is not spelled with an 'e'. :-( ALWAYS_INLINE void eor(RegisterID rd, RegisterID rn, RegisterID rm, ShiftTypeAndAmount shift) { - ASSERT(!BadReg(rd)); - ASSERT(!BadReg(rn)); - ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rn)); + ARMV7_ASM_ASSERT(!BadReg(rm)); m_formatter.twoWordOp12Reg4FourFours(OP_EOR_reg_T2, rn, FourFours(shift.hi4(), rd, shift.lo4(), rm)); } @@ -1116,8 +1158,8 @@ class ARMv7Assembler { // rt == ARMRegisters::pc only allowed if last instruction in IT (if then) block. ALWAYS_INLINE void ldr(RegisterID rt, RegisterID rn, ARMThumbImmediate imm) { - ASSERT(rn != ARMRegisters::pc); // LDR (literal) - ASSERT(imm.isUInt12()); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); // LDR (literal) + ARMV7_ASM_ASSERT(imm.isUInt12()); if (!((rt | rn) & 8) && imm.isUInt7() && !(imm.getUInt7() % 4)) { // We can only use Encoding T1 when imm is a multiple of 4. @@ -1133,16 +1175,18 @@ class ARMv7Assembler { ALWAYS_INLINE void ldrWide8BitImmediate(RegisterID rt, RegisterID rn, uint8_t immediate) { - ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); m_formatter.twoWordOp12Reg4Reg4Imm12(OP_LDR_imm_T3, rn, rt, immediate); } ALWAYS_INLINE void ldrCompact(RegisterID rt, RegisterID rn, ARMThumbImmediate imm) { - ASSERT(rn != ARMRegisters::pc); // LDR (literal) - ASSERT(imm.isUInt7()); - ASSERT(!(imm.getUInt7() % 4)); - ASSERT(!((rt | rn) & 8)); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); // LDR (literal) + ARMV7_ASM_ASSERT(imm.isUInt16()); + ARMV7_ASM_ASSERT(imm.isUInt16()); + ARMV7_ASM_ASSERT(imm.isUInt7()); + ARMV7_ASM_ASSERT(!(imm.getUInt7() % 4)); + ARMV7_ASM_ASSERT(!((rt | rn) & 8)); m_formatter.oneWordOp5Imm5Reg3Reg3(OP_LDR_imm_T1, imm.getUInt7() >> 2, rn, rt); } @@ -1159,17 +1203,19 @@ class ARMv7Assembler { // if (wback) REG[rn] = _tmp ALWAYS_INLINE void ldr(RegisterID rt, RegisterID rn, int offset, bool index, bool wback) { - ASSERT(rt != ARMRegisters::pc); - ASSERT(rn != ARMRegisters::pc); - ASSERT(index || wback); - ASSERT(!wback | (rt != rn)); + ARMV7_ASM_ASSERT(rt != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(index || wback); + ARMV7_ASM_ASSERT(!wback | (rt != rn)); + ARMV7_ASM_ASSERT((offset >= -255) && (offset <= 255)); + ARMV7_ASM_ASSERT((offset >= -255) && (offset <= 255)); bool add = true; if (offset < 0) { add = false; offset = -offset; } - ASSERT((offset & ~0xff) == 0); + ARMV7_ASM_ASSERT((offset & ~0xff) == 0); offset |= (wback << 8); offset |= (add << 9); @@ -1182,9 +1228,9 @@ class ARMv7Assembler { // rt == ARMRegisters::pc only allowed if last instruction in IT (if then) block. ALWAYS_INLINE void ldr(RegisterID rt, RegisterID rn, RegisterID rm, unsigned shift = 0) { - ASSERT(rn != ARMRegisters::pc); // LDR (literal) - ASSERT(!BadReg(rm)); - ASSERT(shift <= 3); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); // LDR (literal) + ARMV7_ASM_ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(shift <= 3); if (!shift && !((rt | rn | rm) & 8)) m_formatter.oneWordOp7Reg3Reg3Reg3(OP_LDR_reg_T1, rm, rn, rt); @@ -1195,8 +1241,9 @@ class ARMv7Assembler { // rt == ARMRegisters::pc only allowed if last instruction in IT (if then) block. ALWAYS_INLINE void ldrh(RegisterID rt, RegisterID rn, ARMThumbImmediate imm) { - ASSERT(rn != ARMRegisters::pc); // LDR (literal) - ASSERT(imm.isUInt12()); + ARMV7_ASM_ASSERT(!BadReg(rt)); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); // LDR (literal) + ARMV7_ASM_ASSERT(imm.isUInt12()); if (!((rt | rn) & 8) && imm.isUInt6() && !(imm.getUInt6() & 1)) m_formatter.oneWordOp5Imm5Reg3Reg3(OP_LDRH_imm_T1, imm.getUInt6() >> 1, rn, rt); @@ -1217,17 +1264,21 @@ class ARMv7Assembler { // if (wback) REG[rn] = _tmp ALWAYS_INLINE void ldrh(RegisterID rt, RegisterID rn, int offset, bool index, bool wback) { - ASSERT(rt != ARMRegisters::pc); - ASSERT(rn != ARMRegisters::pc); - ASSERT(index || wback); - ASSERT(!wback | (rt != rn)); + ARMV7_ASM_ASSERT(rt != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(index || wback); + ARMV7_ASM_ASSERT(!wback | (rt != rn)); + ARMV7_ASM_ASSERT(!BadReg(rt)); + ARMV7_ASM_ASSERT((offset >= -255) && (offset <= 255)); + ARMV7_ASM_ASSERT(!BadReg(rt)); + ARMV7_ASM_ASSERT((offset >= -255) && (offset <= 255)); bool add = true; if (offset < 0) { add = false; offset = -offset; } - ASSERT((offset & ~0xff) == 0); + ARMV7_ASM_ASSERT((offset & ~0xff) == 0); offset |= (wback << 8); offset |= (add << 9); @@ -1239,10 +1290,10 @@ class ARMv7Assembler { ALWAYS_INLINE void ldrh(RegisterID rt, RegisterID rn, RegisterID rm, unsigned shift = 0) { - ASSERT(!BadReg(rt)); // Memory hint - ASSERT(rn != ARMRegisters::pc); // LDRH (literal) - ASSERT(!BadReg(rm)); - ASSERT(shift <= 3); + ARMV7_ASM_ASSERT(!BadReg(rt)); // Memory hint + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); // LDRH (literal) + ARMV7_ASM_ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(shift <= 3); if (!shift && !((rt | rn | rm) & 8)) m_formatter.oneWordOp7Reg3Reg3Reg3(OP_LDRH_reg_T1, rm, rn, rt); @@ -1252,8 +1303,9 @@ class ARMv7Assembler { void ldrb(RegisterID rt, RegisterID rn, ARMThumbImmediate imm) { - ASSERT(rn != ARMRegisters::pc); // LDR (literal) - ASSERT(imm.isUInt12()); + ARMV7_ASM_ASSERT(!BadReg(rt)); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); // LDR (literal) + ARMV7_ASM_ASSERT(imm.isUInt12()); if (!((rt | rn) & 8) && imm.isUInt5()) m_formatter.oneWordOp5Imm5Reg3Reg3(OP_LDRB_imm_T1, imm.getUInt5(), rn, rt); @@ -1263,10 +1315,14 @@ class ARMv7Assembler { void ldrb(RegisterID rt, RegisterID rn, int offset, bool index, bool wback) { - ASSERT(rt != ARMRegisters::pc); - ASSERT(rn != ARMRegisters::pc); - ASSERT(index || wback); - ASSERT(!wback | (rt != rn)); + ARMV7_ASM_ASSERT(rt != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(index || wback); + ARMV7_ASM_ASSERT(!wback | (rt != rn)); + ARMV7_ASM_ASSERT(!BadReg(rt)); + ARMV7_ASM_ASSERT((offset >= -255) && (offset <= 255)); + ARMV7_ASM_ASSERT(!BadReg(rt)); + ARMV7_ASM_ASSERT((offset >= -255) && (offset <= 255)); bool add = true; if (offset < 0) { @@ -1274,7 +1330,7 @@ class ARMv7Assembler { offset = -offset; } - ASSERT(!(offset & ~0xff)); + ARMV7_ASM_ASSERT(!(offset & ~0xff)); offset |= (wback << 8); offset |= (add << 9); @@ -1286,9 +1342,10 @@ class ARMv7Assembler { ALWAYS_INLINE void ldrb(RegisterID rt, RegisterID rn, RegisterID rm, unsigned shift = 0) { - ASSERT(rn != ARMRegisters::pc); // LDR (literal) - ASSERT(!BadReg(rm)); - ASSERT(shift <= 3); + ARMV7_ASM_ASSERT(!BadReg(rt)); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); // LDR (literal) + ARMV7_ASM_ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(shift <= 3); if (!shift && !((rt | rn | rm) & 8)) m_formatter.oneWordOp7Reg3Reg3Reg3(OP_LDRB_reg_T1, rm, rn, rt); @@ -1298,26 +1355,30 @@ class ARMv7Assembler { void ldrsb(RegisterID rt, RegisterID rn, ARMThumbImmediate imm) { - ASSERT(rt != ARMRegisters::sp); - ASSERT(rn != ARMRegisters::pc); // LDR (literal) - ASSERT(imm.isUInt12()); + ARMV7_ASM_ASSERT(rt != ARMRegisters::sp); + ARMV7_ASM_ASSERT(rt != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rt != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); // LDR (literal) + ARMV7_ASM_ASSERT(imm.isUInt12()); m_formatter.twoWordOp12Reg4Reg4Imm12(OP_LDRSB_imm_T1, rn, rt, imm.getUInt12()); } void ldrsb(RegisterID rt, RegisterID rn, int offset, bool index, bool wback) { - ASSERT(!BadReg(rt)); - ASSERT(rn != ARMRegisters::pc); // LDR (literal) - ASSERT(index || wback); - ASSERT(!wback | (rt != rn)); + ARMV7_ASM_ASSERT(!BadReg(rt)); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); // LDR (literal) + ARMV7_ASM_ASSERT(index || wback); + ARMV7_ASM_ASSERT(!wback | (rt != rn)); + ARMV7_ASM_ASSERT((offset >= -255) && (offset <= 255)); + ARMV7_ASM_ASSERT((offset >= -255) && (offset <= 255)); bool add = true; if (offset < 0) { add = false; offset = -offset; } - ASSERT(!(offset & ~0xff)); + ARMV7_ASM_ASSERT(!(offset & ~0xff)); offset |= (wback << 8); offset |= (add << 9); @@ -1329,9 +1390,10 @@ class ARMv7Assembler { void ldrsb(RegisterID rt, RegisterID rn, RegisterID rm, unsigned shift = 0) { - ASSERT(rn != ARMRegisters::pc); - ASSERT(!BadReg(rm)); - ASSERT(shift <= 3); + ARMV7_ASM_ASSERT(!BadReg(rt)); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(shift <= 3); if (!shift && !((rt | rn | rm) & 8)) m_formatter.oneWordOp7Reg3Reg3Reg3(OP_LDRSB_reg_T1, rm, rn, rt); @@ -1341,26 +1403,30 @@ class ARMv7Assembler { void ldrsh(RegisterID rt, RegisterID rn, ARMThumbImmediate imm) { - ASSERT(rt != ARMRegisters::sp); - ASSERT(rn != ARMRegisters::pc); // LDR (literal) - ASSERT(imm.isUInt12()); + ARMV7_ASM_ASSERT(rt != ARMRegisters::sp); + ARMV7_ASM_ASSERT(rt != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rt != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); // LDR (literal) + ARMV7_ASM_ASSERT(imm.isUInt12()); m_formatter.twoWordOp12Reg4Reg4Imm12(OP_LDRSH_imm_T1, rn, rt, imm.getUInt12()); } void ldrsh(RegisterID rt, RegisterID rn, int offset, bool index, bool wback) { - ASSERT(!BadReg(rt)); - ASSERT(rn != ARMRegisters::pc); // LDR (literal) - ASSERT(index || wback); - ASSERT(!wback | (rt != rn)); + ARMV7_ASM_ASSERT(!BadReg(rt)); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); // LDR (literal) + ARMV7_ASM_ASSERT(index || wback); + ARMV7_ASM_ASSERT(!wback | (rt != rn)); + ARMV7_ASM_ASSERT((offset >= -255) && (offset <= 255)); + ARMV7_ASM_ASSERT((offset >= -255) && (offset <= 255)); bool add = true; if (offset < 0) { add = false; offset = -offset; } - ASSERT(!(offset & ~0xff)); + ARMV7_ASM_ASSERT(!(offset & ~0xff)); offset |= (wback << 8); offset |= (add << 9); @@ -1372,9 +1438,10 @@ class ARMv7Assembler { void ldrsh(RegisterID rt, RegisterID rn, RegisterID rm, unsigned shift = 0) { - ASSERT(rn != ARMRegisters::pc); - ASSERT(!BadReg(rm)); - ASSERT(shift <= 3); + ARMV7_ASM_ASSERT(!BadReg(rt)); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(shift <= 3); if (!shift && !((rt | rn | rm) & 8)) m_formatter.oneWordOp7Reg3Reg3Reg3(OP_LDRSH_reg_T1, rm, rn, rt); @@ -1398,14 +1465,16 @@ class ARMv7Assembler { // if (wback) REG[rn] = _tmp ALWAYS_INLINE void ldrd(RegisterID rt, RegisterID rt2, RegisterID rn, int offset, bool index, bool wback) { - ASSERT(!BadReg(rt)); - ASSERT(!BadReg(rt2)); - ASSERT(rn != ARMRegisters::pc); - ASSERT(rt != rt2); - ASSERT(index || wback); - ASSERT(!wback | (rt != rn)); - ASSERT(!wback | (rt2 != rn)); - ASSERT(!(offset & 0x3)); + ARMV7_ASM_ASSERT(!BadReg(rt)); + ARMV7_ASM_ASSERT(!BadReg(rt2)); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rt != rt2); + ARMV7_ASM_ASSERT(index || wback); + ARMV7_ASM_ASSERT(!wback | (rt != rn)); + ARMV7_ASM_ASSERT(!wback | (rt2 != rn)); + ARMV7_ASM_ASSERT(!(offset & 0x3)); + ARMV7_ASM_ASSERT((offset >= -1020) && (offset <= 1020)); + ARMV7_ASM_ASSERT((offset >= -1020) && (offset <= 1020)); bool add = true; if (offset < 0) { @@ -1413,7 +1482,7 @@ class ARMv7Assembler { offset = -offset; } offset >>= 2; - ASSERT(!(offset & ~0xff)); + ARMV7_ASM_ASSERT(!(offset & ~0xff)); uint16_t opcode = OP_LDRD_imm_T1; opcode |= (wback << 5); @@ -1425,71 +1494,74 @@ class ARMv7Assembler { ALWAYS_INLINE void ldrex(RegisterID rt, RegisterID rn, int32_t offset) { - ASSERT(!BadReg(rt)); - ASSERT(rn != ARMRegisters::pc); - ASSERT(!(offset & ~0x3fc)); + ARMV7_ASM_ASSERT(!BadReg(rt)); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(!(offset & ~0x3fc)); m_formatter.twoWordOp12Reg4Reg4Imm12(OP_LDREX_T1, rn, rt, (0xf << 8 | offset >> 2)); } ALWAYS_INLINE void ldrexb(RegisterID rt, RegisterID rn) { - ASSERT(!BadReg(rt)); - ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(!BadReg(rt)); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); m_formatter.twoWordOp12Reg4FourFours(OP_LDREXB_T1, rn, FourFours(rt, 0xf, 0x4, 0xf)); } ALWAYS_INLINE void ldrexh(RegisterID rt, RegisterID rn) { - ASSERT(!BadReg(rt)); - ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(!BadReg(rt)); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); m_formatter.twoWordOp12Reg4FourFours(OP_LDREXH_T1, rn, FourFours(rt, 0xf, 0x5, 0xf)); } ALWAYS_INLINE void ldrexd(RegisterID rt, RegisterID rt2, RegisterID rn) { - ASSERT(!BadReg(rt)); - ASSERT(!BadReg(rt2)); - ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(!BadReg(rt)); + ARMV7_ASM_ASSERT(!BadReg(rt2)); + ARMV7_ASM_ASSERT(rt != rt2); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); m_formatter.twoWordOp12Reg4FourFours(OP_LDREXD_T1, rn, FourFours(rt, rt2, 0x7, 0xf)); } void lsl(RegisterID rd, RegisterID rm, int32_t shiftAmount) { - ASSERT(!BadReg(rd)); - ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(shiftAmount >= 0 && shiftAmount <= 31); ShiftTypeAndAmount shift(SRType_LSL, shiftAmount); m_formatter.twoWordOp16FourFours(OP_LSL_imm_T1, FourFours(shift.hi4(), rd, shift.lo4(), rm)); } ALWAYS_INLINE void lsl(RegisterID rd, RegisterID rn, RegisterID rm) { - ASSERT(!BadReg(rd)); - ASSERT(!BadReg(rn)); - ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rn)); + ARMV7_ASM_ASSERT(!BadReg(rm)); m_formatter.twoWordOp12Reg4FourFours(OP_LSL_reg_T2, rn, FourFours(0xf, rd, 0, rm)); } ALWAYS_INLINE void lsr(RegisterID rd, RegisterID rm, int32_t shiftAmount) { - ASSERT(!BadReg(rd)); - ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(shiftAmount >= 1 && shiftAmount <= 32); ShiftTypeAndAmount shift(SRType_LSR, shiftAmount); m_formatter.twoWordOp16FourFours(OP_LSR_imm_T1, FourFours(shift.hi4(), rd, shift.lo4(), rm)); } ALWAYS_INLINE void lsr(RegisterID rd, RegisterID rn, RegisterID rm) { - ASSERT(!BadReg(rd)); - ASSERT(!BadReg(rn)); - ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rn)); + ARMV7_ASM_ASSERT(!BadReg(rm)); m_formatter.twoWordOp12Reg4FourFours(OP_LSR_reg_T2, rn, FourFours(0xf, rd, 0, rm)); } ALWAYS_INLINE void movT3(RegisterID rd, ARMThumbImmediate imm) { - ASSERT(imm.isValid()); - ASSERT(!imm.isEncodedImm()); - ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(imm.isValid()); + ARMV7_ASM_ASSERT(!imm.isEncodedImm()); + ARMV7_ASM_ASSERT(!BadReg(rd)); m_formatter.twoWordOp5i6Imm4Reg4EncodedImm(OP_MOV_imm_T3, imm.m_value.imm4, rd, imm); } @@ -1498,6 +1570,9 @@ class ARMv7Assembler { static void revertJumpTo_movT3movtcmpT2(void* instructionStart, RegisterID left, RegisterID right, uintptr_t imm) { uint16_t* address = static_cast(instructionStart); + ARMV7_ASM_ASSERT(isEven(instructionStart)); + ARMV7_ASM_ASSERT(!BadReg(left)); + ARMV7_ASM_ASSERT(!BadReg(right)); ARMThumbImmediate lo16 = ARMThumbImmediate::makeUInt16(static_cast(imm)); ARMThumbImmediate hi16 = ARMThumbImmediate::makeUInt16(static_cast(imm >> 16)); uint16_t instruction[] = { @@ -1513,9 +1588,9 @@ class ARMv7Assembler { #else static void revertJumpTo_movT3(void* instructionStart, RegisterID rd, ARMThumbImmediate imm) { - ASSERT(imm.isValid()); - ASSERT(!imm.isEncodedImm()); - ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(imm.isValid()); + ARMV7_ASM_ASSERT(!imm.isEncodedImm()); + ARMV7_ASM_ASSERT(!BadReg(rd)); uint16_t* address = static_cast(instructionStart); uint16_t instruction[] = { @@ -1529,8 +1604,8 @@ class ARMv7Assembler { ALWAYS_INLINE void mov(RegisterID rd, ARMThumbImmediate imm) { - ASSERT(imm.isValid()); - ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(imm.isValid()); + ARMV7_ASM_ASSERT(!BadReg(rd)); if ((rd < 8) && imm.isUInt8()) m_formatter.oneWordOp5Reg3Imm8(OP_MOV_imm_T1, rd, imm.getUInt8()); @@ -1542,29 +1617,29 @@ class ARMv7Assembler { ALWAYS_INLINE void mov(RegisterID rd, RegisterID rm) { - ASSERT(rd != rm); // Use a NOP instead + ARMV7_ASM_ASSERT(rd != rm); // Use a NOP instead m_formatter.oneWordOp8RegReg143(OP_MOV_reg_T1, rm, rd); } ALWAYS_INLINE void movt(RegisterID rd, ARMThumbImmediate imm) { - ASSERT(imm.isUInt16()); - ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(imm.isUInt16()); + ARMV7_ASM_ASSERT(!BadReg(rd)); m_formatter.twoWordOp5i6Imm4Reg4EncodedImm(OP_MOVT, imm.m_value.imm4, rd, imm); } ALWAYS_INLINE void mvn(RegisterID rd, ARMThumbImmediate imm) { - ASSERT(imm.isEncodedImm()); - ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(imm.isEncodedImm()); + ARMV7_ASM_ASSERT(!BadReg(rd)); m_formatter.twoWordOp5i6Imm4Reg4EncodedImm(OP_MVN_imm, 0xf, rd, imm); } ALWAYS_INLINE void mvn(RegisterID rd, RegisterID rm, ShiftTypeAndAmount shift) { - ASSERT(!BadReg(rd)); - ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rm)); m_formatter.twoWordOp16FourFours(OP_MVN_reg_T2, FourFours(shift.hi4(), rd, shift.lo4(), rm)); } @@ -1578,8 +1653,8 @@ class ARMv7Assembler { ALWAYS_INLINE void mrs(RegisterID rd, SPRegisterID specReg) { - ASSERT(specReg == ARMRegisters::apsr); - ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(specReg == ARMRegisters::apsr); + ARMV7_ASM_ASSERT(!BadReg(rd)); unsigned short specialRegisterBit = (specReg == ARMRegisters::apsr) ? 0 : (1 << 4); OpcodeID1 mrsOp = static_cast(OP_MRS_T1 | specialRegisterBit); m_formatter.twoWordOp16FourFours(mrsOp, FourFours(0x8, rd, 0, 0)); @@ -1587,23 +1662,23 @@ class ARMv7Assembler { ALWAYS_INLINE void neg(RegisterID rd, RegisterID rm) { - ARMThumbImmediate zero = ARMThumbImmediate::makeUInt12(0); + ARMThumbImmediate zero = ARMThumbImmediate::makeEncodedImm(0); sub(rd, zero, rm); } ALWAYS_INLINE void orr(RegisterID rd, RegisterID rn, ARMThumbImmediate imm) { - ASSERT(!BadReg(rd)); - ASSERT(!BadReg(rn)); - ASSERT(imm.isEncodedImm()); + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rn)); + ARMV7_ASM_ASSERT(imm.isEncodedImm()); m_formatter.twoWordOp5i6Imm4Reg4EncodedImm(OP_ORR_imm_T1, rn, rd, imm); } ALWAYS_INLINE void orr(RegisterID rd, RegisterID rn, RegisterID rm, ShiftTypeAndAmount shift) { - ASSERT(!BadReg(rd)); - ASSERT(!BadReg(rn)); - ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rn)); + ARMV7_ASM_ASSERT(!BadReg(rm)); m_formatter.twoWordOp12Reg4FourFours(OP_ORR_reg_T2, rn, FourFours(shift.hi4(), rd, shift.lo4(), rm)); } @@ -1619,9 +1694,9 @@ class ARMv7Assembler { ALWAYS_INLINE void orr_S(RegisterID rd, RegisterID rn, RegisterID rm, ShiftTypeAndAmount shift) { - ASSERT(!BadReg(rd)); - ASSERT(!BadReg(rn)); - ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rn)); + ARMV7_ASM_ASSERT(!BadReg(rm)); m_formatter.twoWordOp12Reg4FourFours(OP_ORR_S_reg_T2, rn, FourFours(shift.hi4(), rd, shift.lo4(), rm)); } @@ -1637,24 +1712,25 @@ class ARMv7Assembler { ALWAYS_INLINE void rbit(RegisterID rd, RegisterID rm) { - ASSERT(!BadReg(rd)); - ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rm)); m_formatter.twoWordOp12Reg4FourFours(OP_RBIT, rm, FourFours(0xf, rd, 0xa, rm)); } ALWAYS_INLINE void ror(RegisterID rd, RegisterID rm, int32_t shiftAmount) { - ASSERT(!BadReg(rd)); - ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(shiftAmount >= 1 && shiftAmount <= 31); ShiftTypeAndAmount shift(SRType_ROR, shiftAmount); m_formatter.twoWordOp16FourFours(OP_ROR_imm_T1, FourFours(shift.hi4(), rd, shift.lo4(), rm)); } ALWAYS_INLINE void ror(RegisterID rd, RegisterID rn, RegisterID rm) { - ASSERT(!BadReg(rd)); - ASSERT(!BadReg(rn)); - ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rn)); + ARMV7_ASM_ASSERT(!BadReg(rm)); m_formatter.twoWordOp12Reg4FourFours(OP_ROR_reg_T2, rn, FourFours(0xf, rd, 0, rm)); } @@ -1670,9 +1746,9 @@ class ARMv7Assembler { ALWAYS_INLINE void pop(uint32_t registerList) { - ASSERT(WTF::bitCount(registerList) > 1); - ASSERT(!((1 << ARMRegisters::pc) & registerList) || !((1 << ARMRegisters::lr) & registerList)); - ASSERT(!((1 << ARMRegisters::sp) & registerList)); + ARMV7_ASM_ASSERT(WTF::bitCount(registerList) > 1); + ARMV7_ASM_ASSERT(!((1 << ARMRegisters::pc) & registerList) || !((1 << ARMRegisters::lr) & registerList)); + ARMV7_ASM_ASSERT(!((1 << ARMRegisters::sp) & registerList)); m_formatter.twoWordOp16Imm16(OP_POP_T2, registerList); } @@ -1690,18 +1766,18 @@ class ARMv7Assembler { ALWAYS_INLINE void push(uint32_t registerList) { - ASSERT(WTF::bitCount(registerList) > 1); - ASSERT(!((1 << ARMRegisters::pc) & registerList)); - ASSERT(!((1 << ARMRegisters::sp) & registerList)); + ARMV7_ASM_ASSERT(WTF::bitCount(registerList) > 1); + ARMV7_ASM_ASSERT(!((1 << ARMRegisters::pc) & registerList)); + ARMV7_ASM_ASSERT(!((1 << ARMRegisters::sp) & registerList)); m_formatter.twoWordOp16Imm16(OP_PUSH_T2, registerList); } void sbc(RegisterID rd, RegisterID rn, RegisterID rm, ShiftTypeAndAmount shift = ShiftTypeAndAmount { }) { // Rd can only be SP if Rn is also SP. - ASSERT(!BadReg(rd)); - ASSERT(!BadReg(rn)); - ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rn)); + ARMV7_ASM_ASSERT(!BadReg(rm)); m_formatter.twoWordOp12Reg4FourFours(OP_SBC_reg_T2, rn, FourFours(shift.hi4(), rd, shift.lo4(), rm)); } @@ -1710,39 +1786,39 @@ class ARMv7Assembler { ALWAYS_INLINE void sdiv(RegisterID rd, RegisterID rn, RegisterID rm) { static_assert(datasize == 32, "sdiv datasize must be 32 for armv7s"); - ASSERT(!BadReg(rd)); - ASSERT(!BadReg(rn)); - ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rn)); + ARMV7_ASM_ASSERT(!BadReg(rm)); m_formatter.twoWordOp12Reg4FourFours(OP_SDIV_T1, rn, FourFours(0xf, rd, 0xf, rm)); } #endif ALWAYS_INLINE void smull(RegisterID rdLo, RegisterID rdHi, RegisterID rn, RegisterID rm) { - ASSERT(!BadReg(rdLo)); - ASSERT(!BadReg(rdHi)); - ASSERT(!BadReg(rn)); - ASSERT(!BadReg(rm)); - ASSERT(rdLo != rdHi); + ARMV7_ASM_ASSERT(!BadReg(rdLo)); + ARMV7_ASM_ASSERT(!BadReg(rdHi)); + ARMV7_ASM_ASSERT(!BadReg(rn)); + ARMV7_ASM_ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(rdLo != rdHi); m_formatter.twoWordOp12Reg4FourFours(OP_SMULL_T1, rn, FourFours(rdLo, rdHi, 0, rm)); } ALWAYS_INLINE void umull(RegisterID rdLo, RegisterID rdHi, RegisterID rn, RegisterID rm) { - ASSERT(!BadReg(rdLo)); - ASSERT(!BadReg(rdHi)); - ASSERT(!BadReg(rn)); - ASSERT(!BadReg(rm)); - ASSERT(rdLo != rdHi); + ARMV7_ASM_ASSERT(!BadReg(rdLo)); + ARMV7_ASM_ASSERT(!BadReg(rdHi)); + ARMV7_ASM_ASSERT(!BadReg(rn)); + ARMV7_ASM_ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(rdLo != rdHi); m_formatter.twoWordOp12Reg4FourFours(OP_UMULL_T1, rn, FourFours(rdLo, rdHi, 0, rm)); } // rt == ARMRegisters::pc only allowed if last instruction in IT (if then) block. ALWAYS_INLINE void str(RegisterID rt, RegisterID rn, ARMThumbImmediate imm) { - ASSERT(rt != ARMRegisters::pc); - ASSERT(rn != ARMRegisters::pc); - ASSERT(imm.isUInt12()); + ARMV7_ASM_ASSERT(rt != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(imm.isUInt12()); if (!((rt | rn) & 8) && imm.isUInt7() && !(imm.getUInt7() & 0x3)) m_formatter.oneWordOp5Imm5Reg3Reg3(OP_STR_imm_T1, imm.getUInt7() >> 2, rn, rt); @@ -1765,17 +1841,19 @@ class ARMv7Assembler { // if (wback) REG[rn] = _tmp ALWAYS_INLINE void str(RegisterID rt, RegisterID rn, int offset, bool index, bool wback) { - ASSERT(rt != ARMRegisters::pc); - ASSERT(rn != ARMRegisters::pc); - ASSERT(index || wback); - ASSERT(!wback | (rt != rn)); + ARMV7_ASM_ASSERT(rt != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(index || wback); + ARMV7_ASM_ASSERT(!wback | (rt != rn)); + ARMV7_ASM_ASSERT((offset >= -255) && (offset <= 255)); + ARMV7_ASM_ASSERT((offset >= -255) && (offset <= 255)); bool add = true; if (offset < 0) { add = false; offset = -offset; } - ASSERT((offset & ~0xff) == 0); + ARMV7_ASM_ASSERT((offset & ~0xff) == 0); offset |= (wback << 8); offset |= (add << 9); @@ -1788,9 +1866,10 @@ class ARMv7Assembler { // rt == ARMRegisters::pc only allowed if last instruction in IT (if then) block. ALWAYS_INLINE void str(RegisterID rt, RegisterID rn, RegisterID rm, unsigned shift = 0) { - ASSERT(rn != ARMRegisters::pc); - ASSERT(!BadReg(rm)); - ASSERT(shift <= 3); + ARMV7_ASM_ASSERT(rt != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(shift <= 3); if (!shift && !((rt | rn | rm) & 8)) m_formatter.oneWordOp7Reg3Reg3Reg3(OP_STR_reg_T1, rm, rn, rt); @@ -1801,9 +1880,11 @@ class ARMv7Assembler { // rt == ARMRegisters::pc only allowed if last instruction in IT (if then) block. ALWAYS_INLINE void strb(RegisterID rt, RegisterID rn, ARMThumbImmediate imm) { - ASSERT(rt != ARMRegisters::pc); - ASSERT(rn != ARMRegisters::pc); - ASSERT(imm.isUInt12()); + ARMV7_ASM_ASSERT(rt != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rt != ARMRegisters::sp); + ARMV7_ASM_ASSERT(rt != ARMRegisters::sp); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(imm.isUInt12()); if (!((rt | rn) & 8) && imm.isUInt5()) m_formatter.oneWordOp5Imm5Reg3Reg3(OP_STRB_imm_T1, imm.getUInt5(), rn, rt); @@ -1824,17 +1905,21 @@ class ARMv7Assembler { // if (wback) REG[rn] = _tmp ALWAYS_INLINE void strb(RegisterID rt, RegisterID rn, int offset, bool index, bool wback) { - ASSERT(rt != ARMRegisters::pc); - ASSERT(rn != ARMRegisters::pc); - ASSERT(index || wback); - ASSERT(!wback | (rt != rn)); + ARMV7_ASM_ASSERT(rt != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(index || wback); + ARMV7_ASM_ASSERT(!wback | (rt != rn)); + ARMV7_ASM_ASSERT(!BadReg(rt)); + ARMV7_ASM_ASSERT((offset >= -255) && (offset <= 255)); + ARMV7_ASM_ASSERT(!BadReg(rt)); + ARMV7_ASM_ASSERT((offset >= -255) && (offset <= 255)); bool add = true; if (offset < 0) { add = false; offset = -offset; } - ASSERT((offset & ~0xff) == 0); + ARMV7_ASM_ASSERT((offset & ~0xff) == 0); offset |= (wback << 8); offset |= (add << 9); @@ -1847,9 +1932,10 @@ class ARMv7Assembler { // rt == ARMRegisters::pc only allowed if last instruction in IT (if then) block. ALWAYS_INLINE void strb(RegisterID rt, RegisterID rn, RegisterID rm, unsigned shift = 0) { - ASSERT(rn != ARMRegisters::pc); - ASSERT(!BadReg(rm)); - ASSERT(shift <= 3); + ARMV7_ASM_ASSERT(!BadReg(rt)); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(shift <= 3); if (!shift && !((rt | rn | rm) & 8)) m_formatter.oneWordOp7Reg3Reg3Reg3(OP_STRB_reg_T1, rm, rn, rt); @@ -1860,9 +1946,11 @@ class ARMv7Assembler { // rt == ARMRegisters::pc only allowed if last instruction in IT (if then) block. ALWAYS_INLINE void strh(RegisterID rt, RegisterID rn, ARMThumbImmediate imm) { - ASSERT(rt != ARMRegisters::pc); - ASSERT(rn != ARMRegisters::pc); - ASSERT(imm.isUInt12()); + ARMV7_ASM_ASSERT(rt != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rt != ARMRegisters::sp); + ARMV7_ASM_ASSERT(rt != ARMRegisters::sp); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(imm.isUInt12()); if (!((rt | rn) & 8) && imm.isUInt6() && !(imm.getUInt6() & 0x1)) m_formatter.oneWordOp5Imm5Reg3Reg3(OP_STRH_imm_T1, imm.getUInt6() >> 1, rn, rt); @@ -1883,17 +1971,21 @@ class ARMv7Assembler { // if (wback) REG[rn] = _tmp ALWAYS_INLINE void strh(RegisterID rt, RegisterID rn, int offset, bool index, bool wback) { - ASSERT(rt != ARMRegisters::pc); - ASSERT(rn != ARMRegisters::pc); - ASSERT(index || wback); - ASSERT(!wback | (rt != rn)); + ARMV7_ASM_ASSERT(rt != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(index || wback); + ARMV7_ASM_ASSERT(!wback | (rt != rn)); + ARMV7_ASM_ASSERT(!BadReg(rt)); + ARMV7_ASM_ASSERT((offset >= -255) && (offset <= 255)); + ARMV7_ASM_ASSERT(!BadReg(rt)); + ARMV7_ASM_ASSERT((offset >= -255) && (offset <= 255)); bool add = true; if (offset < 0) { add = false; offset = -offset; } - ASSERT(!(offset & ~0xff)); + ARMV7_ASM_ASSERT(!(offset & ~0xff)); offset |= (wback << 8); offset |= (add << 9); @@ -1906,9 +1998,10 @@ class ARMv7Assembler { // rt == ARMRegisters::pc only allowed if last instruction in IT (if then) block. ALWAYS_INLINE void strh(RegisterID rt, RegisterID rn, RegisterID rm, unsigned shift = 0) { - ASSERT(rn != ARMRegisters::pc); - ASSERT(!BadReg(rm)); - ASSERT(shift <= 3); + ARMV7_ASM_ASSERT(!BadReg(rt)); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(shift <= 3); if (!shift && !((rt | rn | rm) & 8)) m_formatter.oneWordOp7Reg3Reg3Reg3(OP_STRH_reg_T1, rm, rn, rt); @@ -1932,13 +2025,15 @@ class ARMv7Assembler { // if (wback) REG[rn] = _tmp ALWAYS_INLINE void strd(RegisterID rt, RegisterID rt2, RegisterID rn, int offset, bool index, bool wback) { - ASSERT(!BadReg(rt)); - ASSERT(!BadReg(rt2)); - ASSERT(rn != ARMRegisters::pc); - ASSERT(index || wback); - ASSERT(!wback | (rt != rn)); - ASSERT(!wback | (rt2 != rn)); - ASSERT(!(offset & 0x3)); + ARMV7_ASM_ASSERT(!BadReg(rt)); + ARMV7_ASM_ASSERT(!BadReg(rt2)); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(index || wback); + ARMV7_ASM_ASSERT(!wback | (rt != rn)); + ARMV7_ASM_ASSERT(!wback | (rt2 != rn)); + ARMV7_ASM_ASSERT(!(offset & 0x3)); + ARMV7_ASM_ASSERT((offset >= -1020) && (offset <= 1020)); + ARMV7_ASM_ASSERT((offset >= -1020) && (offset <= 1020)); bool add = true; if (offset < 0) { @@ -1946,7 +2041,7 @@ class ARMv7Assembler { offset = -offset; } offset >>= 2; - ASSERT(!(offset & ~0xff)); + ARMV7_ASM_ASSERT(!(offset & ~0xff)); uint16_t opcode = OP_STRD_imm_T1; opcode |= (wback << 5); @@ -1958,57 +2053,56 @@ class ARMv7Assembler { ALWAYS_INLINE void strex(RegisterID rd, RegisterID rt, RegisterID rn, int32_t offset) { - ASSERT(!BadReg(rd)); - ASSERT(!BadReg(rt)); - ASSERT(rn != ARMRegisters::pc); - ASSERT(rd != rn); - ASSERT(rd != rt); - ASSERT(!(offset & ~0x3fc)); + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rt)); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rd != rn); + ARMV7_ASM_ASSERT(rd != rt); + ARMV7_ASM_ASSERT(!(offset & ~0x3fc)); m_formatter.twoWordOp12Reg4Reg4Reg4Imm8(OP_STREX_T1, rn, rt, rd, offset >> 2); } ALWAYS_INLINE void strexb(RegisterID rd, RegisterID rt, RegisterID rn) { - ASSERT(!BadReg(rd)); - ASSERT(!BadReg(rt)); - ASSERT(rn != ARMRegisters::pc); - ASSERT(rd != rn); - ASSERT(rd != rt); + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rt)); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rd != rn); + ARMV7_ASM_ASSERT(rd != rt); m_formatter.twoWordOp12Reg4FourFours(OP_STREXB_T1, rn, FourFours(rt, 0xf, 0x4, rd)); } ALWAYS_INLINE void strexh(RegisterID rd, RegisterID rt, RegisterID rn) { - ASSERT(!BadReg(rd)); - ASSERT(!BadReg(rt)); - ASSERT(rn != ARMRegisters::pc); - ASSERT(rd != rn); - ASSERT(rd != rt); + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rt)); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rd != rn); + ARMV7_ASM_ASSERT(rd != rt); m_formatter.twoWordOp12Reg4FourFours(OP_STREXH_T1, rn, FourFours(rt, 0xf, 0x5, rd)); } ALWAYS_INLINE void strexd(RegisterID rd, RegisterID rt, RegisterID rt2, RegisterID rn) { - ASSERT(!BadReg(rd)); - ASSERT(!BadReg(rt)); - ASSERT(!BadReg(rt2)); - ASSERT(rn != ARMRegisters::pc); - ASSERT(rd != rn); - ASSERT(rd != rt); - ASSERT(rd != rt2); + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rt)); + ARMV7_ASM_ASSERT(!BadReg(rt2)); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rd != rn); + ARMV7_ASM_ASSERT(rd != rt); + ARMV7_ASM_ASSERT(rd != rt2); m_formatter.twoWordOp12Reg4FourFours(OP_STREXD_T1, rn, FourFours(rt, rt2, 0x7, rd)); } ALWAYS_INLINE void sub(RegisterID rd, RegisterID rn, ARMThumbImmediate imm) { // Rd can only be SP if Rn is also SP. - ASSERT((rd != ARMRegisters::sp) || (rn == ARMRegisters::sp)); - ASSERT(rd != ARMRegisters::pc); - ASSERT(rn != ARMRegisters::pc); - ASSERT(imm.isValid()); + ARMV7_ASM_ASSERT((rd != ARMRegisters::sp) || (rn == ARMRegisters::sp)); + ARMV7_ASM_ASSERT(rd != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(imm.isValid()); - if ((rn == ARMRegisters::sp) && (rd == ARMRegisters::sp) && imm.isUInt9()) { - ASSERT(!(imm.getUInt16() & 3)); + if ((rn == ARMRegisters::sp) && (rd == ARMRegisters::sp) && imm.isUInt9() && !(imm.getUInt9() & 3)) { m_formatter.oneWordOp9Imm7(OP_SUB_SP_imm_T1, static_cast(imm.getUInt9() >> 2)); return; } else if (!((rd | rn) & 8)) { @@ -2024,19 +2118,19 @@ class ARMv7Assembler { if (imm.isEncodedImm()) m_formatter.twoWordOp5i6Imm4Reg4EncodedImm(OP_SUB_imm_T3, rn, rd, imm); else { - ASSERT(imm.isUInt12()); + ARMV7_ASM_ASSERT(imm.isUInt12()); m_formatter.twoWordOp5i6Imm4Reg4EncodedImm(OP_SUB_imm_T4, rn, rd, imm); } } ALWAYS_INLINE void sub(RegisterID rd, ARMThumbImmediate imm, RegisterID rn) { - ASSERT(rd != ARMRegisters::pc); - ASSERT(rn != ARMRegisters::pc); - ASSERT(imm.isValid()); - ASSERT(imm.isUInt12()); + ARMV7_ASM_ASSERT(rd != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(imm.isValid()); + ARMV7_ASM_ASSERT(imm.isEncodedImm()); - if (!((rd | rn) & 8) && !imm.getUInt12()) + if (!((rd | rn) & 8) && !imm.asUInt16()) m_formatter.oneWordOp10Reg3Reg3(OP_RSB_imm_T1, rn, rd); else m_formatter.twoWordOp5i6Imm4Reg4EncodedImm(OP_RSB_imm_T2, rn, rd, imm); @@ -2044,10 +2138,10 @@ class ARMv7Assembler { ALWAYS_INLINE void sub(RegisterID rd, RegisterID rn, RegisterID rm, ShiftTypeAndAmount shift) { - ASSERT((rd != ARMRegisters::sp) || (rn == ARMRegisters::sp)); - ASSERT(rd != ARMRegisters::pc); - ASSERT(rn != ARMRegisters::pc); - ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT((rd != ARMRegisters::sp) || (rn == ARMRegisters::sp)); + ARMV7_ASM_ASSERT(rd != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(!BadReg(rm)); m_formatter.twoWordOp12Reg4FourFours(OP_SUB_reg_T2, rn, FourFours(shift.hi4(), rd, shift.lo4(), rm)); } @@ -2064,16 +2158,14 @@ class ARMv7Assembler { void sub_S(RegisterID rd, RegisterID rn, ARMThumbImmediate imm) { // Rd can only be SP if Rn is also SP. - ASSERT((rd != ARMRegisters::sp) || (rn == ARMRegisters::sp)); - ASSERT(rd != ARMRegisters::pc); - ASSERT(rn != ARMRegisters::pc); - ASSERT(imm.isValid()); + ARMV7_ASM_ASSERT((rd != ARMRegisters::sp) || (rn == ARMRegisters::sp)); + ARMV7_ASM_ASSERT(rd != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(imm.isValid()); - if ((rn == ARMRegisters::sp) && (rd == ARMRegisters::sp) && imm.isUInt9()) { - ASSERT(!(imm.getUInt16() & 3)); - m_formatter.oneWordOp9Imm7(OP_SUB_SP_imm_T1, static_cast(imm.getUInt9() >> 2)); - return; - } else if (!((rd | rn) & 8)) { + ARMV7_ASM_ASSERT(imm.isEncodedImm()); + + if (!((rd | rn) & 8)) { if (imm.isUInt3()) { m_formatter.oneWordOp7Reg3Reg3Reg3(OP_SUB_imm_T1, (RegisterID)imm.getUInt3(), rn, rd); return; @@ -2088,10 +2180,10 @@ class ARMv7Assembler { ALWAYS_INLINE void sub_S(RegisterID rd, ARMThumbImmediate imm, RegisterID rn) { - ASSERT(rd != ARMRegisters::pc); - ASSERT(rn != ARMRegisters::pc); - ASSERT(imm.isValid()); - ASSERT(imm.isUInt12()); + ARMV7_ASM_ASSERT(rd != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(imm.isValid()); + ARMV7_ASM_ASSERT(imm.isEncodedImm()); m_formatter.twoWordOp5i6Imm4Reg4EncodedImm(OP_RSB_S_imm_T2, rn, rd, imm); } @@ -2099,10 +2191,10 @@ class ARMv7Assembler { // Not allowed in an IT (if then) block? ALWAYS_INLINE void sub_S(RegisterID rd, RegisterID rn, RegisterID rm, ShiftTypeAndAmount shift) { - ASSERT((rd != ARMRegisters::sp) || (rn == ARMRegisters::sp)); - ASSERT(rd != ARMRegisters::pc); - ASSERT(rn != ARMRegisters::pc); - ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT((rd != ARMRegisters::sp) || (rn == ARMRegisters::sp)); + ARMV7_ASM_ASSERT(rd != ARMRegisters::pc); + ARMV7_ASM_ASSERT(rn != ARMRegisters::pc); + ARMV7_ASM_ASSERT(!BadReg(rm)); m_formatter.twoWordOp12Reg4FourFours(OP_SUB_S_reg_T2, rn, FourFours(shift.hi4(), rd, shift.lo4(), rm)); } @@ -2117,6 +2209,8 @@ class ARMv7Assembler { ALWAYS_INLINE void sxtb(RegisterID rd, RegisterID rm) { + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rm)); if (!((rd | rm) & 8)) m_formatter.oneWordOp10Reg3Reg3(OP_SXTB_T1, rm, rd); else @@ -2125,6 +2219,8 @@ class ARMv7Assembler { ALWAYS_INLINE void sxth(RegisterID rd, RegisterID rm) { + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rm)); if (!((rd | rm) & 8)) m_formatter.oneWordOp10Reg3Reg3(OP_SXTH_T1, rm, rd); else @@ -2133,16 +2229,16 @@ class ARMv7Assembler { ALWAYS_INLINE void tst(RegisterID rn, ARMThumbImmediate imm) { - ASSERT(!BadReg(rn)); - ASSERT(imm.isEncodedImm()); + ARMV7_ASM_ASSERT(!BadReg(rn)); + ARMV7_ASM_ASSERT(imm.isEncodedImm()); m_formatter.twoWordOp5i6Imm4Reg4EncodedImm(OP_TST_imm, rn, (RegisterID)0xf, imm); } ALWAYS_INLINE void tst(RegisterID rn, RegisterID rm, ShiftTypeAndAmount shift) { - ASSERT(!BadReg(rn)); - ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(!BadReg(rn)); + ARMV7_ASM_ASSERT(!BadReg(rm)); m_formatter.twoWordOp12Reg4FourFours(OP_TST_reg_T2, rn, FourFours(shift.hi4(), 0xf, shift.lo4(), rm)); } @@ -2156,24 +2252,28 @@ class ARMv7Assembler { ALWAYS_INLINE void ubfx(RegisterID rd, RegisterID rn, unsigned lsb, unsigned width) { - ASSERT(lsb < 32); - ASSERT((width >= 1) && (width <= 32)); - ASSERT((lsb + width) <= 32); + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rn)); + ARMV7_ASM_ASSERT(lsb < 32); + ARMV7_ASM_ASSERT((width >= 1) && (width <= 32)); + ARMV7_ASM_ASSERT((lsb + width) <= 32); m_formatter.twoWordOp12Reg40Imm3Reg4Imm20Imm5(OP_UBFX_T1, rd, rn, (lsb & 0x1c) << 10, (lsb & 0x3) << 6, (width - 1) & 0x1f); } #if HAVE(ARM_IDIV_INSTRUCTIONS) ALWAYS_INLINE void udiv(RegisterID rd, RegisterID rn, RegisterID rm) { - ASSERT(!BadReg(rd)); - ASSERT(!BadReg(rn)); - ASSERT(!BadReg(rm)); + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rn)); + ARMV7_ASM_ASSERT(!BadReg(rm)); m_formatter.twoWordOp12Reg4FourFours(OP_UDIV_T1, rn, FourFours(0xf, rd, 0xf, rm)); } #endif ALWAYS_INLINE void uxtb(RegisterID rd, RegisterID rm) { + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rm)); if (!((rd | rm) & 8)) m_formatter.oneWordOp10Reg3Reg3(OP_UXTB_T1, rm, rd); else @@ -2182,6 +2282,8 @@ class ARMv7Assembler { ALWAYS_INLINE void uxth(RegisterID rd, RegisterID rm) { + ARMV7_ASM_ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rm)); if (!((rd | rm) & 8)) m_formatter.oneWordOp10Reg3Reg3(OP_UXTH_T1, rm, rd); else @@ -2190,13 +2292,19 @@ class ARMv7Assembler { void vldmia(RegisterID rn, FPDoubleRegisterID rs, uint32_t count) { - ASSERT(count < 16); + ARMV7_ASM_ASSERT(count < 16); + ARMV7_ASM_ASSERT(count); + ARMV7_ASM_ASSERT(rs >= firstFPRegister()); + ARMV7_ASM_ASSERT(static_cast(rs) + static_cast(count) <= static_cast(lastFPRegister()) + 1); m_formatter.vfpMemOp(OP_VLDMIA, OP_VLDMIAb, true, rn, rs, count << 3); } void vstmia(RegisterID rn, FPDoubleRegisterID rs, uint32_t count) { - ASSERT(count < 16); + ARMV7_ASM_ASSERT(count < 16); + ARMV7_ASM_ASSERT(count); + ARMV7_ASM_ASSERT(rs >= firstFPRegister()); + ARMV7_ASM_ASSERT(static_cast(rs) + static_cast(count) <= static_cast(lastFPRegister()) + 1); m_formatter.vfpMemOp(OP_VSTMIA, OP_VSTMIAb, true, rn, rs, count << 3); } @@ -2230,6 +2338,11 @@ class ARMv7Assembler { m_formatter.vfpOp(OP_VCMP, OP_VCMPb, true, VFPOperand(4), rd, rm); } + void vcmpz(FPSingleRegisterID rd) + { + m_formatter.vfpOp(OP_VCMP, OP_VCMPb, false, VFPOperand(5), rd, VFPOperand(0)); + } + void vcmpz(FPDoubleRegisterID rd) { m_formatter.vfpOp(OP_VCMP, OP_VCMPb, true, VFPOperand(5), rd, VFPOperand(0)); @@ -2237,12 +2350,14 @@ class ARMv7Assembler { void vcvt_signedToFloatingPoint(FPDoubleRegisterID rd, FPSingleRegisterID rm, bool toDouble = true) { + ARMV7_ASM_ASSERT(toDouble || rd <= ARMRegisters::d15); // boolean values are 64bit (toInt, unsigned, roundZero) m_formatter.vfpOp(OP_VCVT_FPIVFP, OP_VCVT_FPIVFPb, toDouble, vcvtOp(false, false, false), rd, rm); } void vcvt_unsignedToFloatingPoint(FPDoubleRegisterID rd, FPSingleRegisterID rm, bool toDouble = true) { + ARMV7_ASM_ASSERT(toDouble || rd <= ARMRegisters::d15); // boolean values are 64bit (toInt, unsigned, roundZero) m_formatter.vfpOp(OP_VCVT_FPIVFP, OP_VCVT_FPIVFPb, toDouble, vcvtOp(false, true, false), rd, rm); } @@ -2299,27 +2414,29 @@ class ARMv7Assembler { void vmov(RegisterID rd, FPSingleRegisterID rn) { - ASSERT(!BadReg(rd)); + ARMV7_ASM_ASSERT(!BadReg(rd)); m_formatter.vfpOp(OP_VMOV_StoC, OP_VMOV_StoCb, false, rn, rd, VFPOperand(0)); } void vmov(FPSingleRegisterID rd, RegisterID rn) { - ASSERT(!BadReg(rn)); + ARMV7_ASM_ASSERT(!BadReg(rn)); m_formatter.vfpOp(OP_VMOV_CtoS, OP_VMOV_CtoSb, false, rd, rn, VFPOperand(0)); } void vmov(RegisterID rd1, RegisterID rd2, FPDoubleRegisterID rn) { - ASSERT(!BadReg(rd1)); - ASSERT(!BadReg(rd2)); + ARMV7_ASM_ASSERT(!BadReg(rd1)); + ARMV7_ASM_ASSERT(!BadReg(rd2)); + ARMV7_ASM_ASSERT(rd1 != rd2); + ARMV7_ASM_ASSERT(rd1 != rd2); m_formatter.vfpOp(OP_VMOV_DtoC, OP_VMOV_DtoCb, true, rd2, VFPOperand(rd1 | 16), rn); } void vmov(FPDoubleRegisterID rd, RegisterID rn1, RegisterID rn2) { - ASSERT(!BadReg(rn1)); - ASSERT(!BadReg(rn2)); + ARMV7_ASM_ASSERT(!BadReg(rn1)); + ARMV7_ASM_ASSERT(!BadReg(rn2)); m_formatter.vfpOp(OP_VMOV_CtoD, OP_VMOV_CtoDb, true, rn2, VFPOperand(rn1 | 16), rd); } @@ -2330,7 +2447,8 @@ class ARMv7Assembler { void vmrs(RegisterID reg = ARMRegisters::pc) { - ASSERT(reg != ARMRegisters::sp); + ARMV7_ASM_ASSERT(reg != ARMRegisters::sp); + ARMV7_ASM_ASSERT(reg >= ARMRegisters::r0 && reg <= ARMRegisters::pc); m_formatter.vfpOp(OP_VMRS, OP_VMRSb, false, VFPOperand(1), VFPOperand(0x10 | reg), VFPOperand(0)); } @@ -2344,6 +2462,11 @@ class ARMv7Assembler { m_formatter.vfpOp(OP_VMUL_T2, OP_VMUL_T2b, true, rn, rd, rm); } + void vmla(FPDoubleRegisterID rd, FPDoubleRegisterID rn, FPDoubleRegisterID rm) + { + m_formatter.vfpOp(OP_VMLA_T2, OP_VMLA_T2b, true, rn, rd, rm); + } + void vstr(FPDoubleRegisterID rd, RegisterID rn, int32_t imm) { m_formatter.vfpMemOp(OP_VSTR, OP_VSTRb, true, rn, rd, imm); @@ -2428,6 +2551,8 @@ class ARMv7Assembler { ALWAYS_INLINE static void fillNops(void* base, size_t size) { RELEASE_ASSERT(!(size % sizeof(int16_t))); + ARMV7_ASM_ASSERT(isEven(base)); + ARMV7_ASM_ASSERT(isEven(base)); char* ptr = static_cast(base); const size_t num32s = size / sizeof(int32_t); @@ -2438,8 +2563,8 @@ class ARMv7Assembler { } const size_t num16s = (size % sizeof(int32_t)) / sizeof(int16_t); - ASSERT(num16s == 0 || num16s == 1); - ASSERT(num16s * sizeof(int16_t) + num32s * sizeof(int32_t) == size); + ARMV7_ASM_ASSERT(num16s == 0 || num16s == 1); + ARMV7_ASM_ASSERT(num16s * sizeof(int16_t) + num32s * sizeof(int32_t) == size); if (num16s) { const int16_t insn = nopPseudo16(); machineCodeCopy(ptr, &insn, sizeof(int16_t)); @@ -2450,6 +2575,9 @@ class ARMv7Assembler { ALWAYS_INLINE static void fillNearTailCall(void* from, void* to) { uint16_t* ptr = reinterpret_cast(from) + 2; + ARMV7_ASM_ASSERT(isEven(from)); + ARMV7_ASM_ASSERT(isEven(to)); + ARMV7_ASM_ASSERT(canBeJumpT4(ptr, to)); linkJumpT4(ptr, ptr, to, BranchWithLink::No); cacheFlush(from, sizeof(uint16_t) * 2); } @@ -2515,7 +2643,7 @@ class ARMv7Assembler { static void* getRelocatedAddress(void* code, AssemblerLabel label) { - ASSERT(label.isSet()); + ARMV7_ASM_ASSERT(label.isSet()); return reinterpret_cast(reinterpret_cast(code) + label.offset()); } @@ -2583,7 +2711,7 @@ class ARMv7Assembler { return LinkBX; } - ASSERT(jumpType == JumpCondition); + ARMV7_ASM_ASSERT(jumpType == JumpCondition); return LinkConditionalBX; } @@ -2638,7 +2766,7 @@ class ARMv7Assembler { static unsigned getCallReturnOffset(AssemblerLabel call) { - ASSERT(call.isSet()); + ARMV7_ASM_ASSERT(call.isSet()); return call.offset(); } @@ -2652,14 +2780,14 @@ class ARMv7Assembler { void linkJump(AssemblerLabel from, AssemblerLabel to, JumpType type, Condition condition) { - ASSERT(to.isSet()); - ASSERT(from.isSet()); + ARMV7_ASM_ASSERT(to.isSet()); + ARMV7_ASM_ASSERT(from.isSet()); m_jumpsToLink.append(LinkRecord(from.offset(), to.offset(), type, condition)); } static void linkJump(void* code, AssemblerLabel from, void* to) { - ASSERT(from.isSet()); + ARMV7_ASM_ASSERT(from.isSet()); uint16_t* location = reinterpret_cast(reinterpret_cast(code) + from.offset()); linkJumpAbsolute(location, location, to); @@ -2667,7 +2795,7 @@ class ARMv7Assembler { static void linkTailCall(void* code, AssemblerLabel from, void* to) { - ASSERT(from.isSet()); + ARMV7_ASM_ASSERT(from.isSet()); uint16_t* location = reinterpret_cast(reinterpret_cast(code) + from.offset()); linkBranch(location, location, makeEven(to), BranchWithLink::No); @@ -2675,7 +2803,7 @@ class ARMv7Assembler { static void linkCall(void* code, AssemblerLabel from, void* to) { - ASSERT(from.isSet()); + ARMV7_ASM_ASSERT(from.isSet()); uint16_t* location = reinterpret_cast(reinterpret_cast(code) + from.offset()); linkBranch(location, location, makeEven(to), BranchWithLink::Yes); @@ -2692,8 +2820,8 @@ class ARMv7Assembler { // provided is correct for relative address computation. static void relinkJump(void* from, void* to) { - ASSERT(!(reinterpret_cast(from) & 1)); - ASSERT(!(reinterpret_cast(to) & 1)); + ARMV7_ASM_ASSERT(!(reinterpret_cast(from) & 1)); + ARMV7_ASM_ASSERT(!(reinterpret_cast(to) & 1)); linkJumpAbsolute(reinterpret_cast(from), reinterpret_cast(from), to); @@ -2702,7 +2830,7 @@ class ARMv7Assembler { static void relinkCall(void* from, void* to) { - ASSERT(isEven(from)); + ARMV7_ASM_ASSERT(isEven(from)); uint16_t* location = reinterpret_cast(from); if (isBL(location - 2)) { @@ -2716,7 +2844,7 @@ class ARMv7Assembler { static void relinkTailCall(void* from, void* to) { - ASSERT(isEven(from)); + ARMV7_ASM_ASSERT(isEven(from)); uint16_t* location = reinterpret_cast(from); linkBranch(location, location, to, BranchWithLink::No); @@ -2726,11 +2854,11 @@ class ARMv7Assembler { #if ENABLE(JUMP_ISLANDS) static void* prepareForAtomicRelinkJumpConcurrently(void* from, void* to) { - ASSERT(isEven(from)); - ASSERT(isEven(to)); + ARMV7_ASM_ASSERT(isEven(from)); + ARMV7_ASM_ASSERT(isEven(to)); intptr_t offset = std::bit_cast(to) - std::bit_cast(from); - ASSERT(static_cast(offset) == offset); + ARMV7_ASM_ASSERT(static_cast(offset) == offset); if (isInt<25>(offset)) return to; @@ -2740,7 +2868,7 @@ class ARMv7Assembler { static void* prepareForAtomicRelinkCallConcurrently(void* from, void* to) { - ASSERT(isEven(from)); + ARMV7_ASM_ASSERT(isEven(from)); return prepareForAtomicRelinkJumpConcurrently(from, makeEven(to)); } @@ -2753,7 +2881,7 @@ class ARMv7Assembler { static void repatchPointer(void* where, void* value) { - ASSERT(!(reinterpret_cast(where) & 1)); + ARMV7_ASM_ASSERT(!(reinterpret_cast(where) & 1)); setPointer(where, value, true); } @@ -2765,11 +2893,11 @@ class ARMv7Assembler { static void replaceWithJump(void* instructionStart, void* to) { - ASSERT(!(std::bit_cast(instructionStart) & 1)); - ASSERT(!(std::bit_cast(to) & 1)); + ARMV7_ASM_ASSERT(!(std::bit_cast(instructionStart) & 1)); + ARMV7_ASM_ASSERT(!(std::bit_cast(to) & 1)); #if OS(LINUX) - if (canBeJumpT4(reinterpret_cast(instructionStart), to)) { + if (canBeJumpT4(reinterpret_cast(instructionStart) + 2, to)) { uint16_t* ptr = reinterpret_cast(instructionStart) + 2; linkJumpT4(ptr, ptr, to, BranchWithLink::No); cacheFlush(ptr - 2, sizeof(uint16_t) * 2); @@ -2874,7 +3002,7 @@ class ARMv7Assembler { explicit VFPOperand(uint32_t value) : m_value(value) { - ASSERT(!(m_value & ~0x1f)); + ARMV7_ASM_ASSERT(!(m_value & ~0x1f)); } VFPOperand(FPDoubleRegisterID reg) @@ -2908,7 +3036,7 @@ class ARMv7Assembler { VFPOperand vcvtOp(bool toInteger, bool isUnsigned, bool isRoundZero) { // Cannot specify rounding when converting to float. - ASSERT(toInteger || !isRoundZero); + ARMV7_ASM_ASSERT(toInteger || !isRoundZero); uint32_t op = 0x8; if (toInteger) { @@ -2918,7 +3046,7 @@ class ARMv7Assembler { if (isRoundZero) op |= 0x10; } else { - ASSERT(!isRoundZero); + ARMV7_ASM_ASSERT(!isRoundZero); // 'op' field in instruction is isUnsigned if (!isUnsigned) op |= 0x10; @@ -2929,7 +3057,7 @@ class ARMv7Assembler { static void setInt32(void* code, uint32_t value, bool flush) { uint16_t* location = reinterpret_cast(code); - ASSERT(isMOV_imm_T3(location - 4) && isMOVT(location - 2)); + ARMV7_ASM_ASSERT(isMOV_imm_T3(location - 4) && isMOVT(location - 2)); ARMThumbImmediate lo16 = ARMThumbImmediate::makeUInt16(static_cast(value)); ARMThumbImmediate hi16 = ARMThumbImmediate::makeUInt16(static_cast(value >> 16)); @@ -2947,7 +3075,7 @@ class ARMv7Assembler { static int32_t readInt32(void* code) { uint16_t* location = reinterpret_cast(code); - ASSERT(isMOV_imm_T3(location - 4) && isMOVT(location - 2)); + ARMV7_ASM_ASSERT(isMOV_imm_T3(location - 4) && isMOVT(location - 2)); ARMThumbImmediate lo16; ARMThumbImmediate hi16; @@ -2964,8 +3092,8 @@ class ARMv7Assembler { static void setUInt7ForLoad(void* code, ARMThumbImmediate imm) { // Requires us to have planted a LDR_imm_T1 - ASSERT(imm.isValid()); - ASSERT(imm.isUInt7()); + ARMV7_ASM_ASSERT(imm.isValid()); + ARMV7_ASM_ASSERT(imm.isUInt7()); uint16_t* location = reinterpret_cast(code); uint16_t instruction; instruction = location[0] & ~((static_cast(0x7f) >> 2) << 6); @@ -3023,8 +3151,8 @@ class ARMv7Assembler { static bool canBeJumpT1(const uint16_t* instruction, const void* target) { - ASSERT(!(reinterpret_cast(instruction) & 1)); - ASSERT(!(reinterpret_cast(target) & 1)); + ARMV7_ASM_ASSERT(!(reinterpret_cast(instruction) & 1)); + ARMV7_ASM_ASSERT(!(reinterpret_cast(target) & 1)); intptr_t relative = reinterpret_cast(target) - (reinterpret_cast(instruction)); // It does not appear to be documented in the ARM ARM (big surprise), but @@ -3036,8 +3164,8 @@ class ARMv7Assembler { static bool canBeJumpT2(const uint16_t* instruction, const void* target) { - ASSERT(!(reinterpret_cast(instruction) & 1)); - ASSERT(!(reinterpret_cast(target) & 1)); + ARMV7_ASM_ASSERT(!(reinterpret_cast(instruction) & 1)); + ARMV7_ASM_ASSERT(!(reinterpret_cast(target) & 1)); intptr_t relative = reinterpret_cast(target) - (reinterpret_cast(instruction)); // It does not appear to be documented in the ARM ARM (big surprise), but @@ -3049,8 +3177,8 @@ class ARMv7Assembler { static bool canBeJumpT3(const uint16_t* instruction, const void* target) { - ASSERT(!(reinterpret_cast(instruction) & 1)); - ASSERT(!(reinterpret_cast(target) & 1)); + ARMV7_ASM_ASSERT(!(reinterpret_cast(instruction) & 1)); + ARMV7_ASM_ASSERT(!(reinterpret_cast(target) & 1)); intptr_t relative = reinterpret_cast(target) - (reinterpret_cast(instruction)); return ((relative << 11) >> 11) == relative; @@ -3058,8 +3186,8 @@ class ARMv7Assembler { static bool canBeJumpT4(const uint16_t* instruction, const void* target) { - ASSERT(!(reinterpret_cast(instruction) & 1)); - ASSERT(!(reinterpret_cast(target) & 1)); + ARMV7_ASM_ASSERT(!(reinterpret_cast(instruction) & 1)); + ARMV7_ASM_ASSERT(!(reinterpret_cast(target) & 1)); intptr_t relative = reinterpret_cast(target) - (reinterpret_cast(instruction)); return ((relative << 7) >> 7) == relative; @@ -3069,9 +3197,11 @@ class ARMv7Assembler { static void linkJumpT1(Condition cond, uint16_t* writeTarget, const uint16_t* instruction, void* target) { // FIMXE: this should be up in the MacroAssembler layer. :-( - ASSERT(!(reinterpret_cast(instruction) & 1)); - ASSERT(!(reinterpret_cast(target) & 1)); - ASSERT(canBeJumpT1(instruction, target)); + ARMV7_ASM_ASSERT(!(reinterpret_cast(instruction) & 1)); + ARMV7_ASM_ASSERT(!(reinterpret_cast(writeTarget) & 1)); + ARMV7_ASM_ASSERT(cond < ConditionAL); + ARMV7_ASM_ASSERT(!(reinterpret_cast(target) & 1)); + ARMV7_ASM_ASSERT(canBeJumpT1(instruction, target)); intptr_t relative = reinterpret_cast(target) - (reinterpret_cast(instruction)); // It does not appear to be documented in the ARM ARM (big surprise), but @@ -3080,7 +3210,7 @@ class ARMv7Assembler { relative -= 2; // All branch offsets should be an even distance. - ASSERT(!(relative & 1)); + ARMV7_ASM_ASSERT(!(relative & 1)); uint16_t newInstruction = OP_B_T1 | ((cond & 0xf) << 8) | ((relative & 0x1fe) >> 1); machineCodeCopy(writeTarget - 1, &newInstruction, sizeof(uint16_t)); } @@ -3089,9 +3219,10 @@ class ARMv7Assembler { static void linkJumpT2(uint16_t* writeTarget, const uint16_t* instruction, void* target) { // FIMXE: this should be up in the MacroAssembler layer. :-( - ASSERT(!(reinterpret_cast(instruction) & 1)); - ASSERT(!(reinterpret_cast(target) & 1)); - ASSERT(canBeJumpT2(instruction, target)); + ARMV7_ASM_ASSERT(!(reinterpret_cast(instruction) & 1)); + ARMV7_ASM_ASSERT(!(reinterpret_cast(writeTarget) & 1)); + ARMV7_ASM_ASSERT(!(reinterpret_cast(target) & 1)); + ARMV7_ASM_ASSERT(canBeJumpT2(instruction, target)); intptr_t relative = reinterpret_cast(target) - (reinterpret_cast(instruction)); // It does not appear to be documented in the ARM ARM (big surprise), but @@ -3100,7 +3231,7 @@ class ARMv7Assembler { relative -= 2; // All branch offsets should be an even distance. - ASSERT(!(relative & 1)); + ARMV7_ASM_ASSERT(!(relative & 1)); uint16_t newInstruction = OP_B_T2 | ((relative & 0xffe) >> 1); machineCodeCopy(writeTarget - 1, &newInstruction, sizeof(uint16_t)); } @@ -3109,14 +3240,16 @@ class ARMv7Assembler { static void linkJumpT3(Condition cond, uint16_t* writeTarget, const uint16_t* instruction, void* target) { // FIMXE: this should be up in the MacroAssembler layer. :-( - ASSERT(!(reinterpret_cast(instruction) & 1)); - ASSERT(!(reinterpret_cast(target) & 1)); - ASSERT(canBeJumpT3(instruction, target)); + ARMV7_ASM_ASSERT(!(reinterpret_cast(instruction) & 1)); + ARMV7_ASM_ASSERT(!(reinterpret_cast(writeTarget) & 1)); + ARMV7_ASM_ASSERT(cond < ConditionAL); + ARMV7_ASM_ASSERT(!(reinterpret_cast(target) & 1)); + ARMV7_ASM_ASSERT(canBeJumpT3(instruction, target)); intptr_t relative = reinterpret_cast(target) - (reinterpret_cast(instruction)); // All branch offsets should be an even distance. - ASSERT(!(relative & 1)); + ARMV7_ASM_ASSERT(!(relative & 1)); uint16_t instructions[2]; instructions[0] = OP_B_T3a | ((relative & 0x100000) >> 10) | ((cond & 0xf) << 6) | ((relative & 0x3f000) >> 12); instructions[1] = OP_B_T3b | ((relative & 0x80000) >> 8) | ((relative & 0x40000) >> 5) | ((relative & 0xffe) >> 1); @@ -3127,9 +3260,10 @@ class ARMv7Assembler { static void linkJumpT4(uint16_t* writeTarget, const uint16_t* instruction, void* target, BranchWithLink link) { // FIMXE: this should be up in the MacroAssembler layer. :-( - ASSERT(!(reinterpret_cast(instruction) & 1)); - ASSERT(!(reinterpret_cast(target) & 1)); - ASSERT(canBeJumpT4(instruction, target)); + ARMV7_ASM_ASSERT(!(reinterpret_cast(instruction) & 1)); + ARMV7_ASM_ASSERT(!(reinterpret_cast(writeTarget) & 1)); + ARMV7_ASM_ASSERT(!(reinterpret_cast(target) & 1)); + ARMV7_ASM_ASSERT(canBeJumpT4(instruction, target)); intptr_t relative = reinterpret_cast(target) - (reinterpret_cast(instruction)); // ARM encoding for the top two bits below the sign bit is 'peculiar'. @@ -3137,7 +3271,7 @@ class ARMv7Assembler { relative ^= 0xC00000; // All branch offsets should be an even distance. - ASSERT(!(relative & 1)); + ARMV7_ASM_ASSERT(!(relative & 1)); uint16_t instructions[2]; instructions[0] = OP_B_T4a | ((relative & 0x1000000) >> 14) | ((relative & 0x3ff000) >> 12); instructions[1] = OP_B_T4b | (static_cast(link) << 14) | ((relative & 0x800000) >> 10) | ((relative & 0x400000) >> 11) | ((relative & 0xffe) >> 1); @@ -3148,8 +3282,10 @@ class ARMv7Assembler { static void linkConditionalJumpT4(Condition cond, uint16_t* writeTarget, const uint16_t* instruction, void* target) { // FIMXE: this should be up in the MacroAssembler layer. :-( - ASSERT(!(reinterpret_cast(instruction) & 1)); - ASSERT(!(reinterpret_cast(target) & 1)); + ARMV7_ASM_ASSERT(!(reinterpret_cast(instruction) & 1)); + ARMV7_ASM_ASSERT(!(reinterpret_cast(writeTarget) & 1)); + ARMV7_ASM_ASSERT(cond <= ConditionAL); + ARMV7_ASM_ASSERT(!(reinterpret_cast(target) & 1)); uint16_t newInstruction = ifThenElse(cond) | OP_IT; machineCodeCopy(writeTarget - 3, &newInstruction, sizeof(uint16_t)); @@ -3160,9 +3296,9 @@ class ARMv7Assembler { static void linkBX(uint16_t* writeTarget, const uint16_t* instruction, void* target) { // FIMXE: this should be up in the MacroAssembler layer. :-( - ASSERT_UNUSED(instruction, !(reinterpret_cast(instruction) & 1)); - ASSERT(!(reinterpret_cast(writeTarget) & 1)); - ASSERT(!(reinterpret_cast(target) & 1)); + ARMV7_ASM_ASSERT_UNUSED(instruction, !(reinterpret_cast(instruction) & 1)); + ARMV7_ASM_ASSERT(!(reinterpret_cast(writeTarget) & 1)); + ARMV7_ASM_ASSERT(!(reinterpret_cast(target) & 1)); const uint16_t JUMP_TEMPORARY_REGISTER = ARMRegisters::ip; ARMThumbImmediate lo16 = ARMThumbImmediate::makeUInt16(static_cast(reinterpret_cast(target) + 1)); @@ -3181,8 +3317,10 @@ class ARMv7Assembler { static void linkConditionalBX(Condition cond, uint16_t* writeTarget, const uint16_t* instruction, void* target) { // FIMXE: this should be up in the MacroAssembler layer. :-( - ASSERT(!(reinterpret_cast(instruction) & 1)); - ASSERT(!(reinterpret_cast(target) & 1)); + ARMV7_ASM_ASSERT(!(reinterpret_cast(instruction) & 1)); + ARMV7_ASM_ASSERT(!(reinterpret_cast(writeTarget) & 1)); + ARMV7_ASM_ASSERT(cond <= ConditionAL); + ARMV7_ASM_ASSERT(!(reinterpret_cast(target) & 1)); linkBX(writeTarget, instruction, target); uint16_t newInstruction = ifThenElse(cond, true, true) | OP_IT; @@ -3192,10 +3330,10 @@ class ARMv7Assembler { static void linkJumpAbsolute(uint16_t* writeTarget, const uint16_t* instruction, void* target) { // FIMXE: this should be up in the MacroAssembler layer. :-( - ASSERT(!(reinterpret_cast(instruction) & 1)); - ASSERT(!(reinterpret_cast(target) & 1)); + ARMV7_ASM_ASSERT(!(reinterpret_cast(instruction) & 1)); + ARMV7_ASM_ASSERT(!(reinterpret_cast(target) & 1)); - ASSERT((isMOV_imm_T3(instruction - 5) && isMOVT(instruction - 3) && isBX(instruction - 1)) + ARMV7_ASM_ASSERT((isMOV_imm_T3(instruction - 5) && isMOVT(instruction - 3) && isBX(instruction - 1)) || (isNOP_T1(instruction - 5) && isNOP_T2(instruction - 4) && isB(instruction - 2))); if (canBeJumpT4(instruction, target)) { @@ -3227,10 +3365,10 @@ class ARMv7Assembler { static void linkBranch(uint16_t* from, const uint16_t* fromInstruction, void* to, BranchWithLink link) { - ASSERT(isEven(fromInstruction)); - ASSERT(isEven(from)); - ASSERT(isEven(to)); - ASSERT(link == BranchWithLink::Yes ? isBL(from - 2) : isB(from - 2)); + ARMV7_ASM_ASSERT(isEven(fromInstruction)); + ARMV7_ASM_ASSERT(isEven(from)); + ARMV7_ASM_ASSERT(isEven(to)); + ARMV7_ASM_ASSERT(link == BranchWithLink::Yes ? isBL(from - 2) : isB(from - 2)); intptr_t offset = std::bit_cast(to) - std::bit_cast(fromInstruction); #if ENABLE(JUMP_ISLANDS) @@ -3365,8 +3503,8 @@ class ARMv7Assembler { // Operands provide 5 bit values of the form Aaaaa, Bbbbb, Ccccc. ALWAYS_INLINE void vfpOp(OpcodeID1 op1, OpcodeID2 op2, bool size, VFPOperand a, VFPOperand b, VFPOperand c) { - ASSERT(!(op1 & 0x004f)); - ASSERT(!(op2 & 0xf1af)); + ARMV7_ASM_ASSERT(!(op1 & 0x004f)); + ARMV7_ASM_ASSERT(!(op2 & 0xf1af)); m_buffer.putShort(op1 | b.bits1() << 6 | a.bits4()); m_buffer.putShort(op2 | b.bits4() << 12 | size << 8 | a.bits1() << 7 | c.bits1() << 5 | c.bits4()); } @@ -3376,13 +3514,16 @@ class ARMv7Assembler { ALWAYS_INLINE void vfpMemOp(OpcodeID1 op1, OpcodeID2 op2, bool size, RegisterID rn, VFPOperand rd, int32_t imm) { bool up = true; + ARMV7_ASM_ASSERT(!(op1 & 0x004f)); + ARMV7_ASM_ASSERT(imm >= -1020 && imm <= 1020); + ARMV7_ASM_ASSERT(!(imm & 3)); if (imm < 0) { imm = -imm; up = false; } uint32_t offset = imm; - ASSERT(!(offset & ~0x3fc)); + ARMV7_ASM_ASSERT(!(offset & ~0x3fc)); offset >>= 2; m_buffer.putShort(op1 | (up << 7) | rd.bits1() << 6 | rn); diff --git a/Source/JavaScriptCore/assembler/MacroAssemblerARMv7.h b/Source/JavaScriptCore/assembler/MacroAssemblerARMv7.h index 2cdf4a0ccd3d..b9be807b2dbc 100644 --- a/Source/JavaScriptCore/assembler/MacroAssemblerARMv7.h +++ b/Source/JavaScriptCore/assembler/MacroAssemblerARMv7.h @@ -1,6 +1,7 @@ /* * Copyright (C) 2009-2022 Apple Inc. All rights reserved. * Copyright (C) 2010 University of Szeged + * Copyright (C) 2026 Igalia S.L. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -35,6 +36,21 @@ namespace JSC { +#if !defined(VALIDATE_ARMV7_MACRO_ASSEMBLER) +#define VALIDATE_ARMV7_MACRO_ASSEMBLER ASSERT_ENABLED +#endif + +static constexpr bool validateARMv7MacroAssembler = VALIDATE_ARMV7_MACRO_ASSEMBLER; + +#define ARMV7_MASM_ASSERT(assertion) do { \ + if constexpr (validateARMv7MacroAssembler) { \ + if (UNLIKELY(!(assertion))) { \ + WTFReportAssertionFailure(__FILE__, __LINE__, WTF_PRETTY_FUNCTION, #assertion); \ + CRASH(); \ + } \ + } \ + } while (0) + using Assembler = TARGET_ASSEMBLER; class MacroAssemblerARMv7 : public AbstractMacroAssembler { @@ -164,6 +180,57 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { LessThanOrEqual = ARMv7Assembler::ConditionLE }; + static constexpr ARMv7Assembler::Condition armV7ConditionForHigh32(RelationalCondition cond) + { + switch (cond) { + case GreaterThan: + case GreaterThanOrEqual: + return ARMv7Assembler::ConditionGT; + case LessThan: + case LessThanOrEqual: + return ARMv7Assembler::ConditionLT; + case Above: + case AboveOrEqual: + return ARMv7Assembler::ConditionHI; + case Below: + case BelowOrEqual: + return ARMv7Assembler::ConditionLO; + case NotEqual: + return ARMv7Assembler::ConditionNE; + case Equal: + // Equal can never be determined from high alone (needs both parts to match) + return ARMv7Assembler::ConditionInvalid; + default: + RELEASE_ASSERT_NOT_REACHED(); + return ARMv7Assembler::ConditionInvalid; + } + } + + static constexpr ARMv7Assembler::Condition armV7ConditionForLow32(RelationalCondition cond) + { + switch (cond) { + case GreaterThan: + case Above: + return ARMv7Assembler::ConditionHI; + case GreaterThanOrEqual: + case AboveOrEqual: + return ARMv7Assembler::ConditionHS; + case LessThan: + case Below: + return ARMv7Assembler::ConditionLO; + case LessThanOrEqual: + case BelowOrEqual: + return ARMv7Assembler::ConditionLS; + case NotEqual: + return ARMv7Assembler::ConditionNE; + case Equal: + return ARMv7Assembler::ConditionEQ; + default: + RELEASE_ASSERT_NOT_REACHED(); + return ARMv7Assembler::ConditionInvalid; + } + } + enum ResultCondition { Carry = ARMv7Assembler::ConditionCS, Overflow = ARMv7Assembler::ConditionVS, @@ -218,6 +285,8 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void add32(AbsoluteAddress src, RegisterID dest) { + ARMV7_MASM_ASSERT(dest != dataTempRegister); + ARMV7_MASM_ASSERT(dest != addressTempRegister); load32(setupArmAddress(src), dataTempRegister); add32(dataTempRegister, dest); } @@ -244,6 +313,8 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { return; } + ARMV7_MASM_ASSERT(src != dataTempRegister); + ARMV7_MASM_ASSERT(src != dataTempRegister); move(imm, dataTempRegister); m_assembler.add(dest, src, dataTempRegister); } @@ -256,6 +327,8 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void add32(Address src, RegisterID dest) { + ARMV7_MASM_ASSERT(dest != dataTempRegister); + ARMV7_MASM_ASSERT(dest != addressTempRegister); // load32 will invalidate the cachedDataTempRegister() for us load32(src, dataTempRegister); add32(dataTempRegister, dest); @@ -276,9 +349,14 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void getEffectiveAddress(BaseIndex address, RegisterID dest) { + ARMV7_MASM_ASSERT(address.index != addressTempRegister); RegisterID scratch = getCachedAddressTempRegisterIDAndInvalidate(); m_assembler.lsl(scratch, address.index, static_cast(address.scale)); m_assembler.add(dest, address.base, scratch); + if (dest == addressTempRegister) + invalidateCachedAddressTempRegister(); + else if (dest == dataTempRegister) + cachedDataTempRegister().invalidate(); if (address.offset) add32(TrustedImm32(address.offset), dest); } @@ -292,14 +370,15 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { { RegisterID scratch = getCachedDataTempRegisterIDAndInvalidate(); - load32(setupArmAddress(address), scratch); + move(TrustedImmPtr(address.m_ptr), addressTempRegister); + m_assembler.ldr(scratch, addressTempRegister, ARMThumbImmediate::makeUInt12(0)); ARMThumbImmediate armImm = ARMThumbImmediate::makeEncodedImm(imm.m_value); if (armImm.isValid()) m_assembler.add_S(scratch, scratch, armImm); else { move(imm, addressTempRegister); m_assembler.add_S(scratch, scratch, addressTempRegister); - move(TrustedImmPtr(address.m_ptr), addressTempRegister); + moveFixedWidthEncoding(TrustedImm32(TrustedImmPtr(address.m_ptr)), addressTempRegister); } m_assembler.str(scratch, addressTempRegister, ARMThumbImmediate::makeUInt12(0)); @@ -310,6 +389,8 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void add64(RegisterID op1Hi, RegisterID op1Lo, RegisterID op2Hi, RegisterID op2Lo, RegisterID destHi, RegisterID destLo) { + ARMV7_MASM_ASSERT(op1Hi != dataTempRegister); + ARMV7_MASM_ASSERT(op2Hi != dataTempRegister); RegisterID scratch = getCachedDataTempRegisterIDAndInvalidate(); m_assembler.add_S(scratch, op1Lo, op2Lo); m_assembler.adc(destHi, op1Hi, op2Hi); @@ -318,6 +399,8 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void and16(Address src, RegisterID dest) { + ARMV7_MASM_ASSERT(dest != dataTempRegister); + ARMV7_MASM_ASSERT(dest != addressTempRegister); load16(src, dataTempRegister); and32(dataTempRegister, dest); } @@ -341,6 +424,8 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { return; } + ARMV7_MASM_ASSERT(src != dataTempRegister); + ARMV7_MASM_ASSERT(src != dataTempRegister); move(imm, dataTempRegister); m_assembler.ARM_and(dest, src, dataTempRegister); } @@ -357,6 +442,8 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void and32(Address src, RegisterID dest) { + ARMV7_MASM_ASSERT(dest != dataTempRegister); + ARMV7_MASM_ASSERT(dest != addressTempRegister); load32(src, dataTempRegister); and32(dataTempRegister, dest); } @@ -372,12 +459,148 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { m_assembler.clz(dest, dest); } + void countLeadingZeros64(RegisterID srcHi, RegisterID srcLo, RegisterID destHi, RegisterID destLo) + { + ARMV7_MASM_ASSERT(destHi != destLo); + ARMV7_MASM_ASSERT(destLo != srcLo || srcHi != dataTempRegister); + if (destLo != srcLo) { + m_assembler.clz(destLo, srcHi); + m_assembler.cmp(destLo, ARMThumbImmediate::makeEncodedImm(32)); + Jump done = makeBranch(ARMv7Assembler::ConditionNE); + m_assembler.clz(destLo, srcLo); + m_assembler.add(destLo, destLo, ARMThumbImmediate::makeEncodedImm(32)); + done.link(this); + xor32(destHi, destHi); + } else { + RegisterID scratch = getCachedDataTempRegisterIDAndInvalidate(); + move(srcLo, scratch); + m_assembler.clz(destLo, srcHi); + m_assembler.cmp(destLo, ARMThumbImmediate::makeEncodedImm(32)); + Jump done = makeBranch(ARMv7Assembler::ConditionNE); + m_assembler.clz(destLo, scratch); + m_assembler.add(destLo, destLo, ARMThumbImmediate::makeEncodedImm(32)); + done.link(this); + xor32(destHi, destHi); + } + } + + void countTrailingZeros64(RegisterID srcHi, RegisterID srcLo, RegisterID destHi, RegisterID destLo) + { + ARMV7_MASM_ASSERT(destHi != destLo); + ARMV7_MASM_ASSERT(destLo != srcHi || srcLo != dataTempRegister); + if (destLo != srcHi) { + countTrailingZeros32(srcLo, destLo); + m_assembler.cmp(destLo, ARMThumbImmediate::makeEncodedImm(32)); + Jump done = makeBranch(ARMv7Assembler::ConditionNE); + countTrailingZeros32(srcHi, destLo); + m_assembler.add(destLo, destLo, ARMThumbImmediate::makeEncodedImm(32)); + done.link(this); + xor32(destHi, destHi); + } else { + RegisterID scratch = getCachedDataTempRegisterIDAndInvalidate(); + move(srcHi, scratch); + countTrailingZeros32(srcLo, destLo); + m_assembler.cmp(destLo, ARMThumbImmediate::makeEncodedImm(32)); + Jump done = makeBranch(ARMv7Assembler::ConditionNE); + countTrailingZeros32(scratch, destLo); + m_assembler.add(destLo, destLo, ARMThumbImmediate::makeEncodedImm(32)); + done.link(this); + xor32(destHi, destHi); + } + } + + void compare64(RelationalCondition cond, RegisterID lhsHi, RegisterID lhsLo, RegisterID rhsHi, RegisterID rhsLo, RegisterID dest) + { + ARMV7_MASM_ASSERT(lhsHi != dataTempRegister); + ARMV7_MASM_ASSERT(lhsLo != dataTempRegister); + ARMV7_MASM_ASSERT(rhsHi != dataTempRegister); + ARMV7_MASM_ASSERT(rhsLo != dataTempRegister); + ARMV7_MASM_ASSERT(lhsHi != ARMRegisters::sp && lhsLo != ARMRegisters::sp); + ARMV7_MASM_ASSERT(rhsHi != ARMRegisters::sp && rhsLo != ARMRegisters::sp); + ARMV7_MASM_ASSERT(dest != ARMRegisters::sp && dest != ARMRegisters::pc); + + if (cond == RelationalCondition::Equal || cond == RelationalCondition::NotEqual) { + // For Equal/NotEqual, we can optimize to only set one value conditionally + // NotEqual: default to 1, change to 0 only if both parts equal + // Equal: default to 0, change to 1 only if both parts equal + if (dest != lhsHi && dest != rhsHi && dest != lhsLo && dest != rhsLo) { + m_assembler.mov(dest, ARMThumbImmediate::makeEncodedImm(cond == RelationalCondition::NotEqual ? 1 : 0)); + m_assembler.cmp(lhsHi, rhsHi); + Jump done = makeBranch(ARMv7Assembler::ConditionNE); + m_assembler.cmp(lhsLo, rhsLo); + // Only need to set the "opposite" value when both parts match + m_assembler.it(ARMv7Assembler::ConditionEQ); + m_assembler.mov(dest, ARMThumbImmediate::makeEncodedImm(cond == RelationalCondition::NotEqual ? 0 : 1)); + done.link(this); + } else { + RegisterID scratch = getCachedDataTempRegisterIDAndInvalidate(); + m_assembler.mov(scratch, ARMThumbImmediate::makeEncodedImm(cond == RelationalCondition::NotEqual ? 1 : 0)); + m_assembler.cmp(lhsHi, rhsHi); + Jump done = makeBranch(ARMv7Assembler::ConditionNE); + m_assembler.cmp(lhsLo, rhsLo); + m_assembler.it(ARMv7Assembler::ConditionEQ); + m_assembler.mov(scratch, ARMThumbImmediate::makeEncodedImm(cond == RelationalCondition::NotEqual ? 0 : 1)); + done.link(this); + move(scratch, dest); + } + return; + } + + ARMv7Assembler::Condition hiCondition = armV7ConditionForHigh32(cond); + ARMv7Assembler::Condition loCondition = armV7ConditionForLow32(cond); + ARMV7_MASM_ASSERT(hiCondition != ARMv7Assembler::ConditionInvalid); + ARMV7_MASM_ASSERT(loCondition != ARMv7Assembler::ConditionInvalid); + ARMV7_MASM_ASSERT(hiCondition != ARMv7Assembler::ConditionInvalid); + ARMV7_MASM_ASSERT(loCondition != ARMv7Assembler::ConditionInvalid); + + if (dest != lhsLo && dest != rhsLo && dest != lhsHi && dest != rhsHi) { + // No aliasing - use ITE blocks with 1 branch + m_assembler.cmp(lhsHi, rhsHi); + m_assembler.it(hiCondition, false); + m_assembler.mov(dest, ARMThumbImmediate::makeEncodedImm(1)); + m_assembler.mov(dest, ARMThumbImmediate::makeEncodedImm(0)); + + Jump done = makeBranch(ARMv7Assembler::ConditionNE); + + m_assembler.cmp(lhsLo, rhsLo); + m_assembler.it(loCondition, false); + m_assembler.mov(dest, ARMThumbImmediate::makeEncodedImm(1)); + m_assembler.mov(dest, ARMThumbImmediate::makeEncodedImm(0)); + + done.link(this); + } else { + // dest aliases with source - use scratch + RegisterID scratch = getCachedDataTempRegisterIDAndInvalidate(); + + m_assembler.cmp(lhsHi, rhsHi); + m_assembler.it(hiCondition, false); + m_assembler.mov(scratch, ARMThumbImmediate::makeEncodedImm(1)); + m_assembler.mov(scratch, ARMThumbImmediate::makeEncodedImm(0)); + + Jump done = makeBranch(ARMv7Assembler::ConditionNE); + + m_assembler.cmp(lhsLo, rhsLo); + m_assembler.it(loCondition, false); + m_assembler.mov(scratch, ARMThumbImmediate::makeEncodedImm(1)); + m_assembler.mov(scratch, ARMThumbImmediate::makeEncodedImm(0)); + + done.link(this); + move(scratch, dest); + } + } + + void lshiftUnchecked(RegisterID src, RegisterID shiftAmount, RegisterID dest) + { + m_assembler.lsl(dest, src, shiftAmount); + } + void lshift32(RegisterID src, RegisterID shiftAmount, RegisterID dest) { + ARMV7_MASM_ASSERT(src != dataTempRegister); RegisterID scratch = getCachedDataTempRegisterIDAndInvalidate(); // Clamp the shift to the range 0..31 ARMThumbImmediate armImm = ARMThumbImmediate::makeEncodedImm(0x1f); - ASSERT(armImm.isValid()); + ARMV7_MASM_ASSERT(armImm.isValid()); m_assembler.ARM_and(scratch, shiftAmount, armImm); m_assembler.lsl(dest, src, scratch); @@ -385,11 +608,15 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void lshift32(RegisterID src, TrustedImm32 imm, RegisterID dest) { - m_assembler.lsl(dest, src, imm.m_value & 0x1f); + if (!(imm.m_value & 0x1f)) + move(src, dest); + else + m_assembler.lsl(dest, src, imm.m_value & 0x1f); } void lshift32(TrustedImm32 imm, RegisterID shiftAmount, RegisterID dest) { + ARMV7_MASM_ASSERT(dest != dataTempRegister); // Clamp the shift to the range 0..31 m_assembler.ARM_and(dest, shiftAmount, ARMThumbImmediate::makeEncodedImm(0x1f)); move(imm, getCachedDataTempRegisterIDAndInvalidate()); @@ -408,18 +635,22 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void mul32(RegisterID src, RegisterID dest) { + ARMV7_MASM_ASSERT(dest != dataTempRegister); RegisterID scratch = getCachedDataTempRegisterIDAndInvalidate(); m_assembler.smull(dest, scratch, dest, src); } void mul32(RegisterID left, RegisterID right, RegisterID dest) { + ARMV7_MASM_ASSERT(dest != dataTempRegister); RegisterID scratch = getCachedDataTempRegisterIDAndInvalidate(); m_assembler.smull(dest, scratch, left, right); } void mul32(TrustedImm32 imm, RegisterID src, RegisterID dest) { + ARMV7_MASM_ASSERT(src != dataTempRegister); + ARMV7_MASM_ASSERT(dest != dataTempRegister); move(imm, dataTempRegister); cachedDataTempRegister().invalidate(); m_assembler.smull(dest, dataTempRegister, src, dataTempRegister); @@ -446,7 +677,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { load8(setupArmAddress(address), dataTempRegister); if (armImm.isValid()) { m_assembler.orr(dataTempRegister, dataTempRegister, armImm); - store8(dataTempRegister, Address(addressTempRegister)); + store8(dataTempRegister, address.m_ptr); } else { move(imm, addressTempRegister); m_assembler.orr(dataTempRegister, dataTempRegister, addressTempRegister); @@ -461,7 +692,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { load16(setupArmAddress(dest), dataTempRegister); if (armImm.isValid()) { m_assembler.orr(dataTempRegister, dataTempRegister, armImm); - store16(dataTempRegister, Address(addressTempRegister)); + store16(dataTempRegister, dest.m_ptr); } else { move(imm, addressTempRegister); m_assembler.orr(dataTempRegister, dataTempRegister, addressTempRegister); @@ -472,9 +703,11 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void or16(RegisterID mask, AbsoluteAddress dest) { + ARMV7_MASM_ASSERT(mask != dataTempRegister); + ARMV7_MASM_ASSERT(mask != addressTempRegister); load16(setupArmAddress(dest), dataTempRegister); m_assembler.orr(dataTempRegister, dataTempRegister, mask); - store16(dataTempRegister, Address(addressTempRegister)); + store16(dataTempRegister, dest.m_ptr); } void or32(RegisterID src, RegisterID dest) @@ -484,13 +717,17 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void or32(RegisterID src, AbsoluteAddress dest) { + ARMV7_MASM_ASSERT(src != dataTempRegister); + ARMV7_MASM_ASSERT(src != addressTempRegister); load32(setupArmAddress(dest), dataTempRegister); or32(src, dataTempRegister); - store32(dataTempRegister, Address(addressTempRegister)); + store32(dataTempRegister, dest.m_ptr); } void or32(RegisterID src, Address dest) { + ARMV7_MASM_ASSERT(src != dataTempRegister); + ARMV7_MASM_ASSERT(src != addressTempRegister); load32(dest, dataTempRegister); or32(src, dataTempRegister); store32(dataTempRegister, dest); @@ -502,7 +739,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { load32(setupArmAddress(address), dataTempRegister); if (armImm.isValid()) { m_assembler.orr(dataTempRegister, dataTempRegister, armImm); - store32(dataTempRegister, Address(addressTempRegister)); + store32(dataTempRegister, address.m_ptr); } else { move(imm, addressTempRegister); m_assembler.orr(dataTempRegister, dataTempRegister, addressTempRegister); @@ -534,7 +771,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { if (armImm.isValid()) m_assembler.orr(dest, src, armImm); else { - ASSERT(src != dataTempRegister); + ARMV7_MASM_ASSERT(src != dataTempRegister); move(imm, dataTempRegister); m_assembler.orr(dest, src, dataTempRegister); } @@ -560,7 +797,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void rotateRight32(RegisterID src, TrustedImm32 imm, RegisterID dest) { - if (!imm.m_value) + if (!(imm.m_value & 0x1f)) move(src, dest); else m_assembler.ror(dest, src, imm.m_value & 0x1f); @@ -573,26 +810,35 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void rotateLeft32(RegisterID src, RegisterID shift, RegisterID dest) { + ARMV7_MASM_ASSERT(src != dataTempRegister); RegisterID scratch = getCachedDataTempRegisterIDAndInvalidate(); m_assembler.ARM_and(scratch, shift, ARMThumbImmediate::makeEncodedImm(0x1f)); - m_assembler.sub(scratch, ARMThumbImmediate::makeUInt12(32), scratch); + m_assembler.sub(scratch, ARMThumbImmediate::makeEncodedImm(32), scratch); m_assembler.ror(dest, src, scratch); } void rotateLeft32(RegisterID src, TrustedImm32 shift, RegisterID dest) { + ARMV7_MASM_ASSERT(src != dataTempRegister); RegisterID scratch = getCachedDataTempRegisterIDAndInvalidate(); move(shift, scratch); m_assembler.ARM_and(scratch, scratch, ARMThumbImmediate::makeEncodedImm(0x1f)); - m_assembler.sub(scratch, ARMThumbImmediate::makeUInt12(32), scratch); + m_assembler.sub(scratch, ARMThumbImmediate::makeEncodedImm(32), scratch); m_assembler.ror(dest, src, scratch); } + + void rshiftUnchecked(RegisterID src, RegisterID shiftAmount, RegisterID dest) + { + m_assembler.asr(dest, src, shiftAmount); + } + void rshift32(RegisterID src, RegisterID shiftAmount, RegisterID dest) { + ARMV7_MASM_ASSERT(src != dataTempRegister); RegisterID scratch = getCachedDataTempRegisterIDAndInvalidate(); // Clamp the shift to the range 0..31 ARMThumbImmediate armImm = ARMThumbImmediate::makeEncodedImm(0x1f); - ASSERT(armImm.isValid()); + ARMV7_MASM_ASSERT(armImm.isValid()); m_assembler.ARM_and(scratch, shiftAmount, armImm); m_assembler.asr(dest, src, scratch); @@ -610,23 +856,38 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { { rshift32(dest, shiftAmount, dest); } - + void rshift32(TrustedImm32 imm, RegisterID dest) { rshift32(dest, imm, dest); } + void rshift32(TrustedImm32 imm, RegisterID shiftAmount, RegisterID dest) + { + ARMV7_MASM_ASSERT(dest != dataTempRegister); + // Clamp the shift to the range 0..31 + m_assembler.ARM_and(dest, shiftAmount, ARMThumbImmediate::makeEncodedImm(0x1f)); + move(imm, getCachedDataTempRegisterIDAndInvalidate()); + m_assembler.asr(dest, dataTempRegister, dest); + } + + void urshiftUnchecked(RegisterID src, RegisterID shiftAmount, RegisterID dest) + { + m_assembler.lsr(dest, src, shiftAmount); + } + void urshift32(RegisterID src, RegisterID shiftAmount, RegisterID dest) { + ARMV7_MASM_ASSERT(src != dataTempRegister); RegisterID scratch = getCachedDataTempRegisterIDAndInvalidate(); // Clamp the shift to the range 0..31 ARMThumbImmediate armImm = ARMThumbImmediate::makeEncodedImm(0x1f); - ASSERT(armImm.isValid()); + ARMV7_MASM_ASSERT(armImm.isValid()); m_assembler.ARM_and(scratch, shiftAmount, armImm); - + m_assembler.lsr(dest, src, scratch); } - + void urshift32(RegisterID src, TrustedImm32 imm, RegisterID dest) { if (!(imm.m_value & 0x1f)) @@ -639,7 +900,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { { urshift32(dest, shiftAmount, dest); } - + void urshift32(TrustedImm32 imm, RegisterID dest) { urshift32(dest, imm, dest); @@ -648,6 +909,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void addUnsignedRightShift32(RegisterID src1, RegisterID src2, TrustedImm32 amount, RegisterID dest) { // dest = src1 + (src2 >> amount) + ARMV7_MASM_ASSERT(src1 != dataTempRegister); urshift32(src2, amount, dataTempRegister); add32(src1, dataTempRegister, dest); } @@ -668,6 +930,8 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { if (armImm.isValid()) m_assembler.sub(dest, left, armImm); else { + ARMV7_MASM_ASSERT(left != dataTempRegister); + ARMV7_MASM_ASSERT(left != dataTempRegister); move(right, dataTempRegister); m_assembler.sub(dest, left, dataTempRegister); } @@ -679,11 +943,26 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { if (armImm.isValid()) m_assembler.sub(dest, dest, armImm); else { + ARMV7_MASM_ASSERT(dest != dataTempRegister); + ARMV7_MASM_ASSERT(dest != dataTempRegister); move(imm, dataTempRegister); m_assembler.sub(dest, dest, dataTempRegister); } } + void sub32(TrustedImm32 imm, RegisterID src, RegisterID dest) + { + ARMThumbImmediate armImm = ARMThumbImmediate::makeEncodedImm(imm.m_value); + if (armImm.isValid()) + m_assembler.sub(dest, armImm, src); + else { + ARMV7_MASM_ASSERT(src != dataTempRegister); + ARMV7_MASM_ASSERT(src != dataTempRegister); + move(imm, dataTempRegister); + m_assembler.sub(dest, dataTempRegister, src); + } + } + void sub32(TrustedImm32 imm, Address address) { load32(address, dataTempRegister); @@ -703,6 +982,8 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void sub32(Address src, RegisterID dest) { + ARMV7_MASM_ASSERT(dest != dataTempRegister); + ARMV7_MASM_ASSERT(dest != addressTempRegister); load32(src, dataTempRegister); sub32(dataTempRegister, dest); } @@ -726,6 +1007,8 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void sub64(RegisterID leftHi, RegisterID leftLo, RegisterID rightHi, RegisterID rightLo, RegisterID destHi, RegisterID destLo) { + ARMV7_MASM_ASSERT(leftHi != dataTempRegister); + ARMV7_MASM_ASSERT(rightHi != dataTempRegister); RegisterID scratch = getCachedDataTempRegisterIDAndInvalidate(); m_assembler.sub_S(scratch, leftLo, rightLo); m_assembler.sbc(destHi, leftHi, rightHi); @@ -748,6 +1031,8 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { if (armImm.isValid()) m_assembler.eor(dest, src, armImm); else { + ARMV7_MASM_ASSERT(src != dataTempRegister); + ARMV7_MASM_ASSERT(src != dataTempRegister); move(imm, dataTempRegister); m_assembler.eor(dest, src, dataTempRegister); } @@ -760,6 +1045,8 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void xor32(Address src, RegisterID dest) { + ARMV7_MASM_ASSERT(dest != dataTempRegister); + ARMV7_MASM_ASSERT(dest != addressTempRegister); load32(src, dataTempRegister); xor32(dataTempRegister, dest); } @@ -806,10 +1093,10 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { m_assembler.ldr(dest, address.base, address.u.index, address.u.scale); else if (address.u.offset >= 0) { ARMThumbImmediate armImm = ARMThumbImmediate::makeUInt12(address.u.offset); - ASSERT(armImm.isValid()); + ARMV7_MASM_ASSERT(armImm.isValid()); m_assembler.ldr(dest, address.base, armImm); } else { - ASSERT(address.u.offset >= -255); + ARMV7_MASM_ASSERT(address.u.offset >= -255); m_assembler.ldr(dest, address.base, address.u.offset, true, false); } } @@ -825,10 +1112,10 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { m_assembler.ldrh(dest, address.base, address.u.index, address.u.scale); else if (address.u.offset >= 0) { ARMThumbImmediate armImm = ARMThumbImmediate::makeUInt12(address.u.offset); - ASSERT(armImm.isValid()); + ARMV7_MASM_ASSERT(armImm.isValid()); m_assembler.ldrh(dest, address.base, armImm); } else { - ASSERT(address.u.offset >= -255); + ARMV7_MASM_ASSERT(address.u.offset >= -255); m_assembler.ldrh(dest, address.base, address.u.offset, true, false); } } @@ -844,10 +1131,10 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { m_assembler.ldrsh(dest, address.base, address.u.index, address.u.scale); else if (address.u.offset >= 0) { ARMThumbImmediate armImm = ARMThumbImmediate::makeUInt12(address.u.offset); - ASSERT(armImm.isValid()); + ARMV7_MASM_ASSERT(armImm.isValid()); m_assembler.ldrsh(dest, address.base, armImm); } else { - ASSERT(address.u.offset >= -255); + ARMV7_MASM_ASSERT(address.u.offset >= -255); m_assembler.ldrsh(dest, address.base, address.u.offset, true, false); } } @@ -863,10 +1150,10 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { m_assembler.ldrb(dest, address.base, address.u.index, address.u.scale); else if (address.u.offset >= 0) { ARMThumbImmediate armImm = ARMThumbImmediate::makeUInt12(address.u.offset); - ASSERT(armImm.isValid()); + ARMV7_MASM_ASSERT(armImm.isValid()); m_assembler.ldrb(dest, address.base, armImm); } else { - ASSERT(address.u.offset >= -255); + ARMV7_MASM_ASSERT(address.u.offset >= -255); m_assembler.ldrb(dest, address.base, address.u.offset, true, false); } } @@ -882,10 +1169,10 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { m_assembler.ldrsb(dest, address.base, address.u.index, address.u.scale); else if (address.u.offset >= 0) { ARMThumbImmediate armImm = ARMThumbImmediate::makeUInt12(address.u.offset); - ASSERT(armImm.isValid()); + ARMV7_MASM_ASSERT(armImm.isValid()); m_assembler.ldrsb(dest, address.base, armImm); } else { - ASSERT(address.u.offset >= -255); + ARMV7_MASM_ASSERT(address.u.offset >= -255); m_assembler.ldrsb(dest, address.base, address.u.offset, true, false); } } @@ -897,10 +1184,10 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { m_assembler.str(src, address.base, address.u.index, address.u.scale); else if (address.u.offset >= 0) { ARMThumbImmediate armImm = ARMThumbImmediate::makeUInt12(address.u.offset); - ASSERT(armImm.isValid()); + ARMV7_MASM_ASSERT(armImm.isValid()); m_assembler.str(src, address.base, armImm); } else { - ASSERT(address.u.offset >= -255); + ARMV7_MASM_ASSERT(address.u.offset >= -255); m_assembler.str(src, address.base, address.u.offset, true, false); } } @@ -912,10 +1199,10 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { m_assembler.strb(src, address.base, address.u.index, address.u.scale); else if (address.u.offset >= 0) { ARMThumbImmediate armImm = ARMThumbImmediate::makeUInt12(address.u.offset); - ASSERT(armImm.isValid()); + ARMV7_MASM_ASSERT(armImm.isValid()); m_assembler.strb(src, address.base, armImm); } else { - ASSERT(address.u.offset >= -255); + ARMV7_MASM_ASSERT(address.u.offset >= -255); m_assembler.strb(src, address.base, address.u.offset, true, false); } } @@ -926,10 +1213,10 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { m_assembler.strh(src, address.base, address.u.index, address.u.scale); else if (address.u.offset >= 0) { ARMThumbImmediate armImm = ARMThumbImmediate::makeUInt12(address.u.offset); - ASSERT(armImm.isValid()); + ARMV7_MASM_ASSERT(armImm.isValid()); m_assembler.strh(src, address.base, armImm); } else { - ASSERT(address.u.offset >= -255); + ARMV7_MASM_ASSERT(address.u.offset >= -255); m_assembler.strh(src, address.base, address.u.offset, true, false); } } @@ -981,8 +1268,13 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { ConvertibleLoadLabel convertibleLoadPtr(Address address, RegisterID dest) { ConvertibleLoadLabel result(this); - ASSERT(address.offset >= 0 && address.offset <= 255); + ARMV7_MASM_ASSERT(address.offset >= 0 && address.offset <= 255); m_assembler.ldrWide8BitImmediate(dest, address.base, address.offset); + + if (dest == addressTempRegister) + invalidateCachedAddressTempRegister(); + else if (dest == dataTempRegister) + cachedDataTempRegister().invalidate(); return result; } @@ -1030,7 +1322,14 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void load16(BaseIndex address, RegisterID dest) { + ARMV7_MASM_ASSERT(address.index != addressTempRegister); + ARMV7_MASM_ASSERT(address.scale <= TimesEight); m_assembler.ldrh(dest, makeBaseIndexBase(address), address.index, address.scale); + + if (dest == addressTempRegister) + invalidateCachedAddressTempRegister(); + else if (dest == dataTempRegister) + cachedDataTempRegister().invalidate(); } void load16SignedExtendTo32(BaseIndex address, RegisterID dest) @@ -1044,9 +1343,15 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { if (armImm.isValid()) m_assembler.ldrh(dest, address.base, armImm); else { + ARMV7_MASM_ASSERT(address.base != dataTempRegister); move(TrustedImm32(address.offset), dataTempRegister); m_assembler.ldrh(dest, address.base, dataTempRegister); } + + if (dest == addressTempRegister) + invalidateCachedAddressTempRegister(); + else if (dest == dataTempRegister) + cachedDataTempRegister().invalidate(); } void loadAcq16(Address address, RegisterID dest) @@ -1092,18 +1397,14 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { } loadPair32(Address(scratch), dest1, dest2); } else { - ASSERT(dest1 != dest2); // If it is the same, ldp becomes illegal instruction. + ARMV7_MASM_ASSERT(dest1 != dest2); // If it is the same, ldp becomes illegal instruction. // Check if dest1 or dest2 aliases the base register to avoid UNPREDICTABLE ldrd behavior if (address.base == dest1) { - // Load high word first to avoid clobbering base register - ArmAddress highAddress(address.base, address.u.offset + 4); - load32(highAddress, dest2); + load32(Address(address.base, address.u.offset + 4), dest2); load32(address, dest1); } else if (address.base == dest2) { - // Load low word first to avoid clobbering base register load32(address, dest1); - ArmAddress highAddress(address.base, address.u.offset + 4); - load32(highAddress, dest2); + load32(Address(address.base, address.u.offset + 4), dest2); } else { int32_t absOffset = address.u.offset; if (absOffset < 0) @@ -1116,8 +1417,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { m_assembler.ldrd(dest1, dest2, address.base, address.u.offset, /* index: */ true, /* wback: */ false); } else { load32(address, dest1); - ArmAddress highAddress(address.base, address.u.offset + 4); - load32(highAddress, dest2); + load32(Address(address.base, address.u.offset + 4), dest2); } } } @@ -1125,16 +1425,32 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void loadPair32(Address address, RegisterID dest1, RegisterID dest2) { + ARMV7_MASM_ASSERT(dest1 != dest2); loadPair32(setupArmAddress(address), dest1, dest2); } + void loadPair32Unaligned(Address address, RegisterID dest1, RegisterID dest2) + { + ARMV7_MASM_ASSERT(dest1 != dest2); + if (address.base == dest1) { + load32(address.withOffset(4), dest2); + load32(address, dest1); + } else { + load32(address, dest1); + load32(address.withOffset(4), dest2); + } + } + void loadPair32(BaseIndex address, RegisterID dest1, RegisterID dest2) { + ARMV7_MASM_ASSERT(dest1 != dest2); + ARMV7_MASM_ASSERT(address.scale <= TimesEight); // Using r0-r7 can often be encoded with a shorter (16-bit vs 32-bit) instruction, so use // whichever destination register is in that range (if any) as the address temp register RegisterID scratch = dest1; if (dest1 >= ARMRegisters::r8) scratch = dest2; + ARMV7_MASM_ASSERT(scratch != addressTempRegister); if (address.scale == TimesOne) m_assembler.add(scratch, address.base, address.index); else { @@ -1146,7 +1462,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void loadPair64(RegisterID src, TrustedImm32 offset, FPRegisterID dest1, FPRegisterID dest2) { - ASSERT(dest1 != dest2); + ARMV7_MASM_ASSERT(dest1 != dest2); if ((dest2 == (dest1 + 1)) && !offset.m_value) { // Only emit a VLDMIA if the registers happen to be consecutive and // in the proper order and the offset happens to be zero. Otherwise, @@ -1161,26 +1477,47 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void loadLink8(Address addr, RegisterID dest) { - ASSERT(!addr.offset); + ARMV7_MASM_ASSERT(!addr.offset); m_assembler.ldrexb(dest, addr.base); + if (dest == addressTempRegister) + invalidateCachedAddressTempRegister(); + else if (dest == dataTempRegister) + cachedDataTempRegister().invalidate(); } void loadLink16(Address addr, RegisterID dest) { - ASSERT(!addr.offset); + ARMV7_MASM_ASSERT(!addr.offset); m_assembler.ldrexh(dest, addr.base); + if (dest == addressTempRegister) + invalidateCachedAddressTempRegister(); + else if (dest == dataTempRegister) + cachedDataTempRegister().invalidate(); } void loadLink32(Address addr, RegisterID dest) { - ASSERT(!addr.offset); + ARMV7_MASM_ASSERT(!addr.offset); m_assembler.ldrex(dest, addr.base, 0); + if (dest == addressTempRegister) + invalidateCachedAddressTempRegister(); + else if (dest == dataTempRegister) + cachedDataTempRegister().invalidate(); } void loadLinkPair32(Address addr, RegisterID destLo, RegisterID destHi) { - ASSERT(!addr.offset); + ARMV7_MASM_ASSERT(!addr.offset); + ARMV7_MASM_ASSERT(destLo != destHi); m_assembler.ldrexd(destLo, destHi, addr.base); + if (destLo == addressTempRegister) + invalidateCachedAddressTempRegister(); + else if (destLo == dataTempRegister) + cachedDataTempRegister().invalidate(); + if (destHi == addressTempRegister) + invalidateCachedAddressTempRegister(); + else if (destHi == dataTempRegister) + cachedDataTempRegister().invalidate(); } void storePair64(FPRegisterID src1, FPRegisterID src2, RegisterID dest, TrustedImm32 offset) @@ -1216,6 +1553,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { RegisterID scratch = addressTempRegister; if (armAddress.type == ArmAddress::HasIndex) scratch = dataTempRegister; + RELEASE_ASSERT(armAddress.base != scratch); move(imm, scratch); store32(scratch, armAddress); } @@ -1305,6 +1643,12 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { store16(dataTempRegister, address); } + void store16(TrustedImm32 imm, Address address) + { + move(imm, dataTempRegister); + store16(dataTempRegister, address); + } + void storeRel16(RegisterID src, Address address) { storeFence(); @@ -1325,26 +1669,11 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void storePair32(TrustedImm32 imm1, TrustedImm32 imm2, Address address) { - // We cannot re-use the two register version of `storePair32` defined - // below here because we can only use the `addressTempRegister` as a - // scratch register if the `strd` case is taken. int32_t absOffset = address.offset; if (absOffset < 0) absOffset = -absOffset; - if (!(absOffset & ~0x3fc)) { - RegisterID src1 = getCachedAddressTempRegisterIDAndInvalidate(); - move(imm1, src1); - RegisterID src2 = src1; - if (imm1.m_value != imm2.m_value) { - src2 = getCachedDataTempRegisterIDAndInvalidate(); - move(imm2, src2); - } - ASSERT(src1 != address.base && src2 != address.base); - m_assembler.strd(src1, src2, address.base, address.offset, /* index: */ true, /* wback: */ false); - } else { - store32(imm1, address); - store32(imm2, address.withOffset(4)); - } + store32(imm1, address); + store32(imm2, address.withOffset(4)); } void storePair32(RegisterID src1, RegisterID src2, RegisterID dest) @@ -1362,17 +1691,16 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { int32_t absOffset = address.offset; if (absOffset < 0) absOffset = -absOffset; - if (!(absOffset & ~0x3fc)) - m_assembler.strd(src1, src2, address.base, address.offset, /* index: */ true, /* wback: */ false); - else { - store32(src1, address); - store32(src2, address.withOffset(4)); - } + // strd does not support unaligned accesses on some chips, so we avoid it. + store32(src1, address); + store32(src2, address.withOffset(4)); } void storePair32(RegisterID src1, RegisterID src2, BaseIndex address) { - ASSERT(src1 != dataTempRegister && src2 != dataTempRegister); + ARMV7_MASM_ASSERT(src1 != dataTempRegister && src2 != dataTempRegister); + ARMV7_MASM_ASSERT(src1 != addressTempRegister && src2 != addressTempRegister); + ARMV7_MASM_ASSERT(address.scale <= TimesEight); RegisterID scratch = getCachedDataTempRegisterIDAndInvalidate(); // The 'addressTempRegister' might be used when the offset is wide, so use 'dataTempRegister' if (address.scale == TimesOne) @@ -1394,8 +1722,11 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void storePair32(RegisterID src1, TrustedImm32 imm, const void* address) { ArmAddress armAddress = setupArmAddress(AbsoluteAddress(address)); - ASSERT(armAddress.type == ArmAddress::HasOffset); - RegisterID scratch = getCachedDataTempRegisterIDAndInvalidate(); + ARMV7_MASM_ASSERT(armAddress.type == ArmAddress::HasOffset); + RegisterID scratch = armAddress.base == dataTempRegister + ? getCachedAddressTempRegisterIDAndInvalidate() + : getCachedDataTempRegisterIDAndInvalidate(); + ARMV7_MASM_ASSERT(armAddress.base != scratch && src1 != scratch); move(imm, scratch); storePair32(src1, scratch, Address(armAddress.base, armAddress.u.offset)); } @@ -1403,7 +1734,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void storePair32(RegisterID src1, RegisterID src2, const void* address) { ArmAddress armAddress = setupArmAddress(AbsoluteAddress(address)); - ASSERT(armAddress.type == ArmAddress::HasOffset); + ARMV7_MASM_ASSERT(armAddress.type == ArmAddress::HasOffset); storePair32(src1, src2, Address(armAddress.base, armAddress.u.offset)); } @@ -1484,31 +1815,48 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void storeCond8(RegisterID src, Address addr, RegisterID result) { - ASSERT(!addr.offset); + ARMV7_MASM_ASSERT(!addr.offset); m_assembler.strexb(result, src, addr.base); + if (result == addressTempRegister) + invalidateCachedAddressTempRegister(); + else if (result == dataTempRegister) + cachedDataTempRegister().invalidate(); } void storeCond16(RegisterID src, Address addr, RegisterID result) { - ASSERT(!addr.offset); + ARMV7_MASM_ASSERT(!addr.offset); m_assembler.strexh(result, src, addr.base); + if (result == addressTempRegister) + invalidateCachedAddressTempRegister(); + else if (result == dataTempRegister) + cachedDataTempRegister().invalidate(); } void storeCond32(RegisterID src, Address addr, RegisterID result) { - ASSERT(!addr.offset); + ARMV7_MASM_ASSERT(!addr.offset); m_assembler.strex(result, src, addr.base, 0); + if (result == addressTempRegister) + invalidateCachedAddressTempRegister(); + else if (result == dataTempRegister) + cachedDataTempRegister().invalidate(); } void storeCondPair32(RegisterID srcLo, RegisterID srcHi, Address addr, RegisterID result) { - ASSERT(!addr.offset); + ARMV7_MASM_ASSERT(!addr.offset); m_assembler.strexd(result, srcLo, srcHi, addr.base); + if (result == addressTempRegister) + invalidateCachedAddressTempRegister(); + else if (result == dataTempRegister) + cachedDataTempRegister().invalidate(); } // Possibly clobbers src, but not on this architecture. void moveDoubleToInts(FPRegisterID src, RegisterID dest1, RegisterID dest2) { + ARMV7_MASM_ASSERT(dest1 != dest2); m_assembler.vmov(dest1, dest2, src); } @@ -1534,6 +1882,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void moveDoubleTo64(FPRegisterID src, RegisterID destHi, RegisterID destLo) { + ARMV7_MASM_ASSERT(destHi != destLo); m_assembler.vmov(destLo, destHi, src); } @@ -1571,25 +1920,25 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { NO_RETURN_DUE_TO_CRASH void countPopulation32(RegisterID, RegisterID) { - ASSERT(!supportsCountPopulation()); + ARMV7_MASM_ASSERT(!supportsCountPopulation()); CRASH(); } NO_RETURN_DUE_TO_CRASH void countPopulation32(RegisterID, RegisterID, FPRegisterID) { - ASSERT(!supportsCountPopulation()); + ARMV7_MASM_ASSERT(!supportsCountPopulation()); CRASH(); } NO_RETURN_DUE_TO_CRASH void countPopulation64(RegisterID, RegisterID) { - ASSERT(!supportsCountPopulation()); + ARMV7_MASM_ASSERT(!supportsCountPopulation()); CRASH(); } NO_RETURN_DUE_TO_CRASH void countPopulation64(RegisterID, RegisterID, FPRegisterID) { - ASSERT(!supportsCountPopulation()); + ARMV7_MASM_ASSERT(!supportsCountPopulation()); CRASH(); } @@ -1609,6 +1958,9 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { // Arm vfp addresses can be offset by a 9-bit ones-comp immediate, left shifted by 2. if ((offset & 3) || (offset > (255 * 4)) || (offset < -(255 * 4))) { + ARMV7_MASM_ASSERT(base != dataTempRegister + || ARMThumbImmediate::makeUInt12OrEncodedImm(offset).isValid() + || ARMThumbImmediate::makeUInt12OrEncodedImm(-offset).isValid()); RegisterID scratch = getCachedAddressTempRegisterIDAndInvalidate(); add32(TrustedImm32(offset), base, scratch); base = scratch; @@ -1625,6 +1977,9 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { // Arm vfp addresses can be offset by a 9-bit ones-comp immediate, left shifted by 2. if ((offset & 3) || (offset > (255 * 4)) || (offset < -(255 * 4))) { + ARMV7_MASM_ASSERT(base != dataTempRegister + || ARMThumbImmediate::makeUInt12OrEncodedImm(offset).isValid() + || ARMThumbImmediate::makeUInt12OrEncodedImm(-offset).isValid()); RegisterID scratch = getCachedAddressTempRegisterIDAndInvalidate(); add32(TrustedImm32(offset), base, scratch); base = scratch; @@ -1636,6 +1991,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void loadDouble(BaseIndex address, FPRegisterID dest) { + ARMV7_MASM_ASSERT(address.base != addressTempRegister); move(address.index, addressTempRegister); lshift32(TrustedImm32(address.scale), addressTempRegister); add32(address.base, addressTempRegister); @@ -1645,6 +2001,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void loadFloat(BaseIndex address, FPRegisterID dest) { + ARMV7_MASM_ASSERT(address.base != addressTempRegister); move(address.index, addressTempRegister); lshift32(TrustedImm32(address.scale), addressTempRegister); add32(address.base, addressTempRegister); @@ -1721,6 +2078,9 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { // Arm vfp addresses can be offset by a 9-bit ones-comp immediate, left shifted by 2. if ((offset & 3) || (offset > (255 * 4)) || (offset < -(255 * 4))) { + ARMV7_MASM_ASSERT(base != dataTempRegister + || ARMThumbImmediate::makeUInt12OrEncodedImm(offset).isValid() + || ARMThumbImmediate::makeUInt12OrEncodedImm(-offset).isValid()); RegisterID scratch = getCachedAddressTempRegisterIDAndInvalidate(); add32(TrustedImm32(offset), base, scratch); base = scratch; @@ -1737,6 +2097,9 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { // Arm vfp addresses can be offset by a 9-bit ones-comp immediate, left shifted by 2. if ((offset & 3) || (offset > (255 * 4)) || (offset < -(255 * 4))) { + ARMV7_MASM_ASSERT(base != dataTempRegister + || ARMThumbImmediate::makeUInt12OrEncodedImm(offset).isValid() + || ARMThumbImmediate::makeUInt12OrEncodedImm(-offset).isValid()); RegisterID scratch = getCachedAddressTempRegisterIDAndInvalidate(); add32(TrustedImm32(offset), base, scratch); base = scratch; @@ -1754,6 +2117,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void storeDouble(FPRegisterID src, BaseIndex address) { + ARMV7_MASM_ASSERT(address.base != addressTempRegister); move(address.index, addressTempRegister); lshift32(TrustedImm32(address.scale), addressTempRegister); add32(address.base, addressTempRegister); @@ -1763,6 +2127,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void storeFloat(FPRegisterID src, BaseIndex address) { + ARMV7_MASM_ASSERT(address.base != addressTempRegister); move(address.index, addressTempRegister); lshift32(TrustedImm32(address.scale), addressTempRegister); add32(address.base, addressTempRegister); @@ -1782,6 +2147,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void addDouble(Address src, FPRegisterID dest) { + ARMV7_MASM_ASSERT(dest != fpTempRegister); loadDouble(src, fpTempRegister); addDouble(fpTempRegister, dest); } @@ -1793,6 +2159,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void addDouble(AbsoluteAddress address, FPRegisterID dest) { + ARMV7_MASM_ASSERT(dest != fpTempRegister); loadDouble(TrustedImmPtr(address.m_ptr), fpTempRegister); m_assembler.vadd(dest, dest, fpTempRegister); } @@ -1824,6 +2191,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void subDouble(Address src, FPRegisterID dest) { + ARMV7_MASM_ASSERT(dest != fpTempRegister); loadDouble(src, fpTempRegister); subDouble(fpTempRegister, dest); } @@ -1845,6 +2213,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void mulDouble(Address src, FPRegisterID dest) { + ARMV7_MASM_ASSERT(dest != fpTempRegister); loadDouble(src, fpTempRegister); mulDouble(fpTempRegister, dest); } @@ -1906,43 +2275,43 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { NO_RETURN_DUE_TO_CRASH void ceilFloat(FPRegisterID, FPRegisterID) { - ASSERT(!supportsFloatingPointRounding()); + ARMV7_MASM_ASSERT(!supportsFloatingPointRounding()); CRASH(); } NO_RETURN_DUE_TO_CRASH void floorFloat(FPRegisterID, FPRegisterID) { - ASSERT(!supportsFloatingPointRounding()); + ARMV7_MASM_ASSERT(!supportsFloatingPointRounding()); CRASH(); } NO_RETURN_DUE_TO_CRASH void roundTowardNearestIntFloat(FPRegisterID, FPRegisterID) { - ASSERT(!supportsFloatingPointRounding()); + ARMV7_MASM_ASSERT(!supportsFloatingPointRounding()); CRASH(); } NO_RETURN_DUE_TO_CRASH void roundTowardZeroFloat(FPRegisterID, FPRegisterID) { - ASSERT(!supportsFloatingPointRounding()); + ARMV7_MASM_ASSERT(!supportsFloatingPointRounding()); CRASH(); } NO_RETURN_DUE_TO_CRASH void ceilDouble(FPRegisterID, FPRegisterID) { - ASSERT(!supportsFloatingPointRounding()); + ARMV7_MASM_ASSERT(!supportsFloatingPointRounding()); CRASH(); } NO_RETURN_DUE_TO_CRASH void floorDouble(FPRegisterID, FPRegisterID) { - ASSERT(!supportsFloatingPointRounding()); + ARMV7_MASM_ASSERT(!supportsFloatingPointRounding()); CRASH(); } NO_RETURN_DUE_TO_CRASH void roundTowardZeroDouble(FPRegisterID, FPRegisterID) { - ASSERT(!supportsFloatingPointRounding()); + ARMV7_MASM_ASSERT(!supportsFloatingPointRounding()); CRASH(); } @@ -1958,6 +2327,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void convertInt32ToFloat(RegisterID src, FPRegisterID dest) { + ARMV7_MASM_ASSERT(dest <= ARMRegisters::d15); m_assembler.vmov(fpTempRegister, src, src); m_assembler.vcvt_signedToFloatingPoint(dest, fpTempRegisterAsSingle(), /* toDouble: */ false); } @@ -1992,6 +2362,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void convertUInt32ToFloat(RegisterID src, FPRegisterID dest) { + ARMV7_MASM_ASSERT(dest <= ARMRegisters::d15); m_assembler.vmov(fpTempRegister, src, src); m_assembler.vcvt_unsignedToFloatingPoint(dest, fpTempRegisterAsSingle(), /* toDouble: */ false); } @@ -2006,7 +2377,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { { m_assembler.vcvtds(dst, ARMRegisters::asSingle(src)); } - + void convertDoubleToFloat(FPRegisterID src, FPRegisterID dst) { m_assembler.vcvtsd(ARMRegisters::asSingle(dst), src); @@ -2085,10 +2456,8 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { Jump branchFloatWithZero(DoubleCondition cond, FPRegisterID left) { - UNUSED_PARAM(cond); - UNUSED_PARAM(left); - UNREACHABLE_FOR_PLATFORM(); - return { }; + m_assembler.vcmpz(asSingle(left)); + return makeFPBranch(cond); } Jump branchDouble(DoubleCondition cond, FPRegisterID left, FPRegisterID right) @@ -2099,10 +2468,8 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { Jump branchDoubleWithZero(DoubleCondition cond, FPRegisterID left) { - UNUSED_PARAM(cond); - UNUSED_PARAM(left); - UNREACHABLE_FOR_PLATFORM(); - return { }; + m_assembler.vcmpz(left); + return makeFPBranch(cond); } enum BranchTruncateType { BranchIfTruncateFailed, BranchIfTruncateSuccessful }; @@ -2155,13 +2522,15 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { m_assembler.vcvt_floatingPointToUnsigned(fpTempRegisterAsSingle(), asSingle(src)); m_assembler.vmov(dest, fpTempRegisterAsSingle()); } - + // Convert 'src' to an integer, and places the resulting 'dest'. // If the result is not representable as a 32 bit value, branch. // May also branch for some values that are representable in 32 bits // (specifically, in this case, 0). void branchConvertDoubleToInt32(FPRegisterID src, RegisterID dest, JumpList& failureCases, FPRegisterID, bool negZeroCheck = true) { + ARMV7_MASM_ASSERT(src != fpTempRegister); + ARMV7_MASM_ASSERT(dest != dataTempRegister); m_assembler.vcvt_floatingPointToSigned(fpTempRegisterAsSingle(), src); m_assembler.vmov(dest, fpTempRegisterAsSingle()); @@ -2201,6 +2570,102 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { notEqual.link(this); return result; } +private: + void convertDoubleToUint64(FPRegisterID src, RegisterID destLo, RegisterID destHi, FPRegisterID scratch1, FPRegisterID scratch2) + { + // We override src on the vmla, so we require a temp fp register here + ARMV7_MASM_ASSERT(src == fpTempRegister); + ARMV7_MASM_ASSERT(scratch1 != scratch2); + ARMV7_MASM_ASSERT(scratch1 != src); + ARMV7_MASM_ASSERT(scratch2 != src); + ARMV7_MASM_ASSERT(destLo != destHi); + ARMV7_MASM_ASSERT(scratch1 != scratch2); + ARMV7_MASM_ASSERT(scratch1 != src); + ARMV7_MASM_ASSERT(scratch2 != src); + ARMV7_MASM_ASSERT(destLo != destHi); + + // Constant materialization + move(TrustedImm32(0x00000000), destLo); + move(TrustedImm32(0x3DF00000), destHi); + move64ToDouble(destHi, destLo, scratch1); + move(TrustedImm32(0x00000000), destLo); + move(TrustedImm32(0xC1F00000), destHi); + move64ToDouble(destHi, destLo, scratch2); + + m_assembler.vmul(scratch1, src, scratch1); + + m_assembler.vcvt_floatingPointToUnsigned(ARMRegisters::asSingle(scratch1), scratch1); + m_assembler.vmov(destHi, ARMRegisters::asSingle(scratch1)); + + m_assembler.vcvt_unsignedToFloatingPoint(scratch1, ARMRegisters::asSingle(scratch1)); + + m_assembler.vmla(src, scratch1, scratch2); + + m_assembler.vcvt_floatingPointToUnsigned(ARMRegisters::asSingle(src), src); + m_assembler.vmov(destLo, ARMRegisters::asSingle(src)); + } + +public: + void truncateDoubleToUint64(FPRegisterID src, RegisterID destLo, RegisterID destHi, FPRegisterID scratch1, FPRegisterID scratch2) + { + ARMV7_MASM_ASSERT(scratch1 != fpTempRegister); + ARMV7_MASM_ASSERT(scratch2 != fpTempRegister); + ARMV7_MASM_ASSERT(scratch1 != scratch2); + ARMV7_MASM_ASSERT(destLo != destHi); + Jump notPositive = branchDoubleWithZero(DoubleLessThanOrEqualOrUnordered, src); + + moveDouble(src, fpTempRegister); + convertDoubleToUint64(fpTempRegister, destLo, destHi, scratch1, scratch2); + + Jump done = jump(); + + notPositive.link(this); + move(TrustedImm32(0), destLo); + move(TrustedImm32(0), destHi); + + done.link(this); + } + + void truncateDoubleToInt64(FPRegisterID src, RegisterID destLo, RegisterID destHi, FPRegisterID scratch1, FPRegisterID scratch2) + { + ARMV7_MASM_ASSERT(scratch1 != fpTempRegister); + ARMV7_MASM_ASSERT(scratch2 != fpTempRegister); + ARMV7_MASM_ASSERT(scratch1 != scratch2); + ARMV7_MASM_ASSERT(destLo != destHi); + ARMV7_MASM_ASSERT(destLo != addressTempRegister); + ARMV7_MASM_ASSERT(destHi != addressTempRegister); + RegisterID signFlag = getCachedAddressTempRegisterIDAndInvalidate(); + moveDouble(src, fpTempRegister); + + Jump isNegative = branchDoubleWithZero(DoubleLessThanAndOrdered, fpTempRegister); + move(TrustedImm32(0), signFlag); + Jump join = jump(); + + isNegative.link(this); + m_assembler.vneg(fpTempRegister, fpTempRegister); + move(TrustedImm32(1), signFlag); + + join.link(this); + convertDoubleToUint64(fpTempRegister, destLo, destHi, scratch1, scratch2); + + Jump wasNonNegative = branch32(Equal, signFlag, TrustedImm32(0)); + m_assembler.sub_S(destLo, ARMThumbImmediate::makeEncodedImm(0), destLo); + m_assembler.mvn(destHi, destHi); + m_assembler.adc(destHi, destHi, ARMThumbImmediate::makeEncodedImm(0)); + wasNonNegative.link(this); + } + + void truncateFloatToUint64(FPRegisterID src, RegisterID destLo, RegisterID destHi, FPRegisterID scratch1, FPRegisterID scratch2) + { + convertFloatToDouble(src, fpTempRegister); + truncateDoubleToUint64(fpTempRegister, destLo, destHi, scratch1, scratch2); + } + + void truncateFloatToInt64(FPRegisterID src, RegisterID destLo, RegisterID destHi, FPRegisterID scratch1, FPRegisterID scratch2) + { + convertFloatToDouble(src, fpTempRegister); + truncateDoubleToInt64(fpTempRegister, destLo, destHi, scratch1, scratch2); + } // Stack manipulation operations: // @@ -2209,7 +2674,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { // operations add and remove a single register sized unit of data // to or from the stack. Peek and poke operations read or write // values on the stack, without moving the current stack position. - + void pop(RegisterID dest) { m_assembler.pop(dest); @@ -2351,6 +2816,8 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void swap(RegisterID reg1, RegisterID reg2) { + ARMV7_MASM_ASSERT(reg1 != dataTempRegister); + ARMV7_MASM_ASSERT(reg2 != dataTempRegister); move(reg1, dataTempRegister); move(reg2, reg1); move(dataTempRegister, reg2); @@ -2494,9 +2961,16 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { m_assembler.cmp(left, scratch); } + void compare32AndSetFlags(RegisterID left, RegisterID right) + { + m_assembler.cmp(left, right); + } + void add32Impl(TrustedImm32 imm, Address address, bool updateFlags = false) { - load32(address, dataTempRegister); + ArmAddress armAddress = setupArmAddress(address); + RELEASE_ASSERT(armAddress.base != dataTempRegister && armAddress.base != addressTempRegister); + load32(armAddress, dataTempRegister); ARMThumbImmediate armImm = ARMThumbImmediate::makeEncodedImm(imm.m_value); if (armImm.isValid()) { @@ -2507,14 +2981,20 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { } else { // Hrrrm, since dataTempRegister holds the data loaded, // use addressTempRegister to hold the immediate. + bool addressUsesTemp = armAddress.type == ArmAddress::HasIndex && armAddress.u.index == addressTempRegister; move(imm, addressTempRegister); - if (updateFlags) + if (updateFlags) { m_assembler.add_S(dataTempRegister, dataTempRegister, addressTempRegister); - else + if (addressUsesTemp) + moveFixedWidthEncoding(TrustedImm32(address.offset), addressTempRegister); + } else { m_assembler.add(dataTempRegister, dataTempRegister, addressTempRegister); + if (addressUsesTemp) + move(TrustedImm32(address.offset), addressTempRegister); + } } - store32(dataTempRegister, address); + store32(dataTempRegister, armAddress); } void add32Impl(TrustedImm32 imm, AbsoluteAddress address, bool updateFlags = false) @@ -2545,9 +3025,13 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { { int32_t imm = mask.m_value; - if (imm == -1) - m_assembler.tst(reg, reg); - else { + if (imm == -1) { + if (reg == ARMRegisters::sp) { + move(reg, addressTempRegister); + m_assembler.tst(addressTempRegister, addressTempRegister); + } else + m_assembler.tst(reg, reg); + } else { ARMThumbImmediate armImm = ARMThumbImmediate::makeEncodedImm(imm); if (armImm.isValid()) { if (reg == ARMRegisters::sp) { @@ -2730,16 +3214,106 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { return branch32(cond, addressTempRegister, right16); } +private: + template + Jump branch64Impl(RelationalCondition cond, RegisterID leftHi, RegisterID leftLo, T rightHi, T rightLo) + { + if (cond == Equal) { + // Equal: bne done; cmp lo; beq target; done: + compare32AndSetFlags(leftHi, rightHi); + Jump done = makeBranch(ARMv7Assembler::ConditionNE); + compare32AndSetFlags(leftLo, rightLo); + Jump result = makeBranch(ARMv7Assembler::ConditionEQ); + done.link(this); + return result; + } + + if (cond == NotEqual) { + // NotEqual: branch taken if ANY part differs + compare32AndSetFlags(leftHi, rightHi); + Jump fromHi = makeBranch(ARMv7Assembler::ConditionNE); + compare32AndSetFlags(leftLo, rightLo); + Jump fromLo = makeBranch(ARMv7Assembler::ConditionNE); + Jump notTaken = jump(); + fromHi.link(this); + fromLo.link(this); + Jump result = jump(); + notTaken.link(this); + return result; + } + + ARMv7Assembler::Condition hiCond = armV7ConditionForHigh32(cond); + ARMv7Assembler::Condition loCond = armV7ConditionForLow32(cond); + ARMV7_MASM_ASSERT(hiCond != ARMv7Assembler::ConditionInvalid); + ARMV7_MASM_ASSERT(loCond != ARMv7Assembler::ConditionInvalid); + ARMv7Assembler::Condition inverseLoCond = ARMv7Assembler::invert(loCond); + compare32AndSetFlags(leftHi, rightHi); + Jump fromHi = makeBranch(hiCond); + Jump notTakenHi = makeBranch(ARMv7Assembler::ConditionNE); + compare32AndSetFlags(leftLo, rightLo); + Jump notTakenLo = makeBranch(inverseLoCond); + fromHi.link(this); + Jump result = jump(); + notTakenHi.link(this); + notTakenLo.link(this); + return result; + } + +public: + Jump branch64(RelationalCondition cond, RegisterID leftHi, RegisterID leftLo, RegisterID rightHi, RegisterID rightLo) + { + return branch64Impl(cond, leftHi, leftLo, rightHi, rightLo); + } + + Jump branch64(RelationalCondition cond, RegisterID leftHi, RegisterID leftLo, TrustedImm64 right) + { + TrustedImm32 rightHi(static_cast(right.m_value >> 32)); + TrustedImm32 rightLo(static_cast(right.m_value)); + + // Optimize for comparing with zero (unsigned comparisons only) + if (!rightHi.m_value && !rightLo.m_value) { + if (cond == Below) + return branch32(Below, leftHi, TrustedImm32(0)); + if (cond == AboveOrEqual) + return jump(); // all unsigned values are >= 0 + + if (cond == Equal || cond == BelowOrEqual || cond == NotEqual || cond == Above) { + RegisterID scratch = getCachedDataTempRegisterIDAndInvalidate(); + m_assembler.orr_S(scratch, leftHi, leftLo); + return Jump(makeBranch((cond == Equal || cond == BelowOrEqual) ? ARMv7Assembler::ConditionEQ : ARMv7Assembler::ConditionNE)); + } + } + + return branch64Impl(cond, leftHi, leftLo, rightHi, rightLo); + } + + Jump branchTest64(ResultCondition cond, RegisterID regHi, RegisterID regLo) + { + ARMV7_MASM_ASSERT(regHi != ARMRegisters::sp && regHi != ARMRegisters::pc); + ARMV7_MASM_ASSERT(regLo != ARMRegisters::sp && regLo != ARMRegisters::pc); + ARMV7_MASM_ASSERT(cond == Signed || cond == PositiveOrZero || cond == Zero || cond == NonZero); + if (cond == Signed || cond == PositiveOrZero) { + // For sign tests, only check the sign bit of the high 32 bits + m_assembler.tst(regHi, regHi); + return Jump(makeBranch((cond == Signed) ? ARMv7Assembler::ConditionMI : ARMv7Assembler::ConditionPL)); + } + + ARMV7_MASM_ASSERT(cond == Zero || cond == NonZero); + RegisterID scratch = getCachedDataTempRegisterIDAndInvalidate(); + m_assembler.orr_S(scratch, regHi, regLo); + return Jump(makeBranch(cond == Zero ? ARMv7Assembler::ConditionEQ : ARMv7Assembler::ConditionNE)); + } + Jump branchTest32(ResultCondition cond, RegisterID reg, RegisterID mask) { - ASSERT(cond == Zero || cond == NonZero || cond == Signed || cond == PositiveOrZero); + ARMV7_MASM_ASSERT(cond == Zero || cond == NonZero || cond == Signed || cond == PositiveOrZero); m_assembler.tst(reg, mask); return Jump(makeBranch(cond)); } Jump branchTest32(ResultCondition cond, RegisterID reg, TrustedImm32 mask = TrustedImm32(-1)) { - ASSERT(cond == Zero || cond == NonZero || cond == Signed || cond == PositiveOrZero); + ARMV7_MASM_ASSERT(cond == Zero || cond == NonZero || cond == Signed || cond == PositiveOrZero); test32(reg, mask); return Jump(makeBranch(cond)); } @@ -2875,6 +3449,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { if (armImm.isValid()) m_assembler.add_S(dest, op1, armImm); else { + ARMV7_MASM_ASSERT(op1 != dataTempRegister); move(imm, dataTempRegister); m_assembler.add_S(dest, op1, dataTempRegister); } @@ -2913,6 +3488,8 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { Jump branchMul32(ResultCondition cond, RegisterID src1, RegisterID src2, RegisterID dest) { + ARMV7_MASM_ASSERT(dest != dataTempRegister); + ARMV7_MASM_ASSERT((cond != Overflow) || (dest != addressTempRegister)); m_assembler.smull(dest, dataTempRegister, src1, src2); // The invalidation of cachedDataTempRegister is handled by the branch. if (cond == Overflow) { @@ -2931,13 +3508,14 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { Jump branchMul32(ResultCondition cond, RegisterID src, TrustedImm32 imm, RegisterID dest) { + ARMV7_MASM_ASSERT(src != dataTempRegister); move(imm, dataTempRegister); return branchMul32(cond, dataTempRegister, src, dest); } Jump branchNeg32(ResultCondition cond, RegisterID srcDest) { - ARMThumbImmediate zero = ARMThumbImmediate::makeUInt12(0); + ARMThumbImmediate zero = ARMThumbImmediate::makeEncodedImm(0); m_assembler.sub_S(srcDest, zero, srcDest); return Jump(makeBranch(cond)); } @@ -2960,6 +3538,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { if (armImm.isValid()) m_assembler.sub_S(dest, op1, armImm); else { + ARMV7_MASM_ASSERT(op1 != dataTempRegister); move(imm, dataTempRegister); m_assembler.sub_S(dest, op1, dataTempRegister); } @@ -2978,7 +3557,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void relativeTableJump(RegisterID index, int scale) { - ASSERT(scale >= 0 && scale <= 31); + ARMV7_MASM_ASSERT(scale >= 0 && scale <= 31); // dataTempRegister will point after the jump if index register contains zero move(ARMRegisters::pc, dataTempRegister); @@ -3084,6 +3663,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void compare32(RelationalCondition cond, Address left, RegisterID right, RegisterID dest) { + ARMV7_MASM_ASSERT(right != addressTempRegister); load32(left, addressTempRegister); compare32(cond, addressTempRegister, right, dest); } @@ -3201,6 +3781,8 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void moveConditionally32(RelationalCondition cond, RegisterID left, TrustedImm32 right, RegisterID thenCase, RegisterID elseCase, RegisterID dest) { + ARMV7_MASM_ASSERT(thenCase != dataTempRegister && thenCase != addressTempRegister); + ARMV7_MASM_ASSERT(elseCase != dataTempRegister && elseCase != addressTempRegister); compare32AndSetFlags(left, right); m_assembler.it(armV7Condition(cond), false); moveOrNop(thenCase, dest); @@ -3243,6 +3825,8 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void moveConditionallyTest32(ResultCondition cond, RegisterID left, TrustedImm32 right, RegisterID thenCase, RegisterID elseCase, RegisterID dest) { + ARMV7_MASM_ASSERT(thenCase != dataTempRegister && thenCase != addressTempRegister); + ARMV7_MASM_ASSERT(elseCase != dataTempRegister && elseCase != addressTempRegister); test32(left, right); m_assembler.it(armV7Condition(cond), false); moveOrNop(thenCase, dest); @@ -3548,6 +4132,8 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { ALWAYS_INLINE Jump makeBranch(ARMv7Assembler::Condition cond) { + ARMV7_MASM_ASSERT(cond != ARMv7Assembler::ConditionInvalid); + ARMV7_MASM_ASSERT(cond != ARMv7Assembler::ConditionAL); m_assembler.label(); // Force nop-padding if we're in the middle of a watchpoint. m_assembler.it(cond, true, true); moveFixedWidthEncoding(TrustedImm32(0), dataTempRegister); @@ -3560,13 +4146,16 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { ArmAddress setupArmAddress(BaseIndex address) { + ARMV7_MASM_ASSERT(address.scale <= TimesEight); if (address.offset) { + ARMV7_MASM_ASSERT(address.index != addressTempRegister); ARMThumbImmediate imm = ARMThumbImmediate::makeUInt12OrEncodedImm(address.offset); if (imm.isValid()) { RegisterID scratch = getCachedAddressTempRegisterIDAndInvalidate(); m_assembler.add(scratch, address.base, imm); } else { RELEASE_ASSERT(m_allowScratchRegister); + ARMV7_MASM_ASSERT(address.base != addressTempRegister); move(TrustedImm32(address.offset), addressTempRegister); m_assembler.add(addressTempRegister, addressTempRegister, address.base); cachedAddressTempRegister().invalidate(); @@ -3583,6 +4172,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { return ArmAddress(address.base, address.offset); RELEASE_ASSERT(m_allowScratchRegister); + ARMV7_MASM_ASSERT(address.base != addressTempRegister); move(TrustedImm32(address.offset), addressTempRegister); return ArmAddress(address.base, addressTempRegister); } @@ -3616,10 +4206,13 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { if (!address.offset) return address.base; + RELEASE_ASSERT(m_allowScratchRegister); + ARMV7_MASM_ASSERT(address.index != addressTempRegister); ARMThumbImmediate imm = ARMThumbImmediate::makeUInt12OrEncodedImm(address.offset); if (imm.isValid()) m_assembler.add(addressTempRegister, address.base, imm); else { + ARMV7_MASM_ASSERT(address.base != addressTempRegister); move(TrustedImm32(address.offset), addressTempRegister); m_assembler.add(addressTempRegister, addressTempRegister, address.base); } diff --git a/Source/JavaScriptCore/assembler/testmasm.cpp b/Source/JavaScriptCore/assembler/testmasm.cpp index fc3d7e0493c4..24a2f4237fb2 100644 --- a/Source/JavaScriptCore/assembler/testmasm.cpp +++ b/Source/JavaScriptCore/assembler/testmasm.cpp @@ -97,6 +97,12 @@ static Vector floatOperands() } #endif +template +static T* bitwise_cast_ptr(U* p) +{ + return static_cast(static_cast(p)); +} + static Vector int32Operands() { return Vector { @@ -4164,171 +4170,6782 @@ void testSub32ArgImm() } } -#if CPU(ARM64) -void testLoadStorePair64Int64() +#if CPU(ARM_THUMB2) +void testSub32ImmArg() { - constexpr uint64_t initialValue = 0x5555aaaabbbb8800ull; - constexpr uint64_t value1 = 42; - constexpr uint64_t value2 = 0xcafebabe12345678ull; + for (auto immediate : { 0, 1, 32, 0xff, 0x100, 0x101, 0x1ff, 0x200, 0x3ff, 0x400, 0x555, + 0xabc, 0xfff, 0x1000, -1, -0x101, std::numeric_limits::max(), std::numeric_limits::min() }) { + auto sub = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); - uint64_t buffer[10]; + jit.sub32(CCallHelpers::TrustedImm32(immediate), GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); - auto initBuffer = [&] { - for (unsigned i = 0; i < 10; ++i) - buffer[i] = initialValue + i; - }; + emitFunctionEpilogue(jit); + jit.ret(); + }); - struct Pair { - uint64_t value1; - uint64_t value2; + for (auto value : int32Operands()) + CHECK_EQ(invoke(sub, value), static_cast(immediate - value)); + } +} +#endif + +static Vector aluImmediates() +{ + return Vector { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 0x7f, 0xfe, 0xff, 0x100, 0x101, 0x1fc, 0x1fd, 0x1ff, + 0x200, 0x3fc, 0x3fd, 0x3ff, 0x400, 0x555, 0xabc, 0xffc, 0xfff, 0x1000, 0x1001, + 0x10001, 0xff00, 0x00ff00ff, static_cast(0xff00ff00), + -1, -2, -6, -0x101, -0x1000, + std::numeric_limits::max(), std::numeric_limits::min() }; +} - Pair pair; - auto initPair = [&] { - pair = { 0, 0 }; +static Vector aluValues() +{ + return Vector { + 0u, 1u, 2u, 7u, 0xffu, 0x100u, 0xfffu, 0x1000u, 0x12345678u, + 0x7fffffffu, 0x80000000u, 0xfffffffeu, 0xffffffffu }; +} - // Test loadPair64. - auto testLoadPair = [] (CCallHelpers& jit, int offset) { - emitFunctionPrologue(jit); +void testALUAdd32Immediate() +{ + for (auto immediate : aluImmediates()) { + auto aliased = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.add32(CCallHelpers::TrustedImm32(immediate), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); - constexpr GPRReg bufferGPR = GPRInfo::argumentGPR0; - constexpr GPRReg pairGPR = GPRInfo::argumentGPR1; - jit.loadPair64(bufferGPR, CCallHelpers::TrustedImm32(offset * sizeof(CPURegister)), GPRInfo::regT2, GPRInfo::regT3); + auto distinct = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1); + jit.add32(CCallHelpers::TrustedImm32(immediate), GPRInfo::argumentGPR1, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); - jit.store64(GPRInfo::regT2, CCallHelpers::Address(pairGPR, 0)); - jit.store64(GPRInfo::regT3, CCallHelpers::Address(pairGPR, sizeof(uint64_t))); + for (auto value : aluValues()) { + uint32_t expected = value + static_cast(immediate); + CHECK_EQ(invoke(aliased, value), expected); + CHECK_EQ(invoke(distinct, value), expected); + } + } +} - emitFunctionEpilogue(jit); - jit.ret(); - }; +void testALUSub32Immediate() +{ + for (auto immediate : aluImmediates()) { + auto aliased = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.sub32(CCallHelpers::TrustedImm32(immediate), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); - auto testLoadPair0 = compile([&] (CCallHelpers& jit) { - testLoadPair(jit, 0); - }); + auto distinct = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1); + jit.sub32(GPRInfo::argumentGPR1, CCallHelpers::TrustedImm32(immediate), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); - initBuffer(); +#if CPU(ARM_THUMB2) + auto reversed = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1); + jit.sub32(CCallHelpers::TrustedImm32(immediate), GPRInfo::argumentGPR1, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); - initPair(); - invoke(testLoadPair0, &buffer[4], &pair); - CHECK_EQ(pair.value1, initialValue + 4); - CHECK_EQ(pair.value2, initialValue + 5); + auto reversedAliased = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.sub32(CCallHelpers::TrustedImm32(immediate), GPRInfo::returnValueGPR, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); +#endif - initPair(); - buffer[4] = value1; - buffer[5] = value2; - invoke(testLoadPair0, &buffer[4], &pair); - CHECK_EQ(pair.value1, value1); - CHECK_EQ(pair.value2, value2); + for (auto value : aluValues()) { + uint32_t expected = value - static_cast(immediate); + CHECK_EQ(invoke(aliased, value), expected); + CHECK_EQ(invoke(distinct, value), expected); +#if CPU(ARM_THUMB2) + uint32_t expectedReversed = static_cast(immediate) - value; + CHECK_EQ(invoke(reversed, value), expectedReversed); + CHECK_EQ(invoke(reversedAliased, value), expectedReversed); +#endif + } + } +} - auto testLoadPairMinus2 = compile([&] (CCallHelpers& jit) { - testLoadPair(jit, -2); - }); +void testALUAnd32Immediate() +{ + for (auto immediate : aluImmediates()) { + auto aliased = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.and32(CCallHelpers::TrustedImm32(immediate), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); - initPair(); - invoke(testLoadPairMinus2, &buffer[4], &pair); - CHECK_EQ(pair.value1, initialValue + 4 - 2); - CHECK_EQ(pair.value2, initialValue + 5 - 2); + auto distinct = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1); + jit.and32(CCallHelpers::TrustedImm32(immediate), GPRInfo::argumentGPR1, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); - initPair(); - buffer[4 - 2] = value2; - buffer[5 - 2] = value1; - invoke(testLoadPairMinus2, &buffer[4], &pair); - CHECK_EQ(pair.value1, value2); - CHECK_EQ(pair.value2, value1); + for (auto value : aluValues()) { + uint32_t expected = value & static_cast(immediate); + CHECK_EQ(invoke(aliased, value), expected); + CHECK_EQ(invoke(distinct, value), expected); + } + } +} - auto testLoadPairPlus3 = compile([&] (CCallHelpers& jit) { - testLoadPair(jit, 3); - }); +void testALUOr32Immediate() +{ + for (auto immediate : aluImmediates()) { + auto aliased = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.or32(CCallHelpers::TrustedImm32(immediate), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); - initPair(); - invoke(testLoadPairPlus3, &buffer[4], &pair); - CHECK_EQ(pair.value1, initialValue + 4 + 3); - CHECK_EQ(pair.value2, initialValue + 5 + 3); + auto distinct = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1); + jit.or32(CCallHelpers::TrustedImm32(immediate), GPRInfo::argumentGPR1, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); - initPair(); - buffer[4 + 3] = value1; - buffer[5 + 3] = value2; - invoke(testLoadPairPlus3, &buffer[4], &pair); - CHECK_EQ(pair.value1, value1); - CHECK_EQ(pair.value2, value2); + for (auto value : aluValues()) { + uint32_t expected = value | static_cast(immediate); + CHECK_EQ(invoke(aliased, value), expected); + CHECK_EQ(invoke(distinct, value), expected); + } + } +} - // Test loadPair64 using a buffer register as a destination. - auto testLoadPairUsingBufferRegisterAsDestination = [] (CCallHelpers& jit, int offset) { - emitFunctionPrologue(jit); +void testALUXor32Immediate() +{ + for (auto immediate : aluImmediates()) { + auto aliased = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.xor32(CCallHelpers::TrustedImm32(immediate), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); - constexpr GPRReg bufferGPR = GPRInfo::argumentGPR0; - constexpr GPRReg pairGPR = GPRInfo::argumentGPR1; - jit.loadPair64(bufferGPR, CCallHelpers::TrustedImm32(offset * sizeof(CPURegister)), GPRInfo::argumentGPR0, GPRInfo::regT2); + auto distinct = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1); + jit.xor32(CCallHelpers::TrustedImm32(immediate), GPRInfo::argumentGPR1, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); - jit.store64(GPRInfo::argumentGPR0, CCallHelpers::Address(pairGPR, 0)); - jit.store64(GPRInfo::regT2, CCallHelpers::Address(pairGPR, sizeof(uint64_t))); + for (auto value : aluValues()) { + uint32_t expected = value ^ static_cast(immediate); + CHECK_EQ(invoke(aliased, value), expected); + CHECK_EQ(invoke(distinct, value), expected); + } + } +} +void testALURegisterAliasing() +{ + auto addSame = compile([] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.add32(GPRInfo::returnValueGPR, GPRInfo::returnValueGPR, GPRInfo::returnValueGPR); emitFunctionEpilogue(jit); jit.ret(); - }; + }); - auto testLoadPairUsingBufferRegisterAsDestination0 = compile([&] (CCallHelpers& jit) { - testLoadPairUsingBufferRegisterAsDestination(jit, 0); + auto addDestIsLeft = compile([] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.move(GPRInfo::argumentGPR1, GPRInfo::argumentGPR2); + jit.add32(GPRInfo::returnValueGPR, GPRInfo::argumentGPR2, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); }); - initBuffer(); + auto addDestIsRight = compile([] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.move(GPRInfo::argumentGPR1, GPRInfo::argumentGPR2); + jit.add32(GPRInfo::argumentGPR2, GPRInfo::returnValueGPR, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); - initPair(); - invoke(testLoadPairUsingBufferRegisterAsDestination0, &buffer[4], &pair); - CHECK_EQ(pair.value1, initialValue + 4); - CHECK_EQ(pair.value2, initialValue + 5); + auto subSame = compile([] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.sub32(GPRInfo::returnValueGPR, GPRInfo::returnValueGPR, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); - // Test storePair64. - auto testStorePair = [] (CCallHelpers& jit, int offset) { + auto subDestIsRight = compile([] (CCallHelpers& jit) { emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR1, GPRInfo::argumentGPR2); + jit.sub32(GPRInfo::argumentGPR0, GPRInfo::argumentGPR2, GPRInfo::argumentGPR2); + jit.move(GPRInfo::argumentGPR2, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); - constexpr GPRReg bufferGPR = GPRInfo::argumentGPR2; - jit.storePair64(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, bufferGPR, CCallHelpers::TrustedImm32(offset * sizeof(CPURegister))); + auto andSame = compile([] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.and32(GPRInfo::returnValueGPR, GPRInfo::returnValueGPR, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + auto xorSame = compile([] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.xor32(GPRInfo::returnValueGPR, GPRInfo::returnValueGPR, GPRInfo::returnValueGPR); emitFunctionEpilogue(jit); jit.ret(); - }; + }); - auto testStorePair0 = compile([&] (CCallHelpers& jit) { - testStorePair(jit, 0); + auto orDestIsRight = compile([] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.move(GPRInfo::argumentGPR1, GPRInfo::argumentGPR2); + jit.or32(GPRInfo::argumentGPR2, GPRInfo::returnValueGPR, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); }); - initBuffer(); - invoke(testStorePair0, value1, value2, &buffer[4]); - CHECK_EQ(buffer[0], initialValue + 0); - CHECK_EQ(buffer[1], initialValue + 1); - CHECK_EQ(buffer[2], initialValue + 2); - CHECK_EQ(buffer[3], initialValue + 3); - CHECK_EQ(buffer[4], value1); - CHECK_EQ(buffer[5], value2); - CHECK_EQ(buffer[6], initialValue + 6); - CHECK_EQ(buffer[7], initialValue + 7); - CHECK_EQ(buffer[8], initialValue + 8); - CHECK_EQ(buffer[9], initialValue + 9); + for (auto left : aluValues()) { + CHECK_EQ(invoke(addSame, left), static_cast(left + left)); + CHECK_EQ(invoke(subSame, left), 0u); + CHECK_EQ(invoke(andSame, left), left); + CHECK_EQ(invoke(xorSame, left), 0u); + for (auto right : aluValues()) { + CHECK_EQ(invoke(addDestIsLeft, left, right), static_cast(left + right)); + CHECK_EQ(invoke(addDestIsRight, left, right), static_cast(left + right)); + CHECK_EQ(invoke(subDestIsRight, left, right), static_cast(left - right)); + CHECK_EQ(invoke(orDestIsRight, left, right), left | right); + } + } +} - auto testStorePairMinus2 = compile([&] (CCallHelpers& jit) { - testStorePair(jit, -2); +void testALUNeg32AndNot32() +{ + auto negAliased = compile([] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.neg32(GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); }); - initBuffer(); - invoke(testStorePairMinus2, value1, value2, &buffer[4]); - CHECK_EQ(buffer[0], initialValue + 0); - CHECK_EQ(buffer[1], initialValue + 1); - CHECK_EQ(buffer[2], value1); - CHECK_EQ(buffer[3], value2); - CHECK_EQ(buffer[4], initialValue + 4); - CHECK_EQ(buffer[5], initialValue + 5); - CHECK_EQ(buffer[6], initialValue + 6); - CHECK_EQ(buffer[7], initialValue + 7); - CHECK_EQ(buffer[8], initialValue + 8); - CHECK_EQ(buffer[9], initialValue + 9); - - auto testStorePairPlus3 = compile([&] (CCallHelpers& jit) { - testStorePair(jit, 3); + auto notAliased = compile([] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.not32(GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); }); - initBuffer(); +#if CPU(ARM_THUMB2) || CPU(ARM64) || CPU(RISCV64) + auto notDistinct = compile([] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1); + jit.not32(GPRInfo::argumentGPR1, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); +#endif + + auto xorMinusOne = compile([] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.xor32(CCallHelpers::TrustedImm32(-1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : aluValues()) { + CHECK_EQ(invoke(negAliased, value), static_cast(0u - value)); + CHECK_EQ(invoke(notAliased, value), ~value); +#if CPU(ARM_THUMB2) || CPU(ARM64) || CPU(RISCV64) + CHECK_EQ(invoke(notDistinct, value), ~value); +#endif + CHECK_EQ(invoke(xorMinusOne, value), ~value); + } +} + +void testALUMul32() +{ + auto mulAliased = compile([] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.mul32(GPRInfo::returnValueGPR, GPRInfo::returnValueGPR, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto mulDestIsLeft = compile([] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.move(GPRInfo::argumentGPR1, GPRInfo::argumentGPR2); + jit.mul32(GPRInfo::argumentGPR2, GPRInfo::returnValueGPR, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto mulDestIsRight = compile([] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.move(GPRInfo::argumentGPR1, GPRInfo::argumentGPR2); + jit.mul32(GPRInfo::returnValueGPR, GPRInfo::argumentGPR2, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto left : aluValues()) { + CHECK_EQ(invoke(mulAliased, left), static_cast(left * left)); + for (auto right : aluValues()) { + CHECK_EQ(invoke(mulDestIsLeft, left, right), static_cast(left * right)); + CHECK_EQ(invoke(mulDestIsRight, left, right), static_cast(left * right)); + } + } +} + +void testALUMul32Immediate() +{ + for (auto immediate : aluImmediates()) { + auto test = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1); + jit.mul32(CCallHelpers::TrustedImm32(immediate), GPRInfo::argumentGPR1, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : aluValues()) + CHECK_EQ(invoke(test, value), static_cast(value * static_cast(immediate))); + } +} + +void testALUCountLeadingZeros32() +{ + auto aliased = compile([] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.countLeadingZeros32(GPRInfo::returnValueGPR, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto distinct = compile([] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1); + jit.countLeadingZeros32(GPRInfo::argumentGPR1, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : aluValues()) { + uint32_t expected = 32; + for (uint32_t bit = 0; bit < 32; ++bit) { + if (value & (1u << (31 - bit))) { + expected = bit; + break; + } + } + CHECK_EQ(invoke(aliased, value), expected); + CHECK_EQ(invoke(distinct, value), expected); + } +} + +void testALUShift32Boundaries() +{ + for (int32_t shiftAmount : { 0, 1, 2, 15, 16, 30, 31 }) { + auto lshiftAliased = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.lshift32(CCallHelpers::TrustedImm32(shiftAmount), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto lshiftDistinct = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1); + jit.lshift32(GPRInfo::argumentGPR1, CCallHelpers::TrustedImm32(shiftAmount), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto rshiftAliased = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.rshift32(CCallHelpers::TrustedImm32(shiftAmount), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto rshiftDistinct = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1); + jit.rshift32(GPRInfo::argumentGPR1, CCallHelpers::TrustedImm32(shiftAmount), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto urshiftAliased = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.urshift32(CCallHelpers::TrustedImm32(shiftAmount), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto urshiftDistinct = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1); + jit.urshift32(GPRInfo::argumentGPR1, CCallHelpers::TrustedImm32(shiftAmount), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto lshiftByRegister = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::argumentGPR2); + jit.move(CCallHelpers::TrustedImm32(shiftAmount), GPRInfo::argumentGPR1); + jit.lshift32(GPRInfo::argumentGPR2, GPRInfo::argumentGPR1, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto rshiftByRegister = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::argumentGPR2); + jit.move(CCallHelpers::TrustedImm32(shiftAmount), GPRInfo::argumentGPR1); + jit.rshift32(GPRInfo::argumentGPR2, GPRInfo::argumentGPR1, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto urshiftByRegister = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::argumentGPR2); + jit.move(CCallHelpers::TrustedImm32(shiftAmount), GPRInfo::argumentGPR1); + jit.urshift32(GPRInfo::argumentGPR2, GPRInfo::argumentGPR1, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : aluValues()) { + uint32_t expectedLeft = shiftAmount ? (value << shiftAmount) : value; + uint32_t expectedLogicalRight = shiftAmount ? (value >> shiftAmount) : value; + uint32_t expectedArithmeticRight = shiftAmount + ? static_cast(static_cast(value) >> shiftAmount) + : value; + + CHECK_EQ(invoke(lshiftAliased, value), expectedLeft); + CHECK_EQ(invoke(lshiftDistinct, value), expectedLeft); + CHECK_EQ(invoke(lshiftByRegister, value), expectedLeft); + CHECK_EQ(invoke(rshiftAliased, value), expectedArithmeticRight); + CHECK_EQ(invoke(rshiftDistinct, value), expectedArithmeticRight); + CHECK_EQ(invoke(rshiftByRegister, value), expectedArithmeticRight); + CHECK_EQ(invoke(urshiftAliased, value), expectedLogicalRight); + CHECK_EQ(invoke(urshiftDistinct, value), expectedLogicalRight); + CHECK_EQ(invoke(urshiftByRegister, value), expectedLogicalRight); + } + } +} + +void testALUBranchAdd32Immediate() +{ + for (auto immediate : aluImmediates()) { + auto test = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1); + auto overflow = jit.branchAdd32(CCallHelpers::Overflow, GPRInfo::argumentGPR1, + CCallHelpers::TrustedImm32(immediate), GPRInfo::returnValueGPR); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + auto done = jit.jump(); + overflow.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + done.link(&jit); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : aluValues()) { + int64_t wide = static_cast(static_cast(value)) + static_cast(immediate); + uint32_t expected = (wide < INT32_MIN || wide > INT32_MAX) ? 1 : 0; + CHECK_EQ(invoke(test, value), expected); + } + } +} + +void testALUBranchSub32Immediate() +{ + for (auto immediate : aluImmediates()) { + auto test = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1); + auto overflow = jit.branchSub32(CCallHelpers::Overflow, GPRInfo::argumentGPR1, + CCallHelpers::TrustedImm32(immediate), GPRInfo::returnValueGPR); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + auto done = jit.jump(); + overflow.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + done.link(&jit); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto result = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1); + auto overflow = jit.branchSub32(CCallHelpers::Overflow, GPRInfo::argumentGPR1, + CCallHelpers::TrustedImm32(immediate), GPRInfo::returnValueGPR); + overflow.link(&jit); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : aluValues()) { + int64_t wide = static_cast(static_cast(value)) - static_cast(immediate); + uint32_t expected = (wide < INT32_MIN || wide > INT32_MAX) ? 1 : 0; + CHECK_EQ(invoke(test, value), expected); + CHECK_EQ(invoke(result, value), value - static_cast(immediate)); + } + } +} + +void testALUBranchNeg32() +{ + auto test = compile([] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + auto overflow = jit.branchNeg32(CCallHelpers::Overflow, GPRInfo::returnValueGPR); + overflow.link(&jit); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : aluValues()) + CHECK_EQ(invoke(test, value), static_cast(0u - value)); +} + +void testALUBranchMul32Overflow() +{ + auto test = compile([] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::argumentGPR2); + auto overflow = jit.branchMul32(CCallHelpers::Overflow, GPRInfo::argumentGPR1, + GPRInfo::argumentGPR2, GPRInfo::returnValueGPR); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + auto done = jit.jump(); + overflow.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + done.link(&jit); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto left : aluValues()) { + for (auto right : aluValues()) { + int64_t wide = static_cast(static_cast(left)) * static_cast(static_cast(right)); + uint32_t expected = (wide < INT32_MIN || wide > INT32_MAX) ? 1 : 0; + CHECK_EQ(invoke(test, left, right), expected); + } + } +} + +void testALUCompare32Immediate() +{ + for (auto immediate : aluImmediates()) { + auto equal = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.compare32(CCallHelpers::Equal, GPRInfo::argumentGPR0, + CCallHelpers::TrustedImm32(immediate), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto below = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.compare32(CCallHelpers::Below, GPRInfo::argumentGPR0, + CCallHelpers::TrustedImm32(immediate), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto lessThan = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.compare32(CCallHelpers::LessThan, GPRInfo::argumentGPR0, + CCallHelpers::TrustedImm32(immediate), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto testNonZero = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.test32(CCallHelpers::NonZero, GPRInfo::argumentGPR0, + CCallHelpers::TrustedImm32(immediate), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : aluValues()) { + CHECK_EQ(invoke(equal, value), static_cast(value == static_cast(immediate))); + CHECK_EQ(invoke(below, value), static_cast(value < static_cast(immediate))); + CHECK_EQ(invoke(lessThan, value), static_cast(static_cast(value) < immediate)); + CHECK_EQ(invoke(testNonZero, value), static_cast(!!(value & static_cast(immediate)))); + } + } +} + +#if CPU(ARM_THUMB2) +void testALUARMv7Add32ToStackPointerImmediate() +{ + for (int32_t immediate : { 0, 1, 2, 3, 4, 5, 6, 7, 8, 0xfd, 0xff, 0x100, 0x1fc, 0x1fd, + 0x1ff, 0x200, 0x3fc, 0x3fd, 0x3ff, 0x400, 0xffd, 0xfff, -1, -6, -0x400 }) { + auto test = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(MacroAssembler::stackPointerRegister, GPRInfo::argumentGPR1); + jit.add32(CCallHelpers::TrustedImm32(immediate), MacroAssembler::stackPointerRegister); + jit.move(MacroAssembler::stackPointerRegister, GPRInfo::returnValueGPR); + jit.move(GPRInfo::argumentGPR1, MacroAssembler::stackPointerRegister); + jit.sub32(GPRInfo::argumentGPR1, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + CHECK_EQ(invoke(test), immediate); + } +} + +void testALUARMv7Sub32FromStackPointerImmediate() +{ + for (int32_t immediate : { 0, 1, 2, 3, 4, 5, 6, 7, 8, 0xfd, 0xff, 0x100, 0x1fc, 0x1fd, + 0x1ff, 0x200, 0x3fc, 0x3fd, 0x3ff, 0x400, 0xffd, 0xfff, -1, -6, -0x400 }) { + auto test = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(MacroAssembler::stackPointerRegister, GPRInfo::argumentGPR1); + jit.sub32(CCallHelpers::TrustedImm32(immediate), MacroAssembler::stackPointerRegister); + jit.move(MacroAssembler::stackPointerRegister, GPRInfo::returnValueGPR); + jit.move(GPRInfo::argumentGPR1, MacroAssembler::stackPointerRegister); + jit.sub32(GPRInfo::returnValueGPR, GPRInfo::argumentGPR1); + jit.move(GPRInfo::argumentGPR1, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + CHECK_EQ(invoke(test), immediate); + } +} + +void testALUARMv7Add32FromStackPointerImmediate() +{ + for (int32_t immediate : { 0, 1, 2, 3, 4, 5, 6, 7, 8, 0xfd, 0xff, 0x100, 0x1fc, 0x1fd, + 0x1ff, 0x200, 0x3fc, 0x3fd, 0x3ff, 0x400, 0xffd, 0xfff, -1, -6, -0x400 }) { + auto lowDestination = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(MacroAssembler::stackPointerRegister, GPRInfo::argumentGPR1); + jit.add32(CCallHelpers::TrustedImm32(immediate), MacroAssembler::stackPointerRegister, GPRInfo::returnValueGPR); + jit.sub32(GPRInfo::argumentGPR1, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto highDestination = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.push(ARMRegisters::r8); + jit.push(ARMRegisters::r10); + jit.move(MacroAssembler::stackPointerRegister, GPRInfo::argumentGPR1); + jit.add32(CCallHelpers::TrustedImm32(immediate), MacroAssembler::stackPointerRegister, ARMRegisters::r8); + jit.move(ARMRegisters::r8, GPRInfo::returnValueGPR); + jit.sub32(GPRInfo::argumentGPR1, GPRInfo::returnValueGPR); + jit.pop(ARMRegisters::r10); + jit.pop(ARMRegisters::r8); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + CHECK_EQ(invoke(lowDestination), immediate); + CHECK_EQ(invoke(highDestination), immediate); + } +} + +void testALUARMv7HighRegisterOperands() +{ + auto test = compile([] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.push(ARMRegisters::r8); + jit.push(ARMRegisters::r10); + jit.move(GPRInfo::argumentGPR0, ARMRegisters::r8); + jit.move(GPRInfo::argumentGPR1, ARMRegisters::r10); + jit.add32(ARMRegisters::r10, ARMRegisters::r8, ARMRegisters::r8); + jit.sub32(ARMRegisters::r10, ARMRegisters::r8, ARMRegisters::r8); + jit.and32(ARMRegisters::r10, ARMRegisters::r8, ARMRegisters::r10); + jit.or32(ARMRegisters::r8, ARMRegisters::r10, ARMRegisters::r10); + jit.xor32(ARMRegisters::r8, ARMRegisters::r10, ARMRegisters::r10); + jit.add32(ARMRegisters::r10, ARMRegisters::r8, GPRInfo::returnValueGPR); + jit.pop(ARMRegisters::r10); + jit.pop(ARMRegisters::r8); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto left : aluValues()) { + for (auto right : aluValues()) { + uint32_t r8 = left; + uint32_t r10 = right; + r8 = r10 + r8; + r8 = r10 - r8; + r10 = r10 & r8; + r10 = r8 | r10; + r10 = r8 ^ r10; + CHECK_EQ(invoke(test, left, right), static_cast(r10 + r8)); + } + } +} + +void testALUARMv7RotateRight32MultipleOf32() +{ + for (int32_t rotation : { 0, 1, 31, 32, 33, 63, 64, -32, -64 }) { + auto test = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1); + jit.rotateRight32(GPRInfo::argumentGPR1, CCallHelpers::TrustedImm32(rotation), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto aliased = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.rotateRight32(CCallHelpers::TrustedImm32(rotation), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + uint32_t amount = static_cast(rotation) & 0x1f; + for (auto value : aluValues()) { + uint32_t expected = amount ? ((value >> amount) | (value << (32 - amount))) : value; + CHECK_EQ(invoke(test, value), expected); + CHECK_EQ(invoke(aliased, value), expected); + } + } +} + +void testALUARMv7RotateLeft32() +{ + for (int32_t rotation : { 0, 1, 31, 32, 33, -32 }) { + auto test = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1); + jit.rotateLeft32(GPRInfo::argumentGPR1, CCallHelpers::TrustedImm32(rotation), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + uint32_t amount = static_cast(rotation) & 0x1f; + for (auto value : aluValues()) { + uint32_t expected = amount ? ((value << amount) | (value >> (32 - amount))) : value; + CHECK_EQ(invoke(test, value), expected); + } + } +} + +void testALUARMv7TruncateDoubleToInt64Negative() +{ + auto test = compile([] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.push(ARMRegisters::r8); + jit.push(ARMRegisters::r10); + jit.loadDouble(CCallHelpers::Address(GPRInfo::argumentGPR0), FPRInfo::fpRegT0); + jit.truncateDoubleToInt64(FPRInfo::fpRegT0, ARMRegisters::r8, ARMRegisters::r10, FPRInfo::fpRegT1, FPRInfo::fpRegT2); + jit.move(ARMRegisters::r8, GPRInfo::returnValueGPR); + jit.move(ARMRegisters::r10, GPRInfo::returnValueGPR2); + jit.pop(ARMRegisters::r10); + jit.pop(ARMRegisters::r8); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (double value : { -1.0, -2.5, -1000.0, -4294967296.0, -4294967297.0, 0.0, 1.0, 1000.0 }) { + double storage = value; + int64_t expected = static_cast(value); + CHECK_EQ(invoke(test, &storage), expected); + } +} +#endif + +#if CPU(ARM_THUMB2) + +static MacroAssemblerCodeRef armv7CompileBranchOverPadding(unsigned paddingBytes, bool conditional) +{ + return compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + CCallHelpers::Jump taken; + if (conditional) + taken = jit.branch32(CCallHelpers::Equal, GPRInfo::argumentGPR1, CCallHelpers::TrustedImm32(0)); + else + taken = jit.jump(); + jit.move(CCallHelpers::TrustedImm32(2), GPRInfo::returnValueGPR); + for (unsigned i = 0; i < paddingBytes / 2; ++i) + jit.nop(); + taken.link(&jit); + emitFunctionEpilogue(jit); + jit.ret(); + }); +} + +void testARMv7BranchRangeConditional() +{ + for (unsigned padding : { 0u, 2u, 4u, 244u, 248u, 250u, 252u, 254u, 256u, 258u, 260u, 262u, 512u, + 2040u, 2044u, 2046u, 2048u, 2050u, 2052u, 4096u, 8192u }) { + auto code = armv7CompileBranchOverPadding(padding, true); + CHECK_EQ(invoke(code, 0, 0), 1); + CHECK_EQ(invoke(code, 0, 1), 2); + CHECK_EQ(invoke(code, 0, -1), 2); + } +} + +void testARMv7BranchRangeUnconditional() +{ + for (unsigned padding : { 0u, 2u, 4u, 244u, 250u, 254u, 256u, 258u, 262u, 512u, + 2038u, 2042u, 2044u, 2046u, 2048u, 2050u, 2052u, 2054u, 4096u, 8192u }) { + auto code = armv7CompileBranchOverPadding(padding, false); + CHECK_EQ(invoke(code, 0, 0), 1); + CHECK_EQ(invoke(code, 0, 1), 1); + } +} + +void testARMv7BranchRangeConditionalLong() +{ + for (unsigned padding : { 1048572u, 1048576u, 1048580u }) { + auto code = armv7CompileBranchOverPadding(padding, true); + CHECK_EQ(invoke(code, 0, 0), 1); + CHECK_EQ(invoke(code, 0, 1), 2); + } +} + +void testARMv7BranchRangeUnconditionalLong() +{ + for (unsigned padding : { 1048572u, 1048576u, 1048580u }) { + auto code = armv7CompileBranchOverPadding(padding, false); + CHECK_EQ(invoke(code, 0, 0), 1); + } +} + +void testARMv7BranchRangeBackward() +{ + for (unsigned padding : { 0u, 2u, 244u, 250u, 254u, 256u, 258u, 262u, 2040u, 2046u, 2048u, 2052u, 4096u }) { + auto code = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + auto top = jit.label(); + jit.add32(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + for (unsigned i = 0; i < padding / 2; ++i) + jit.nop(); + auto back = jit.branch32(CCallHelpers::LessThan, GPRInfo::returnValueGPR, CCallHelpers::TrustedImm32(5)); + back.linkTo(top, &jit); + emitFunctionEpilogue(jit); + jit.ret(); + }); + CHECK_EQ(invoke(code), 5); + } +} + +void testARMv7BranchRangeManyJumpsToOneLabel() +{ + auto code = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + CCallHelpers::JumpList done; + for (unsigned i = 0; i < 64; ++i) { + done.append(jit.branch32(CCallHelpers::Equal, GPRInfo::argumentGPR1, CCallHelpers::TrustedImm32(static_cast(i)))); + jit.add32(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + for (unsigned j = 0; j < 64; ++j) + jit.nop(); + } + done.link(&jit); + emitFunctionEpilogue(jit); + jit.ret(); + }); + for (int32_t i = 0; i < 64; ++i) + CHECK_EQ(invoke(code, 0, i), i); + CHECK_EQ(invoke(code, 0, 1000), 64); +} + +void testARMv7LoadDoubleOffsets() +{ + double buffer[768]; + for (unsigned i = 0; i < 768; ++i) + buffer[i] = static_cast(i) + 0.5; + + for (int offset : { -2052, -2048, -1028, -1024, -1023, -1021, -1020, -1016, -8, -4, -3, -1, + 0, 1, 3, 4, 8, 1016, 1020, 1021, 1023, 1024, 1028, 2048, 2052 }) { + auto code = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadDouble(CCallHelpers::Address(GPRInfo::argumentGPR0, offset), FPRInfo::fpRegT0); + jit.storeDouble(FPRInfo::fpRegT0, CCallHelpers::Address(GPRInfo::argumentGPR1)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + char* base = reinterpret_cast(&buffer[384]) - offset; + double out = 0; + invoke(code, base, &out); + CHECK_EQ(out, buffer[384]); + } +} + +void testARMv7StoreDoubleOffsets() +{ + double buffer[768]; + double source = 1234.5; + + for (int offset : { -2052, -2048, -1028, -1024, -1023, -1021, -1020, -1016, -8, -4, -3, -1, + 0, 1, 3, 4, 8, 1016, 1020, 1021, 1023, 1024, 1028, 2048, 2052 }) { + auto code = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadDouble(CCallHelpers::Address(GPRInfo::argumentGPR0), FPRInfo::fpRegT0); + jit.storeDouble(FPRInfo::fpRegT0, CCallHelpers::Address(GPRInfo::argumentGPR1, offset)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + buffer[384] = 0; + char* base = reinterpret_cast(&buffer[384]) - offset; + invoke(code, &source, base); + CHECK_EQ(buffer[384], source); + } +} + +void testARMv7LoadStoreFloatOffsets() +{ + float buffer[1536]; + for (unsigned i = 0; i < 1536; ++i) + buffer[i] = static_cast(i) + 0.5f; + + for (int offset : { -2052, -2048, -1024, -1023, -1021, -1020, -8, -4, -3, -1, + 0, 1, 3, 4, 8, 1016, 1020, 1021, 1023, 1024, 1028, 2048 }) { + auto code = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadFloat(CCallHelpers::Address(GPRInfo::argumentGPR0, offset), FPRInfo::fpRegT0); + jit.storeFloat(FPRInfo::fpRegT0, CCallHelpers::Address(GPRInfo::argumentGPR1, offset)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + float outBuffer[1536]; + for (unsigned i = 0; i < 1536; ++i) + outBuffer[i] = 0; + char* inBase = reinterpret_cast(&buffer[768]) - offset; + char* outBase = reinterpret_cast(&outBuffer[768]) - offset; + invoke(code, inBase, outBase); + CHECK_EQ(outBuffer[768], buffer[768]); + } +} + +void testARMv7DoubleArithAliasing() +{ + MacroAssembler::FPRegisterID dstReg[5] = { FPRInfo::fpRegT2, FPRInfo::fpRegT0, FPRInfo::fpRegT1, FPRInfo::fpRegT0, FPRInfo::fpRegT2 }; + MacroAssembler::FPRegisterID lhsReg[5] = { FPRInfo::fpRegT0, FPRInfo::fpRegT0, FPRInfo::fpRegT0, FPRInfo::fpRegT0, FPRInfo::fpRegT0 }; + MacroAssembler::FPRegisterID rhsReg[5] = { FPRInfo::fpRegT1, FPRInfo::fpRegT1, FPRInfo::fpRegT1, FPRInfo::fpRegT0, FPRInfo::fpRegT0 }; + + double io[3]; + for (unsigned op = 0; op < 4; ++op) { + for (unsigned m = 0; m < 5; ++m) { + MacroAssembler::FPRegisterID d = dstReg[m]; + MacroAssembler::FPRegisterID l = lhsReg[m]; + MacroAssembler::FPRegisterID r = rhsReg[m]; + auto code = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadDouble(CCallHelpers::Address(GPRInfo::argumentGPR0, 0), FPRInfo::fpRegT0); + jit.loadDouble(CCallHelpers::Address(GPRInfo::argumentGPR0, 8), FPRInfo::fpRegT1); + switch (op) { + case 0: + jit.addDouble(l, r, d); + break; + case 1: + jit.subDouble(l, r, d); + break; + case 2: + jit.mulDouble(l, r, d); + break; + default: + jit.divDouble(l, r, d); + break; + } + jit.storeDouble(d, CCallHelpers::Address(GPRInfo::argumentGPR0, 16)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + for (double a : { 0.0, -0.0, 1.0, -1.0, 3.5, -7.25, 1e300, 1e-300 }) { + for (double b : { 1.0, -1.0, 2.0, 0.5, -4.75, 1e300 }) { + io[0] = a; + io[1] = b; + io[2] = 99.0; + double lv = a; + double rv = (r == FPRInfo::fpRegT0) ? a : b; + double expected = 0; + switch (op) { + case 0: + expected = lv + rv; + break; + case 1: + expected = lv - rv; + break; + case 2: + expected = lv * rv; + break; + default: + expected = lv / rv; + break; + } + invoke(code, io); + if (expected == expected) + CHECK_EQ(io[2], expected); + else + CHECK_EQ(static_cast(io[2] == io[2]), 0); + } + } + } + } +} + +void testARMv7FloatArithAliasing() +{ + MacroAssembler::FPRegisterID dstReg[5] = { FPRInfo::fpRegT2, FPRInfo::fpRegT0, FPRInfo::fpRegT1, FPRInfo::fpRegT0, FPRInfo::fpRegT2 }; + MacroAssembler::FPRegisterID rhsReg[5] = { FPRInfo::fpRegT1, FPRInfo::fpRegT1, FPRInfo::fpRegT1, FPRInfo::fpRegT0, FPRInfo::fpRegT0 }; + + float io[3]; + for (unsigned op = 0; op < 4; ++op) { + for (unsigned m = 0; m < 5; ++m) { + MacroAssembler::FPRegisterID d = dstReg[m]; + MacroAssembler::FPRegisterID l = FPRInfo::fpRegT0; + MacroAssembler::FPRegisterID r = rhsReg[m]; + auto code = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadFloat(CCallHelpers::Address(GPRInfo::argumentGPR0, 0), FPRInfo::fpRegT0); + jit.loadFloat(CCallHelpers::Address(GPRInfo::argumentGPR0, 4), FPRInfo::fpRegT1); + switch (op) { + case 0: + jit.addFloat(l, r, d); + break; + case 1: + jit.subFloat(l, r, d); + break; + case 2: + jit.mulFloat(l, r, d); + break; + default: + jit.divFloat(l, r, d); + break; + } + jit.storeFloat(d, CCallHelpers::Address(GPRInfo::argumentGPR0, 8)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + for (float a : { 0.0f, -0.0f, 1.0f, -1.0f, 3.5f, -7.25f, 1e30f }) { + for (float b : { 1.0f, -1.0f, 2.0f, 0.5f, -4.75f }) { + io[0] = a; + io[1] = b; + io[2] = 99.0f; + float rv = (r == FPRInfo::fpRegT0) ? a : b; + float expected = 0; + switch (op) { + case 0: + expected = a + rv; + break; + case 1: + expected = a - rv; + break; + case 2: + expected = a * rv; + break; + default: + expected = a / rv; + break; + } + invoke(code, io); + if (expected == expected) + CHECK_EQ(io[2], expected); + else + CHECK_EQ(static_cast(io[2] == io[2]), 0); + } + } + } + } +} + +void testARMv7AbsNegSqrtDouble() +{ + double io[2]; + for (unsigned op = 0; op < 3; ++op) { + for (unsigned aliased = 0; aliased < 2; ++aliased) { + MacroAssembler::FPRegisterID dest = aliased ? FPRInfo::fpRegT0 : FPRInfo::fpRegT1; + auto code = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadDouble(CCallHelpers::Address(GPRInfo::argumentGPR0, 0), FPRInfo::fpRegT0); + switch (op) { + case 0: + jit.absDouble(FPRInfo::fpRegT0, dest); + break; + case 1: + jit.negateDouble(FPRInfo::fpRegT0, dest); + break; + default: + jit.sqrtDouble(FPRInfo::fpRegT0, dest); + break; + } + jit.storeDouble(dest, CCallHelpers::Address(GPRInfo::argumentGPR0, 8)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + for (double a : { 0.0, -0.0, 1.0, -1.0, 4.0, 2.25, -2.25, 1e300, -1e300, 1e-300 }) { + io[0] = a; + io[1] = 99.0; + invoke(code, io); + if (!op) + CHECK_EQ(std::bit_cast(io[1]), std::bit_cast(a) & 0x7fffffffffffffffull); + else if (op == 1) + CHECK_EQ(std::bit_cast(io[1]), std::bit_cast(a) ^ 0x8000000000000000ull); + else if (a == 0.0 || a == 1.0 || a == 4.0 || a == 2.25) + CHECK_EQ(io[1] * io[1], a); + } + } + } +} + +void testARMv7AbsNegSqrtFloat() +{ + float io[2]; + for (unsigned op = 0; op < 3; ++op) { + for (unsigned aliased = 0; aliased < 2; ++aliased) { + MacroAssembler::FPRegisterID dest = aliased ? FPRInfo::fpRegT0 : FPRInfo::fpRegT1; + auto code = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadFloat(CCallHelpers::Address(GPRInfo::argumentGPR0, 0), FPRInfo::fpRegT0); + switch (op) { + case 0: + jit.absFloat(FPRInfo::fpRegT0, dest); + break; + case 1: + jit.negateFloat(FPRInfo::fpRegT0, dest); + break; + default: + jit.sqrtFloat(FPRInfo::fpRegT0, dest); + break; + } + jit.storeFloat(dest, CCallHelpers::Address(GPRInfo::argumentGPR0, 4)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + for (float a : { 0.0f, -0.0f, 1.0f, -1.0f, 4.0f, 2.25f, -2.25f, 1e30f }) { + io[0] = a; + io[1] = 99.0f; + invoke(code, io); + if (!op) + CHECK_EQ(std::bit_cast(io[1]), std::bit_cast(a) & 0x7fffffffu); + else if (op == 1) + CHECK_EQ(std::bit_cast(io[1]), std::bit_cast(a) ^ 0x80000000u); + else if (a == 0.0f || a == 1.0f || a == 4.0f || a == 2.25f) + CHECK_EQ(io[1] * io[1], a); + } + } + } +} + +void testARMv7MoveDoubleAliasing() +{ + double io[2]; + for (unsigned aliased = 0; aliased < 2; ++aliased) { + MacroAssembler::FPRegisterID dest = aliased ? FPRInfo::fpRegT0 : FPRInfo::fpRegT3; + auto code = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadDouble(CCallHelpers::Address(GPRInfo::argumentGPR0, 0), FPRInfo::fpRegT0); + jit.moveDouble(FPRInfo::fpRegT0, dest); + jit.storeDouble(dest, CCallHelpers::Address(GPRInfo::argumentGPR0, 8)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + for (double a : { 0.0, -0.0, 1.0, -1.0, 1e300, -1e-300 }) { + io[0] = a; + io[1] = 99.0; + invoke(code, io); + CHECK_EQ(std::bit_cast(io[1]), std::bit_cast(a)); + } + } +} + +void testARMv7ConvertInt32ToDouble() +{ + auto code = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.convertInt32ToDouble(GPRInfo::argumentGPR1, FPRInfo::fpRegT0); + jit.storeDouble(FPRInfo::fpRegT0, CCallHelpers::Address(GPRInfo::argumentGPR0)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + double out = 0; + for (int32_t value : { 0, 1, -1, 2, -2, 255, 256, 4095, 4096, 65535, 65536, 0x7ffffffe, + std::numeric_limits::max(), std::numeric_limits::min(), -65536 }) { + out = 99; + invoke(code, &out, value); + CHECK_EQ(out, static_cast(value)); + } +} + +void testARMv7ConvertUInt32ToDouble() +{ + auto code = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.convertUInt32ToDouble(GPRInfo::argumentGPR1, FPRInfo::fpRegT0); + jit.storeDouble(FPRInfo::fpRegT0, CCallHelpers::Address(GPRInfo::argumentGPR0)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + double out = 0; + for (uint32_t value : { 0u, 1u, 255u, 256u, 65535u, 65536u, 0x7fffffffu, 0x80000000u, 0xfffffffeu, 0xffffffffu }) { + out = 99; + invoke(code, &out, value); + CHECK_EQ(out, static_cast(value)); + } +} + +void testARMv7ConvertInt32ToFloat() +{ + auto code = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.convertInt32ToFloat(GPRInfo::argumentGPR1, FPRInfo::fpRegT0); + jit.storeFloat(FPRInfo::fpRegT0, CCallHelpers::Address(GPRInfo::argumentGPR0)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + float out = 0; + for (int32_t value : { 0, 1, -1, 255, 256, 65535, 65536, 0x1000000, + std::numeric_limits::max(), std::numeric_limits::min() }) { + out = 99; + invoke(code, &out, value); + CHECK_EQ(out, static_cast(value)); + } +} + +void testARMv7ConvertUInt32ToFloat() +{ + auto code = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.convertUInt32ToFloat(GPRInfo::argumentGPR1, FPRInfo::fpRegT0); + jit.storeFloat(FPRInfo::fpRegT0, CCallHelpers::Address(GPRInfo::argumentGPR0)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + float out = 0; + for (uint32_t value : { 0u, 1u, 255u, 65536u, 0x7fffffffu, 0x80000000u, 0xffffffffu }) { + out = 99; + invoke(code, &out, value); + CHECK_EQ(out, static_cast(value)); + } +} + +void testARMv7TruncateDoubleToInt32() +{ + auto code = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadDouble(CCallHelpers::Address(GPRInfo::argumentGPR0), FPRInfo::fpRegT0); + jit.truncateDoubleToInt32(FPRInfo::fpRegT0, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + double in = 0; + for (double value : { 0.0, -0.0, 1.0, -1.0, 1.9, -1.9, 42.5, -42.5, 2147483647.0, -2147483648.0, + 0.5, -0.5, 65536.25, -65536.25 }) { + in = value; + CHECK_EQ(invoke(code, &in), static_cast(value)); + } +} + +void testARMv7TruncateDoubleToUint32() +{ + auto code = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadDouble(CCallHelpers::Address(GPRInfo::argumentGPR0), FPRInfo::fpRegT0); + jit.truncateDoubleToUint32(FPRInfo::fpRegT0, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + double in = 0; + for (double value : { 0.0, 1.0, 1.9, 42.5, 65536.25, 2147483647.0, 2147483648.0, 4294967295.0 }) { + in = value; + CHECK_EQ(invoke(code, &in), static_cast(value)); + } +} + +void testARMv7ConvertDoubleFloatRoundTrip() +{ + double io[2]; + for (unsigned aliased = 0; aliased < 2; ++aliased) { + MacroAssembler::FPRegisterID mid = aliased ? FPRInfo::fpRegT0 : FPRInfo::fpRegT1; + auto code = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadDouble(CCallHelpers::Address(GPRInfo::argumentGPR0, 0), FPRInfo::fpRegT0); + jit.convertDoubleToFloat(FPRInfo::fpRegT0, mid); + jit.convertFloatToDouble(mid, FPRInfo::fpRegT2); + jit.storeDouble(FPRInfo::fpRegT2, CCallHelpers::Address(GPRInfo::argumentGPR0, 8)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + for (double a : { 0.0, -0.0, 1.0, -1.0, 0.5, -0.5, 3.25, 65536.0, 1e30, -1e30 }) { + io[0] = a; + io[1] = 99.0; + invoke(code, io); + CHECK_EQ(io[1], static_cast(static_cast(a))); + } + } +} + +void testARMv7BranchDoubleConditions() +{ + MacroAssembler::DoubleCondition conds[12] = { + MacroAssembler::DoubleEqualAndOrdered, + MacroAssembler::DoubleNotEqualAndOrdered, + MacroAssembler::DoubleGreaterThanAndOrdered, + MacroAssembler::DoubleGreaterThanOrEqualAndOrdered, + MacroAssembler::DoubleLessThanAndOrdered, + MacroAssembler::DoubleLessThanOrEqualAndOrdered, + MacroAssembler::DoubleEqualOrUnordered, + MacroAssembler::DoubleNotEqualOrUnordered, + MacroAssembler::DoubleGreaterThanOrUnordered, + MacroAssembler::DoubleGreaterThanOrEqualOrUnordered, + MacroAssembler::DoubleLessThanOrUnordered, + MacroAssembler::DoubleLessThanOrEqualOrUnordered + }; + + double io[2]; + for (unsigned c = 0; c < 12; ++c) { + MacroAssembler::DoubleCondition cond = conds[c]; + auto code = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadDouble(CCallHelpers::Address(GPRInfo::argumentGPR0, 0), FPRInfo::fpRegT0); + jit.loadDouble(CCallHelpers::Address(GPRInfo::argumentGPR0, 8), FPRInfo::fpRegT1); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + auto taken = jit.branchDouble(cond, FPRInfo::fpRegT0, FPRInfo::fpRegT1); + auto done = jit.jump(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + done.link(&jit); + emitFunctionEpilogue(jit); + jit.ret(); + }); + for (double a : { 0.0, -0.0, 1.0, -1.0, 42.0, std::numeric_limits::infinity(), + -std::numeric_limits::infinity(), std::numeric_limits::quiet_NaN() }) { + for (double b : { 0.0, 1.0, -1.0, 42.0, std::numeric_limits::infinity(), + std::numeric_limits::quiet_NaN() }) { + io[0] = a; + io[1] = b; + bool unordered = !(a == a) || !(b == b); + bool expected = false; + switch (c) { + case 0: + expected = a == b; + break; + case 1: + expected = !unordered && a != b; + break; + case 2: + expected = a > b; + break; + case 3: + expected = a >= b; + break; + case 4: + expected = a < b; + break; + case 5: + expected = a <= b; + break; + case 6: + expected = unordered || a == b; + break; + case 7: + expected = unordered || a != b; + break; + case 8: + expected = unordered || a > b; + break; + case 9: + expected = unordered || a >= b; + break; + case 10: + expected = unordered || a < b; + break; + default: + expected = unordered || a <= b; + break; + } + CHECK_EQ(invoke(code, io), expected ? 1 : 0); + } + } + } +} + +void testARMv7BranchDoubleWithZero() +{ + MacroAssembler::DoubleCondition conds[6] = { + MacroAssembler::DoubleEqualAndOrdered, + MacroAssembler::DoubleGreaterThanAndOrdered, + MacroAssembler::DoubleGreaterThanOrEqualAndOrdered, + MacroAssembler::DoubleLessThanAndOrdered, + MacroAssembler::DoubleLessThanOrEqualAndOrdered, + MacroAssembler::DoubleNotEqualOrUnordered + }; + + double in = 0; + for (unsigned c = 0; c < 6; ++c) { + MacroAssembler::DoubleCondition cond = conds[c]; + auto code = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadDouble(CCallHelpers::Address(GPRInfo::argumentGPR0), FPRInfo::fpRegT0); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + auto taken = jit.branchDoubleWithZero(cond, FPRInfo::fpRegT0); + auto done = jit.jump(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + done.link(&jit); + emitFunctionEpilogue(jit); + jit.ret(); + }); + for (double a : { 0.0, -0.0, 1.0, -1.0, std::numeric_limits::infinity(), + -std::numeric_limits::infinity(), std::numeric_limits::quiet_NaN() }) { + in = a; + bool unordered = !(a == a); + bool expected = false; + switch (c) { + case 0: + expected = a == 0; + break; + case 1: + expected = a > 0; + break; + case 2: + expected = a >= 0; + break; + case 3: + expected = a < 0; + break; + case 4: + expected = a <= 0; + break; + default: + expected = unordered || a != 0; + break; + } + CHECK_EQ(invoke(code, &in), expected ? 1 : 0); + } + } +} + +void testARMv7CompareDouble() +{ + MacroAssembler::DoubleCondition conds[12] = { + MacroAssembler::DoubleEqualAndOrdered, + MacroAssembler::DoubleNotEqualAndOrdered, + MacroAssembler::DoubleGreaterThanAndOrdered, + MacroAssembler::DoubleGreaterThanOrEqualAndOrdered, + MacroAssembler::DoubleLessThanAndOrdered, + MacroAssembler::DoubleLessThanOrEqualAndOrdered, + MacroAssembler::DoubleEqualOrUnordered, + MacroAssembler::DoubleNotEqualOrUnordered, + MacroAssembler::DoubleGreaterThanOrUnordered, + MacroAssembler::DoubleGreaterThanOrEqualOrUnordered, + MacroAssembler::DoubleLessThanOrUnordered, + MacroAssembler::DoubleLessThanOrEqualOrUnordered + }; + + double io[2]; + for (unsigned c = 0; c < 12; ++c) { + MacroAssembler::DoubleCondition cond = conds[c]; + auto code = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadDouble(CCallHelpers::Address(GPRInfo::argumentGPR0, 0), FPRInfo::fpRegT0); + jit.loadDouble(CCallHelpers::Address(GPRInfo::argumentGPR0, 8), FPRInfo::fpRegT1); + jit.compareDouble(cond, FPRInfo::fpRegT0, FPRInfo::fpRegT1, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + for (double a : { 0.0, 1.0, -1.0, 42.0, std::numeric_limits::quiet_NaN() }) { + for (double b : { 0.0, 1.0, -1.0, 42.0, std::numeric_limits::quiet_NaN() }) { + io[0] = a; + io[1] = b; + bool unordered = !(a == a) || !(b == b); + bool expected = false; + switch (c) { + case 0: + expected = a == b; + break; + case 1: + expected = !unordered && a != b; + break; + case 2: + expected = a > b; + break; + case 3: + expected = a >= b; + break; + case 4: + expected = a < b; + break; + case 5: + expected = a <= b; + break; + case 6: + expected = unordered || a == b; + break; + case 7: + expected = unordered || a != b; + break; + case 8: + expected = unordered || a > b; + break; + case 9: + expected = unordered || a >= b; + break; + case 10: + expected = unordered || a < b; + break; + default: + expected = unordered || a <= b; + break; + } + CHECK_EQ(invoke(code, io), expected ? 1 : 0); + } + } + } +} + +void testARMv7LoadStorePair64Double() +{ + double buffer[8]; + for (int32_t offset : { 0, 8, 16, -16 }) { + for (unsigned consecutive = 0; consecutive < 2; ++consecutive) { + MacroAssembler::FPRegisterID d1 = FPRInfo::fpRegT2; + MacroAssembler::FPRegisterID d2 = consecutive ? FPRInfo::fpRegT3 : FPRInfo::fpRegT5; + auto code = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadPair64(GPRInfo::argumentGPR0, CCallHelpers::TrustedImm32(offset), d1, d2); + jit.storePair64(d1, d2, GPRInfo::argumentGPR1, CCallHelpers::TrustedImm32(offset)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + for (unsigned i = 0; i < 8; ++i) + buffer[i] = static_cast(i) + 0.25; + double out[8]; + for (unsigned i = 0; i < 8; ++i) + out[i] = 0; + char* inBase = reinterpret_cast(&buffer[4]) - offset; + char* outBase = reinterpret_cast(&out[4]) - offset; + invoke(code, inBase, outBase); + CHECK_EQ(out[4], buffer[4]); + CHECK_EQ(out[5], buffer[5]); + } + } +} + +void testARMv7MoveDoubleBits() +{ + uint32_t io[4]; + auto code = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load32(CCallHelpers::Address(GPRInfo::argumentGPR0, 0), GPRInfo::argumentGPR2); + jit.load32(CCallHelpers::Address(GPRInfo::argumentGPR0, 4), GPRInfo::argumentGPR3); + jit.move64ToDouble(GPRInfo::argumentGPR3, GPRInfo::argumentGPR2, FPRInfo::fpRegT0); + jit.moveDoubleTo64(FPRInfo::fpRegT0, GPRInfo::argumentGPR3, GPRInfo::argumentGPR2); + jit.store32(GPRInfo::argumentGPR2, CCallHelpers::Address(GPRInfo::argumentGPR0, 8)); + jit.store32(GPRInfo::argumentGPR3, CCallHelpers::Address(GPRInfo::argumentGPR0, 12)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + for (uint32_t lo : { 0u, 1u, 0xffffffffu, 0x80000000u, 0x7ff80000u }) { + for (uint32_t hi : { 0u, 1u, 0xffffffffu, 0x80000000u, 0x7ff80000u }) { + io[0] = lo; + io[1] = hi; + io[2] = 0; + io[3] = 0; + invoke(code, io); + CHECK_EQ(io[2], lo); + CHECK_EQ(io[3], hi); + } + } +} + +void testARMv7Move32ToFloatBits() +{ + uint32_t io[2]; + auto code = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load32(CCallHelpers::Address(GPRInfo::argumentGPR0, 0), GPRInfo::argumentGPR2); + jit.move32ToFloat(GPRInfo::argumentGPR2, FPRInfo::fpRegT0); + jit.moveFloatTo32(FPRInfo::fpRegT0, GPRInfo::argumentGPR3); + jit.store32(GPRInfo::argumentGPR3, CCallHelpers::Address(GPRInfo::argumentGPR0, 4)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + for (uint32_t value : { 0u, 1u, 0x3f800000u, 0x80000000u, 0x7f800000u, 0x7fc00000u, 0xffffffffu }) { + io[0] = value; + io[1] = 0; + invoke(code, io); + CHECK_EQ(io[1], value); + } +} + +void testARMv7MoveZeroToDoubleAndFloat() +{ + double outDouble = 1.0; + float outFloat = 1.0f; + auto code = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.moveZeroToDouble(FPRInfo::fpRegT0); + jit.moveZeroToFloat(FPRInfo::fpRegT1); + jit.storeDouble(FPRInfo::fpRegT0, CCallHelpers::Address(GPRInfo::argumentGPR0)); + jit.storeFloat(FPRInfo::fpRegT1, CCallHelpers::Address(GPRInfo::argumentGPR1)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + invoke(code, &outDouble, &outFloat); + CHECK_EQ(std::bit_cast(outDouble), static_cast(0)); + CHECK_EQ(std::bit_cast(outFloat), static_cast(0)); +} + +#endif + +void testAsmMemStoreLoad32OffsetBoundaries() +{ + alignas(8) uint8_t buffer[8448]; + uint8_t* base = buffer + 4096; + + for (auto offset : { 0, 4, 8, 124, 128, 252, 256, 1016, 1020, 1024, 4088, 4092, 4096, -4, -8, -252, -255, -256, -1020, -1024 }) { + auto test = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.store32(GPRInfo::argumentGPR1, CCallHelpers::Address(GPRInfo::argumentGPR0, offset)); + jit.load32(CCallHelpers::Address(GPRInfo::argumentGPR0, offset), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : { 0u, 1u, 0x0000ffffu, 0x7fffffffu, 0x80000000u, 0xffffffffu, 0x12345678u }) { + for (unsigned i = 0; i < sizeof(buffer); ++i) + buffer[i] = 0xa5; + CHECK_EQ(invoke(test, base, value), value); + CHECK_EQ(*bitwise_cast_ptr(base + offset), value); + } + } +} + +void testAsmMemStoreLoad16OffsetBoundaries() +{ + alignas(8) uint8_t buffer[8448]; + uint8_t* base = buffer + 4096; + + for (auto offset : { 0, 2, 30, 62, 64, 126, 128, 254, 256, 1020, 1024, 4094, 4096, -2, -30, -62, -254, -256, -1024 }) { + auto test = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.store16(GPRInfo::argumentGPR1, CCallHelpers::Address(GPRInfo::argumentGPR0, offset)); + jit.load16(CCallHelpers::Address(GPRInfo::argumentGPR0, offset), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto testSigned = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.store16(GPRInfo::argumentGPR1, CCallHelpers::Address(GPRInfo::argumentGPR0, offset)); + jit.load16SignedExtendTo32(CCallHelpers::Address(GPRInfo::argumentGPR0, offset), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : { 0u, 1u, 0x00ffu, 0x0100u, 0x7fffu, 0x8000u, 0xffffu, 0x1234u }) { + for (unsigned i = 0; i < sizeof(buffer); ++i) + buffer[i] = 0xa5; + CHECK_EQ(invoke(test, base, value), value); + CHECK_EQ(*bitwise_cast_ptr(base + offset), static_cast(value)); + CHECK_EQ(invoke(testSigned, base, value), static_cast(static_cast(value))); + } + } +} + +void testAsmMemStoreLoad8OffsetBoundaries() +{ + alignas(8) uint8_t buffer[8448]; + uint8_t* base = buffer + 4096; + + for (auto offset : { 0, 1, 30, 31, 32, 63, 64, 127, 128, 254, 255, 256, 4094, 4095, 4096, -1, -31, -32, -255, -256, -1024 }) { + auto test = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.store8(GPRInfo::argumentGPR1, CCallHelpers::Address(GPRInfo::argumentGPR0, offset)); + jit.load8(CCallHelpers::Address(GPRInfo::argumentGPR0, offset), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto testSigned = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.store8(GPRInfo::argumentGPR1, CCallHelpers::Address(GPRInfo::argumentGPR0, offset)); + jit.load8SignedExtendTo32(CCallHelpers::Address(GPRInfo::argumentGPR0, offset), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : { 0u, 1u, 0x7fu, 0x80u, 0xffu, 0x5au }) { + for (unsigned i = 0; i < sizeof(buffer); ++i) + buffer[i] = 0xa5; + CHECK_EQ(invoke(test, base, value), value); + CHECK_EQ(static_cast(*(base + offset)), value); + CHECK_EQ(invoke(testSigned, base, value), static_cast(static_cast(value))); + } + } +} + +void testAsmMemStoreLoad32BaseIndexScales() +{ + alignas(8) uint8_t buffer[16640]; + uint8_t* base = buffer + 8192; + + for (auto scale : { CCallHelpers::TimesOne, CCallHelpers::TimesTwo, CCallHelpers::TimesFour, CCallHelpers::TimesEight }) { + for (auto offset : { 0, 4, 252, 256, 1020, 1024, 4092, 4096, -4, -252, -256, -1024 }) { + auto test = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.store32(GPRInfo::argumentGPR2, CCallHelpers::BaseIndex(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, scale, offset)); + jit.load32(CCallHelpers::BaseIndex(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, scale, offset), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto index : { 0, 4, 128 }) { + for (auto value : { 0u, 1u, 0x80000000u, 0xffffffffu, 0x12345678u }) { + for (unsigned i = 0; i < sizeof(buffer); ++i) + buffer[i] = 0xa5; + uint8_t* effective = base + (static_cast(index) << static_cast(scale)) + offset; + CHECK_EQ(invoke(test, base, static_cast(index), value), value); + CHECK_EQ(*bitwise_cast_ptr(effective), value); + } + } + } + } +} + +void testAsmMemStoreLoad8BaseIndexScales() +{ + alignas(8) uint8_t buffer[16640]; + uint8_t* base = buffer + 8192; + + for (auto scale : { CCallHelpers::TimesOne, CCallHelpers::TimesTwo, CCallHelpers::TimesFour, CCallHelpers::TimesEight }) { + for (auto offset : { 0, 1, 31, 32, 255, 256, 4095, 4096, -1, -255, -256 }) { + auto test = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.store8(GPRInfo::argumentGPR2, CCallHelpers::BaseIndex(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, scale, offset)); + jit.load8(CCallHelpers::BaseIndex(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, scale, offset), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto index : { 0, 3, 128 }) { + for (auto value : { 0u, 1u, 0x7fu, 0x80u, 0xffu }) { + for (unsigned i = 0; i < sizeof(buffer); ++i) + buffer[i] = 0xa5; + uint8_t* effective = base + (static_cast(index) << static_cast(scale)) + offset; + CHECK_EQ(invoke(test, base, static_cast(index), value), value); + CHECK_EQ(static_cast(*effective), value); + } + } + } + } +} + +void testAsmMemLoadPair32OffsetBoundaries() +{ + alignas(8) uint8_t buffer[8448]; + uint8_t* base = buffer + 4096; + + for (auto offset : { 0, 4, 8, 1012, 1016, 1020, 1024, 2048, 4084, 4088, 4092, -4, -8, -252, -256, -1020, -1024 }) { + auto test = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadPair32(GPRInfo::argumentGPR0, CCallHelpers::TrustedImm32(offset), GPRInfo::argumentGPR2, GPRInfo::argumentGPR3); + jit.store32(GPRInfo::argumentGPR2, CCallHelpers::Address(GPRInfo::argumentGPR1, 0)); + jit.store32(GPRInfo::argumentGPR3, CCallHelpers::Address(GPRInfo::argumentGPR1, 4)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (unsigned i = 0; i < sizeof(buffer); ++i) + buffer[i] = 0xa5; + uint32_t* words = bitwise_cast_ptr(base + offset); + words[0] = 0x11223344u; + words[1] = 0x55667788u; + + uint32_t out[2] = { 0, 0 }; + invoke(test, base, &out[0]); + CHECK_EQ(out[0], 0x11223344u); + CHECK_EQ(out[1], 0x55667788u); + } +} + +void testAsmMemLoadPair32Aliasing() +{ + alignas(8) uint8_t buffer[8448]; + uint8_t* base = buffer + 4096; + + for (auto offset : { 0, 4, 1020, 1024, 4088, 4092, -4, -256, -1024 }) { + auto aliasLow = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadPair32(GPRInfo::argumentGPR0, CCallHelpers::TrustedImm32(offset), GPRInfo::argumentGPR0, GPRInfo::argumentGPR3); + jit.store32(GPRInfo::argumentGPR0, CCallHelpers::Address(GPRInfo::argumentGPR1, 0)); + jit.store32(GPRInfo::argumentGPR3, CCallHelpers::Address(GPRInfo::argumentGPR1, 4)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto aliasHigh = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadPair32(GPRInfo::argumentGPR0, CCallHelpers::TrustedImm32(offset), GPRInfo::argumentGPR3, GPRInfo::argumentGPR0); + jit.store32(GPRInfo::argumentGPR3, CCallHelpers::Address(GPRInfo::argumentGPR1, 0)); + jit.store32(GPRInfo::argumentGPR0, CCallHelpers::Address(GPRInfo::argumentGPR1, 4)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (unsigned i = 0; i < sizeof(buffer); ++i) + buffer[i] = 0xa5; + uint32_t* words = bitwise_cast_ptr(base + offset); + words[0] = 0x0badf00du; + words[1] = 0xfeedfaceu; + + uint32_t out[2] = { 0, 0 }; + invoke(aliasLow, base, &out[0]); + CHECK_EQ(out[0], 0x0badf00du); + CHECK_EQ(out[1], 0xfeedfaceu); + + out[0] = 0; + out[1] = 0; + invoke(aliasHigh, base, &out[0]); + CHECK_EQ(out[0], 0x0badf00du); + CHECK_EQ(out[1], 0xfeedfaceu); + } +} + +void testAsmMemStorePair32OffsetBoundaries() +{ + alignas(8) uint8_t buffer[8448]; + uint8_t* base = buffer + 4096; + + for (auto offset : { 0, 4, 8, 1016, 1020, 1024, 4084, 4088, 4092, -4, -8, -252, -256, -1020, -1024 }) { + auto test = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.storePair32(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, GPRInfo::argumentGPR2, CCallHelpers::TrustedImm32(offset)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (unsigned i = 0; i < sizeof(buffer); ++i) + buffer[i] = 0xa5; + invoke(test, 0x11223344u, 0x55667788u, base); + uint32_t* words = bitwise_cast_ptr(base + offset); + CHECK_EQ(words[0], 0x11223344u); + CHECK_EQ(words[1], 0x55667788u); + } +} + +void testAsmMemShift32ImmediateBoundaries() +{ + for (auto shift : { 0, 1, 2, 15, 16, 30, 31, 32, 33, 63, 64, 96, -1, -31, -32 }) { + auto lshift = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.lshift32(GPRInfo::argumentGPR0, CCallHelpers::TrustedImm32(shift), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto rshift = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.rshift32(GPRInfo::argumentGPR0, CCallHelpers::TrustedImm32(shift), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto urshift = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.urshift32(GPRInfo::argumentGPR0, CCallHelpers::TrustedImm32(shift), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto lshiftInPlace = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.lshift32(CCallHelpers::TrustedImm32(shift), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + unsigned amount = static_cast(shift) & 31; + for (auto value : { 0u, 1u, 2u, 0x0000ffffu, 0x7fffffffu, 0x80000000u, 0xffffffffu, 0x12345678u }) { + CHECK_EQ(invoke(lshift, value), static_cast(value << amount)); + CHECK_EQ(invoke(rshift, value), static_cast(static_cast(value) >> amount)); + CHECK_EQ(invoke(urshift, value), static_cast(value >> amount)); + CHECK_EQ(invoke(lshiftInPlace, value), static_cast(value << amount)); + } + } +} + +void testAsmMemShift32RegisterBoundaries() +{ + auto lshift = compile([] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.lshift32(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto rshift = compile([] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.rshift32(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto urshift = compile([] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.urshift32(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto lshiftAliased = compile([] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.lshift32(GPRInfo::argumentGPR0, GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto shift : { 0u, 1u, 15u, 31u, 32u, 33u, 63u, 64u, 0xffffffffu }) { + unsigned amount = shift & 31; + for (auto value : { 0u, 1u, 0x7fffffffu, 0x80000000u, 0xffffffffu, 0x12345678u }) { + CHECK_EQ(invoke(lshift, value, shift), static_cast(value << amount)); + CHECK_EQ(invoke(rshift, value, shift), static_cast(static_cast(value) >> amount)); + CHECK_EQ(invoke(urshift, value, shift), static_cast(value >> amount)); + } + } + + for (auto value : { 0u, 1u, 3u, 31u, 32u, 33u, 0xffffffffu }) + CHECK_EQ(invoke(lshiftAliased, value), static_cast(value << (value & 31))); +} + +void testAsmMemCountZeros32Boundaries() +{ + auto clz = compile([] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.countLeadingZeros32(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto ctz = compile([] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.countTrailingZeros32(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto clzAliased = compile([] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.countLeadingZeros32(GPRInfo::returnValueGPR, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : { 0u, 1u, 2u, 3u, 0x0000ffffu, 0x00010000u, 0x7fffffffu, 0x80000000u, 0xffffffffu, 0x12345678u }) { + uint32_t expectedClz = 32; + for (unsigned i = 0; i < 32; ++i) { + if (value & (1u << (31 - i))) { + expectedClz = i; + break; + } + } + + uint32_t expectedCtz = 32; + for (unsigned i = 0; i < 32; ++i) { + if (value & (1u << i)) { + expectedCtz = i; + break; + } + } + + CHECK_EQ(invoke(clz, value), expectedClz); + CHECK_EQ(invoke(ctz, value), expectedCtz); + CHECK_EQ(invoke(clzAliased, value), expectedClz); + } +} + +#if CPU(ARM_THUMB2) + +void testAsmMemRotateRight32ImmediateBoundaries() +{ + for (auto shift : { 0, 1, 2, 15, 16, 31, 32, 33, 63, 64, 96, -1, -31, -32 }) { + auto rotate = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.rotateRight32(GPRInfo::argumentGPR0, CCallHelpers::TrustedImm32(shift), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto rotateInPlace = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.rotateRight32(CCallHelpers::TrustedImm32(shift), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + unsigned amount = static_cast(shift) & 31; + for (auto value : { 0u, 1u, 2u, 0x7fffffffu, 0x80000000u, 0xffffffffu, 0x12345678u, 0xdeadbeefu }) { + uint32_t expected = amount ? ((value >> amount) | (value << (32 - amount))) : value; + CHECK_EQ(invoke(rotate, value), expected); + CHECK_EQ(invoke(rotateInPlace, value), expected); + } + } +} + +void testAsmMemRotate32RegisterBoundaries() +{ + auto rotateRight = compile([] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.rotateRight32(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto rotateLeft = compile([] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.rotateLeft32(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto shift : { 0u, 1u, 15u, 31u, 32u, 33u, 63u, 64u, 0xffffffffu }) { + unsigned amount = shift & 31; + for (auto value : { 0u, 1u, 0x7fffffffu, 0x80000000u, 0xffffffffu, 0x12345678u, 0xdeadbeefu }) { + uint32_t expectedRight = amount ? ((value >> amount) | (value << (32 - amount))) : value; + uint32_t expectedLeft = amount ? ((value << amount) | (value >> (32 - amount))) : value; + CHECK_EQ(invoke(rotateRight, value, shift), expectedRight); + CHECK_EQ(invoke(rotateLeft, value, shift), expectedLeft); + } + } +} + +void testAsmMemRotateLeft32ImmediateBoundaries() +{ + for (auto shift : { 0, 1, 15, 31, 32, 33, 63, 64, -1 }) { + auto rotate = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.rotateLeft32(GPRInfo::argumentGPR0, CCallHelpers::TrustedImm32(shift), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + unsigned amount = static_cast(shift) & 31; + for (auto value : { 0u, 1u, 0x80000000u, 0xffffffffu, 0x12345678u }) { + uint32_t expected = amount ? ((value << amount) | (value >> (32 - amount))) : value; + CHECK_EQ(invoke(rotate, value), expected); + } + } +} + +#endif + +#if CPU(ARM_THUMB2) + +static Vector armv7ImmediateOperands() +{ + return Vector { + 0, + 1, + 2, + 7, + 8, + 0xff, + 0x100, + 0x101, + 0x1ff, + 0x200, + 0x3ff, + 0x400, + 0x555, + 0xabc, + 0xfff, + 0x1000, + 0x1001, + 0xff00, + 0xff0000, + static_cast(0xff000000), + 0x00ff00ff, + static_cast(0xff00ff00), + 0x01010101, + static_cast(0xf000000f), + -1, + -2, + -0x100, + -0x101, + -0x1000, + std::numeric_limits::max(), + std::numeric_limits::min(), + }; +} + +static Vector armv7ShiftAmounts() +{ + return Vector { 0, 1, 2, 15, 16, 17, 30, 31, 32, 33, 63, 64, -1, -31, -32 }; +} + +void testARMv7ImmAdd32ArgImm() +{ + for (int32_t immediate : armv7ImmediateOperands()) { + auto add = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + + jit.add32(CCallHelpers::TrustedImm32(immediate), GPRInfo::argumentGPR1, GPRInfo::returnValueGPR); + + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : int32Operands()) + CHECK_EQ(invoke(add, 0, value), static_cast(value) + static_cast(immediate)); + } +} + +void testARMv7ImmAdd32ImmDest() +{ + for (int32_t immediate : armv7ImmediateOperands()) { + auto add = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + + jit.add32(CCallHelpers::TrustedImm32(immediate), GPRInfo::returnValueGPR); + + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : int32Operands()) + CHECK_EQ(invoke(add, value), static_cast(value) + static_cast(immediate)); + } +} + +void testARMv7ImmAddSubRoundTrip() +{ + for (int32_t immediate : armv7ImmediateOperands()) { + auto roundTrip = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + + jit.add32(CCallHelpers::TrustedImm32(immediate), GPRInfo::returnValueGPR); + jit.sub32(CCallHelpers::TrustedImm32(immediate), GPRInfo::returnValueGPR); + + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : int32Operands()) + CHECK_EQ(invoke(roundTrip, value), static_cast(value)); + } +} + +void testARMv7ImmAnd32ArgImm() +{ + for (int32_t immediate : armv7ImmediateOperands()) { + auto andOp = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + + jit.and32(CCallHelpers::TrustedImm32(immediate), GPRInfo::argumentGPR1, GPRInfo::returnValueGPR); + + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto andSame = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + + jit.and32(CCallHelpers::TrustedImm32(immediate), GPRInfo::returnValueGPR); + + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : int32Operands()) { + uint32_t expected = static_cast(value) & static_cast(immediate); + CHECK_EQ(invoke(andOp, 0, value), expected); + CHECK_EQ(invoke(andSame, value), expected); + } + } +} + +void testARMv7ImmOr32ArgImm() +{ + for (int32_t immediate : armv7ImmediateOperands()) { + auto orOp = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + + jit.or32(CCallHelpers::TrustedImm32(immediate), GPRInfo::argumentGPR1, GPRInfo::returnValueGPR); + + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto orSame = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + + jit.or32(CCallHelpers::TrustedImm32(immediate), GPRInfo::returnValueGPR); + + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : int32Operands()) { + uint32_t expected = static_cast(value) | static_cast(immediate); + CHECK_EQ(invoke(orOp, 0, value), expected); + CHECK_EQ(invoke(orSame, value), expected); + } + } +} + +void testARMv7ImmXor32ArgImm() +{ + for (int32_t immediate : armv7ImmediateOperands()) { + auto xorOp = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + + jit.xor32(CCallHelpers::TrustedImm32(immediate), GPRInfo::argumentGPR1, GPRInfo::returnValueGPR); + + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto xorSame = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + + jit.xor32(CCallHelpers::TrustedImm32(immediate), GPRInfo::returnValueGPR); + + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : int32Operands()) { + uint32_t expected = static_cast(value) ^ static_cast(immediate); + CHECK_EQ(invoke(xorOp, 0, value), expected); + CHECK_EQ(invoke(xorSame, value), expected); + } + } +} + +void testARMv7ImmMove32() +{ + for (int32_t immediate : armv7ImmediateOperands()) { + auto moveOp = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + + jit.move(CCallHelpers::TrustedImm32(immediate), GPRInfo::returnValueGPR); + + emitFunctionEpilogue(jit); + jit.ret(); + }); + + CHECK_EQ(invoke(moveOp), static_cast(immediate)); + } +} + +void testARMv7ImmBranch32() +{ + for (int32_t immediate : armv7ImmediateOperands()) { + auto branch = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + + auto equal = jit.branch32(CCallHelpers::Equal, GPRInfo::argumentGPR0, CCallHelpers::TrustedImm32(immediate)); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + auto done = jit.jump(); + equal.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + done.link(&jit); + + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : int32Operands()) + CHECK_EQ(invoke(branch, value), value == immediate ? 1 : 0); + CHECK_EQ(invoke(branch, immediate), 1); + } +} + +void testARMv7ImmCompare32() +{ + for (int32_t immediate : armv7ImmediateOperands()) { + auto compare = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + + jit.compare32(CCallHelpers::LessThan, GPRInfo::argumentGPR0, CCallHelpers::TrustedImm32(immediate), GPRInfo::returnValueGPR); + + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : int32Operands()) + CHECK_EQ(invoke(compare, value), value < immediate ? 1 : 0); + } +} + +void testARMv7ImmRotateRight32() +{ + for (int32_t amount : armv7ShiftAmounts()) { + auto rotate = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + + jit.rotateRight32(GPRInfo::argumentGPR1, CCallHelpers::TrustedImm32(amount), GPRInfo::returnValueGPR); + + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto rotateSame = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + + jit.rotateRight32(CCallHelpers::TrustedImm32(amount), GPRInfo::returnValueGPR); + + emitFunctionEpilogue(jit); + jit.ret(); + }); + + unsigned shift = static_cast(amount) & 31; + for (auto value : int32Operands()) { + uint32_t unsignedValue = static_cast(value); + uint32_t expected = shift ? ((unsignedValue >> shift) | (unsignedValue << (32 - shift))) : unsignedValue; + CHECK_EQ(invoke(rotate, 0, value), expected); + CHECK_EQ(invoke(rotateSame, value), expected); + } + } +} + +void testARMv7ImmRotateLeft32() +{ + for (int32_t amount : armv7ShiftAmounts()) { + auto rotate = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + + jit.rotateLeft32(GPRInfo::argumentGPR1, CCallHelpers::TrustedImm32(amount), GPRInfo::returnValueGPR); + + emitFunctionEpilogue(jit); + jit.ret(); + }); + + unsigned shift = static_cast(amount) & 31; + for (auto value : int32Operands()) { + uint32_t unsignedValue = static_cast(value); + uint32_t expected = shift ? ((unsignedValue << shift) | (unsignedValue >> (32 - shift))) : unsignedValue; + CHECK_EQ(invoke(rotate, 0, value), expected); + } + } +} + +void testARMv7ImmShift32Amounts() +{ + for (int32_t amount : armv7ShiftAmounts()) { + auto left = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + + jit.lshift32(GPRInfo::argumentGPR1, CCallHelpers::TrustedImm32(amount), GPRInfo::returnValueGPR); + + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto arithmeticRight = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + + jit.rshift32(GPRInfo::argumentGPR1, CCallHelpers::TrustedImm32(amount), GPRInfo::returnValueGPR); + + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto logicalRight = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + + jit.urshift32(GPRInfo::argumentGPR1, CCallHelpers::TrustedImm32(amount), GPRInfo::returnValueGPR); + + emitFunctionEpilogue(jit); + jit.ret(); + }); + + unsigned shift = static_cast(amount) & 31; + for (auto value : int32Operands()) { + uint32_t unsignedValue = static_cast(value); + CHECK_EQ(invoke(left, 0, value), unsignedValue << shift); + CHECK_EQ(invoke(arithmeticRight, 0, value), value >> shift); + CHECK_EQ(invoke(logicalRight, 0, value), unsignedValue >> shift); + } + } +} + +#endif + + + +void testRotateRight32Immediate() +{ + for (auto shift : { 0, 1, 4, 7, 15, 16, 30, 31, 32, 33, 47, 63, 64, 96, 128, + -1, -7, -31, -32, -33, -64 }) { + auto rotateToOther = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.rotateRight32(GPRInfo::argumentGPR0, CCallHelpers::TrustedImm32(shift), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto rotateInPlace = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.rotateRight32(CCallHelpers::TrustedImm32(shift), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto rotateSameRegister = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.rotateRight32(GPRInfo::returnValueGPR, CCallHelpers::TrustedImm32(shift), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : int32Operands()) { + uint32_t v = static_cast(value); + uint32_t n = static_cast(shift) & 31; + uint32_t expected = n ? ((v >> n) | (v << (32 - n))) : v; + CHECK_EQ(invoke(rotateToOther, value), expected); + CHECK_EQ(invoke(rotateInPlace, value), expected); + CHECK_EQ(invoke(rotateSameRegister, value), expected); + } + } +} + +void testShift32Immediate() +{ + for (auto shift : { 0, 1, 4, 15, 16, 30, 31, 32, 33, 63, 64, -1, -31, -32, -33 }) { + auto left = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.lshift32(GPRInfo::argumentGPR0, CCallHelpers::TrustedImm32(shift), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto leftInPlace = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.lshift32(CCallHelpers::TrustedImm32(shift), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto arithmeticRight = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.rshift32(GPRInfo::argumentGPR0, CCallHelpers::TrustedImm32(shift), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto arithmeticRightInPlace = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.rshift32(CCallHelpers::TrustedImm32(shift), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto logicalRight = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.urshift32(GPRInfo::argumentGPR0, CCallHelpers::TrustedImm32(shift), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto logicalRightInPlace = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + jit.urshift32(CCallHelpers::TrustedImm32(shift), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : int32Operands()) { + uint32_t v = static_cast(value); + uint32_t n = static_cast(shift) & 31; + CHECK_EQ(invoke(left, value), static_cast(v << n)); + CHECK_EQ(invoke(leftInPlace, value), static_cast(v << n)); + CHECK_EQ(invoke(arithmeticRight, value), static_cast(value >> n)); + CHECK_EQ(invoke(arithmeticRightInPlace, value), static_cast(value >> n)); + CHECK_EQ(invoke(logicalRight, value), static_cast(v >> n)); + CHECK_EQ(invoke(logicalRightInPlace, value), static_cast(v >> n)); + } + } +} + +void testOrAbsoluteAddressAdjacent() +{ + uint32_t words[4]; + uint16_t halves[4]; + uint8_t bytes[4]; + + auto reset = [&] { + for (unsigned i = 0; i < 4; ++i) { + words[i] = 0x11110000u * (i + 1); + halves[i] = static_cast(0x1100 * (i + 1)); + bytes[i] = static_cast(0x10 * (i + 1)); + } + }; + + reset(); + auto orImm32 = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.or32(CCallHelpers::TrustedImm32(1), CCallHelpers::AbsoluteAddress(&words[0])); + jit.or32(CCallHelpers::TrustedImm32(2), CCallHelpers::AbsoluteAddress(&words[1])); + jit.or32(CCallHelpers::TrustedImm32(0x12345678), CCallHelpers::AbsoluteAddress(&words[2])); + jit.or32(CCallHelpers::TrustedImm32(4), CCallHelpers::AbsoluteAddress(&words[3])); + emitFunctionEpilogue(jit); + jit.ret(); + }); + invoke(orImm32); + CHECK_EQ(words[0], 0x11110000u | 1); + CHECK_EQ(words[1], 0x22220000u | 2); + CHECK_EQ(words[2], 0x33330000u | 0x12345678u); + CHECK_EQ(words[3], 0x44440000u | 4); + + reset(); + auto orReg32 = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(CCallHelpers::TrustedImm32(0xf), GPRInfo::argumentGPR0); + jit.or32(GPRInfo::argumentGPR0, CCallHelpers::AbsoluteAddress(&words[0])); + jit.or32(GPRInfo::argumentGPR0, CCallHelpers::AbsoluteAddress(&words[1])); + jit.or32(GPRInfo::argumentGPR0, CCallHelpers::AbsoluteAddress(&words[3])); + emitFunctionEpilogue(jit); + jit.ret(); + }); + invoke(orReg32); + CHECK_EQ(words[0], 0x11110000u | 0xf); + CHECK_EQ(words[1], 0x22220000u | 0xf); + CHECK_EQ(words[2], 0x33330000u); + CHECK_EQ(words[3], 0x44440000u | 0xf); + + reset(); + auto orImm16 = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.or16(CCallHelpers::TrustedImm32(1), CCallHelpers::AbsoluteAddress(&halves[0])); + jit.or16(CCallHelpers::TrustedImm32(2), CCallHelpers::AbsoluteAddress(&halves[1])); + jit.or16(CCallHelpers::TrustedImm32(0x4444), CCallHelpers::AbsoluteAddress(&halves[2])); + emitFunctionEpilogue(jit); + jit.ret(); + }); + invoke(orImm16); + CHECK_EQ(static_cast(halves[0]), static_cast(0x1100 | 1)); + CHECK_EQ(static_cast(halves[1]), static_cast(0x2200 | 2)); + CHECK_EQ(static_cast(halves[2]), static_cast(0x3300 | 0x4444)); + CHECK_EQ(static_cast(halves[3]), static_cast(0x4400)); + + reset(); + auto orReg16 = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(CCallHelpers::TrustedImm32(0xf), GPRInfo::argumentGPR0); + jit.or16(GPRInfo::argumentGPR0, CCallHelpers::AbsoluteAddress(&halves[0])); + jit.or16(GPRInfo::argumentGPR0, CCallHelpers::AbsoluteAddress(&halves[2])); + emitFunctionEpilogue(jit); + jit.ret(); + }); + invoke(orReg16); + CHECK_EQ(static_cast(halves[0]), static_cast(0x1100 | 0xf)); + CHECK_EQ(static_cast(halves[1]), static_cast(0x2200)); + CHECK_EQ(static_cast(halves[2]), static_cast(0x3300 | 0xf)); + CHECK_EQ(static_cast(halves[3]), static_cast(0x4400)); + + reset(); + auto orImm8 = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.or8(CCallHelpers::TrustedImm32(1), CCallHelpers::AbsoluteAddress(&bytes[0])); + jit.or8(CCallHelpers::TrustedImm32(2), CCallHelpers::AbsoluteAddress(&bytes[1])); + jit.or8(CCallHelpers::TrustedImm32(4), CCallHelpers::AbsoluteAddress(&bytes[3])); + emitFunctionEpilogue(jit); + jit.ret(); + }); + invoke(orImm8); + CHECK_EQ(static_cast(bytes[0]), static_cast(0x10 | 1)); + CHECK_EQ(static_cast(bytes[1]), static_cast(0x20 | 2)); + CHECK_EQ(static_cast(bytes[2]), static_cast(0x30)); + CHECK_EQ(static_cast(bytes[3]), static_cast(0x40 | 4)); +} + +void testAdd64ImmAbsoluteAddressAdjacent() +{ + uint64_t counters[3]; + + auto reset = [&] { + counters[0] = 0; + counters[1] = 0x00000000ffffffffull; + counters[2] = 0x1000000000000000ull; + }; + + reset(); + auto smallImmediates = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.add64(CCallHelpers::TrustedImm32(1), CCallHelpers::AbsoluteAddress(&counters[0])); + jit.add64(CCallHelpers::TrustedImm32(1), CCallHelpers::AbsoluteAddress(&counters[1])); + jit.add64(CCallHelpers::TrustedImm32(1), CCallHelpers::AbsoluteAddress(&counters[2])); + emitFunctionEpilogue(jit); + jit.ret(); + }); + invoke(smallImmediates); + CHECK_EQ(counters[0], 1ull); + CHECK_EQ(counters[1], 0x0000000100000000ull); + CHECK_EQ(counters[2], 0x1000000000000001ull); + + reset(); + auto largeImmediates = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.add64(CCallHelpers::TrustedImm32(0x12345678), CCallHelpers::AbsoluteAddress(&counters[0])); + jit.add64(CCallHelpers::TrustedImm32(0x12345678), CCallHelpers::AbsoluteAddress(&counters[1])); + emitFunctionEpilogue(jit); + jit.ret(); + }); + invoke(largeImmediates); + CHECK_EQ(counters[0], 0x12345678ull); + CHECK_EQ(counters[1], 0x0000000112345677ull); + + reset(); + auto negativeImmediates = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.add64(CCallHelpers::TrustedImm32(-1), CCallHelpers::AbsoluteAddress(&counters[0])); + jit.add64(CCallHelpers::TrustedImm32(-1), CCallHelpers::AbsoluteAddress(&counters[1])); + jit.add64(CCallHelpers::TrustedImm32(-0x12345678), CCallHelpers::AbsoluteAddress(&counters[2])); + emitFunctionEpilogue(jit); + jit.ret(); + }); + invoke(negativeImmediates); + CHECK_EQ(counters[0], 0xffffffffffffffffull); + CHECK_EQ(counters[1], 0x00000000fffffffeull); + CHECK_EQ(counters[2], 0x1000000000000000ull - 0x12345678ull); +} + + +#if CPU(ARM_THUMB2) + +static uint32_t masmaluClz32(uint32_t value) +{ + uint32_t count = 0; + while (count < 32 && !(value & 0x80000000u)) { + ++count; + value <<= 1; + } + return count; +} + +static uint32_t masmaluCtz32(uint32_t value) +{ + uint32_t count = 0; + while (count < 32 && !(value & 1)) { + ++count; + value >>= 1; + } + return count; +} + +void testArmv7Arith32Aliasing() +{ + uint32_t inLeft = 0; + uint32_t inRight = 0; + uint32_t out = 0; + uint32_t effective[16]; + + for (auto operation : { 0, 1, 2, 3, 4, 5 }) { + for (auto left : { ARMRegisters::r0, ARMRegisters::r1, ARMRegisters::r2 }) { + for (auto right : { ARMRegisters::r0, ARMRegisters::r1, ARMRegisters::r2 }) { + for (auto dest : { ARMRegisters::r0, ARMRegisters::r1, ARMRegisters::r2, ARMRegisters::r3 }) { + auto code = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load32(&inLeft, left); + jit.load32(&inRight, right); + switch (operation) { + case 0: + jit.add32(left, right, dest); + break; + case 1: + jit.sub32(left, right, dest); + break; + case 2: + jit.and32(left, right, dest); + break; + case 3: + jit.or32(left, right, dest); + break; + case 4: + jit.xor32(left, right, dest); + break; + default: + jit.mul32(left, right, dest); + break; + } + jit.store32(dest, &out); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto a : int32Operands()) { + for (auto b : { 0, 1, -1, 3, 42, -42, 0x7fffffff }) { + inLeft = static_cast(a); + inRight = static_cast(b); + out = 0; + effective[left] = inLeft; + effective[right] = inRight; + uint32_t x = effective[left]; + uint32_t y = effective[right]; + uint32_t expected = 0; + switch (operation) { + case 0: + expected = x + y; + break; + case 1: + expected = x - y; + break; + case 2: + expected = x & y; + break; + case 3: + expected = x | y; + break; + case 4: + expected = x ^ y; + break; + default: + expected = x * y; + break; + } + invoke(code); + CHECK_EQ(out, expected); + } + } + } + } + } + } +} + +void testArmv7Arith32ImmediateAliasing() +{ + uint32_t in = 0; + uint32_t out = 0; + + for (auto immediate : { 0, 1, 2, 7, 8, 0xff, 0x100, 0x101, 0x1ff, 0x200, 0x3ff, + 0x400, 0xabc, 0xfff, 0x1000, 0x1001, 0x10000, 0x12345678, -1, -2, -0xff, + -0x1000, std::numeric_limits::max(), std::numeric_limits::min() }) { + for (auto operation : { 0, 1, 2, 3, 4, 5, 6 }) { + for (auto src : { ARMRegisters::r0, ARMRegisters::r1 }) { + for (auto dest : { ARMRegisters::r0, ARMRegisters::r1, ARMRegisters::r2 }) { + auto code = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load32(&in, src); + switch (operation) { + case 0: + jit.add32(CCallHelpers::TrustedImm32(immediate), src, dest); + break; + case 1: + jit.and32(CCallHelpers::TrustedImm32(immediate), src, dest); + break; + case 2: + jit.or32(CCallHelpers::TrustedImm32(immediate), src, dest); + break; + case 3: + jit.xor32(CCallHelpers::TrustedImm32(immediate), src, dest); + break; + case 4: + jit.sub32(src, CCallHelpers::TrustedImm32(immediate), dest); + break; + case 5: + jit.sub32(CCallHelpers::TrustedImm32(immediate), src, dest); + break; + default: + jit.mul32(CCallHelpers::TrustedImm32(immediate), src, dest); + break; + } + jit.store32(dest, &out); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : int32Operands()) { + in = static_cast(value); + out = 0; + uint32_t v = in; + uint32_t i = static_cast(immediate); + uint32_t expected = 0; + switch (operation) { + case 0: + expected = v + i; + break; + case 1: + expected = v & i; + break; + case 2: + expected = v | i; + break; + case 3: + expected = v ^ i; + break; + case 4: + expected = v - i; + break; + case 5: + expected = i - v; + break; + default: + expected = v * i; + break; + } + invoke(code); + CHECK_EQ(out, expected); + } + } + } + } + } +} + +void testArmv7Arith32InPlaceImmediate() +{ + uint32_t in = 0; + uint32_t out = 0; + + for (auto immediate : { 0, 1, 0xff, 0x100, 0x3ff, 0x400, 0xfff, 0x1000, 0x12345678, + -1, -0x1000, std::numeric_limits::max(), std::numeric_limits::min() }) { + for (auto operation : { 0, 1, 2, 3, 4 }) { + for (auto dest : { ARMRegisters::r0, ARMRegisters::r1 }) { + auto code = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load32(&in, dest); + switch (operation) { + case 0: + jit.add32(CCallHelpers::TrustedImm32(immediate), dest); + break; + case 1: + jit.and32(CCallHelpers::TrustedImm32(immediate), dest); + break; + case 2: + jit.or32(CCallHelpers::TrustedImm32(immediate), dest); + break; + case 3: + jit.xor32(CCallHelpers::TrustedImm32(immediate), dest); + break; + default: + jit.sub32(CCallHelpers::TrustedImm32(immediate), dest); + break; + } + jit.store32(dest, &out); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : int32Operands()) { + in = static_cast(value); + out = 0; + uint32_t v = in; + uint32_t i = static_cast(immediate); + uint32_t expected = 0; + switch (operation) { + case 0: + expected = v + i; + break; + case 1: + expected = v & i; + break; + case 2: + expected = v | i; + break; + case 3: + expected = v ^ i; + break; + default: + expected = v - i; + break; + } + invoke(code); + CHECK_EQ(out, expected); + } + } + } + } +} + +void testArmv7Neg32Not32Aliasing() +{ + uint32_t in = 0; + uint32_t out = 0; + + for (auto src : { ARMRegisters::r0, ARMRegisters::r1 }) { + for (auto dest : { ARMRegisters::r0, ARMRegisters::r1, ARMRegisters::r2 }) { + auto negate = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load32(&in, src); + jit.neg32(src, dest); + jit.store32(dest, &out); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto invert = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load32(&in, src); + jit.not32(src, dest); + jit.store32(dest, &out); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto negateInPlace = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load32(&in, dest); + jit.neg32(dest); + jit.store32(dest, &out); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto invertInPlace = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load32(&in, dest); + jit.not32(dest); + jit.store32(dest, &out); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : int32Operands()) { + in = static_cast(value); + out = 0; + invoke(negate); + CHECK_EQ(out, static_cast(0) - static_cast(value)); + out = 0; + invoke(invert); + CHECK_EQ(out, ~static_cast(value)); + out = 0; + invoke(negateInPlace); + CHECK_EQ(out, static_cast(0) - static_cast(value)); + out = 0; + invoke(invertInPlace); + CHECK_EQ(out, ~static_cast(value)); + } + } + } +} + +void testArmv7CountZeros32Aliasing() +{ + uint32_t in = 0; + uint32_t out = 0; + + for (auto src : { ARMRegisters::r0, ARMRegisters::r1 }) { + for (auto dest : { ARMRegisters::r0, ARMRegisters::r1, ARMRegisters::r2 }) { + auto leading = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load32(&in, src); + jit.countLeadingZeros32(src, dest); + jit.store32(dest, &out); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto trailing = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load32(&in, src); + jit.countTrailingZeros32(src, dest); + jit.store32(dest, &out); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : { 0u, 1u, 2u, 3u, 0x80000000u, 0xffffffffu, 0x00010000u, + 0x00008000u, 0x40000000u, 0x0000ffffu }) { + in = value; + out = 0; + invoke(leading); + CHECK_EQ(out, masmaluClz32(value)); + out = 0; + invoke(trailing); + CHECK_EQ(out, masmaluCtz32(value)); + } + } + } +} + +void testArmv7Shift32RegisterAliasing() +{ + uint32_t inValue = 0; + uint32_t inShift = 0; + uint32_t out = 0; + uint32_t effective[16]; + + for (auto operation : { 0, 1, 2 }) { + for (auto src : { ARMRegisters::r0, ARMRegisters::r1 }) { + for (auto shiftAmount : { ARMRegisters::r0, ARMRegisters::r1 }) { + for (auto dest : { ARMRegisters::r0, ARMRegisters::r1, ARMRegisters::r2 }) { + auto code = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load32(&inValue, src); + jit.load32(&inShift, shiftAmount); + switch (operation) { + case 0: + jit.lshift32(src, shiftAmount, dest); + break; + case 1: + jit.rshift32(src, shiftAmount, dest); + break; + default: + jit.urshift32(src, shiftAmount, dest); + break; + } + jit.store32(dest, &out); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : int32Operands()) { + for (auto shift : { 0, 1, 5, 31, 32, 33, 63, 64, 255, 256, -1 }) { + inValue = static_cast(value); + inShift = static_cast(shift); + out = 0; + effective[src] = inValue; + effective[shiftAmount] = inShift; + uint32_t v = effective[src]; + uint32_t n = effective[shiftAmount] & 31; + uint32_t expected = 0; + switch (operation) { + case 0: + expected = v << n; + break; + case 1: + expected = static_cast(static_cast(v) >> n); + break; + default: + expected = v >> n; + break; + } + invoke(code); + CHECK_EQ(out, expected); + } + } + } + } + } + } +} + +void testArmv7Shift32ImmediateValueRegisterAmount() +{ + uint32_t inShift = 0; + uint32_t out = 0; + + for (auto immediate : { 0, 1, 0xff, 0x100, 0x3ff, 0x400, 0xfff, 0x1000, 0x12345678, + -1, std::numeric_limits::min() }) { + for (auto operation : { 0, 1 }) { + for (auto shiftAmount : { ARMRegisters::r0, ARMRegisters::r1 }) { + for (auto dest : { ARMRegisters::r0, ARMRegisters::r1, ARMRegisters::r2 }) { + auto code = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load32(&inShift, shiftAmount); + if (!operation) + jit.lshift32(CCallHelpers::TrustedImm32(immediate), shiftAmount, dest); + else + jit.rshift32(CCallHelpers::TrustedImm32(immediate), shiftAmount, dest); + jit.store32(dest, &out); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto shift : { 0, 1, 5, 31, 32, 33, 63, 255, -1 }) { + inShift = static_cast(shift); + out = 0; + uint32_t n = inShift & 31; + uint32_t v = static_cast(immediate); + uint32_t expected = operation + ? static_cast(static_cast(v) >> n) + : (v << n); + invoke(code); + CHECK_EQ(out, expected); + } + } + } + } + } +} + +void testArmv7ShiftUnchecked() +{ + uint32_t inValue = 0; + uint32_t inShift = 0; + uint32_t out = 0; + uint32_t effective[16]; + + for (auto operation : { 0, 1, 2 }) { + for (auto src : { ARMRegisters::r0, ARMRegisters::r1 }) { + for (auto shiftAmount : { ARMRegisters::r0, ARMRegisters::r1 }) { + for (auto dest : { ARMRegisters::r0, ARMRegisters::r1, ARMRegisters::r2 }) { + auto code = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load32(&inValue, src); + jit.load32(&inShift, shiftAmount); + switch (operation) { + case 0: + jit.lshiftUnchecked(src, shiftAmount, dest); + break; + case 1: + jit.rshiftUnchecked(src, shiftAmount, dest); + break; + default: + jit.urshiftUnchecked(src, shiftAmount, dest); + break; + } + jit.store32(dest, &out); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : int32Operands()) { + for (auto shift : { 0, 1, 5, 30, 31 }) { + inValue = static_cast(value); + inShift = static_cast(shift); + out = 0; + effective[src] = inValue; + effective[shiftAmount] = inShift; + uint32_t v = effective[src]; + uint32_t n = effective[shiftAmount] & 31; + uint32_t expected = 0; + switch (operation) { + case 0: + expected = v << n; + break; + case 1: + expected = static_cast(static_cast(v) >> n); + break; + default: + expected = v >> n; + break; + } + invoke(code); + CHECK_EQ(out, expected); + } + } + } + } + } + } +} + +void testArmv7RotateLeft32() +{ + uint32_t inValue = 0; + uint32_t inShift = 0; + uint32_t out = 0; + uint32_t effective[16]; + + for (auto src : { ARMRegisters::r0, ARMRegisters::r1 }) { + for (auto shiftAmount : { ARMRegisters::r0, ARMRegisters::r1 }) { + for (auto dest : { ARMRegisters::r0, ARMRegisters::r1, ARMRegisters::r2 }) { + auto code = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load32(&inValue, src); + jit.load32(&inShift, shiftAmount); + jit.rotateLeft32(src, shiftAmount, dest); + jit.store32(dest, &out); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : int32Operands()) { + for (auto shift : { 0, 1, 5, 31, 32, 33, 63, 255, -1 }) { + inValue = static_cast(value); + inShift = static_cast(shift); + out = 0; + effective[src] = inValue; + effective[shiftAmount] = inShift; + uint32_t v = effective[src]; + uint32_t n = effective[shiftAmount] & 31; + uint32_t expected = n ? ((v << n) | (v >> (32 - n))) : v; + invoke(code); + CHECK_EQ(out, expected); + } + } + } + } + } +} + +void testArmv7RotateLeft32Immediate() +{ + uint32_t in = 0; + uint32_t out = 0; + + for (auto shift : { 0, 1, 5, 31, 32, 33, 63, 64, 96, 255, -1, -31, -32, -33 }) { + for (auto src : { ARMRegisters::r0, ARMRegisters::r1 }) { + for (auto dest : { ARMRegisters::r0, ARMRegisters::r1, ARMRegisters::r2 }) { + auto code = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load32(&in, src); + jit.rotateLeft32(src, CCallHelpers::TrustedImm32(shift), dest); + jit.store32(dest, &out); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : int32Operands()) { + in = static_cast(value); + out = 0; + uint32_t v = in; + uint32_t n = static_cast(shift) & 31; + uint32_t expected = n ? ((v << n) | (v >> (32 - n))) : v; + invoke(code); + CHECK_EQ(out, expected); + } + } + } + } +} + +void testArmv7Add64Sub64RegisterPairs() +{ + uint32_t inA[2]; + uint32_t inB[2]; + uint32_t outHi = 0; + uint32_t outLo = 0; + uint32_t effective[16]; + + for (auto subtract : { false, true }) { + for (auto op2Hi : { ARMRegisters::r0, ARMRegisters::r1, ARMRegisters::r2 }) { + for (auto op2Lo : { ARMRegisters::r1, ARMRegisters::r2, ARMRegisters::r3 }) { + for (auto destHi : { ARMRegisters::r0, ARMRegisters::r2, ARMRegisters::r3 }) { + for (auto destLo : { ARMRegisters::r0, ARMRegisters::r1, ARMRegisters::r2, ARMRegisters::r3 }) { + if (destHi == destLo) + continue; + + auto op1Hi = ARMRegisters::r0; + auto op1Lo = ARMRegisters::r1; + + auto code = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load32(&inA[1], op1Hi); + jit.load32(&inA[0], op1Lo); + jit.load32(&inB[1], op2Hi); + jit.load32(&inB[0], op2Lo); + if (subtract) + jit.sub64(op1Hi, op1Lo, op2Hi, op2Lo, destHi, destLo); + else + jit.add64(op1Hi, op1Lo, op2Hi, op2Lo, destHi, destLo); + jit.store32(destLo, &outLo); + jit.store32(destHi, &outHi); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto pair : { 0, 1, 2, 3, 4 }) { + switch (pair) { + case 0: + inA[1] = 0; + inA[0] = 1; + inB[1] = 0; + inB[0] = 2; + break; + case 1: + inA[1] = 0; + inA[0] = 0xffffffffu; + inB[1] = 0; + inB[0] = 1; + break; + case 2: + inA[1] = 0xffffffffu; + inA[0] = 0xffffffffu; + inB[1] = 0; + inB[0] = 1; + break; + case 3: + inA[1] = 0x12345678u; + inA[0] = 0x9abcdef0u; + inB[1] = 0x0fedcba9u; + inB[0] = 0x87654321u; + break; + default: + inA[1] = 0; + inA[0] = 0; + inB[1] = 0; + inB[0] = 1; + break; + } + + outHi = 0; + outLo = 0; + effective[op1Hi] = inA[1]; + effective[op1Lo] = inA[0]; + effective[op2Hi] = inB[1]; + effective[op2Lo] = inB[0]; + uint64_t a = (static_cast(effective[op1Hi]) << 32) | effective[op1Lo]; + uint64_t b = (static_cast(effective[op2Hi]) << 32) | effective[op2Lo]; + uint64_t expected = subtract ? (a - b) : (a + b); + + invoke(code); + CHECK_EQ((static_cast(outHi) << 32) | outLo, expected); + } + } + } + } + } + } +} + +void testArmv7CountZeros64RegisterPairs() +{ + uint32_t inHi = 0; + uint32_t inLo = 0; + uint32_t outHi = 0; + uint32_t outLo = 0; + uint32_t effective[16]; + + for (auto trailing : { false, true }) { + for (auto srcHi : { ARMRegisters::r0, ARMRegisters::r1 }) { + for (auto srcLo : { ARMRegisters::r0, ARMRegisters::r1 }) { + for (auto destHi : { ARMRegisters::r0, ARMRegisters::r1, ARMRegisters::r2, ARMRegisters::r3 }) { + for (auto destLo : { ARMRegisters::r0, ARMRegisters::r1, ARMRegisters::r2, ARMRegisters::r3 }) { + if (destHi == destLo) + continue; + + auto code = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load32(&inHi, srcHi); + jit.load32(&inLo, srcLo); + if (trailing) + jit.countTrailingZeros64(srcHi, srcLo, destHi, destLo); + else + jit.countLeadingZeros64(srcHi, srcLo, destHi, destLo); + jit.store32(destLo, &outLo); + jit.store32(destHi, &outHi); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto hi : { 0u, 1u, 0x80000000u, 0xffffffffu, 0x00010000u }) { + for (auto lo : { 0u, 1u, 0x80000000u, 0xffffffffu, 0x00010000u }) { + inHi = hi; + inLo = lo; + outHi = 0; + outLo = 0; + effective[srcHi] = inHi; + effective[srcLo] = inLo; + uint32_t h = effective[srcHi]; + uint32_t l = effective[srcLo]; + uint32_t expected; + if (trailing) { + uint32_t low = masmaluCtz32(l); + expected = low != 32 ? low : 32 + masmaluCtz32(h); + } else { + uint32_t high = masmaluClz32(h); + expected = high != 32 ? high : 32 + masmaluClz32(l); + } + invoke(code); + CHECK_EQ(outLo, expected); + CHECK_EQ(outHi, 0u); + } + } + } + } + } + } + } +} + +void testArmv7AddUnsignedRightShift32() +{ + uint32_t in1 = 0; + uint32_t in2 = 0; + uint32_t out = 0; + uint32_t effective[16]; + + for (auto amount : { 0, 1, 5, 31, 32, 33, -1 }) { + for (auto src1 : { ARMRegisters::r0, ARMRegisters::r1 }) { + for (auto src2 : { ARMRegisters::r0, ARMRegisters::r1 }) { + for (auto dest : { ARMRegisters::r0, ARMRegisters::r1, ARMRegisters::r2 }) { + auto code = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load32(&in1, src1); + jit.load32(&in2, src2); + jit.addUnsignedRightShift32(src1, src2, CCallHelpers::TrustedImm32(amount), dest); + jit.store32(dest, &out); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : int32Operands()) { + in1 = static_cast(value); + in2 = 0xdeadbeefu; + out = 0; + effective[src1] = in1; + effective[src2] = in2; + uint32_t n = static_cast(amount) & 31; + uint32_t expected = effective[src1] + (effective[src2] >> n); + invoke(code); + CHECK_EQ(out, expected); + } + } + } + } + } +} + +void testArmv7Add32Sub32MemoryOperands() +{ + uint32_t words[4]; + uint32_t out = 0; + + auto reset = [&] { + for (unsigned i = 0; i < 4; ++i) + words[i] = 0x11110000u * (i + 1); + }; + + for (auto dest : { ARMRegisters::r0, ARMRegisters::r1 }) { + reset(); + auto addAbsolute = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(CCallHelpers::TrustedImm32(7), dest); + jit.add32(CCallHelpers::AbsoluteAddress(&words[0]), dest); + jit.add32(CCallHelpers::AbsoluteAddress(&words[2]), dest); + jit.store32(dest, &out); + emitFunctionEpilogue(jit); + jit.ret(); + }); + out = 0; + invoke(addAbsolute); + CHECK_EQ(out, 7u + 0x11110000u + 0x33330000u); + + reset(); + auto memoryImmediates = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.add32(CCallHelpers::TrustedImm32(1), CCallHelpers::AbsoluteAddress(&words[0])); + jit.add32(CCallHelpers::TrustedImm32(0x12345678), CCallHelpers::AbsoluteAddress(&words[1])); + jit.sub32(CCallHelpers::TrustedImm32(1), CCallHelpers::AbsoluteAddress(&words[2])); + jit.sub32(CCallHelpers::TrustedImm32(0x12345678), CCallHelpers::AbsoluteAddress(&words[3])); + emitFunctionEpilogue(jit); + jit.ret(); + }); + invoke(memoryImmediates); + CHECK_EQ(words[0], 0x11110000u + 1); + CHECK_EQ(words[1], 0x22220000u + 0x12345678u); + CHECK_EQ(words[2], 0x33330000u - 1); + CHECK_EQ(words[3], 0x44440000u - 0x12345678u); + } +} + +#endif + +static bool evaluateRelational32(MacroAssembler::RelationalCondition cond, int32_t a, int32_t b) +{ + switch (cond) { + case MacroAssembler::Equal: return a == b; + case MacroAssembler::NotEqual: return a != b; + case MacroAssembler::Above: return static_cast(a) > static_cast(b); + case MacroAssembler::AboveOrEqual: return static_cast(a) >= static_cast(b); + case MacroAssembler::Below: return static_cast(a) < static_cast(b); + case MacroAssembler::BelowOrEqual: return static_cast(a) <= static_cast(b); + case MacroAssembler::GreaterThan: return a > b; + case MacroAssembler::GreaterThanOrEqual: return a >= b; + case MacroAssembler::LessThan: return a < b; + case MacroAssembler::LessThanOrEqual: return a <= b; + } + RELEASE_ASSERT_NOT_REACHED(); + return false; +} + +static bool evaluateResult32(MacroAssembler::ResultCondition cond, int32_t result, bool carry, bool overflow) +{ + switch (cond) { + case MacroAssembler::Zero: return !result; + case MacroAssembler::NonZero: return !!result; + case MacroAssembler::Signed: return result < 0; + case MacroAssembler::PositiveOrZero: return result >= 0; + case MacroAssembler::Carry: return carry; + case MacroAssembler::Overflow: return overflow; + } + RELEASE_ASSERT_NOT_REACHED(); + return false; +} + +static Vector masmbrRelationalConditions() +{ + return Vector { + MacroAssembler::Equal, + MacroAssembler::NotEqual, + MacroAssembler::Above, + MacroAssembler::AboveOrEqual, + MacroAssembler::Below, + MacroAssembler::BelowOrEqual, + MacroAssembler::GreaterThan, + MacroAssembler::GreaterThanOrEqual, + MacroAssembler::LessThan, + MacroAssembler::LessThanOrEqual, + }; +} + +static Vector masmbrBoundary32() +{ + return Vector { + 0, 1, -1, 2, -2, 0x7f, 0x80, 0xff, 0x100, 0x3ff, 0x400, 0xfff, 0x1000, 0x1001, + -0xff, -0x100, -0x1000, 0x7ffffffe, + std::numeric_limits::max(), + std::numeric_limits::min(), + static_cast(0x80000001u), + static_cast(0xfffffffeu), + }; +} + +void testBranch32RegReg() +{ + for (auto cond : masmbrRelationalConditions()) { + auto test = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branch32(cond, GPRInfo::argumentGPR0, GPRInfo::argumentGPR1); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto a : masmbrBoundary32()) { + for (auto b : masmbrBoundary32()) + CHECK_EQ(invoke(test, a, b), evaluateRelational32(cond, a, b) ? 1 : 0); + CHECK_EQ(invoke(test, a, a), evaluateRelational32(cond, a, a) ? 1 : 0); + } + } + + for (auto cond : masmbrRelationalConditions()) { + auto test = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branch32(cond, GPRInfo::argumentGPR0, GPRInfo::argumentGPR0); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto a : masmbrBoundary32()) + CHECK_EQ(invoke(test, a), evaluateRelational32(cond, a, a) ? 1 : 0); + } +} + +void testBranch32RegImm() +{ + for (auto cond : masmbrRelationalConditions()) { + for (auto imm : masmbrBoundary32()) { + auto test = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branch32(cond, GPRInfo::argumentGPR0, CCallHelpers::TrustedImm32(imm)); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto a : masmbrBoundary32()) + CHECK_EQ(invoke(test, a), evaluateRelational32(cond, a, imm) ? 1 : 0); + } + } +} + +void testBranch32Address() +{ + for (auto cond : masmbrRelationalConditions()) { + for (auto offset : { 0, 4, 0x100, 0xffc, 0x1000, 0x1004 }) { + auto testAddrImm = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branch32(cond, CCallHelpers::Address(GPRInfo::argumentGPR0, offset), CCallHelpers::TrustedImm32(0x1234)); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto testAddrReg = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branch32(cond, CCallHelpers::Address(GPRInfo::argumentGPR0, offset), GPRInfo::argumentGPR1); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto testRegAddr = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branch32(cond, GPRInfo::argumentGPR1, CCallHelpers::Address(GPRInfo::argumentGPR0, offset)); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + Vector buffer(0x1010 + sizeof(int32_t)); + for (auto a : masmbrBoundary32()) { + *std::bit_cast(buffer.data() + offset) = a; + CHECK_EQ(invoke(testAddrImm, buffer.data()), evaluateRelational32(cond, a, 0x1234) ? 1 : 0); + for (auto b : masmbrBoundary32()) { + CHECK_EQ(invoke(testAddrReg, buffer.data(), b), evaluateRelational32(cond, a, b) ? 1 : 0); + CHECK_EQ(invoke(testRegAddr, buffer.data(), b), evaluateRelational32(cond, b, a) ? 1 : 0); + } + } + } + } +} + +void testBranch8And16() +{ + for (auto cond : masmbrRelationalConditions()) { + for (auto imm : { 0, 1, 0x7f, 0x80, 0xfe, 0xff }) { + auto test8 = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branch8(cond, CCallHelpers::Address(GPRInfo::argumentGPR0, 3), CCallHelpers::TrustedImm32(imm)); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + uint8_t buffer[8] = { 0, 0, 0, 0, 0, 0, 0, 0 }; + for (auto v : { 0, 1, 0x7f, 0x80, 0xfe, 0xff }) { + buffer[3] = static_cast(v); + int32_t left = (cond == MacroAssembler::GreaterThan || cond == MacroAssembler::GreaterThanOrEqual + || cond == MacroAssembler::LessThan || cond == MacroAssembler::LessThanOrEqual) + ? static_cast(static_cast(v)) : static_cast(static_cast(v)); + int32_t right = (cond == MacroAssembler::GreaterThan || cond == MacroAssembler::GreaterThanOrEqual + || cond == MacroAssembler::LessThan || cond == MacroAssembler::LessThanOrEqual) + ? static_cast(static_cast(imm)) : static_cast(static_cast(imm)); + CHECK_EQ(invoke(test8, buffer), evaluateRelational32(cond, left, right) ? 1 : 0); + } + } + } + + for (auto cond : masmbrRelationalConditions()) { + for (auto imm : { 0, 1, 0x7fff, 0x8000, 0xfffe, 0xffff }) { + auto test16 = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branch16(cond, CCallHelpers::Address(GPRInfo::argumentGPR0, 2), CCallHelpers::TrustedImm32(imm)); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + uint16_t buffer[4] = { 0, 0, 0, 0 }; + for (auto v : { 0, 1, 0x7fff, 0x8000, 0xfffe, 0xffff }) { + buffer[1] = static_cast(v); + bool isSigned = (cond == MacroAssembler::GreaterThan || cond == MacroAssembler::GreaterThanOrEqual + || cond == MacroAssembler::LessThan || cond == MacroAssembler::LessThanOrEqual); + int32_t left = isSigned ? static_cast(static_cast(v)) : static_cast(static_cast(v)); + int32_t right = isSigned ? static_cast(static_cast(imm)) : static_cast(static_cast(imm)); + CHECK_EQ(invoke(test16, buffer), evaluateRelational32(cond, left, right) ? 1 : 0); + } + } + } +} + +void testCompare32Setter() +{ + for (auto cond : masmbrRelationalConditions()) { + auto testNoAlias = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(CCallHelpers::TrustedImm32(0x5a5a5a5a), GPRInfo::argumentGPR2); + jit.compare32(cond, GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, GPRInfo::argumentGPR2); + jit.move(GPRInfo::argumentGPR2, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto testDestIsLeft = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.compare32(cond, GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, GPRInfo::argumentGPR0); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto testDestIsRight = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.compare32(cond, GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, GPRInfo::argumentGPR1); + jit.move(GPRInfo::argumentGPR1, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto testAllSame = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.compare32(cond, GPRInfo::argumentGPR0, GPRInfo::argumentGPR0, GPRInfo::argumentGPR0); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto a : masmbrBoundary32()) { + CHECK_EQ(invoke(testAllSame, a), evaluateRelational32(cond, a, a) ? 1 : 0); + for (auto b : masmbrBoundary32()) { + int expected = evaluateRelational32(cond, a, b) ? 1 : 0; + CHECK_EQ(invoke(testNoAlias, a, b), expected); + CHECK_EQ(invoke(testDestIsLeft, a, b), expected); + CHECK_EQ(invoke(testDestIsRight, a, b), expected); + } + } + } +} + +void testCompare32SetterImm() +{ + for (auto cond : masmbrRelationalConditions()) { + for (auto imm : masmbrBoundary32()) { + auto test = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.compare32(cond, GPRInfo::argumentGPR0, CCallHelpers::TrustedImm32(imm), GPRInfo::argumentGPR0); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto a : masmbrBoundary32()) + CHECK_EQ(invoke(test, a), evaluateRelational32(cond, a, imm) ? 1 : 0); + } + } +} + +void testBranchTest32Masmbr() +{ + for (auto cond : { MacroAssembler::Zero, MacroAssembler::NonZero, MacroAssembler::Signed, MacroAssembler::PositiveOrZero }) { + auto testRegReg = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branchTest32(cond, GPRInfo::argumentGPR0, GPRInfo::argumentGPR1); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto testSameReg = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branchTest32(cond, GPRInfo::argumentGPR0, GPRInfo::argumentGPR0); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto a : masmbrBoundary32()) { + CHECK_EQ(invoke(testSameReg, a), evaluateResult32(cond, a, false, false) ? 1 : 0); + for (auto b : masmbrBoundary32()) + CHECK_EQ(invoke(testRegReg, a, b), evaluateResult32(cond, a & b, false, false) ? 1 : 0); + } + + for (auto mask : masmbrBoundary32()) { + auto testRegImm = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branchTest32(cond, GPRInfo::argumentGPR0, CCallHelpers::TrustedImm32(mask)); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto a : masmbrBoundary32()) + CHECK_EQ(invoke(testRegImm, a), evaluateResult32(cond, a & mask, false, false) ? 1 : 0); + } + + auto testDefaultMask = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branchTest32(cond, GPRInfo::argumentGPR0); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto a : masmbrBoundary32()) + CHECK_EQ(invoke(testDefaultMask, a), evaluateResult32(cond, a, false, false) ? 1 : 0); + } +} + +void testTest32Setter() +{ + for (auto cond : { MacroAssembler::Zero, MacroAssembler::NonZero, MacroAssembler::Signed, MacroAssembler::PositiveOrZero }) { + auto testRegReg = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.test32(cond, GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, GPRInfo::argumentGPR0); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto a : masmbrBoundary32()) { + for (auto b : masmbrBoundary32()) + CHECK_EQ(invoke(testRegReg, a, b), evaluateResult32(cond, a & b, false, false) ? 1 : 0); + } + + for (auto mask : masmbrBoundary32()) { + auto testRegImm = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.test32(cond, GPRInfo::argumentGPR0, CCallHelpers::TrustedImm32(mask), GPRInfo::argumentGPR1); + jit.move(GPRInfo::argumentGPR1, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto a : masmbrBoundary32()) + CHECK_EQ(invoke(testRegImm, a), evaluateResult32(cond, a & mask, false, false) ? 1 : 0); + } + } +} + +void testBranchAdd32Masmbr() +{ + for (auto cond : { MacroAssembler::Zero, MacroAssembler::NonZero, MacroAssembler::Signed, MacroAssembler::PositiveOrZero, MacroAssembler::Overflow }) { + auto testThreeArg = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branchAdd32(cond, GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, GPRInfo::argumentGPR2); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto testDestIsOp1 = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branchAdd32(cond, GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, GPRInfo::argumentGPR0); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto testAllSame = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branchAdd32(cond, GPRInfo::argumentGPR0, GPRInfo::argumentGPR0, GPRInfo::argumentGPR0); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto a : masmbrBoundary32()) { + int64_t wideSame = static_cast(a) + static_cast(a); + int32_t sumSame = static_cast(static_cast(a) + static_cast(a)); + CHECK_EQ(invoke(testAllSame, a), evaluateResult32(cond, sumSame, false, wideSame != sumSame) ? 1 : 0); + + for (auto b : masmbrBoundary32()) { + int64_t wide = static_cast(a) + static_cast(b); + int32_t sum = static_cast(static_cast(a) + static_cast(b)); + int expected = evaluateResult32(cond, sum, false, wide != sum) ? 1 : 0; + CHECK_EQ(invoke(testThreeArg, a, b), expected); + CHECK_EQ(invoke(testDestIsOp1, a, b), expected); + } + } + + for (auto imm : masmbrBoundary32()) { + auto testImm = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branchAdd32(cond, GPRInfo::argumentGPR0, CCallHelpers::TrustedImm32(imm), GPRInfo::argumentGPR0); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto a : masmbrBoundary32()) { + int64_t wide = static_cast(a) + static_cast(imm); + int32_t sum = static_cast(static_cast(a) + static_cast(imm)); + CHECK_EQ(invoke(testImm, a), evaluateResult32(cond, sum, false, wide != sum) ? 1 : 0); + } + } + } +} + +void testBranchAdd32Address() +{ + for (auto cond : { MacroAssembler::Zero, MacroAssembler::NonZero, MacroAssembler::Signed, MacroAssembler::PositiveOrZero, MacroAssembler::Overflow }) { + for (auto offset : { 0, 4, 0x100, 0xffc, 0x1000, 5000 }) { + for (auto imm : { 1, -1, 0xff, 0x100, 0x1000, 5100, 0x12345 }) { + auto test = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branchAdd32(cond, CCallHelpers::TrustedImm32(imm), CCallHelpers::Address(GPRInfo::argumentGPR0, offset)); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + Vector buffer(6000 + sizeof(int32_t)); + for (auto a : masmbrBoundary32()) { + *std::bit_cast(buffer.data() + offset) = a; + int64_t wide = static_cast(a) + static_cast(imm); + int32_t sum = static_cast(static_cast(a) + static_cast(imm)); + CHECK_EQ(invoke(test, buffer.data()), evaluateResult32(cond, sum, false, wide != sum) ? 1 : 0); + CHECK_EQ(*std::bit_cast(buffer.data() + offset), sum); + } + } + } + } +} + +void testBranchSub32Masmbr() +{ + for (auto cond : { MacroAssembler::Zero, MacroAssembler::NonZero, MacroAssembler::Signed, MacroAssembler::PositiveOrZero, MacroAssembler::Overflow }) { + auto testThreeArg = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branchSub32(cond, GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, GPRInfo::argumentGPR2); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto testAllSame = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branchSub32(cond, GPRInfo::argumentGPR0, GPRInfo::argumentGPR0, GPRInfo::argumentGPR0); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto a : masmbrBoundary32()) { + CHECK_EQ(invoke(testAllSame, a), evaluateResult32(cond, 0, false, false) ? 1 : 0); + for (auto b : masmbrBoundary32()) { + int64_t wide = static_cast(a) - static_cast(b); + int32_t diff = static_cast(static_cast(a) - static_cast(b)); + CHECK_EQ(invoke(testThreeArg, a, b), evaluateResult32(cond, diff, false, wide != diff) ? 1 : 0); + } + } + + for (auto imm : masmbrBoundary32()) { + auto testImm = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branchSub32(cond, GPRInfo::argumentGPR0, CCallHelpers::TrustedImm32(imm), GPRInfo::argumentGPR0); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto a : masmbrBoundary32()) { + int64_t wide = static_cast(a) - static_cast(imm); + int32_t diff = static_cast(static_cast(a) - static_cast(imm)); + CHECK_EQ(invoke(testImm, a), evaluateResult32(cond, diff, false, wide != diff) ? 1 : 0); + } + } + } +} + +void testBranchMul32Masmbr() +{ + for (auto cond : { MacroAssembler::Zero, MacroAssembler::NonZero, MacroAssembler::Signed, MacroAssembler::PositiveOrZero, MacroAssembler::Overflow }) { + auto testThreeArg = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branchMul32(cond, GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, GPRInfo::argumentGPR2); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto testDestIsSrc2 = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branchMul32(cond, GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, GPRInfo::argumentGPR1); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto testAllSame = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branchMul32(cond, GPRInfo::argumentGPR0, GPRInfo::argumentGPR0, GPRInfo::argumentGPR0); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto a : masmbrBoundary32()) { + int64_t wideSame = static_cast(a) * static_cast(a); + int32_t prodSame = static_cast(static_cast(a) * static_cast(a)); + CHECK_EQ(invoke(testAllSame, a), evaluateResult32(cond, prodSame, false, wideSame != prodSame) ? 1 : 0); + + for (auto b : masmbrBoundary32()) { + int64_t wide = static_cast(a) * static_cast(b); + int32_t prod = static_cast(static_cast(a) * static_cast(b)); + int expected = evaluateResult32(cond, prod, false, wide != prod) ? 1 : 0; + CHECK_EQ(invoke(testThreeArg, a, b), expected); + CHECK_EQ(invoke(testDestIsSrc2, a, b), expected); + } + } + + for (auto imm : masmbrBoundary32()) { + auto testImm = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branchMul32(cond, GPRInfo::argumentGPR0, CCallHelpers::TrustedImm32(imm), GPRInfo::argumentGPR1); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto a : masmbrBoundary32()) { + int64_t wide = static_cast(a) * static_cast(imm); + int32_t prod = static_cast(static_cast(a) * static_cast(imm)); + CHECK_EQ(invoke(testImm, a), evaluateResult32(cond, prod, false, wide != prod) ? 1 : 0); + } + } + } +} + +void testBranchNegAndOr32Masmbr() +{ + for (auto cond : { MacroAssembler::Zero, MacroAssembler::NonZero, MacroAssembler::Signed, MacroAssembler::PositiveOrZero, MacroAssembler::Overflow }) { + auto testNeg = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branchNeg32(cond, GPRInfo::argumentGPR0); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto testNegValue = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.branchNeg32(cond, GPRInfo::argumentGPR0).link(&jit); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto a : masmbrBoundary32()) { + int64_t wide = -static_cast(a); + int32_t neg = static_cast(0u - static_cast(a)); + CHECK_EQ(invoke(testNeg, a), evaluateResult32(cond, neg, false, wide != neg) ? 1 : 0); + CHECK_EQ(invoke(testNegValue, a), neg); + } + } + +#if CPU(X86) || CPU(X86_64) || CPU(ARM_THUMB2) + for (auto cond : { MacroAssembler::Zero, MacroAssembler::NonZero, MacroAssembler::Signed, MacroAssembler::PositiveOrZero }) { + auto testOr = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branchOr32(cond, GPRInfo::argumentGPR0, GPRInfo::argumentGPR1); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto testOrSame = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branchOr32(cond, GPRInfo::argumentGPR0, GPRInfo::argumentGPR0); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto a : masmbrBoundary32()) { + CHECK_EQ(invoke(testOrSame, a), evaluateResult32(cond, a, false, false) ? 1 : 0); + for (auto b : masmbrBoundary32()) + CHECK_EQ(invoke(testOr, a, b), evaluateResult32(cond, a | b, false, false) ? 1 : 0); + } + } +#endif +} + +void testMoveConditionally32Masmbr() +{ + for (auto cond : masmbrRelationalConditions()) { + auto testNoAlias = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.moveConditionally32(cond, GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, GPRInfo::argumentGPR2, GPRInfo::argumentGPR3, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto testThenIsDest = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.moveConditionally32(cond, GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, GPRInfo::argumentGPR2, GPRInfo::argumentGPR3, GPRInfo::argumentGPR2); + jit.move(GPRInfo::argumentGPR2, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto testElseIsDest = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.moveConditionally32(cond, GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, GPRInfo::argumentGPR2, GPRInfo::argumentGPR3, GPRInfo::argumentGPR3); + jit.move(GPRInfo::argumentGPR3, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto testThenEqualsElse = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.moveConditionally32(cond, GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, GPRInfo::argumentGPR2, GPRInfo::argumentGPR2, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto a : masmbrBoundary32()) { + for (auto b : masmbrBoundary32()) { + int32_t thenCase = 0x1234; + int32_t elseCase = -0x4321; + int32_t expected = evaluateRelational32(cond, a, b) ? thenCase : elseCase; + CHECK_EQ(invoke(testNoAlias, a, b, thenCase, elseCase), expected); + CHECK_EQ(invoke(testThenIsDest, a, b, thenCase, elseCase), expected); + CHECK_EQ(invoke(testElseIsDest, a, b, thenCase, elseCase), expected); + CHECK_EQ(invoke(testThenEqualsElse, a, b, thenCase, elseCase), thenCase); + } + } + } +} + +void testMoveConditionallyTest32Masmbr() +{ + for (auto cond : { MacroAssembler::Zero, MacroAssembler::NonZero, MacroAssembler::Signed, MacroAssembler::PositiveOrZero }) { + auto testNoAlias = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.moveConditionallyTest32(cond, GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, GPRInfo::argumentGPR2, GPRInfo::argumentGPR3, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto testThenIsDest = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.moveConditionallyTest32(cond, GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, GPRInfo::argumentGPR2, GPRInfo::argumentGPR3, GPRInfo::argumentGPR2); + jit.move(GPRInfo::argumentGPR2, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto testElseIsDest = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.moveConditionallyTest32(cond, GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, GPRInfo::argumentGPR2, GPRInfo::argumentGPR3, GPRInfo::argumentGPR3); + jit.move(GPRInfo::argumentGPR3, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto a : masmbrBoundary32()) { + for (auto b : masmbrBoundary32()) { + int32_t thenCase = 0x1234; + int32_t elseCase = -0x4321; + int32_t expected = evaluateResult32(cond, a & b, false, false) ? thenCase : elseCase; + CHECK_EQ(invoke(testNoAlias, a, b, thenCase, elseCase), expected); + CHECK_EQ(invoke(testThenIsDest, a, b, thenCase, elseCase), expected); + CHECK_EQ(invoke(testElseIsDest, a, b, thenCase, elseCase), expected); + } + } + } +} + +void testPatchableBranchMasmbr() +{ + for (auto cond : masmbrRelationalConditions()) { + for (auto imm : { 0, 1, -1, 0xff, 0x100, 0x1000 }) { + auto test = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.patchableBranch32(cond, GPRInfo::argumentGPR0, CCallHelpers::TrustedImm32(imm)); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.m_jump.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto a : masmbrBoundary32()) + CHECK_EQ(invoke(test, a), evaluateRelational32(cond, a, imm) ? 1 : 0); + } + } + + for (auto cond : { MacroAssembler::Zero, MacroAssembler::NonZero, MacroAssembler::Signed, MacroAssembler::PositiveOrZero }) { + for (auto mask : { -1, 0, 1, 0xff, 0x100, static_cast(0x80000000u) }) { + auto test = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.patchableBranchTest32(cond, GPRInfo::argumentGPR0, CCallHelpers::TrustedImm32(mask)); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.m_jump.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto a : masmbrBoundary32()) + CHECK_EQ(invoke(test, a), evaluateResult32(cond, a & mask, false, false) ? 1 : 0); + } + } +} + +#if CPU(ARM_THUMB2) +static bool evaluateRelational64(MacroAssembler::RelationalCondition cond, int64_t a, int64_t b) +{ + switch (cond) { + case MacroAssembler::Equal: return a == b; + case MacroAssembler::NotEqual: return a != b; + case MacroAssembler::Above: return static_cast(a) > static_cast(b); + case MacroAssembler::AboveOrEqual: return static_cast(a) >= static_cast(b); + case MacroAssembler::Below: return static_cast(a) < static_cast(b); + case MacroAssembler::BelowOrEqual: return static_cast(a) <= static_cast(b); + case MacroAssembler::GreaterThan: return a > b; + case MacroAssembler::GreaterThanOrEqual: return a >= b; + case MacroAssembler::LessThan: return a < b; + case MacroAssembler::LessThanOrEqual: return a <= b; + } + RELEASE_ASSERT_NOT_REACHED(); + return false; +} + +static Vector masmbrBoundary64() +{ + return Vector { + 0LL, + 1LL, + -1LL, + 2LL, + -2LL, + static_cast(std::numeric_limits::max()), + static_cast(std::numeric_limits::min()), + 0x00000000ffffffffLL, + 0x0000000100000000LL, + 0x0000000100000001LL, + 0x00000001ffffffffLL, + 0x0000000200000000LL, + 0x7fffffff00000000LL, + 0x7fffffff00000001LL, + 0x7ffffffffffffffeLL, + std::numeric_limits::max(), + std::numeric_limits::min(), + static_cast(0x8000000000000001ULL), + static_cast(0xffffffff00000000ULL), + static_cast(0xfffffffffffffffeULL), + }; +} + +void testBranch64ARMv7() +{ + for (auto cond : masmbrRelationalConditions()) { + auto test = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branch64(cond, GPRInfo::argumentGPR1, GPRInfo::argumentGPR0, GPRInfo::argumentGPR3, GPRInfo::argumentGPR2); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto testSameOperand = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branch64(cond, GPRInfo::argumentGPR1, GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, GPRInfo::argumentGPR0); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto a : masmbrBoundary64()) { + int32_t aLo = static_cast(a); + int32_t aHi = static_cast(a >> 32); + CHECK_EQ(invoke(testSameOperand, aLo, aHi), evaluateRelational64(cond, a, a) ? 1 : 0); + + for (auto b : masmbrBoundary64()) { + int32_t bLo = static_cast(b); + int32_t bHi = static_cast(b >> 32); + CHECK_EQ(invoke(test, aLo, aHi, bLo, bHi), evaluateRelational64(cond, a, b) ? 1 : 0); + } + } + } +} + +void testBranch64ImmARMv7() +{ + for (auto cond : masmbrRelationalConditions()) { + for (auto imm : masmbrBoundary64()) { + auto test = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branch64(cond, GPRInfo::argumentGPR1, GPRInfo::argumentGPR0, CCallHelpers::TrustedImm64(imm)); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto a : masmbrBoundary64()) { + int32_t aLo = static_cast(a); + int32_t aHi = static_cast(a >> 32); + CHECK_EQ(invoke(test, aLo, aHi), evaluateRelational64(cond, a, imm) ? 1 : 0); + } + } + } +} + +void testCompare64ARMv7() +{ + for (auto cond : masmbrRelationalConditions()) { + auto testNoAlias = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.push(GPRInfo::regT4); + jit.move(CCallHelpers::TrustedImm32(0x5a5a5a5a), GPRInfo::regT4); + jit.compare64(cond, GPRInfo::argumentGPR1, GPRInfo::argumentGPR0, GPRInfo::argumentGPR3, GPRInfo::argumentGPR2, GPRInfo::regT4); + jit.move(GPRInfo::regT4, GPRInfo::returnValueGPR); + jit.pop(GPRInfo::regT4); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto testDestIsLhsLo = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.compare64(cond, GPRInfo::argumentGPR1, GPRInfo::argumentGPR0, GPRInfo::argumentGPR3, GPRInfo::argumentGPR2, GPRInfo::argumentGPR0); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto testDestIsLhsHi = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.compare64(cond, GPRInfo::argumentGPR1, GPRInfo::argumentGPR0, GPRInfo::argumentGPR3, GPRInfo::argumentGPR2, GPRInfo::argumentGPR1); + jit.move(GPRInfo::argumentGPR1, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto testDestIsRhsLo = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.compare64(cond, GPRInfo::argumentGPR1, GPRInfo::argumentGPR0, GPRInfo::argumentGPR3, GPRInfo::argumentGPR2, GPRInfo::argumentGPR2); + jit.move(GPRInfo::argumentGPR2, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto testDestIsRhsHi = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.compare64(cond, GPRInfo::argumentGPR1, GPRInfo::argumentGPR0, GPRInfo::argumentGPR3, GPRInfo::argumentGPR2, GPRInfo::argumentGPR3); + jit.move(GPRInfo::argumentGPR3, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto testSameOperand = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.compare64(cond, GPRInfo::argumentGPR1, GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, GPRInfo::argumentGPR0, GPRInfo::argumentGPR0); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto a : masmbrBoundary64()) { + int32_t aLo = static_cast(a); + int32_t aHi = static_cast(a >> 32); + CHECK_EQ(invoke(testSameOperand, aLo, aHi), evaluateRelational64(cond, a, a) ? 1 : 0); + + for (auto b : masmbrBoundary64()) { + int32_t bLo = static_cast(b); + int32_t bHi = static_cast(b >> 32); + int32_t expected = evaluateRelational64(cond, a, b) ? 1 : 0; + CHECK_EQ(invoke(testNoAlias, aLo, aHi, bLo, bHi), expected); + CHECK_EQ(invoke(testDestIsLhsLo, aLo, aHi, bLo, bHi), expected); + CHECK_EQ(invoke(testDestIsLhsHi, aLo, aHi, bLo, bHi), expected); + CHECK_EQ(invoke(testDestIsRhsLo, aLo, aHi, bLo, bHi), expected); + CHECK_EQ(invoke(testDestIsRhsHi, aLo, aHi, bLo, bHi), expected); + } + } + } +} + +void testBranchTest64ARMv7() +{ + for (auto cond : { MacroAssembler::Zero, MacroAssembler::NonZero, MacroAssembler::Signed, MacroAssembler::PositiveOrZero }) { + auto test = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branchTest64(cond, GPRInfo::argumentGPR1, GPRInfo::argumentGPR0); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto testSameReg = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branchTest64(cond, GPRInfo::argumentGPR0, GPRInfo::argumentGPR0); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto a : masmbrBoundary64()) { + int32_t aLo = static_cast(a); + int32_t aHi = static_cast(a >> 32); + bool expected = false; + switch (cond) { + case MacroAssembler::Zero: expected = !a; break; + case MacroAssembler::NonZero: expected = !!a; break; + case MacroAssembler::Signed: expected = a < 0; break; + case MacroAssembler::PositiveOrZero: expected = a >= 0; break; + default: RELEASE_ASSERT_NOT_REACHED(); + } + CHECK_EQ(invoke(test, aLo, aHi), expected ? 1 : 0); + + bool expectedSame = false; + int64_t doubled = (static_cast(aLo) << 32) | static_cast(aLo); + switch (cond) { + case MacroAssembler::Zero: expectedSame = !doubled; break; + case MacroAssembler::NonZero: expectedSame = !!doubled; break; + case MacroAssembler::Signed: expectedSame = doubled < 0; break; + case MacroAssembler::PositiveOrZero: expectedSame = doubled >= 0; break; + default: RELEASE_ASSERT_NOT_REACHED(); + } + CHECK_EQ(invoke(testSameReg, aLo), expectedSame ? 1 : 0); + } + } +} +#endif + +#if CPU(ARM_THUMB2) + +static Vector armv7DoubleBitPatterns() +{ + return Vector { + 0x0000000000000000ull, + 0x8000000000000000ull, + 0x3ff0000000000000ull, + 0xbff0000000000000ull, + 0x0000000000000001ull, + 0x8000000000000001ull, + 0x000fffffffffffffull, + 0x0010000000000000ull, + 0x7fefffffffffffffull, + 0xffefffffffffffffull, + 0x7ff0000000000000ull, + 0xfff0000000000000ull, + 0x7ff8000000000000ull, + 0xfff8000000000000ull, + 0x7ff800000abcdef1ull, + 0xfff123456789abcdull, + 0x0123456789abcdefull, + 0xfedcba9876543210ull, + 0xffffffffffffffffull, + }; +} + +static Vector armv7FloatBitPatterns() +{ + return Vector { + 0x00000000u, + 0x80000000u, + 0x3f800000u, + 0xbf800000u, + 0x00000001u, + 0x80000001u, + 0x007fffffu, + 0x00800000u, + 0x7f7fffffu, + 0xff7fffffu, + 0x7f800000u, + 0xff800000u, + 0x7fc00000u, + 0xffc00000u, + 0x7fc0abcdu, + 0x4b000001u, + 0x4f000000u, + 0xcf000000u, + }; +} + +static Vector armv7EdgeDoubles() +{ + return Vector { + 0.0, + -0.0, + 1.0, + -1.0, + 0.5, + -0.5, + 1.5, + -1.5, + 2.5, + -2.5, + 0.3, + -0.3, + std::numeric_limits::denorm_min(), + -std::numeric_limits::denorm_min(), + std::numeric_limits::min(), + -std::numeric_limits::min(), + std::numeric_limits::max(), + std::numeric_limits::lowest(), + std::numeric_limits::infinity(), + -std::numeric_limits::infinity(), + std::numeric_limits::quiet_NaN(), + -std::numeric_limits::quiet_NaN(), + std::bit_cast(0x7ff800000abcdef1ull), + 2147483646.0, + 2147483647.0, + 2147483648.0, + -2147483648.0, + -2147483649.0, + 4294967295.0, + 4294967296.0, + 2147483647.5, + -2147483648.5, + }; +} + +static bool armv7IsNaN(double value) +{ + return value != value; +} + +static bool armv7IsNegativeZero(double value) +{ + return std::bit_cast(value) == 0x8000000000000000ull; +} + +static int32_t armv7SaturatingToInt32(double value) +{ + if (armv7IsNaN(value)) + return 0; + if (value >= 2147483647.0) + return std::numeric_limits::max(); + if (value <= -2147483648.0) + return std::numeric_limits::min(); + return static_cast(value); +} + +static uint32_t armv7SaturatingToUint32(double value) +{ + if (armv7IsNaN(value)) + return 0; + if (value >= 4294967295.0) + return std::numeric_limits::max(); + if (value <= 0.0) + return 0; + return static_cast(value); +} + +static int32_t armv7NearestEvenToInt32(double value) +{ + if (armv7IsNaN(value)) + return 0; + if (value >= 2147483647.0) + return std::numeric_limits::max(); + if (value <= -2147483648.0) + return std::numeric_limits::min(); + + int32_t towardZero = static_cast(value); + double truncated = static_cast(towardZero); + int32_t low = towardZero; + if (value < truncated) + low = towardZero - 1; + double fraction = value - static_cast(low); + if (fraction > 0.5) + return low + 1; + if (fraction < 0.5) + return low; + return (low & 1) ? low + 1 : low; +} + +static void armv7CheckDouble(uint64_t actualBits, double expected) +{ + double actual = std::bit_cast(actualBits); + if (armv7IsNaN(expected)) { + CHECK_EQ(armv7IsNaN(actual), true); + return; + } + CHECK_EQ(actualBits, std::bit_cast(expected)); +} + +static void armv7CheckFloat(uint32_t actualBits, float expected) +{ + float actual = std::bit_cast(actualBits); + if (expected != expected) { + CHECK_EQ(actual != actual, true); + return; + } + CHECK_EQ(actualBits, std::bit_cast(expected)); +} + +static void armv7StoreFloatToPointer(CCallHelpers& jit, FPRReg src, void* address) +{ + jit.move(CCallHelpers::TrustedImmPtr(address), GPRInfo::regT3); + jit.storeFloat(src, CCallHelpers::Address(GPRInfo::regT3)); +} + +static int armv7ExpectedDoubleCondition(MacroAssembler::DoubleCondition condition, double a, double b) +{ + bool unordered = armv7IsNaN(a) || armv7IsNaN(b); + switch (condition) { + case MacroAssembler::DoubleEqualAndOrdered: + return !unordered && (a == b); + case MacroAssembler::DoubleNotEqualAndOrdered: + return !unordered && (a != b); + case MacroAssembler::DoubleGreaterThanAndOrdered: + return !unordered && (a > b); + case MacroAssembler::DoubleGreaterThanOrEqualAndOrdered: + return !unordered && (a >= b); + case MacroAssembler::DoubleLessThanAndOrdered: + return !unordered && (a < b); + case MacroAssembler::DoubleLessThanOrEqualAndOrdered: + return !unordered && (a <= b); + case MacroAssembler::DoubleEqualOrUnordered: + return unordered || (a == b); + case MacroAssembler::DoubleNotEqualOrUnordered: + return unordered || (a != b); + case MacroAssembler::DoubleGreaterThanOrUnordered: + return unordered || (a > b); + case MacroAssembler::DoubleGreaterThanOrEqualOrUnordered: + return unordered || (a >= b); + case MacroAssembler::DoubleLessThanOrUnordered: + return unordered || (a < b); + case MacroAssembler::DoubleLessThanOrEqualOrUnordered: + return unordered || (a <= b); + } + RELEASE_ASSERT_NOT_REACHED(); +} + +void testARMv7FPMoveDoubleInts() +{ + uint64_t input = 0; + uint64_t viaInts = 0; + uint64_t via64 = 0; + uint64_t hiInserted = 0; + uint32_t lo = 0; + uint32_t hi = 0; + uint32_t hiOnly = 0; + + auto test = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + + jit.loadDouble(CCallHelpers::TrustedImmPtr(&input), FPRInfo::fpRegT0); + + jit.moveDoubleToInts(FPRInfo::fpRegT0, GPRInfo::regT0, GPRInfo::regT1); + jit.store32(GPRInfo::regT0, &lo); + jit.store32(GPRInfo::regT1, &hi); + jit.moveIntsToDouble(GPRInfo::regT0, GPRInfo::regT1, FPRInfo::fpRegT1); + jit.storeDouble(FPRInfo::fpRegT1, CCallHelpers::TrustedImmPtr(&viaInts)); + + jit.moveDoubleTo64(FPRInfo::fpRegT0, GPRInfo::regT2, GPRInfo::regT3); + jit.move64ToDouble(GPRInfo::regT2, GPRInfo::regT3, FPRInfo::fpRegT2); + jit.storeDouble(FPRInfo::fpRegT2, CCallHelpers::TrustedImmPtr(&via64)); + + jit.moveDoubleHiTo32(FPRInfo::fpRegT0, GPRInfo::regT0); + jit.store32(GPRInfo::regT0, &hiOnly); + jit.moveZeroToDouble(FPRInfo::fpRegT3); + jit.move32ToDoubleHi(GPRInfo::regT0, FPRInfo::fpRegT3); + jit.storeDouble(FPRInfo::fpRegT3, CCallHelpers::TrustedImmPtr(&hiInserted)); + + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto bits : armv7DoubleBitPatterns()) { + input = bits; + invoke(test); + CHECK_EQ(lo, static_cast(bits)); + CHECK_EQ(hi, static_cast(bits >> 32)); + CHECK_EQ(viaInts, bits); + CHECK_EQ(via64, bits); + CHECK_EQ(hiOnly, static_cast(bits >> 32)); + CHECK_EQ(hiInserted, (bits >> 32) << 32); + } +} + +void testARMv7FPMoveDoubleAliasing() +{ + uint64_t input = 0; + uint64_t moved = 0; + uint64_t movedOrNop = 0; + uint64_t selfMoved = 0; + + auto test = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadDouble(CCallHelpers::TrustedImmPtr(&input), FPRInfo::fpRegT0); + jit.moveDouble(FPRInfo::fpRegT0, FPRInfo::fpRegT1); + jit.storeDouble(FPRInfo::fpRegT1, CCallHelpers::TrustedImmPtr(&moved)); + jit.moveDoubleOrNop(FPRInfo::fpRegT0, FPRInfo::fpRegT2); + jit.storeDouble(FPRInfo::fpRegT2, CCallHelpers::TrustedImmPtr(&movedOrNop)); + jit.moveDouble(FPRInfo::fpRegT0, FPRInfo::fpRegT0); + jit.moveDoubleOrNop(FPRInfo::fpRegT0, FPRInfo::fpRegT0); + jit.storeDouble(FPRInfo::fpRegT0, CCallHelpers::TrustedImmPtr(&selfMoved)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto bits : armv7DoubleBitPatterns()) { + input = bits; + invoke(test); + CHECK_EQ(moved, bits); + CHECK_EQ(movedOrNop, bits); + CHECK_EQ(selfMoved, bits); + } +} + +void testARMv7FPMoveFloatAndImmediates() +{ + for (auto bits : armv7FloatBitPatterns()) { + uint32_t input = bits; + uint32_t roundTrip = ~bits; + uint32_t fromImm = ~bits; + uint32_t zeroFloat = 0xdeadbeefu; + + auto test = compile([&, bits] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + + jit.loadFloat(CCallHelpers::TrustedImmPtr(&input), FPRInfo::fpRegT0); + jit.moveFloatTo32(FPRInfo::fpRegT0, GPRInfo::regT0); + jit.move32ToFloat(GPRInfo::regT0, FPRInfo::fpRegT1); + armv7StoreFloatToPointer(jit, FPRInfo::fpRegT1, &roundTrip); + + jit.move32ToFloat(CCallHelpers::TrustedImm32(static_cast(bits)), FPRInfo::fpRegT2); + armv7StoreFloatToPointer(jit, FPRInfo::fpRegT2, &fromImm); + + jit.moveZeroToFloat(FPRInfo::fpRegT3); + armv7StoreFloatToPointer(jit, FPRInfo::fpRegT3, &zeroFloat); + + emitFunctionEpilogue(jit); + jit.ret(); + }); + + invoke(test); + CHECK_EQ(roundTrip, bits); + CHECK_EQ(fromImm, bits); + CHECK_EQ(zeroFloat, 0u); + } +} + +void testARMv7FPMove64ToDoubleImmediate() +{ + for (auto bits : armv7DoubleBitPatterns()) { + uint64_t result = ~bits; + uint64_t zeroDouble = 0xffffffffffffffffull; + + auto test = compile([&, bits] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move64ToDouble(CCallHelpers::TrustedImm64(static_cast(bits)), FPRInfo::fpRegT0); + jit.storeDouble(FPRInfo::fpRegT0, CCallHelpers::TrustedImmPtr(&result)); + jit.moveZeroToDouble(FPRInfo::fpRegT1); + jit.storeDouble(FPRInfo::fpRegT1, CCallHelpers::TrustedImmPtr(&zeroDouble)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + invoke(test); + CHECK_EQ(result, bits); + CHECK_EQ(zeroDouble, 0ull); + } +} + +void testARMv7FPLoadStoreOffsets() +{ + Vector storage(0x8000); + uint8_t* buffer = std::bit_cast(storage.data()); + uint8_t* base = buffer + 0x20000; + + for (int32_t offset : { 0, 4, 8, 1016, 1020, 1024, 2048, -4, -1020, -1024, -2048, 0x12348, -0x12348 }) { + for (int32_t skew : { 0, 2 }) { + uint8_t* skewedBase = base - skew; + int32_t skewedOffset = offset + skew; + double source = 1.0 + offset; + uint32_t floatSource = 0x40490fdbu; + uint64_t loaded = 0; + uint32_t loadedFloat = 0; + + auto test = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(CCallHelpers::TrustedImmPtr(skewedBase), GPRInfo::regT0); + + jit.loadDouble(CCallHelpers::TrustedImmPtr(&source), FPRInfo::fpRegT0); + jit.storeDouble(FPRInfo::fpRegT0, CCallHelpers::Address(GPRInfo::regT0, skewedOffset)); + jit.loadDouble(CCallHelpers::Address(GPRInfo::regT0, skewedOffset), FPRInfo::fpRegT1); + jit.storeDouble(FPRInfo::fpRegT1, CCallHelpers::TrustedImmPtr(&loaded)); + + jit.move(CCallHelpers::TrustedImmPtr(skewedBase), GPRInfo::regT0); + jit.loadFloat(CCallHelpers::TrustedImmPtr(&floatSource), FPRInfo::fpRegT2); + jit.storeFloat(FPRInfo::fpRegT2, CCallHelpers::Address(GPRInfo::regT0, skewedOffset)); + jit.loadFloat(CCallHelpers::Address(GPRInfo::regT0, skewedOffset), FPRInfo::fpRegT3); + armv7StoreFloatToPointer(jit, FPRInfo::fpRegT3, &loadedFloat); + + emitFunctionEpilogue(jit); + jit.ret(); + }); + + invoke(test); + CHECK_EQ(loaded, std::bit_cast(source)); + CHECK_EQ(loadedFloat, floatSource); + } + } +} + +void testARMv7FPLoadStoreBaseIndex() +{ + Vector storage(0x400); + uint8_t* base = std::bit_cast(storage.data()); + + for (auto scale : { CCallHelpers::TimesOne, CCallHelpers::TimesTwo, CCallHelpers::TimesFour, CCallHelpers::TimesEight }) { + for (int32_t offset : { 0, 8, 1024 }) { + int32_t index = 8; + double source = 12345.678; + uint32_t floatSource = 0x40490fdbu; + uint64_t loaded = 0; + uint32_t loadedFloat = 0; + + auto test = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(CCallHelpers::TrustedImmPtr(base), GPRInfo::regT0); + jit.move(CCallHelpers::TrustedImm32(index), GPRInfo::regT1); + + jit.loadDouble(CCallHelpers::TrustedImmPtr(&source), FPRInfo::fpRegT0); + jit.storeDouble(FPRInfo::fpRegT0, CCallHelpers::BaseIndex(GPRInfo::regT0, GPRInfo::regT1, scale, offset)); + jit.loadDouble(CCallHelpers::BaseIndex(GPRInfo::regT0, GPRInfo::regT1, scale, offset), FPRInfo::fpRegT1); + jit.storeDouble(FPRInfo::fpRegT1, CCallHelpers::TrustedImmPtr(&loaded)); + + jit.move(CCallHelpers::TrustedImmPtr(base), GPRInfo::regT0); + jit.move(CCallHelpers::TrustedImm32(index), GPRInfo::regT1); + jit.loadFloat(CCallHelpers::TrustedImmPtr(&floatSource), FPRInfo::fpRegT2); + jit.storeFloat(FPRInfo::fpRegT2, CCallHelpers::BaseIndex(GPRInfo::regT0, GPRInfo::regT1, scale, offset + 8)); + jit.loadFloat(CCallHelpers::BaseIndex(GPRInfo::regT0, GPRInfo::regT1, scale, offset + 8), FPRInfo::fpRegT3); + armv7StoreFloatToPointer(jit, FPRInfo::fpRegT3, &loadedFloat); + + emitFunctionEpilogue(jit); + jit.ret(); + }); + + invoke(test); + CHECK_EQ(loaded, std::bit_cast(source)); + CHECK_EQ(loadedFloat, floatSource); + + uint8_t* slot = base + (index << static_cast(scale)) + offset; + CHECK_EQ(*std::bit_cast(slot), std::bit_cast(source)); + CHECK_EQ(*std::bit_cast(slot + 8), floatSource); + } + } +} + +void testARMv7BranchDoubleEdges(MacroAssembler::DoubleCondition condition) +{ + double arg1 = 0; + double arg2 = 0; + + auto branchTest = compile([&, condition] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadDouble(CCallHelpers::TrustedImmPtr(&arg1), FPRInfo::fpRegT0); + jit.loadDouble(CCallHelpers::TrustedImmPtr(&arg2), FPRInfo::fpRegT1); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + auto jump = jit.branchDouble(condition, FPRInfo::fpRegT0, FPRInfo::fpRegT1); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + jump.link(&jit); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto withZeroTest = compile([&, condition] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadDouble(CCallHelpers::TrustedImmPtr(&arg1), FPRInfo::fpRegT0); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + auto jump = jit.branchDoubleWithZero(condition, FPRInfo::fpRegT0); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + jump.link(&jit); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto sameRegisterTest = compile([&, condition] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadDouble(CCallHelpers::TrustedImmPtr(&arg1), FPRInfo::fpRegT0); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + auto jump = jit.branchDouble(condition, FPRInfo::fpRegT0, FPRInfo::fpRegT0); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + jump.link(&jit); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto operands = armv7EdgeDoubles(); + for (auto a : operands) { + arg1 = a; + arg2 = 0; + CHECK_EQ(invoke(withZeroTest), armv7ExpectedDoubleCondition(condition, a, 0.0)); + CHECK_EQ(invoke(sameRegisterTest), armv7ExpectedDoubleCondition(condition, a, a)); + for (auto b : operands) { + arg1 = a; + arg2 = b; + CHECK_EQ(invoke(branchTest), armv7ExpectedDoubleCondition(condition, a, b)); + } + } +} + +void testARMv7BranchFloatEdges(MacroAssembler::DoubleCondition condition) +{ + uint32_t arg1 = 0; + uint32_t arg2 = 0; + + auto branchTest = compile([&, condition] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadFloat(CCallHelpers::TrustedImmPtr(&arg1), FPRInfo::fpRegT0); + jit.loadFloat(CCallHelpers::TrustedImmPtr(&arg2), FPRInfo::fpRegT1); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + auto jump = jit.branchFloat(condition, FPRInfo::fpRegT0, FPRInfo::fpRegT1); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + jump.link(&jit); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto withZeroTest = compile([&, condition] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadFloat(CCallHelpers::TrustedImmPtr(&arg1), FPRInfo::fpRegT0); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + auto jump = jit.branchFloatWithZero(condition, FPRInfo::fpRegT0); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + jump.link(&jit); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto operands = armv7FloatBitPatterns(); + for (auto a : operands) { + arg1 = a; + arg2 = 0; + double aValue = static_cast(std::bit_cast(a)); + CHECK_EQ(invoke(withZeroTest), armv7ExpectedDoubleCondition(condition, aValue, 0.0)); + for (auto b : operands) { + arg1 = a; + arg2 = b; + double bValue = static_cast(std::bit_cast(b)); + CHECK_EQ(invoke(branchTest), armv7ExpectedDoubleCondition(condition, aValue, bValue)); + } + } +} + +void testARMv7BranchDoubleNonZeroAndZeroOrNaN() +{ + double input = 0; + + auto nonZeroTest = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadDouble(CCallHelpers::TrustedImmPtr(&input), FPRInfo::fpRegT0); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + auto jump = jit.branchDoubleNonZero(FPRInfo::fpRegT0, FPRInfo::fpRegT1); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + jump.link(&jit); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto zeroOrNaNTest = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadDouble(CCallHelpers::TrustedImmPtr(&input), FPRInfo::fpRegT0); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + auto jump = jit.branchDoubleZeroOrNaN(FPRInfo::fpRegT0, FPRInfo::fpRegT1); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + jump.link(&jit); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : armv7EdgeDoubles()) { + input = value; + CHECK_EQ(invoke(nonZeroTest), !armv7IsNaN(value) && value != 0.0); + CHECK_EQ(invoke(zeroOrNaNTest), armv7IsNaN(value) || value == 0.0); + } +} + +void testARMv7TruncateDoubleAndFloatToInt32() +{ + double input = 0; + int32_t signedResult = 0; + uint32_t unsignedResult = 0; + int32_t floatSigned = 0; + uint32_t floatUnsigned = 0; + + auto test = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadDouble(CCallHelpers::TrustedImmPtr(&input), FPRInfo::fpRegT0); + + jit.truncateDoubleToInt32(FPRInfo::fpRegT0, GPRInfo::regT0); + jit.store32(GPRInfo::regT0, &signedResult); + jit.truncateDoubleToUint32(FPRInfo::fpRegT0, GPRInfo::regT0); + jit.store32(GPRInfo::regT0, &unsignedResult); + + jit.convertDoubleToFloat(FPRInfo::fpRegT0, FPRInfo::fpRegT1); + jit.truncateFloatToInt32(FPRInfo::fpRegT1, GPRInfo::regT0); + jit.store32(GPRInfo::regT0, &floatSigned); + jit.truncateFloatToUint32(FPRInfo::fpRegT1, GPRInfo::regT0); + jit.store32(GPRInfo::regT0, &floatUnsigned); + + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : armv7EdgeDoubles()) { + input = value; + invoke(test); + CHECK_EQ(signedResult, armv7SaturatingToInt32(value)); + CHECK_EQ(unsignedResult, armv7SaturatingToUint32(value)); + double asFloat = static_cast(static_cast(value)); + CHECK_EQ(floatSigned, armv7SaturatingToInt32(asFloat)); + CHECK_EQ(floatUnsigned, armv7SaturatingToUint32(asFloat)); + } +} + +void testARMv7BranchTruncateDoubleToInt32() +{ + double input = 0; + int32_t dest = 0; + + auto failedTest = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadDouble(CCallHelpers::TrustedImmPtr(&input), FPRInfo::fpRegT0); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::regT2); + auto failure = jit.branchTruncateDoubleToInt32(FPRInfo::fpRegT0, GPRInfo::regT1, CCallHelpers::BranchIfTruncateFailed); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::regT2); + failure.link(&jit); + jit.store32(GPRInfo::regT1, &dest); + jit.move(GPRInfo::regT2, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto successfulTest = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadDouble(CCallHelpers::TrustedImmPtr(&input), FPRInfo::fpRegT0); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::regT2); + auto success = jit.branchTruncateDoubleToInt32(FPRInfo::fpRegT0, GPRInfo::regT1, CCallHelpers::BranchIfTruncateSuccessful); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::regT2); + success.link(&jit); + jit.store32(GPRInfo::regT1, &dest); + jit.move(GPRInfo::regT2, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : armv7EdgeDoubles()) { + input = value; + int32_t truncated = armv7SaturatingToInt32(value); + int reportedFailure = truncated == 0 + || truncated == -1 + || truncated == std::numeric_limits::max() + || truncated == std::numeric_limits::min(); + + dest = 0x5a5a5a5a; + CHECK_EQ(invoke(failedTest), reportedFailure); + CHECK_EQ(dest, truncated); + + dest = 0x5a5a5a5a; + CHECK_EQ(invoke(successfulTest), reportedFailure); + CHECK_EQ(dest, truncated); + } +} + +void testARMv7BranchConvertDoubleToInt32(bool negZeroCheck) +{ + double input = 0; + int32_t dest = 0; + + auto test = compile([&, negZeroCheck] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadDouble(CCallHelpers::TrustedImmPtr(&input), FPRInfo::fpRegT0); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::regT2); + CCallHelpers::JumpList failureCases; + jit.branchConvertDoubleToInt32(FPRInfo::fpRegT0, GPRInfo::regT1, failureCases, FPRInfo::fpRegT3, negZeroCheck); + jit.store32(GPRInfo::regT1, &dest); + auto done = jit.jump(); + failureCases.link(&jit); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::regT2); + done.link(&jit); + jit.move(GPRInfo::regT2, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : armv7EdgeDoubles()) { + input = value; + dest = 0x5a5a5a5a; + + bool exact = !armv7IsNaN(value) + && value >= -2147483648.0 + && value <= 2147483647.0 + && static_cast(static_cast(value)) == value; + int expected = exact && !(negZeroCheck && armv7IsNegativeZero(value)); + + CHECK_EQ(invoke(test), expected); + if (expected) + CHECK_EQ(dest, static_cast(value)); + } +} + +void testARMv7ConvertInt32AndUint32ToFloatingPoint() +{ + int32_t input = 0; + uint64_t signedDouble = 0; + uint64_t unsignedDouble = 0; + uint32_t signedFloat = 0; + uint32_t unsignedFloat = 0; + uint64_t fromAddress = 0; + uint64_t fromAbsolute = 0; + + auto test = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(CCallHelpers::TrustedImmPtr(&input), GPRInfo::regT0); + jit.load32(CCallHelpers::Address(GPRInfo::regT0), GPRInfo::regT1); + + jit.convertInt32ToDouble(GPRInfo::regT1, FPRInfo::fpRegT0); + jit.storeDouble(FPRInfo::fpRegT0, CCallHelpers::TrustedImmPtr(&signedDouble)); + jit.convertUInt32ToDouble(GPRInfo::regT1, FPRInfo::fpRegT0); + jit.storeDouble(FPRInfo::fpRegT0, CCallHelpers::TrustedImmPtr(&unsignedDouble)); + + jit.convertInt32ToFloat(GPRInfo::regT1, FPRInfo::fpRegT0); + armv7StoreFloatToPointer(jit, FPRInfo::fpRegT0, &signedFloat); + jit.convertUInt32ToFloat(GPRInfo::regT1, FPRInfo::fpRegT0); + armv7StoreFloatToPointer(jit, FPRInfo::fpRegT0, &unsignedFloat); + + jit.move(CCallHelpers::TrustedImmPtr(&input), GPRInfo::regT0); + jit.convertInt32ToDouble(CCallHelpers::Address(GPRInfo::regT0), FPRInfo::fpRegT1); + jit.storeDouble(FPRInfo::fpRegT1, CCallHelpers::TrustedImmPtr(&fromAddress)); + jit.convertInt32ToDouble(CCallHelpers::AbsoluteAddress(&input), FPRInfo::fpRegT1); + jit.storeDouble(FPRInfo::fpRegT1, CCallHelpers::TrustedImmPtr(&fromAbsolute)); + + emitFunctionEpilogue(jit); + jit.ret(); + }); + + Vector values = int32Operands(); + values.append(16777217); + values.append(-16777217); + values.append(0x40000001); + values.append(static_cast(0x80000001u)); + values.append(static_cast(0xffffff81u)); + values.append(static_cast(0x7fffff81u)); + + for (auto value : values) { + input = value; + invoke(test); + armv7CheckDouble(signedDouble, static_cast(value)); + armv7CheckDouble(unsignedDouble, static_cast(static_cast(value))); + armv7CheckFloat(signedFloat, static_cast(value)); + armv7CheckFloat(unsignedFloat, static_cast(static_cast(value))); + armv7CheckDouble(fromAddress, static_cast(value)); + armv7CheckDouble(fromAbsolute, static_cast(value)); + } +} + +void testARMv7ConvertInt32ImmediateToDouble() +{ + for (auto value : int32Operands()) { + uint64_t result = 0xffffffffffffffffull; + + auto test = compile([&, value] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.convertInt32ToDouble(CCallHelpers::TrustedImm32(value), FPRInfo::fpRegT0); + jit.storeDouble(FPRInfo::fpRegT0, CCallHelpers::TrustedImmPtr(&result)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + invoke(test); + armv7CheckDouble(result, static_cast(value)); + } +} + +void testARMv7ConvertFloatDouble() +{ + uint32_t floatInput = 0; + double doubleInput = 0; + uint64_t toDouble = 0; + uint64_t toDoubleAliased = 0; + uint32_t toFloat = 0; + uint32_t toFloatAliased = 0; + + auto test = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + + jit.loadFloat(CCallHelpers::TrustedImmPtr(&floatInput), FPRInfo::fpRegT0); + jit.convertFloatToDouble(FPRInfo::fpRegT0, FPRInfo::fpRegT1); + jit.storeDouble(FPRInfo::fpRegT1, CCallHelpers::TrustedImmPtr(&toDouble)); + jit.convertFloatToDouble(FPRInfo::fpRegT0, FPRInfo::fpRegT0); + jit.storeDouble(FPRInfo::fpRegT0, CCallHelpers::TrustedImmPtr(&toDoubleAliased)); + + jit.loadDouble(CCallHelpers::TrustedImmPtr(&doubleInput), FPRInfo::fpRegT2); + jit.convertDoubleToFloat(FPRInfo::fpRegT2, FPRInfo::fpRegT1); + armv7StoreFloatToPointer(jit, FPRInfo::fpRegT1, &toFloat); + jit.convertDoubleToFloat(FPRInfo::fpRegT2, FPRInfo::fpRegT2); + armv7StoreFloatToPointer(jit, FPRInfo::fpRegT2, &toFloatAliased); + + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto floats = armv7FloatBitPatterns(); + auto doubles = armv7EdgeDoubles(); + for (auto bits : floats) { + for (auto value : doubles) { + floatInput = bits; + doubleInput = value; + invoke(test); + + float asFloat = std::bit_cast(bits); + armv7CheckDouble(toDouble, static_cast(asFloat)); + CHECK_EQ(toDoubleAliased, toDouble); + armv7CheckFloat(toFloat, static_cast(value)); + CHECK_EQ(toFloatAliased, toFloat); + } + } +} + +static double armv7ApplyBinary(int operation, double left, double right) +{ + switch (operation) { + case 0: + return left + right; + case 1: + return left - right; + case 2: + return left * right; + default: + return left / right; + } +} + +void testARMv7DoubleBinaryAliasing() +{ + double lhs = 0; + double rhs = 0; + uint64_t result = 0; + + for (int operation = 0; operation < 4; ++operation) { + for (int aliasing = 0; aliasing < 4; ++aliasing) { + auto test = compile([&, operation, aliasing] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + + FPRReg a = FPRInfo::fpRegT0; + FPRReg b = FPRInfo::fpRegT1; + FPRReg dest = FPRInfo::fpRegT2; + if (aliasing == 1) + dest = a; + else if (aliasing == 2) + dest = b; + else if (aliasing == 3) { + b = a; + dest = a; + } + + jit.loadDouble(CCallHelpers::TrustedImmPtr(&lhs), a); + jit.loadDouble(CCallHelpers::TrustedImmPtr(&rhs), b); + + switch (operation) { + case 0: + jit.addDouble(a, b, dest); + break; + case 1: + jit.subDouble(a, b, dest); + break; + case 2: + jit.mulDouble(a, b, dest); + break; + default: + jit.divDouble(a, b, dest); + break; + } + + jit.storeDouble(dest, CCallHelpers::TrustedImmPtr(&result)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto x : armv7EdgeDoubles()) { + for (auto y : armv7EdgeDoubles()) { + lhs = x; + rhs = y; + invoke(test); + armv7CheckDouble(result, armv7ApplyBinary(operation, (aliasing == 3) ? y : x, y)); + } + } + } + } +} + +void testARMv7DoubleAccumulateForms() +{ + double lhs = 0; + double rhs = 0; + uint64_t registerForm = 0; + uint64_t selfForm = 0; + uint64_t addressForm = 0; + uint64_t absoluteForm = 0; + + for (int operation = 0; operation < 4; ++operation) { + auto test = compile([&, operation] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + + jit.loadDouble(CCallHelpers::TrustedImmPtr(&lhs), FPRInfo::fpRegT0); + jit.loadDouble(CCallHelpers::TrustedImmPtr(&rhs), FPRInfo::fpRegT1); + switch (operation) { + case 0: + jit.addDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0); + break; + case 1: + jit.subDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0); + break; + case 2: + jit.mulDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0); + break; + default: + jit.divDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0); + break; + } + jit.storeDouble(FPRInfo::fpRegT0, CCallHelpers::TrustedImmPtr(®isterForm)); + + jit.loadDouble(CCallHelpers::TrustedImmPtr(&lhs), FPRInfo::fpRegT2); + switch (operation) { + case 0: + jit.addDouble(FPRInfo::fpRegT2, FPRInfo::fpRegT2); + break; + case 1: + jit.subDouble(FPRInfo::fpRegT2, FPRInfo::fpRegT2); + break; + case 2: + jit.mulDouble(FPRInfo::fpRegT2, FPRInfo::fpRegT2); + break; + default: + jit.divDouble(FPRInfo::fpRegT2, FPRInfo::fpRegT2); + break; + } + jit.storeDouble(FPRInfo::fpRegT2, CCallHelpers::TrustedImmPtr(&selfForm)); + + jit.move(CCallHelpers::TrustedImmPtr(&rhs), GPRInfo::regT0); + jit.loadDouble(CCallHelpers::TrustedImmPtr(&lhs), FPRInfo::fpRegT3); + switch (operation) { + case 0: + jit.addDouble(CCallHelpers::Address(GPRInfo::regT0), FPRInfo::fpRegT3); + break; + case 1: + jit.subDouble(CCallHelpers::Address(GPRInfo::regT0), FPRInfo::fpRegT3); + break; + default: + jit.mulDouble(CCallHelpers::Address(GPRInfo::regT0), FPRInfo::fpRegT3); + break; + } + jit.storeDouble(FPRInfo::fpRegT3, CCallHelpers::TrustedImmPtr(&addressForm)); + + jit.loadDouble(CCallHelpers::TrustedImmPtr(&lhs), FPRInfo::fpRegT3); + jit.addDouble(CCallHelpers::AbsoluteAddress(&rhs), FPRInfo::fpRegT3); + jit.storeDouble(FPRInfo::fpRegT3, CCallHelpers::TrustedImmPtr(&absoluteForm)); + + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto x : armv7EdgeDoubles()) { + for (auto y : armv7EdgeDoubles()) { + lhs = x; + rhs = y; + invoke(test); + armv7CheckDouble(registerForm, armv7ApplyBinary(operation, x, y)); + armv7CheckDouble(selfForm, armv7ApplyBinary(operation, x, x)); + armv7CheckDouble(addressForm, armv7ApplyBinary(operation > 2 ? 2 : operation, x, y)); + armv7CheckDouble(absoluteForm, x + y); + } + } + } +} + +void testARMv7DoubleUnaryAliasing() +{ + double input = 0; + uint64_t sqrtSeparate = 0; + uint64_t sqrtAliased = 0; + uint64_t absSeparate = 0; + uint64_t absAliased = 0; + uint64_t negSeparate = 0; + uint64_t negAliased = 0; + + auto test = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + + jit.loadDouble(CCallHelpers::TrustedImmPtr(&input), FPRInfo::fpRegT0); + jit.sqrtDouble(FPRInfo::fpRegT0, FPRInfo::fpRegT1); + jit.storeDouble(FPRInfo::fpRegT1, CCallHelpers::TrustedImmPtr(&sqrtSeparate)); + jit.absDouble(FPRInfo::fpRegT0, FPRInfo::fpRegT1); + jit.storeDouble(FPRInfo::fpRegT1, CCallHelpers::TrustedImmPtr(&absSeparate)); + jit.negateDouble(FPRInfo::fpRegT0, FPRInfo::fpRegT1); + jit.storeDouble(FPRInfo::fpRegT1, CCallHelpers::TrustedImmPtr(&negSeparate)); + + jit.loadDouble(CCallHelpers::TrustedImmPtr(&input), FPRInfo::fpRegT2); + jit.sqrtDouble(FPRInfo::fpRegT2, FPRInfo::fpRegT2); + jit.storeDouble(FPRInfo::fpRegT2, CCallHelpers::TrustedImmPtr(&sqrtAliased)); + jit.loadDouble(CCallHelpers::TrustedImmPtr(&input), FPRInfo::fpRegT2); + jit.absDouble(FPRInfo::fpRegT2, FPRInfo::fpRegT2); + jit.storeDouble(FPRInfo::fpRegT2, CCallHelpers::TrustedImmPtr(&absAliased)); + jit.loadDouble(CCallHelpers::TrustedImmPtr(&input), FPRInfo::fpRegT2); + jit.negateDouble(FPRInfo::fpRegT2, FPRInfo::fpRegT2); + jit.storeDouble(FPRInfo::fpRegT2, CCallHelpers::TrustedImmPtr(&negAliased)); + + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : armv7EdgeDoubles()) { + input = value; + invoke(test); + + uint64_t bits = std::bit_cast(value); + CHECK_EQ(absSeparate, bits & 0x7fffffffffffffffull); + CHECK_EQ(absAliased, bits & 0x7fffffffffffffffull); + CHECK_EQ(negSeparate, bits ^ 0x8000000000000000ull); + CHECK_EQ(negAliased, bits ^ 0x8000000000000000ull); + CHECK_EQ(sqrtAliased, sqrtSeparate); + + if (armv7IsNaN(value) || value < 0.0) + CHECK_EQ(armv7IsNaN(std::bit_cast(sqrtSeparate)), true); + else if (value == 0.0) + CHECK_EQ(sqrtSeparate, bits); + } +} + +void testARMv7FloatArithmeticIsSinglePrecision() +{ + uint32_t lhsBits = 0; + uint32_t rhsBits = 0; + uint32_t addResult = 0; + uint32_t subResult = 0; + uint32_t mulResult = 0; + uint32_t divResult = 0; + uint32_t sqrtResult = 0; + uint32_t absResult = 0; + uint32_t negResult = 0; + uint32_t addAliased = 0; + + auto test = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + + jit.loadFloat(CCallHelpers::TrustedImmPtr(&lhsBits), FPRInfo::fpRegT0); + jit.loadFloat(CCallHelpers::TrustedImmPtr(&rhsBits), FPRInfo::fpRegT1); + + jit.addFloat(FPRInfo::fpRegT0, FPRInfo::fpRegT1, FPRInfo::fpRegT2); + armv7StoreFloatToPointer(jit, FPRInfo::fpRegT2, &addResult); + jit.subFloat(FPRInfo::fpRegT0, FPRInfo::fpRegT1, FPRInfo::fpRegT2); + armv7StoreFloatToPointer(jit, FPRInfo::fpRegT2, &subResult); + jit.mulFloat(FPRInfo::fpRegT0, FPRInfo::fpRegT1, FPRInfo::fpRegT2); + armv7StoreFloatToPointer(jit, FPRInfo::fpRegT2, &mulResult); + jit.divFloat(FPRInfo::fpRegT0, FPRInfo::fpRegT1, FPRInfo::fpRegT2); + armv7StoreFloatToPointer(jit, FPRInfo::fpRegT2, &divResult); + + jit.sqrtFloat(FPRInfo::fpRegT0, FPRInfo::fpRegT2); + armv7StoreFloatToPointer(jit, FPRInfo::fpRegT2, &sqrtResult); + jit.absFloat(FPRInfo::fpRegT0, FPRInfo::fpRegT2); + armv7StoreFloatToPointer(jit, FPRInfo::fpRegT2, &absResult); + jit.negateFloat(FPRInfo::fpRegT0, FPRInfo::fpRegT2); + armv7StoreFloatToPointer(jit, FPRInfo::fpRegT2, &negResult); + + jit.loadFloat(CCallHelpers::TrustedImmPtr(&lhsBits), FPRInfo::fpRegT2); + jit.addFloat(FPRInfo::fpRegT2, FPRInfo::fpRegT1, FPRInfo::fpRegT2); + armv7StoreFloatToPointer(jit, FPRInfo::fpRegT2, &addAliased); + + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto operands = armv7FloatBitPatterns(); + for (auto a : operands) { + for (auto b : operands) { + lhsBits = a; + rhsBits = b; + invoke(test); + + float x = std::bit_cast(a); + float y = std::bit_cast(b); + armv7CheckFloat(addResult, x + y); + armv7CheckFloat(subResult, x - y); + armv7CheckFloat(mulResult, x * y); + armv7CheckFloat(divResult, x / y); + CHECK_EQ(absResult, a & 0x7fffffffu); + CHECK_EQ(negResult, a ^ 0x80000000u); + CHECK_EQ(addAliased, addResult); + if (x != x || x < 0.0f) + CHECK_EQ(std::bit_cast(sqrtResult) != std::bit_cast(sqrtResult), true); + } + } + + lhsBits = 0x3f800000u; + rhsBits = 0x33000000u; + invoke(test); + CHECK_EQ(addResult, 0x3f800000u); + CHECK_EQ(mulResult, 0x33000000u); + + lhsBits = 0x7f7fffffu; + rhsBits = 0x40000000u; + invoke(test); + CHECK_EQ(mulResult, 0x7f800000u); +} + +void testARMv7RoundTowardNearestIntDouble() +{ + double input = 0; + uint64_t result = 0; + uint64_t aliased = 0; + + auto test = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadDouble(CCallHelpers::TrustedImmPtr(&input), FPRInfo::fpRegT0); + jit.roundTowardNearestIntDouble(FPRInfo::fpRegT0, FPRInfo::fpRegT1); + jit.storeDouble(FPRInfo::fpRegT1, CCallHelpers::TrustedImmPtr(&result)); + jit.loadDouble(CCallHelpers::TrustedImmPtr(&input), FPRInfo::fpRegT2); + jit.roundTowardNearestIntDouble(FPRInfo::fpRegT2, FPRInfo::fpRegT2); + jit.storeDouble(FPRInfo::fpRegT2, CCallHelpers::TrustedImmPtr(&aliased)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : armv7EdgeDoubles()) { + input = value; + invoke(test); + double expected = static_cast(armv7NearestEvenToInt32(value)); + CHECK_EQ(std::bit_cast(result), expected); + CHECK_EQ(aliased, result); + } + + for (auto value : { 0.5, -0.5, 1.5, -1.5, 2.5, -2.5, 3.5, -3.5, 254.5, 255.5, 0.49999999999999994, 1.0000000000000002 }) { + input = value; + invoke(test); + CHECK_EQ(std::bit_cast(result), static_cast(armv7NearestEvenToInt32(value))); + } +} + +void testARMv7TruncateDoubleToInt64() +{ + double input = 0; + uint32_t lo = 0; + uint32_t hi = 0; + + auto test = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadDouble(CCallHelpers::TrustedImmPtr(&input), FPRInfo::fpRegT0); + jit.truncateDoubleToInt64(FPRInfo::fpRegT0, GPRInfo::regT0, GPRInfo::regT1, FPRInfo::fpRegT1, FPRInfo::fpRegT2); + jit.store32(GPRInfo::regT0, &lo); + jit.store32(GPRInfo::regT1, &hi); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + auto floatTest = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadDouble(CCallHelpers::TrustedImmPtr(&input), FPRInfo::fpRegT0); + jit.convertDoubleToFloat(FPRInfo::fpRegT0, FPRInfo::fpRegT3); + jit.truncateFloatToInt64(FPRInfo::fpRegT3, GPRInfo::regT0, GPRInfo::regT1, FPRInfo::fpRegT1, FPRInfo::fpRegT2); + jit.store32(GPRInfo::regT0, &lo); + jit.store32(GPRInfo::regT1, &hi); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : { 0.0, -0.0, 1.0, -1.0, 0.9, -0.9, 1.5, -1.5, 42.0, -42.0, + 2147483647.0, -2147483648.0, 2147483648.0, -2147483649.0, + 4294967295.0, 4294967296.0, -4294967296.0, -4294967297.0, + 4503599627370496.0, -4503599627370496.0, 1099511627776.0, -1099511627776.0 }) { + input = value; + invoke(test); + uint64_t actual = (static_cast(hi) << 32) | lo; + CHECK_EQ(actual, static_cast(static_cast(value))); + + float asFloat = static_cast(value); + invoke(floatTest); + actual = (static_cast(hi) << 32) | lo; + CHECK_EQ(actual, static_cast(static_cast(asFloat))); + } +} + +void testARMv7TruncateDoubleToUint64() +{ + double input = 0; + uint32_t lo = 0; + uint32_t hi = 0; + + auto test = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadDouble(CCallHelpers::TrustedImmPtr(&input), FPRInfo::fpRegT0); + jit.truncateDoubleToUint64(FPRInfo::fpRegT0, GPRInfo::regT0, GPRInfo::regT1, FPRInfo::fpRegT1, FPRInfo::fpRegT2); + jit.store32(GPRInfo::regT0, &lo); + jit.store32(GPRInfo::regT1, &hi); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + for (auto value : { 0.0, -0.0, 1.0, 0.9, 42.0, 4294967295.0, 4294967296.0, 4294967297.0, + 4503599627370496.0, 1099511627776.0, 18446744073709549568.0, 9223372036854775808.0 }) { + input = value; + invoke(test); + uint64_t actual = (static_cast(hi) << 32) | lo; + CHECK_EQ(actual, static_cast(value)); + } + + for (auto value : { -0.0, -1.0, -42.0, -4294967296.0, std::numeric_limits::quiet_NaN(), + -std::numeric_limits::infinity() }) { + input = value; + invoke(test); + uint64_t actual = (static_cast(hi) << 32) | lo; + CHECK_EQ(actual, 0ull); + } +} + +void testARMv7TransferDouble() +{ + Vector storage(16); + uint8_t* base = std::bit_cast(storage.data()); + + for (auto bits : armv7DoubleBitPatterns()) { + *std::bit_cast(base) = bits; + *std::bit_cast(base + 8) = 0; + *std::bit_cast(base + 16) = static_cast(bits); + *std::bit_cast(base + 20) = 0; + + auto test = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(CCallHelpers::TrustedImmPtr(base), GPRInfo::regT0); + jit.transferDouble(CCallHelpers::Address(GPRInfo::regT0, 0), CCallHelpers::Address(GPRInfo::regT0, 8)); + jit.transferDouble(CCallHelpers::Address(GPRInfo::regT0, 8), CCallHelpers::Address(GPRInfo::regT0, 8)); + jit.transferFloat(CCallHelpers::Address(GPRInfo::regT0, 16), CCallHelpers::Address(GPRInfo::regT0, 20)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + invoke(test); + CHECK_EQ(*std::bit_cast(base + 8), bits); + CHECK_EQ(*std::bit_cast(base + 20), static_cast(bits)); + } +} + +#endif + +#if CPU(ARM_THUMB2) + +void testARMv7Load32StoreOffsetBoundaries() +{ + constexpr int words = 5000; + constexpr int mid = 2500; + static uint32_t buffer[words]; + + auto initBuffer = [&] { + for (int i = 0; i < words; ++i) + buffer[i] = 0xa5000000u + static_cast(i); + }; + + for (auto offset : { -8192, -4100, -4096, -1028, -1024, -1020, -260, -256, -252, -8, -4, 0, 4, 252, 256, 1020, 1024, 4088, 4092, 4096, 8188 }) { + int slot = mid + offset / 4; + + auto load = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load32(CCallHelpers::Address(GPRInfo::argumentGPR0, offset), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + CHECK_EQ(invoke(load, &buffer[mid]), buffer[slot]); + + auto store = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.store32(GPRInfo::argumentGPR1, CCallHelpers::Address(GPRInfo::argumentGPR0, offset)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + invoke(store, &buffer[mid], 0x5eed1234u); + CHECK_EQ(buffer[slot], 0x5eed1234u); + CHECK_EQ(buffer[slot - 1], 0xa5000000u + static_cast(slot - 1)); + CHECK_EQ(buffer[slot + 1], 0xa5000000u + static_cast(slot + 1)); + + auto storeImm = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.store32(CCallHelpers::TrustedImm32(0x0badf00d), CCallHelpers::Address(GPRInfo::argumentGPR0, offset)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + invoke(storeImm, &buffer[mid]); + CHECK_EQ(buffer[slot], 0x0badf00du); + CHECK_EQ(buffer[slot - 1], 0xa5000000u + static_cast(slot - 1)); + CHECK_EQ(buffer[slot + 1], 0xa5000000u + static_cast(slot + 1)); + } +} + +void testARMv7Load8Load16OffsetBoundaries() +{ + constexpr int size = 20000; + constexpr int mid = 10000; + static uint8_t buffer[size]; + + auto initBuffer = [&] { + for (int i = 0; i < size; ++i) + buffer[i] = static_cast(i * 7 + 3); + }; + + for (auto offset : { -8192, -4096, -1024, -257, -256, -255, -254, -2, -1, 0, 1, 2, 254, 255, 256, 4094, 4095, 4096, 8191 }) { + int slot = mid + offset; + + auto load = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load8(CCallHelpers::Address(GPRInfo::argumentGPR0, offset), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + CHECK_EQ(invoke(load, &buffer[mid]), static_cast(buffer[slot])); + + auto loadSigned = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load8SignedExtendTo32(CCallHelpers::Address(GPRInfo::argumentGPR0, offset), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + CHECK_EQ(invoke(loadSigned, &buffer[mid]), static_cast(static_cast(buffer[slot]))); + + auto store = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.store8(GPRInfo::argumentGPR1, CCallHelpers::Address(GPRInfo::argumentGPR0, offset)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + invoke(store, &buffer[mid], 0xa7u); + CHECK_EQ(buffer[slot], static_cast(0xa7)); + CHECK_EQ(buffer[slot - 1], static_cast((slot - 1) * 7 + 3)); + CHECK_EQ(buffer[slot + 1], static_cast((slot + 1) * 7 + 3)); + + auto storeImm = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.store8(CCallHelpers::TrustedImm32(0x3c), CCallHelpers::Address(GPRInfo::argumentGPR0, offset)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + invoke(storeImm, &buffer[mid]); + CHECK_EQ(buffer[slot], static_cast(0x3c)); + } + + for (auto offset : { -8192, -4096, -1024, -256, -254, -2, 0, 2, 254, 256, 4094, 4096, 8190 }) { + int slot = mid + offset; + + initBuffer(); + uint32_t expected = static_cast(buffer[slot]) | (static_cast(buffer[slot + 1]) << 8); + + auto load = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load16(CCallHelpers::Address(GPRInfo::argumentGPR0, offset), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + CHECK_EQ(invoke(load, &buffer[mid]), expected); + + auto loadSigned = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load16SignedExtendTo32(CCallHelpers::Address(GPRInfo::argumentGPR0, offset), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + CHECK_EQ(invoke(loadSigned, &buffer[mid]), static_cast(static_cast(expected))); + + auto store = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.store16(GPRInfo::argumentGPR1, CCallHelpers::Address(GPRInfo::argumentGPR0, offset)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + invoke(store, &buffer[mid], 0xbeefu); + CHECK_EQ(buffer[slot], static_cast(0xef)); + CHECK_EQ(buffer[slot + 1], static_cast(0xbe)); + CHECK_EQ(buffer[slot - 1], static_cast((slot - 1) * 7 + 3)); + CHECK_EQ(buffer[slot + 2], static_cast((slot + 2) * 7 + 3)); + + auto storeImm = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.store16(CCallHelpers::TrustedImm32(0x1234), CCallHelpers::Address(GPRInfo::argumentGPR0, offset)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + invoke(storeImm, &buffer[mid]); + CHECK_EQ(buffer[slot], static_cast(0x34)); + CHECK_EQ(buffer[slot + 1], static_cast(0x12)); + } +} + +void testARMv7LoadStoreBaseIndexScales() +{ + constexpr int words = 8192; + static uint32_t buffer[words]; + constexpr int index = 100; + + auto initBuffer = [&] { + for (int i = 0; i < words; ++i) + buffer[i] = 0xc3000000u + static_cast(i); + }; + + for (int shift = 0; shift < 4; ++shift) { + auto scale = static_cast(shift); + for (auto offset : { 0, 4, 252, 256, 1020, 1024, 4092, 4096, 8192 }) { + int byteOffset = index * (1 << shift) + offset; + uint32_t* slot = bitwise_cast_ptr(reinterpret_cast(&buffer[0]) + byteOffset); + + auto load = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load32(CCallHelpers::BaseIndex(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, scale, offset), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + CHECK_EQ(invoke(load, &buffer[0], index), *slot); + + auto loadDestIsBase = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load32(CCallHelpers::BaseIndex(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, scale, offset), GPRInfo::argumentGPR0); + jit.move(GPRInfo::argumentGPR0, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + CHECK_EQ(invoke(loadDestIsBase, &buffer[0], index), *slot); + + auto loadDestIsIndex = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load32(CCallHelpers::BaseIndex(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, scale, offset), GPRInfo::argumentGPR1); + jit.move(GPRInfo::argumentGPR1, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + CHECK_EQ(invoke(loadDestIsIndex, &buffer[0], index), *slot); + + auto load16BaseIndex = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load16(CCallHelpers::BaseIndex(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, scale, offset), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + CHECK_EQ(invoke(load16BaseIndex, &buffer[0], index), *slot & 0xffffu); + + auto load8BaseIndex = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load8(CCallHelpers::BaseIndex(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, scale, offset), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + CHECK_EQ(invoke(load8BaseIndex, &buffer[0], index), *slot & 0xffu); + + auto store = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.store32(GPRInfo::argumentGPR2, CCallHelpers::BaseIndex(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, scale, offset)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + invoke(store, &buffer[0], index, 0x77665544u); + CHECK_EQ(*slot, 0x77665544u); + + auto storeImm = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.store32(CCallHelpers::TrustedImm32(0x11223344), CCallHelpers::BaseIndex(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, scale, offset)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + invoke(storeImm, &buffer[0], index); + CHECK_EQ(*slot, 0x11223344u); + + auto store16 = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.store16(GPRInfo::argumentGPR2, CCallHelpers::BaseIndex(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, scale, offset)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + uint32_t before = *slot; + invoke(store16, &buffer[0], index, 0xdeadu); + CHECK_EQ(*slot, (before & 0xffff0000u) | 0xdeadu); + + auto store8 = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.store8(GPRInfo::argumentGPR2, CCallHelpers::BaseIndex(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, scale, offset)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + before = *slot; + invoke(store8, &buffer[0], index, 0x9au); + CHECK_EQ(*slot, (before & 0xffffff00u) | 0x9au); + } + } +} + +void testARMv7LoadPair32OffsetBoundaries() +{ + constexpr int words = 5000; + constexpr int mid = 2500; + static uint32_t buffer[words]; + + auto initBuffer = [&] { + for (int i = 0; i < words; ++i) + buffer[i] = 0x77000000u + static_cast(i); + }; + + for (auto offset : { -8192, -4096, -4092, -1028, -1024, -1020, -1016, -256, -252, -8, -4, 0, 4, 8, 252, 256, 1016, 1020, 1024, 4084, 4088, 4092 }) { + int slot = mid + offset / 4; + uint32_t out[2]; + + auto load = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadPair32(CCallHelpers::Address(GPRInfo::argumentGPR0, offset), GPRInfo::regT2, GPRInfo::regT3); + jit.store32(GPRInfo::regT2, CCallHelpers::Address(GPRInfo::argumentGPR1, 0)); + jit.store32(GPRInfo::regT3, CCallHelpers::Address(GPRInfo::argumentGPR1, 4)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + out[0] = 0; + out[1] = 0; + invoke(load, &buffer[mid], &out[0]); + CHECK_EQ(out[0], buffer[slot]); + CHECK_EQ(out[1], buffer[slot + 1]); + + auto loadUnaligned = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadPair32Unaligned(CCallHelpers::Address(GPRInfo::argumentGPR0, offset), GPRInfo::regT2, GPRInfo::regT3); + jit.store32(GPRInfo::regT2, CCallHelpers::Address(GPRInfo::argumentGPR1, 0)); + jit.store32(GPRInfo::regT3, CCallHelpers::Address(GPRInfo::argumentGPR1, 4)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + out[0] = 0; + out[1] = 0; + invoke(loadUnaligned, &buffer[mid], &out[0]); + CHECK_EQ(out[0], buffer[slot]); + CHECK_EQ(out[1], buffer[slot + 1]); + + auto store = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.storePair32(GPRInfo::argumentGPR1, GPRInfo::argumentGPR2, CCallHelpers::Address(GPRInfo::argumentGPR0, offset)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + invoke(store, &buffer[mid], 0x01020304u, 0x05060708u); + CHECK_EQ(buffer[slot], 0x01020304u); + CHECK_EQ(buffer[slot + 1], 0x05060708u); + CHECK_EQ(buffer[slot - 1], 0x77000000u + static_cast(slot - 1)); + CHECK_EQ(buffer[slot + 2], 0x77000000u + static_cast(slot + 2)); + + auto storeImms = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.storePair32(CCallHelpers::TrustedImm32(0x0a0b0c0d), CCallHelpers::TrustedImm32(0x0e0f1011), CCallHelpers::Address(GPRInfo::argumentGPR0, offset)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + invoke(storeImms, &buffer[mid]); + CHECK_EQ(buffer[slot], 0x0a0b0c0du); + CHECK_EQ(buffer[slot + 1], 0x0e0f1011u); + CHECK_EQ(buffer[slot - 1], 0x77000000u + static_cast(slot - 1)); + CHECK_EQ(buffer[slot + 2], 0x77000000u + static_cast(slot + 2)); + + auto storeRegImm = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.storePair32(GPRInfo::argumentGPR1, CCallHelpers::TrustedImm32(0x12131415), CCallHelpers::Address(GPRInfo::argumentGPR0, offset)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + invoke(storeRegImm, &buffer[mid], 0x16171819u); + CHECK_EQ(buffer[slot], 0x16171819u); + CHECK_EQ(buffer[slot + 1], 0x12131415u); + } +} + +void testARMv7LoadPair32Aliasing() +{ + constexpr int words = 5000; + constexpr int mid = 2500; + static uint32_t buffer[words]; + + auto initBuffer = [&] { + for (int i = 0; i < words; ++i) + buffer[i] = 0x88000000u + static_cast(i); + }; + + for (auto offset : { -1024, -1020, -256, -8, 0, 4, 1016, 1020, 1024, 4088, 4092 }) { + int slot = mid + offset / 4; + uint32_t out[2]; + + auto destIsBaseFirst = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadPair32(CCallHelpers::Address(GPRInfo::argumentGPR0, offset), GPRInfo::argumentGPR0, GPRInfo::regT2); + jit.store32(GPRInfo::argumentGPR0, CCallHelpers::Address(GPRInfo::argumentGPR1, 0)); + jit.store32(GPRInfo::regT2, CCallHelpers::Address(GPRInfo::argumentGPR1, 4)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + out[0] = 0; + out[1] = 0; + invoke(destIsBaseFirst, &buffer[mid], &out[0]); + CHECK_EQ(out[0], buffer[slot]); + CHECK_EQ(out[1], buffer[slot + 1]); + + auto destIsBaseSecond = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadPair32(CCallHelpers::Address(GPRInfo::argumentGPR0, offset), GPRInfo::regT2, GPRInfo::argumentGPR0); + jit.store32(GPRInfo::regT2, CCallHelpers::Address(GPRInfo::argumentGPR1, 0)); + jit.store32(GPRInfo::argumentGPR0, CCallHelpers::Address(GPRInfo::argumentGPR1, 4)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + out[0] = 0; + out[1] = 0; + invoke(destIsBaseSecond, &buffer[mid], &out[0]); + CHECK_EQ(out[0], buffer[slot]); + CHECK_EQ(out[1], buffer[slot + 1]); + + auto unalignedDestIsBase = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadPair32Unaligned(CCallHelpers::Address(GPRInfo::argumentGPR0, offset), GPRInfo::argumentGPR0, GPRInfo::regT2); + jit.store32(GPRInfo::argumentGPR0, CCallHelpers::Address(GPRInfo::argumentGPR1, 0)); + jit.store32(GPRInfo::regT2, CCallHelpers::Address(GPRInfo::argumentGPR1, 4)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + out[0] = 0; + out[1] = 0; + invoke(unalignedDestIsBase, &buffer[mid], &out[0]); + CHECK_EQ(out[0], buffer[slot]); + CHECK_EQ(out[1], buffer[slot + 1]); + } + + for (int shift = 0; shift < 4; ++shift) { + auto scale = static_cast(shift); + for (auto offset : { 0, 4, 1020, 1024, 4092 }) { + constexpr int index = 64; + int byteOffset = index * (1 << shift) + offset; + uint32_t* slot = bitwise_cast_ptr(reinterpret_cast(&buffer[0]) + byteOffset); + uint32_t out[2]; + + auto load = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.push(ARMRegisters::r4); + jit.push(ARMRegisters::r5); + jit.loadPair32(CCallHelpers::BaseIndex(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, scale, offset), ARMRegisters::r4, ARMRegisters::r5); + jit.store32(ARMRegisters::r4, CCallHelpers::Address(GPRInfo::argumentGPR2, 0)); + jit.store32(ARMRegisters::r5, CCallHelpers::Address(GPRInfo::argumentGPR2, 4)); + jit.pop(ARMRegisters::r5); + jit.pop(ARMRegisters::r4); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + out[0] = 0; + out[1] = 0; + invoke(load, &buffer[0], index, &out[0]); + CHECK_EQ(out[0], slot[0]); + CHECK_EQ(out[1], slot[1]); + + auto loadDestIsBase = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadPair32(CCallHelpers::BaseIndex(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, scale, offset), GPRInfo::argumentGPR0, GPRInfo::regT3); + jit.store32(GPRInfo::argumentGPR0, CCallHelpers::Address(GPRInfo::argumentGPR2, 0)); + jit.store32(GPRInfo::regT3, CCallHelpers::Address(GPRInfo::argumentGPR2, 4)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + out[0] = 0; + out[1] = 0; + invoke(loadDestIsBase, &buffer[0], index, &out[0]); + CHECK_EQ(out[0], slot[0]); + CHECK_EQ(out[1], slot[1]); + + auto loadDestIsIndex = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadPair32(CCallHelpers::BaseIndex(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, scale, offset), GPRInfo::regT3, GPRInfo::argumentGPR1); + jit.store32(GPRInfo::regT3, CCallHelpers::Address(GPRInfo::argumentGPR2, 0)); + jit.store32(GPRInfo::argumentGPR1, CCallHelpers::Address(GPRInfo::argumentGPR2, 4)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + out[0] = 0; + out[1] = 0; + invoke(loadDestIsIndex, &buffer[0], index, &out[0]); + CHECK_EQ(out[0], slot[0]); + CHECK_EQ(out[1], slot[1]); + + auto store = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.storePair32(GPRInfo::regT2, GPRInfo::regT3, CCallHelpers::BaseIndex(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, scale, offset)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + invoke(store, &buffer[0], index, 0xaabbccddu, 0x11223344u); + CHECK_EQ(slot[0], 0xaabbccddu); + CHECK_EQ(slot[1], 0x11223344u); + } + } +} + +void testARMv7CachedTempRegisterInvalidatedByMemoryOps() +{ + constexpr int words = 4096; + constexpr int mid = 2048; + static uint32_t buffer[words]; + for (int i = 0; i < words; ++i) + buffer[i] = 0xffffffffu - static_cast(i); + + constexpr uint32_t marker = 0x12345678u; + + auto check = [&] (auto emitMemoryOp, MacroAssembler::RegisterID temp) { + auto code = compile([&] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(CCallHelpers::TrustedImm32(marker), temp); + emitMemoryOp(jit, temp); + jit.move(CCallHelpers::TrustedImm32(marker), temp); + jit.move(temp, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + CHECK_EQ(invoke(code, &buffer[mid], 0u), marker); + }; + + for (auto temp : { ARMRegisters::ip, ARMRegisters::r6 }) { + check([] (CCallHelpers& jit, MacroAssembler::RegisterID dest) { + jit.load32(CCallHelpers::Address(GPRInfo::argumentGPR0, 0), dest); + }, temp); + + check([] (CCallHelpers& jit, MacroAssembler::RegisterID dest) { + jit.load8(CCallHelpers::Address(GPRInfo::argumentGPR0, 0), dest); + }, temp); + + check([] (CCallHelpers& jit, MacroAssembler::RegisterID dest) { + jit.load8SignedExtendTo32(CCallHelpers::Address(GPRInfo::argumentGPR0, 0), dest); + }, temp); + + check([] (CCallHelpers& jit, MacroAssembler::RegisterID dest) { + jit.load16(CCallHelpers::Address(GPRInfo::argumentGPR0, 0), dest); + }, temp); + + check([] (CCallHelpers& jit, MacroAssembler::RegisterID dest) { + jit.load16(CCallHelpers::Address(GPRInfo::argumentGPR0, 4096), dest); + }, temp); + + check([] (CCallHelpers& jit, MacroAssembler::RegisterID dest) { + jit.load16SignedExtendTo32(CCallHelpers::Address(GPRInfo::argumentGPR0, 0), dest); + }, temp); + + check([] (CCallHelpers& jit, MacroAssembler::RegisterID dest) { + jit.load16(CCallHelpers::BaseIndex(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, CCallHelpers::TimesTwo, 0), dest); + }, temp); + + check([] (CCallHelpers& jit, MacroAssembler::RegisterID dest) { + jit.load16(CCallHelpers::BaseIndex(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, CCallHelpers::TimesTwo, 8), dest); + }, temp); + + check([] (CCallHelpers& jit, MacroAssembler::RegisterID dest) { + jit.load32(CCallHelpers::BaseIndex(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, CCallHelpers::TimesFour, 0), dest); + }, temp); + + check([] (CCallHelpers& jit, MacroAssembler::RegisterID dest) { + jit.load16Unaligned(CCallHelpers::BaseIndex(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, CCallHelpers::TimesOne, 0), dest); + }, temp); + + check([] (CCallHelpers& jit, MacroAssembler::RegisterID dest) { + jit.convertibleLoadPtr(CCallHelpers::Address(GPRInfo::argumentGPR0, 4), dest); + }, temp); + + check([] (CCallHelpers& jit, MacroAssembler::RegisterID dest) { + jit.loadPair32(CCallHelpers::Address(GPRInfo::argumentGPR0, 0), dest, GPRInfo::regT2); + }, temp); + + check([] (CCallHelpers& jit, MacroAssembler::RegisterID dest) { + jit.loadPair32(CCallHelpers::Address(GPRInfo::argumentGPR0, 0), GPRInfo::regT2, dest); + }, temp); + + check([] (CCallHelpers& jit, MacroAssembler::RegisterID dest) { + jit.loadLink32(CCallHelpers::Address(GPRInfo::argumentGPR0, 0), dest); + }, temp); + + check([] (CCallHelpers& jit, MacroAssembler::RegisterID dest) { + jit.loadLink8(CCallHelpers::Address(GPRInfo::argumentGPR0, 0), dest); + }, temp); + + check([] (CCallHelpers& jit, MacroAssembler::RegisterID dest) { + jit.loadLink16(CCallHelpers::Address(GPRInfo::argumentGPR0, 0), dest); + }, temp); + + check([] (CCallHelpers& jit, MacroAssembler::RegisterID result) { + jit.loadLink32(CCallHelpers::Address(GPRInfo::argumentGPR0, 0), GPRInfo::regT2); + jit.storeCond32(GPRInfo::regT2, CCallHelpers::Address(GPRInfo::argumentGPR0, 0), result); + }, temp); + + check([] (CCallHelpers& jit, MacroAssembler::RegisterID result) { + jit.loadLink8(CCallHelpers::Address(GPRInfo::argumentGPR0, 0), GPRInfo::regT2); + jit.storeCond8(GPRInfo::regT2, CCallHelpers::Address(GPRInfo::argumentGPR0, 0), result); + }, temp); + + check([] (CCallHelpers& jit, MacroAssembler::RegisterID dest) { + jit.getEffectiveAddress(CCallHelpers::BaseIndex(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, CCallHelpers::TimesFour, 0), dest); + }, temp); + } +} + +void testARMv7TransferMemory() +{ + constexpr int words = 3000; + constexpr int mid = 1000; + static uint32_t buffer[words]; + + auto initBuffer = [&] { + for (int i = 0; i < words; ++i) + buffer[i] = 0x99000000u + static_cast(i); + }; + + for (auto offset : { -1024, -256, -4, 0, 4, 1020, 4092, 4096 }) { + int srcSlot = mid + offset / 4; + + auto transfer = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.transfer32(CCallHelpers::Address(GPRInfo::argumentGPR0, offset), CCallHelpers::Address(GPRInfo::argumentGPR1, 0)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + uint32_t out = 0; + invoke(transfer, &buffer[mid], &out); + CHECK_EQ(out, buffer[srcSlot]); + + auto transfer64 = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.transfer64(CCallHelpers::Address(GPRInfo::argumentGPR0, offset), CCallHelpers::Address(GPRInfo::argumentGPR1, 0)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + uint32_t out64[2]; + out64[0] = 0; + out64[1] = 0; + invoke(transfer64, &buffer[mid], &out64[0]); + CHECK_EQ(out64[0], buffer[srcSlot]); + CHECK_EQ(out64[1], buffer[srcSlot + 1]); + } + + for (int shift = 0; shift < 4; ++shift) { + auto scale = static_cast(shift); + constexpr int index = 32; + int byteOffset = index * (1 << shift); + uint32_t* slot = bitwise_cast_ptr(reinterpret_cast(&buffer[0]) + byteOffset); + + auto transfer = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.transfer32(CCallHelpers::BaseIndex(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, scale, 0), CCallHelpers::BaseIndex(GPRInfo::argumentGPR2, GPRInfo::argumentGPR3, CCallHelpers::TimesOne, 0)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + uint32_t out = 0; + invoke(transfer, &buffer[0], index, &out, 0); + CHECK_EQ(out, *slot); + } +} + +void testARMv7SwapRegisters() +{ + for (auto variant : { 0, 1, 2 }) { + auto code = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + if (!variant) + jit.swap(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1); + else if (variant == 1) { + jit.swap(GPRInfo::argumentGPR0, GPRInfo::argumentGPR0); + jit.swap(GPRInfo::argumentGPR1, GPRInfo::argumentGPR1); + jit.swap(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1); + } else { + jit.swap(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1); + jit.swap(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1); + jit.swap(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1); + } + jit.store32(GPRInfo::argumentGPR0, CCallHelpers::Address(GPRInfo::argumentGPR2, 0)); + jit.store32(GPRInfo::argumentGPR1, CCallHelpers::Address(GPRInfo::argumentGPR2, 4)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + uint32_t out[2]; + out[0] = 0; + out[1] = 0; + invoke(code, 0x11112222u, 0x33334444u, &out[0]); + CHECK_EQ(out[0], 0x33334444u); + CHECK_EQ(out[1], 0x11112222u); + } +} + +void testARMv7LoadStoreAbsoluteAddress() +{ + static uint32_t buffer[16]; + + auto initBuffer = [&] { + for (int i = 0; i < 16; ++i) + buffer[i] = 0x44000000u + static_cast(i); + }; + + for (auto index : { 0, 1, 2, 15 }) { + auto load = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load32(&buffer[index], GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + CHECK_EQ(invoke(load), buffer[index]); + + auto load8 = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load8(&buffer[index], GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + CHECK_EQ(invoke(load8), buffer[index] & 0xffu); + + auto load16 = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.load16(&buffer[index], GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + CHECK_EQ(invoke(load16), buffer[index] & 0xffffu); + + auto store = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.store32(GPRInfo::argumentGPR0, &buffer[index]); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + invoke(store, 0xfeedbeefu); + CHECK_EQ(buffer[index], 0xfeedbeefu); + + auto storeImm = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.store32(CCallHelpers::TrustedImm32(0x0c0d0e0f), &buffer[index]); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + invoke(storeImm); + CHECK_EQ(buffer[index], 0x0c0d0e0fu); + + uint32_t out[2]; + + auto loadPair = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadPair32(CCallHelpers::AbsoluteAddress(&buffer[0]), GPRInfo::regT2, GPRInfo::regT3); + jit.store32(GPRInfo::regT2, CCallHelpers::Address(GPRInfo::argumentGPR0, 0)); + jit.store32(GPRInfo::regT3, CCallHelpers::Address(GPRInfo::argumentGPR0, 4)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + out[0] = 0; + out[1] = 0; + invoke(loadPair, &out[0]); + CHECK_EQ(out[0], buffer[0]); + CHECK_EQ(out[1], buffer[1]); + + auto storePair = compile([=] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.storePair32(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, &buffer[0]); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + initBuffer(); + invoke(storePair, 0x21222324u, 0x25262728u); + CHECK_EQ(buffer[0], 0x21222324u); + CHECK_EQ(buffer[1], 0x25262728u); + } +} + +#endif + +#if CPU(ARM_THUMB2) +void testARMv7StorePair32AbsoluteTempAliasing() +{ + static uint32_t buffer[8]; + for (auto& word : buffer) + word = 0; + + auto test = compile([] (CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(CCallHelpers::TrustedImmPtr(&buffer[0]), MacroAssembler::dataTempRegister); + jit.move(CCallHelpers::TrustedImm32(0x11111111), GPRInfo::regT2); + jit.storePair32(GPRInfo::regT2, CCallHelpers::TrustedImm32(0x22222222), &buffer[2]); + emitFunctionEpilogue(jit); + jit.ret(); + }); + + invoke(test); + CHECK_EQ(buffer[2], 0x11111111u); + CHECK_EQ(buffer[3], 0x22222222u); +} +#endif + +#if CPU(ARM64) +void testLoadStorePair64Int64() +{ + constexpr uint64_t initialValue = 0x5555aaaabbbb8800ull; + constexpr uint64_t value1 = 42; + constexpr uint64_t value2 = 0xcafebabe12345678ull; + + uint64_t buffer[10]; + + auto initBuffer = [&] { + for (unsigned i = 0; i < 10; ++i) + buffer[i] = initialValue + i; + }; + + struct Pair { + uint64_t value1; + uint64_t value2; + }; + + Pair pair; + auto initPair = [&] { + pair = { 0, 0 }; + }; + + // Test loadPair64. + auto testLoadPair = [] (CCallHelpers& jit, int offset) { + emitFunctionPrologue(jit); + + constexpr GPRReg bufferGPR = GPRInfo::argumentGPR0; + constexpr GPRReg pairGPR = GPRInfo::argumentGPR1; + jit.loadPair64(bufferGPR, CCallHelpers::TrustedImm32(offset * sizeof(CPURegister)), GPRInfo::regT2, GPRInfo::regT3); + + jit.store64(GPRInfo::regT2, CCallHelpers::Address(pairGPR, 0)); + jit.store64(GPRInfo::regT3, CCallHelpers::Address(pairGPR, sizeof(uint64_t))); + + emitFunctionEpilogue(jit); + jit.ret(); + }; + + auto testLoadPair0 = compile([&] (CCallHelpers& jit) { + testLoadPair(jit, 0); + }); + + initBuffer(); + + initPair(); + invoke(testLoadPair0, &buffer[4], &pair); + CHECK_EQ(pair.value1, initialValue + 4); + CHECK_EQ(pair.value2, initialValue + 5); + + initPair(); + buffer[4] = value1; + buffer[5] = value2; + invoke(testLoadPair0, &buffer[4], &pair); + CHECK_EQ(pair.value1, value1); + CHECK_EQ(pair.value2, value2); + + auto testLoadPairMinus2 = compile([&] (CCallHelpers& jit) { + testLoadPair(jit, -2); + }); + + initPair(); + invoke(testLoadPairMinus2, &buffer[4], &pair); + CHECK_EQ(pair.value1, initialValue + 4 - 2); + CHECK_EQ(pair.value2, initialValue + 5 - 2); + + initPair(); + buffer[4 - 2] = value2; + buffer[5 - 2] = value1; + invoke(testLoadPairMinus2, &buffer[4], &pair); + CHECK_EQ(pair.value1, value2); + CHECK_EQ(pair.value2, value1); + + auto testLoadPairPlus3 = compile([&] (CCallHelpers& jit) { + testLoadPair(jit, 3); + }); + + initPair(); + invoke(testLoadPairPlus3, &buffer[4], &pair); + CHECK_EQ(pair.value1, initialValue + 4 + 3); + CHECK_EQ(pair.value2, initialValue + 5 + 3); + + initPair(); + buffer[4 + 3] = value1; + buffer[5 + 3] = value2; + invoke(testLoadPairPlus3, &buffer[4], &pair); + CHECK_EQ(pair.value1, value1); + CHECK_EQ(pair.value2, value2); + + // Test loadPair64 using a buffer register as a destination. + auto testLoadPairUsingBufferRegisterAsDestination = [] (CCallHelpers& jit, int offset) { + emitFunctionPrologue(jit); + + constexpr GPRReg bufferGPR = GPRInfo::argumentGPR0; + constexpr GPRReg pairGPR = GPRInfo::argumentGPR1; + jit.loadPair64(bufferGPR, CCallHelpers::TrustedImm32(offset * sizeof(CPURegister)), GPRInfo::argumentGPR0, GPRInfo::regT2); + + jit.store64(GPRInfo::argumentGPR0, CCallHelpers::Address(pairGPR, 0)); + jit.store64(GPRInfo::regT2, CCallHelpers::Address(pairGPR, sizeof(uint64_t))); + + emitFunctionEpilogue(jit); + jit.ret(); + }; + + auto testLoadPairUsingBufferRegisterAsDestination0 = compile([&] (CCallHelpers& jit) { + testLoadPairUsingBufferRegisterAsDestination(jit, 0); + }); + + initBuffer(); + + initPair(); + invoke(testLoadPairUsingBufferRegisterAsDestination0, &buffer[4], &pair); + CHECK_EQ(pair.value1, initialValue + 4); + CHECK_EQ(pair.value2, initialValue + 5); + + // Test storePair64. + auto testStorePair = [] (CCallHelpers& jit, int offset) { + emitFunctionPrologue(jit); + + constexpr GPRReg bufferGPR = GPRInfo::argumentGPR2; + jit.storePair64(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, bufferGPR, CCallHelpers::TrustedImm32(offset * sizeof(CPURegister))); + + emitFunctionEpilogue(jit); + jit.ret(); + }; + + auto testStorePair0 = compile([&] (CCallHelpers& jit) { + testStorePair(jit, 0); + }); + + initBuffer(); + invoke(testStorePair0, value1, value2, &buffer[4]); + CHECK_EQ(buffer[0], initialValue + 0); + CHECK_EQ(buffer[1], initialValue + 1); + CHECK_EQ(buffer[2], initialValue + 2); + CHECK_EQ(buffer[3], initialValue + 3); + CHECK_EQ(buffer[4], value1); + CHECK_EQ(buffer[5], value2); + CHECK_EQ(buffer[6], initialValue + 6); + CHECK_EQ(buffer[7], initialValue + 7); + CHECK_EQ(buffer[8], initialValue + 8); + CHECK_EQ(buffer[9], initialValue + 9); + + auto testStorePairMinus2 = compile([&] (CCallHelpers& jit) { + testStorePair(jit, -2); + }); + + initBuffer(); + invoke(testStorePairMinus2, value1, value2, &buffer[4]); + CHECK_EQ(buffer[0], initialValue + 0); + CHECK_EQ(buffer[1], initialValue + 1); + CHECK_EQ(buffer[2], value1); + CHECK_EQ(buffer[3], value2); + CHECK_EQ(buffer[4], initialValue + 4); + CHECK_EQ(buffer[5], initialValue + 5); + CHECK_EQ(buffer[6], initialValue + 6); + CHECK_EQ(buffer[7], initialValue + 7); + CHECK_EQ(buffer[8], initialValue + 8); + CHECK_EQ(buffer[9], initialValue + 9); + + auto testStorePairPlus3 = compile([&] (CCallHelpers& jit) { + testStorePair(jit, 3); + }); + + initBuffer(); invoke(testStorePairPlus3, value1, value2, &buffer[4]); CHECK_EQ(buffer[0], initialValue + 0); CHECK_EQ(buffer[1], initialValue + 1); @@ -6065,6 +12682,201 @@ void run(const char* filter) WTF_IGNORES_THREAD_SAFETY_ANALYSIS RUN(testMul32WithImmediates()); RUN(testLoadStorePair32()); RUN(testSub32ArgImm()); +#if CPU(ARM_THUMB2) + RUN(testSub32ImmArg()); +#endif +#if CPU(ARM_THUMB2) + RUN(testARMv7StorePair32AbsoluteTempAliasing()); +#endif +RUN(testALUAdd32Immediate()); + RUN(testALUSub32Immediate()); + RUN(testALUAnd32Immediate()); + RUN(testALUOr32Immediate()); + RUN(testALUXor32Immediate()); + RUN(testALURegisterAliasing()); + RUN(testALUNeg32AndNot32()); + RUN(testALUMul32()); + RUN(testALUMul32Immediate()); + RUN(testALUCountLeadingZeros32()); + RUN(testALUShift32Boundaries()); + RUN(testALUBranchAdd32Immediate()); + RUN(testALUBranchSub32Immediate()); + RUN(testALUBranchNeg32()); + RUN(testALUBranchMul32Overflow()); + RUN(testALUCompare32Immediate()); +#if CPU(ARM_THUMB2) + RUN(testALUARMv7Add32ToStackPointerImmediate()); + RUN(testALUARMv7Sub32FromStackPointerImmediate()); + RUN(testALUARMv7Add32FromStackPointerImmediate()); + RUN(testALUARMv7HighRegisterOperands()); + RUN(testALUARMv7RotateRight32MultipleOf32()); + RUN(testALUARMv7RotateLeft32()); + RUN(testALUARMv7TruncateDoubleToInt64Negative()); +#endif +#if CPU(ARM_THUMB2) + RUN(testARMv7BranchRangeConditional()); + RUN(testARMv7BranchRangeUnconditional()); + RUN(testARMv7BranchRangeConditionalLong()); + RUN(testARMv7BranchRangeUnconditionalLong()); + RUN(testARMv7BranchRangeBackward()); + RUN(testARMv7BranchRangeManyJumpsToOneLabel()); + RUN(testARMv7LoadDoubleOffsets()); + RUN(testARMv7StoreDoubleOffsets()); + RUN(testARMv7LoadStoreFloatOffsets()); + RUN(testARMv7DoubleArithAliasing()); + RUN(testARMv7FloatArithAliasing()); + RUN(testARMv7AbsNegSqrtDouble()); + RUN(testARMv7AbsNegSqrtFloat()); + RUN(testARMv7MoveDoubleAliasing()); + RUN(testARMv7ConvertInt32ToDouble()); + RUN(testARMv7ConvertUInt32ToDouble()); + RUN(testARMv7ConvertInt32ToFloat()); + RUN(testARMv7ConvertUInt32ToFloat()); + RUN(testARMv7TruncateDoubleToInt32()); + RUN(testARMv7TruncateDoubleToUint32()); + RUN(testARMv7ConvertDoubleFloatRoundTrip()); + RUN(testARMv7BranchDoubleConditions()); + RUN(testARMv7BranchDoubleWithZero()); + RUN(testARMv7CompareDouble()); + RUN(testARMv7LoadStorePair64Double()); + RUN(testARMv7MoveDoubleBits()); + RUN(testARMv7Move32ToFloatBits()); + RUN(testARMv7MoveZeroToDoubleAndFloat()); +#endif +RUN(testAsmMemStoreLoad32OffsetBoundaries()); + RUN(testAsmMemStoreLoad16OffsetBoundaries()); + RUN(testAsmMemStoreLoad8OffsetBoundaries()); + RUN(testAsmMemStoreLoad32BaseIndexScales()); + RUN(testAsmMemStoreLoad8BaseIndexScales()); + RUN(testAsmMemLoadPair32OffsetBoundaries()); + RUN(testAsmMemLoadPair32Aliasing()); + RUN(testAsmMemStorePair32OffsetBoundaries()); + RUN(testAsmMemShift32ImmediateBoundaries()); + RUN(testAsmMemShift32RegisterBoundaries()); + RUN(testAsmMemCountZeros32Boundaries()); + +#if CPU(ARM_THUMB2) + RUN(testAsmMemRotateRight32ImmediateBoundaries()); + RUN(testAsmMemRotate32RegisterBoundaries()); + RUN(testAsmMemRotateLeft32ImmediateBoundaries()); +#endif +#if CPU(ARM_THUMB2) + RUN(testARMv7ImmAdd32ArgImm()); + RUN(testARMv7ImmAdd32ImmDest()); + RUN(testARMv7ImmAddSubRoundTrip()); + RUN(testARMv7ImmAnd32ArgImm()); + RUN(testARMv7ImmOr32ArgImm()); + RUN(testARMv7ImmXor32ArgImm()); + RUN(testARMv7ImmMove32()); + RUN(testARMv7ImmBranch32()); + RUN(testARMv7ImmCompare32()); + RUN(testARMv7ImmRotateRight32()); + RUN(testARMv7ImmRotateLeft32()); + RUN(testARMv7ImmShift32Amounts()); +#endif +RUN(testRotateRight32Immediate()); +RUN(testShift32Immediate()); +RUN(testOrAbsoluteAddressAdjacent()); +RUN(testAdd64ImmAbsoluteAddressAdjacent()); + +#if CPU(ARM_THUMB2) +RUN(testArmv7Arith32Aliasing()); +RUN(testArmv7Arith32ImmediateAliasing()); +RUN(testArmv7Arith32InPlaceImmediate()); +RUN(testArmv7Neg32Not32Aliasing()); +RUN(testArmv7CountZeros32Aliasing()); +RUN(testArmv7Shift32RegisterAliasing()); +RUN(testArmv7Shift32ImmediateValueRegisterAmount()); +RUN(testArmv7ShiftUnchecked()); +RUN(testArmv7RotateLeft32()); +RUN(testArmv7RotateLeft32Immediate()); +RUN(testArmv7Add64Sub64RegisterPairs()); +RUN(testArmv7CountZeros64RegisterPairs()); +RUN(testArmv7AddUnsignedRightShift32()); +RUN(testArmv7Add32Sub32MemoryOperands()); +#endif +RUN(testBranch32RegReg()); + RUN(testBranch32RegImm()); + RUN(testBranch32Address()); + RUN(testBranch8And16()); + RUN(testCompare32Setter()); + RUN(testCompare32SetterImm()); + RUN(testBranchTest32Masmbr()); + RUN(testTest32Setter()); + RUN(testBranchAdd32Masmbr()); + RUN(testBranchAdd32Address()); + RUN(testBranchSub32Masmbr()); + RUN(testBranchMul32Masmbr()); + RUN(testBranchNegAndOr32Masmbr()); + RUN(testMoveConditionally32Masmbr()); + RUN(testMoveConditionallyTest32Masmbr()); + RUN(testPatchableBranchMasmbr()); + +#if CPU(ARM_THUMB2) + RUN(testBranch64ARMv7()); + RUN(testBranch64ImmARMv7()); + RUN(testCompare64ARMv7()); + RUN(testBranchTest64ARMv7()); +#endif +#if CPU(ARM_THUMB2) + RUN(testARMv7FPMoveDoubleInts()); + RUN(testARMv7FPMoveDoubleAliasing()); + RUN(testARMv7FPMoveFloatAndImmediates()); + RUN(testARMv7FPMove64ToDoubleImmediate()); + RUN(testARMv7FPLoadStoreOffsets()); + RUN(testARMv7FPLoadStoreBaseIndex()); + RUN(testARMv7BranchDoubleEdges(MacroAssembler::DoubleEqualAndOrdered)); + RUN(testARMv7BranchDoubleEdges(MacroAssembler::DoubleNotEqualAndOrdered)); + RUN(testARMv7BranchDoubleEdges(MacroAssembler::DoubleGreaterThanAndOrdered)); + RUN(testARMv7BranchDoubleEdges(MacroAssembler::DoubleGreaterThanOrEqualAndOrdered)); + RUN(testARMv7BranchDoubleEdges(MacroAssembler::DoubleLessThanAndOrdered)); + RUN(testARMv7BranchDoubleEdges(MacroAssembler::DoubleLessThanOrEqualAndOrdered)); + RUN(testARMv7BranchDoubleEdges(MacroAssembler::DoubleEqualOrUnordered)); + RUN(testARMv7BranchDoubleEdges(MacroAssembler::DoubleNotEqualOrUnordered)); + RUN(testARMv7BranchDoubleEdges(MacroAssembler::DoubleGreaterThanOrUnordered)); + RUN(testARMv7BranchDoubleEdges(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered)); + RUN(testARMv7BranchDoubleEdges(MacroAssembler::DoubleLessThanOrUnordered)); + RUN(testARMv7BranchDoubleEdges(MacroAssembler::DoubleLessThanOrEqualOrUnordered)); + RUN(testARMv7BranchFloatEdges(MacroAssembler::DoubleEqualAndOrdered)); + RUN(testARMv7BranchFloatEdges(MacroAssembler::DoubleNotEqualAndOrdered)); + RUN(testARMv7BranchFloatEdges(MacroAssembler::DoubleGreaterThanAndOrdered)); + RUN(testARMv7BranchFloatEdges(MacroAssembler::DoubleGreaterThanOrEqualAndOrdered)); + RUN(testARMv7BranchFloatEdges(MacroAssembler::DoubleLessThanAndOrdered)); + RUN(testARMv7BranchFloatEdges(MacroAssembler::DoubleLessThanOrEqualAndOrdered)); + RUN(testARMv7BranchFloatEdges(MacroAssembler::DoubleEqualOrUnordered)); + RUN(testARMv7BranchFloatEdges(MacroAssembler::DoubleNotEqualOrUnordered)); + RUN(testARMv7BranchFloatEdges(MacroAssembler::DoubleGreaterThanOrUnordered)); + RUN(testARMv7BranchFloatEdges(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered)); + RUN(testARMv7BranchFloatEdges(MacroAssembler::DoubleLessThanOrUnordered)); + RUN(testARMv7BranchFloatEdges(MacroAssembler::DoubleLessThanOrEqualOrUnordered)); + RUN(testARMv7BranchDoubleNonZeroAndZeroOrNaN()); + RUN(testARMv7TruncateDoubleAndFloatToInt32()); + RUN(testARMv7BranchTruncateDoubleToInt32()); + RUN(testARMv7BranchConvertDoubleToInt32(true)); + RUN(testARMv7BranchConvertDoubleToInt32(false)); + RUN(testARMv7ConvertInt32AndUint32ToFloatingPoint()); + RUN(testARMv7ConvertInt32ImmediateToDouble()); + RUN(testARMv7ConvertFloatDouble()); + RUN(testARMv7DoubleBinaryAliasing()); + RUN(testARMv7DoubleAccumulateForms()); + RUN(testARMv7DoubleUnaryAliasing()); + RUN(testARMv7FloatArithmeticIsSinglePrecision()); + RUN(testARMv7RoundTowardNearestIntDouble()); + RUN(testARMv7TruncateDoubleToInt64()); + RUN(testARMv7TruncateDoubleToUint64()); + RUN(testARMv7TransferDouble()); +#endif +#if CPU(ARM_THUMB2) + RUN(testARMv7Load32StoreOffsetBoundaries()); + RUN(testARMv7Load8Load16OffsetBoundaries()); + RUN(testARMv7LoadStoreBaseIndexScales()); + RUN(testARMv7LoadPair32OffsetBoundaries()); + RUN(testARMv7LoadPair32Aliasing()); + RUN(testARMv7CachedTempRegisterInvalidatedByMemoryOps()); + RUN(testARMv7TransferMemory()); + RUN(testARMv7SwapRegisters()); + RUN(testARMv7LoadStoreAbsoluteAddress()); +#endif RUN(testBranchTest8()); RUN(testBranchTest16()); diff --git a/Source/JavaScriptCore/bytecode/BytecodeList.rb b/Source/JavaScriptCore/bytecode/BytecodeList.rb index 40ae1bad147f..1e0deba18225 100644 --- a/Source/JavaScriptCore/bytecode/BytecodeList.rb +++ b/Source/JavaScriptCore/bytecode/BytecodeList.rb @@ -1481,6 +1481,7 @@ op :op_put_by_val_direct_return_location op :op_in_by_id_return_location op :op_in_by_val_return_location +op :op_instanceof_return_location op :op_enumerator_get_by_val_return_location op :op_enumerator_put_by_val_return_location op :op_enumerator_in_by_val_return_location diff --git a/Source/JavaScriptCore/bytecode/DeferredCompilationCallback.cpp b/Source/JavaScriptCore/bytecode/DeferredCompilationCallback.cpp index 67ff2b2ffd92..f879fbc47933 100644 --- a/Source/JavaScriptCore/bytecode/DeferredCompilationCallback.cpp +++ b/Source/JavaScriptCore/bytecode/DeferredCompilationCallback.cpp @@ -26,6 +26,8 @@ #include "config.h" #include "DeferredCompilationCallback.h" +#include "StrongInlines.h" + namespace JSC { DeferredCompilationCallback::DeferredCompilationCallback() = default; diff --git a/Source/JavaScriptCore/bytecode/InlineCacheCompiler.cpp b/Source/JavaScriptCore/bytecode/InlineCacheCompiler.cpp index 31ee5b3feff5..38b9c3df4681 100644 --- a/Source/JavaScriptCore/bytecode/InlineCacheCompiler.cpp +++ b/Source/JavaScriptCore/bytecode/InlineCacheCompiler.cpp @@ -1050,6 +1050,11 @@ const ScalarRegisterSet& InlineCacheCompiler::calculateLiveRegistersForCallAndEx if (m_stubInfo.useDataIC) liveRegistersForCall.add(m_stubInfo.m_stubInfoGPR, IgnoreVectors); liveRegistersForCall.exclude(calleeSaveRegisters().buildAndValidate().includeWholeRegisterWidth()); +#if CPU(ARM_THUMB2) + // These are callee save on armv7 only, so DFG treats them as caller-saved. + liveRegistersForCall.add(GPRInfo::metadataTableRegister, IgnoreVectors); + liveRegistersForCall.add(GPRInfo::jitDataRegister, IgnoreVectors); +#endif m_liveRegistersForCall = liveRegistersForCall.buildScalarRegisterSet(); } return m_liveRegistersForCall; @@ -4192,10 +4197,6 @@ void InlineCacheCompiler::emitProxyObjectAccess(unsigned index, AccessCase& acce #if USE(JSVALUE32_64) // We *always* know that the proxy function, if non-null, is a cell. jit.move(CCallHelpers::TrustedImm32(JSValue::CellTag), BaselineJITRegisters::Call::calleeJSR.tagGPR()); -#endif -#if CPU(ARM_THUMB2) - // ARMv7 clobbers metadataTable register. Thus we need to restore them back here. - JIT::emitMaterializeMetadataAndConstantPoolRegisters(jit); #endif m_callLinkInfos[index] = makeUnique(m_stubInfo.codeOrigin, nullptr); auto* callLinkInfo = m_callLinkInfos[index].get(); diff --git a/Source/JavaScriptCore/dfg/DFGAdaptiveInferredPropertyValueWatchpoint.cpp b/Source/JavaScriptCore/dfg/DFGAdaptiveInferredPropertyValueWatchpoint.cpp index 492bac685f22..8f901f638b93 100644 --- a/Source/JavaScriptCore/dfg/DFGAdaptiveInferredPropertyValueWatchpoint.cpp +++ b/Source/JavaScriptCore/dfg/DFGAdaptiveInferredPropertyValueWatchpoint.cpp @@ -29,6 +29,7 @@ #if ENABLE(DFG_JIT) #include "CodeBlock.h" +#include "CodeBlockInlines.h" #include "DFGCommon.h" #include diff --git a/Source/JavaScriptCore/dfg/DFGOSRExitCompilerCommon.cpp b/Source/JavaScriptCore/dfg/DFGOSRExitCompilerCommon.cpp index 3b1f802f04cd..69cb5de8e343 100644 --- a/Source/JavaScriptCore/dfg/DFGOSRExitCompilerCommon.cpp +++ b/Source/JavaScriptCore/dfg/DFGOSRExitCompilerCommon.cpp @@ -193,6 +193,8 @@ static CodePtr callerReturnPC(CodeBlock* baselineCodeBlockForCall jumpTarget = LLINT_RETURN_LOCATION(op_get_by_val); else if (callInstruction.opcodeID() == op_enumerator_get_by_val) jumpTarget = LLINT_RETURN_LOCATION(op_enumerator_get_by_val); + else if (callInstruction.opcodeID() == op_instanceof) + jumpTarget = LLINT_RETURN_LOCATION(op_instanceof); else RELEASE_ASSERT_NOT_REACHED(); break; diff --git a/Source/JavaScriptCore/heap/BlockDirectory.cpp b/Source/JavaScriptCore/heap/BlockDirectory.cpp index 2e5bf80bb5ce..baeef2d509c6 100644 --- a/Source/JavaScriptCore/heap/BlockDirectory.cpp +++ b/Source/JavaScriptCore/heap/BlockDirectory.cpp @@ -28,6 +28,7 @@ #include "BlockDirectoryInlines.h" #include "Heap.h" +#include "HeapInlines.h" #include "MarkedSpaceInlines.h" #include "SubspaceInlines.h" #include "SuperSampler.h" diff --git a/Source/JavaScriptCore/jit/JITOpcodes.cpp b/Source/JavaScriptCore/jit/JITOpcodes.cpp index 9764a06b6ef6..e66dd889193c 100644 --- a/Source/JavaScriptCore/jit/JITOpcodes.cpp +++ b/Source/JavaScriptCore/jit/JITOpcodes.cpp @@ -1210,10 +1210,14 @@ void JIT::emit_op_catch(const JSInstruction* currentInstruction) // we updated the exception handlers to point here). Because the LLInt uses a different value // inside GPRInfo::jitDataRegister, the callee saves we restore above may not contain the correct register. // So we replenish it here. +#if CPU(ARM_THUMB2) + emitMaterializeMetadataAndConstantPoolRegisters(); +#else { loadPtr(addressFor(CallFrameSlot::codeBlock), regT0); loadPtr(Address(regT0, CodeBlock::offsetOfJITData()), GPRInfo::jitDataRegister); } +#endif callOperationNoExceptionCheck(operationRetrieveAndClearExceptionIfCatchable, TrustedImmPtr(&vm())); Jump isCatchableException = branchTest32(NonZero, returnValueGPR); diff --git a/Source/JavaScriptCore/llint/LLIntOpcode.h b/Source/JavaScriptCore/llint/LLIntOpcode.h index 1503dc64f5e1..c742db314000 100644 --- a/Source/JavaScriptCore/llint/LLIntOpcode.h +++ b/Source/JavaScriptCore/llint/LLIntOpcode.h @@ -75,4 +75,5 @@ macro(op_enumerator_get_by_val) \ macro(op_enumerator_put_by_val) \ macro(op_enumerator_in_by_val) \ + macro(op_instanceof) \ diff --git a/Source/JavaScriptCore/llint/LLIntSlowPaths.cpp b/Source/JavaScriptCore/llint/LLIntSlowPaths.cpp index da635a652120..3049c6373731 100644 --- a/Source/JavaScriptCore/llint/LLIntSlowPaths.cpp +++ b/Source/JavaScriptCore/llint/LLIntSlowPaths.cpp @@ -2728,38 +2728,22 @@ extern "C" UGPRPair SYSV_ABI llint_slow_path_checkpoint_osr_exit_from_inlined_ca } case op_instanceof: { - auto& dst = callFrame->uncheckedR(destinationFor(pc->as(), bytecodeIndex.checkpoint()).virtualRegister()); const auto& bytecode = pc->as(); - auto value = getOperand(callFrame, bytecode.m_value); - auto hasInstanceOrPrototype = JSValue::decode(result); - switch (bytecodeIndex.checkpoint()) { - case OpInstanceof::getHasInstance: { + case OpInstanceof::getHasInstance: // First one is not handled by checkpoint. + case OpInstanceof::instanceof: // No inlined calls exist at the last checkpoint. RELEASE_ASSERT_NOT_REACHED(); break; - } case OpInstanceof::getPrototype: { - auto constructor = getOperand(callFrame, bytecode.m_constructor); - ASSERT(constructor.isObject()); - if (hasInstanceOrPrototype != globalObject->functionProtoHasInstanceSymbolFunction() || !constructor.getObject()->structure()->typeInfo().implementsDefaultHasInstance()) { - dst = jsBoolean(constructor.getObject()->hasInstance(globalObject, value, hasInstanceOrPrototype)); - RETURN_IF_EXCEPTION(throwScope, { }); - break; - } - if (!value.isObject()) { - dst = jsBoolean(false); - break; - } - hasInstanceOrPrototype = constructor.get(globalObject, vm.propertyNames->prototype); - RETURN_IF_EXCEPTION(throwScope, { }); - FALLTHROUGH; - } - case OpInstanceof::instanceof: - bool result = JSObject::defaultHasInstance(globalObject, value, hasInstanceOrPrototype); + auto& dst = callFrame->uncheckedR(bytecode.m_dst); + auto value = getOperand(callFrame, bytecode.m_value); + auto prototype = JSValue::decode(result); + bool result = JSObject::defaultHasInstance(globalObject, value, prototype); RETURN_IF_EXCEPTION(throwScope, { }); dst = jsBoolean(result); break; } + } break; } diff --git a/Source/JavaScriptCore/llint/LowLevelInterpreter32_64.asm b/Source/JavaScriptCore/llint/LowLevelInterpreter32_64.asm index 39c765f5d9a1..6d72182b06c7 100644 --- a/Source/JavaScriptCore/llint/LowLevelInterpreter32_64.asm +++ b/Source/JavaScriptCore/llint/LowLevelInterpreter32_64.asm @@ -3423,10 +3423,19 @@ llintOpWithReturn(op_enumerator_put_by_val, OpEnumeratorPutByVal, macro (size, g dispatch() end) +llintOpWithReturn(op_instanceof, OpInstanceof, macro (size, get, dispatch, return) + callSlowPath(_llint_slow_path_instanceof) + dispatch() +.osrReturnPoint: + # We do not checkpoint properly on 32-bit. + getterSetterOSRExitReturnPoint(op_instanceof, size) + callSlowPath(_llint_slow_path_instanceof) + dispatch() +end) + slowPathOp(get_property_enumerator) slowPathOp(enumerator_next) slowPathOp(enumerator_has_own_property) slowPathOp(mod) llintSlowPathOp(has_structure_with_flags) -llintSlowPathOp(instanceof) diff --git a/Source/JavaScriptCore/llint/LowLevelInterpreter64.asm b/Source/JavaScriptCore/llint/LowLevelInterpreter64.asm index 78b0f6a7c78a..9d17320bb27d 100644 --- a/Source/JavaScriptCore/llint/LowLevelInterpreter64.asm +++ b/Source/JavaScriptCore/llint/LowLevelInterpreter64.asm @@ -3278,6 +3278,15 @@ llintOpWithMetadata(op_instanceof, OpInstanceof, macro (size, get, dispatch, met store(result, m_hasInstanceOrPrototype) jmp .getPrototype end) + jmp .getPrototype + +.getHasInstanceInlinedGetterOSRReturnPoint: + # This can only be reached if we're exiting to the LLInt and we're exiting + # from an inlined getter for Symbol.hasInstance. Other exits e.g. for a "prototype" + # getter will exit directly to op_checkpoint_osr_exit_from_inlined_call_trampoline. + getterSetterOSRExitReturnPoint(op_instanceof, size) + valueProfile(size, OpInstanceof, m_hasInstanceValueProfile, r0, t2) + store(r0, m_hasInstanceOrPrototype) .getPrototype: overridesHasInstance(m_hasInstanceOrPrototype, m_constructor, .instanceofCustom) diff --git a/Source/JavaScriptCore/parser/SourceTaintedOrigin.cpp b/Source/JavaScriptCore/parser/SourceTaintedOrigin.cpp index 52923690182a..47a83a9e69e7 100644 --- a/Source/JavaScriptCore/parser/SourceTaintedOrigin.cpp +++ b/Source/JavaScriptCore/parser/SourceTaintedOrigin.cpp @@ -27,6 +27,7 @@ #include "SourceTaintedOrigin.h" #include "CodeBlock.h" +#include "JSCellInlines.h" #include "StackVisitor.h" #include "VM.h" diff --git a/Source/JavaScriptCore/runtime/ExceptionScope.cpp b/Source/JavaScriptCore/runtime/ExceptionScope.cpp index 89af72bab77d..fe22f6895aec 100644 --- a/Source/JavaScriptCore/runtime/ExceptionScope.cpp +++ b/Source/JavaScriptCore/runtime/ExceptionScope.cpp @@ -28,6 +28,7 @@ #include "ErrorInstance.h" #include "Exception.h" +#include "JSCJSValueInlines.h" #include #include #include diff --git a/Source/JavaScriptCore/wasm/WasmBBQJIT.cpp b/Source/JavaScriptCore/wasm/WasmBBQJIT.cpp index 351f838ab15b..267e6b988fd9 100644 --- a/Source/JavaScriptCore/wasm/WasmBBQJIT.cpp +++ b/Source/JavaScriptCore/wasm/WasmBBQJIT.cpp @@ -4657,6 +4657,396 @@ ALWAYS_INLINE void BBQJIT::didParseOpcode() { } +BBQJIT::BranchFoldResult BBQJIT::tryFoldFusedBranchCompare(OpType opType, ExpressionType operand) +{ + if (!operand.isConst()) + return BranchNotFolded; + switch (opType) { + case OpType::I32Eqz: + return operand.asI32() ? BranchNeverTaken : BranchAlwaysTaken; + case OpType::I64Eqz: + return operand.asI64() ? BranchNeverTaken : BranchAlwaysTaken; + default: + RELEASE_ASSERT_NOT_REACHED_WITH_MESSAGE("Op type '%s' is not a unary comparison and should not have been fused.\n", makeString(opType).characters()); + } + return BranchNotFolded; +} + +BBQJIT::Jump BBQJIT::emitFusedBranchCompareBranch(OpType opType, ExpressionType, Location operandLocation) +{ + // Emit the negation of the intended branch. + switch (opType) { + case OpType::I32Eqz: + return m_jit.branchTest32(ResultCondition::NonZero, operandLocation.asGPR()); + case OpType::I64Eqz: +#if USE(JSVALUE64) + return m_jit.branchTest64(ResultCondition::NonZero, operandLocation.asGPR()); +#else + return m_jit.branchTest64(ResultCondition::NonZero, operandLocation.asGPRhi(), operandLocation.asGPRlo()); +#endif + default: + RELEASE_ASSERT_NOT_REACHED_WITH_MESSAGE("Op type '%s' is not a unary comparison and should not have been fused.\n", makeString(opType).characters()); + } +} + +PartialResult BBQJIT::addFusedBranchCompare(OpType opType, ControlType& target, ExpressionType operand, Stack& results) +{ + ASSERT(!operand.isNone()); + + switch (tryFoldFusedBranchCompare(opType, operand)) { + case BranchNeverTaken: + return { }; + case BranchAlwaysTaken: + currentControlData().flushAndSingleExit(*this, target, results, false, false); + target.addBranch(m_jit.jump()); + return { }; + case BranchNotFolded: + break; + } + + { + // Like in normal addBranch(), we can directly use the operand location + // because it shouldn't interfere with flushAtBlockBoundary(). + Location operandLocation = loadIfNecessary(operand); + consume(operand); + + LOG_INSTRUCTION("BranchCompare", makeString(opType).characters(), operand, operandLocation); + + currentControlData().flushAtBlockBoundary(*this, 0, results, false); + Jump ifNotTaken = emitFusedBranchCompareBranch(opType, operand, operandLocation); + currentControlData().addExit(*this, target.targetLocations(), results); + target.addBranch(m_jit.jump()); + ifNotTaken.link(&m_jit); + currentControlData().finalizeBlock(*this, target.targetLocations().size(), results, true); + } + + return { }; +} + +PartialResult WARN_UNUSED_RETURN BBQJIT::addFusedIfCompare(OpType op, ExpressionType operand, BlockSignature signature, Stack& enclosingStack, ControlData& result, Stack& newStack) +{ + BranchFoldResult foldResult = tryFoldFusedBranchCompare(op, operand); + + ScratchScope<0, 1> scratches(*this); + Location operandLocation; + RegisterSet liveScratchGPRs, liveScratchFPRs; + if (foldResult == BranchNotFolded) { + if (!operand.isConst()) + operandLocation = loadIfNecessary(operand); + else if (operand.isFloat()) { + operandLocation = Location::fromFPR(scratches.fpr(0)); + emitMove(operand, operandLocation); + } + + if (operandLocation.isGPR()) + liveScratchGPRs.add(operandLocation.asGPR(), IgnoreVectors); + else if (operandLocation.isGPR2()) { + liveScratchGPRs.add(operandLocation.asGPRlo(), IgnoreVectors); + liveScratchGPRs.add(operandLocation.asGPRhi(), IgnoreVectors); + } else if (operandLocation.isFPR()) + liveScratchFPRs.add(operandLocation.asFPR(), operand.type() == TypeKind::V128 ? Width128 : Width64); + } + if (!liveScratchFPRs.contains(scratches.fpr(0), IgnoreVectors)) + scratches.unbindEarly(); + + consume(operand); + + result = ControlData(*this, BlockType::If, signature, currentControlData().enclosedHeight() + currentControlData().implicitSlots() + enclosingStack.size() - signature.m_signature->argumentCount(), liveScratchGPRs, liveScratchFPRs); + + // Despite being conditional, if doesn't need to worry about diverging expression stacks at block boundaries, so it doesn't need multiple exits. + currentControlData().flushAndSingleExit(*this, result, enclosingStack, true, false); + + LOG_INSTRUCTION("IfCompare", makeString(op).characters(), *signature.m_signature, operand, operandLocation); + LOG_INDENT(); + splitStack(signature, enclosingStack, newStack); + + result.startBlock(*this, newStack); + if (foldResult == BranchNeverTaken) + result.setIfBranch(m_jit.jump()); // Emit direct branch if we know the condition is false. + else if (foldResult == BranchNotFolded) // Otherwise, we only emit a branch at all if we don't know the condition statically. + result.setIfBranch(emitFusedBranchCompareBranch(op, operand, operandLocation)); + return { }; +} + +BBQJIT::BranchFoldResult BBQJIT::tryFoldFusedBranchCompare(OpType opType, ExpressionType left, ExpressionType right) +{ + if (!left.isConst() || !right.isConst()) + return BranchNotFolded; + switch (opType) { + case OpType::I32LtS: + return left.asI32() < right.asI32() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I32LtU: + return static_cast(left.asI32()) < static_cast(right.asI32()) ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I32GtS: + return left.asI32() > right.asI32() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I32GtU: + return static_cast(left.asI32()) > static_cast(right.asI32()) ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I32LeS: + return left.asI32() <= right.asI32() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I32LeU: + return static_cast(left.asI32()) <= static_cast(right.asI32()) ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I32GeS: + return left.asI32() >= right.asI32() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I32GeU: + return static_cast(left.asI32()) >= static_cast(right.asI32()) ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I32Eq: + return left.asI32() == right.asI32() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I32Ne: + return left.asI32() != right.asI32() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I64LtS: + return left.asI64() < right.asI64() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I64LtU: + return static_cast(left.asI64()) < static_cast(right.asI64()) ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I64GtS: + return left.asI64() > right.asI64() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I64GtU: + return static_cast(left.asI64()) > static_cast(right.asI64()) ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I64LeS: + return left.asI64() <= right.asI64() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I64LeU: + return static_cast(left.asI64()) <= static_cast(right.asI64()) ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I64GeS: + return left.asI64() >= right.asI64() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I64GeU: + return static_cast(left.asI64()) >= static_cast(right.asI64()) ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I64Eq: + return left.asI64() == right.asI64() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I64Ne: + return left.asI64() != right.asI64() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::F32Lt: + return left.asF32() < right.asF32() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::F32Gt: + return left.asF32() > right.asF32() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::F32Le: + return left.asF32() <= right.asF32() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::F32Ge: + return left.asF32() >= right.asF32() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::F32Eq: + return left.asF32() == right.asF32() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::F32Ne: + return left.asF32() != right.asF32() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::F64Lt: + return left.asF64() < right.asF64() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::F64Gt: + return left.asF64() > right.asF64() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::F64Le: + return left.asF64() <= right.asF64() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::F64Ge: + return left.asF64() >= right.asF64() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::F64Eq: + return left.asF64() == right.asF64() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::F64Ne: + return left.asF64() != right.asF64() ? BranchAlwaysTaken : BranchNeverTaken; + default: + RELEASE_ASSERT_NOT_REACHED_WITH_MESSAGE("Op type '%s' is not a binary comparison and should not have been fused.\n", makeString(opType).characters()); + } +} + +static MacroAssembler::Jump emitBranchI32(CCallHelpers& jit, MacroAssembler::RelationalCondition condition, Value left, Location leftLocation, Value right, Location rightLocation) +{ + if (right.isConst()) + return jit.branch32(condition, leftLocation.asGPR(), MacroAssembler::TrustedImm32(right.asI32())); + if (left.isConst()) + return jit.branch32(condition, MacroAssembler::TrustedImm32(left.asI32()), rightLocation.asGPR()); + return jit.branch32(condition, leftLocation.asGPR(), rightLocation.asGPR()); +} + +static MacroAssembler::Jump emitBranchI64(CCallHelpers& jit, MacroAssembler::RelationalCondition condition, Value left, Location leftLocation, Value right, Location rightLocation) +{ +#if USE(JSVALUE64) + if (right.isConst()) + return jit.branch64(condition, leftLocation.asGPR(), MacroAssembler::Imm64(right.asI64())); + if (left.isConst()) + return jit.branch64(MacroAssembler::commute(condition), rightLocation.asGPR(), MacroAssembler::Imm64(left.asI64())); + return jit.branch64(condition, leftLocation.asGPR(), rightLocation.asGPR()); +#else + if (right.isConst()) + return jit.branch64(condition, leftLocation.asGPRhi(), leftLocation.asGPRlo(), MacroAssembler::TrustedImm64(right.asI64())); + if (left.isConst()) + return jit.branch64(MacroAssembler::commute(condition), rightLocation.asGPRhi(), rightLocation.asGPRlo(), MacroAssembler::TrustedImm64(left.asI64())); + return jit.branch64(condition, leftLocation.asGPRhi(), leftLocation.asGPRlo(), rightLocation.asGPRhi(), rightLocation.asGPRlo()); +#endif +} + +static MacroAssembler::Jump emitBranchF32(CCallHelpers& jit, MacroAssembler::DoubleCondition condition, Value, Location leftLocation, Value, Location rightLocation) +{ + return jit.branchFloat(condition, leftLocation.asFPR(), rightLocation.asFPR()); +} + +static MacroAssembler::Jump emitBranchF64(CCallHelpers& jit, MacroAssembler::DoubleCondition condition, Value, Location leftLocation, Value, Location rightLocation) +{ + return jit.branchDouble(condition, leftLocation.asFPR(), rightLocation.asFPR()); +} + +BBQJIT::Jump BBQJIT::emitFusedBranchCompareBranch(OpType opType, ExpressionType left, Location leftLocation, ExpressionType right, Location rightLocation) +{ + // Emit a branch with the inverse of the comparison. We're generating the "branch-if-false" case. + switch (opType) { + case OpType::I32LtS: + return emitBranchI32(m_jit, RelationalCondition::GreaterThanOrEqual, left, leftLocation, right, rightLocation); + case OpType::I32LtU: + return emitBranchI32(m_jit, RelationalCondition::AboveOrEqual, left, leftLocation, right, rightLocation); + case OpType::I32GtS: + return emitBranchI32(m_jit, RelationalCondition::LessThanOrEqual, left, leftLocation, right, rightLocation); + case OpType::I32GtU: + return emitBranchI32(m_jit, RelationalCondition::BelowOrEqual, left, leftLocation, right, rightLocation); + case OpType::I32LeS: + return emitBranchI32(m_jit, RelationalCondition::GreaterThan, left, leftLocation, right, rightLocation); + case OpType::I32LeU: + return emitBranchI32(m_jit, RelationalCondition::Above, left, leftLocation, right, rightLocation); + case OpType::I32GeS: + return emitBranchI32(m_jit, RelationalCondition::LessThan, left, leftLocation, right, rightLocation); + case OpType::I32GeU: + return emitBranchI32(m_jit, RelationalCondition::Below, left, leftLocation, right, rightLocation); + case OpType::I32Eq: + return emitBranchI32(m_jit, RelationalCondition::NotEqual, left, leftLocation, right, rightLocation); + case OpType::I32Ne: + return emitBranchI32(m_jit, RelationalCondition::Equal, left, leftLocation, right, rightLocation); + case OpType::I64LtS: + return emitBranchI64(m_jit, RelationalCondition::GreaterThanOrEqual, left, leftLocation, right, rightLocation); + case OpType::I64LtU: + return emitBranchI64(m_jit, RelationalCondition::AboveOrEqual, left, leftLocation, right, rightLocation); + case OpType::I64GtS: + return emitBranchI64(m_jit, RelationalCondition::LessThanOrEqual, left, leftLocation, right, rightLocation); + case OpType::I64GtU: + return emitBranchI64(m_jit, RelationalCondition::BelowOrEqual, left, leftLocation, right, rightLocation); + case OpType::I64LeS: + return emitBranchI64(m_jit, RelationalCondition::GreaterThan, left, leftLocation, right, rightLocation); + case OpType::I64LeU: + return emitBranchI64(m_jit, RelationalCondition::Above, left, leftLocation, right, rightLocation); + case OpType::I64GeS: + return emitBranchI64(m_jit, RelationalCondition::LessThan, left, leftLocation, right, rightLocation); + case OpType::I64GeU: + return emitBranchI64(m_jit, RelationalCondition::Below, left, leftLocation, right, rightLocation); + case OpType::I64Eq: + return emitBranchI64(m_jit, RelationalCondition::NotEqual, left, leftLocation, right, rightLocation); + case OpType::I64Ne: + return emitBranchI64(m_jit, RelationalCondition::Equal, left, leftLocation, right, rightLocation); + case OpType::F32Lt: + return emitBranchF32(m_jit, MacroAssembler::invert(DoubleCondition::DoubleLessThanAndOrdered), left, leftLocation, right, rightLocation); + case OpType::F32Gt: + return emitBranchF32(m_jit, MacroAssembler::invert(DoubleCondition::DoubleGreaterThanAndOrdered), left, leftLocation, right, rightLocation); + case OpType::F32Le: + return emitBranchF32(m_jit, MacroAssembler::invert(DoubleCondition::DoubleLessThanOrEqualAndOrdered), left, leftLocation, right, rightLocation); + case OpType::F32Ge: + return emitBranchF32(m_jit, MacroAssembler::invert(DoubleCondition::DoubleGreaterThanOrEqualAndOrdered), left, leftLocation, right, rightLocation); + case OpType::F32Eq: + return emitBranchF32(m_jit, MacroAssembler::invert(DoubleCondition::DoubleEqualAndOrdered), left, leftLocation, right, rightLocation); + case OpType::F32Ne: + return emitBranchF32(m_jit, MacroAssembler::invert(DoubleCondition::DoubleNotEqualOrUnordered), left, leftLocation, right, rightLocation); + case OpType::F64Lt: + return emitBranchF64(m_jit, MacroAssembler::invert(DoubleCondition::DoubleLessThanAndOrdered), left, leftLocation, right, rightLocation); + case OpType::F64Gt: + return emitBranchF64(m_jit, MacroAssembler::invert(DoubleCondition::DoubleGreaterThanAndOrdered), left, leftLocation, right, rightLocation); + case OpType::F64Le: + return emitBranchF64(m_jit, MacroAssembler::invert(DoubleCondition::DoubleLessThanOrEqualAndOrdered), left, leftLocation, right, rightLocation); + case OpType::F64Ge: + return emitBranchF64(m_jit, MacroAssembler::invert(DoubleCondition::DoubleGreaterThanOrEqualAndOrdered), left, leftLocation, right, rightLocation); + case OpType::F64Eq: + return emitBranchF64(m_jit, MacroAssembler::invert(DoubleCondition::DoubleEqualAndOrdered), left, leftLocation, right, rightLocation); + case OpType::F64Ne: + return emitBranchF64(m_jit, MacroAssembler::invert(DoubleCondition::DoubleNotEqualOrUnordered), left, leftLocation, right, rightLocation); + default: + RELEASE_ASSERT_NOT_REACHED_WITH_MESSAGE("Op type '%s' is not a binary comparison and should not have been fused.\n", makeString(opType).characters()); + } +} + +PartialResult BBQJIT::addFusedBranchCompare(OpType opType, ControlType& target, ExpressionType left, ExpressionType right, Stack& results) +{ + switch (tryFoldFusedBranchCompare(opType, left, right)) { + case BranchNeverTaken: + return { }; + case BranchAlwaysTaken: + currentControlData().flushAndSingleExit(*this, target, results, false, false); + target.addBranch(m_jit.jump()); + return { }; + case BranchNotFolded: + break; + } + + { + Location leftLocation, rightLocation; + + if (!left.isConst()) + leftLocation = loadIfNecessary(left); + else if (left.isFloat()) // Materialize floats here too, since they don't have a good immediate lowering. + emitMove(left, leftLocation = Location::fromFPR(wasmScratchFPR)); + if (!right.isConst()) + rightLocation = loadIfNecessary(right); + else if (right.isFloat()) + emitMove(right, rightLocation = Location::fromFPR(wasmScratchFPR)); + + consume(left); + consume(right); + + LOG_INSTRUCTION("BranchCompare", makeString(opType).characters(), left, leftLocation, right, rightLocation); + + currentControlData().flushAtBlockBoundary(*this, 0, results, false); + Jump ifNotTaken = emitFusedBranchCompareBranch(opType, left, leftLocation, right, rightLocation); + currentControlData().addExit(*this, target.targetLocations(), results); + target.addBranch(m_jit.jump()); + if (ifNotTaken.isSet()) + ifNotTaken.link(&m_jit); + currentControlData().finalizeBlock(*this, target.targetLocations().size(), results, true); + } + + return { }; +} + +PartialResult WARN_UNUSED_RETURN BBQJIT::addFusedIfCompare(OpType op, ExpressionType left, ExpressionType right, BlockSignature signature, Stack& enclosingStack, ControlData& result, Stack& newStack) +{ + BranchFoldResult foldResult = tryFoldFusedBranchCompare(op, left, right); + + Location leftLocation, rightLocation; + RegisterSet liveScratchGPRs, liveScratchFPRs; + if (foldResult == BranchNotFolded) { + ASSERT(!left.isConst() || !right.isConst()); // If they're both constants, we should have folded. + + if (!left.isConst()) + leftLocation = loadIfNecessary(left); + else if (left.isFloat()) + emitMove(left, leftLocation = Location::fromFPR(wasmScratchFPR)); + if (leftLocation.isGPR()) + liveScratchGPRs.add(leftLocation.asGPR(), IgnoreVectors); + else if (leftLocation.isGPR2()) { + liveScratchGPRs.add(leftLocation.asGPRlo(), IgnoreVectors); + liveScratchGPRs.add(leftLocation.asGPRhi(), IgnoreVectors); + } else if (leftLocation.isFPR()) + liveScratchFPRs.add(leftLocation.asFPR(), left.type() == TypeKind::V128 ? Width128 : Width64); + + if (!right.isConst()) + rightLocation = loadIfNecessary(right); + else if (right.isFloat()) + emitMove(right, rightLocation = Location::fromFPR(wasmScratchFPR)); + if (rightLocation.isGPR()) + liveScratchGPRs.add(rightLocation.asGPR(), IgnoreVectors); + else if (rightLocation.isGPR2()) { + liveScratchGPRs.add(rightLocation.asGPRlo(), IgnoreVectors); + liveScratchGPRs.add(rightLocation.asGPRhi(), IgnoreVectors); + } else if (rightLocation.isFPR()) + liveScratchFPRs.add(rightLocation.asFPR(), right.type() == TypeKind::V128 ? Width128 : Width64); + } + consume(left); + consume(right); + + + result = ControlData(*this, BlockType::If, signature, currentControlData().enclosedHeight() + currentControlData().implicitSlots() + enclosingStack.size() - signature.m_signature->argumentCount(), liveScratchGPRs, liveScratchFPRs); + + // Despite being conditional, if doesn't need to worry about diverging expression stacks at block boundaries, so it doesn't need multiple exits. + currentControlData().flushAndSingleExit(*this, result, enclosingStack, true, false); + + LOG_INSTRUCTION("IfCompare", makeString(op).characters(), *signature.m_signature, left, leftLocation, right, rightLocation); + LOG_INDENT(); + splitStack(signature, enclosingStack, newStack); + + result.startBlock(*this, newStack); + if (foldResult == BranchNeverTaken) + result.setIfBranch(m_jit.jump()); // Emit direct branch if we know the condition is false. + else if (foldResult == BranchNotFolded) // Otherwise, we only emit a branch at all if we don't know the condition statically. + result.setIfBranch(emitFusedBranchCompareBranch(op, left, leftLocation, right, rightLocation)); + return { }; +} + // SIMD bool BBQJIT::usesSIMD() diff --git a/Source/JavaScriptCore/wasm/WasmBBQJIT.h b/Source/JavaScriptCore/wasm/WasmBBQJIT.h index ea45678cd87c..ae7980168119 100644 --- a/Source/JavaScriptCore/wasm/WasmBBQJIT.h +++ b/Source/JavaScriptCore/wasm/WasmBBQJIT.h @@ -861,8 +861,8 @@ class BBQJIT { #define LOG_DEDENT() do { if (UNLIKELY(Options::verboseBBQJITInstructions())) { m_loggingIndent -= 2; } } while (false); public: - // FIXME: Support fused branch compare on 32-bit platforms. - static constexpr bool shouldFuseBranchCompare = is64Bit(); + // Enable fused branch compare for all platforms + static constexpr bool shouldFuseBranchCompare = true; static constexpr bool tierSupportsSIMD = true; static constexpr bool validateFunctionBodySize = true; @@ -1163,7 +1163,6 @@ class BBQJIT { FloatingPointRange lookupTruncationRange(TruncationKind truncationKind); void truncInBounds(TruncationKind truncationKind, Location operandLocation, Location resultLocation, FPRReg scratch1FPR, FPRReg scratch2FPR); - void truncInBounds(TruncationKind truncationKind, Location operandLocation, Value& result, Location resultLocation); PartialResult WARN_UNUSED_RETURN truncTrapping(OpType truncationOp, Value operand, Value& result, Type returnType, Type operandType); PartialResult WARN_UNUSED_RETURN truncSaturated(Ext1OpType truncationOp, Value operand, Value& result, Type returnType, Type operandType); @@ -2248,9 +2247,6 @@ class BBQJIT { constexpr static int tempSlotSize = 16; // Size of the stack slot for a stack temporary. Currently the size of the largest possible temporary (a v128). - enum class ShiftI64HelperOp { Lshift, Urshift, Rshift }; - void shiftI64Helper(ShiftI64HelperOp op, Location lhsLocation, Location rhsLocation, Location resultLocation); - enum class RotI64HelperOp { Left, Right }; void rotI64Helper(RotI64HelperOp op, Location lhsLocation, Location rhsLocation, Location resultLocation); diff --git a/Source/JavaScriptCore/wasm/WasmBBQJIT32_64.cpp b/Source/JavaScriptCore/wasm/WasmBBQJIT32_64.cpp index d7d42ccd063d..5b6f1381c5b5 100644 --- a/Source/JavaScriptCore/wasm/WasmBBQJIT32_64.cpp +++ b/Source/JavaScriptCore/wasm/WasmBBQJIT32_64.cpp @@ -477,14 +477,14 @@ PartialResult WARN_UNUSED_RETURN BBQJIT::load(LoadOpType loadOp, Value pointer, m_jit.rshift32(resultLocation.asGPRlo(), TrustedImm32(31), resultLocation.asGPRhi()); break; case LoadOpType::I64Load: - m_jit.loadPair32(location, resultLocation.asGPRlo(), resultLocation.asGPRhi()); + m_jit.loadPair32Unaligned(location, resultLocation.asGPRlo(), resultLocation.asGPRhi()); break; case LoadOpType::F32Load: m_jit.load32(location, wasmScratchGPR); m_jit.move32ToFloat(wasmScratchGPR, resultLocation.asFPR()); break; case LoadOpType::F64Load: - m_jit.loadPair32(location, wasmScratchGPR, wasmScratchGPR2); + m_jit.loadPair32Unaligned(location, wasmScratchGPR, wasmScratchGPR2); m_jit.move64ToDouble(wasmScratchGPR2, wasmScratchGPR, resultLocation.asFPR()); break; } @@ -513,42 +513,57 @@ PartialResult WARN_UNUSED_RETURN BBQJIT::store(StoreOpType storeOp, Value pointe ScratchScope<0, 1> scratches(*this); valueLocation = Location::fromFPR(scratches.fpr(0)); emitMoveConst(value, valueLocation); - } else if (value.isConst() && typeNeedsGPR2(value.type())) { - ScratchScope<2, 0> scratches(*this); - valueLocation = Location::fromGPR2(scratches.gpr(1), scratches.gpr(0)); - emitMoveConst(value, valueLocation); - } else if (value.isConst()) { - ScratchScope<1, 0> scratches(*this); - valueLocation = Location::fromGPR(scratches.gpr(0)); - emitMoveConst(value, valueLocation); - } else + } else if (!value.isConst()) { valueLocation = loadIfNecessary(value); - ASSERT(valueLocation.isRegister()); + ASSERT(valueLocation.isRegister()); + } consume(value); consume(pointer); switch (storeOp) { case StoreOpType::I64Store8: - m_jit.store8(valueLocation.asGPRlo(), location); + if (value.isConst()) + m_jit.store8(TrustedImm32(static_cast(value.asI64())), location); + else + m_jit.store8(valueLocation.asGPRlo(), location); return; case StoreOpType::I32Store8: - m_jit.store8(valueLocation.asGPR(), location); + if (value.isConst()) + m_jit.store8(TrustedImm32(value.asI32()), location); + else + m_jit.store8(valueLocation.asGPR(), location); return; case StoreOpType::I64Store16: - m_jit.store16(valueLocation.asGPRlo(), location); + if (value.isConst()) + m_jit.store16(TrustedImm32(static_cast(value.asI64())), location); + else + m_jit.store16(valueLocation.asGPRlo(), location); return; case StoreOpType::I32Store16: - m_jit.store16(valueLocation.asGPR(), location); + if (value.isConst()) + m_jit.store16(TrustedImm32(value.asI32()), location); + else + m_jit.store16(valueLocation.asGPR(), location); return; case StoreOpType::I64Store32: - m_jit.store32(valueLocation.asGPRlo(), location); + if (value.isConst()) + m_jit.store32(TrustedImm32(static_cast(value.asI64())), location); + else + m_jit.store32(valueLocation.asGPRlo(), location); return; case StoreOpType::I32Store: - m_jit.store32(valueLocation.asGPR(), location); + if (value.isConst()) + m_jit.store32(TrustedImm32(value.asI32()), location); + else + m_jit.store32(valueLocation.asGPR(), location); return; case StoreOpType::I64Store: - m_jit.storePair32(valueLocation.asGPRlo(), valueLocation.asGPRhi(), location); + if (value.isConst()) { + int64_t val = value.asI64(); + m_jit.storePair32(TrustedImm32(static_cast(val)), TrustedImm32(static_cast(val >> 32)), location); + } else + m_jit.storePair32(valueLocation.asGPRlo(), valueLocation.asGPRhi(), location); return; case StoreOpType::F32Store: { ScratchScope<1, 0> scratches(*this); @@ -1161,7 +1176,7 @@ Value WARN_UNUSED_RETURN BBQJIT::emitAtomicCompareExchange(ExtAtomicOpType op, T return result; } -void BBQJIT::truncInBounds(TruncationKind truncationKind, Location operandLocation, Value& result, Location resultLocation) +void BBQJIT::truncInBounds(TruncationKind truncationKind, Location operandLocation, Location resultLocation, FPRReg scratch1FPR, FPRReg scratch2FPR) { switch (truncationKind) { case TruncationKind::I32TruncF32S: @@ -1177,27 +1192,19 @@ void BBQJIT::truncInBounds(TruncationKind truncationKind, Location operandLocati m_jit.truncateDoubleToUint32(operandLocation.asFPR(), resultLocation.asGPR()); break; case TruncationKind::I64TruncF32S: { - auto operand = Value::pinned(TypeKind::F32, operandLocation); - consume(result); - emitCCall(Math::i64_trunc_s_f32, ArgumentList { operand }, result); + m_jit.truncateFloatToInt64(operandLocation.asFPR(), resultLocation.asGPRlo(), resultLocation.asGPRhi(), scratch1FPR, scratch2FPR); break; } case TruncationKind::I64TruncF64S: { - auto operand = Value::pinned(TypeKind::F64, operandLocation); - consume(result); - emitCCall(Math::i64_trunc_s_f64, ArgumentList { operand }, result); + m_jit.truncateDoubleToInt64(operandLocation.asFPR(), resultLocation.asGPRlo(), resultLocation.asGPRhi(), scratch1FPR, scratch2FPR); break; } case TruncationKind::I64TruncF32U: { - auto operand = Value::pinned(TypeKind::F32, operandLocation); - consume(result); - emitCCall(Math::i64_trunc_u_f32, ArgumentList { operand }, result); + m_jit.truncateFloatToUint64(operandLocation.asFPR(), resultLocation.asGPRlo(), resultLocation.asGPRhi(), scratch1FPR, scratch2FPR); break; } case TruncationKind::I64TruncF64U: { - auto operand = Value::pinned(TypeKind::F64, operandLocation); - consume(result); - emitCCall(Math::i64_trunc_u_f64, ArgumentList { operand }, result); + m_jit.truncateDoubleToUint64(operandLocation.asFPR(), resultLocation.asGPRlo(), resultLocation.asGPRhi(), scratch1FPR, scratch2FPR); break; } } @@ -1231,7 +1238,7 @@ PartialResult WARN_UNUSED_RETURN BBQJIT::truncTrapping(OpType truncationOp, Valu emitMoveConst(minFloatConst, minFloat); emitMoveConst(maxFloatConst, maxFloat); - LOG_INSTRUCTION("TruncSaturated", operand, operandLocation, RESULT(result)); + LOG_INSTRUCTION("TruncTrapping", operand, operandLocation, RESULT(result)); DoubleCondition minCondition = range.closedLowerEndpoint ? DoubleCondition::DoubleLessThanOrUnordered : DoubleCondition::DoubleLessThanOrEqualOrUnordered; Jump belowMin = operandType == Types::F32 @@ -1244,7 +1251,7 @@ PartialResult WARN_UNUSED_RETURN BBQJIT::truncTrapping(OpType truncationOp, Valu : m_jit.branchDouble(DoubleCondition::DoubleGreaterThanOrEqualOrUnordered, operandLocation.asFPR(), maxFloat.asFPR()); throwExceptionIf(ExceptionType::OutOfBoundsTrunc, aboveMax); - truncInBounds(kind, operandLocation, result, resultLocation); + truncInBounds(kind, operandLocation, resultLocation, scratches.fpr(0), scratches.fpr(1)); return { }; } @@ -1314,7 +1321,7 @@ PartialResult WARN_UNUSED_RETURN BBQJIT::truncSaturated(Ext1OpType truncationOp, : m_jit.branchDouble(DoubleCondition::DoubleGreaterThanOrEqualOrUnordered, operandLocation.asFPR(), maxFloat.asFPR()); // In-bounds case. Emit normal truncation instructions. - truncInBounds(kind, operandLocation, result, resultLocation); + truncInBounds(kind, operandLocation, resultLocation, scratches.fpr(0), scratches.fpr(1)); resultLocation = locationOf(result); Jump afterInBounds = m_jit.jump(); @@ -2250,120 +2257,132 @@ PartialResult WARN_UNUSED_RETURN BBQJIT::addI64Or(Value lhs, Value rhs, Value& r PartialResult WARN_UNUSED_RETURN BBQJIT::addI64Shl(Value lhs, Value rhs, Value& result) { - PREPARE_FOR_SHIFT; + auto emitI64Shl = [&](Location lhsLocation, Location rhsLocation, Location resultLocation) { + ScratchScope<2, 0> scratches(*this, lhsLocation, rhsLocation, resultLocation); + + auto shiftReg = rhsLocation.asGPRlo(); + auto resultLo = resultLocation.asGPRlo(); + auto resultHi = resultLocation.asGPRhi(); + auto lhsLo = lhsLocation.asGPRlo(); + auto lhsHi = lhsLocation.asGPRhi(); + + auto shift = scratches.gpr(0); + auto tmp = scratches.gpr(1); + + m_jit.and32(TrustedImm32(63), shiftReg, shift); + + m_jit.sub32(shift, TrustedImm32(32), tmp); + m_jit.lshiftUnchecked(lhsHi, shift, resultHi); + m_jit.lshiftUnchecked(lhsLo, tmp, tmp); + m_jit.or32(resultHi, tmp, resultHi); + + m_jit.sub32(TrustedImm32(32), shift, tmp); + m_jit.urshiftUnchecked(lhsLo, tmp, tmp); + m_jit.or32(resultHi, tmp, resultHi); + m_jit.lshiftUnchecked(lhsLo, shift, resultLo); + }; + EMIT_BINARY( "I64Shl", TypeKind::I64, BLOCK(Value::fromI64(lhs.asI64() << rhs.asI64())), BLOCK( - shiftI64Helper(ShiftI64HelperOp::Lshift, lhsLocation, rhsLocation, resultLocation); + emitI64Shl(lhsLocation, rhsLocation, resultLocation); ), BLOCK( ImmHelpers::immLocation(lhsLocation, rhsLocation) = Location::fromGPR2(wasmScratchGPR, wasmScratchGPR2); emitMoveConst(ImmHelpers::imm(lhs, rhs), Location::fromGPR2(wasmScratchGPR, wasmScratchGPR2)); - shiftI64Helper(ShiftI64HelperOp::Lshift, lhsLocation, rhsLocation, resultLocation); + emitI64Shl(lhsLocation, rhsLocation, resultLocation); ) ); } PartialResult WARN_UNUSED_RETURN BBQJIT::addI64ShrS(Value lhs, Value rhs, Value& result) { - PREPARE_FOR_SHIFT; + auto emitI64ShrS = [&](Location lhsLocation, Location rhsLocation, Location resultLocation) { + ScratchScope<2, 0> scratches(*this, lhsLocation, rhsLocation, resultLocation); + + auto shiftReg = rhsLocation.asGPRlo(); + auto resultLo = resultLocation.asGPRlo(); + auto resultHi = resultLocation.asGPRhi(); + auto lhsLo = lhsLocation.asGPRlo(); + auto lhsHi = lhsLocation.asGPRhi(); + + auto shift = scratches.gpr(0); + auto tmp = scratches.gpr(1); + + m_jit.and32(TrustedImm32(63), shiftReg, shift); + + m_jit.urshiftUnchecked(lhsLo, shift, resultLo); + + m_jit.sub32(TrustedImm32(32), shift, tmp); + m_jit.lshiftUnchecked(lhsHi, tmp, tmp); + m_jit.or32(tmp, resultLo); + + m_jit.sub32(shift, TrustedImm32(32), tmp); + m_jit.rshiftUnchecked(lhsHi, tmp, tmp); + m_jit.or32(resultLo, tmp, tmp); + m_jit.moveConditionally32(RelationalCondition::AboveOrEqual, shift, TrustedImm32(32), tmp, resultLo, resultLo); + + m_jit.rshiftUnchecked(lhsHi, shift, resultHi); + }; + EMIT_BINARY( "I64ShrS", TypeKind::I64, BLOCK(Value::fromI64(lhs.asI64() >> rhs.asI64())), BLOCK( - shiftI64Helper(ShiftI64HelperOp::Rshift, lhsLocation, rhsLocation, resultLocation); + emitI64ShrS(lhsLocation, rhsLocation, resultLocation); ), BLOCK( ImmHelpers::immLocation(lhsLocation, rhsLocation) = Location::fromGPR2(wasmScratchGPR, wasmScratchGPR2); emitMoveConst(ImmHelpers::imm(lhs, rhs), Location::fromGPR2(wasmScratchGPR, wasmScratchGPR2)); - shiftI64Helper(ShiftI64HelperOp::Rshift, lhsLocation, rhsLocation, resultLocation); + emitI64ShrS(lhsLocation, rhsLocation, resultLocation); ) ); } PartialResult WARN_UNUSED_RETURN BBQJIT::addI64ShrU(Value lhs, Value rhs, Value& result) { - PREPARE_FOR_SHIFT; + auto emitI64ShrU = [&](Location lhsLocation, Location rhsLocation, Location resultLocation) { + ScratchScope<2, 0> scratches(*this, lhsLocation, rhsLocation, resultLocation); + + auto shiftReg = rhsLocation.asGPRlo(); + auto resultLo = resultLocation.asGPRlo(); + auto resultHi = resultLocation.asGPRhi(); + auto lhsLo = lhsLocation.asGPRlo(); + auto lhsHi = lhsLocation.asGPRhi(); + + auto shift = scratches.gpr(0); + auto tmp = scratches.gpr(1); + + m_jit.and32(TrustedImm32(63), shiftReg, shift); + + m_jit.urshiftUnchecked(lhsLo, shift, resultLo); + + m_jit.sub32(TrustedImm32(32), shift, tmp); + m_jit.lshiftUnchecked(lhsHi, tmp, tmp); + m_jit.or32(tmp, resultLo); + + m_jit.sub32(shift, TrustedImm32(32), tmp); + m_jit.urshiftUnchecked(lhsHi, tmp, tmp); + m_jit.or32(tmp, resultLo); + + m_jit.urshiftUnchecked(lhsHi, shift, resultHi); + }; + EMIT_BINARY( "I64ShrU", TypeKind::I64, BLOCK(Value::fromI64(static_cast(lhs.asI64()) >> static_cast(rhs.asI64()))), BLOCK( - shiftI64Helper(ShiftI64HelperOp::Urshift, lhsLocation, rhsLocation, resultLocation); + emitI64ShrU(lhsLocation, rhsLocation, resultLocation); ), BLOCK( ImmHelpers::immLocation(lhsLocation, rhsLocation) = Location::fromGPR2(wasmScratchGPR, wasmScratchGPR2); emitMoveConst(ImmHelpers::imm(lhs, rhs), Location::fromGPR2(wasmScratchGPR, wasmScratchGPR2)); - shiftI64Helper(ShiftI64HelperOp::Urshift, lhsLocation, rhsLocation, resultLocation); + emitI64ShrU(lhsLocation, rhsLocation, resultLocation); ) ); } -void BBQJIT::shiftI64Helper(ShiftI64HelperOp op, Location lhsLocation, Location rhsLocation, Location resultLocation) -{ - ScratchScope<1, 0> scratches(*this, lhsLocation, rhsLocation, resultLocation); - - auto shift = rhsLocation.asGPRlo(); - m_jit.and32(TrustedImm32(63), rhsLocation.asGPRlo(), shift); - auto zero = m_jit.branch32(RelationalCondition::Equal, shift, TrustedImm32(0)); - auto aboveOrEqual32 = m_jit.branch32(RelationalCondition::AboveOrEqual, shift, TrustedImm32(32)); - - // shift < 32 - auto carry = scratches.gpr(0); - m_jit.move(TrustedImm32(32), carry); - m_jit.sub32(carry, shift, carry); - if (op == ShiftI64HelperOp::Lshift) { - ASSERT(resultLocation.asGPRhi() != shift); - ASSERT(resultLocation.asGPRhi() != lhsLocation.asGPRlo()); - m_jit.lshift32(lhsLocation.asGPRhi(), shift, resultLocation.asGPRhi()); - m_jit.urshift32(lhsLocation.asGPRlo(), carry, carry); - m_jit.or32(carry, resultLocation.asGPRhi()); - m_jit.lshift32(lhsLocation.asGPRlo(), shift, resultLocation.asGPRlo()); - } else if (op == ShiftI64HelperOp::Urshift) { - m_jit.lshift32(lhsLocation.asGPRhi(), carry, carry); - ASSERT(resultLocation.asGPRhi() != shift); - ASSERT(resultLocation.asGPRhi() != lhsLocation.asGPRlo()); - m_jit.urshift32(lhsLocation.asGPRhi(), shift, resultLocation.asGPRhi()); - m_jit.urshift32(lhsLocation.asGPRlo(), shift, resultLocation.asGPRlo()); - m_jit.or32(carry, resultLocation.asGPRlo()); - } else if (op == ShiftI64HelperOp::Rshift) { - m_jit.lshift32(lhsLocation.asGPRhi(), carry, carry); - ASSERT(resultLocation.asGPRhi() != shift); - ASSERT(resultLocation.asGPRhi() != lhsLocation.asGPRlo()); - m_jit.rshift32(lhsLocation.asGPRhi(), shift, resultLocation.asGPRhi()); - m_jit.urshift32(lhsLocation.asGPRlo(), shift, resultLocation.asGPRlo()); - m_jit.or32(carry, resultLocation.asGPRlo()); - } - - JumpList done; - done.append(m_jit.jump()); - - // shift >= 32 - aboveOrEqual32.link(&m_jit); - m_jit.sub32(shift, TrustedImm32(32), shift); - if (op == ShiftI64HelperOp::Lshift) { - m_jit.lshift32(lhsLocation.asGPRlo(), shift, resultLocation.asGPRhi()); - m_jit.xor32(resultLocation.asGPRlo(), resultLocation.asGPRlo()); - } else if (op == ShiftI64HelperOp::Urshift) { - m_jit.urshift32(lhsLocation.asGPRhi(), shift, resultLocation.asGPRlo()); - m_jit.xor32(resultLocation.asGPRhi(), resultLocation.asGPRhi()); - } else if (op == ShiftI64HelperOp::Rshift) { - ASSERT(resultLocation.asGPRlo() != lhsLocation.asGPRhi()); - m_jit.rshift32(lhsLocation.asGPRhi(), shift, resultLocation.asGPRlo()); - m_jit.rshift32(lhsLocation.asGPRhi(), TrustedImm32(31), resultLocation.asGPRhi()); - } - // The following move is only emitted if the lhs != res, so when they are the same, we - // can just fallthrough - if (lhsLocation != resultLocation) - done.append(m_jit.jump()); - - // shift == 0 - zero.link(&m_jit); - emitMove(TypeKind::I64, lhsLocation, resultLocation); - - done.link(&m_jit); -} - PartialResult WARN_UNUSED_RETURN BBQJIT::addI64Rotl(Value lhs, Value rhs, Value& result) { PREPARE_FOR_SHIFT; @@ -3123,46 +3142,6 @@ PartialResult WARN_UNUSED_RETURN BBQJIT::addRethrow(unsigned, ControlType& data) return { }; } -BBQJIT::BranchFoldResult BBQJIT::tryFoldFusedBranchCompare(OpType, ExpressionType) -{ - RELEASE_ASSERT_NOT_REACHED(); -} - -BBQJIT::Jump BBQJIT::emitFusedBranchCompareBranch(OpType, ExpressionType, Location) -{ - RELEASE_ASSERT_NOT_REACHED(); -} - -BBQJIT::BranchFoldResult BBQJIT::tryFoldFusedBranchCompare(OpType, ExpressionType, ExpressionType) -{ - RELEASE_ASSERT_NOT_REACHED(); -} - -BBQJIT::Jump BBQJIT::emitFusedBranchCompareBranch(OpType, ExpressionType, Location, ExpressionType, Location) -{ - RELEASE_ASSERT_NOT_REACHED(); -} - -PartialResult BBQJIT::addFusedBranchCompare(OpType, ControlType&, ExpressionType, Stack&) -{ - RELEASE_ASSERT_NOT_REACHED(); -} - -PartialResult BBQJIT::addFusedBranchCompare(OpType, ControlType&, ExpressionType, ExpressionType, Stack&) -{ - RELEASE_ASSERT_NOT_REACHED(); -} - -PartialResult BBQJIT::addFusedIfCompare(OpType, ExpressionType, BlockSignature, Stack&, ControlType&, Stack&) -{ - RELEASE_ASSERT_NOT_REACHED(); -} - -PartialResult BBQJIT::addFusedIfCompare(OpType, ExpressionType, ExpressionType, BlockSignature, Stack&, ControlType&, Stack&) -{ - RELEASE_ASSERT_NOT_REACHED(); -} - PartialResult WARN_UNUSED_RETURN BBQJIT::addBranchNull(ControlData& data, ExpressionType reference, Stack& returnValues, bool shouldNegate, ExpressionType& result) { if (reference.isConst() && (reference.asRef() == JSValue::encode(jsNull())) == shouldNegate) { diff --git a/Source/JavaScriptCore/wasm/WasmBBQJIT32_64.h b/Source/JavaScriptCore/wasm/WasmBBQJIT32_64.h index 862440659b65..e4de7b1014f6 100644 --- a/Source/JavaScriptCore/wasm/WasmBBQJIT32_64.h +++ b/Source/JavaScriptCore/wasm/WasmBBQJIT32_64.h @@ -70,21 +70,68 @@ auto BBQJIT::emitCheckAndPrepareAndMaterializePointerApply(Value pointer, uint32 } return functor(CCallHelpers::Address(wasmBaseMemoryPointer, static_cast(finalOffset))); } - pointerLocation = Location::fromGPR(scratches.gpr(0)); - emitMoveConst(pointer, pointerLocation); + // Constant pointer, but finalOffset doesn't fit in addressing mode + // Fold constantPointer + boundary into a single immediate for bounds checking + GPRReg boundsCheckReg = wasmScratchGPR; + if (boundary) { + uint64_t foldedValue = constantPointer + boundary; + // Check for 32-bit overflow at compile time + if (foldedValue > std::numeric_limits::max()) { + // Overflow occurred - unconditional throw + throwExceptionIf(ExceptionType::OutOfBoundsMemoryAccess, m_jit.jump()); + } else { + m_jit.move(TrustedImm32(foldedValue), boundsCheckReg); + switch (m_mode) { + case MemoryMode::BoundsChecking: { + throwExceptionIf(ExceptionType::OutOfBoundsMemoryAccess, m_jit.branchPtr(RelationalCondition::AboveOrEqual, boundsCheckReg, wasmBoundsCheckingSizeRegister)); + break; + } + case MemoryMode::Signaling: { + if (uoffset >= Memory::fastMappedRedzoneBytes()) { + uint64_t maximum = m_info.memory.maximum() ? m_info.memory.maximum().bytes() : std::numeric_limits::max(); + throwExceptionIf(ExceptionType::OutOfBoundsMemoryAccess, m_jit.branchPtr(RelationalCondition::AboveOrEqual, boundsCheckReg, TrustedImmPtr(static_cast(maximum)))); + } + break; + } + } + } + } else { + switch (m_mode) { + case MemoryMode::BoundsChecking: + m_jit.move(TrustedImm32(constantPointer), boundsCheckReg); + throwExceptionIf(ExceptionType::OutOfBoundsMemoryAccess, m_jit.branchPtr(RelationalCondition::AboveOrEqual, boundsCheckReg, wasmBoundsCheckingSizeRegister)); + break; + case MemoryMode::Signaling: + if (uoffset >= Memory::fastMappedRedzoneBytes()) { + uint64_t maximum = m_info.memory.maximum() ? m_info.memory.maximum().bytes() : std::numeric_limits::max(); + m_jit.move(TrustedImm32(constantPointer), boundsCheckReg); + throwExceptionIf(ExceptionType::OutOfBoundsMemoryAccess, m_jit.branchPtr(RelationalCondition::AboveOrEqual, boundsCheckReg, TrustedImmPtr(static_cast(maximum)))); + } + break; + } + } + // Use original constantPointer (not folded) for address calculation + m_jit.add32(TrustedImm32(constantPointer), wasmBaseMemoryPointer, wasmScratchGPR); + if (static_cast(uoffset) > static_cast(std::numeric_limits::max()) || !B3::Air::Arg::isValidAddrForm(B3::Air::Move, static_cast(uoffset), Width::Width128)) { + m_jit.addPtr(TrustedImmPtr(static_cast(uoffset)), wasmScratchGPR); + return functor(Address(wasmScratchGPR)); + } + return functor(Address(wasmScratchGPR, static_cast(uoffset))); } else pointerLocation = loadIfNecessary(pointer); ASSERT(pointerLocation.isGPR()); + GPRReg pointerReg = pointerLocation.asGPR(); switch (m_mode) { case MemoryMode::BoundsChecking: { // We're not using signal handling only when the memory is not shared. // Regardless of signaling, we must check that no memory access exceeds the current memory size. - m_jit.zeroExtend32ToWord(pointerLocation.asGPR(), wasmScratchGPR); - if (boundary) - // NB: On 32-bit we have to check the addition for overflow - throwExceptionIf(ExceptionType::OutOfBoundsMemoryAccess, m_jit.branchAdd32(ResultCondition::Carry, wasmScratchGPR, TrustedImm32(boundary), wasmScratchGPR)); - throwExceptionIf(ExceptionType::OutOfBoundsMemoryAccess, m_jit.branchPtr(RelationalCondition::AboveOrEqual, wasmScratchGPR, wasmBoundsCheckingSizeRegister)); + if (boundary) { + m_jit.add32(TrustedImm32(boundary), pointerReg, wasmScratchGPR); + throwExceptionIf(ExceptionType::OutOfBoundsMemoryAccess, m_jit.branch32(RelationalCondition::Below, wasmScratchGPR, pointerReg)); + throwExceptionIf(ExceptionType::OutOfBoundsMemoryAccess, m_jit.branchPtr(RelationalCondition::AboveOrEqual, wasmScratchGPR, wasmBoundsCheckingSizeRegister)); + } else + throwExceptionIf(ExceptionType::OutOfBoundsMemoryAccess, m_jit.branchPtr(RelationalCondition::AboveOrEqual, pointerReg, wasmBoundsCheckingSizeRegister)); break; } @@ -101,17 +148,16 @@ auto BBQJIT::emitCheckAndPrepareAndMaterializePointerApply(Value pointer, uint32 // any access equal to or greater than 4GiB will trap, no need to add the redzone. if (uoffset >= Memory::fastMappedRedzoneBytes()) { uint64_t maximum = m_info.memory.maximum() ? m_info.memory.maximum().bytes() : std::numeric_limits::max(); - m_jit.zeroExtend32ToWord(pointerLocation.asGPR(), wasmScratchGPR); - if (boundary) - m_jit.addPtr(TrustedImmPtr(boundary), wasmScratchGPR); - throwExceptionIf(ExceptionType::OutOfBoundsMemoryAccess, m_jit.branchPtr(RelationalCondition::AboveOrEqual, wasmScratchGPR, TrustedImmPtr(static_cast(maximum)))); + if (boundary) { + m_jit.add32(TrustedImm32(boundary), pointerReg, wasmScratchGPR); + throwExceptionIf(ExceptionType::OutOfBoundsMemoryAccess, m_jit.branchPtr(RelationalCondition::AboveOrEqual, wasmScratchGPR, TrustedImmPtr(static_cast(maximum)))); + } else + throwExceptionIf(ExceptionType::OutOfBoundsMemoryAccess, m_jit.branchPtr(RelationalCondition::AboveOrEqual, pointerReg, TrustedImmPtr(static_cast(maximum)))); } break; } } - - m_jit.zeroExtend32ToWord(pointerLocation.asGPR(), wasmScratchGPR); - m_jit.addPtr(wasmBaseMemoryPointer, wasmScratchGPR); + m_jit.add32(pointerReg, wasmBaseMemoryPointer, wasmScratchGPR); if (static_cast(uoffset) > static_cast(std::numeric_limits::max()) || !B3::Air::Arg::isValidAddrForm(B3::Air::Move, uoffset, Width::Width128)) { m_jit.addPtr(TrustedImmPtr(static_cast(uoffset)), wasmScratchGPR); diff --git a/Source/JavaScriptCore/wasm/WasmBBQJIT64.cpp b/Source/JavaScriptCore/wasm/WasmBBQJIT64.cpp index 264b54786897..54d386036897 100644 --- a/Source/JavaScriptCore/wasm/WasmBBQJIT64.cpp +++ b/Source/JavaScriptCore/wasm/WasmBBQJIT64.cpp @@ -3175,376 +3175,6 @@ PartialResult WARN_UNUSED_RETURN BBQJIT::addRethrow(unsigned, ControlType& data) return { }; } -BBQJIT::BranchFoldResult BBQJIT::tryFoldFusedBranchCompare(OpType opType, ExpressionType operand) -{ - if (!operand.isConst()) - return BranchNotFolded; - switch (opType) { - case OpType::I32Eqz: - return operand.asI32() ? BranchNeverTaken : BranchAlwaysTaken; - case OpType::I64Eqz: - return operand.asI64() ? BranchNeverTaken : BranchAlwaysTaken; - default: - RELEASE_ASSERT_NOT_REACHED_WITH_MESSAGE("Op type '%s' is not a unary comparison and should not have been fused.\n", makeString(opType).characters()); - } - return BranchNotFolded; -} - -BBQJIT::Jump BBQJIT::emitFusedBranchCompareBranch(OpType opType, ExpressionType, Location operandLocation) -{ - // Emit the negation of the intended branch. - switch (opType) { - case OpType::I32Eqz: - return m_jit.branchTest32(ResultCondition::NonZero, operandLocation.asGPR()); - case OpType::I64Eqz: - return m_jit.branchTest64(ResultCondition::NonZero, operandLocation.asGPR()); - default: - RELEASE_ASSERT_NOT_REACHED_WITH_MESSAGE("Op type '%s' is not a unary comparison and should not have been fused.\n", makeString(opType).characters()); - } -} - -PartialResult BBQJIT::addFusedBranchCompare(OpType opType, ControlType& target, ExpressionType operand, Stack& results) -{ - ASSERT(!operand.isNone()); - - switch (tryFoldFusedBranchCompare(opType, operand)) { - case BranchNeverTaken: - return { }; - case BranchAlwaysTaken: - currentControlData().flushAndSingleExit(*this, target, results, false, false); - target.addBranch(m_jit.jump()); - return { }; - case BranchNotFolded: - break; - } - - { - // Like in normal addBranch(), we can directly use the operand location - // because it shouldn't interfere with flushAtBlockBoundary(). - Location operandLocation = loadIfNecessary(operand); - consume(operand); - - LOG_INSTRUCTION("BranchCompare", makeString(opType).characters(), operand, operandLocation); - - currentControlData().flushAtBlockBoundary(*this, 0, results, false); - Jump ifNotTaken = emitFusedBranchCompareBranch(opType, operand, operandLocation); - currentControlData().addExit(*this, target.targetLocations(), results); - target.addBranch(m_jit.jump()); - ifNotTaken.link(&m_jit); - currentControlData().finalizeBlock(*this, target.targetLocations().size(), results, true); - } - - return { }; -} - -PartialResult WARN_UNUSED_RETURN BBQJIT::addFusedIfCompare(OpType op, ExpressionType operand, BlockSignature signature, Stack& enclosingStack, ControlData& result, Stack& newStack) -{ - BranchFoldResult foldResult = tryFoldFusedBranchCompare(op, operand); - - ScratchScope<0, 1> scratches(*this); - Location operandLocation; - RegisterSet liveScratchGPRs, liveScratchFPRs; - if (foldResult == BranchNotFolded) { - if (!operand.isConst()) - operandLocation = loadIfNecessary(operand); - else if (operand.isFloat()) { - operandLocation = Location::fromFPR(scratches.fpr(0)); - emitMove(operand, operandLocation); - } - - if (operandLocation.isGPR()) - liveScratchGPRs.add(operandLocation.asGPR(), IgnoreVectors); - else - liveScratchFPRs.add(operandLocation.asFPR(), operand.type() == TypeKind::V128 ? Width128 : Width64); - } - if (!liveScratchFPRs.contains(scratches.fpr(0), IgnoreVectors)) - scratches.unbindEarly(); - - consume(operand); - - result = ControlData(*this, BlockType::If, signature, currentControlData().enclosedHeight() + currentControlData().implicitSlots() + enclosingStack.size() - signature.m_signature->argumentCount(), liveScratchGPRs, liveScratchFPRs); - - // Despite being conditional, if doesn't need to worry about diverging expression stacks at block boundaries, so it doesn't need multiple exits. - currentControlData().flushAndSingleExit(*this, result, enclosingStack, true, false); - - LOG_INSTRUCTION("IfCompare", makeString(op).characters(), *signature.m_signature, operand, operandLocation); - LOG_INDENT(); - splitStack(signature, enclosingStack, newStack); - - result.startBlock(*this, newStack); - if (foldResult == BranchNeverTaken) - result.setIfBranch(m_jit.jump()); // Emit direct branch if we know the condition is false. - else if (foldResult == BranchNotFolded) // Otherwise, we only emit a branch at all if we don't know the condition statically. - result.setIfBranch(emitFusedBranchCompareBranch(op, operand, operandLocation)); - return { }; -} - -BBQJIT::BranchFoldResult BBQJIT::tryFoldFusedBranchCompare(OpType opType, ExpressionType left, ExpressionType right) -{ - if (!left.isConst() || !right.isConst()) - return BranchNotFolded; - switch (opType) { - case OpType::I32LtS: - return left.asI32() < right.asI32() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I32LtU: - return static_cast(left.asI32()) < static_cast(right.asI32()) ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I32GtS: - return left.asI32() > right.asI32() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I32GtU: - return static_cast(left.asI32()) > static_cast(right.asI32()) ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I32LeS: - return left.asI32() <= right.asI32() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I32LeU: - return static_cast(left.asI32()) <= static_cast(right.asI32()) ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I32GeS: - return left.asI32() >= right.asI32() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I32GeU: - return static_cast(left.asI32()) >= static_cast(right.asI32()) ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I32Eq: - return left.asI32() == right.asI32() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I32Ne: - return left.asI32() == right.asI32() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I64LtS: - return left.asI64() < right.asI64() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I64LtU: - return static_cast(left.asI64()) < static_cast(right.asI64()) ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I64GtS: - return left.asI64() > right.asI64() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I64GtU: - return static_cast(left.asI64()) > static_cast(right.asI64()) ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I64LeS: - return left.asI64() <= right.asI64() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I64LeU: - return static_cast(left.asI64()) <= static_cast(right.asI64()) ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I64GeS: - return left.asI64() >= right.asI64() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I64GeU: - return static_cast(left.asI64()) >= static_cast(right.asI64()) ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I64Eq: - return left.asI64() == right.asI64() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I64Ne: - return left.asI64() == right.asI64() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::F32Lt: - return left.asF32() < right.asF32() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::F32Gt: - return left.asF32() > right.asF32() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::F32Le: - return left.asF32() <= right.asF32() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::F32Ge: - return left.asF32() >= right.asF32() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::F32Eq: - return left.asF32() == right.asF32() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::F32Ne: - return left.asF32() != right.asF32() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::F64Lt: - return left.asF64() < right.asF64() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::F64Gt: - return left.asF64() > right.asF64() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::F64Le: - return left.asF64() <= right.asF64() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::F64Ge: - return left.asF64() >= right.asF64() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::F64Eq: - return left.asF64() == right.asF64() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::F64Ne: - return left.asF64() != right.asF64() ? BranchAlwaysTaken : BranchNeverTaken; - default: - RELEASE_ASSERT_NOT_REACHED_WITH_MESSAGE("Op type '%s' is not a binary comparison and should not have been fused.\n", makeString(opType).characters()); - } -} - -static MacroAssembler::Jump emitBranchI32(CCallHelpers& jit, MacroAssembler::RelationalCondition condition, Value left, Location leftLocation, Value right, Location rightLocation) -{ - if (right.isConst()) - return jit.branch32(condition, leftLocation.asGPR(), MacroAssembler::TrustedImm32(right.asI32())); - if (left.isConst()) - return jit.branch32(condition, MacroAssembler::TrustedImm32(left.asI32()), rightLocation.asGPR()); - return jit.branch32(condition, leftLocation.asGPR(), rightLocation.asGPR()); -} - -static MacroAssembler::Jump emitBranchI64(CCallHelpers& jit, MacroAssembler::RelationalCondition condition, Value left, Location leftLocation, Value right, Location rightLocation) -{ - if (right.isConst()) - return jit.branch64(condition, leftLocation.asGPR(), MacroAssembler::Imm64(right.asI64())); - if (left.isConst()) - return jit.branch64(MacroAssembler::commute(condition), rightLocation.asGPR(), MacroAssembler::Imm64(left.asI64())); - return jit.branch64(condition, leftLocation.asGPR(), rightLocation.asGPR()); -} - -static MacroAssembler::Jump emitBranchF32(CCallHelpers& jit, MacroAssembler::DoubleCondition condition, Value, Location leftLocation, Value, Location rightLocation) -{ - return jit.branchFloat(condition, leftLocation.asFPR(), rightLocation.asFPR()); -} - -static MacroAssembler::Jump emitBranchF64(CCallHelpers& jit, MacroAssembler::DoubleCondition condition, Value, Location leftLocation, Value, Location rightLocation) -{ - return jit.branchDouble(condition, leftLocation.asFPR(), rightLocation.asFPR()); -} - -BBQJIT::Jump BBQJIT::emitFusedBranchCompareBranch(OpType opType, ExpressionType left, Location leftLocation, ExpressionType right, Location rightLocation) -{ - // Emit a branch with the inverse of the comparison. We're generating the "branch-if-false" case. - switch (opType) { - case OpType::I32LtS: - return emitBranchI32(m_jit, RelationalCondition::GreaterThanOrEqual, left, leftLocation, right, rightLocation); - case OpType::I32LtU: - return emitBranchI32(m_jit, RelationalCondition::AboveOrEqual, left, leftLocation, right, rightLocation); - case OpType::I32GtS: - return emitBranchI32(m_jit, RelationalCondition::LessThanOrEqual, left, leftLocation, right, rightLocation); - case OpType::I32GtU: - return emitBranchI32(m_jit, RelationalCondition::BelowOrEqual, left, leftLocation, right, rightLocation); - case OpType::I32LeS: - return emitBranchI32(m_jit, RelationalCondition::GreaterThan, left, leftLocation, right, rightLocation); - case OpType::I32LeU: - return emitBranchI32(m_jit, RelationalCondition::Above, left, leftLocation, right, rightLocation); - case OpType::I32GeS: - return emitBranchI32(m_jit, RelationalCondition::LessThan, left, leftLocation, right, rightLocation); - case OpType::I32GeU: - return emitBranchI32(m_jit, RelationalCondition::Below, left, leftLocation, right, rightLocation); - case OpType::I32Eq: - return emitBranchI32(m_jit, RelationalCondition::NotEqual, left, leftLocation, right, rightLocation); - case OpType::I32Ne: - return emitBranchI32(m_jit, RelationalCondition::Equal, left, leftLocation, right, rightLocation); - case OpType::I64LtS: - return emitBranchI64(m_jit, RelationalCondition::GreaterThanOrEqual, left, leftLocation, right, rightLocation); - case OpType::I64LtU: - return emitBranchI64(m_jit, RelationalCondition::AboveOrEqual, left, leftLocation, right, rightLocation); - case OpType::I64GtS: - return emitBranchI64(m_jit, RelationalCondition::LessThanOrEqual, left, leftLocation, right, rightLocation); - case OpType::I64GtU: - return emitBranchI64(m_jit, RelationalCondition::BelowOrEqual, left, leftLocation, right, rightLocation); - case OpType::I64LeS: - return emitBranchI64(m_jit, RelationalCondition::GreaterThan, left, leftLocation, right, rightLocation); - case OpType::I64LeU: - return emitBranchI64(m_jit, RelationalCondition::Above, left, leftLocation, right, rightLocation); - case OpType::I64GeS: - return emitBranchI64(m_jit, RelationalCondition::LessThan, left, leftLocation, right, rightLocation); - case OpType::I64GeU: - return emitBranchI64(m_jit, RelationalCondition::Below, left, leftLocation, right, rightLocation); - case OpType::I64Eq: - return emitBranchI64(m_jit, RelationalCondition::NotEqual, left, leftLocation, right, rightLocation); - case OpType::I64Ne: - return emitBranchI64(m_jit, RelationalCondition::Equal, left, leftLocation, right, rightLocation); - case OpType::F32Lt: - return emitBranchF32(m_jit, MacroAssembler::invert(DoubleCondition::DoubleLessThanAndOrdered), left, leftLocation, right, rightLocation); - case OpType::F32Gt: - return emitBranchF32(m_jit, MacroAssembler::invert(DoubleCondition::DoubleGreaterThanAndOrdered), left, leftLocation, right, rightLocation); - case OpType::F32Le: - return emitBranchF32(m_jit, MacroAssembler::invert(DoubleCondition::DoubleLessThanOrEqualAndOrdered), left, leftLocation, right, rightLocation); - case OpType::F32Ge: - return emitBranchF32(m_jit, MacroAssembler::invert(DoubleCondition::DoubleGreaterThanOrEqualAndOrdered), left, leftLocation, right, rightLocation); - case OpType::F32Eq: - return emitBranchF32(m_jit, MacroAssembler::invert(DoubleCondition::DoubleEqualAndOrdered), left, leftLocation, right, rightLocation); - case OpType::F32Ne: - return emitBranchF32(m_jit, MacroAssembler::invert(DoubleCondition::DoubleNotEqualOrUnordered), left, leftLocation, right, rightLocation); - case OpType::F64Lt: - return emitBranchF64(m_jit, MacroAssembler::invert(DoubleCondition::DoubleLessThanAndOrdered), left, leftLocation, right, rightLocation); - case OpType::F64Gt: - return emitBranchF64(m_jit, MacroAssembler::invert(DoubleCondition::DoubleGreaterThanAndOrdered), left, leftLocation, right, rightLocation); - case OpType::F64Le: - return emitBranchF64(m_jit, MacroAssembler::invert(DoubleCondition::DoubleLessThanOrEqualAndOrdered), left, leftLocation, right, rightLocation); - case OpType::F64Ge: - return emitBranchF64(m_jit, MacroAssembler::invert(DoubleCondition::DoubleGreaterThanOrEqualAndOrdered), left, leftLocation, right, rightLocation); - case OpType::F64Eq: - return emitBranchF64(m_jit, MacroAssembler::invert(DoubleCondition::DoubleEqualAndOrdered), left, leftLocation, right, rightLocation); - case OpType::F64Ne: - return emitBranchF64(m_jit, MacroAssembler::invert(DoubleCondition::DoubleNotEqualOrUnordered), left, leftLocation, right, rightLocation); - default: - RELEASE_ASSERT_NOT_REACHED_WITH_MESSAGE("Op type '%s' is not a binary comparison and should not have been fused.\n", makeString(opType).characters()); - } -} - -PartialResult BBQJIT::addFusedBranchCompare(OpType opType, ControlType& target, ExpressionType left, ExpressionType right, Stack& results) -{ - switch (tryFoldFusedBranchCompare(opType, left, right)) { - case BranchNeverTaken: - return { }; - case BranchAlwaysTaken: - currentControlData().flushAndSingleExit(*this, target, results, false, false); - target.addBranch(m_jit.jump()); - return { }; - case BranchNotFolded: - break; - } - - { - Location leftLocation, rightLocation; - - if (!left.isConst()) - leftLocation = loadIfNecessary(left); - else if (left.isFloat()) // Materialize floats here too, since they don't have a good immediate lowering. - emitMove(left, leftLocation = Location::fromFPR(wasmScratchFPR)); - if (!right.isConst()) - rightLocation = loadIfNecessary(right); - else if (right.isFloat()) - emitMove(right, rightLocation = Location::fromFPR(wasmScratchFPR)); - - consume(left); - consume(right); - - LOG_INSTRUCTION("BranchCompare", makeString(opType).characters(), left, leftLocation, right, rightLocation); - - currentControlData().flushAtBlockBoundary(*this, 0, results, false); - Jump ifNotTaken = emitFusedBranchCompareBranch(opType, left, leftLocation, right, rightLocation); - currentControlData().addExit(*this, target.targetLocations(), results); - target.addBranch(m_jit.jump()); - ifNotTaken.link(&m_jit); - currentControlData().finalizeBlock(*this, target.targetLocations().size(), results, true); - } - - return { }; -} - -PartialResult WARN_UNUSED_RETURN BBQJIT::addFusedIfCompare(OpType op, ExpressionType left, ExpressionType right, BlockSignature signature, Stack& enclosingStack, ControlData& result, Stack& newStack) -{ - BranchFoldResult foldResult = tryFoldFusedBranchCompare(op, left, right); - - ScratchScope<0, 1> scratches(*this, RegisterSetBuilder::argumentGPRs(), RegisterSetBuilder::argumentFPRs()); - scratches.unbindPreserved(); - Location leftLocation, rightLocation; - RegisterSet liveScratchGPRs, liveScratchFPRs; - if (foldResult == BranchNotFolded) { - ASSERT(!left.isConst() || !right.isConst()); // If they're both constants, we should have folded. - - if (!left.isConst()) - leftLocation = loadIfNecessary(left); - else if (left.isFloat()) - emitMove(left, leftLocation = Location::fromFPR(scratches.fpr(0))); - if (leftLocation.isGPR()) - liveScratchGPRs.add(leftLocation.asGPR(), IgnoreVectors); - else if (leftLocation.isFPR()) - liveScratchFPRs.add(leftLocation.asFPR(), left.type() == TypeKind::V128 ? Width128 : Width64); - - if (!right.isConst()) - rightLocation = loadIfNecessary(right); - else if (right.isFloat()) - emitMove(right, rightLocation = Location::fromFPR(scratches.fpr(0))); - if (rightLocation.isGPR()) - liveScratchGPRs.add(rightLocation.asGPR(), IgnoreVectors); - else if (rightLocation.isFPR()) - liveScratchFPRs.add(rightLocation.asFPR(), right.type() == TypeKind::V128 ? Width128 : Width64); - } - consume(left); - consume(right); - - - result = ControlData(*this, BlockType::If, signature, currentControlData().enclosedHeight() + currentControlData().implicitSlots() + enclosingStack.size() - signature.m_signature->argumentCount(), liveScratchGPRs, liveScratchFPRs); - - // Despite being conditional, if doesn't need to worry about diverging expression stacks at block boundaries, so it doesn't need multiple exits. - currentControlData().flushAndSingleExit(*this, result, enclosingStack, true, false); - - LOG_INSTRUCTION("IfCompare", makeString(op).characters(), *signature.m_signature, left, leftLocation, right, rightLocation); - LOG_INDENT(); - splitStack(signature, enclosingStack, newStack); - - result.startBlock(*this, newStack); - if (foldResult == BranchNeverTaken) - result.setIfBranch(m_jit.jump()); // Emit direct branch if we know the condition is false. - else if (foldResult == BranchNotFolded) // Otherwise, we only emit a branch at all if we don't know the condition statically. - result.setIfBranch(emitFusedBranchCompareBranch(op, left, leftLocation, right, rightLocation)); - return { }; -} - PartialResult WARN_UNUSED_RETURN BBQJIT::addBranchNull(ControlData& data, ExpressionType reference, Stack& returnValues, bool shouldNegate, ExpressionType& result) { if (reference.isConst() && (reference.asRef() == JSValue::encode(jsNull())) == shouldNegate) { diff --git a/Source/JavaScriptCore/wasm/WasmOperationsInlines.h b/Source/JavaScriptCore/wasm/WasmOperationsInlines.h index 3be526d74075..12e5c5f3c508 100644 --- a/Source/JavaScriptCore/wasm/WasmOperationsInlines.h +++ b/Source/JavaScriptCore/wasm/WasmOperationsInlines.h @@ -433,7 +433,12 @@ inline bool arrayInitElem(JSWebAssemblyInstance* instance, EncodedJSValue dst, u if (lastSrcElementIndexChecked > lengthOfElementSegment) return false; - instance->copyElementSegment(dstObject, *instance->elementAt(srcElementIndex), srcOffset, size, dstObject->reftypeData() + dstOffset); + auto* elementSegment = instance->elementAt(srcElementIndex); + if (elementSegment) + instance->copyElementSegment(dstObject, *elementSegment, srcOffset, size, dstObject->reftypeData() + dstOffset); + else + ASSERT(!lastSrcElementIndexChecked); + return true; } diff --git a/Source/WTF/wtf/Threading.cpp b/Source/WTF/wtf/Threading.cpp index 6ddaea460c69..0cac58e99d1b 100644 --- a/Source/WTF/wtf/Threading.cpp +++ b/Source/WTF/wtf/Threading.cpp @@ -210,7 +210,6 @@ const char* Thread::normalizeThreadName(const char* threadName) result = result.right(kLinuxThreadNameLimit); #endif auto characters = result.span8(); - ASSERT(characters[characters.size()] == '\0'); return byteCast(characters.data()); #endif } diff --git a/Source/cmake/OptionsCommon.cmake b/Source/cmake/OptionsCommon.cmake index 78b5f052586e..0b7c8087398a 100644 --- a/Source/cmake/OptionsCommon.cmake +++ b/Source/cmake/OptionsCommon.cmake @@ -204,7 +204,7 @@ if (DEBUG_FISSION) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -gsplit-dwarf") if (LD_SUPPORTS_GDB_INDEX) set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gdb-index") - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gdb-index") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--gdb-index") endif () if (WTF_CPU_ARM) string(REPLACE "-fdebug-types-section" "" CMAKE_C_FLAGS ${CMAKE_C_FLAGS})