diff --git a/cranelift/assembler-x64/meta/src/dsl.rs b/cranelift/assembler-x64/meta/src/dsl.rs index 69e7778d386c..293d914620d7 100644 --- a/cranelift/assembler-x64/meta/src/dsl.rs +++ b/cranelift/assembler-x64/meta/src/dsl.rs @@ -10,8 +10,8 @@ mod features; pub mod format; pub use custom::{Custom, Customization}; +pub use encoding::{ApxClass, Evex, Length, Vex, VexEscape, VexPrefix, evex, vex}; pub use encoding::{Encoding, ModRmKind, OpcodeMod}; -pub use encoding::{Evex, Length, Vex, VexEscape, VexPrefix, evex, vex}; pub use encoding::{ Group1Prefix, Group2Prefix, Group3Prefix, Group4Prefix, Opcodes, Prefixes, Rex, TupleType, rex, }; diff --git a/cranelift/assembler-x64/meta/src/dsl/encoding.rs b/cranelift/assembler-x64/meta/src/dsl/encoding.rs index c843e609d7eb..2d254be9c46e 100644 --- a/cranelift/assembler-x64/meta/src/dsl/encoding.rs +++ b/cranelift/assembler-x64/meta/src/dsl/encoding.rs @@ -57,6 +57,9 @@ pub fn evex(length: Length, tuple_type: TupleType) -> Evex { modrm: None, imm: Imm::None, tuple_type, + apx: None, + nd: None, + nf: None, } } @@ -86,6 +89,13 @@ impl Encoding { Encoding::Evex(evex) => evex.opcode, } } + + /// Return whether this encoding sets the APX `ND` ("new data destination") + /// bit, meaning the architectural destination is the `vvvv`-encoded + /// register rather than an operand named by ModRM. + pub fn is_nd(&self) -> bool { + matches!(self, Encoding::Evex(evex) if evex.nd == Some(true)) + } } impl fmt::Display for Encoding { @@ -841,6 +851,10 @@ pub enum VexEscape { _0F, _0F3A, _0F38, + /// APX "opcode map 4"; only valid for APX legacy-GPR (Extended EVEX) + /// encodings, never for VEX. This is the map that enables promoting legacy + /// general-purpose-register instructions into the EVEX space. + _MAP4, } impl VexEscape { @@ -851,6 +865,7 @@ impl VexEscape { Self::_0F => 0b01, Self::_0F38 => 0b10, Self::_0F3A => 0b11, + Self::_MAP4 => 0b100, } } } @@ -861,6 +876,7 @@ impl fmt::Display for VexEscape { Self::_0F => write!(f, "0F"), Self::_0F3A => write!(f, "0F3A"), Self::_0F38 => write!(f, "0F38"), + Self::_MAP4 => write!(f, "MAP4"), } } } @@ -1247,6 +1263,30 @@ pub struct Evex { /// The "Tuple Type" corresponding to scaling of the 8-bit displacement /// parameter for memory operands. See [`TupleType`] for more information. pub tuple_type: TupleType, + /// The APX "Extended EVEX" class, when this instruction is an APX + /// promotion. + /// + /// Intel APX (Advanced Performance Extensions) reuses the `0x62` EVEX + /// identifier but does *not* define a single static payload layout. + /// Instead, the meaning of the payload bytes (`P0`, `P1`, `P2`) is + /// re-mapped depending on which class of instruction is being promoted (see + /// [`ApxClass`]). `None` denotes a standard (AVX-512) EVEX encoding; + /// `Some(_)` selects one of the APX layouts and enables addressing of the + /// extended general-purpose registers `R16`–`R31` ("EGPR"). + pub apx: Option, + /// The APX `ND` (New Data destination) bit. + /// + /// When set, a legacy destructive two-operand instruction is promoted into + /// a non-destructive three-operand form (the extra destination is encoded + /// in the `EVEX.vvvv` field). `None` when the bit is not part of this + /// encoding; only valid for [`ApxClass::LegacyGpr`]. + pub nd: Option, + /// The APX `NF` (No Flags) bit. + /// + /// When set, the status-flag writes that a legacy integer instruction would + /// otherwise perform are suppressed. `None` when the bit is not part of + /// this encoding; only valid for [`ApxClass::LegacyGpr`]. + pub nf: Option, } impl Evex { @@ -1310,6 +1350,27 @@ impl Evex { } } + /// Select the APX "opcode map 4", promoting a legacy general-purpose-register + /// instruction into the Extended EVEX space; equivalent to `.MAP4` in the + /// manual. + /// + /// This both sets the `mmm` map bits and marks the encoding as + /// [`ApxClass::LegacyGpr`], which in turn enables addressing of the extended + /// GPRs `R16`–`R31` ("EGPR") and permits the `ND`/`NF` bits. + /// + /// # Panics + /// + /// Panics if the map (`mmm`) or APX class has already been set. + pub fn map4(self) -> Self { + assert!(self.mmm.is_none()); + assert!(self.apx.is_none()); + Self { + mmm: Some(VexEscape::_MAP4), + apx: Some(ApxClass::LegacyGpr), + ..self + } + } + /// Set the `W` bit to `0`; equivalent to `.W0` in the manual. pub fn w0(self) -> Self { assert!(self.w.is_ignored()); @@ -1352,9 +1413,72 @@ impl Evex { } } + /// Mark this as an APX "Extended EVEX" encoding of the given [`ApxClass`]. + /// + /// This selects the APX payload layout to emit and enables addressing of + /// the extended general-purpose registers `R16`–`R31` ("EGPR"). + /// + /// # Panics + /// + /// Panics if an APX class has already been set. + pub fn apx(self, class: ApxClass) -> Self { + assert!(self.apx.is_none()); + Self { + apx: Some(class), + ..self + } + } + + /// Set the APX `ND` (New Data destination) bit; equivalent to `.ND` in the + /// manual. + /// + /// # Panics + /// + /// Panics if this is not an [`ApxClass::LegacyGpr`] encoding, or if the bit + /// has already been set. + pub fn nd(self) -> Self { + assert_eq!( + self.apx, + Some(ApxClass::LegacyGpr), + "the ND bit is only valid for APX legacy-GPR promotions" + ); + assert!(self.nd.is_none()); + Self { + nd: Some(true), + ..self + } + } + + /// Set the APX `NF` (No Flags) bit; equivalent to `.NF` in the manual. + /// + /// # Panics + /// + /// Panics if this is not an [`ApxClass::LegacyGpr`] encoding, or if the bit + /// has already been set. + pub fn nf(self) -> Self { + assert_eq!( + self.apx, + Some(ApxClass::LegacyGpr), + "the NF bit is only valid for APX legacy-GPR promotions" + ); + assert!(self.nf.is_none()); + Self { + nf: Some(true), + ..self + } + } + fn validate(&self, _operands: &[Operand]) { assert!(self.opcode != u8::MAX); assert!(self.mmm.is_some()); + // The `ND`/`NF` bits are only defined for APX legacy-GPR promotions. + if self.nd.is_some() || self.nf.is_some() { + assert_eq!( + self.apx, + Some(ApxClass::LegacyGpr), + "the ND/NF bits require an APX legacy-GPR encoding" + ); + } } /// Retrieve the digit extending the opcode, if available. @@ -1412,7 +1536,14 @@ impl fmt::Display for Evex { if let Some(mmmmm) = self.mmm { write!(f, ".{mmmmm}")?; } - write!(f, ".{} {:#04X}", self.w, self.opcode)?; + write!(f, ".{}", self.w)?; + if self.nd == Some(true) { + write!(f, ".ND")?; + } + if self.nf == Some(true) { + write!(f, ".NF")?; + } + write!(f, " {:#04X}", self.opcode)?; if let Some(modrm) = self.modrm { write!(f, " {modrm}")?; } @@ -1423,6 +1554,35 @@ impl fmt::Display for Evex { } } +/// The class of an APX "Extended EVEX" encoding. +/// +/// Intel APX does not define a single static Extended-EVEX layout. Although +/// every APX instruction still begins with the `0x62` EVEX identifier, the +/// meaning of the payload bytes (`P0`, `P1`, `P2`) is re-mapped depending on +/// the class of instruction being promoted. This enum selects which layout the +/// assembler should emit; all classes gain access to the extended +/// general-purpose registers `R16`–`R31` ("EGPR"). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum ApxClass { + /// Promotion of a legacy general-purpose-register (GPR) instruction using + /// extended opcode map 4. Only this class may set the `ND` and `NF` bits. + LegacyGpr, + /// Promotion of a legacy SSE / VEX vector instruction into the EVEX space. + Vector, + /// An existing AVX-512 (EVEX) instruction extended with APX register bits. + Avx512, +} + +impl fmt::Display for ApxClass { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Self::LegacyGpr => write!(f, "LegacyGpr"), + Self::Vector => write!(f, "Vector"), + Self::Avx512 => write!(f, "Avx512"), + } + } +} + /// Tuple Type definitions used in EVEX encodings. /// /// This enumeration corresponds to table 2-34 and 2-35 in the Intel manual. diff --git a/cranelift/assembler-x64/meta/src/dsl/features.rs b/cranelift/assembler-x64/meta/src/dsl/features.rs index d7cfdbe5ebfd..e9266d7db5b8 100644 --- a/cranelift/assembler-x64/meta/src/dsl/features.rs +++ b/cranelift/assembler-x64/meta/src/dsl/features.rs @@ -98,6 +98,7 @@ pub enum Feature { fma, avx_vnni, avx512vnni, + apx, } /// List all CPU features. @@ -131,6 +132,7 @@ pub const ALL_FEATURES: &[Feature] = &[ Feature::fma, Feature::avx_vnni, Feature::avx512vnni, + Feature::apx, ]; impl fmt::Display for Feature { diff --git a/cranelift/assembler-x64/meta/src/generate/format.rs b/cranelift/assembler-x64/meta/src/generate/format.rs index e1e4a8bb3bfb..6cb6c1cd5b71 100644 --- a/cranelift/assembler-x64/meta/src/generate/format.rs +++ b/cranelift/assembler-x64/meta/src/generate/format.rs @@ -54,29 +54,46 @@ impl dsl::Format { /// once Cranelift has switched to using this assembler predominantly /// (TODO). #[must_use] - pub(crate) fn generate_att_style_operands(&self) -> String { - let ordered_ops: Vec<_> = self - .operands - .iter() - .filter(|o| !o.implicit) - .rev() - .map(|o| format!("{{{}}}", o.location)) - .collect(); - ordered_ops.join(", ") + pub(crate) fn generate_att_style_operands(&self, nd: bool) -> String { + self.ordered_operands(nd, false) } /// Like [`Self::generate_att_style_operands`], but omits the fixed `%xmm0` /// mask operand, which XED leaves implicit. #[must_use] - pub(crate) fn generate_xed_style_operands(&self) -> String { - let ordered_ops: Vec<_> = self + pub(crate) fn generate_xed_style_operands(&self, nd: bool) -> String { + self.ordered_operands(nd, true) + } + + /// Shared operand ordering for AT&T-style printing. + /// + /// Normally this is just the reverse of the DSL's Intel-style order. APX + /// "new data destination" (NDD) forms are the exception: their + /// architectural destination is the `vvvv`-encoded register, which the DSL + /// must list in the middle so that the positional slot assignment in + /// `generate_vex_or_evex_prefix` maps operands to `reg`/`vvvv`/`rm` + /// correctly. Blindly reversing would therefore print the destination in + /// the middle. Instead, for `ND = 1` the sources keep their DSL order and + /// the destination is printed last, which matches both AT&T convention and + /// the XED disassembly used as a fuzzing oracle. + #[must_use] + fn ordered_operands(&self, nd: bool, xed_style: bool) -> String { + let ops = self .operands .iter() - .filter(|o| !o.implicit && o.location != dsl::Location::xmm0) - .rev() + .filter(|o| !o.implicit && !(xed_style && o.location == dsl::Location::xmm0)); + let ordered: Vec<&dsl::Operand> = if nd { + let (dst, srcs): (Vec<_>, Vec<_>) = + ops.partition(|o| matches!(o.mutability, dsl::Mutability::Write)); + srcs.into_iter().chain(dst).collect() + } else { + ops.rev().collect() + }; + ordered + .into_iter() .map(|o| format!("{{{}}}", o.location)) - .collect(); - ordered_ops.join(", ") + .collect::>() + .join(", ") } #[must_use] @@ -243,24 +260,51 @@ impl dsl::Format { fmtln!(f, "let w = {};", vex.w.as_bool()); let bits = "len, pp, mmmmm, w"; - self.generate_vex_or_evex_prefix(f, "VexPrefix", &bits, vex.is4, None, || { - vex.unwrap_digit() - }) + self.generate_vex_or_evex_prefix( + f, + "VexPrefix", + &bits, + vex.is4, + None, + "two_op", + "three_op", + || vex.unwrap_digit(), + ) } fn generate_evex_prefix(&self, f: &mut Formatter, evex: &dsl::Evex) -> ModRmStyle { f.empty_line(); f.comment("Emit EVEX prefix."); - let ll = evex.length.evex_bits(); - fmtln!(f, "let ll = {ll:#04b};"); + + // Intel APX promotes legacy GPR instructions into "EVEX map 4" using a + // re-purposed payload layout (see `EvexPrefix::legacy` in the runtime + // assembler and section 3.1.2.3.1 of the APX spec). In that case we + // emit the ND/NF bits and select the `legacy_*` constructors instead of + // the AVX-512 ones. + let apx_legacy = matches!(evex.apx, Some(dsl::ApxClass::LegacyGpr)); + fmtln!(f, "let pp = {:#04b};", evex.pp.map_or(0b00, |pp| pp.bits())); fmtln!(f, "let mmm = {:#07b};", evex.mmm.unwrap().bits()); fmtln!(f, "let w = {};", evex.w.as_bool()); - // NB: when bcast is supported in the future the `evex_scaling` - // calculation for `Full` and `Half` below need to be updated. + + let (bits, two_op, three_op); + if apx_legacy { + fmtln!(f, "let nd = {};", evex.nd == Some(true)); + fmtln!(f, "let nf = {};", evex.nf == Some(true)); + bits = String::from("pp, mmm, w, nd, nf"); + two_op = "legacy_two_op"; + three_op = "legacy_three_op"; + } else { + let ll = evex.length.evex_bits(); + fmtln!(f, "let ll = {ll:#04b};"); + // NB: when bcast is supported in the future the `evex_scaling` + // calculation for `Full` and `Half` below need to be updated. + fmtln!(f, "let bcast = false;"); + bits = String::from("ll, pp, mmm, w, bcast"); + two_op = "two_op"; + three_op = "three_op"; + } let bcast = false; - fmtln!(f, "let bcast = {bcast};"); - let bits = format!("ll, pp, mmm, w, bcast"); let is4 = false; let length_bytes = match evex.length { @@ -273,39 +317,59 @@ impl dsl::Format { // Figure out, according to table 2-34 and 2-35 in the Intel manual, // what the scaling factor is for 8-bit displacements to pass through to // encoding. - let evex_scaling = Some(match evex.tuple_type { - dsl::TupleType::Full => { - assert!(!bcast); - length_bytes - } - dsl::TupleType::Half => { - assert!(!bcast); - length_bytes / 2 - } - dsl::TupleType::FullMem => length_bytes, - // FIXME: according to table 2-35 this needs to take into account - // "InputSize" which isn't accounted for in our `Evex` structure at - // this time. - dsl::TupleType::Tuple1Scalar => unimplemented!(), - dsl::TupleType::Tuple1Fixed => unimplemented!(), - dsl::TupleType::Tuple2 => unimplemented!(), - dsl::TupleType::Tuple4 => unimplemented!(), - dsl::TupleType::Tuple8 => 32, - dsl::TupleType::HalfMem => length_bytes / 2, - dsl::TupleType::QuarterMem => length_bytes / 4, - dsl::TupleType::EigthMem => length_bytes / 8, - dsl::TupleType::Mem128 => 16, - dsl::TupleType::Movddup => match evex.length { - dsl::Length::LZ | dsl::Length::LIG => unimplemented!(), - dsl::Length::L128 => 8, - dsl::Length::L256 => 32, - dsl::Length::L512 => 64, - }, - }); + // + // The compressed-displacement scheme of section 2.7.5 only applies to + // the vector EVEX encodings. APX promotes legacy general-purpose- + // register instructions into extended-EVEX "map 4", and those keep + // legacy displacement semantics: `disp8` is a plain byte offset rather + // than a multiple of a tuple-derived scaling factor `N`. Scaling them + // would emit an offset wrong by a factor of `N`. Note this is specific + // to the legacy-GPR class; promoted vector instructions and APX-extended + // AVX-512 instructions still use compressed displacements. + let evex_scaling = if matches!(evex.apx, Some(dsl::ApxClass::LegacyGpr)) { + None + } else { + Some(match evex.tuple_type { + dsl::TupleType::Full => { + assert!(!bcast); + length_bytes + } + dsl::TupleType::Half => { + assert!(!bcast); + length_bytes / 2 + } + dsl::TupleType::FullMem => length_bytes, + // FIXME: according to table 2-35 this needs to take into account + // "InputSize" which isn't accounted for in our `Evex` structure at + // this time. + dsl::TupleType::Tuple1Scalar => unimplemented!(), + dsl::TupleType::Tuple1Fixed => unimplemented!(), + dsl::TupleType::Tuple2 => unimplemented!(), + dsl::TupleType::Tuple4 => unimplemented!(), + dsl::TupleType::Tuple8 => 32, + dsl::TupleType::HalfMem => length_bytes / 2, + dsl::TupleType::QuarterMem => length_bytes / 4, + dsl::TupleType::EigthMem => length_bytes / 8, + dsl::TupleType::Mem128 => 16, + dsl::TupleType::Movddup => match evex.length { + dsl::Length::LZ | dsl::Length::LIG => unimplemented!(), + dsl::Length::L128 => 8, + dsl::Length::L256 => 32, + dsl::Length::L512 => 64, + }, + }) + }; - self.generate_vex_or_evex_prefix(f, "EvexPrefix", &bits, is4, evex_scaling, || { - evex.unwrap_digit() - }) + self.generate_vex_or_evex_prefix( + f, + "EvexPrefix", + &bits, + is4, + evex_scaling, + two_op, + three_op, + || evex.unwrap_digit(), + ) } /// Helper function to generate either a vex or evex prefix, mostly handling @@ -318,6 +382,8 @@ impl dsl::Format { bits: &str, is4: bool, evex_scaling: Option, + two_op: &str, + three_op: &str, unwrap_digit: impl Fn() -> Option, ) -> ModRmStyle { use dsl::OperandKind::{FixedReg, Imm, Mem, Reg, RegMem}; @@ -330,7 +396,7 @@ impl dsl::Format { fmtln!(f, "let rm = self.{rm}.encode_bx_regs();"); fmtln!( f, - "let prefix = {prefix_type}::three_op(reg, vvvv, rm, {bits});" + "let prefix = {prefix_type}::{three_op}(reg, vvvv, rm, {bits});" ); ModRmStyle::Reg { reg: ModRmReg::Reg(*reg), @@ -347,7 +413,7 @@ impl dsl::Format { fmtln!(f, "let rm = self.{rm}.encode_bx_regs();"); fmtln!( f, - "let prefix = {prefix_type}::three_op(reg, vvvv, rm, {bits});" + "let prefix = {prefix_type}::{three_op}(reg, vvvv, rm, {bits});" ); ModRmStyle::RegMem { reg: ModRmReg::Reg(*reg), @@ -362,7 +428,7 @@ impl dsl::Format { fmtln!(f, "let rm = self.{rm}.encode_bx_regs();"); fmtln!( f, - "let prefix = {prefix_type}::three_op(reg, vvvv, rm, {bits});" + "let prefix = {prefix_type}::{three_op}(reg, vvvv, rm, {bits});" ); ModRmStyle::RegMemIs4 { reg: ModRmReg::Reg(*reg), @@ -382,7 +448,7 @@ impl dsl::Format { fmtln!(f, "let rm = self.{rm}.encode_bx_regs();"); fmtln!( f, - "let prefix = {prefix_type}::three_op(reg, vvvv, rm, {bits});" + "let prefix = {prefix_type}::{three_op}(reg, vvvv, rm, {bits});" ); ModRmStyle::RegMem { reg: ModRmReg::Digit(digit), @@ -395,7 +461,7 @@ impl dsl::Format { let reg = reg_or_vvvv; fmtln!(f, "let reg = self.{reg}.enc();"); fmtln!(f, "let rm = self.{rm}.encode_bx_regs();"); - fmtln!(f, "let prefix = {prefix_type}::two_op(reg, rm, {bits});"); + fmtln!(f, "let prefix = {prefix_type}::{two_op}(reg, rm, {bits});"); ModRmStyle::RegMem { reg: ModRmReg::Reg(*reg), rm: *rm, @@ -413,7 +479,7 @@ impl dsl::Format { fmtln!(f, "let rm = self.{rm}.encode_bx_regs();"); fmtln!( f, - "let prefix = {prefix_type}::three_op(reg, vvvv, rm, {bits});" + "let prefix = {prefix_type}::{three_op}(reg, vvvv, rm, {bits});" ); ModRmStyle::Reg { reg: ModRmReg::Digit(digit), @@ -425,7 +491,7 @@ impl dsl::Format { let reg = reg_or_vvvv; fmtln!(f, "let reg = self.{reg}.enc();"); fmtln!(f, "let rm = self.{rm}.encode_bx_regs();"); - fmtln!(f, "let prefix = {prefix_type}::two_op(reg, rm, {bits});"); + fmtln!(f, "let prefix = {prefix_type}::{two_op}(reg, rm, {bits});"); ModRmStyle::Reg { reg: ModRmReg::Reg(*reg), rm: *rm, @@ -437,7 +503,7 @@ impl dsl::Format { assert!(!is4); fmtln!(f, "let reg = self.{reg}.enc();"); fmtln!(f, "let rm = self.{rm}.encode_bx_regs();"); - fmtln!(f, "let prefix = {prefix_type}::two_op(reg, rm, {bits});"); + fmtln!(f, "let prefix = {prefix_type}::{two_op}(reg, rm, {bits});"); ModRmStyle::RegMem { reg: ModRmReg::Reg(*reg), rm: *rm, diff --git a/cranelift/assembler-x64/meta/src/generate/inst.rs b/cranelift/assembler-x64/meta/src/generate/inst.rs index a2eb137356ef..71ec243abd0a 100644 --- a/cranelift/assembler-x64/meta/src/generate/inst.rs +++ b/cranelift/assembler-x64/meta/src/generate/inst.rs @@ -324,8 +324,9 @@ impl dsl::Inst { None => fmtln!(f, "let {location} = {to_string};"), } } - let ordered_ops = self.format.generate_att_style_operands(); - let xed_ops = self.format.generate_xed_style_operands(); + let nd = self.encoding.is_nd(); + let ordered_ops = self.format.generate_att_style_operands(nd); + let xed_ops = self.format.generate_xed_style_operands(nd); let mut implicit_ops = self.format.generate_implicit_operands(); if self.has_trap { fmtln!(f, "let trap = self.trap;"); diff --git a/cranelift/assembler-x64/meta/src/instructions/add.rs b/cranelift/assembler-x64/meta/src/instructions/add.rs index f5969061b053..f4b6b4fb296f 100644 --- a/cranelift/assembler-x64/meta/src/instructions/add.rs +++ b/cranelift/assembler-x64/meta/src/instructions/add.rs @@ -97,5 +97,12 @@ pub fn list() -> Vec { inst("vphaddw", fmt("B", [w(xmm1), r(xmm2), r(xmm_m128)]), vex(L128)._66()._0f38().op(0x01).r(), (_64b | compat) & avx), inst("vphaddd", fmt("B", [w(xmm1), r(xmm2), r(xmm_m128)]), vex(L128)._66()._0f38().op(0x02).r(), (_64b | compat) & avx), inst("vaddpd", fmt("C", [w(xmm1), r(xmm2), r(xmm_m128)]), evex(L128, Full)._66()._0f().w1().op(0x58).r(), (_64b | compat) & avx512vl), + // APX: NDD (new data destination) form of `ADD`, promoted into EVEX + // "map 4" via the extended-EVEX prefix. With `ND = 1` the architectural + // destination is the `vvvv`-encoded register, so the written operand is + // placed in the `V` slot (the middle operand of the `RVM` format) while + // the two sources occupy ModRM.reg and ModRM.rm. `addq` is 64-bit so + // the `W` bit is set (`.w1()`); use `.w0()` for the 32-bit `addl` form. + inst("addq", fmt("RVM", [r(r64b), w(r64a), r(rm64)]), evex(L128, Full).map4().w1().nd().op(0x01).r(), _64b & apx), ] } diff --git a/cranelift/assembler-x64/src/evex.rs b/cranelift/assembler-x64/src/evex.rs index bf89dd13bdc1..2734fae7ac22 100644 --- a/cranelift/assembler-x64/src/evex.rs +++ b/cranelift/assembler-x64/src/evex.rs @@ -92,6 +92,107 @@ impl EvexPrefix { EvexPrefix::new(reg, vvvv, (b, x), ll, pp, mmm, w, broadcast) } + // --------------------------------------------------------------------- + // Intel APX "Extended EVEX" prefix for promoted *legacy* GPR instructions + // (EVEX map 4). See the Intel APX Architecture Specification (rev 8), + // section 3.1.2.3.1 "EVEX Extension of Legacy Instructions", Figure 3.3. + // + // The four bytes still begin with `0x62`, but several payload bits are + // re-purposed relative to the AVX-512 layout above: + // + // ┌────┬────┬────┬────┬────┬────┬────┬────┐ + // Byte 1: │ R3 │ X3 │ B3 │ R4 │ B4 │ 1 │ 0 │ 0 │ (map id = 4) + // ├────┼────┼────┼────┼────┼────┼────┼────┤ + // Byte 2: │ W │ V3 │ V2 │ V1 │ V0 │ U │ p │ p │ (U = ~X4) + // ├────┼────┼────┼────┼────┼────┼────┼────┤ + // Byte 3: │ 0 │ 0 │ 0 │ ND │ V4 │ NF │ 0 │ 0 │ + // └────┴────┴────┴────┴────┴────┴────┴────┘ + // + // The "underlined" fields (`R3`, `X3`, `B3`, `R4`, the `vvvv` bits and `V4`) + // are stored inverted, exactly as in the AVX-512 layout. `B4` and `X4` are + // newly repurposed reserved bits: `B4` uses *true* polarity (fixed value 0) + // and `X4` is carried inverted in the `U` bit (`EVEX.X4 = ~EVEX.U`), so a + // register-form instruction (ModRM.Mod = 3, no index) has `U = 1`. + + /// Construct the extended-EVEX (APX map 4) prefix for a legacy GPR + /// instruction. + /// + /// `reg` is the ModRM.reg register, `vvvv` is the `V` register identifier + /// (the NDD register when `nd` is set), and `(b, x)` are the ModRM.r/m base + /// and (optional) SIB index registers. `nd`/`nf` select the New Data + /// destination and No Flags bits respectively. + pub fn legacy( + reg: u8, + vvvv: u8, + (b, x): (Option, Option), + pp: u8, + mmm: u8, + w: bool, + nd: bool, + nf: bool, + ) -> Self { + let base = b.unwrap_or(0); + let index = x.unwrap_or(0); + + // byte1 (P0) + let r3 = invert_top_bit(reg); + let x3 = invert_top_bit(index); + let b3 = invert_top_bit(base); + let r4 = invert_top_bit(reg >> 1); + let b4 = (base >> 4) & 1; // true polarity + debug_assert!(mmm <= 0b111); + let byte1 = r3 << 7 | x3 << 6 | b3 << 5 | r4 << 4 | b4 << 3 | mmm; + + // byte2 (P1) + debug_assert!(vvvv <= 0b11111); + debug_assert!(pp <= 0b11); + let vvvv_value = !vvvv & 0b1111; + // `EVEX.X4 = ~EVEX.U`; with no index register X4 = 0 so U = 1. + let x4 = (index >> 4) & 1; + let u = (!x4) & 1; + let byte2 = (w as u8) << 7 | vvvv_value << 3 | u << 2 | (pp & 0b11); + + // byte3 (P2) + let v_prime = invert_top_bit(vvvv >> 1); // V4, inverted + let byte3 = (nd as u8) << 4 | v_prime << 3 | (nf as u8) << 2; + + Self { + byte1, + byte2, + byte3, + } + } + + /// Construct the extended-EVEX (APX map 4) prefix for a two-operand legacy + /// GPR instruction (no NDD); the `V` register identifier is unused. + #[allow(dead_code, reason = "not all APX legacy forms are emitted yet")] + pub fn legacy_two_op( + reg: u8, + (b, x): (Option, Option), + pp: u8, + mmm: u8, + w: bool, + nd: bool, + nf: bool, + ) -> Self { + EvexPrefix::legacy(reg, 0, (b, x), pp, mmm, w, nd, nf) + } + + /// Construct the extended-EVEX (APX map 4) prefix for a three-operand + /// legacy GPR instruction; `vvvv` carries the NDD register. + pub fn legacy_three_op( + reg: u8, + vvvv: u8, + (b, x): (Option, Option), + pp: u8, + mmm: u8, + w: bool, + nd: bool, + nf: bool, + ) -> Self { + EvexPrefix::legacy(reg, vvvv, (b, x), pp, mmm, w, nd, nf) + } + pub(crate) fn encode(&self, sink: &mut impl CodeSink) { sink.put1(0x62); sink.put1(self.byte1); diff --git a/cranelift/assembler-x64/src/fuzz.rs b/cranelift/assembler-x64/src/fuzz.rs index 3efe0acfc472..99166feca756 100644 --- a/cranelift/assembler-x64/src/fuzz.rs +++ b/cranelift/assembler-x64/src/fuzz.rs @@ -34,6 +34,13 @@ pub fn roundtrip(inst: &Inst) { return; } + // Likewise, capstone cannot decode the APX extended-EVEX ("map 4") + // encodings at all, returning zero instructions. These are instead checked + // against XED by `roundtrip_xed`, which does understand map 4. + if features_mention(inst.features(), Feature::apx) { + return; + } + roundtrip_with( inst, "capstone", @@ -589,4 +596,62 @@ mod test { }) .budget_ms(1_000); } + + /// Byte-level encoding check for the APX NDD (new-data-destination) form of + /// `ADD`, promoted into EVEX "map 4" via the extended-EVEX prefix. + /// + /// We assert the exact bytes rather than round-tripping through Capstone + /// because the bundled disassembler does not yet understand APX. With + /// `ND = 1` the architectural destination is the `vvvv`-encoded register, so + /// the `RVM` operands are `[ModRM.reg source, vvvv destination, ModRM.rm + /// source]`. For `addq %rax, %rcx, %rdx` (destination `%rax` = 0 in `vvvv`, + /// source `%rcx` = 1 in ModRM.reg, source `%rdx` = 2 in ModRM.rm), `W = 1`, + /// `ND = 1`, `NF = 0`, the expected sequence is: + /// + /// ```text + /// 62 F4 FC 18 01 CA + /// ^^ ^^ ^^ ^^ ^^ ^^ + /// | | | | | └ ModRM: mod=11 reg=rcx r/m=rdx + /// | | | | └ opcode 0x01 (ADD r/m, reg) + /// | | | └ P2: ND=1 (bit4), V4=1 (bit3, inverted), NF=0 + /// | | └ P1: W=1, vvvv=~rax=1111, U=1, pp=00 + /// | └ P0: R3 X3 B3 R4 = 1111, B4=0, map=100 (map 4) + /// └ EVEX identifier + /// ``` + #[test] + fn apx_addq_rvm_ndd_encoding() { + use crate::inst::addq_rvm; + // Format is `RVM` = [ModRM.reg source, vvvv destination, ModRM.rm + // source], so the constructor arguments are (reg source = %rcx, + // destination = %rax, r/m source = %rdx). + let inst = addq_rvm::::new(FuzzReg::new(1), FuzzReg::new(0), FuzzReg::new(2)); + let assembled = assemble(&inst.into()); + assert_eq!(pretty_print_hexadecimal(&assembled), "62F4FC1801CA"); + } + + /// Companion to [`apx_addq_rvm_ndd_encoding`] covering a memory operand. + /// + /// APX map-4 instructions are legacy instructions promoted into EVEX, so + /// their `disp8` keeps legacy semantics: a plain byte offset. The + /// compressed-displacement scheme that divides `disp8` by a tuple-derived + /// factor `N` applies only to the vector EVEX encodings. Encoding + /// `0x50(%rdi)` must therefore emit `0x50` and not `0x50 / 16 = 0x05`, + /// which would silently address the wrong memory. + #[test] + fn apx_addq_rvm_ndd_disp8_is_unscaled() { + use crate::inst::addq_rvm; + use crate::mem::{Amode, AmodeOffset, AmodeOffsetPlusKnownOffset, GprMem}; + + let mem: GprMem = GprMem::Mem(Amode::ImmReg { + base: FuzzReg::new(7), + simm32: AmodeOffsetPlusKnownOffset { + simm32: AmodeOffset::new(0x50), + offset: None, + }, + trap: None, + }); + let inst = addq_rvm::::new(FuzzReg::new(7), FuzzReg::new(7), mem); + let assembled = assemble(&inst.into()); + assert_eq!(pretty_print_hexadecimal(&assembled), "62F4C418017F50"); + } } diff --git a/cranelift/codegen/meta/src/isa/x86.rs b/cranelift/codegen/meta/src/isa/x86.rs index d72be07de2c4..836338027f40 100644 --- a/cranelift/codegen/meta/src/isa/x86.rs +++ b/cranelift/codegen/meta/src/isa/x86.rs @@ -95,6 +95,15 @@ pub(crate) fn define() -> TargetIsa { "AVX512VNNI: CPUID.07H:ECX.AVX512_VNNI[bit 11]", false, ); + // Registered so `target x86_64 has_apx` parses; not yet referenced by any + // preset because no shipping microarchitecture ships APX today. Drop the + // leading underscore once a preset needs it. + let _has_apx = settings.add_bool( + "has_apx", + "Has support for APX.", + "APX_F: CPUID.(EAX=07H,ECX=1H):EDX.APX_F[bit 21]", + false, + ); let has_popcnt = settings.add_bool( "has_popcnt", "Has support for POPCNT.", diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index 57639f6ff06f..ca321a0a6089 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -1336,6 +1336,9 @@ (decl pure use_avx2 () bool) (extern constructor use_avx2 use_avx2) +(decl pure use_apx () bool) +(extern constructor use_apx use_apx) + (decl pure has_cmpxchg16b () bool) (extern constructor has_cmpxchg16b has_cmpxchg16b) diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index 8134eb5dc6e7..a2e6306c94fd 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -1595,6 +1595,10 @@ impl asm::AvailableFeatures for &EmitInfo { fn avx512vnni(&self) -> bool { self.isa_flags.has_avx512vnni() } + + fn apx(&self) -> bool { + self.isa_flags.has_apx() + } } impl MachInstEmit for Inst { diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index 8739c6fb6c27..5f99cf7ab020 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -94,6 +94,15 @@ (rule iadd_base_case_32_or_64_lea -5 (lower (iadd (ty_32_or_64 ty) x y)) (x64_lea ty (to_amode_add (mem_flags_trusted_data) x y (zero_offset)))) +;; APX: when the `has_apx` feature is enabled, use the NDD (new-data- +;; destination) form of `add` for 64-bit adds. Unlike the legacy two-operand +;; `add`, the NDD form writes its result to a fresh register encoded in the +;; EVEX `vvvv` field, so the register allocator does not need to insert a move +;; to preserve either input. +(rule iadd_apx_ndd 1 (lower (iadd $I64 x y)) + (if-let true (use_apx)) + (x64_addq_rvm x y)) + ;; Higher-priority cases than the previous two where a load can be sunk into ;; the add instruction itself. Note that both operands are tested for ;; sink-ability since addition is commutative diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs index 8c04e7f89369..c5b33e162304 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle.rs @@ -492,6 +492,11 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> { self.backend.x64_flags.has_avx512vbmi() } + #[inline] + fn use_apx(&mut self) -> bool { + self.backend.x64_flags.has_apx() + } + #[inline] fn has_lzcnt(&mut self) -> bool { self.backend.x64_flags.has_lzcnt() diff --git a/cranelift/filetests/filetests/isa/x64/apx-add.clif b/cranelift/filetests/filetests/isa/x64/apx-add.clif new file mode 100644 index 000000000000..94fcbfd18071 --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/apx-add.clif @@ -0,0 +1,26 @@ +test compile precise-output +target x86_64 has_apx + +function %add_i64(i64, i64) -> i64 { +block0(v0: i64, v1: i64): + v2 = iadd v0, v1 + return v2 +} + +; VCode: +; pushq %rbp +; movq %rsp, %rbp +; block0: +; addq %rdi, %rsi, %rax +; movq %rbp, %rsp +; popq %rbp +; retq +; +; Disassembled: +; block0: ; offset 0x0 +; pushq %rbp +; movq %rsp, %rbp +; block1: ; offset 0x4 +; .byte 0x62, 0xf4, 0xfc, 0x18, 0x01, 0xfe, 0x48, 0x89 +; .byte 0xec, 0x5d, 0xc3 +