Add TLS support for ELF and MachO (bytecodealliance#1174)

* Add TLS support
* Add binemit and legalize tests
* Spill all caller-saved registers when necessary

cranelift/codegen/meta/src/cdsl/instructions.rs

@@ -340,7 +340,7 @@ impl InstructionBuilder {
         let polymorphic_info =
             verify_polymorphic(&operands_in, &operands_out, &self.format, &value_opnums);
 
-        // Infer from output operands whether an instruciton clobbers CPU flags or not.
+        // Infer from output operands whether an instruction clobbers CPU flags or not.
         let writes_cpu_flags = operands_out.iter().any(|op| op.is_cpu_flags());
 
         let camel_name = camel_case(&self.name);

cranelift/codegen/meta/src/isa/x86/encodings.rs

@@ -2407,5 +2407,14 @@ pub(crate) fn define(
     define_control_flow(&mut e, shared_defs, settings, r);
     define_reftypes(&mut e, shared_defs, r);
 
+    let x86_elf_tls_get_addr = x86.by_name("x86_elf_tls_get_addr");
+    let x86_macho_tls_get_addr = x86.by_name("x86_macho_tls_get_addr");
+
+    let rec_elf_tls_get_addr = r.recipe("elf_tls_get_addr");
+    let rec_macho_tls_get_addr = r.recipe("macho_tls_get_addr");
+
+    e.enc64_rec(x86_elf_tls_get_addr, rec_elf_tls_get_addr, 0);
+    e.enc64_rec(x86_macho_tls_get_addr, rec_macho_tls_get_addr, 0);
+
     e
 }

cranelift/codegen/meta/src/isa/x86/instructions.rs

@@ -7,6 +7,7 @@ use crate::cdsl::operands::Operand;
 use crate::cdsl::types::ValueType;
 use crate::cdsl::typevar::{Interval, TypeSetBuilder, TypeVar};
 
+use crate::shared::entities::EntityRefs;
 use crate::shared::formats::Formats;
 use crate::shared::immediates::Immediates;
 use crate::shared::types;
@@ -16,6 +17,7 @@ pub(crate) fn define(
     mut all_instructions: &mut AllInstructions,
     formats: &Formats,
     immediates: &Immediates,
+    entities: &EntityRefs,
 ) -> InstructionGroup {
     let mut ig = InstructionGroupBuilder::new(&mut all_instructions);
 
@@ -542,5 +544,39 @@ pub(crate) fn define(
         .operands_out(vec![a]),
     );
 
+    let i64_t = &TypeVar::new(
+        "i64_t",
+        "A scalar 64bit integer",
+        TypeSetBuilder::new().ints(64..64).build(),
+    );
+
+    let GV = &Operand::new("GV", &entities.global_value);
+    let addr = &Operand::new("addr", i64_t);
+
+    ig.push(
+        Inst::new(
+            "x86_elf_tls_get_addr",
+            r#"
+        Elf tls get addr -- This implements the GD TLS model for ELF. The clobber output should
+        not be used.
+            "#,
+            &formats.unary_global_value,
+        )
+        .operands_in(vec![GV])
+        .operands_out(vec![addr]),
+    );
+    ig.push(
+        Inst::new(
+            "x86_macho_tls_get_addr",
+            r#"
+        Mach-O tls get addr -- This implements TLS access for Mach-O. The clobber output should
+        not be used.
+            "#,
+            &formats.unary_global_value,
+        )
+        .operands_in(vec![GV])
+        .operands_out(vec![addr]),
+    );
+
     ig.build()
 }

cranelift/codegen/meta/src/isa/x86/legalize.rs

@@ -61,6 +61,7 @@ pub(crate) fn define(shared: &mut SharedDefinitions, x86_instructions: &Instruct
     let shuffle = insts.by_name("shuffle");
     let srem = insts.by_name("srem");
     let sshr = insts.by_name("sshr");
+    let tls_value = insts.by_name("tls_value");
     let trueif = insts.by_name("trueif");
     let udiv = insts.by_name("udiv");
     let umax = insts.by_name("umax");
@@ -326,6 +327,8 @@ pub(crate) fn define(shared: &mut SharedDefinitions, x86_instructions: &Instruct
 
     group.custom_legalize(ineg, "convert_ineg");
 
+    group.custom_legalize(tls_value, "expand_tls_value");
+
     group.build_and_add_to(&mut shared.transform_groups);
 
     let mut narrow = TransformGroupBuilder::new(

cranelift/codegen/meta/src/isa/x86/mod.rs

@@ -24,6 +24,7 @@ pub(crate) fn define(shared_defs: &mut SharedDefinitions) -> TargetIsa {
         &mut shared_defs.all_instructions,
         &shared_defs.formats,
         &shared_defs.imm,
+        &shared_defs.entities,
     );
     legalize::define(shared_defs, &inst_group);
 

cranelift/codegen/meta/src/isa/x86/recipes.rs

@@ -3259,10 +3259,73 @@ pub(crate) fn define<'shared>(
     recipes.add_recipe(
         EncodingRecipeBuilder::new("safepoint", &formats.multiary, 0).emit(
             r#"
-            sink.add_stackmap(args, func, isa);
-        "#,
+                sink.add_stackmap(args, func, isa);
+            "#,
         ),
     );
 
+    // Both `elf_tls_get_addr` and `macho_tls_get_addr` require all caller-saved registers to be spilled.
+    // This is currently special cased in `regalloc/spilling.rs` in the `visit_inst` function.
+
+    recipes.add_recipe(
+        EncodingRecipeBuilder::new("elf_tls_get_addr", &formats.unary_global_value, 16)
+            // FIXME Correct encoding for non rax registers
+            .operands_out(vec![reg_rax])
+            .emit(
+                r#"
+                    // output %rax
+                    // clobbers %rdi
+
+                    // Those data16 prefixes are necessary to pad to 16 bytes.
+
+                    // data16 lea gv@tlsgd(%rip),%rdi
+                    sink.put1(0x66); // data16
+                    sink.put1(0b01001000); // rex.w
+                    const LEA: u8 = 0x8d;
+                    sink.put1(LEA); // lea
+                    modrm_riprel(0b111/*out_reg0*/, sink); // 0x3d
+                    sink.reloc_external(Reloc::ElfX86_64TlsGd,
+                                        &func.global_values[global_value].symbol_name(),
+                                        -4);
+                    sink.put4(0);
+
+                    // data16 data16 callq __tls_get_addr-4
+                    sink.put1(0x66); // data16
+                    sink.put1(0x66); // data16
+                    sink.put1(0b01001000); // rex.w
+                    sink.put1(0xe8); // call
+                    sink.reloc_external(Reloc::X86CallPLTRel4,
+                                        &ExternalName::LibCall(LibCall::ElfTlsGetAddr),
+                                        -4);
+                    sink.put4(0);
+                "#,
+            ),
+    );
+
+    recipes.add_recipe(
+        EncodingRecipeBuilder::new("macho_tls_get_addr", &formats.unary_global_value, 9)
+            // FIXME Correct encoding for non rax registers
+            .operands_out(vec![reg_rax])
+            .emit(
+                r#"
+                    // output %rax
+                    // clobbers %rdi
+
+                    // movq gv@tlv(%rip), %rdi
+                    sink.put1(0x48); // rex
+                    sink.put1(0x8b); // mov
+                    modrm_riprel(0b111/*out_reg0*/, sink); // 0x3d
+                    sink.reloc_external(Reloc::MachOX86_64Tlv,
+                                        &func.global_values[global_value].symbol_name(),
+                                        -4);
+                    sink.put4(0);
+
+                    // callq *(%rdi)
+                    sink.put1(0xff);
+                    sink.put1(0x17);
+                "#,
+            ),
+    );
+
     recipes
 }

cranelift/codegen/meta/src/shared/instructions.rs

@@ -1127,6 +1127,18 @@ pub(crate) fn define(
         .operands_out(vec![a]),
     );
 
+    ig.push(
+        Inst::new(
+            "tls_value",
+            r#"
+        Compute the value of global GV, which is a TLS (thread local storage) value.
+        "#,
+            &formats.unary_global_value,
+        )
+        .operands_in(vec![GV])
+        .operands_out(vec![a]),
+    );
+
     let HeapOffset = &TypeVar::new(
         "HeapOffset",
         "An unsigned heap offset",

cranelift/codegen/meta/src/shared/mod.rs

@@ -1,6 +1,6 @@
 //! Shared definitions for the Cranelift intermediate language.
 
-mod entities;
+pub mod entities;
 pub mod formats;
 pub mod immediates;
 pub mod instructions;
@@ -28,6 +28,7 @@ pub(crate) struct Definitions {
     pub imm: Immediates,
     pub formats: Formats,
     pub transform_groups: TransformGroups,
+    pub entities: EntityRefs,
 }
 
 pub(crate) fn define() -> Definitions {
@@ -47,6 +48,7 @@ pub(crate) fn define() -> Definitions {
         imm: immediates,
         formats,
         transform_groups,
+        entities,
     }
 }
 

cranelift/codegen/meta/src/shared/settings.rs

@@ -131,6 +131,14 @@ pub(crate) fn define() -> SettingGroup {
         false,
     );
 
+    settings.add_enum(
+        "tls_model",
+        r#"
+            Defines the model used to perform TLS accesses.
+        "#,
+        vec!["none", "elf_gd", "macho", "coff"],
+    );
+
     // Settings specific to the `baldrdash` calling convention.
 
     settings.add_enum(

cranelift/codegen/src/binemit/mod.rs

@@ -56,6 +56,12 @@ pub enum Reloc {
     Arm64Call,
     /// RISC-V call target
     RiscvCall,
+
+    /// Elf x86_64 32 bit signed PC relative offset to two GOT entries for GD symbol.
+    ElfX86_64TlsGd,
+
+    /// Mach-O x86_64 32 bit signed PC relative offset to a `__thread_vars` entry.
+    MachOX86_64Tlv,
 }
 
 impl fmt::Display for Reloc {
@@ -71,6 +77,9 @@ impl fmt::Display for Reloc {
             Self::X86CallPLTRel4 => write!(f, "CallPLTRel4"),
             Self::X86GOTPCRel4 => write!(f, "GOTPCRel4"),
             Self::Arm32Call | Self::Arm64Call | Self::RiscvCall => write!(f, "Call"),
+
+            Self::ElfX86_64TlsGd => write!(f, "ElfX86_64TlsGd"),
+            Self::MachOX86_64Tlv => write!(f, "MachOX86_64Tlv"),
         }
     }
 }

cranelift/codegen/src/ir/globalvalue.rs

@@ -63,6 +63,9 @@ pub enum GlobalValueData {
         /// away, after linking? If so, references to it can avoid going through a GOT. Note that
         /// symbols meant to be preemptible cannot be colocated.
         colocated: bool,
+
+        /// Does this symbol refer to a thread local storage value?
+        tls: bool,
     },
 }
 
@@ -110,11 +113,13 @@ impl fmt::Display for GlobalValueData {
                 ref name,
                 offset,
                 colocated,
+                tls,
             } => {
                 write!(
                     f,
-                    "symbol {}{}",
+                    "symbol {}{}{}",
                     if colocated { "colocated " } else { "" },
+                    if tls { "tls " } else { "" },
                     name
                 )?;
                 let offset_val: i64 = offset.into();

cranelift/codegen/src/ir/libcall.rs

@@ -46,6 +46,9 @@ pub enum LibCall {
     Memset,
     /// libc.memmove
     Memmove,
+
+    /// Elf __tls_get_addr
+    ElfTlsGetAddr,
 }
 
 impl fmt::Display for LibCall {
@@ -71,6 +74,8 @@ impl FromStr for LibCall {
             "Memcpy" => Ok(Self::Memcpy),
             "Memset" => Ok(Self::Memset),
             "Memmove" => Ok(Self::Memmove),
+
+            "ElfTlsGetAddr" => Ok(Self::ElfTlsGetAddr),
             _ => Err(()),
         }
     }

cranelift/codegen/src/isa/x86/binemit.rs

@@ -4,7 +4,10 @@ use super::enc_tables::{needs_offset, needs_sib_byte};
 use super::registers::RU;
 use crate::binemit::{bad_encoding, CodeSink, Reloc};
 use crate::ir::condcodes::{CondCode, FloatCC, IntCC};
-use crate::ir::{Block, Constant, Function, Inst, InstructionData, JumpTable, Opcode, TrapCode};
+use crate::ir::{
+    Block, Constant, ExternalName, Function, Inst, InstructionData, JumpTable, LibCall, Opcode,
+    TrapCode,
+};
 use crate::isa::{RegUnit, StackBase, StackBaseMask, StackRef, TargetIsa};
 use crate::regalloc::RegDiversions;
 

cranelift/codegen/src/isa/x86/enc_tables.rs

@@ -1288,3 +1288,40 @@ fn convert_ineg(
         unreachable!()
     }
 }
+
+fn expand_tls_value(
+    inst: ir::Inst,
+    func: &mut ir::Function,
+    _cfg: &mut ControlFlowGraph,
+    isa: &dyn TargetIsa,
+) {
+    use crate::settings::TlsModel;
+
+    assert!(
+        isa.triple().architecture == target_lexicon::Architecture::X86_64,
+        "Not yet implemented for {:?}",
+        isa.triple(),
+    );
+
+    if let ir::InstructionData::UnaryGlobalValue {
+        opcode: ir::Opcode::TlsValue,
+        global_value,
+    } = func.dfg[inst]
+    {
+        let ctrl_typevar = func.dfg.ctrl_typevar(inst);
+        assert_eq!(ctrl_typevar, ir::types::I64);
+
+        match isa.flags().tls_model() {
+            TlsModel::None => panic!("tls_model flag is not set."),
+            TlsModel::ElfGd => {
+                func.dfg.replace(inst).x86_elf_tls_get_addr(global_value);
+            }
+            TlsModel::Macho => {
+                func.dfg.replace(inst).x86_macho_tls_get_addr(global_value);
+            }
+            model => unimplemented!("tls_value for tls model {:?}", model),
+        }
+    } else {
+        unreachable!();
+    }
+}

cranelift/codegen/src/legalizer/globalvalue.rs

@@ -40,7 +40,7 @@ pub fn expand_global_value(
             global_type,
             readonly,
         } => load_addr(inst, func, base, offset, global_type, readonly, isa),
-        ir::GlobalValueData::Symbol { .. } => symbol(inst, func, gv, isa),
+        ir::GlobalValueData::Symbol { tls, .. } => symbol(inst, func, gv, isa, tls),
     }
 }
 
@@ -123,7 +123,18 @@ fn load_addr(
 }
 
 /// Expand a `global_value` instruction for a symbolic name global.
-fn symbol(inst: ir::Inst, func: &mut ir::Function, gv: ir::GlobalValue, isa: &dyn TargetIsa) {
+fn symbol(
+    inst: ir::Inst,
+    func: &mut ir::Function,
+    gv: ir::GlobalValue,
+    isa: &dyn TargetIsa,
+    tls: bool,
+) {
     let ptr_ty = isa.pointer_type();
-    func.dfg.replace(inst).symbol_value(ptr_ty, gv);
+
+    if tls {
+        func.dfg.replace(inst).tls_value(ptr_ty, gv);
+    } else {
+        func.dfg.replace(inst).symbol_value(ptr_ty, gv);
+    }
 }