@mulCarryless() - add a builtin to llvm backend

addresses ziglang#9631. only works with llvm backend/x86 so far. allows new test/behavior/mul_carryless.zig to pass with -Denable-llvm. doesn't do any backend/arch/cpu-feature testing.
travisstaloch · Nov 8, 2022 · 4ebc8d9 · 4ebc8d9
1 parent 694d883
commit 4ebc8d9
Show file tree

Hide file tree

Showing 21 changed files with 245 additions and 35 deletions.
diff --git a/lib/std/os/linux.zig b/lib/std/os/linux.zig
@@ -2823,46 +2823,46 @@ pub const DT = struct {
 
 pub const T = struct {
     pub const CGETS = if (is_mips) 0x540D else 0x5401;
-    pub const CSETS = 0x5402;
-    pub const CSETSW = 0x5403;
-    pub const CSETSF = 0x5404;
-    pub const CGETA = 0x5405;
-    pub const CSETA = 0x5406;
-    pub const CSETAW = 0x5407;
-    pub const CSETAF = 0x5408;
-    pub const CSBRK = 0x5409;
-    pub const CXONC = 0x540A;
-    pub const CFLSH = 0x540B;
-    pub const IOCEXCL = 0x540C;
-    pub const IOCNXCL = 0x540D;
-    pub const IOCSCTTY = 0x540E;
-    pub const IOCGPGRP = 0x540F;
-    pub const IOCSPGRP = 0x5410;
+    pub const CSETS = if (is_mips) 0x540e else 0x5402;
+    pub const CSETSW = if (is_mips) 0x540f else 0x5403;
+    pub const CSETSF = if (is_mips) 0x5410 else 0x5404;
+    pub const CGETA = if (is_mips) 0x5401 else 0x5405;
+    pub const CSETA = if (is_mips) 0x5402 else 0x5406;
+    pub const CSETAW = if (is_mips) 0x5403 else 0x5407;
+    pub const CSETAF = if (is_mips) 0x5404 else 0x5408;
+    pub const CSBRK = if (is_mips) 0x5405 else 0x5409;
+    pub const CXONC = if (is_mips) 0x5406 else 0x540A;
+    pub const CFLSH = if (is_mips) 0x5407 else 0x540B;
+    pub const IOCEXCL = if (is_mips) 0x740d else 0x540C;
+    pub const IOCNXCL = if (is_mips) 0x740e else 0x540D;
+    pub const IOCSCTTY = if (is_mips) 0x7472 else 0x540E;
+    pub const IOCGPGRP = if (is_mips) 0x5472 else 0x540F;
+    pub const IOCSPGRP = if (is_mips) 0x741d else 0x5410;
     pub const IOCOUTQ = if (is_mips) 0x7472 else 0x5411;
-    pub const IOCSTI = 0x5412;
+    pub const IOCSTI = if (is_mips) 0x5472 else 0x5412;
     pub const IOCGWINSZ = if (is_mips or is_ppc64) 0x40087468 else 0x5413;
     pub const IOCSWINSZ = if (is_mips or is_ppc64) 0x80087467 else 0x5414;
-    pub const IOCMGET = 0x5415;
-    pub const IOCMBIS = 0x5416;
-    pub const IOCMBIC = 0x5417;
-    pub const IOCMSET = 0x5418;
-    pub const IOCGSOFTCAR = 0x5419;
-    pub const IOCSSOFTCAR = 0x541A;
+    pub const IOCMGET = if (is_mips) 0x741d else 0x5415;
+    pub const IOCMBIS = if (is_mips) 0x741b else 0x5416;
+    pub const IOCMBIC = if (is_mips) 0x741c else 0x5417;
+    pub const IOCMSET = if (is_mips) 0x741a else 0x5418;
+    pub const IOCGSOFTCAR = if (is_mips) 0x5481 else 0x5419;
+    pub const IOCSSOFTCAR = if (is_mips) 0x5482 else 0x541A;
     pub const FIONREAD = if (is_mips) 0x467F else 0x541B;
     pub const IOCINQ = FIONREAD;
-    pub const IOCLINUX = 0x541C;
-    pub const IOCCONS = 0x541D;
-    pub const IOCGSERIAL = 0x541E;
-    pub const IOCSSERIAL = 0x541F;
-    pub const IOCPKT = 0x5420;
-    pub const FIONBIO = 0x5421;
-    pub const IOCNOTTY = 0x5422;
-    pub const IOCSETD = 0x5423;
-    pub const IOCGETD = 0x5424;
-    pub const CSBRKP = 0x5425;
+    pub const IOCLINUX = if (is_mips) 0x5483 else 0x541C;
+    pub const IOCCONS = if (is_mips) IOCTL.IOW('t', 120, c_int) else 0x541D;
+    pub const IOCGSERIAL = if (is_mips) 0x5484 else 0x541E;
+    pub const IOCSSERIAL = if (is_mips) 0x5485 else 0x541F;
+    pub const IOCPKT = if (is_mips) 0x5470 else 0x5420;
+    pub const FIONBIO = if (is_mips) 0x667e else 0x5421;
+    pub const IOCNOTTY = if (is_mips) 0x5471 else 0x5422;
+    pub const IOCSETD = if (is_mips) 0x7401 else 0x5423;
+    pub const IOCGETD = if (is_mips) 0x7400 else 0x5424;
+    pub const CSBRKP = if (is_mips) 0x5486 else 0x5425;
     pub const IOCSBRK = 0x5427;
     pub const IOCCBRK = 0x5428;
-    pub const IOCGSID = 0x5429;
+    pub const IOCGSID = if (is_mips) 0x7416 else 0x5429;
     pub const IOCGRS485 = 0x542E;
     pub const IOCSRS485 = 0x542F;
     pub const IOCGPTN = IOCTL.IOR('T', 0x30, c_uint);

diff --git a/src/Air.zig b/src/Air.zig
@@ -178,6 +178,11 @@ pub const Inst = struct {
         /// and if an overflow happens, ov is 1. Otherwise ov is 0.
         /// Uses the `ty_pl` field. Payload is `Bin`.
         shl_with_overflow,
+        /// Carryless multiplication. Both operands are guaranteed to be the same type,
+        /// Result type is the same as both operands.
+        /// Uses the `ty_pl` field. Payload is `MulCarryless`.
+        /// Uses the `ty` field.
+        mul_carryless,
         /// Allocates stack local memory.
         /// Uses the `ty` field.
         alloc,
@@ -897,6 +902,12 @@ pub const Shuffle = struct {
     mask_len: u32,
 };
 
+pub const MulCarryless = struct {
+    a: Inst.Ref,
+    b: Inst.Ref,
+    imm: Inst.Ref,
+};
+
 pub const VectorCmp = struct {
     lhs: Inst.Ref,
     rhs: Inst.Ref,
@@ -1222,7 +1233,11 @@ pub fn typeOfIndex(air: Air, inst: Air.Inst.Index) Type {
             const extra = air.extraData(Air.Bin, datas[inst].pl_op.payload).data;
             return air.typeOf(extra.lhs);
         },
-
+        .mul_carryless => {
+            // const extra = air.extraData(Air.MulCarryless, datas[inst].ty_pl.payload).data;
+            // return air.typeOf(extra.a);
+            return air.getRefType(datas[inst].ty_pl.ty);
+        },
         .@"try" => {
             const err_union_ty = air.typeOf(datas[inst].pl_op.operand);
             return err_union_ty.errorUnionPayload();

diff --git a/src/AstGen.zig b/src/AstGen.zig
@@ -8100,7 +8100,15 @@ fn builtinCall(
             });
             return rvalue(gz, ri, result, node);
         },
-
+        .mul_carryless => {
+            const result = try gz.addExtendedPayload(.mul_carryless, Zir.Inst.MulCarryless{
+                .node = gz.nodeIndexToRelative(node),
+                .a = try expr(gz, scope, .{ .rl = .none }, params[0]),
+                .b = try expr(gz, scope, .{ .rl = .none }, params[1]),
+                .imm = try expr(gz, scope, .{ .rl = .none }, params[2]),
+            });
+            return rvalue(gz, ri, result, node);
+        },
         .atomic_load => {
             const result = try gz.addPlNode(.atomic_load, node, Zir.Inst.AtomicLoad{
                 // zig fmt: off

diff --git a/src/BuiltinFn.zig b/src/BuiltinFn.zig
@@ -66,6 +66,7 @@ pub const Tag = enum {
     wasm_memory_grow,
     mod,
     mul_with_overflow,
+    mul_carryless,
     panic,
     pop_count,
     prefetch,
@@ -611,6 +612,13 @@ pub const list = list: {
                 .param_count = 4,
             },
         },
+        .{
+            "@mulCarryless",
+            .{
+                .tag = .mul_carryless,
+                .param_count = 3,
+            },
+        },
         .{
             "@panic",
             .{

diff --git a/src/Liveness.zig b/src/Liveness.zig
@@ -405,6 +405,13 @@ pub fn categorizeOperand(
             if (extra.b == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
             return .none;
         },
+        .mul_carryless => {
+            const extra = air.extraData(Air.MulCarryless, air_datas[inst].ty_pl.payload).data;
+            if (extra.a == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
+            if (extra.b == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
+            if (extra.imm == operand_ref) return matchOperandSmallIndex(l, inst, 2, .none);
+            return .none;
+        },
         .reduce, .reduce_optimized => {
             const reduce = air_datas[inst].reduce;
             if (reduce.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
@@ -905,6 +912,11 @@ fn analyzeInst(
             const extra = a.air.extraData(Air.Bin, ty_pl.payload).data;
             return trackOperands(a, new_set, inst, main_tomb, .{ extra.lhs, extra.rhs, .none });
         },
+        .mul_carryless => {
+            const ty_pl = inst_datas[inst].ty_pl;
+            const extra = a.air.extraData(Air.MulCarryless, ty_pl.payload).data;
+            return trackOperands(a, new_set, inst, main_tomb, .{ extra.a, extra.b, extra.imm });
+        },
 
         .dbg_var_ptr,
         .dbg_var_val,

diff --git a/src/Sema.zig b/src/Sema.zig
@@ -1036,6 +1036,7 @@ fn analyzeBodyInner(
                     .sub_with_overflow     => try sema.zirOverflowArithmetic(block, extended, extended.opcode),
                     .mul_with_overflow     => try sema.zirOverflowArithmetic(block, extended, extended.opcode),
                     .shl_with_overflow     => try sema.zirOverflowArithmetic(block, extended, extended.opcode),
+                    .mul_carryless         => try sema.zirMulCarryless(      block, extended),
                     .c_undef               => try sema.zirCUndef(            block, extended),
                     .c_include             => try sema.zirCInclude(          block, extended),
                     .c_define              => try sema.zirCDefine(           block, extended),
@@ -13404,6 +13405,69 @@ fn zirRem(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Ins
     return block.addBinOp(air_tag, casted_lhs, casted_rhs);
 }
 
+fn zirMulCarryless(
+    sema: *Sema,
+    block: *Block,
+    extended: Zir.Inst.Extended.InstData,
+) CompileError!Air.Inst.Ref {
+    const tracy = trace(@src());
+    defer tracy.end();
+
+    const extra = sema.code.extraData(Zir.Inst.MulCarryless, extended.operand).data;
+    const src = LazySrcLoc.nodeOffset(extra.node);
+    sema.src = src;
+    const a_src: LazySrcLoc = .{ .node_offset_builtin_call_arg0 = extra.node };
+    const b_src: LazySrcLoc = .{ .node_offset_builtin_call_arg1 = extra.node };
+    const imm_src: LazySrcLoc = .{ .node_offset_builtin_call_arg2 = extra.node };
+
+    const a = try sema.resolveInst(extra.a);
+    const b = try sema.resolveInst(extra.b);
+    const imm = try sema.resolveInst(extra.imm);
+
+    const a_ty = sema.typeOf(a);
+    const b_ty = sema.typeOf(b);
+    const imm_ty = sema.typeOf(imm);
+    const mod = sema.mod;
+    const target = mod.getTarget();
+
+    try sema.checkVectorizableBinaryOperands(block, src, a_ty, b_ty, a_src, b_src);
+
+    if (a_ty.scalarType().zigTypeTag() != .Int)
+        return sema.fail(block, a_src, "expected vector of 64 bit integers, found '{}'", .{a_ty.fmt(mod)});
+    const a_bit_size = try a_ty.scalarType().bitSizeAdvanced(target, sema.kit(block, src));
+    if (a_bit_size != 64)
+        return sema.fail(block, a_src, "expected vector of 64 bit integers, found '{}'", .{a_ty.fmt(mod)});
+
+    if (!imm_ty.isInt())
+        return sema.fail(block, imm_src, "imm must be a comptime known 8 bit integer", .{});
+    const bit_size = try imm_ty.bitSizeAdvanced(target, sema.kit(block, src));
+    if (bit_size != 8)
+        return sema.fail(block, imm_src, "imm must be a comptime known 8 bit integer", .{});
+    if (!try sema.isComptimeKnown(block, imm_src, imm))
+        return sema.fail(block, imm_src, "imm must be a comptime known 8 bit integer", .{});
+
+    const instructions = &[_]Air.Inst.Ref{ a, b };
+    const resolved_type = try sema.resolvePeerTypes(block, src, instructions, .{
+        .override = &[_]LazySrcLoc{ a_src, b_src },
+    });
+    const casted_a = try sema.coerce(block, resolved_type, a, a_src);
+    const casted_b = try sema.coerce(block, resolved_type, b, b_src);
+
+    return block.addInst(.{
+        .tag = .mul_carryless,
+        .data = .{
+            .ty_pl = .{
+                .ty = try block.sema.addType(sema.typeOf(casted_a)),
+                .payload = try block.sema.addExtra(Air.MulCarryless{
+                    .a = casted_a,
+                    .b = casted_b,
+                    .imm = imm,
+                }),
+            },
+        },
+    });
+}
+
 fn zirOverflowArithmetic(
     sema: *Sema,
     block: *Block,

diff --git a/src/Zir.zig b/src/Zir.zig
@@ -1927,6 +1927,10 @@ pub const Inst = struct {
         /// `operand` is payload index to `OverflowArithmetic`.
         /// `small` is unused.
         mul_with_overflow,
+        /// Implements the `@mulCarryless` builtin.
+        /// `operand` is payload index to `MulCarryless`.
+        /// `small` is unused.
+        mul_carryless,
         /// Implements the `@shlWithOverflow` builtin.
         /// `operand` is payload index to `OverflowArithmetic`.
         /// `small` is unused.
@@ -3424,6 +3428,13 @@ pub const Inst = struct {
         ptr: Ref,
     };
 
+    pub const MulCarryless = struct {
+        node: i32,
+        a: Ref,
+        b: Ref,
+        imm: Ref,
+    };
+
     pub const Cmpxchg = struct {
         node: i32,
         ptr: Ref,

diff --git a/src/arch/aarch64/CodeGen.zig b/src/arch/aarch64/CodeGen.zig
@@ -594,6 +594,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
             .mul_with_overflow => try self.airMulWithOverflow(inst),
             .shl_with_overflow => try self.airShlWithOverflow(inst),
 
+            .mul_carryless => @panic("TODO"),
+
             .cmp_lt  => try self.airCmp(inst, .lt),
             .cmp_lte => try self.airCmp(inst, .lte),
             .cmp_eq  => try self.airCmp(inst, .eq),

diff --git a/src/arch/arm/CodeGen.zig b/src/arch/arm/CodeGen.zig
@@ -619,6 +619,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
             .mul_with_overflow => try self.airMulWithOverflow(inst),
             .shl_with_overflow => try self.airShlWithOverflow(inst),
 
+            .mul_carryless => @panic("TODO"),
+
             .cmp_lt  => try self.airCmp(inst, .lt),
             .cmp_lte => try self.airCmp(inst, .lte),
             .cmp_eq  => try self.airCmp(inst, .eq),

diff --git a/src/arch/riscv64/CodeGen.zig b/src/arch/riscv64/CodeGen.zig
@@ -524,6 +524,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
             .mul_with_overflow => try self.airMulWithOverflow(inst),
             .shl_with_overflow => try self.airShlWithOverflow(inst),
 
+            .mul_carryless => @panic("TODO"),
+
             .div_float, .div_trunc, .div_floor, .div_exact => try self.airDiv(inst),
 
             .cmp_lt  => try self.airCmp(inst, .lt),

diff --git a/src/arch/sparc64/CodeGen.zig b/src/arch/sparc64/CodeGen.zig
@@ -546,6 +546,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
             .mul_with_overflow => try self.airMulWithOverflow(inst),
             .shl_with_overflow => try self.airShlWithOverflow(inst),
 
+            .mul_carryless => @panic("TODO"),
+
             .div_float, .div_trunc, .div_floor, .div_exact => try self.airDiv(inst),
 
             .cmp_lt  => try self.airCmp(inst, .lt),

diff --git a/src/arch/wasm/CodeGen.zig b/src/arch/wasm/CodeGen.zig
@@ -1703,6 +1703,8 @@ fn genInst(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
         .shl_with_overflow => func.airShlWithOverflow(inst),
         .mul_with_overflow => func.airMulWithOverflow(inst),
 
+        .mul_carryless => @panic("TODO"),
+
         .clz => func.airClz(inst),
         .ctz => func.airCtz(inst),
 

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
@@ -622,6 +622,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
             .mul_with_overflow => try self.airMulWithOverflow(inst),
             .shl_with_overflow => try self.airAddSubShlWithOverflow(inst),
 
+            .mul_carryless => @panic("TODO"),
+
             .div_float, .div_trunc, .div_floor, .div_exact => try self.airMulDivBinOp(inst),
 
             .cmp_lt  => try self.airCmp(inst, .lt),

diff --git a/src/codegen/c.zig b/src/codegen/c.zig
@@ -2413,6 +2413,8 @@ fn genBody(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail, OutO
             .mul_with_overflow => try airOverflow(f, inst, "mul", .Bits),
             .shl_with_overflow => try airOverflow(f, inst, "shl", .Bits),
 
+            .mul_carryless => return f.fail("TODO: C backend: implement @mulCarryless()", .{}),
+
             .min => try airMinMax(f, inst, '<', "fmin"),
             .max => try airMinMax(f, inst, '>', "fmax"),
 

diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig
@@ -4484,6 +4484,8 @@ pub const FuncGen = struct {
                 .mul_with_overflow => try self.airOverflow(inst, "llvm.smul.with.overflow", "llvm.umul.with.overflow"),
                 .shl_with_overflow => try self.airShlWithOverflow(inst),
 
+                .mul_carryless      => try self.airMulCarryless(inst),
+
                 .bit_and, .bool_and => try self.airAnd(inst),
                 .bit_or, .bool_or   => try self.airOr(inst),
                 .xor                => try self.airXor(inst),
@@ -8785,6 +8787,17 @@ pub const FuncGen = struct {
         return self.builder.buildVectorSplat(len, scalar, "");
     }
 
+    fn airMulCarryless(self: *FuncGen, inst: Air.Inst.Index) !?*llvm.Value {
+        if (self.liveness.isUnused(inst)) return null;
+
+        const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
+        const extra = self.air.extraData(Air.MulCarryless, ty_pl.payload).data;
+        const a = try self.resolveInst(extra.a);
+        const b = try self.resolveInst(extra.b);
+        const imm = try self.resolveInst(extra.imm);
+        return self.builder.buildMulcl(a, b, imm, "");
+    }
+
     fn airSelect(self: *FuncGen, inst: Air.Inst.Index) !?*llvm.Value {
         if (self.liveness.isUnused(inst)) return null;
 

diff --git a/src/codegen/llvm/bindings.zig b/src/codegen/llvm/bindings.zig
@@ -736,6 +736,15 @@ pub const Builder = opaque {
         Name: [*:0]const u8,
     ) *Value;
 
+    pub const buildMulcl = ZigLLVMBuildMulcl;
+    extern fn ZigLLVMBuildMulcl(
+        *Builder,
+        A: *Value,
+        B: *Value,
+        IMM: *Value,
+        Name: [*:0]const u8,
+    ) *Value;
+
     pub const buildPtrToInt = LLVMBuildPtrToInt;
     extern fn LLVMBuildPtrToInt(
         *Builder,