Skip to content

Commit

Permalink
Merge pull request #9458 from SuperAuguste/popcount-ctz-clz
Browse files Browse the repository at this point in the history 
Vector support for `@popCount`, `@ctz`, and `@clz`
  • Loading branch information
@andrewrk
andrewrk authored Jul 26, 2021
2 parents a9a4fd3 + dd79615 commit fc105f2
Show file tree
Hide file tree
Showing 7 changed files with 258 additions and 49 deletions.
32 changes: 19 additions & 13 deletions doc/langref.html.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7366,18 +7366,20 @@ pub const CallOptions = struct {
{#header_close#}

{#header_open|@clz#}
<pre>{#syntax#}@clz(comptime T: type, integer: T){#endsyntax#}</pre>
<pre>{#syntax#}@clz(comptime T: type, operand: T){#endsyntax#}</pre>
<p>{#syntax#}T{#endsyntax#} must be an integer type.</p>
<p>{#syntax#}operand{#endsyntax#} may be an {#link|integer|Integers#} or {#link|vector|Vectors#}.</p>
<p>
This function counts the number of most-significant (leading in a big-Endian sense) zeroes in {#syntax#}integer{#endsyntax#}.
This function counts the number of most-significant (leading in a big-Endian sense) zeroes in an integer.
</p>
<p>
If {#syntax#}integer{#endsyntax#} is known at {#link|comptime#},
If {#syntax#}operand{#endsyntax#} is a {#link|comptime#}-known integer,
the return type is {#syntax#}comptime_int{#endsyntax#}.
Otherwise, the return type is an unsigned integer with the minimum number
Otherwise, the return type is an unsigned integer or vector of unsigned integers with the minimum number
of bits that can represent the bit count of the integer type.
</p>
<p>
If {#syntax#}integer{#endsyntax#} is zero, {#syntax#}@clz{#endsyntax#} returns the bit width
If {#syntax#}operand{#endsyntax#} is zero, {#syntax#}@clz{#endsyntax#} returns the bit width
of integer type {#syntax#}T{#endsyntax#}.
</p>
{#see_also|@ctz|@popCount#}
Expand Down Expand Up @@ -7509,18 +7511,20 @@ test "main" {
{#header_close#}

{#header_open|@ctz#}
<pre>{#syntax#}@ctz(comptime T: type, integer: T){#endsyntax#}</pre>
<pre>{#syntax#}@ctz(comptime T: type, operand: T){#endsyntax#}</pre>
<p>{#syntax#}T{#endsyntax#} must be an integer type.</p>
<p>{#syntax#}operand{#endsyntax#} may be an {#link|integer|Integers#} or {#link|vector|Vectors#}.</p>
<p>
This function counts the number of least-significant (trailing in a big-Endian sense) zeroes in {#syntax#}integer{#endsyntax#}.
This function counts the number of least-significant (trailing in a big-Endian sense) zeroes in an integer.
</p>
<p>
If {#syntax#}integer{#endsyntax#} is known at {#link|comptime#},
If {#syntax#}operand{#endsyntax#} is a {#link|comptime#}-known integer,
the return type is {#syntax#}comptime_int{#endsyntax#}.
Otherwise, the return type is an unsigned integer with the minimum number
Otherwise, the return type is an unsigned integer or vector of unsigned integers with the minimum number
of bits that can represent the bit count of the integer type.
</p>
<p>
If {#syntax#}integer{#endsyntax#} is zero, {#syntax#}@ctz{#endsyntax#} returns
If {#syntax#}operand{#endsyntax#} is zero, {#syntax#}@ctz{#endsyntax#} returns
the bit width of integer type {#syntax#}T{#endsyntax#}.
</p>
{#see_also|@clz|@popCount#}
Expand Down Expand Up @@ -8105,12 +8109,14 @@ test "@wasmMemoryGrow" {
{#header_close#}

{#header_open|@popCount#}
<pre>{#syntax#}@popCount(comptime T: type, integer: T){#endsyntax#}</pre>
<pre>{#syntax#}@popCount(comptime T: type, operand: T){#endsyntax#}</pre>
<p>{#syntax#}T{#endsyntax#} must be an integer type.</p>
<p>{#syntax#}operand{#endsyntax#} may be an {#link|integer|Integers#} or {#link|vector|Vectors#}.</p>
<p>Counts the number of bits set in an integer.</p>
<p>
If {#syntax#}integer{#endsyntax#} is known at {#link|comptime#},
If {#syntax#}operand{#endsyntax#} is a {#link|comptime#}-known integer,
the return type is {#syntax#}comptime_int{#endsyntax#}.
Otherwise, the return type is an unsigned integer with the minimum number
Otherwise, the return type is an unsigned integer or vector of unsigned integers with the minimum number
of bits that can represent the bit count of the integer type.
</p>
{#see_also|@ctz|@clz#}
Expand Down
3 changes: 3 additions & 0 deletions src/stage1/all_types.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1907,12 +1907,15 @@ struct ZigLLVMFnKey {
union {
struct {
uint32_t bit_count;
uint32_t vector_len; // 0 means not a vector
} ctz;
struct {
uint32_t bit_count;
uint32_t vector_len; // 0 means not a vector
} clz;
struct {
uint32_t bit_count;
uint32_t vector_len; // 0 means not a vector
} pop_count;
struct {
BuiltinFnId op;
Expand Down
9 changes: 6 additions & 3 deletions src/stage1/analyze.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7883,11 +7883,14 @@ bool type_id_eql(TypeId const *a, TypeId const *b) {
uint32_t zig_llvm_fn_key_hash(ZigLLVMFnKey const *x) {
switch (x->id) {
case ZigLLVMFnIdCtz:
return (uint32_t)(x->data.ctz.bit_count) * (uint32_t)810453934;
return (uint32_t)(x->data.ctz.bit_count) * (uint32_t)810453934 +
(uint32_t)(x->data.ctz.vector_len) * (((uint32_t)x->id << 5) + 1025);
case ZigLLVMFnIdClz:
return (uint32_t)(x->data.clz.bit_count) * (uint32_t)2428952817;
return (uint32_t)(x->data.clz.bit_count) * (uint32_t)2428952817 +
(uint32_t)(x->data.clz.vector_len) * (((uint32_t)x->id << 5) + 1025);
case ZigLLVMFnIdPopCount:
return (uint32_t)(x->data.clz.bit_count) * (uint32_t)101195049;
return (uint32_t)(x->data.pop_count.bit_count) * (uint32_t)101195049 +
(uint32_t)(x->data.pop_count.vector_len) * (((uint32_t)x->id << 5) + 1025);
case ZigLLVMFnIdFloatOp:
return (uint32_t)(x->data.floating.bit_count) * ((uint32_t)x->id + 1025) +
(uint32_t)(x->data.floating.vector_len) * (((uint32_t)x->id << 5) + 1025) +
Expand Down
1 change: 1 addition & 0 deletions src/stage1/codegen.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5070,6 +5070,7 @@ static LLVMValueRef get_int_builtin_fn(CodeGen *g, ZigType *expr_type, BuiltinFn
n_args = 1;
key.id = ZigLLVMFnIdPopCount;
key.data.pop_count.bit_count = (uint32_t)int_type->data.integral.bit_count;
key.data.pop_count.vector_len = vector_len;
} else if (fn_id == BuiltinFnIdBswap) {
fn_name = "bswap";
n_args = 1;
Expand Down
182 changes: 168 additions & 14 deletions src/stage1/ir.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15945,85 +15945,239 @@ static Stage1AirInst *ir_analyze_instruction_optional_unwrap_ptr(IrAnalyze *ira,
}

static Stage1AirInst *ir_analyze_instruction_ctz(IrAnalyze *ira, Stage1ZirInstCtz *instruction) {
Error err;

ZigType *int_type = ir_resolve_int_type(ira, instruction->type->child);
if (type_is_invalid(int_type))
return ira->codegen->invalid_inst_gen;

Stage1AirInst *op = ir_implicit_cast(ira, instruction->op->child, int_type);
Stage1AirInst *uncasted_op = instruction->op->child;
if (type_is_invalid(uncasted_op->value->type))
return ira->codegen->invalid_inst_gen;

uint32_t vector_len = UINT32_MAX; // means not a vector
if (uncasted_op->value->type->id == ZigTypeIdArray) {
bool can_be_vec_elem;
if ((err = is_valid_vector_elem_type(ira->codegen, uncasted_op->value->type->data.array.child_type,
&can_be_vec_elem)))
{
return ira->codegen->invalid_inst_gen;
}
if (can_be_vec_elem) {
vector_len = uncasted_op->value->type->data.array.len;
}
} else if (uncasted_op->value->type->id == ZigTypeIdVector) {
vector_len = uncasted_op->value->type->data.vector.len;
}

bool is_vector = (vector_len != UINT32_MAX);
ZigType *op_type = is_vector ? get_vector_type(ira->codegen, vector_len, int_type) : int_type;

Stage1AirInst *op = ir_implicit_cast(ira, uncasted_op, op_type);
if (type_is_invalid(op->value->type))
return ira->codegen->invalid_inst_gen;

if (int_type->data.integral.bit_count == 0)
return ir_const_unsigned(ira, instruction->base.scope, instruction->base.source_node, 0);

ZigType *smallest_type = get_smallest_unsigned_int_type(ira->codegen, int_type->data.integral.bit_count);

if (instr_is_comptime(op)) {
ZigValue *val = ir_resolve_const(ira, op, UndefOk);
if (val == nullptr)
return ira->codegen->invalid_inst_gen;
if (val->special == ConstValSpecialUndef)
return ir_const_undef(ira, instruction->base.scope, instruction->base.source_node, ira->codegen->builtin_types.entry_num_lit_int);
size_t result_usize = bigint_ctz(&op->value->data.x_bigint, int_type->data.integral.bit_count);
return ir_const_unsigned(ira, instruction->base.scope, instruction->base.source_node, result_usize);

if (is_vector) {
ZigType *smallest_vec_type = get_vector_type(ira->codegen, vector_len, smallest_type);
Stage1AirInst *result = ir_const(ira, instruction->base.scope, instruction->base.source_node, smallest_vec_type);
expand_undef_array(ira->codegen, val);
result->value->data.x_array.data.s_none.elements = ira->codegen->pass1_arena->allocate<ZigValue>(smallest_vec_type->data.vector.len);
for (unsigned i = 0; i < smallest_vec_type->data.vector.len; i += 1) {
ZigValue *op_elem_val = &val->data.x_array.data.s_none.elements[i];
if ((err = ir_resolve_const_val(ira->codegen, ira->new_irb.exec, instruction->base.source_node,
op_elem_val, UndefOk)))
{
return ira->codegen->invalid_inst_gen;
}
ZigValue *result_elem_val = &result->value->data.x_array.data.s_none.elements[i];
result_elem_val->type = smallest_type;
result_elem_val->special = op_elem_val->special;
if (op_elem_val->special == ConstValSpecialUndef)
continue;
size_t value = bigint_ctz(&op_elem_val->data.x_bigint, int_type->data.integral.bit_count);
bigint_init_unsigned(&result->value->data.x_array.data.s_none.elements[i].data.x_bigint, value);
}
return result;
} else {
size_t result_usize = bigint_ctz(&op->value->data.x_bigint, int_type->data.integral.bit_count);
return ir_const_unsigned(ira, instruction->base.scope, instruction->base.source_node, result_usize);
}
}

ZigType *return_type = get_smallest_unsigned_int_type(ira->codegen, int_type->data.integral.bit_count);
ZigType *return_type = is_vector ? get_vector_type(ira->codegen, vector_len, smallest_type) : smallest_type;
return ir_build_ctz_gen(ira, instruction->base.scope, instruction->base.source_node, return_type, op);
}

static Stage1AirInst *ir_analyze_instruction_clz(IrAnalyze *ira, Stage1ZirInstClz *instruction) {
Error err;

ZigType *int_type = ir_resolve_int_type(ira, instruction->type->child);
if (type_is_invalid(int_type))
return ira->codegen->invalid_inst_gen;

Stage1AirInst *op = ir_implicit_cast(ira, instruction->op->child, int_type);
Stage1AirInst *uncasted_op = instruction->op->child;
if (type_is_invalid(uncasted_op->value->type))
return ira->codegen->invalid_inst_gen;

uint32_t vector_len = UINT32_MAX; // means not a vector
if (uncasted_op->value->type->id == ZigTypeIdArray) {
bool can_be_vec_elem;
if ((err = is_valid_vector_elem_type(ira->codegen, uncasted_op->value->type->data.array.child_type,
&can_be_vec_elem)))
{
return ira->codegen->invalid_inst_gen;
}
if (can_be_vec_elem) {
vector_len = uncasted_op->value->type->data.array.len;
}
} else if (uncasted_op->value->type->id == ZigTypeIdVector) {
vector_len = uncasted_op->value->type->data.vector.len;
}

bool is_vector = (vector_len != UINT32_MAX);
ZigType *op_type = is_vector ? get_vector_type(ira->codegen, vector_len, int_type) : int_type;

Stage1AirInst *op = ir_implicit_cast(ira, uncasted_op, op_type);
if (type_is_invalid(op->value->type))
return ira->codegen->invalid_inst_gen;

if (int_type->data.integral.bit_count == 0)
return ir_const_unsigned(ira, instruction->base.scope, instruction->base.source_node, 0);

ZigType *smallest_type = get_smallest_unsigned_int_type(ira->codegen, int_type->data.integral.bit_count);

if (instr_is_comptime(op)) {
ZigValue *val = ir_resolve_const(ira, op, UndefOk);
if (val == nullptr)
return ira->codegen->invalid_inst_gen;
if (val->special == ConstValSpecialUndef)
return ir_const_undef(ira, instruction->base.scope, instruction->base.source_node, ira->codegen->builtin_types.entry_num_lit_int);
size_t result_usize = bigint_clz(&op->value->data.x_bigint, int_type->data.integral.bit_count);
return ir_const_unsigned(ira, instruction->base.scope, instruction->base.source_node, result_usize);

if (is_vector) {
ZigType *smallest_vec_type = get_vector_type(ira->codegen, vector_len, smallest_type);
Stage1AirInst *result = ir_const(ira, instruction->base.scope, instruction->base.source_node, smallest_vec_type);
expand_undef_array(ira->codegen, val);
result->value->data.x_array.data.s_none.elements = ira->codegen->pass1_arena->allocate<ZigValue>(smallest_vec_type->data.vector.len);
for (unsigned i = 0; i < smallest_vec_type->data.vector.len; i += 1) {
ZigValue *op_elem_val = &val->data.x_array.data.s_none.elements[i];
if ((err = ir_resolve_const_val(ira->codegen, ira->new_irb.exec, instruction->base.source_node,
op_elem_val, UndefOk)))
{
return ira->codegen->invalid_inst_gen;
}
ZigValue *result_elem_val = &result->value->data.x_array.data.s_none.elements[i];
result_elem_val->type = smallest_type;
result_elem_val->special = op_elem_val->special;
if (op_elem_val->special == ConstValSpecialUndef)
continue;
size_t value = bigint_clz(&op_elem_val->data.x_bigint, int_type->data.integral.bit_count);
bigint_init_unsigned(&result->value->data.x_array.data.s_none.elements[i].data.x_bigint, value);
}
return result;
} else {
size_t result_usize = bigint_clz(&op->value->data.x_bigint, int_type->data.integral.bit_count);
return ir_const_unsigned(ira, instruction->base.scope, instruction->base.source_node, result_usize);
}
}

ZigType *return_type = get_smallest_unsigned_int_type(ira->codegen, int_type->data.integral.bit_count);
ZigType *return_type = is_vector ? get_vector_type(ira->codegen, vector_len, smallest_type) : smallest_type;
return ir_build_clz_gen(ira, instruction->base.scope, instruction->base.source_node, return_type, op);
}

static Stage1AirInst *ir_analyze_instruction_pop_count(IrAnalyze *ira, Stage1ZirInstPopCount *instruction) {
Error err;

ZigType *int_type = ir_resolve_int_type(ira, instruction->type->child);
if (type_is_invalid(int_type))
return ira->codegen->invalid_inst_gen;

Stage1AirInst *op = ir_implicit_cast(ira, instruction->op->child, int_type);
Stage1AirInst *uncasted_op = instruction->op->child;
if (type_is_invalid(uncasted_op->value->type))
return ira->codegen->invalid_inst_gen;

uint32_t vector_len = UINT32_MAX; // means not a vector
if (uncasted_op->value->type->id == ZigTypeIdArray) {
bool can_be_vec_elem;
if ((err = is_valid_vector_elem_type(ira->codegen, uncasted_op->value->type->data.array.child_type,
&can_be_vec_elem)))
{
return ira->codegen->invalid_inst_gen;
}
if (can_be_vec_elem) {
vector_len = uncasted_op->value->type->data.array.len;
}
} else if (uncasted_op->value->type->id == ZigTypeIdVector) {
vector_len = uncasted_op->value->type->data.vector.len;
}

bool is_vector = (vector_len != UINT32_MAX);
ZigType *op_type = is_vector ? get_vector_type(ira->codegen, vector_len, int_type) : int_type;

Stage1AirInst *op = ir_implicit_cast(ira, uncasted_op, op_type);
if (type_is_invalid(op->value->type))
return ira->codegen->invalid_inst_gen;

if (int_type->data.integral.bit_count == 0)
return ir_const_unsigned(ira, instruction->base.scope, instruction->base.source_node, 0);

ZigType *smallest_type = get_smallest_unsigned_int_type(ira->codegen, int_type->data.integral.bit_count);

if (instr_is_comptime(op)) {
ZigValue *val = ir_resolve_const(ira, op, UndefOk);
if (val == nullptr)
return ira->codegen->invalid_inst_gen;
if (val->special == ConstValSpecialUndef)
return ir_const_undef(ira, instruction->base.scope, instruction->base.source_node, ira->codegen->builtin_types.entry_num_lit_int);

if (is_vector) {
ZigType *smallest_vec_type = get_vector_type(ira->codegen, vector_len, smallest_type);
Stage1AirInst *result = ir_const(ira, instruction->base.scope, instruction->base.source_node, smallest_vec_type);
expand_undef_array(ira->codegen, val);
result->value->data.x_array.data.s_none.elements = ira->codegen->pass1_arena->allocate<ZigValue>(smallest_vec_type->data.vector.len);
for (unsigned i = 0; i < smallest_vec_type->data.vector.len; i += 1) {
ZigValue *op_elem_val = &val->data.x_array.data.s_none.elements[i];
if ((err = ir_resolve_const_val(ira->codegen, ira->new_irb.exec, instruction->base.source_node,
op_elem_val, UndefOk)))
{
return ira->codegen->invalid_inst_gen;
}
ZigValue *result_elem_val = &result->value->data.x_array.data.s_none.elements[i];
result_elem_val->type = smallest_type;
result_elem_val->special = op_elem_val->special;
if (op_elem_val->special == ConstValSpecialUndef)
continue;

if (bigint_cmp_zero(&val->data.x_bigint) != CmpLT) {
size_t result = bigint_popcount_unsigned(&val->data.x_bigint);
if (bigint_cmp_zero(&op_elem_val->data.x_bigint) != CmpLT) {
size_t value = bigint_popcount_unsigned(&op_elem_val->data.x_bigint);
bigint_init_unsigned(&result->value->data.x_array.data.s_none.elements[i].data.x_bigint, value);
}
size_t value = bigint_popcount_signed(&op_elem_val->data.x_bigint, int_type->data.integral.bit_count);
bigint_init_unsigned(&result->value->data.x_array.data.s_none.elements[i].data.x_bigint, value);
}
return result;
} else {
if (bigint_cmp_zero(&val->data.x_bigint) != CmpLT) {
size_t result = bigint_popcount_unsigned(&val->data.x_bigint);
return ir_const_unsigned(ira, instruction->base.scope, instruction->base.source_node, result);
}
size_t result = bigint_popcount_signed(&val->data.x_bigint, int_type->data.integral.bit_count);
return ir_const_unsigned(ira, instruction->base.scope, instruction->base.source_node, result);
}
size_t result = bigint_popcount_signed(&val->data.x_bigint, int_type->data.integral.bit_count);
return ir_const_unsigned(ira, instruction->base.scope, instruction->base.source_node, result);
}

ZigType *return_type = get_smallest_unsigned_int_type(ira->codegen, int_type->data.integral.bit_count);
ZigType *return_type = is_vector ? get_vector_type(ira->codegen, vector_len, smallest_type) : smallest_type;
return ir_build_pop_count_gen(ira, instruction->base.scope, instruction->base.source_node, return_type, op);
}

Expand Down
Loading

0 comments on commit fc105f2

Please sign in to comment.