Skip to content

Commit

Permalink
[vm/compiler] Various 64-bit operator improvements.
Browse files Browse the repository at this point in the history 
Rationale:
Improves the slow path of 64-bit REM/TRUNCDIV on
X64 and ARM64. Also introduces 64-bit negate operator,
which was needed to expose negative contants
(viz. x / -3 was represented as x / - (3) first).
The negate operator is not recognized in AOT mode yet,
since shifts by out-of-range constants should learn
how to throw rather than deopt....


#33967
flutter/flutter#19677

Change-Id: I7d81c9b1c72d99e8c4018f68c0501c7b599e073f
Reviewed-on: https://dart-review.googlesource.com/68280
Commit-Queue: Aart Bik <ajcbik@google.com>
Reviewed-by: Alexander Markov <alexmarkov@google.com>
Reviewed-by: Vyacheslav Egorov <vegorov@google.com>
  • Loading branch information
@aartbik
aartbik authored and commit-bot@chromium.org committed Aug 3, 2018
1 parent 92a7094 commit bfa890d
Show file tree
Hide file tree
Showing 4 changed files with 173 additions and 66 deletions.
3 changes: 0 additions & 3 deletions runtime/vm/compiler/aot/aot_call_specializer.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -565,7 +565,6 @@ bool AotCallSpecializer::TryOptimizeStaticCallUsingStaticTypes(
}
break;
}
// TODO(dartbug.com/30480): Enable 64-bit integer shifts (SHL, SHR).
case Token::kBIT_OR:
case Token::kBIT_XOR:
case Token::kBIT_AND:
Expand Down Expand Up @@ -608,7 +607,6 @@ bool AotCallSpecializer::TryOptimizeStaticCallUsingStaticTypes(
}
break;
}

case Token::kSHL:
case Token::kSHR: {
Value* left_value = instr->PushArgumentAt(receiver_index)->value();
Expand All @@ -622,7 +620,6 @@ bool AotCallSpecializer::TryOptimizeStaticCallUsingStaticTypes(
}
break;
}

default:
break;
}
Expand Down
18 changes: 15 additions & 3 deletions runtime/vm/compiler/backend/il.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6076,9 +6076,13 @@ class UnaryUint32OpInstr : public UnaryIntegerOpInstr {

class UnaryInt64OpInstr : public UnaryIntegerOpInstr {
public:
UnaryInt64OpInstr(Token::Kind op_kind, Value* value, intptr_t deopt_id)
: UnaryIntegerOpInstr(op_kind, value, deopt_id) {
ASSERT(op_kind == Token::kBIT_NOT);
UnaryInt64OpInstr(Token::Kind op_kind,
Value* value,
intptr_t deopt_id,
SpeculativeMode speculative_mode = kGuardInputs)
: UnaryIntegerOpInstr(op_kind, value, deopt_id),
speculative_mode_(speculative_mode) {
ASSERT(op_kind == Token::kBIT_NOT || op_kind == Token::kNEGATE);
}

virtual bool ComputeCanDeoptimize() const { return false; }
Expand All @@ -6092,9 +6096,17 @@ class UnaryInt64OpInstr : public UnaryIntegerOpInstr {
return kUnboxedInt64;
}

virtual bool AttributesEqual(Instruction* other) const {
return UnaryIntegerOpInstr::AttributesEqual(other) &&
(speculative_mode() == other->speculative_mode());
}

virtual SpeculativeMode speculative_mode() const { return speculative_mode_; }

DECLARE_INSTRUCTION(UnaryInt64Op)

private:
const SpeculativeMode speculative_mode_;
DISALLOW_COPY_AND_ASSIGN(UnaryInt64OpInstr);
};

Expand Down
88 changes: 64 additions & 24 deletions runtime/vm/compiler/backend/il_arm64.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5047,7 +5047,8 @@ class Int64DivideSlowPath : public ThrowErrorSlowPathCode {
static const intptr_t kNumberOfArguments = 0;

Int64DivideSlowPath(BinaryInt64OpInstr* instruction,
Register right,
Register divisor,
Range* divisor_range,
Register tmp,
Register out,
intptr_t try_index)
Expand All @@ -5056,36 +5057,63 @@ class Int64DivideSlowPath : public ThrowErrorSlowPathCode {
kNumberOfArguments,
try_index),
is_mod_(instruction->op_kind() == Token::kMOD),
right_(right),
divisor_(divisor),
divisor_range_(divisor_range),
tmp_(tmp),
out_(out),
adjust_sign_label_() {}

void EmitNativeCode(FlowGraphCompiler* compiler) override {
// Main entry throws, use code of superclass.
ThrowErrorSlowPathCode::EmitNativeCode(compiler);
// Adjust modulo for negative sign.
// if (right < 0)
// out -= right;
// Handle modulo/division by zero, if needed. Use superclass code.
if (has_divide_by_zero()) {
ThrowErrorSlowPathCode::EmitNativeCode(compiler);
} else {
__ Bind(entry_label()); // not used, but keeps destructor happy
if (Assembler::EmittingComments()) {
__ Comment("slow path %s operation (no throw)", name());
}
}
// Adjust modulo for negative sign, optimized for known ranges.
// if (divisor < 0)
// out -= divisor;
// else
// out += right;
if (is_mod_) {
// out += divisor;
if (has_adjust_sign()) {
__ Bind(adjust_sign_label());
__ CompareRegisters(right_, ZR);
__ sub(tmp_, out_, Operand(right_));
__ add(out_, out_, Operand(right_));
__ csel(out_, tmp_, out_, LT);
if (RangeUtils::Overlaps(divisor_range_, -1, 1)) {
// General case.
__ CompareRegisters(divisor_, ZR);
__ sub(tmp_, out_, Operand(divisor_));
__ add(out_, out_, Operand(divisor_));
__ csel(out_, tmp_, out_, LT);
} else if (divisor_range_->IsPositive()) {
// Always positive.
__ add(out_, out_, Operand(divisor_));
} else {
// Always negative.
__ sub(out_, out_, Operand(divisor_));
}
__ b(exit_label());
}
}

const char* name() override { return "int64 divide"; }

Label* adjust_sign_label() { return &adjust_sign_label_; }
bool has_divide_by_zero() { return RangeUtils::CanBeZero(divisor_range_); }

bool has_adjust_sign() { return is_mod_; }

bool is_needed() { return has_divide_by_zero() || has_adjust_sign(); }

Label* adjust_sign_label() {
ASSERT(has_adjust_sign());
return &adjust_sign_label_;
}

private:
bool is_mod_;
Register right_;
Register divisor_;
Range* divisor_range_;
Register tmp_;
Register out_;
Label adjust_sign_label_;
Expand All @@ -5100,14 +5128,16 @@ static void EmitInt64ModTruncDiv(FlowGraphCompiler* compiler,
Register out) {
ASSERT(op_kind == Token::kMOD || op_kind == Token::kTRUNCDIV);

// Set up a slow path.
// Prepare a slow path.
Range* right_range = instruction->right()->definition()->range();
Int64DivideSlowPath* slow_path = new (Z) Int64DivideSlowPath(
instruction, right, tmp, out, compiler->CurrentTryIndex());
compiler->AddSlowPathCode(slow_path);
instruction, right, right_range, tmp, out, compiler->CurrentTryIndex());

// Handle modulo/division by zero exception on slow path.
__ CompareRegisters(right, ZR);
__ b(slow_path->entry_label(), EQ);
if (slow_path->has_divide_by_zero()) {
__ CompareRegisters(right, ZR);
__ b(slow_path->entry_label(), EQ);
}

// Perform actual operation
// out = left % right
Expand All @@ -5124,7 +5154,10 @@ static void EmitInt64ModTruncDiv(FlowGraphCompiler* compiler,
__ sdiv(out, left, right);
}

__ Bind(slow_path->exit_label());
if (slow_path->is_needed()) {
__ Bind(slow_path->exit_label());
compiler->AddSlowPathCode(slow_path);
}
}

LocationSummary* BinaryInt64OpInstr::MakeLocationSummary(Zone* zone,
Expand Down Expand Up @@ -5571,11 +5604,18 @@ LocationSummary* UnaryInt64OpInstr::MakeLocationSummary(Zone* zone,
}

void UnaryInt64OpInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
ASSERT(op_kind() == Token::kBIT_NOT);
const Register left = locs()->in(0).reg();
const Register out = locs()->out(0).reg();
ASSERT(out == left);
__ mvn(out, left);
switch (op_kind()) {
case Token::kBIT_NOT:
__ mvn(out, left);
break;
case Token::kNEGATE:
__ sub(out, ZR, Operand(left));
break;
default:
UNREACHABLE();
}
}

CompileType BinaryUint32OpInstr::ComputeType() const {
Expand Down
130 changes: 94 additions & 36 deletions runtime/vm/compiler/backend/il_x64.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5201,54 +5201,96 @@ class Int64DivideSlowPath : public ThrowErrorSlowPathCode {
static const intptr_t kNumberOfArguments = 0;

Int64DivideSlowPath(BinaryInt64OpInstr* instruction,
Register right,
Register divisor,
Range* divisor_range,
intptr_t try_index)
: ThrowErrorSlowPathCode(instruction,
kIntegerDivisionByZeroExceptionRuntimeEntry,
kNumberOfArguments,
try_index),
is_mod_(instruction->op_kind() == Token::kMOD),
right_(right),
divisor_(divisor),
divisor_range_(divisor_range),
div_by_minus_one_label_(),
adjust_sign_label_() {}

void EmitNativeCode(FlowGraphCompiler* compiler) override {
// Main entry throws, use code of superclass.
ThrowErrorSlowPathCode::EmitNativeCode(compiler);
// Handle modulo/division by minus one.
__ Bind(div_by_minus_one_label());
if (is_mod_) {
__ xorq(RDX, RDX); // x % -1 = 0
// Handle modulo/division by zero, if needed. Use superclass code.
if (has_divide_by_zero()) {
ThrowErrorSlowPathCode::EmitNativeCode(compiler);
} else {
__ negq(RAX); // x / -1 = -x
__ Bind(entry_label()); // not used, but keeps destructor happy
if (Assembler::EmittingComments()) {
__ Comment("slow path %s operation (no throw)", name());
}
}
__ jmp(exit_label());
// Adjust modulo for negative sign.
// if (right < 0)
// out -= right;
// Handle modulo/division by minus one, if needed.
// Note: an exact -1 divisor is best optimized prior to codegen.
if (has_divide_by_minus_one()) {
__ Bind(div_by_minus_one_label());
if (is_mod_) {
__ xorq(RDX, RDX); // x % -1 = 0
} else {
__ negq(RAX); // x / -1 = -x
}
__ jmp(exit_label());
}
// Adjust modulo for negative sign, optimized for known ranges.
// if (divisor < 0)
// out -= divisor;
// else
// out += right;
if (is_mod_) {
Label subtract;
// out += divisor;
if (has_adjust_sign()) {
__ Bind(adjust_sign_label());
__ testq(right_, right_);
__ j(LESS, &subtract, Assembler::kNearJump);
__ addq(RDX, right_);
__ jmp(exit_label());
__ Bind(&subtract);
__ subq(RDX, right_);
if (RangeUtils::Overlaps(divisor_range_, -1, 1)) {
// General case.
Label subtract;
__ testq(divisor_, divisor_);
__ j(LESS, &subtract, Assembler::kNearJump);
__ addq(RDX, divisor_);
__ jmp(exit_label());
__ Bind(&subtract);
__ subq(RDX, divisor_);
} else if (divisor_range_->IsPositive()) {
// Always positive.
__ addq(RDX, divisor_);
} else {
// Always negative.
__ subq(RDX, divisor_);
}
__ jmp(exit_label());
}
}

const char* name() override { return "int64 divide"; }

Label* div_by_minus_one_label() { return &div_by_minus_one_label_; }
Label* adjust_sign_label() { return &adjust_sign_label_; }
bool has_divide_by_zero() { return RangeUtils::CanBeZero(divisor_range_); }

bool has_divide_by_minus_one() {
return RangeUtils::Overlaps(divisor_range_, -1, -1);
}

bool has_adjust_sign() { return is_mod_; }

bool is_needed() {
return has_divide_by_zero() || has_divide_by_minus_one() ||
has_adjust_sign();
}

Label* div_by_minus_one_label() {
ASSERT(has_divide_by_minus_one());
return &div_by_minus_one_label_;
}

Label* adjust_sign_label() {
ASSERT(has_adjust_sign());
return &adjust_sign_label_;
}

private:
bool is_mod_;
Register right_;
Register divisor_;
Range* divisor_range_;
Label div_by_minus_one_label_;
Label adjust_sign_label_;
};
Expand All @@ -5262,19 +5304,23 @@ static void EmitInt64ModTruncDiv(FlowGraphCompiler* compiler,
Register out) {
ASSERT(op_kind == Token::kMOD || op_kind == Token::kTRUNCDIV);

// Set up a slow path.
Int64DivideSlowPath* slow_path = new (Z)
Int64DivideSlowPath(instruction, right, compiler->CurrentTryIndex());
compiler->AddSlowPathCode(slow_path);
// Prepare a slow path.
Range* right_range = instruction->right()->definition()->range();
Int64DivideSlowPath* slow_path = new (Z) Int64DivideSlowPath(
instruction, right, right_range, compiler->CurrentTryIndex());

// Handle modulo/division by zero exception on slow path.
__ testq(right, right);
__ j(EQUAL, slow_path->entry_label());
if (slow_path->has_divide_by_zero()) {
__ testq(right, right);
__ j(EQUAL, slow_path->entry_label());
}

// Handle modulo/division by minus one explicitly on slow path
// (to avoid arithmetic exception on 0x8000000000000000 / -1).
__ cmpq(right, Immediate(-1));
__ j(EQUAL, slow_path->div_by_minus_one_label());
if (slow_path->has_divide_by_minus_one()) {
__ cmpq(right, Immediate(-1));
__ j(EQUAL, slow_path->div_by_minus_one_label());
}

// Perform actual operation
// out = left % right
Expand All @@ -5294,7 +5340,11 @@ static void EmitInt64ModTruncDiv(FlowGraphCompiler* compiler,
ASSERT(out == RAX);
ASSERT(tmp == RDX);
}
__ Bind(slow_path->exit_label());

if (slow_path->is_needed()) {
__ Bind(slow_path->exit_label());
compiler->AddSlowPathCode(slow_path);
}
}

template <typename OperandType>
Expand Down Expand Up @@ -5393,11 +5443,19 @@ LocationSummary* UnaryInt64OpInstr::MakeLocationSummary(Zone* zone,
}

void UnaryInt64OpInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
ASSERT(op_kind() == Token::kBIT_NOT);
const Register left = locs()->in(0).reg();
const Register out = locs()->out(0).reg();
ASSERT(out == left);
__ notq(left);
switch (op_kind()) {
case Token::kBIT_NOT:
__ notq(left);
break;
case Token::kNEGATE:
__ negq(left);
break;
default:
UNREACHABLE();
}
}

static void EmitShiftInt64ByConstant(FlowGraphCompiler* compiler,
Expand Down

0 comments on commit bfa890d

Please sign in to comment.