cmd/compile: hoist some loop invariants

Conservatively hoist some loop invariants outside the loop.

Updates #15808

src/cmd/compile/internal/ssa/branchelim.go

@@ -424,6 +424,14 @@ func shouldElimIfElse(no, yes, post *Block, arch string) bool {
 	}
 }
 
+func hasSideEffect(v *Value) bool {
+	if v.Op == OpPhi || isDivMod(v.Op) || isPtrArithmetic(v.Op) || v.Type.IsMemory() ||
+		v.MemoryArg() != nil || opcodeTable[v.Op].hasSideEffects {
+		return true
+	}
+	return false
+}
+
 // canSpeculativelyExecute reports whether every value in the block can
 // be evaluated without causing any observable side effects (memory
 // accesses, panics and so on) except for execution time changes. It
@@ -436,8 +444,7 @@ func canSpeculativelyExecute(b *Block) bool {
 	// don't fuse memory ops, Phi ops, divides (can panic),
 	// or anything else with side-effects
 	for _, v := range b.Values {
-		if v.Op == OpPhi || isDivMod(v.Op) || isPtrArithmetic(v.Op) || v.Type.IsMemory() ||
-			v.MemoryArg() != nil || opcodeTable[v.Op].hasSideEffects {
+		if hasSideEffect(v) {
 			return false
 		}
 	}

src/cmd/compile/internal/ssa/compile.go

@@ -485,6 +485,7 @@ var passes = [...]pass{
 	{name: "writebarrier", fn: writebarrier, required: true}, // expand write barrier ops
 	{name: "insert resched checks", fn: insertLoopReschedChecks,
 		disabled: !buildcfg.Experiment.PreemptibleLoops}, // insert resched checks in loops.
+	{name: "hoist loop invariant", fn: hoistLoopInvariant},
 	{name: "lower", fn: lower, required: true},
 	{name: "addressing modes", fn: addressingModes, required: false},
 	{name: "late lower", fn: lateLower, required: true},

src/cmd/compile/internal/ssa/graphkit.go

@@ -0,0 +1,46 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package ssa
+
+// ----------------------------------------------------------------------------
+// Graph transformation
+
+// replaceUses replaces all uses of old in b with new.
+func (b *Block) replaceUses(old, new *Value) {
+	for _, v := range b.Values {
+		for i, a := range v.Args {
+			if a == old {
+				v.SetArg(i, new)
+			}
+		}
+	}
+	for i, v := range b.ControlValues() {
+		if v == old {
+			b.ReplaceControl(i, new)
+		}
+	}
+}
+
+// moveTo moves v to dst, adjusting the appropriate Block.Values slices.
+// The caller is responsible for ensuring that this is safe.
+// i is the index of v in v.Block.Values.
+func (v *Value) moveTo(dst *Block, i int) {
+	if dst.Func.scheduled {
+		v.Fatalf("moveTo after scheduling")
+	}
+	src := v.Block
+	if src.Values[i] != v {
+		v.Fatalf("moveTo bad index %d", v, i)
+	}
+	if src == dst {
+		return
+	}
+	v.Block = dst
+	dst.Values = append(dst.Values, v)
+	last := len(src.Values) - 1
+	src.Values[i] = src.Values[last]
+	src.Values[last] = nil
+	src.Values = src.Values[:last]
+}

src/cmd/compile/internal/ssa/hoistloopiv.go

@@ -0,0 +1,132 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package ssa
+
+import "fmt"
+
+const MaxLoopBlockSize = 8
+
+func printInvariant(val *Value, block *Block, domBlock *Block) {
+	fmt.Printf("== Hoist %v(%v) from b%v to b%v in %v\n",
+		val.Op.String(), val.String(),
+		block.ID, domBlock.ID, block.Func.Name)
+	fmt.Printf("  %v\n", val.LongString())
+}
+
+func isCandidate(block *Block, val *Value) bool {
+	if len(val.Args) == 0 {
+		// not a profitable expression, e.g. constant
+		return false
+	}
+	if block.Likely == BranchUnlikely {
+		// all values are excluded as candidate when branch becomes unlikely to reach
+		return false
+	}
+	return true
+}
+
+func isInsideLoop(loopBlocks []*Block, v *Value) bool {
+	for _, block := range loopBlocks {
+		for _, val := range block.Values {
+			if val == v {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// tryHoist hoists profitable loop invariant to block that dominates the entire loop.
+// Value is considered as loop invariant if all its inputs are defined outside the loop
+// or all its inputs are loop invariants. Since loop invariant will immediately moved
+// to dominator block of loop, the first rule actually already implies the second rule
+func tryHoist(loopnest *loopnest, loop *loop, loopBlocks []*Block) {
+	for _, block := range loopBlocks {
+		// if basic block is located in a nested loop rather than directly in the
+		// current loop, it will not be processed.
+		if loopnest.b2l[block.ID] != loop {
+			continue
+		}
+		for i := 0; i < len(block.Values); i++ {
+			var val *Value = block.Values[i]
+			if !isCandidate(block, val) {
+				continue
+			}
+			// value can hoist because it may causes observable side effects
+			if hasSideEffect(val) {
+				continue
+			}
+			// consider the following operation as pinned anyway
+			switch val.Op {
+			case OpInlMark,
+				OpAtomicLoad8, OpAtomicLoad32, OpAtomicLoad64,
+				OpAtomicLoadPtr, OpAtomicLoadAcq32, OpAtomicLoadAcq64:
+				continue
+			}
+			// input def is inside loop, consider as variant
+			isInvariant := true
+			loopnest.assembleChildren()
+			for _, arg := range val.Args {
+				if isInsideLoop(loopBlocks, arg) {
+					isInvariant = false
+					break
+				}
+			}
+			if isInvariant {
+				for valIdx, v := range block.Values {
+					if val != v {
+						continue
+					}
+					domBlock := loopnest.sdom.Parent(loop.header)
+					if block.Func.pass.debug >= 1 {
+						printInvariant(val, block, domBlock)
+					}
+					val.moveTo(domBlock, valIdx)
+					i--
+					break
+				}
+			}
+		}
+	}
+}
+
+// hoistLoopInvariant hoists expressions that computes the same value
+// while has no effect outside loop
+func hoistLoopInvariant(f *Func) {
+	loopnest := f.loopnest()
+	if loopnest.hasIrreducible {
+		return
+	}
+	if len(loopnest.loops) == 0 {
+		return
+	}
+	for _, loop := range loopnest.loops {
+		loopBlocks := loopnest.findLoopBlocks(loop)
+		if len(loopBlocks) >= MaxLoopBlockSize {
+			continue
+		}
+
+		// check if it's too complicated for such optmization
+		tooComplicated := false
+	Out:
+		for _, block := range loopBlocks {
+			for _, val := range block.Values {
+				if val.Op.IsCall() || val.Op.HasSideEffects() {
+					tooComplicated = true
+					break Out
+				}
+				switch val.Op {
+				case OpLoad, OpStore:
+					tooComplicated = true
+					break Out
+				}
+			}
+		}
+		// try to hoist loop invariant outside the loop
+		if !tooComplicated {
+			tryHoist(loopnest, loop, loopBlocks)
+		}
+	}
+}

src/cmd/compile/internal/ssa/hoistloopiv_test.go

@@ -0,0 +1,111 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package ssa
+
+import (
+	"cmd/compile/internal/types"
+	"testing"
+)
+
+func checkValueMotion(t *testing.T, fun fun, valName, expectedBlock string) {
+	for _, b := range fun.f.Blocks {
+		for _, v := range b.Values {
+			if v == fun.values[valName] {
+				if fun.blocks[expectedBlock] != b {
+					t.Errorf("Error: %v\n", v.LongString())
+				}
+			}
+		}
+	}
+}
+
+//	var d int
+//	var p = 15
+//
+//	for i := 0; i < 10; i++ {
+//		t := 1 * p
+//		d = i + t
+//	}
+//
+// t should be hoisted to dominator block of loop header
+func TestHoistLoopIVSimple(t *testing.T) {
+	c := testConfig(t)
+	fun := c.Fun("b1",
+		Bloc("b1",
+			Valu("mem", OpInitMem, types.TypeMem, 0, nil),
+			Valu("zero", OpConst64, c.config.Types.Int64, 0, nil),
+			Valu("one", OpConst64, c.config.Types.Int64, 1, nil),
+			Valu("ten", OpConst64, c.config.Types.Int64, 10, nil),
+			Valu("p", OpConst64, c.config.Types.Int64, 15, nil),
+			Goto("b2")),
+		Bloc("b2",
+			Valu("i", OpPhi, c.config.Types.Int64, 0, nil, "one", "i2"),
+			Valu("d", OpPhi, c.config.Types.Int64, 0, nil, "zero", "d2"),
+			Valu("cmp", OpLess64, c.config.Types.Bool, 0, nil, "i", "ten"),
+			If("cmp", "b3", "b4")),
+		Bloc("b3",
+			Valu("loopiv", OpMul64, c.config.Types.Int64, 0, nil, "one", "p"),
+			Valu("d2", OpAdd64, c.config.Types.Int64, 0, nil, "loopiv", "d"),
+			Valu("i2", OpAdd64, c.config.Types.Int64, 0, nil, "i", "one"),
+			Goto("b2")),
+		Bloc("b4",
+			Exit("mem")))
+
+	CheckFunc(fun.f)
+	hoistLoopInvariant(fun.f)
+	CheckFunc(fun.f)
+	checkValueMotion(t, fun, "loopiv", "b1")
+}
+
+func BenchmarkHoistIV1Opt(b *testing.B) {
+	var d = 0
+	var a = 3
+
+	for i := 0; i < b.N; i++ {
+		d = i + (a*10 - a + 3)
+	}
+	_ = d
+}
+
+func BenchmarkHoistIV1Manual(b *testing.B) {
+	var d = 0
+	var a = 3
+	val := (a*10 - a + 3)
+	for i := 0; i < b.N; i++ {
+		d = i + val
+	}
+	_ = d
+}
+
+//go:noinline
+func hoistLoopIV2Opt(n, d int) {
+	t := 0
+	for i := 0; i < n*d; i++ {
+		t += 1
+	}
+	_ = t
+}
+
+//go:noinline
+func hoistLoopIV2Manual(n, d int) {
+	t := 0
+	val := n * d
+	for i := 0; i < val; i++ {
+		t += 1
+	}
+	_ = t
+}
+
+func BenchmarkHoistIV2Opt(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		hoistLoopIV2Opt(i%10, i%5)
+	}
+}
+
+func BenchmarkHoistIV2Manual(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		hoistLoopIV2Manual(i%10, i%5)
+	}
+}