cmd/compile/internal/types2: avoid infinite recursion in unification

If the type T inferred for a type parameter P is P itself (or a derived
type containing P), a subsequent unification step leads to infinite
recursion: at each encounter of P with the already inferred type T
(which is or contains P), P stands for that T and the recursive matching
process continues with T, which inevitably contains P again and recursion
never terminates.

This CL introduces a set of masks, one for each type parameter.
When a type parameter is encountered for which a type has already
been inferred, the type parameter is "masked" for the recursive
matching of the inferred type. Masking makes the type parameter
"invisible" such that it will be handled like any other type and
not unpacked further.

Fixes #48619.
For #48656.

Change-Id: Ic1d938322be51fd44323ea14f925303f58b27c97
Reviewed-on: https://go-review.googlesource.com/c/go/+/352832
Trust: Robert Griesemer <gri@golang.org>
Run-TryBot: Robert Griesemer <gri@golang.org>
Reviewed-by: Robert Findley <rfindley@google.com>

Author: griesemer

src/cmd/compile/internal/types2/testdata/fixedbugs/issue48619.go2

@@ -0,0 +1,22 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package p
+
+func f[P any](a, _ P) {
+	var x int
+	f(a, x /* ERROR type int of x does not match P */)
+	f(x, a /* ERROR type P of a does not match inferred type int for P */)
+}
+
+func g[P any](a, b P) {
+	g(a, b)
+	g(&a, &b)
+	g([]P{}, []P{})
+}
+
+func h[P any](a, b P) {
+	h(&a, &b)
+	h([]P{a}, []P{b})
+}

src/cmd/compile/internal/types2/testdata/fixedbugs/issue48656.go2

@@ -0,0 +1,13 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package p
+
+// TODO(gri) Still need better error positions and message here.
+//           But this doesn't crash anymore.
+
+func f[P /* ERROR does not match \*Q */ interface{*Q}, Q any](p P, q Q) {
+	_ = f[P]
+        _ = f[/* ERROR cannot infer P */ *P]
+}

src/cmd/compile/internal/types2/unify.go

@@ -63,6 +63,10 @@ func (u *unifier) unify(x, y Type) bool {
 type tparamsList struct {
 	unifier *unifier
 	tparams []*TypeParam
+	// For each tparams element, there is a corresponding mask bit in masks.
+	// If set, the corresponding type parameter is masked and doesn't appear
+	// as a type parameter with tparamsList.index.
+	masks []bool
 	// For each tparams element, there is a corresponding type slot index in indices.
 	// index  < 0: unifier.types[-index-1] == nil
 	// index == 0: no type slot allocated yet
@@ -103,9 +107,14 @@ func (d *tparamsList) init(tparams []*TypeParam) {
 		}
 	}
 	d.tparams = tparams
+	d.masks = make([]bool, len(tparams))
 	d.indices = make([]int, len(tparams))
 }
 
+// mask and unmask permit the masking/unmasking of the i'th type parameter of d.
+func (d *tparamsList) mask(i int)   { d.masks[i] = true }
+func (d *tparamsList) unmask(i int) { d.masks[i] = false }
+
 // join unifies the i'th type parameter of x with the j'th type parameter of y.
 // If both type parameters already have a type associated with them and they are
 // not joined, join fails and returns false.
@@ -149,11 +158,13 @@ func (u *unifier) join(i, j int) bool {
 	return true
 }
 
-// If typ is a type parameter of d, index returns the type parameter index.
+// If typ is an unmasked type parameter of d, index returns the type parameter index.
 // Otherwise, the result is < 0.
 func (d *tparamsList) index(typ Type) int {
 	if tpar, ok := typ.(*TypeParam); ok {
-		return tparamIndex(d.tparams, tpar)
+		if i := tparamIndex(d.tparams, tpar); i >= 0 && !d.masks[i] {
+			return i
+		}
 	}
 	return -1
 }
@@ -246,7 +257,7 @@ func (u *unifier) nify(x, y Type, p *ifacePair) bool {
 		}
 	}
 
-	// Cases where at least one of x or y is a type parameter.
+	// Cases where at least one of x or y is an (unmasked) type parameter.
 	switch i, j := u.x.index(x), u.y.index(y); {
 	case i >= 0 && j >= 0:
 		// both x and y are type parameters
@@ -259,6 +270,12 @@ func (u *unifier) nify(x, y Type, p *ifacePair) bool {
 	case i >= 0:
 		// x is a type parameter, y is not
 		if tx := u.x.at(i); tx != nil {
+			// The inferred type tx may be or contain x again but we don't
+			// want to "unpack" it again when unifying tx with y: tx is the
+			// inferred type. Mask type parameter x for this recursion, so
+			// that subsequent encounters treat x like an ordinary type.
+			u.x.mask(i)
+			defer u.x.unmask(i)
 			return u.nifyEq(tx, y, p)
 		}
 		// otherwise, infer type from y
@@ -268,6 +285,9 @@ func (u *unifier) nify(x, y Type, p *ifacePair) bool {
 	case j >= 0:
 		// y is a type parameter, x is not
 		if ty := u.y.at(j); ty != nil {
+			// see comment above
+			u.y.mask(j)
+			defer u.y.unmask(j)
 			return u.nifyEq(x, ty, p)
 		}
 		// otherwise, infer type from x