staticcheck: don't chase infinite chain of instantiations

(cherry picked from commit 2746778)

Author: dominikh

staticcheck/fakejson/encode.go

@@ -17,6 +17,7 @@ import (
 	"unicode"
 
 	"golang.org/x/exp/typeparams"
+	"honnef.co/go/tools/go/types/typeutil"
 	"honnef.co/go/tools/knowledge"
 	"honnef.co/go/tools/staticcheck/fakereflect"
 )
@@ -31,9 +32,7 @@ func parseTag(tag string) string {
 }
 
 func Marshal(v types.Type) *UnsupportedTypeError {
-	enc := encoder{
-		seen: map[fakereflect.TypeAndCanAddr]struct{}{},
-	}
+	enc := encoder{}
 	return enc.newTypeEncoder(fakereflect.TypeAndCanAddr{Type: v}, "x")
 }
 
@@ -45,14 +44,23 @@ type UnsupportedTypeError struct {
 }
 
 type encoder struct {
-	seen map[fakereflect.TypeAndCanAddr]struct{}
+	// TODO we track addressable and non-addressable instances separately out of an abundance of caution. We don't know
+	// if this is actually required for correctness.
+	seenCanAddr  typeutil.Map
+	seenCantAddr typeutil.Map
 }
 
 func (enc *encoder) newTypeEncoder(t fakereflect.TypeAndCanAddr, stack string) *UnsupportedTypeError {
-	if _, ok := enc.seen[t]; ok {
+	var m *typeutil.Map
+	if t.CanAddr() {
+		m = &enc.seenCanAddr
+	} else {
+		m = &enc.seenCantAddr
+	}
+	if ok := m.At(t.Type); ok != nil {
 		return nil
 	}
-	enc.seen[t] = struct{}{}
+	m.Set(t.Type, struct{}{})
 
 	if t.Implements(knowledge.Interfaces["encoding/json.Marshaler"]) {
 		return nil

staticcheck/fakexml/marshal.go

@@ -26,13 +26,14 @@ func Marshal(v types.Type) error {
 }
 
 type Encoder struct {
-	seen map[fakereflect.TypeAndCanAddr]struct{}
+	// TODO we track addressable and non-addressable instances separately out of an abundance of caution. We don't know
+	// if this is actually required for correctness.
+	seenCanAddr  typeutil.Map
+	seenCantAddr typeutil.Map
 }
 
 func NewEncoder() *Encoder {
-	e := &Encoder{
-		seen: map[fakereflect.TypeAndCanAddr]struct{}{},
-	}
+	e := &Encoder{}
 	return e
 }
 
@@ -113,10 +114,16 @@ func (err *CyclicTypeError) Error() string {
 // marshalValue writes one or more XML elements representing val.
 // If val was obtained from a struct field, finfo must have its details.
 func (e *Encoder) marshalValue(val fakereflect.TypeAndCanAddr, finfo *fieldInfo, startTemplate *StartElement, stack string) error {
-	if _, ok := e.seen[val]; ok {
+	var m *typeutil.Map
+	if val.CanAddr() {
+		m = &e.seenCanAddr
+	} else {
+		m = &e.seenCantAddr
+	}
+	if ok := m.At(val.Type); ok != nil {
 		return nil
 	}
-	e.seen[val] = struct{}{}
+	m.Set(val.Type, struct{}{})
 
 	// Drill into interfaces and pointers.
 	seen := map[fakereflect.TypeAndCanAddr]struct{}{}

staticcheck/lint.go

@@ -524,18 +524,16 @@ func checkPrintfCallImpl(carg *Argument, f ir.Value, args []ir.Value) {
 		return true
 	}
 
-	seen := map[types.Type]bool{}
+	var seen typeutil.Map
 	var checkType func(verb rune, T types.Type, top bool) bool
 	checkType = func(verb rune, T types.Type, top bool) bool {
 		if top {
-			for k := range seen {
-				delete(seen, k)
-			}
+			seen = typeutil.Map{}
 		}
-		if seen[T] {
+		if ok := seen.At(T); ok != nil {
 			return true
 		}
-		seen[T] = true
+		seen.Set(T, struct{}{})
 		if int(verb) >= len(verbs) {
 			// Unknown verb
 			return true

staticcheck/testdata/src/CheckUnsupportedMarshal/generics.go

@@ -2,7 +2,10 @@
 
 package pkg
 
-import "encoding/json"
+import (
+	"encoding/json"
+	"encoding/xml"
+)
 
 type LMap[K comparable, V any] struct {
 	M1 map[K]V
@@ -13,3 +16,18 @@ func (lm *LMap[K, V]) MarshalJSON() {
 	json.Marshal(lm.M1)
 	json.Marshal(lm.M2) //@ diag(`unsupported type`)
 }
+
+func recursiveGeneric() {
+	// don't recurse infinitely
+	var t Tree[int]
+	json.Marshal(t)
+	xml.Marshal(t)
+}
+
+type Tree[T any] struct {
+	Node *Node[T]
+}
+
+type Node[T any] struct {
+	Tree *Tree[T]
+}