diff --git a/api/internal/accumulator/namereferencetransformer_test.go b/api/internal/accumulator/namereferencetransformer_test.go
index 39fa9b0375..31ec435362 100644
--- a/api/internal/accumulator/namereferencetransformer_test.go
+++ b/api/internal/accumulator/namereferencetransformer_test.go
@@ -897,7 +897,9 @@ func TestNameReferenceClusterWide(t *testing.T) {
 		t.Fatalf("unexpected error: %v", err)
 	}
 
+	expected.RemoveBuildAnnotations()
 	m.RemoveBuildAnnotations()
+
 	if err = expected.ErrorIfNotEqualLists(m); err != nil {
 		t.Fatalf(notEqualErrFmt, err)
 	}
diff --git a/api/internal/utils/makeResIds.go b/api/internal/utils/makeResIds.go
index 3f6b17a3f4..a249fc9b83 100644
--- a/api/internal/utils/makeResIds.go
+++ b/api/internal/utils/makeResIds.go
@@ -15,6 +15,7 @@ const (
 	BuildAnnotationPrefixes           = konfig.ConfigAnnoDomain + "/prefixes"
 	BuildAnnotationSuffixes           = konfig.ConfigAnnoDomain + "/suffixes"
 	BuildAnnotationPreviousNamespaces = konfig.ConfigAnnoDomain + "/previousNamespaces"
+	BuildAnnotationsRefBy             = konfig.ConfigAnnoDomain + "/refBy"
 
 	// the following are only for patches, to specify whether they can change names
 	// and kinds of their targets
diff --git a/api/resource/resource.go b/api/resource/resource.go
index c7065a3859..2ec0209ec8 100644
--- a/api/resource/resource.go
+++ b/api/resource/resource.go
@@ -23,7 +23,6 @@ import (
 type Resource struct {
 	kyaml.RNode
 	options     *types.GenArgs
-	refBy       []resid.ResId
 	refVarNames []string
 }
 
@@ -35,6 +34,7 @@ var BuildAnnotations = []string{
 	utils.BuildAnnotationPreviousNamespaces,
 	utils.BuildAnnotationAllowNameChange,
 	utils.BuildAnnotationAllowKindChange,
+	utils.BuildAnnotationsRefBy,
 }
 
 func (r *Resource) ResetRNode(incoming *Resource) {
@@ -102,7 +102,6 @@ func (r *Resource) CopyMergeMetaDataFieldsFrom(other *Resource) error {
 
 func (r *Resource) copyKustomizeSpecificFields(other *Resource) {
 	r.options = other.options
-	r.refBy = other.copyRefBy()
 	r.refVarNames = copyStringSlice(other.refVarNames)
 }
 
@@ -144,10 +143,10 @@ func (r *Resource) ErrIfNotEquals(o *Resource) error {
 func (r *Resource) ReferencesEqual(other *Resource) bool {
 	setSelf := make(map[resid.ResId]bool)
 	setOther := make(map[resid.ResId]bool)
-	for _, ref := range other.refBy {
+	for _, ref := range other.GetRefBy() {
 		setOther[ref] = true
 	}
-	for _, ref := range r.refBy {
+	for _, ref := range r.GetRefBy() {
 		if _, ok := setOther[ref]; !ok {
 			return false
 		}
@@ -156,15 +155,6 @@ func (r *Resource) ReferencesEqual(other *Resource) bool {
 	return len(setSelf) == len(setOther)
 }
 
-func (r *Resource) copyRefBy() []resid.ResId {
-	if r.refBy == nil {
-		return nil
-	}
-	s := make([]resid.ResId, len(r.refBy))
-	copy(s, r.refBy)
-	return s
-}
-
 func copyStringSlice(s []string) []string {
 	if s == nil {
 		return nil
@@ -363,12 +353,18 @@ func (r *Resource) CurId() resid.ResId {
 
 // GetRefBy returns the ResIds that referred to current resource
 func (r *Resource) GetRefBy() []resid.ResId {
-	return r.refBy
+	var resIds []resid.ResId
+	asStrings := r.getCsvAnnotation(utils.BuildAnnotationsRefBy)
+	for _, s := range asStrings {
+		resIds = append(resIds, resid.FromString(s))
+	}
+	return resIds
 }
 
 // AppendRefBy appends a ResId into the refBy list
-func (r *Resource) AppendRefBy(id resid.ResId) {
-	r.refBy = append(r.refBy, id)
+// Using any type except fmt.Stringer here results in a compilation error
+func (r *Resource) AppendRefBy(id fmt.Stringer) {
+	r.appendCsvAnnotation(utils.BuildAnnotationsRefBy, id.String())
 }
 
 // GetRefVarNames returns vars that refer to current resource
diff --git a/api/resource/resource_test.go b/api/resource/resource_test.go
index ec462e75f6..4b5f2571ca 100644
--- a/api/resource/resource_test.go
+++ b/api/resource/resource_test.go
@@ -1133,3 +1133,41 @@ spec:
 		t.Fatalf("expected '%s', got '%s'", expected, actual)
 	}
 }
+
+func TestRefBy(t *testing.T) {
+	r, err := factory.FromBytes([]byte(`
+apiVersion: v1
+kind: Deployment
+metadata:
+  name: clown
+spec:
+  numReplicas: 1
+`))
+	assert.NoError(t, err)
+	r.AppendRefBy(resid.FromString("gr1_ver1_knd1|ns1|name1"))
+	assert.Equal(t, r.RNode.MustString(), `apiVersion: v1
+kind: Deployment
+metadata:
+  name: clown
+  annotations:
+    config.kubernetes.io/refBy: gr1_ver1_knd1|ns1|name1
+spec:
+  numReplicas: 1
+`)
+	assert.Equal(t, r.GetRefBy(), []resid.ResId{resid.FromString("gr1_ver1_knd1|ns1|name1")})
+
+	r.AppendRefBy(resid.FromString("gr2_ver2_knd2|ns2|name2"))
+	assert.Equal(t, r.RNode.MustString(), `apiVersion: v1
+kind: Deployment
+metadata:
+  name: clown
+  annotations:
+    config.kubernetes.io/refBy: gr1_ver1_knd1|ns1|name1,gr2_ver2_knd2|ns2|name2
+spec:
+  numReplicas: 1
+`)
+	assert.Equal(t, r.GetRefBy(), []resid.ResId{
+		resid.FromString("gr1_ver1_knd1|ns1|name1"),
+		resid.FromString("gr2_ver2_knd2|ns2|name2"),
+	})
+}