Merge branch 'master' into bpf-iface-autodetect

.gitignore

@@ -19,6 +19,7 @@ node/windows-packaging/nssm.exe
 _output
 builder.coverprofile
 *.log
+.release-*.*
 
 /* Created by local kind cluster */
 hack/test/kind/kind

.semaphore/release/hashrelease.yml

@@ -40,12 +40,10 @@ blocks:
       jobs:
         - name: Build and publish hashrelease
           commands:
+            - if [[ ${SEMAPHORE_WORKFLOW_TRIGGERED_BY_SCHEDULE} == "true" ]]; then export BUILD_IMAGES=true; export SKIP_PUBLISH_IMAGES=false; fi
             - make hashrelease
       prologue:
         commands:
           - export GITHUB_TOKEN=${MARVIN_GITHUB_TOKEN}
           - cd release
           - make build
-      env_vars:
-        - name: IS_HASHRELEASE
-          value: "true"

.semaphore/vms/create-test-vm

@@ -28,11 +28,11 @@ gcloud auth activate-service-account --key-file=$gcp_secret_key
 function create-vm() {
   gcloud --quiet compute instances create "${vm_name}" \
            --zone=${zone} \
-           --machine-type=n1-standard-4 \
-           --image=ubuntu-2004-focal-v20211102 \
+           --machine-type=n4-standard-4 \
+           --image=ubuntu-2004-focal-v20241115 \
            --image-project=ubuntu-os-cloud \
            --boot-disk-size=$disk_size \
-           --boot-disk-type=pd-standard && \
+           --boot-disk-type=hyperdisk-balanced && \
   ssh_cmd="gcloud --quiet compute ssh --zone=${zone} ubuntu@${vm_name}"
   for ssh_try in $(seq 1 10); do
     echo "Trying to SSH in: $ssh_try"

Makefile

@@ -119,7 +119,7 @@ release/bin/release: $(shell find ./release -type f -name '*.go')
 
 # Install ghr for publishing to github.
 bin/ghr:
-	$(DOCKER_RUN) -e GOBIN=/go/src/$(PACKAGE_NAME)/bin/ $(CALICO_BUILD) go install github.com/tcnksm/ghr@v0.14.0
+	$(DOCKER_RUN) -e GOBIN=/go/src/$(PACKAGE_NAME)/bin/ $(CALICO_BUILD) go install github.com/tcnksm/ghr@$(GHR_VERSION)
 
 # Build a release.
 release: release/bin/release

apiserver/Makefile

@@ -223,16 +223,6 @@ release-build: .release-$(VERSION).created
 	$(MAKE) FIPS=true retag-build-images-with-registries IMAGETAG=latest-fips RELEASE=true LATEST_IMAGE_TAG=latest-fips
 	touch $@
 
-## Verifies the release artifacts produces by `make release-build` are correct.
-release-verify: release-prereqs
-	# Check the reported version is correct for each release artifact.
-	if ! docker run calico/apiserver | grep 'Version:\s*$(VERSION)$$'; then \
-	  echo "Reported version:" `docker run calico/apiserver` "\nExpected version: $(VERSION)"; \
-	  false; \
-	else \
-	  echo "Version check passed\n"; \
-	fi
-
 ## Pushes a github release and release artifacts produced by `make release-build`.
 release-publish: release-prereqs .release-$(VERSION).published
 .release-$(VERSION).published:

app-policy/Makefile

@@ -75,7 +75,7 @@ build-all: $(VALIDARCHES)
 
 .PHONY: build
 ## Build the binary for the current architecture and platform
-build: 
+build:
 	$(MAKE) $(BINDIR)/dikastes-$(ARCH) ARCH=$(ARCH)
 	$(MAKE) $(BINDIR)/healthz-$(ARCH) ARCH=$(ARCH)
 
@@ -103,7 +103,7 @@ endif
 
 
 ###############################################################################
-# Protobufs, 
+# Protobufs,
 #
 # 1. defer to felix's makefile for felixbackend stuff
 # 2. build proto for healthz
@@ -202,16 +202,6 @@ release-build: .release-$(VERSION).created
 	$(MAKE) FIPS=true retag-build-images-with-registries IMAGETAG=latest-fips RELEASE=true LATEST_IMAGE_TAG=latest-fips
 	touch $@
 
-## Verifies the release artifacts produces by `make release-build` are correct.
-release-verify: release-prereqs
-	# Check the reported version is correct for each release artifact.
-	if ! docker run $(DIKASTES_IMAGE):$(VERSION)-$(ARCH) /dikastes --version | grep '^$(VERSION)$$'; then \
-	  echo "Reported version:" `docker run $(DIKASTES_IMAGE):$(VERSION)-$(ARCH) /dikastes --version` "\nExpected version: $(VERSION)"; \
-	  false; \
-	else \
-	  echo "Version check passed\n"; \
-	fi
-
 ## Pushes a github release and release artifacts produced by `make release-build`.
 release-publish: release-prereqs .release-$(VERSION).published
 .release-$(VERSION).published:

calicoctl/Makefile

@@ -260,16 +260,6 @@ release-build: .release-$(VERSION).created
 	$(MAKE) retag-build-images-with-registries IMAGETAG=latest RELEASE=true
 	touch $@
 
-## Verifies the release artifacts produces by `make release-build` are correct.
-release-verify: release-prereqs
-	# Check the reported version is correct for each release artifact.
-	if ! docker run $(CALICOCTL_IMAGE):$(VERSION)-$(ARCH) version | grep 'Version:\s*$(VERSION)$$'; then \
-	  echo "Reported version:" `docker run $(CALICOCTL_IMAGE):$(VERSION)-$(ARCH) version` "\nExpected version: $(VERSION)"; \
-	  false; \
-	else \
-	  echo "Version check passed\n"; \
-	fi
-
 ## Pushes a github release and release artifacts produced by `make release-build`.
 release-publish: release-prereqs .release-$(VERSION).published
 .release-$(VERSION).published:

calicoctl/calicoctl/commands/datastore/migrate/export.go

@@ -48,38 +48,35 @@ var title = cases.Title(language.English)
 var allV3Resources []string = []string{
 	"ippools",
 	"bgppeers",
+	"tiers", // Must come before policies since policies reference tiers.
 	"globalnetworkpolicies",
 	"globalnetworksets",
-	"heps",
-	"kubecontrollersconfigs",
+	"hostendpoints",
+	"kubecontrollersconfigurations",
 	"networkpolicies",
 	"networksets",
-	"nodes",
-	"bgpconfigs",
-	"felixconfigs",
+	"nodes", // Must be before resources that reference nodes.
+	"bgpconfigurations",
+	"felixconfigurations",
 	"ipreservations",
 	"bgpfilters",
 }
 
 var resourceDisplayMap map[string]string = map[string]string{
-	"ipamBlocks":             "IPAMBlocks",
-	"blockaffinities":        "BlockAffinities",
-	"ipamhandles":            "IPAMHandles",
-	"ipamconfigs":            "IPAMConfigurations",
-	"ippools":                "IPPools",
-	"bgpconfigs":             "BGPConfigurations",
-	"bgppeers":               "BGPPeers",
-	"clusterinfos":           "ClusterInformations",
-	"felixconfigs":           "FelixConfigurations",
-	"globalnetworkpolicies":  "GlobalNetworkPolicies",
-	"globalnetworksets":      "GlobalNetworkSets",
-	"heps":                   "HostEndpoints",
-	"kubecontrollersconfigs": "KubeControllersConfigurations",
-	"networkpolicies":        "NetworkPolicies",
-	"networksets":            "Networksets",
-	"nodes":                  "Nodes",
-	"ipreservations":         "IPReservations",
-	"bgpfilters":             "BGPFilters",
+	"ippools":                       "IPPools",
+	"bgpconfigurations":             "BGPConfigurations",
+	"bgppeers":                      "BGPPeers",
+	"felixconfigurations":           "FelixConfigurations",
+	"globalnetworkpolicies":         "GlobalNetworkPolicies",
+	"globalnetworksets":             "GlobalNetworkSets",
+	"hostendpoints":                 "HostEndpoints",
+	"kubecontrollersconfigurations": "KubeControllersConfigurations",
+	"networkpolicies":               "NetworkPolicies",
+	"networksets":                   "NetworkSets",
+	"nodes":                         "Nodes",
+	"ipreservations":                "IPReservations",
+	"bgpfilters":                    "BGPFilters",
+	"tiers":                         "Tiers",
 }
 
 var namespacedResources map[string]struct{} = map[string]struct{}{
@@ -289,7 +286,7 @@ Description:
 
 			// Felix configs may also need to be modified if node names do not match the Kubernetes node names.
 			// Felix configs must come after nodes in the allV3Resources list since we populate the node mapping when nodes are exported.
-			if r == "felixconfigs" {
+			if r == "felixconfigurations" {
 				err := meta.EachListItem(resource, func(obj runtime.Object) error {
 					felixConfig, ok := obj.(*apiv3.FelixConfiguration)
 					if !ok {
@@ -315,7 +312,7 @@ Description:
 
 			// BGP configs may also need to be modified if node names do not match the Kubernetes node names.
 			// BGP configs must come after nodes in the allV3Resources list since we populate the node mapping when nodes are exported.
-			if r == "bgpconfigs" {
+			if r == "bgpconfigurations" {
 				err := meta.EachListItem(resource, func(obj runtime.Object) error {
 					bgpConfig, ok := obj.(*apiv3.BGPConfiguration)
 					if !ok {

calicoctl/calicoctl/commands/datastore/migrate/export_test.go

@@ -12,14 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package migrate_test
+package migrate
 
 import (
+	"strings"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3"
 
-	"github.com/projectcalico/calico/calicoctl/calicoctl/commands/datastore/migrate"
+	"github.com/projectcalico/calico/libcalico-go/lib/backend/model"
+	"github.com/projectcalico/calico/libcalico-go/lib/set"
 )
 
 var _ = Describe("Etcd to KDD Migration Export handling", func() {
@@ -33,7 +36,7 @@ var _ = Describe("Etcd to KDD Migration Export handling", func() {
 				IptablesFilterDenyAction:    "DROP",
 			}
 
-			migrate.ConvertIptablesFields(felixConfig)
+			ConvertIptablesFields(felixConfig)
 			Expect(felixConfig.Spec.DefaultEndpointToHostAction).To(Equal("Drop"))
 			Expect(felixConfig.Spec.IptablesFilterAllowAction).To(Equal("Accept"))
 			Expect(felixConfig.Spec.IptablesMangleAllowAction).To(Equal("Return"))
@@ -49,7 +52,7 @@ var _ = Describe("Etcd to KDD Migration Export handling", func() {
 				IptablesFilterDenyAction:    "Drop",
 			}
 
-			migrate.ConvertIptablesFields(felixConfig)
+			ConvertIptablesFields(felixConfig)
 			Expect(felixConfig.Spec.DefaultEndpointToHostAction).To(Equal("Drop"))
 			Expect(felixConfig.Spec.IptablesFilterAllowAction).To(Equal("Accept"))
 			Expect(felixConfig.Spec.IptablesMangleAllowAction).To(Equal("Return"))
@@ -60,11 +63,48 @@ var _ = Describe("Etcd to KDD Migration Export handling", func() {
 			felixConfig := apiv3.NewFelixConfiguration()
 			felixConfig.Spec = apiv3.FelixConfigurationSpec{}
 
-			migrate.ConvertIptablesFields(felixConfig)
+			ConvertIptablesFields(felixConfig)
 			Expect(felixConfig.Spec.DefaultEndpointToHostAction).To(Equal(""))
 			Expect(felixConfig.Spec.IptablesFilterAllowAction).To(Equal(""))
 			Expect(felixConfig.Spec.IptablesMangleAllowAction).To(Equal(""))
 			Expect(felixConfig.Spec.IptablesFilterDenyAction).To(Equal(""))
 		})
 	})
+
+	It("should cover all calico resources", func() {
+		allPlurals := set.FromArray(model.AllResourcePlurals())
+
+		// Profiles are backed by k8s resources in KDD.  User cannot create
+		// their own.
+		allPlurals.Discard("profiles")
+		// WEPs are backed by Pods in KDD.
+		allPlurals.Discard("workloadendpoints")
+		// ClusterInformation is generated fresh in the new cluster.
+		allPlurals.Discard("clusterinformations")
+		// Not supported in KDD (OpenStack only).
+		allPlurals.Discard("caliconodestatuses")
+		// Handled by IPAM migration code.
+		allPlurals.Discard("ipamconfigs")
+		allPlurals.Discard("blockaffinities")
+
+		allPlurals.Iter(func(resource string) error {
+			if strings.HasPrefix(resource, "kubernetes") {
+				// "kubernetes"-prefixed resources are backed by Kubernetes API
+				// objects, not Calico objects.
+				return set.RemoveItem
+			}
+			return nil
+		})
+
+		Expect(allV3Resources).To(ConsistOf(allPlurals.Slice()))
+	})
+
+	It("should have names for all resources", func() {
+		var keys []string
+		for k := range resourceDisplayMap {
+			keys = append(keys, k)
+		}
+		Expect(keys).To(ConsistOf(allV3Resources),
+			"expected to see names for the listed calico resources (only)")
+	})
 })

calicoctl/calicoctl/commands/datastore/migrate/migrate_suite_test.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package migrate_test
+package migrate
 
 import (
 	"testing"

calicoctl/calicoctl/commands/datastore/migrate/migrateipam_test.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package migrate_test
+package migrate
 
 import (
 	"context"
@@ -21,7 +21,6 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/projectcalico/calico/calicoctl/calicoctl/commands/datastore/migrate"
 	bapi "github.com/projectcalico/calico/libcalico-go/lib/backend/api"
 	"github.com/projectcalico/calico/libcalico-go/lib/backend/model"
 	client "github.com/projectcalico/calico/libcalico-go/lib/clientv3"
@@ -99,7 +98,7 @@ var _ = Describe("IPAM migration handling", func() {
 
 		bc := NewMockIPAMBackendClient(blocks, affinities, handles)
 		client := NewMockIPAMClient(bc)
-		migrateIPAM := migrate.NewMigrateIPAM(client)
+		migrateIPAM := NewMigrateIPAM(client)
 		migrateIPAM.SetNodeMap(map[string]string{nodeName: newNodeName})
 		err := migrateIPAM.PullFromDatastore()
 		Expect(err).NotTo(HaveOccurred())
@@ -144,7 +143,7 @@ var _ = Describe("IPAM migration handling", func() {
 
 		bc := NewMockIPAMBackendClient(blocks, affinities, handles)
 		client := NewMockIPAMClient(bc)
-		migrateIPAM := migrate.NewMigrateIPAM(client)
+		migrateIPAM := NewMigrateIPAM(client)
 		migrateIPAM.SetNodeMap(map[string]string{nodeName: nodeName})
 		err := migrateIPAM.PullFromDatastore()
 		Expect(err).NotTo(HaveOccurred())

cni-plugin/Makefile

@@ -284,29 +284,7 @@ release-build: .release-$(VERSION).created
 	$(MAKE) FIPS=true retag-build-images-with-registries RELEASE=true IMAGETAG=latest-fips LATEST_IMAGE_TAG=latest-fips
 	touch $@
 
-## Verifies the release artifacts produces by `make release-build` are correct.
-release-verify: release-prereqs
-	# Check the reported version is correct for each release artifact.
-	$(MAKE) release-verify-version IMAGE=calico/cni:$(VERSION)-$(ARCH)
-	$(MAKE) release-verify-version IMAGE=calico/cni:$(VERSION)-fips-$(ARCH)
-	$(MAKE) release-verify-version IMAGE=quay.io/calico/cni:$(VERSION)-$(ARCH)
-	$(MAKE) release-verify-version IMAGE=quay.io/calico/cni:$(VERSION)-fips-$(ARCH)
-	# Check that the FIPS binaries have the correct symbols.
-	$(MAKE) release-verify-fips IMAGE=calico/cni:$(VERSION)-fips-$(ARCH)
-	$(MAKE) release-verify-fips IMAGE=quay.io/calico/cni:$(VERSION)-fips-$(ARCH)
-
-release-verify-version:
-	docker run --rm $(IMAGE) calico -v | grep -x $(VERSION) || ( echo "Reported version does not match" && exit 1 )
-	docker run --rm $(IMAGE) calico-ipam -v | grep -x $(VERSION) || ( echo "Reported version does not match" && exit 1 )
-
-release-verify-fips:
-	rm -rf .tmp && mkdir -p .tmp
-	# Copy binaries from the image so we can analyze them.
-	sh -c "docker create --name calico-cni-verify $(IMAGE); docker cp calico-cni-verify:/opt/cni/bin/install .tmp/calico; docker rm -f calico-cni-verify"
-	go tool nm .tmp/calico | grep '_Cfunc__goboringcrypto_' 1> /dev/null || echo "ERROR: Binary in image '$(IMAGE)' is missing expected goboring symbols"
-	rm -rf .tmp
-
-release-publish: release-prereqs release-verify .release-$(VERSION).published
+release-publish: release-prereqs .release-$(VERSION).published
 .release-$(VERSION).published:
 	$(MAKE) push-images-to-registries push-manifests IMAGETAG=$(VERSION) RELEASE=$(RELEASE) CONFIRM=$(CONFIRM)
 	$(MAKE) FIPS=true push-images-to-registries push-manifests IMAGETAG=$(VERSION)-fips RELEASE=$(RELEASE) CONFIRM=$(CONFIRM)
@@ -318,7 +296,7 @@ release-publish: release-prereqs release-verify .release-$(VERSION).published
 # WARNING: Only run this target if this release is the latest stable release. Do NOT
 # run this target for alpha / beta / release candidate builds, or patches to earlier Calico versions.
 ## Pushes `latest` release images. WARNING: Only run this for latest stable releases.
-release-publish-latest: release-prereqs release-verify
+release-publish-latest: release-prereqs
 	# Check latest versions match.
 	if ! docker run $(CNI_PLUGIN_IMAGE):latest-$(ARCH) calico -v | grep '^$(VERSION)$$'; then echo "Reported version:" `docker run $(CNI_PLUGIN_IMAGE):latest-$(ARCH) calico -v` "\nExpected version: $(VERSION)"; false; else echo "\nVersion check passed\n"; fi
 	if ! docker run quay.io/$(CNI_PLUGIN_IMAGE):latest-$(ARCH) calico -v | grep '^$(VERSION)$$'; then echo "Reported version:" `docker run quay.io/$(CNI_PLUGIN_IMAGE):latest-$(ARCH) calico -v` "\nExpected version: $(VERSION)"; false; else echo "\nVersion check passed\n"; fi