Backport of [NET-5622] build: consolidate Envoy version management to release/1.19.x

.github/workflows/reusable-get-envoy-versions.yml

@@ -0,0 +1,71 @@
+name: get-envoy-versions
+
+# Reads the canonical ENVOY_VERSIONS file for either the current branch or a specified version of Consul,
+# and returns both the max and all supported Envoy versions.
+
+on:
+  workflow_call:
+    inputs:
+      ref:
+        description: |
+          The Consul ref/branch (e.g. release/1.18.x) for which to determine supported Envoy versions.
+          If not provided, the default actions/checkout value (current ref) is used.
+        type: string
+    outputs:
+      max-envoy-version:
+        description: The max supported Envoy version for the specified Consul version
+        value: ${{ jobs.get-envoy-versions.outputs.max-envoy-version }}
+      envoy-versions:
+        description: |
+          All supported Envoy versions for the specified Consul version (formatted as multiline string with one version 
+          per line, in descending order)
+        value: ${{ jobs.get-envoy-versions.outputs.envoy-versions }}
+      envoy-versions-json:
+        description: |
+          All supported Envoy versions for the specified Consul version (formatted as JSON array)
+        value: ${{ jobs.get-envoy-versions.outputs.envoy-versions-json }}
+
+jobs:
+  get-envoy-versions:
+    name: "Determine supported Envoy versions"
+    runs-on: ubuntu-latest
+    outputs:
+      max-envoy-version: ${{ steps.get-envoy-versions.outputs.max-envoy-version }}
+      envoy-versions: ${{ steps.get-envoy-versions.outputs.envoy-versions }}
+      envoy-versions-json: ${{ steps.get-envoy-versions.outputs.envoy-versions-json }}
+    steps:
+      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+        with:
+          # If not set, will default to current branch.
+          ref: ${{ inputs.ref }}
+      - name: Determine Envoy versions
+        id: get-envoy-versions
+        # Note that this script assumes that the ENVOY_VERSIONS file is in the envoyextensions/xdscommon directory.
+        # If in the future this file moves between branches, we could introduce a workflow input for the path that
+        # defaults to the new value, and manually configure the old value as needed.
+        run: |
+          MAX_ENVOY_VERSION=$(cat envoyextensions/xdscommon/ENVOY_VERSIONS | grep '^[[:digit:]]' | sort -nr | head -n 1)
+          ENVOY_VERSIONS=$(cat envoyextensions/xdscommon/ENVOY_VERSIONS | grep '^[[:digit:]]' | sort -nr)
+          ENVOY_VERSIONS_JSON=$(echo -n '[' && echo "${ENVOY_VERSIONS}" | awk '{printf "\"%s\",", $0}' | sed 's/,$//' && echo -n ']')
+
+          # Loop through each line of ENVOY_VERSIONS and compare it to the regex
+          while IFS= read -r version; do
+            if ! [[ $version =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+              echo 'Invalid version in ENVOY_VERSIONS: '$version' does not match the pattern ^[0-9]+\.[0-9]+\.[0-9]+$'
+              exit 1
+            fi
+          done <<< "$ENVOY_VERSIONS"
+          if ! [[ $MAX_ENVOY_VERSION =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo 'Invalid MAX_ENVOY_VERSION: '$MAX_ENVOY_VERSION' does not match the pattern ^[0-9]+\.[0-9]+\.[0-9]+$'
+            exit 1
+          fi
+
+          echo "Supported Envoy versions:"
+          echo "${ENVOY_VERSIONS}"
+          echo "envoy-versions<<EOF" >> $GITHUB_OUTPUT
+          echo "${ENVOY_VERSIONS}" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+          echo "Supported Envoy versions JSON: ${ENVOY_VERSIONS_JSON}"
+          echo "envoy-versions-json=${ENVOY_VERSIONS_JSON}" >> $GITHUB_OUTPUT
+          echo "Max supported Envoy version: ${MAX_ENVOY_VERSION}"
+          echo "max-envoy-version=${MAX_ENVOY_VERSION}" >> $GITHUB_OUTPUT

.github/workflows/reusable-get-go-version.yml

@@ -2,6 +2,12 @@ name: get-go-version
 
 on:
   workflow_call:
+    inputs:
+      ref:
+        description: |
+          The Consul ref/branch (e.g. release/1.18.x) for which to determine the Go version.
+          If not provided, the default actions/checkout value (current ref) is used.
+        type: string
     outputs:
       go-version:
         description: "The Go version detected by this workflow"
@@ -19,6 +25,9 @@ jobs:
       go-version-previous: ${{ steps.get-go-version.outputs.go-version-previous }}
     steps:
       - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+        with:
+          # If not set, will default to current branch.
+          ref: ${{ inputs.ref }}
       - name: Determine Go version
         id: get-go-version
         # We use .go-version as our source of truth for current Go

.github/workflows/test-integrations.yml

@@ -63,6 +63,9 @@ jobs:
   get-go-version:
     uses: ./.github/workflows/reusable-get-go-version.yml
 
+  get-envoy-versions:
+    uses: ./.github/workflows/reusable-get-envoy-versions.yml
+
   dev-build:
     needs:
     - setup
@@ -269,28 +272,25 @@ jobs:
       - name: Generate Envoy Job Matrix
         id: set-matrix
         env:
-          # this is further going to multiplied in envoy-integration tests by the
-          # other dimensions in the matrix.  Currently TOTAL_RUNNERS would be
-          # multiplied by 2 based on these values:
-          # envoy-version: ["1.29.5"]
-          # xds-target: ["server", "client"]
-          TOTAL_RUNNERS: 2
+          # TEST_SPLITS sets the number of test case splits to use in the matrix. This will be
+          # further multiplied in envoy-integration tests by the other dimensions in the matrix
+          # to determine the total number of runners used.
+          TEST_SPLITS: 4
           JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]'
         run: |
-          NUM_RUNNERS=$TOTAL_RUNNERS
           NUM_DIRS=$(find ./test/integration/connect/envoy -mindepth 1 -maxdepth 1 -type d | wc -l)
 
-          if [ "$NUM_DIRS" -lt "$NUM_RUNNERS" ]; then
-            echo "TOTAL_RUNNERS is larger than the number of tests/packages to split."
-            NUM_RUNNERS=$((NUM_DIRS-1))
+          if [ "$NUM_DIRS" -lt "$TEST_SPLITS" ]; then
+            echo "TEST_SPLITS is larger than the number of tests/packages to split."
+            TEST_SPLITS=$((NUM_DIRS-1))
           fi
-          # fix issue where test splitting calculation generates 1 more split than TOTAL_RUNNERS.
-          NUM_RUNNERS=$((NUM_RUNNERS-1))
+          # fix issue where test splitting calculation generates 1 more split than TEST_SPLITS.
+          TEST_SPLITS=$((TEST_SPLITS-1))
           {
             echo -n "envoy-matrix="
             find ./test/integration/connect/envoy -maxdepth 1 -type d -print0 \
               | xargs -0 -n 1 basename \
-              | jq --raw-input --argjson runnercount "$NUM_RUNNERS" "$JQ_SLICER" \
+              | jq --raw-input --argjson runnercount "$TEST_SPLITS" "$JQ_SLICER" \
               | jq --compact-output 'map(join("|"))'
           } >> "$GITHUB_OUTPUT"
 
@@ -299,6 +299,7 @@ jobs:
     needs:
       - setup
       - get-go-version
+      - get-envoy-versions
       - generate-envoy-job-matrices
       - dev-build
     permissions:
@@ -307,11 +308,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        envoy-version: ["1.29.5"]
         xds-target: ["server", "client"]
         test-cases: ${{ fromJSON(needs.generate-envoy-job-matrices.outputs.envoy-matrix) }}
     env:
-      ENVOY_VERSION: ${{ matrix.envoy-version }}
+      ENVOY_VERSION: ${{ needs.get-envoy-versions.outputs.max-envoy-version }}
       XDS_TARGET: ${{ matrix.xds-target }}
       AWS_LAMBDA_REGION: us-west-2
     steps:
@@ -392,13 +392,14 @@ jobs:
     needs:
       - setup
       - get-go-version
+      - get-envoy-versions
       - dev-build
     permissions:
       id-token: write # NOTE: this permission is explicitly required for Vault auth.
       contents: read
     env:
-      ENVOY_VERSION: "1.29.5"
-      CONSUL_DATAPLANE_IMAGE: "docker.io/hashicorppreview/consul-dataplane:1.3-dev-ubi"
+      ENVOY_VERSION: ${{ needs.get-envoy-versions.outputs.max-envoy-version }}
+      CONSUL_DATAPLANE_IMAGE: "docker.io/hashicorppreview/consul-dataplane:1.5-dev-ubi"
     steps:
       - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
       # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos.
@@ -511,7 +512,7 @@ jobs:
     strategy:
       fail-fast: false
     env:
-      DEPLOYER_CONSUL_DATAPLANE_IMAGE: "docker.mirror.hashicorp.services/hashicorppreview/consul-dataplane:1.3-dev"
+      DEPLOYER_CONSUL_DATAPLANE_IMAGE: "docker.mirror.hashicorp.services/hashicorppreview/consul-dataplane:1.5-dev"
     steps:
       - name: Checkout code
         uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4

Makefile

@@ -71,7 +71,8 @@ CONSUL_IMAGE_VERSION?=latest
 # When changing the method of Go version detection, also update
 # version detection in CI workflows (reusable-get-go-version.yml).
 GOLANG_VERSION?=$(shell head -n 1 .go-version)
-ENVOY_VERSION?='1.29.5'
+# Takes the highest version from the ENVOY_VERSIONS file.
+ENVOY_VERSION?=$(shell cat envoyextensions/xdscommon/ENVOY_VERSIONS | grep '^[[:digit:]]' | sort -nr | head -n 1)
 CONSUL_DATAPLANE_IMAGE := $(or $(CONSUL_DATAPLANE_IMAGE),"docker.io/hashicorppreview/consul-dataplane:1.3-dev-ubi")
 DEPLOYER_CONSUL_DATAPLANE_IMAGE := $(or $(DEPLOYER_CONSUL_DATAPLANE_IMAGE), "docker.io/hashicorppreview/consul-dataplane:1.3-dev")
 

command/connect/envoy/envoy.go

@@ -1052,7 +1052,7 @@ func checkEnvoyVersionCompatibility(envoyVersion string, unsupportedList []strin
 
 	// Next build the constraint string using the bounds, make sure that we are less than but not equal to
 	// maxSupported since we will add 1. Need to add one to the max minor version so that we accept all patches
-	splitS := strings.Split(xdscommon.GetMaxEnvoyMinorVersion(), ".")
+	splitS := strings.Split(xdscommon.GetMaxEnvoyMajorVersion(), ".")
 	minor, err := strconv.Atoi(splitS[1])
 	if err != nil {
 		return envoyCompat{}, err
@@ -1061,7 +1061,7 @@ func checkEnvoyVersionCompatibility(envoyVersion string, unsupportedList []strin
 	maxSupported := fmt.Sprintf("%s.%d", splitS[0], minor)
 
 	cs.Reset()
-	cs.WriteString(fmt.Sprintf(">= %s, < %s", xdscommon.GetMinEnvoyMinorVersion(), maxSupported))
+	cs.WriteString(fmt.Sprintf(">= %s, < %s", xdscommon.GetMinEnvoyMajorVersion(), maxSupported))
 	constraints, err := version.NewConstraint(cs.String())
 	if err != nil {
 		return envoyCompat{}, err

command/connect/envoy/envoy_test.go

@@ -1850,7 +1850,7 @@ func TestCheckEnvoyVersionCompatibility(t *testing.T) {
 		},
 		{
 			name:            "supported-at-max",
-			envoyVersion:    xdscommon.GetMaxEnvoyMinorVersion(),
+			envoyVersion:    xdscommon.GetMaxEnvoyMajorVersion(),
 			unsupportedList: xdscommon.UnsupportedEnvoyVersions,
 			expectedCompat: envoyCompat{
 				isCompatible: true,

envoyextensions/xdscommon/ENVOY_VERSIONS

@@ -0,0 +1,14 @@
+# This file represents the canonical list of supported Envoy versions for this version of Consul.
+#
+# Every line must contain a valid version number in the format "x.y.z" where x, y, and z are integers.
+# All other lines must be comments beginning with a "#", or a blank line.
+#
+# Every prior "minor" version for a given "major" (x.y) version is implicitly supported unless excluded by
+# `xdscommon.UnsupportedEnvoyVersions`. For example, 1.28.3 implies support for 1.28.0, 1.28.1, and 1.28.2.
+#
+# See https://www.consul.io/docs/connect/proxies/envoy#supported-versions for more information on Consul's Envoy
+# version support.
+1.29.5
+1.28.4
+1.27.6
+1.26.8

envoyextensions/xdscommon/envoy_versioning.go

@@ -13,7 +13,7 @@ import (
 var (
 	// minSupportedVersion is the oldest mainline version we support. This should always be
 	// the zero'th point release of the last element of xdscommon.EnvoyVersions.
-	minSupportedVersion = version.Must(version.NewVersion(GetMinEnvoyMinorVersion()))
+	minSupportedVersion = version.Must(version.NewVersion(GetMinEnvoyMajorVersion()))
 
 	specificUnsupportedVersions = []unsupportedVersion{}
 )

envoyextensions/xdscommon/envoy_versioning_test.go

@@ -4,6 +4,8 @@
 package xdscommon
 
 import (
+	"fmt"
+	"slices"
 	"testing"
 
 	envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
@@ -70,99 +72,53 @@ func TestDetermineEnvoyVersionFromNode(t *testing.T) {
 }
 
 func TestDetermineSupportedProxyFeaturesFromString(t *testing.T) {
-	const (
-		errTooOld = "is too old and is not supported by Consul"
-	)
+	const errTooOld = "is too old and is not supported by Consul"
 
 	type testcase struct {
+		name      string
 		expect    SupportedProxyFeatures
 		expectErr string
 	}
+	var cases []testcase
 
-	// Just the bad versions
-	cases := map[string]testcase{
-		"1.9.0":   {expectErr: "Envoy 1.9.0 " + errTooOld},
-		"1.10.0":  {expectErr: "Envoy 1.10.0 " + errTooOld},
-		"1.11.0":  {expectErr: "Envoy 1.11.0 " + errTooOld},
-		"1.12.0":  {expectErr: "Envoy 1.12.0 " + errTooOld},
-		"1.12.1":  {expectErr: "Envoy 1.12.1 " + errTooOld},
-		"1.12.2":  {expectErr: "Envoy 1.12.2 " + errTooOld},
-		"1.12.3":  {expectErr: "Envoy 1.12.3 " + errTooOld},
-		"1.12.4":  {expectErr: "Envoy 1.12.4 " + errTooOld},
-		"1.12.5":  {expectErr: "Envoy 1.12.5 " + errTooOld},
-		"1.12.6":  {expectErr: "Envoy 1.12.6 " + errTooOld},
-		"1.12.7":  {expectErr: "Envoy 1.12.7 " + errTooOld},
-		"1.13.0":  {expectErr: "Envoy 1.13.0 " + errTooOld},
-		"1.13.1":  {expectErr: "Envoy 1.13.1 " + errTooOld},
-		"1.13.2":  {expectErr: "Envoy 1.13.2 " + errTooOld},
-		"1.13.3":  {expectErr: "Envoy 1.13.3 " + errTooOld},
-		"1.13.4":  {expectErr: "Envoy 1.13.4 " + errTooOld},
-		"1.13.5":  {expectErr: "Envoy 1.13.5 " + errTooOld},
-		"1.13.6":  {expectErr: "Envoy 1.13.6 " + errTooOld},
-		"1.13.7":  {expectErr: "Envoy 1.13.7 " + errTooOld},
-		"1.14.0":  {expectErr: "Envoy 1.14.0 " + errTooOld},
-		"1.14.1":  {expectErr: "Envoy 1.14.1 " + errTooOld},
-		"1.14.2":  {expectErr: "Envoy 1.14.2 " + errTooOld},
-		"1.14.3":  {expectErr: "Envoy 1.14.3 " + errTooOld},
-		"1.14.4":  {expectErr: "Envoy 1.14.4 " + errTooOld},
-		"1.14.5":  {expectErr: "Envoy 1.14.5 " + errTooOld},
-		"1.14.6":  {expectErr: "Envoy 1.14.6 " + errTooOld},
-		"1.14.7":  {expectErr: "Envoy 1.14.7 " + errTooOld},
-		"1.15.0":  {expectErr: "Envoy 1.15.0 " + errTooOld},
-		"1.15.1":  {expectErr: "Envoy 1.15.1 " + errTooOld},
-		"1.15.2":  {expectErr: "Envoy 1.15.2 " + errTooOld},
-		"1.15.3":  {expectErr: "Envoy 1.15.3 " + errTooOld},
-		"1.15.4":  {expectErr: "Envoy 1.15.4 " + errTooOld},
-		"1.15.5":  {expectErr: "Envoy 1.15.5 " + errTooOld},
-		"1.16.1":  {expectErr: "Envoy 1.16.1 " + errTooOld},
-		"1.16.2":  {expectErr: "Envoy 1.16.2 " + errTooOld},
-		"1.16.3":  {expectErr: "Envoy 1.16.3 " + errTooOld},
-		"1.16.4":  {expectErr: "Envoy 1.16.4 " + errTooOld},
-		"1.16.5":  {expectErr: "Envoy 1.16.5 " + errTooOld},
-		"1.16.6":  {expectErr: "Envoy 1.16.6 " + errTooOld},
-		"1.17.4":  {expectErr: "Envoy 1.17.4 " + errTooOld},
-		"1.18.6":  {expectErr: "Envoy 1.18.6 " + errTooOld},
-		"1.19.5":  {expectErr: "Envoy 1.19.5 " + errTooOld},
-		"1.20.7":  {expectErr: "Envoy 1.20.7 " + errTooOld},
-		"1.21.5":  {expectErr: "Envoy 1.21.5 " + errTooOld},
-		"1.22.0":  {expectErr: "Envoy 1.22.0 " + errTooOld},
-		"1.22.1":  {expectErr: "Envoy 1.22.1 " + errTooOld},
-		"1.22.2":  {expectErr: "Envoy 1.22.2 " + errTooOld},
-		"1.22.3":  {expectErr: "Envoy 1.22.3 " + errTooOld},
-		"1.22.4":  {expectErr: "Envoy 1.22.4 " + errTooOld},
-		"1.22.5":  {expectErr: "Envoy 1.22.5 " + errTooOld},
-		"1.22.6":  {expectErr: "Envoy 1.22.6 " + errTooOld},
-		"1.22.7":  {expectErr: "Envoy 1.22.7 " + errTooOld},
-		"1.22.8":  {expectErr: "Envoy 1.22.8 " + errTooOld},
-		"1.22.9":  {expectErr: "Envoy 1.22.9 " + errTooOld},
-		"1.22.10": {expectErr: "Envoy 1.22.10 " + errTooOld},
-		"1.22.11": {expectErr: "Envoy 1.22.11 " + errTooOld},
+	// Bad versions.
+	minMajorVersion := version.Must(version.NewVersion(getMinEnvoyVersion()))
+	minMajorVersionMajorPart := minMajorVersion.Segments()[len(minMajorVersion.Segments())-2]
+	for major := 9; major < minMajorVersionMajorPart; major++ {
+		for minor := 0; minor < 10; minor++ {
+			cases = append(cases, testcase{
+				name:      version.Must(version.NewVersion(fmt.Sprintf("1.%d.%d", major, minor))).String(),
+				expectErr: errTooOld,
+			})
+		}
 	}
 
-	// Insert a bunch of valid versions.
-	// Populate feature flags here when appropriate. See consul 1.10.x for reference.
-	/* Example from 1.18
-	for _, v := range []string{
-		"1.18.0", "1.18.1", "1.18.2", "1.18.3", "1.18.4", "1.18.5", "1.18.6",
-	} {
-		cases[v] = testcase{expect: SupportedProxyFeatures{
-			ForceLDSandCDSToAlwaysUseWildcardsOnReconnect: true,
-		}}
-	}
-	*/
-	for _, v := range []string{
-		"1.26.0", "1.26.1", "1.26.2", "1.26.3", "1.26.4", "1.26.5", "1.26.6", "1.26.7", "1.26.8",
-		"1.27.0", "1.27.1", "1.27.2", "1.27.3", "1.27.4", "1.27.5", "1.27.6",
-		"1.28.0", "1.28.1", "1.28.2", "1.28.3", "1.28.4",
-		"1.29.0", "1.29.1", "1.29.2", "1.29.3", "1.29.4", "1.29.5",
-	} {
-		cases[v] = testcase{expect: SupportedProxyFeatures{}}
+	// Good versions.
+	// Sort ascending so test output is ordered like bad cases above.
+	var supportedVersionsAscending []string
+	supportedVersionsAscending = append(supportedVersionsAscending, EnvoyVersions...)
+	slices.Reverse(supportedVersionsAscending)
+	for _, v := range supportedVersionsAscending {
+		envoyVersion := version.Must(version.NewVersion(v))
+		// e.g. this is 27 in 1.27.4
+		versionMajorPart := envoyVersion.Segments()[len(envoyVersion.Segments())-2]
+		// e.g. this is 4 in 1.27.4
+		versionMinorPart := envoyVersion.Segments()[len(envoyVersion.Segments())-1]
+
+		// Create synthetic minor versions from .0 through the actual configured version.
+		for minor := 0; minor <= versionMinorPart; minor++ {
+			minorVersion := version.Must(version.NewVersion(fmt.Sprintf("1.%d.%d", versionMajorPart, minor)))
+			cases = append(cases, testcase{
+				name:   minorVersion.String(),
+				expect: SupportedProxyFeatures{},
+			})
+		}
 	}
 
-	for name, tc := range cases {
+	for _, tc := range cases {
 		tc := tc
-		t.Run(name, func(t *testing.T) {
-			sf, err := DetermineSupportedProxyFeaturesFromString(name)
+		t.Run(tc.name, func(t *testing.T) {
+			sf, err := DetermineSupportedProxyFeaturesFromString(tc.name)
 			if tc.expectErr == "" {
 				require.NoError(t, err)
 				require.Equal(t, tc.expect, sf)