Merge branch 'main' into NET-2678/xw-http-api-write

hashicorp · Jul 25, 2023 · 884b79d · 884b79d
2 parents 1c2efc2 + 9b540e2
commit 884b79d
Show file tree

Hide file tree

Showing 46 changed files with 6,165 additions and 3,710 deletions.
diff --git a/.changelog/18184.txt b/.changelog/18184.txt
@@ -0,0 +1,3 @@
+```release-note:bug
+api: Fix client deserialization errors by marking new Enterprise-only prepared query fields as omit empty
+```
diff --git a/.changelog/18223.txt b/.changelog/18223.txt
@@ -0,0 +1,3 @@
+```release-note:feature
+cli: `consul members` command uses `-filter` expression to filter members based on bexpr.
+```
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -282,7 +282,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        arch: ["386", "amd64", "arm", "arm64"]
+        include:
+          - { arch: "386" }
+          - { arch: "arm" }
+          - { arch: "amd64" }
+          - { arch: "arm64" }
     env:
       repo: ${{github.event.repository.name}}
       version: ${{needs.set-product-version.outputs.product-version}}
@@ -371,7 +375,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        arch: ["386", "amd64", "arm", "arm64"]
+        include:
+          - { arch: "386" }
+          - { arch: "arm" }
+          - { arch: "amd64" }
+          - { arch: "arm64" }
       fail-fast: true
     env:
       version: ${{ needs.set-product-version.outputs.product-version }}
@@ -432,7 +440,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        arch: ["i386", "amd64", "armhf", "arm64"]
+        include:
+          - { arch: "i386" }
+          - { arch: "armhf" }
+          - { arch: "amd64" }
+          - { arch: "arm64" }
       # fail-fast: true
     env:
       version: ${{ needs.set-product-version.outputs.product-version }}
@@ -469,8 +481,12 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
+        include:
+          - { arch: "i386" }
+          - { arch: "x86_64" }
         # TODO(eculver): re-enable when there is a smaller verification container available
-        arch: ["i386", "x86_64"] #, "armv7hl", "aarch64"]
+          # - { arch: "armv7hl" }
+          # - { arch: "aarch64" }
     env:
       version: ${{ needs.set-product-version.outputs.product-version }}
 

diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml
@@ -184,18 +184,18 @@ jobs:
     secrets:
       elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
 
-  dev-build-s390x:
-    if: ${{ endsWith(github.repository, '-enterprise') }}
-    needs:
-    - setup
-    uses: ./.github/workflows/reusable-dev-build.yml
-    with:
-      uploaded-binary-name: 'consul-bin-s390x'
-      runs-on: ${{ needs.setup.outputs.compute-xl }}
-      go-arch: "s390x"
-      repository-name: ${{ github.repository }}
-    secrets:
-      elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
+  # dev-build-s390x:
+  #   if: ${{ endsWith(github.repository, '-enterprise') }}
+  #   needs:
+  #   - setup
+  #   uses: ./.github/workflows/reusable-dev-build.yml
+  #   with:
+  #     uploaded-binary-name: 'consul-bin-s390x'
+  #     runs-on: ${{ needs.setup.outputs.compute-xl }}
+  #     go-arch: "s390x"
+  #     repository-name: ${{ github.repository }}
+  #   secrets:
+  #     elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
 
   # dev-build-arm64:
   #   # only run on enterprise because GHA does not have arm64 runners in OSS
@@ -309,26 +309,26 @@ jobs:
       consul-license: ${{secrets.CONSUL_LICENSE}}
       datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}"
 
-  go-test-s390x:
-    if: ${{ endsWith(github.repository, '-enterprise') }}
-    needs:
-    - setup
-    - dev-build-s390x
-    uses: ./.github/workflows/reusable-unit.yml
-    with:
-      uploaded-binary-name: 'consul-bin-s390x'
-      directory: .
-      go-test-flags: 'export GO_TEST_FLAGS="-short"'
-      runs-on: ${{ needs.setup.outputs.compute-xl }}
-      repository-name: ${{ github.repository }}
-      go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
-    permissions:
-      id-token: write # NOTE: this permission is explicitly required for Vault auth.
-      contents: read
-    secrets:
-      elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
-      consul-license: ${{secrets.CONSUL_LICENSE}}
-      datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}"
+  # go-test-s390x:
+  #   if: ${{ endsWith(github.repository, '-enterprise') }}
+  #   needs:
+  #   - setup
+  #   - dev-build-s390x
+  #   uses: ./.github/workflows/reusable-unit.yml
+  #   with:
+  #     uploaded-binary-name: 'consul-bin-s390x'
+  #     directory: .
+  #     go-test-flags: 'export GO_TEST_FLAGS="-short"'
+  #     runs-on: ${{ needs.setup.outputs.compute-xl }}
+  #     repository-name: ${{ github.repository }}
+  #     go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
+  #   permissions:
+  #     id-token: write # NOTE: this permission is explicitly required for Vault auth.
+  #     contents: read
+  #   secrets:
+  #     elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
+  #     consul-license: ${{secrets.CONSUL_LICENSE}}
+  #     datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}"
 
   go-test-envoyextensions:
     needs:
@@ -483,7 +483,7 @@ jobs:
     - go-test-sdk-1-19
     - go-test-sdk-1-20
     - go-test-32bit
-    - go-test-s390x
+    # - go-test-s390x
     runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
     if: ${{ always() }}
     steps:

diff --git a/.github/workflows/test-integrations-windows.yml b/.github/workflows/test-integrations-windows.yml
@@ -1197,7 +1197,6 @@ jobs:
   test-integrations-success:
     needs:
       - envoy-integration-test
-      - upgrade-integration-test
     runs-on: 'ubuntu-latest'
     if: ${{ always() }}
     steps:

diff --git a/agent/agent_endpoint.go b/agent/agent_endpoint.go
@@ -619,6 +619,21 @@ func (s *HTTPHandlers) AgentMembers(resp http.ResponseWriter, req *http.Request)
 		}
 	}
 
+	// filter the members by parsed filter expression
+	var filterExpression string
+	s.parseFilter(req, &filterExpression)
+	if filterExpression != "" {
+		filter, err := bexpr.CreateFilter(filterExpression, nil, members)
+		if err != nil {
+			return nil, err
+		}
+		raw, err := filter.Execute(members)
+		if err != nil {
+			return nil, err
+		}
+		members = raw.([]serf.Member)
+	}
+
 	total := len(members)
 	if err := s.agent.filterMembers(token, &members); err != nil {
 		return nil, err

diff --git a/agent/proxycfg/testing.go b/agent/proxycfg/testing.go
@@ -167,7 +167,7 @@ func TestUpstreamNodes(t testing.T, service string) structs.CheckServiceNodes {
 				Datacenter: "dc1",
 				Partition:  structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty(),
 			},
-			Service: structs.TestNodeServiceWithName(t, service),
+			Service: structs.TestNodeServiceWithName(service),
 		},
 		structs.CheckServiceNode{
 			Node: &structs.Node{
@@ -177,7 +177,47 @@ func TestUpstreamNodes(t testing.T, service string) structs.CheckServiceNodes {
 				Datacenter: "dc1",
 				Partition:  structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty(),
 			},
-			Service: structs.TestNodeServiceWithName(t, service),
+			Service: structs.TestNodeServiceWithName(service),
+		},
+	}
+}
+
+// TestUpstreamNodesWithServiceSubset returns a sample service discovery result with one instance tagged v1
+// and the other tagged v2
+func TestUpstreamNodesWithServiceSubset(t testing.T, service string) structs.CheckServiceNodes {
+	return structs.CheckServiceNodes{
+		structs.CheckServiceNode{
+			Node: &structs.Node{
+				ID:         "test1",
+				Node:       "test1",
+				Address:    "10.10.1.3",
+				Datacenter: "dc1",
+				Partition:  structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty(),
+			},
+			Service: &structs.NodeService{
+				Kind:    structs.ServiceKindTypical,
+				Service: service,
+				Port:    8080,
+				Meta:    map[string]string{"Version": "1"},
+				Weights: &structs.Weights{
+					Passing: 300, // Check that this gets normalized to 128
+				},
+			},
+		},
+		structs.CheckServiceNode{
+			Node: &structs.Node{
+				ID:         "test2",
+				Node:       "test2",
+				Address:    "10.10.1.4",
+				Datacenter: "dc1",
+				Partition:  structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty(),
+			},
+			Service: &structs.NodeService{
+				Kind:    structs.ServiceKindTypical,
+				Service: service,
+				Port:    8080,
+				Meta:    map[string]string{"Version": "2"},
+			},
 		},
 	}
 }
@@ -231,7 +271,7 @@ func TestUpstreamNodesInStatus(t testing.T, status string) structs.CheckServiceN
 				Address:    "10.10.1.1",
 				Datacenter: "dc1",
 			},
-			Service: structs.TestNodeService(t),
+			Service: structs.TestNodeService(),
 			Checks: structs.HealthChecks{
 				&structs.HealthCheck{
 					Node:        "test1",
@@ -248,7 +288,7 @@ func TestUpstreamNodesInStatus(t testing.T, status string) structs.CheckServiceN
 				Address:    "10.10.1.2",
 				Datacenter: "dc1",
 			},
-			Service: structs.TestNodeService(t),
+			Service: structs.TestNodeService(),
 			Checks: structs.HealthChecks{
 				&structs.HealthCheck{
 					Node:        "test2",
@@ -270,7 +310,7 @@ func TestUpstreamNodesDC2(t testing.T) structs.CheckServiceNodes {
 				Address:    "10.20.1.1",
 				Datacenter: "dc2",
 			},
-			Service: structs.TestNodeService(t),
+			Service: structs.TestNodeService(),
 		},
 		structs.CheckServiceNode{
 			Node: &structs.Node{
@@ -279,7 +319,7 @@ func TestUpstreamNodesDC2(t testing.T) structs.CheckServiceNodes {
 				Address:    "10.20.1.2",
 				Datacenter: "dc2",
 			},
-			Service: structs.TestNodeService(t),
+			Service: structs.TestNodeService(),
 		},
 	}
 }
@@ -293,7 +333,7 @@ func TestUpstreamNodesInStatusDC2(t testing.T, status string) structs.CheckServi
 				Address:    "10.20.1.1",
 				Datacenter: "dc2",
 			},
-			Service: structs.TestNodeService(t),
+			Service: structs.TestNodeService(),
 			Checks: structs.HealthChecks{
 				&structs.HealthCheck{
 					Node:        "test1",
@@ -310,7 +350,7 @@ func TestUpstreamNodesInStatusDC2(t testing.T, status string) structs.CheckServi
 				Address:    "10.20.1.2",
 				Datacenter: "dc2",
 			},
-			Service: structs.TestNodeService(t),
+			Service: structs.TestNodeService(),
 			Checks: structs.HealthChecks{
 				&structs.HealthCheck{
 					Node:        "test2",
@@ -332,7 +372,7 @@ func TestUpstreamNodesAlternate(t testing.T) structs.CheckServiceNodes {
 				Address:    "10.20.1.1",
 				Datacenter: "dc1",
 			},
-			Service: structs.TestNodeService(t),
+			Service: structs.TestNodeService(),
 		},
 		structs.CheckServiceNode{
 			Node: &structs.Node{
@@ -341,7 +381,7 @@ func TestUpstreamNodesAlternate(t testing.T) structs.CheckServiceNodes {
 				Address:    "10.20.1.2",
 				Datacenter: "dc1",
 			},
-			Service: structs.TestNodeService(t),
+			Service: structs.TestNodeService(),
 		},
 	}
 }

diff --git a/agent/proxycfg/testing_mesh_gateway.go b/agent/proxycfg/testing_mesh_gateway.go
@@ -766,8 +766,12 @@ func TestConfigSnapshotPeeredMeshGateway(t testing.T, variant string, nsFn func(
 				Kind: structs.ServiceResolver,
 				Name: "api",
 				Subsets: map[string]structs.ServiceResolverSubset{
+					"v1": {
+						Filter: "Service.Meta.Version == 1",
+					},
 					"v2": {
-						Filter: "Service.Meta.version == v2",
+						Filter:      "Service.Meta.Version == 2",
+						OnlyPassing: true,
 					},
 				},
 			},
@@ -817,6 +821,7 @@ func TestConfigSnapshotPeeredMeshGateway(t testing.T, variant string, nsFn func(
 		var (
 			dbSN  = structs.NewServiceName("db", nil)
 			altSN = structs.NewServiceName("alt", nil)
+			apiSN = structs.NewServiceName("api", nil)
 
 			dbChain = discoverychain.TestCompileConfigEntries(t, "db", "default", "default", "dc1", connect.TestClusterID+".consul", nil, set)
 		)
@@ -826,6 +831,7 @@ func TestConfigSnapshotPeeredMeshGateway(t testing.T, variant string, nsFn func(
 		discoChains[dbSN] = dbChain
 		endpoints[dbSN] = TestUpstreamNodes(t, "db")
 		endpoints[altSN] = TestUpstreamNodes(t, "alt")
+		endpoints[apiSN] = TestUpstreamNodesWithServiceSubset(t, "api")
 
 		extraUpdates = append(extraUpdates,
 			UpdateEvent{
@@ -849,7 +855,29 @@ func TestConfigSnapshotPeeredMeshGateway(t testing.T, variant string, nsFn func(
 					},
 				},
 			},
+			UpdateEvent{
+				CorrelationID: serviceResolversWatchID,
+				Result: &structs.IndexedConfigEntries{
+					Kind: structs.ServiceResolver,
+					Entries: []structs.ConfigEntry{
+						&structs.ServiceResolverConfigEntry{
+							Kind: structs.ServiceResolver,
+							Name: "api",
+							Subsets: map[string]structs.ServiceResolverSubset{
+								"v1": {
+									Filter: "Service.Meta.Version == 1",
+								},
+								"v2": {
+									Filter:      "Service.Meta.Version == 2",
+									OnlyPassing: true,
+								},
+							},
+						},
+					},
+				},
+			},
 		)
+
 	case "peer-through-mesh-gateway":
 
 		extraUpdates = append(extraUpdates,

diff --git a/agent/structs/structs_test.go b/agent/structs/structs_test.go
@@ -1358,7 +1358,7 @@ func TestStructs_NodeService_ValidateSidecarService(t *testing.T) {
 }
 
 func TestStructs_NodeService_ConnectNativeEmptyPortError(t *testing.T) {
-	ns := TestNodeService(t)
+	ns := TestNodeService()
 	ns.Connect.Native = true
 	ns.Port = 0
 	err := ns.Validate()