Merge branch 'main' into docs/ce-514-envoy-constraints

.changelog/18719.txt

@@ -0,0 +1,7 @@
+```release-note:feature
+acl: Add BindRule support for templated policies. Add new BindType: templated-policy and BindVar field for templated policy variables.
+```
+
+```release-note:feature
+cli: Add `bind-var` flag to `consul acl binding-rule` for templated policy variables.
+```

.github/scripts/filter_changed_files_go_test.sh

@@ -1,4 +1,7 @@
 #!/bin/bash
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
 
 # Get the list of changed files
 files_to_check=$(git diff --name-only origin/$GITHUB_BASE_REF)

.github/scripts/get_runner_classes_windows.sh

@@ -10,11 +10,11 @@ set -euo pipefail
 case "$GITHUB_REPOSITORY" in
 *-enterprise)
 	# shellcheck disable=SC2129
-	echo "compute-small=['self-hosted', 'ondemand', 'os=windows-2019']" >>"$GITHUB_OUTPUT"
-	echo "compute-medium=['self-hosted', 'ondemand', 'os=windows-2019']" >>"$GITHUB_OUTPUT"
-	echo "compute-large=['self-hosted', 'ondemand', 'os=windows-2019']" >>"$GITHUB_OUTPUT"
+	echo "compute-small=['self-hosted', 'ondemand', 'os=windows-2019', 'type=m6a.4xlarge']" >>"$GITHUB_OUTPUT"
+	echo "compute-medium=['self-hosted', 'ondemand', 'os=windows-2019', 'type=m6a.8xlarge']" >>"$GITHUB_OUTPUT"
+	echo "compute-large=['self-hosted', 'ondemand', 'os=windows-2019', 'type=m6a.12xlarge']" >>"$GITHUB_OUTPUT"
 	# m5d.8xlarge is equivalent to our xl custom runner in CE
-	echo "compute-xl=['self-hosted', 'ondemand', 'os=windows-2019', 'type=m6a.4xlarge']" >>"$GITHUB_OUTPUT"
+	echo "compute-xl=['self-hosted', 'ondemand', 'os=windows-2019', 'type=m6a.16xlarge']" >>"$GITHUB_OUTPUT"
 	;;
 *)
 	# shellcheck disable=SC2129

.github/workflows/go-tests.yml

@@ -96,29 +96,28 @@ jobs:
     - name: Notify Slack
       if: ${{ failure() }}
       run: .github/scripts/notify_slack.sh
-  # Temporarily changing until the situation with license headers in the generated code can be worked out
-  # check-generated-deep-copy:
-  #   needs:
-  #   - setup
-  #   runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }}
-  #   steps:
-  #   - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
-  #   # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos.
-  #   - name: Setup Git
-  #     if: ${{ endsWith(github.repository, '-enterprise') }}
-  #     run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com"
-  #   - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
-  #     with:
-  #       go-version-file: 'go.mod'
-  #   - run: make --always-make deep-copy
-  #   - run: |
-  #       if ! git diff --exit-code; then
-  #         echo "Generated code was not updated correctly"
-  #         exit 1
-  #       fi
-  #   - name: Notify Slack
-  #     if: ${{ failure() }}
-  #     run: .github/scripts/notify_slack.sh
+  check-codegen:
+     needs:
+     - setup
+     runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }}
+     steps:
+     - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+     # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos.
+     - name: Setup Git
+       if: ${{ endsWith(github.repository, '-enterprise') }}
+       run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com"
+     - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
+       with:
+         go-version-file: 'go.mod'
+     - run: make --always-make codegen
+     - run: |
+         if ! git diff --exit-code; then
+           echo "Generated code was not updated correctly"
+           exit 1
+         fi
+     - name: Notify Slack
+       if: ${{ failure() }}
+       run: .github/scripts/notify_slack.sh
 
   lint-enums:
     needs:
@@ -486,8 +485,7 @@ jobs:
     needs:
     - conditional-skip
     - setup
-    # Reenable later
-    #- check-generated-deep-copy
+    - check-codegen
     - check-generated-protobuf
     - check-go-mod
     - lint-consul-retry

Makefile

@@ -19,6 +19,7 @@ MOG_VERSION='v0.4.0'
 PROTOC_GO_INJECT_TAG_VERSION='v1.3.0'
 PROTOC_GEN_GO_BINARY_VERSION="v0.1.0"
 DEEP_COPY_VERSION='bc3f5aa5735d8a54961580a3a24422c308c831c2'
+COPYWRITE_TOOL_VERSION='v0.16.4'
 
 MOCKED_PB_DIRS= pbdns
 
@@ -427,12 +428,13 @@ lint-tools: ## Install tools for linting
 codegen-tools: ## Install tools for codegen
 	@$(SHELL) $(CURDIR)/build-support/scripts/devtools.sh -codegen
 
-.PHONY: deep-copy
-deep-copy: codegen-tools ## Deep copy
+.PHONY: codegen
+codegen: codegen-tools ## Deep copy
 	@$(SHELL) $(CURDIR)/agent/structs/deep-copy.sh
 	@$(SHELL) $(CURDIR)/agent/proxycfg/deep-copy.sh
 	@$(SHELL) $(CURDIR)/agent/consul/state/deep-copy.sh
 	@$(SHELL) $(CURDIR)/agent/config/deep-copy.sh
+	copywrite headers
 
 print-%  : ; @echo $($*) ## utility to echo a makefile variable (i.e. 'make print-GOPATH')
 

agent/acl_endpoint.go

@@ -1133,3 +1133,150 @@ func (s *HTTPHandlers) ACLAuthorize(resp http.ResponseWriter, req *http.Request)
 
 	return responses, nil
 }
+
+type ACLTemplatedPolicyResponse struct {
+	TemplateName string
+	Schema       string
+	Template     string
+}
+
+func (s *HTTPHandlers) ACLTemplatedPoliciesList(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
+	if s.checkACLDisabled() {
+		return nil, aclDisabled
+	}
+
+	var token string
+	s.parseToken(req, &token)
+
+	var entMeta acl.EnterpriseMeta
+	if err := s.parseEntMetaNoWildcard(req, &entMeta); err != nil {
+		return nil, err
+	}
+
+	s.defaultMetaPartitionToAgent(&entMeta)
+	var authzContext acl.AuthorizerContext
+	authz, err := s.agent.delegate.ResolveTokenAndDefaultMeta(token, &entMeta, &authzContext)
+	if err != nil {
+		return nil, err
+	}
+
+	// Only ACLRead privileges are required to list templated policies
+	if err := authz.ToAllowAuthorizer().ACLReadAllowed(&authzContext); err != nil {
+		return nil, err
+	}
+
+	templatedPolicies := make(map[string]ACLTemplatedPolicyResponse)
+
+	for tp, tmpBase := range structs.GetACLTemplatedPolicyList() {
+		templatedPolicies[tp] = ACLTemplatedPolicyResponse{
+			TemplateName: tmpBase.TemplateName,
+			Schema:       tmpBase.Schema,
+			Template:     tmpBase.Template,
+		}
+	}
+
+	return templatedPolicies, nil
+}
+
+func (s *HTTPHandlers) ACLTemplatedPolicyRead(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
+	if s.checkACLDisabled() {
+		return nil, aclDisabled
+	}
+
+	templateName := strings.TrimPrefix(req.URL.Path, "/v1/acl/templated-policy/name/")
+	if templateName == "" {
+		return nil, HTTPError{StatusCode: http.StatusBadRequest, Reason: "Missing templated policy Name"}
+	}
+
+	var token string
+	s.parseToken(req, &token)
+
+	var entMeta acl.EnterpriseMeta
+	if err := s.parseEntMetaNoWildcard(req, &entMeta); err != nil {
+		return nil, err
+	}
+
+	s.defaultMetaPartitionToAgent(&entMeta)
+	var authzContext acl.AuthorizerContext
+	authz, err := s.agent.delegate.ResolveTokenAndDefaultMeta(token, &entMeta, &authzContext)
+	if err != nil {
+		return nil, err
+	}
+
+	// Only ACLRead privileges are required to read templated policies
+	if err := authz.ToAllowAuthorizer().ACLReadAllowed(&authzContext); err != nil {
+		return nil, err
+	}
+
+	baseTemplate, ok := structs.GetACLTemplatedPolicyBase(templateName)
+	if !ok {
+		return nil, HTTPError{StatusCode: http.StatusBadRequest, Reason: fmt.Sprintf("Invalid templated policy Name: %s", templateName)}
+	}
+
+	return ACLTemplatedPolicyResponse{
+		TemplateName: baseTemplate.TemplateName,
+		Schema:       baseTemplate.Schema,
+		Template:     baseTemplate.Template,
+	}, nil
+}
+
+func (s *HTTPHandlers) ACLTemplatedPolicyPreview(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
+	if s.checkACLDisabled() {
+		return nil, aclDisabled
+	}
+
+	templateName := strings.TrimPrefix(req.URL.Path, "/v1/acl/templated-policy/preview/")
+	if templateName == "" {
+		return nil, HTTPError{StatusCode: http.StatusBadRequest, Reason: "Missing templated policy Name"}
+	}
+
+	var token string
+	s.parseToken(req, &token)
+
+	var entMeta acl.EnterpriseMeta
+	if err := s.parseEntMetaNoWildcard(req, &entMeta); err != nil {
+		return nil, err
+	}
+
+	s.defaultMetaPartitionToAgent(&entMeta)
+	var authzContext acl.AuthorizerContext
+	authz, err := s.agent.delegate.ResolveTokenAndDefaultMeta(token, &entMeta, &authzContext)
+	if err != nil {
+		return nil, err
+	}
+
+	// Only ACLRead privileges are required to read/preview templated policies
+	if err := authz.ToAllowAuthorizer().ACLReadAllowed(&authzContext); err != nil {
+		return nil, err
+	}
+
+	baseTemplate, ok := structs.GetACLTemplatedPolicyBase(templateName)
+	if !ok {
+		return nil, HTTPError{StatusCode: http.StatusBadRequest, Reason: fmt.Sprintf("templated policy %q does not exist", templateName)}
+	}
+
+	var tpRequest structs.ACLTemplatedPolicyVariables
+
+	if err := decodeBody(req.Body, &tpRequest); err != nil {
+		return nil, HTTPError{StatusCode: http.StatusBadRequest, Reason: fmt.Sprintf("Failed to decode request body: %s", err.Error())}
+	}
+
+	templatedPolicy := structs.ACLTemplatedPolicy{
+		TemplateID:        baseTemplate.TemplateID,
+		TemplateName:      baseTemplate.TemplateName,
+		TemplateVariables: &tpRequest,
+	}
+
+	err = templatedPolicy.ValidateTemplatedPolicy(baseTemplate.Schema)
+	if err != nil {
+		return nil, HTTPError{StatusCode: http.StatusBadRequest, Reason: fmt.Sprintf("validation error for templated policy: %q: %s", templatedPolicy.TemplateName, err.Error())}
+	}
+
+	renderedPolicy, err := templatedPolicy.SyntheticPolicy(&entMeta)
+
+	if err != nil {
+		return nil, HTTPError{StatusCode: http.StatusInternalServerError, Reason: fmt.Sprintf("Failed to generate synthetic policy: %q: %s", templatedPolicy.TemplateName, err.Error())}
+	}
+
+	return renderedPolicy, nil
+}

agent/acl_endpoint_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/hashicorp/consul/acl"
 	"github.com/hashicorp/consul/agent/consul/authmethod/testauth"
 	"github.com/hashicorp/consul/agent/structs"
+	"github.com/hashicorp/consul/api"
 	"github.com/hashicorp/consul/internal/go-sso/oidcauth/oidcauthtest"
 	"github.com/hashicorp/consul/sdk/testutil"
 	"github.com/hashicorp/consul/testrpc"
@@ -1361,6 +1362,88 @@ func TestACL_HTTP(t *testing.T) {
 			require.Equal(t, "sn1", token.ServiceIdentities[0].ServiceName)
 		})
 	})
+
+	t.Run("ACLTemplatedPolicy", func(t *testing.T) {
+		t.Run("List", func(t *testing.T) {
+			req, _ := http.NewRequest("GET", "/v1/acl/templated-policies", nil)
+			req.Header.Add("X-Consul-Token", "root")
+			resp := httptest.NewRecorder()
+			a.srv.h.ServeHTTP(resp, req)
+
+			require.Equal(t, http.StatusOK, resp.Code)
+
+			var list map[string]ACLTemplatedPolicyResponse
+			require.NoError(t, json.NewDecoder(resp.Body).Decode(&list))
+			require.Len(t, list, 3)
+
+			require.Equal(t, ACLTemplatedPolicyResponse{
+				TemplateName: api.ACLTemplatedPolicyServiceName,
+				Schema:       structs.ACLTemplatedPolicyIdentitiesSchema,
+				Template:     structs.ACLTemplatedPolicyService,
+			}, list[api.ACLTemplatedPolicyServiceName])
+		})
+		t.Run("Read", func(t *testing.T) {
+			t.Run("With non existing templated policy", func(t *testing.T) {
+				req, _ := http.NewRequest("GET", "/v1/acl/templated-policy/name/fake", nil)
+				req.Header.Add("X-Consul-Token", "root")
+				resp := httptest.NewRecorder()
+				a.srv.h.ServeHTTP(resp, req)
+				require.Equal(t, http.StatusBadRequest, resp.Code)
+			})
+
+			t.Run("With existing templated policy", func(t *testing.T) {
+				req, _ := http.NewRequest("GET", "/v1/acl/templated-policy/name/"+api.ACLTemplatedPolicyDNSName, nil)
+				req.Header.Add("X-Consul-Token", "root")
+				resp := httptest.NewRecorder()
+
+				a.srv.h.ServeHTTP(resp, req)
+				require.Equal(t, http.StatusOK, resp.Code)
+
+				var templatedPolicy ACLTemplatedPolicyResponse
+				require.NoError(t, json.NewDecoder(resp.Body).Decode(&templatedPolicy))
+				require.Equal(t, structs.ACLTemplatedPolicyDNSSchema, templatedPolicy.Schema)
+				require.Equal(t, api.ACLTemplatedPolicyDNSName, templatedPolicy.TemplateName)
+				require.Equal(t, structs.ACLTemplatedPolicyDNS, templatedPolicy.Template)
+			})
+		})
+		t.Run("preview", func(t *testing.T) {
+			t.Run("When missing required variables", func(t *testing.T) {
+				previewInput := &structs.ACLTemplatedPolicyVariables{}
+				req, _ := http.NewRequest(
+					"POST",
+					fmt.Sprintf("/v1/acl/templated-policy/preview/%s", api.ACLTemplatedPolicyServiceName),
+					jsonBody(previewInput),
+				)
+				req.Header.Add("X-Consul-Token", "root")
+				resp := httptest.NewRecorder()
+
+				a.srv.h.ServeHTTP(resp, req)
+				require.Equal(t, http.StatusBadRequest, resp.Code)
+			})
+
+			t.Run("Correct input", func(t *testing.T) {
+				previewInput := &structs.ACLTemplatedPolicyVariables{Name: "web"}
+				req, _ := http.NewRequest(
+					"POST",
+					fmt.Sprintf("/v1/acl/templated-policy/preview/%s", api.ACLTemplatedPolicyServiceName),
+					jsonBody(previewInput),
+				)
+				req.Header.Add("X-Consul-Token", "root")
+				resp := httptest.NewRecorder()
+
+				a.srv.h.ServeHTTP(resp, req)
+				require.Equal(t, http.StatusOK, resp.Code)
+
+				var syntheticPolicy *structs.ACLPolicy
+				require.NoError(t, json.NewDecoder(resp.Body).Decode(&syntheticPolicy))
+
+				require.NotEmpty(t, syntheticPolicy.ID)
+				require.NotEmpty(t, syntheticPolicy.Hash)
+				require.Equal(t, "synthetic policy generated from templated policy: builtin/service", syntheticPolicy.Description)
+				require.Contains(t, syntheticPolicy.Name, "synthetic-policy-")
+			})
+		})
+	})
 }
 
 func TestACL_LoginProcedure_HTTP(t *testing.T) {