Merge pull request #1694 from alanjcastonguay/aks-credentials-for-acr

AKS credentials for ACR

chart/flux/README.md

@@ -223,6 +223,8 @@ The following tables lists the configurable parameters of the Weave Flux chart a
 | `registry.ecr.region` | Restrict ECR scanning to these AWS regions; if empty, only the cluster's region will be scanned | `None`
 | `registry.ecr.includeId` | Restrict ECR scanning to these AWS account IDs; if empty, all account IDs that aren't excluded may be scanned | `None`
 | `registry.ecr.excludeId` | Do not scan ECR for images in these AWS account IDs; the default is to exclude the EKS system account | `602401143452`
+| `registry.acr.enabled` | Mount `azure.json` via HostPath into the Flux Pod, enabling Flux to use AKS's service principal for ACR authentication | `false`
+| `registry.acr.hostPath` | Alternative location of `azure.json` on the host | `/etc/kubernetes/azure.json`
 | `memcached.verbose` | Enable request logging in memcached | `false`
 | `memcached.maxItemSize` | Maximum size for one item | `1m`
 | `memcached.maxMemory` | Maximum memory to use, in megabytes | `64`

chart/flux/templates/deployment.yaml

@@ -49,6 +49,12 @@ spec:
       - name: git-keygen
         emptyDir:
           medium: Memory
+      {{- if .Values.registry.acr.enabled }}
+      - name: acr-credentials
+        hostPath:
+          path: "{{ .Values.registry.acr.hostPath }}"
+          type: ""
+      {{- end }}
       containers:
         - name: {{ .Chart.Name }}
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
@@ -70,6 +76,11 @@ spec:
             readOnly: true
           - name: git-keygen
             mountPath: /var/fluxd/keygen
+          {{- if .Values.registry.acr.enabled }}
+          - name: acr-credentials
+            mountPath: /etc/kubernetes/azure.json
+            readOnly: true
+          {{- end }}
           env:
           - name: KUBECONFIG
             value: /root/.kubectl/config

chart/flux/values.yaml

@@ -140,6 +140,10 @@ registry:
     region:
     includeId:
     excludeId:
+  # Azure ACR settings
+  acr:
+    enabled: false
+    hostPath: /etc/kubernetes/azure.json
 
 memcached:
   repository: memcached

registry/azure.go

@@ -0,0 +1,53 @@
+package registry
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"strings"
+)
+
+const (
+	// Mount volume from hostpath.
+	azureCloudConfigJsonFile = "/etc/kubernetes/azure.json"
+)
+
+type azureCloudConfig struct {
+	AADClientId     string `json:"aadClientId"`
+	AADClientSecret string `json:"aadClientSecret"`
+}
+
+// Fetch Azure Active Directory clientid/secret pair from azure.json, usable for container registry authentication.
+//
+// Note: azure.json is populated by AKS/AKS-Engine script kubernetesconfigs.sh. The file is then passed to kubelet via
+// --azure-container-registry-config=/etc/kubernetes/azure.json, parsed by kubernetes/kubernetes' azure_credentials.go
+// https://github.com/kubernetes/kubernetes/issues/58034 seeks to deprecate this kubelet command-line argument, possibly
+// replacing it with managed identity for the Node VMs. See https://github.com/Azure/acr/blob/master/docs/AAD-OAuth.md
+func getAzureCloudConfigAADToken(host string) (creds, error) {
+	jsonFile, err := ioutil.ReadFile(azureCloudConfigJsonFile)
+	if err != nil {
+		return creds{}, err
+	}
+
+	var token azureCloudConfig
+
+	err = json.Unmarshal(jsonFile, &token)
+	if err != nil {
+		return creds{}, err
+	}
+
+	return creds{
+		registry:   host,
+		provenance: "azure.json",
+		username:   token.AADClientId,
+		password:   token.AADClientSecret}, nil
+}
+
+// List from https://github.com/kubernetes/kubernetes/blob/master/pkg/credentialprovider/azure/azure_credentials.go
+func hostIsAzureContainerRegistry(host string) bool {
+	for _, v := range []string{".azurecr.io", ".azurecr.cn", ".azurecr.de", ".azurecr.us"} {
+		if strings.HasSuffix(host, v) {
+			return true
+		}
+	}
+	return false
+}

registry/azure_test.go

@@ -0,0 +1,56 @@
+package registry
+
+import (
+	"testing"
+)
+
+func Test_HostIsAzureContainerRegistry(t *testing.T) {
+	for _, v := range []struct {
+		host  string
+		isACR bool
+	}{
+		{
+			host:  "azurecr.io",
+			isACR: false,
+		},
+		{
+			host:  "",
+			isACR: false,
+		},
+		{
+			host:  "gcr.io",
+			isACR: false,
+		},
+		{
+			host:  "notazurecr.io",
+			isACR: false,
+		},
+		{
+			host:  "example.azurecr.io.not",
+			isACR: false,
+		},
+		// Public cloud
+		{
+			host:  "example.azurecr.io",
+			isACR: true,
+		},
+		// Sovereign clouds
+		{
+			host:  "example.azurecr.cn",
+			isACR: true,
+		},
+		{
+			host:  "example.azurecr.de",
+			isACR: true,
+		},
+		{
+			host:  "example.azurecr.us",
+			isACR: true,
+		},
+	} {
+		result := hostIsAzureContainerRegistry(v.host)
+		if result != v.isACR {
+			t.Fatalf("For test %q, expected isACR = %v but got %v", v.host, v.isACR, result)
+		}
+	}
+}

registry/credentials.go

@@ -144,6 +144,13 @@ func (cs Credentials) credsFor(host string) creds {
 			return cred
 		}
 	}
+
+    if hostIsAzureContainerRegistry(host) {
+        if cred, err := getAzureCloudConfigAADToken(host); err == nil {
+            return cred
+        }
+    }
+
 	return creds{}
 }
 

site/faq.md

@@ -247,7 +247,7 @@ happen:
    if you've only just started using a particular image in a workload.
  - Flux can't get suitable credentials for the image repository. At
    present, it looks at `imagePullSecret`s attached to workloads,
-   service accounts, platform-provided credentials on GCP or AWS, and
+   service accounts, platform-provided credentials on GCP, AWS or Azure, and
    a Docker config file if you mount one into the fluxd container (see
    the [command-line usage](./daemon.md)).
  - When using images in ECR, from EC2, the `NodeInstanceRole` for the
@@ -257,6 +257,25 @@ happen:
    [`kops`](https://github.com/kubernetes/kops) (with
    [`.iam.allowContainerRegistry=true`](https://github.com/kubernetes/kops/blob/master/docs/iam_roles.md#iam-roles))
    both make sure this is the case.
+ - When using images from ACR in AKS, the HostPath `/etc/kubernetes/azure.json`
+   should be [mounted](https://kubernetes.io/docs/concepts/storage/volumes/) into the Flux Pod.
+   Set `registry.acr.enabled=True` in the [helm chart](../chart/flux/README.md)
+   or alter the [Deployment](../deploy/flux-deployment.yaml):
+   ```yaml
+    spec:
+      containers:
+        image: quay.io/weaveworks/flux
+        ...
+        volumeMounts:
+        - name: acr-credentials
+          mountPath: /etc/kubernetes/azure.json
+          readOnly: true
+      volumes:
+      - name: acr-credentials
+        hostPath:
+          path: /etc/kubernetes/azure.json
+          type: ""
+   ```
  - Flux excludes images with no suitable manifest (linux amd64) in manifestlist
  - Flux doesn't yet understand image refs that use digests instead of
    tags; see