From 41b7b6c8b6bee0faf2eb64f3959237a83fe79d50 Mon Sep 17 00:00:00 2001
From: Hidekazu Nakamura <hidekazuna@gmail.com>
Date: Tue, 5 Feb 2019 05:44:03 +0000
Subject: [PATCH] Support keystone with cacert

---
 .../examples/openstack/generate-yaml.sh       |  9 ++++-
 .../clouds-secrets/kustomization.yaml         |  4 +++
 .../ubuntu/templates/master-user-data.sh      |  8 +++++
 .../ubuntu/templates/worker-user-data.sh      |  2 ++
 .../ubuntu/provider-components.yaml.template  | 36 +++++++++++++++++++
 config/manager/manager.yaml                   |  5 +++
 pkg/cloud/openstack/clients/machineservice.go | 24 ++++++++++++-
 7 files changed, 86 insertions(+), 2 deletions(-)
 create mode 100644 cmd/clusterctl/examples/openstack/ubuntu/provider-components.yaml.template

diff --git a/cmd/clusterctl/examples/openstack/generate-yaml.sh b/cmd/clusterctl/examples/openstack/generate-yaml.sh
index 4a4e6c810c..c7e154193c 100755
--- a/cmd/clusterctl/examples/openstack/generate-yaml.sh
+++ b/cmd/clusterctl/examples/openstack/generate-yaml.sh
@@ -120,6 +120,7 @@ OPENSTACK_CLOUD_CONFIG_PLAIN=$(cat "$CLOUDS_PATH")
 
 MACHINE_CONTROLLER_SSH_PRIVATE_FILE=openstack_tmp
 MACHINE_CONTROLLER_SSH_HOME=${HOME}/.ssh/
+CACERT="/etc/certs/cacert"
 
 # Set up the output dir if it does not yet exist
 mkdir -p out
@@ -143,7 +144,7 @@ PASSWORD=$(echo "$OPENSTACK_CLOUD_CONFIG_PLAIN" | yq r - clouds.$CLOUD.auth.pass
 REGION=$(echo "$OPENSTACK_CLOUD_CONFIG_PLAIN" | yq r - clouds.$CLOUD.region_name)
 PROJECT_ID=$(echo "$OPENSTACK_CLOUD_CONFIG_PLAIN" | yq r - clouds.$CLOUD.auth.project_id)
 DOMAIN_NAME=$(echo "$OPENSTACK_CLOUD_CONFIG_PLAIN" | yq r - clouds.$CLOUD.auth.user_domain_name)
-
+CACERT_ORIGINAL=$(echo "$OPENSTACK_CLOUD_CONFIG_PLAIN" | yq r - clouds.$CLOUD.cacert)
 
 # Basic cloud.conf, no LB configuration as that data is not known yet.
 OPENSTACK_CLOUD_PROVIDER_CONF_PLAIN="[Global]
@@ -153,13 +154,16 @@ password=\"$PASSWORD\"
 region=\"$REGION\"
 tenant-id=\"$PROJECT_ID\"
 domain-name=\"$DOMAIN_NAME\"
+ca-file=\"$CACERT\"
 "
 
 OS=$(uname)
 if [[ "$OS" =~ "Linux" ]]; then
   OPENSTACK_CLOUD_PROVIDER_CONF=$(echo "$OPENSTACK_CLOUD_PROVIDER_CONF_PLAIN"|base64 -w0)
+  OPENSTACK_CLOUD_CACERT_CONFIG=$(cat "$CACERT_ORIGINAL"|base64 -w0)
 elif [[ "$OS" =~ "Darwin" ]]; then
   OPENSTACK_CLOUD_PROVIDER_CONF=$(echo "$OPENSTACK_CLOUD_PROVIDER_CONF_PLAIN"|base64)
+  OPENSTACK_CLOUD_CACERT_CONFIG=$(cat "$CACERT_ORIGINAL"|base64)
 else
   echo "Unrecognized OS : $OS"
   exit 1
@@ -167,13 +171,16 @@ fi
 
 cat "$MASTER_USER_DATA" \
     | sed -e "s#\$OPENSTACK_CLOUD_PROVIDER_CONF#$OPENSTACK_CLOUD_PROVIDER_CONF#" \
+    | sed -e "s#\$OPENSTACK_CLOUD_CACERT_CONFIG#$OPENSTACK_CLOUD_CACERT_CONFIG#" \
     > $USERDATA/$PROVIDER_OS/master-user-data.sh
 cat "$WORKER_USER_DATA" \
   | sed -e "s#\$OPENSTACK_CLOUD_PROVIDER_CONF#$OPENSTACK_CLOUD_PROVIDER_CONF#" \
+  | sed -e "s#\$OPENSTACK_CLOUD_CACERT_CONFIG#$OPENSTACK_CLOUD_CACERT_CONFIG#" \
   > $USERDATA/$PROVIDER_OS/worker-user-data.sh
 
 printf $CLOUD > $CONFIG_DIR/os_cloud.txt
 echo "$OPENSTACK_CLOUD_CONFIG_PLAIN" > $CONFIG_DIR/clouds.yaml
+cat "$CACERT_ORIGINAL" > $CONFIG_DIR/cacert
 
 # Build provider-components.yaml with kustomize
 kustomize build ../../../../config -o out/provider-components.yaml
diff --git a/cmd/clusterctl/examples/openstack/provider-component/clouds-secrets/kustomization.yaml b/cmd/clusterctl/examples/openstack/provider-component/clouds-secrets/kustomization.yaml
index b827b58782..5b280d4e54 100644
--- a/cmd/clusterctl/examples/openstack/provider-component/clouds-secrets/kustomization.yaml
+++ b/cmd/clusterctl/examples/openstack/provider-component/clouds-secrets/kustomization.yaml
@@ -12,5 +12,9 @@ secretGenerator:
   commands:
     OS_CLOUD: "cat configs/os_cloud.txt"
   type: Opaque
+- name: cloud-cacert-config
+  commands:
+    cacert: "cat configs/cacert"
+  type: Opaque
 
 namespace: openstack-provider-system
diff --git a/cmd/clusterctl/examples/openstack/provider-component/user-data/ubuntu/templates/master-user-data.sh b/cmd/clusterctl/examples/openstack/provider-component/user-data/ubuntu/templates/master-user-data.sh
index 3feaf8e11a..193576cc6f 100644
--- a/cmd/clusterctl/examples/openstack/provider-component/user-data/ubuntu/templates/master-user-data.sh
+++ b/cmd/clusterctl/examples/openstack/provider-component/user-data/ubuntu/templates/master-user-data.sh
@@ -103,6 +103,8 @@ Environment="KUBELET_DNS_ARGS=--cluster-dns=${CLUSTER_DNS_SERVER} --cluster-doma
 EOF
 
 echo $OPENSTACK_CLOUD_PROVIDER_CONF | base64 -d > /etc/kubernetes/cloud.conf
+mkdir /etc/certs
+echo $OPENSTACK_CLOUD_CACERT_CONFIG | base64 -d > /etc/certs/cacert
 
 systemctl daemon-reload
 systemctl restart kubelet.service
@@ -138,6 +140,9 @@ apiServerExtraVolumes:
 - name: cloud
   hostPath: "/etc/kubernetes/cloud.conf"
   mountPath: "/etc/kubernetes/cloud.conf"
+- name: cacert
+  hostPath: "/etc/certs/cacert"
+  mountPath: "/etc/certs/cacert"
 controllerManagerExtraArgs:
   cluster-cidr: ${POD_CIDR}
   service-cluster-ip-range: ${SERVICE_CIDR}
@@ -148,6 +153,9 @@ controllerManagerExtraVolumes:
 - name: cloud
   hostPath: "/etc/kubernetes/cloud.conf"
   mountPath: "/etc/kubernetes/cloud.conf"
+- name: cacert
+  hostPath: "/etc/certs/cacert"
+  mountPath: "/etc/certs/cacert"
 EOF
 
 # Create and set bridge-nf-call-iptables to 1 to pass the kubeadm preflight check.
diff --git a/cmd/clusterctl/examples/openstack/provider-component/user-data/ubuntu/templates/worker-user-data.sh b/cmd/clusterctl/examples/openstack/provider-component/user-data/ubuntu/templates/worker-user-data.sh
index f00c37bf3e..a8875aadf3 100644
--- a/cmd/clusterctl/examples/openstack/provider-component/user-data/ubuntu/templates/worker-user-data.sh
+++ b/cmd/clusterctl/examples/openstack/provider-component/user-data/ubuntu/templates/worker-user-data.sh
@@ -76,6 +76,8 @@ CLUSTER_DNS_SERVER=$(prips ${SERVICE_CIDR} | head -n 11 | tail -n 1)
 
 # Write the cloud.conf so that the kubelet can use it.
 echo $OPENSTACK_CLOUD_PROVIDER_CONF | base64 -d > /etc/kubernetes/cloud.conf
+mkdir /etc/certs
+echo $OPENSTACK_CLOUD_CACERT_CONFIG | base64 -d > /etc/certs/cacert
 
 # Set up kubeadm config file to pass to kubeadm join.
 cat > /etc/kubernetes/kubeadm_config.yaml <<EOF
diff --git a/cmd/clusterctl/examples/openstack/ubuntu/provider-components.yaml.template b/cmd/clusterctl/examples/openstack/ubuntu/provider-components.yaml.template
new file mode 100644
index 0000000000..29dfc3ce21
--- /dev/null
+++ b/cmd/clusterctl/examples/openstack/ubuntu/provider-components.yaml.template
@@ -0,0 +1,36 @@
+---
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: cloud-config
+  namespace: openstack-provider-system
+data:
+  clouds.yaml: $OPENSTACK_CLOUD_CONFIG
+---
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: cloud-cacert-config
+  namespace: openstack-provider-system
+data:
+  cacert: $OPENSTACK_CLOUD_CACERT_CONFIG
+---
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: worker-user-data
+  namespace: openstack-provider-system
+data:
+  userData: $WORKER_USER_DATA
+---
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: master-user-data
+  namespace: openstack-provider-system
+data:
+  userData: $MASTER_USER_DATA
diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml
index 24687b340d..e28c1c1fba 100644
--- a/config/manager/manager.yaml
+++ b/config/manager/manager.yaml
@@ -55,6 +55,8 @@ spec:
           mountPath: /etc/kubernetes
         - name: cloud-config
           mountPath: /etc/openstack
+        - name: cloud-cacert-config
+          mountPath: /etc/certs
         - name: kubeadm
           mountPath: /usr/bin/kubeadm
         resources:
@@ -79,6 +81,9 @@ spec:
       - name: cloud-config
         secret:
           secretName: cloud-config
+      - name: cloud-cacert-config
+        secret:
+          secretName: cloud-cacert-config
       - name: kubeadm
         hostPath:
           path: /usr/bin/kubeadm
diff --git a/pkg/cloud/openstack/clients/machineservice.go b/pkg/cloud/openstack/clients/machineservice.go
index ae09893a83..3023ee7a16 100644
--- a/pkg/cloud/openstack/clients/machineservice.go
+++ b/pkg/cloud/openstack/clients/machineservice.go
@@ -21,6 +21,9 @@ import (
 	"fmt"
 	"time"
 
+	"crypto/tls"
+	"net/http"
+
 	"gopkg.in/yaml.v2"
 	"k8s.io/client-go/kubernetes"
 
@@ -40,6 +43,7 @@ import (
 	"github.com/gophercloud/gophercloud/pagination"
 	"github.com/gophercloud/utils/openstack/clientconfig"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	certutil "k8s.io/client-go/util/cert"
 	"k8s.io/klog"
 	openstackconfigv1 "sigs.k8s.io/cluster-api-provider-openstack/pkg/apis/openstackproviderconfig/v1alpha1"
 	clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1"
@@ -48,6 +52,7 @@ import (
 
 const (
 	CloudsSecretKey = "clouds.yaml"
+	caFile          = "/etc/certs/cacert"
 
 	TimeoutTrunkDelete       = 3 * time.Minute
 	RetryIntervalTrunkDelete = 5 * time.Second
@@ -155,7 +160,24 @@ func NewInstanceServiceFromCloud(cloud clientconfig.Cloud) (*InstanceService, er
 
 	opts.AllowReauth = true
 
-	provider, err := openstack.AuthenticatedClient(*opts)
+	provider, err := openstack.NewClient(opts.IdentityEndpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	cloudFromYaml, err := clientconfig.GetCloudFromYAML(clientOpts)
+	if cloudFromYaml.CACertFile != "" {
+		roots, err := certutil.NewPool(caFile)
+		if err != nil {
+			return nil, err
+		}
+		config := &tls.Config{}
+		config.RootCAs = roots
+
+		transport := &http.Transport{TLSClientConfig: config}
+		provider.HTTPClient = http.Client{Transport: transport}
+	}
+	err = openstack.Authenticate(provider, *opts)
 	if err != nil {
 		return nil, fmt.Errorf("Create providerClient err: %v", err)
 	}