Support keystone with certificate

cmd/clusterctl/examples/openstack/generate-yaml.sh

@@ -136,6 +136,7 @@ OPENSTACK_CLOUD_CONFIG_PLAIN=$(cat "$CLOUDS_PATH")
 
 MACHINE_CONTROLLER_SSH_PRIVATE_FILE=openstack_tmp
 MACHINE_CONTROLLER_SSH_HOME=${HOME}/.ssh/
+CACERT="/etc/certs/cacert"
 
 # Set up the output dir if it does not yet exist
 mkdir -p $PWD/$OUTPUT
@@ -159,7 +160,7 @@ PASSWORD=$(echo "$OPENSTACK_CLOUD_CONFIG_PLAIN" | yq r - clouds.$CLOUD.auth.pass
 REGION=$(echo "$OPENSTACK_CLOUD_CONFIG_PLAIN" | yq r - clouds.$CLOUD.region_name)
 PROJECT_ID=$(echo "$OPENSTACK_CLOUD_CONFIG_PLAIN" | yq r - clouds.$CLOUD.auth.project_id)
 DOMAIN_NAME=$(echo "$OPENSTACK_CLOUD_CONFIG_PLAIN" | yq r - clouds.$CLOUD.auth.user_domain_name)
-
+CACERT_ORIGINAL=$(echo "$OPENSTACK_CLOUD_CONFIG_PLAIN" | yq r - clouds.$CLOUD.cacert)
 
 # Basic cloud.conf, no LB configuration as that data is not known yet.
 OPENSTACK_CLOUD_PROVIDER_CONF_PLAIN="[Global]
@@ -171,11 +172,23 @@ tenant-id=\"$PROJECT_ID\"
 domain-name=\"$DOMAIN_NAME\"
 "
 
+if [ "$CACERT_ORIGINAL" != "null" ]; then
+  OPENSTACK_CLOUD_PROVIDER_CONF_PLAIN="$OPENSTACK_CLOUD_PROVIDER_CONF_PLAIN
+  ca-file=\"$CACERT\"
+  "
+fi
+
 OS=$(uname)
 if [[ "$OS" =~ "Linux" ]]; then
   OPENSTACK_CLOUD_PROVIDER_CONF=$(echo "$OPENSTACK_CLOUD_PROVIDER_CONF_PLAIN"|base64 -w0)
+  if [ "$CACERT_ORIGINAL" != "null" ]; then
+    OPENSTACK_CLOUD_CACERT_CONFIG=$(cat "$CACERT_ORIGINAL"|base64 -w0)
+  fi
 elif [[ "$OS" =~ "Darwin" ]]; then
   OPENSTACK_CLOUD_PROVIDER_CONF=$(echo "$OPENSTACK_CLOUD_PROVIDER_CONF_PLAIN"|base64)
+  if [ "$CACERT_ORIGINAL" != "null" ]; then
+    OPENSTACK_CLOUD_CACERT_CONFIG=$(cat "$CACERT_ORIGINAL"|base64)
+  fi
 else
   echo "Unrecognized OS : $OS"
   exit 1
@@ -184,24 +197,32 @@ fi
 if [[ "$PROVIDER_OS" == "coreos" ]]; then
   cat $COREOS_COMMON_SECTION \
     | sed -e "s#\$OPENSTACK_CLOUD_PROVIDER_CONF#$OPENSTACK_CLOUD_PROVIDER_CONF#" \
+    | sed -e "s#\$OPENSTACK_CLOUD_CACERT_CONFIG#$OPENSTACK_CLOUD_CACERT_CONFIG#" \
     | yq m -a - $COREOS_MASTER_SECTION  \
     > $COREOS_MASTER_USER_DATA
   cat $COREOS_COMMON_SECTION \
     | sed -e "s#\$OPENSTACK_CLOUD_PROVIDER_CONF#$OPENSTACK_CLOUD_PROVIDER_CONF#" \
+    | sed -e "s#\$OPENSTACK_CLOUD_CACERT_CONFIG#$OPENSTACK_CLOUD_CACERT_CONFIG#" \
     | yq m -a - $COREOS_WORKER_SECTION  \
     > $COREOS_WORKER_USER_DATA
 else
   cat "$MASTER_USER_DATA" \
       | sed -e "s#\$OPENSTACK_CLOUD_PROVIDER_CONF#$OPENSTACK_CLOUD_PROVIDER_CONF#" \
+      | sed -e "s#\$OPENSTACK_CLOUD_CACERT_CONFIG#$OPENSTACK_CLOUD_CACERT_CONFIG#" \
       > $USERDATA/$PROVIDER_OS/master-user-data.sh
   cat "$WORKER_USER_DATA" \
     | sed -e "s#\$OPENSTACK_CLOUD_PROVIDER_CONF#$OPENSTACK_CLOUD_PROVIDER_CONF#" \
+    | sed -e "s#\$OPENSTACK_CLOUD_CACERT_CONFIG#$OPENSTACK_CLOUD_CACERT_CONFIG#" \
     > $USERDATA/$PROVIDER_OS/worker-user-data.sh
 fi
 
 printf $CLOUD > $CONFIG_DIR/os_cloud.txt
 echo "$OPENSTACK_CLOUD_CONFIG_PLAIN" > $CONFIG_DIR/clouds.yaml
-
+if [ "$CACERT_ORIGINAL" != "null" ]; then
+  cat "$CACERT_ORIGINAL" > $CONFIG_DIR/cacert
+else
+  echo "dummy" > $CONFIG_DIR/cacert
+fi
 
 # Build provider-components.yaml with kustomize
 # Coreos has a different kubeadm path (/usr is read-only) so gets a different kustomization.

cmd/clusterctl/examples/openstack/provider-component/clouds-secrets/kustomization.yaml

@@ -7,6 +7,7 @@ secretGenerator:
 - name: cloud-config
   commands:
     clouds.yaml: "cat configs/clouds.yaml"
+    cacert: "cat configs/cacert"
   type: Opaque
 - name: cloud-selector
   commands:

cmd/clusterctl/examples/openstack/provider-component/user-data/centos/templates/master-user-data.sh

@@ -86,6 +86,8 @@ echo '1' > /proc/sys/net/bridge/bridge-nf-call-iptables
 echo '1' > /proc/sys/net/ipv4/ip_forward
 
 echo $OPENSTACK_CLOUD_PROVIDER_CONF | base64 -d > /etc/kubernetes/cloud.conf
+mkdir /etc/certs
+echo $OPENSTACK_CLOUD_CACERT_CONFIG | base64 -d > /etc/certs/cacert
 
 # Set up kubeadm config file to pass parameters to kubeadm init.
 cat > /etc/kubernetes/kubeadm_config.yaml <<EOF
@@ -122,6 +124,10 @@ apiServer:
     mountPath: /etc/kubernetes/cloud.conf
     name: cloud
     readOnly: true
+  - hostPath: "/etc/certs/cacert"
+    mountPath: "/etc/certs/cacert"
+    name: cacert
+    readOnly: true
   timeoutForControlPlane: 4m0s
 certificatesDir: /etc/kubernetes/pki
 clusterName: kubernetes
@@ -138,6 +144,10 @@ controllerManager:
     mountPath: /etc/kubernetes/cloud.conf
     name: cloud
     readOnly: true
+  - hostPath: "/etc/certs/cacert"
+    mountPath: "/etc/certs/cacert"
+    name: cacert
+    readOnly: true
 dns:
   type: CoreDNS
 etcd:

cmd/clusterctl/examples/openstack/provider-component/user-data/centos/templates/worker-user-data.sh

@@ -45,6 +45,8 @@ install_configure_docker
 
 # Write the cloud.conf so that the kubelet can use it.
 echo $OPENSTACK_CLOUD_PROVIDER_CONF | base64 -d > /etc/kubernetes/cloud.conf
+mkdir /etc/certs
+echo $OPENSTACK_CLOUD_CACERT_CONFIG | base64 -d > /etc/certs/cacert
 
 # Set up kubeadm config file to pass to kubeadm join.
 cat > /etc/kubernetes/kubeadm_config.yaml <<EOF

cmd/clusterctl/examples/openstack/provider-component/user-data/coreos/templates/common.yaml

@@ -120,6 +120,16 @@ storage:
         id: 0
       group:
         id: 0
+    - path: /etc/certs/cacert
+      filesystem: root
+      contents:
+        inline: !!binary |
+          $OPENSTACK_CLOUD_CACERT_CONFIG
+      mode: 0600
+      user:
+        id: 0
+      group:
+        id: 0
     - path: /etc/modules-load.d/ipvs.conf
       filesystem: root
       contents:

cmd/clusterctl/examples/openstack/provider-component/user-data/coreos/templates/master.yaml

@@ -31,6 +31,9 @@ storage:
           - name: cloud
             hostPath: "/etc/kubernetes/cloud.conf"
             mountPath: "/etc/kubernetes/cloud.conf"
+          - name: cacert
+            hostPath: "/etc/certs/cacert"
+            mountPath: "/etc/certs/cacert"
         controllerManager:
           extraArgs:
             cluster-cidr: {{ .PodCIDR }}
@@ -42,6 +45,9 @@ storage:
           - name: cloud
             hostPath: "/etc/kubernetes/cloud.conf"
             mountPath: "/etc/kubernetes/cloud.conf"
+          - name: cacert
+            hostPath: "/etc/certs/cacert"
+            mountPath: "/etc/certs/cacert"
     user:
       id: 0
     group:

cmd/clusterctl/examples/openstack/provider-component/user-data/ubuntu/templates/master-user-data.sh

@@ -107,6 +107,8 @@ EOF
 
 echo $OPENSTACK_CLOUD_PROVIDER_CONF | base64 -d > /etc/kubernetes/cloud.conf
 chmod 600 /etc/kubernetes/cloud.conf
+mkdir /etc/certs
+echo $OPENSTACK_CLOUD_CACERT_CONFIG | base64 -d > /etc/certs/cacert
 
 systemctl daemon-reload
 systemctl restart kubelet.service
@@ -150,6 +152,10 @@ apiServer:
     mountPath: /etc/kubernetes/cloud.conf
     name: cloud
     readOnly: true
+  - hostPath: "/etc/certs/cacert"
+    mountPath: "/etc/certs/cacert"
+    name: cacert
+    readOnly: true
   timeoutForControlPlane: 4m0s
 certificatesDir: /etc/kubernetes/pki
 clusterName: kubernetes
@@ -166,6 +172,10 @@ controllerManager:
     mountPath: /etc/kubernetes/cloud.conf
     name: cloud
     readOnly: true
+  - hostPath: "/etc/certs/cacert"
+    mountPath: "/etc/certs/cacert"
+    name: cacert
+    readOnly: true
 dns:
   type: CoreDNS
 etcd:

cmd/clusterctl/examples/openstack/provider-component/user-data/ubuntu/templates/worker-user-data.sh

@@ -79,6 +79,8 @@ CLUSTER_DNS_SERVER=$(prips ${SERVICE_CIDR} | head -n 11 | tail -n 1)
 
 # Write the cloud.conf so that the kubelet can use it.
 echo $OPENSTACK_CLOUD_PROVIDER_CONF | base64 -d > /etc/kubernetes/cloud.conf
+mkdir /etc/certs
+echo $OPENSTACK_CLOUD_CACERT_CONFIG | base64 -d > /etc/certs/cacert
 
 # Set up kubeadm config file to pass to kubeadm join.
 cat > /etc/kubernetes/kubeadm_config.yaml <<EOF

config/crds/openstackproviderconfig_v1alpha1_openstackclusterproviderspec.yaml

@@ -16,6 +16,10 @@ spec:
       properties:
         apiVersion:
           type: string
+        cloudName:
+          type: string
+        cloudsSecret:
+          type: object
         disableServerTags:
           type: boolean
         dnsNameservers:
@@ -37,6 +41,8 @@ spec:
             type: string
           type: array
       required:
+      - cloudsSecret
+      - cloudName
       - managedSecurityGroups
   version: v1alpha1
 status:

pkg/apis/openstackproviderconfig/v1alpha1/types.go

@@ -187,6 +187,12 @@ type OpenstackClusterProviderSpec struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`
 
+	// The name of the secret containing the openstack credentials
+	CloudsSecret *corev1.SecretReference `json:"cloudsSecret"`
+
+	// The name of the cloud to use from the clouds secret
+	CloudName string `json:"cloudName"`
+
 	// NodeCIDR is the OpenStack Subnet to be created. Cluster actuator will create a
 	// network, a subnet with NodeCIDR, and a router connected to this subnet.
 	// If you leave this empty, no network will be created.

pkg/cloud/openstack/clients/machineservice.go

@@ -17,8 +17,11 @@ limitations under the License.
 package clients
 
 import (
+	"crypto/tls"
+	"crypto/x509"
 	"encoding/base64"
 	"fmt"
+	"net/http"
 	"time"
 
 	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/groups"
@@ -53,6 +56,7 @@ import (
 
 const (
 	CloudsSecretKey = "clouds.yaml"
+	CaSecretKey     = "cacert"
 
 	TimeoutTrunkDelete       = 3 * time.Minute
 	RetryIntervalTrunkDelete = 5 * time.Second
@@ -91,34 +95,40 @@ type InstanceListOpts struct {
 	Name string `q:"name"`
 }
 
-func GetCloudFromSecret(kubeClient kubernetes.Interface, namespace string, secretName string, cloudName string) (clientconfig.Cloud, error) {
+func GetCloudFromSecret(kubeClient kubernetes.Interface, namespace string, secretName string, cloudName string) (clientconfig.Cloud, []byte, error) {
 	emptyCloud := clientconfig.Cloud{}
 
 	if secretName == "" {
-		return emptyCloud, nil
+		return emptyCloud, nil, nil
 	}
 
 	if secretName != "" && cloudName == "" {
-		return emptyCloud, fmt.Errorf("Secret name set to %v but no cloud was specified. Please set cloud_name in your machine spec.", secretName)
+		return emptyCloud, nil, fmt.Errorf("Secret name set to %v but no cloud was specified. Please set cloud_name in your machine spec.", secretName)
 	}
 
 	secret, err := kubeClient.CoreV1().Secrets(namespace).Get(secretName, metav1.GetOptions{})
 	if err != nil {
-		return emptyCloud, err
+		return emptyCloud, nil, err
 	}
 
 	content, ok := secret.Data[CloudsSecretKey]
 	if !ok {
-		return emptyCloud, fmt.Errorf("OpenStack credentials secret %v did not contain key %v",
+		return emptyCloud, nil, fmt.Errorf("OpenStack credentials secret %v did not contain key %v",
 			secretName, CloudsSecretKey)
 	}
 	var clouds clientconfig.Clouds
 	err = yaml.Unmarshal(content, &clouds)
 	if err != nil {
-		return emptyCloud, fmt.Errorf("failed to unmarshal clouds credentials stored in secret %v: %v", secretName, err)
+		return emptyCloud, nil, fmt.Errorf("failed to unmarshal clouds credentials stored in secret %v: %v", secretName, err)
 	}
 
-	return clouds.Clouds[cloudName], nil
+	// get cacert
+	cacert, ok := secret.Data[CaSecretKey]
+	if !ok {
+		return emptyCloud, nil, err
+	}
+
+	return clouds.Clouds[cloudName], cacert, nil
 }
 
 // TODO: Eventually we'll have a NewInstanceServiceFromCluster too
@@ -128,25 +138,28 @@ func NewInstanceServiceFromMachine(kubeClient kubernetes.Interface, machine *clu
 		return nil, err
 	}
 	cloud := clientconfig.Cloud{}
+	var cacert []byte
+
 	if machineSpec.CloudsSecret != nil && machineSpec.CloudsSecret.Name != "" {
 		namespace := machineSpec.CloudsSecret.Namespace
 		if namespace == "" {
 			namespace = machine.Namespace
 		}
-		cloud, err = GetCloudFromSecret(kubeClient, namespace, machineSpec.CloudsSecret.Name, machineSpec.CloudName)
+		cloud, cacert, err = GetCloudFromSecret(kubeClient, namespace, machineSpec.CloudsSecret.Name, machineSpec.CloudName)
 		if err != nil {
 			return nil, err
 		}
 	}
-	return NewInstanceServiceFromCloud(cloud)
+	return NewInstanceServiceFromCloud(cloud, cacert)
 }
 
 func NewInstanceService() (*InstanceService, error) {
 	cloud := clientconfig.Cloud{}
-	return NewInstanceServiceFromCloud(cloud)
+	var cacert []byte
+	return NewInstanceServiceFromCloud(cloud, cacert)
 }
 
-func NewInstanceServiceFromCloud(cloud clientconfig.Cloud) (*InstanceService, error) {
+func NewInstanceServiceFromCloud(cloud clientconfig.Cloud, cacert []byte) (*InstanceService, error) {
 	clientOpts := new(clientconfig.ClientOpts)
 	var opts *gophercloud.AuthOptions
 
@@ -165,9 +178,26 @@ func NewInstanceServiceFromCloud(cloud clientconfig.Cloud) (*InstanceService, er
 
 	opts.AllowReauth = true
 
-	provider, err := openstack.AuthenticatedClient(*opts)
+	provider, err := openstack.NewClient(opts.IdentityEndpoint)
+	if err != nil {
+		return nil, fmt.Errorf("create providerClient err: %v", err)
+	}
+
+	config := &tls.Config{}
+	cloudFromYaml, err := clientconfig.GetCloudFromYAML(clientOpts)
+	if cloudFromYaml.CACertFile != "" {
+		caCertPool := x509.NewCertPool()
+		caCertPool.AppendCertsFromPEM(cacert)
+		config.RootCAs = caCertPool
+	}
+
+	config.InsecureSkipVerify = *cloudFromYaml.Verify
+	transport := &http.Transport{Proxy: http.ProxyFromEnvironment, TLSClientConfig: config}
+	provider.HTTPClient.Transport = transport
+
+	err = openstack.Authenticate(provider, *opts)
 	if err != nil {
-		return nil, fmt.Errorf("Create providerClient err: %v", err)
+		return nil, fmt.Errorf("providerClient authentication err: %v", err)
 	}
 
 	identityClient, err := openstack.NewIdentityV3(provider, gophercloud.EndpointOpts{