From 75fc12a37bb59e89131f5183301d5cc78b6432f7 Mon Sep 17 00:00:00 2001 From: PARKIN Date: Tue, 5 Sep 2023 12:04:22 -0700 Subject: [PATCH 1/5] Added project ID support to cloustack-csi-driver. Project ID is retrieved from instance metadata --- pkg/cloud/metadata.go | 31 +++++++++++++++++++++++++++++++ pkg/cloud/vms.go | 16 ++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/pkg/cloud/metadata.go b/pkg/cloud/metadata.go index b2b5638..e420779 100644 --- a/pkg/cloud/metadata.go +++ b/pkg/cloud/metadata.go @@ -49,8 +49,31 @@ func (c *client) metadataInstanceID(ctx context.Context) string { return "" } +func (c *client) metadataProjectID(ctx context.Context) (string) { + slog := ctxzap.Extract(ctx).Sugar() + + // Try cloud-init + slog.Debug("Try with cloud-init") + if _, err := os.Stat(cloudInitInstanceFilePath); err == nil { + slog.Debugf("File %s exists", cloudInitInstanceFilePath) + ciData, err := c.readCloudInit(ctx, cloudInitInstanceFilePath) + if err != nil { + slog.Errorf("Cannot read cloud-init instance data: %v", err) + } else { + if ciData.Ds.Metadata.ProjectID != "" { + return ciData.Ds.Metadata.ProjectID + } + } + slog.Error("cloud-init project ID is not provided") + } + + slog.Debug("CloudStack project ID not found in meta-data.") + return "" +} + type cloudInitInstanceData struct { V1 cloudInitV1 `json:"v1"` + Ds cloudInitDs `json:"ds"` } type cloudInitV1 struct { @@ -59,6 +82,14 @@ type cloudInitV1 struct { Zone string `json:"availability_zone"` } +type cloudInitDs struct { + Metadata cloudInitMetadata `json:"meta_data"` +} + +type cloudInitMetadata struct { + ProjectID string `json:"project-uuid"` +} + func (c *client) readCloudInit(ctx context.Context, instanceFilePath string) (*cloudInitInstanceData, error) { slog := ctxzap.Extract(ctx).Sugar() diff --git a/pkg/cloud/vms.go b/pkg/cloud/vms.go index 29124c3..f4dc954 100644 --- a/pkg/cloud/vms.go +++ b/pkg/cloud/vms.go @@ -12,6 +12,14 @@ func (c *client) GetVMByID(ctx context.Context, vmID string) (*VM, error) { ctxzap.Extract(ctx).Sugar().Infow("CloudStack API call", "command", "ListVirtualMachines", "params", map[string]string{ "id": vmID, }) + + //get projectid from metadata + projectID := c.metadataProjectID(ctx) + + if projectID != "" { + p.SetProjectid(projectID) + } + l, err := c.VirtualMachine.ListVirtualMachines(p) if err != nil { return nil, err @@ -35,6 +43,14 @@ func (c *client) getVMByName(ctx context.Context, name string) (*VM, error) { ctxzap.Extract(ctx).Sugar().Infow("CloudStack API call", "command", "ListVirtualMachines", "params", map[string]string{ "name": name, }) + + //get projectid from metadata + projectID := c.metadataProjectID(ctx) + + if projectID != "" { + p.SetProjectid(projectID) + } + l, err := c.VirtualMachine.ListVirtualMachines(p) if err != nil { return nil, err From 548df936631fa0749f8d2c8387496f8cd290ae2d Mon Sep 17 00:00:00 2001 From: PARKIN Date: Wed, 20 Sep 2023 13:24:36 -0700 Subject: [PATCH 2/5] Added project ID support for attach volume --- pkg/cloud/cloud.go | 3 ++- pkg/cloud/config.go | 2 ++ pkg/cloud/metadata.go | 12 ++++++------ pkg/cloud/vms.go | 28 ++++++++++++++++------------ pkg/cloud/volumes.go | 29 +++++++++++++++++++++++++++++ 5 files changed, 55 insertions(+), 19 deletions(-) diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go index c376303..2738c9b 100644 --- a/pkg/cloud/cloud.go +++ b/pkg/cloud/cloud.go @@ -54,10 +54,11 @@ var ( // client is the implementation of Interface. type client struct { *cloudstack.CloudStackClient + projectID string } // New creates a new cloud connector, given its configuration. func New(config *Config) Interface { csClient := cloudstack.NewAsyncClient(config.APIURL, config.APIKey, config.SecretKey, config.VerifySSL) - return &client{csClient} + return &client{csClient, config.ProjectID} } diff --git a/pkg/cloud/config.go b/pkg/cloud/config.go index 691be99..069de49 100644 --- a/pkg/cloud/config.go +++ b/pkg/cloud/config.go @@ -12,6 +12,7 @@ type Config struct { APIKey string SecretKey string VerifySSL bool + ProjectID string } // csConfig wraps the config for the CloudStack cloud provider. @@ -42,5 +43,6 @@ func ReadConfig(configFilePath string) (*Config, error) { APIKey: cfg.Global.APIKey, SecretKey: cfg.Global.SecretKey, VerifySSL: cfg.Global.SSLNoVerify, + ProjectID: cfg.Global.ProjectID, }, nil } diff --git a/pkg/cloud/metadata.go b/pkg/cloud/metadata.go index e420779..33028de 100644 --- a/pkg/cloud/metadata.go +++ b/pkg/cloud/metadata.go @@ -49,9 +49,9 @@ func (c *client) metadataInstanceID(ctx context.Context) string { return "" } -func (c *client) metadataProjectID(ctx context.Context) (string) { +func (c *client) metadataProjectID(ctx context.Context) string { slog := ctxzap.Extract(ctx).Sugar() - + // Try cloud-init slog.Debug("Try with cloud-init") if _, err := os.Stat(cloudInitInstanceFilePath); err == nil { @@ -60,12 +60,12 @@ func (c *client) metadataProjectID(ctx context.Context) (string) { if err != nil { slog.Errorf("Cannot read cloud-init instance data: %v", err) } else { - if ciData.Ds.Metadata.ProjectID != "" { + if ciData.Ds.Metadata.ProjectID != "" { return ciData.Ds.Metadata.ProjectID } } slog.Error("cloud-init project ID is not provided") - } + } slog.Debug("CloudStack project ID not found in meta-data.") return "" @@ -83,11 +83,11 @@ type cloudInitV1 struct { } type cloudInitDs struct { - Metadata cloudInitMetadata `json:"meta_data"` + Metadata cloudInitMetadata `json:"meta_data"` } type cloudInitMetadata struct { - ProjectID string `json:"project-uuid"` + ProjectID string `json:"project-uuid"` } func (c *client) readCloudInit(ctx context.Context, instanceFilePath string) (*cloudInitInstanceData, error) { diff --git a/pkg/cloud/vms.go b/pkg/cloud/vms.go index f4dc954..1ba4d77 100644 --- a/pkg/cloud/vms.go +++ b/pkg/cloud/vms.go @@ -12,14 +12,16 @@ func (c *client) GetVMByID(ctx context.Context, vmID string) (*VM, error) { ctxzap.Extract(ctx).Sugar().Infow("CloudStack API call", "command", "ListVirtualMachines", "params", map[string]string{ "id": vmID, }) - - //get projectid from metadata - projectID := c.metadataProjectID(ctx) - if projectID != "" { - p.SetProjectid(projectID) + if c.projectID == "" { + //get projectid from metadata + c.projectID = c.metadataProjectID(ctx) } - + + if c.projectID != "" { + p.SetProjectid(c.projectID) + } + l, err := c.VirtualMachine.ListVirtualMachines(p) if err != nil { return nil, err @@ -43,14 +45,16 @@ func (c *client) getVMByName(ctx context.Context, name string) (*VM, error) { ctxzap.Extract(ctx).Sugar().Infow("CloudStack API call", "command", "ListVirtualMachines", "params", map[string]string{ "name": name, }) - - //get projectid from metadata - projectID := c.metadataProjectID(ctx) - if projectID != "" { - p.SetProjectid(projectID) + if c.projectID == "" { + //get projectid from metadata + c.projectID = c.metadataProjectID(ctx) } - + + if c.projectID != "" { + p.SetProjectid(c.projectID) + } + l, err := c.VirtualMachine.ListVirtualMachines(p) if err != nil { return nil, err diff --git a/pkg/cloud/volumes.go b/pkg/cloud/volumes.go index 394be39..fd0f2d7 100644 --- a/pkg/cloud/volumes.go +++ b/pkg/cloud/volumes.go @@ -14,6 +14,16 @@ func (c *client) GetVolumeByID(ctx context.Context, volumeID string) (*Volume, e ctxzap.Extract(ctx).Sugar().Infow("CloudStack API call", "command", "ListVolumes", "params", map[string]string{ "id": volumeID, }) + + if c.projectID == "" { + //get projectid from metadata + c.projectID = c.metadataProjectID(ctx) + } + + if c.projectID != "" { + p.SetProjectid(c.projectID) + } + l, err := c.Volume.ListVolumes(p) if err != nil { return nil, err @@ -43,6 +53,16 @@ func (c *client) GetVolumeByName(ctx context.Context, name string) (*Volume, err ctxzap.Extract(ctx).Sugar().Infow("CloudStack API call", "command", "ListVolumes", "params", map[string]string{ "name": name, }) + + if c.projectID == "" { + //get projectid from metadata + c.projectID = c.metadataProjectID(ctx) + } + + if c.projectID != "" { + p.SetProjectid(c.projectID) + } + l, err := c.Volume.ListVolumes(p) if err != nil { return nil, err @@ -72,6 +92,15 @@ func (c *client) CreateVolume(ctx context.Context, diskOfferingID, zoneID, name p.SetZoneid(zoneID) p.SetName(name) p.SetSize(sizeInGB) + + if c.projectID == "" { + //get projectid from metadata + c.projectID = c.metadataProjectID(ctx) + } + + if c.projectID != "" { + p.SetProjectid(c.projectID) + } ctxzap.Extract(ctx).Sugar().Infow("CloudStack API call", "command", "CreateVolume", "params", map[string]string{ "diskofferingid": diskOfferingID, "zoneid": zoneID, From aea6ca25d144d2370cdcd321e5396b49390e0b6c Mon Sep 17 00:00:00 2001 From: Hongmei Parkin Date: Tue, 26 Sep 2023 13:58:10 -0700 Subject: [PATCH 3/5] Added cloud-init mount to example yaml --- deploy/k8s/controller-deployment.yaml | 168 +++++++++++++------------- 1 file changed, 87 insertions(+), 81 deletions(-) diff --git a/deploy/k8s/controller-deployment.yaml b/deploy/k8s/controller-deployment.yaml index 2ab16d5..49341fc 100644 --- a/deploy/k8s/controller-deployment.yaml +++ b/deploy/k8s/controller-deployment.yaml @@ -1,81 +1,87 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: cloudstack-csi-controller - namespace: kube-system -spec: - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - maxSurge: 0 - selector: - matchLabels: - app.kubernetes.io/name: cloudstack-csi-controller - template: - metadata: - labels: - app.kubernetes.io/name: cloudstack-csi-controller - app.kubernetes.io/part-of: cloudstack-csi-driver - spec: - serviceAccountName: cloudstack-csi-controller - nodeSelector: - kubernetes.io/os: linux - node-role.kubernetes.io/master: "" - tolerations: - - effect: NoExecute - operator: Exists - - effect: NoSchedule - operator: Exists - - containers: - - name: cloudstack-csi-controller - image: cloudstack-csi-driver - imagePullPolicy: Always - args: - - "-endpoint=$(CSI_ENDPOINT)" - - "-cloudstackconfig=/etc/cloudstack-csi-driver/cloud-config" - - "-debug" - env: - - name: CSI_ENDPOINT - value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - - name: cloudstack-conf - mountPath: /etc/cloudstack-csi-driver - - - name: external-provisioner - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.4 - imagePullPolicy: IfNotPresent - args: - - "--csi-address=$(ADDRESS)" - - "--v=5" - - "--feature-gates=Topology=true" - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - - - name: external-attacher - image: k8s.gcr.io/sig-storage/csi-attacher:v3.0.2 - imagePullPolicy: IfNotPresent - args: - - "--csi-address=$(ADDRESS)" - - "--v=5" - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - - volumes: - - name: socket-dir - emptyDir: {} - - name: cloudstack-conf - secret: - secretName: cloudstack-secret +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cloudstack-csi-controller + namespace: kube-system +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + maxSurge: 0 + selector: + matchLabels: + app.kubernetes.io/name: cloudstack-csi-controller + template: + metadata: + labels: + app.kubernetes.io/name: cloudstack-csi-controller + app.kubernetes.io/part-of: cloudstack-csi-driver + spec: + serviceAccountName: cloudstack-csi-controller + nodeSelector: + kubernetes.io/os: linux + node-role.kubernetes.io/master: "" + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + operator: Exists + + containers: + - name: cloudstack-csi-controller + image: cloudstack-csi-driver + imagePullPolicy: Always + args: + - "-endpoint=$(CSI_ENDPOINT)" + - "-cloudstackconfig=/etc/cloudstack-csi-driver/cloud-config" + - "-debug" + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: cloudstack-conf + mountPath: /etc/cloudstack-csi-driver + - name: cloud-init-dir + mountPath: /run/cloud-init/ + + - name: external-provisioner + image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.4 + imagePullPolicy: IfNotPresent + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + - "--feature-gates=Topology=true" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + + - name: external-attacher + image: k8s.gcr.io/sig-storage/csi-attacher:v3.0.2 + imagePullPolicy: IfNotPresent + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + + volumes: + - name: socket-dir + emptyDir: {} + - name: cloudstack-conf + secret: + secretName: cloudstack-secret + - name: cloud-init-dir + hostPath: + path: /run/cloud-init/ + type: Directory \ No newline at end of file From 7a851911ffc4f2b3000d77343beb710abcb83240 Mon Sep 17 00:00:00 2001 From: Hongmei Parkin Date: Tue, 26 Sep 2023 14:08:33 -0700 Subject: [PATCH 4/5] revert change --- deploy/k8s/controller-deployment.yaml | 168 +++++++++++++------------- 1 file changed, 81 insertions(+), 87 deletions(-) diff --git a/deploy/k8s/controller-deployment.yaml b/deploy/k8s/controller-deployment.yaml index 49341fc..2ab16d5 100644 --- a/deploy/k8s/controller-deployment.yaml +++ b/deploy/k8s/controller-deployment.yaml @@ -1,87 +1,81 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: cloudstack-csi-controller - namespace: kube-system -spec: - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - maxSurge: 0 - selector: - matchLabels: - app.kubernetes.io/name: cloudstack-csi-controller - template: - metadata: - labels: - app.kubernetes.io/name: cloudstack-csi-controller - app.kubernetes.io/part-of: cloudstack-csi-driver - spec: - serviceAccountName: cloudstack-csi-controller - nodeSelector: - kubernetes.io/os: linux - node-role.kubernetes.io/master: "" - tolerations: - - effect: NoExecute - operator: Exists - - effect: NoSchedule - operator: Exists - - containers: - - name: cloudstack-csi-controller - image: cloudstack-csi-driver - imagePullPolicy: Always - args: - - "-endpoint=$(CSI_ENDPOINT)" - - "-cloudstackconfig=/etc/cloudstack-csi-driver/cloud-config" - - "-debug" - env: - - name: CSI_ENDPOINT - value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - - name: cloudstack-conf - mountPath: /etc/cloudstack-csi-driver - - name: cloud-init-dir - mountPath: /run/cloud-init/ - - - name: external-provisioner - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.4 - imagePullPolicy: IfNotPresent - args: - - "--csi-address=$(ADDRESS)" - - "--v=5" - - "--feature-gates=Topology=true" - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - - - name: external-attacher - image: k8s.gcr.io/sig-storage/csi-attacher:v3.0.2 - imagePullPolicy: IfNotPresent - args: - - "--csi-address=$(ADDRESS)" - - "--v=5" - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - - volumes: - - name: socket-dir - emptyDir: {} - - name: cloudstack-conf - secret: - secretName: cloudstack-secret - - name: cloud-init-dir - hostPath: - path: /run/cloud-init/ - type: Directory \ No newline at end of file +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cloudstack-csi-controller + namespace: kube-system +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + maxSurge: 0 + selector: + matchLabels: + app.kubernetes.io/name: cloudstack-csi-controller + template: + metadata: + labels: + app.kubernetes.io/name: cloudstack-csi-controller + app.kubernetes.io/part-of: cloudstack-csi-driver + spec: + serviceAccountName: cloudstack-csi-controller + nodeSelector: + kubernetes.io/os: linux + node-role.kubernetes.io/master: "" + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + operator: Exists + + containers: + - name: cloudstack-csi-controller + image: cloudstack-csi-driver + imagePullPolicy: Always + args: + - "-endpoint=$(CSI_ENDPOINT)" + - "-cloudstackconfig=/etc/cloudstack-csi-driver/cloud-config" + - "-debug" + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: cloudstack-conf + mountPath: /etc/cloudstack-csi-driver + + - name: external-provisioner + image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.4 + imagePullPolicy: IfNotPresent + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + - "--feature-gates=Topology=true" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + + - name: external-attacher + image: k8s.gcr.io/sig-storage/csi-attacher:v3.0.2 + imagePullPolicy: IfNotPresent + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + + volumes: + - name: socket-dir + emptyDir: {} + - name: cloudstack-conf + secret: + secretName: cloudstack-secret From 523b8eb6a11565038218cfec7db5790d52a7137a Mon Sep 17 00:00:00 2001 From: PARKIN Date: Mon, 2 Oct 2023 10:50:53 -0700 Subject: [PATCH 5/5] Added cloud-init directory mount to the controller deployment yaml --- deploy/k8s/controller-deployment.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/deploy/k8s/controller-deployment.yaml b/deploy/k8s/controller-deployment.yaml index 2ab16d5..ab20ca2 100644 --- a/deploy/k8s/controller-deployment.yaml +++ b/deploy/k8s/controller-deployment.yaml @@ -45,6 +45,8 @@ spec: mountPath: /var/lib/csi/sockets/pluginproxy/ - name: cloudstack-conf mountPath: /etc/cloudstack-csi-driver + - name: cloud-init-dir + mountPath: /run/cloud-init/ - name: external-provisioner image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.4 @@ -79,3 +81,7 @@ spec: - name: cloudstack-conf secret: secretName: cloudstack-secret + - name: cloud-init-dir + hostPath: + path: /run/cloud-init/ + type: Directory \ No newline at end of file