From 3c240ac2585c3b6486b72f0fb9a7f52ae5dc852d Mon Sep 17 00:00:00 2001 From: Kautilya Tripathi Date: Mon, 31 Jan 2022 07:03:01 +0530 Subject: [PATCH] azure: Add new platform This adds new platform azure to lokomotive. Signed-off-by: Kautilya Tripathi --- cli/cmd/cluster/utils.go | 2 + pkg/platform/azure/azure.go | 423 +++++++++++++++++++++++++++++++++ pkg/platform/azure/template.go | 240 +++++++++++++++++++ 3 files changed, 665 insertions(+) create mode 100644 pkg/platform/azure/azure.go create mode 100644 pkg/platform/azure/template.go diff --git a/cli/cmd/cluster/utils.go b/cli/cmd/cluster/utils.go index 51231c441..d7b389d7e 100644 --- a/cli/cmd/cluster/utils.go +++ b/cli/cmd/cluster/utils.go @@ -30,6 +30,7 @@ import ( "github.com/kinvolk/lokomotive/pkg/platform" "github.com/kinvolk/lokomotive/pkg/platform/aks" "github.com/kinvolk/lokomotive/pkg/platform/aws" + "github.com/kinvolk/lokomotive/pkg/platform/azure" "github.com/kinvolk/lokomotive/pkg/platform/baremetal" "github.com/kinvolk/lokomotive/pkg/platform/equinixmetal" "github.com/kinvolk/lokomotive/pkg/platform/tinkerbell" @@ -79,6 +80,7 @@ func getPlatform(name string) (platform.Platform, error) { platforms := map[string]platform.Platform{ aks.Name: aks.NewConfig(), aws.Name: aws.NewConfig(), + azure.Name: azure.NewConfig(), equinixmetal.Name: equinixmetal.NewConfig(), baremetal.Name: baremetal.NewConfig(), tinkerbell.Name: tinkerbell.NewConfig(), diff --git a/pkg/platform/azure/azure.go b/pkg/platform/azure/azure.go new file mode 100644 index 000000000..091383330 --- /dev/null +++ b/pkg/platform/azure/azure.go @@ -0,0 +1,423 @@ +// Copyright 2022 The Lokomotive Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package azure is a Platform implementation for creating a Kubernetes cluster using +// Azure AKS. +package azure + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "text/template" + "time" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/gohcl" + "github.com/mitchellh/go-homedir" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + v1 "k8s.io/client-go/kubernetes/typed/storage/v1" + + "github.com/kinvolk/lokomotive/pkg/k8sutil" + "github.com/kinvolk/lokomotive/pkg/oidc" + "github.com/kinvolk/lokomotive/pkg/platform" + "github.com/kinvolk/lokomotive/pkg/terraform" +) + +// workerPool defines "worker_pool" block. +type workerPool struct { + Name string `hcl:"pool_name,label"` + SSHPubKeys []string `hcl:"ssh_pubkeys"` + Count int `hcl:"count,optional"` + VMType string `hcl:"vm_type,optional"` + OSImage string `hcl:"os_image,optional"` + CLCSnippets []string `hcl:"clc_snippets,optional"` + Priority string `hcl:"priority,optional"` + Tags map[string]string `hcl:"tags,optional"` +} + +// config defines "cluster" block for Azure. +type config struct { //nolint:maligned + AssetDir string `hcl:"asset_dir"` + ClusterName string `hcl:"cluster_name"` + ControllerType string `hcl:"controller_type,optional"` + ControllerCLCSnippets []string `hcl:"controller_clc_snippets,optional"` + WorkerType string `hcl:"worker_type,optional"` + SSHPubKeys []string `hcl:"ssh_pubkeys"` + Tags map[string]string `hcl:"tags,optional"` + ControllerCount int `hcl:"controller_count"` + DNSZone string `hcl:"dns_zone"` + DNSZoneGroup string `hcl:"dns_zone_group"` + Region string `hcl:"region,optional"` + EnableAggregation bool `hcl:"enable_aggregation,optional"` + EnableReporting bool `hcl:"enable_reporting,optional"` + OSImage string `hcl:"os_image,optional"` + ClusterDomainSuffix string `hcl:"cluster_domain_suffix,optional"` + // CustomImageResourceGroupName string `hcl:"custom_image_resource_group_name,optional"` + // CustomImageName string `hcl:"custom_image_name,optional"` + EnableNodeLocalDNS bool `hcl:"enable_node_local_dns,optional"` + DisableSelfHostedKubelet bool `hcl:"disable_self_hosted_kubelet,optional"` + OIDC *oidc.Config `hcl:"oidc,block"` + EnableTLSBootstrap bool `hcl:"enable_tls_bootstrap,optional"` + EncryptPodTraffic bool `hcl:"encrypt_pod_traffic,optional"` + PodCIDR string `hcl:"pod_cidr,optional"` + ServiceCIDR string `hcl:"service_cidr,optional"` + CertsValidityPeriodHours int `hcl:"certs_validity_period_hours,optional"` + ConntrackMaxPerCore int `hcl:"conntrack_max_per_core,optional"` + WorkerPools []workerPool `hcl:"worker_pool,block"` + WorkerPriority string `hcl:"worker_priority,optional"` + Networking string `hcl:"networking,optional"` + + KubernetesVersion string + + // Not exposed to the user + KubeAPIServerExtraFlags []string +} + +const ( + // Name represents azure platform name as it should be referenced in function calls and configuration. + Name = "azure" + + kubernetesVersion = "1.21.2" +) + +// NewConfig returns new Azure platform configuration with default values set. +// +//nolint:golint +func NewConfig() *config { + return &config{ + Region: "West Europe", + KubernetesVersion: kubernetesVersion, + ConntrackMaxPerCore: platform.ConntrackMaxPerCore, + } +} + +// LoadConfig loads configuration values into the config struct from given HCL configuration. +func (c *config) LoadConfig(configBody *hcl.Body, evalContext *hcl.EvalContext) hcl.Diagnostics { + if configBody == nil { + return hcl.Diagnostics{} + } + + if d := gohcl.DecodeBody(*configBody, evalContext, c); d.HasErrors() { + return d + } + + return c.checkValidConfig() +} + +func (c *config) clusterDomain() string { + return fmt.Sprintf("%s.%s", c.ClusterName, c.DNSZone) +} + +// Meta is part of Platform interface and returns common information about the platform configuration. +func (c *config) Meta() platform.Meta { + nodes := c.ControllerCount + for _, workerpool := range c.WorkerPools { + nodes += workerpool.Count + } + + charts := platform.CommonControlPlaneCharts(platform.ControlPlanCharts{ + Kubelet: !c.DisableSelfHostedKubelet, + NodeLocalDNS: c.EnableNodeLocalDNS, + }) + + return platform.Meta{ + AssetDir: c.AssetDir, + ExpectedNodes: nodes, + ControlplaneCharts: charts, + ControllerModuleName: fmt.Sprintf("%s-%s", Name, c.ClusterName), + Deployments: platform.CommonDeployments(c.ControllerCount), + DaemonSets: platform.CommonDaemonSets(c.ControllerCount, c.DisableSelfHostedKubelet), + } +} + +// Apply creates Azure infrastructure via Terraform. +func (c *config) Apply(ex *terraform.Executor) error { + if err := c.Initialize(ex); err != nil { + return err + } + + return ex.Apply([]string{terraform.WithParallelism}) +} + +// ApplyWithoutParallel applies Terraform configuration without parallel execution. +func (c *config) ApplyWithoutParallel(ex *terraform.Executor) error { + if err := c.Initialize(ex); err != nil { + return fmt.Errorf("initializing Terraform configuration: %w", err) + } + + return ex.Apply([]string{terraform.WithoutParallelism}) +} + +// Destroy destroys Azure infrastructure via Terraform. +func (c *config) Destroy(ex *terraform.Executor) error { + if err := c.Initialize(ex); err != nil { + return err + } + + return ex.Destroy() +} + +// Initialize creates Terrafrom files required for Azure. +func (c *config) Initialize(ex *terraform.Executor) error { + assetDir, err := homedir.Expand(c.AssetDir) + if err != nil { + return err + } + + terraformRootDir := terraform.GetTerraformRootDir(assetDir) + + return createTerraformConfigFile(c, terraformRootDir) +} + +func createTerraformConfigFile(cfg *config, terraformRootDir string) error { //nolint:funlen + workerpoolCfgList := []map[string]string{} + tmplName := "cluster.tf" + t := template.New(tmplName) + + t, err := t.Parse(terraformConfigTmpl) + if err != nil { + // TODO: Use template.Must(). + return fmt.Errorf("parsing template: %w", err) + } + + path := filepath.Join(terraformRootDir, tmplName) + + f, err := os.Create(path) + if err != nil { + return fmt.Errorf("creating file %q: %w", path, err) + } + + defer f.Close() //nolint:errcheck,gosec + + keyListBytes, err := json.Marshal(cfg.SSHPubKeys) + if err != nil { + // TODO: Render manually instead of marshaling. + return fmt.Errorf("marshaling SSH public keys: %w", err) + } + + controllerCLCSnippetsBytes, err := json.Marshal(cfg.ControllerCLCSnippets) + if err != nil { + // TODO: Render manually instead of marshaling. + return fmt.Errorf("marshaling CLC snippets: %w", err) + } + + // Configure oidc flags and set it to KubeAPIServerExtraFlags. + if cfg.OIDC != nil { + // Skipping the error checking here because its done in checkValidConfig(). + oidcFlags, _ := cfg.OIDC.ToKubeAPIServerFlags(cfg.clusterDomain()) + // TODO: Use append instead of setting the oidcFlags to KubeAPIServerExtraFlags + // append is not used for now because Initialize is called in cli/cmd/cluster.go + // and again in Apply which duplicates the values. + cfg.KubeAPIServerExtraFlags = oidcFlags + } + + platform.AppendVersionTag(&cfg.Tags) + + tags, err := json.Marshal(cfg.Tags) + if err != nil { + // TODO: Render manually instead of marshaling. + return fmt.Errorf("marshaling tags: %w", err) + } + + for _, workerpool := range cfg.WorkerPools { + input := map[string]interface{}{ + "clc_snippets": workerpool.CLCSnippets, + "ssh_pub_keys": workerpool.SSHPubKeys, + "tags": workerpool.Tags, + } + + output := map[string]string{} + + platform.AppendVersionTag(&workerpool.Tags) + + for k, v := range input { + bytes, err := json.Marshal(v) + if err != nil { + return fmt.Errorf("marshaling %q for worker pool %q failed: %w", k, workerpool.Name, err) + } + + output[k] = string(bytes) + } + + workerpoolCfgList = append(workerpoolCfgList, output) + } + + terraformCfg := struct { + Config config + Tags string + SSHPublicKeys string + ControllerCLCSnippets string + WorkerCLCSnippets string + WorkerTargetGroups string + WorkerpoolCfg []map[string]string + }{ + Config: *cfg, + Tags: string(tags), + SSHPublicKeys: string(keyListBytes), + ControllerCLCSnippets: string(controllerCLCSnippetsBytes), + WorkerpoolCfg: workerpoolCfgList, + } + + if err := t.Execute(f, terraformCfg); err != nil { + return fmt.Errorf("executing template: %w", err) + } + + return nil +} + +// checkValidConfig validates cluster configuration. +func (c *config) checkValidConfig() hcl.Diagnostics { + var d hcl.Diagnostics + + d = append(d, c.checkNotEmptyWorkers()...) + d = append(d, c.checkWorkerPoolNamesUnique()...) + d = append(d, c.checkWorkerPools()...) + d = append(d, c.checkRequiredFields()...) + + if c.ConntrackMaxPerCore < 0 { + d = append(d, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "conntrack_max_per_core can't be negative value", + Detail: fmt.Sprintf("'conntrack_max_per_core' value is %d", c.ConntrackMaxPerCore), + }) + } + + if c.OIDC != nil { + _, diags := c.OIDC.ToKubeAPIServerFlags(c.clusterDomain()) + d = append(d, diags...) + } + + return d +} + +// checkNotEmptyWorkers checks if the cluster has at least 1 node pool defined. +func (c *config) checkNotEmptyWorkers() hcl.Diagnostics { + var diagnostics hcl.Diagnostics + + if len(c.WorkerPools) == 0 { + diagnostics = append(diagnostics, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "At least one worker pool must be defined", + Detail: "Make sure to define at least one worker pool block in your cluster block", + }) + } + + return diagnostics +} + +// checkWorkerPoolNamesUnique verifies that all worker pool names are unique. +func (c *config) checkWorkerPoolNamesUnique() hcl.Diagnostics { + var diagnostics hcl.Diagnostics + + dup := make(map[string]bool) + + for _, w := range c.WorkerPools { + if !dup[w.Name] { + dup[w.Name] = true + + continue + } + + // It is duplicated. + diagnostics = append(diagnostics, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Worker pool names should be unique", + Detail: fmt.Sprintf("Worker pool '%v' is duplicated", w.Name), + }) + } + + return diagnostics +} + +// checkWorkerPools validates all configured worker pool fields. +func (c *config) checkWorkerPools() hcl.Diagnostics { + var d hcl.Diagnostics + + for _, w := range c.WorkerPools { + if w.Count <= 0 { + d = append(d, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("pool %q: count must be bigger than 0", w.Name), + }) + } + } + + return d +} + +// checkRequiredFields checks if that all required fields are populated in the top level configuration. +func (c *config) checkRequiredFields() hcl.Diagnostics { + var d hcl.Diagnostics + + f := map[string]string{ + "AssetDir": c.AssetDir, + "ClusterName": c.ClusterName, + } + + for k, v := range f { + if v == "" { + d = append(d, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("field %q can't be empty", k), + }) + } + } + + return d +} + +const ( + retryInterval = 5 * time.Second + timeout = 10 * time.Minute +) + +// PostApplyHook implements platform.PlatformWithPostApplyHook interface and defines hooks +// which should be executed after Azure cluster is created. +func (c *config) PostApplyHook(kubeconfig []byte) error { + client, err := k8sutil.NewClientset(kubeconfig) + if err != nil { + return fmt.Errorf("creating clientset from kubeconfig: %w", err) + } + + return waitForDefaultStorageClass(client.StorageV1().StorageClasses()) +} + +// waitForDefaultStorageClass waits until the default storage class appears on a given cluster. +// If it doesn't appear within a defined time range, an error is returned. +func waitForDefaultStorageClass(sci v1.StorageClassInterface) error { + defaultStorageClassAnnotation := "storageclass.kubernetes.io/is-default-class" + + if err := wait.PollImmediate(retryInterval, timeout, func() (done bool, err error) { + scs, err := sci.List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return false, fmt.Errorf("getting storage classes: %w", err) + } + + for _, sc := range scs.Items { + if v, ok := sc.ObjectMeta.Annotations[defaultStorageClassAnnotation]; ok && v == "true" { + return true, nil + } + } + + return false, nil + }); err != nil { + return fmt.Errorf("waiting for the default storage class to be configured: %w", err) + } + + return nil +} diff --git a/pkg/platform/azure/template.go b/pkg/platform/azure/template.go new file mode 100644 index 000000000..d875a968e --- /dev/null +++ b/pkg/platform/azure/template.go @@ -0,0 +1,240 @@ +// Copyright 2022 The Lokomotive Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azure + +var terraformConfigTmpl = ` +module "azure-{{.Config.ClusterName}}" { + source = "../terraform-modules/azure/flatcar-linux/kubernetes" + + dns_zone = "{{.Config.DNSZone}}" + dns_zone_group = "{{.Config.DNSZoneGroup}}" + + region = "{{.Config.Region}}" + + ssh_keys = {{.SSHPublicKeys}} + asset_dir = "../cluster-assets" + + cluster_name = "{{.Config.ClusterName}}" + tags = {{.Tags}} + + {{- if .Config.ClusterDomainSuffix }} + cluster_domain_suffix = "{{.Config.ClusterDomainSuffix}}" + {{- end }} + + controller_count = {{.Config.ControllerCount}} + {{- if .Config.ControllerType }} + controller_type = "{{ .Config.ControllerType }}" + {{- end }} + + {{- if .Config.WorkerType }} + worker_type = "{{ .Config.WorkerType }}" + {{- end }} + + {{- if .Config.WorkerPriority }} + worker_priority = "{{ .Config.WorkerPriority }}" + {{- end }} + + {{- if .Config.OSImage }} + os_image = "{{ .Config.OSImage }}" + {{- end }} + + enable_aggregation = {{.Config.EnableAggregation}} + + enable_reporting = {{.Config.EnableReporting}} + + {{- if .Config.PodCIDR }} + pod_cidr = "{{.Config.PodCIDR}}" + {{- end }} + + {{- if .Config.ServiceCIDR }} + service_cidr = "{{.Config.ServiceCIDR}}" + {{- end }} + + {{- if .Config.ControllerCLCSnippets}} + controller_clc_snippets = [ + {{- range $clc_snippet := .Config.ControllerCLCSnippets}} + <