Put versioned API of cluster into state store

cmd/kops-controller/controllers/node_controller.go

@@ -189,6 +189,7 @@ func (r *NodeReconciler) patchNodeLabels(ctx context.Context, node *corev1.Node,
 // getClusterForNode returns the kops.Cluster object for the node
 // The cluster is actually loaded when we first start
 func (r *NodeReconciler) getClusterForNode(node *corev1.Node) (*kops.Cluster, error) {
+	// todo have to read the versioned format
 	clusterPath := r.configBase.Join(registry.PathClusterCompleted)
 	cluster, err := r.loadCluster(clusterPath)
 	if err != nil {
@@ -261,6 +262,7 @@ func (r *NodeReconciler) loadNamedInstanceGroup(name string) (*kops.InstanceGrou
 		return nil, fmt.Errorf("error loading InstanceGroup %q: %v", p, err)
 	}
 
+	// todo bug: The on-disk format is the versioned API but this is deserializing as the unversioned API
 	instanceGroup := &kops.InstanceGroup{}
 	if err := utils.YamlUnmarshal(b, instanceGroup); err != nil {
 		return nil, fmt.Errorf("error parsing InstanceGroup %q: %v", p, err)

cmd/kops/create_cluster.go

@@ -1354,7 +1354,7 @@ func RunCreateCluster(ctx context.Context, f *util.Factory, out io.Writer, c *Cr
 		return fmt.Errorf("error writing updated configuration: %v", err)
 	}
 
-	err = registry.WriteConfigDeprecated(cluster, configBase.Join(registry.PathClusterCompleted), fullCluster)
+	err = registry.WriteConfigDeprecated(cluster, configBase.Join(registry.PathLegacyClusterCompleted), fullCluster)
 	if err != nil {
 		return fmt.Errorf("error writing completed cluster spec: %v", err)
 	}

cmd/kops/edit_cluster.go

@@ -250,7 +250,7 @@ func RunEditCluster(ctx context.Context, f *util.Factory, cmd *cobra.Command, ar
 			return preservedFile(err, file, out)
 		}
 
-		err = registry.WriteConfigDeprecated(newCluster, configBase.Join(registry.PathClusterCompleted), fullCluster)
+		err = registry.WriteConfigDeprecated(newCluster, configBase.Join(registry.PathLegacyClusterCompleted), fullCluster)
 		if err != nil {
 			return preservedFile(fmt.Errorf("error writing completed cluster spec: %v", err), file, out)
 		}

pkg/apis/kops/registry/registry.go

@@ -28,7 +28,9 @@ const (
 	// Path for the user-specified cluster spec
 	PathCluster = "config"
 	// Path for completed cluster spec in the state store
-	PathClusterCompleted = "cluster.spec"
+	PathClusterCompleted = "cluster-completed.spec"
+	// Path for unversioned completed cluster spec in the state store
+	PathLegacyClusterCompleted = "cluster.spec"
 )
 
 func ConfigBase(c *api.Cluster) (vfs.Path, error) {

pkg/apis/kops/registry/statestore.go

@@ -28,6 +28,7 @@ import (
 	"k8s.io/kops/util/pkg/vfs"
 )
 
+// todo remove this function
 func ReadConfigDeprecated(configPath vfs.Path, config interface{}) error {
 	data, err := configPath.ReadFile()
 	if err != nil {

pkg/bundle/builder.go

@@ -64,6 +64,7 @@ func (b *Builder) Build(cluster *kops.Cluster, ig *kops.InstanceGroup) (*Data, e
 			return nil, fmt.Errorf("error building ConfigBase for cluster: %v", err)
 		}
 
+		// todo have to read the versioned format
 		p := configBase.Join(registry.PathClusterCompleted)
 
 		b, err := p.ReadFile()
@@ -87,13 +88,13 @@ func (b *Builder) Build(cluster *kops.Cluster, ig *kops.InstanceGroup) (*Data, e
 	var files []*DataFile
 
 	{
-		data, err := utils.YamlMarshal(fullCluster)
+		data, err := kopscodecs.ToVersionedYaml(fullCluster)
 		if err != nil {
 			return nil, fmt.Errorf("error marshaling configuration: %v", err)
 		}
 
 		file := &DataFile{}
-		file.Header.Name = "cluster.spec"
+		file.Header.Name = "cluster-completed.spec"
 		file.Header.Size = int64(len(data))
 		file.Header.Mode = 0644
 		file.Data = data

pkg/client/simple/api/clientset.go

@@ -68,6 +68,12 @@ func (c *RESTClientset) UpdateCluster(ctx context.Context, cluster *kops.Cluster
 	return c.KopsClient.Clusters(namespace).Update(ctx, cluster, metav1.UpdateOptions{})
 }
 
+// UpdateCompletedCluster implements the UpdateCompletedCluster method of Clientset for a kubernetes-API state store
+func (c *RESTClientset) UpdateCompletedCluster(ctx context.Context, cluster *kops.Cluster) error {
+	namespace := restNamespaceForClusterName(cluster.Name)
+	return c.KopsClient.Clusters(namespace).UpdateCompleted(ctx, cluster, metav1.UpdateOptions{})
+}
+
 // ConfigBaseFor implements the ConfigBaseFor method of Clientset for a kubernetes-API state store
 func (c *RESTClientset) ConfigBaseFor(cluster *kops.Cluster) (vfs.Path, error) {
 	if cluster.Spec.ConfigBase != "" {

pkg/client/simple/clientset.go

@@ -36,6 +36,9 @@ type Clientset interface {
 	// UpdateCluster updates a cluster
 	UpdateCluster(ctx context.Context, cluster *kops.Cluster, status *kops.ClusterStatus) (*kops.Cluster, error)
 
+	// UpdateCompletedCluster updates a completed cluster.
+	UpdateCompletedCluster(ctx context.Context, cluster *kops.Cluster) error
+
 	// ListClusters returns all clusters
 	ListClusters(ctx context.Context, options metav1.ListOptions) (*kops.ClusterList, error)
 

pkg/client/simple/vfsclientset/clientset.go

@@ -51,6 +51,11 @@ func (c *VFSClientset) UpdateCluster(ctx context.Context, cluster *kops.Cluster,
 	return c.clusters().Update(cluster, status)
 }
 
+// UpdateCompletedCluster implements the UpdateCluster method of simple.Clientset for a VFS-backed state store.
+func (c *VFSClientset) UpdateCompletedCluster(ctx context.Context, cluster *kops.Cluster) error {
+	return c.clusters().UpdateCompleted(cluster)
+}
+
 // CreateCluster implements the CreateCluster method of simple.Clientset for a VFS-backed state store
 func (c *VFSClientset) CreateCluster(ctx context.Context, cluster *kops.Cluster) (*kops.Cluster, error) {
 	return c.clusters().Create(cluster)
@@ -117,7 +122,7 @@ func DeleteAllClusterState(basePath vfs.Path) error {
 			continue
 		}
 
-		if relativePath == "config" || relativePath == "cluster.spec" {
+		if relativePath == "config" || relativePath == "cluster.spec" || relativePath == "cluster-completed.spec" {
 			continue
 		}
 		if strings.HasPrefix(relativePath, "addons/") {

pkg/client/simple/vfsclientset/cluster.go

@@ -152,6 +152,22 @@ func (r *ClusterVFS) Update(c *api.Cluster, status *api.ClusterStatus) (*api.Clu
 	return c, nil
 }
 
+func (r *ClusterVFS) UpdateCompleted(c *api.Cluster) error {
+	clusterName := c.ObjectMeta.Name
+	if clusterName == "" {
+		return field.Required(field.NewPath("objectMeta", "name"), "clusterName is required")
+	}
+
+	if err := r.writeConfig(c, r.basePath.Join(clusterName, registry.PathClusterCompleted), c); err != nil {
+		if os.IsNotExist(err) {
+			return err
+		}
+		return fmt.Errorf("error writing Cluster: %v", err)
+	}
+
+	return nil
+}
+
 // List returns a slice containing all the cluster names
 // It skips directories that don't look like clusters
 func (r *ClusterVFS) listNames() ([]string, error) {

pkg/model/alimodel/policy_builder.go

@@ -276,7 +276,7 @@ func (b *PolicyBuilder) AddOSSPermissions(p *Policy) (*Policy, error) {
 			} else if b.Role == kops.InstanceGroupRoleNode {
 				resources := []string{
 					strings.Join([]string{b.RAMPrefix(), ":oss:*:*:", ramOSSPath, "/addons/*"}, ""),
-					strings.Join([]string{b.RAMPrefix(), ":oss:*:*:", ramOSSPath, "/cluster.spec"}, ""),
+					strings.Join([]string{b.RAMPrefix(), ":oss:*:*:", ramOSSPath, "/cluster-completed.spec"}, ""),
 					strings.Join([]string{b.RAMPrefix(), ":oss:*:*:", ramOSSPath, "/config"}, ""),
 					strings.Join([]string{b.RAMPrefix(), ":oss:*:*:", ramOSSPath, "/instancegroup/*"}, ""),
 					strings.Join([]string{b.RAMPrefix(), ":oss:*:*:", ramOSSPath, "/pki/issued/*"}, ""),

pkg/model/iam/iam_builder.go

@@ -359,7 +359,7 @@ func (b *PolicyBuilder) AddS3Permissions(p *Policy) (*Policy, error) {
 				} else if b.Role == kops.InstanceGroupRoleNode {
 					resources := []string{
 						strings.Join([]string{b.IAMPrefix(), ":s3:::", iamS3Path, "/addons/*"}, ""),
-						strings.Join([]string{b.IAMPrefix(), ":s3:::", iamS3Path, "/cluster.spec"}, ""),
+						strings.Join([]string{b.IAMPrefix(), ":s3:::", iamS3Path, "/cluster-completed.spec"}, ""),
 						strings.Join([]string{b.IAMPrefix(), ":s3:::", iamS3Path, "/config"}, ""),
 						strings.Join([]string{b.IAMPrefix(), ":s3:::", iamS3Path, "/instancegroup/*"}, ""),
 						strings.Join([]string{b.IAMPrefix(), ":s3:::", iamS3Path, "/pki/issued/*"}, ""),

pkg/model/iam/tests/iam_builder_node_strict.json

@@ -30,7 +30,7 @@
       ],
       "Resource": [
         "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/addons/*",
-        "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/cluster.spec",
+        "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/cluster-completed.spec",
         "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/config",
         "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/instancegroup/*",
         "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/issued/*",

pkg/model/iam/tests/iam_builder_node_strict_ecr.json

@@ -30,7 +30,7 @@
       ],
       "Resource": [
         "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/addons/*",
-        "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/cluster.spec",
+        "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/cluster-completed.spec",
         "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/config",
         "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/instancegroup/*",
         "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/issued/*",

upup/pkg/fi/cloudup/apply_cluster.go

@@ -924,11 +924,16 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) error {
 	c.Target = target
 
 	if !dryRun {
-		err = registry.WriteConfigDeprecated(cluster, configBase.Join(registry.PathClusterCompleted), c.Cluster)
+		err = c.Clientset.UpdateCompletedCluster(ctx, c.Cluster)
 		if err != nil {
 			return fmt.Errorf("error writing completed cluster spec: %v", err)
 		}
 
+		err = registry.WriteConfigDeprecated(cluster, configBase.Join(registry.PathLegacyClusterCompleted), c.Cluster)
+		if err != nil {
+			return fmt.Errorf("error writing legacy completed cluster spec: %v", err)
+		}
+
 		vfsMirror := vfsclientset.NewInstanceGroupMirror(cluster, configBase)
 
 		for _, g := range c.InstanceGroups {

upup/pkg/fi/nodeup/command.go

@@ -135,6 +135,7 @@ func (c *NodeUpCommand) Run(out io.Writer) error {
 			p = configBase.Join(registry.PathClusterCompleted)
 		}
 
+		// todo have to read the versioned format
 		b, err := p.ReadFile()
 		if err != nil {
 			return fmt.Errorf("error loading Cluster %q: %v", p, err)
@@ -155,6 +156,7 @@ func (c *NodeUpCommand) Run(out io.Writer) error {
 			return fmt.Errorf("error loading InstanceGroup %q: %v", instanceGroupLocation, err)
 		}
 
+		// todo bug: The on-disk format is the versioned API but this is deserializing as the unversioned API
 		if err = utils.YamlUnmarshal(b, c.instanceGroup); err != nil {
 			return fmt.Errorf("error parsing InstanceGroup %q: %v", instanceGroupLocation, err)
 		}

upup/pkg/kutil/convert_kubeup_cluster.go

@@ -471,11 +471,7 @@ func (x *ConvertKubeupCluster) Upgrade(ctx context.Context) error {
 	}
 
 	// TODO: No longer needed?
-	err = registry.WriteConfigDeprecated(cluster, newConfigBase.Join(registry.PathClusterCompleted), fullCluster)
-	if err != nil {
-		return fmt.Errorf("error writing completed cluster spec: %v", err)
-	}
-
+	err = registry.WriteConfigDeprecated(cluster, newConfigBase.Join(registry.PathLegacyClusterCompleted), fullCluster)
 	if err != nil {
 		return fmt.Errorf("error writing completed cluster spec: %v", err)
 	}