Add a feature flag for formatting assets

Image rewriting involves a yaml format of the manifests, which makes for
a large and hard to read diff.  Add a feature flag to disable it, along
with a workaround to the release notes.

docs/releases/1.7-NOTES.md

@@ -2,6 +2,15 @@ _This is a WIP document describing changes to the upcoming kops 1.7 release_
 
 # Significant changes
 
+* Manifests are rewritten by default, which includes a normalization phase.
+  This can make it hard to understand the actual changes (as opposed to just the formatting changes).
+  A feature flag has been added, `export KOPS_FEATURE_FLAGS="-RewriteManifests"` which can be used
+  to disable manifest rewriting.  A recommendation: you can run `kops update` twice, once without
+  manifest formatting to show the real changes, and then immediately afterwards with manifest changes,
+  which will be just formatting changes.  Run `KOPS_FEATURE_FLAGS="-RewriteManifests" kops update cluster`,
+  to show the real changes, apply them with `KOPS_FEATURE_FLAGS="-RewriteManifests" kops update cluster --yes`,
+  then run `kops update cluster` to show the formatting changes, followed by `kops update cluster --yes`
+
 * Default disk size increased to 64GB (masters) and 128GB (nodes).  This does have a higher cost, but also gives us more inodes & more iops (and more disk space, of course!)
 * Calico now configured with the correct pod CIDR: #2768. Please refer to the *Required Actions* section for details regarding this.
 

pkg/assets/builder.go

@@ -19,12 +19,18 @@ package assets
 import (
 	"bytes"
 	"fmt"
+	"github.com/golang/glog"
 	"os"
 	"strings"
 
+	"k8s.io/kops/pkg/featureflag"
 	"k8s.io/kops/pkg/kubemanifest"
 )
 
+// RewriteManifests controls whether we rewrite manifests
+// Because manifest rewriting converts everything to and from YAML, we normalize everything by doing so
+var RewriteManifests = featureflag.New("RewriteManifests", featureflag.Bool(true))
+
 // AssetBuilder discovers and remaps assets
 type AssetBuilder struct {
 	Assets []*Asset
@@ -40,6 +46,9 @@ func NewAssetBuilder() *AssetBuilder {
 }
 
 func (a *AssetBuilder) RemapManifest(data []byte) ([]byte, error) {
+	if !RewriteManifests.Enabled() {
+		return data, nil
+	}
 	manifests, err := kubemanifest.LoadManifestsFrom(data)
 	if err != nil {
 		return nil, err