MCO: manage the user-data secret

With this we let the installer take care of installing the initial
user-data secret and we then take over with our managed secret. if we
are upgrading (and thus no installer at play), we just create the new
(managed) secret.

This is a cherry-pick of the original patch
1f52e48, which was later reverted by
1c65355.

Signed-off-by: Antonio Murdaca <runcom@linux.com>
Co-authored-by: Zane Bitter <zbitter@redhat.com>

cmd/machine-config-operator/start.go

@@ -76,6 +76,7 @@ func runStartCmd(cmd *cobra.Command, args []string) {
 			ctrlctx.ClientBuilder.ConfigClientOrDie(componentName),
 			ctrlctx.OpenShiftKubeAPIServerKubeNamespacedInformerFactory.Core().V1().ConfigMaps(),
 			ctrlctx.KubeInformerFactory.Core().V1().Nodes(),
+			ctrlctx.KubeMAOSharedInformer.Core().V1().Secrets(),
 		)
 
 		ctrlctx.NamespacedInformerFactory.Start(ctrlctx.Stop)
@@ -85,6 +86,7 @@ func runStartCmd(cmd *cobra.Command, args []string) {
 		ctrlctx.ConfigInformerFactory.Start(ctrlctx.Stop)
 		ctrlctx.OpenShiftKubeAPIServerKubeNamespacedInformerFactory.Start(ctrlctx.Stop)
 		ctrlctx.OperatorInformerFactory.Start(ctrlctx.Stop)
+		ctrlctx.KubeMAOSharedInformer.Start(ctrlctx.Stop)
 		close(ctrlctx.InformersStarted)
 
 		go controller.Run(2, ctrlctx.Stop)

manifests/userdata_secret.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{.Role}}-user-data-managed
+  namespace: openshift-machine-api
+type: Opaque
+data:
+  disableTemplating: "dHJ1ZQo="
+  userData: {{.PointerConfig}}

pkg/controller/common/controller_context.go

@@ -45,6 +45,7 @@ type ControllerContext struct {
 	APIExtInformerFactory                               apiextinformers.SharedInformerFactory
 	ConfigInformerFactory                               configinformers.SharedInformerFactory
 	OperatorInformerFactory                             operatorinformers.SharedInformerFactory
+	KubeMAOSharedInformer                               informers.SharedInformerFactory
 
 	AvailableResources map[schema.GroupVersionResource]bool
 
@@ -74,6 +75,8 @@ func CreateControllerContext(cb *clients.Builder, stop <-chan struct{}, targetNa
 			opt.FieldSelector = fields.OneTermEqualSelector("metadata.name", "kube-apiserver-to-kubelet-client-ca").String()
 		},
 	)
+	// this is needed to listen for changes in MAO user data secrets to re-apply the ones we define in the MCO (since we manage them)
+	kubeMAOSharedInformer := informers.NewFilteredSharedInformerFactory(kubeClient, resyncPeriod()(), "openshift-machine-api", nil)
 
 	// filter out CRDs that do not have the MCO label
 	assignFilterLabels := func(opts *metav1.ListOptions) {
@@ -103,5 +106,6 @@ func CreateControllerContext(cb *clients.Builder, stop <-chan struct{}, targetNa
 		Stop:                                                stop,
 		InformersStarted:                                    make(chan struct{}),
 		ResyncPeriod:                                        resyncPeriod(),
+		KubeMAOSharedInformer:                               kubeMAOSharedInformer,
 	}
 }

pkg/controller/common/helpers.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
+	"net/url"
 	"reflect"
 	"sort"
 
@@ -29,6 +30,7 @@ import (
 	"github.com/ghodss/yaml"
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
+	"github.com/vincent-petithory/dataurl"
 	kerr "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -131,6 +133,39 @@ func MergeMachineConfigs(configs []*mcfgv1.MachineConfig, osImageURL string) (*m
 	}, nil
 }
 
+// PointerConfig generates the stub ignition for the machine to boot properly
+// NOTE: If you change this, you also need to change the pointer configuration in openshift/installer, see
+// https://github.com/openshift/installer/blob/master/pkg/asset/ignition/machine/node.go#L20
+func PointerConfig(ignitionHost string, rootCA []byte) (ign2types.Config, error) {
+	configSourceURL := &url.URL{
+		Scheme: "https",
+		Host:   ignitionHost,
+		Path:   "/config/{{.Role}}",
+	}
+	// we do decoding here as curly brackets are escaped to %7B and breaks golang's templates
+	ignitionHostTmpl, err := url.QueryUnescape(configSourceURL.String())
+	if err != nil {
+		return ign2types.Config{}, err
+	}
+	return ign2types.Config{
+		Ignition: ign2types.Ignition{
+			Version: ign2types.MaxVersion.String(),
+			Config: ign2types.IgnitionConfig{
+				Append: []ign2types.ConfigReference{{
+					Source: ignitionHostTmpl,
+				}},
+			},
+			Security: ign2types.Security{
+				TLS: ign2types.TLS{
+					CertificateAuthorities: []ign2types.CaReference{{
+						Source: dataurl.EncodeBytes(rootCA),
+					}},
+				},
+			},
+		},
+	}, nil
+}
+
 // NewIgnConfig returns an empty ignition config with version set as latest version
 func NewIgnConfig() ign3types.Config {
 	return ign3types.Config{

pkg/operator/bootstrap.go

@@ -151,7 +151,7 @@ func RenderBootstrap(
 		templatectrl.BaremetalRuntimeCfgKey: imgs.BaremetalRuntimeCfg,
 	}
 
-	config := getRenderConfig("", string(filesData[kubeAPIServerServingCA]), spec, &imgs.RenderConfigImages, infra.Status.APIServerInternalURL)
+	config := getRenderConfig("", string(filesData[kubeAPIServerServingCA]), spec, &imgs.RenderConfigImages, infra.Status.APIServerInternalURL, nil)
 
 	manifests := []manifest{
 		{

pkg/operator/operator.go

@@ -101,6 +101,7 @@ type Operator struct {
 	oseKubeAPIListerSynced           cache.InformerSynced
 	nodeListerSynced                 cache.InformerSynced
 	dnsListerSynced                  cache.InformerSynced
+	maoSecretInformerSynced          cache.InformerSynced
 
 	// queue only ever has one item, but it has nice error handling backoff/retry semantics
 	queue workqueue.RateLimitingInterface
@@ -134,6 +135,7 @@ func New(
 	configClient configclientset.Interface,
 	oseKubeAPIInformer coreinformersv1.ConfigMapInformer,
 	nodeInformer coreinformersv1.NodeInformer,
+	maoSecretInformer coreinformersv1.SecretInformer,
 ) *Operator {
 	eventBroadcaster := record.NewBroadcaster()
 	eventBroadcaster.StartLogging(glog.Infof)
@@ -168,6 +170,7 @@ func New(
 		oseKubeAPIInformer.Informer(),
 		nodeInformer.Informer(),
 		dnsInformer.Informer(),
+		maoSecretInformer.Informer(),
 	} {
 		i.AddEventHandler(optr.eventHandler())
 	}
@@ -189,6 +192,7 @@ func New(
 	optr.nodeLister = nodeInformer.Lister()
 	optr.nodeListerSynced = nodeInformer.Informer().HasSynced
 
+	optr.maoSecretInformerSynced = maoSecretInformer.Informer().HasSynced
 	optr.serviceAccountInformerSynced = serviceAccountInfomer.Informer().HasSynced
 	optr.clusterRoleInformerSynced = clusterRoleInformer.Informer().HasSynced
 	optr.clusterRoleBindingInformerSynced = clusterRoleBindingInformer.Informer().HasSynced
@@ -237,6 +241,7 @@ func (optr *Operator) Run(workers int, stopCh <-chan struct{}) {
 		optr.clusterCmListerSynced,
 		optr.serviceAccountInformerSynced,
 		optr.clusterRoleInformerSynced,
+		optr.maoSecretInformerSynced,
 		optr.clusterRoleBindingInformerSynced,
 		optr.networkListerSynced,
 		optr.proxyListerSynced,

pkg/operator/render.go

@@ -31,6 +31,7 @@ type renderConfig struct {
 	KubeAPIServerServingCA string
 	Infra                  configv1.Infrastructure
 	Constants              map[string]string
+	PointerConfig          string
 }
 
 type assetRenderer struct {

pkg/operator/sync.go

@@ -8,6 +8,9 @@ import (
 	"encoding/pem"
 	"fmt"
 	"io/ioutil"
+	"net"
+	"net/url"
+	"strconv"
 	"strings"
 	"time"
 
@@ -29,8 +32,10 @@ import (
 	"github.com/openshift/machine-config-operator/lib/resourceread"
 	"github.com/openshift/machine-config-operator/manifests"
 	mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
+	ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
 	templatectrl "github.com/openshift/machine-config-operator/pkg/controller/template"
 	daemonconsts "github.com/openshift/machine-config-operator/pkg/daemon/constants"
+	"github.com/openshift/machine-config-operator/pkg/server"
 	"github.com/openshift/machine-config-operator/pkg/version"
 )
 
@@ -274,11 +279,48 @@ func (optr *Operator) syncRenderConfig(_ *renderConfig) error {
 		templatectrl.BaremetalRuntimeCfgKey: imgs.BaremetalRuntimeCfg,
 	}
 
+	ignitionHost, err := getIgnitionHost(&infra.Status)
+	if err != nil {
+		return err
+	}
+
+	pointerConfig, err := ctrlcommon.PointerConfig(ignitionHost, rootCA)
+	if err != nil {
+		return err
+	}
+	pointerConfigData, err := json.Marshal(pointerConfig)
+	if err != nil {
+		return err
+	}
+
 	// create renderConfig
-	optr.renderConfig = getRenderConfig(optr.namespace, string(kubeAPIServerServingCABytes), spec, &imgs.RenderConfigImages, infra.Status.APIServerInternalURL)
+	optr.renderConfig = getRenderConfig(optr.namespace, string(kubeAPIServerServingCABytes), spec, &imgs.RenderConfigImages, infra.Status.APIServerInternalURL, pointerConfigData)
 	return nil
 }
 
+func getIgnitionHost(infraStatus *configv1.InfrastructureStatus) (string, error) {
+	internalURL := infraStatus.APIServerInternalURL
+	internalURLParsed, err := url.Parse(internalURL)
+	if err != nil {
+		return "", err
+	}
+	securePortStr := strconv.Itoa(server.SecurePort)
+	ignitionHost := fmt.Sprintf("%s:%s", internalURLParsed.Hostname(), securePortStr)
+	switch infraStatus.PlatformStatus.Type {
+	case configv1.BareMetalPlatformType:
+		ignitionHost = net.JoinHostPort(infraStatus.PlatformStatus.BareMetal.APIServerInternalIP, securePortStr)
+	case configv1.OpenStackPlatformType:
+		ignitionHost = net.JoinHostPort(infraStatus.PlatformStatus.OpenStack.APIServerInternalIP, securePortStr)
+	case configv1.OvirtPlatformType:
+		ignitionHost = net.JoinHostPort(infraStatus.PlatformStatus.Ovirt.APIServerInternalIP, securePortStr)
+	case configv1.VSpherePlatformType:
+		if infraStatus.PlatformStatus.VSphere.APIServerInternalIP != "" {
+			ignitionHost = net.JoinHostPort(infraStatus.PlatformStatus.VSphere.APIServerInternalIP, securePortStr)
+		}
+	}
+	return ignitionHost, nil
+}
+
 func (optr *Operator) syncCustomResourceDefinitions() error {
 	crds := []string{
 		"manifests/controllerconfig.crd.yaml",
@@ -322,6 +364,39 @@ func (optr *Operator) syncMachineConfigPools(config *renderConfig) error {
 		}
 	}
 
+	userDataTemplatePath := "manifests/userdata_secret.yaml"
+	pools, err := optr.mcpLister.List(labels.Everything())
+	if err != nil {
+		return err
+	}
+	// base64.StdEncoding.EncodeToString
+	for _, pool := range pools {
+		pointerConfigAsset := newAssetRenderer("pointer-config")
+		pointerConfigAsset.templateData = config.PointerConfig
+		pointerConfigData, err := pointerConfigAsset.render(struct{ Role string }{pool.Name})
+		if err != nil {
+			return err
+		}
+
+		userDataAsset := newAssetRenderer(userDataTemplatePath)
+		if err := userDataAsset.read(); err != nil {
+			return err
+		}
+		userDataAsset.addTemplateFuncs()
+		userdataBytes, err := userDataAsset.render(struct{ Role, PointerConfig string }{
+			pool.Name,
+			base64.StdEncoding.EncodeToString(pointerConfigData),
+		})
+		if err != nil {
+			return err
+		}
+		p := resourceread.ReadSecretV1OrDie(userdataBytes)
+		_, _, err = resourceapply.ApplySecret(optr.kubeClient.CoreV1(), p)
+		if err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -840,14 +915,15 @@ func (optr *Operator) getGlobalConfig() (*configv1.Infrastructure, *configv1.Net
 	return infra, network, proxy, dns, nil
 }
 
-func getRenderConfig(tnamespace, kubeAPIServerServingCA string, ccSpec *mcfgv1.ControllerConfigSpec, imgs *RenderConfigImages, apiServerURL string) *renderConfig {
+func getRenderConfig(tnamespace, kubeAPIServerServingCA string, ccSpec *mcfgv1.ControllerConfigSpec, imgs *RenderConfigImages, apiServerURL string, pointerConfigData []byte) *renderConfig {
 	return &renderConfig{
 		TargetNamespace:        tnamespace,
 		Version:                version.Raw,
 		ControllerConfig:       *ccSpec,
 		Images:                 imgs,
 		APIServerURL:           apiServerURL,
 		KubeAPIServerServingCA: kubeAPIServerServingCA,
+		PointerConfig:          string(pointerConfigData),
 	}
 }