Merge pull request #3270 from enxebre/spike-opentonic-green

Spike opentonic green

config.tf

@@ -49,7 +49,7 @@ variable "tectonic_image_re" {
 EOF
 
   type    = "string"
-  default = "/^([^/]+/[^/]+/[^/]+):(.*)$/"
+  default = "/^([^/]+/[^/]+):(.*)$/"
 }
 
 variable "tectonic_container_images" {
@@ -61,20 +61,19 @@ variable "tectonic_container_images" {
     awscli                               = "quay.io/coreos/awscli:025a357f05242fdad6a81e8a6b520098aa65a600"
     gcloudsdk                            = "google/cloud-sdk:178.0.0-alpine"
     bootkube                             = "quay.io/coreos/bootkube:v0.10.0"
-    tnc_operator                         = "quay.io/coreos/tectonic-node-controller-operator-dev:ce870e6b3cc17f09a99d46150865e81c0997d224"
+    tnc_operator                         = "quay.io/coreos/tectonic-node-controller-operator-dev:8a9cde8fdb4baef4050b4770c837c151647bb99a"
     etcd_cert_signer                     = "quay.io/coreos/kube-etcd-signer-server:678cc8e6841e2121ebfdb6e2db568fce290b67d6"
     etcd                                 = "quay.io/coreos/etcd:v3.2.14"
-    hyperkube                            = "quay.io/coreos/hyperkube:v1.9.3_coreos.0"
-    kube_core_renderer                   = "quay.io/coreos/kube-core-renderer:beryllium-m2"
-    kube_core_operator                   = "quay.io/coreos/kube-core-operator:beryllium-m2"
-    tectonic_channel_operator            = "quay.io/coreos/tectonic-channel-operator:beryllium-m2"
-    tectonic_prometheus_operator         = "quay.io/coreos/tectonic-prometheus-operator:v1.9.3"
+    hyperkube                            = "openshift/origin-node:latest"
+    kube_core_renderer                   = "quay.io/coreos/kube-core-renderer-dev:8a9cde8fdb4baef4050b4770c837c151647bb99a"
+    kube_core_operator                   = "quay.io/coreos/kube-core-operator-dev:8a9cde8fdb4baef4050b4770c837c151647bb99a"
+    tectonic_channel_operator            = "quay.io/coreos/tectonic-channel-operator-dev:8a9cde8fdb4baef4050b4770c837c151647bb99a"
     tectonic_torcx                       = "quay.io/coreos/tectonic-torcx:v0.2.1"
-    kube_addon_operator                  = "quay.io/coreos/kube-addon-operator:beryllium-m2"
-    tectonic_alm_operator                = "quay.io/coreos/tectonic-alm-operator:v0.4.0"
-    tectonic_ingress_controller_operator = "quay.io/coreos/tectonic-ingress-controller-operator:d6b0848118e3b7c78d7d1728ee8846d5c6af2412"
-    tectonic_utility_operator            = "quay.io/coreos/tectonic-utility-operator:beryllium-m2"
-    tectonic_network_operator            = "quay.io/coreos/tectonic-network-operator:beryllium-m2"
+    kube_addon_operator                  = "quay.io/coreos/kube-addon-operator-dev:8a9cde8fdb4baef4050b4770c837c151647bb99a"
+    tectonic_alm_operator                = "quay.io/coreos/tectonic-alm-operator:v0.3.1"
+    tectonic_ingress_controller_operator = "quay.io/coreos/tectonic-ingress-controller-operator-dev:8a9cde8fdb4baef4050b4770c837c151647bb99a"
+    tectonic_utility_operator            = "quay.io/coreos/tectonic-utility-operator-dev:8a9cde8fdb4baef4050b4770c837c151647bb99a"
+    tectonic_network_operator            = "quay.io/coreos/tectonic-network-operator-dev:8a9cde8fdb4baef4050b4770c837c151647bb99a"
   }
 }
 
@@ -103,9 +102,8 @@ variable "tectonic_versions" {
   type        = "map"
 
   default = {
-    monitoring = "1.9.3"
-    tectonic   = "1.8.4-tectonic.2"
-    alm        = "0.4.0"
+    tectonic = "1.8.4-tectonic.2"
+    alm      = "0.4.0"
   }
 }
 

glide.yaml

@@ -11,7 +11,7 @@ import:
 - package: gopkg.in/alecthomas/kingpin.v2
   version: 947dcec5ba9c011838740e680966fd7087a71d0d
 - package: github.com/coreos/tectonic-config
-  version: cad3130928ece512f1145d09f95ffd5da1be92e6
+  version: 0d649ebfd3552dfa5c6cc2cf053e17ba924b7024
 - package: k8s.io/apimachinery
   version: kubernetes-1.9.0
 - package: golang.org/x/crypto

installer/pkg/config-generator/fixtures/kube-system.yaml

@@ -12,12 +12,16 @@ data:
       cloud_provider_profile: aws
     clusterConfig:
       apiserver_url: https://test-api.cluster.com:443
+    dnsConfig:
+      clusterIP: 10.3.0.10
     kind: KubeCoreOperatorConfig
     networkConfig:
       advertise_address: 0.0.0.0
       cluster_cidr: 10.2.0.0/16
       etcd_servers: https://test-etcd-0.cluster.com:2379,https://test-etcd-1.cluster.com:2379,https://test-etcd-2.cluster.com:2379
       service_cidr: 10.3.0.0/16
+    routingConfig:
+      subdomain: test.cluster.com
   network-config: |
     apiVersion: v1
     calicoConfig:

installer/pkg/config-generator/generator.go

@@ -16,7 +16,6 @@ import (
 	tnco "github.com/coreos/tectonic-config/config/tectonic-node-controller"
 	"github.com/coreos/tectonic-config/config/tectonic-utility"
 	"github.com/ghodss/yaml"
-	"golang.org/x/crypto/bcrypt"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/coreos/tectonic-installer/installer/pkg/config"
@@ -30,7 +29,7 @@ const (
 	identityConfigConsoleClientID = "tectonic-console"
 	identityConfigKubectlClientID = "tectonic-kubectl"
 	statsEmitterConfigStatsURL    = "https://stats-collector.tectonic.com"
-	ingressConfigIngressKind      = "NodePort"
+	ingressConfigIngressKind      = "haproxy-router"
 	certificatesStrategy          = "userProvidedCA"
 	identityAPIService            = "tectonic-identity-api.tectonic-system.svc.cluster.local"
 )
@@ -69,9 +68,13 @@ func (c *ConfigGenerator) KubeSystem() (string, error) {
 	if err != nil {
 		return "", err
 	}
+	coreConfig, err := c.coreConfig()
+	if err != nil {
+		return "", err
+	}
 
 	return configMap("kube-system", genericData{
-		"kco-config":     c.coreConfig(),
+		"kco-config":     coreConfig,
 		"network-config": c.networkConfig(),
 		"tnco-config":    tncoConfig,
 	})
@@ -95,7 +98,11 @@ func (c *ConfigGenerator) TectonicSystem() (string, error) {
 
 // CoreConfig returns, if successful, a yaml string for the on-disk kco-config.
 func (c *ConfigGenerator) CoreConfig() (string, error) {
-	return marshalYAML(c.coreConfig())
+	coreConfig, err := c.coreConfig()
+	if err != nil {
+		return "", err
+	}
+	return marshalYAML(coreConfig)
 }
 
 // TncoConfig returns, if successful, a yaml string for the on-disk tnco-config.
@@ -114,16 +121,17 @@ func (c *ConfigGenerator) addonConfig() (*kubeaddon.OperatorConfig, error) {
 			Kind:       kubeaddon.Kind,
 		},
 	}
-	cidrhost, err := cidrhost(c.Cluster.Networking.ServiceCIDR, 10)
+	addonConfig.CloudProvider = c.Platform.String()
+	addonConfig.ClusterConfig.APIServerURL = c.getAPIServerURL()
+	registrySecret, err := generateRandomID(16)
 	if err != nil {
 		return nil, err
 	}
-	addonConfig.DNSConfig.ClusterIP = cidrhost
-	addonConfig.CloudProvider = cloudProvider(c.Platform)
+	addonConfig.RegistryHTTPSecret = registrySecret
 	return &addonConfig, nil
 }
 
-func (c *ConfigGenerator) coreConfig() *kubecore.OperatorConfig {
+func (c *ConfigGenerator) coreConfig() (*kubecore.OperatorConfig, error) {
 	coreConfig := kubecore.OperatorConfig{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: kubecore.APIVersion,
@@ -136,15 +144,23 @@ func (c *ConfigGenerator) coreConfig() *kubecore.OperatorConfig {
 	coreConfig.AuthConfig.OIDCGroupsClaim = authConfigOIDCGroupsClaim
 	coreConfig.AuthConfig.OIDCUsernameClaim = authConfigOIDCUsernameClaim
 
+	cidrhost, err := cidrhost(c.Cluster.Networking.ServiceCIDR, 10)
+	if err != nil {
+		return nil, err
+	}
+	coreConfig.DNSConfig.ClusterIP = cidrhost
+
 	coreConfig.CloudProviderConfig.CloudConfigPath = ""
 	coreConfig.CloudProviderConfig.CloudProviderProfile = cloudProvider(c.Cluster.Platform)
 
+	coreConfig.RoutingConfig.Subdomain = c.getBaseAddress()
+
 	coreConfig.NetworkConfig.ClusterCIDR = c.Cluster.Networking.PodCIDR
 	coreConfig.NetworkConfig.ServiceCIDR = c.Cluster.Networking.ServiceCIDR
 	coreConfig.NetworkConfig.AdvertiseAddress = networkConfigAdvertiseAddress
 	coreConfig.NetworkConfig.EtcdServers = c.getEtcdServersURLs()
 
-	return &coreConfig
+	return &coreConfig, nil
 }
 
 func (c *ConfigGenerator) networkConfig() *tectonicnetwork.OperatorConfig {
@@ -183,9 +199,9 @@ func (c *ConfigGenerator) tncoConfig() (*tnco.OperatorConfig, error) {
 	}
 
 	tncoConfig.ControllerConfig.ClusterDNSIP = cidrhost
+	tncoConfig.ControllerConfig.Platform = c.Platform.String()
 	tncoConfig.ControllerConfig.CloudProviderConfig = "" // TODO(yifan): Get CloudProviderConfig.
 	tncoConfig.ControllerConfig.ClusterName = c.Cluster.Name
-	tncoConfig.ControllerConfig.Platform = string(c.Cluster.Platform)
 	tncoConfig.ControllerConfig.BaseDomain = c.Cluster.BaseDomain
 	tncoConfig.ControllerConfig.EtcdInitialCount = c.Cluster.NodeCount(c.Cluster.Etcd.NodePools)
 	tncoConfig.ControllerConfig.AdditionalConfigs = []string{} // TODO(yifan): Get additional configs.
@@ -202,46 +218,11 @@ func (c *ConfigGenerator) utilityConfig() (*tectonicutility.OperatorConfig, erro
 		},
 	}
 
-	var err error
-	bytes, err := bcrypt.GenerateFromPassword([]byte(c.Admin.Password), 12)
-	if err != nil {
-		return nil, err
-	}
-	hashedAdminPassword := string(bytes)
-	adminUserID, err := generateRandomID(16)
-	if err != nil {
-		return nil, err
-	}
-	consoleSecret, err := generateRandomID(16)
-	if err != nil {
-		return nil, err
-	}
-	KubectlSecret, err := generateRandomID(16)
-	if err != nil {
-		return nil, err
-	}
-
-	if err != nil {
-		return nil, err
-	}
-	utilityConfig.IdentityConfig.AdminEmail = c.Admin.Email
-	utilityConfig.IdentityConfig.AdminPasswordHash = hashedAdminPassword
-	utilityConfig.IdentityConfig.AdminUserID = adminUserID
-	utilityConfig.IdentityConfig.ConsoleClientID = identityConfigConsoleClientID
-	utilityConfig.IdentityConfig.ConsoleSecret = consoleSecret
-	utilityConfig.IdentityConfig.KubectlClientID = identityConfigKubectlClientID
-	utilityConfig.IdentityConfig.KubectlSecret = KubectlSecret
-
-	utilityConfig.IngressConfig.ConsoleBaseHost = c.getBaseAddress()
-	utilityConfig.IngressConfig.IngressKind = ingressConfigIngressKind
-
 	utilityConfig.StatsEmitterConfig.StatsURL = statsEmitterConfigStatsURL
 
-	utilityConfig.TectonicConfigMapConfig.BaseAddress = c.getBaseAddress()
 	utilityConfig.TectonicConfigMapConfig.CertificatesStrategy = certificatesStrategy
 	utilityConfig.TectonicConfigMapConfig.ClusterID = c.Cluster.Internal.ClusterID
 	utilityConfig.TectonicConfigMapConfig.ClusterName = c.Cluster.Name
-	utilityConfig.TectonicConfigMapConfig.IdentityAPIService = identityAPIService
 	utilityConfig.TectonicConfigMapConfig.InstallerPlatform = c.Platform.String()
 	utilityConfig.TectonicConfigMapConfig.KubeAPIServerURL = c.getAPIServerURL()
 	// TODO: Speficy what's a version in ut2 and set it here

modules/aws/vpc/sg-master.tf

@@ -65,7 +65,7 @@ resource "aws_security_group_rule" "master_ingress_https" {
   protocol    = "tcp"
   cidr_blocks = ["0.0.0.0/0"]
   from_port   = 6443
-  to_port     = 6443
+  to_port     = 6445
 }
 
 resource "aws_security_group_rule" "master_ingress_heapster" {

modules/bootkube/manifests.tf

@@ -2,13 +2,15 @@ variable "manifest_names" {
   default = [
     "01-tectonic-namespace.yaml",
     "02-ingress-namespace.yaml",
+    "03-openshift-web-console-namespace.yaml",
     "app-version-kind.yaml",
     "app-version-tectonic-network.yaml",
     "app-version-tnc.yaml",
     "kube-apiserver-secret.yaml",
     "kube-cloud-config.yaml",
     "kube-controller-manager-secret.yaml",
     "node-config-kind.yaml",
+    "openshift-apiserver-secret.yaml",
     "pull.json",
     "tectonic-network-operator.yaml",
     "tectonic-node-controller-operator.yaml",
@@ -26,19 +28,26 @@ data "template_file" "manifest_file_list" {
 
     cloud_provider_config = "${var.cloud_provider_config}"
 
-    root_ca_cert         = "${base64encode(var.root_ca_cert_pem)}"
-    aggregator_ca_cert   = "${base64encode(var.aggregator_ca_cert_pem)}"
-    kube_ca_cert         = "${base64encode(var.kube_ca_cert_pem)}"
-    kube_ca_key          = "${base64encode(var.kube_ca_key_pem)}"
-    apiserver_key        = "${base64encode(var.apiserver_key_pem)}"
-    apiserver_cert       = "${base64encode(var.apiserver_cert_pem)}"
-    apiserver_proxy_key  = "${base64encode(var.apiserver_proxy_key_pem)}"
-    apiserver_proxy_cert = "${base64encode(var.apiserver_proxy_cert_pem)}"
-    oidc_ca_cert         = "${base64encode(var.oidc_ca_cert)}"
-    pull_secret          = "${base64encode(file(var.pull_secret_path))}"
-    serviceaccount_pub   = "${base64encode(tls_private_key.service_account.public_key_pem)}"
-    serviceaccount_key   = "${base64encode(tls_private_key.service_account.private_key_pem)}"
-    kube_dns_service_ip  = "${cidrhost(var.service_cidr, 10)}"
+    root_ca_cert             = "${base64encode(var.root_ca_cert_pem)}"
+    aggregator_ca_cert       = "${base64encode(var.aggregator_ca_cert_pem)}"
+    aggregator_ca_key        = "${base64encode(var.aggregator_ca_key_pem)}"
+    kube_ca_cert             = "${base64encode(var.kube_ca_cert_pem)}"
+    kube_ca_key              = "${base64encode(var.kube_ca_key_pem)}"
+    service_serving_ca_cert  = "${base64encode(var.service_serving_ca_cert_pem)}"
+    service_serving_ca_key   = "${base64encode(var.service_serving_ca_key_pem)}"
+    apiserver_key            = "${base64encode(var.apiserver_key_pem)}"
+    apiserver_cert           = "${base64encode(var.apiserver_cert_pem)}"
+    openshift_apiserver_key  = "${base64encode(var.openshift_apiserver_key_pem)}"
+    openshift_apiserver_cert = "${base64encode(var.openshift_apiserver_cert_pem)}"
+    apiserver_proxy_key      = "${base64encode(var.apiserver_proxy_key_pem)}"
+    apiserver_proxy_cert     = "${base64encode(var.apiserver_proxy_cert_pem)}"
+    oidc_ca_cert             = "${base64encode(var.oidc_ca_cert)}"
+    pull_secret              = "${base64encode(file(var.pull_secret_path))}"
+    serviceaccount_pub       = "${base64encode(tls_private_key.service_account.public_key_pem)}"
+    serviceaccount_key       = "${base64encode(tls_private_key.service_account.private_key_pem)}"
+    kube_dns_service_ip      = "${cidrhost(var.service_cidr, 10)}"
+
+    openshift_loopback_kubeconfig = "${base64encode(data.template_file.kubeconfig.rendered)}"
 
     etcd_ca_cert     = "${base64encode(var.etcd_ca_cert_pem)}"
     etcd_client_cert = "${base64encode(var.etcd_client_cert_pem)}"

modules/bootkube/resources/bootkube.sh

@@ -58,9 +58,10 @@ export ETCDCTL_API=
 
 # shellcheck disable=SC2154
 /usr/bin/docker kill $${signer_id}
-
 rm /etc/kubernetes/manifests/tectonic-node-controller-pod.yaml
 
+cp -r $(pwd)/bootstrap-configs /etc/kubernetes/bootstrap-configs
+
 # shellcheck disable=SC2154
 /usr/bin/docker run \
     --volume "$(pwd)":/assets \

modules/bootkube/resources/manifests/01-tectonic-namespace.yaml

@@ -4,3 +4,4 @@ metadata:
   name: tectonic-system # Create the namespace first.
   labels: # network policy can only select by labels
     name: tectonic-system
+    openshift.io/run-level: "1"

modules/bootkube/resources/manifests/02-ingress-namespace.yaml

@@ -7,3 +7,4 @@ metadata:
   labels:
     kubernetes.io/ingress.class: tectonic
     name: tectonic-ingress
+    openshift.io/run-level: "1"

modules/bootkube/resources/manifests/03-openshift-web-console-namespace.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  # This is the namespace used to hold the openshift console.
+  # They require openshift console run in this namespace.
+  name: openshift-web-console
+  labels:
+    name: openshift-web-console

modules/bootkube/resources/manifests/app-version-tectonic-network.yaml

@@ -2,7 +2,7 @@ apiVersion: tco.coreos.com/v1
 kind: AppVersion
 metadata:
   name: tectonic-network
-  namespace: tectonic-system
+  namespace: kube-system
   labels:
     managed-by-channel-operator: "true"
 spec:

modules/bootkube/resources/manifests/kube-apiserver-secret.yaml

@@ -6,14 +6,19 @@ metadata:
 type: Opaque
 data:
   aggregator-ca.crt: ${aggregator_ca_cert}
+  aggregator-ca.key: ${aggregator_ca_key}
   apiserver.key: ${apiserver_key}
   apiserver.crt: ${apiserver_cert}
   apiserver-proxy.key: ${apiserver_proxy_key}
   apiserver-proxy.crt: ${apiserver_proxy_cert}
   service-account.pub: ${serviceaccount_pub}
+  service-account.key: ${serviceaccount_key}
   root-ca.crt: ${root_ca_cert}
   kube-ca.crt: ${kube_ca_cert}
   etcd-client-ca.crt: ${etcd_ca_cert}
   etcd-client.crt: ${etcd_client_cert}
   etcd-client.key: ${etcd_client_key}
   oidc-ca.crt: ${oidc_ca_cert}
+  service-serving-ca.crt: ${service_serving_ca_cert}
+  service-serving-ca.key: ${service_serving_ca_key}
+  kubeconfig: ${openshift_loopback_kubeconfig}