diff --git a/manifests/ovirt/coredns-corefile.tmpl b/manifests/ovirt/coredns-corefile.tmpl new file mode 100644 index 0000000000..e2e2758ca7 --- /dev/null +++ b/manifests/ovirt/coredns-corefile.tmpl @@ -0,0 +1,12 @@ +. { + errors + health :18080 + mdns {{ .ControllerConfig.EtcdDiscoveryDomain }} {{`{{.Cluster.MasterAmount}}`}} {{`{{.Cluster.Name}}`}} + forward . {{`{{- range $upstream := .DNSUpstreams}} {{$upstream}}{{- end}}`}} + cache 30 + reload + hosts /etc/coredns/api-int.hosts {{ .ControllerConfig.EtcdDiscoveryDomain }} { + {{ .ControllerConfig.Infra.Status.PlatformStatus.Ovirt.APIServerInternalIP }} api-int.{{ .ControllerConfig.EtcdDiscoveryDomain }} api.{{ .ControllerConfig.EtcdDiscoveryDomain }} + fallthrough + } +} diff --git a/manifests/ovirt/coredns.yaml b/manifests/ovirt/coredns.yaml new file mode 100644 index 0000000000..2d904714e5 --- /dev/null +++ b/manifests/ovirt/coredns.yaml @@ -0,0 +1,92 @@ +--- +kind: Pod +apiVersion: v1 +metadata: + name: coredns + namespace: openshift-kni-infra + creationTimestamp: + deletionGracePeriodSeconds: 65 + labels: + app: kni-infra-mdns +spec: + volumes: + - name: resource-dir + hostPath: + path: "/etc/kubernetes/static-pod-resources/coredns" + - name: kubeconfig + hostPath: + path: "/etc/kubernetes/kubeconfig" + - name: conf-dir + empty-dir: {} + - name: manifests + hostPath: + path: "/opt/openshift/manifests" + initContainers: + - name: render-config + image: {{ .Images.BaremetalRuntimeCfgBootstrap }} + command: + - runtimecfg + - render + - "/etc/kubernetes/kubeconfig" + - "--api-vip" + - "{{ .ControllerConfig.Infra.Status.PlatformStatus.Ovirt.APIServerInternalIP }}" + - "--dns-vip" + - "{{ .ControllerConfig.Infra.Status.PlatformStatus.Ovirt.NodeDNSIP }}" + - "--ingress-vip" + - "{{ .ControllerConfig.Infra.Status.PlatformStatus.Ovirt.IngressIP }}" + - "/config" + - "--out-dir" + - "/etc/coredns" + - "--cluster-config" + - "/opt/openshift/manifests/cluster-config.yaml" + resources: {} + volumeMounts: + - name: kubeconfig + mountPath: "/etc/kubernetes/kubeconfig" + - name: resource-dir + mountPath: "/config" + - name: conf-dir + mountPath: "/etc/coredns" + - name: manifests + mountPath: "/opt/openshift/manifests" + imagePullPolicy: IfNotPresent + containers: + - name: coredns + securityContext: + privileged: true + image: {{ .Images.CorednsBootstrap }} + args: + - "--conf" + - "/etc/coredns/Corefile" + resources: + requests: + cpu: 150m + memory: 1Gi + volumeMounts: + - name: conf-dir + mountPath: "/etc/coredns" + readinessProbe: + httpGet: + path: /health + port: 18080 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 10 + livenessProbe: + httpGet: + path: /health + port: 18080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + terminationMessagePolicy: FallbackToLogsOnError + hostNetwork: true + tolerations: + - operator: Exists + priorityClassName: system-node-critical +status: {} diff --git a/manifests/ovirt/keepalived.conf.tmpl b/manifests/ovirt/keepalived.conf.tmpl new file mode 100644 index 0000000000..7ae696c17d --- /dev/null +++ b/manifests/ovirt/keepalived.conf.tmpl @@ -0,0 +1,35 @@ +# Configuration template for Keepalived, which is used to manage the DNS and +# API VIPs. +# +# For more information, see installer/data/data/bootstrap/baremetal/README.md +# in the installer repo. + +vrrp_instance {{`{{.Cluster.Name}}`}}_API { + state BACKUP + interface {{`{{.VRRPInterface}}`}} + virtual_router_id {{`{{.Cluster.APIVirtualRouterID }}`}} + priority 50 + advert_int 1 + authentication { + auth_type PASS + auth_pass {{`{{.Cluster.Name}}`}}_api_vip + } + virtual_ipaddress { + {{`{{ .Cluster.APIVIP }}`}}/{{`{{ .Cluster.VIPNetmask }}`}} + } +} + +vrrp_instance {{`{{.Cluster.Name}}`}}_DNS { + state MASTER + interface {{`{{.VRRPInterface}}`}} + virtual_router_id {{`{{.Cluster.DNSVirtualRouterID }}`}} + priority 140 + advert_int 1 + authentication { + auth_type PASS + auth_pass {{`{{.Cluster.Name}}`}}_dns_vip + } + virtual_ipaddress { + {{`{{ .Cluster.DNSVIP }}`}}/{{`{{ .Cluster.VIPNetmask }}`}} + } +} diff --git a/manifests/ovirt/keepalived.yaml b/manifests/ovirt/keepalived.yaml new file mode 100644 index 0000000000..5479a5ae8c --- /dev/null +++ b/manifests/ovirt/keepalived.yaml @@ -0,0 +1,83 @@ +--- +kind: Pod +apiVersion: v1 +metadata: + name: keepalived + namespace: openshift-kni-infra + creationTimestamp: + deletionGracePeriodSeconds: 65 + labels: + app: kni-infra-vrrp +spec: + volumes: + - name: resource-dir + hostPath: + path: "/etc/kubernetes/static-pod-resources/keepalived" + - name: kubeconfig + hostPath: + path: "/etc/kubernetes/kubeconfig" + - name: conf-dir + empty-dir: {} + - name: manifests + hostPath: + path: "/opt/openshift/manifests" + initContainers: + - name: render-config + image: {{ .Images.BaremetalRuntimeCfgBootstrap }} + command: + - runtimecfg + - render + - "/etc/kubernetes/kubeconfig" + - "--api-vip" + - "{{ .ControllerConfig.Infra.Status.PlatformStatus.Ovirt.APIServerInternalIP }}" + - "--dns-vip" + - "{{ .ControllerConfig.Infra.Status.PlatformStatus.Ovirt.NodeDNSIP }}" + - "--ingress-vip" + - "{{ .ControllerConfig.Infra.Status.PlatformStatus.Ovirt.IngressIP }}" + - "/config" + - "--out-dir" + - "/etc/keepalived" + - "--cluster-config" + - "/opt/openshift/manifests/cluster-config.yaml" + resources: {} + volumeMounts: + - name: resource-dir + mountPath: "/config" + - name: kubeconfig + mountPath: "/etc/kubernetes/kubeconfig" + - name: conf-dir + mountPath: "/etc/keepalived" + - name: manifests + mountPath: "/opt/openshift/manifests" + imagePullPolicy: IfNotPresent + containers: + - name: keepalived + securityContext: + privileged: true + image: {{ .Images.KeepalivedBootstrap }} + env: + - name: NSS_SDB_USE_CACHE + value: "no" + command: + - /usr/sbin/keepalived + args: + - "-f" + - "/etc/keepalived/keepalived.conf" + - "--dont-fork" + - "--vrrp" + - "--log-detail" + - "--log-console" + resources: + requests: + cpu: 150m + memory: 1Gi + volumeMounts: + - name: conf-dir + mountPath: "/etc/keepalived" + terminationMessagePolicy: FallbackToLogsOnError + imagePullPolicy: IfNotPresent + hostNetwork: true + tolerations: + - operator: Exists + priorityClassName: system-node-critical +status: {} diff --git a/pkg/controller/template/render.go b/pkg/controller/template/render.go index 8980cf170a..b8a33bca13 100644 --- a/pkg/controller/template/render.go +++ b/pkg/controller/template/render.go @@ -43,6 +43,7 @@ const ( platformNone = "none" platformVSphere = "vsphere" platformBase = "_base" + platformOvirt = "ovirt" ) // generateTemplateMachineConfigs returns MachineConfig objects from the templateDir and a config object @@ -128,7 +129,7 @@ func platformFromControllerConfigSpec(ic *mcfgv1.ControllerConfigSpec) (string, return "", fmt.Errorf("cannot generate MachineConfigs when no platform is set") case platformBase: return "", fmt.Errorf("platform _base unsupported") - case platformAWS, platformAzure, platformBaremetal, platformGCP, platformOpenStack, platformLibvirt, platformNone: + case platformAWS, platformAzure, platformBaremetal, platformGCP, platformOpenStack, platformLibvirt, platformOvirt, platformNone: return ic.Platform, nil default: // platformNone is used for a non-empty, but currently unsupported platform. diff --git a/pkg/controller/template/test_data/controller_config_ovirt.yaml b/pkg/controller/template/test_data/controller_config_ovirt.yaml new file mode 100644 index 0000000000..5e634fa952 --- /dev/null +++ b/pkg/controller/template/test_data/controller_config_ovirt.yaml @@ -0,0 +1,21 @@ +apiVersion: "machineconfigurations.openshift.io/v1" +kind: "ControllerConfig" +spec: + clusterDNSIP: "10.3.0.10" + cloudProviderConfig: | + testing + multi-line cloud config + [test] + option = dummy + etcdDiscoveryDomain: "my-test-cluster.example.org" + etcdInitialCount: 3 + platform: "ovirt" + etcdCAData: OHVtbXkgZXRjZC1jYQo= + rootCAData: OHVtbXkgcm9vdC1jYQo= + pullSecret: + data: OHVtbXkgZXRjZC1jYQo= + images: + etcd: image/etcd:1 + setupEtcdEnv: image/setupEtcdEnv:1 + infraImage: image/infraImage:1 + kubeClientAgentImage: image/kubeClientAgentImage:1 diff --git a/pkg/operator/assets/bindata.go b/pkg/operator/assets/bindata.go index f343dadfaf..0784d8932a 100644 --- a/pkg/operator/assets/bindata.go +++ b/pkg/operator/assets/bindata.go @@ -37,6 +37,10 @@ // manifests/openstack/coredns.yaml // manifests/openstack/keepalived.conf.tmpl // manifests/openstack/keepalived.yaml +// manifests/ovirt/coredns-corefile.tmpl +// manifests/ovirt/coredns.yaml +// manifests/ovirt/keepalived.conf.tmpl +// manifests/ovirt/keepalived.yaml // manifests/worker.machineconfigpool.yaml package assets @@ -2011,6 +2015,296 @@ func manifestsOpenstackKeepalivedYaml() (*asset, error) { return a, nil } +var _manifestsOvirtCorednsCorefileTmpl = []byte(`. { + errors + health :18080 + mdns {{ .ControllerConfig.EtcdDiscoveryDomain }} {{`+"`"+`{{.Cluster.MasterAmount}}`+"`"+`}} {{`+"`"+`{{.Cluster.Name}}`+"`"+`}} + forward . {{`+"`"+`{{- range $upstream := .DNSUpstreams}} {{$upstream}}{{- end}}`+"`"+`}} + cache 30 + reload + hosts /etc/coredns/api-int.hosts {{ .ControllerConfig.EtcdDiscoveryDomain }} { + {{ .ControllerConfig.Infra.Status.PlatformStatus.Ovirt.APIServerInternalIP }} api-int.{{ .ControllerConfig.EtcdDiscoveryDomain }} api.{{ .ControllerConfig.EtcdDiscoveryDomain }} + fallthrough + } +} +`) + +func manifestsOvirtCorednsCorefileTmplBytes() ([]byte, error) { + return _manifestsOvirtCorednsCorefileTmpl, nil +} + +func manifestsOvirtCorednsCorefileTmpl() (*asset, error) { + bytes, err := manifestsOvirtCorednsCorefileTmplBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "manifests/ovirt/coredns-corefile.tmpl", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _manifestsOvirtCorednsYaml = []byte(`--- +kind: Pod +apiVersion: v1 +metadata: + name: coredns + namespace: openshift-kni-infra + creationTimestamp: + deletionGracePeriodSeconds: 65 + labels: + app: kni-infra-mdns +spec: + volumes: + - name: resource-dir + hostPath: + path: "/etc/kubernetes/static-pod-resources/coredns" + - name: kubeconfig + hostPath: + path: "/etc/kubernetes/kubeconfig" + - name: conf-dir + empty-dir: {} + - name: manifests + hostPath: + path: "/opt/openshift/manifests" + initContainers: + - name: render-config + image: {{ .Images.BaremetalRuntimeCfgBootstrap }} + command: + - runtimecfg + - render + - "/etc/kubernetes/kubeconfig" + - "--api-vip" + - "{{ .ControllerConfig.Infra.Status.PlatformStatus.Ovirt.APIServerInternalIP }}" + - "--dns-vip" + - "{{ .ControllerConfig.Infra.Status.PlatformStatus.Ovirt.NodeDNSIP }}" + - "--ingress-vip" + - "{{ .ControllerConfig.Infra.Status.PlatformStatus.Ovirt.IngressIP }}" + - "/config" + - "--out-dir" + - "/etc/coredns" + - "--cluster-config" + - "/opt/openshift/manifests/cluster-config.yaml" + resources: {} + volumeMounts: + - name: kubeconfig + mountPath: "/etc/kubernetes/kubeconfig" + - name: resource-dir + mountPath: "/config" + - name: conf-dir + mountPath: "/etc/coredns" + - name: manifests + mountPath: "/opt/openshift/manifests" + imagePullPolicy: IfNotPresent + containers: + - name: coredns + securityContext: + privileged: true + image: {{ .Images.CorednsBootstrap }} + args: + - "--conf" + - "/etc/coredns/Corefile" + resources: + requests: + cpu: 150m + memory: 1Gi + volumeMounts: + - name: conf-dir + mountPath: "/etc/coredns" + readinessProbe: + httpGet: + path: /health + port: 18080 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 10 + livenessProbe: + httpGet: + path: /health + port: 18080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + terminationMessagePolicy: FallbackToLogsOnError + hostNetwork: true + tolerations: + - operator: Exists + priorityClassName: system-node-critical +status: {} +`) + +func manifestsOvirtCorednsYamlBytes() ([]byte, error) { + return _manifestsOvirtCorednsYaml, nil +} + +func manifestsOvirtCorednsYaml() (*asset, error) { + bytes, err := manifestsOvirtCorednsYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "manifests/ovirt/coredns.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _manifestsOvirtKeepalivedConfTmpl = []byte(`# Configuration template for Keepalived, which is used to manage the DNS and +# API VIPs. +# +# For more information, see installer/data/data/bootstrap/baremetal/README.md +# in the installer repo. + +vrrp_instance {{`+"`"+`{{.Cluster.Name}}`+"`"+`}}_API { + state BACKUP + interface {{`+"`"+`{{.VRRPInterface}}`+"`"+`}} + virtual_router_id {{`+"`"+`{{.Cluster.APIVirtualRouterID }}`+"`"+`}} + priority 50 + advert_int 1 + authentication { + auth_type PASS + auth_pass {{`+"`"+`{{.Cluster.Name}}`+"`"+`}}_api_vip + } + virtual_ipaddress { + {{`+"`"+`{{ .Cluster.APIVIP }}`+"`"+`}}/{{`+"`"+`{{ .Cluster.VIPNetmask }}`+"`"+`}} + } +} + +vrrp_instance {{`+"`"+`{{.Cluster.Name}}`+"`"+`}}_DNS { + state MASTER + interface {{`+"`"+`{{.VRRPInterface}}`+"`"+`}} + virtual_router_id {{`+"`"+`{{.Cluster.DNSVirtualRouterID }}`+"`"+`}} + priority 140 + advert_int 1 + authentication { + auth_type PASS + auth_pass {{`+"`"+`{{.Cluster.Name}}`+"`"+`}}_dns_vip + } + virtual_ipaddress { + {{`+"`"+`{{ .Cluster.DNSVIP }}`+"`"+`}}/{{`+"`"+`{{ .Cluster.VIPNetmask }}`+"`"+`}} + } +} +`) + +func manifestsOvirtKeepalivedConfTmplBytes() ([]byte, error) { + return _manifestsOvirtKeepalivedConfTmpl, nil +} + +func manifestsOvirtKeepalivedConfTmpl() (*asset, error) { + bytes, err := manifestsOvirtKeepalivedConfTmplBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "manifests/ovirt/keepalived.conf.tmpl", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _manifestsOvirtKeepalivedYaml = []byte(`--- +kind: Pod +apiVersion: v1 +metadata: + name: keepalived + namespace: openshift-kni-infra + creationTimestamp: + deletionGracePeriodSeconds: 65 + labels: + app: kni-infra-vrrp +spec: + volumes: + - name: resource-dir + hostPath: + path: "/etc/kubernetes/static-pod-resources/keepalived" + - name: kubeconfig + hostPath: + path: "/etc/kubernetes/kubeconfig" + - name: conf-dir + empty-dir: {} + - name: manifests + hostPath: + path: "/opt/openshift/manifests" + initContainers: + - name: render-config + image: {{ .Images.BaremetalRuntimeCfgBootstrap }} + command: + - runtimecfg + - render + - "/etc/kubernetes/kubeconfig" + - "--api-vip" + - "{{ .ControllerConfig.Infra.Status.PlatformStatus.Ovirt.APIServerInternalIP }}" + - "--dns-vip" + - "{{ .ControllerConfig.Infra.Status.PlatformStatus.Ovirt.NodeDNSIP }}" + - "--ingress-vip" + - "{{ .ControllerConfig.Infra.Status.PlatformStatus.Ovirt.IngressIP }}" + - "/config" + - "--out-dir" + - "/etc/keepalived" + - "--cluster-config" + - "/opt/openshift/manifests/cluster-config.yaml" + resources: {} + volumeMounts: + - name: resource-dir + mountPath: "/config" + - name: kubeconfig + mountPath: "/etc/kubernetes/kubeconfig" + - name: conf-dir + mountPath: "/etc/keepalived" + - name: manifests + mountPath: "/opt/openshift/manifests" + imagePullPolicy: IfNotPresent + containers: + - name: keepalived + securityContext: + privileged: true + image: {{ .Images.KeepalivedBootstrap }} + env: + - name: NSS_SDB_USE_CACHE + value: "no" + command: + - /usr/sbin/keepalived + args: + - "-f" + - "/etc/keepalived/keepalived.conf" + - "--dont-fork" + - "--vrrp" + - "--log-detail" + - "--log-console" + resources: + requests: + cpu: 150m + memory: 1Gi + volumeMounts: + - name: conf-dir + mountPath: "/etc/keepalived" + terminationMessagePolicy: FallbackToLogsOnError + imagePullPolicy: IfNotPresent + hostNetwork: true + tolerations: + - operator: Exists + priorityClassName: system-node-critical +status: {} +`) + +func manifestsOvirtKeepalivedYamlBytes() ([]byte, error) { + return _manifestsOvirtKeepalivedYaml, nil +} + +func manifestsOvirtKeepalivedYaml() (*asset, error) { + bytes, err := manifestsOvirtKeepalivedYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "manifests/ovirt/keepalived.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + var _manifestsWorkerMachineconfigpoolYaml = []byte(`apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfigPool metadata: @@ -2129,6 +2423,10 @@ var _bindata = map[string]func() (*asset, error){ "manifests/openstack/coredns.yaml": manifestsOpenstackCorednsYaml, "manifests/openstack/keepalived.conf.tmpl": manifestsOpenstackKeepalivedConfTmpl, "manifests/openstack/keepalived.yaml": manifestsOpenstackKeepalivedYaml, + "manifests/ovirt/coredns-corefile.tmpl": manifestsOvirtCorednsCorefileTmpl, + "manifests/ovirt/coredns.yaml": manifestsOvirtCorednsYaml, + "manifests/ovirt/keepalived.conf.tmpl": manifestsOvirtKeepalivedConfTmpl, + "manifests/ovirt/keepalived.yaml": manifestsOvirtKeepalivedYaml, "manifests/worker.machineconfigpool.yaml": manifestsWorkerMachineconfigpoolYaml, } @@ -2221,6 +2519,12 @@ var _bintree = &bintree{nil, map[string]*bintree{ "keepalived.conf.tmpl": &bintree{manifestsOpenstackKeepalivedConfTmpl, map[string]*bintree{}}, "keepalived.yaml": &bintree{manifestsOpenstackKeepalivedYaml, map[string]*bintree{}}, }}, + "ovirt": &bintree{nil, map[string]*bintree{ + "coredns-corefile.tmpl": &bintree{manifestsOvirtCorednsCorefileTmpl, map[string]*bintree{}}, + "coredns.yaml": &bintree{manifestsOvirtCorednsYaml, map[string]*bintree{}}, + "keepalived.conf.tmpl": &bintree{manifestsOvirtKeepalivedConfTmpl, map[string]*bintree{}}, + "keepalived.yaml": &bintree{manifestsOvirtKeepalivedYaml, map[string]*bintree{}}, + }}, "worker.machineconfigpool.yaml": &bintree{manifestsWorkerMachineconfigpoolYaml, map[string]*bintree{}}, }}, }} diff --git a/pkg/operator/bootstrap.go b/pkg/operator/bootstrap.go index 1a413cc0a9..24553655a5 100644 --- a/pkg/operator/bootstrap.go +++ b/pkg/operator/bootstrap.go @@ -17,6 +17,12 @@ import ( templatectrl "github.com/openshift/machine-config-operator/pkg/controller/template" ) +type manifest struct { + name string + data []byte + filename string +} + // RenderBootstrap writes to destinationDir static Pods. func RenderBootstrap( additionalTrustBundleFile, @@ -144,72 +150,31 @@ func RenderBootstrap( config := getRenderConfig("", string(filesData[kubeAPIServerServingCA]), spec, &imgs.RenderConfigImages, infra.Status.APIServerInternalURL) - manifests := []struct { - name string - data []byte - filename string - }{{ - name: "manifests/machineconfigcontroller/controllerconfig.yaml", - filename: "bootstrap/manifests/machineconfigcontroller-controllerconfig.yaml", - }, { - name: "manifests/master.machineconfigpool.yaml", - filename: "bootstrap/manifests/master.machineconfigpool.yaml", - }, { - name: "manifests/worker.machineconfigpool.yaml", - filename: "bootstrap/manifests/worker.machineconfigpool.yaml", - }, { - name: "manifests/bootstrap-pod-v2.yaml", - filename: "bootstrap/machineconfigoperator-bootstrap-pod.yaml", - }, { - data: filesData[pullSecretFile], - filename: "bootstrap/manifests/machineconfigcontroller-pull-secret", - }, { - name: "manifests/machineconfigserver/csr-bootstrap-role-binding.yaml", - filename: "manifests/csr-bootstrap-role-binding.yaml", - }, { - name: "manifests/machineconfigserver/kube-apiserver-serving-ca-configmap.yaml", - filename: "manifests/kube-apiserver-serving-ca-configmap.yaml", - }} - - if infra.Status.PlatformStatus.BareMetal != nil { - manifests = append(manifests, []struct { - name string - data []byte - filename string - }{{ - name: "manifests/baremetal/coredns.yaml", - filename: "baremetal/manifests/coredns.yaml", + manifests := []manifest{ + { + name: "manifests/machineconfigcontroller/controllerconfig.yaml", + filename: "bootstrap/manifests/machineconfigcontroller-controllerconfig.yaml", }, { - name: "manifests/baremetal/coredns-corefile.tmpl", - filename: "baremetal/static-pod-resources/coredns/Corefile.tmpl", + name: "manifests/master.machineconfigpool.yaml", + filename: "bootstrap/manifests/master.machineconfigpool.yaml", }, { - name: "manifests/baremetal/keepalived.yaml", - filename: "baremetal/manifests/keepalived.yaml", + name: "manifests/worker.machineconfigpool.yaml", + filename: "bootstrap/manifests/worker.machineconfigpool.yaml", }, { - name: "manifests/baremetal/keepalived.conf.tmpl", - filename: "baremetal/static-pod-resources/keepalived/keepalived.conf.tmpl", - }}...) - } - - if infra.Status.PlatformStatus.OpenStack != nil { - manifests = append(manifests, []struct { - name string - data []byte - filename string - }{{ - name: "manifests/openstack/coredns.yaml", - filename: "openstack/manifests/coredns.yaml", + name: "manifests/bootstrap-pod-v2.yaml", + filename: "bootstrap/machineconfigoperator-bootstrap-pod.yaml", }, { - name: "manifests/openstack/coredns-corefile.tmpl", - filename: "openstack/static-pod-resources/coredns/Corefile.tmpl", + data: filesData[pullSecretFile], + filename: "bootstrap/manifests/machineconfigcontroller-pull-secret", }, { - name: "manifests/openstack/keepalived.yaml", - filename: "openstack/manifests/keepalived.yaml", + name: "manifests/machineconfigserver/csr-bootstrap-role-binding.yaml", + filename: "manifests/csr-bootstrap-role-binding.yaml", }, { - name: "manifests/openstack/keepalived.conf.tmpl", - filename: "openstack/static-pod-resources/keepalived/keepalived.conf.tmpl", - }}...) - } + name: "manifests/machineconfigserver/kube-apiserver-serving-ca-configmap.yaml", + filename: "manifests/kube-apiserver-serving-ca-configmap.yaml", + }} + + manifests = appendManifestsByPlatform(manifests, *infra) for _, m := range manifests { var b []byte @@ -238,3 +203,71 @@ func RenderBootstrap( } return nil } + +func appendManifestsByPlatform(manifests []manifest, infra configv1.Infrastructure) []manifest { + + if infra.Status.PlatformStatus.BareMetal != nil { + manifests = append(manifests, + manifest{ + name: "manifests/baremetal/coredns.yaml", + filename: "baremetal/manifests/coredns.yaml", + }, + manifest{ + name: "manifests/baremetal/coredns-corefile.tmpl", + filename: "baremetal/static-pod-resources/coredns/Corefile.tmpl", + }, + manifest{ + name: "manifests/baremetal/keepalived.yaml", + filename: "baremetal/manifests/keepalived.yaml", + }, + manifest{ + name: "manifests/baremetal/keepalived.conf.tmpl", + filename: "baremetal/static-pod-resources/keepalived/keepalived.conf.tmpl", + }, + ) + } + + if infra.Status.PlatformStatus.OpenStack != nil { + manifests = append(manifests, + manifest{ + name: "manifests/openstack/coredns.yaml", + filename: "openstack/manifests/coredns.yaml", + }, + manifest{ + name: "manifests/openstack/coredns-corefile.tmpl", + filename: "openstack/static-pod-resources/coredns/Corefile.tmpl", + }, + manifest{ + name: "manifests/openstack/keepalived.yaml", + filename: "openstack/manifests/keepalived.yaml", + }, + manifest{ + name: "manifests/openstack/keepalived.conf.tmpl", + filename: "openstack/static-pod-resources/keepalived/keepalived.conf.tmpl", + }, + ) + } + + if infra.Status.PlatformStatus.Ovirt != nil { + manifests = append(manifests, + manifest{ + name: "manifests/ovirt/coredns.yaml", + filename: "ovirt/manifests/coredns.yaml", + }, + manifest{ + name: "manifests/ovirt/coredns-corefile.tmpl", + filename: "ovirt/static-pod-resources/coredns/Corefile.tmpl", + }, + manifest{ + name: "manifests/ovirt/keepalived.yaml", + filename: "ovirt/manifests/keepalived.yaml", + }, + manifest{ + name: "manifests/ovirt/keepalived.conf.tmpl", + filename: "ovirt/static-pod-resources/keepalived/keepalived.conf.tmpl", + }, + ) + } + + return manifests +} diff --git a/templates/common/ovirt/files/ovirt-NetworkManager-kni-conf.yaml b/templates/common/ovirt/files/ovirt-NetworkManager-kni-conf.yaml new file mode 100644 index 0000000000..004c551320 --- /dev/null +++ b/templates/common/ovirt/files/ovirt-NetworkManager-kni-conf.yaml @@ -0,0 +1,7 @@ +filesystem: "root" +mode: 0644 +path: "/etc/NetworkManager/conf.d/99-kni.conf" +contents: + inline: | + [main] + dhcp=dhclient diff --git a/templates/common/ovirt/files/ovirt-coredns-corefile.yaml b/templates/common/ovirt/files/ovirt-coredns-corefile.yaml new file mode 100644 index 0000000000..0f938cf89c --- /dev/null +++ b/templates/common/ovirt/files/ovirt-coredns-corefile.yaml @@ -0,0 +1,14 @@ +filesystem: "root" +mode: 0644 +path: "/etc/kubernetes/static-pod-resources/coredns/Corefile.tmpl" +contents: + inline: | + . { + errors + health :18080 + mdns {{ .EtcdDiscoveryDomain }} 0 {{`{{.Cluster.Name}}`}} + forward . {{`{{- range $upstream := .DNSUpstreams}} {{$upstream}}{{- end}}`}} + cache 30 + reload + file /etc/coredns/node-dns-db {{ .EtcdDiscoveryDomain }} + } diff --git a/templates/common/ovirt/files/ovirt-coredns-db.yaml b/templates/common/ovirt/files/ovirt-coredns-db.yaml new file mode 100644 index 0000000000..100e44b1d7 --- /dev/null +++ b/templates/common/ovirt/files/ovirt-coredns-db.yaml @@ -0,0 +1,17 @@ +filesystem: "root" +mode: 0644 +path: "/etc/coredns/node-dns-db" +contents: + inline: | + $ORIGIN {{ .EtcdDiscoveryDomain }}. + @ 3600 IN SOA host.{{ .EtcdDiscoveryDomain }}. hostmaster ( + 2017042752 ; serial + 7200 ; refresh (2 hours) + 3600 ; retry (1 hour) + 1209600 ; expire (2 weeks) + 3600 ; minimum (1 hour) + ) + api-int IN A {{ .Infra.Status.PlatformStatus.Ovirt.APIServerInternalIP }} + api IN A {{ .Infra.Status.PlatformStatus.Ovirt.APIServerInternalIP }} + + *.apps IN A {{ .Infra.Status.PlatformStatus.Ovirt.IngressIP }} diff --git a/templates/common/ovirt/files/ovirt-coredns.yaml b/templates/common/ovirt/files/ovirt-coredns.yaml new file mode 100644 index 0000000000..8a188d8e03 --- /dev/null +++ b/templates/common/ovirt/files/ovirt-coredns.yaml @@ -0,0 +1,91 @@ +filesystem: "root" +mode: 0644 +path: "/etc/kubernetes/manifests/coredns.yaml" +contents: + inline: | + kind: Pod + apiVersion: v1 + metadata: + name: coredns + namespace: openshift-kni-infra + creationTimestamp: + deletionGracePeriodSeconds: 65 + labels: + app: kni-infra-mdns + spec: + volumes: + - name: resource-dir + hostPath: + path: "/etc/kubernetes/static-pod-resources/coredns" + - name: kubeconfig + hostPath: + path: "/etc/kubernetes/kubeconfig" + - name: conf-dir + hostPath: + path: "/etc/coredns" + initContainers: + - name: render-config + image: {{ .Images.baremetalRuntimeCfgImage }} + command: + - runtimecfg + - render + - "/etc/kubernetes/kubeconfig" + - "--api-vip" + - "{{ .Infra.Status.PlatformStatus.Ovirt.APIServerInternalIP }}" + - "--dns-vip" + - "{{ .Infra.Status.PlatformStatus.Ovirt.NodeDNSIP }}" + - "--ingress-vip" + - "{{ .Infra.Status.PlatformStatus.Ovirt.IngressIP }}" + - "/config" + - "--out-dir" + - "/etc/coredns" + resources: {} + volumeMounts: + - name: kubeconfig + mountPath: "/etc/kubernetes/kubeconfig" + - name: resource-dir + mountPath: "/config" + - name: conf-dir + mountPath: "/etc/coredns" + imagePullPolicy: IfNotPresent + containers: + - name: coredns + securityContext: + privileged: true + image: {{.Images.corednsImage}} + args: + - "--conf" + - "/etc/coredns/Corefile" + resources: + requests: + cpu: 150m + memory: 1Gi + volumeMounts: + - name: conf-dir + mountPath: "/etc/coredns" + readinessProbe: + httpGet: + path: /health + port: 18080 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 10 + livenessProbe: + httpGet: + path: /health + port: 18080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + terminationMessagePolicy: FallbackToLogsOnError + imagePullPolicy: IfNotPresent + hostNetwork: true + tolerations: + - operator: Exists + priorityClassName: system-node-critical + status: {} diff --git a/templates/common/ovirt/files/ovirt-keepalived.yaml b/templates/common/ovirt/files/ovirt-keepalived.yaml new file mode 100644 index 0000000000..efab0c85fc --- /dev/null +++ b/templates/common/ovirt/files/ovirt-keepalived.yaml @@ -0,0 +1,118 @@ +filesystem: "root" +mode: 0644 +path: "/etc/kubernetes/manifests/keepalived.yaml" +contents: + inline: | + kind: Pod + apiVersion: v1 + metadata: + name: keepalived + namespace: openshift-kni-infra + creationTimestamp: + deletionGracePeriodSeconds: 65 + labels: + app: kni-infra-vrrp + spec: + volumes: + - name: resource-dir + hostPath: + path: "/etc/kubernetes/static-pod-resources/keepalived" + - name: kubeconfig + hostPath: + path: "/etc/kubernetes/kubeconfig" + - name: conf-dir + hostPath: + path: "/etc/keepalived" + - name: run-dir + empty-dir: {} + containers: + - name: keepalived + securityContext: + privileged: true + image: {{.Images.keepalivedImage}} + env: + - name: NSS_SDB_USE_CACHE + value: "no" + command: + - /bin/bash + - -c + - | + #/bin/bash + reload_keepalived() + { + if pid=$(pgrep -o keepalived); then + kill -s SIGHUP "$pid" + else + /usr/sbin/keepalived -f /etc/keepalived/keepalived.conf --dont-fork --vrrp --log-detail --log-console & + fi + } + + msg_handler() + { + while read -r line; do + echo "The client sent: $line" >&2 + # currently only 'reload' msg is supported + if [ "$line" = reload ]; then + reload_keepalived + fi + done + } + + set -ex + declare -r keepalived_sock="/var/run/keepalived/keepalived.sock" + export -f msg_handler + export -f reload_keepalived + if [ -s "/etc/keepalived/keepalived.conf" ]; then + /usr/sbin/keepalived -f /etc/keepalived/keepalived.conf --dont-fork --vrrp --log-detail --log-console & + fi + + rm -f "$keepalived_sock" + socat UNIX-LISTEN:${keepalived_sock},fork system:'bash -c msg_handler' + resources: + requests: + cpu: 150m + memory: 1Gi + volumeMounts: + - name: conf-dir + mountPath: "/etc/keepalived" + - name: run-dir + mountPath: "/var/run/keepalived" + livenessProbe: + exec: + command: + - /bin/sh + - -c + - | + [[ -s /etc/keepalived/keepalived.conf ]] || \ + kill -s SIGUSR1 "$(pgrep -o keepalived)" && ! grep -q "State = FAULT" /tmp/keepalived.data + initialDelaySeconds: 10 + terminationMessagePolicy: FallbackToLogsOnError + imagePullPolicy: IfNotPresent + - name: keepalived-monitor + image: {{ .Images.baremetalRuntimeCfgImage }} + command: + - dynkeepalived + - "/etc/kubernetes/kubeconfig" + - "/config/keepalived.conf.tmpl" + - "/etc/keepalived/keepalived.conf" + - "--api-vip" + - "{{ .Infra.Status.PlatformStatus.Ovirt.APIServerInternalIP }}" + - "--dns-vip" + - "{{ .Infra.Status.PlatformStatus.Ovirt.NodeDNSIP }}" + - "--ingress-vip" + - "{{ .Infra.Status.PlatformStatus.Ovirt.IngressIP }}" + volumeMounts: + - name: resource-dir + mountPath: "/config" + - name: kubeconfig + mountPath: "/etc/kubernetes/kubeconfig" + - name: conf-dir + mountPath: "/etc/keepalived" + - name: run-dir + mountPath: "/var/run/keepalived" + imagePullPolicy: IfNotPresent + hostNetwork: true + tolerations: + - operator: Exists + priorityClassName: system-node-critical + status: {} diff --git a/templates/common/ovirt/files/ovirt-mdns-publisher.yaml b/templates/common/ovirt/files/ovirt-mdns-publisher.yaml new file mode 100644 index 0000000000..d425c106d3 --- /dev/null +++ b/templates/common/ovirt/files/ovirt-mdns-publisher.yaml @@ -0,0 +1,75 @@ +filesystem: "root" +mode: 0644 +path: "/etc/kubernetes/manifests/mdns-publisher.yaml" +contents: + inline: | + kind: Pod + apiVersion: v1 + metadata: + name: mdns-publisher + namespace: openshift-kni-infra + creationTimestamp: + deletionGracePeriodSeconds: 65 + labels: + app: kni-infra-mdns + spec: + volumes: + - name: resource-dir + hostPath: + path: "/etc/kubernetes/static-pod-resources/mdns" + - name: kubeconfig + hostPath: + path: "/etc/kubernetes/kubeconfig" + - name: conf-dir + hostPath: + path: "/etc/mdns" + initContainers: + - name: render-config + image: {{ .Images.baremetalRuntimeCfgImage }} + command: + - runtimecfg + - render + - "/etc/kubernetes/kubeconfig" + - "--api-vip" + - "{{ .Infra.Status.PlatformStatus.Ovirt.APIServerInternalIP }}" + - "--dns-vip" + - "{{ .Infra.Status.PlatformStatus.Ovirt.NodeDNSIP }}" + - "--ingress-vip" + - "{{ .Infra.Status.PlatformStatus.Ovirt.IngressIP }}" + - "/config" + - "--out-dir" + - "/etc/mdns" + resources: {} + volumeMounts: + - name: kubeconfig + mountPath: "/etc/kubernetes/kubeconfig" + - name: resource-dir + mountPath: "/config" + - name: conf-dir + mountPath: "/etc/mdns" + imagePullPolicy: IfNotPresent + containers: + - name: mdns-publisher + image: {{.Images.mdnsPublisherImage}} + args: + - "--debug" + resources: + requests: + cpu: 150m + memory: 1Gi + volumeMounts: + - name: conf-dir + mountPath: "/etc/mdns" + livenessProbe: + exec: + command: + - pgrep + - mdns-publisher + initialDelaySeconds: 10 + terminationMessagePolicy: FallbackToLogsOnError + imagePullPolicy: IfNotPresent + hostNetwork: true + tolerations: + - operator: Exists + priorityClassName: system-node-critical + status: {} diff --git a/templates/master/00-master/ovirt/files/dhcp-dhclient-conf.yaml b/templates/master/00-master/ovirt/files/dhcp-dhclient-conf.yaml new file mode 100644 index 0000000000..8a75bbd1e9 --- /dev/null +++ b/templates/master/00-master/ovirt/files/dhcp-dhclient-conf.yaml @@ -0,0 +1,7 @@ +filesystem: "root" +mode: 0644 +path: "/etc/dhcp/dhclient.conf" +contents: + inline: | + supersede domain-search "{{ .EtcdDiscoveryDomain }}"; + prepend domain-name-servers {{ .Infra.Status.PlatformStatus.Ovirt.NodeDNSIP }}; diff --git a/templates/master/00-master/ovirt/files/ovirt-haproxy-haproxy.yaml b/templates/master/00-master/ovirt/files/ovirt-haproxy-haproxy.yaml new file mode 100644 index 0000000000..f6e8d14718 --- /dev/null +++ b/templates/master/00-master/ovirt/files/ovirt-haproxy-haproxy.yaml @@ -0,0 +1,40 @@ +filesystem: "root" +mode: 0644 +path: "/etc/kubernetes/static-pod-resources/haproxy/haproxy.cfg.tmpl" +contents: + inline: | + defaults + maxconn 20000 + mode tcp + log /var/run/haproxy/haproxy-log.sock local0 + option dontlognull + retries 3 + timeout http-request 10s + timeout queue 1m + timeout connect 10s + timeout client 86400s + timeout server 86400s + timeout tunnel 86400s + frontend main + bind :{{`{{ .LBConfig.LbPort }}`}} + default_backend masters + listen health_check_http_url + bind :50936 + mode http + monitor-uri /healthz + option dontlognull + listen stats + bind 127.0.0.1:{{`{{ .LBConfig.StatPort }}`}} + mode http + stats enable + stats hide-version + stats uri /haproxy_stats + stats refresh 30s + stats auth Username:Password + backend masters + option httpchk GET /healthz HTTP/1.0 + option log-health-checks + balance roundrobin + {{`{{- range .LBConfig.Backends }} + server {{ .Host }} {{ .Address }}:{{ .Port }} weight 1 verify none check check-ssl inter 3s fall 2 rise 3 + {{- end }}`}} diff --git a/templates/master/00-master/ovirt/files/ovirt-haproxy.yaml b/templates/master/00-master/ovirt/files/ovirt-haproxy.yaml new file mode 100644 index 0000000000..629ee9f25e --- /dev/null +++ b/templates/master/00-master/ovirt/files/ovirt-haproxy.yaml @@ -0,0 +1,110 @@ +filesystem: "root" +mode: 0644 +path: "/etc/kubernetes/manifests/haproxy.yaml" +contents: + inline: | + kind: Pod + apiVersion: v1 + metadata: + name: haproxy + namespace: openshift-kni-infra + creationTimestamp: + deletionGracePeriodSeconds: 65 + labels: + app: openstack-infra-api-lb + spec: + volumes: + - name: resource-dir + hostPath: + path: "/etc/kubernetes/static-pod-resources/haproxy" + - name: kubeconfig + hostPath: + path: "/etc/kubernetes/kubeconfig" + - name: run-dir + empty-dir: {} + - name: conf-dir + hostPath: + path: "/etc/haproxy" + - name: chroot-host + hostPath: + path: "/" + containers: + - name: haproxy + image: {{.Images.haproxyImage}} + command: + - "/bin/bash" + - "-c" + - | + #/bin/bash + reload_haproxy() + { + old_pids=$(pidof haproxy) + if [ -n "$old_pids" ]; then + /usr/sbin/haproxy -W -db -f /etc/haproxy/haproxy.cfg -p /var/lib/haproxy/run/haproxy.pid -x /var/lib/haproxy/run/haproxy.sock -sf $old_pids & + else + /usr/sbin/haproxy -W -db -f /etc/haproxy/haproxy.cfg -p /var/lib/haproxy/run/haproxy.pid & + fi + } + + msg_handler() + { + while read -r line; do + echo "The client send: $line" >&2 + # currently only 'reload' msg is supported + if [ "$line" = reload ]; then + reload_haproxy + fi + done + } + set -ex + declare -r haproxy_sock="/var/run/haproxy/haproxy-master.sock" + declare -r haproxy_log_sock="/var/run/haproxy/haproxy-log.sock" + export -f msg_handler + export -f reload_haproxy + rm -f "$haproxy_sock" "$haproxy_log_sock" + socat UNIX-RECV:${haproxy_log_sock} STDOUT & + if [ -s "/etc/haproxy/haproxy.cfg" ]; then + /usr/sbin/haproxy -W -db -f /etc/haproxy/haproxy.cfg -p /var/lib/haproxy/run/haproxy.pid & + fi + socat UNIX-LISTEN:${haproxy_sock},fork system:'bash -c msg_handler' + volumeMounts: + - name: conf-dir + mountPath: "/etc/haproxy" + - name: run-dir + mountPath: "/var/run/haproxy" + livenessProbe: + initialDelaySeconds: 10 + httpGet: + path: /healthz + port: 50936 + terminationMessagePolicy: FallbackToLogsOnError + imagePullPolicy: IfNotPresent + - name: haproxy-monitor + securityContext: + privileged: true + image: {{ .Images.baremetalRuntimeCfgImage }} + command: + - monitor + - "/etc/kubernetes/kubeconfig" + - "/config/haproxy.cfg.tmpl" + - "/etc/haproxy/haproxy.cfg" + - "--api-vip" + - "{{ .Infra.Status.PlatformStatus.Ovirt.APIServerInternalIP }}" + volumeMounts: + - name: conf-dir + mountPath: "/etc/haproxy" + - name: run-dir + mountPath: "/var/run/haproxy" + - name: resource-dir + mountPath: "/config" + - name: chroot-host + mountPath: "/host" + - name: kubeconfig + mountPath: "/etc/kubernetes/kubeconfig" + terminationMessagePolicy: FallbackToLogsOnError + imagePullPolicy: IfNotPresent + hostNetwork: true + tolerations: + - operator: Exists + priorityClassName: system-node-critical + status: {} diff --git a/templates/master/00-master/ovirt/files/ovirt-keepalived-keepalived.yaml b/templates/master/00-master/ovirt/files/ovirt-keepalived-keepalived.yaml new file mode 100644 index 0000000000..a51f45eef4 --- /dev/null +++ b/templates/master/00-master/ovirt/files/ovirt-keepalived-keepalived.yaml @@ -0,0 +1,78 @@ +filesystem: "root" +mode: 0644 +path: "/etc/kubernetes/static-pod-resources/keepalived/keepalived.conf.tmpl" +contents: + inline: | + vrrp_script chk_ocp { + script "/usr/bin/curl -o /dev/null -kLs https://0:6443/readyz" + interval 1 + weight 50 + } + + vrrp_script chk_dns { + script "/usr/bin/host -t SRV _etcd-server-ssl._tcp.{{ .EtcdDiscoveryDomain }} localhost" + interval 1 + weight 50 + } + + # TODO: Improve this check. The port is assumed to be alive. + # Need to assess what is the ramification if the port is not there. + vrrp_script chk_ingress { + script "/usr/bin/curl -o /dev/null -kLs http://0:1936/healthz" + interval 1 + weight 50 + } + + vrrp_instance {{`{{ .Cluster.Name }}`}}_API { + state BACKUP + interface {{`{{ .VRRPInterface }}`}} + virtual_router_id {{`{{ .Cluster.APIVirtualRouterID }}`}} + priority 40 + advert_int 1 + authentication { + auth_type PASS + auth_pass {{`{{ .Cluster.Name }}`}}_api_vip + } + virtual_ipaddress { + {{`{{ .Cluster.APIVIP }}`}}/{{`{{ .Cluster.VIPNetmask }}`}} + } + track_script { + chk_ocp + } + } + + vrrp_instance {{`{{ .Cluster.Name }}`}}_DNS { + state BACKUP + interface {{`{{ .VRRPInterface }}`}} + virtual_router_id {{`{{ .Cluster.DNSVirtualRouterID }}`}} + priority 40 + advert_int 1 + authentication { + auth_type PASS + auth_pass {{`{{ .Cluster.Name }}`}}_dns_vip + } + virtual_ipaddress { + {{`{{ .Cluster.DNSVIP }}`}}/{{`{{ .Cluster.VIPNetmask }}`}} + } + track_script { + chk_dns + } + } + + vrrp_instance {{`{{ .Cluster.Name }}`}}_INGRESS { + state BACKUP + interface {{`{{ .VRRPInterface }}`}} + virtual_router_id {{`{{ .Cluster.IngressVirtualRouterID }}`}} + priority 40 + advert_int 1 + authentication { + auth_type PASS + auth_pass cluster_uuid_ingress_vip + } + virtual_ipaddress { + {{`{{ .Cluster.IngressVIP }}`}}/{{`{{ .Cluster.VIPNetmask }}`}} + } + track_script { + chk_ingress + } + } diff --git a/templates/master/00-master/ovirt/files/ovirt-mdns-config.yaml b/templates/master/00-master/ovirt/files/ovirt-mdns-config.yaml new file mode 100644 index 0000000000..bd9020df1f --- /dev/null +++ b/templates/master/00-master/ovirt/files/ovirt-mdns-config.yaml @@ -0,0 +1,34 @@ +filesystem: "root" +mode: 0644 +path: "/etc/kubernetes/static-pod-resources/mdns/config.hcl.tmpl" +contents: + inline: | + bind_address = "{{`{{ .NonVirtualIP }}`}}" + collision_avoidance = "hostname" + + service { + name = "{{`{{ .Cluster.Name }}`}} Etcd" + host_name = "{{`{{ .EtcdShortHostname }}`}}.local." + type = "_etcd-server-ssl._tcp" + domain = "local." + port = 2380 + ttl = 3200 + } + + service { + name = "{{`{{ .Cluster.Name }}`}} Workstation" + host_name = "{{`{{ .ShortHostname }}`}}.local." + type = "_workstation._tcp" + domain = "local." + port = 42424 + ttl = 3200 + } + + service { + name = "{{`{{ .Cluster.Name }}`}} EtcdWorkstation" + host_name = "{{`{{ .EtcdShortHostname }}`}}.local." + type = "_workstation._tcp" + domain = "local." + port = 42424 + ttl = 300 + } diff --git a/templates/worker/00-worker/ovirt/files/NetworkManager-non-virtual-ip-prepender.yaml b/templates/worker/00-worker/ovirt/files/NetworkManager-non-virtual-ip-prepender.yaml new file mode 100644 index 0000000000..a704c26618 --- /dev/null +++ b/templates/worker/00-worker/ovirt/files/NetworkManager-non-virtual-ip-prepender.yaml @@ -0,0 +1,41 @@ +filesystem: "root" +mode: 0755 +path: "/etc/NetworkManager/dispatcher.d/pre-up.d/non-virtual-ip-prepender" +contents: + inline: | + #!/bin/bash + IFACE=$1 + STATUS=$2 + case "$STATUS" in + pre-up) + logger -s "NM non-virtual-ip-prepender triggered by pre-upping ${1}." + NON_VIRTUAL_IP=$(/usr/local/bin/non_virtual_ip \ + "{{.Infra.Status.PlatformStatus.Ovirt.APIServerInternalIP}}" \ + "{{.Infra.Status.PlatformStatus.Ovirt.NodeDNSIP}}" \ + "{{.Infra.Status.PlatformStatus.Ovirt.IngressIP}}") + set +e + if [[ -n $NON_VIRTUAL_IP ]]; then + logger -s "NM non-virtual-ip-prepender: Checking if worker non virtual IP is the first entry in resolv.conf" + if grep nameserver /etc/resolv.conf | head -n 1 | grep -q "$NON_VIRTUAL_IP" ; then + logger -s "NM non-virtual-ip-prepender: worker node non virtual IP already is the first entry in resolv.conf" + exit 0 + else + logger -s "NM non-virtual-ip-prepender: Setting dhclient to prepend non virtual IP in resolv.conf" + sed "s/{{`{{ .NonVirtualIP }}`}}/$NON_VIRTUAL_IP/" /etc/dhcp/dhclient.conf.template > /etc/dhcp/dhclient.conf + logger -s "NM non-virtual-ip-prepender: Looking for 'search' in /etc/resolv.conf to place 'nameserver $NON_VIRTUAL_IP'" + sed -i "/^search .*$/a nameserver $NON_VIRTUAL_IP" /etc/resolv.conf + fi + fi + ;; + down) + logger -s "NM non-virtual-ip-prepender triggered by downing $IFACE" + ;; + up) + logger -s "NM non-virtual-ip-prepender triggered by upping $IFACE" + ;; + post-down) + logger -s "NM non-virtual-ip-prepender triggered by post-downing $IFACE" + ;; + *) + ;; + esac \ No newline at end of file diff --git a/templates/worker/00-worker/ovirt/files/dhcp-dhclient-conf.yaml b/templates/worker/00-worker/ovirt/files/dhcp-dhclient-conf.yaml new file mode 100644 index 0000000000..ee141be526 --- /dev/null +++ b/templates/worker/00-worker/ovirt/files/dhcp-dhclient-conf.yaml @@ -0,0 +1,7 @@ +filesystem: "root" +mode: 0644 +path: "/etc/dhcp/dhclient.conf.template" +contents: + inline: | + supersede domain-search "{{ .EtcdDiscoveryDomain }}"; + prepend domain-name-servers {{`{{ .NonVirtualIP }}`}}; diff --git a/templates/worker/00-worker/ovirt/files/ovirt-keepalived-keepalived.yaml b/templates/worker/00-worker/ovirt/files/ovirt-keepalived-keepalived.yaml new file mode 100644 index 0000000000..952e2e0017 --- /dev/null +++ b/templates/worker/00-worker/ovirt/files/ovirt-keepalived-keepalived.yaml @@ -0,0 +1,30 @@ +filesystem: "root" +mode: 0644 +path: "/etc/kubernetes/static-pod-resources/keepalived/keepalived.conf.tmpl" +contents: + inline: | + # TODO: Improve this check. The port is assumed to be alive. + # Need to assess what is the ramification if the port is not there. + vrrp_script chk_ingress { + script "/usr/bin/curl -o /dev/null -kLs http://0:1936/healthz" + interval 1 + weight 50 + } + + vrrp_instance {{`{{ .Cluster.Name }}`}}_INGRESS { + state BACKUP + interface {{`{{ .VRRPInterface }}`}} + virtual_router_id {{`{{ .Cluster.IngressVirtualRouterID }}`}} + priority 40 + advert_int 1 + authentication { + auth_type PASS + auth_pass cluster_uuid_ingress_vip + } + virtual_ipaddress { + {{`{{ .Cluster.IngressVIP }}`}}/{{`{{ .Cluster.VIPNetmask }}`}} + } + track_script { + chk_ingress + } + } diff --git a/templates/worker/00-worker/ovirt/files/ovirt-mdns-config.yaml b/templates/worker/00-worker/ovirt/files/ovirt-mdns-config.yaml new file mode 100644 index 0000000000..b1baf471db --- /dev/null +++ b/templates/worker/00-worker/ovirt/files/ovirt-mdns-config.yaml @@ -0,0 +1,16 @@ +filesystem: "root" +mode: 0644 +path: "/etc/kubernetes/static-pod-resources/mdns/config.hcl.tmpl" +contents: + inline: | + bind_address = "{{`{{ .NonVirtualIP }}`}}" + collision_avoidance = "hostname" + + service { + name = "{{`{{ .Cluster.Name }}`}} Workstation" + host_name = "{{`{{ .ShortHostname }}`}}.local." + type = "_workstation._tcp" + domain = "local." + port = 42424 + ttl = 3200 + } diff --git a/templates/worker/00-worker/ovirt/files/ovirt-non-virtual-ip.yaml b/templates/worker/00-worker/ovirt/files/ovirt-non-virtual-ip.yaml new file mode 100644 index 0000000000..c66cd2c5ba --- /dev/null +++ b/templates/worker/00-worker/ovirt/files/ovirt-non-virtual-ip.yaml @@ -0,0 +1,138 @@ +filesystem: "root" +mode: 0755 +path: "/usr/local/bin/non_virtual_ip" +contents: + inline: | + #!/usr/libexec/platform-python + import collections + import socket + import struct + import subprocess + import sys + + + class SubnetNotFoundException(Exception): + """ + Exception raised when no subnet in the systems ifaces is on the VIP subnet + """ + pass + + + Address = collections.namedtuple('Address', 'index name family cidr scope') + + SUBNET_MASK_LEN = { + 'inet': 32, + 'inet6': 128 + } + + + def ntoa(family, num): + if family == 'inet': + result = socket.inet_ntoa(struct.pack("!I", num)) + else: + lo_half = num & 0xFFFFFFFFFFFFFFFF + hi_half = num >> 64 + result = socket.inet_ntop(socket.AF_INET6, + struct.pack(">QQ", hi_half, lo_half)) + return result + + + def aton(family, rep): + if family == 'inet': + result = struct.unpack("!I", socket.inet_aton(rep))[0] + else: + hi_half, lo_half = struct.unpack(">QQ", socket.inet_pton(socket.AF_INET6, rep)) + result = (hi_half << 64) | lo_half + return result + + + def addr_subnet_int_min_max(addr): + ip, prefix = addr.cidr.split('/') + ip_int = aton(addr.family, ip) + + prefix_int = int(prefix) + mask = int('1' * prefix_int + + '0' * (SUBNET_MASK_LEN[addr.family] - prefix_int), 2) + + subnet_ip_int_min = ip_int & mask + + remainder = '1' * (SUBNET_MASK_LEN[addr.family] - prefix_int) + subnet_ip_int_max = subnet_ip_int_min | ( + 0 if remainder == '' else int(remainder, 2)) + return subnet_ip_int_min, subnet_ip_int_max + + + def vip_subnet_cidr(vip, addrs): + try: + vip_int = aton('inet', vip) + except Exception: + vip_int = aton('inet6', vip) + for addr in addrs: + subnet_ip_int_min, subnet_ip_int_max = addr_subnet_int_min_max(addr) + subnet_ip = ntoa(addr.family, subnet_ip_int_min) + subnet_ip_max = ntoa(addr.family, subnet_ip_int_max) + + sys.stderr.write('Is %s between %s and %s\n' % + (vip, subnet_ip, subnet_ip_max)) + if subnet_ip_int_min < vip_int < subnet_ip_int_max: + subnet_ip = socket.inet_ntoa(struct.pack("!I", subnet_ip_int_min)) + return Address(index="-1", + name="subnet", + cidr='%s/%s' % (subnet_ip, addr.cidr.split('/')[1]), + family=addr.family, + scope='') + raise SubnetNotFoundException() + + + def line_to_address(line): + spl = line.split() + return Address(index=spl[0][:-1], + name=spl[1], + family=spl[2], + cidr=spl[3], + scope=spl[5]) + + + def interface_cidrs(filter_func=None): + try: + out = subprocess.check_output(["ip", "-o", "addr", "show"], encoding=sys.stdout.encoding) + except TypeError: # python2 + out = subprocess.check_output(["ip", "-o", "addr", "show"]) + for addr in (line_to_address(line) for line in out.splitlines()): + if filter_func(addr): + yield addr + + + def non_host_scope(addr): + return addr.scope != 'host' + + + def in_subnet(subnet): + subnet_ip_int_min, subnet_ip_int_max = addr_subnet_int_min_max(subnet) + + def filt(addr): + ip, _ = addr.cidr.split('/') + ip_int = aton(addr.family, ip) + return subnet_ip_int_min < ip_int < subnet_ip_int_max + return filt + + + def main(): + api_vip, dns_vip, ingress_vip = sys.argv[1:4] + vips = set(sys.argv[1:4]) + iface_cidrs = list(interface_cidrs(non_host_scope)) + try: + subnet = vip_subnet_cidr(api_vip, iface_cidrs) + + sys.stderr.write('VIP Subnet %s\n' % subnet.cidr) + + for addr in interface_cidrs(in_subnet(subnet)): + ip, prefix = addr.cidr.split('/') + if ip not in vips: + print(addr.cidr.split('/')[0]) + sys.exit(0) + except SubnetNotFoundException: + sys.exit(1) + + if __name__ == '__main__': + main()