Skip to content
This repository has been archived by the owner on Jun 15, 2021. It is now read-only.

Commit

Permalink
Calico self hosted integration (kubernetes-retired#124)
Browse files Browse the repository at this point in the history 
feat: Calico self hosted integration
* Migrated Calico to self hosted install
* Updated Calico to v2.0 versions
* Bumped the policy controller to 0.5.2 to get a NoneType bugfix in
  • Loading branch information
@heschlie @mumoshu
heschlie authored and mumoshu committed Jan 15, 2017
1 parent 1ee950b commit 10c01fa
Show file tree
Hide file tree
Showing 2 changed files with 247 additions and 144 deletions.
323 changes: 241 additions & 82 deletions config/templates/cloud-config-controller
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,10 +90,13 @@ coreos:
--volume stage,kind=host,source=/tmp \
--mount volume=stage,target=/tmp \
--volume var-log,kind=host,source=/var/log \
--mount volume=var-log,target=/var/log"
--mount volume=var-log,target=/var/log{{ if .UseCalico }} \
--volume cni-bin,kind=host,source=/opt/cni/bin \
--mount volume=cni-bin,target=/opt/cni/bin{{ end }}"
ExecStartPre=/usr/bin/systemctl is-active flanneld.service
ExecStartPre=/usr/bin/mkdir -p /var/lib/cni
ExecStartPre=/usr/bin/mkdir -p /var/log/containers
ExecStartPre=/usr/bin/mkdir -p /opt/cni/bin
ExecStart=/usr/lib/coreos/kubelet-wrapper \
--api-servers=http://localhost:8080 \
--cni-conf-dir=/etc/kubernetes/cni/net.d \
Expand Down Expand Up @@ -143,45 +146,6 @@ coreos:
RequiredBy=rkt-api.service
{{ end }}

{{ if .UseCalico }}
- name: calico-node.service
command: start
runtime: true
content: |
[Unit]
Description=Calico per-host agent
Requires=network-online.target
After=network-online.target

[Service]
Slice=machine.slice
Environment=CALICO_DISABLE_FILE_LOGGING=true
Environment=HOSTNAME=$private_ipv4
Environment=IP=$private_ipv4
Environment=FELIX_FELIXHOSTNAME=$private_ipv4
Environment=CALICO_NETWORKING=false
Environment=NO_DEFAULT_POOLS=true
Environment=ETCD_SCHEME=https
Environment=ETCD_CA_CERT_FILE=/etc/etcd2/ssl/ca.pem
Environment=ETCD_CERT_FILE=/etc/etcd2/ssl/etcd-client.pem
Environment=ETCD_KEY_FILE=/etc/etcd2/ssl/etcd-client-key.pem
Environment=ETCD_ENDPOINTS={{ .EtcdEndpoints }}
ExecStart=/usr/bin/rkt run --inherit-env --stage1-from-dir=stage1-fly.aci \
--volume=modules,kind=host,source=/lib/modules,readOnly=false \
--mount=volume=modules,target=/lib/modules \
--volume=dns,kind=host,source=/etc/resolv.conf,readOnly=true \
--mount=volume=dns,target=/etc/resolv.conf \
--volume=ssl,kind=host,source=/etc/kubernetes/ssl,readOnly=true \
--mount=volume=ssl,target=/etc/etcd2/ssl \
--trust-keys-from-https quay.io/calico/node:v0.22.0
KillMode=mixed
Restart=always
TimeoutStartSec=0

[Install]
WantedBy=multi-user.target
{{ end }}

- name: install-kube-system.service
command: start
runtime: true
Expand All @@ -199,24 +163,6 @@ coreos:
ExecStartPre=/usr/bin/curl -s -f http://127.0.0.1:8080/version
ExecStart=/opt/bin/install-kube-system

{{ if .UseCalico }}
- name: install-calico-system.service
command: start
runtime: true
content: |
[Unit]
Wants=kubelet.service docker.service

[Service]
Type=simple
StartLimitInterval=0
RestartSec=10
Restart=on-failure
ExecStartPre=/usr/bin/systemctl is-active kubelet.service
ExecStartPre=/usr/bin/systemctl is-active docker.service
ExecStartPre=/usr/bin/curl -s -f http://127.0.0.1:8080/version
ExecStart=/opt/bin/install-calico-system
{{ end }}
{{ if $.ElasticFileSystemID }}
- name: rpc-statd.service
command: start
Expand Down Expand Up @@ -266,7 +212,7 @@ coreos:
Type=oneshot
ExecStartPre=/usr/bin/bash -c "while sleep 1; do if /usr/bin/curl -s -m 20 -f http://127.0.0.1:8080/healthz > /dev/null && /usr/bin/curl -s -m 20 -f http://127.0.0.1:10252/healthz > /dev/null && /usr/bin/curl -s -m 20 -f http://127.0.0.1:10251/healthz > /dev/null && /usr/bin/curl --insecure -s -m 20 -f https://127.0.0.1:10250/healthz > /dev/null ; then break ; fi; done"
{{ if .UseCalico }}
ExecStartPre=/usr/bin/systemctl is-active calico-node
ExecStartPre=/usr/bin/bash -c "until /usr/bin/docker run --net=host --pid=host --rm calico/ctl:v1.0.0 node status > /dev/null; do sleep 3; done && echo Calico running"
{{ end }}
ExecStart=/usr/bin/rkt run \
--uuid-file-save=/var/run/coreos/cfn-signal.uuid \
Expand Down Expand Up @@ -367,6 +313,10 @@ write_files:
owner: root:root
content: |
#!/bin/bash -e
{{ if .UseCalico }}
/bin/bash /opt/bin/populate-tls-calico-etcd
/usr/bin/docker run --rm --net=host -v /srv/kubernetes/manifests:/host/manifests {{.HyperkubeImageRepo}}:{{.K8sVer}} /hyperkube kubectl apply -f /host/manifests/calico.yaml
{{ end }}

post_yaml() {
/usr/bin/curl -H "Content-Type: application/yaml" -XPOST \
Expand Down Expand Up @@ -408,14 +358,241 @@ write_files:
exec nsenter -m -u -i -n -p -t 1 -- /usr/bin/rkt "$@"

{{ if .UseCalico }}
- path: /opt/bin/install-calico-system
permissions: 0700
- path: /srv/kubernetes/manifests/calico.yaml
content: |
kind: ConfigMap
apiVersion: v1
metadata:
name: calico-config
namespace: kube-system
data:
etcd_endpoints: "{{ .EtcdEndpoints }}"
cni_network_config: |-
{
"name": "calico",
"type": "flannel",
"delegate": {
"type": "calico",
"etcd_endpoints": "__ETCD_ENDPOINTS__",
"etcd_key_file": "__ETCD_KEY_FILE__",
"etcd_cert_file": "__ETCD_CERT_FILE__",
"etcd_ca_cert_file": "__ETCD_CA_CERT_FILE__",
"log_level": "info",
"policy": {
"type": "k8s",
"k8s_api_root": "https://__KUBERNETES_SERVICE_HOST__:__KUBERNETES_SERVICE_PORT__",
"k8s_auth_token": "__SERVICEACCOUNT_TOKEN__"
},
"kubernetes": {
"kubeconfig": "__KUBECONFIG_FILEPATH__"
}
}
}

etcd_ca: "/calico-secrets/etcd-ca"
etcd_cert: "/calico-secrets/etcd-cert"
etcd_key: "/calico-secrets/etcd-key"

---

apiVersion: v1
kind: Secret
type: Opaque
metadata:
name: calico-etcd-secrets
namespace: kube-system
data:
etcd-key: "$ETCDKEY"
etcd-cert: "$ETCDCERT"
etcd-ca: "$ETCDCA"

---

kind: DaemonSet
apiVersion: extensions/v1beta1
metadata:
name: calico-node
namespace: kube-system
labels:
k8s-app: calico-node
spec:
selector:
matchLabels:
k8s-app: calico-node
template:
metadata:
labels:
k8s-app: calico-node
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ''
scheduler.alpha.kubernetes.io/tolerations: |
[{"key": "node.alpha.kubernetes.io/role", "value": "master", "effect": "NoSchedule" },
{"key":"CriticalAddonsOnly", "operator":"Exists"}]
spec:
hostNetwork: true
containers:
- name: calico-node
image: quay.io/calico/node:v1.0.0
env:
- name: ETCD_ENDPOINTS
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_endpoints
- name: CALICO_NETWORKING_BACKEND
value: "none"
- name: CALICO_DISABLE_FILE_LOGGING
value: "true"
- name: NO_DEFAULT_POOLS
value: "true"
- name: ETCD_CA_CERT_FILE
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_ca
- name: ETCD_KEY_FILE
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_key
- name: ETCD_CERT_FILE
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_cert
securityContext:
privileged: true
volumeMounts:
- mountPath: /lib/modules
name: lib-modules
readOnly: true
- mountPath: /var/run/calico
name: var-run-calico
readOnly: false
- mountPath: /calico-secrets
name: etcd-certs
- mountPath: /etc/resolv.conf
name: dns
readOnly: true
- name: install-cni
image: calico/cni:v1.5.5
imagePullPolicy: Always
command: ["/install-cni.sh"]
env:
- name: ETCD_ENDPOINTS
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_endpoints
- name: CNI_NETWORK_CONFIG
valueFrom:
configMapKeyRef:
name: calico-config
key: cni_network_config
- name: CNI_NET_DIR
value: "/etc/kubernetes/cni/net.d"
volumeMounts:
- mountPath: /host/opt/cni/bin
name: cni-bin-dir
- mountPath: /host/etc/cni/net.d
name: cni-net-dir
- mountPath: /calico-secrets
name: etcd-certs
volumes:
- name: lib-modules
hostPath:
path: /lib/modules
- name: var-run-calico
hostPath:
path: /var/run/calico
- name: cni-bin-dir
hostPath:
path: /opt/cni/bin
- name: cni-net-dir
hostPath:
path: /etc/kubernetes/cni/net.d
- name: etcd-certs
secret:
secretName: calico-etcd-secrets
- name: dns
hostPath:
path: /etc/resolv.conf

---

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: calico-policy-controller
namespace: kube-system
labels:
k8s-app: calico-policy
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ''
scheduler.alpha.kubernetes.io/tolerations: |
[{"key": "node.alpha.kubernetes.io/role", "value": "master", "effect": "NoSchedule" },
{"key":"CriticalAddonsOnly", "operator":"Exists"}]

spec:
replicas: 1
template:
metadata:
name: calico-policy-controller
namespace: kube-system
labels:
k8s-app: calico-policy
spec:
hostNetwork: true
containers:
- name: calico-policy-controller
image: calico/kube-policy-controller:v0.5.2
env:
- name: ETCD_ENDPOINTS
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_endpoints
- name: ETCD_CA_CERT_FILE
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_ca
- name: ETCD_KEY_FILE
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_key
- name: ETCD_CERT_FILE
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_cert
- name: K8S_API
value: "https://kubernetes.default:443"
- name: CONFIGURE_ETC_HOSTS
value: "true"
volumeMounts:
- mountPath: /calico-secrets
name: etcd-certs
volumes:
- name: etcd-certs
secret:
secretName: calico-etcd-secrets

- path: /opt/bin/populate-tls-calico-etcd
owner: root:root
permissions: 0700
content: |
#!/bin/bash -e
/usr/bin/curl -H "Content-Type: application/json" -XPOST --data-binary @"/srv/kubernetes/manifests/calico-system.json" "http://127.0.0.1:8080/api/v1/namespaces"

/usr/bin/cp /srv/kubernetes/manifests/calico-policy-controller.yaml /etc/kubernetes/manifests
etcd_ca=$(cat /etc/kubernetes/ssl/ca.pem | base64 | tr -d '\n')
etcd_key=$(cat /etc/kubernetes/ssl/etcd-client-key.pem | base64 | tr -d '\n')
etcd_cert=$(cat /etc/kubernetes/ssl/etcd-client.pem | base64 | tr -d '\n')

sed -i -e "s#\$ETCDCA#$etcd_ca#g" /srv/kubernetes/manifests/calico.yaml
sed -i -e "s#\$ETCDCERT#$etcd_cert#g" /srv/kubernetes/manifests/calico.yaml
sed -i -e "s#\$ETCDKEY#$etcd_key#g" /srv/kubernetes/manifests/calico.yaml

{{ end }}

- path: /opt/bin/decrypt-tls-assets
Expand Down Expand Up @@ -1097,25 +1274,7 @@ write_files:
content: {{.TLSConfig.EtcdClientKey}}


{{ if .UseCalico }}
- path: /etc/kubernetes/cni/net.d/10-calico.conf
content: |
{
"name": "calico",
"type": "flannel",
"delegate": {
"type": "calico",
"etcd_endpoints": "{{ .EtcdEndpoints }}",
"log_level": "none",
"log_level_stderr": "info",
"hostname": "$private_ipv4",
"policy": {
"type": "k8s",
"k8s_api_root": "http://127.0.0.1:8080/api/v1/"
}
}
}
{{ else }}
{{ if not .UseCalico }}
- path: /etc/kubernetes/cni/net.d/10-flannel.conf
content: |
{
Expand Down
Loading

0 comments on commit 10c01fa

Please sign in to comment.