From 943f8dbe1861448eac17da8fcb1f42139a843edf Mon Sep 17 00:00:00 2001 From: "bingchang.tbc" Date: Tue, 29 Jun 2021 11:35:13 +0800 Subject: [PATCH] Add subcommand join to yurtctl. --- README.md | 5 + docs/tutorial/yurtctl.md | 14 + go.mod | 5 + go.sum | 18 + pkg/yurtctl/cmd/cmd.go | 3 + pkg/yurtctl/cmd/convert/edgenode.go | 2 +- pkg/yurtctl/cmd/join/join.go | 399 ++++++++++++++++++ pkg/yurtctl/cmd/join/phases/constants.go | 92 ++++ .../cmd/join/phases/join-cloud-node.go | 224 ++++++++++ pkg/yurtctl/cmd/join/phases/join-edge-node.go | 167 ++++++++ pkg/yurtctl/cmd/join/phases/postcheck.go | 136 ++++++ pkg/yurtctl/cmd/join/phases/prepare.go | 219 ++++++++++ pkg/yurtctl/cmd/join/phases/type.go | 27 ++ pkg/yurtctl/util/edgenode/common.go | 1 + pkg/yurtctl/util/edgenode/util.go | 7 +- pkg/yurtctl/util/util.go | 137 ++++++ pkg/yurtctl/util/util_test.go | 54 +++ 17 files changed, 1506 insertions(+), 4 deletions(-) create mode 100644 pkg/yurtctl/cmd/join/join.go create mode 100644 pkg/yurtctl/cmd/join/phases/constants.go create mode 100644 pkg/yurtctl/cmd/join/phases/join-cloud-node.go create mode 100644 pkg/yurtctl/cmd/join/phases/join-edge-node.go create mode 100644 pkg/yurtctl/cmd/join/phases/postcheck.go create mode 100644 pkg/yurtctl/cmd/join/phases/prepare.go create mode 100644 pkg/yurtctl/cmd/join/phases/type.go create mode 100644 pkg/yurtctl/util/util.go create mode 100644 pkg/yurtctl/util/util_test.go diff --git a/README.md b/README.md index 8ba33d978d8..2b00e9b9bb5 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,11 @@ To uninstall OpenYurt and revert back to the original Kubernetes cluster setting _output/bin/yurtctl revert ``` +To join nodes to OpenYurt, you can run the following command: +```bash +_output/bin/yurtctl join +``` + Please check [yurtctl tutorial](./docs/tutorial/yurtctl.md) for more details. ## Usage diff --git a/docs/tutorial/yurtctl.md b/docs/tutorial/yurtctl.md index a1b66da9e5c..4bf54f47f6a 100644 --- a/docs/tutorial/yurtctl.md +++ b/docs/tutorial/yurtctl.md @@ -216,6 +216,20 @@ Note that before performing the uninstall, please make sure all edge nodes are r In addition, the path of the kubelet service configuration can be set by the option `--kubeadm-conf-path`, and the path of the directory on edge node containing static pod files can be set by the option `--pod-manifest-path`. +## Join Edge-Node/Cloud-Node to OpenYurt + +`yurtctl join` will automatically install the corresponding kubelet according to the cluster version, but the user needs to install the runtime in advance and ensure that the swap partition of the node has been closed. + +Using `yurtctl` to join an Edge-Node to OpenYurt cluster can be by doing the following: +``` +$ _output/bin/yurtctl join 1.2.3.4:6443 --token=zffaj3.a5vjzf09qn9ft3gt --node-type=edge-node --discovery-token-unsafe-skip-ca-verification --v=5 +``` + +Using `yurtctl` to join a Cloud-Node to OpenYurt cluster can be by doing the following: +``` +$ _output/bin/yurtctl join 1.2.3.4:6443 --token=zffaj3.a5vjzf09qn9ft3gt --node-type=cloud-node --discovery-token-unsafe-skip-ca-verification --v=5 +``` + ## Subcommand ### Convert a Kubernetes node to Yurt edge node diff --git a/go.mod b/go.mod index 2aee1f37af2..61348bee43f 100644 --- a/go.mod +++ b/go.mod @@ -11,17 +11,22 @@ require ( github.com/go-openapi/spec v0.19.8 // indirect github.com/google/uuid v1.1.1 github.com/gorilla/mux v1.7.4 + github.com/lithammer/dedent v1.1.0 github.com/onsi/ginkgo v1.13.0 github.com/onsi/gomega v1.10.1 github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/selinux v1.3.1-0.20190929122143-5215b1806f52 github.com/openyurtio/yurt-app-manager-api v0.18.8 + github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v1.7.1 github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 + github.com/stretchr/testify v1.4.0 github.com/vishvananda/netlink v1.0.0 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 // indirect google.golang.org/grpc v1.27.0 + gopkg.in/cheggaaa/pb.v1 v1.0.25 gopkg.in/square/go-jose.v2 v2.5.1 // indirect k8s.io/api v0.19.7 k8s.io/apimachinery v0.19.7 diff --git a/go.sum b/go.sum index f0f0c1fb49f..8fd8aa67d11 100644 --- a/go.sum +++ b/go.sum @@ -26,6 +26,7 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20190822182118-27a4ced34534/go.mod h1:iroGtC8B3tQiqtds1l+mgk/BBOrxbqjH+eUfFQYRc14= github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5hodBMJ5+l/7J4xAyMeuM2PNuepvHlGs8yilUCA= github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= +github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Microsoft/hcsshim v0.0.0-20190417211021-672e52e9209d/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= @@ -71,6 +72,7 @@ github.com/blang/semver v3.5.0+incompatible h1:CGxCgetQ64DKk7rdZ++Vfnb1+ogGNnB17 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= +github.com/caddyserver/caddy v1.0.3 h1:i9gRhBgvc5ifchwWtSe7pDpsdS9+Q0Rw9oYQmYUTw1w= github.com/caddyserver/caddy v1.0.3/go.mod h1:G+ouvOY32gENkJC+jhgl62TyhvqEsFaDiZ4uw0RzP1E= github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -89,6 +91,7 @@ github.com/containerd/console v0.0.0-20170925154832-84eeaae905fa/go.mod h1:Tj/on github.com/containerd/containerd v1.0.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/typeurl v0.0.0-20190228175220-2a93cfde8c20/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/coredns/corefile-migration v1.0.6 h1:hB6vclp2g/KeXe9n1oz/PafgieUahsOYeHMQA+RJ4Hg= github.com/coredns/corefile-migration v1.0.6/go.mod h1:OFwBp/Wc9dJt5cAZzHWMNhK1r5L0p0jDwIBc6j8NC8E= github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= @@ -149,6 +152,7 @@ github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= @@ -334,6 +338,7 @@ github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= github.com/gostaticanalysis/analysisutil v0.0.3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -402,7 +407,9 @@ github.com/kubernetes/kubernetes v1.18.8 h1:wpG4RPPyuNRgQ/L8yEAHrsH1FKprqE8vPNLf github.com/kubernetes/kubernetes v1.18.8/go.mod h1:SU7bBi8ZNHRjqzNhY4U78gClS1O7Q7avCrfF5aSiDko= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/libopenstorage/openstorage v1.0.0/go.mod h1:Sp1sIObHjat1BeXhfMqLZ14wnOzEhNx2YQedreMcUyc= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= +github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/lpabon/godbc v0.1.1/go.mod h1:Jo9QV0cf3U6jZABgiJ2skINAXb9j8m51r07g4KI92ZA= @@ -422,10 +429,13 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk= +github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-runewidth v0.0.2 h1:UnlwIPBGaTZfPQ6T1IGzPI0EkYAQmT9fAEJ/poFC63o= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-shellwords v1.0.5/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= @@ -489,6 +499,7 @@ github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3I github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v1.0.0-rc10/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runtime-spec v1.0.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/selinux v1.3.1-0.20190929122143-5215b1806f52 h1:B8hYj3NxHmjsC3T+tnlZ1UhInqUgnyF1zlGPmzNg2Qk= github.com/opencontainers/selinux v1.3.1-0.20190929122143-5215b1806f52/go.mod h1:+BLncwf63G4dgOzykXAxcmnFlUaOlkDdmw/CqsW6pjs= github.com/openyurtio/apiserver-network-proxy v1.18.8 h1:xXqaP8DAOvCHD7DNIqtBOhuWxCnwULLc1PqOMoJ7UeI= github.com/openyurtio/apiserver-network-proxy v1.18.8/go.mod h1:X5Au3jBNIgYL2uK0IHeNGnZqlUlVSCFQhi/npPgkKRg= @@ -498,6 +509,7 @@ github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -827,6 +839,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.25 h1:Ev7yu1/f6+d+b3pi5vPdRPc6nNtP1umSfcWiEfRqv6I= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -870,6 +883,7 @@ k8s.io/apimachinery v0.18.9-rc.0 h1:RnhcqZsTFI3l4z1iXNJhP0PrC4Gnrw5WlSjRhoRYifs= k8s.io/apimachinery v0.18.9-rc.0/go.mod h1:6sQd+iHEqmOtALqOFjSWp2KZ9F0wlU/nWm0ZgsYWMig= k8s.io/apiserver v0.18.8 h1:Au4kMn8sb1zFdyKqc8iMHLsYLxRI6Y+iAhRNKKQtlBY= k8s.io/apiserver v0.18.8/go.mod h1:12u5FuGql8Cc497ORNj79rhPdiXQC4bf53X/skR/1YM= +k8s.io/cli-runtime v0.18.8 h1:ycmbN3hs7CfkJIYxJAOB10iW7BVPmXGXkfEyiV9NJ+k= k8s.io/cli-runtime v0.18.8/go.mod h1:7EzWiDbS9PFd0hamHHVoCY4GrokSTPSL32MA4rzIu0M= k8s.io/client-go v0.18.8 h1:SdbLpIxk5j5YbFr1b7fq8S7mDgDjYmUxSbszyoesoDM= k8s.io/client-go v0.18.8/go.mod h1:HqFqMllQ5NnQJNwjro9k5zMyfhZlOwpuTLVrxjkYSxU= @@ -897,16 +911,19 @@ k8s.io/kube-aggregator v0.18.8/go.mod h1:CyLoGZB+io8eEwnn+6RbV7QWJQhj8a3TBH8ZM8s k8s.io/kube-controller-manager v0.18.8/go.mod h1:IYZteddXJFD1TVgAw8eRP3c9OOA2WtHdXdE8aH6gXnc= k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6 h1:Oh3Mzx5pJ+yIumsAD0MOECPVeXsVot0UkiaCGVyfGQY= k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= +k8s.io/kube-proxy v0.18.8 h1:b7+JbH0WbMh9Y41qqxEULby2RVVuZ6dp2oYHijGWv9Y= k8s.io/kube-proxy v0.18.8/go.mod h1:u4E8OsUpUzfZ9CEFf9rdLsbYiusZr8utbtF4WQrX+qs= k8s.io/kube-scheduler v0.18.8 h1:t08qdlU0UzBoKaOmNu0ripIHawR4emSJUTFdRZMg6Zk= k8s.io/kube-scheduler v0.18.8/go.mod h1:OeliYiILv1XkSq0nmQjRewgt5NimKsTidZFEhfL5fqA= k8s.io/kubectl v0.18.8 h1:qTkHCz21YmK0+S0oE6TtjtxmjeDP42gJcZJyRKsIenA= k8s.io/kubectl v0.18.8/go.mod h1:PlEgIAjOMua4hDFTEkVf+W5M0asHUKfE4y7VDZkpLHM= +k8s.io/kubelet v0.18.8 h1:81oth5jMmDScFUbtap7fEElsG5hx5o/GSOsin4Nmm+M= k8s.io/kubelet v0.18.8/go.mod h1:6z1jHCk0NPE6WshFStfqcgQ1bnD3tetcPmhC2915aio= k8s.io/legacy-cloud-providers v0.18.8/go.mod h1:tgp4xYf6lvjrWnjQwTOPvWQE9IVqSBGPF4on0IyICQE= k8s.io/metrics v0.18.8/go.mod h1:j7JzZdiyhLP2BsJm/Fzjs+j5Lb1Y7TySjhPWqBPwRXA= k8s.io/repo-infra v0.0.1-alpha.1/go.mod h1:wO1t9WaB99V80ljbeENTnayuEEwNZt7gECYh/CEyOJ8= k8s.io/sample-apiserver v0.18.8/go.mod h1:qXPfVwaZwM2owoSMNRRm9vw+HNJGLNsBpGckv1uxWy4= +k8s.io/system-validators v1.0.4 h1:sW57tJ/ciqOVbbTLN+ZNy64MJMNqUuiwrirQv8IR2Kw= k8s.io/system-validators v1.0.4/go.mod h1:HgSgTg4NAGNoYYjKsUyk52gdNi2PVDswQ9Iyn66R7NI= k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20200603063816-c1c6865ac451 h1:v8ud2Up6QK1lNOKFgiIVrZdMg7MpmSnvtrOieolJKoE= @@ -924,6 +941,7 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7 h1:uuHDyjllyzRyCI sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0= sigs.k8s.io/controller-runtime v0.5.7 h1:QcB8YQTyMshLLspHiqAkKHO74PgmUAUmTDhol4VccOw= sigs.k8s.io/controller-runtime v0.5.7/go.mod h1:KjjGQrdWFaSTHwB5A5VDmX9sMLlvkXjVazxVbfOI3a8= +sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbLXqhyOyX0= sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E= diff --git a/pkg/yurtctl/cmd/cmd.go b/pkg/yurtctl/cmd/cmd.go index 2831483555f..0cca604e294 100644 --- a/pkg/yurtctl/cmd/cmd.go +++ b/pkg/yurtctl/cmd/cmd.go @@ -19,6 +19,7 @@ package cmd import ( goflag "flag" "fmt" + "os" "github.com/spf13/cobra" flag "github.com/spf13/pflag" @@ -27,6 +28,7 @@ import ( "github.com/openyurtio/openyurt/pkg/projectinfo" "github.com/openyurtio/openyurt/pkg/yurtctl/cmd/clusterinfo" "github.com/openyurtio/openyurt/pkg/yurtctl/cmd/convert" + "github.com/openyurtio/openyurt/pkg/yurtctl/cmd/join" "github.com/openyurtio/openyurt/pkg/yurtctl/cmd/markautonomous" "github.com/openyurtio/openyurt/pkg/yurtctl/cmd/revert" ) @@ -55,6 +57,7 @@ func NewYurtctlCommand() *cobra.Command { cmds.AddCommand(revert.NewRevertCmd()) cmds.AddCommand(markautonomous.NewMarkAutonomousCmd()) cmds.AddCommand(clusterinfo.NewClusterInfoCmd()) + cmds.AddCommand(join.NewCmdJoin(os.Stdout, nil)) klog.InitFlags(nil) // goflag.Parse() diff --git a/pkg/yurtctl/cmd/convert/edgenode.go b/pkg/yurtctl/cmd/convert/edgenode.go index e204dac46fc..6a5dd33e619 100644 --- a/pkg/yurtctl/cmd/convert/edgenode.go +++ b/pkg/yurtctl/cmd/convert/edgenode.go @@ -352,7 +352,7 @@ func (c *ConvertEdgeNodeOptions) ResetKubelet() error { // 2. revise the kubelet.service drop-in // 2.1 make a backup for the origin kubelet.service bkfile := c.getKubeletSvcBackup() - err = enutil.CopyFile(c.KubeadmConfPath, bkfile) + err = enutil.CopyFile(c.KubeadmConfPath, bkfile, 0666) if err != nil { return err } diff --git a/pkg/yurtctl/cmd/join/join.go b/pkg/yurtctl/cmd/join/join.go new file mode 100644 index 00000000000..e49df8b2092 --- /dev/null +++ b/pkg/yurtctl/cmd/join/join.go @@ -0,0 +1,399 @@ +/* +Copyright 2021 The OpenYurt Authors. +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package join + +import ( + "fmt" + "io" + "os" + + "k8s.io/apimachinery/pkg/util/sets" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + "k8s.io/klog" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" + kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" + "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" + kubeadmPhase "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/join" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/discovery" + configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" + kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" + + "github.com/lithammer/dedent" + yurtphase "github.com/openyurtio/openyurt/pkg/yurtctl/cmd/join/phases" + "github.com/pkg/errors" + "github.com/spf13/cobra" + flag "github.com/spf13/pflag" +) + +var ( + joinWorkerNodeDoneMsg = dedent.Dedent(` + This node has joined the cluster: + * Certificate signing request was sent to apiserver and a response was received. + * The Kubelet was informed of the new secure connection details. + + Run 'kubectl get nodes' on the control-plane to see this node join the cluster. + + `) +) + +type joinOptions struct { + cfgPath string + token string + controlPlane bool + ignorePreflightErrors []string + externalcfg *kubeadmapiv1beta2.JoinConfiguration + kustomizeDir string + nodeType string + yurthubImage string +} + +// newJoinOptions returns a struct ready for being used for creating cmd join flags. +func newJoinOptions() *joinOptions { + // initialize the public kubeadm config API by applying defaults + externalcfg := &kubeadmapiv1beta2.JoinConfiguration{} + + // Add optional config objects to host flags. + // un-set objects will be cleaned up afterwards (into newJoinData func) + externalcfg.Discovery.File = &kubeadmapiv1beta2.FileDiscovery{} + externalcfg.Discovery.BootstrapToken = &kubeadmapiv1beta2.BootstrapTokenDiscovery{} + externalcfg.ControlPlane = &kubeadmapiv1beta2.JoinControlPlane{} + + // Apply defaults + kubeadmscheme.Scheme.Default(externalcfg) + + return &joinOptions{ + externalcfg: externalcfg, + } +} + +// NewJoinCmd returns "yurtctl join" command. +func NewCmdJoin(out io.Writer, joinOptions *joinOptions) *cobra.Command { + if joinOptions == nil { + joinOptions = newJoinOptions() + } + joinRunner := workflow.NewRunner() + + cmd := &cobra.Command{ + Use: "join [api-server-endpoint]", + Short: "Run this on any machine you wish to join an existing cluster", + RunE: func(cmd *cobra.Command, args []string) error { + c, err := joinRunner.InitData(args) + if err != nil { + return err + } + data := c.(*joinData) + if err := joinRunner.Run(args); err != nil { + return err + } + fmt.Fprint(data.outputWriter, joinWorkerNodeDoneMsg) + return nil + }, + } + + addJoinConfigFlags(cmd.Flags(), joinOptions) + + joinRunner.AppendPhase(yurtphase.NewPreparePhase()) + joinRunner.AppendPhase(kubeadmPhase.NewPreflightPhase()) + joinRunner.AppendPhase(yurtphase.NewEdgeNodePhase()) + joinRunner.AppendPhase(yurtphase.NewCloudNodePhase()) + joinRunner.AppendPhase(yurtphase.NewPostcheckPhase()) + + joinRunner.SetDataInitializer(func(cmd *cobra.Command, args []string) (workflow.RunData, error) { + return newJoinData(cmd, args, joinOptions, out) + }) + joinRunner.BindToCommand(cmd) + return cmd +} + +// addJoinConfigFlags adds join flags bound to the config to the specified flagset +func addJoinConfigFlags(flagSet *flag.FlagSet, joinOptions *joinOptions) { + flagSet.StringVar( + &joinOptions.externalcfg.NodeRegistration.Name, options.NodeName, joinOptions.externalcfg.NodeRegistration.Name, + `Specify the node name.`, + ) + flagSet.StringSliceVar( + &joinOptions.externalcfg.Discovery.BootstrapToken.CACertHashes, options.TokenDiscoveryCAHash, []string{}, + "For token-based discovery, validate that the root CA public key matches this hash (format: \":\").", + ) + flagSet.BoolVar( + &joinOptions.externalcfg.Discovery.BootstrapToken.UnsafeSkipCAVerification, options.TokenDiscoverySkipCAHash, false, + "For token-based discovery, allow joining without --discovery-token-ca-cert-hash pinning.", + ) + flagSet.StringSliceVar( + &joinOptions.ignorePreflightErrors, options.IgnorePreflightErrors, joinOptions.ignorePreflightErrors, + "A list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks.", + ) + flagSet.StringVar( + &joinOptions.token, options.TokenStr, "", + "Use this token for both discovery-token and tls-bootstrap-token when those values are not provided.", + ) + flagSet.StringVar( + &joinOptions.nodeType, "node-type", "", + "Sets the node is edge-node or cloud-node", + ) + flagSet.StringVar( + &joinOptions.yurthubImage, "yurthub-image", "", + "Sets the image version of yurthub component", + ) + cmdutil.AddCRISocketFlag(flagSet, &joinOptions.externalcfg.NodeRegistration.CRISocket) +} + +type joinData struct { + cfg *kubeadmapi.JoinConfiguration + initCfg *kubeadmapi.InitConfiguration + tlsBootstrapCfg *clientcmdapi.Config + clientSet *clientset.Clientset + ignorePreflightErrors sets.String + outputWriter io.Writer + kustomizeDir string + nodeType string + yurthubImage string +} + +// newJoinData returns a new joinData struct to be used for the execution of the kubeadm join workflow. +// This func takes care of validating joinOptions passed to the command, and then it converts +// options into the internal JoinConfiguration type that is used as input all the phases in the kubeadm join workflow +func newJoinData(cmd *cobra.Command, args []string, opt *joinOptions, out io.Writer) (*joinData, error) { + // Re-apply defaults to the public kubeadm API (this will set only values not exposed/not set as a flags) + kubeadmscheme.Scheme.Default(opt.externalcfg) + + // Validate standalone flags values and/or combination of flags and then assigns + // validated values to the public kubeadm config API when applicable + + // if a token is provided, use this value for both discovery-token and tls-bootstrap-token when those values are not provided + if len(opt.token) > 0 { + if len(opt.externalcfg.Discovery.TLSBootstrapToken) == 0 { + opt.externalcfg.Discovery.TLSBootstrapToken = opt.token + } + if len(opt.externalcfg.Discovery.BootstrapToken.Token) == 0 { + opt.externalcfg.Discovery.BootstrapToken.Token = opt.token + } + } + + // if a file or URL from which to load cluster information was not provided, unset the Discovery.File object + if len(opt.externalcfg.Discovery.File.KubeConfigPath) == 0 { + opt.externalcfg.Discovery.File = nil + } + + // if an APIServerEndpoint from which to retrieve cluster information was not provided, unset the Discovery.BootstrapToken object + if len(args) == 0 { + opt.externalcfg.Discovery.BootstrapToken = nil + } else { + if len(opt.cfgPath) == 0 && len(args) > 1 { + klog.Warningf("[preflight] WARNING: More than one API server endpoint supplied on command line %v. Using the first one.", args) + } + opt.externalcfg.Discovery.BootstrapToken.APIServerEndpoint = args[0] + } + + // if not joining a control plane, unset the ControlPlane object + if !opt.controlPlane { + if opt.externalcfg.ControlPlane != nil { + klog.Warningf("[preflight] WARNING: JoinControlPane.controlPlane settings will be ignored when %s flag is not set.", options.ControlPlane) + } + opt.externalcfg.ControlPlane = nil + } + + // if the admin.conf file already exists, use it for skipping the discovery process. + // NB. this case can happen when we are joining a control-plane node only (and phases are invoked atomically) + var adminKubeConfigPath = kubeadmconstants.GetAdminKubeConfigPath() + var tlsBootstrapCfg *clientcmdapi.Config + if _, err := os.Stat(adminKubeConfigPath); err == nil && opt.controlPlane { + // use the admin.conf as tlsBootstrapCfg, that is the kubeconfig file used for reading the kubeadm-config during discovery + klog.V(1).Infof("[preflight] found %s. Use it for skipping discovery", adminKubeConfigPath) + tlsBootstrapCfg, err = clientcmd.LoadFromFile(adminKubeConfigPath) + if err != nil { + return nil, errors.Wrapf(err, "Error loading %s", adminKubeConfigPath) + } + } + + if err := validation.ValidateMixedArguments(cmd.Flags()); err != nil { + return nil, err + } + + // Either use the config file if specified, or convert public kubeadm API to the internal JoinConfiguration + // and validates JoinConfiguration + if opt.externalcfg.NodeRegistration.Name == "" { + klog.V(1).Infoln("[preflight] found NodeName empty; using OS hostname as NodeName") + } + + if opt.externalcfg.ControlPlane != nil && opt.externalcfg.ControlPlane.LocalAPIEndpoint.AdvertiseAddress == "" { + klog.V(1).Infoln("[preflight] found advertiseAddress empty; using default interface's IP address as advertiseAddress") + } + + cfg, err := configutil.LoadOrDefaultJoinConfiguration(opt.cfgPath, opt.externalcfg) + if err != nil { + return nil, err + } + + ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(opt.ignorePreflightErrors, cfg.NodeRegistration.IgnorePreflightErrors) + if err != nil { + return nil, err + } + // Also set the union of pre-flight errors to JoinConfiguration, to provide a consistent view of the runtime configuration: + cfg.NodeRegistration.IgnorePreflightErrors = ignorePreflightErrorsSet.List() + + // override node name and CRI socket from the command line opt + if opt.externalcfg.NodeRegistration.Name != "" { + cfg.NodeRegistration.Name = opt.externalcfg.NodeRegistration.Name + } + if opt.externalcfg.NodeRegistration.CRISocket != "" { + cfg.NodeRegistration.CRISocket = opt.externalcfg.NodeRegistration.CRISocket + } + + if cfg.ControlPlane != nil { + if err := configutil.VerifyAPIServerBindAddress(cfg.ControlPlane.LocalAPIEndpoint.AdvertiseAddress); err != nil { + return nil, err + } + } + + return &joinData{ + cfg: cfg, + tlsBootstrapCfg: tlsBootstrapCfg, + ignorePreflightErrors: ignorePreflightErrorsSet, + outputWriter: out, + kustomizeDir: opt.kustomizeDir, + nodeType: opt.nodeType, + yurthubImage: opt.yurthubImage, + }, nil +} + +// CertificateKey returns the key used to encrypt the certs. +func (j *joinData) CertificateKey() string { + if j.cfg.ControlPlane != nil { + return j.cfg.ControlPlane.CertificateKey + } + return "" +} + +// Cfg returns the JoinConfiguration. +func (j *joinData) Cfg() *kubeadmapi.JoinConfiguration { + return j.cfg +} + +// TLSBootstrapCfg returns the cluster-info (kubeconfig). +func (j *joinData) TLSBootstrapCfg() (*clientcmdapi.Config, error) { + if j.tlsBootstrapCfg != nil { + return j.tlsBootstrapCfg, nil + } + klog.V(1).Infoln("[preflight] Discovering cluster-info") + tlsBootstrapCfg, err := discovery.For(j.cfg) + j.tlsBootstrapCfg = tlsBootstrapCfg + return tlsBootstrapCfg, err +} + +// InitCfg returns the InitConfiguration. +func (j *joinData) InitCfg() (*kubeadmapi.InitConfiguration, error) { + if j.initCfg != nil { + return j.initCfg, nil + } + if _, err := j.TLSBootstrapCfg(); err != nil { + return nil, err + } + klog.V(1).Infoln("[preflight] Fetching init configuration") + initCfg, err := fetchInitConfigurationFromJoinConfiguration(j.cfg, j.tlsBootstrapCfg) + j.initCfg = initCfg + return initCfg, err +} + +// fetchInitConfigurationFromJoinConfiguration retrieves the init configuration from a join configuration, performing the discovery +func fetchInitConfigurationFromJoinConfiguration(cfg *kubeadmapi.JoinConfiguration, tlsBootstrapCfg *clientcmdapi.Config) (*kubeadmapi.InitConfiguration, error) { + // Retrieves the kubeadm configuration + klog.V(1).Infoln("[preflight] Retrieving KubeConfig objects") + initConfiguration, err := fetchInitConfiguration(tlsBootstrapCfg) + if err != nil { + return nil, err + } + + // Create the final KubeConfig file with the cluster name discovered after fetching the cluster configuration + clusterinfo := kubeconfigutil.GetClusterFromKubeConfig(tlsBootstrapCfg) + tlsBootstrapCfg.Clusters = map[string]*clientcmdapi.Cluster{ + initConfiguration.ClusterName: clusterinfo, + } + tlsBootstrapCfg.Contexts[tlsBootstrapCfg.CurrentContext].Cluster = initConfiguration.ClusterName + + // injects into the kubeadm configuration the information about the joining node + initConfiguration.NodeRegistration = cfg.NodeRegistration + if cfg.ControlPlane != nil { + initConfiguration.LocalAPIEndpoint = cfg.ControlPlane.LocalAPIEndpoint + } + + return initConfiguration, nil +} + +// fetchInitConfiguration reads the cluster configuration from the kubeadm-admin configMap +func fetchInitConfiguration(tlsBootstrapCfg *clientcmdapi.Config) (*kubeadmapi.InitConfiguration, error) { + // creates a client to access the cluster using the bootstrap token identity + tlsClient, err := kubeconfigutil.ToClientSet(tlsBootstrapCfg) + if err != nil { + return nil, errors.Wrap(err, "unable to access the cluster") + } + + // Fetches the init configuration + initConfiguration, err := configutil.FetchInitConfigurationFromCluster(tlsClient, os.Stdout, "preflight", true) + if err != nil { + return nil, errors.Wrap(err, "unable to fetch the kubeadm-config ConfigMap") + } + + return initConfiguration, nil +} + +// ClientSet returns the ClientSet for accessing the cluster with the identity defined in admin.conf. +func (j *joinData) ClientSet() (*clientset.Clientset, error) { + if j.clientSet != nil { + return j.clientSet, nil + } + path := kubeadmconstants.GetAdminKubeConfigPath() + client, err := kubeconfigutil.ClientSetFromFile(path) + if err != nil { + return nil, err + } + j.clientSet = client + return client, nil +} + +// IgnorePreflightErrors returns the list of preflight errors to ignore. +func (j *joinData) IgnorePreflightErrors() sets.String { + return j.ignorePreflightErrors +} + +// OutputWriter returns the io.Writer used to write messages such as the "join done" message. +func (j *joinData) OutputWriter() io.Writer { + return j.outputWriter +} + +// KustomizeDir returns the folder where kustomize patches for static pod manifest are stored +func (j *joinData) KustomizeDir() string { + return j.kustomizeDir +} + +//NodeType returns the node is cloud-node or edge-node. +func (j *joinData) NodeType() string { + return j.nodeType +} + +//YurtHubImage returns the YurtHub image. +func (j *joinData) YurtHubImage() string { + return j.yurthubImage +} diff --git a/pkg/yurtctl/cmd/join/phases/constants.go b/pkg/yurtctl/cmd/join/phases/constants.go new file mode 100644 index 00000000000..7c58e63e979 --- /dev/null +++ b/pkg/yurtctl/cmd/join/phases/constants.go @@ -0,0 +1,92 @@ +package phases + +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +const ( + ip_forward = "/proc/sys/net/ipv4/ip_forward" + + bridgenf = "/proc/sys/net/bridge/bridge-nf-call-iptables" + bridgenf6 = "/proc/sys/net/bridge/bridge-nf-call-ip6tables" + sysctl_k8s_config = "/etc/sysctl.d/k8s.conf" + kubernetsBridgeSetting = `net.bridge.bridge-nf-call-ip6tables = 1 +net.bridge.bridge-nf-call-iptables = 1` + tmpDownloadDir = "/tmp" + yurtHubStaticPodYamlFile = "/etc/kubernetes/manifests/yurthub.yaml" + defaultYurthubImage = "registry.cn-hangzhou.aliyuncs.com/openyurt/yurthub:v0.4.0" + + kubeCniDir = "/opt/cni/bin" + kubeCniVersion = "v0.8.0" + cniUrlFormat = "https://aliacs-edge-k8s-cn-hangzhou.oss-cn-hangzhou.aliyuncs.com/public/pkg/openyurt/cni/%s/cni-plugins-linux-%s-%s.tgz" + kubeUrlFormat = "https://dl.k8s.io/%s/kubernetes-node-linux-%s.tar.gz" + staticPodPath = "/etc/kubernetes/manifests" + + EdgeNode = "edge-node" + CloudNode = "cloud-node" +) + +const ( + KubeletServiceFilepath string = "/etc/systemd/system/kubelet.service" + kubeletServiceContent = `[Unit] +Description=kubelet: The Kubernetes Node Agent +Documentation=http://kubernetes.io/docs/ + +[Service] +ExecStartPre=/sbin/swapoff -a +ExecStart=/usr/bin/kubelet +Restart=always +StartLimitInterval=0 +RestartSec=10 + +[Install] +WantedBy=multi-user.target` + + edgeKubeletUnitConfig = ` +[Service] +Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf" +Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" +EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env +EnvironmentFile=-/etc/default/kubelet +ExecStart= +ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS +` + cloudKubeletUnitConfig = ` +[Service] +Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf" +Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" +EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env +EnvironmentFile=-/etc/default/kubelet +ExecStart= +ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS +` + + kubeletConfForEdgeNode = ` +apiVersion: v1 +clusters: +- cluster: + server: http://127.0.0.1:10261 + name: default-cluster +contexts: +- context: + cluster: default-cluster + namespace: default + user: default-auth + name: default-context +current-context: default-context +kind: Config +preferences: {} +` +) diff --git a/pkg/yurtctl/cmd/join/phases/join-cloud-node.go b/pkg/yurtctl/cmd/join/phases/join-cloud-node.go new file mode 100644 index 00000000000..9d8729effae --- /dev/null +++ b/pkg/yurtctl/cmd/join/phases/join-cloud-node.go @@ -0,0 +1,224 @@ +/* +Copyright 2021 The OpenYurt Authors. +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package phases + +import ( + "context" + "fmt" + "os" + + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/version" + "k8s.io/apimachinery/pkg/util/wait" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + certutil "k8s.io/client-go/util/cert" + "k8s.io/klog" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" + patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode" + "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" + kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" + + "github.com/lithammer/dedent" + "github.com/pkg/errors" +) + +var ( + kubeadmJoinFailMsg = dedent.Dedent(` + Unfortunately, an error has occurred: + %v + + This error is likely caused by: + - The kubelet is not running + - The kubelet is unhealthy due to a misconfiguration of the node in some way (required cgroups disabled) + + If you are on a systemd-powered system, you can try to troubleshoot the error with the following commands: + - 'systemctl status kubelet' + - 'journalctl -xeu kubelet' + `) +) + +// NewCloudNodePhase creates a yurtctl workflow phase that start kubelet on a cloud node. +func NewCloudNodePhase() workflow.Phase { + return workflow.Phase{ + Name: "kubelet-start [api-server-endpoint]", + Short: "Write kubelet settings, certificates and (re)start the kubelet", + Long: "Write a file with KubeletConfiguration and an environment file with node specific kubelet settings, and then (re)start kubelet.", + Run: runKubeletStartJoinPhase, + InheritFlags: []string{ + options.CfgPath, + options.NodeCRISocket, + options.NodeName, + options.FileDiscovery, + options.TokenDiscovery, + options.TokenDiscoveryCAHash, + options.TokenDiscoverySkipCAHash, + options.TLSBootstrapToken, + options.TokenStr, + }, + } +} + +//getCloudNodeJoinData get node configuration for cloud-node. +func getCloudNodeJoinData(c workflow.RunData) (*kubeadmapi.JoinConfiguration, *kubeadmapi.InitConfiguration, *clientcmdapi.Config, error) { + data, ok := c.(YurtJoinData) + if !ok { + return nil, nil, nil, errors.New("kubelet-start phase invoked with an invalid data struct") + } + cfg := data.Cfg() + initCfg, err := data.InitCfg() + if err != nil { + return nil, nil, nil, err + } + tlsBootstrapCfg, err := data.TLSBootstrapCfg() + if err != nil { + return nil, nil, nil, err + } + return cfg, initCfg, tlsBootstrapCfg, nil +} + +// runKubeletStartJoinPhase executes the kubelet TLS bootstrap process. +// This process is executed by the kubelet and completes with the node joining the cluster +// with a dedicates set of credentials as required by the node authorizer +func runKubeletStartJoinPhase(c workflow.RunData) (returnErr error) { + data, ok := c.(YurtJoinData) + if !ok { + return errors.New("kubelet-start phase invoked with an invalid data struct") + } + if data.NodeType() != CloudNode { + return + } + cfg, initCfg, tlsBootstrapCfg, err := getCloudNodeJoinData(c) + if err != nil { + return err + } + bootstrapKubeConfigFile := kubeadmconstants.GetBootstrapKubeletKubeConfigPath() + + // Deletes the bootstrapKubeConfigFile, so the credential used for TLS bootstrap is removed from disk + defer os.Remove(bootstrapKubeConfigFile) + + // Write the bootstrap kubelet config file or the TLS-Bootstrapped kubelet config file down to disk + klog.V(1).Infof("[kubelet-start] writing bootstrap kubelet config file at %s", bootstrapKubeConfigFile) + if err := kubeconfigutil.WriteToDisk(bootstrapKubeConfigFile, tlsBootstrapCfg); err != nil { + return errors.Wrap(err, "couldn't save bootstrap-kubelet.conf to disk") + } + + // Write the ca certificate to disk so kubelet can use it for authentication + cluster := tlsBootstrapCfg.Contexts[tlsBootstrapCfg.CurrentContext].Cluster + if _, err := os.Stat(cfg.CACertPath); os.IsNotExist(err) { + klog.V(1).Infof("[kubelet-start] writing CA certificate at %s", cfg.CACertPath) + if err := certutil.WriteCert(cfg.CACertPath, tlsBootstrapCfg.Clusters[cluster].CertificateAuthorityData); err != nil { + return errors.Wrap(err, "couldn't save the CA certificate to disk") + } + } + + kubeletVersion, err := version.ParseSemantic(initCfg.ClusterConfiguration.KubernetesVersion) + if err != nil { + return err + } + + bootstrapClient, err := kubeconfigutil.ClientSetFromFile(bootstrapKubeConfigFile) + if err != nil { + return errors.Errorf("couldn't create client from kubeconfig file %q", bootstrapKubeConfigFile) + } + + // Obtain the name of this Node. + nodeName, _, err := kubeletphase.GetNodeNameAndHostname(&cfg.NodeRegistration) + if err != nil { + klog.Warning(err) + } + + // Make sure to exit before TLS bootstrap if a Node with the same name exist in the cluster + // and it has the "Ready" status. + // A new Node with the same name as an existing control-plane Node can cause undefined + // behavior and ultimately control-plane failure. + klog.V(1).Infof("[kubelet-start] Checking for an existing Node in the cluster with name %q and status %q", nodeName, v1.NodeReady) + node, err := bootstrapClient.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return errors.Wrapf(err, "cannot get Node %q", nodeName) + } + for _, cond := range node.Status.Conditions { + if cond.Type == v1.NodeReady && cond.Status == v1.ConditionTrue { + return errors.Errorf("a Node with name %q and status %q already exists in the cluster. "+ + "You must delete the existing Node or change the name of this new joining Node", nodeName, v1.NodeReady) + } + } + + // Configure the kubelet. In this short timeframe, kubeadm is trying to stop/restart the kubelet + // Try to stop the kubelet service so no race conditions occur when configuring it + klog.V(1).Infoln("[kubelet-start] Stopping the kubelet") + kubeletphase.TryStopKubelet() + + // Write the configuration for the kubelet (using the bootstrap token credentials) to disk so the kubelet can start + if err := kubeletphase.DownloadConfig(bootstrapClient, kubeletVersion, kubeadmconstants.KubeletRunDirectory); err != nil { + return err + } + + // Write env file with flags for the kubelet to use. We only want to + // register the joining node with the specified taints if the node + // is not a control-plane. The mark-control-plane phase will register the taints otherwise. + registerTaintsUsingFlags := cfg.ControlPlane == nil + if err := kubeletphase.WriteKubeletDynamicEnvFile(&initCfg.ClusterConfiguration, &initCfg.NodeRegistration, registerTaintsUsingFlags, kubeadmconstants.KubeletRunDirectory); err != nil { + return err + } + + // Try to start the kubelet service in case it's inactive + fmt.Println("[kubelet-start] Starting the kubelet") + kubeletphase.TryStartKubelet() + + // Now the kubelet will perform the TLS Bootstrap, transforming /etc/kubernetes/bootstrap-kubelet.conf to /etc/kubernetes/kubelet.conf + // Wait for the kubelet to create the /etc/kubernetes/kubelet.conf kubeconfig file. If this process + // times out, display a somewhat user-friendly message. + waiter := apiclient.NewKubeWaiter(nil, kubeadmconstants.TLSBootstrapTimeout, os.Stdout) + if err := waiter.WaitForKubeletAndFunc(waitForTLSBootstrappedClient); err != nil { + fmt.Printf(kubeadmJoinFailMsg, err) + return err + } + + // When we know the /etc/kubernetes/kubelet.conf file is available, get the client + client, err := kubeconfigutil.ClientSetFromFile(kubeadmconstants.GetKubeletKubeConfigPath()) + if err != nil { + return err + } + + klog.V(1).Infoln("[kubelet-start] preserving the crisocket information for the node") + if err := patchnodephase.AnnotateCRISocket(client, cfg.NodeRegistration.Name, cfg.NodeRegistration.CRISocket); err != nil { + return errors.Wrap(err, "error uploading crisocket") + } + + return nil +} + +// waitForTLSBootstrappedClient waits for the /etc/kubernetes/kubelet.conf file to be available +func waitForTLSBootstrappedClient() error { + fmt.Println("[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...") + + // Loop on every falsy return. Return with an error if raised. Exit successfully if true is returned. + return wait.PollImmediate(kubeadmconstants.TLSBootstrapRetryInterval, kubeadmconstants.TLSBootstrapTimeout, func() (bool, error) { + // Check that we can create a client set out of the kubelet kubeconfig. This ensures not + // only that the kubeconfig file exists, but that other files required by it also exist (like + // client certificate and key) + _, err := kubeconfigutil.ClientSetFromFile(kubeadmconstants.GetKubeletKubeConfigPath()) + return (err == nil), nil + }) +} diff --git a/pkg/yurtctl/cmd/join/phases/join-edge-node.go b/pkg/yurtctl/cmd/join/phases/join-edge-node.go new file mode 100644 index 00000000000..8c783c0e16c --- /dev/null +++ b/pkg/yurtctl/cmd/join/phases/join-edge-node.go @@ -0,0 +1,167 @@ +package phases + +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + + "k8s.io/apimachinery/pkg/util/version" + clientset "k8s.io/client-go/kubernetes" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + certutil "k8s.io/client-go/util/cert" + "k8s.io/klog" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" + kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" + + "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode" +) + +// NewEdgeNodePhase creates a yurtctl workflow phase that start kubelet on a edge node. +func NewEdgeNodePhase() workflow.Phase { + return workflow.Phase{ + Name: "Join edge-node to OpenYurt cluster. ", + Short: "Join edge-node", + Run: runJoinEdgeNode, + } +} + +//runJoinEdgeNode executes the edge node join process. +func runJoinEdgeNode(c workflow.RunData) error { + data, ok := c.(YurtJoinData) + if !ok { + return fmt.Errorf("Join edge-node phase invoked with an invalid data struct. ") + } + if data.NodeType() != EdgeNode { + return nil + } + cfg, initCfg, tlsBootstrapCfg, err := getEdgeNodeJoinData(data) + if err != nil { + return err + } + + if err := setKubeleConfigForEdgeNode(); err != nil { + return err + } + if err := addYurthubStaticYaml(cfg, data.YurtHubImage()); err != nil { + return err + } + clusterinfo := kubeconfigutil.GetClusterFromKubeConfig(tlsBootstrapCfg) + if err := certutil.WriteCert(edgenode.KubeCaFile, clusterinfo.CertificateAuthorityData); err != nil { + return err + } + + tlsClient, err := kubeconfigutil.ToClientSet(tlsBootstrapCfg) + if err != nil { + return err + } + if err := getKubeletConfig(cfg, initCfg, tlsClient); err != nil { + return err + } + klog.Info("[kubelet-start] Starting the kubelet") + kubeletphase.TryStartKubelet() + return nil +} + +//setKubeleConfigForEdgeNode write kubelet.conf for edge-node. +func setKubeleConfigForEdgeNode() error { + kubeletConfigDir := filepath.Dir(edgenode.KubeCondfigPath) + if _, err := os.Stat(kubeletConfigDir); err != nil { + if os.IsNotExist(err) { + if err := os.MkdirAll(kubeletConfigDir, os.ModePerm); err != nil { + klog.Errorf("Create dir %s fail: %v", kubeletConfigDir, err) + return err + } + } else { + klog.Errorf("Describe dir %s fail: %v", kubeletConfigDir, err) + return err + } + } + if err := ioutil.WriteFile(edgenode.KubeCondfigPath, []byte(kubeletConfForEdgeNode), 0755); err != nil { + return err + } + return nil +} + +//addYurthubStaticYaml generate YurtHub static yaml for edge-node. +func addYurthubStaticYaml(cfg *kubeadmapi.JoinConfiguration, yurthubImage string) error { + klog.Info("[join-node] Adding edge hub static yaml") + if len(yurthubImage) == 0 { + yurthubImage = defaultYurthubImage + } + kubeletStaticPodDir := filepath.Dir(yurtHubStaticPodYamlFile) + if _, err := os.Stat(kubeletStaticPodDir); err != nil { + if os.IsNotExist(err) { + err = os.MkdirAll(kubeletStaticPodDir, os.ModePerm) + if err != nil { + return err + } + } else { + klog.Errorf("Describe dir %s fail: %v", kubeletStaticPodDir, err) + return err + } + } + + yurthubTemplate := edgenode.ReplaceRegularExpression(edgenode.YurthubTemplate, + map[string]string{ + "__kubernetes_service_addr__": fmt.Sprintf("https://%s", cfg.Discovery.BootstrapToken.APIServerEndpoint), + "__yurthub_image__": yurthubImage, + "__join_token__": cfg.Discovery.BootstrapToken.Token, + }) + + if err := ioutil.WriteFile(yurtHubStaticPodYamlFile, []byte(yurthubTemplate), 0600); err != nil { + return err + } + klog.Info("[join-node] Add edge hub static yaml is ok") + return nil +} + +//getKubeletConfig get kubelet configure from master. +func getKubeletConfig(cfg *kubeadmapi.JoinConfiguration, initCfg *kubeadmapi.InitConfiguration, tlsClient *clientset.Clientset) error { + kubeletVersion, err := version.ParseSemantic(initCfg.ClusterConfiguration.KubernetesVersion) + if err != nil { + return err + } + + // Write the configuration for the kubelet (using the bootstrap token credentials) to disk so the kubelet can start + if err := kubeletphase.DownloadConfig(tlsClient, kubeletVersion, kubeadmconstants.KubeletRunDirectory); err != nil { + return err + } + if err := kubeletphase.WriteKubeletDynamicEnvFile(&initCfg.ClusterConfiguration, &cfg.NodeRegistration, false, kubeadmconstants.KubeletRunDirectory); err != nil { + return err + } + return nil +} + +//getEdgeNodeJoinData get edge-node join configuration. +func getEdgeNodeJoinData(data YurtJoinData) (*kubeadmapi.JoinConfiguration, *kubeadmapi.InitConfiguration, *clientcmdapi.Config, error) { + cfg := data.Cfg() + initCfg, err := data.InitCfg() + if err != nil { + return nil, nil, nil, err + } + tlsBootstrapCfg, err := data.TLSBootstrapCfg() + if err != nil { + return nil, nil, nil, err + } + return cfg, initCfg, tlsBootstrapCfg, nil +} diff --git a/pkg/yurtctl/cmd/join/phases/postcheck.go b/pkg/yurtctl/cmd/join/phases/postcheck.go new file mode 100644 index 00000000000..091d4d3d754 --- /dev/null +++ b/pkg/yurtctl/cmd/join/phases/postcheck.go @@ -0,0 +1,136 @@ +package phases + +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import ( + "fmt" + "io/ioutil" + "net/http" + "time" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog" + "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode" + "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" + "k8s.io/kubernetes/cmd/kubeadm/app/util/initsystem" + kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" + + "github.com/openyurtio/openyurt/pkg/projectinfo" + "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode" +) + +//NewPostcheckPhase creates a yurtctl workflow phase that check the health status of node components. +func NewPostcheckPhase() workflow.Phase { + return workflow.Phase{ + Name: "postcheck", + Short: "postcheck", + Run: runPostcheck, + } +} + +//runPostcheck executes the node health check process. +func runPostcheck(c workflow.RunData) error { + j, ok := c.(YurtJoinData) + if !ok { + return fmt.Errorf("Postcheck edge-node phase invoked with an invalid data struct. ") + } + + klog.V(1).Infof("check kubelet status.") + if err := checkKubeletStatus(); err != nil { + return err + } + + cfg := j.Cfg() + if j.NodeType() == EdgeNode { + klog.V(1).Infof("waiting yurt hub ready.") + if err := checkYurthubHealthz(); err != nil { + return err + } + return patchEdgeNode(cfg) + } + return patchCloudNode(cfg) +} + +//checkKubeletStatus check if kubelet is healthy. +func checkKubeletStatus() error { + initSystem, err := initsystem.GetInitSystem() + if err != nil { + return err + } + if ok := initSystem.ServiceIsActive("kubelet"); !ok { + return fmt.Errorf("kubelet is not active. ") + } + return nil +} + +//checkYurthubHealthz check if YurtHub is healthy. +func checkYurthubHealthz() error { + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s%s", edgenode.ServerHealthzServer, edgenode.ServerHealthzURLPath), nil) + if err != nil { + return err + } + client := &http.Client{} + return wait.PollImmediate(time.Second*5, 300*time.Second, func() (bool, error) { + resp, err := client.Do(req) + if err != nil { + return false, nil + } + ok, err := ioutil.ReadAll(resp.Body) + if err != nil { + return false, nil + } + return string(ok) == "OK", nil + }) +} + +//patchEdgeNode patch labels and annotations for edge-node. +func patchEdgeNode(cfg *kubeadm.JoinConfiguration) error { + client, err := kubeconfigutil.ClientSetFromFile(kubeadmconstants.GetKubeletKubeConfigPath()) + if err != nil { + return err + } + if err := patchnodephase.AnnotateCRISocket(client, cfg.NodeRegistration.Name, cfg.NodeRegistration.CRISocket); err != nil { + return err + } + if err := apiclient.PatchNode(client, cfg.NodeRegistration.Name, func(n *v1.Node) { + n.Labels[projectinfo.GetEdgeWorkerLabelKey()] = "true" + }); err != nil { + return err + } + return nil +} + +//patchCloudNode patch labels and annotations for cloud-node. +func patchCloudNode(cfg *kubeadm.JoinConfiguration) error { + client, err := kubeconfigutil.ClientSetFromFile(kubeadmconstants.GetKubeletKubeConfigPath()) + if err != nil { + return err + } + if err := patchnodephase.AnnotateCRISocket(client, cfg.NodeRegistration.Name, cfg.NodeRegistration.CRISocket); err != nil { + return err + } + if err := apiclient.PatchNode(client, cfg.NodeRegistration.Name, func(n *v1.Node) { + n.Labels[projectinfo.GetEdgeWorkerLabelKey()] = "false" + }); err != nil { + return err + } + return nil +} diff --git a/pkg/yurtctl/cmd/join/phases/prepare.go b/pkg/yurtctl/cmd/join/phases/prepare.go new file mode 100644 index 00000000000..c28756b4b88 --- /dev/null +++ b/pkg/yurtctl/cmd/join/phases/prepare.go @@ -0,0 +1,219 @@ +package phases + +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + + "k8s.io/klog" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + + selinux "github.com/opencontainers/selinux/go-selinux" + "github.com/openyurtio/openyurt/pkg/yurtctl/util" + "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode" +) + +// NewEdgeNodePhase creates a yurtctl workflow phase that initialize the node environment. +func NewPreparePhase() workflow.Phase { + return workflow.Phase{ + Name: "Initialize system environment.", + Short: "Initialize system environment.", + Run: runPrepare, + } +} + +//runPrepare executes the node initialization process. +func runPrepare(c workflow.RunData) error { + data, ok := c.(YurtJoinData) + if !ok { + return fmt.Errorf("Prepare phase invoked with an invalid data struct. ") + } + + initCfg, err := data.InitCfg() + if err != nil { + return err + } + + if err := setIpv4Forward(); err != nil { + return err + } + if err := setBridgeSetting(); err != nil { + return err + } + if err := setSELinux(); err != nil { + return err + } + if err := checkAndInstallKubelet(initCfg.ClusterConfiguration.KubernetesVersion); err != nil { + return err + } + if err := setKubeletService(); err != nil { + return err + } + if err := setKubeletUnitConfig(data.NodeType()); err != nil { + return err + } + return nil +} + +//setIpv4Forward turn on the node ipv4 forward. +func setIpv4Forward() error { + klog.Infof("Setting ipv4 forward") + if err := ioutil.WriteFile(ip_forward, []byte("1"), 0644); err != nil { + return fmt.Errorf("Write content 1 to file %s fail: %v ", ip_forward, err) + } + return nil +} + +//setBridgeSetting turn on the node bridge-nf-call-iptables. +func setBridgeSetting() error { + klog.Info("Setting bridge settings for kubernetes.") + if err := ioutil.WriteFile(sysctl_k8s_config, []byte(kubernetsBridgeSetting), 0644); err != nil { + return fmt.Errorf("Write file %s fail: %v ", sysctl_k8s_config, err) + } + if err := ioutil.WriteFile(bridgenf, []byte("1"), 0644); err != nil { + return fmt.Errorf("Write file %s fail: %v ", bridgenf, err) + } + if err := ioutil.WriteFile(bridgenf6, []byte("1"), 0644); err != nil { + return fmt.Errorf("Write file %s fail: %v ", bridgenf, err) + } + return nil +} + +// setSELinux turn off the node selinux. +func setSELinux() error { + klog.Info("Disabling SELinux.") + selinux.SetDisabled() + return nil +} + +//checkAndInstallKubelet install kubelet and kubernetes-cni, skip install if they exist. +func checkAndInstallKubelet(clusterVersion string) error { + klog.Info("Check and install kubelet.") + kubeletExist := false + if _, err := exec.LookPath("kubelet"); err == nil { + if b, err := exec.Command("kubelet", "--version").CombinedOutput(); err == nil { + kubeletVersion := strings.Split(string(b), " ")[1] + kubeletVersion = strings.TrimSpace(kubeletVersion) + klog.Infof("kubelet --version: %s", kubeletVersion) + if strings.Contains(string(b), clusterVersion) { + klog.Infof("Kubelet %s already exist, skip install.", clusterVersion) + kubeletExist = true + } else { + return fmt.Errorf("The existing kubelet version %s of the node is inconsistent with cluster version %s, please clean it. ", kubeletVersion, clusterVersion) + } + } + } + + if !kubeletExist { + //download and install kubernetes-node + packageUrl := fmt.Sprintf(kubeUrlFormat, clusterVersion, runtime.GOARCH) + savePath := fmt.Sprintf("%s/kubernetes-node-linux-%s.tar.gz", tmpDownloadDir, runtime.GOARCH) + klog.V(1).Infof("Download kubelet from: %s", packageUrl) + if err := util.DownloadFile(packageUrl, savePath, 3); err != nil { + return fmt.Errorf("Download kuelet fail: %v", err) + } + if err := util.Untar(savePath, tmpDownloadDir); err != nil { + return err + } + for _, comp := range []string{"kubectl", "kubeadm", "kubelet"} { + target := fmt.Sprintf("/usr/bin/%s", comp) + if err := edgenode.CopyFile(tmpDownloadDir+"/kubernetes/node/bin/"+comp, target, 0755); err != nil { + return err + } + } + } + if _, err := os.Stat(staticPodPath); os.IsNotExist(err) { + if err := os.MkdirAll(staticPodPath, 0755); err != nil { + return err + } + } + + if _, err := os.Stat(kubeCniDir); err == nil { + klog.Infof("Cni dir %s already exist, skip install.", kubeCniDir) + return nil + } + //download and install kubernetes-cni + cniUrl := fmt.Sprintf(cniUrlFormat, kubeCniVersion, runtime.GOARCH, kubeCniVersion) + savePath := fmt.Sprintf("%s/cni-plugins-linux-%s-%s.tgz", tmpDownloadDir, runtime.GOARCH, kubeCniVersion) + klog.V(1).Infof("Download cni from: %s", cniUrl) + if err := util.DownloadFile(cniUrl, savePath, 3); err != nil { + return err + } + + if err := os.MkdirAll(kubeCniDir, 0600); err != nil { + return err + } + if err := util.Untar(savePath, kubeCniDir); err != nil { + return err + } + return nil +} + +// setKubeletService configure kubelet service. +func setKubeletService() error { + klog.Info("Setting kubelet service.") + kubeletServiceDir := filepath.Dir(KubeletServiceFilepath) + if _, err := os.Stat(kubeletServiceDir); err != nil { + if os.IsNotExist(err) { + if err := os.MkdirAll(kubeletServiceDir, os.ModePerm); err != nil { + klog.Errorf("Create dir %s fail: %v", kubeletServiceDir, err) + return err + } + } else { + klog.Errorf("Describe dir %s fail: %v", kubeletServiceDir, err) + return err + } + } + if err := ioutil.WriteFile(KubeletServiceFilepath, []byte(kubeletServiceContent), 0644); err != nil { + klog.Errorf("Write file %s fail: %v", KubeletServiceFilepath, err) + return err + } + return nil +} + +//setKubeletUnitConfig configure kubelet startup parameters. +func setKubeletUnitConfig(nodeType string) error { + kubeletUnitDir := filepath.Dir(edgenode.KubeletSvcPath) + if _, err := os.Stat(kubeletUnitDir); err != nil { + if os.IsNotExist(err) { + if err := os.MkdirAll(kubeletUnitDir, os.ModePerm); err != nil { + klog.Errorf("Create dir %s fail: %v", kubeletUnitDir, err) + return err + } + } else { + klog.Errorf("Describe dir %s fail: %v", kubeletUnitDir, err) + return err + } + } + if nodeType == EdgeNode { + if err := ioutil.WriteFile(edgenode.KubeletSvcPath, []byte(edgeKubeletUnitConfig), 0600); err != nil { + return err + } + } else { + if err := ioutil.WriteFile(edgenode.KubeletSvcPath, []byte(cloudKubeletUnitConfig), 0600); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/yurtctl/cmd/join/phases/type.go b/pkg/yurtctl/cmd/join/phases/type.go new file mode 100644 index 00000000000..851c6a65621 --- /dev/null +++ b/pkg/yurtctl/cmd/join/phases/type.go @@ -0,0 +1,27 @@ +package phases + +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import ( + joinphases "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/join" +) + +type YurtJoinData interface { + joinphases.JoinData + NodeType() string + YurtHubImage() string +} diff --git a/pkg/yurtctl/util/edgenode/common.go b/pkg/yurtctl/util/edgenode/common.go index bb9a93be96c..95e9428a000 100644 --- a/pkg/yurtctl/util/edgenode/common.go +++ b/pkg/yurtctl/util/edgenode/common.go @@ -21,6 +21,7 @@ const ( OpenyurtDir = "/var/lib/openyurt" StaticPodPath = "/etc/kubernetes/manifests" KubeCondfigPath = "/etc/kubernetes/kubelet.conf" + KubeCaFile = "/etc/kubernetes/pki/ca.crt" YurthubYamlName = "yurt-hub.yaml" KubeletConfName = "kubelet.conf" KubeletSvcBackup = "%s.bk" diff --git a/pkg/yurtctl/util/edgenode/util.go b/pkg/yurtctl/util/edgenode/util.go index 8954387b88a..899db26d83d 100644 --- a/pkg/yurtctl/util/edgenode/util.go +++ b/pkg/yurtctl/util/edgenode/util.go @@ -25,11 +25,12 @@ import ( "regexp" "strings" - "github.com/spf13/pflag" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/util/homedir" "k8s.io/klog" + + "github.com/spf13/pflag" ) // FileExists determines whether the file exists @@ -83,12 +84,12 @@ func DirExists(dirname string) (bool, error) { } // CopyFile copys sourceFile to destinationFile -func CopyFile(sourceFile string, destinationFile string) error { +func CopyFile(sourceFile string, destinationFile string, perm os.FileMode) error { content, err := ioutil.ReadFile(sourceFile) if err != nil { return fmt.Errorf("failed to read source file %s: %v", sourceFile, err) } - err = ioutil.WriteFile(destinationFile, content, 0666) + err = ioutil.WriteFile(destinationFile, content, perm) if err != nil { return fmt.Errorf("failed to write destination file %s: %v", destinationFile, err) } diff --git a/pkg/yurtctl/util/util.go b/pkg/yurtctl/util/util.go new file mode 100644 index 00000000000..31bcb2cdbba --- /dev/null +++ b/pkg/yurtctl/util/util.go @@ -0,0 +1,137 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "archive/tar" + "compress/gzip" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strconv" + "time" + + "k8s.io/klog" + + pb "gopkg.in/cheggaaa/pb.v1" +) + +// DownloadFile try to download file from URL and save to savePath multiple times. +func DownloadFile(URL, savePath string, retry int) error { + var count int + var err error + for count = 1; count <= retry; count++ { + if _, err := os.Stat(savePath); os.IsExist(err) { + if err := os.Remove(savePath); err != nil { + return err + } + } + + if err = downloadFileOnce(URL, savePath); err != nil { + continue + } else { + break + } + } + if count > retry { + return err + } + return nil +} + +//downloadFileOnce download file from URL and save to savePath. +func downloadFileOnce(URL, savePath string) error { + client := http.Client{Timeout: time.Minute * 10} + resp, err := client.Get(URL) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("Server return non-200 status: %v\n", resp.Status) + } + len, _ := strconv.Atoi(resp.Header.Get("Content-Length")) + + out, err := os.Create(savePath) + if err != nil { + return err + } + defer out.Close() + + bar := pb.New(int(len)).SetUnits(pb.U_BYTES).SetRefreshRate(time.Second * 1) + bar.ShowSpeed = true + bar.Start() + + reader := bar.NewProxyReader(resp.Body) + if _, err = io.Copy(out, reader); err != nil { + return err + } + + bar.Finish() + return nil +} + +//Untar unzip the file to the target directory. +func Untar(tarFile, dest string) error { + srcFile, err := os.Open(tarFile) + if err != nil { + return err + } + defer srcFile.Close() + + gr, err := gzip.NewReader(srcFile) + if err != nil { + return err + } + defer gr.Close() + + tr := tar.NewReader(gr) + for { + hdr, err := tr.Next() + if err != nil { + if err == io.EOF { + return nil + } + return err + } + destFile := filepath.Join(dest, hdr.Name) + if hdr.Typeflag == tar.TypeDir { + if _, err := os.Stat(destFile); err != nil { + if os.IsNotExist(err) { + if err := os.MkdirAll(destFile, 0775); err != nil { + return err + } + continue + } + return err + } + } else if hdr.Typeflag == tar.TypeReg { + file, err := os.OpenFile(destFile, os.O_CREATE|os.O_RDWR, os.FileMode(hdr.Mode)) + if err != nil { + klog.Errorf("open file %s error: %v", destFile, err) + return err + } + defer file.Close() + if _, err := io.Copy(file, tr); err != nil { + return err + } + } + } +} diff --git a/pkg/yurtctl/util/util_test.go b/pkg/yurtctl/util/util_test.go new file mode 100644 index 00000000000..367edbc3adc --- /dev/null +++ b/pkg/yurtctl/util/util_test.go @@ -0,0 +1,54 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_DownloadFile(t *testing.T) { + testCase := []struct { + name string + url string + retry int + expected bool + }{ + {"invalid download url", "http://1.2.3.4", 2, false}, + } + + for _, tc := range testCase { + err := DownloadFile(tc.url, "", tc.retry) + assert.Equal(t, tc.expected, err == nil) + } +} + +func Test_Untar(t *testing.T) { + testCase := []struct { + name string + filePath string + expected bool + }{ + {"invalid tar file", "/tmp", false}, + } + + for _, tc := range testCase { + err := Untar(tc.name, "") + assert.Equal(t, tc.expected, err == nil) + } +}