diff --git a/cmd/localkube/cmd/options.go b/cmd/localkube/cmd/options.go index 172036e97d3e..de0d2ec23ba6 100644 --- a/cmd/localkube/cmd/options.go +++ b/cmd/localkube/cmd/options.go @@ -68,6 +68,8 @@ func AddFlags(s *localkube.LocalkubeServer) { flag.Var(&s.RuntimeConfig, "runtime-config", "A set of key=value pairs that describe runtime configuration that may be passed to apiserver. apis/ key can be used to turn on/off specific api versions. apis// can be used to turn on/off specific resources. api/all and api/legacy are special keys to control all and legacy api versions respectively.") flag.IPVar(&s.NodeIP, "node-ip", s.NodeIP, "IP address of the node. If set, kubelet will use this IP address for the node.") flag.StringVar(&s.ContainerRuntime, "container-runtime", "", "The container runtime to be used") + flag.StringVar(&s.RemoteRuntimeEndpoint, "remote-runtime-endpoint", "", "The container runtime endpoint (CRI) to be used (if this is set, then --container-runtime is forced as 'remote')") + flag.StringVar(&s.RemoteImageEndpoint, "remote-image-endpoint", "", "The container image endpoint (CRI) to be used (if this is set, then --container-runtime is forced as 'remote')") flag.StringVar(&s.NetworkPlugin, "network-plugin", "", "The name of the network plugin") flag.StringVar(&s.FeatureGates, "feature-gates", "", "A set of key=value pairs that describe feature gates for alpha/experimental features.") flag.Var(&s.ExtraConfig, "extra-config", "A set of key=value pairs that describe configuration that may be passed to different components. The key should be '.' separated, and the first part before the dot is the component to apply the configuration to.") diff --git a/cmd/localkube/cmd/start.go b/cmd/localkube/cmd/start.go index 5d98659b2140..1f4cf0f4de2f 100644 --- a/cmd/localkube/cmd/start.go +++ b/cmd/localkube/cmd/start.go @@ -61,6 +61,18 @@ func StartLocalkube() { } func SetupServer(s *localkube.LocalkubeServer) { + if s.ContainerRuntime == "remote" && s.RemoteRuntimeEndpoint == "" { + panic("Failed to connect to --container-runtime='remote' with no --container-runtime-endpoint") + } + // localkube flags can handle `--container-runtime=remote --remote-runtime-endpoint=/var/run/crio.sock --remote-image-endpoint=/var/run/crio.sock`, + // but this allows for a convenience of just e.g.`--container-runtime=crio` and the same for minikube + switch s.ContainerRuntime { + case "crio", "cri-o": + s.ContainerRuntime = "remote" + s.RemoteRuntimeEndpoint = "unix:///var/run/crio.sock" + s.RemoteImageEndpoint = "unix:///var/run/crio.sock" + } + if s.ShouldGenerateCerts { if err := s.GenerateCerts(); err != nil { fmt.Println("Failed to create certificates!") diff --git a/deploy/iso/minikube-iso/package/Config.in b/deploy/iso/minikube-iso/package/Config.in index 6ea6ead251ee..87db356c048b 100644 --- a/deploy/iso/minikube-iso/package/Config.in +++ b/deploy/iso/minikube-iso/package/Config.in @@ -1,5 +1,7 @@ menu "System tools" source "$BR2_EXTERNAL_MINIKUBE_PATH/package/rkt-bin/Config.in" + source "$BR2_EXTERNAL_MINIKUBE_PATH/package/runc-master/Config.in" + source "$BR2_EXTERNAL_MINIKUBE_PATH/package/crio-bin/Config.in" source "$BR2_EXTERNAL_MINIKUBE_PATH/package/automount/Config.in" source "$BR2_EXTERNAL_MINIKUBE_PATH/package/docker-bin/Config.in" source "$BR2_EXTERNAL_MINIKUBE_PATH/package/cni-bin/Config.in" diff --git a/deploy/iso/minikube-iso/package/automount/minikube-automount b/deploy/iso/minikube-iso/package/automount/minikube-automount index 494f86e189ca..de593a45bf5f 100755 --- a/deploy/iso/minikube-iso/package/automount/minikube-automount +++ b/deploy/iso/minikube-iso/package/automount/minikube-automount @@ -181,6 +181,10 @@ if [ -n "$BOOT2DOCKER_DATA" ]; then mkdir -p /mnt/$PARTNAME/var/lib/localkube mkdir /var/lib/localkube mount --bind /mnt/$PARTNAME/var/lib/localkube /var/lib/localkube + + ## make an env file for other services to discover this PARTNAME easier + mkdir -p /var/run/minikube + echo "PERSISTENT_DIR=\"/mnt/$PARTNAME\"" > /var/run/minikube/env fi swapon "${UNPARTITIONED_HD}2" diff --git a/deploy/iso/minikube-iso/package/crio-bin/Config.in b/deploy/iso/minikube-iso/package/crio-bin/Config.in new file mode 100644 index 000000000000..ce6f773776ac --- /dev/null +++ b/deploy/iso/minikube-iso/package/crio-bin/Config.in @@ -0,0 +1,17 @@ +config BR2_PACKAGE_CRIO_BIN + bool "crio-bin" + default y + depends on BR2_x86_64 + depends on BR2_PACKAGE_HOST_GO_ARCH_SUPPORTS + depends on BR2_PACKAGE_HOST_GO_CGO_LINKING_SUPPORTS + depends on BR2_TOOLCHAIN_HAS_THREADS + depends on BR2_USE_MMU # lvm2 + depends on !BR2_STATIC_LIBS # lvm2 + depends on !BR2_TOOLCHAIN_USES_MUSL # lvm2 + select BR2_PACKAGE_RUNC_MASTER + select BR2_PACKAGE_BTRFS_PROGS + select BR2_PACKAGE_LIBSECCOMP + select BR2_PACKAGE_LIBGPGME + select BR2_PACKAGE_BTRFS_PROGS + select BR2_PACKAGE_LVM2 + select BR2_PACKAGE_LVM2_APP_LIBRARY diff --git a/deploy/iso/minikube-iso/package/crio-bin/crio-bin.hash b/deploy/iso/minikube-iso/package/crio-bin/crio-bin.hash new file mode 100644 index 000000000000..808fb23a4c60 --- /dev/null +++ b/deploy/iso/minikube-iso/package/crio-bin/crio-bin.hash @@ -0,0 +1,2 @@ +#sha256 a4f423aeede371d21a1b32c0f2287ae5d0c4cfb5ed2927e4b3930675b3c8e986 5e3c53c1721e9fdd6a3c52fcd953dfa35af846d5.tar.gz +sha256 b5fd99ab615ca3f11ed3735ab275ca66930ce3030fad1dcaddf0abed7af90c90 no-pivot.tar.gz diff --git a/deploy/iso/minikube-iso/package/crio-bin/crio-bin.mk b/deploy/iso/minikube-iso/package/crio-bin/crio-bin.mk new file mode 100644 index 000000000000..999438f9b770 --- /dev/null +++ b/deploy/iso/minikube-iso/package/crio-bin/crio-bin.mk @@ -0,0 +1,58 @@ +################################################################################ +# +# cri-o +# +################################################################################ + +#CRIO_BIN_VERSION = 5e3c53c1721e9fdd6a3c52fcd953dfa35af846d5 +CRIO_BIN_VERSION = no-pivot +#CRIO_BIN_SITE = https://github.com/kubernetes-incubator/cri-o/archive +CRIO_BIN_SITE = https://github.com/vbatts/cri-o/archive +CRIO_BIN_SOURCE = $(CRIO_BIN_VERSION).tar.gz +CRIO_BIN_DEPENDENCIES = libgpgme +CRIO_BIN_GOPATH = $(@D)/_output + +define CRIO_BIN_USERS + - -1 crio-admin -1 - - - - - + - -1 crio -1 - - - - - +endef + +define CRIO_BIN_CONFIGURE_CMDS + mkdir -p $(CRIO_BIN_GOPATH)/src/github.com/kubernetes-incubator + ln -sf $(@D) $(CRIO_BIN_GOPATH)/src/github.com/kubernetes-incubator/cri-o +endef + +define CRIO_BIN_BUILD_CMDS + GOPATH=$(CRIO_BIN_GOPATH) $(MAKE) PREFIX=/usr BUILDTAGS="containers_image_ostree_stub" $(TARGET_CONFIGURE_OPTS) -C $(@D) +endef + +define CRIO_BIN_INSTALL_TARGET_CMDS + mkdir -p $(TARGET_DIR)/usr/share/containers/oci/hooks.d + mkdir -p $(TARGET_DIR)/etc/containers/oci/hooks.d + mkdir -p $(TARGET_DIR)/etc/sysconfig + mkdir -p $(TARGET_DIR)/etc/crio + + GOPATH=$(CRIO_BIN_GOPATH) $(MAKE) $(TARGET_CONFIGURE_OPTS) -C $(@D) install DESTDIR=$(TARGET_DIR) PREFIX=$(TARGET_DIR)/usr + $(MAKE) $(TARGET_CONFIGURE_OPTS) -C $(@D) install.config DESTDIR=$(TARGET_DIR) PREFIX=$(TARGET_DIR)/usr + $(MAKE) $(TARGET_CONFIGURE_OPTS) -C $(@D) install.completions DESTDIR=$(TARGET_DIR) PREFIX=$(TARGET_DIR)/usr + + $(INSTALL) -Dm755 \ + $(BR2_EXTERNAL_MINIKUBE_PATH)/package/crio-bin/crio.conf \ + $(TARGET_DIR)/etc/crio/crio.conf + $(INSTALL) -Dm755 \ + $(BR2_EXTERNAL_MINIKUBE_PATH)/package/crio-bin/policy.json \ + $(TARGET_DIR)/etc/containers/policy.json + echo 'CRIO_OPTIONS="--storage-driver=overlay2 --debug"' > $(TARGET_DIR)/etc/sysconfig/crio + +endef + +define CRIO_BIN_INSTALL_INIT_SYSTEMD + $(MAKE) $(TARGET_CONFIGURE_OPTS) -C $(@D) install.systemd DESTDIR=$(TARGET_DIR) PREFIX=$(TARGET_DIR)/usr + $(INSTALL) -Dm755 \ + $(BR2_EXTERNAL_MINIKUBE_PATH)/package/crio-bin/crio.service \ + $(TARGET_DIR)/usr/lib/systemd/system/crio.service + $(call link-service,crio.service) + $(call link-service,crio-shutdown.service) +endef + +$(eval $(generic-package)) diff --git a/deploy/iso/minikube-iso/package/crio-bin/crio.conf b/deploy/iso/minikube-iso/package/crio-bin/crio.conf new file mode 100644 index 000000000000..2bc6d88b7108 --- /dev/null +++ b/deploy/iso/minikube-iso/package/crio-bin/crio.conf @@ -0,0 +1,149 @@ + +# The "crio" table contains all of the server options. +[crio] + +# root is a path to the "root directory". CRIO stores all of its data, +# including container images, in this directory. +root = "/var/lib/containers/storage" + +# run is a path to the "run directory". CRIO stores all of its state +# in this directory. +runroot = "/var/run/containers/storage" + +# storage_driver select which storage driver is used to manage storage +# of images and containers. +storage_driver = "" + +# storage_option is used to pass an option to the storage driver. +storage_option = [ +] + +# The "crio.api" table contains settings for the kubelet/gRPC +# interface (which is also used by crioctl). +[crio.api] + +# listen is the path to the AF_LOCAL socket on which crio will listen. +listen = "/var/run/crio.sock" + +# stream_address is the IP address on which the stream server will listen +stream_address = "" + +# stream_port is the port on which the stream server will listen +stream_port = "10010" + +# file_locking is whether file-based locking will be used instead of +# in-memory locking +file_locking = true + +# The "crio.runtime" table contains settings pertaining to the OCI +# runtime used and options for how to set up and manage the OCI runtime. +[crio.runtime] + +# runtime is the OCI compatible runtime used for trusted container workloads. +# This is a mandatory setting as this runtime will be the default one +# and will also be used for untrusted container workloads if +# runtime_untrusted_workload is not set. +runtime = "/usr/bin/runc" + +# runtime_untrusted_workload is the OCI compatible runtime used for untrusted +# container workloads. This is an optional setting, except if +# default_container_trust is set to "untrusted". +runtime_untrusted_workload = "" + +# default_workload_trust is the default level of trust crio puts in container +# workloads. It can either be "trusted" or "untrusted", and the default +# is "trusted". +# Containers can be run through different container runtimes, depending on +# the trust hints we receive from kubelet: +# - If kubelet tags a container workload as untrusted, crio will try first to +# run it through the untrusted container workload runtime. If it is not set, +# crio will use the trusted runtime. +# - If kubelet does not provide any information about the container workload trust +# level, the selected runtime will depend on the default_container_trust setting. +# If it is set to "untrusted", then all containers except for the host privileged +# ones, will be run by the runtime_untrusted_workload runtime. Host privileged +# containers are by definition trusted and will always use the trusted container +# runtime. If default_container_trust is set to "trusted", crio will use the trusted +# container runtime for all containers. +default_workload_trust = "trusted" + +# no_pivot instructs the runtime to not use pivot_root, but instead use MS_MOVE +no_pivot = true + +# conmon is the path to conmon binary, used for managing the runtime. +conmon = "/usr/libexec/crio/conmon" + +# conmon_env is the environment variable list for conmon process, +# used for passing necessary environment variable to conmon or runtime. +conmon_env = [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", +] + +# selinux indicates whether or not SELinux will be used for pod +# separation on the host. If you enable this flag, SELinux must be running +# on the host. +selinux = false + +# seccomp_profile is the seccomp json profile path which is used as the +# default for the runtime. +seccomp_profile = "/etc/crio/seccomp.json" + +# apparmor_profile is the apparmor profile name which is used as the +# default for the runtime. +apparmor_profile = "crio-default" + +# cgroup_manager is the cgroup management implementation to be used +# for the runtime. +cgroup_manager = "cgroupfs" + +# hooks_dir_path is the oci hooks directory for automatically executed hooks +hooks_dir_path = "/usr/share/containers/oci/hooks.d" + +# pids_limit is the number of processes allowed in a container +pids_limit = 1024 + +# The "crio.image" table contains settings pertaining to the +# management of OCI images. +[crio.image] + +# default_transport is the prefix we try prepending to an image name if the +# image name as we receive it can't be parsed as a valid source reference +default_transport = "docker://" + +# pause_image is the image which we use to instantiate infra containers. +pause_image = "kubernetes/pause" + +# pause_command is the command to run in a pause_image to have a container just +# sit there. If the image contains the necessary information, this value need +# not be specified. +pause_command = "/pause" + +# signature_policy is the name of the file which decides what sort of policy we +# use when deciding whether or not to trust an image that we've pulled. +# Outside of testing situations, it is strongly advised that this be left +# unspecified so that the default system-wide policy will be used. +signature_policy = "" + +# image_volumes controls how image volumes are handled. +# The valid values are mkdir and ignore. +image_volumes = "mkdir" + +# insecure_registries is used to skip TLS verification when pulling images. +insecure_registries = [ +] + +# registries is used to specify a comma separated list of registries to be used +# when pulling an unqualified image (e.g. fedora:rawhide). +registries = [ +] + +# The "crio.network" table contains settings pertaining to the +# management of CNI plugins. +[crio.network] + +# network_dir is is where CNI network configuration +# files are stored. +network_dir = "/etc/cni/net.d/" + +# plugin_dir is is where CNI plugin binaries are stored. +plugin_dir = "/opt/cni/bin/" diff --git a/deploy/iso/minikube-iso/package/crio-bin/crio.service b/deploy/iso/minikube-iso/package/crio-bin/crio.service new file mode 100644 index 000000000000..4b3dd2a1d0d3 --- /dev/null +++ b/deploy/iso/minikube-iso/package/crio-bin/crio.service @@ -0,0 +1,26 @@ +[Unit] +Description=Open Container Initiative Daemon +Documentation=https://github.com/kubernetes-incubator/cri-o +After=network-online.target minikube-automount.service +Requires=minikube-automount.service + +[Service] +Type=notify +EnvironmentFile=-/etc/sysconfig/crio +EnvironmentFile=/var/run/minikube/env +Environment=GOTRACEBACK=crash +ExecStartPre=/bin/mkdir -p ${PERSISTENT_DIR}/var/lib/containers +ExecStart=/usr/bin/crio \ + $CRIO_OPTIONS \ + --root ${PERSISTENT_DIR}/var/lib/containers +ExecReload=/bin/kill -s HUP $MAINPID +TasksMax=8192 +LimitNOFILE=1048576 +LimitNPROC=1048576 +LimitCORE=infinity +OOMScoreAdjust=-999 +TimeoutStartSec=0 +Restart=on-abnormal + +[Install] +WantedBy=multi-user.target diff --git a/deploy/iso/minikube-iso/package/crio-bin/policy.json b/deploy/iso/minikube-iso/package/crio-bin/policy.json new file mode 100644 index 000000000000..9333053f93bf --- /dev/null +++ b/deploy/iso/minikube-iso/package/crio-bin/policy.json @@ -0,0 +1,7 @@ +{ + "default": [ + { + "type": "insecureAcceptAnything" + } + ] +} diff --git a/deploy/iso/minikube-iso/package/rkt-bin/rkt-bin.mk b/deploy/iso/minikube-iso/package/rkt-bin/rkt-bin.mk index 4e58a5dd0fef..af6d7378934c 100644 --- a/deploy/iso/minikube-iso/package/rkt-bin/rkt-bin.mk +++ b/deploy/iso/minikube-iso/package/rkt-bin/rkt-bin.mk @@ -52,18 +52,21 @@ define RKT_BIN_INSTALL_INIT_SYSTEMD $(@D)/init/systemd/tmpfiles.d/rkt.conf \ $(TARGET_DIR)/usr/lib/tmpfiles.d/rkt.conf - $(call install-service,rkt-api.service) - $(call install-service,rkt-gc.timer) - $(call install-service,rkt-gc.service) - $(call install-service,rkt-metadata.socket) - $(call install-service,rkt-metadata.service) + $(call rkt-install-service,rkt-api.service) + $(call rkt-install-service,rkt-gc.timer) + $(call rkt-install-service,rkt-gc.service) + $(call rkt-install-service,rkt-metadata.socket) + $(call rkt-install-service,rkt-metadata.service) endef -define install-service +define rkt-install-service $(INSTALL) -D -m 644 \ $(@D)/init/systemd/$(1) \ $(TARGET_DIR)/usr/lib/systemd/system/$(1) + $(call link-service,$(1)) +endef +define link-service ln -fs /usr/lib/systemd/system/$(1) \ $(TARGET_DIR)/etc/systemd/system/multi-user.target.wants/$(1) endef diff --git a/deploy/iso/minikube-iso/package/runc-master/Config.in b/deploy/iso/minikube-iso/package/runc-master/Config.in new file mode 100644 index 000000000000..c33153880ebc --- /dev/null +++ b/deploy/iso/minikube-iso/package/runc-master/Config.in @@ -0,0 +1,17 @@ +config BR2_PACKAGE_RUNC_MASTER + bool "runc-master" + depends on BR2_PACKAGE_HOST_GO_ARCH_SUPPORTS + depends on BR2_PACKAGE_HOST_GO_CGO_LINKING_SUPPORTS + depends on BR2_TOOLCHAIN_HAS_THREADS + help + runC is a CLI tool for spawning and running containers + according to the OCP specification. + + This is just a newer build of runc than the buildroot version. + + https://github.com/opencontainers/runc + +comment "runc needs a toolchain w/ threads" + depends on BR2_PACKAGE_HOST_GO_ARCH_SUPPORTS && \ + BR2_PACKAGE_HOST_GO_CGO_LINKING_SUPPORTS + depends on !BR2_TOOLCHAIN_HAS_THREADS diff --git a/deploy/iso/minikube-iso/package/runc-master/runc-master.hash b/deploy/iso/minikube-iso/package/runc-master/runc-master.hash new file mode 100644 index 000000000000..f62965d0cc88 --- /dev/null +++ b/deploy/iso/minikube-iso/package/runc-master/runc-master.hash @@ -0,0 +1,2 @@ +# Locally computed +sha256 e9ad8aa5590f65a23326b7e9944d8b9881fa002ccb4a8e2cd40712a89a40ee45 runc-master-593914b8bd5448a93f7c3e4902a03408b6d5c0ce.tar.gz diff --git a/deploy/iso/minikube-iso/package/runc-master/runc-master.mk b/deploy/iso/minikube-iso/package/runc-master/runc-master.mk new file mode 100644 index 000000000000..539b83a4ef9c --- /dev/null +++ b/deploy/iso/minikube-iso/package/runc-master/runc-master.mk @@ -0,0 +1,50 @@ +################################################################################ +# +# runc +# +################################################################################ + +RUNC_MASTER_VERSION = 593914b8bd5448a93f7c3e4902a03408b6d5c0ce +RUNC_MASTER_SITE = $(call github,opencontainers,runc,$(RUNC_MASTER_VERSION)) +RUNC_MASTER_LICENSE = Apache-2.0 +RUNC_MASTER_LICENSE_FILES = LICENSE + +RUNC_MASTER_DEPENDENCIES = host-go + +RUNC_MASTER_GOPATH = "$(@D)/Godeps/_workspace" +RUNC_MASTER_MAKE_ENV = $(HOST_GO_TARGET_ENV) \ + CGO_ENABLED=1 \ + GOBIN="$(@D)/bin" \ + GOPATH="$(RUNC_MASTER_GOPATH)" \ + PATH=$(BR_PATH) + +RUNC_MASTER_GLDFLAGS = \ + -buildmode=pie -X main.gitCommit=$(RUNC_MASTER_VERSION) + +ifeq ($(BR2_STATIC_LIBS),y) +RUNC_MASTER_GLDFLAGS += -extldflags '-static' +endif + +RUNC_MASTER_GOTAGS = cgo static_build + +ifeq ($(BR2_PACKAGE_LIBSECCOMP),y) +RUNC_MASTER_GOTAGS += seccomp +RUNC_MASTER_DEPENDENCIES += libseccomp host-pkgconf +endif + +define RUNC_MASTER_CONFIGURE_CMDS + mkdir -p $(RUNC_MASTER_GOPATH)/src/github.com/opencontainers + ln -s $(@D) $(RUNC_MASTER_GOPATH)/src/github.com/opencontainers/runc +endef + +define RUNC_MASTER_BUILD_CMDS + cd $(@D) && $(RUNC_MASTER_MAKE_ENV) $(HOST_DIR)/usr/bin/go \ + build -v -o $(@D)/bin/runc \ + -tags "$(RUNC_MASTER_GOTAGS)" -ldflags "$(RUNC_MASTER_GLDFLAGS)" github.com/opencontainers/runc +endef + +define RUNC_MASTER_INSTALL_TARGET_CMDS + $(INSTALL) -D -m 0755 $(@D)/bin/runc $(TARGET_DIR)/usr/bin/runc +endef + +$(eval $(generic-package)) diff --git a/pkg/localkube/kubelet.go b/pkg/localkube/kubelet.go index 199a041f5dfe..fe6e76bd44e1 100644 --- a/pkg/localkube/kubelet.go +++ b/pkg/localkube/kubelet.go @@ -56,6 +56,12 @@ func StartKubeletServer(lk LocalkubeServer) func() error { if lk.ContainerRuntime != "" { config.ContainerRuntime = lk.ContainerRuntime } + if lk.RemoteRuntimeEndpoint != "" { + config.RemoteRuntimeEndpoint = lk.RemoteRuntimeEndpoint + } + if lk.RemoteImageEndpoint != "" { + config.RemoteImageEndpoint = lk.RemoteImageEndpoint + } lk.SetExtraConfigForComponent("kubelet", &config) // Use the host's resolver config diff --git a/pkg/localkube/localkube.go b/pkg/localkube/localkube.go index aa1da9154fff..4e70e2dfb7ac 100644 --- a/pkg/localkube/localkube.go +++ b/pkg/localkube/localkube.go @@ -57,6 +57,8 @@ type LocalkubeServer struct { RuntimeConfig flag.ConfigurationMap NodeIP net.IP ContainerRuntime string + RemoteRuntimeEndpoint string + RemoteImageEndpoint string NetworkPlugin string FeatureGates string ExtraConfig util.ExtraOptionSlice