diff --git a/.github/workflows/k8s-e2e.yml b/.github/workflows/k8s-e2e.yml index fb46a55e31..d7a134890c 100644 --- a/.github/workflows/k8s-e2e.yml +++ b/.github/workflows/k8s-e2e.yml @@ -81,13 +81,7 @@ jobs: # change snapshotter to nydus after nydus snapshotter started registry_ip=$(ip addr show eth0 | grep 'inet ' | awk '{print $2}' | cut -d/ -f1) docker cp kind-control-plane:/etc/containerd/config.toml containerd.config.toml.bak - sed -i -e 's|snapshotter = "overlayfs"|snapshotter = "nydus"|' containerd.config.toml.bak cat << EOF >> containerd.config.toml.bak - [proxy_plugins] - [proxy_plugins.nydus] - type = "snapshot" - address = "/run/containerd-nydus/containerd-nydus-grpc.sock" - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."${registry_ip}:5000"] endpoint = ["http://${registry_ip}:5000"] EOF diff --git a/docs/setup_snapshotter_by_daemonset.md b/docs/setup_snapshotter_by_daemonset.md new file mode 100644 index 0000000000..71eca5a62d --- /dev/null +++ b/docs/setup_snapshotter_by_daemonset.md @@ -0,0 +1,140 @@ +# Setup Nydus Snapshotter by DaemonSet + +This document will guide you through the simple steps of setting up and cleaning up the nydus snapshotter in a kubernetes cluster that runs on the host. + +## Steps for Setting up Nydus Snapshotter + +To begin, let's clone the Nydus Snapshotter repository. + +```bash +git clone https://github.com/containerd/nydus-snapshotter +cd nydus-snapshotter +``` + +We can build the docker image locally. (optional) +```bash +$ export NYDUS_VER=$(curl -s "https://api.github.com/repos/dragonflyoss/nydus/releases/latest" | jq -r .tag_name) +$ make # build snapshotter binaries +$ cp bin/* misc/snapshotter/ +$ pushd misc/snapshotter/ +$ docker build --build-arg NYDUS_VER="${NYDUS_VER}" -t ghcr.io/containerd/nydus-snapshotter:latest . +$ popd +``` +**NOTE:** By default, the nydus snapshotter would use the latest release nydus version. If you want to use a specific version, you can set `NYDUS_VER` on your side. + +Next, we can configure access control for nydus snapshotter. +```bash +kubectl apply -f misc/snapshotter/nydus-snapshotter-rbac.yaml +``` + +Afterward, we can deploy a DaemonSet for nydus snapshotter. + +```bash +kubectl apply -f misc/snapshotter/nydus-snapshotter.yaml +``` + +Then, we can confirm that nydus snapshotter is running through the DaemonSet. +```bash +$ kubectl get pods -n nydus-system +NAME READY STATUS RESTARTS AGE +nydus-snapshotter-26rf7 1/1 Running 0 18s +``` + +Finally, we can view the logs in the pod. +```bash +$ kubectl logs nydus-snapshotter-26rf7 -n nydus-system +install nydus snapshotter artifacts +there is no proxy plugin! +Created symlink /etc/systemd/system/multi-user.target.wants/nydus-snapshotter.service → /etc/systemd/system/nydus-snapshotter.service. +``` + +And we can see the nydus snapshotter service on the host. +```bash +$ systemctl status nydus-snapshotter +● nydus-snapshotter.service - nydus snapshotter + Loaded: loaded (/etc/systemd/system/nydus-snapshotter.service; enabled; vendor preset: enabled) + Drop-In: /etc/systemd/system/nydus-snapshotter.service.d + └─proxy.conf + Active: active (running) since Wed 2024-01-17 16:14:22 UTC; 56s ago + Main PID: 1100169 (containerd-nydu) + Tasks: 11 (limit: 96376) + Memory: 8.6M + CPU: 35ms + CGroup: /system.slice/nydus-snapshotter.service + └─1100169 /opt/nydus/bin/containerd-nydus-grpc --config /etc/nydus/config.toml + +Jan 17 16:14:22 worker systemd[1]: Started nydus snapshotter. +Jan 17 16:14:22 worker containerd-nydus-grpc[1100169]: time="2024-01-17T16:14:22.998798369Z" level=info msg="Start nydus-snapshotter. Version: v0.7.0-308-g106a6cb, PID: 1100169, FsDriver: fusedev, DaemonMode: dedicated" +Jan 17 16:14:23 worker containerd-nydus-grpc[1100169]: time="2024-01-17T16:14:23.000186538Z" level=info msg="Run daemons monitor..." +``` + +**NOTE:** By default, the nydus snapshotter operates as a systemd service. If you prefer to run nydus snapshotter as a standalone process, you can set `ENABLE_SYSTEMD_SERVICE` to `false` in `nydus-snapshotter.yaml`. + +## Steps for Cleaning up Nydus Snapshotter + +We use `preStop`` hook in the DaemonSet to uninstall nydus snapshotter and roll back the containerd configuration. + +```bash +$ kubectl delete -f misc/snapshotter/nydus-snapshotter.yaml +$ kubectl delete -f misc/snapshotter/nydus-snapshotter-rbac.yaml +$ systemd restart containerd.service +``` + +## Customized Setup + +As we know, nydus snapshotter supports four filesystem drivers (fs_driver): `fusedev`, `fscache`, `blockdev`, `proxy`. Within the container image, we have included configurations for these snapshotter drivers, as well as the corresponding nydusd configurations. By default, the fusedev driver is enabled in the nydus snapshotter, using the snapshotter configuration [`config-fusedev.toml`](../misc/snapshotter/config-fusedev.toml) and the nydusd configuration [`nydusd-config.fusedev.json`](../misc/snapshotter/nydusd-config.fusedev.json). + +### Other filesystem driver with related default configuration + +If we want to setup the nydus snapshotter with the default configuration for different fs_driver (such as `proxy`), we can modify the values in the `Configmap` in `nydus-snapshotter.yaml`: +```yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: nydus-snapshotter-configs + labels: + app: nydus-snapshotter + namespace: nydus-snapshotter +data: + FS_DRIVER: "proxy" + NYDUSD_DAEMON_MODE: "none" +``` + +Then we can run the nydus snapshotter enabling `proxy` `fs_driver` with the snapshotter configuration [`config-proxy.toml`](../misc/snapshotter/config-proxy.toml). + +**NOTE:** The fs_driver (`blockdev` and `proxy`) do not need nydusd, so they do not need nydusd config. + +### Same filesystem with different snapshotter configuration and different nydusd configuration + +If we want to setup the nydus snapshotter for the same fs_driver (such as `fusedev`) with different snapshotter configuration and different nydusd configuration, we can enable `ENABLE_CONFIG_FROM_VOLUME` and add the snapshotter configuration [`config.toml`](../misc/snapshotter/config.toml) and nydusd configuration [`nydusd-config.json`](../misc/snapshotter/nydusd-config.fusedev.json) in the `Configmap` in `nydus-snapshotter.yaml`: + +```yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: nydus-snapshotter-configs + labels: + app: nydus-snapshotter + namespace: nydus-snapshotter +data: + ENABLE_CONFIG_FROM_VOLUME: "true" + + config.toml: |- + # The snapshotter config content copied here + + nydusd-config.json: |- + # The nydusd config content copied here +``` + +**NOTE:** We need to set `nydusd_config` to `/etc/nydus/nydusd-config.json` in the `config.toml`, so that snapshotter can find the nydusd configuration from configmap. + +### Customized Options + +| Options | Type | Default | Comment | +| ----------------------------------- | ------ | ------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | +| FS_DRIVER | string | "fusedev" | the filesystem driver of snapshotter | +| ENABLE_CONFIG_FROM_VOLUME | bool | false | enabling to use the configurations from volume | +| ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER | bool | false | enabling to skip to set `plugins."io.containerd.grpc.v1.cri".containerd` to `nydus` for runtime specific snapshotter feature in containerd 1.7+ | +| ENABLE_SYSTEMD_SERVICE | bool | true | enabling to run nydus snapshotter as a systemd service | diff --git a/misc/snapshotter/Dockerfile b/misc/snapshotter/Dockerfile index 8a404ba0ce..29b050c1cc 100644 --- a/misc/snapshotter/Dockerfile +++ b/misc/snapshotter/Dockerfile @@ -2,30 +2,35 @@ FROM alpine:3.17.0 AS sourcer ARG NYDUS_VER=v2.1.5 -RUN apk add --no-cache curl -RUN apk add --no-cache --upgrade grep -RUN wget https://github.com/dragonflyoss/nydus/releases/download/$NYDUS_VER/nydus-static-$NYDUS_VER-linux-amd64.tgz && \ +RUN apk add --no-cache curl && \ + apk add --no-cache --upgrade grep && \ + curl -OL https://github.com/dragonflyoss/nydus/releases/download/$NYDUS_VER/nydus-static-$NYDUS_VER-linux-amd64.tgz && \ echo $NYDUS_VER > /.nydus_version && \ tar xzf nydus-static-$NYDUS_VER-linux-amd64.tgz && \ - rm nydus-static-$NYDUS_VER-linux-amd64.tgz -RUN mv nydus-static/* / + rm nydus-static-$NYDUS_VER-linux-amd64.tgz && \ + mv nydus-static/* / FROM alpine:3.17.0 +ARG DESTINATION=/opt/nydus-artifacts +ARG CONFIG_DESTINATION=${DESTINATION}/etc/nydus +ARG BINARY_DESTINATION=${DESTINATION}/usr/local/bin +ARG SCRIPT_DESTINATION=${DESTINATION}/opt/nydus WORKDIR /root/ - -RUN apk add --no-cache libc6-compat +RUN apk add --no-cache libc6-compat bash VOLUME /var/lib/containerd-nydus /run/containerd-nydus COPY --from=sourcer /.nydus_version /.nydus_version -RUN mkdir -p /usr/local/bin/ /etc/nydus/ /var/lib/containerd-nydus/cache /tmp/blobs/ -COPY --from=sourcer /nydus* /usr/local/bin/ -COPY containerd-nydus-grpc /usr/local/bin/ -RUN chmod +x /usr/local/bin/containerd-nydus-grpc -COPY nydusd-config.fusedev.json /etc/nydus/config.json -COPY nydusd-config-localfs.json /etc/nydus/localfs.json -COPY entrypoint.sh / +RUN mkdir -p ${CONFIG_DESTINATION} ${BINARY_DESTINATION} ${SCRIPT_DESTINATION} /var/lib/containerd-nydus/cache /tmp/blobs/ +COPY --from=sourcer /nydus* ${BINARY_DESTINATION}/ +COPY containerd-nydus-grpc ${BINARY_DESTINATION}/ +COPY snapshotter.sh ${SCRIPT_DESTINATION}/snapshotter.sh +RUN chmod +x ${BINARY_DESTINATION}/containerd-nydus-grpc ${SCRIPT_DESTINATION}/snapshotter.sh +COPY nydusd-config.fusedev.json ${CONFIG_DESTINATION}/nydusd-fusedev.json +COPY nydusd-config-localfs.json ${CONFIG_DESTINATION}/nydusd-localfs.json +COPY nydusd-config.fscache.json ${CONFIG_DESTINATION}/nydusd-fscache.json +COPY config.toml ${CONFIG_DESTINATION}/config.toml +COPY nydus-snapshotter.service ${DESTINATION}/etc/systemd/system/nydus-snapshotter.service -ENTRYPOINT ["/entrypoint.sh"] diff --git a/misc/snapshotter/config-coco-host-sharing.toml b/misc/snapshotter/config-blockdev.toml similarity index 97% rename from misc/snapshotter/config-coco-host-sharing.toml rename to misc/snapshotter/config-blockdev.toml index e52812d243..280ec074f7 100644 --- a/misc/snapshotter/config-coco-host-sharing.toml +++ b/misc/snapshotter/config-blockdev.toml @@ -36,4 +36,4 @@ mount_tarfs_on_host = false # - "image_block": generate a raw block disk image with tarfs for an image # - "layer_block_with_verity": generate a raw block disk image with tarfs for a layer with dm-verity info # - "image_block_with_verity": generate a raw block disk image with tarfs for an image with dm-verity info -export_mode = "image_block_with_verity" \ No newline at end of file +export_mode = "image_block_with_verity" diff --git a/misc/snapshotter/config-coco-guest-pulling.toml b/misc/snapshotter/config-proxy.toml similarity index 94% rename from misc/snapshotter/config-coco-guest-pulling.toml rename to misc/snapshotter/config-proxy.toml index eeba9a7b2e..8ae9d62c93 100644 --- a/misc/snapshotter/config-coco-guest-pulling.toml +++ b/misc/snapshotter/config-proxy.toml @@ -12,4 +12,4 @@ fs_driver = "proxy" [snapshot] # Insert Kata volume information to `Mount.Options` -enable_kata_volume = true \ No newline at end of file +enable_kata_volume = true diff --git a/misc/snapshotter/config.toml b/misc/snapshotter/config.toml index b0c82ca429..29a2aafe8b 100644 --- a/misc/snapshotter/config.toml +++ b/misc/snapshotter/config.toml @@ -27,7 +27,8 @@ pprof_address = "" nydusd_config = "/etc/nydus/nydusd-config.fusedev.json" nydusd_path = "/usr/local/bin/nydusd" nydusimage_path = "/usr/local/bin/nydus-image" -# fusedev or fscache +# The fs driver can be one of the following options: fusedev, fscache, blockdev, proxy, or nodev. +# If `fs_driver` option is not specified, the default value is fusedev. fs_driver = "fusedev" # How to process when daemon dies: "none", "restart" or "failover" recover_policy = "restart" diff --git a/misc/snapshotter/entrypoint.sh b/misc/snapshotter/entrypoint.sh deleted file mode 100755 index 6ecf8de684..0000000000 --- a/misc/snapshotter/entrypoint.sh +++ /dev/null @@ -1,22 +0,0 @@ -#! /bin/sh - -NYDUS_LIB="${NYDUS_LIB:-/var/lib/containerd-nydus}" -NYDUS_RUN="${NYDUS_RUN:-/run/containerd-nydus}" -LEVEL="${LEVEL:-info}" - -set -eu -BACKEND_TYPE="${BACKEND_TYPE:-config}" -NYDUSD_DAEMON_MODE="${NYDUSD_DAEMON_MODE:-multiple}" - -if [ "$#" -eq 0 ]; then - containerd-nydus-grpc \ - --nydusd /usr/local/bin/nydusd \ - --nydusd-config /etc/nydus/${BACKEND_TYPE}.json \ - --root ${NYDUS_LIB} \ - --address ${NYDUS_RUN}/containerd-nydus-grpc.sock \ - --log-level ${LEVEL} \ - --daemon-mode ${NYDUSD_DAEMON_MODE} \ - --log-to-stdout -fi - -exec $@ diff --git a/misc/snapshotter/nydus-snapshotter-rbac.yaml b/misc/snapshotter/nydus-snapshotter-rbac.yaml new file mode 100644 index 0000000000..d321fc5fe1 --- /dev/null +++ b/misc/snapshotter/nydus-snapshotter-rbac.yaml @@ -0,0 +1,34 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: nydus-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nydus-snapshotter-sa + namespace: nydus-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nydus-snapshotter-role +rules: +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "patch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nydus-snapshotter-role-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nydus-snapshotter-role +subjects: +- kind: ServiceAccount + name: nydus-snapshotter-sa + namespace: nydus-system diff --git a/misc/snapshotter/nydus-snapshotter.service b/misc/snapshotter/nydus-snapshotter.service new file mode 100644 index 0000000000..46ef2fd1f7 --- /dev/null +++ b/misc/snapshotter/nydus-snapshotter.service @@ -0,0 +1,18 @@ +[Unit] +Description=nydus snapshotter +After=network.target +Before=containerd.service + +[Service] +Type=simple +Environment=HOME=/root +ExecStart=/usr/local/bin/containerd-nydus-grpc --config /etc/nydus/config-proxy.toml +Restart=always +RestartSec=1 +KillMode=process +OOMScoreAdjust=-999 +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/misc/snapshotter/nydus-snapshotter.yaml b/misc/snapshotter/nydus-snapshotter.yaml new file mode 100644 index 0000000000..8e72f6136f --- /dev/null +++ b/misc/snapshotter/nydus-snapshotter.yaml @@ -0,0 +1,140 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: nydus-snapshotter-configs + labels: + app: nydus-snapshotter + namespace: nydus-system +data: + FS_DRIVER: "fusedev" + ENABLE_CONFIG_FROM_VOLUME: "false" + ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER: "false" + ENABLE_SYSTEMD_SERVICE: "true" + +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: nydus-snapshotter + namespace: nydus-system + labels: + app: nydus-snapshotter +spec: + selector: + matchLabels: + app: nydus-snapshotter + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + app: nydus-snapshotter + + spec: + serviceAccountName: nydus-snapshotter-sa + hostNetwork: true + hostPID: true + containers: + - name: nydus-snapshotter + image: "ghcr.io/containerd/nydus-snapshotter:latest" + imagePullPolicy: Always + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: FS_DRIVER + valueFrom: + configMapKeyRef: + name: nydus-snapshotter-configs + key: FS_DRIVER + optional: true + - name: ENABLE_CONFIG_FROM_VOLUME + valueFrom: + configMapKeyRef: + name: nydus-snapshotter-configs + key: ENABLE_CONFIG_FROM_VOLUME + optional: true + - name: ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER + valueFrom: + configMapKeyRef: + name: nydus-snapshotter-configs + key: ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER + optional: true + - name: ENABLE_SYSTEMD_SERVICE + valueFrom: + configMapKeyRef: + name: nydus-snapshotter-configs + key: ENABLE_SYSTEMD_SERVICE + optional: true + lifecycle: + preStop: + exec: + command: + - "bash" + - "-c" + - | + /opt/nydus-artifacts/opt/nydus/snapshotter.sh cleanup + command: + - bash + - -c + - |- + /opt/nydus-artifacts/opt/nydus/snapshotter.sh deploy + volumeMounts: + - name: config-volume + mountPath: "/etc/nydus-snapshotter" + - name: nydus-lib + mountPath: "/var/lib/containerd-nydus" + mountPropagation: Bidirectional + - name: nydus-run + mountPath: "/run/containerd-nydus" + mountPropagation: Bidirectional + - name: nydus-opt + mountPath: "/opt/nydus" + mountPropagation: Bidirectional + - name: nydus-etc + mountPath: "/etc/nydus" + mountPropagation: Bidirectional + - name: containerd-conf + mountPath: "/etc/containerd/config.toml" + - name: local-bin + mountPath: "/usr/local/bin/" + - name: etc-systemd-system + mountPath: "/etc/systemd/system/" + securityContext: + privileged: true + + volumes: + - name: config-volume + configMap: + name: nydus-snapshotter-configs + optional: true + - name: nydus-run + hostPath: + path: /run/containerd-nydus + type: DirectoryOrCreate + - name: nydus-lib + hostPath: + path: /var/lib/containerd-nydus + type: DirectoryOrCreate + - name: nydus-opt + hostPath: + path: /opt/nydus + type: DirectoryOrCreate + - name: nydus-etc + hostPath: + path: /etc/nydus + type: DirectoryOrCreate + - name: containerd-conf + hostPath: + path: /etc/containerd/config.toml + - name: local-bin + hostPath: + path: /usr/local/bin/ + - name: etc-systemd-system + hostPath: + path: /etc/systemd/system/ + diff --git a/misc/snapshotter/snapshotter.sh b/misc/snapshotter/snapshotter.sh new file mode 100644 index 0000000000..bd0b3aad9a --- /dev/null +++ b/misc/snapshotter/snapshotter.sh @@ -0,0 +1,259 @@ +#!/usr/bin/env bash +# Copyright (c) 2023. Nydus Developers. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit +set -o pipefail +set -o nounset + +SNAPSHOTTER_ARTIFACTS_DIR="/opt/nydus-artifacts" + +# Container runtime config, the default container runtime is containerd +CONTAINER_RUNTIME="${CONTAINER_RUNTIME:-containerd}" +CONTAINER_RUNTIME_CONFIG="${CONTAINER_RUNTIME_CONFIG:-/etc/containerd/config.toml}" + +# Common nydus snapshotter config options +FS_DRIVER="${FS_DRIVER:-fusedev}" +SNAPSHOTTER_GRPC_SOCKET="${SNAPSHOTTER_GRPC_SOCKET:-/run/containerd-nydus/containerd-nydus-grpc.sock}" + +# The directory about nydus and nydus snapshotter +NYDUS_CONFIG_DIR="${NYDUS_CONFIG_DIR:-/etc/nydus}" +NYDUS_LIB_DIR="${NYDUS_LIB_DIR:-/var/lib/containerd-nydus}" +NYDUS_BINARY_DIR="${NYDUS_BINARY_DIR:-/usr/local/bin}" +SNAPSHOTTER_SCRYPT_DIR="${SNAPSHOTTER_SCRYPT_DIR:-/opt/nydus}" + +# The binary about nydus-snapshotter +SNAPSHOTTER_BINARY="${SNAPSHOTTER_BINARY:-${NYDUS_BINARY_DIR}/containerd-nydus-grpc}" + +# The config about nydus snapshotter +SNAPSHOTTER_CONFIG="${SNAPSHOTTER_CONFIG:-${NYDUS_CONFIG_DIR}/config.toml}" +# The systemd service config about nydus snapshotter +SNAPSHOTTER_SERVICE="${SNAPSHOTTER_SERVICE:-/etc/systemd/system/nydus-snapshotter.service}" +# If true, the script would read the config from env. +ENABLE_CONFIG_FROM_VOLUME="${ENABLE_CONFIG_FROM_VOLUME:-false}" +# If true, the script would enable the "runtime specific snapshotter" in containerd config. +ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER="${ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER:-false}" +# If true, the snapshotter would be running as a systemd service +ENABLE_SYSTEMD_SERVICE="${ENABLE_SYSTEMD_SERVICE:-false}" + +COMMANDLINE="" + +# If we fail for any reason a message will be displayed +die() { + msg="$*" + echo "ERROR: $msg" >&2 + exit 1 +} + +print_usage() { + echo "Usage: $0 [deploy/cleanup]" +} + +wait_service_active(){ + local wait_time="$1" + local sleep_time="$2" + local service="$3" + + nsenter -t 1 -m systemctl restart $service + + # Wait for containerd to be running + while [ "$wait_time" -gt 0 ]; do + if nsenter -t 1 -m systemctl is-active --quiet $service; then + echo "$service is running" + return 0 + else + sleep "$sleep_time" + wait_time=$((wait_time-sleep_time)) + fi + done + + echo "Timeout reached. $service may not be running." + nsenter -t 1 -m systemctl status $service + return 1 +} + +function fs_driver_handler() { + if [ "${ENABLE_CONFIG_FROM_VOLUME}" == "true" ]; then + SNAPSHOTTER_CONFIG="${NYDUS_CONFIG_DIR}/config.toml" + else + case "${FS_DRIVER}" in + fusedev) + sed -i -e "s|nydusd_config = .*|nydusd_config = \"${NYDUS_CONFIG_DIR}/nydusd-fusedev.json\"|" "${SNAPSHOTTER_CONFIG}" + sed -i -e "s|fs_driver = .*|fs_driver = \"fusedev\"|" "${SNAPSHOTTER_CONFIG}" + sed -i -e "s|daemon_mode = .*|daemon_mode = \"multiple\"|" "${SNAPSHOTTER_CONFIG}" + ;; + fscache) + sed -i -e "s|nydusd_config = .*|nydusd_config = \"${NYDUS_CONFIG_DIR}/nydusd-fscache.json\"|" "${SNAPSHOTTER_CONFIG}" + sed -i -e "s|fs_driver = .*|fs_driver = \"fscache\"|" "${SNAPSHOTTER_CONFIG}" + sed -i -e "s|daemon_mode = .*|daemon_mode = \"multiple\"|" "${SNAPSHOTTER_CONFIG}" + ;; + blockdev) + sed -i -e "s|fs_driver = .*|fs_driver = \"blockdev\"|" "${SNAPSHOTTER_CONFIG}" + sed -i -e "s|enable_kata_volume = .*|enable_kata_volume = true|" "${SNAPSHOTTER_CONFIG}" + sed -i -e "s|enable_tarfs = .*|enable_tarfs = true|" "${SNAPSHOTTER_CONFIG}" + sed -i -e "s|daemon_mode = .*|daemon_mode = \"none\"|" "${SNAPSHOTTER_CONFIG}" + sed -i -e "s|export_mode = .*|export_mode = \"layer_block_with_verity\"|" "${SNAPSHOTTER_CONFIG}" + ;; + proxy) + sed -i -e "s|fs_driver = .*|fs_driver = \"proxy\"|" "${SNAPSHOTTER_CONFIG}" + sed -i -e "s|enable_kata_volume = .*|enable_kata_volume = true|" "${SNAPSHOTTER_CONFIG}" + sed -i -e "s|daemon_mode = .*|daemon_mode = \"none\"|" "${SNAPSHOTTER_CONFIG}" + ;; + *) die "invalid fs driver ${FS_DRIVER}" ;; + esac + fi + COMMANDLINE+=" --config ${SNAPSHOTTER_CONFIG}" +} + +function configure_snapshotter() { + + echo "configuring snapshotter" + if [ "${CONTAINER_RUNTIME}" != "containerd" ]; then + die "not supported container runtime: ${CONTAINER_RUNTIME}" + fi + + # Copy the container runtime config to a backup + cp "$CONTAINER_RUNTIME_CONFIG" "$CONTAINER_RUNTIME_CONFIG".bak.nydus + + + # When trying to edit the config file that is mounted by docker with `sed -i`, the error would happend: + # sed: cannot rename /etc/containerd/config.tomlpmdkIP: Device or resource busy + # The reason is that `sed`` with option `-i` creates new file, and then replaces the old file with the new one, + # which definitely will change the file inode. But the file is mounted by docker, which means we are not allowed to + # change its inode from within docker container. + # + # So we copy the original file to a backup, make changes to the backup, and then overwrite the original file with the backup. + cp "$CONTAINER_RUNTIME_CONFIG" "$CONTAINER_RUNTIME_CONFIG".bak + # Check and add nydus proxy plugin in the config + if grep -q '\[proxy_plugins.nydus\]' "$CONTAINER_RUNTIME_CONFIG".bak; then + echo "the config has configured the nydus proxy plugin!" + else + echo "Not found nydus proxy plugin!" + cat <>"$CONTAINER_RUNTIME_CONFIG".bak + + [proxy_plugins.nydus] + type = "snapshot" + address = "$SNAPSHOTTER_GRPC_SOCKET" +EOF + fi + + if grep -q 'disable_snapshot_annotations' "$CONTAINER_RUNTIME_CONFIG".bak; then + sed -i -e "s|disable_snapshot_annotations = .*|disable_snapshot_annotations = false|" \ + "${CONTAINER_RUNTIME_CONFIG}".bak + else + sed -i '/\[plugins\..*\.containerd\]/a\disable_snapshot_annotations = false' \ + "${CONTAINER_RUNTIME_CONFIG}".bak + fi + if grep -q 'discard_unpacked_layers' "$CONTAINER_RUNTIME_CONFIG".bak; then + sed -i -e "s|discard_unpacked_layers = .*|discard_unpacked_layers = false|" \ + "${CONTAINER_RUNTIME_CONFIG}".bak + else + sed -i '/\[plugins\..*\.containerd\]/a\discard_unpacked_layers = false' \ + "${CONTAINER_RUNTIME_CONFIG}".bak + fi + + if [ "${ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER}" == "false" ]; then + sed -i -e '/\[plugins\..*\.containerd\]/,/snapshotter =/ s/snapshotter = "[^"]*"/snapshotter = "nydus"/' "${CONTAINER_RUNTIME_CONFIG}".bak + fi + + cat "${CONTAINER_RUNTIME_CONFIG}".bak > "${CONTAINER_RUNTIME_CONFIG}" +} + +function install_snapshotter() { + echo "install nydus snapshotter artifacts" + find "${SNAPSHOTTER_ARTIFACTS_DIR}${NYDUS_BINARY_DIR}" -type f -exec install -Dm 755 -t "${NYDUS_BINARY_DIR}" "{}" \; + find "${SNAPSHOTTER_ARTIFACTS_DIR}${NYDUS_CONFIG_DIR}" -type f -exec install -Dm 644 -t "${NYDUS_CONFIG_DIR}" "{}" \; + install -D -m 644 "${SNAPSHOTTER_ARTIFACTS_DIR}${SNAPSHOTTER_SCRYPT_DIR}/snapshotter.sh" "${SNAPSHOTTER_SCRYPT_DIR}/snapshotter.sh" + if [ "${ENABLE_SYSTEMD_SERVICE}" == "true" ]; then + install -D -m 644 "${SNAPSHOTTER_ARTIFACTS_DIR}${SNAPSHOTTER_SERVICE}" "${SNAPSHOTTER_SERVICE}" + fi + if [ "${ENABLE_CONFIG_FROM_VOLUME}" == "true" ]; then + find "/etc/nydus-snapshotter" -type f -exec install -Dm 644 -t "${NYDUS_CONFIG_DIR}" "{}" \; + fi +} + +function deploy_snapshotter() { + echo "deploying snapshotter" + if [ ! -f "${CONTAINER_RUNTIME_CONFIG}" ] && [ "${CONTAINER_RUNTIME}" == "containerd" ]; then + mkdir -p /etc/containerd || true + containerd config default >/etc/containerd/config.toml + fi + + install_snapshotter + + COMMANDLINE="${SNAPSHOTTER_BINARY}" + fs_driver_handler + configure_snapshotter + if [ "${ENABLE_SYSTEMD_SERVICE}" == "true" ]; then + echo "running snapshotter as systemd service" + sed -i "s|^ExecStart=.*$|ExecStart=$COMMANDLINE|" "${SNAPSHOTTER_SERVICE}" + nsenter -t 1 -m systemctl daemon-reload + nsenter -t 1 -m systemctl enable nydus-snapshotter.service + wait_service_active 30 5 nydus-snapshotter + else + echo "running snapshotter as standalone process" + ${COMMANDLINE} & + fi + wait_service_active 30 5 containerd + +} + +function cleanup_snapshotter() { + echo "cleaning up snapshotter" + + pid=$(ps -ef | grep containerd-nydus-grpc | grep -v grep | awk '{print $1}') + if [ ! -z "$pid" ]; then + for i in $(nsenter -t 1 -m ctr -n k8s.io snapshot --snapshotter nydus list | grep -v KEY | cut -d' ' -f1); do + nsenter -t 1 -m ctr -n k8s.io snapshot --snapshotter nydus rm $i || true + done + fi + echo "Recover containerd config" + cat "$CONTAINER_RUNTIME_CONFIG".bak.nydus >"$CONTAINER_RUNTIME_CONFIG" + if [ "${ENABLE_SYSTEMD_SERVICE}" == "true" ]; then + nsenter -t 1 -m systemctl stop nydus-snapshotter.service + nsenter -t 1 -m systemctl disable --now nydus-snapshotter.service + rm -f "${SNAPSHOTTER_SERVICE}" + else + kill -9 $pid || true + fi + wait_service_active 30 5 containerd + echo "Removing nydus-snapshotter artifacts from host" + rm -f "${SNAPSHOTTER_BINARY}" + rm -f "${NYDUS_BINARY_DIR}/nydus*" + rm -rf "${NYDUS_CONFIG_DIR}/*" + rm -rf "${SNAPSHOTTER_SCRYPT_DIR}/*" + rm -rf "${NYDUS_LIB_DIR}/*" +} + +function main() { + # script requires that user is root + euid=$(id -u) + if [[ $euid -ne 0 ]]; then + die "This script must be run as root" + fi + + action=${1:-} + if [ -z "$action" ]; then + print_usage + die "invalid arguments" + fi + + case "$action" in + deploy) + deploy_snapshotter + ;; + cleanup) + cleanup_snapshotter + ;; + *) + die "invalid arguments" + print_usage + ;; + esac + + sleep infinity +} + +main "$@" diff --git a/tests/e2e/k8s/snapshotter-cri.yaml b/tests/e2e/k8s/snapshotter-cri.yaml index 31641a5122..d011d26f67 100644 --- a/tests/e2e/k8s/snapshotter-cri.yaml +++ b/tests/e2e/k8s/snapshotter-cri.yaml @@ -1,3 +1,4 @@ +--- apiVersion: v1 kind: Namespace metadata: @@ -15,113 +16,126 @@ kind: Pod metadata: name: nydus-snapshotter namespace: nydus-system + labels: + app: nydus-snapshotter spec: + serviceAccountName: nydus-snapshotter-sa + hostNetwork: true + hostPID: true containers: - name: nydus-snapshotter - image: local-dev:e2e + image: "local-dev:e2e" imagePullPolicy: IfNotPresent - args: - - containerd-nydus-grpc - - --nydusd /usr/local/bin/nydusd - - --nydusd-config /etc/nydus/nydusd.json - - --config /etc/nydus/config.toml - - --log-level debug - - --daemon-mode shared - - --log-to-stdout - securityContext: - privileged: true + env: + - name: FS_DRIVER + valueFrom: + configMapKeyRef: + name: nydus-snapshotter-configs + key: FS_DRIVER + optional: true + - name: ENABLE_CONFIG_FROM_VOLUME + valueFrom: + configMapKeyRef: + name: nydus-snapshotter-configs + key: ENABLE_CONFIG_FROM_VOLUME + optional: true + - name: ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER + valueFrom: + configMapKeyRef: + name: nydus-snapshotter-configs + key: ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER + optional: true + - name: ENABLE_SYSTEMD_SERVICE + valueFrom: + configMapKeyRef: + name: nydus-snapshotter-configs + key: ENABLE_SYSTEMD_SERVICE + optional: true + lifecycle: + preStop: + exec: + command: + - "bash" + - "-c" + - | + /opt/nydus-artifacts/opt/nydus/snapshotter.sh cleanup + command: + - bash + - -c + - |- + /opt/nydus-artifacts/opt/nydus/snapshotter.sh deploy volumeMounts: - - mountPath: /etc/nydus/ - name: nydus-all-config - - mountPath: /var/lib/containerd-nydus + - name: config-volume + mountPath: "/etc/nydus-snapshotter" + - name: nydus-lib + mountPath: "/var/lib/containerd-nydus" mountPropagation: Bidirectional - name: nydus-lib - - mountPath: /run/containerd-nydus + - name: nydus-run + mountPath: "/run/containerd-nydus" mountPropagation: Bidirectional - name: nydus-run - - mountPath: /run/containerd + - name: nydus-opt + mountPath: "/opt/nydus" mountPropagation: Bidirectional - name: containerd-run - - mountPath: /dev/fuse - name: fuse - - mountPath: /etc/containerd/config.toml - name: containerd-conf - hostNetwork: true - hostPID: true - serviceAccountName: nydus-snapshotter-sa + - name: nydus-etc + mountPath: "/etc/nydus" + mountPropagation: Bidirectional + - name: containerd-conf + mountPath: "/etc/containerd/config.toml" + - name: local-bin + mountPath: "/usr/local/bin/" + - name: etc-systemd-system + mountPath: "/etc/systemd/system/" + - name: fuse + mountPath: /dev/fuse + securityContext: + privileged: true + volumes: - - name: nydus-all-config - projected: - sources: - - configMap: - name: nydusd-config - - configMap: - name: snapshotter-config - - hostPath: + - name: config-volume + configMap: + name: nydus-snapshotter-configs + optional: true + - name: nydus-run + hostPath: path: /run/containerd-nydus type: DirectoryOrCreate - name: nydus-run - - hostPath: + - name: nydus-lib + hostPath: path: /var/lib/containerd-nydus type: DirectoryOrCreate - name: nydus-lib - - hostPath: - path: /run/containerd - type: Directory - name: containerd-run - - hostPath: - path: /dev/fuse - name: fuse - - hostPath: + - name: nydus-etc + hostPath: + path: /etc/nydus + type: DirectoryOrCreate + - name: nydus-opt + hostPath: + path: /opt/nydus + type: DirectoryOrCreate + - name: containerd-conf + hostPath: path: /etc/containerd/config.toml - type: "" - name: containerd-conf ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: nydusd-config - namespace: nydus-system -data: - nydusd.json: |- - { - "device": { - "backend": { - "type": "registry", - "config": { - "scheme": "", - "skip_verify": true, - "timeout": 10, - "connect_timeout": 10, - "retry_limit": 2 - } - }, - "cache": { - "type": "blobcache", - "config": { - "work_dir": "/var/lib/nydus/cache/" - } - } - }, - "mode": "direct", - "digest_validate": false, - "iostats_files": false, - "enable_xattr": true, - "amplify_io": 1048576, - "fs_prefetch": { - "enable": true, - "threads_count": 10, - "merging_size": 131072, - "bandwidth_rate": 1048576 - } - } + - name: local-bin + hostPath: + path: /usr/local/bin/ + - name: etc-systemd-system + hostPath: + path: /etc/systemd/system/ + - name: fuse + hostPath: + path: /dev/fuse --- apiVersion: v1 kind: ConfigMap metadata: - name: snapshotter-config + name: nydus-snapshotter-configs + labels: + app: nydus-snapshotter namespace: nydus-system data: + FS_DRIVER: "fusedev" + ENABLE_CONFIG_FROM_VOLUME: "true" + ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER: "false" + ENABLE_SYSTEMD_SERVICE: "true" config.toml: |- version = 1 root = "/var/lib/containerd-nydus" @@ -134,7 +148,6 @@ data: enable_stargz = false # Whether snapshotter should try to clean up resources when it is closed cleanup_on_close = false - [daemon] nydusd_path = "/usr/local/bin/nydusd" nydusimage_path = "/usr/local/bin/nydus-image" @@ -148,7 +161,6 @@ data: nydusd_config = "/etc/nydus/nydusd.json" # The fuse or fscache IO working threads started by nydusd threads_number = 4 - [log] # Snapshotter's log level level = "info" @@ -160,10 +172,8 @@ data: # In unit MB(megabytes) log_rotation_max_size = 1 log_to_stdout = false - [remote] convert_vpc_registry = false - [remote.auth] # Fetch the private registry auth by listening to K8s API server enable_kubeconfig_keychain = false @@ -173,17 +183,47 @@ data: enable_cri_keychain = true # the target image service when using image proxy image_service_address = "" - [snapshot] enable_nydus_overlayfs = false # Whether to remove resources when a snapshot is removed sync_remove = false - [cache_manager] disable = false gc_period = "24h" cache_dir = "" - [image] public_key_file = "" validate_signature = false + + nydusd.json: |- + { + "device": { + "backend": { + "type": "registry", + "config": { + "scheme": "", + "skip_verify": true, + "timeout": 10, + "connect_timeout": 10, + "retry_limit": 2 + } + }, + "cache": { + "type": "blobcache", + "config": { + "work_dir": "/var/lib/nydus/cache/" + } + } + }, + "mode": "direct", + "digest_validate": false, + "iostats_files": false, + "enable_xattr": true, + "amplify_io": 1048576, + "fs_prefetch": { + "enable": true, + "threads_count": 10, + "merging_size": 131072, + "bandwidth_rate": 1048576 + } + } diff --git a/tests/e2e/k8s/snapshotter-kubeconf.yaml b/tests/e2e/k8s/snapshotter-kubeconf.yaml index 247155a914..7b1d4d60cb 100644 --- a/tests/e2e/k8s/snapshotter-kubeconf.yaml +++ b/tests/e2e/k8s/snapshotter-kubeconf.yaml @@ -1,15 +1,14 @@ +--- apiVersion: v1 kind: Namespace metadata: name: nydus-system - --- apiVersion: v1 kind: ServiceAccount metadata: name: nydus-snapshotter-sa namespace: nydus-system - --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 @@ -31,9 +30,9 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: nydus-snapshotter-role-binding roleRef: + apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: nydus-snapshotter-role - apiGroup: rbac.authorization.k8s.io subjects: - kind: ServiceAccount name: nydus-snapshotter-sa @@ -45,106 +44,126 @@ kind: Pod metadata: name: nydus-snapshotter namespace: nydus-system + labels: + app: nydus-snapshotter spec: + serviceAccountName: nydus-snapshotter-sa + hostNetwork: true + hostPID: true containers: - name: nydus-snapshotter - image: local-dev:e2e + image: "local-dev:e2e" imagePullPolicy: IfNotPresent - args: - - containerd-nydus-grpc - - --nydusd /usr/local/bin/nydusd - - --nydusd-config /etc/nydus/nydusd.json - - --config /etc/nydus/config.toml - - --log-level debug - - --daemon-mode shared - - --log-to-stdout - securityContext: - privileged: true + env: + - name: FS_DRIVER + valueFrom: + configMapKeyRef: + name: nydus-snapshotter-configs + key: FS_DRIVER + optional: true + - name: ENABLE_CONFIG_FROM_VOLUME + valueFrom: + configMapKeyRef: + name: nydus-snapshotter-configs + key: ENABLE_CONFIG_FROM_VOLUME + optional: true + - name: ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER + valueFrom: + configMapKeyRef: + name: nydus-snapshotter-configs + key: ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER + optional: true + - name: ENABLE_SYSTEMD_SERVICE + valueFrom: + configMapKeyRef: + name: nydus-snapshotter-configs + key: ENABLE_SYSTEMD_SERVICE + optional: true + lifecycle: + preStop: + exec: + command: + - "bash" + - "-c" + - | + /opt/nydus-artifacts/opt/nydus/snapshotter.sh cleanup + command: + - bash + - -c + - |- + /opt/nydus-artifacts/opt/nydus/snapshotter.sh deploy volumeMounts: - - mountPath: /etc/nydus/ - name: nydus-all-config - - mountPath: /var/lib/containerd-nydus + - name: config-volume + mountPath: "/etc/nydus-snapshotter" + - name: nydus-lib + mountPath: "/var/lib/containerd-nydus" mountPropagation: Bidirectional - name: nydus-lib - - mountPath: /run/containerd-nydus + - name: nydus-run + mountPath: "/run/containerd-nydus" mountPropagation: Bidirectional - name: nydus-run - - mountPath: /dev/fuse - name: fuse - - mountPath: /etc/containerd/config.toml - name: containerd-conf - hostNetwork: true - hostPID: true - serviceAccountName: nydus-snapshotter-sa + - name: nydus-opt + mountPath: "/opt/nydus" + mountPropagation: Bidirectional + - name: nydus-etc + mountPath: "/etc/nydus" + mountPropagation: Bidirectional + - name: containerd-conf + mountPath: "/etc/containerd/config.toml" + - name: local-bin + mountPath: "/usr/local/bin/" + - name: etc-systemd-system + mountPath: "/etc/systemd/system/" + - name: fuse + mountPath: /dev/fuse + securityContext: + privileged: true + volumes: - - name: nydus-all-config - projected: - sources: - - configMap: - name: nydusd-config - - configMap: - name: snapshotter-config - - hostPath: + - name: config-volume + configMap: + name: nydus-snapshotter-configs + optional: true + - name: nydus-run + hostPath: path: /run/containerd-nydus type: DirectoryOrCreate - name: nydus-run - - hostPath: + - name: nydus-lib + hostPath: path: /var/lib/containerd-nydus type: DirectoryOrCreate - name: nydus-lib - - hostPath: - path: /dev/fuse - name: fuse - - hostPath: + - name: nydus-etc + hostPath: + path: /etc/nydus + type: DirectoryOrCreate + - name: nydus-opt + hostPath: + path: /opt/nydus + type: DirectoryOrCreate + - name: containerd-conf + hostPath: path: /etc/containerd/config.toml - type: "" - name: containerd-conf ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: nydusd-config - namespace: nydus-system -data: - nydusd.json: |- - { - "device": { - "backend": { - "type": "registry", - "config": { - "scheme": "", - "skip_verify": true, - "timeout": 10, - "connect_timeout": 10, - "retry_limit": 2 - } - }, - "cache": { - "type": "blobcache", - "config": { - "work_dir": "/var/lib/nydus/cache/" - } - } - }, - "mode": "direct", - "digest_validate": false, - "iostats_files": false, - "enable_xattr": true, - "amplify_io": 1048576, - "fs_prefetch": { - "enable": true, - "threads_count": 10, - "merging_size": 131072, - "bandwidth_rate": 1048576 - } - } + - name: local-bin + hostPath: + path: /usr/local/bin/ + - name: etc-systemd-system + hostPath: + path: /etc/systemd/system/ + - name: fuse + hostPath: + path: /dev/fuse --- apiVersion: v1 kind: ConfigMap metadata: - name: snapshotter-config + name: nydus-snapshotter-configs + labels: + app: nydus-snapshotter namespace: nydus-system data: + FS_DRIVER: "fusedev" + ENABLE_CONFIG_FROM_VOLUME: "true" + ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER: "false" + ENABLE_SYSTEMD_SERVICE: "false" config.toml: |- version = 1 root = "/var/lib/containerd-nydus" @@ -157,7 +176,6 @@ data: enable_stargz = false # Whether snapshotter should try to clean up resources when it is closed cleanup_on_close = false - [daemon] nydusd_path = "/usr/local/bin/nydusd" nydusimage_path = "/usr/local/bin/nydus-image" @@ -171,7 +189,6 @@ data: nydusd_config = "/etc/nydus/nydusd.json" # The fuse or fscache IO working threads started by nydusd threads_number = 4 - [log] # Snapshotter's log level level = "info" @@ -183,10 +200,8 @@ data: # In unit MB(megabytes) log_rotation_max_size = 1 log_to_stdout = false - [remote] convert_vpc_registry = false - [remote.auth] # Fetch the private registry auth by listening to K8s API server enable_kubeconfig_keychain = true @@ -196,17 +211,47 @@ data: enable_cri_keychain = false # the target image service when using image proxy image_service_address = "" - [snapshot] enable_nydus_overlayfs = false # Whether to remove resources when a snapshot is removed sync_remove = false - [cache_manager] disable = false gc_period = "24h" cache_dir = "" - [image] public_key_file = "" validate_signature = false + + nydusd.json: |- + { + "device": { + "backend": { + "type": "registry", + "config": { + "scheme": "", + "skip_verify": true, + "timeout": 10, + "connect_timeout": 10, + "retry_limit": 2 + } + }, + "cache": { + "type": "blobcache", + "config": { + "work_dir": "/var/lib/nydus/cache/" + } + } + }, + "mode": "direct", + "digest_validate": false, + "iostats_files": false, + "enable_xattr": true, + "amplify_io": 1048576, + "fs_prefetch": { + "enable": true, + "threads_count": 10, + "merging_size": 131072, + "bandwidth_rate": 1048576 + } + } \ No newline at end of file