From 9db676b41fd4546c06d7fd1df6afbac44d29bde5 Mon Sep 17 00:00:00 2001 From: Mathis Marcotte Date: Tue, 9 Jul 2024 15:54:13 +0000 Subject: [PATCH] added meta-fuse-csi-plugin code and made dockerfile --- .github/workflows/build.yaml | 0 Dockerfile | 35 ++ Makefile | 11 + go.work | 6 + go.work.sum | 63 ++++ .../cmd/fusermount3-proxy/main.go | 141 ++++++++ meta-fuse-csi-plugin/go.mod | 11 + meta-fuse-csi-plugin/go.sum | 8 + .../pkg/fuse_starter/fuse_starter.go | 105 ++++++ meta-fuse-csi-plugin/pkg/util/fdchannel.go | 84 +++++ meta-fuse-csi-plugin/pkg/util/util.go | 149 +++++++++ meta-fuse-csi-plugin/pkg/util/util_test.go | 306 ++++++++++++++++++ meta-fuse-csi-plugin/pkg/util/volume_lock.go | 61 ++++ 13 files changed, 980 insertions(+) create mode 100644 .github/workflows/build.yaml create mode 100644 Dockerfile create mode 100644 go.work create mode 100644 go.work.sum create mode 100644 meta-fuse-csi-plugin/cmd/fusermount3-proxy/main.go create mode 100644 meta-fuse-csi-plugin/go.mod create mode 100644 meta-fuse-csi-plugin/go.sum create mode 100644 meta-fuse-csi-plugin/pkg/fuse_starter/fuse_starter.go create mode 100644 meta-fuse-csi-plugin/pkg/util/fdchannel.go create mode 100644 meta-fuse-csi-plugin/pkg/util/util.go create mode 100644 meta-fuse-csi-plugin/pkg/util/util_test.go create mode 100755 meta-fuse-csi-plugin/pkg/util/volume_lock.go diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 00000000..e69de29b diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..0bd23caf --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ +FROM golang:1.20.7 as fusermount3-proxy-builder + +WORKDIR /meta-fuse-csi-plugin +ADD . . +# Builds the meta-fuse-csi-plugin app +RUN make fusermount3-proxy BINDIR=/bin +# Builds the goofys app +RUN CGO_ENABLED=0 GOOS=linux go build -o goofys + +FROM ubuntu:22.04 + +RUN apt update && apt upgrade -y +RUN apt install -y ca-certificates wget libfuse2 fuse3 + +# prepare for MinIO +RUN wget https://dl.min.io/client/mc/release/linux-amd64/mc -O /usr/bin/mc && chmod +x /usr/bin/mc + +COPY </dev/null || git rev-list -n1 HEAD) +export BUILD_DATE ?= $(shell date --iso-8601=minutes) +BINDIR ?= bin +LDFLAGS ?= -s -w -X main.version=${STAGINGVERSION} -X main.builddate=${BUILD_DATE} -extldflags '-static' +FUSERMOUNT3PROXY_BINARY = fusermount3-proxy + run-test: s3proxy.jar ./test/run-tests.sh @@ -14,3 +20,8 @@ build: install: go install -ldflags "-X main.Version=`git rev-parse HEAD`" + +fusermount3-proxy: + mkdir -p ${BINDIR} + CGO_ENABLED=0 GOOS=linux GOARCH=$(shell dpkg --print-architecture) go build -ldflags "${LDFLAGS}" -o ${BINDIR}/${FUSERMOUNT3PROXY_BINARY} meta-fuse-csi-plugin/cmd/fusermount3-proxy/main.go + diff --git a/go.work b/go.work new file mode 100644 index 00000000..57ee83a7 --- /dev/null +++ b/go.work @@ -0,0 +1,6 @@ +go 1.20 + +use ( + . + ./meta-fuse-csi-plugin +) diff --git a/go.work.sum b/go.work.sum new file mode 100644 index 00000000..04e9e374 --- /dev/null +++ b/go.work.sum @@ -0,0 +1,63 @@ +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fxamacker/cbor/v2 v2.6.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLywzIhbKM= +github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM= +github.com/onsi/gomega v1.31.0/go.mod h1:DW9aCi7U6Yi40wNVAvT6kzFnEVEI5n3DloYBiKiT6zk= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +golang.org/x/net v0.13.0 h1:Nvo8UFsZ8X3BhAC9699Z1j7XQ3rsZnUUm7jfBEk1ueY= +golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= +golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/apimachinery v0.28.1 h1:EJD40og3GizBSV3mkIoXQBsws32okPOy+MkRyzh6nPY= +k8s.io/apimachinery v0.28.1/go.mod h1:X0xh/chESs2hP9koe+SdIAcXWcQ+RM5hy0ZynB+yEvw= +k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/meta-fuse-csi-plugin/cmd/fusermount3-proxy/main.go b/meta-fuse-csi-plugin/cmd/fusermount3-proxy/main.go new file mode 100644 index 00000000..09d621a4 --- /dev/null +++ b/meta-fuse-csi-plugin/cmd/fusermount3-proxy/main.go @@ -0,0 +1,141 @@ +/* +Copyright 2023 Preferred Networks, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fusermount3proxy + +import ( + "fmt" + "os" + "strconv" + "syscall" + + starter "github.com/StatCan/goofys/meta-fuse-csi-plugin/pkg/fuse_starter" + "github.com/StatCan/goofys/meta-fuse-csi-plugin/pkg/util" + flag "github.com/spf13/pflag" + + "k8s.io/klog/v2" +) + +var ( + optUnmount = flag.BoolP("unmount", "u", false, "unmount (NOT SUPPORTED)") + optAutoUnmount = flag.BoolP("auto-unmount", "U", false, "auto-unmount (NOT SUPPORTED)") + optLazy = flag.BoolP("lazy", "z", false, "lazy umount (NOT SUPPORTED)") + optQuiet = flag.BoolP("quiet", "q", false, "quiet (NOT SUPPORTED)") + optHelp = flag.BoolP("help", "h", false, "print help") + optVersion = flag.BoolP("version", "V", false, "print version") + optOptions = flag.StringP("options", "o", "", "mount options") + // This is set at compile time. + version = "unknown" + builddate = "unknown" +) + +var ignoredOptions = map[string]*bool{ + "unmount": optUnmount, + "auto-unmount": optAutoUnmount, + "lazy": optLazy, + "optQuiet": optQuiet, +} + +const ( + ENV_FUSE_COMMFD = "_FUSE_COMMFD" + ENV_FUSERMOUNT3PROXY_FDPASSING_SOCKPATH = "FUSERMOUNT3PROXY_FDPASSING_SOCKPATH" +) + +func main() { + klog.InitFlags(nil) + flag.Parse() + + if *optHelp { + flag.PrintDefaults() + os.Exit(0) + } + + if *optVersion { + fmt.Printf("fusermount3-dummy version %v (BuildDate %v)\n", version, builddate) + os.Exit(0) + } + + klog.Infof("Running meta-fuse-csi-plugin fusermount3-dummy version %v (BuildDate %v)", version, builddate) + + if *optUnmount { + klog.Warning("'unmount' is not supported.") + os.Exit(0) + } + + if len(flag.Args()) == 0 { + klog.Error("mountpoint is not specified.") + os.Exit(1) + } + + if *optOptions == "" { + klog.Error("options is not specified.") + os.Exit(1) + } + + // fd-passing socket between fusermount3-dummy and csi-driver is passed as env var + fdPassingSocketPath := os.Getenv(ENV_FUSERMOUNT3PROXY_FDPASSING_SOCKPATH) + if fdPassingSocketPath == "" { + klog.Errorf("environment variable %q is not specified.", ENV_FUSERMOUNT3PROXY_FDPASSING_SOCKPATH) + os.Exit(1) + } + klog.Infof("fd-passing socket path is %q", fdPassingSocketPath) + + mntPoint := flag.Args()[0] + klog.Infof("mountpoint is %q, but ignored.", mntPoint) + + for k, v := range ignoredOptions { + if *v { + klog.Warningf("opiton %q is true, but ignored.", k) + } + } + + // TODO: send options to csi-driver and use them? + klog.Infof("options=%q", *optOptions) + + // get unix domain socket from caller + commFdStr := os.Getenv(ENV_FUSE_COMMFD) + commFd, err := strconv.Atoi(commFdStr) + if err != nil { + klog.Errorf("failed to get commFd _FUSE_COMMFD=%q", commFdStr) + os.Exit(1) + } + klog.Infof("commFd from %q is %d", ENV_FUSE_COMMFD, commFd) + + commConn, err := util.GetNetConnFromRawUnixSocketFd(commFd) + if err != nil { + klog.Errorf("failed to convert commFd to net.Conn: %w", err) + os.Exit(1) + } + klog.Infof("net.Conn is acquired from fd %d", commFd) + + // get fd for /dev/fuse from csi-driver + mc, err := starter.PrepareMountConfig(fdPassingSocketPath) + if err != nil { + klog.Errorf("failed to prepare mount config: socket path %q: %w", fdPassingSocketPath, err) + os.Exit(1) + } + defer syscall.Close(mc.FileDescriptor) + klog.Infof("received fd for /dev/fuse from csi-driver via socket %q", fdPassingSocketPath) + + // now already FUSE-fs mounted and fd is ready. + err = util.SendMsg(commConn, mc.FileDescriptor, []byte{0}) + if err != nil { + klog.Errorf("failed to send fd via commFd: %w", err) + os.Exit(1) + } + klog.Infof("sent fd for /dev/fuse via commFd %d", commFd) + klog.Info("exiting fusermount3-dummy...") +} diff --git a/meta-fuse-csi-plugin/go.mod b/meta-fuse-csi-plugin/go.mod new file mode 100644 index 00000000..37c5a69a --- /dev/null +++ b/meta-fuse-csi-plugin/go.mod @@ -0,0 +1,11 @@ +module github.com/StatCan/goofys/meta-fuse-csi-plugin + +go 1.20 + +require ( + github.com/spf13/pflag v1.0.5 + k8s.io/apimachinery v0.28.1 + k8s.io/klog/v2 v2.130.1 +) + +require github.com/go-logr/logr v1.4.1 // indirect diff --git a/meta-fuse-csi-plugin/go.sum b/meta-fuse-csi-plugin/go.sum new file mode 100644 index 00000000..32b660b7 --- /dev/null +++ b/meta-fuse-csi-plugin/go.sum @@ -0,0 +1,8 @@ +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +k8s.io/apimachinery v0.28.1 h1:EJD40og3GizBSV3mkIoXQBsws32okPOy+MkRyzh6nPY= +k8s.io/apimachinery v0.28.1/go.mod h1:X0xh/chESs2hP9koe+SdIAcXWcQ+RM5hy0ZynB+yEvw= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= diff --git a/meta-fuse-csi-plugin/pkg/fuse_starter/fuse_starter.go b/meta-fuse-csi-plugin/pkg/fuse_starter/fuse_starter.go new file mode 100644 index 00000000..7a367f3f --- /dev/null +++ b/meta-fuse-csi-plugin/pkg/fuse_starter/fuse_starter.go @@ -0,0 +1,105 @@ +/* +Copyright 2018 The Kubernetes Authors. +Copyright 2022 Google LLC +Copyright 2023 Preferred Networks, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fusestarter + +import ( + "encoding/json" + "fmt" + "net" + "os" + "os/exec" + "syscall" + + "github.com/StatCan/goofys/meta-fuse-csi-plugin/pkg/util" + "k8s.io/klog/v2" +) + +// FuseStarter will be used in the sidecar container to invoke fuse impl. +type FuseStarter struct { + mounterPath string + mounterArgs []string + Cmd *exec.Cmd +} + +// New returns a FuseStarter for the current system. +// It provides an option to specify the path to fuse binary. +func New(mounterPath string, mounterArgs []string) *FuseStarter { + return &FuseStarter{ + mounterPath: mounterPath, + mounterArgs: mounterArgs, + Cmd: nil, + } +} + +type MountConfig struct { + FileDescriptor int `json:"-"` + VolumeName string `json:"volumeName,omitempty"` +} + +func (m *FuseStarter) Mount(mc *MountConfig) (*exec.Cmd, error) { + klog.Infof("start to invoke fuse impl for volume %q", mc.VolumeName) + + klog.Infof("%s mounting with args %v...", m.mounterPath, m.mounterArgs) + cmd := exec.Cmd{ + Path: m.mounterPath, + Args: append([]string{m.mounterPath}, m.mounterArgs...), + ExtraFiles: []*os.File{os.NewFile(uintptr(mc.FileDescriptor), "/dev/fuse")}, + Stdout: os.Stdout, + Stderr: os.Stderr, + } + + m.Cmd = &cmd + + return &cmd, nil +} + +// Fetch the following information from a given socket path: +// 1. Pod volume name +// 2. The file descriptor +// 3. Mount options passing to mounter (passed by the csi mounter). +func PrepareMountConfig(sp string) (*MountConfig, error) { + mc := MountConfig{} + + klog.Infof("connecting to socket %q", sp) + c, err := net.Dial("unix", sp) + if err != nil { + return nil, fmt.Errorf("failed to connect to the socket %q: %w", sp, err) + } + defer func() { + // as we got all the information from the socket, closing the connection and deleting the socket + c.Close() + if err = syscall.Unlink(sp); err != nil { + // csi driver may already removed the socket. + klog.Warningf("failed to close socket %q: %v", sp, err) + } + }() + + fd, msg, err := util.RecvMsg(c) + if err != nil { + return nil, fmt.Errorf("failed to receive mount options from the socket %q: %w", sp, err) + } + + mc.FileDescriptor = fd + + if err := json.Unmarshal(msg, &mc); err != nil { + return nil, fmt.Errorf("failed to unmarshal the mount config: %w", err) + } + + return &mc, nil +} diff --git a/meta-fuse-csi-plugin/pkg/util/fdchannel.go b/meta-fuse-csi-plugin/pkg/util/fdchannel.go new file mode 100644 index 00000000..721349aa --- /dev/null +++ b/meta-fuse-csi-plugin/pkg/util/fdchannel.go @@ -0,0 +1,84 @@ +/* +Copyright 2018 The Kubernetes Authors. +Copyright 2022 Google LLC +Copyright 2023 Preferred Networks, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" + "net" + "syscall" + + "k8s.io/klog/v2" +) + +func SendMsg(via net.Conn, fd int, msg []byte) error { + klog.V(4).Info("get the underlying socket") + conn, ok := via.(*net.UnixConn) + if !ok { + return fmt.Errorf("failed to cast via to *net.UnixConn") + } + connf, err := conn.File() + if err != nil { + return err + } + socket := int(connf.Fd()) + defer connf.Close() + + klog.V(4).Infof("calling sendmsg...") + rights := syscall.UnixRights(fd) + + return syscall.Sendmsg(socket, msg, rights, nil, 0) +} + +func RecvMsg(via net.Conn) (int, []byte, error) { + klog.V(4).Info("get the underlying socket") + conn, ok := via.(*net.UnixConn) + if !ok { + return 0, nil, fmt.Errorf("failed to cast via to *net.UnixConn") + } + connf, err := conn.File() + if err != nil { + return 0, nil, err + } + socket := int(connf.Fd()) + defer connf.Close() + + klog.V(4).Info("calling recvmsg...") + buf := make([]byte, syscall.CmsgSpace(4)) + b := make([]byte, 500) + //nolint:dogsled + n, _, _, _, err := syscall.Recvmsg(socket, b, buf, 0) + if err != nil { + return 0, nil, err + } + + klog.V(4).Info("parsing SCM...") + var msgs []syscall.SocketControlMessage + msgs, err = syscall.ParseSocketControlMessage(buf) + if err != nil { + return 0, nil, err + } + + klog.V(4).Info("parsing SCM_RIGHTS...") + fds, err := syscall.ParseUnixRights(&msgs[0]) + if err != nil { + return 0, nil, err + } + + return fds[0], b[:n], err +} diff --git a/meta-fuse-csi-plugin/pkg/util/util.go b/meta-fuse-csi-plugin/pkg/util/util.go new file mode 100644 index 00000000..7c418f78 --- /dev/null +++ b/meta-fuse-csi-plugin/pkg/util/util.go @@ -0,0 +1,149 @@ +/* +Copyright 2018 The Kubernetes Authors. +Copyright 2022 Google LLC +Copyright 2023 Preferred Networks, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" + "net" + "net/url" + "os" + "regexp" + "strings" + + "k8s.io/klog/v2" +) + +const ( + Mb = 1024 * 1024 +) + +// ConvertLabelsStringToMap converts the labels from string to map +// example: "key1=value1,key2=value2" gets converted into {"key1": "value1", "key2": "value2"} +func ConvertLabelsStringToMap(labels string) (map[string]string, error) { + const labelsDelimiter = "," + const labelsKeyValueDelimiter = "=" + + labelsMap := make(map[string]string) + if labels == "" { + return labelsMap, nil + } + + // Following rules enforced for label keys + // 1. Keys have a minimum length of 1 character and a maximum length of 63 characters, and cannot be empty. + // 2. Keys and values can contain only lowercase letters, numeric characters, underscores, and dashes. + // 3. Keys must start with a lowercase letter. + regexKey := regexp.MustCompile(`^\p{Ll}[\p{Ll}0-9_-]{0,62}$`) + checkLabelKeyFn := func(key string) error { + if !regexKey.MatchString(key) { + return fmt.Errorf("label value %q is invalid (should start with lowercase letter / lowercase letter, digit, _ and - chars are allowed / 1-63 characters", key) + } + + return nil + } + + // Values can be empty, and have a maximum length of 63 characters. + regexValue := regexp.MustCompile(`^[\p{Ll}0-9_-]{0,63}$`) + checkLabelValueFn := func(value string) error { + if !regexValue.MatchString(value) { + return fmt.Errorf("label value %q is invalid (lowercase letter, digit, _ and - chars are allowed / 0-63 characters", value) + } + + return nil + } + + keyValueStrings := strings.Split(labels, labelsDelimiter) + for _, keyValue := range keyValueStrings { + keyValue := strings.Split(keyValue, labelsKeyValueDelimiter) + + if len(keyValue) != 2 { + return nil, fmt.Errorf("labels %q are invalid, correct format: 'key1=value1,key2=value2'", labels) + } + + key := strings.TrimSpace(keyValue[0]) + if err := checkLabelKeyFn(key); err != nil { + return nil, err + } + + value := strings.TrimSpace(keyValue[1]) + if err := checkLabelValueFn(value); err != nil { + return nil, err + } + + labelsMap[key] = value + } + + const maxNumberOfLabels = 64 + if len(labelsMap) > maxNumberOfLabels { + return nil, fmt.Errorf("more than %d labels is not allowed, given: %d", maxNumberOfLabels, len(labelsMap)) + } + + return labelsMap, nil +} + +func ParseEndpoint(endpoint string, cleanupSocket bool) (string, string, error) { + u, err := url.Parse(endpoint) + if err != nil { + klog.Fatal(err.Error()) + } + + var addr string + switch u.Scheme { + case "unix": + addr = u.Path + if cleanupSocket { + if err := os.Remove(addr); err != nil && !os.IsNotExist(err) { + klog.Fatalf("Failed to remove %s, error: %s", addr, err) + } + } + case "tcp": + addr = u.Host + default: + klog.Fatalf("%v endpoint scheme not supported", u.Scheme) + } + + return u.Scheme, addr, nil +} + +func ParsePodIDVolumeFromTargetpath(targetPath string) (string, string, error) { + r := regexp.MustCompile(`/var/lib/kubelet/pods/(.*)/volumes/kubernetes\.io~csi/(.*)/mount`) + matched := r.FindStringSubmatch(targetPath) + if len(matched) < 3 { + return "", "", fmt.Errorf("targetPath %v does not contain Pod ID or volume information", targetPath) + } + podID := matched[1] + volume := matched[2] + + return podID, volume, nil +} + +func GetEmptyDirPath(podId, emptyDirName string) string { + return fmt.Sprintf("/var/lib/kubelet/pods/%s/volumes/kubernetes.io~empty-dir/%s", podId, emptyDirName) +} + +func GetNetConnFromRawUnixSocketFd(fd int) (net.Conn, error) { + f := os.NewFile(uintptr(fd), "unix_socket") + defer f.Close() + + c, err := net.FileConn(f) + if err != nil { + return nil, err + } + + return c, err +} diff --git a/meta-fuse-csi-plugin/pkg/util/util_test.go b/meta-fuse-csi-plugin/pkg/util/util_test.go new file mode 100644 index 00000000..759cdda6 --- /dev/null +++ b/meta-fuse-csi-plugin/pkg/util/util_test.go @@ -0,0 +1,306 @@ +/* +Copyright 2018 The Kubernetes Authors. +Copyright 2022 Google LLC +Copyright 2023 Preferred Networks, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "reflect" + "testing" +) + +func TestConvertLabelsStringToMap(t *testing.T) { + t.Parallel() + t.Run("parsing labels string into map", func(t *testing.T) { + t.Parallel() + testCases := []struct { + name string + labels string + expectedOutput map[string]string + expectedError bool + }{ + // Success test cases + { + name: "should return empty map when labels string is empty", + labels: "", + expectedOutput: map[string]string{}, + expectedError: false, + }, + { + name: "single label string", + labels: "key=value", + expectedOutput: map[string]string{ + "key": "value", + }, + expectedError: false, + }, + { + name: "multiple label string", + labels: "key1=value1,key2=value2", + expectedOutput: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + expectedError: false, + }, + { + name: "multiple labels string with whitespaces gets trimmed", + labels: "key1=value1, key2=value2", + expectedOutput: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + expectedError: false, + }, + // Failure test cases + { + name: "malformed labels string (no keys and values)", + labels: ",,", + expectedOutput: nil, + expectedError: true, + }, + { + name: "malformed labels string (incorrect format)", + labels: "foo,bar", + expectedOutput: nil, + expectedError: true, + }, + { + name: "malformed labels string (missing key)", + labels: "key1=value1,=bar", + expectedOutput: nil, + expectedError: true, + }, + { + name: "malformed labels string (missing key and value)", + labels: "key1=value1,=bar,=", + expectedOutput: nil, + expectedError: true, + }, + } + + for _, tc := range testCases { + t.Logf("test case: %s", tc.name) + output, err := ConvertLabelsStringToMap(tc.labels) + if tc.expectedError && err == nil { + t.Errorf("Expected error but got none") + } + if err != nil { + if !tc.expectedError { + t.Errorf("Did not expect error but got: %v", err) + } + + continue + } + + if !reflect.DeepEqual(output, tc.expectedOutput) { + t.Errorf("Got labels %v, but expected %v", output, tc.expectedOutput) + } + } + }) + + t.Run("checking google requirements", func(t *testing.T) { + t.Parallel() + testCases := []struct { + name string + labels string + expectedError bool + }{ + { + name: "64 labels at most", + labels: `k1=v,k2=v,k3=v,k4=v,k5=v,k6=v,k7=v,k8=v,k9=v,k10=v,k11=v,k12=v,k13=v,k14=v,k15=v,k16=v,k17=v,k18=v,k19=v,k20=v, + k21=v,k22=v,k23=v,k24=v,k25=v,k26=v,k27=v,k28=v,k29=v,k30=v,k31=v,k32=v,k33=v,k34=v,k35=v,k36=v,k37=v,k38=v,k39=v,k40=v, + k41=v,k42=v,k43=v,k44=v,k45=v,k46=v,k47=v,k48=v,k49=v,k50=v,k51=v,k52=v,k53=v,k54=v,k55=v,k56=v,k57=v,k58=v,k59=v,k60=v, + k61=v,k62=v,k63=v,k64=v,k65=v`, + expectedError: true, + }, + { + name: "label key must have atleast 1 char", + labels: "=v", + expectedError: true, + }, + { + name: "label key can only contain lowercase chars, digits, _ and -)", + labels: "k*=v", + expectedError: true, + }, + { + name: "label key can only contain lowercase chars)", + labels: "K=v", + expectedError: true, + }, + { + name: "label key may not have over 63 characters", + labels: "abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij1234=v", + expectedError: true, + }, + { + name: "label value can only contain lowercase chars, digits, _ and -)", + labels: "k1=###", + expectedError: true, + }, + { + name: "label value can only contain lowercase chars)", + labels: "k1=V", + expectedError: true, + }, + { + name: "label key cannot contain . and /", + labels: "kubernetes.io/created-for/pvc/namespace=v", + expectedError: true, + }, + { + name: "label value cannot contain . and /", + labels: "kubernetes_io_created-for_pvc_namespace=v./", + expectedError: true, + }, + { + name: "label value may not have over 63 chars", + labels: "v=abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij1234", + expectedError: true, + }, + { + name: "label key can have up to 63 chars", + labels: "abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij123=v", + expectedError: false, + }, + { + name: "label value can have up to 63 chars", + labels: "k=abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij123", + expectedError: false, + }, + { + name: "label key can contain - and _", + labels: "abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij-_=v", + expectedError: false, + }, + { + name: "label value can contain - and _", + labels: "k=abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij-_", + expectedError: false, + }, + { + name: "label value can have 0 chars", + labels: "kubernetes_io_created-for_pvc_namespace=", + expectedError: false, + }, + } + + for _, tc := range testCases { + t.Logf("test case: %s", tc.name) + _, err := ConvertLabelsStringToMap(tc.labels) + + if tc.expectedError && err == nil { + t.Errorf("Expected error but got none") + } + + if !tc.expectedError && err != nil { + t.Errorf("Did not expect error but got: %v", err) + } + } + }) +} + +func TestParseEndpoint(t *testing.T) { + t.Parallel() + testCases := []struct { + name string + endpoint string + expectedScheme string + expectedAddress string + expectedError bool + }{ + { + name: "should parse unix endpoint correctly", + endpoint: "unix:/csi/csi.sock", + expectedScheme: "unix", + expectedAddress: "/csi/csi.sock", + expectedError: false, + }, + } + + for _, tc := range testCases { + t.Logf("test case: %s", tc.name) + scheme, address, err := ParseEndpoint(tc.endpoint, false) + if tc.expectedError && err == nil { + t.Errorf("Expected error but got none") + } + if err != nil { + if !tc.expectedError { + t.Errorf("Did not expect error but got: %v", err) + } + + continue + } + + if !reflect.DeepEqual(scheme, tc.expectedScheme) { + t.Errorf("Got scheme %v, but expected %v", scheme, tc.expectedScheme) + } + + if !reflect.DeepEqual(address, tc.expectedAddress) { + t.Errorf("Got address %v, but expected %v", address, tc.expectedAddress) + } + } +} + +func TestParsePodIDVolumeFromTargetpath(t *testing.T) { + t.Parallel() + testCases := []struct { + name string + targetPath string + expectedPodID string + expectedVolume string + expectedError bool + }{ + { + name: "should parse Pod ID correctly", + targetPath: "/var/lib/kubelet/pods/d2013878-3d56-45f9-89ec-0826612c89b6/volumes/kubernetes.io~csi/test-volume/mount", + expectedPodID: "d2013878-3d56-45f9-89ec-0826612c89b6", + expectedVolume: "test-volume", + expectedError: false, + }, + { + name: "should return error", + targetPath: "/foo/bar/volumes", + expectedPodID: "", + expectedVolume: "", + expectedError: true, + }, + } + + for _, tc := range testCases { + t.Logf("test case: %s", tc.name) + podID, volume, err := ParsePodIDVolumeFromTargetpath(tc.targetPath) + if tc.expectedError && err == nil { + t.Errorf("Expected error but got none") + } + if err != nil { + if !tc.expectedError { + t.Errorf("Did not expect error but got: %v", err) + } + + continue + } + + if !reflect.DeepEqual(podID, tc.expectedPodID) { + t.Errorf("Got pod ID %v, but expected %v", podID, tc.expectedPodID) + } + if !reflect.DeepEqual(volume, tc.expectedVolume) { + t.Errorf("Got volume %v, but expected %v", volume, tc.expectedVolume) + } + } +} diff --git a/meta-fuse-csi-plugin/pkg/util/volume_lock.go b/meta-fuse-csi-plugin/pkg/util/volume_lock.go new file mode 100755 index 00000000..c64aa7de --- /dev/null +++ b/meta-fuse-csi-plugin/pkg/util/volume_lock.go @@ -0,0 +1,61 @@ +/* +Copyright 2018 The Kubernetes Authors. +Copyright 2022 Google LLC +Copyright 2023 Preferred Networks, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "sync" + + "k8s.io/apimachinery/pkg/util/sets" +) + +const ( + VolumeOperationAlreadyExistsFmt = "An operation with the given volume key %s already exists" +) + +// VolumeLocks implements a map with atomic operations. It stores a set of all volume IDs +// with an ongoing operation. +type VolumeLocks struct { + locks sets.Set[string] + mux sync.Mutex +} + +func NewVolumeLocks() *VolumeLocks { + return &VolumeLocks{ + locks: sets.Set[string]{}, + } +} + +// TryAcquire tries to acquire the lock for operating on volumeID and returns true if successful. +// If another operation is already using volumeID, returns false. +func (vl *VolumeLocks) TryAcquire(volumeID string) bool { + vl.mux.Lock() + defer vl.mux.Unlock() + if vl.locks.Has(volumeID) { + return false + } + vl.locks.Insert(volumeID) + + return true +} + +func (vl *VolumeLocks) Release(volumeID string) { + vl.mux.Lock() + defer vl.mux.Unlock() + vl.locks.Delete(volumeID) +}