From 379ea7091248ea088ac711acd9b967b88821660e Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 26 Jul 2024 03:04:47 +0000 Subject: [PATCH] fix(deps): update module github.com/containers/storage to v1.55.0 Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 14 +- go.sum | 28 +- vendor/github.com/BurntSushi/toml/README.md | 2 +- vendor/github.com/BurntSushi/toml/decode.go | 93 +-- .../BurntSushi/toml/decode_go116.go | 19 - .../github.com/BurntSushi/toml/deprecated.go | 12 +- vendor/github.com/BurntSushi/toml/doc.go | 3 - vendor/github.com/BurntSushi/toml/encode.go | 45 +- vendor/github.com/BurntSushi/toml/error.go | 111 ++- vendor/github.com/BurntSushi/toml/lex.go | 50 +- vendor/github.com/BurntSushi/toml/meta.go | 49 +- vendor/github.com/BurntSushi/toml/parse.go | 269 ++++--- .../github.com/BurntSushi/toml/type_fields.go | 8 +- .../github.com/BurntSushi/toml/type_toml.go | 11 +- .../Microsoft/hcsshim/hnsaccelnet.go | 46 ++ .../hcsshim/internal/hns/hnsaccelnet.go | 60 ++ .../hcsshim/internal/hns/hnsendpoint.go | 23 + .../hcsshim/internal/hns/hnspolicy.go | 7 +- .../github.com/containers/storage/.cirrus.yml | 2 +- .../containers/storage/.golangci.yml | 6 +- vendor/github.com/containers/storage/Makefile | 2 + vendor/github.com/containers/storage/VERSION | 2 +- vendor/github.com/containers/storage/check.go | 17 +- .../containers/storage/drivers/aufs/aufs.go | 43 +- .../containers/storage/drivers/btrfs/btrfs.go | 26 +- .../containers/storage/drivers/chown.go | 10 +- .../containers/storage/drivers/chroot_unix.go | 4 +- .../storage/drivers/copy/copy_linux.go | 6 +- .../containers/storage/drivers/driver.go | 16 +- .../storage/drivers/overlay/check.go | 6 +- .../storage/drivers/overlay/composefs.go | 12 +- .../storage/drivers/overlay/overlay.go | 273 ++++--- .../{overlay_cgo.go => overlay_disk_quota.go} | 4 +- .../overlay/overlay_disk_quota_unsupported.go | 18 + .../storage/drivers/overlay/overlay_nocgo.go | 10 - .../drivers/quota/projectquota_supported.go | 11 - .../containers/storage/drivers/vfs/driver.go | 31 +- .../containers/storage/drivers/zfs/zfs.go | 20 +- .../github.com/containers/storage/layers.go | 4 +- .../containers/storage/lockfile_compat.go | 2 +- .../containers/storage/pkg/archive/archive.go | 88 ++- .../storage/pkg/archive/archive_bsd.go | 4 +- .../storage/pkg/archive/archive_linux.go | 4 +- .../storage/pkg/archive/changes_linux.go | 8 +- .../storage/pkg/archive/changes_unix.go | 4 +- .../containers/storage/pkg/archive/filter.go | 73 ++ .../pkg/chrootarchive/archive_darwin.go | 2 + .../storage/pkg/chrootarchive/archive_unix.go | 7 +- .../storage/pkg/chrootarchive/chroot_linux.go | 5 +- .../storage/pkg/chrootarchive/diff_unix.go | 6 +- ...{bloom_filter.go => bloom_filter_linux.go} | 9 + .../storage/pkg/chunked/cache_linux.go | 32 +- .../storage/pkg/chunked/compression_linux.go | 149 +++- .../pkg/chunked/compressor/compressor.go | 31 +- .../storage/pkg/chunked/dump/dump.go | 77 +- .../storage/pkg/chunked/filesystem_linux.go | 605 ++++++++++++++++ .../pkg/chunked/internal/compression.go | 116 ++- .../storage/pkg/chunked/storage_linux.go | 670 ++---------------- .../storage/pkg/chunked/tar_split_linux.go | 68 ++ .../containers/storage/pkg/config/config.go | 11 - .../storage/pkg/directory/directory_unix.go | 4 +- .../storage/pkg/homedir/homedir_others.go | 38 - .../storage/pkg/idmap/idmapped_utils.go | 15 +- .../containers/storage/pkg/idtools/idtools.go | 72 +- .../storage/pkg/ioutils/fswriters.go | 5 +- .../storage/pkg/lockfile/lockfile.go | 11 +- .../storage/pkg/lockfile/lockfile_unix.go | 4 +- .../storage/pkg/loopback/attach_loopback.go | 56 +- .../storage/pkg/mount/mountinfo_linux.go | 17 +- .../storage/pkg/system/syscall_unix.go | 4 +- .../storage/pkg/unshare/unshare_linux.go | 12 +- .../containers/storage/storage.conf | 26 +- .../containers/storage/storage.conf-freebsd | 21 - vendor/github.com/containers/storage/store.go | 102 ++- .../containers/storage/types/options.go | 29 +- .../{options_freebsd.go => options_bsd.go} | 4 +- .../storage/types/options_darwin.go | 2 +- .../containers/storage/types/options_linux.go | 2 +- .../storage/types/options_windows.go | 2 +- .../storage/types/storage_test.conf | 10 - .../containers/storage/types/utils.go | 5 +- .../cyphar/filepath-securejoin/CHANGELOG.md | 138 ++++ .../cyphar/filepath-securejoin/LICENSE | 2 +- .../cyphar/filepath-securejoin/README.md | 139 +++- .../cyphar/filepath-securejoin/VERSION | 2 +- .../cyphar/filepath-securejoin/join.go | 8 +- .../filepath-securejoin/lookup_linux.go | 389 ++++++++++ .../cyphar/filepath-securejoin/mkdir_linux.go | 229 ++++++ .../cyphar/filepath-securejoin/open_linux.go | 101 +++ .../filepath-securejoin/openat2_linux.go | 141 ++++ .../filepath-securejoin/openat_linux.go | 59 ++ .../filepath-securejoin/procfs_linux.go | 474 +++++++++++++ .../testing_mocks_linux.go | 68 ++ .../cyphar/filepath-securejoin/vfs.go | 2 +- .../klauspost/compress/flate/matchlen_amd64.s | 10 +- .../klauspost/compress/zstd/dict.go | 31 + .../zstd/internal/xxhash/xxhash_arm64.s | 4 +- .../klauspost/compress/zstd/matchlen_amd64.s | 10 +- .../moby/sys/mountinfo/mounted_linux.go | 2 +- vendor/github.com/moby/sys/user/user.go | 1 - .../selinux/pkg/pwalk/README.md | 48 -- .../opencontainers/selinux/pkg/pwalk/pwalk.go | 123 ---- vendor/modules.txt | 23 +- 103 files changed, 4146 insertions(+), 1643 deletions(-) delete mode 100644 vendor/github.com/BurntSushi/toml/decode_go116.go create mode 100644 vendor/github.com/Microsoft/hcsshim/hnsaccelnet.go create mode 100644 vendor/github.com/Microsoft/hcsshim/internal/hns/hnsaccelnet.go rename vendor/github.com/containers/storage/drivers/overlay/{overlay_cgo.go => overlay_disk_quota.go} (86%) create mode 100644 vendor/github.com/containers/storage/drivers/overlay/overlay_disk_quota_unsupported.go create mode 100644 vendor/github.com/containers/storage/pkg/archive/filter.go rename vendor/github.com/containers/storage/pkg/chunked/{bloom_filter.go => bloom_filter_linux.go} (87%) create mode 100644 vendor/github.com/containers/storage/pkg/chunked/filesystem_linux.go create mode 100644 vendor/github.com/containers/storage/pkg/chunked/tar_split_linux.go delete mode 100644 vendor/github.com/containers/storage/pkg/homedir/homedir_others.go rename vendor/github.com/containers/storage/types/{options_freebsd.go => options_bsd.go} (90%) create mode 100644 vendor/github.com/cyphar/filepath-securejoin/CHANGELOG.md create mode 100644 vendor/github.com/cyphar/filepath-securejoin/lookup_linux.go create mode 100644 vendor/github.com/cyphar/filepath-securejoin/mkdir_linux.go create mode 100644 vendor/github.com/cyphar/filepath-securejoin/open_linux.go create mode 100644 vendor/github.com/cyphar/filepath-securejoin/openat2_linux.go create mode 100644 vendor/github.com/cyphar/filepath-securejoin/openat_linux.go create mode 100644 vendor/github.com/cyphar/filepath-securejoin/procfs_linux.go create mode 100644 vendor/github.com/cyphar/filepath-securejoin/testing_mocks_linux.go delete mode 100644 vendor/github.com/opencontainers/selinux/pkg/pwalk/README.md delete mode 100644 vendor/github.com/opencontainers/selinux/pkg/pwalk/pwalk.go diff --git a/go.mod b/go.mod index 4abcf172fb..37d4131581 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/containers/common v0.59.2 github.com/containers/image/v5 v5.31.1 github.com/containers/ocicrypt v1.2.0 - github.com/containers/storage v1.54.0 + github.com/containers/storage v1.55.0 github.com/docker/distribution v2.8.3+incompatible github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.1.0 @@ -28,9 +28,9 @@ require ( require ( dario.cat/mergo v1.0.0 // indirect - github.com/BurntSushi/toml v1.3.2 // indirect + github.com/BurntSushi/toml v1.4.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/Microsoft/hcsshim v0.12.3 // indirect + github.com/Microsoft/hcsshim v0.12.5 // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect @@ -40,7 +40,7 @@ require ( github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 // indirect github.com/coreos/go-oidc/v3 v3.10.0 // indirect github.com/cyberphone/json-canonicalization v0.0.0-20231217050601-ba74d44ecf5f // indirect - github.com/cyphar/filepath-securejoin v0.2.5 // indirect + github.com/cyphar/filepath-securejoin v0.3.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/docker v26.1.3+incompatible // indirect @@ -77,7 +77,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.8 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/letsencrypt/boulder v0.0.0-20230907030200-6d76a0f91e1e // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -87,8 +87,8 @@ require ( github.com/mistifyio/go-zfs/v3 v3.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect - github.com/moby/sys/mountinfo v0.7.1 // indirect - github.com/moby/sys/user v0.1.0 // indirect + github.com/moby/sys/mountinfo v0.7.2 // indirect + github.com/moby/sys/user v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/oklog/ulid v1.3.1 // indirect diff --git a/go.sum b/go.sum index 0922f0ff8b..a8cde8fe6b 100644 --- a/go.sum +++ b/go.sum @@ -6,14 +6,14 @@ github.com/14rcole/gopopulate v0.0.0-20180821133914-b175b219e774/go.mod h1:6/0dY github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= -github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= +github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/Microsoft/hcsshim v0.12.3 h1:LS9NXqXhMoqNCplK1ApmVSfB4UnVLRDWRapB6EIlxE0= -github.com/Microsoft/hcsshim v0.12.3/go.mod h1:Iyl1WVpZzr+UkzjekHZbV8o5Z9ZkxNGx6CtY2Qg/JVQ= +github.com/Microsoft/hcsshim v0.12.5 h1:bpTInLlDy/nDRWFVcefDZZ1+U8tS+rz3MxjKgu9boo0= +github.com/Microsoft/hcsshim v0.12.5/go.mod h1:tIUGego4G1EN5Hb6KC90aDYiUI2dqLSTTOCjVNpOgZ8= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= @@ -45,15 +45,15 @@ github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 h1:Qzk5C6cYgle github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY= github.com/containers/ocicrypt v1.2.0 h1:X14EgRK3xNFvJEfI5O4Qn4T3E25ANudSOZz/sirVuPM= github.com/containers/ocicrypt v1.2.0/go.mod h1:ZNviigQajtdlxIZGibvblVuIFBKIuUI2M0QM12SD31U= -github.com/containers/storage v1.54.0 h1:xwYAlf6n9OnIlURQLLg3FYHbO74fQ/2W2N6EtQEUM4I= -github.com/containers/storage v1.54.0/go.mod h1:PlMOoinRrBSnhYODLxt4EXl0nmJt+X0kjG0Xdt9fMTw= +github.com/containers/storage v1.55.0 h1:wTWZ3YpcQf1F+dSP4KxG9iqDfpQY1otaUXjPpffuhgg= +github.com/containers/storage v1.55.0/go.mod h1:28cB81IDk+y7ok60Of6u52RbCeBRucbFOeLunhER1RQ= github.com/coreos/go-oidc/v3 v3.10.0 h1:tDnXHnLyiTVyT/2zLDGj09pFPkhND8Gl8lnTRhoEaJU= github.com/coreos/go-oidc/v3 v3.10.0/go.mod h1:5j11xcw0D3+SGxn6Z/WFADsgcWVMyNAlSQupk0KK3ac= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cyberphone/json-canonicalization v0.0.0-20231217050601-ba74d44ecf5f h1:eHnXnuK47UlSTOQexbzxAZfekVz6i+LKRdj1CU5DPaM= github.com/cyberphone/json-canonicalization v0.0.0-20231217050601-ba74d44ecf5f/go.mod h1:uzvlm1mxhHkdfqitSA92i7Se+S9ksOn3a3qmv/kyOCw= -github.com/cyphar/filepath-securejoin v0.2.5 h1:6iR5tXJ/e6tJZzzdMc1km3Sa7RRIVBKAK32O2s7AYfo= -github.com/cyphar/filepath-securejoin v0.2.5/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= +github.com/cyphar/filepath-securejoin v0.3.1 h1:1V7cHiaW+C+39wEfpH6XlLBQo3j/PciWFrgfCLS8XrE= +github.com/cyphar/filepath-securejoin v0.3.1/go.mod h1:F7i41x/9cBF7lzCrVsYs9fuzwRZm4NQsGTBdpp6mETc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -185,8 +185,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= -github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= @@ -214,10 +214,10 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= -github.com/moby/sys/mountinfo v0.7.1 h1:/tTvQaSJRr2FshkhXiIpux6fQ2Zvc4j7tAhMTStAG2g= -github.com/moby/sys/mountinfo v0.7.1/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= -github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= -github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= +github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= +github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= +github.com/moby/sys/user v0.2.0 h1:OnpapJsRp25vkhw8TFG6OLJODNh/3rEwRWtJ3kakwRM= +github.com/moby/sys/user v0.2.0/go.mod h1:RYstrcWOJpVh+6qzUqp2bU3eaRpdiQeKGlKitaH0PM8= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= diff --git a/vendor/github.com/BurntSushi/toml/README.md b/vendor/github.com/BurntSushi/toml/README.md index 3651cfa960..639e6c3998 100644 --- a/vendor/github.com/BurntSushi/toml/README.md +++ b/vendor/github.com/BurntSushi/toml/README.md @@ -9,7 +9,7 @@ See the [releases page](https://github.com/BurntSushi/toml/releases) for a changelog; this information is also in the git tag annotations (e.g. `git show v0.4.0`). -This library requires Go 1.13 or newer; add it to your go.mod with: +This library requires Go 1.18 or newer; add it to your go.mod with: % go get github.com/BurntSushi/toml@latest diff --git a/vendor/github.com/BurntSushi/toml/decode.go b/vendor/github.com/BurntSushi/toml/decode.go index 4d38f3bfce..7aaf462c94 100644 --- a/vendor/github.com/BurntSushi/toml/decode.go +++ b/vendor/github.com/BurntSushi/toml/decode.go @@ -6,7 +6,7 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" + "io/fs" "math" "os" "reflect" @@ -18,13 +18,13 @@ import ( // Unmarshaler is the interface implemented by objects that can unmarshal a // TOML description of themselves. type Unmarshaler interface { - UnmarshalTOML(interface{}) error + UnmarshalTOML(any) error } // Unmarshal decodes the contents of data in TOML format into a pointer v. // // See [Decoder] for a description of the decoding process. -func Unmarshal(data []byte, v interface{}) error { +func Unmarshal(data []byte, v any) error { _, err := NewDecoder(bytes.NewReader(data)).Decode(v) return err } @@ -32,12 +32,12 @@ func Unmarshal(data []byte, v interface{}) error { // Decode the TOML data in to the pointer v. // // See [Decoder] for a description of the decoding process. -func Decode(data string, v interface{}) (MetaData, error) { +func Decode(data string, v any) (MetaData, error) { return NewDecoder(strings.NewReader(data)).Decode(v) } // DecodeFile reads the contents of a file and decodes it with [Decode]. -func DecodeFile(path string, v interface{}) (MetaData, error) { +func DecodeFile(path string, v any) (MetaData, error) { fp, err := os.Open(path) if err != nil { return MetaData{}, err @@ -46,6 +46,17 @@ func DecodeFile(path string, v interface{}) (MetaData, error) { return NewDecoder(fp).Decode(v) } +// DecodeFS reads the contents of a file from [fs.FS] and decodes it with +// [Decode]. +func DecodeFS(fsys fs.FS, path string, v any) (MetaData, error) { + fp, err := fsys.Open(path) + if err != nil { + return MetaData{}, err + } + defer fp.Close() + return NewDecoder(fp).Decode(v) +} + // Primitive is a TOML value that hasn't been decoded into a Go value. // // This type can be used for any value, which will cause decoding to be delayed. @@ -58,7 +69,7 @@ func DecodeFile(path string, v interface{}) (MetaData, error) { // overhead of reflection. They can be useful when you don't know the exact type // of TOML data until runtime. type Primitive struct { - undecoded interface{} + undecoded any context Key } @@ -122,7 +133,7 @@ var ( ) // Decode TOML data in to the pointer `v`. -func (dec *Decoder) Decode(v interface{}) (MetaData, error) { +func (dec *Decoder) Decode(v any) (MetaData, error) { rv := reflect.ValueOf(v) if rv.Kind() != reflect.Ptr { s := "%q" @@ -136,8 +147,8 @@ func (dec *Decoder) Decode(v interface{}) (MetaData, error) { return MetaData{}, fmt.Errorf("toml: cannot decode to nil value of %q", reflect.TypeOf(v)) } - // Check if this is a supported type: struct, map, interface{}, or something - // that implements UnmarshalTOML or UnmarshalText. + // Check if this is a supported type: struct, map, any, or something that + // implements UnmarshalTOML or UnmarshalText. rv = indirect(rv) rt := rv.Type() if rv.Kind() != reflect.Struct && rv.Kind() != reflect.Map && @@ -148,7 +159,7 @@ func (dec *Decoder) Decode(v interface{}) (MetaData, error) { // TODO: parser should read from io.Reader? Or at the very least, make it // read from []byte rather than string - data, err := ioutil.ReadAll(dec.r) + data, err := io.ReadAll(dec.r) if err != nil { return MetaData{}, err } @@ -179,7 +190,7 @@ func (dec *Decoder) Decode(v interface{}) (MetaData, error) { // will only reflect keys that were decoded. Namely, any keys hidden behind a // Primitive will be considered undecoded. Executing this method will update the // undecoded keys in the meta data. (See the example.) -func (md *MetaData) PrimitiveDecode(primValue Primitive, v interface{}) error { +func (md *MetaData) PrimitiveDecode(primValue Primitive, v any) error { md.context = primValue.context defer func() { md.context = nil }() return md.unify(primValue.undecoded, rvalue(v)) @@ -190,7 +201,7 @@ func (md *MetaData) PrimitiveDecode(primValue Primitive, v interface{}) error { // // Any type mismatch produces an error. Finding a type that we don't know // how to handle produces an unsupported type error. -func (md *MetaData) unify(data interface{}, rv reflect.Value) error { +func (md *MetaData) unify(data any, rv reflect.Value) error { // Special case. Look for a `Primitive` value. // TODO: #76 would make this superfluous after implemented. if rv.Type() == primitiveType { @@ -207,7 +218,11 @@ func (md *MetaData) unify(data interface{}, rv reflect.Value) error { rvi := rv.Interface() if v, ok := rvi.(Unmarshaler); ok { - return v.UnmarshalTOML(data) + err := v.UnmarshalTOML(data) + if err != nil { + return md.parseErr(err) + } + return nil } if v, ok := rvi.(encoding.TextUnmarshaler); ok { return md.unifyText(data, v) @@ -227,14 +242,6 @@ func (md *MetaData) unify(data interface{}, rv reflect.Value) error { return md.unifyInt(data, rv) } switch k { - case reflect.Ptr: - elem := reflect.New(rv.Type().Elem()) - err := md.unify(data, reflect.Indirect(elem)) - if err != nil { - return err - } - rv.Set(elem) - return nil case reflect.Struct: return md.unifyStruct(data, rv) case reflect.Map: @@ -258,14 +265,13 @@ func (md *MetaData) unify(data interface{}, rv reflect.Value) error { return md.e("unsupported type %s", rv.Kind()) } -func (md *MetaData) unifyStruct(mapping interface{}, rv reflect.Value) error { - tmap, ok := mapping.(map[string]interface{}) +func (md *MetaData) unifyStruct(mapping any, rv reflect.Value) error { + tmap, ok := mapping.(map[string]any) if !ok { if mapping == nil { return nil } - return md.e("type mismatch for %s: expected table but found %T", - rv.Type().String(), mapping) + return md.e("type mismatch for %s: expected table but found %s", rv.Type().String(), fmtType(mapping)) } for key, datum := range tmap { @@ -304,14 +310,14 @@ func (md *MetaData) unifyStruct(mapping interface{}, rv reflect.Value) error { return nil } -func (md *MetaData) unifyMap(mapping interface{}, rv reflect.Value) error { +func (md *MetaData) unifyMap(mapping any, rv reflect.Value) error { keyType := rv.Type().Key().Kind() if keyType != reflect.String && keyType != reflect.Interface { return fmt.Errorf("toml: cannot decode to a map with non-string key type (%s in %q)", keyType, rv.Type()) } - tmap, ok := mapping.(map[string]interface{}) + tmap, ok := mapping.(map[string]any) if !ok { if tmap == nil { return nil @@ -347,7 +353,7 @@ func (md *MetaData) unifyMap(mapping interface{}, rv reflect.Value) error { return nil } -func (md *MetaData) unifyArray(data interface{}, rv reflect.Value) error { +func (md *MetaData) unifyArray(data any, rv reflect.Value) error { datav := reflect.ValueOf(data) if datav.Kind() != reflect.Slice { if !datav.IsValid() { @@ -361,7 +367,7 @@ func (md *MetaData) unifyArray(data interface{}, rv reflect.Value) error { return md.unifySliceArray(datav, rv) } -func (md *MetaData) unifySlice(data interface{}, rv reflect.Value) error { +func (md *MetaData) unifySlice(data any, rv reflect.Value) error { datav := reflect.ValueOf(data) if datav.Kind() != reflect.Slice { if !datav.IsValid() { @@ -388,7 +394,7 @@ func (md *MetaData) unifySliceArray(data, rv reflect.Value) error { return nil } -func (md *MetaData) unifyString(data interface{}, rv reflect.Value) error { +func (md *MetaData) unifyString(data any, rv reflect.Value) error { _, ok := rv.Interface().(json.Number) if ok { if i, ok := data.(int64); ok { @@ -408,7 +414,7 @@ func (md *MetaData) unifyString(data interface{}, rv reflect.Value) error { return md.badtype("string", data) } -func (md *MetaData) unifyFloat64(data interface{}, rv reflect.Value) error { +func (md *MetaData) unifyFloat64(data any, rv reflect.Value) error { rvk := rv.Kind() if num, ok := data.(float64); ok { @@ -429,7 +435,7 @@ func (md *MetaData) unifyFloat64(data interface{}, rv reflect.Value) error { if num, ok := data.(int64); ok { if (rvk == reflect.Float32 && (num < -maxSafeFloat32Int || num > maxSafeFloat32Int)) || (rvk == reflect.Float64 && (num < -maxSafeFloat64Int || num > maxSafeFloat64Int)) { - return md.parseErr(errParseRange{i: num, size: rvk.String()}) + return md.parseErr(errUnsafeFloat{i: num, size: rvk.String()}) } rv.SetFloat(float64(num)) return nil @@ -438,7 +444,7 @@ func (md *MetaData) unifyFloat64(data interface{}, rv reflect.Value) error { return md.badtype("float", data) } -func (md *MetaData) unifyInt(data interface{}, rv reflect.Value) error { +func (md *MetaData) unifyInt(data any, rv reflect.Value) error { _, ok := rv.Interface().(time.Duration) if ok { // Parse as string duration, and fall back to regular integer parsing @@ -481,7 +487,7 @@ func (md *MetaData) unifyInt(data interface{}, rv reflect.Value) error { return nil } -func (md *MetaData) unifyBool(data interface{}, rv reflect.Value) error { +func (md *MetaData) unifyBool(data any, rv reflect.Value) error { if b, ok := data.(bool); ok { rv.SetBool(b) return nil @@ -489,12 +495,12 @@ func (md *MetaData) unifyBool(data interface{}, rv reflect.Value) error { return md.badtype("boolean", data) } -func (md *MetaData) unifyAnything(data interface{}, rv reflect.Value) error { +func (md *MetaData) unifyAnything(data any, rv reflect.Value) error { rv.Set(reflect.ValueOf(data)) return nil } -func (md *MetaData) unifyText(data interface{}, v encoding.TextUnmarshaler) error { +func (md *MetaData) unifyText(data any, v encoding.TextUnmarshaler) error { var s string switch sdata := data.(type) { case Marshaler: @@ -523,13 +529,13 @@ func (md *MetaData) unifyText(data interface{}, v encoding.TextUnmarshaler) erro return md.badtype("primitive (string-like)", data) } if err := v.UnmarshalText([]byte(s)); err != nil { - return err + return md.parseErr(err) } return nil } -func (md *MetaData) badtype(dst string, data interface{}) error { - return md.e("incompatible types: TOML value has type %T; destination has type %s", data, dst) +func (md *MetaData) badtype(dst string, data any) error { + return md.e("incompatible types: TOML value has type %s; destination has type %s", fmtType(data), dst) } func (md *MetaData) parseErr(err error) error { @@ -543,7 +549,7 @@ func (md *MetaData) parseErr(err error) error { } } -func (md *MetaData) e(format string, args ...interface{}) error { +func (md *MetaData) e(format string, args ...any) error { f := "toml: " if len(md.context) > 0 { f = fmt.Sprintf("toml: (last key %q): ", md.context) @@ -556,7 +562,7 @@ func (md *MetaData) e(format string, args ...interface{}) error { } // rvalue returns a reflect.Value of `v`. All pointers are resolved. -func rvalue(v interface{}) reflect.Value { +func rvalue(v any) reflect.Value { return indirect(reflect.ValueOf(v)) } @@ -600,3 +606,8 @@ func isUnifiable(rv reflect.Value) bool { } return false } + +// fmt %T with "interface {}" replaced with "any", which is far more readable. +func fmtType(t any) string { + return strings.ReplaceAll(fmt.Sprintf("%T", t), "interface {}", "any") +} diff --git a/vendor/github.com/BurntSushi/toml/decode_go116.go b/vendor/github.com/BurntSushi/toml/decode_go116.go deleted file mode 100644 index 086d0b6866..0000000000 --- a/vendor/github.com/BurntSushi/toml/decode_go116.go +++ /dev/null @@ -1,19 +0,0 @@ -//go:build go1.16 -// +build go1.16 - -package toml - -import ( - "io/fs" -) - -// DecodeFS reads the contents of a file from [fs.FS] and decodes it with -// [Decode]. -func DecodeFS(fsys fs.FS, path string, v interface{}) (MetaData, error) { - fp, err := fsys.Open(path) - if err != nil { - return MetaData{}, err - } - defer fp.Close() - return NewDecoder(fp).Decode(v) -} diff --git a/vendor/github.com/BurntSushi/toml/deprecated.go b/vendor/github.com/BurntSushi/toml/deprecated.go index b9e309717e..155709a80b 100644 --- a/vendor/github.com/BurntSushi/toml/deprecated.go +++ b/vendor/github.com/BurntSushi/toml/deprecated.go @@ -15,15 +15,15 @@ type TextMarshaler encoding.TextMarshaler // Deprecated: use encoding.TextUnmarshaler type TextUnmarshaler encoding.TextUnmarshaler +// DecodeReader is an alias for NewDecoder(r).Decode(v). +// +// Deprecated: use NewDecoder(reader).Decode(&value). +func DecodeReader(r io.Reader, v any) (MetaData, error) { return NewDecoder(r).Decode(v) } + // PrimitiveDecode is an alias for MetaData.PrimitiveDecode(). // // Deprecated: use MetaData.PrimitiveDecode. -func PrimitiveDecode(primValue Primitive, v interface{}) error { +func PrimitiveDecode(primValue Primitive, v any) error { md := MetaData{decoded: make(map[string]struct{})} return md.unify(primValue.undecoded, rvalue(v)) } - -// DecodeReader is an alias for NewDecoder(r).Decode(v). -// -// Deprecated: use NewDecoder(reader).Decode(&value). -func DecodeReader(r io.Reader, v interface{}) (MetaData, error) { return NewDecoder(r).Decode(v) } diff --git a/vendor/github.com/BurntSushi/toml/doc.go b/vendor/github.com/BurntSushi/toml/doc.go index 81a7c0fe9f..82c90a9057 100644 --- a/vendor/github.com/BurntSushi/toml/doc.go +++ b/vendor/github.com/BurntSushi/toml/doc.go @@ -2,9 +2,6 @@ // // This package supports TOML v1.0.0, as specified at https://toml.io // -// There is also support for delaying decoding with the Primitive type, and -// querying the set of keys in a TOML document with the MetaData type. -// // The github.com/BurntSushi/toml/cmd/tomlv package implements a TOML validator, // and can be used to verify if TOML document is valid. It can also be used to // print the type of each key. diff --git a/vendor/github.com/BurntSushi/toml/encode.go b/vendor/github.com/BurntSushi/toml/encode.go index 9cd25d7571..73366c0d9a 100644 --- a/vendor/github.com/BurntSushi/toml/encode.go +++ b/vendor/github.com/BurntSushi/toml/encode.go @@ -2,6 +2,7 @@ package toml import ( "bufio" + "bytes" "encoding" "encoding/json" "errors" @@ -76,6 +77,17 @@ type Marshaler interface { MarshalTOML() ([]byte, error) } +// Marshal returns a TOML representation of the Go value. +// +// See [Encoder] for a description of the encoding process. +func Marshal(v any) ([]byte, error) { + buff := new(bytes.Buffer) + if err := NewEncoder(buff).Encode(v); err != nil { + return nil, err + } + return buff.Bytes(), nil +} + // Encoder encodes a Go to a TOML document. // // The mapping between Go values and TOML values should be precisely the same as @@ -115,26 +127,21 @@ type Marshaler interface { // NOTE: only exported keys are encoded due to the use of reflection. Unexported // keys are silently discarded. type Encoder struct { - // String to use for a single indentation level; default is two spaces. - Indent string - + Indent string // string for a single indentation level; default is two spaces. + hasWritten bool // written any output to w yet? w *bufio.Writer - hasWritten bool // written any output to w yet? } // NewEncoder create a new Encoder. func NewEncoder(w io.Writer) *Encoder { - return &Encoder{ - w: bufio.NewWriter(w), - Indent: " ", - } + return &Encoder{w: bufio.NewWriter(w), Indent: " "} } // Encode writes a TOML representation of the Go value to the [Encoder]'s writer. // // An error is returned if the value given cannot be encoded to a valid TOML // document. -func (enc *Encoder) Encode(v interface{}) error { +func (enc *Encoder) Encode(v any) error { rv := eindirect(reflect.ValueOf(v)) err := enc.safeEncode(Key([]string{}), rv) if err != nil { @@ -280,18 +287,30 @@ func (enc *Encoder) eElement(rv reflect.Value) { case reflect.Float32: f := rv.Float() if math.IsNaN(f) { + if math.Signbit(f) { + enc.wf("-") + } enc.wf("nan") } else if math.IsInf(f, 0) { - enc.wf("%cinf", map[bool]byte{true: '-', false: '+'}[math.Signbit(f)]) + if math.Signbit(f) { + enc.wf("-") + } + enc.wf("inf") } else { enc.wf(floatAddDecimal(strconv.FormatFloat(f, 'f', -1, 32))) } case reflect.Float64: f := rv.Float() if math.IsNaN(f) { + if math.Signbit(f) { + enc.wf("-") + } enc.wf("nan") } else if math.IsInf(f, 0) { - enc.wf("%cinf", map[bool]byte{true: '-', false: '+'}[math.Signbit(f)]) + if math.Signbit(f) { + enc.wf("-") + } + enc.wf("inf") } else { enc.wf(floatAddDecimal(strconv.FormatFloat(f, 'f', -1, 64))) } @@ -304,7 +323,7 @@ func (enc *Encoder) eElement(rv reflect.Value) { case reflect.Interface: enc.eElement(rv.Elem()) default: - encPanic(fmt.Errorf("unexpected type: %T", rv.Interface())) + encPanic(fmt.Errorf("unexpected type: %s", fmtType(rv.Interface()))) } } @@ -712,7 +731,7 @@ func (enc *Encoder) writeKeyValue(key Key, val reflect.Value, inline bool) { } } -func (enc *Encoder) wf(format string, v ...interface{}) { +func (enc *Encoder) wf(format string, v ...any) { _, err := fmt.Fprintf(enc.w, format, v...) if err != nil { encPanic(err) diff --git a/vendor/github.com/BurntSushi/toml/error.go b/vendor/github.com/BurntSushi/toml/error.go index efd68865bb..b45a3f45f6 100644 --- a/vendor/github.com/BurntSushi/toml/error.go +++ b/vendor/github.com/BurntSushi/toml/error.go @@ -114,13 +114,22 @@ func (pe ParseError) ErrorWithPosition() string { msg, pe.Position.Line, col, col+pe.Position.Len) } if pe.Position.Line > 2 { - fmt.Fprintf(b, "% 7d | %s\n", pe.Position.Line-2, lines[pe.Position.Line-3]) + fmt.Fprintf(b, "% 7d | %s\n", pe.Position.Line-2, expandTab(lines[pe.Position.Line-3])) } if pe.Position.Line > 1 { - fmt.Fprintf(b, "% 7d | %s\n", pe.Position.Line-1, lines[pe.Position.Line-2]) + fmt.Fprintf(b, "% 7d | %s\n", pe.Position.Line-1, expandTab(lines[pe.Position.Line-2])) } - fmt.Fprintf(b, "% 7d | %s\n", pe.Position.Line, lines[pe.Position.Line-1]) - fmt.Fprintf(b, "% 10s%s%s\n", "", strings.Repeat(" ", col), strings.Repeat("^", pe.Position.Len)) + + /// Expand tabs, so that the ^^^s are at the correct position, but leave + /// "column 10-13" intact. Adjusting this to the visual column would be + /// better, but we don't know the tabsize of the user in their editor, which + /// can be 8, 4, 2, or something else. We can't know. So leaving it as the + /// character index is probably the "most correct". + expanded := expandTab(lines[pe.Position.Line-1]) + diff := len(expanded) - len(lines[pe.Position.Line-1]) + + fmt.Fprintf(b, "% 7d | %s\n", pe.Position.Line, expanded) + fmt.Fprintf(b, "% 10s%s%s\n", "", strings.Repeat(" ", col+diff), strings.Repeat("^", pe.Position.Len)) return b.String() } @@ -159,17 +168,47 @@ func (pe ParseError) column(lines []string) int { return col } +func expandTab(s string) string { + var ( + b strings.Builder + l int + fill = func(n int) string { + b := make([]byte, n) + for i := range b { + b[i] = ' ' + } + return string(b) + } + ) + b.Grow(len(s)) + for _, r := range s { + switch r { + case '\t': + tw := 8 - l%8 + b.WriteString(fill(tw)) + l += tw + default: + b.WriteRune(r) + l += 1 + } + } + return b.String() +} + type ( errLexControl struct{ r rune } errLexEscape struct{ r rune } errLexUTF8 struct{ b byte } - errLexInvalidNum struct{ v string } - errLexInvalidDate struct{ v string } + errParseDate struct{ v string } errLexInlineTableNL struct{} errLexStringNL struct{} errParseRange struct { - i interface{} // int or float - size string // "int64", "uint16", etc. + i any // int or float + size string // "int64", "uint16", etc. + } + errUnsafeFloat struct { + i interface{} // float32 or float64 + size string // "float32" or "float64" } errParseDuration struct{ d string } ) @@ -183,18 +222,20 @@ func (e errLexEscape) Error() string { return fmt.Sprintf(`invalid escape func (e errLexEscape) Usage() string { return usageEscape } func (e errLexUTF8) Error() string { return fmt.Sprintf("invalid UTF-8 byte: 0x%02x", e.b) } func (e errLexUTF8) Usage() string { return "" } -func (e errLexInvalidNum) Error() string { return fmt.Sprintf("invalid number: %q", e.v) } -func (e errLexInvalidNum) Usage() string { return "" } -func (e errLexInvalidDate) Error() string { return fmt.Sprintf("invalid date: %q", e.v) } -func (e errLexInvalidDate) Usage() string { return "" } +func (e errParseDate) Error() string { return fmt.Sprintf("invalid datetime: %q", e.v) } +func (e errParseDate) Usage() string { return usageDate } func (e errLexInlineTableNL) Error() string { return "newlines not allowed within inline tables" } func (e errLexInlineTableNL) Usage() string { return usageInlineNewline } func (e errLexStringNL) Error() string { return "strings cannot contain newlines" } func (e errLexStringNL) Usage() string { return usageStringNewline } func (e errParseRange) Error() string { return fmt.Sprintf("%v is out of range for %s", e.i, e.size) } func (e errParseRange) Usage() string { return usageIntOverflow } -func (e errParseDuration) Error() string { return fmt.Sprintf("invalid duration: %q", e.d) } -func (e errParseDuration) Usage() string { return usageDuration } +func (e errUnsafeFloat) Error() string { + return fmt.Sprintf("%v is out of the safe %s range", e.i, e.size) +} +func (e errUnsafeFloat) Usage() string { return usageUnsafeFloat } +func (e errParseDuration) Error() string { return fmt.Sprintf("invalid duration: %q", e.d) } +func (e errParseDuration) Usage() string { return usageDuration } const usageEscape = ` A '\' inside a "-delimited string is interpreted as an escape character. @@ -251,19 +292,35 @@ bug in the program that uses too small of an integer. The maximum and minimum values are: size │ lowest │ highest - ───────┼────────────────┼────────── + ───────┼────────────────┼────────────── int8 │ -128 │ 127 int16 │ -32,768 │ 32,767 int32 │ -2,147,483,648 │ 2,147,483,647 int64 │ -9.2 × 10¹⁷ │ 9.2 × 10¹⁷ uint8 │ 0 │ 255 - uint16 │ 0 │ 65535 - uint32 │ 0 │ 4294967295 + uint16 │ 0 │ 65,535 + uint32 │ 0 │ 4,294,967,295 uint64 │ 0 │ 1.8 × 10¹⁸ int refers to int32 on 32-bit systems and int64 on 64-bit systems. ` +const usageUnsafeFloat = ` +This number is outside of the "safe" range for floating point numbers; whole +(non-fractional) numbers outside the below range can not always be represented +accurately in a float, leading to some loss of accuracy. + +Explicitly mark a number as a fractional unit by adding ".0", which will incur +some loss of accuracy; for example: + + f = 2_000_000_000.0 + +Accuracy ranges: + + float32 = 16,777,215 + float64 = 9,007,199,254,740,991 +` + const usageDuration = ` A duration must be as "number", without any spaces. Valid units are: @@ -277,3 +334,23 @@ A duration must be as "number", without any spaces. Valid units are: You can combine multiple units; for example "5m10s" for 5 minutes and 10 seconds. ` + +const usageDate = ` +A TOML datetime must be in one of the following formats: + + 2006-01-02T15:04:05Z07:00 Date and time, with timezone. + 2006-01-02T15:04:05 Date and time, but without timezone. + 2006-01-02 Date without a time or timezone. + 15:04:05 Just a time, without any timezone. + +Seconds may optionally have a fraction, up to nanosecond precision: + + 15:04:05.123 + 15:04:05.856018510 +` + +// TOML 1.1: +// The seconds part in times is optional, and may be omitted: +// 2006-01-02T15:04Z07:00 +// 2006-01-02T15:04 +// 15:04 diff --git a/vendor/github.com/BurntSushi/toml/lex.go b/vendor/github.com/BurntSushi/toml/lex.go index 3545a6ad66..a1016d98a8 100644 --- a/vendor/github.com/BurntSushi/toml/lex.go +++ b/vendor/github.com/BurntSushi/toml/lex.go @@ -17,6 +17,7 @@ const ( itemEOF itemText itemString + itemStringEsc itemRawString itemMultilineString itemRawMultilineString @@ -53,6 +54,7 @@ type lexer struct { state stateFn items chan item tomlNext bool + esc bool // Allow for backing up up to 4 runes. This is necessary because TOML // contains 3-rune tokens (""" and '''). @@ -164,7 +166,7 @@ func (lx *lexer) next() (r rune) { } r, w := utf8.DecodeRuneInString(lx.input[lx.pos:]) - if r == utf8.RuneError { + if r == utf8.RuneError && w == 1 { lx.error(errLexUTF8{lx.input[lx.pos]}) return utf8.RuneError } @@ -270,7 +272,7 @@ func (lx *lexer) errorPos(start, length int, err error) stateFn { } // errorf is like error, and creates a new error. -func (lx *lexer) errorf(format string, values ...interface{}) stateFn { +func (lx *lexer) errorf(format string, values ...any) stateFn { if lx.atEOF { pos := lx.getPos() pos.Line-- @@ -333,9 +335,7 @@ func lexTopEnd(lx *lexer) stateFn { lx.emit(itemEOF) return nil } - return lx.errorf( - "expected a top-level item to end with a newline, comment, or EOF, but got %q instead", - r) + return lx.errorf("expected a top-level item to end with a newline, comment, or EOF, but got %q instead", r) } // lexTable lexes the beginning of a table. Namely, it makes sure that @@ -698,7 +698,12 @@ func lexString(lx *lexer) stateFn { return lexStringEscape case r == '"': lx.backup() - lx.emit(itemString) + if lx.esc { + lx.esc = false + lx.emit(itemStringEsc) + } else { + lx.emit(itemString) + } lx.next() lx.ignore() return lx.pop() @@ -748,6 +753,7 @@ func lexMultilineString(lx *lexer) stateFn { lx.backup() /// backup: don't include the """ in the item. lx.backup() lx.backup() + lx.esc = false lx.emit(itemMultilineString) lx.next() /// Read over ''' again and discard it. lx.next() @@ -837,6 +843,7 @@ func lexMultilineStringEscape(lx *lexer) stateFn { } func lexStringEscape(lx *lexer) stateFn { + lx.esc = true r := lx.next() switch r { case 'e': @@ -879,10 +886,8 @@ func lexHexEscape(lx *lexer) stateFn { var r rune for i := 0; i < 2; i++ { r = lx.next() - if !isHexadecimal(r) { - return lx.errorf( - `expected two hexadecimal digits after '\x', but got %q instead`, - lx.current()) + if !isHex(r) { + return lx.errorf(`expected two hexadecimal digits after '\x', but got %q instead`, lx.current()) } } return lx.pop() @@ -892,10 +897,8 @@ func lexShortUnicodeEscape(lx *lexer) stateFn { var r rune for i := 0; i < 4; i++ { r = lx.next() - if !isHexadecimal(r) { - return lx.errorf( - `expected four hexadecimal digits after '\u', but got %q instead`, - lx.current()) + if !isHex(r) { + return lx.errorf(`expected four hexadecimal digits after '\u', but got %q instead`, lx.current()) } } return lx.pop() @@ -905,10 +908,8 @@ func lexLongUnicodeEscape(lx *lexer) stateFn { var r rune for i := 0; i < 8; i++ { r = lx.next() - if !isHexadecimal(r) { - return lx.errorf( - `expected eight hexadecimal digits after '\U', but got %q instead`, - lx.current()) + if !isHex(r) { + return lx.errorf(`expected eight hexadecimal digits after '\U', but got %q instead`, lx.current()) } } return lx.pop() @@ -975,7 +976,7 @@ func lexDatetime(lx *lexer) stateFn { // lexHexInteger consumes a hexadecimal integer after seeing the '0x' prefix. func lexHexInteger(lx *lexer) stateFn { r := lx.next() - if isHexadecimal(r) { + if isHex(r) { return lexHexInteger } switch r { @@ -1109,7 +1110,7 @@ func lexBaseNumberOrDate(lx *lexer) stateFn { return lexOctalInteger case 'x': r = lx.peek() - if !isHexadecimal(r) { + if !isHex(r) { lx.errorf("not a hexidecimal number: '%s%c'", lx.current(), r) } return lexHexInteger @@ -1207,7 +1208,7 @@ func (itype itemType) String() string { return "EOF" case itemText: return "Text" - case itemString, itemRawString, itemMultilineString, itemRawMultilineString: + case itemString, itemStringEsc, itemRawString, itemMultilineString, itemRawMultilineString: return "String" case itemBool: return "Bool" @@ -1240,7 +1241,7 @@ func (itype itemType) String() string { } func (item item) String() string { - return fmt.Sprintf("(%s, %s)", item.typ.String(), item.val) + return fmt.Sprintf("(%s, %s)", item.typ, item.val) } func isWhitespace(r rune) bool { return r == '\t' || r == ' ' } @@ -1256,10 +1257,7 @@ func isControl(r rune) bool { // Control characters except \t, \r, \n func isDigit(r rune) bool { return r >= '0' && r <= '9' } func isBinary(r rune) bool { return r == '0' || r == '1' } func isOctal(r rune) bool { return r >= '0' && r <= '7' } -func isHexadecimal(r rune) bool { - return (r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F') -} - +func isHex(r rune) bool { return (r >= '0' && r <= '9') || (r|0x20 >= 'a' && r|0x20 <= 'f') } func isBareKeyChar(r rune, tomlNext bool) bool { if tomlNext { return (r >= 'A' && r <= 'Z') || diff --git a/vendor/github.com/BurntSushi/toml/meta.go b/vendor/github.com/BurntSushi/toml/meta.go index 2e78b24e95..e614537300 100644 --- a/vendor/github.com/BurntSushi/toml/meta.go +++ b/vendor/github.com/BurntSushi/toml/meta.go @@ -13,7 +13,7 @@ type MetaData struct { context Key // Used only during decoding. keyInfo map[string]keyInfo - mapping map[string]interface{} + mapping map[string]any keys []Key decoded map[string]struct{} data []byte // Input file; for errors. @@ -31,12 +31,12 @@ func (md *MetaData) IsDefined(key ...string) bool { } var ( - hash map[string]interface{} + hash map[string]any ok bool - hashOrVal interface{} = md.mapping + hashOrVal any = md.mapping ) for _, k := range key { - if hash, ok = hashOrVal.(map[string]interface{}); !ok { + if hash, ok = hashOrVal.(map[string]any); !ok { return false } if hashOrVal, ok = hash[k]; !ok { @@ -94,28 +94,55 @@ func (md *MetaData) Undecoded() []Key { type Key []string func (k Key) String() string { - ss := make([]string, len(k)) - for i := range k { - ss[i] = k.maybeQuoted(i) + // This is called quite often, so it's a bit funky to make it faster. + var b strings.Builder + b.Grow(len(k) * 25) +outer: + for i, kk := range k { + if i > 0 { + b.WriteByte('.') + } + if kk == "" { + b.WriteString(`""`) + } else { + for _, r := range kk { + // "Inline" isBareKeyChar + if !((r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '_' || r == '-') { + b.WriteByte('"') + b.WriteString(dblQuotedReplacer.Replace(kk)) + b.WriteByte('"') + continue outer + } + } + b.WriteString(kk) + } } - return strings.Join(ss, ".") + return b.String() } func (k Key) maybeQuoted(i int) string { if k[i] == "" { return `""` } - for _, c := range k[i] { - if !isBareKeyChar(c, false) { - return `"` + dblQuotedReplacer.Replace(k[i]) + `"` + for _, r := range k[i] { + if (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '_' || r == '-' { + continue } + return `"` + dblQuotedReplacer.Replace(k[i]) + `"` } return k[i] } +// Like append(), but only increase the cap by 1. func (k Key) add(piece string) Key { + if cap(k) > len(k) { + return append(k, piece) + } newKey := make(Key, len(k)+1) copy(newKey, k) newKey[len(k)] = piece return newKey } + +func (k Key) parent() Key { return k[:len(k)-1] } // all except the last piece. +func (k Key) last() string { return k[len(k)-1] } // last piece of this key. diff --git a/vendor/github.com/BurntSushi/toml/parse.go b/vendor/github.com/BurntSushi/toml/parse.go index 9c19153698..11ac3108be 100644 --- a/vendor/github.com/BurntSushi/toml/parse.go +++ b/vendor/github.com/BurntSushi/toml/parse.go @@ -2,6 +2,7 @@ package toml import ( "fmt" + "math" "os" "strconv" "strings" @@ -20,9 +21,9 @@ type parser struct { ordered []Key // List of keys in the order that they appear in the TOML data. - keyInfo map[string]keyInfo // Map keyname → info about the TOML key. - mapping map[string]interface{} // Map keyname → key value. - implicits map[string]struct{} // Record implicit keys (e.g. "key.group.names"). + keyInfo map[string]keyInfo // Map keyname → info about the TOML key. + mapping map[string]any // Map keyname → key value. + implicits map[string]struct{} // Record implicit keys (e.g. "key.group.names"). } type keyInfo struct { @@ -49,6 +50,7 @@ func parse(data string) (p *parser, err error) { // it anyway. if strings.HasPrefix(data, "\xff\xfe") || strings.HasPrefix(data, "\xfe\xff") { // UTF-16 data = data[2:] + //lint:ignore S1017 https://github.com/dominikh/go-tools/issues/1447 } else if strings.HasPrefix(data, "\xef\xbb\xbf") { // UTF-8 data = data[3:] } @@ -71,7 +73,7 @@ func parse(data string) (p *parser, err error) { p = &parser{ keyInfo: make(map[string]keyInfo), - mapping: make(map[string]interface{}), + mapping: make(map[string]any), lx: lex(data, tomlNext), ordered: make([]Key, 0), implicits: make(map[string]struct{}), @@ -97,7 +99,7 @@ func (p *parser) panicErr(it item, err error) { }) } -func (p *parser) panicItemf(it item, format string, v ...interface{}) { +func (p *parser) panicItemf(it item, format string, v ...any) { panic(ParseError{ Message: fmt.Sprintf(format, v...), Position: it.pos, @@ -106,7 +108,7 @@ func (p *parser) panicItemf(it item, format string, v ...interface{}) { }) } -func (p *parser) panicf(format string, v ...interface{}) { +func (p *parser) panicf(format string, v ...any) { panic(ParseError{ Message: fmt.Sprintf(format, v...), Position: p.pos, @@ -139,7 +141,7 @@ func (p *parser) nextPos() item { return it } -func (p *parser) bug(format string, v ...interface{}) { +func (p *parser) bug(format string, v ...any) { panic(fmt.Sprintf("BUG: "+format+"\n\n", v...)) } @@ -194,11 +196,11 @@ func (p *parser) topLevel(item item) { p.assertEqual(itemKeyEnd, k.typ) /// The current key is the last part. - p.currentKey = key[len(key)-1] + p.currentKey = key.last() /// All the other parts (if any) are the context; need to set each part /// as implicit. - context := key[:len(key)-1] + context := key.parent() for i := range context { p.addImplicitContext(append(p.context, context[i:i+1]...)) } @@ -207,7 +209,8 @@ func (p *parser) topLevel(item item) { /// Set value. vItem := p.next() val, typ := p.value(vItem, false) - p.set(p.currentKey, val, typ, vItem.pos) + p.setValue(p.currentKey, val) + p.setType(p.currentKey, typ, vItem.pos) /// Remove the context we added (preserving any context from [tbl] lines). p.context = outerContext @@ -222,7 +225,7 @@ func (p *parser) keyString(it item) string { switch it.typ { case itemText: return it.val - case itemString, itemMultilineString, + case itemString, itemStringEsc, itemMultilineString, itemRawString, itemRawMultilineString: s, _ := p.value(it, false) return s.(string) @@ -239,9 +242,11 @@ var datetimeRepl = strings.NewReplacer( // value translates an expected value from the lexer into a Go value wrapped // as an empty interface. -func (p *parser) value(it item, parentIsArray bool) (interface{}, tomlType) { +func (p *parser) value(it item, parentIsArray bool) (any, tomlType) { switch it.typ { case itemString: + return it.val, p.typeOfPrimitive(it) + case itemStringEsc: return p.replaceEscapes(it, it.val), p.typeOfPrimitive(it) case itemMultilineString: return p.replaceEscapes(it, p.stripEscapedNewlines(stripFirstNewline(it.val))), p.typeOfPrimitive(it) @@ -274,7 +279,7 @@ func (p *parser) value(it item, parentIsArray bool) (interface{}, tomlType) { panic("unreachable") } -func (p *parser) valueInteger(it item) (interface{}, tomlType) { +func (p *parser) valueInteger(it item) (any, tomlType) { if !numUnderscoresOK(it.val) { p.panicItemf(it, "Invalid integer %q: underscores must be surrounded by digits", it.val) } @@ -298,7 +303,7 @@ func (p *parser) valueInteger(it item) (interface{}, tomlType) { return num, p.typeOfPrimitive(it) } -func (p *parser) valueFloat(it item) (interface{}, tomlType) { +func (p *parser) valueFloat(it item) (any, tomlType) { parts := strings.FieldsFunc(it.val, func(r rune) bool { switch r { case '.', 'e', 'E': @@ -322,7 +327,9 @@ func (p *parser) valueFloat(it item) (interface{}, tomlType) { p.panicItemf(it, "Invalid float %q: '.' must be followed by one or more digits", it.val) } val := strings.Replace(it.val, "_", "", -1) - if val == "+nan" || val == "-nan" { // Go doesn't support this, but TOML spec does. + signbit := false + if val == "+nan" || val == "-nan" { + signbit = val == "-nan" val = "nan" } num, err := strconv.ParseFloat(val, 64) @@ -333,6 +340,9 @@ func (p *parser) valueFloat(it item) (interface{}, tomlType) { p.panicItemf(it, "Invalid float value: %q", it.val) } } + if signbit { + num = math.Copysign(num, -1) + } return num, p.typeOfPrimitive(it) } @@ -352,7 +362,7 @@ var dtTypes = []struct { {"15:04", internal.LocalTime, true}, } -func (p *parser) valueDatetime(it item) (interface{}, tomlType) { +func (p *parser) valueDatetime(it item) (any, tomlType) { it.val = datetimeRepl.Replace(it.val) var ( t time.Time @@ -365,26 +375,44 @@ func (p *parser) valueDatetime(it item) (interface{}, tomlType) { } t, err = time.ParseInLocation(dt.fmt, it.val, dt.zone) if err == nil { + if missingLeadingZero(it.val, dt.fmt) { + p.panicErr(it, errParseDate{it.val}) + } ok = true break } } if !ok { - p.panicItemf(it, "Invalid TOML Datetime: %q.", it.val) + p.panicErr(it, errParseDate{it.val}) } return t, p.typeOfPrimitive(it) } -func (p *parser) valueArray(it item) (interface{}, tomlType) { +// Go's time.Parse() will accept numbers without a leading zero; there isn't any +// way to require it. https://github.com/golang/go/issues/29911 +// +// Depend on the fact that the separators (- and :) should always be at the same +// location. +func missingLeadingZero(d, l string) bool { + for i, c := range []byte(l) { + if c == '.' || c == 'Z' { + return false + } + if (c < '0' || c > '9') && d[i] != c { + return true + } + } + return false +} + +func (p *parser) valueArray(it item) (any, tomlType) { p.setType(p.currentKey, tomlArray, it.pos) var ( - types []tomlType - - // Initialize to a non-nil empty slice. This makes it consistent with - // how S = [] decodes into a non-nil slice inside something like struct - // { S []string }. See #338 - array = []interface{}{} + // Initialize to a non-nil slice to make it consistent with how S = [] + // decodes into a non-nil slice inside something like struct { S + // []string }. See #338 + array = make([]any, 0, 2) ) for it = p.next(); it.typ != itemArrayEnd; it = p.next() { if it.typ == itemCommentStart { @@ -394,21 +422,20 @@ func (p *parser) valueArray(it item) (interface{}, tomlType) { val, typ := p.value(it, true) array = append(array, val) - types = append(types, typ) - // XXX: types isn't used here, we need it to record the accurate type + // XXX: type isn't used here, we need it to record the accurate type // information. // // Not entirely sure how to best store this; could use "key[0]", // "key[1]" notation, or maybe store it on the Array type? - _ = types + _ = typ } return array, tomlArray } -func (p *parser) valueInlineTable(it item, parentIsArray bool) (interface{}, tomlType) { +func (p *parser) valueInlineTable(it item, parentIsArray bool) (any, tomlType) { var ( - hash = make(map[string]interface{}) + topHash = make(map[string]any) outerContext = p.context outerKey = p.currentKey ) @@ -436,11 +463,11 @@ func (p *parser) valueInlineTable(it item, parentIsArray bool) (interface{}, tom p.assertEqual(itemKeyEnd, k.typ) /// The current key is the last part. - p.currentKey = key[len(key)-1] + p.currentKey = key.last() /// All the other parts (if any) are the context; need to set each part /// as implicit. - context := key[:len(key)-1] + context := key.parent() for i := range context { p.addImplicitContext(append(p.context, context[i:i+1]...)) } @@ -448,7 +475,21 @@ func (p *parser) valueInlineTable(it item, parentIsArray bool) (interface{}, tom /// Set the value. val, typ := p.value(p.next(), false) - p.set(p.currentKey, val, typ, it.pos) + p.setValue(p.currentKey, val) + p.setType(p.currentKey, typ, it.pos) + + hash := topHash + for _, c := range context { + h, ok := hash[c] + if !ok { + h = make(map[string]any) + hash[c] = h + } + hash, ok = h.(map[string]any) + if !ok { + p.panicf("%q is not a table", p.context) + } + } hash[p.currentKey] = val /// Restore context. @@ -456,7 +497,7 @@ func (p *parser) valueInlineTable(it item, parentIsArray bool) (interface{}, tom } p.context = outerContext p.currentKey = outerKey - return hash, tomlHash + return topHash, tomlHash } // numHasLeadingZero checks if this number has leading zeroes, allowing for '0', @@ -486,9 +527,9 @@ func numUnderscoresOK(s string) bool { } } - // isHexadecimal is a superset of all the permissable characters - // surrounding an underscore. - accept = isHexadecimal(r) + // isHexis a superset of all the permissable characters surrounding an + // underscore. + accept = isHex(r) } return accept } @@ -511,21 +552,19 @@ func numPeriodsOK(s string) bool { // Establishing the context also makes sure that the key isn't a duplicate, and // will create implicit hashes automatically. func (p *parser) addContext(key Key, array bool) { - var ok bool - - // Always start at the top level and drill down for our context. + /// Always start at the top level and drill down for our context. hashContext := p.mapping - keyContext := make(Key, 0) + keyContext := make(Key, 0, len(key)-1) - // We only need implicit hashes for key[0:-1] - for _, k := range key[0 : len(key)-1] { - _, ok = hashContext[k] + /// We only need implicit hashes for the parents. + for _, k := range key.parent() { + _, ok := hashContext[k] keyContext = append(keyContext, k) // No key? Make an implicit hash and move on. if !ok { p.addImplicit(keyContext) - hashContext[k] = make(map[string]interface{}) + hashContext[k] = make(map[string]any) } // If the hash context is actually an array of tables, then set @@ -534,9 +573,9 @@ func (p *parser) addContext(key Key, array bool) { // Otherwise, it better be a table, since this MUST be a key group (by // virtue of it not being the last element in a key). switch t := hashContext[k].(type) { - case []map[string]interface{}: + case []map[string]any: hashContext = t[len(t)-1] - case map[string]interface{}: + case map[string]any: hashContext = t default: p.panicf("Key '%s' was already created as a hash.", keyContext) @@ -547,39 +586,33 @@ func (p *parser) addContext(key Key, array bool) { if array { // If this is the first element for this array, then allocate a new // list of tables for it. - k := key[len(key)-1] + k := key.last() if _, ok := hashContext[k]; !ok { - hashContext[k] = make([]map[string]interface{}, 0, 4) + hashContext[k] = make([]map[string]any, 0, 4) } // Add a new table. But make sure the key hasn't already been used // for something else. - if hash, ok := hashContext[k].([]map[string]interface{}); ok { - hashContext[k] = append(hash, make(map[string]interface{})) + if hash, ok := hashContext[k].([]map[string]any); ok { + hashContext[k] = append(hash, make(map[string]any)) } else { p.panicf("Key '%s' was already created and cannot be used as an array.", key) } } else { - p.setValue(key[len(key)-1], make(map[string]interface{})) + p.setValue(key.last(), make(map[string]any)) } - p.context = append(p.context, key[len(key)-1]) -} - -// set calls setValue and setType. -func (p *parser) set(key string, val interface{}, typ tomlType, pos Position) { - p.setValue(key, val) - p.setType(key, typ, pos) + p.context = append(p.context, key.last()) } // setValue sets the given key to the given value in the current context. // It will make sure that the key hasn't already been defined, account for // implicit key groups. -func (p *parser) setValue(key string, value interface{}) { +func (p *parser) setValue(key string, value any) { var ( - tmpHash interface{} + tmpHash any ok bool hash = p.mapping - keyContext Key + keyContext = make(Key, 0, len(p.context)+1) ) for _, k := range p.context { keyContext = append(keyContext, k) @@ -587,11 +620,11 @@ func (p *parser) setValue(key string, value interface{}) { p.bug("Context for key '%s' has not been established.", keyContext) } switch t := tmpHash.(type) { - case []map[string]interface{}: + case []map[string]any: // The context is a table of hashes. Pick the most recent table // defined as the current hash. hash = t[len(t)-1] - case map[string]interface{}: + case map[string]any: hash = t default: p.panicf("Key '%s' has already been defined.", keyContext) @@ -618,9 +651,8 @@ func (p *parser) setValue(key string, value interface{}) { p.removeImplicit(keyContext) return } - - // Otherwise, we have a concrete key trying to override a previous - // key, which is *always* wrong. + // Otherwise, we have a concrete key trying to override a previous key, + // which is *always* wrong. p.panicf("Key '%s' has already been defined.", keyContext) } @@ -683,8 +715,11 @@ func stripFirstNewline(s string) string { // the next newline. After a line-ending backslash, all whitespace is removed // until the next non-whitespace character. func (p *parser) stripEscapedNewlines(s string) string { - var b strings.Builder - var i int + var ( + b strings.Builder + i int + ) + b.Grow(len(s)) for { ix := strings.Index(s[i:], `\`) if ix < 0 { @@ -714,9 +749,8 @@ func (p *parser) stripEscapedNewlines(s string) string { continue } if !strings.Contains(s[i:j], "\n") { - // This is not a line-ending backslash. - // (It's a bad escape sequence, but we can let - // replaceEscapes catch it.) + // This is not a line-ending backslash. (It's a bad escape sequence, + // but we can let replaceEscapes catch it.) i++ continue } @@ -727,79 +761,78 @@ func (p *parser) stripEscapedNewlines(s string) string { } func (p *parser) replaceEscapes(it item, str string) string { - replaced := make([]rune, 0, len(str)) - s := []byte(str) - r := 0 - for r < len(s) { - if s[r] != '\\' { - c, size := utf8.DecodeRune(s[r:]) - r += size - replaced = append(replaced, c) + var ( + b strings.Builder + skip = 0 + ) + b.Grow(len(str)) + for i, c := range str { + if skip > 0 { + skip-- continue } - r += 1 - if r >= len(s) { + if c != '\\' { + b.WriteRune(c) + continue + } + + if i >= len(str) { p.bug("Escape sequence at end of string.") return "" } - switch s[r] { + switch str[i+1] { default: - p.bug("Expected valid escape code after \\, but got %q.", s[r]) + p.bug("Expected valid escape code after \\, but got %q.", str[i+1]) case ' ', '\t': - p.panicItemf(it, "invalid escape: '\\%c'", s[r]) + p.panicItemf(it, "invalid escape: '\\%c'", str[i+1]) case 'b': - replaced = append(replaced, rune(0x0008)) - r += 1 + b.WriteByte(0x08) + skip = 1 case 't': - replaced = append(replaced, rune(0x0009)) - r += 1 + b.WriteByte(0x09) + skip = 1 case 'n': - replaced = append(replaced, rune(0x000A)) - r += 1 + b.WriteByte(0x0a) + skip = 1 case 'f': - replaced = append(replaced, rune(0x000C)) - r += 1 + b.WriteByte(0x0c) + skip = 1 case 'r': - replaced = append(replaced, rune(0x000D)) - r += 1 + b.WriteByte(0x0d) + skip = 1 case 'e': if p.tomlNext { - replaced = append(replaced, rune(0x001B)) - r += 1 + b.WriteByte(0x1b) + skip = 1 } case '"': - replaced = append(replaced, rune(0x0022)) - r += 1 + b.WriteByte(0x22) + skip = 1 case '\\': - replaced = append(replaced, rune(0x005C)) - r += 1 + b.WriteByte(0x5c) + skip = 1 + // The lexer guarantees the correct number of characters are present; + // don't need to check here. case 'x': if p.tomlNext { - escaped := p.asciiEscapeToUnicode(it, s[r+1:r+3]) - replaced = append(replaced, escaped) - r += 3 + escaped := p.asciiEscapeToUnicode(it, str[i+2:i+4]) + b.WriteRune(escaped) + skip = 3 } case 'u': - // At this point, we know we have a Unicode escape of the form - // `uXXXX` at [r, r+5). (Because the lexer guarantees this - // for us.) - escaped := p.asciiEscapeToUnicode(it, s[r+1:r+5]) - replaced = append(replaced, escaped) - r += 5 + escaped := p.asciiEscapeToUnicode(it, str[i+2:i+6]) + b.WriteRune(escaped) + skip = 5 case 'U': - // At this point, we know we have a Unicode escape of the form - // `uXXXX` at [r, r+9). (Because the lexer guarantees this - // for us.) - escaped := p.asciiEscapeToUnicode(it, s[r+1:r+9]) - replaced = append(replaced, escaped) - r += 9 + escaped := p.asciiEscapeToUnicode(it, str[i+2:i+10]) + b.WriteRune(escaped) + skip = 9 } } - return string(replaced) + return b.String() } -func (p *parser) asciiEscapeToUnicode(it item, bs []byte) rune { - s := string(bs) +func (p *parser) asciiEscapeToUnicode(it item, s string) rune { hex, err := strconv.ParseUint(strings.ToLower(s), 16, 32) if err != nil { p.bug("Could not parse '%s' as a hexadecimal number, but the lexer claims it's OK: %s", s, err) diff --git a/vendor/github.com/BurntSushi/toml/type_fields.go b/vendor/github.com/BurntSushi/toml/type_fields.go index 254ca82e54..10c51f7eeb 100644 --- a/vendor/github.com/BurntSushi/toml/type_fields.go +++ b/vendor/github.com/BurntSushi/toml/type_fields.go @@ -25,10 +25,8 @@ type field struct { // breaking ties with index sequence. type byName []field -func (x byName) Len() int { return len(x) } - +func (x byName) Len() int { return len(x) } func (x byName) Swap(i, j int) { x[i], x[j] = x[j], x[i] } - func (x byName) Less(i, j int) bool { if x[i].name != x[j].name { return x[i].name < x[j].name @@ -45,10 +43,8 @@ func (x byName) Less(i, j int) bool { // byIndex sorts field by index sequence. type byIndex []field -func (x byIndex) Len() int { return len(x) } - +func (x byIndex) Len() int { return len(x) } func (x byIndex) Swap(i, j int) { x[i], x[j] = x[j], x[i] } - func (x byIndex) Less(i, j int) bool { for k, xik := range x[i].index { if k >= len(x[j].index) { diff --git a/vendor/github.com/BurntSushi/toml/type_toml.go b/vendor/github.com/BurntSushi/toml/type_toml.go index 4e90d77373..1c090d331e 100644 --- a/vendor/github.com/BurntSushi/toml/type_toml.go +++ b/vendor/github.com/BurntSushi/toml/type_toml.go @@ -22,13 +22,8 @@ func typeIsTable(t tomlType) bool { type tomlBaseType string -func (btype tomlBaseType) typeString() string { - return string(btype) -} - -func (btype tomlBaseType) String() string { - return btype.typeString() -} +func (btype tomlBaseType) typeString() string { return string(btype) } +func (btype tomlBaseType) String() string { return btype.typeString() } var ( tomlInteger tomlBaseType = "Integer" @@ -54,7 +49,7 @@ func (p *parser) typeOfPrimitive(lexItem item) tomlType { return tomlFloat case itemDatetime: return tomlDatetime - case itemString: + case itemString, itemStringEsc: return tomlString case itemMultilineString: return tomlString diff --git a/vendor/github.com/Microsoft/hcsshim/hnsaccelnet.go b/vendor/github.com/Microsoft/hcsshim/hnsaccelnet.go new file mode 100644 index 0000000000..86c7c22aa3 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/hnsaccelnet.go @@ -0,0 +1,46 @@ +//go:build windows + +package hcsshim + +import ( + "errors" + + "github.com/Microsoft/hcsshim/internal/hns" +) + +// HNSNnvManagementMacAddress represents management mac address +// which needs to be excluded from VF reassignment +type HNSNnvManagementMacAddress = hns.HNSNnvManagementMacAddress + +// HNSNnvManagementMacList represents a list of management +// mac addresses for exclusion from VF reassignment +type HNSNnvManagementMacList = hns.HNSNnvManagementMacList + +var ( + ErrorEmptyMacAddressList = errors.New("management mac_address list is empty") +) + +// SetNnvManagementMacAddresses sets a list of +// management mac addresses in hns for exclusion from VF reassignment. +func SetNnvManagementMacAddresses(managementMacAddresses []string) (*HNSNnvManagementMacList, error) { + if len(managementMacAddresses) == 0 { + return nil, ErrorEmptyMacAddressList + } + nnvManagementMacList := &HNSNnvManagementMacList{} + for _, mac := range managementMacAddresses { + nnvManagementMacList.MacAddressList = append(nnvManagementMacList.MacAddressList, HNSNnvManagementMacAddress{MacAddress: mac}) + } + return nnvManagementMacList.Set() +} + +// GetNnvManagementMacAddresses retrieves a list of +// management mac addresses in hns for exclusion from VF reassignment. +func GetNnvManagementMacAddresses() (*HNSNnvManagementMacList, error) { + return hns.GetNnvManagementMacAddressList() +} + +// DeleteNnvManagementMacAddresses delete list of +// management mac addresses in hns which are excluded from VF reassignment. +func DeleteNnvManagementMacAddresses() (*HNSNnvManagementMacList, error) { + return hns.DeleteNnvManagementMacAddressList() +} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hns/hnsaccelnet.go b/vendor/github.com/Microsoft/hcsshim/internal/hns/hnsaccelnet.go new file mode 100644 index 0000000000..82ca5baefd --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/internal/hns/hnsaccelnet.go @@ -0,0 +1,60 @@ +//go:build windows + +package hns + +import ( + "encoding/json" + + "github.com/sirupsen/logrus" +) + +// HNSNnvManagementMacAddress represents management mac address +// which needs to be excluded from VF reassignment +type HNSNnvManagementMacAddress struct { + MacAddress string `json:",omitempty"` +} + +// HNSNnvManagementMacList represents a list of management +// mac addresses for exclusion from VF reassignment +type HNSNnvManagementMacList struct { + MacAddressList []HNSNnvManagementMacAddress `json:",omitempty"` +} + +// HNSNnvManagementMacRequest makes a HNS call to modify/query NnvManagementMacList +func HNSNnvManagementMacRequest(method, path, request string) (*HNSNnvManagementMacList, error) { + nnvManagementMacList := &HNSNnvManagementMacList{} + err := hnsCall(method, "/accelnet/"+path, request, &nnvManagementMacList) + if err != nil { + return nil, err + } + return nnvManagementMacList, nil +} + +// Set ManagementMacAddressList by sending "POST" NnvManagementMacRequest to HNS. +func (nnvManagementMacList *HNSNnvManagementMacList) Set() (*HNSNnvManagementMacList, error) { + operation := "Set" + title := "hcsshim::nnvManagementMacList::" + operation + logrus.Debugf(title+" id=%s", nnvManagementMacList.MacAddressList) + + jsonString, err := json.Marshal(nnvManagementMacList) + if err != nil { + return nil, err + } + return HNSNnvManagementMacRequest("POST", "", string(jsonString)) +} + +// Get ManagementMacAddressList by sending "GET" NnvManagementMacRequest to HNS. +func GetNnvManagementMacAddressList() (*HNSNnvManagementMacList, error) { + operation := "Get" + title := "hcsshim::nnvManagementMacList::" + operation + logrus.Debugf(title) + return HNSNnvManagementMacRequest("GET", "", "") +} + +// Delete ManagementMacAddressList by sending "DELETE" NnvManagementMacRequest to HNS. +func DeleteNnvManagementMacAddressList() (*HNSNnvManagementMacList, error) { + operation := "Delete" + title := "hcsshim::nnvManagementMacList::" + operation + logrus.Debugf(title) + return HNSNnvManagementMacRequest("DELETE", "", "") +} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hns/hnsendpoint.go b/vendor/github.com/Microsoft/hcsshim/internal/hns/hnsendpoint.go index 593664419d..6238e103be 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/hns/hnsendpoint.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/hns/hnsendpoint.go @@ -10,6 +10,28 @@ import ( "github.com/sirupsen/logrus" ) +// EndpointState represents the states of an HNS Endpoint lifecycle. +type EndpointState uint16 + +// EndpointState const +// The lifecycle of an Endpoint goes through created, attached, AttachedSharing - endpoint is being shared with other containers, +// detached, after being attached, degraded and finally destroyed. +// Note: This attribute is used by calico to define stale containers and is dependent on HNS v1 api, if we move to HNS v2 api we will need +// to update the current calico code and cordinate the change with calico. Reach out to Microsoft to facilate the change via HNS. +const ( + Uninitialized EndpointState = iota + Created EndpointState = 1 + Attached EndpointState = 2 + AttachedSharing EndpointState = 3 + Detached EndpointState = 4 + Degraded EndpointState = 5 + Destroyed EndpointState = 6 +) + +func (es EndpointState) String() string { + return [...]string{"Uninitialized", "Attached", "AttachedSharing", "Detached", "Degraded", "Destroyed"}[es] +} + // HNSEndpoint represents a network endpoint in HNS type HNSEndpoint struct { Id string `json:"ID,omitempty"` @@ -34,6 +56,7 @@ type HNSEndpoint struct { Namespace *Namespace `json:",omitempty"` EncapOverhead uint16 `json:",omitempty"` SharedContainers []string `json:",omitempty"` + State EndpointState `json:",omitempty"` } // SystemType represents the type of the system on which actions are done diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hns/hnspolicy.go b/vendor/github.com/Microsoft/hcsshim/internal/hns/hnspolicy.go index 082c018a4e..e97e4f6319 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/hns/hnspolicy.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/hns/hnspolicy.go @@ -57,9 +57,10 @@ type PaPolicy struct { type OutboundNatPolicy struct { Policy - VIP string `json:"VIP,omitempty"` - Exceptions []string `json:"ExceptionList,omitempty"` - Destinations []string `json:",omitempty"` + VIP string `json:"VIP,omitempty"` + Exceptions []string `json:"ExceptionList,omitempty"` + Destinations []string `json:",omitempty"` + MaxPortPoolUsage uint16 `json:",omitempty"` } type ProxyPolicy struct { diff --git a/vendor/github.com/containers/storage/.cirrus.yml b/vendor/github.com/containers/storage/.cirrus.yml index c2474c7f29..50b9876169 100644 --- a/vendor/github.com/containers/storage/.cirrus.yml +++ b/vendor/github.com/containers/storage/.cirrus.yml @@ -23,7 +23,7 @@ env: # GCE project where images live IMAGE_PROJECT: "libpod-218412" # VM Image built in containers/automation_images - IMAGE_SUFFIX: "c20240513t140131z-f40f39d13" + IMAGE_SUFFIX: "c20240529t141726z-f40f39d13" FEDORA_CACHE_IMAGE_NAME: "fedora-${IMAGE_SUFFIX}" DEBIAN_CACHE_IMAGE_NAME: "debian-${IMAGE_SUFFIX}" diff --git a/vendor/github.com/containers/storage/.golangci.yml b/vendor/github.com/containers/storage/.golangci.yml index 20968466c3..ec11f5dae3 100644 --- a/vendor/github.com/containers/storage/.golangci.yml +++ b/vendor/github.com/containers/storage/.golangci.yml @@ -1,11 +1,7 @@ --- run: concurrency: 6 - deadline: 5m - skip-dirs-use-default: true + timeout: 5m linters: enable: - gofumpt - disable: - - errcheck - - staticcheck diff --git a/vendor/github.com/containers/storage/Makefile b/vendor/github.com/containers/storage/Makefile index 7ee2642fc3..96ea9f9a57 100644 --- a/vendor/github.com/containers/storage/Makefile +++ b/vendor/github.com/containers/storage/Makefile @@ -53,6 +53,8 @@ local-cross cross: ## cross build the binaries for arm, darwin, and freebsd os=`echo $${target} | cut -f1 -d/` ; \ arch=`echo $${target} | cut -f2 -d/` ; \ suffix=$${os}.$${arch} ; \ + echo env CGO_ENABLED=0 GOOS=$${os} GOARCH=$${arch} $(GO) build -compiler gc -tags \"$(NATIVETAGS) $(TAGS)\" $(FLAGS) ./... ; \ + env CGO_ENABLED=0 GOOS=$${os} GOARCH=$${arch} $(GO) build -compiler gc -tags "$(NATIVETAGS) $(TAGS)" $(FLAGS) ./... || exit 1 ; \ echo env CGO_ENABLED=0 GOOS=$${os} GOARCH=$${arch} $(GO) build -compiler gc -tags \"$(NATIVETAGS) $(TAGS)\" $(FLAGS) -o containers-storage.$${suffix} ./cmd/containers-storage ; \ env CGO_ENABLED=0 GOOS=$${os} GOARCH=$${arch} $(GO) build -compiler gc -tags "$(NATIVETAGS) $(TAGS)" $(FLAGS) -o containers-storage.$${suffix} ./cmd/containers-storage || exit 1 ; \ done diff --git a/vendor/github.com/containers/storage/VERSION b/vendor/github.com/containers/storage/VERSION index b7921ae87b..094d6ad00c 100644 --- a/vendor/github.com/containers/storage/VERSION +++ b/vendor/github.com/containers/storage/VERSION @@ -1 +1 @@ -1.54.0 +1.55.0 diff --git a/vendor/github.com/containers/storage/check.go b/vendor/github.com/containers/storage/check.go index e58084fc7c..7176ba361f 100644 --- a/vendor/github.com/containers/storage/check.go +++ b/vendor/github.com/containers/storage/check.go @@ -304,7 +304,14 @@ func (s *store) Check(options *CheckOptions) (CheckReport, error) { archiveErr = err } // consume any trailer after the EOF marker - io.Copy(io.Discard, diffReader) + if _, err := io.Copy(io.Discard, diffReader); err != nil { + err = fmt.Errorf("layer %s: consume any trailer after the EOF marker: %w", layerID, err) + if isReadWrite { + report.Layers[layerID] = append(report.Layers[layerID], err) + } else { + report.ROLayers[layerID] = append(report.ROLayers[layerID], err) + } + } wg.Done() }(id, reader) wg.Wait() @@ -366,7 +373,7 @@ func (s *store) Check(options *CheckOptions) (CheckReport, error) { if options.LayerMountable { func() { // Mount the layer. - mountPoint, err := s.graphDriver.Get(id, drivers.MountOpts{MountLabel: layer.MountLabel}) + mountPoint, err := s.graphDriver.Get(id, drivers.MountOpts{MountLabel: layer.MountLabel, Options: []string{"ro"}}) if err != nil { err := fmt.Errorf("%slayer %s: %w", readWriteDesc, id, err) if isReadWrite { @@ -955,6 +962,9 @@ func (c *checkDirectory) add(path string, typeflag byte, uid, gid int, size int6 mtime: mtime, } } + case tar.TypeXGlobalHeader: + // ignore, since even though it looks like a valid pathname, it doesn't end + // up on the filesystem default: // treat these as TypeReg items delete(c.directory, components[0]) @@ -966,9 +976,6 @@ func (c *checkDirectory) add(path string, typeflag byte, uid, gid int, size int6 mode: mode, mtime: mtime, } - case tar.TypeXGlobalHeader: - // ignore, since even though it looks like a valid pathname, it doesn't end - // up on the filesystem } return } diff --git a/vendor/github.com/containers/storage/drivers/aufs/aufs.go b/vendor/github.com/containers/storage/drivers/aufs/aufs.go index e00314d3fd..c12d4ca54c 100644 --- a/vendor/github.com/containers/storage/drivers/aufs/aufs.go +++ b/vendor/github.com/containers/storage/drivers/aufs/aufs.go @@ -30,7 +30,6 @@ import ( "io" "io/fs" "os" - "os/exec" "path" "path/filepath" "strings" @@ -75,8 +74,6 @@ func init() { type Driver struct { sync.Mutex root string - uidMaps []idtools.IDMap - gidMaps []idtools.IDMap ctr *graphdriver.RefCounter pathCacheLock sync.Mutex pathCache map[string]string @@ -129,22 +126,16 @@ func Init(home string, options graphdriver.Options) (graphdriver.Driver, error) a := &Driver{ root: home, - uidMaps: options.UIDMaps, - gidMaps: options.GIDMaps, pathCache: make(map[string]string), ctr: graphdriver.NewRefCounter(graphdriver.NewFsChecker(graphdriver.FsMagicAufs)), locker: locker.New(), mountOptions: mountOptions, } - rootUID, rootGID, err := idtools.GetRootUIDGID(options.UIDMaps, options.GIDMaps) - if err != nil { - return nil, err - } // Create the root aufs driver dir and return // if it already exists // If not populate the dir structure - if err := idtools.MkdirAllAs(home, 0o700, rootUID, rootGID); err != nil { + if err := os.MkdirAll(home, 0o700); err != nil { if os.IsExist(err) { return a, nil } @@ -157,7 +148,7 @@ func Init(home string, options graphdriver.Options) (graphdriver.Driver, error) // Populate the dir structure for _, p := range paths { - if err := idtools.MkdirAllAs(path.Join(home, p), 0o700, rootUID, rootGID); err != nil { + if err := os.MkdirAll(path.Join(home, p), 0o700); err != nil { return nil, err } } @@ -191,13 +182,7 @@ func Init(home string, options graphdriver.Options) (graphdriver.Driver, error) } // Return a nil error if the kernel supports aufs -// We cannot modprobe because inside dind modprobe fails -// to run func supportsAufs() error { - // We can try to modprobe aufs first before looking at - // proc/filesystems for when aufs is supported - exec.Command("modprobe", "aufs").Run() - if unshare.IsRootless() { return ErrAufsNested } @@ -334,7 +319,7 @@ func (a *Driver) createDirsFor(id, parent string) error { // The path of directories are /mnt/ // and /diff/ for _, p := range paths { - rootPair := idtools.NewIDMappingsFromMaps(a.uidMaps, a.gidMaps).RootPair() + rootPair := idtools.IDPair{UID: 0, GID: 0} rootPerms := defaultPerms if parent != "" { st, err := system.Stat(path.Join(a.rootPath(), p, parent)) @@ -355,7 +340,9 @@ func (a *Driver) createDirsFor(id, parent string) error { // Remove will unmount and remove the given id. func (a *Driver) Remove(id string) error { a.locker.Lock(id) - defer a.locker.Unlock(id) + defer func() { + _ = a.locker.Unlock(id) + }() a.pathCacheLock.Lock() mountpoint, exists := a.pathCache[id] a.pathCacheLock.Unlock() @@ -446,7 +433,10 @@ func atomicRemove(source string) error { // This will mount the dir at its given path func (a *Driver) Get(id string, options graphdriver.MountOpts) (string, error) { a.locker.Lock(id) - defer a.locker.Unlock(id) + defer func() { + _ = a.locker.Unlock(id) + }() + parents, err := a.getParentLayerPaths(id) if err != nil && !os.IsNotExist(err) { return "", err @@ -483,7 +473,10 @@ func (a *Driver) Get(id string, options graphdriver.MountOpts) (string, error) { // Put unmounts and updates list of active mounts. func (a *Driver) Put(id string) error { a.locker.Lock(id) - defer a.locker.Unlock(id) + defer func() { + _ = a.locker.Unlock(id) + }() + a.pathCacheLock.Lock() m, exists := a.pathCache[id] if !exists { @@ -506,7 +499,9 @@ func (a *Driver) Put(id string) error { // For AUFS, it queries the mountpoint for this ID. func (a *Driver) ReadWriteDiskUsage(id string) (*directory.DiskUsage, error) { a.locker.Lock(id) - defer a.locker.Unlock(id) + defer func() { + _ = a.locker.Unlock(id) + }() a.pathCacheLock.Lock() m, exists := a.pathCache[id] if !exists { @@ -689,7 +684,9 @@ func (a *Driver) Cleanup() error { func (a *Driver) aufsMount(ro []string, rw, target string, options graphdriver.MountOpts) (err error) { defer func() { if err != nil { - Unmount(target) + if err1 := Unmount(target); err1 != nil { + logrus.Warnf("Unmount %q: %v", target, err1) + } } }() diff --git a/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go b/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go index 11ae563647..7a0f9cbc7d 100644 --- a/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go +++ b/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go @@ -66,11 +66,7 @@ func Init(home string, options graphdriver.Options) (graphdriver.Driver, error) return nil, fmt.Errorf("%q is not on a btrfs filesystem: %w", home, graphdriver.ErrPrerequisites) } - rootUID, rootGID, err := idtools.GetRootUIDGID(options.UIDMaps, options.GIDMaps) - if err != nil { - return nil, err - } - if err := idtools.MkdirAllAs(filepath.Join(home, "subvolumes"), 0o700, rootUID, rootGID); err != nil { + if err := os.MkdirAll(filepath.Join(home, "subvolumes"), 0o700); err != nil { return nil, err } @@ -85,8 +81,6 @@ func Init(home string, options graphdriver.Options) (graphdriver.Driver, error) driver := &Driver{ home: home, - uidMaps: options.UIDMaps, - gidMaps: options.GIDMaps, options: opt, } @@ -129,8 +123,6 @@ func parseOptions(opt []string) (btrfsOptions, bool, error) { type Driver struct { // root of the file system home string - uidMaps []idtools.IDMap - gidMaps []idtools.IDMap options btrfsOptions quotaEnabled bool once sync.Once @@ -481,11 +473,7 @@ func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error { quotas := d.quotasDir() subvolumes := d.subvolumesDir() - rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps) - if err != nil { - return err - } - if err := idtools.MkdirAllAs(subvolumes, 0o700, rootUID, rootGID); err != nil { + if err := os.MkdirAll(subvolumes, 0o700); err != nil { return err } if parent == "" { @@ -523,7 +511,7 @@ func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error { if err := d.setStorageSize(path.Join(subvolumes, id), driver); err != nil { return err } - if err := idtools.MkdirAllAs(quotas, 0o700, rootUID, rootGID); err != nil { + if err := os.MkdirAll(quotas, 0o700); err != nil { return err } if err := os.WriteFile(path.Join(quotas, id), []byte(fmt.Sprint(driver.options.size)), 0o644); err != nil { @@ -531,14 +519,6 @@ func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error { } } - // if we have a remapped root (user namespaces enabled), change the created snapshot - // dir ownership to match - if rootUID != 0 || rootGID != 0 { - if err := os.Chown(path.Join(subvolumes, id), rootUID, rootGID); err != nil { - return err - } - } - mountLabel := "" if opts != nil { mountLabel = opts.MountLabel diff --git a/vendor/github.com/containers/storage/drivers/chown.go b/vendor/github.com/containers/storage/drivers/chown.go index ca43c3f057..d728e919ba 100644 --- a/vendor/github.com/containers/storage/drivers/chown.go +++ b/vendor/github.com/containers/storage/drivers/chown.go @@ -4,11 +4,12 @@ import ( "bytes" "errors" "fmt" + "io/fs" "os" "github.com/containers/storage/pkg/idtools" "github.com/containers/storage/pkg/reexec" - "github.com/opencontainers/selinux/pkg/pwalk" + "github.com/opencontainers/selinux/pkg/pwalkdir" ) const ( @@ -54,13 +55,14 @@ func chownByMapsMain() { chowner := newLChowner() - chown := func(path string, info os.FileInfo, _ error) error { - if path == "." { + var chown fs.WalkDirFunc = func(path string, d fs.DirEntry, _ error) error { + info, err := d.Info() + if path == "." || err != nil { return nil } return chowner.LChown(path, info, toHost, toContainer) } - if err := pwalk.Walk(".", chown); err != nil { + if err := pwalkdir.Walk(".", chown); err != nil { fmt.Fprintf(os.Stderr, "error during chown: %v", err) os.Exit(1) } diff --git a/vendor/github.com/containers/storage/drivers/chroot_unix.go b/vendor/github.com/containers/storage/drivers/chroot_unix.go index 9a1c6751f8..a65f0d58d2 100644 --- a/vendor/github.com/containers/storage/drivers/chroot_unix.go +++ b/vendor/github.com/containers/storage/drivers/chroot_unix.go @@ -1,5 +1,5 @@ -//go:build linux || darwin || freebsd || solaris -// +build linux darwin freebsd solaris +//go:build !windows +// +build !windows package graphdriver diff --git a/vendor/github.com/containers/storage/drivers/copy/copy_linux.go b/vendor/github.com/containers/storage/drivers/copy/copy_linux.go index 9c3d7c668f..8789409451 100644 --- a/vendor/github.com/containers/storage/drivers/copy/copy_linux.go +++ b/vendor/github.com/containers/storage/drivers/copy/copy_linux.go @@ -50,13 +50,13 @@ func CopyRegularToFile(srcPath string, dstFile *os.File, fileinfo os.FileInfo, c defer srcFile.Close() if *copyWithFileClone { - _, _, err = unix.Syscall(unix.SYS_IOCTL, dstFile.Fd(), C.FICLONE, srcFile.Fd()) - if err == nil { + _, _, errno := unix.Syscall(unix.SYS_IOCTL, dstFile.Fd(), C.FICLONE, srcFile.Fd()) + if errno == 0 { return nil } *copyWithFileClone = false - if err == unix.EXDEV { + if errno == unix.EXDEV { *copyWithFileRange = false } } diff --git a/vendor/github.com/containers/storage/drivers/driver.go b/vendor/github.com/containers/storage/drivers/driver.go index b42ba0757d..b37837b001 100644 --- a/vendor/github.com/containers/storage/drivers/driver.go +++ b/vendor/github.com/containers/storage/drivers/driver.go @@ -193,6 +193,7 @@ type DriverWithDifferOutput struct { UIDs []uint32 GIDs []uint32 UncompressedDigest digest.Digest + CompressedDigest digest.Digest Metadata string BigData map[string][]byte TarSplit []byte @@ -215,20 +216,25 @@ const ( DifferOutputFormatFlat ) +// DifferFsVerity is a part of the experimental Differ interface and should not be used from outside of c/storage. +// It configures the fsverity requirement. type DifferFsVerity int const ( // DifferFsVerityDisabled means no fs-verity is used DifferFsVerityDisabled = iota - // DifferFsVerityEnabled means fs-verity is used when supported - DifferFsVerityEnabled + // DifferFsVerityIfAvailable means fs-verity is used when supported by + // the underlying kernel and filesystem. + DifferFsVerityIfAvailable - // DifferFsVerityRequired means fs-verity is required + // DifferFsVerityRequired means fs-verity is required. Note this is not + // currently set or exposed by the overlay driver. DifferFsVerityRequired ) -// DifferOptions overrides how the differ work +// DifferOptions is a part of the experimental Differ interface and should not be used from outside of c/storage. +// It overrides how the differ works. type DifferOptions struct { // Format defines the destination directory layout format Format DifferOutputFormat @@ -377,8 +383,6 @@ type Options struct { ImageStore string DriverPriority []string DriverOptions []string - UIDMaps []idtools.IDMap - GIDMaps []idtools.IDMap ExperimentalEnabled bool } diff --git a/vendor/github.com/containers/storage/drivers/overlay/check.go b/vendor/github.com/containers/storage/drivers/overlay/check.go index d8139f6566..d75c5fdf5e 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/check.go +++ b/vendor/github.com/containers/storage/drivers/overlay/check.go @@ -263,7 +263,11 @@ func supportsIdmappedLowerLayers(home string) (bool, error) { if err := idmap.CreateIDMappedMount(lowerDir, lowerMappedDir, int(pid)); err != nil { return false, fmt.Errorf("create mapped mount: %w", err) } - defer unix.Unmount(lowerMappedDir, unix.MNT_DETACH) + defer func() { + if err := unix.Unmount(lowerMappedDir, unix.MNT_DETACH); err != nil { + logrus.Warnf("Unmount %q: %v", lowerMappedDir, err) + } + }() opts := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", lowerMappedDir, upperDir, workDir) flags := uintptr(0) diff --git a/vendor/github.com/containers/storage/drivers/overlay/composefs.go b/vendor/github.com/containers/storage/drivers/overlay/composefs.go index 8f07c23602..973cd786c3 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/composefs.go +++ b/vendor/github.com/containers/storage/drivers/overlay/composefs.go @@ -8,6 +8,7 @@ import ( "encoding/binary" "errors" "fmt" + "io/fs" "os" "os/exec" "path/filepath" @@ -56,7 +57,7 @@ func generateComposeFsBlob(verityDigests map[string]string, toc interface{}, com fd, err := unix.Openat(unix.AT_FDCWD, destFile, unix.O_WRONLY|unix.O_CREAT|unix.O_TRUNC|unix.O_EXCL|unix.O_CLOEXEC, 0o644) if err != nil { - return fmt.Errorf("failed to open output file %q: %w", destFile, err) + return &fs.PathError{Op: "openat", Path: destFile, Err: err} } outFd := os.NewFile(uintptr(fd), "outFd") @@ -117,7 +118,7 @@ func hasACL(path string) (bool, error) { fd, err := unix.Openat(unix.AT_FDCWD, path, unix.O_RDONLY|unix.O_CLOEXEC, 0) if err != nil { - return false, err + return false, &fs.PathError{Op: "openat", Path: path, Err: err} } defer unix.Close(fd) // do not worry about checking the magic number, if the file is invalid @@ -125,7 +126,7 @@ func hasACL(path string) (bool, error) { flags := make([]byte, 4) nread, err := unix.Pread(fd, flags, 8) if err != nil { - return false, err + return false, fmt.Errorf("pread %q: %w", path, err) } if nread != 4 { return false, fmt.Errorf("failed to read flags from %q", path) @@ -150,5 +151,8 @@ func mountComposefsBlob(dataDir, mountPoint string) error { mountOpts += ",noacl" } - return unix.Mount(loop.Name(), mountPoint, "erofs", unix.MS_RDONLY, mountOpts) + if err := unix.Mount(loop.Name(), mountPoint, "erofs", unix.MS_RDONLY, mountOpts); err != nil { + return fmt.Errorf("failed to mount erofs image at %q: %w", mountPoint, err) + } + return nil } diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay.go b/vendor/github.com/containers/storage/drivers/overlay/overlay.go index 8b6f64b1ee..8ed35745f6 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "io" + "io/fs" "os" "os/exec" "path" @@ -119,8 +120,6 @@ type Driver struct { home string runhome string imageStore string - uidMaps []idtools.IDMap - gidMaps []idtools.IDMap ctr *graphdriver.RefCounter quotaCtl *quota.Control options overlayOptions @@ -332,13 +331,9 @@ func Init(home string, options graphdriver.Options) (graphdriver.Driver, error) backingFs = fsName runhome := filepath.Join(options.RunRoot, filepath.Base(home)) - rootUID, rootGID, err := idtools.GetRootUIDGID(options.UIDMaps, options.GIDMaps) - if err != nil { - return nil, err - } // Create the driver home dir - if err := idtools.MkdirAllAs(path.Join(home, linkDir), 0o755, 0, 0); err != nil { + if err := os.MkdirAll(path.Join(home, linkDir), 0o755); err != nil { return nil, err } @@ -348,7 +343,7 @@ func Init(home string, options graphdriver.Options) (graphdriver.Driver, error) } } - if err := idtools.MkdirAllAs(runhome, 0o700, rootUID, rootGID); err != nil { + if err := os.MkdirAll(runhome, 0o700); err != nil { return nil, err } @@ -373,9 +368,6 @@ func Init(home string, options graphdriver.Options) (graphdriver.Driver, error) return nil, err } } else { - if opts.forceMask != nil { - return nil, errors.New("'force_mask' is supported only with 'mount_program'") - } // check if they are running over btrfs, aufs, overlay, or ecryptfs switch fsMagic { case graphdriver.FsMagicAufs, graphdriver.FsMagicOverlay, graphdriver.FsMagicEcryptfs: @@ -457,8 +449,6 @@ func Init(home string, options graphdriver.Options) (graphdriver.Driver, error) home: home, imageStore: options.ImageStore, runhome: runhome, - uidMaps: options.UIDMaps, - gidMaps: options.GIDMaps, ctr: graphdriver.NewRefCounter(graphdriver.NewFsChecker(fileSystemType)), supportsDType: supportsDType, usingMetacopy: usingMetacopy, @@ -698,12 +688,8 @@ func SupportsNativeOverlay(home, runhome string) (bool, error) { } func supportsOverlay(home string, homeMagic graphdriver.FsMagic, rootUID, rootGID int) (supportsDType bool, err error) { - // We can try to modprobe overlay first - selinuxLabelTest := selinux.PrivContainerMountLabel() - exec.Command("modprobe", "overlay").Run() - logLevel := logrus.ErrorLevel if unshare.IsRootless() { logLevel = logrus.DebugLevel @@ -831,7 +817,9 @@ func (d *Driver) useNaiveDiff() bool { logrus.Info(nativeDiffCacheText) useNaiveDiffOnly = true } - cachedFeatureRecord(d.runhome, feature, !useNaiveDiffOnly, nativeDiffCacheText) + if err := cachedFeatureRecord(d.runhome, feature, !useNaiveDiffOnly, nativeDiffCacheText); err != nil { + logrus.Warnf("Recording overlay native-diff support status: %v", err) + } }) return useNaiveDiffOnly } @@ -860,14 +848,14 @@ func (d *Driver) Status() [][2]string { // Metadata returns meta data about the overlay driver such as // LowerDir, UpperDir, WorkDir and MergeDir used to store data. func (d *Driver) Metadata(id string) (map[string]string, error) { - dir := d.dir(id) + dir, _, inAdditionalStore := d.dir2(id, false) if err := fileutils.Exists(dir); err != nil { return nil, err } metadata := map[string]string{ "WorkDir": path.Join(dir, "work"), - "MergedDir": path.Join(dir, "merged"), + "MergedDir": d.getMergedDir(id, dir, inAdditionalStore), "UpperDir": path.Join(dir, "diff"), } @@ -983,6 +971,10 @@ func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts } } + if d.options.forceMask != nil && d.options.mountProgram == "" { + return fmt.Errorf("overlay: force_mask option for writeable layers is only supported with a mount_program") + } + if _, ok := opts.StorageOpt["size"]; !ok { if opts.StorageOpt == nil { opts.StorageOpt = map[string]string{} @@ -1021,8 +1013,8 @@ func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts, readOnl disableQuota := readOnly - uidMaps := d.uidMaps - gidMaps := d.gidMaps + var uidMaps []idtools.IDMap + var gidMaps []idtools.IDMap if opts != nil && opts.IDMappings != nil { uidMaps = opts.IDMappings.UIDs() @@ -1047,14 +1039,23 @@ func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts, readOnl if err := idtools.MkdirAllAndChownNew(path.Dir(dir), 0o755, idPair); err != nil { return err } + + st := idtools.Stat{IDs: idPair, Mode: defaultPerms} + if parent != "" { parentBase := d.dir(parent) - st, err := system.Stat(filepath.Join(parentBase, "diff")) - if err != nil { - return err + parentDiff := filepath.Join(parentBase, "diff") + if xSt, err := idtools.GetContainersOverrideXattr(parentDiff); err == nil { + st = xSt + } else { + systemSt, err := system.Stat(parentDiff) + if err != nil { + return err + } + st.IDs.UID = int(systemSt.UID()) + st.IDs.GID = int(systemSt.GID()) + st.Mode = os.FileMode(systemSt.Mode()) } - rootUID = int(st.UID()) - rootGID = int(st.GID()) } if err := fileutils.Lexists(dir); err == nil { @@ -1100,22 +1101,21 @@ func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts, readOnl } } - perms := defaultPerms + forcedSt := st if d.options.forceMask != nil { - perms = *d.options.forceMask + forcedSt.IDs = idPair + forcedSt.Mode = *d.options.forceMask } - if parent != "" { - parentBase := d.dir(parent) - st, err := system.Stat(filepath.Join(parentBase, "diff")) - if err != nil { - return err - } - perms = os.FileMode(st.Mode()) + diff := path.Join(dir, "diff") + if err := idtools.MkdirAs(diff, forcedSt.Mode, forcedSt.IDs.UID, forcedSt.IDs.GID); err != nil { + return err } - if err := idtools.MkdirAs(path.Join(dir, "diff"), perms, rootUID, rootGID); err != nil { - return err + if d.options.forceMask != nil { + if err := idtools.SetContainersOverrideXattr(diff, st); err != nil { + return err + } } lid := generateID(idLength) @@ -1130,16 +1130,16 @@ func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts, readOnl return err } - if err := idtools.MkdirAs(path.Join(dir, "work"), 0o700, rootUID, rootGID); err != nil { + if err := idtools.MkdirAs(path.Join(dir, "work"), 0o700, forcedSt.IDs.UID, forcedSt.IDs.GID); err != nil { return err } - if err := idtools.MkdirAs(path.Join(dir, "merged"), 0o700, rootUID, rootGID); err != nil { + if err := idtools.MkdirAs(path.Join(dir, "merged"), 0o700, forcedSt.IDs.UID, forcedSt.IDs.GID); err != nil { return err } // if no parent directory, create a dummy lower directory and skip writing a "lowers" file if parent == "" { - return idtools.MkdirAs(path.Join(dir, "empty"), 0o700, rootUID, rootGID) + return idtools.MkdirAs(path.Join(dir, "empty"), 0o700, forcedSt.IDs.UID, forcedSt.IDs.GID) } lower, err := d.getLower(parent) @@ -1283,12 +1283,6 @@ func (d *Driver) getLowerDirs(id string) ([]string, error) { } func (d *Driver) optsAppendMappings(opts string, uidMaps, gidMaps []idtools.IDMap) string { - if uidMaps == nil { - uidMaps = d.uidMaps - } - if gidMaps == nil { - gidMaps = d.gidMaps - } if uidMaps != nil { var uids, gids bytes.Buffer if len(uidMaps) == 1 && uidMaps[0].Size == 1 { @@ -1539,11 +1533,7 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO for err == nil { absLowers = append(absLowers, filepath.Join(dir, nameWithSuffix("diff", diffN))) diffN++ - st, err = os.Stat(filepath.Join(dir, nameWithSuffix("diff", diffN))) - if err == nil && !permsKnown { - perms = os.FileMode(st.Mode()) - permsKnown = true - } + err = fileutils.Exists(filepath.Join(dir, nameWithSuffix("diff", diffN))) } idmappedMountProcessPid := -1 @@ -1561,7 +1551,11 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO composefsMounts := []string{} defer func() { for _, m := range composefsMounts { - defer unix.Unmount(m, unix.MNT_DETACH) + defer func(m string) { + if err := unix.Unmount(m, unix.MNT_DETACH); err != nil { + logrus.Warnf("Unmount %q: %v", m, err) + } + }(m) } }() @@ -1665,7 +1659,11 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO skipIDMappingLayers[composefsMount] = composefsMount // overlay takes a reference on the mount, so it is safe to unmount // the mapped idmounts as soon as the final overlay file system is mounted. - defer unix.Unmount(composefsMount, unix.MNT_DETACH) + defer func() { + if err := unix.Unmount(composefsMount, unix.MNT_DETACH); err != nil { + logrus.Warnf("Unmount %q: %v", composefsMount, err) + } + }() } absLowers = append(absLowers, composefsMount) continue @@ -1705,10 +1703,10 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO } } - mergedDir := path.Join(dir, "merged") + mergedDir := d.getMergedDir(id, dir, inAdditionalStore) // Attempt to create the merged dir only if it doesn't exist. if err := fileutils.Exists(mergedDir); err != nil && os.IsNotExist(err) { - if err := idtools.MkdirAs(mergedDir, 0o700, rootUID, rootGID); err != nil && !os.IsExist(err) { + if err := idtools.MkdirAllAs(mergedDir, 0o700, rootUID, rootGID); err != nil && !os.IsExist(err) { return "", err } } @@ -1772,7 +1770,11 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO // overlay takes a reference on the mount, so it is safe to unmount // the mapped idmounts as soon as the final overlay file system is mounted. - defer unix.Unmount(root, unix.MNT_DETACH) + defer func() { + if err := unix.Unmount(root, unix.MNT_DETACH); err != nil { + logrus.Warnf("Unmount %q: %v", root, err) + } + }() } // relative path to the layer through the id mapped mount @@ -1854,7 +1856,9 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO mountFunc = func(source string, target string, mType string, flags uintptr, label string) error { return mountOverlayFrom(d.home, source, target, mType, flags, label) } - mountTarget = path.Join(id, "merged") + if !inAdditionalStore { + mountTarget = path.Join(id, "merged") + } } // overlay has a check in place to prevent mounting the same file system twice @@ -1873,13 +1877,26 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO return mergedDir, nil } +// getMergedDir returns the directory path that should be used as the mount point for the overlayfs. +func (d *Driver) getMergedDir(id, dir string, inAdditionalStore bool) string { + // If the layer is in an additional store, the lock we might hold only a reading lock. To prevent + // races with other processes, use a private directory under the main store rundir. At this point, the + // current process is holding an exclusive lock on the store, and since the rundir cannot be shared for + // different stores, it is safe to assume the current process has exclusive access to it. + if inAdditionalStore { + return path.Join(d.runhome, id, "merged") + } + return path.Join(dir, "merged") +} + // Put unmounts the mount path created for the give id. func (d *Driver) Put(id string) error { dir, _, inAdditionalStore := d.dir2(id, false) if err := fileutils.Exists(dir); err != nil { return err } - mountpoint := path.Join(dir, "merged") + mountpoint := d.getMergedDir(id, dir, inAdditionalStore) + if count := d.ctr.Decrement(mountpoint); count > 0 { return nil } @@ -1936,7 +1953,15 @@ func (d *Driver) Put(id string) error { } } - if !inAdditionalStore { + if inAdditionalStore { + // check the base name for extra safety + if strings.HasPrefix(mountpoint, d.runhome) && filepath.Base(mountpoint) == "merged" { + err := os.RemoveAll(filepath.Dir(mountpoint)) + if err != nil { + logrus.Warningf("Failed to remove mountpoint %s overlay: %s: %v", id, mountpoint, err) + } + } + } else { uid, gid := int(0), int(0) fi, err := os.Stat(mountpoint) if err != nil { @@ -1953,7 +1978,7 @@ func (d *Driver) Put(id string) error { // rename(2) can be used on an empty directory, as it is the mountpoint after umount, and it retains // its atomic semantic. In this way the "merged" directory is never removed. if err := unix.Rename(tmpMountpoint, mountpoint); err != nil { - logrus.Debugf("Failed to replace mountpoint %s overlay: %s - %v", id, mountpoint, err) + logrus.Debugf("Failed to replace mountpoint %s overlay: %s: %v", id, mountpoint, err) return fmt.Errorf("replacing mount point %q: %w", mountpoint, err) } } @@ -2024,11 +2049,27 @@ func (d *Driver) getWhiteoutFormat() archive.WhiteoutFormat { } type overlayFileGetter struct { - diffDirs []string + diffDirs []string + composefsMounts map[string]*os.File // map from diff dir to the directory with the composefs blob mounted } func (g *overlayFileGetter) Get(path string) (io.ReadCloser, error) { + buf := make([]byte, unix.PathMax) for _, d := range g.diffDirs { + if f, found := g.composefsMounts[d]; found { + // there is no *at equivalent for getxattr, but it can be emulated by opening the file under /proc/self/fd/$FD/$PATH + len, err := unix.Getxattr(fmt.Sprintf("/proc/self/fd/%d/%s", int(f.Fd()), path), "trusted.overlay.redirect", buf) + if err != nil { + if errors.Is(err, unix.ENODATA) { + continue + } + return nil, &fs.PathError{Op: "getxattr", Path: path, Err: err} + } + + // the xattr value is the path to the file in the composefs layer diff directory + return os.Open(filepath.Join(d, string(buf[:len]))) + } + f, err := os.Open(filepath.Join(d, path)) if err == nil { return f, nil @@ -2041,7 +2082,16 @@ func (g *overlayFileGetter) Get(path string) (io.ReadCloser, error) { } func (g *overlayFileGetter) Close() error { - return nil + var errs *multierror.Error + for _, f := range g.composefsMounts { + if err := f.Close(); err != nil { + errs = multierror.Append(errs, err) + } + if err := unix.Rmdir(f.Name()); err != nil { + errs = multierror.Append(errs, err) + } + } + return errs.ErrorOrNil() } func (d *Driver) getStagingDir(id string) string { @@ -2052,10 +2102,7 @@ func (d *Driver) getStagingDir(id string) string { // DiffGetter returns a FileGetCloser that can read files from the directory that // contains files for the layer differences, either for this layer, or one of our // lowers if we're just a template directory. Used for direct access for tar-split. -func (d *Driver) DiffGetter(id string) (graphdriver.FileGetCloser, error) { - if d.usingComposefs { - return nil, nil - } +func (d *Driver) DiffGetter(id string) (_ graphdriver.FileGetCloser, Err error) { p, err := d.getDiffPath(id) if err != nil { return nil, err @@ -2064,7 +2111,43 @@ func (d *Driver) DiffGetter(id string) (graphdriver.FileGetCloser, error) { if err != nil { return nil, err } - return &overlayFileGetter{diffDirs: append([]string{p}, paths...)}, nil + + // map from diff dir to the directory with the composefs blob mounted + composefsMounts := make(map[string]*os.File) + defer func() { + if Err != nil { + for _, f := range composefsMounts { + f.Close() + if err := unix.Rmdir(f.Name()); err != nil && !os.IsNotExist(err) { + logrus.Warnf("Failed to remove %s: %v", f.Name(), err) + } + } + } + }() + diffDirs := append([]string{p}, paths...) + for _, diffDir := range diffDirs { + // diffDir has the form $GRAPH_ROOT/overlay/$ID/diff, so grab the $ID from the parent directory + id := path.Base(path.Dir(diffDir)) + composefsBlob := d.getComposefsData(id) + if fileutils.Exists(composefsBlob) != nil { + // not a composefs layer, ignore it + continue + } + dir, err := os.MkdirTemp(d.runhome, "composefs-mnt") + if err != nil { + return nil, err + } + if err := mountComposefsBlob(composefsBlob, dir); err != nil { + return nil, err + } + fd, err := os.Open(dir) + if err != nil { + return nil, err + } + composefsMounts[diffDir] = fd + _ = unix.Unmount(dir, unix.MNT_DETACH) + } + return &overlayFileGetter{diffDirs: diffDirs, composefsMounts: composefsMounts}, nil } // CleanupStagingDirectory cleanups the staging directory. @@ -2100,9 +2183,16 @@ func supportsDataOnlyLayersCached(home, runhome string) (bool, error) { // ApplyDiffWithDiffer applies the changes in the new layer using the specified function func (d *Driver) ApplyDiffWithDiffer(id, parent string, options *graphdriver.ApplyDiffWithDifferOpts, differ graphdriver.Differ) (output graphdriver.DriverWithDifferOutput, errRet error) { var idMappings *idtools.IDMappings + var forceMask *os.FileMode + if options != nil { idMappings = options.Mappings + forceMask = options.ForceMask + } + if d.options.forceMask != nil { + forceMask = d.options.forceMask } + if idMappings == nil { idMappings = &idtools.IDMappings{} } @@ -2120,8 +2210,8 @@ func (d *Driver) ApplyDiffWithDiffer(id, parent string, options *graphdriver.App return graphdriver.DriverWithDifferOutput{}, err } perms := defaultPerms - if d.options.forceMask != nil { - perms = *d.options.forceMask + if forceMask != nil { + perms = *forceMask } applyDir = filepath.Join(layerDir, "dir") if err := os.Mkdir(applyDir, perms); err != nil { @@ -2155,7 +2245,7 @@ func (d *Driver) ApplyDiffWithDiffer(id, parent string, options *graphdriver.App } if d.usingComposefs { differOptions.Format = graphdriver.DifferOutputFormatFlat - differOptions.UseFsVerity = graphdriver.DifferFsVerityEnabled + differOptions.UseFsVerity = graphdriver.DifferFsVerityIfAvailable } out, err := differ.ApplyDiff(applyDir, &archive.TarOptions{ UIDMaps: idMappings.UIDs(), @@ -2163,6 +2253,7 @@ func (d *Driver) ApplyDiffWithDiffer(id, parent string, options *graphdriver.App IgnoreChownErrors: d.options.ignoreChownErrors, WhiteoutFormat: d.getWhiteoutFormat(), InUserNS: unshare.IsRootless(), + ForceMask: forceMask, }, &differOptions) out.Target = applyDir @@ -2342,14 +2433,18 @@ func (d *Driver) Changes(id string, idMappings *idtools.IDMappings, parent strin // layers. diffPath, err := d.getDiffPath(id) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to get diff path: %w", err) } layers, err := d.getLowerDiffPaths(id) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to get lower diff path: %w", err) } - return archive.OverlayChanges(layers, diffPath) + c, err := archive.OverlayChanges(layers, diffPath) + if err != nil { + return nil, fmt.Errorf("computing changes: %w", err) + } + return c, nil } // AdditionalImageStores returns additional image stores supported by the driver @@ -2476,6 +2571,19 @@ func nameWithSuffix(name string, number int) string { return fmt.Sprintf("%s%d", name, number) } +func validateOneAdditionalLayerPath(target string) error { + for _, p := range []string{ + filepath.Join(target, "diff"), + filepath.Join(target, "info"), + filepath.Join(target, "blob"), + } { + if err := fileutils.Exists(p); err != nil { + return err + } + } + return nil +} + func (d *Driver) getAdditionalLayerPath(tocDigest digest.Digest, ref string) (string, error) { refElem := base64.StdEncoding.EncodeToString([]byte(ref)) for _, ls := range d.options.layerStores { @@ -2484,18 +2592,11 @@ func (d *Driver) getAdditionalLayerPath(tocDigest digest.Digest, ref string) (st ref = refElem } target := path.Join(ls.path, ref, tocDigest.String()) - // Check if all necessary files exist - for _, p := range []string{ - filepath.Join(target, "diff"), - filepath.Join(target, "info"), - filepath.Join(target, "blob"), - } { - if err := fileutils.Exists(p); err != nil { - wrapped := fmt.Errorf("failed to stat additional layer %q: %w", p, err) - return "", fmt.Errorf("%v: %w", wrapped, graphdriver.ErrLayerUnknown) - } - } - return target, nil + err := validateOneAdditionalLayerPath(target) + if err == nil { + return target, nil + } + logrus.Debugf("additional Layer Store %v failed to stat additional layer: %v", ls, err) } return "", fmt.Errorf("additional layer (%q, %q) not found: %w", tocDigest, ref, graphdriver.ErrLayerUnknown) diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay_cgo.go b/vendor/github.com/containers/storage/drivers/overlay/overlay_disk_quota.go similarity index 86% rename from vendor/github.com/containers/storage/drivers/overlay/overlay_cgo.go rename to vendor/github.com/containers/storage/drivers/overlay/overlay_disk_quota.go index 88bfbf9c74..bc80301d49 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay_cgo.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay_disk_quota.go @@ -1,5 +1,5 @@ -//go:build linux && cgo -// +build linux,cgo +//go:build linux && cgo && !exclude_disk_quota +// +build linux,cgo,!exclude_disk_quota package overlay diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay_disk_quota_unsupported.go b/vendor/github.com/containers/storage/drivers/overlay/overlay_disk_quota_unsupported.go new file mode 100644 index 0000000000..1340c45b33 --- /dev/null +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay_disk_quota_unsupported.go @@ -0,0 +1,18 @@ +//go:build linux && (!cgo || exclude_disk_quota) +// +build linux +// +build !cgo exclude_disk_quota + +package overlay + +import ( + "path" + + "github.com/containers/storage/pkg/directory" +) + +// ReadWriteDiskUsage returns the disk usage of the writable directory for the ID. +// For Overlay, it attempts to check the XFS quota for size, and falls back to +// finding the size of the "diff" directory. +func (d *Driver) ReadWriteDiskUsage(id string) (*directory.DiskUsage, error) { + return directory.Usage(path.Join(d.dir(id), "diff")) +} diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay_nocgo.go b/vendor/github.com/containers/storage/drivers/overlay/overlay_nocgo.go index d4f540c9cc..0577711b3d 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay_nocgo.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay_nocgo.go @@ -5,18 +5,8 @@ package overlay import ( "fmt" - "path" - - "github.com/containers/storage/pkg/directory" ) -// ReadWriteDiskUsage returns the disk usage of the writable directory for the ID. -// For Overlay, it attempts to check the XFS quota for size, and falls back to -// finding the size of the "diff" directory. -func (d *Driver) ReadWriteDiskUsage(id string) (*directory.DiskUsage, error) { - return directory.Usage(path.Join(d.dir(id), "diff")) -} - func getComposeFsHelper() (string, error) { return "", fmt.Errorf("composefs not supported on this build") } diff --git a/vendor/github.com/containers/storage/drivers/quota/projectquota_supported.go b/vendor/github.com/containers/storage/drivers/quota/projectquota_supported.go index b0623bdacc..92e4001d76 100644 --- a/vendor/github.com/containers/storage/drivers/quota/projectquota_supported.go +++ b/vendor/github.com/containers/storage/drivers/quota/projectquota_supported.go @@ -19,16 +19,6 @@ package quota #include #include -#ifndef FS_XFLAG_PROJINHERIT -struct fsxattr { - __u32 fsx_xflags; - __u32 fsx_extsize; - __u32 fsx_nextents; - __u32 fsx_projid; - unsigned char fsx_pad[12]; -}; -#define FS_XFLAG_PROJINHERIT 0x00000200 -#endif #ifndef FS_IOC_FSGETXATTR #define FS_IOC_FSGETXATTR _IOR ('X', 31, struct fsxattr) #endif @@ -357,7 +347,6 @@ func setProjectID(targetPath string, projectID uint32) error { return fmt.Errorf("failed to get projid for %s: %w", targetPath, errno) } fsx.fsx_projid = C.__u32(projectID) - fsx.fsx_xflags |= C.FS_XFLAG_PROJINHERIT _, _, errno = unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSSETXATTR, uintptr(unsafe.Pointer(&fsx))) if errno != 0 { diff --git a/vendor/github.com/containers/storage/drivers/vfs/driver.go b/vendor/github.com/containers/storage/drivers/vfs/driver.go index db9032117d..f60ec17b3f 100644 --- a/vendor/github.com/containers/storage/drivers/vfs/driver.go +++ b/vendor/github.com/containers/storage/drivers/vfs/driver.go @@ -33,12 +33,10 @@ func Init(home string, options graphdriver.Options) (graphdriver.Driver, error) d := &Driver{ name: "vfs", home: home, - idMappings: idtools.NewIDMappingsFromMaps(options.UIDMaps, options.GIDMaps), imageStore: options.ImageStore, } - rootIDs := d.idMappings.RootPair() - if err := idtools.MkdirAllAndChown(filepath.Join(home, "dir"), 0o700, rootIDs); err != nil { + if err := os.MkdirAll(filepath.Join(home, "dir"), 0o700); err != nil { return nil, err } for _, option := range options.DriverOptions { @@ -79,7 +77,6 @@ type Driver struct { name string home string additionalHomes []string - idMappings *idtools.IDMappings ignoreChownErrors bool naiveDiff graphdriver.DiffDriver updater graphdriver.LayerIDMapUpdater @@ -152,14 +149,21 @@ func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts, ro bool return fmt.Errorf("--storage-opt is not supported for vfs") } - idMappings := d.idMappings + var uidMaps []idtools.IDMap + var gidMaps []idtools.IDMap + if opts != nil && opts.IDMappings != nil { - idMappings = opts.IDMappings + uidMaps = opts.IDMappings.UIDs() + gidMaps = opts.IDMappings.GIDs() + } + + rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps) + if err != nil { + return err } dir := d.dir2(id, ro) - rootIDs := idMappings.RootPair() - if err := idtools.MkdirAllAndChown(filepath.Dir(dir), 0o700, rootIDs); err != nil { + if err := os.MkdirAll(filepath.Dir(dir), 0o700); err != nil { return err } @@ -174,21 +178,24 @@ func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts, ro bool rootPerms = os.FileMode(0o700) } + idPair := idtools.IDPair{UID: rootUID, GID: rootGID} if parent != "" { st, err := system.Stat(d.dir(parent)) if err != nil { return err } rootPerms = os.FileMode(st.Mode()) - rootIDs.UID = int(st.UID()) - rootIDs.GID = int(st.GID()) + idPair.UID = int(st.UID()) + idPair.GID = int(st.GID()) } - if err := idtools.MkdirAndChown(dir, rootPerms, rootIDs); err != nil { + if err := idtools.MkdirAllAndChownNew(dir, rootPerms, idPair); err != nil { return err } labelOpts := []string{"level:s0"} if _, mountLabel, err := label.InitLabels(labelOpts); err == nil { - label.SetFileLabel(dir, mountLabel) + if err := label.SetFileLabel(dir, mountLabel); err != nil { + logrus.Debugf("Set %s label to %q file ended with error: %v", mountLabel, dir, err) + } } if parent != "" { parentDir, err := d.Get(parent, graphdriver.MountOpts{}) diff --git a/vendor/github.com/containers/storage/drivers/zfs/zfs.go b/vendor/github.com/containers/storage/drivers/zfs/zfs.go index e022897848..0e859a93b8 100644 --- a/vendor/github.com/containers/storage/drivers/zfs/zfs.go +++ b/vendor/github.com/containers/storage/drivers/zfs/zfs.go @@ -106,11 +106,7 @@ func Init(base string, opt graphdriver.Options) (graphdriver.Driver, error) { return nil, fmt.Errorf("zfs get all -t filesystem -rHp '%s' should contain '%s'", options.fsName, options.fsName) } - rootUID, rootGID, err := idtools.GetRootUIDGID(opt.UIDMaps, opt.GIDMaps) - if err != nil { - return nil, fmt.Errorf("failed to get root uid/gid: %w", err) - } - if err := idtools.MkdirAllAs(base, 0o700, rootUID, rootGID); err != nil { + if err := os.MkdirAll(base, 0o700); err != nil { return nil, fmt.Errorf("failed to create '%s': %w", base, err) } @@ -118,8 +114,6 @@ func Init(base string, opt graphdriver.Options) (graphdriver.Driver, error) { dataset: rootDataset, options: options, filesystemsCache: filesystemsCache, - uidMaps: opt.UIDMaps, - gidMaps: opt.GIDMaps, ctr: graphdriver.NewRefCounter(graphdriver.NewDefaultChecker()), } return graphdriver.NewNaiveDiffDriver(d, graphdriver.NewNaiveLayerIDMapUpdater(d)), nil @@ -177,8 +171,6 @@ type Driver struct { options zfsOptions sync.Mutex // protects filesystem cache against concurrent access filesystemsCache map[string]bool - uidMaps []idtools.IDMap - gidMaps []idtools.IDMap ctr *graphdriver.RefCounter } @@ -248,7 +240,9 @@ func (d *Driver) cloneFilesystem(name, parentName string) error { } if err != nil { - snapshot.Destroy(zfs.DestroyDeferDeletion) + if err1 := snapshot.Destroy(zfs.DestroyDeferDeletion); err1 != nil { + logrus.Warnf("Destroy zfs.DestroyDeferDeletion: %v", err1) + } return err } return snapshot.Destroy(zfs.DestroyDeferDeletion) @@ -448,12 +442,8 @@ func (d *Driver) Get(id string, options graphdriver.MountOpts) (_ string, retErr opts := label.FormatMountLabel(mountOptions, options.MountLabel) logrus.WithField("storage-driver", "zfs").Debugf(`mount("%s", "%s", "%s")`, filesystem, mountpoint, opts) - rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps) - if err != nil { - return "", err - } // Create the target directories if they don't exist - if err := idtools.MkdirAllAs(mountpoint, 0o755, rootUID, rootGID); err != nil { + if err := os.MkdirAll(mountpoint, 0o755); err != nil { return "", err } diff --git a/vendor/github.com/containers/storage/layers.go b/vendor/github.com/containers/storage/layers.go index f1325262b5..6caf28ab71 100644 --- a/vendor/github.com/containers/storage/layers.go +++ b/vendor/github.com/containers/storage/layers.go @@ -2529,7 +2529,9 @@ func (r *layerStore) applyDiffFromStagingDirectory(id string, diffOutput *driver layer.GIDs = diffOutput.GIDs updateDigestMap(&r.byuncompressedsum, layer.UncompressedDigest, diffOutput.UncompressedDigest, layer.ID) layer.UncompressedDigest = diffOutput.UncompressedDigest - updateDigestMap(&r.bytocsum, diffOutput.TOCDigest, diffOutput.TOCDigest, layer.ID) + updateDigestMap(&r.bycompressedsum, layer.CompressedDigest, diffOutput.CompressedDigest, layer.ID) + layer.CompressedDigest = diffOutput.CompressedDigest + updateDigestMap(&r.bytocsum, layer.TOCDigest, diffOutput.TOCDigest, layer.ID) layer.TOCDigest = diffOutput.TOCDigest layer.UncompressedSize = diffOutput.Size layer.Metadata = diffOutput.Metadata diff --git a/vendor/github.com/containers/storage/lockfile_compat.go b/vendor/github.com/containers/storage/lockfile_compat.go index 640203881a..ec98b40ceb 100644 --- a/vendor/github.com/containers/storage/lockfile_compat.go +++ b/vendor/github.com/containers/storage/lockfile_compat.go @@ -5,7 +5,7 @@ import ( ) // Deprecated: Use lockfile.*LockFile. -type Locker = lockfile.Locker //lint:ignore SA1019 // lockfile.Locker is deprecated +type Locker = lockfile.Locker //nolint:staticcheck // SA1019 lockfile.Locker is deprecated // Deprecated: Use lockfile.GetLockFile. func GetLockfile(path string) (lockfile.Locker, error) { diff --git a/vendor/github.com/containers/storage/pkg/archive/archive.go b/vendor/github.com/containers/storage/pkg/archive/archive.go index 77c9c818c9..13a458956c 100644 --- a/vendor/github.com/containers/storage/pkg/archive/archive.go +++ b/vendor/github.com/containers/storage/pkg/archive/archive.go @@ -70,6 +70,8 @@ type ( } ) +const PaxSchilyXattr = "SCHILY.xattr." + const ( tarExt = "tar" solaris = "solaris" @@ -169,10 +171,17 @@ func DetectCompression(source []byte) Compression { } // DecompressStream decompresses the archive and returns a ReaderCloser with the decompressed archive. -func DecompressStream(archive io.Reader) (io.ReadCloser, error) { +func DecompressStream(archive io.Reader) (_ io.ReadCloser, Err error) { p := pools.BufioReader32KPool buf := p.Get(archive) bs, err := buf.Peek(10) + + defer func() { + if Err != nil { + p.Put(buf) + } + }() + if err != nil && err != io.EOF { // Note: we'll ignore any io.EOF error because there are some odd // cases where the layer.tar file will be empty (zero bytes) and @@ -189,6 +198,12 @@ func DecompressStream(archive io.Reader) (io.ReadCloser, error) { readBufWrapper := p.NewReadCloserWrapper(buf, buf) return readBufWrapper, nil case Gzip: + cleanup := func() { + p.Put(buf) + } + if rc, canUse := tryProcFilter([]string{"pigz", "-d"}, buf, cleanup); canUse { + return rc, nil + } gzReader, err := gzip.NewReader(buf) if err != nil { return nil, err @@ -207,6 +222,12 @@ func DecompressStream(archive io.Reader) (io.ReadCloser, error) { readBufWrapper := p.NewReadCloserWrapper(buf, xzReader) return readBufWrapper, nil case Zstd: + cleanup := func() { + p.Put(buf) + } + if rc, canUse := tryProcFilter([]string{"zstd", "-d"}, buf, cleanup); canUse { + return rc, nil + } return zstdReader(buf) default: return nil, fmt.Errorf("unsupported compression format %s", (&compression).Extension()) @@ -214,9 +235,16 @@ func DecompressStream(archive io.Reader) (io.ReadCloser, error) { } // CompressStream compresses the dest with specified compression algorithm. -func CompressStream(dest io.Writer, compression Compression) (io.WriteCloser, error) { +func CompressStream(dest io.Writer, compression Compression) (_ io.WriteCloser, Err error) { p := pools.BufioWriter32KPool buf := p.Get(dest) + + defer func() { + if Err != nil { + p.Put(buf) + } + }() + switch compression { case Uncompressed: writeBufWrapper := p.NewWriteCloserWrapper(buf, buf) @@ -391,11 +419,11 @@ func FileInfoHeader(name string, fi os.FileInfo, link string) (*tar.Header, erro return hdr, nil } -// ReadSecurityXattrToTarHeader reads security.capability, security,image +// readSecurityXattrToTarHeader reads security.capability, security,image // xattrs from filesystem to a tar header -func ReadSecurityXattrToTarHeader(path string, hdr *tar.Header) error { - if hdr.Xattrs == nil { - hdr.Xattrs = make(map[string]string) +func readSecurityXattrToTarHeader(path string, hdr *tar.Header) error { + if hdr.PAXRecords == nil { + hdr.PAXRecords = make(map[string]string) } for _, xattr := range []string{"security.capability", "security.ima"} { capability, err := system.Lgetxattr(path, xattr) @@ -403,14 +431,14 @@ func ReadSecurityXattrToTarHeader(path string, hdr *tar.Header) error { return fmt.Errorf("failed to read %q attribute from %q: %w", xattr, path, err) } if capability != nil { - hdr.Xattrs[xattr] = string(capability) + hdr.PAXRecords[PaxSchilyXattr+xattr] = string(capability) } } return nil } -// ReadUserXattrToTarHeader reads user.* xattr from filesystem to a tar header -func ReadUserXattrToTarHeader(path string, hdr *tar.Header) error { +// readUserXattrToTarHeader reads user.* xattr from filesystem to a tar header +func readUserXattrToTarHeader(path string, hdr *tar.Header) error { xattrs, err := system.Llistxattr(path) if err != nil && !errors.Is(err, system.EOPNOTSUPP) && err != system.ErrNotSupportedPlatform { return err @@ -425,10 +453,10 @@ func ReadUserXattrToTarHeader(path string, hdr *tar.Header) error { } return err } - if hdr.Xattrs == nil { - hdr.Xattrs = make(map[string]string) + if hdr.PAXRecords == nil { + hdr.PAXRecords = make(map[string]string) } - hdr.Xattrs[key] = string(value) + hdr.PAXRecords[PaxSchilyXattr+key] = string(value) } } return nil @@ -516,10 +544,10 @@ func (ta *tarAppender) addTarFile(path, name string) error { if err != nil { return err } - if err := ReadSecurityXattrToTarHeader(path, hdr); err != nil { + if err := readSecurityXattrToTarHeader(path, hdr); err != nil { return err } - if err := ReadUserXattrToTarHeader(path, hdr); err != nil { + if err := readUserXattrToTarHeader(path, hdr); err != nil { return err } if err := ReadFileFlagsToTarHeader(path, hdr); err != nil { @@ -642,7 +670,7 @@ func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, L } } - case tar.TypeReg, tar.TypeRegA: + case tar.TypeReg: // Source is regular file. We use system.OpenFileSequential to use sequential // file access to avoid depleting the standby list on Windows. // On Linux, this equates to a regular os.OpenFile @@ -701,8 +729,11 @@ func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, L } if forceMask != nil && (hdr.Typeflag != tar.TypeSymlink || runtime.GOOS == "darwin") { - value := fmt.Sprintf("%d:%d:0%o", hdr.Uid, hdr.Gid, hdrInfo.Mode()&0o7777) - if err := system.Lsetxattr(path, idtools.ContainersOverrideXattr, []byte(value), 0); err != nil { + value := idtools.Stat{ + IDs: idtools.IDPair{UID: hdr.Uid, GID: hdr.Gid}, + Mode: hdrInfo.Mode() & 0o7777, + } + if err := idtools.SetContainersOverrideXattr(path, value); err != nil { return err } } @@ -753,11 +784,15 @@ func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, L } var errs []string - for key, value := range hdr.Xattrs { - if _, found := xattrsToIgnore[key]; found { + for key, value := range hdr.PAXRecords { + xattrKey, ok := strings.CutPrefix(key, PaxSchilyXattr) + if !ok { + continue + } + if _, found := xattrsToIgnore[xattrKey]; found { continue } - if err := system.Lsetxattr(path, key, []byte(value), 0); err != nil { + if err := system.Lsetxattr(path, xattrKey, []byte(value), 0); err != nil { if errors.Is(err, syscall.ENOTSUP) || (inUserns && errors.Is(err, syscall.EPERM)) { // We ignore errors here because not all graphdrivers support // xattrs *cough* old versions of AUFS *cough*. However only @@ -1113,9 +1148,14 @@ loop: } } - if options.ForceMask != nil && rootHdr != nil { - value := fmt.Sprintf("%d:%d:0%o", rootHdr.Uid, rootHdr.Gid, rootHdr.Mode) - if err := system.Lsetxattr(dest, idtools.ContainersOverrideXattr, []byte(value), 0); err != nil { + if options.ForceMask != nil { + value := idtools.Stat{Mode: 0o755} + if rootHdr != nil { + value.IDs.UID = rootHdr.Uid + value.IDs.GID = rootHdr.Gid + value.Mode = os.FileMode(rootHdr.Mode) + } + if err := idtools.SetContainersOverrideXattr(dest, value); err != nil { return err } } @@ -1337,7 +1377,7 @@ func remapIDs(readIDMappings, writeIDMappings *idtools.IDMappings, chownOpts *id } } else if runtime.GOOS == darwin { uid, gid = hdr.Uid, hdr.Gid - if xstat, ok := hdr.Xattrs[idtools.ContainersOverrideXattr]; ok { + if xstat, ok := hdr.PAXRecords[PaxSchilyXattr+idtools.ContainersOverrideXattr]; ok { attrs := strings.Split(string(xstat), ":") if len(attrs) == 3 { val, err := strconv.ParseUint(attrs[0], 10, 32) diff --git a/vendor/github.com/containers/storage/pkg/archive/archive_bsd.go b/vendor/github.com/containers/storage/pkg/archive/archive_bsd.go index 4d362f0750..a754d0d4a3 100644 --- a/vendor/github.com/containers/storage/pkg/archive/archive_bsd.go +++ b/vendor/github.com/containers/storage/pkg/archive/archive_bsd.go @@ -1,5 +1,5 @@ -//go:build freebsd || darwin -// +build freebsd darwin +//go:build netbsd || freebsd || darwin +// +build netbsd freebsd darwin package archive diff --git a/vendor/github.com/containers/storage/pkg/archive/archive_linux.go b/vendor/github.com/containers/storage/pkg/archive/archive_linux.go index 02995d767a..eae60a3055 100644 --- a/vendor/github.com/containers/storage/pkg/archive/archive_linux.go +++ b/vendor/github.com/containers/storage/pkg/archive/archive_linux.go @@ -48,8 +48,8 @@ func (o overlayWhiteoutConverter) ConvertWrite(hdr *tar.Header, path string, fi return nil, err } if len(opaque) == 1 && opaque[0] == 'y' { - if hdr.Xattrs != nil { - delete(hdr.Xattrs, getOverlayOpaqueXattrName()) + if hdr.PAXRecords != nil { + delete(hdr.PAXRecords, PaxSchilyXattr+getOverlayOpaqueXattrName()) } // If there are no lower layers, then it can't have been deleted in this layer. if len(o.rolayers) == 0 { diff --git a/vendor/github.com/containers/storage/pkg/archive/changes_linux.go b/vendor/github.com/containers/storage/pkg/archive/changes_linux.go index f8414717b4..dc308120dc 100644 --- a/vendor/github.com/containers/storage/pkg/archive/changes_linux.go +++ b/vendor/github.com/containers/storage/pkg/archive/changes_linux.go @@ -316,7 +316,11 @@ func parseDirent(buf []byte, names []nameIno) (consumed int, newnames []nameIno) // with respect to the parent layers func OverlayChanges(layers []string, rw string) ([]Change, error) { dc := func(root, path string, fi os.FileInfo) (string, error) { - return overlayDeletedFile(layers, root, path, fi) + r, err := overlayDeletedFile(layers, root, path, fi) + if err != nil { + return "", fmt.Errorf("overlay deleted file query: %w", err) + } + return r, nil } return changes(layers, rw, dc, nil, overlayLowerContainsWhiteout) } @@ -351,7 +355,7 @@ func overlayDeletedFile(layers []string, root, path string, fi os.FileInfo) (str // If the directory isn't marked as opaque, then it's just a normal directory. opaque, err := system.Lgetxattr(filepath.Join(root, path), getOverlayOpaqueXattrName()) if err != nil { - return "", err + return "", fmt.Errorf("failed querying overlay opaque xattr: %w", err) } if len(opaque) != 1 || opaque[0] != 'y' { return "", err diff --git a/vendor/github.com/containers/storage/pkg/archive/changes_unix.go b/vendor/github.com/containers/storage/pkg/archive/changes_unix.go index 6b2e593807..25429406f8 100644 --- a/vendor/github.com/containers/storage/pkg/archive/changes_unix.go +++ b/vendor/github.com/containers/storage/pkg/archive/changes_unix.go @@ -31,9 +31,9 @@ func statDifferent(oldStat *system.StatT, oldInfo *FileInfo, newStat *system.Sta ownerChanged || oldStat.Rdev() != newStat.Rdev() || oldStat.Flags() != newStat.Flags() || + !sameFsTimeSpec(oldStat.Mtim(), newStat.Mtim()) || // Don't look at size for dirs, its not a good measure of change - (oldStat.Mode()&unix.S_IFDIR != unix.S_IFDIR && - (!sameFsTimeSpec(oldStat.Mtim(), newStat.Mtim()) || (oldStat.Size() != newStat.Size()))) { + ((oldStat.Mode()&unix.S_IFDIR != unix.S_IFDIR) && (oldStat.Size() != newStat.Size())) { return true } return false diff --git a/vendor/github.com/containers/storage/pkg/archive/filter.go b/vendor/github.com/containers/storage/pkg/archive/filter.go new file mode 100644 index 0000000000..9902a1ef57 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/archive/filter.go @@ -0,0 +1,73 @@ +package archive + +import ( + "bytes" + "fmt" + "io" + "os/exec" + "strings" + "sync" +) + +var filterPath sync.Map + +func getFilterPath(name string) string { + path, ok := filterPath.Load(name) + if ok { + return path.(string) + } + + path, err := exec.LookPath(name) + if err != nil { + path = "" + } + + filterPath.Store(name, path) + return path.(string) +} + +type errorRecordingReader struct { + r io.Reader + err error +} + +func (r *errorRecordingReader) Read(p []byte) (int, error) { + n, err := r.r.Read(p) + if r.err == nil && err != io.EOF { + r.err = err + } + return n, err +} + +// tryProcFilter tries to run the command specified in args, passing input to its stdin and returning its stdout. +// cleanup() is a caller provided function that will be called when the command finishes running, regardless of +// whether it succeeds or fails. +// If the command is not found, it returns (nil, false) and the cleanup function is not called. +func tryProcFilter(args []string, input io.Reader, cleanup func()) (io.ReadCloser, bool) { + path := getFilterPath(args[0]) + if path == "" { + return nil, false + } + + var stderrBuf bytes.Buffer + + inputWithError := &errorRecordingReader{r: input} + + r, w := io.Pipe() + cmd := exec.Command(path, args[1:]...) + cmd.Stdin = inputWithError + cmd.Stdout = w + cmd.Stderr = &stderrBuf + go func() { + err := cmd.Run() + // if there is an error reading from input, prefer to return that error + if inputWithError.err != nil { + err = inputWithError.err + } else if err != nil && stderrBuf.Len() > 0 { + err = fmt.Errorf("%s: %w", strings.TrimRight(stderrBuf.String(), "\n"), err) + } + w.CloseWithError(err) // CloseWithErr(nil) == Close() + cleanup() + }() + return r, true +} diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/archive_darwin.go b/vendor/github.com/containers/storage/pkg/chrootarchive/archive_darwin.go index f7a16e9f98..a0a578d3e7 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/archive_darwin.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/archive_darwin.go @@ -10,9 +10,11 @@ func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.TarOptions, root string, ) error { + _ = root // Restricting the operation to this root is not implemented on macOS return archive.Unpack(decompressedArchive, dest, options) } func invokePack(srcPath string, options *archive.TarOptions, root string) (io.ReadCloser, error) { + _ = root // Restricting the operation to this root is not implemented on macOS return archive.TarWithOptions(srcPath, options) } diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/archive_unix.go b/vendor/github.com/containers/storage/pkg/chrootarchive/archive_unix.go index 259f8c99a1..9907d24c30 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/archive_unix.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/archive_unix.go @@ -107,12 +107,15 @@ func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.T w.Close() if err := cmd.Wait(); err != nil { + errorOut := fmt.Errorf("unpacking failed (error: %w; output: %s)", err, output) // when `xz -d -c -q | storage-untar ...` failed on storage-untar side, // we need to exhaust `xz`'s output, otherwise the `xz` side will be // pending on write pipe forever - io.Copy(io.Discard, decompressedArchive) + if _, err := io.Copy(io.Discard, decompressedArchive); err != nil { + return fmt.Errorf("%w\nexhausting input failed (error: %w)", errorOut, err) + } - return fmt.Errorf("processing tar file(%s): %w", output, err) + return errorOut } return nil } diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_linux.go b/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_linux.go index 09ef6d5de4..5b8acdaba5 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_linux.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_linux.go @@ -19,10 +19,13 @@ import ( // Old root is removed after the call to pivot_root so it is no longer available under the new root. // This is similar to how libcontainer sets up a container's rootfs func chroot(path string) (err error) { - caps, err := capability.NewPid(0) + caps, err := capability.NewPid2(0) if err != nil { return err } + if err := caps.Load(); err != nil { + return err + } // initialize nss libraries in Glibc so that the dynamic libraries are loaded in the host // environment not in the chroot from untrusted files. diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/diff_unix.go b/vendor/github.com/containers/storage/pkg/chrootarchive/diff_unix.go index 71ed094d1e..cdcd9fdc3b 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/diff_unix.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/diff_unix.go @@ -40,11 +40,13 @@ func applyLayer() { } // We need to be able to set any perms - oldmask, err := system.Umask(0) - defer system.Umask(oldmask) + oldMask, err := system.Umask(0) if err != nil { fatal(err) } + defer func() { + _, _ = system.Umask(oldMask) // Ignore err. This can only fail with ErrNotSupportedPlatform, in which case we would have failed above. + }() if err := json.Unmarshal([]byte(os.Getenv("OPT")), &options); err != nil { fatal(err) diff --git a/vendor/github.com/containers/storage/pkg/chunked/bloom_filter.go b/vendor/github.com/containers/storage/pkg/chunked/bloom_filter_linux.go similarity index 87% rename from vendor/github.com/containers/storage/pkg/chunked/bloom_filter.go rename to vendor/github.com/containers/storage/pkg/chunked/bloom_filter_linux.go index 45d76ec30e..09e75680a8 100644 --- a/vendor/github.com/containers/storage/pkg/chunked/bloom_filter.go +++ b/vendor/github.com/containers/storage/pkg/chunked/bloom_filter_linux.go @@ -2,10 +2,15 @@ package chunked import ( "encoding/binary" + "fmt" "hash/crc32" "io" + + "github.com/docker/go-units" ) +const bloomFilterMaxLength = 100 * units.MB // max size for bloom filter + type bloomFilter struct { bitArray []uint64 k uint32 @@ -79,6 +84,10 @@ func readBloomFilter(reader io.Reader) (*bloomFilter, error) { if err := binary.Read(reader, binary.LittleEndian, &k); err != nil { return nil, err } + // sanity check + if bloomFilterLen > bloomFilterMaxLength { + return nil, fmt.Errorf("bloom filter length %d exceeds max length %d", bloomFilterLen, bloomFilterMaxLength) + } bloomFilterArray := make([]uint64, bloomFilterLen) if err := binary.Read(reader, binary.LittleEndian, &bloomFilterArray); err != nil { return nil, err diff --git a/vendor/github.com/containers/storage/pkg/chunked/cache_linux.go b/vendor/github.com/containers/storage/pkg/chunked/cache_linux.go index 34d1b92f4e..d49ddfed03 100644 --- a/vendor/github.com/containers/storage/pkg/chunked/cache_linux.go +++ b/vendor/github.com/containers/storage/pkg/chunked/cache_linux.go @@ -18,6 +18,7 @@ import ( graphdriver "github.com/containers/storage/drivers" "github.com/containers/storage/pkg/chunked/internal" "github.com/containers/storage/pkg/ioutils" + "github.com/docker/go-units" jsoniter "github.com/json-iterator/go" digest "github.com/opencontainers/go-digest" "github.com/sirupsen/logrus" @@ -34,6 +35,8 @@ const ( // https://pages.cs.wisc.edu/~cao/papers/summary-cache/node8.html bloomFilterScale = 10 // how much bigger is the bloom filter than the number of entries bloomFilterHashes = 3 // number of hash functions for the bloom filter + + maxTagsLen = 100 * units.MB // max size for tags len ) type cacheFile struct { @@ -77,7 +80,9 @@ var ( func (c *layer) release() { runtime.SetFinalizer(c, nil) if c.mmapBuffer != nil { - unix.Munmap(c.mmapBuffer) + if err := unix.Munmap(c.mmapBuffer); err != nil { + logrus.Warnf("Error Munmap: layer %q: %v", c.id, err) + } } } @@ -189,7 +194,9 @@ func (c *layersCache) loadLayerCache(layerID string) (_ *layer, errRet error) { } defer func() { if errRet != nil && mmapBuffer != nil { - unix.Munmap(mmapBuffer) + if err := unix.Munmap(mmapBuffer); err != nil { + logrus.Warnf("Error Munmap: layer %q: %v", layerID, err) + } } }() cacheFile, err := readCacheFileFromMemory(buffer) @@ -280,6 +287,13 @@ func (c *layersCache) load() error { newLayers = append(newLayers, l) continue } + + if r.ReadOnly { + // if the layer is coming from a read-only store, do not attempt + // to write to it. + continue + } + // the cache file is either not present or broken. Try to generate it from the TOC. l, err = c.createCacheFileFromTOC(r.ID) if err != nil { @@ -635,6 +649,14 @@ func readCacheFileFromMemory(bigDataBuffer []byte) (*cacheFile, error) { if err := binary.Read(bigData, binary.LittleEndian, &fnamesLen); err != nil { return nil, err } + + if tagsLen > maxTagsLen { + return nil, fmt.Errorf("tags len %d exceeds the maximum allowed size %d", tagsLen, maxTagsLen) + } + if digestLen > tagLen { + return nil, fmt.Errorf("digest len %d exceeds the tag len %d", digestLen, tagLen) + } + tags := make([]byte, tagsLen) if _, err := bigData.Read(tags); err != nil { return nil, err @@ -643,6 +665,10 @@ func readCacheFileFromMemory(bigDataBuffer []byte) (*cacheFile, error) { // retrieve the unread part of the buffer. remaining := bigDataBuffer[len(bigDataBuffer)-bigData.Len():] + if vdataLen >= uint64(len(remaining)) { + return nil, fmt.Errorf("vdata len %d exceeds the remaining buffer size %d", vdataLen, len(remaining)) + } + vdata := remaining[:vdataLen] fnames := remaining[vdataLen:] @@ -901,7 +927,7 @@ func unmarshalToc(manifest []byte) (*internal.TOC, error) { s := iter.ReadString() d, err := digest.Parse(s) if err != nil { - return nil, fmt.Errorf("Invalid tarSplitDigest %q: %w", s, err) + return nil, fmt.Errorf("invalid tarSplitDigest %q: %w", s, err) } toc.TarSplitDigest = d diff --git a/vendor/github.com/containers/storage/pkg/chunked/compression_linux.go b/vendor/github.com/containers/storage/pkg/chunked/compression_linux.go index 7b3879a994..633740a280 100644 --- a/vendor/github.com/containers/storage/pkg/chunked/compression_linux.go +++ b/vendor/github.com/containers/storage/pkg/chunked/compression_linux.go @@ -5,13 +5,16 @@ import ( "errors" "fmt" "io" + "maps" "strconv" + "time" "github.com/containers/storage/pkg/chunked/internal" "github.com/klauspost/compress/zstd" "github.com/klauspost/pgzip" digest "github.com/opencontainers/go-digest" "github.com/vbatts/tar-split/archive/tar" + expMaps "golang.org/x/exp/maps" ) var typesToTar = map[string]byte{ @@ -209,20 +212,162 @@ func readZstdChunkedManifest(blobStream ImageSourceSeekable, tocDigest digest.Di } decodedTarSplit := []byte{} - if tarSplitChunk.Offset > 0 { + if toc.TarSplitDigest != "" { + if tarSplitChunk.Offset <= 0 { + return nil, nil, nil, 0, fmt.Errorf("TOC requires a tar-split, but the %s annotation does not describe a position", internal.TarSplitInfoKey) + } tarSplit, err := readBlob(tarSplitChunk.Length) if err != nil { return nil, nil, nil, 0, err } - decodedTarSplit, err = decodeAndValidateBlob(tarSplit, tarSplitLengthUncompressed, toc.TarSplitDigest.String()) if err != nil { return nil, nil, nil, 0, fmt.Errorf("validating and decompressing tar-split: %w", err) } + // We use the TOC for creating on-disk files, but the tar-split for creating metadata + // when exporting the layer contents. Ensure the two match, otherwise local inspection of a container + // might be misleading about the exported contents. + if err := ensureTOCMatchesTarSplit(toc, decodedTarSplit); err != nil { + return nil, nil, nil, 0, fmt.Errorf("tar-split and TOC data is inconsistent: %w", err) + } + } else if tarSplitChunk.Offset > 0 { + // We must ignore the tar-split when the digest is not present in the TOC, because we can’t authenticate it. + // + // But if we asked for the chunk, now we must consume the data to not block the producer. + // Ideally the GetBlobAt API should be changed so that this is not necessary. + _, err := readBlob(tarSplitChunk.Length) + if err != nil { + return nil, nil, nil, 0, err + } } return decodedBlob, toc, decodedTarSplit, int64(manifestChunk.Offset), err } +// ensureTOCMatchesTarSplit validates that toc and tarSplit contain _exactly_ the same entries. +func ensureTOCMatchesTarSplit(toc *internal.TOC, tarSplit []byte) error { + pendingFiles := map[string]*internal.FileMetadata{} // Name -> an entry in toc.Entries + for i := range toc.Entries { + e := &toc.Entries[i] + if e.Type != internal.TypeChunk { + if _, ok := pendingFiles[e.Name]; ok { + return fmt.Errorf("TOC contains duplicate entries for path %q", e.Name) + } + pendingFiles[e.Name] = e + } + } + + if err := iterateTarSplit(tarSplit, func(hdr *tar.Header) error { + e, ok := pendingFiles[hdr.Name] + if !ok { + return fmt.Errorf("tar-split contains an entry for %q missing in TOC", hdr.Name) + } + delete(pendingFiles, hdr.Name) + expected, err := internal.NewFileMetadata(hdr) + if err != nil { + return fmt.Errorf("determining expected metadata for %q: %w", hdr.Name, err) + } + if err := ensureFileMetadataAttributesMatch(e, &expected); err != nil { + return fmt.Errorf("TOC and tar-split metadata doesn’t match: %w", err) + } + + return nil + }); err != nil { + return err + } + if len(pendingFiles) != 0 { + remaining := expMaps.Keys(pendingFiles) + if len(remaining) > 5 { + remaining = remaining[:5] // Just to limit the size of the output. + } + return fmt.Errorf("TOC contains entries not present in tar-split, incl. %q", remaining) + } + return nil +} + +// ensureTimePointersMatch ensures that a and b are equal +func ensureTimePointersMatch(a, b *time.Time) error { + // We didn’t always use “timeIfNotZero” when creating the TOC, so treat time.IsZero the same as nil. + // The archive/tar code turns time.IsZero() timestamps into an Unix timestamp of 0 when writing, but turns an Unix timestamp of 0 + // when writing into a (local-timezone) Jan 1 1970, which is not IsZero(). So, treat that the same as IsZero as well. + unixZero := time.Unix(0, 0) + if a != nil && (a.IsZero() || a.Equal(unixZero)) { + a = nil + } + if b != nil && (b.IsZero() || b.Equal(unixZero)) { + b = nil + } + switch { + case a == nil && b == nil: + return nil + case a == nil: + return fmt.Errorf("nil != %v", *b) + case b == nil: + return fmt.Errorf("%v != nil", *a) + default: + if a.Equal(*b) { + return nil + } + return fmt.Errorf("%v != %v", *a, *b) + } +} + +// ensureFileMetadataAttributesMatch ensures that a and b match in file attributes (it ignores entries relevant to locating data +// in the tar stream or matching contents) +func ensureFileMetadataAttributesMatch(a, b *internal.FileMetadata) error { + // Keep this in sync with internal.FileMetadata! + + if a.Type != b.Type { + return fmt.Errorf("mismatch of Type: %q != %q", a.Type, b.Type) + } + if a.Name != b.Name { + return fmt.Errorf("mismatch of Name: %q != %q", a.Name, b.Name) + } + if a.Linkname != b.Linkname { + return fmt.Errorf("mismatch of Linkname: %q != %q", a.Linkname, b.Linkname) + } + if a.Mode != b.Mode { + return fmt.Errorf("mismatch of Mode: %q != %q", a.Mode, b.Mode) + } + if a.Size != b.Size { + return fmt.Errorf("mismatch of Size: %q != %q", a.Size, b.Size) + } + if a.UID != b.UID { + return fmt.Errorf("mismatch of UID: %q != %q", a.UID, b.UID) + } + if a.GID != b.GID { + return fmt.Errorf("mismatch of GID: %q != %q", a.GID, b.GID) + } + + if err := ensureTimePointersMatch(a.ModTime, b.ModTime); err != nil { + return fmt.Errorf("mismatch of ModTime: %w", err) + } + if err := ensureTimePointersMatch(a.AccessTime, b.AccessTime); err != nil { + return fmt.Errorf("mismatch of AccessTime: %w", err) + } + if err := ensureTimePointersMatch(a.ChangeTime, b.ChangeTime); err != nil { + return fmt.Errorf("mismatch of ChangeTime: %w", err) + } + if a.Devmajor != b.Devmajor { + return fmt.Errorf("mismatch of Devmajor: %q != %q", a.Devmajor, b.Devmajor) + } + if a.Devminor != b.Devminor { + return fmt.Errorf("mismatch of Devminor: %q != %q", a.Devminor, b.Devminor) + } + if !maps.Equal(a.Xattrs, b.Xattrs) { + return fmt.Errorf("mismatch of Xattrs: %q != %q", a.Xattrs, b.Xattrs) + } + + // Digest is not compared + // Offset is not compared + // EndOffset is not compared + + // ChunkSize is not compared + // ChunkOffset is not compared + // ChunkDigest is not compared + // ChunkType is not compared + return nil +} + func decodeAndValidateBlob(blob []byte, lengthUncompressed uint64, expectedCompressedChecksum string) ([]byte, error) { d, err := digest.Parse(expectedCompressedChecksum) if err != nil { diff --git a/vendor/github.com/containers/storage/pkg/chunked/compressor/compressor.go b/vendor/github.com/containers/storage/pkg/chunked/compressor/compressor.go index a2fd904caf..654969749b 100644 --- a/vendor/github.com/containers/storage/pkg/chunked/compressor/compressor.go +++ b/vendor/github.com/containers/storage/pkg/chunked/compressor/compressor.go @@ -7,7 +7,6 @@ package compressor import ( "bufio" "bytes" - "encoding/base64" "io" "github.com/containers/storage/pkg/chunked/internal" @@ -369,34 +368,14 @@ func writeZstdChunkedStream(destFile io.Writer, outMetadata map[string]string, r } } - typ, err := internal.GetType(hdr.Typeflag) + mainEntry, err := internal.NewFileMetadata(hdr) if err != nil { return err } - xattrs := make(map[string]string) - for k, v := range hdr.Xattrs { - xattrs[k] = base64.StdEncoding.EncodeToString([]byte(v)) - } - entries := []internal.FileMetadata{ - { - Type: typ, - Name: hdr.Name, - Linkname: hdr.Linkname, - Mode: hdr.Mode, - Size: hdr.Size, - UID: hdr.Uid, - GID: hdr.Gid, - ModTime: &hdr.ModTime, - AccessTime: &hdr.AccessTime, - ChangeTime: &hdr.ChangeTime, - Devmajor: hdr.Devmajor, - Devminor: hdr.Devminor, - Xattrs: xattrs, - Digest: checksum, - Offset: startOffset, - EndOffset: lastOffset, - }, - } + mainEntry.Digest = checksum + mainEntry.Offset = startOffset + mainEntry.EndOffset = lastOffset + entries := []internal.FileMetadata{mainEntry} for i := 1; i < len(chunks); i++ { entries = append(entries, internal.FileMetadata{ Type: internal.TypeChunk, diff --git a/vendor/github.com/containers/storage/pkg/chunked/dump/dump.go b/vendor/github.com/containers/storage/pkg/chunked/dump/dump.go index 701b6aa53b..d98cee09de 100644 --- a/vendor/github.com/containers/storage/pkg/chunked/dump/dump.go +++ b/vendor/github.com/containers/storage/pkg/chunked/dump/dump.go @@ -1,13 +1,16 @@ +//go:build unix + package dump import ( "bufio" + "encoding/base64" "fmt" "io" "path/filepath" + "reflect" "strings" "time" - "unicode" "github.com/containers/storage/pkg/chunked/internal" "golang.org/x/sys/unix" @@ -20,20 +23,26 @@ const ( ESCAPE_LONE_DASH ) -func escaped(val string, escape int) string { +func escaped(val []byte, escape int) string { noescapeSpace := escape&NOESCAPE_SPACE != 0 escapeEqual := escape&ESCAPE_EQUAL != 0 escapeLoneDash := escape&ESCAPE_LONE_DASH != 0 - length := len(val) - - if escapeLoneDash && val == "-" { + if escapeLoneDash && len(val) == 1 && val[0] == '-' { return fmt.Sprintf("\\x%.2x", val[0]) } + // This is intended to match the C isprint API with LC_CTYPE=C + isprint := func(c byte) bool { + return c >= 32 && c < 127 + } + // This is intended to match the C isgraph API with LC_CTYPE=C + isgraph := func(c byte) bool { + return c > 32 && c < 127 + } + var result string - for i := 0; i < length; i++ { - c := val[i] + for _, c := range []byte(val) { hexEscape := false var special string @@ -50,9 +59,9 @@ func escaped(val string, escape int) string { hexEscape = escapeEqual default: if noescapeSpace { - hexEscape = !unicode.IsPrint(rune(c)) + hexEscape = !isprint(c) } else { - hexEscape = !unicode.IsPrint(rune(c)) || unicode.IsSpace(rune(c)) + hexEscape = !isgraph(c) } } @@ -67,8 +76,8 @@ func escaped(val string, escape int) string { return result } -func escapedOptional(val string, escape int) string { - if val == "" { +func escapedOptional(val []byte, escape int) string { + if len(val) == 0 { return "-" } return escaped(val, escape) @@ -104,10 +113,31 @@ func sanitizeName(name string) string { return path } -func dumpNode(out io.Writer, links map[string]int, verityDigests map[string]string, entry *internal.FileMetadata) error { +func dumpNode(out io.Writer, added map[string]*internal.FileMetadata, links map[string]int, verityDigests map[string]string, entry *internal.FileMetadata) error { path := sanitizeName(entry.Name) - if _, err := fmt.Fprint(out, escaped(path, ESCAPE_STANDARD)); err != nil { + parent := filepath.Dir(path) + if _, found := added[parent]; !found && path != "/" { + parentEntry := &internal.FileMetadata{ + Name: parent, + Type: internal.TypeDir, + Mode: 0o755, + } + if err := dumpNode(out, added, links, verityDigests, parentEntry); err != nil { + return err + } + + } + if e, found := added[path]; found { + // if the entry was already added, make sure it has the same data + if !reflect.DeepEqual(*e, *entry) { + return fmt.Errorf("entry %q already added with different data", path) + } + return nil + } + added[path] = entry + + if _, err := fmt.Fprint(out, escaped([]byte(path), ESCAPE_STANDARD)); err != nil { return err } @@ -151,7 +181,7 @@ func dumpNode(out io.Writer, links map[string]int, verityDigests map[string]stri } } - if _, err := fmt.Fprintf(out, escapedOptional(payload, ESCAPE_LONE_DASH)); err != nil { + if _, err := fmt.Fprint(out, escapedOptional([]byte(payload), ESCAPE_LONE_DASH)); err != nil { return err } @@ -165,14 +195,18 @@ func dumpNode(out io.Writer, links map[string]int, verityDigests map[string]stri return err } digest := verityDigests[payload] - if _, err := fmt.Fprintf(out, escapedOptional(digest, ESCAPE_LONE_DASH)); err != nil { + if _, err := fmt.Fprint(out, escapedOptional([]byte(digest), ESCAPE_LONE_DASH)); err != nil { return err } - for k, v := range entry.Xattrs { - name := escaped(k, ESCAPE_EQUAL) - value := escaped(v, ESCAPE_EQUAL) + for k, vEncoded := range entry.Xattrs { + v, err := base64.StdEncoding.DecodeString(vEncoded) + if err != nil { + return fmt.Errorf("decode xattr %q: %w", k, err) + } + name := escaped([]byte(k), ESCAPE_EQUAL) + value := escaped(v, ESCAPE_EQUAL) if _, err := fmt.Fprintf(out, " %s=%s", name, value); err != nil { return err } @@ -201,6 +235,7 @@ func GenerateDump(tocI interface{}, verityDigests map[string]string) (io.Reader, }() links := make(map[string]int) + added := make(map[string]*internal.FileMetadata) for _, e := range toc.Entries { if e.Linkname == "" { continue @@ -211,14 +246,14 @@ func GenerateDump(tocI interface{}, verityDigests map[string]string) (io.Reader, links[e.Linkname] = links[e.Linkname] + 1 } - if len(toc.Entries) == 0 || (sanitizeName(toc.Entries[0].Name) != "/") { + if len(toc.Entries) == 0 { root := &internal.FileMetadata{ Name: "/", Type: internal.TypeDir, Mode: 0o755, } - if err := dumpNode(w, links, verityDigests, root); err != nil { + if err := dumpNode(w, added, links, verityDigests, root); err != nil { pipeW.CloseWithError(err) closed = true return @@ -229,7 +264,7 @@ func GenerateDump(tocI interface{}, verityDigests map[string]string) (io.Reader, if e.Type == internal.TypeChunk { continue } - if err := dumpNode(w, links, verityDigests, &e); err != nil { + if err := dumpNode(w, added, links, verityDigests, &e); err != nil { pipeW.CloseWithError(err) closed = true return diff --git a/vendor/github.com/containers/storage/pkg/chunked/filesystem_linux.go b/vendor/github.com/containers/storage/pkg/chunked/filesystem_linux.go new file mode 100644 index 0000000000..4fc4864a28 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/chunked/filesystem_linux.go @@ -0,0 +1,605 @@ +package chunked + +import ( + "encoding/base64" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "sync/atomic" + "syscall" + "time" + + driversCopy "github.com/containers/storage/drivers/copy" + "github.com/containers/storage/pkg/archive" + "github.com/containers/storage/pkg/chunked/internal" + securejoin "github.com/cyphar/filepath-securejoin" + "github.com/vbatts/tar-split/archive/tar" + "golang.org/x/sys/unix" +) + +// procPathForFile returns an absolute path in /proc which +// refers to the file; see procPathForFd. +func procPathForFile(f *os.File) string { + return procPathForFd(int(f.Fd())) +} + +// procPathForFd returns an absolute path in /proc which +// refers to the file; this allows passing a file descriptor +// in places that don't accept a file descriptor. +func procPathForFd(fd int) string { + return fmt.Sprintf("/proc/self/fd/%d", fd) +} + +// fileMetadata is a wrapper around internal.FileMetadata with additional private fields that +// are not part of the TOC document. +// Type: TypeChunk entries are stored in Chunks, the primary [fileMetadata] entries never use TypeChunk. +type fileMetadata struct { + internal.FileMetadata + + // chunks stores the TypeChunk entries relevant to this entry when FileMetadata.Type == TypeReg. + chunks []*internal.FileMetadata + + // skipSetAttrs is set when the file attributes must not be + // modified, e.g. it is a hard link from a different source, + // or a composefs file. + skipSetAttrs bool +} + +func doHardLink(dirfd, srcFd int, destFile string) error { + destDir, destBase := filepath.Split(destFile) + destDirFd := dirfd + if destDir != "" && destDir != "." { + f, err := openOrCreateDirUnderRoot(dirfd, destDir, 0) + if err != nil { + return err + } + defer f.Close() + destDirFd = int(f.Fd()) + } + + doLink := func() error { + // Using unix.AT_EMPTY_PATH requires CAP_DAC_READ_SEARCH while this variant that uses + // /proc/self/fd doesn't and can be used with rootless. + srcPath := procPathForFd(srcFd) + err := unix.Linkat(unix.AT_FDCWD, srcPath, destDirFd, destBase, unix.AT_SYMLINK_FOLLOW) + if err != nil { + return &fs.PathError{Op: "linkat", Path: destFile, Err: err} + } + return nil + } + + err := doLink() + + // if the destination exists, unlink it first and try again + if err != nil && os.IsExist(err) { + if err := unix.Unlinkat(destDirFd, destBase, 0); err != nil { + return err + } + return doLink() + } + return err +} + +func copyFileContent(srcFd int, fileMetadata *fileMetadata, dirfd int, mode os.FileMode, useHardLinks bool) (*os.File, int64, error) { + destFile := fileMetadata.Name + src := procPathForFd(srcFd) + st, err := os.Stat(src) + if err != nil { + return nil, -1, fmt.Errorf("copy file content for %q: %w", destFile, err) + } + + copyWithFileRange, copyWithFileClone := true, true + + if useHardLinks { + err := doHardLink(dirfd, srcFd, destFile) + if err == nil { + // if the file was deduplicated with a hard link, skip overriding file metadata. + fileMetadata.skipSetAttrs = true + return nil, st.Size(), nil + } + } + + // If the destination file already exists, we shouldn't blow it away + dstFile, err := openFileUnderRoot(dirfd, destFile, newFileFlags, mode) + if err != nil { + return nil, -1, fmt.Errorf("open file %q under rootfs for copy: %w", destFile, err) + } + + err = driversCopy.CopyRegularToFile(src, dstFile, st, ©WithFileRange, ©WithFileClone) + if err != nil { + dstFile.Close() + return nil, -1, fmt.Errorf("copy to file %q under rootfs: %w", destFile, err) + } + return dstFile, st.Size(), nil +} + +func timeToTimespec(time *time.Time) (ts unix.Timespec) { + if time == nil || time.IsZero() { + // Return UTIME_OMIT special value + ts.Sec = 0 + ts.Nsec = ((1 << 30) - 2) + return + } + return unix.NsecToTimespec(time.UnixNano()) +} + +// chown changes the owner and group of the file at the specified path under the directory +// pointed by dirfd. +// If nofollow is true, the function will not follow symlinks. +// If path is empty, the function will change the owner and group of the file descriptor. +// absolutePath is the absolute path of the file, used only for error messages. +func chown(dirfd int, path string, uid, gid int, nofollow bool, absolutePath string) error { + var err error + flags := 0 + if nofollow { + flags |= unix.AT_SYMLINK_NOFOLLOW + } else if path == "" { + flags |= unix.AT_EMPTY_PATH + } + err = unix.Fchownat(dirfd, path, uid, gid, flags) + if err == nil { + return nil + } + if errors.Is(err, syscall.EINVAL) { + return fmt.Errorf(`potentially insufficient UIDs or GIDs available in the user namespace (requested %d:%d for %s): Check /etc/subuid and /etc/subgid if configured locally and run "podman system migrate": %w`, uid, gid, path, err) + } + return &fs.PathError{Op: "fchownat", Path: absolutePath, Err: err} +} + +// setFileAttrs sets the file attributes for file given metadata +func setFileAttrs(dirfd int, file *os.File, mode os.FileMode, metadata *fileMetadata, options *archive.TarOptions, usePath bool) error { + if metadata.skipSetAttrs { + return nil + } + if file == nil { + return errors.New("invalid file") + } + fd := int(file.Fd()) + + t, err := typeToTarType(metadata.Type) + if err != nil { + return err + } + + // If it is a symlink, force to use the path + if t == tar.TypeSymlink { + usePath = true + } + + baseName := "" + if usePath { + dirName := filepath.Dir(metadata.Name) + if dirName != "" { + parentFd, err := openFileUnderRoot(dirfd, dirName, unix.O_PATH|unix.O_DIRECTORY, 0) + if err != nil { + return err + } + defer parentFd.Close() + + dirfd = int(parentFd.Fd()) + } + baseName = filepath.Base(metadata.Name) + } + + doChown := func() error { + var err error + if usePath { + err = chown(dirfd, baseName, metadata.UID, metadata.GID, true, metadata.Name) + } else { + err = chown(fd, "", metadata.UID, metadata.GID, false, metadata.Name) + } + if options.IgnoreChownErrors { + return nil + } + return err + } + + doSetXattr := func(k string, v []byte) error { + err := unix.Fsetxattr(fd, k, v, 0) + if err != nil { + return &fs.PathError{Op: "fsetxattr", Path: metadata.Name, Err: err} + } + return nil + } + + doUtimes := func() error { + ts := []unix.Timespec{timeToTimespec(metadata.AccessTime), timeToTimespec(metadata.ModTime)} + var err error + if usePath { + err = unix.UtimesNanoAt(dirfd, baseName, ts, unix.AT_SYMLINK_NOFOLLOW) + } else { + err = unix.UtimesNanoAt(unix.AT_FDCWD, procPathForFd(fd), ts, 0) + } + if err != nil { + return &fs.PathError{Op: "utimensat", Path: metadata.Name, Err: err} + } + return nil + } + + doChmod := func() error { + var err error + op := "" + if usePath { + err = unix.Fchmodat(dirfd, baseName, uint32(mode), unix.AT_SYMLINK_NOFOLLOW) + op = "fchmodat" + } else { + err = unix.Fchmod(fd, uint32(mode)) + op = "fchmod" + } + if err != nil { + return &fs.PathError{Op: op, Path: metadata.Name, Err: err} + } + return nil + } + + if err := doChown(); err != nil { + return err + } + + canIgnore := func(err error) bool { + return err == nil || errors.Is(err, unix.ENOSYS) || errors.Is(err, unix.ENOTSUP) + } + + for k, v := range metadata.Xattrs { + if _, found := xattrsToIgnore[k]; found { + continue + } + data, err := base64.StdEncoding.DecodeString(v) + if err != nil { + return fmt.Errorf("decode xattr %q: %w", v, err) + } + if err := doSetXattr(k, data); !canIgnore(err) { + return fmt.Errorf("set xattr %s=%q for %q: %w", k, data, metadata.Name, err) + } + } + + if err := doUtimes(); !canIgnore(err) { + return err + } + + if err := doChmod(); !canIgnore(err) { + return err + } + return nil +} + +func openFileUnderRootFallback(dirfd int, name string, flags uint64, mode os.FileMode) (int, error) { + root := procPathForFd(dirfd) + + targetRoot, err := os.Readlink(root) + if err != nil { + return -1, err + } + + hasNoFollow := (flags & unix.O_NOFOLLOW) != 0 + + var fd int + // If O_NOFOLLOW is specified in the flags, then resolve only the parent directory and use the + // last component as the path to openat(). + if hasNoFollow { + dirName, baseName := filepath.Split(name) + if dirName != "" && dirName != "." { + newRoot, err := securejoin.SecureJoin(root, dirName) + if err != nil { + return -1, err + } + root = newRoot + } + + parentDirfd, err := unix.Open(root, unix.O_PATH|unix.O_CLOEXEC, 0) + if err != nil { + return -1, &fs.PathError{Op: "open", Path: root, Err: err} + } + defer unix.Close(parentDirfd) + + fd, err = unix.Openat(parentDirfd, baseName, int(flags), uint32(mode)) + if err != nil { + return -1, &fs.PathError{Op: "openat", Path: name, Err: err} + } + } else { + newPath, err := securejoin.SecureJoin(root, name) + if err != nil { + return -1, err + } + fd, err = unix.Openat(dirfd, newPath, int(flags), uint32(mode)) + if err != nil { + return -1, &fs.PathError{Op: "openat", Path: newPath, Err: err} + } + } + + target, err := os.Readlink(procPathForFd(fd)) + if err != nil { + unix.Close(fd) + return -1, err + } + + // Add an additional check to make sure the opened fd is inside the rootfs + if !strings.HasPrefix(target, targetRoot) { + unix.Close(fd) + return -1, fmt.Errorf("while resolving %q. It resolves outside the root directory", name) + } + + return fd, err +} + +func openFileUnderRootOpenat2(dirfd int, name string, flags uint64, mode os.FileMode) (int, error) { + how := unix.OpenHow{ + Flags: flags, + Mode: uint64(mode & 0o7777), + Resolve: unix.RESOLVE_IN_ROOT, + } + fd, err := unix.Openat2(dirfd, name, &how) + if err != nil { + return -1, &fs.PathError{Op: "openat2", Path: name, Err: err} + } + return fd, nil +} + +// skipOpenat2 is set when openat2 is not supported by the underlying kernel and avoid +// using it again. +var skipOpenat2 int32 + +// openFileUnderRootRaw tries to open a file using openat2 and if it is not supported fallbacks to a +// userspace lookup. +func openFileUnderRootRaw(dirfd int, name string, flags uint64, mode os.FileMode) (int, error) { + var fd int + var err error + if name == "" { + fd, err := unix.Dup(dirfd) + if err != nil { + return -1, fmt.Errorf("failed to duplicate file descriptor %d: %w", dirfd, err) + } + return fd, nil + } + if atomic.LoadInt32(&skipOpenat2) > 0 { + fd, err = openFileUnderRootFallback(dirfd, name, flags, mode) + } else { + fd, err = openFileUnderRootOpenat2(dirfd, name, flags, mode) + // If the function failed with ENOSYS, switch off the support for openat2 + // and fallback to using safejoin. + if err != nil && errors.Is(err, unix.ENOSYS) { + atomic.StoreInt32(&skipOpenat2, 1) + fd, err = openFileUnderRootFallback(dirfd, name, flags, mode) + } + } + return fd, err +} + +// openFileUnderRoot safely opens a file under the specified root directory using openat2 +// dirfd is an open file descriptor to the target checkout directory. +// name is the path to open relative to dirfd. +// flags are the flags to pass to the open syscall. +// mode specifies the mode to use for newly created files. +func openFileUnderRoot(dirfd int, name string, flags uint64, mode os.FileMode) (*os.File, error) { + fd, err := openFileUnderRootRaw(dirfd, name, flags, mode) + if err == nil { + return os.NewFile(uintptr(fd), name), nil + } + + hasCreate := (flags & unix.O_CREAT) != 0 + if errors.Is(err, unix.ENOENT) && hasCreate { + parent := filepath.Dir(name) + if parent != "" { + newDirfd, err2 := openOrCreateDirUnderRoot(dirfd, parent, 0) + if err2 == nil { + defer newDirfd.Close() + fd, err := openFileUnderRootRaw(int(newDirfd.Fd()), filepath.Base(name), flags, mode) + if err == nil { + return os.NewFile(uintptr(fd), name), nil + } + } + } + } + return nil, fmt.Errorf("open %q under the rootfs: %w", name, err) +} + +// openOrCreateDirUnderRoot safely opens a directory or create it if it is missing. +// dirfd is an open file descriptor to the target checkout directory. +// name is the path to open relative to dirfd. +// mode specifies the mode to use for newly created files. +func openOrCreateDirUnderRoot(dirfd int, name string, mode os.FileMode) (*os.File, error) { + fd, err := openFileUnderRootRaw(dirfd, name, unix.O_DIRECTORY|unix.O_RDONLY, 0) + if err == nil { + return os.NewFile(uintptr(fd), name), nil + } + + if errors.Is(err, unix.ENOENT) { + parent := filepath.Dir(name) + if parent != "" { + pDir, err2 := openOrCreateDirUnderRoot(dirfd, parent, mode) + if err2 != nil { + return nil, err + } + defer pDir.Close() + + baseName := filepath.Base(name) + + if err2 := unix.Mkdirat(int(pDir.Fd()), baseName, uint32(mode)); err2 != nil { + return nil, &fs.PathError{Op: "mkdirat", Path: name, Err: err2} + } + + fd, err = openFileUnderRootRaw(int(pDir.Fd()), baseName, unix.O_DIRECTORY|unix.O_RDONLY, 0) + if err == nil { + return os.NewFile(uintptr(fd), name), nil + } + } + } + return nil, err +} + +// appendHole creates a hole with the specified size at the open fd. +// fd is the open file descriptor. +// name is the path to use for error messages. +// size is the size of the hole to create. +func appendHole(fd int, name string, size int64) error { + off, err := unix.Seek(fd, size, unix.SEEK_CUR) + if err != nil { + return &fs.PathError{Op: "seek", Path: name, Err: err} + } + // Make sure the file size is changed. It might be the last hole and no other data written afterwards. + if err := unix.Ftruncate(fd, off); err != nil { + return &fs.PathError{Op: "ftruncate", Path: name, Err: err} + } + return nil +} + +func safeMkdir(dirfd int, mode os.FileMode, name string, metadata *fileMetadata, options *archive.TarOptions) error { + parent, base := filepath.Split(name) + parentFd := dirfd + if parent != "" && parent != "." { + parentFile, err := openOrCreateDirUnderRoot(dirfd, parent, 0) + if err != nil { + return err + } + defer parentFile.Close() + parentFd = int(parentFile.Fd()) + } + + if err := unix.Mkdirat(parentFd, base, uint32(mode)); err != nil { + if !os.IsExist(err) { + return &fs.PathError{Op: "mkdirat", Path: name, Err: err} + } + } + + file, err := openFileUnderRoot(parentFd, base, unix.O_DIRECTORY|unix.O_RDONLY, 0) + if err != nil { + return err + } + defer file.Close() + + return setFileAttrs(dirfd, file, mode, metadata, options, false) +} + +func safeLink(dirfd int, mode os.FileMode, metadata *fileMetadata, options *archive.TarOptions) error { + sourceFile, err := openFileUnderRoot(dirfd, metadata.Linkname, unix.O_PATH|unix.O_RDONLY|unix.O_NOFOLLOW, 0) + if err != nil { + return err + } + defer sourceFile.Close() + + err = doHardLink(dirfd, int(sourceFile.Fd()), metadata.Name) + if err != nil { + return err + } + + newFile, err := openFileUnderRoot(dirfd, metadata.Name, unix.O_WRONLY|unix.O_NOFOLLOW, 0) + if err != nil { + // If the target is a symlink, open the file with O_PATH. + if errors.Is(err, unix.ELOOP) { + newFile, err := openFileUnderRoot(dirfd, metadata.Name, unix.O_PATH|unix.O_NOFOLLOW, 0) + if err != nil { + return err + } + defer newFile.Close() + + return setFileAttrs(dirfd, newFile, mode, metadata, options, true) + } + return err + } + defer newFile.Close() + + return setFileAttrs(dirfd, newFile, mode, metadata, options, false) +} + +func safeSymlink(dirfd int, metadata *fileMetadata) error { + destDir, destBase := filepath.Split(metadata.Name) + destDirFd := dirfd + if destDir != "" && destDir != "." { + f, err := openOrCreateDirUnderRoot(dirfd, destDir, 0) + if err != nil { + return err + } + defer f.Close() + destDirFd = int(f.Fd()) + } + + if err := unix.Symlinkat(metadata.Linkname, destDirFd, destBase); err != nil { + return &fs.PathError{Op: "symlinkat", Path: metadata.Name, Err: err} + } + return nil +} + +type whiteoutHandler struct { + Dirfd int + Root string +} + +func (d whiteoutHandler) Setxattr(path, name string, value []byte) error { + file, err := openOrCreateDirUnderRoot(d.Dirfd, path, 0) + if err != nil { + return err + } + defer file.Close() + + if err := unix.Fsetxattr(int(file.Fd()), name, value, 0); err != nil { + return &fs.PathError{Op: "fsetxattr", Path: path, Err: err} + } + return nil +} + +func (d whiteoutHandler) Mknod(path string, mode uint32, dev int) error { + dir, base := filepath.Split(path) + dirfd := d.Dirfd + if dir != "" && dir != "." { + dir, err := openOrCreateDirUnderRoot(d.Dirfd, dir, 0) + if err != nil { + return err + } + defer dir.Close() + + dirfd = int(dir.Fd()) + } + + if err := unix.Mknodat(dirfd, base, mode, dev); err != nil { + return &fs.PathError{Op: "mknodat", Path: path, Err: err} + } + + return nil +} + +func (d whiteoutHandler) Chown(path string, uid, gid int) error { + file, err := openFileUnderRoot(d.Dirfd, path, unix.O_PATH, 0) + if err != nil { + return err + } + defer file.Close() + + return chown(int(file.Fd()), "", uid, gid, false, path) +} + +type readerAtCloser interface { + io.ReaderAt + io.Closer +} + +// seekableFile is a struct that wraps an *os.File to provide an ImageSourceSeekable. +type seekableFile struct { + reader readerAtCloser +} + +func (f *seekableFile) Close() error { + return f.reader.Close() +} + +func (f *seekableFile) GetBlobAt(chunks []ImageSourceChunk) (chan io.ReadCloser, chan error, error) { + streams := make(chan io.ReadCloser) + errs := make(chan error) + + go func() { + for _, chunk := range chunks { + streams <- io.NopCloser(io.NewSectionReader(f.reader, int64(chunk.Offset), int64(chunk.Length))) + } + close(streams) + close(errs) + }() + + return streams, errs, nil +} + +func newSeekableFile(reader readerAtCloser) *seekableFile { + return &seekableFile{reader: reader} +} diff --git a/vendor/github.com/containers/storage/pkg/chunked/internal/compression.go b/vendor/github.com/containers/storage/pkg/chunked/internal/compression.go index 5decbfb639..2a24e4bf2e 100644 --- a/vendor/github.com/containers/storage/pkg/chunked/internal/compression.go +++ b/vendor/github.com/containers/storage/pkg/chunked/internal/compression.go @@ -5,25 +5,57 @@ package internal // larger software like the graph drivers. import ( - "archive/tar" "bytes" + "encoding/base64" "encoding/binary" "fmt" "io" + "strings" "time" + "github.com/containers/storage/pkg/archive" jsoniter "github.com/json-iterator/go" "github.com/klauspost/compress/zstd" "github.com/opencontainers/go-digest" + "github.com/vbatts/tar-split/archive/tar" ) +// TOC is short for Table of Contents and is used by the zstd:chunked +// file format to effectively add an overall index into the contents +// of a tarball; it also includes file metadata. type TOC struct { - Version int `json:"version"` - Entries []FileMetadata `json:"entries"` - TarSplitDigest digest.Digest `json:"tarSplitDigest,omitempty"` + // Version is currently expected to be 1 + Version int `json:"version"` + // Entries is the list of file metadata in this TOC. + // The ordering in this array currently defaults to being the same + // as that of the tar stream; however, this should not be relied on. + Entries []FileMetadata `json:"entries"` + // TarSplitDigest is the checksum of the "tar-split" data which + // is included as a distinct skippable zstd frame before the TOC. + TarSplitDigest digest.Digest `json:"tarSplitDigest,omitempty"` } +// FileMetadata is an entry in the TOC that includes both generic file metadata +// that duplicates what can found in the tar header (and should match), but +// also special/custom content (see below). +// +// Regular files may optionally be represented as a sequence of “chunks”, +// which may be ChunkTypeData or ChunkTypeZeros (and ChunkTypeData boundaries +// are heuristically determined to increase chance of chunk matching / reuse +// similar to rsync). In that case, the regular file is represented +// as an initial TypeReg entry (with all metadata for the file as a whole) +// immediately followed by zero or more TypeChunk entries (containing only Type, +// Name and Chunk* fields); if there is at least one TypeChunk entry, the Chunk* +// fields are relevant in all of these entries, including the initial +// TypeReg one. +// +// Note that the metadata here, when fetched by a zstd:chunked aware client, +// is used instead of that in the tar stream. The contents of the tar stream +// are not used in this scenario. type FileMetadata struct { + // If you add any fields, update ensureFileMetadataMatches as well! + + // The metadata below largely duplicates that in the tar headers. Type string `json:"type"` Name string `json:"name"` Linkname string `json:"linkName,omitempty"` @@ -37,9 +69,11 @@ type FileMetadata struct { Devmajor int64 `json:"devMajor,omitempty"` Devminor int64 `json:"devMinor,omitempty"` Xattrs map[string]string `json:"xattrs,omitempty"` - Digest string `json:"digest,omitempty"` - Offset int64 `json:"offset,omitempty"` - EndOffset int64 `json:"endOffset,omitempty"` + // Digest is a hexadecimal sha256 checksum of the file contents; it + // is empty for empty files + Digest string `json:"digest,omitempty"` + Offset int64 `json:"offset,omitempty"` + EndOffset int64 `json:"endOffset,omitempty"` ChunkSize int64 `json:"chunkSize,omitempty"` ChunkOffset int64 `json:"chunkOffset,omitempty"` @@ -53,19 +87,23 @@ const ( ) const ( + // The following types correspond to regular types of entries that can + // appear in a tar archive. TypeReg = "reg" - TypeChunk = "chunk" TypeLink = "hardlink" TypeChar = "char" TypeBlock = "block" TypeDir = "dir" TypeFifo = "fifo" TypeSymlink = "symlink" + // TypeChunk is special; in zstd:chunked not only are files individually + // compressed and indexable, there is a "rolling checksum" used to compute + // "chunks" of individual file contents, that are also added to the TOC + TypeChunk = "chunk" ) var TarTypes = map[byte]string{ tar.TypeReg: TypeReg, - tar.TypeRegA: TypeReg, tar.TypeLink: TypeLink, tar.TypeChar: TypeChar, tar.TypeBlock: TypeBlock, @@ -83,11 +121,23 @@ func GetType(t byte) (string, error) { } const ( + // ManifestChecksumKey is a hexadecimal sha256 digest of the compressed manifest digest. ManifestChecksumKey = "io.github.containers.zstd-chunked.manifest-checksum" - ManifestInfoKey = "io.github.containers.zstd-chunked.manifest-position" - TarSplitInfoKey = "io.github.containers.zstd-chunked.tarsplit-position" - - TarSplitChecksumKey = "io.github.containers.zstd-chunked.tarsplit-checksum" // Deprecated: Use the TOC.TarSplitDigest field instead, this annotation is no longer read nor written. + // ManifestInfoKey is an annotation that signals the start of the TOC (manifest) + // contents which are embedded as a skippable zstd frame. It has a format of + // four decimal integers separated by `:` as follows: + // ::: + // The is ManifestTypeCRFS which should have the value `1`. + ManifestInfoKey = "io.github.containers.zstd-chunked.manifest-position" + // TarSplitInfoKey is an annotation that signals the start of the "tar-split" metadata + // contents which are embedded as a skippable zstd frame. It has a format of + // three decimal integers separated by `:` as follows: + // :: + TarSplitInfoKey = "io.github.containers.zstd-chunked.tarsplit-position" + + // TarSplitChecksumKey is no longer used and is replaced by the TOC.TarSplitDigest field instead. + // The value is retained here as a constant as a historical reference for older zstd:chunked images. + // TarSplitChecksumKey = "io.github.containers.zstd-chunked.tarsplit-checksum" // ManifestTypeCRFS is a manifest file compatible with the CRFS TOC file. ManifestTypeCRFS = 1 @@ -232,3 +282,43 @@ func footerDataToBlob(footer ZstdChunkedFooterData) []byte { return manifestDataLE } + +// timeIfNotZero returns a pointer to the time.Time if it is not zero, otherwise it returns nil. +func timeIfNotZero(t *time.Time) *time.Time { + if t == nil || t.IsZero() { + return nil + } + return t +} + +// NewFileMetadata creates a basic FileMetadata entry for hdr. +// The caller must set DigestOffset/EndOffset, and the Chunk* values, separately. +func NewFileMetadata(hdr *tar.Header) (FileMetadata, error) { + typ, err := GetType(hdr.Typeflag) + if err != nil { + return FileMetadata{}, err + } + xattrs := make(map[string]string) + for k, v := range hdr.PAXRecords { + xattrKey, ok := strings.CutPrefix(k, archive.PaxSchilyXattr) + if !ok { + continue + } + xattrs[xattrKey] = base64.StdEncoding.EncodeToString([]byte(v)) + } + return FileMetadata{ + Type: typ, + Name: hdr.Name, + Linkname: hdr.Linkname, + Mode: hdr.Mode, + Size: hdr.Size, + UID: hdr.Uid, + GID: hdr.Gid, + ModTime: timeIfNotZero(&hdr.ModTime), + AccessTime: timeIfNotZero(&hdr.AccessTime), + ChangeTime: timeIfNotZero(&hdr.ChangeTime), + Devmajor: hdr.Devmajor, + Devminor: hdr.Devminor, + Xattrs: xattrs, + }, nil +} diff --git a/vendor/github.com/containers/storage/pkg/chunked/storage_linux.go b/vendor/github.com/containers/storage/pkg/chunked/storage_linux.go index e001022cbe..403d7d5aa3 100644 --- a/vendor/github.com/containers/storage/pkg/chunked/storage_linux.go +++ b/vendor/github.com/containers/storage/pkg/chunked/storage_linux.go @@ -8,20 +8,18 @@ import ( "fmt" "hash" "io" + "io/fs" "os" "path/filepath" "reflect" "sort" "strings" "sync" - "sync/atomic" "syscall" - "time" "github.com/containerd/stargz-snapshotter/estargz" storage "github.com/containers/storage" graphdriver "github.com/containers/storage/drivers" - driversCopy "github.com/containers/storage/drivers/copy" "github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/chunked/compressor" "github.com/containers/storage/pkg/chunked/internal" @@ -29,8 +27,6 @@ import ( "github.com/containers/storage/pkg/fsverity" "github.com/containers/storage/pkg/idtools" "github.com/containers/storage/pkg/system" - "github.com/containers/storage/types" - securejoin "github.com/cyphar/filepath-securejoin" jsoniter "github.com/json-iterator/go" "github.com/klauspost/compress/zstd" "github.com/klauspost/pgzip" @@ -42,9 +38,8 @@ import ( const ( maxNumberMissingChunks = 1024 - autoMergePartsThreshold = 128 // if the gap between two ranges is below this threshold, automatically merge them. + autoMergePartsThreshold = 1024 // if the gap between two ranges is below this threshold, automatically merge them. newFileFlags = (unix.O_CREAT | unix.O_TRUNC | unix.O_EXCL | unix.O_WRONLY) - containersOverrideXattr = "user.containers.override_stat" bigDataKey = "zstd-chunked-manifest" chunkedData = "zstd-chunked-data" chunkedLayerDataKey = "zstd-chunked-layer-data" @@ -59,21 +54,6 @@ const ( copyGoRoutines = 32 ) -// fileMetadata is a wrapper around internal.FileMetadata with additional private fields that -// are not part of the TOC document. -// Type: TypeChunk entries are stored in Chunks, the primary [fileMetadata] entries never use TypeChunk. -type fileMetadata struct { - internal.FileMetadata - - // chunks stores the TypeChunk entries relevant to this entry when FileMetadata.Type == TypeReg. - chunks []*internal.FileMetadata - - // skipSetAttrs is set when the file attributes must not be - // modified, e.g. it is a hard link from a different source, - // or a composefs file. - skipSetAttrs bool -} - type compressedFileType int type chunkedDiffer struct { @@ -111,7 +91,7 @@ type chunkedDiffer struct { blobSize int64 - storeOpts *types.StoreOptions + pullOptions map[string]string useFsVerity graphdriver.DifferFsVerity fsVerityDigests map[string]string @@ -127,98 +107,7 @@ type chunkedLayerData struct { Format graphdriver.DifferOutputFormat `json:"format"` } -func timeToTimespec(time *time.Time) (ts unix.Timespec) { - if time == nil || time.IsZero() { - // Return UTIME_OMIT special value - ts.Sec = 0 - ts.Nsec = ((1 << 30) - 2) - return - } - return unix.NsecToTimespec(time.UnixNano()) -} - -func doHardLink(srcFd int, destDirFd int, destBase string) error { - doLink := func() error { - // Using unix.AT_EMPTY_PATH requires CAP_DAC_READ_SEARCH while this variant that uses - // /proc/self/fd doesn't and can be used with rootless. - srcPath := fmt.Sprintf("/proc/self/fd/%d", srcFd) - return unix.Linkat(unix.AT_FDCWD, srcPath, destDirFd, destBase, unix.AT_SYMLINK_FOLLOW) - } - - err := doLink() - - // if the destination exists, unlink it first and try again - if err != nil && os.IsExist(err) { - unix.Unlinkat(destDirFd, destBase, 0) - return doLink() - } - return err -} - -func copyFileContent(srcFd int, fileMetadata *fileMetadata, dirfd int, mode os.FileMode, useHardLinks bool) (*os.File, int64, error) { - destFile := fileMetadata.Name - src := fmt.Sprintf("/proc/self/fd/%d", srcFd) - st, err := os.Stat(src) - if err != nil { - return nil, -1, fmt.Errorf("copy file content for %q: %w", destFile, err) - } - - copyWithFileRange, copyWithFileClone := true, true - - if useHardLinks { - destDirPath := filepath.Dir(destFile) - destBase := filepath.Base(destFile) - destDir, err := openFileUnderRoot(destDirPath, dirfd, 0, mode) - if err == nil { - defer destDir.Close() - - err := doHardLink(srcFd, int(destDir.Fd()), destBase) - if err == nil { - // if the file was deduplicated with a hard link, skip overriding file metadata. - fileMetadata.skipSetAttrs = true - return nil, st.Size(), nil - } - } - } - - // If the destination file already exists, we shouldn't blow it away - dstFile, err := openFileUnderRoot(destFile, dirfd, newFileFlags, mode) - if err != nil { - return nil, -1, fmt.Errorf("open file %q under rootfs for copy: %w", destFile, err) - } - - err = driversCopy.CopyRegularToFile(src, dstFile, st, ©WithFileRange, ©WithFileClone) - if err != nil { - dstFile.Close() - return nil, -1, fmt.Errorf("copy to file %q under rootfs: %w", destFile, err) - } - return dstFile, st.Size(), nil -} - -type seekableFile struct { - file *os.File -} - -func (f *seekableFile) Close() error { - return f.file.Close() -} - -func (f *seekableFile) GetBlobAt(chunks []ImageSourceChunk) (chan io.ReadCloser, chan error, error) { - streams := make(chan io.ReadCloser) - errs := make(chan error) - - go func() { - for _, chunk := range chunks { - streams <- io.NopCloser(io.NewSectionReader(f.file, int64(chunk.Offset), int64(chunk.Length))) - } - close(streams) - close(errs) - }() - - return streams, errs, nil -} - -func convertTarToZstdChunked(destDirectory string, payload *os.File) (int64, *seekableFile, digest.Digest, map[string]string, error) { +func (c *chunkedDiffer) convertTarToZstdChunked(destDirectory string, payload *os.File) (int64, *seekableFile, digest.Digest, map[string]string, error) { diff, err := archive.DecompressStream(payload) if err != nil { return 0, nil, "", nil, err @@ -226,7 +115,7 @@ func convertTarToZstdChunked(destDirectory string, payload *os.File) (int64, *se fd, err := unix.Open(destDirectory, unix.O_TMPFILE|unix.O_RDWR|unix.O_CLOEXEC, 0o600) if err != nil { - return 0, nil, "", nil, err + return 0, nil, "", nil, &fs.PathError{Op: "open", Path: destDirectory, Err: err} } f := os.NewFile(uintptr(fd), destDirectory) @@ -240,7 +129,7 @@ func convertTarToZstdChunked(destDirectory string, payload *os.File) (int64, *se } convertedOutputDigester := digest.Canonical.Digester() - copied, err := io.Copy(io.MultiWriter(chunked, convertedOutputDigester.Hash()), diff) + copied, err := io.CopyBuffer(io.MultiWriter(chunked, convertedOutputDigester.Hash()), diff, c.copyBuffer) if err != nil { f.Close() return 0, nil, "", nil, err @@ -249,21 +138,15 @@ func convertTarToZstdChunked(destDirectory string, payload *os.File) (int64, *se f.Close() return 0, nil, "", nil, err } - is := seekableFile{ - file: f, - } - return copied, &is, convertedOutputDigester.Digest(), newAnnotations, nil + return copied, newSeekableFile(f), convertedOutputDigester.Digest(), newAnnotations, nil } // GetDiffer returns a differ than can be used with ApplyDiffWithDiffer. func GetDiffer(ctx context.Context, store storage.Store, blobDigest digest.Digest, blobSize int64, annotations map[string]string, iss ImageSourceSeekable) (graphdriver.Differ, error) { - storeOpts, err := types.DefaultStoreOptions() - if err != nil { - return nil, err - } + pullOptions := store.PullOptions() - if !parseBooleanPullOption(&storeOpts, "enable_partial_images", true) { + if !parseBooleanPullOption(pullOptions, "enable_partial_images", true) { return nil, errors.New("enable_partial_images not configured") } @@ -279,21 +162,21 @@ func GetDiffer(ctx context.Context, store storage.Store, blobDigest digest.Diges if err != nil { return nil, fmt.Errorf("parsing zstd:chunked TOC digest %q: %w", zstdChunkedTOCDigestString, err) } - return makeZstdChunkedDiffer(ctx, store, blobSize, zstdChunkedTOCDigest, annotations, iss, &storeOpts) + return makeZstdChunkedDiffer(store, blobSize, zstdChunkedTOCDigest, annotations, iss, pullOptions) } if hasEstargzTOC { estargzTOCDigest, err := digest.Parse(estargzTOCDigestString) if err != nil { return nil, fmt.Errorf("parsing estargz TOC digest %q: %w", estargzTOCDigestString, err) } - return makeEstargzChunkedDiffer(ctx, store, blobSize, estargzTOCDigest, iss, &storeOpts) + return makeEstargzChunkedDiffer(store, blobSize, estargzTOCDigest, iss, pullOptions) } - return makeConvertFromRawDiffer(ctx, store, blobDigest, blobSize, annotations, iss, &storeOpts) + return makeConvertFromRawDiffer(store, blobDigest, blobSize, iss, pullOptions) } -func makeConvertFromRawDiffer(ctx context.Context, store storage.Store, blobDigest digest.Digest, blobSize int64, annotations map[string]string, iss ImageSourceSeekable, storeOpts *types.StoreOptions) (*chunkedDiffer, error) { - if !parseBooleanPullOption(storeOpts, "convert_images", false) { +func makeConvertFromRawDiffer(store storage.Store, blobDigest digest.Digest, blobSize int64, iss ImageSourceSeekable, pullOptions map[string]string) (*chunkedDiffer, error) { + if !parseBooleanPullOption(pullOptions, "convert_images", false) { return nil, errors.New("convert_images not configured") } @@ -309,12 +192,12 @@ func makeConvertFromRawDiffer(ctx context.Context, store storage.Store, blobDige convertToZstdChunked: true, copyBuffer: makeCopyBuffer(), layersCache: layersCache, - storeOpts: storeOpts, + pullOptions: pullOptions, stream: iss, }, nil } -func makeZstdChunkedDiffer(ctx context.Context, store storage.Store, blobSize int64, tocDigest digest.Digest, annotations map[string]string, iss ImageSourceSeekable, storeOpts *types.StoreOptions) (*chunkedDiffer, error) { +func makeZstdChunkedDiffer(store storage.Store, blobSize int64, tocDigest digest.Digest, annotations map[string]string, iss ImageSourceSeekable, pullOptions map[string]string) (*chunkedDiffer, error) { manifest, toc, tarSplit, tocOffset, err := readZstdChunkedManifest(iss, tocDigest, annotations) if err != nil { return nil, fmt.Errorf("read zstd:chunked manifest: %w", err) @@ -333,14 +216,14 @@ func makeZstdChunkedDiffer(ctx context.Context, store storage.Store, blobSize in layersCache: layersCache, manifest: manifest, toc: toc, - storeOpts: storeOpts, + pullOptions: pullOptions, stream: iss, tarSplit: tarSplit, tocOffset: tocOffset, }, nil } -func makeEstargzChunkedDiffer(ctx context.Context, store storage.Store, blobSize int64, tocDigest digest.Digest, iss ImageSourceSeekable, storeOpts *types.StoreOptions) (*chunkedDiffer, error) { +func makeEstargzChunkedDiffer(store storage.Store, blobSize int64, tocDigest digest.Digest, iss ImageSourceSeekable, pullOptions map[string]string) (*chunkedDiffer, error) { manifest, tocOffset, err := readEstargzChunkedManifest(iss, blobSize, tocDigest) if err != nil { return nil, fmt.Errorf("read zstd:chunked manifest: %w", err) @@ -358,7 +241,7 @@ func makeEstargzChunkedDiffer(ctx context.Context, store storage.Store, blobSize fileType: fileTypeEstargz, layersCache: layersCache, manifest: manifest, - storeOpts: storeOpts, + pullOptions: pullOptions, stream: iss, tocOffset: tocOffset, }, nil @@ -375,15 +258,15 @@ func makeCopyBuffer() []byte { // dirfd is an open file descriptor to the destination root directory. // useHardLinks defines whether the deduplication can be performed using hard links. func copyFileFromOtherLayer(file *fileMetadata, source string, name string, dirfd int, useHardLinks bool) (bool, *os.File, int64, error) { - srcDirfd, err := unix.Open(source, unix.O_RDONLY, 0) + srcDirfd, err := unix.Open(source, unix.O_RDONLY|unix.O_CLOEXEC, 0) if err != nil { - return false, nil, 0, fmt.Errorf("open source file: %w", err) + return false, nil, 0, &fs.PathError{Op: "open", Path: source, Err: err} } defer unix.Close(srcDirfd) - srcFile, err := openFileUnderRoot(name, srcDirfd, unix.O_RDONLY, 0) + srcFile, err := openFileUnderRoot(srcDirfd, name, unix.O_RDONLY|syscall.O_CLOEXEC, 0) if err != nil { - return false, nil, 0, fmt.Errorf("open source file under target rootfs (%s): %w", name, err) + return false, nil, 0, err } defer srcFile.Close() @@ -420,7 +303,7 @@ func canDedupFileWithHardLink(file *fileMetadata, fd int, s os.FileInfo) bool { return false } - path := fmt.Sprintf("/proc/self/fd/%d", fd) + path := procPathForFd(fd) listXattrs, err := system.Llistxattr(path) if err != nil { @@ -476,7 +359,7 @@ func findFileInOSTreeRepos(file *fileMetadata, ostreeRepos []string, dirfd int, if st.Size() != file.Size { continue } - fd, err := unix.Open(sourceFile, unix.O_RDONLY|unix.O_NONBLOCK, 0) + fd, err := unix.Open(sourceFile, unix.O_RDONLY|unix.O_NONBLOCK|unix.O_CLOEXEC, 0) if err != nil { logrus.Debugf("could not open sourceFile %s: %v", sourceFile, err) return false, nil, 0, nil @@ -585,15 +468,15 @@ type missingPart struct { } func (o *originFile) OpenFile() (io.ReadCloser, error) { - srcDirfd, err := unix.Open(o.Root, unix.O_RDONLY, 0) + srcDirfd, err := unix.Open(o.Root, unix.O_RDONLY|unix.O_CLOEXEC, 0) if err != nil { - return nil, fmt.Errorf("open source file: %w", err) + return nil, &fs.PathError{Op: "open", Path: o.Root, Err: err} } defer unix.Close(srcDirfd) - srcFile, err := openFileUnderRoot(o.Path, srcDirfd, unix.O_RDONLY, 0) + srcFile, err := openFileUnderRoot(srcDirfd, o.Path, unix.O_RDONLY|unix.O_CLOEXEC, 0) if err != nil { - return nil, fmt.Errorf("open source file under target rootfs: %w", err) + return nil, err } if _, err := srcFile.Seek(o.Offset, 0); err != nil { @@ -603,253 +486,6 @@ func (o *originFile) OpenFile() (io.ReadCloser, error) { return srcFile, nil } -// setFileAttrs sets the file attributes for file given metadata -func setFileAttrs(dirfd int, file *os.File, mode os.FileMode, metadata *fileMetadata, options *archive.TarOptions, usePath bool) error { - if metadata.skipSetAttrs { - return nil - } - if file == nil || file.Fd() < 0 { - return errors.New("invalid file") - } - fd := int(file.Fd()) - - t, err := typeToTarType(metadata.Type) - if err != nil { - return err - } - - // If it is a symlink, force to use the path - if t == tar.TypeSymlink { - usePath = true - } - - baseName := "" - if usePath { - dirName := filepath.Dir(metadata.Name) - if dirName != "" { - parentFd, err := openFileUnderRoot(dirName, dirfd, unix.O_PATH|unix.O_DIRECTORY, 0) - if err != nil { - return err - } - defer parentFd.Close() - - dirfd = int(parentFd.Fd()) - } - baseName = filepath.Base(metadata.Name) - } - - doChown := func() error { - if usePath { - return unix.Fchownat(dirfd, baseName, metadata.UID, metadata.GID, unix.AT_SYMLINK_NOFOLLOW) - } - return unix.Fchown(fd, metadata.UID, metadata.GID) - } - - doSetXattr := func(k string, v []byte) error { - return unix.Fsetxattr(fd, k, v, 0) - } - - doUtimes := func() error { - ts := []unix.Timespec{timeToTimespec(metadata.AccessTime), timeToTimespec(metadata.ModTime)} - if usePath { - return unix.UtimesNanoAt(dirfd, baseName, ts, unix.AT_SYMLINK_NOFOLLOW) - } - return unix.UtimesNanoAt(unix.AT_FDCWD, fmt.Sprintf("/proc/self/fd/%d", fd), ts, 0) - } - - doChmod := func() error { - if usePath { - return unix.Fchmodat(dirfd, baseName, uint32(mode), unix.AT_SYMLINK_NOFOLLOW) - } - return unix.Fchmod(fd, uint32(mode)) - } - - if err := doChown(); err != nil { - if !options.IgnoreChownErrors { - return fmt.Errorf("chown %q to %d:%d: %w", metadata.Name, metadata.UID, metadata.GID, err) - } - } - - canIgnore := func(err error) bool { - return err == nil || errors.Is(err, unix.ENOSYS) || errors.Is(err, unix.ENOTSUP) - } - - for k, v := range metadata.Xattrs { - if _, found := xattrsToIgnore[k]; found { - continue - } - data, err := base64.StdEncoding.DecodeString(v) - if err != nil { - return fmt.Errorf("decode xattr %q: %w", v, err) - } - if err := doSetXattr(k, data); !canIgnore(err) { - return fmt.Errorf("set xattr %s=%q for %q: %w", k, data, metadata.Name, err) - } - } - - if err := doUtimes(); !canIgnore(err) { - return fmt.Errorf("set utimes for %q: %w", metadata.Name, err) - } - - if err := doChmod(); !canIgnore(err) { - return fmt.Errorf("chmod %q: %w", metadata.Name, err) - } - return nil -} - -func openFileUnderRootFallback(dirfd int, name string, flags uint64, mode os.FileMode) (int, error) { - root := fmt.Sprintf("/proc/self/fd/%d", dirfd) - - targetRoot, err := os.Readlink(root) - if err != nil { - return -1, err - } - - hasNoFollow := (flags & unix.O_NOFOLLOW) != 0 - - var fd int - // If O_NOFOLLOW is specified in the flags, then resolve only the parent directory and use the - // last component as the path to openat(). - if hasNoFollow { - dirName := filepath.Dir(name) - if dirName != "" { - newRoot, err := securejoin.SecureJoin(root, filepath.Dir(name)) - if err != nil { - return -1, err - } - root = newRoot - } - - parentDirfd, err := unix.Open(root, unix.O_PATH, 0) - if err != nil { - return -1, err - } - defer unix.Close(parentDirfd) - - fd, err = unix.Openat(parentDirfd, filepath.Base(name), int(flags), uint32(mode)) - if err != nil { - return -1, err - } - } else { - newPath, err := securejoin.SecureJoin(root, name) - if err != nil { - return -1, err - } - fd, err = unix.Openat(dirfd, newPath, int(flags), uint32(mode)) - if err != nil { - return -1, err - } - } - - target, err := os.Readlink(fmt.Sprintf("/proc/self/fd/%d", fd)) - if err != nil { - unix.Close(fd) - return -1, err - } - - // Add an additional check to make sure the opened fd is inside the rootfs - if !strings.HasPrefix(target, targetRoot) { - unix.Close(fd) - return -1, fmt.Errorf("while resolving %q. It resolves outside the root directory", name) - } - - return fd, err -} - -func openFileUnderRootOpenat2(dirfd int, name string, flags uint64, mode os.FileMode) (int, error) { - how := unix.OpenHow{ - Flags: flags, - Mode: uint64(mode & 0o7777), - Resolve: unix.RESOLVE_IN_ROOT, - } - return unix.Openat2(dirfd, name, &how) -} - -// skipOpenat2 is set when openat2 is not supported by the underlying kernel and avoid -// using it again. -var skipOpenat2 int32 - -// openFileUnderRootRaw tries to open a file using openat2 and if it is not supported fallbacks to a -// userspace lookup. -func openFileUnderRootRaw(dirfd int, name string, flags uint64, mode os.FileMode) (int, error) { - var fd int - var err error - if atomic.LoadInt32(&skipOpenat2) > 0 { - fd, err = openFileUnderRootFallback(dirfd, name, flags, mode) - } else { - fd, err = openFileUnderRootOpenat2(dirfd, name, flags, mode) - // If the function failed with ENOSYS, switch off the support for openat2 - // and fallback to using safejoin. - if err != nil && errors.Is(err, unix.ENOSYS) { - atomic.StoreInt32(&skipOpenat2, 1) - fd, err = openFileUnderRootFallback(dirfd, name, flags, mode) - } - } - return fd, err -} - -// openFileUnderRoot safely opens a file under the specified root directory using openat2 -// name is the path to open relative to dirfd. -// dirfd is an open file descriptor to the target checkout directory. -// flags are the flags to pass to the open syscall. -// mode specifies the mode to use for newly created files. -func openFileUnderRoot(name string, dirfd int, flags uint64, mode os.FileMode) (*os.File, error) { - fd, err := openFileUnderRootRaw(dirfd, name, flags, mode) - if err == nil { - return os.NewFile(uintptr(fd), name), nil - } - - hasCreate := (flags & unix.O_CREAT) != 0 - if errors.Is(err, unix.ENOENT) && hasCreate { - parent := filepath.Dir(name) - if parent != "" { - newDirfd, err2 := openOrCreateDirUnderRoot(parent, dirfd, 0) - if err2 == nil { - defer newDirfd.Close() - fd, err := openFileUnderRootRaw(int(newDirfd.Fd()), filepath.Base(name), flags, mode) - if err == nil { - return os.NewFile(uintptr(fd), name), nil - } - } - } - } - return nil, fmt.Errorf("open %q under the rootfs: %w", name, err) -} - -// openOrCreateDirUnderRoot safely opens a directory or create it if it is missing. -// name is the path to open relative to dirfd. -// dirfd is an open file descriptor to the target checkout directory. -// mode specifies the mode to use for newly created files. -func openOrCreateDirUnderRoot(name string, dirfd int, mode os.FileMode) (*os.File, error) { - fd, err := openFileUnderRootRaw(dirfd, name, unix.O_DIRECTORY|unix.O_RDONLY, mode) - if err == nil { - return os.NewFile(uintptr(fd), name), nil - } - - if errors.Is(err, unix.ENOENT) { - parent := filepath.Dir(name) - if parent != "" { - pDir, err2 := openOrCreateDirUnderRoot(parent, dirfd, mode) - if err2 != nil { - return nil, err - } - defer pDir.Close() - - baseName := filepath.Base(name) - - if err2 := unix.Mkdirat(int(pDir.Fd()), baseName, 0o755); err2 != nil { - return nil, err - } - - fd, err = openFileUnderRootRaw(int(pDir.Fd()), baseName, unix.O_DIRECTORY|unix.O_RDONLY, mode) - if err == nil { - return os.NewFile(uintptr(fd), name), nil - } - } - } - return nil, err -} - func (c *chunkedDiffer) prepareCompressedStreamToFile(partCompression compressedFileType, from io.Reader, mf *missingFileChunk) (compressedFileType, error) { switch { case partCompression == fileTypeHole: @@ -918,23 +554,14 @@ func hashHole(h hash.Hash, size int64, copyBuffer []byte) error { return nil } -// appendHole creates a hole with the specified size at the open fd. -func appendHole(fd int, size int64) error { - off, err := unix.Seek(fd, size, unix.SEEK_CUR) - if err != nil { - return err - } - // Make sure the file size is changed. It might be the last hole and no other data written afterwards. - if err := unix.Ftruncate(fd, off); err != nil { - return err - } - return nil -} - func (c *chunkedDiffer) appendCompressedStreamToFile(compression compressedFileType, destFile *destinationFile, size int64) error { switch compression { case fileTypeZstdChunked: - defer c.zstdReader.Reset(nil) + defer func() { + if err := c.zstdReader.Reset(nil); err != nil { + logrus.Warnf("release of references to the previous zstd reader failed: %v", err) + } + }() if _, err := io.CopyBuffer(destFile.to, io.LimitReader(c.zstdReader, size), c.copyBuffer); err != nil { return err } @@ -948,7 +575,7 @@ func (c *chunkedDiffer) appendCompressedStreamToFile(compression compressedFileT return err } case fileTypeHole: - if err := appendHole(int(destFile.file.Fd()), size); err != nil { + if err := appendHole(int(destFile.file.Fd()), destFile.metadata.Name, size); err != nil { return err } if destFile.hash != nil { @@ -977,7 +604,7 @@ type destinationFile struct { } func openDestinationFile(dirfd int, metadata *fileMetadata, options *archive.TarOptions, skipValidation bool, recordFsVerity recordFsVerityFunc) (*destinationFile, error) { - file, err := openFileUnderRoot(metadata.Name, dirfd, newFileFlags, 0) + file, err := openFileUnderRoot(dirfd, metadata.Name, newFileFlags, 0) if err != nil { return nil, err } @@ -1080,7 +707,7 @@ func (c *chunkedDiffer) recordFsVerity(path string, roFile *os.File) error { return nil } -func (c *chunkedDiffer) storeMissingFiles(streams chan io.ReadCloser, errs chan error, dest string, dirfd int, missingParts []missingPart, options *archive.TarOptions) (Err error) { +func (c *chunkedDiffer) storeMissingFiles(streams chan io.ReadCloser, errs chan error, dirfd int, missingParts []missingPart, options *archive.TarOptions) (Err error) { var destFile *destinationFile filesToClose := make(chan *destinationFile, 3) @@ -1294,7 +921,7 @@ func mergeMissingChunks(missingParts []missingPart, target int) []missingPart { return newMissingParts } -func (c *chunkedDiffer) retrieveMissingFiles(stream ImageSourceSeekable, dest string, dirfd int, missingParts []missingPart, options *archive.TarOptions) error { +func (c *chunkedDiffer) retrieveMissingFiles(stream ImageSourceSeekable, dirfd int, missingParts []missingPart, options *archive.TarOptions) error { var chunksToRequest []ImageSourceChunk calculateChunksToRequest := func() { @@ -1333,164 +960,9 @@ func (c *chunkedDiffer) retrieveMissingFiles(stream ImageSourceSeekable, dest st return err } - if err := c.storeMissingFiles(streams, errs, dest, dirfd, missingParts, options); err != nil { - return err - } - return nil -} - -func safeMkdir(dirfd int, mode os.FileMode, name string, metadata *fileMetadata, options *archive.TarOptions) error { - parent := filepath.Dir(name) - base := filepath.Base(name) - - parentFd := dirfd - if parent != "." { - parentFile, err := openOrCreateDirUnderRoot(parent, dirfd, 0) - if err != nil { - return err - } - defer parentFile.Close() - parentFd = int(parentFile.Fd()) - } - - if err := unix.Mkdirat(parentFd, base, uint32(mode)); err != nil { - if !os.IsExist(err) { - return fmt.Errorf("mkdir %q: %w", name, err) - } - } - - file, err := openFileUnderRoot(base, parentFd, unix.O_DIRECTORY|unix.O_RDONLY, 0) - if err != nil { - return err - } - defer file.Close() - - return setFileAttrs(dirfd, file, mode, metadata, options, false) -} - -func safeLink(dirfd int, mode os.FileMode, metadata *fileMetadata, options *archive.TarOptions) error { - sourceFile, err := openFileUnderRoot(metadata.Linkname, dirfd, unix.O_PATH|unix.O_RDONLY|unix.O_NOFOLLOW, 0) - if err != nil { - return err - } - defer sourceFile.Close() - - destDir, destBase := filepath.Dir(metadata.Name), filepath.Base(metadata.Name) - destDirFd := dirfd - if destDir != "." { - f, err := openOrCreateDirUnderRoot(destDir, dirfd, 0) - if err != nil { - return err - } - defer f.Close() - destDirFd = int(f.Fd()) - } - - err = doHardLink(int(sourceFile.Fd()), destDirFd, destBase) - if err != nil { - return fmt.Errorf("create hardlink %q pointing to %q: %w", metadata.Name, metadata.Linkname, err) - } - - newFile, err := openFileUnderRoot(metadata.Name, dirfd, unix.O_WRONLY|unix.O_NOFOLLOW, 0) - if err != nil { - // If the target is a symlink, open the file with O_PATH. - if errors.Is(err, unix.ELOOP) { - newFile, err := openFileUnderRoot(metadata.Name, dirfd, unix.O_PATH|unix.O_NOFOLLOW, 0) - if err != nil { - return err - } - defer newFile.Close() - - return setFileAttrs(dirfd, newFile, mode, metadata, options, true) - } - return err - } - defer newFile.Close() - - return setFileAttrs(dirfd, newFile, mode, metadata, options, false) -} - -func safeSymlink(dirfd int, mode os.FileMode, metadata *fileMetadata, options *archive.TarOptions) error { - destDir, destBase := filepath.Dir(metadata.Name), filepath.Base(metadata.Name) - destDirFd := dirfd - if destDir != "." { - f, err := openOrCreateDirUnderRoot(destDir, dirfd, 0) - if err != nil { - return err - } - defer f.Close() - destDirFd = int(f.Fd()) - } - - if err := unix.Symlinkat(metadata.Linkname, destDirFd, destBase); err != nil { - return fmt.Errorf("create symlink %q pointing to %q: %w", metadata.Name, metadata.Linkname, err) - } - return nil -} - -type whiteoutHandler struct { - Dirfd int - Root string -} - -func (d whiteoutHandler) Setxattr(path, name string, value []byte) error { - file, err := openOrCreateDirUnderRoot(path, d.Dirfd, 0) - if err != nil { + if err := c.storeMissingFiles(streams, errs, dirfd, missingParts, options); err != nil { return err } - defer file.Close() - - if err := unix.Fsetxattr(int(file.Fd()), name, value, 0); err != nil { - return fmt.Errorf("set xattr %s=%q for %q: %w", name, value, path, err) - } - return nil -} - -func (d whiteoutHandler) Mknod(path string, mode uint32, dev int) error { - dir := filepath.Dir(path) - base := filepath.Base(path) - - dirfd := d.Dirfd - if dir != "" { - dir, err := openOrCreateDirUnderRoot(dir, d.Dirfd, 0) - if err != nil { - return err - } - defer dir.Close() - - dirfd = int(dir.Fd()) - } - - if err := unix.Mknodat(dirfd, base, mode, dev); err != nil { - return fmt.Errorf("mknod %q: %w", path, err) - } - - return nil -} - -func checkChownErr(err error, name string, uid, gid int) error { - if errors.Is(err, syscall.EINVAL) { - return fmt.Errorf(`potentially insufficient UIDs or GIDs available in user namespace (requested %d:%d for %s): Check /etc/subuid and /etc/subgid if configured locally and run "podman system migrate": %w`, uid, gid, name, err) - } - return err -} - -func (d whiteoutHandler) Chown(path string, uid, gid int) error { - file, err := openFileUnderRoot(path, d.Dirfd, unix.O_PATH, 0) - if err != nil { - return err - } - defer file.Close() - - if err := unix.Fchownat(int(file.Fd()), "", uid, gid, unix.AT_EMPTY_PATH); err != nil { - var stat unix.Stat_t - if unix.Fstat(int(file.Fd()), &stat) == nil { - if stat.Uid == uint32(uid) && stat.Gid == uint32(gid) { - return nil - } - } - return checkChownErr(err, path, uid, gid) - } return nil } @@ -1501,8 +973,8 @@ type hardLinkToCreate struct { metadata *fileMetadata } -func parseBooleanPullOption(storeOpts *storage.StoreOptions, name string, def bool) bool { - if value, ok := storeOpts.PullOptions[name]; ok { +func parseBooleanPullOption(pullOptions map[string]string, name string, def bool) bool { + if value, ok := pullOptions[name]; ok { return strings.ToLower(value) == "true" } return def @@ -1515,10 +987,10 @@ type findAndCopyFileOptions struct { } func reopenFileReadOnly(f *os.File) (*os.File, error) { - path := fmt.Sprintf("/proc/self/fd/%d", f.Fd()) + path := procPathForFile(f) fd, err := unix.Open(path, unix.O_RDONLY|unix.O_CLOEXEC, 0) if err != nil { - return nil, err + return nil, &fs.PathError{Op: "open", Path: path, Err: err} } return os.NewFile(uintptr(fd), f.Name()), nil } @@ -1636,7 +1108,7 @@ func (c *chunkedDiffer) copyAllBlobToFile(destination *os.File) (digest.Digest, r := io.TeeReader(payload, originalRawDigester.Hash()) // copy the entire tarball and compute its digest - _, err = io.Copy(destination, r) + _, err = io.CopyBuffer(destination, r, c.copyBuffer) return originalRawDigester.Digest(), err } @@ -1654,13 +1126,14 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff // stream to use for reading the zstd:chunked or Estargz file. stream := c.stream + var compressedDigest digest.Digest var uncompressedDigest digest.Digest var convertedBlobSize int64 if c.convertToZstdChunked { fd, err := unix.Open(dest, unix.O_TMPFILE|unix.O_RDWR|unix.O_CLOEXEC, 0o600) if err != nil { - return graphdriver.DriverWithDifferOutput{}, err + return graphdriver.DriverWithDifferOutput{}, &fs.PathError{Op: "open", Path: dest, Err: err} } blobFile := os.NewFile(uintptr(fd), "blob-file") defer func() { @@ -1670,7 +1143,7 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff }() // calculate the checksum before accessing the file. - compressedDigest, err := c.copyAllBlobToFile(blobFile) + compressedDigest, err = c.copyAllBlobToFile(blobFile) if err != nil { return graphdriver.DriverWithDifferOutput{}, err } @@ -1683,7 +1156,7 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff return graphdriver.DriverWithDifferOutput{}, err } - tarSize, fileSource, diffID, annotations, err := convertTarToZstdChunked(dest, blobFile) + tarSize, fileSource, diffID, annotations, err := c.convertTarToZstdChunked(dest, blobFile) if err != nil { return graphdriver.DriverWithDifferOutput{}, err } @@ -1756,14 +1229,15 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff }, TOCDigest: c.tocDigest, UncompressedDigest: uncompressedDigest, + CompressedDigest: compressedDigest, } // When the hard links deduplication is used, file attributes are ignored because setting them // modifies the source file as well. - useHardLinks := parseBooleanPullOption(c.storeOpts, "use_hard_links", false) + useHardLinks := parseBooleanPullOption(c.pullOptions, "use_hard_links", false) // List of OSTree repositories to use for deduplication - ostreeRepos := strings.Split(c.storeOpts.PullOptions["ostree_repos"], ":") + ostreeRepos := strings.Split(c.pullOptions["ostree_repos"], ":") whiteoutConverter := archive.GetWhiteoutConverter(options.WhiteoutFormat, options.WhiteoutData) @@ -1790,16 +1264,19 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff if options.ForceMask != nil { uid, gid, mode, err := archive.GetFileOwner(dest) if err == nil { - value := fmt.Sprintf("%d:%d:0%o", uid, gid, mode) - if err := unix.Setxattr(dest, containersOverrideXattr, []byte(value), 0); err != nil { + value := idtools.Stat{ + IDs: idtools.IDPair{UID: int(uid), GID: int(gid)}, + Mode: os.FileMode(mode), + } + if err := idtools.SetContainersOverrideXattr(dest, value); err != nil { return output, err } } } - dirfd, err := unix.Open(dest, unix.O_RDONLY|unix.O_PATH, 0) + dirfd, err := unix.Open(dest, unix.O_RDONLY|unix.O_PATH|unix.O_CLOEXEC, 0) if err != nil { - return output, fmt.Errorf("cannot open %q: %w", dest, err) + return output, &fs.PathError{Op: "open", Path: dest, Err: err} } defer unix.Close(dirfd) @@ -1812,7 +1289,9 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff for _, e := range mergedEntries { d := e.Name[0:2] if _, found := createdDirs[d]; !found { - unix.Mkdirat(dirfd, d, 0o755) + if err := unix.Mkdirat(dirfd, d, 0o755); err != nil { + return output, &fs.PathError{Op: "mkdirat", Path: d, Err: err} + } createdDirs[d] = struct{}{} } } @@ -1868,11 +1347,14 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff } filesToWaitFor := 0 - for i, r := range mergedEntries { + for i := range mergedEntries { + r := &mergedEntries[i] if options.ForceMask != nil { - value := fmt.Sprintf("%d:%d:0%o", r.UID, r.GID, r.Mode&0o7777) - r.Xattrs[containersOverrideXattr] = base64.StdEncoding.EncodeToString([]byte(value)) - r.Mode = int64(*options.ForceMask) + value := idtools.FormatContainersOverrideXattr(r.UID, r.GID, int(r.Mode)) + if r.Xattrs == nil { + r.Xattrs = make(map[string]string) + } + r.Xattrs[idtools.ContainersOverrideXattr] = base64.StdEncoding.EncodeToString([]byte(value)) } mode := os.FileMode(r.Mode) @@ -1916,12 +1398,12 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff if r.Size == 0 { // Used to have a scope for cleanup. createEmptyFile := func() error { - file, err := openFileUnderRoot(r.Name, dirfd, newFileFlags, 0) + file, err := openFileUnderRoot(dirfd, r.Name, newFileFlags, 0) if err != nil { return err } defer file.Close() - if err := setFileAttrs(dirfd, file, mode, &r, options, false); err != nil { + if err := setFileAttrs(dirfd, file, mode, r, options, false); err != nil { return err } return nil @@ -1936,7 +1418,7 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff if r.Name == "" || r.Name == "." { output.RootDirMode = &mode } - if err := safeMkdir(dirfd, mode, r.Name, &r, options); err != nil { + if err := safeMkdir(dirfd, mode, r.Name, r, options); err != nil { return output, err } continue @@ -1950,12 +1432,12 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff dest: dest, dirfd: dirfd, mode: mode, - metadata: &r, + metadata: r, }) continue case tar.TypeSymlink: - if err := safeSymlink(dirfd, mode, &r, options); err != nil { + if err := safeSymlink(dirfd, r); err != nil { return output, err } continue @@ -2057,7 +1539,7 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff } // There are some missing files. Prepare a multirange request for the missing chunks. if len(missingParts) > 0 { - if err := c.retrieveMissingFiles(stream, dest, dirfd, missingParts, options); err != nil { + if err := c.retrieveMissingFiles(stream, dirfd, missingParts, options); err != nil { return output, err } } @@ -2167,13 +1649,13 @@ func (c *chunkedDiffer) mergeTocEntries(fileType compressedFileType, entries []i // validateChunkChecksum checks if the file at $root/$path[offset:chunk.ChunkSize] has the // same digest as chunk.ChunkDigest func validateChunkChecksum(chunk *internal.FileMetadata, root, path string, offset int64, copyBuffer []byte) bool { - parentDirfd, err := unix.Open(root, unix.O_PATH, 0) + parentDirfd, err := unix.Open(root, unix.O_PATH|unix.O_CLOEXEC, 0) if err != nil { return false } defer unix.Close(parentDirfd) - fd, err := openFileUnderRoot(path, parentDirfd, unix.O_RDONLY, 0) + fd, err := openFileUnderRoot(parentDirfd, path, unix.O_RDONLY|unix.O_CLOEXEC, 0) if err != nil { return false } diff --git a/vendor/github.com/containers/storage/pkg/chunked/tar_split_linux.go b/vendor/github.com/containers/storage/pkg/chunked/tar_split_linux.go new file mode 100644 index 0000000000..aeb1698db2 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/chunked/tar_split_linux.go @@ -0,0 +1,68 @@ +package chunked + +import ( + "bytes" + "fmt" + "io" + + "github.com/vbatts/tar-split/archive/tar" + "github.com/vbatts/tar-split/tar/storage" +) + +// iterateTarSplit calls handler for each tar header in tarSplit +func iterateTarSplit(tarSplit []byte, handler func(hdr *tar.Header) error) error { + // This, strictly speaking, hard-codes undocumented assumptions about how github.com/vbatts/tar-split/tar/asm.NewInputTarStream + // forms the tar-split contents. Pragmatically, NewInputTarStream should always produce storage.FileType entries at least + // for every non-empty file, which constraints it basically to the output we expect. + // + // Specifically, we assume: + // - There is a separate SegmentType entry for every tar header, but only one SegmentType entry for the full header incl. any extensions + // - (There is a FileType entry for every tar header, we ignore it) + // - Trailing padding of a file, if any, is included in the next SegmentType entry + // - At the end, there may be SegmentType entries just for the terminating zero blocks. + + unpacker := storage.NewJSONUnpacker(bytes.NewReader(tarSplit)) + for { + tsEntry, err := unpacker.Next() + if err != nil { + if err == io.EOF { + return nil + } + return fmt.Errorf("reading tar-split entries: %w", err) + } + switch tsEntry.Type { + case storage.SegmentType: + payload := tsEntry.Payload + // This is horrible, but we don’t know how much padding to skip. (It can be computed from the previous hdr.Size for non-sparse + // files, but for sparse files that is set to the logical size.) + // + // First, assume that all padding is zero bytes. + // A tar header starts with a file name, which might in principle be empty, but + // at least https://github.com/opencontainers/image-spec/blob/main/layer.md#populate-initial-filesystem suggests that + // the tar name should never be empty (it should be ".", or maybe "./"). + // + // This will cause us to skip all zero bytes in the trailing blocks, but that’s fine. + i := 0 + for i < len(payload) && payload[i] == 0 { + i++ + } + payload = payload[i:] + tr := tar.NewReader(bytes.NewReader(payload)) + hdr, err := tr.Next() + if err != nil { + if err == io.EOF { // Probably the last entry, but let’s let the unpacker drive that. + break + } + return fmt.Errorf("decoding a tar header from a tar-split entry: %w", err) + } + if err := handler(hdr); err != nil { + return err + } + + case storage.FileType: + // Nothing + default: + return fmt.Errorf("unexpected tar-split entry type %q", tsEntry.Type) + } + } +} diff --git a/vendor/github.com/containers/storage/pkg/config/config.go b/vendor/github.com/containers/storage/pkg/config/config.go index 7f49d029d5..560df3cf5e 100644 --- a/vendor/github.com/containers/storage/pkg/config/config.go +++ b/vendor/github.com/containers/storage/pkg/config/config.go @@ -75,10 +75,6 @@ type OptionsConfig struct { // Size Size string `toml:"size,omitempty"` - // RemapUIDs is a list of default UID mappings to use for layers. - RemapUIDs string `toml:"remap-uids,omitempty"` - // RemapGIDs is a list of default GID mappings to use for layers. - RemapGIDs string `toml:"remap-gids,omitempty"` // IgnoreChownErrors is a flag for whether chown errors should be // ignored when building an image. IgnoreChownErrors string `toml:"ignore_chown_errors,omitempty"` @@ -90,13 +86,6 @@ type OptionsConfig struct { // files and directories. ForceMask os.FileMode `toml:"force_mask,omitempty"` - // RemapUser is the name of one or more entries in /etc/subuid which - // should be used to set up default UID mappings. - RemapUser string `toml:"remap-user,omitempty"` - // RemapGroup is the name of one or more entries in /etc/subgid which - // should be used to set up default GID mappings. - RemapGroup string `toml:"remap-group,omitempty"` - // RootAutoUsernsUser is the name of one or more entries in /etc/subuid and // /etc/subgid which should be used to set up automatically a userns. RootAutoUsernsUser string `toml:"root-auto-userns-user,omitempty"` diff --git a/vendor/github.com/containers/storage/pkg/directory/directory_unix.go b/vendor/github.com/containers/storage/pkg/directory/directory_unix.go index 36e1bdd5fc..dd6c02a774 100644 --- a/vendor/github.com/containers/storage/pkg/directory/directory_unix.go +++ b/vendor/github.com/containers/storage/pkg/directory/directory_unix.go @@ -1,5 +1,5 @@ -//go:build linux || darwin || freebsd || solaris -// +build linux darwin freebsd solaris +//go:build !windows +// +build !windows package directory diff --git a/vendor/github.com/containers/storage/pkg/homedir/homedir_others.go b/vendor/github.com/containers/storage/pkg/homedir/homedir_others.go deleted file mode 100644 index 9057fe1b2c..0000000000 --- a/vendor/github.com/containers/storage/pkg/homedir/homedir_others.go +++ /dev/null @@ -1,38 +0,0 @@ -//go:build !linux && !darwin && !freebsd && !windows -// +build !linux,!darwin,!freebsd,!windows - -package homedir - -// Copyright 2013-2018 Docker, Inc. -// NOTE: this package has originally been copied from github.com/docker/docker. - -import ( - "errors" - "os" - "path/filepath" -) - -// GetRuntimeDir is unsupported on non-linux system. -func GetRuntimeDir() (string, error) { - return "", errors.New("homedir.GetRuntimeDir() is not supported on this system") -} - -// StickRuntimeDirContents is unsupported on non-linux system. -func StickRuntimeDirContents(files []string) ([]string, error) { - return nil, errors.New("homedir.StickRuntimeDirContents() is not supported on this system") -} - -// GetConfigHome returns XDG_CONFIG_HOME. -// GetConfigHome returns $HOME/.config and nil error if XDG_CONFIG_HOME is not set. -// -// See also https://standards.freedesktop.org/basedir-spec/latest/ar01s03.html -func GetConfigHome() (string, error) { - if xdgConfigHome := os.Getenv("XDG_CONFIG_HOME"); xdgConfigHome != "" { - return xdgConfigHome, nil - } - home := Get() - if home == "" { - return "", errors.New("could not get either XDG_CONFIG_HOME or HOME") - } - return filepath.Join(home, ".config"), nil -} diff --git a/vendor/github.com/containers/storage/pkg/idmap/idmapped_utils.go b/vendor/github.com/containers/storage/pkg/idmap/idmapped_utils.go index 87484d95b6..35b93e6263 100644 --- a/vendor/github.com/containers/storage/pkg/idmap/idmapped_utils.go +++ b/vendor/github.com/containers/storage/pkg/idmap/idmapped_utils.go @@ -10,6 +10,7 @@ import ( "syscall" "github.com/containers/storage/pkg/idtools" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) @@ -61,12 +62,20 @@ func CreateUsernsProcess(uidMaps []idtools.IDMap, gidMaps []idtools.IDMap) (int, _ = unix.Prctl(unix.PR_SET_PDEATHSIG, uintptr(unix.SIGKILL), 0, 0, 0) // just wait for the SIGKILL for { - syscall.Pause() + _ = syscall.Pause() } } cleanupFunc := func() { - unix.Kill(int(pid), unix.SIGKILL) - _, _ = unix.Wait4(int(pid), nil, 0, nil) + err1 := unix.Kill(int(pid), unix.SIGKILL) + if err1 != nil && err1 != syscall.ESRCH { + logrus.Warnf("kill process pid: %d with SIGKILL ended with error: %v", int(pid), err1) + } + if err1 != nil { + return + } + if _, err := unix.Wait4(int(pid), nil, 0, nil); err != nil { + logrus.Warnf("wait4 pid: %d ended with error: %v", int(pid), err) + } } writeMappings := func(fname string, idmap []idtools.IDMap) error { mappings := "" diff --git a/vendor/github.com/containers/storage/pkg/idtools/idtools.go b/vendor/github.com/containers/storage/pkg/idtools/idtools.go index ef5a952546..dc963481a3 100644 --- a/vendor/github.com/containers/storage/pkg/idtools/idtools.go +++ b/vendor/github.com/containers/storage/pkg/idtools/idtools.go @@ -367,21 +367,77 @@ func checkChownErr(err error, name string, uid, gid int) error { return err } +// Stat contains file states that can be overriden with ContainersOverrideXattr. +type Stat struct { + IDs IDPair + Mode os.FileMode +} + +// FormatContainersOverrideXattr will format the given uid, gid, and mode into a string +// that can be used as the value for the ContainersOverrideXattr xattr. +func FormatContainersOverrideXattr(uid, gid, mode int) string { + return fmt.Sprintf("%d:%d:0%o", uid, gid, mode&0o7777) +} + +// GetContainersOverrideXattr will get and decode ContainersOverrideXattr. +func GetContainersOverrideXattr(path string) (Stat, error) { + var stat Stat + xstat, err := system.Lgetxattr(path, ContainersOverrideXattr) + if err != nil { + return stat, err + } + + attrs := strings.Split(string(xstat), ":") + if len(attrs) != 3 { + return stat, fmt.Errorf("The number of clons in %s does not equal to 3", + ContainersOverrideXattr) + } + + value, err := strconv.ParseUint(attrs[0], 10, 32) + if err != nil { + return stat, fmt.Errorf("Failed to parse UID: %w", err) + } + + stat.IDs.UID = int(value) + + value, err = strconv.ParseUint(attrs[0], 10, 32) + if err != nil { + return stat, fmt.Errorf("Failed to parse GID: %w", err) + } + + stat.IDs.GID = int(value) + + value, err = strconv.ParseUint(attrs[2], 8, 32) + if err != nil { + return stat, fmt.Errorf("Failed to parse mode: %w", err) + } + + stat.Mode = os.FileMode(value) + + return stat, nil +} + +// SetContainersOverrideXattr will encode and set ContainersOverrideXattr. +func SetContainersOverrideXattr(path string, stat Stat) error { + value := FormatContainersOverrideXattr(stat.IDs.UID, stat.IDs.GID, int(stat.Mode)) + return system.Lsetxattr(path, ContainersOverrideXattr, []byte(value), 0) +} + func SafeChown(name string, uid, gid int) error { if runtime.GOOS == "darwin" { - var mode uint64 = 0o0700 + var mode os.FileMode = 0o0700 xstat, err := system.Lgetxattr(name, ContainersOverrideXattr) if err == nil { attrs := strings.Split(string(xstat), ":") if len(attrs) == 3 { val, err := strconv.ParseUint(attrs[2], 8, 32) if err == nil { - mode = val + mode = os.FileMode(val) } } } - value := fmt.Sprintf("%d:%d:0%o", uid, gid, mode) - if err = system.Lsetxattr(name, ContainersOverrideXattr, []byte(value), 0); err != nil { + value := Stat{IDPair{uid, gid}, mode} + if err = SetContainersOverrideXattr(name, value); err != nil { return err } uid = os.Getuid() @@ -397,19 +453,19 @@ func SafeChown(name string, uid, gid int) error { func SafeLchown(name string, uid, gid int) error { if runtime.GOOS == "darwin" { - var mode uint64 = 0o0700 + var mode os.FileMode = 0o0700 xstat, err := system.Lgetxattr(name, ContainersOverrideXattr) if err == nil { attrs := strings.Split(string(xstat), ":") if len(attrs) == 3 { val, err := strconv.ParseUint(attrs[2], 8, 32) if err == nil { - mode = val + mode = os.FileMode(val) } } } - value := fmt.Sprintf("%d:%d:0%o", uid, gid, mode) - if err = system.Lsetxattr(name, ContainersOverrideXattr, []byte(value), 0); err != nil { + value := Stat{IDPair{uid, gid}, mode} + if err = SetContainersOverrideXattr(name, value); err != nil { return err } uid = os.Getuid() diff --git a/vendor/github.com/containers/storage/pkg/ioutils/fswriters.go b/vendor/github.com/containers/storage/pkg/ioutils/fswriters.go index 2a8c85ad44..fd6addd73b 100644 --- a/vendor/github.com/containers/storage/pkg/ioutils/fswriters.go +++ b/vendor/github.com/containers/storage/pkg/ioutils/fswriters.go @@ -150,10 +150,13 @@ func (w *atomicFileWriter) complete(commit bool) (retErr error) { } defer func() { - w.closeTempFile() + err := w.closeTempFile() if retErr != nil || w.writeErr != nil { os.Remove(w.f.Name()) } + if retErr == nil { + retErr = err + } }() if commit { diff --git a/vendor/github.com/containers/storage/pkg/lockfile/lockfile.go b/vendor/github.com/containers/storage/pkg/lockfile/lockfile.go index 5101475786..25a71ac908 100644 --- a/vendor/github.com/containers/storage/pkg/lockfile/lockfile.go +++ b/vendor/github.com/containers/storage/pkg/lockfile/lockfile.go @@ -415,7 +415,9 @@ func (l *LockFile) lock(lType lockType) { // Optimization: only use the (expensive) syscall when // the counter is 0. In this case, we're either the first // reader lock or a writer lock. - lockHandle(l.fd, lType, false) + if err := lockHandle(l.fd, lType, false); err != nil { + panic(err) + } } l.lockType = lType l.locked = true @@ -426,10 +428,13 @@ func (l *LockFile) lock(lType lockType) { // command. func (l *LockFile) tryLock(lType lockType) error { var success bool + var rwMutexUnlocker func() if lType == readLock { success = l.rwMutex.TryRLock() + rwMutexUnlocker = l.rwMutex.RUnlock } else { success = l.rwMutex.TryLock() + rwMutexUnlocker = l.rwMutex.Unlock } if !success { return fmt.Errorf("resource temporarily unavailable") @@ -440,7 +445,7 @@ func (l *LockFile) tryLock(lType lockType) error { // If we're the first reference on the lock, we need to open the file again. fd, err := openLock(l.file, l.ro) if err != nil { - l.rwMutex.Unlock() + rwMutexUnlocker() return err } l.fd = fd @@ -450,7 +455,7 @@ func (l *LockFile) tryLock(lType lockType) error { // reader lock or a writer lock. if err = lockHandle(l.fd, lType, true); err != nil { closeHandle(fd) - l.rwMutex.Unlock() + rwMutexUnlocker() return err } } diff --git a/vendor/github.com/containers/storage/pkg/lockfile/lockfile_unix.go b/vendor/github.com/containers/storage/pkg/lockfile/lockfile_unix.go index 0eff003bcd..6c8399f9ea 100644 --- a/vendor/github.com/containers/storage/pkg/lockfile/lockfile_unix.go +++ b/vendor/github.com/containers/storage/pkg/lockfile/lockfile_unix.go @@ -1,5 +1,5 @@ -//go:build linux || solaris || darwin || freebsd -// +build linux solaris darwin freebsd +//go:build !windows +// +build !windows package lockfile diff --git a/vendor/github.com/containers/storage/pkg/loopback/attach_loopback.go b/vendor/github.com/containers/storage/pkg/loopback/attach_loopback.go index 40d8fd2b89..067dd7cd90 100644 --- a/vendor/github.com/containers/storage/pkg/loopback/attach_loopback.go +++ b/vendor/github.com/containers/storage/pkg/loopback/attach_loopback.go @@ -6,10 +6,12 @@ package loopback import ( "errors" "fmt" + "io/fs" "os" "syscall" "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" ) // Loopback related errors @@ -39,7 +41,7 @@ func getNextFreeLoopbackIndex() (int, error) { return index, err } -func openNextAvailableLoopback(index int, sparseName string, sparseFile *os.File) (loopFile *os.File, err error) { +func openNextAvailableLoopback(sparseName string, sparseFile *os.File) (loopFile *os.File, err error) { // Read information about the loopback file. var st syscall.Stat_t err = syscall.Fstat(int(sparseFile.Fd()), &st) @@ -48,31 +50,51 @@ func openNextAvailableLoopback(index int, sparseName string, sparseFile *os.File return nil, ErrAttachLoopbackDevice } + // upper bound to avoid infinite loop + remaining := 1000 + // Start looking for a free /dev/loop for { - target := fmt.Sprintf("/dev/loop%d", index) - index++ - - fi, err := os.Stat(target) - if err != nil { - if os.IsNotExist(err) { - logrus.Error("There are no more loopback devices available.") - } + if remaining == 0 { + logrus.Errorf("No free loopback devices available") return nil, ErrAttachLoopbackDevice } + remaining-- - if fi.Mode()&os.ModeDevice != os.ModeDevice { - logrus.Errorf("Loopback device %s is not a block device.", target) - continue + index, err := getNextFreeLoopbackIndex() + if err != nil { + logrus.Debugf("Error retrieving the next available loopback: %s", err) + return nil, err } + target := fmt.Sprintf("/dev/loop%d", index) + // OpenFile adds O_CLOEXEC loopFile, err = os.OpenFile(target, os.O_RDWR, 0o644) if err != nil { + // The kernel returns ENXIO when opening a device that is in the "deleting" or "rundown" state, so + // just treat ENXIO as if the device does not exist. + if errors.Is(err, fs.ErrNotExist) || errors.Is(err, unix.ENXIO) { + // Another process could have taken the loopback device in the meantime. So repeat + // the process with the next loopback device. + continue + } logrus.Errorf("Opening loopback device: %s", err) return nil, ErrAttachLoopbackDevice } + fi, err := loopFile.Stat() + if err != nil { + loopFile.Close() + logrus.Errorf("Stat loopback device: %s", err) + return nil, ErrAttachLoopbackDevice + } + if fi.Mode()&os.ModeDevice != os.ModeDevice { + loopFile.Close() + logrus.Errorf("Loopback device %s is not a block device.", target) + continue + } + // Try to attach to the loop file if err := ioctlLoopSetFd(loopFile.Fd(), sparseFile.Fd()); err != nil { loopFile.Close() @@ -124,14 +146,6 @@ func AttachLoopDeviceRO(sparseName string) (loop *os.File, err error) { } func attachLoopDevice(sparseName string, readonly bool) (loop *os.File, err error) { - // Try to retrieve the next available loopback device via syscall. - // If it fails, we discard error and start looping for a - // loopback from index 0. - startIndex, err := getNextFreeLoopbackIndex() - if err != nil { - logrus.Debugf("Error retrieving the next available loopback: %s", err) - } - var sparseFile *os.File // OpenFile adds O_CLOEXEC @@ -146,7 +160,7 @@ func attachLoopDevice(sparseName string, readonly bool) (loop *os.File, err erro } defer sparseFile.Close() - loopFile, err := openNextAvailableLoopback(startIndex, sparseName, sparseFile) + loopFile, err := openNextAvailableLoopback(sparseName, sparseFile) if err != nil { return nil, err } diff --git a/vendor/github.com/containers/storage/pkg/mount/mountinfo_linux.go b/vendor/github.com/containers/storage/pkg/mount/mountinfo_linux.go index cbc0299fb5..2d9e75ea10 100644 --- a/vendor/github.com/containers/storage/pkg/mount/mountinfo_linux.go +++ b/vendor/github.com/containers/storage/pkg/mount/mountinfo_linux.go @@ -1,5 +1,18 @@ package mount -import "github.com/moby/sys/mountinfo" +import ( + "fmt" + "os" -var PidMountInfo = mountinfo.PidMountInfo + "github.com/moby/sys/mountinfo" +) + +func PidMountInfo(pid int) ([]*Info, error) { + f, err := os.Open(fmt.Sprintf("/proc/%d/mountinfo", pid)) + if err != nil { + return nil, err + } + defer f.Close() + + return mountinfo.GetMountsFromReader(f, nil) +} diff --git a/vendor/github.com/containers/storage/pkg/system/syscall_unix.go b/vendor/github.com/containers/storage/pkg/system/syscall_unix.go index c4816c133e..217e2fe834 100644 --- a/vendor/github.com/containers/storage/pkg/system/syscall_unix.go +++ b/vendor/github.com/containers/storage/pkg/system/syscall_unix.go @@ -1,5 +1,5 @@ -//go:build linux || freebsd || darwin -// +build linux freebsd darwin +//go:build !windows +// +build !windows package system diff --git a/vendor/github.com/containers/storage/pkg/unshare/unshare_linux.go b/vendor/github.com/containers/storage/pkg/unshare/unshare_linux.go index a8dc1ba038..32e8d7dca3 100644 --- a/vendor/github.com/containers/storage/pkg/unshare/unshare_linux.go +++ b/vendor/github.com/containers/storage/pkg/unshare/unshare_linux.go @@ -526,8 +526,11 @@ func MaybeReexecUsingUserNamespace(evenForRoot bool) { } else { // If we have CAP_SYS_ADMIN, then we don't need to create a new namespace in order to be able // to use unshare(), so don't bother creating a new user namespace at this point. - capabilities, err := capability.NewPid(0) + capabilities, err := capability.NewPid2(0) + bailOnError(err, "Initializing a new Capabilities object of pid 0") + err = capabilities.Load() bailOnError(err, "Reading the current capabilities sets") + if capabilities.Get(capability.EFFECTIVE, capability.CAP_SYS_ADMIN) { return } @@ -587,7 +590,12 @@ func MaybeReexecUsingUserNamespace(evenForRoot bool) { cmd.Hook = func(int) error { go func() { for receivedSignal := range interrupted { - cmd.Cmd.Process.Signal(receivedSignal) + if err := cmd.Cmd.Process.Signal(receivedSignal); err != nil { + logrus.Warnf( + "Failed to send a signal '%d' to the Process (PID: %d): %v", + receivedSignal, cmd.Cmd.Process.Pid, err, + ) + } } }() return nil diff --git a/vendor/github.com/containers/storage/storage.conf b/vendor/github.com/containers/storage/storage.conf index 0f8d1f0249..94a9b36d48 100644 --- a/vendor/github.com/containers/storage/storage.conf +++ b/vendor/github.com/containers/storage/storage.conf @@ -19,6 +19,10 @@ driver = "overlay" # Temporary storage location runroot = "/run/containers/storage" +# Priority list for the storage drivers that will be tested one +# after the other to pick the storage driver if it is not defined. +# driver_priority = ["overlay", "btrfs"] + # Primary Read/Write location of container storage # When changing the graphroot location on an SELINUX system, you must # ensure the labeling matches the default locations labels with the @@ -77,28 +81,6 @@ additionalimagestores = [ # operation so it is not enabled by default. pull_options = {enable_partial_images = "true", use_hard_links = "false", ostree_repos=""} -# Remap-UIDs/GIDs is the mapping from UIDs/GIDs as they should appear inside of -# a container, to the UIDs/GIDs as they should appear outside of the container, -# and the length of the range of UIDs/GIDs. Additional mapped sets can be -# listed and will be heeded by libraries, but there are limits to the number of -# mappings which the kernel will allow when you later attempt to run a -# container. -# -# remap-uids = "0:1668442479:65536" -# remap-gids = "0:1668442479:65536" - -# Remap-User/Group is a user name which can be used to look up one or more UID/GID -# ranges in the /etc/subuid or /etc/subgid file. Mappings are set up starting -# with an in-container ID of 0 and then a host-level ID taken from the lowest -# range that matches the specified name, and using the length of that range. -# Additional ranges are then assigned, using the ranges which specify the -# lowest host-level IDs first, to the lowest not-yet-mapped in-container ID, -# until all of the entries have been used for maps. This setting overrides the -# Remap-UIDs/GIDs setting. -# -# remap-user = "containers" -# remap-group = "containers" - # Root-auto-userns-user is a user name which can be used to look up one or more UID/GID # ranges in the /etc/subuid and /etc/subgid file. These ranges will be partitioned # to containers configured to create automatically a user namespace. Containers diff --git a/vendor/github.com/containers/storage/storage.conf-freebsd b/vendor/github.com/containers/storage/storage.conf-freebsd index 43278a1fc5..5f421e0c13 100644 --- a/vendor/github.com/containers/storage/storage.conf-freebsd +++ b/vendor/github.com/containers/storage/storage.conf-freebsd @@ -39,27 +39,6 @@ graphroot = "/var/db/containers/storage" additionalimagestores = [ ] -# Remap-UIDs/GIDs is the mapping from UIDs/GIDs as they should appear inside of -# a container, to the UIDs/GIDs as they should appear outside of the container, -# and the length of the range of UIDs/GIDs. Additional mapped sets can be -# listed and will be heeded by libraries, but there are limits to the number of -# mappings which the kernel will allow when you later attempt to run a -# container. -# -# remap-uids = 0:1668442479:65536 -# remap-gids = 0:1668442479:65536 - -# Remap-User/Group is a user name which can be used to look up one or more UID/GID -# ranges in the /etc/subuid or /etc/subgid file. Mappings are set up starting -# with an in-container ID of 0 and then a host-level ID taken from the lowest -# range that matches the specified name, and using the length of that range. -# Additional ranges are then assigned, using the ranges which specify the -# lowest host-level IDs first, to the lowest not-yet-mapped in-container ID, -# until all of the entries have been used for maps. -# -# remap-user = "containers" -# remap-group = "containers" - # Root-auto-userns-user is a user name which can be used to look up one or more UID/GID # ranges in the /etc/subuid and /etc/subgid file. These ranges will be partitioned # to containers configured to create automatically a user namespace. Containers diff --git a/vendor/github.com/containers/storage/store.go b/vendor/github.com/containers/storage/store.go index 957675ba46..efcecfae87 100644 --- a/vendor/github.com/containers/storage/store.go +++ b/vendor/github.com/containers/storage/store.go @@ -1088,8 +1088,6 @@ func (s *store) createGraphDriverLocked() (drivers.Driver, error) { RunRoot: s.runRoot, DriverPriority: s.graphDriverPriority, DriverOptions: s.graphOptions, - UIDMaps: s.uidMap, - GIDMaps: s.gidMap, } return drivers.New(s.graphDriverName, config) } @@ -1437,7 +1435,9 @@ func (s *store) canUseShifting(uidmap, gidmap []idtools.IDMap) bool { return true } -// putLayer requires the rlstore, rlstores, as well as s.containerStore (even if not an argument to this function) to be locked for write. +// On entry: +// - rlstore must be locked for writing +// - rlstores MUST NOT be locked func (s *store) putLayer(rlstore rwLayerStore, rlstores []roLayerStore, id, parent string, names []string, mountLabel string, writeable bool, lOptions *LayerOptions, diff io.Reader, slo *stagedLayerOptions) (*Layer, int64, error) { var parentLayer *Layer var options LayerOptions @@ -1474,6 +1474,11 @@ func (s *store) putLayer(rlstore rwLayerStore, rlstores []roLayerStore, id, pare return nil, -1, ErrLayerUnknown } parentLayer = ilayer + + if err := s.containerStore.startWriting(); err != nil { + return nil, -1, err + } + defer s.containerStore.stopWriting() containers, err := s.containerStore.Containers() if err != nil { return nil, -1, err @@ -1490,6 +1495,13 @@ func (s *store) putLayer(rlstore rwLayerStore, rlstores []roLayerStore, id, pare gidMap = ilayer.GIDMap } } else { + // FIXME? It’s unclear why we are holding containerStore locked here at all + // (and because we are not modifying it, why it is a write lock, not a read lock). + if err := s.containerStore.startWriting(); err != nil { + return nil, -1, err + } + defer s.containerStore.stopWriting() + if !options.HostUIDMapping && len(options.UIDMap) == 0 { uidMap = s.uidMap } @@ -1497,23 +1509,17 @@ func (s *store) putLayer(rlstore rwLayerStore, rlstores []roLayerStore, id, pare gidMap = s.gidMap } } - layerOptions := LayerOptions{ - OriginalDigest: options.OriginalDigest, - OriginalSize: options.OriginalSize, - UncompressedDigest: options.UncompressedDigest, - Flags: options.Flags, - } if s.canUseShifting(uidMap, gidMap) { - layerOptions.IDMappingOptions = types.IDMappingOptions{HostUIDMapping: true, HostGIDMapping: true, UIDMap: nil, GIDMap: nil} + options.IDMappingOptions = types.IDMappingOptions{HostUIDMapping: true, HostGIDMapping: true, UIDMap: nil, GIDMap: nil} } else { - layerOptions.IDMappingOptions = types.IDMappingOptions{ + options.IDMappingOptions = types.IDMappingOptions{ HostUIDMapping: options.HostUIDMapping, HostGIDMapping: options.HostGIDMapping, UIDMap: copyIDMap(uidMap), GIDMap: copyIDMap(gidMap), } } - return rlstore.create(id, parentLayer, names, mountLabel, nil, &layerOptions, writeable, diff, slo) + return rlstore.create(id, parentLayer, names, mountLabel, nil, &options, writeable, diff, slo) } func (s *store) PutLayer(id, parent string, names []string, mountLabel string, writeable bool, lOptions *LayerOptions, diff io.Reader) (*Layer, int64, error) { @@ -1525,10 +1531,6 @@ func (s *store) PutLayer(id, parent string, names []string, mountLabel string, w return nil, -1, err } defer rlstore.stopWriting() - if err := s.containerStore.startWriting(); err != nil { - return nil, -1, err - } - defer s.containerStore.stopWriting() return s.putLayer(rlstore, rlstores, id, parent, names, mountLabel, writeable, lOptions, diff, nil) } @@ -2844,7 +2846,7 @@ func (s *store) mount(id string, options drivers.MountOpts) (string, error) { exists := store.Exists(id) store.stopReading() if exists { - return "", fmt.Errorf("mounting read/only store images is not allowed: %w", ErrLayerUnknown) + return "", fmt.Errorf("mounting read/only store images is not allowed: %w", ErrStoreIsReadOnly) } } @@ -2928,14 +2930,40 @@ func (s *store) Unmount(id string, force bool) (bool, error) { } func (s *store) Changes(from, to string) ([]archive.Change, error) { - if res, done, err := readAllLayerStores(s, func(store roLayerStore) ([]archive.Change, bool, error) { + // NaiveDiff could cause mounts to happen without a lock, so be safe + // and treat the .Diff operation as a Mount. + // We need to make sure the home mount is present when the Mount is done, which happens by possibly reinitializing the graph driver + // in startUsingGraphDriver(). + if err := s.startUsingGraphDriver(); err != nil { + return nil, err + } + defer s.stopUsingGraphDriver() + + rlstore, lstores, err := s.bothLayerStoreKindsLocked() + if err != nil { + return nil, err + } + if err := rlstore.startWriting(); err != nil { + return nil, err + } + if rlstore.Exists(to) { + res, err := rlstore.Changes(from, to) + rlstore.stopWriting() + return res, err + } + rlstore.stopWriting() + + for _, s := range lstores { + store := s + if err := store.startReading(); err != nil { + return nil, err + } if store.Exists(to) { res, err := store.Changes(from, to) - return res, true, err + store.stopReading() + return res, err } - return nil, false, nil - }); done { - return res, err + store.stopReading() } return nil, ErrLayerUnknown } @@ -2966,12 +2994,30 @@ func (s *store) Diff(from, to string, options *DiffOptions) (io.ReadCloser, erro } defer s.stopUsingGraphDriver() - layerStores, err := s.allLayerStoresLocked() + rlstore, lstores, err := s.bothLayerStoreKindsLocked() if err != nil { return nil, err } - for _, s := range layerStores { + if err := rlstore.startWriting(); err != nil { + return nil, err + } + if rlstore.Exists(to) { + rc, err := rlstore.Diff(from, to, options) + if rc != nil && err == nil { + wrapped := ioutils.NewReadCloserWrapper(rc, func() error { + err := rc.Close() + rlstore.stopWriting() + return err + }) + return wrapped, nil + } + rlstore.stopWriting() + return rc, err + } + rlstore.stopWriting() + + for _, s := range lstores { store := s if err := store.startReading(); err != nil { return nil, err @@ -3009,16 +3055,14 @@ func (s *store) ApplyStagedLayer(args ApplyStagedLayerOptions) (*Layer, error) { return layer, err } if err == nil { + // This code path exists only for cmd/containers/storage.applyDiffUsingStagingDirectory; we have tests that + // assume layer creation and applying a staged layer are separate steps. Production pull code always uses the + // other path, where layer creation is atomic. return layer, rlstore.applyDiffFromStagingDirectory(args.ID, args.DiffOutput, args.DiffOptions) } // if the layer doesn't exist yet, try to create it. - if err := s.containerStore.startWriting(); err != nil { - return nil, err - } - defer s.containerStore.stopWriting() - slo := stagedLayerOptions{ DiffOutput: args.DiffOutput, DiffOptions: args.DiffOptions, diff --git a/vendor/github.com/containers/storage/types/options.go b/vendor/github.com/containers/storage/types/options.go index 03e5f7ab64..f1a900b8d8 100644 --- a/vendor/github.com/containers/storage/types/options.go +++ b/vendor/github.com/containers/storage/types/options.go @@ -352,7 +352,7 @@ func getRootlessStorageOpts(systemOpts StoreOptions) (StoreOptions, error) { } if opts.GraphDriverName == "" { - if canUseRootlessOverlay(opts.GraphRoot, opts.RunRoot) { + if canUseRootlessOverlay() { opts.GraphDriverName = overlayDriver } else { opts.GraphDriverName = "vfs" @@ -481,33 +481,6 @@ func ReloadConfigurationFile(configFile string, storeOptions *StoreOptions) erro if config.Storage.Options.MountOpt != "" { storeOptions.GraphDriverOptions = append(storeOptions.GraphDriverOptions, fmt.Sprintf("%s.mountopt=%s", config.Storage.Driver, config.Storage.Options.MountOpt)) } - - uidmap, err := idtools.ParseIDMap([]string{config.Storage.Options.RemapUIDs}, "remap-uids") - if err != nil { - return err - } - gidmap, err := idtools.ParseIDMap([]string{config.Storage.Options.RemapGIDs}, "remap-gids") - if err != nil { - return err - } - - if config.Storage.Options.RemapUser != "" && config.Storage.Options.RemapGroup == "" { - config.Storage.Options.RemapGroup = config.Storage.Options.RemapUser - } - if config.Storage.Options.RemapGroup != "" && config.Storage.Options.RemapUser == "" { - config.Storage.Options.RemapUser = config.Storage.Options.RemapGroup - } - if config.Storage.Options.RemapUser != "" && config.Storage.Options.RemapGroup != "" { - mappings, err := idtools.NewIDMappings(config.Storage.Options.RemapUser, config.Storage.Options.RemapGroup) - if err != nil { - logrus.Warningf("Error initializing ID mappings for %s:%s %v\n", config.Storage.Options.RemapUser, config.Storage.Options.RemapGroup, err) - return err - } - uidmap = mappings.UIDs() - gidmap = mappings.GIDs() - } - storeOptions.UIDMap = uidmap - storeOptions.GIDMap = gidmap storeOptions.RootAutoNsUser = config.Storage.Options.RootAutoUsernsUser if config.Storage.Options.AutoUsernsMinSize > 0 { storeOptions.AutoNsMinSize = config.Storage.Options.AutoUsernsMinSize diff --git a/vendor/github.com/containers/storage/types/options_freebsd.go b/vendor/github.com/containers/storage/types/options_bsd.go similarity index 90% rename from vendor/github.com/containers/storage/types/options_freebsd.go rename to vendor/github.com/containers/storage/types/options_bsd.go index be2bc2f27d..040fdc797d 100644 --- a/vendor/github.com/containers/storage/types/options_freebsd.go +++ b/vendor/github.com/containers/storage/types/options_bsd.go @@ -1,3 +1,5 @@ +//go:build freebsd || netbsd + package types const ( @@ -14,6 +16,6 @@ var ( ) // canUseRootlessOverlay returns true if the overlay driver can be used for rootless containers -func canUseRootlessOverlay(home, runhome string) bool { +func canUseRootlessOverlay() bool { return false } diff --git a/vendor/github.com/containers/storage/types/options_darwin.go b/vendor/github.com/containers/storage/types/options_darwin.go index 3eecc2b827..27ba6a061d 100644 --- a/vendor/github.com/containers/storage/types/options_darwin.go +++ b/vendor/github.com/containers/storage/types/options_darwin.go @@ -11,6 +11,6 @@ const ( var defaultOverrideConfigFile = "/etc/containers/storage.conf" // canUseRootlessOverlay returns true if the overlay driver can be used for rootless containers -func canUseRootlessOverlay(home, runhome string) bool { +func canUseRootlessOverlay() bool { return false } diff --git a/vendor/github.com/containers/storage/types/options_linux.go b/vendor/github.com/containers/storage/types/options_linux.go index a28e82883c..09cbae54b6 100644 --- a/vendor/github.com/containers/storage/types/options_linux.go +++ b/vendor/github.com/containers/storage/types/options_linux.go @@ -22,7 +22,7 @@ var ( ) // canUseRootlessOverlay returns true if the overlay driver can be used for rootless containers -func canUseRootlessOverlay(home, runhome string) bool { +func canUseRootlessOverlay() bool { // we check first for fuse-overlayfs since it is cheaper. if path, _ := exec.LookPath("fuse-overlayfs"); path != "" { return true diff --git a/vendor/github.com/containers/storage/types/options_windows.go b/vendor/github.com/containers/storage/types/options_windows.go index c1bea9fac0..99a67ff210 100644 --- a/vendor/github.com/containers/storage/types/options_windows.go +++ b/vendor/github.com/containers/storage/types/options_windows.go @@ -14,6 +14,6 @@ var ( ) // canUseRootlessOverlay returns true if the overlay driver can be used for rootless containers -func canUseRootlessOverlay(home, runhome string) bool { +func canUseRootlessOverlay() bool { return false } diff --git a/vendor/github.com/containers/storage/types/storage_test.conf b/vendor/github.com/containers/storage/types/storage_test.conf index c42d33fb9e..761b3a795d 100644 --- a/vendor/github.com/containers/storage/types/storage_test.conf +++ b/vendor/github.com/containers/storage/types/storage_test.conf @@ -25,16 +25,6 @@ rootless_storage_path = "$HOME/$UID/containers/storage" additionalimagestores = [ ] -# Remap-UIDs/GIDs is the mapping from UIDs/GIDs as they should appear inside of -# a container, to the UIDs/GIDs as they should appear outside of the container, -# and the length of the range of UIDs/GIDs. Additional mapped sets can be -# listed and will be heeded by libraries, but there are limits to the number of -# mappings which the kernel will allow when you later attempt to run a -# container. -# -remap-uids = "0:1000000000:30000" -remap-gids = "0:1500000000:60000" - [storage.options.overlay] # mountopt specifies comma separated list of extra mount options diff --git a/vendor/github.com/containers/storage/types/utils.go b/vendor/github.com/containers/storage/types/utils.go index b313a47288..73fcd2405e 100644 --- a/vendor/github.com/containers/storage/types/utils.go +++ b/vendor/github.com/containers/storage/types/utils.go @@ -66,7 +66,10 @@ func reloadConfigurationFileIfNeeded(configFile string, storeOptions *StoreOptio return } - ReloadConfigurationFile(configFile, storeOptions) + if err := ReloadConfigurationFile(configFile, storeOptions); err != nil { + logrus.Warningf("Failed to reload %q %v\n", configFile, err) + return + } prevReloadConfig.storeOptions = storeOptions prevReloadConfig.mod = mtime diff --git a/vendor/github.com/cyphar/filepath-securejoin/CHANGELOG.md b/vendor/github.com/cyphar/filepath-securejoin/CHANGELOG.md new file mode 100644 index 0000000000..7436896e13 --- /dev/null +++ b/vendor/github.com/cyphar/filepath-securejoin/CHANGELOG.md @@ -0,0 +1,138 @@ +# Changelog # +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/) +and this project adheres to [Semantic Versioning](http://semver.org/). + +## [Unreleased] ## + +## [0.3.1] - 2024-07-23 ## + +### Changed ### +- By allowing `Open(at)InRoot` to opt-out of the extra work done by `MkdirAll` + to do the necessary "partial lookups", `Open(at)InRoot` now does less work + for both implementations (resulting in a many-fold decrease in the number of + operations for `openat2`, and a modest improvement for non-`openat2`) and is + far more guaranteed to match the correct `openat2(RESOLVE_IN_ROOT)` + behaviour. +- We now use `readlinkat(fd, "")` where possible. For `Open(at)InRoot` this + effectively just means that we no longer risk getting spurious errors during + rename races. However, for our hardened procfs handler, this in theory should + prevent mount attacks from tricking us when doing magic-link readlinks (even + when using the unsafe host `/proc` handle). Unfortunately `Reopen` is still + potentially vulnerable to those kinds of somewhat-esoteric attacks. + + Technically this [will only work on post-2.6.39 kernels][linux-readlinkat-emptypath] + but it seems incredibly unlikely anyone is using `filepath-securejoin` on a + pre-2011 kernel. + +### Fixed ### +- Several improvements were made to the errors returned by `Open(at)InRoot` and + `MkdirAll` when dealing with invalid paths under the emulated (ie. + non-`openat2`) implementation. Previously, some paths would return the wrong + error (`ENOENT` when the last component was a non-directory), and other paths + would be returned as though they were acceptable (trailing-slash components + after a non-directory would be ignored by `Open(at)InRoot`). + + These changes were done to match `openat2`'s behaviour and purely is a + consistency fix (most users are going to be using `openat2` anyway). + +[linux-readlinkat-emptypath]: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=65cfc6722361570bfe255698d9cd4dccaf47570d + +## [0.3.0] - 2024-07-11 ## + +### Added ### +- A new set of `*os.File`-based APIs have been added. These are adapted from + [libpathrs][] and we strongly suggest using them if possible (as they provide + far more protection against attacks than `SecureJoin`): + + - `Open(at)InRoot` resolves a path inside a rootfs and returns an `*os.File` + handle to the path. Note that the handle returned is an `O_PATH` handle, + which cannot be used for reading or writing (as well as some other + operations -- [see open(2) for more details][open.2]) + + - `Reopen` takes an `O_PATH` file handle and safely re-opens it to upgrade + it to a regular handle. This can also be used with non-`O_PATH` handles, + but `O_PATH` is the most obvious application. + + - `MkdirAll` is an implementation of `os.MkdirAll` that is safe to use to + create a directory tree within a rootfs. + + As these are new APIs, they may change in the future. However, they should be + safe to start migrating to as we have extensive tests ensuring they behave + correctly and are safe against various races and other attacks. + +[libpathrs]: https://github.com/openSUSE/libpathrs +[open.2]: https://www.man7.org/linux/man-pages/man2/open.2.html + +## [0.2.5] - 2024-05-03 ## + +### Changed ### +- Some minor changes were made to how lexical components (like `..` and `.`) + are handled during path generation in `SecureJoin`. There is no behaviour + change as a result of this fix (the resulting paths are the same). + +### Fixed ### +- The error returned when we hit a symlink loop now references the correct + path. (#10) + +## [0.2.4] - 2023-09-06 ## + +### Security ### +- This release fixes a potential security issue in filepath-securejoin when + used on Windows ([GHSA-6xv5-86q9-7xr8][], which could be used to generate + paths outside of the provided rootfs in certain cases), as well as improving + the overall behaviour of filepath-securejoin when dealing with Windows paths + that contain volume names. Thanks to Paulo Gomes for discovering and fixing + these issues. + +### Fixed ### +- Switch to GitHub Actions for CI so we can test on Windows as well as Linux + and MacOS. + +[GHSA-6xv5-86q9-7xr8]: https://github.com/advisories/GHSA-6xv5-86q9-7xr8 + +## [0.2.3] - 2021-06-04 ## + +### Changed ### +- Switch to Go 1.13-style `%w` error wrapping, letting us drop the dependency + on `github.com/pkg/errors`. + +## [0.2.2] - 2018-09-05 ## + +### Changed ### +- Use `syscall.ELOOP` as the base error for symlink loops, rather than our own + (internal) error. This allows callers to more easily use `errors.Is` to check + for this case. + +## [0.2.1] - 2018-09-05 ## + +### Fixed ### +- Use our own `IsNotExist` implementation, which lets us handle `ENOTDIR` + properly within `SecureJoin`. + +## [0.2.0] - 2017-07-19 ## + +We now have 100% test coverage! + +### Added ### +- Add a `SecureJoinVFS` API that can be used for mocking (as we do in our new + tests) or for implementing custom handling of lookup operations (such as for + rootless containers, where work is necessary to access directories with weird + modes because we don't have `CAP_DAC_READ_SEARCH` or `CAP_DAC_OVERRIDE`). + +## 0.1.0 - 2017-07-19 + +This is our first release of `github.com/cyphar/filepath-securejoin`, +containing a full implementation with a coverage of 93.5% (the only missing +cases are the error cases, which are hard to mocktest at the moment). + +[Unreleased]: https://github.com/cyphar/filepath-securejoin/compare/v0.3.1...HEAD +[0.3.1]: https://github.com/cyphar/filepath-securejoin/compare/v0.3.0...v0.3.1 +[0.3.0]: https://github.com/cyphar/filepath-securejoin/compare/v0.2.5...v0.3.0 +[0.2.5]: https://github.com/cyphar/filepath-securejoin/compare/v0.2.4...v0.2.5 +[0.2.4]: https://github.com/cyphar/filepath-securejoin/compare/v0.2.3...v0.2.4 +[0.2.3]: https://github.com/cyphar/filepath-securejoin/compare/v0.2.2...v0.2.3 +[0.2.2]: https://github.com/cyphar/filepath-securejoin/compare/v0.2.1...v0.2.2 +[0.2.1]: https://github.com/cyphar/filepath-securejoin/compare/v0.2.0...v0.2.1 +[0.2.0]: https://github.com/cyphar/filepath-securejoin/compare/v0.1.0...v0.2.0 diff --git a/vendor/github.com/cyphar/filepath-securejoin/LICENSE b/vendor/github.com/cyphar/filepath-securejoin/LICENSE index bec842f294..cb1ab88da0 100644 --- a/vendor/github.com/cyphar/filepath-securejoin/LICENSE +++ b/vendor/github.com/cyphar/filepath-securejoin/LICENSE @@ -1,5 +1,5 @@ Copyright (C) 2014-2015 Docker Inc & Go Authors. All rights reserved. -Copyright (C) 2017 SUSE LLC. All rights reserved. +Copyright (C) 2017-2024 SUSE LLC. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/vendor/github.com/cyphar/filepath-securejoin/README.md b/vendor/github.com/cyphar/filepath-securejoin/README.md index 4eca0f2355..253956f865 100644 --- a/vendor/github.com/cyphar/filepath-securejoin/README.md +++ b/vendor/github.com/cyphar/filepath-securejoin/README.md @@ -2,31 +2,24 @@ [![Build Status](https://github.com/cyphar/filepath-securejoin/actions/workflows/ci.yml/badge.svg)](https://github.com/cyphar/filepath-securejoin/actions/workflows/ci.yml) -An implementation of `SecureJoin`, a [candidate for inclusion in the Go -standard library][go#20126]. The purpose of this function is to be a "secure" -alternative to `filepath.Join`, and in particular it provides certain -guarantees that are not provided by `filepath.Join`. - -> **NOTE**: This code is *only* safe if you are not at risk of other processes -> modifying path components after you've used `SecureJoin`. If it is possible -> for a malicious process to modify path components of the resolved path, then -> you will be vulnerable to some fairly trivial TOCTOU race conditions. [There -> are some Linux kernel patches I'm working on which might allow for a better -> solution.][lwn-obeneath] -> -> In addition, with a slightly modified API it might be possible to use -> `O_PATH` and verify that the opened path is actually the resolved one -- but -> I have not done that yet. I might add it in the future as a helper function -> to help users verify the path (we can't just return `/proc/self/fd/` -> because that doesn't always work transparently for all users). - -This is the function prototype: +### Old API ### -```go -func SecureJoin(root, unsafePath string) (string, error) -``` +This library was originally just an implementation of `SecureJoin` which was +[intended to be included in the Go standard library][go#20126] as a safer +`filepath.Join` that would restrict the path lookup to be inside a root +directory. + +The implementation was based on code that existed in several container +runtimes. Unfortunately, this API is **fundamentally unsafe** against attackers +that can modify path components after `SecureJoin` returns and before the +caller uses the path, allowing for some fairly trivial TOCTOU attacks. + +`SecureJoin` (and `SecureJoinVFS`) are still provided by this library to +support legacy users, but new users are strongly suggested to avoid using +`SecureJoin` and instead use the [new api](#new-api) or switch to +[libpathrs][libpathrs]. -This library **guarantees** the following: +With the above limitations in mind, this library guarantees the following: * If no error is set, the resulting string **must** be a child path of `root` and will not contain any symlink path components (they will all be @@ -47,7 +40,7 @@ This library **guarantees** the following: A (trivial) implementation of this function on GNU/Linux systems could be done with the following (note that this requires root privileges and is far more opaque than the implementation in this library, and also requires that -`readlink` is inside the `root` path): +`readlink` is inside the `root` path and is trustworthy): ```go package securejoin @@ -70,9 +63,105 @@ func SecureJoin(root, unsafePath string) (string, error) { } ``` -[lwn-obeneath]: https://lwn.net/Articles/767547/ +[libpathrs]: https://github.com/openSUSE/libpathrs [go#20126]: https://github.com/golang/go/issues/20126 +### New API ### + +While we recommend users switch to [libpathrs][libpathrs] as soon as it has a +stable release, some methods implemented by libpathrs have been ported to this +library to ease the transition. These APIs are only supported on Linux. + +These APIs are implemented such that `filepath-securejoin` will +opportunistically use certain newer kernel APIs that make these operations far +more secure. In particular: + +* All of the lookup operations will use [`openat2`][openat2.2] on new enough + kernels (Linux 5.6 or later) to restrict lookups through magic-links and + bind-mounts (for certain operations) and to make use of `RESOLVE_IN_ROOT` to + efficiently resolve symlinks within a rootfs. + +* The APIs provide hardening against a malicious `/proc` mount to either detect + or avoid being tricked by a `/proc` that is not legitimate. This is done + using [`openat2`][openat2.2] for all users, and privileged users will also be + further protected by using [`fsopen`][fsopen.2] and [`open_tree`][open_tree.2] + (Linux 4.18 or later). + +[openat2.2]: https://www.man7.org/linux/man-pages/man2/openat2.2.html +[fsopen.2]: https://github.com/brauner/man-pages-md/blob/main/fsopen.md +[open_tree.2]: https://github.com/brauner/man-pages-md/blob/main/open_tree.md + +#### `OpenInRoot` #### + +```go +func OpenInRoot(root, unsafePath string) (*os.File, error) +func OpenatInRoot(root *os.File, unsafePath string) (*os.File, error) +func Reopen(handle *os.File, flags int) (*os.File, error) +``` + +`OpenInRoot` is a much safer version of + +```go +path, err := securejoin.SecureJoin(root, unsafePath) +file, err := os.OpenFile(path, unix.O_PATH|unix.O_CLOEXEC) +``` + +that protects against various race attacks that could lead to serious security +issues, depending on the application. Note that the returned `*os.File` is an +`O_PATH` file descriptor, which is quite restricted. Callers will probably need +to use `Reopen` to get a more usable handle (this split is done to provide +useful features like PTY spawning and to avoid users accidentally opening bad +inodes that could cause a DoS). + +Callers need to be careful in how they use the returned `*os.File`. Usually it +is only safe to operate on the handle directly, and it is very easy to create a +security issue. [libpathrs][libpathrs] provides far more helpers to make using +these handles safer -- there is currently no plan to port them to +`filepath-securejoin`. + +`OpenatInRoot` is like `OpenInRoot` except that the root is provided using an +`*os.File`. This allows you to ensure that multiple `OpenatInRoot` (or +`MkdirAllHandle`) calls are operating on the same rootfs. + +> **NOTE**: Unlike `SecureJoin`, `OpenInRoot` will error out as soon as it hits +> a dangling symlink or non-existent path. This is in contrast to `SecureJoin` +> which treated non-existent components as though they were real directories, +> and would allow for partial resolution of dangling symlinks. These behaviours +> are at odds with how Linux treats non-existent paths and dangling symlinks, +> and so these are no longer allowed. + +#### `MkdirAll` #### + +```go +func MkdirAll(root, unsafePath string, mode int) error +func MkdirAllHandle(root *os.File, unsafePath string, mode int) (*os.File, error) +``` + +`MkdirAll` is a much safer version of + +```go +path, err := securejoin.SecureJoin(root, unsafePath) +err = os.MkdirAll(path, mode) +``` + +that protects against the same kinds of races that `OpenInRoot` protects +against. + +`MkdirAllHandle` is like `MkdirAll` except that the root is provided using an +`*os.File` (the reason for this is the same as with `OpenatInRoot`) and an +`*os.File` of the final created directory is returned (this directory is +guaranteed to be effectively identical to the directory created by +`MkdirAllHandle`, which is not possible to ensure by just using `OpenatInRoot` +after `MkdirAll`). + +> **NOTE**: Unlike `SecureJoin`, `MkdirAll` will error out as soon as it hits +> a dangling symlink or non-existent path. This is in contrast to `SecureJoin` +> which treated non-existent components as though they were real directories, +> and would allow for partial resolution of dangling symlinks. These behaviours +> are at odds with how Linux treats non-existent paths and dangling symlinks, +> and so these are no longer allowed. This means that `MkdirAll` will not +> create non-existent directories referenced by a dangling symlink. + ### License ### The license of this project is the same as Go, which is a BSD 3-clause license diff --git a/vendor/github.com/cyphar/filepath-securejoin/VERSION b/vendor/github.com/cyphar/filepath-securejoin/VERSION index 3a4036fb45..9e11b32fca 100644 --- a/vendor/github.com/cyphar/filepath-securejoin/VERSION +++ b/vendor/github.com/cyphar/filepath-securejoin/VERSION @@ -1 +1 @@ -0.2.5 +0.3.1 diff --git a/vendor/github.com/cyphar/filepath-securejoin/join.go b/vendor/github.com/cyphar/filepath-securejoin/join.go index 5ac23b9983..bd86a48b0c 100644 --- a/vendor/github.com/cyphar/filepath-securejoin/join.go +++ b/vendor/github.com/cyphar/filepath-securejoin/join.go @@ -1,5 +1,5 @@ // Copyright (C) 2014-2015 Docker Inc & Go Authors. All rights reserved. -// Copyright (C) 2017 SUSE LLC. All rights reserved. +// Copyright (C) 2017-2024 SUSE LLC. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -41,6 +41,12 @@ func IsNotExist(err error) bool { // replaced with symlinks on the filesystem) after this function has returned. // Such a symlink race is necessarily out-of-scope of SecureJoin. // +// NOTE: Due to the above limitation, Linux users are strongly encouraged to +// use OpenInRoot instead, which does safely protect against these kinds of +// attacks. There is no way to solve this problem with SecureJoinVFS because +// the API is fundamentally wrong (you cannot return a "safe" path string and +// guarantee it won't be modified afterwards). +// // Volume names in unsafePath are always discarded, regardless if they are // provided via direct input or when evaluating symlinks. Therefore: // diff --git a/vendor/github.com/cyphar/filepath-securejoin/lookup_linux.go b/vendor/github.com/cyphar/filepath-securejoin/lookup_linux.go new file mode 100644 index 0000000000..290befa154 --- /dev/null +++ b/vendor/github.com/cyphar/filepath-securejoin/lookup_linux.go @@ -0,0 +1,389 @@ +//go:build linux + +// Copyright (C) 2024 SUSE LLC. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package securejoin + +import ( + "errors" + "fmt" + "os" + "path" + "path/filepath" + "slices" + "strings" + + "golang.org/x/sys/unix" +) + +type symlinkStackEntry struct { + // (dir, remainingPath) is what we would've returned if the link didn't + // exist. This matches what openat2(RESOLVE_IN_ROOT) would return in + // this case. + dir *os.File + remainingPath string + // linkUnwalked is the remaining path components from the original + // Readlink which we have yet to walk. When this slice is empty, we + // drop the link from the stack. + linkUnwalked []string +} + +func (se symlinkStackEntry) String() string { + return fmt.Sprintf("<%s>/%s [->%s]", se.dir.Name(), se.remainingPath, strings.Join(se.linkUnwalked, "/")) +} + +func (se symlinkStackEntry) Close() { + _ = se.dir.Close() +} + +type symlinkStack []*symlinkStackEntry + +func (s *symlinkStack) IsEmpty() bool { + return s == nil || len(*s) == 0 +} + +func (s *symlinkStack) Close() { + if s != nil { + for _, link := range *s { + link.Close() + } + // TODO: Switch to clear once we switch to Go 1.21. + *s = nil + } +} + +var ( + errEmptyStack = errors.New("[internal] stack is empty") + errBrokenSymlinkStack = errors.New("[internal error] broken symlink stack") +) + +func (s *symlinkStack) popPart(part string) error { + if s == nil || s.IsEmpty() { + // If there is nothing in the symlink stack, then the part was from the + // real path provided by the user, and this is a no-op. + return errEmptyStack + } + if part == "." { + // "." components are no-ops -- we drop them when doing SwapLink. + return nil + } + + tailEntry := (*s)[len(*s)-1] + + // Double-check that we are popping the component we expect. + if len(tailEntry.linkUnwalked) == 0 { + return fmt.Errorf("%w: trying to pop component %q of empty stack entry %s", errBrokenSymlinkStack, part, tailEntry) + } + headPart := tailEntry.linkUnwalked[0] + if headPart != part { + return fmt.Errorf("%w: trying to pop component %q but the last stack entry is %s (%q)", errBrokenSymlinkStack, part, tailEntry, headPart) + } + + // Drop the component, but keep the entry around in case we are dealing + // with a "tail-chained" symlink. + tailEntry.linkUnwalked = tailEntry.linkUnwalked[1:] + return nil +} + +func (s *symlinkStack) PopPart(part string) error { + if err := s.popPart(part); err != nil { + if errors.Is(err, errEmptyStack) { + // Skip empty stacks. + err = nil + } + return err + } + + // Clean up any of the trailing stack entries that are empty. + for lastGood := len(*s) - 1; lastGood >= 0; lastGood-- { + entry := (*s)[lastGood] + if len(entry.linkUnwalked) > 0 { + break + } + entry.Close() + (*s) = (*s)[:lastGood] + } + return nil +} + +func (s *symlinkStack) push(dir *os.File, remainingPath, linkTarget string) error { + if s == nil { + return nil + } + // Split the link target and clean up any "" parts. + linkTargetParts := slices.DeleteFunc( + strings.Split(linkTarget, "/"), + func(part string) bool { return part == "" || part == "." }) + + // Copy the directory so the caller doesn't close our copy. + dirCopy, err := dupFile(dir) + if err != nil { + return err + } + + // Add to the stack. + *s = append(*s, &symlinkStackEntry{ + dir: dirCopy, + remainingPath: remainingPath, + linkUnwalked: linkTargetParts, + }) + return nil +} + +func (s *symlinkStack) SwapLink(linkPart string, dir *os.File, remainingPath, linkTarget string) error { + // If we are currently inside a symlink resolution, remove the symlink + // component from the last symlink entry, but don't remove the entry even + // if it's empty. If we are a "tail-chained" symlink (a trailing symlink we + // hit during a symlink resolution) we need to keep the old symlink until + // we finish the resolution. + if err := s.popPart(linkPart); err != nil { + if !errors.Is(err, errEmptyStack) { + return err + } + // Push the component regardless of whether the stack was empty. + } + return s.push(dir, remainingPath, linkTarget) +} + +func (s *symlinkStack) PopTopSymlink() (*os.File, string, bool) { + if s == nil || s.IsEmpty() { + return nil, "", false + } + tailEntry := (*s)[0] + *s = (*s)[1:] + return tailEntry.dir, tailEntry.remainingPath, true +} + +// partialLookupInRoot tries to lookup as much of the request path as possible +// within the provided root (a-la RESOLVE_IN_ROOT) and opens the final existing +// component of the requested path, returning a file handle to the final +// existing component and a string containing the remaining path components. +func partialLookupInRoot(root *os.File, unsafePath string) (*os.File, string, error) { + return lookupInRoot(root, unsafePath, true) +} + +func completeLookupInRoot(root *os.File, unsafePath string) (*os.File, error) { + handle, remainingPath, err := lookupInRoot(root, unsafePath, false) + if remainingPath != "" && err == nil { + // should never happen + err = fmt.Errorf("[bug] non-empty remaining path when doing a non-partial lookup: %q", remainingPath) + } + // lookupInRoot(partial=false) will always close the handle if an error is + // returned, so no need to double-check here. + return handle, err +} + +func lookupInRoot(root *os.File, unsafePath string, partial bool) (Handle *os.File, _ string, _ error) { + unsafePath = filepath.ToSlash(unsafePath) // noop + + // This is very similar to SecureJoin, except that we operate on the + // components using file descriptors. We then return the last component we + // managed open, along with the remaining path components not opened. + + // Try to use openat2 if possible. + if hasOpenat2() { + return lookupOpenat2(root, unsafePath, partial) + } + + // Get the "actual" root path from /proc/self/fd. This is necessary if the + // root is some magic-link like /proc/$pid/root, in which case we want to + // make sure when we do checkProcSelfFdPath that we are using the correct + // root path. + logicalRootPath, err := procSelfFdReadlink(root) + if err != nil { + return nil, "", fmt.Errorf("get real root path: %w", err) + } + + currentDir, err := dupFile(root) + if err != nil { + return nil, "", fmt.Errorf("clone root fd: %w", err) + } + defer func() { + // If a handle is not returned, close the internal handle. + if Handle == nil { + _ = currentDir.Close() + } + }() + + // symlinkStack is used to emulate how openat2(RESOLVE_IN_ROOT) treats + // dangling symlinks. If we hit a non-existent path while resolving a + // symlink, we need to return the (dir, remainingPath) that we had when we + // hit the symlink (treating the symlink as though it were a regular file). + // The set of (dir, remainingPath) sets is stored within the symlinkStack + // and we add and remove parts when we hit symlink and non-symlink + // components respectively. We need a stack because of recursive symlinks + // (symlinks that contain symlink components in their target). + // + // Note that the stack is ONLY used for book-keeping. All of the actual + // path walking logic is still based on currentPath/remainingPath and + // currentDir (as in SecureJoin). + var symStack *symlinkStack + if partial { + symStack = new(symlinkStack) + defer symStack.Close() + } + + var ( + linksWalked int + currentPath string + remainingPath = unsafePath + ) + for remainingPath != "" { + // Save the current remaining path so if the part is not real we can + // return the path including the component. + oldRemainingPath := remainingPath + + // Get the next path component. + var part string + if i := strings.IndexByte(remainingPath, '/'); i == -1 { + part, remainingPath = remainingPath, "" + } else { + part, remainingPath = remainingPath[:i], remainingPath[i+1:] + } + // If we hit an empty component, we need to treat it as though it is + // "." so that trailing "/" and "//" components on a non-directory + // correctly return the right error code. + if part == "" { + part = "." + } + + // Apply the component lexically to the path we are building. + // currentPath does not contain any symlinks, and we are lexically + // dealing with a single component, so it's okay to do a filepath.Clean + // here. + nextPath := path.Join("/", currentPath, part) + // If we logically hit the root, just clone the root rather than + // opening the part and doing all of the other checks. + if nextPath == "/" { + if err := symStack.PopPart(part); err != nil { + return nil, "", fmt.Errorf("walking into root with part %q failed: %w", part, err) + } + // Jump to root. + rootClone, err := dupFile(root) + if err != nil { + return nil, "", fmt.Errorf("clone root fd: %w", err) + } + _ = currentDir.Close() + currentDir = rootClone + currentPath = nextPath + continue + } + + // Try to open the next component. + nextDir, err := openatFile(currentDir, part, unix.O_PATH|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0) + switch { + case err == nil: + st, err := nextDir.Stat() + if err != nil { + _ = nextDir.Close() + return nil, "", fmt.Errorf("stat component %q: %w", part, err) + } + + switch st.Mode() & os.ModeType { + case os.ModeSymlink: + // readlinkat implies AT_EMPTY_PATH since Linux 2.6.39. See + // Linux commit 65cfc6722361 ("readlinkat(), fchownat() and + // fstatat() with empty relative pathnames"). + linkDest, err := readlinkatFile(nextDir, "") + // We don't need the handle anymore. + _ = nextDir.Close() + if err != nil { + return nil, "", err + } + + linksWalked++ + if linksWalked > maxSymlinkLimit { + return nil, "", &os.PathError{Op: "securejoin.lookupInRoot", Path: logicalRootPath + "/" + unsafePath, Err: unix.ELOOP} + } + + // Swap out the symlink's component for the link entry itself. + if err := symStack.SwapLink(part, currentDir, oldRemainingPath, linkDest); err != nil { + return nil, "", fmt.Errorf("walking into symlink %q failed: push symlink: %w", part, err) + } + + // Update our logical remaining path. + remainingPath = linkDest + "/" + remainingPath + // Absolute symlinks reset any work we've already done. + if path.IsAbs(linkDest) { + // Jump to root. + rootClone, err := dupFile(root) + if err != nil { + return nil, "", fmt.Errorf("clone root fd: %w", err) + } + _ = currentDir.Close() + currentDir = rootClone + currentPath = "/" + } + + default: + // If we are dealing with a directory, simply walk into it. + _ = currentDir.Close() + currentDir = nextDir + currentPath = nextPath + + // The part was real, so drop it from the symlink stack. + if err := symStack.PopPart(part); err != nil { + return nil, "", fmt.Errorf("walking into directory %q failed: %w", part, err) + } + + // If we are operating on a .., make sure we haven't escaped. + // We only have to check for ".." here because walking down + // into a regular component component cannot cause you to + // escape. This mirrors the logic in RESOLVE_IN_ROOT, except we + // have to check every ".." rather than only checking after a + // rename or mount on the system. + if part == ".." { + // Make sure the root hasn't moved. + if err := checkProcSelfFdPath(logicalRootPath, root); err != nil { + return nil, "", fmt.Errorf("root path moved during lookup: %w", err) + } + // Make sure the path is what we expect. + fullPath := logicalRootPath + nextPath + if err := checkProcSelfFdPath(fullPath, currentDir); err != nil { + return nil, "", fmt.Errorf("walking into %q had unexpected result: %w", part, err) + } + } + } + + default: + if !partial { + return nil, "", err + } + // If there are any remaining components in the symlink stack, we + // are still within a symlink resolution and thus we hit a dangling + // symlink. So pretend that the first symlink in the stack we hit + // was an ENOENT (to match openat2). + if oldDir, remainingPath, ok := symStack.PopTopSymlink(); ok { + _ = currentDir.Close() + return oldDir, remainingPath, err + } + // We have hit a final component that doesn't exist, so we have our + // partial open result. Note that we have to use the OLD remaining + // path, since the lookup failed. + return currentDir, oldRemainingPath, err + } + } + + // If the unsafePath had a trailing slash, we need to make sure we try to + // do a relative "." open so that we will correctly return an error when + // the final component is a non-directory (to match openat2). In the + // context of openat2, a trailing slash and a trailing "/." are completely + // equivalent. + if strings.HasSuffix(unsafePath, "/") { + nextDir, err := openatFile(currentDir, ".", unix.O_PATH|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0) + if err != nil { + if !partial { + _ = currentDir.Close() + currentDir = nil + } + return currentDir, "", err + } + _ = currentDir.Close() + currentDir = nextDir + } + + // All of the components existed! + return currentDir, "", nil +} diff --git a/vendor/github.com/cyphar/filepath-securejoin/mkdir_linux.go b/vendor/github.com/cyphar/filepath-securejoin/mkdir_linux.go new file mode 100644 index 0000000000..ad2bd7973a --- /dev/null +++ b/vendor/github.com/cyphar/filepath-securejoin/mkdir_linux.go @@ -0,0 +1,229 @@ +//go:build linux + +// Copyright (C) 2024 SUSE LLC. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package securejoin + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "slices" + "strings" + + "golang.org/x/sys/unix" +) + +var ( + errInvalidMode = errors.New("invalid permission mode") + errPossibleAttack = errors.New("possible attack detected") +) + +// MkdirAllHandle is equivalent to MkdirAll, except that it is safer to use in +// two respects: +// +// - The caller provides the root directory as an *os.File (preferably O_PATH) +// handle. This means that the caller can be sure which root directory is +// being used. Note that this can be emulated by using /proc/self/fd/... as +// the root path with MkdirAll. +// +// - Once all of the directories have been created, an *os.File (O_PATH) handle +// to the directory at unsafePath is returned to the caller. This is done in +// an effectively-race-free way (an attacker would only be able to swap the +// final directory component), which is not possible to emulate with +// MkdirAll. +// +// In addition, the returned handle is obtained far more efficiently than doing +// a brand new lookup of unsafePath (such as with SecureJoin or openat2) after +// doing MkdirAll. If you intend to open the directory after creating it, you +// should use MkdirAllHandle. +func MkdirAllHandle(root *os.File, unsafePath string, mode int) (_ *os.File, Err error) { + // Make sure there are no os.FileMode bits set. + if mode&^0o7777 != 0 { + return nil, fmt.Errorf("%w for mkdir 0o%.3o", errInvalidMode, mode) + } + + // Try to open as much of the path as possible. + currentDir, remainingPath, err := partialLookupInRoot(root, unsafePath) + defer func() { + if Err != nil { + _ = currentDir.Close() + } + }() + if err != nil && !errors.Is(err, unix.ENOENT) { + return nil, fmt.Errorf("find existing subpath of %q: %w", unsafePath, err) + } + + // If there is an attacker deleting directories as we walk into them, + // detect this proactively. Note this is guaranteed to detect if the + // attacker deleted any part of the tree up to currentDir. + // + // Once we walk into a dead directory, partialLookupInRoot would not be + // able to walk further down the tree (directories must be empty before + // they are deleted), and if the attacker has removed the entire tree we + // can be sure that anything that was originally inside a dead directory + // must also be deleted and thus is a dead directory in its own right. + // + // This is mostly a quality-of-life check, because mkdir will simply fail + // later if the attacker deletes the tree after this check. + if err := isDeadInode(currentDir); err != nil { + return nil, fmt.Errorf("finding existing subpath of %q: %w", unsafePath, err) + } + + // Re-open the path to match the O_DIRECTORY reopen loop later (so that we + // always return a non-O_PATH handle). We also check that we actually got a + // directory. + if reopenDir, err := Reopen(currentDir, unix.O_DIRECTORY|unix.O_CLOEXEC); errors.Is(err, unix.ENOTDIR) { + return nil, fmt.Errorf("cannot create subdirectories in %q: %w", currentDir.Name(), unix.ENOTDIR) + } else if err != nil { + return nil, fmt.Errorf("re-opening handle to %q: %w", currentDir.Name(), err) + } else { + _ = currentDir.Close() + currentDir = reopenDir + } + + remainingParts := strings.Split(remainingPath, string(filepath.Separator)) + if slices.Contains(remainingParts, "..") { + // The path contained ".." components after the end of the "real" + // components. We could try to safely resolve ".." here but that would + // add a bunch of extra logic for something that it's not clear even + // needs to be supported. So just return an error. + // + // If we do filepath.Clean(remainingPath) then we end up with the + // problem that ".." can erase a trailing dangling symlink and produce + // a path that doesn't quite match what the user asked for. + return nil, fmt.Errorf("%w: yet-to-be-created path %q contains '..' components", unix.ENOENT, remainingPath) + } + + // Make sure the mode doesn't have any type bits. + mode &^= unix.S_IFMT + // What properties do we expect any newly created directories to have? + var ( + // While umask(2) is a per-thread property, and thus this value could + // vary between threads, a functioning Go program would LockOSThread + // threads with different umasks and so we don't need to LockOSThread + // for this entire mkdirat loop (if we are in the locked thread with a + // different umask, we are already locked and there's nothing for us to + // do -- and if not then it doesn't matter which thread we run on and + // there's nothing for us to do). + expectedMode = uint32(unix.S_IFDIR | (mode &^ getUmask())) + + // We would want to get the fs[ug]id here, but we can't access those + // from userspace. In practice, nobody uses setfs[ug]id() anymore, so + // just use the effective [ug]id (which is equivalent to the fs[ug]id + // for programs that don't use setfs[ug]id). + expectedUid = uint32(unix.Geteuid()) + expectedGid = uint32(unix.Getegid()) + ) + + // Create the remaining components. + for _, part := range remainingParts { + switch part { + case "", ".": + // Skip over no-op paths. + continue + } + + // NOTE: mkdir(2) will not follow trailing symlinks, so we can safely + // create the finaly component without worrying about symlink-exchange + // attacks. + if err := unix.Mkdirat(int(currentDir.Fd()), part, uint32(mode)); err != nil { + err = &os.PathError{Op: "mkdirat", Path: currentDir.Name() + "/" + part, Err: err} + // Make the error a bit nicer if the directory is dead. + if err2 := isDeadInode(currentDir); err2 != nil { + err = fmt.Errorf("%w (%w)", err, err2) + } + return nil, err + } + + // Get a handle to the next component. O_DIRECTORY means we don't need + // to use O_PATH. + var nextDir *os.File + if hasOpenat2() { + nextDir, err = openat2File(currentDir, part, &unix.OpenHow{ + Flags: unix.O_NOFOLLOW | unix.O_DIRECTORY | unix.O_CLOEXEC, + Resolve: unix.RESOLVE_BENEATH | unix.RESOLVE_NO_SYMLINKS | unix.RESOLVE_NO_XDEV, + }) + } else { + nextDir, err = openatFile(currentDir, part, unix.O_NOFOLLOW|unix.O_DIRECTORY|unix.O_CLOEXEC, 0) + } + if err != nil { + return nil, err + } + _ = currentDir.Close() + currentDir = nextDir + + // Make sure that the directory matches what we expect. An attacker + // could have swapped the directory between us making it and opening + // it. There's no way for us to be sure that the directory is + // _precisely_ the same as the directory we created, but if we are in + // an empty directory with the same owner and mode as the one we + // created then there is nothing the attacker could do with this new + // directory that they couldn't do with the old one. + if stat, err := fstat(currentDir); err != nil { + return nil, fmt.Errorf("check newly created directory: %w", err) + } else { + if stat.Mode != expectedMode { + return nil, fmt.Errorf("%w: newly created directory %q has incorrect mode 0o%.3o (expected 0o%.3o)", errPossibleAttack, currentDir.Name(), stat.Mode, expectedMode) + } + if stat.Uid != expectedUid || stat.Gid != expectedGid { + return nil, fmt.Errorf("%w: newly created directory %q has incorrect owner %d:%d (expected %d:%d)", errPossibleAttack, currentDir.Name(), stat.Uid, stat.Gid, expectedUid, expectedGid) + } + // Check that the directory is empty. We only need to check for + // a single entry, and we should get EOF if the directory is + // empty. + _, err := currentDir.Readdirnames(1) + if !errors.Is(err, io.EOF) { + if err == nil { + err = fmt.Errorf("%w: newly created directory %q is non-empty", errPossibleAttack, currentDir.Name()) + } + return nil, fmt.Errorf("check if newly created directory %q is empty: %w", currentDir.Name(), err) + } + // Reset the offset. + _, _ = currentDir.Seek(0, unix.SEEK_SET) + } + } + return currentDir, nil +} + +// MkdirAll is a race-safe alternative to the Go stdlib's os.MkdirAll function, +// where the new directory is guaranteed to be within the root directory (if an +// attacker can move directories from inside the root to outside the root, the +// created directory tree might be outside of the root but the key constraint +// is that at no point will we walk outside of the directory tree we are +// creating). +// +// Effectively, MkdirAll(root, unsafePath, mode) is equivalent to +// +// path, _ := securejoin.SecureJoin(root, unsafePath) +// err := os.MkdirAll(path, mode) +// +// But is much safer. The above implementation is unsafe because if an attacker +// can modify the filesystem tree between SecureJoin and MkdirAll, it is +// possible for MkdirAll to resolve unsafe symlink components and create +// directories outside of the root. +// +// If you plan to open the directory after you have created it or want to use +// an open directory handle as the root, you should use MkdirAllHandle instead. +// This function is a wrapper around MkdirAllHandle. +// +// NOTE: The mode argument must be set the unix mode bits (unix.S_I...), not +// the Go generic mode bits (os.Mode...). +func MkdirAll(root, unsafePath string, mode int) error { + rootDir, err := os.OpenFile(root, unix.O_PATH|unix.O_DIRECTORY|unix.O_CLOEXEC, 0) + if err != nil { + return err + } + defer rootDir.Close() + + f, err := MkdirAllHandle(rootDir, unsafePath, mode) + if err != nil { + return err + } + _ = f.Close() + return nil +} diff --git a/vendor/github.com/cyphar/filepath-securejoin/open_linux.go b/vendor/github.com/cyphar/filepath-securejoin/open_linux.go new file mode 100644 index 0000000000..52dce76f3f --- /dev/null +++ b/vendor/github.com/cyphar/filepath-securejoin/open_linux.go @@ -0,0 +1,101 @@ +//go:build linux + +// Copyright (C) 2024 SUSE LLC. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package securejoin + +import ( + "fmt" + "os" + "strconv" + + "golang.org/x/sys/unix" +) + +// OpenatInRoot is equivalent to OpenInRoot, except that the root is provided +// using an *os.File handle, to ensure that the correct root directory is used. +func OpenatInRoot(root *os.File, unsafePath string) (*os.File, error) { + handle, err := completeLookupInRoot(root, unsafePath) + if err != nil { + return nil, &os.PathError{Op: "securejoin.OpenInRoot", Path: unsafePath, Err: err} + } + return handle, nil +} + +// OpenInRoot safely opens the provided unsafePath within the root. +// Effectively, OpenInRoot(root, unsafePath) is equivalent to +// +// path, _ := securejoin.SecureJoin(root, unsafePath) +// handle, err := os.OpenFile(path, unix.O_PATH|unix.O_CLOEXEC) +// +// But is much safer. The above implementation is unsafe because if an attacker +// can modify the filesystem tree between SecureJoin and OpenFile, it is +// possible for the returned file to be outside of the root. +// +// Note that the returned handle is an O_PATH handle, meaning that only a very +// limited set of operations will work on the handle. This is done to avoid +// accidentally opening an untrusted file that could cause issues (such as a +// disconnected TTY that could cause a DoS, or some other issue). In order to +// use the returned handle, you can "upgrade" it to a proper handle using +// Reopen. +func OpenInRoot(root, unsafePath string) (*os.File, error) { + rootDir, err := os.OpenFile(root, unix.O_PATH|unix.O_DIRECTORY|unix.O_CLOEXEC, 0) + if err != nil { + return nil, err + } + defer rootDir.Close() + return OpenatInRoot(rootDir, unsafePath) +} + +// Reopen takes an *os.File handle and re-opens it through /proc/self/fd. +// Reopen(file, flags) is effectively equivalent to +// +// fdPath := fmt.Sprintf("/proc/self/fd/%d", file.Fd()) +// os.OpenFile(fdPath, flags|unix.O_CLOEXEC) +// +// But with some extra hardenings to ensure that we are not tricked by a +// maliciously-configured /proc mount. While this attack scenario is not +// common, in container runtimes it is possible for higher-level runtimes to be +// tricked into configuring an unsafe /proc that can be used to attack file +// operations. See CVE-2019-19921 for more details. +func Reopen(handle *os.File, flags int) (*os.File, error) { + procRoot, err := getProcRoot() + if err != nil { + return nil, err + } + + // We can't operate on /proc/thread-self/fd/$n directly when doing a + // re-open, so we need to open /proc/thread-self/fd and then open a single + // final component. + procFdDir, closer, err := procThreadSelf(procRoot, "fd/") + if err != nil { + return nil, fmt.Errorf("get safe /proc/thread-self/fd handle: %w", err) + } + defer procFdDir.Close() + defer closer() + + // Try to detect if there is a mount on top of the magic-link we are about + // to open. If we are using unsafeHostProcRoot(), this could change after + // we check it (and there's nothing we can do about that) but for + // privateProcRoot() this should be guaranteed to be safe (at least since + // Linux 5.12[1], when anonymous mount namespaces were completely isolated + // from external mounts including mount propagation events). + // + // [1]: Linux commit ee2e3f50629f ("mount: fix mounting of detached mounts + // onto targets that reside on shared mounts"). + fdStr := strconv.Itoa(int(handle.Fd())) + if err := checkSymlinkOvermount(procRoot, procFdDir, fdStr); err != nil { + return nil, fmt.Errorf("check safety of /proc/thread-self/fd/%s magiclink: %w", fdStr, err) + } + + flags |= unix.O_CLOEXEC + // Rather than just wrapping openatFile, open-code it so we can copy + // handle.Name(). + reopenFd, err := unix.Openat(int(procFdDir.Fd()), fdStr, flags, 0) + if err != nil { + return nil, fmt.Errorf("reopen fd %d: %w", handle.Fd(), err) + } + return os.NewFile(uintptr(reopenFd), handle.Name()), nil +} diff --git a/vendor/github.com/cyphar/filepath-securejoin/openat2_linux.go b/vendor/github.com/cyphar/filepath-securejoin/openat2_linux.go new file mode 100644 index 0000000000..921b3e1d44 --- /dev/null +++ b/vendor/github.com/cyphar/filepath-securejoin/openat2_linux.go @@ -0,0 +1,141 @@ +//go:build linux + +// Copyright (C) 2024 SUSE LLC. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package securejoin + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "golang.org/x/sys/unix" +) + +var ( + hasOpenat2Bool bool + hasOpenat2Once sync.Once + + testingForceHasOpenat2 *bool +) + +func hasOpenat2() bool { + if testing.Testing() && testingForceHasOpenat2 != nil { + return *testingForceHasOpenat2 + } + hasOpenat2Once.Do(func() { + fd, err := unix.Openat2(unix.AT_FDCWD, ".", &unix.OpenHow{ + Flags: unix.O_PATH | unix.O_CLOEXEC, + Resolve: unix.RESOLVE_NO_SYMLINKS | unix.RESOLVE_IN_ROOT, + }) + if err == nil { + hasOpenat2Bool = true + _ = unix.Close(fd) + } + }) + return hasOpenat2Bool +} + +func scopedLookupShouldRetry(how *unix.OpenHow, err error) bool { + // RESOLVE_IN_ROOT (and RESOLVE_BENEATH) can return -EAGAIN if we resolve + // ".." while a mount or rename occurs anywhere on the system. This could + // happen spuriously, or as the result of an attacker trying to mess with + // us during lookup. + // + // In addition, scoped lookups have a "safety check" at the end of + // complete_walk which will return -EXDEV if the final path is not in the + // root. + return how.Resolve&(unix.RESOLVE_IN_ROOT|unix.RESOLVE_BENEATH) != 0 && + (errors.Is(err, unix.EAGAIN) || errors.Is(err, unix.EXDEV)) +} + +const scopedLookupMaxRetries = 10 + +func openat2File(dir *os.File, path string, how *unix.OpenHow) (*os.File, error) { + fullPath := dir.Name() + "/" + path + // Make sure we always set O_CLOEXEC. + how.Flags |= unix.O_CLOEXEC + var tries int + for tries < scopedLookupMaxRetries { + fd, err := unix.Openat2(int(dir.Fd()), path, how) + if err != nil { + if scopedLookupShouldRetry(how, err) { + // We retry a couple of times to avoid the spurious errors, and + // if we are being attacked then returning -EAGAIN is the best + // we can do. + tries++ + continue + } + return nil, &os.PathError{Op: "openat2", Path: fullPath, Err: err} + } + // If we are using RESOLVE_IN_ROOT, the name we generated may be wrong. + // NOTE: The procRoot code MUST NOT use RESOLVE_IN_ROOT, otherwise + // you'll get infinite recursion here. + if how.Resolve&unix.RESOLVE_IN_ROOT == unix.RESOLVE_IN_ROOT { + if actualPath, err := rawProcSelfFdReadlink(fd); err == nil { + fullPath = actualPath + } + } + return os.NewFile(uintptr(fd), fullPath), nil + } + return nil, &os.PathError{Op: "openat2", Path: fullPath, Err: errPossibleAttack} +} + +func lookupOpenat2(root *os.File, unsafePath string, partial bool) (*os.File, string, error) { + if !partial { + file, err := openat2File(root, unsafePath, &unix.OpenHow{ + Flags: unix.O_PATH | unix.O_CLOEXEC, + Resolve: unix.RESOLVE_IN_ROOT | unix.RESOLVE_NO_MAGICLINKS, + }) + return file, "", err + } + return partialLookupOpenat2(root, unsafePath) +} + +// partialLookupOpenat2 is an alternative implementation of +// partialLookupInRoot, using openat2(RESOLVE_IN_ROOT) to more safely get a +// handle to the deepest existing child of the requested path within the root. +func partialLookupOpenat2(root *os.File, unsafePath string) (*os.File, string, error) { + // TODO: Implement this as a git-bisect-like binary search. + + unsafePath = filepath.ToSlash(unsafePath) // noop + endIdx := len(unsafePath) + var lastError error + for endIdx > 0 { + subpath := unsafePath[:endIdx] + + handle, err := openat2File(root, subpath, &unix.OpenHow{ + Flags: unix.O_PATH | unix.O_CLOEXEC, + Resolve: unix.RESOLVE_IN_ROOT | unix.RESOLVE_NO_MAGICLINKS, + }) + if err == nil { + // Jump over the slash if we have a non-"" remainingPath. + if endIdx < len(unsafePath) { + endIdx += 1 + } + // We found a subpath! + return handle, unsafePath[endIdx:], lastError + } + if errors.Is(err, unix.ENOENT) || errors.Is(err, unix.ENOTDIR) { + // That path doesn't exist, let's try the next directory up. + endIdx = strings.LastIndexByte(subpath, '/') + lastError = err + continue + } + return nil, "", fmt.Errorf("open subpath: %w", err) + } + // If we couldn't open anything, the whole subpath is missing. Return a + // copy of the root fd so that the caller doesn't close this one by + // accident. + rootClone, err := dupFile(root) + if err != nil { + return nil, "", err + } + return rootClone, unsafePath, lastError +} diff --git a/vendor/github.com/cyphar/filepath-securejoin/openat_linux.go b/vendor/github.com/cyphar/filepath-securejoin/openat_linux.go new file mode 100644 index 0000000000..949fb5f2d8 --- /dev/null +++ b/vendor/github.com/cyphar/filepath-securejoin/openat_linux.go @@ -0,0 +1,59 @@ +//go:build linux + +// Copyright (C) 2024 SUSE LLC. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package securejoin + +import ( + "os" + "path/filepath" + + "golang.org/x/sys/unix" +) + +func dupFile(f *os.File) (*os.File, error) { + fd, err := unix.FcntlInt(f.Fd(), unix.F_DUPFD_CLOEXEC, 0) + if err != nil { + return nil, os.NewSyscallError("fcntl(F_DUPFD_CLOEXEC)", err) + } + return os.NewFile(uintptr(fd), f.Name()), nil +} + +func openatFile(dir *os.File, path string, flags int, mode int) (*os.File, error) { + // Make sure we always set O_CLOEXEC. + flags |= unix.O_CLOEXEC + fd, err := unix.Openat(int(dir.Fd()), path, flags, uint32(mode)) + if err != nil { + return nil, &os.PathError{Op: "openat", Path: dir.Name() + "/" + path, Err: err} + } + // All of the paths we use with openatFile(2) are guaranteed to be + // lexically safe, so we can use path.Join here. + fullPath := filepath.Join(dir.Name(), path) + return os.NewFile(uintptr(fd), fullPath), nil +} + +func fstatatFile(dir *os.File, path string, flags int) (unix.Stat_t, error) { + var stat unix.Stat_t + if err := unix.Fstatat(int(dir.Fd()), path, &stat, flags); err != nil { + return stat, &os.PathError{Op: "fstatat", Path: dir.Name() + "/" + path, Err: err} + } + return stat, nil +} + +func readlinkatFile(dir *os.File, path string) (string, error) { + size := 4096 + for { + linkBuf := make([]byte, size) + n, err := unix.Readlinkat(int(dir.Fd()), path, linkBuf) + if err != nil { + return "", &os.PathError{Op: "readlinkat", Path: dir.Name() + "/" + path, Err: err} + } + if n != size { + return string(linkBuf[:n]), nil + } + // Possible truncation, resize the buffer. + size *= 2 + } +} diff --git a/vendor/github.com/cyphar/filepath-securejoin/procfs_linux.go b/vendor/github.com/cyphar/filepath-securejoin/procfs_linux.go new file mode 100644 index 0000000000..adf0bd08f3 --- /dev/null +++ b/vendor/github.com/cyphar/filepath-securejoin/procfs_linux.go @@ -0,0 +1,474 @@ +//go:build linux + +// Copyright (C) 2024 SUSE LLC. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package securejoin + +import ( + "errors" + "fmt" + "os" + "runtime" + "strconv" + "sync" + + "golang.org/x/sys/unix" +) + +func fstat(f *os.File) (unix.Stat_t, error) { + var stat unix.Stat_t + if err := unix.Fstat(int(f.Fd()), &stat); err != nil { + return stat, &os.PathError{Op: "fstat", Path: f.Name(), Err: err} + } + return stat, nil +} + +func fstatfs(f *os.File) (unix.Statfs_t, error) { + var statfs unix.Statfs_t + if err := unix.Fstatfs(int(f.Fd()), &statfs); err != nil { + return statfs, &os.PathError{Op: "fstatfs", Path: f.Name(), Err: err} + } + return statfs, nil +} + +// The kernel guarantees that the root inode of a procfs mount has an +// f_type of PROC_SUPER_MAGIC and st_ino of PROC_ROOT_INO. +const ( + procSuperMagic = 0x9fa0 // PROC_SUPER_MAGIC + procRootIno = 1 // PROC_ROOT_INO +) + +func verifyProcRoot(procRoot *os.File) error { + if statfs, err := fstatfs(procRoot); err != nil { + return err + } else if statfs.Type != procSuperMagic { + return fmt.Errorf("%w: incorrect procfs root filesystem type 0x%x", errUnsafeProcfs, statfs.Type) + } + if stat, err := fstat(procRoot); err != nil { + return err + } else if stat.Ino != procRootIno { + return fmt.Errorf("%w: incorrect procfs root inode number %d", errUnsafeProcfs, stat.Ino) + } + return nil +} + +var ( + hasNewMountApiBool bool + hasNewMountApiOnce sync.Once +) + +func hasNewMountApi() bool { + hasNewMountApiOnce.Do(func() { + // All of the pieces of the new mount API we use (fsopen, fsconfig, + // fsmount, open_tree) were added together in Linux 5.1[1,2], so we can + // just check for one of the syscalls and the others should also be + // available. + // + // Just try to use open_tree(2) to open a file without OPEN_TREE_CLONE. + // This is equivalent to openat(2), but tells us if open_tree is + // available (and thus all of the other basic new mount API syscalls). + // open_tree(2) is most light-weight syscall to test here. + // + // [1]: merge commit 400913252d09 + // [2]: + fd, err := unix.OpenTree(-int(unix.EBADF), "/", unix.OPEN_TREE_CLOEXEC) + if err == nil { + hasNewMountApiBool = true + _ = unix.Close(fd) + } + }) + return hasNewMountApiBool +} + +func fsopen(fsName string, flags int) (*os.File, error) { + // Make sure we always set O_CLOEXEC. + flags |= unix.FSOPEN_CLOEXEC + fd, err := unix.Fsopen(fsName, flags) + if err != nil { + return nil, os.NewSyscallError("fsopen "+fsName, err) + } + return os.NewFile(uintptr(fd), "fscontext:"+fsName), nil +} + +func fsmount(ctx *os.File, flags, mountAttrs int) (*os.File, error) { + // Make sure we always set O_CLOEXEC. + flags |= unix.FSMOUNT_CLOEXEC + fd, err := unix.Fsmount(int(ctx.Fd()), flags, mountAttrs) + if err != nil { + return nil, os.NewSyscallError("fsmount "+ctx.Name(), err) + } + return os.NewFile(uintptr(fd), "fsmount:"+ctx.Name()), nil +} + +func newPrivateProcMount() (*os.File, error) { + procfsCtx, err := fsopen("proc", unix.FSOPEN_CLOEXEC) + if err != nil { + return nil, err + } + defer procfsCtx.Close() + + // Try to configure hidepid=ptraceable,subset=pid if possible, but ignore errors. + _ = unix.FsconfigSetString(int(procfsCtx.Fd()), "hidepid", "ptraceable") + _ = unix.FsconfigSetString(int(procfsCtx.Fd()), "subset", "pid") + + // Get an actual handle. + if err := unix.FsconfigCreate(int(procfsCtx.Fd())); err != nil { + return nil, os.NewSyscallError("fsconfig create procfs", err) + } + return fsmount(procfsCtx, unix.FSMOUNT_CLOEXEC, unix.MS_RDONLY|unix.MS_NODEV|unix.MS_NOEXEC|unix.MS_NOSUID) +} + +func openTree(dir *os.File, path string, flags uint) (*os.File, error) { + dirFd := -int(unix.EBADF) + dirName := "." + if dir != nil { + dirFd = int(dir.Fd()) + dirName = dir.Name() + } + // Make sure we always set O_CLOEXEC. + flags |= unix.OPEN_TREE_CLOEXEC + fd, err := unix.OpenTree(dirFd, path, flags) + if err != nil { + return nil, &os.PathError{Op: "open_tree", Path: path, Err: err} + } + return os.NewFile(uintptr(fd), dirName+"/"+path), nil +} + +func clonePrivateProcMount() (_ *os.File, Err error) { + // Try to make a clone without using AT_RECURSIVE if we can. If this works, + // we can be sure there are no over-mounts and so if the root is valid then + // we're golden. Otherwise, we have to deal with over-mounts. + procfsHandle, err := openTree(nil, "/proc", unix.OPEN_TREE_CLONE) + if err != nil || testingForcePrivateProcRootOpenTreeAtRecursive(procfsHandle) { + procfsHandle, err = openTree(nil, "/proc", unix.OPEN_TREE_CLONE|unix.AT_RECURSIVE) + } + if err != nil { + return nil, fmt.Errorf("creating a detached procfs clone: %w", err) + } + defer func() { + if Err != nil { + _ = procfsHandle.Close() + } + }() + if err := verifyProcRoot(procfsHandle); err != nil { + return nil, err + } + return procfsHandle, nil +} + +func privateProcRoot() (*os.File, error) { + if !hasNewMountApi() || testingForceGetProcRootUnsafe() { + return nil, fmt.Errorf("new mount api: %w", unix.ENOTSUP) + } + // Try to create a new procfs mount from scratch if we can. This ensures we + // can get a procfs mount even if /proc is fake (for whatever reason). + procRoot, err := newPrivateProcMount() + if err != nil || testingForcePrivateProcRootOpenTree(procRoot) { + // Try to clone /proc then... + procRoot, err = clonePrivateProcMount() + } + return procRoot, err +} + +var ( + procRootHandle *os.File + procRootError error + procRootOnce sync.Once + + errUnsafeProcfs = errors.New("unsafe procfs detected") +) + +func unsafeHostProcRoot() (_ *os.File, Err error) { + procRoot, err := os.OpenFile("/proc", unix.O_PATH|unix.O_NOFOLLOW|unix.O_DIRECTORY|unix.O_CLOEXEC, 0) + if err != nil { + return nil, err + } + defer func() { + if Err != nil { + _ = procRoot.Close() + } + }() + if err := verifyProcRoot(procRoot); err != nil { + return nil, err + } + return procRoot, nil +} + +func doGetProcRoot() (*os.File, error) { + procRoot, err := privateProcRoot() + if err != nil { + // Fall back to using a /proc handle if making a private mount failed. + // If we have openat2, at least we can avoid some kinds of over-mount + // attacks, but without openat2 there's not much we can do. + procRoot, err = unsafeHostProcRoot() + } + return procRoot, err +} + +func getProcRoot() (*os.File, error) { + procRootOnce.Do(func() { + procRootHandle, procRootError = doGetProcRoot() + }) + return procRootHandle, procRootError +} + +var ( + haveProcThreadSelf bool + haveProcThreadSelfOnce sync.Once +) + +type procThreadSelfCloser func() + +// procThreadSelf returns a handle to /proc/thread-self/ (or an +// equivalent handle on older kernels where /proc/thread-self doesn't exist). +// Once finished with the handle, you must call the returned closer function +// (runtime.UnlockOSThread). You must not pass the returned *os.File to other +// Go threads or use the handle after calling the closer. +// +// This is similar to ProcThreadSelf from runc, but with extra hardening +// applied and using *os.File. +func procThreadSelf(procRoot *os.File, subpath string) (_ *os.File, _ procThreadSelfCloser, Err error) { + haveProcThreadSelfOnce.Do(func() { + // If the kernel doesn't support thread-self, it doesn't matter which + // /proc handle we use. + _, err := fstatatFile(procRoot, "thread-self", unix.AT_SYMLINK_NOFOLLOW) + haveProcThreadSelf = (err == nil) + }) + + // We need to lock our thread until the caller is done with the handle + // because between getting the handle and using it we could get interrupted + // by the Go runtime and hit the case where the underlying thread is + // swapped out and the original thread is killed, resulting in + // pull-your-hair-out-hard-to-debug issues in the caller. + runtime.LockOSThread() + defer func() { + if Err != nil { + runtime.UnlockOSThread() + } + }() + + // Figure out what prefix we want to use. + threadSelf := "thread-self/" + if !haveProcThreadSelf || testingForceProcSelfTask() { + /// Pre-3.17 kernels don't have /proc/thread-self, so do it manually. + threadSelf = "self/task/" + strconv.Itoa(unix.Gettid()) + "/" + if _, err := fstatatFile(procRoot, threadSelf, unix.AT_SYMLINK_NOFOLLOW); err != nil || testingForceProcSelf() { + // In this case, we running in a pid namespace that doesn't match + // the /proc mount we have. This can happen inside runc. + // + // Unfortunately, there is no nice way to get the correct TID to + // use here because of the age of the kernel, so we have to just + // use /proc/self and hope that it works. + threadSelf = "self/" + } + } + + // Grab the handle. + var ( + handle *os.File + err error + ) + if hasOpenat2() { + // We prefer being able to use RESOLVE_NO_XDEV if we can, to be + // absolutely sure we are operating on a clean /proc handle that + // doesn't have any cheeky overmounts that could trick us (including + // symlink mounts on top of /proc/thread-self). RESOLVE_BENEATH isn't + // stricly needed, but just use it since we have it. + // + // NOTE: /proc/self is technically a magic-link (the contents of the + // symlink are generated dynamically), but it doesn't use + // nd_jump_link() so RESOLVE_NO_MAGICLINKS allows it. + // + // NOTE: We MUST NOT use RESOLVE_IN_ROOT here, as openat2File uses + // procSelfFdReadlink to clean up the returned f.Name() if we use + // RESOLVE_IN_ROOT (which would lead to an infinite recursion). + handle, err = openat2File(procRoot, threadSelf+subpath, &unix.OpenHow{ + Flags: unix.O_PATH | unix.O_NOFOLLOW | unix.O_CLOEXEC, + Resolve: unix.RESOLVE_BENEATH | unix.RESOLVE_NO_XDEV | unix.RESOLVE_NO_MAGICLINKS, + }) + if err != nil { + return nil, nil, fmt.Errorf("%w: %w", errUnsafeProcfs, err) + } + } else { + handle, err = openatFile(procRoot, threadSelf+subpath, unix.O_PATH|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0) + if err != nil { + return nil, nil, fmt.Errorf("%w: %w", errUnsafeProcfs, err) + } + defer func() { + if Err != nil { + _ = handle.Close() + } + }() + // We can't detect bind-mounts of different parts of procfs on top of + // /proc (a-la RESOLVE_NO_XDEV), but we can at least be sure that we + // aren't on the wrong filesystem here. + if statfs, err := fstatfs(handle); err != nil { + return nil, nil, err + } else if statfs.Type != procSuperMagic { + return nil, nil, fmt.Errorf("%w: incorrect /proc/self/fd filesystem type 0x%x", errUnsafeProcfs, statfs.Type) + } + } + return handle, runtime.UnlockOSThread, nil +} + +var ( + hasStatxMountIdBool bool + hasStatxMountIdOnce sync.Once +) + +func hasStatxMountId() bool { + hasStatxMountIdOnce.Do(func() { + var ( + stx unix.Statx_t + // We don't care which mount ID we get. The kernel will give us the + // unique one if it is supported. + wantStxMask uint32 = unix.STATX_MNT_ID_UNIQUE | unix.STATX_MNT_ID + ) + err := unix.Statx(-int(unix.EBADF), "/", 0, int(wantStxMask), &stx) + hasStatxMountIdBool = (err == nil && (stx.Mask&wantStxMask != 0)) + }) + return hasStatxMountIdBool +} + +func getMountId(dir *os.File, path string) (uint64, error) { + // If we don't have statx(STATX_MNT_ID*) support, we can't do anything. + if !hasStatxMountId() { + return 0, nil + } + + var ( + stx unix.Statx_t + // We don't care which mount ID we get. The kernel will give us the + // unique one if it is supported. + wantStxMask uint32 = unix.STATX_MNT_ID_UNIQUE | unix.STATX_MNT_ID + ) + + err := unix.Statx(int(dir.Fd()), path, unix.AT_EMPTY_PATH|unix.AT_SYMLINK_NOFOLLOW, int(wantStxMask), &stx) + if stx.Mask&wantStxMask == 0 { + // It's not a kernel limitation, for some reason we couldn't get a + // mount ID. Assume it's some kind of attack. + err = fmt.Errorf("%w: could not get mount id", errUnsafeProcfs) + } + if err != nil { + return 0, &os.PathError{Op: "statx(STATX_MNT_ID_...)", Path: dir.Name() + "/" + path, Err: err} + } + return stx.Mnt_id, nil +} + +func checkSymlinkOvermount(procRoot *os.File, dir *os.File, path string) error { + // Get the mntId of our procfs handle. + expectedMountId, err := getMountId(procRoot, "") + if err != nil { + return err + } + // Get the mntId of the target magic-link. + gotMountId, err := getMountId(dir, path) + if err != nil { + return err + } + // As long as the directory mount is alive, even with wrapping mount IDs, + // we would expect to see a different mount ID here. (Of course, if we're + // using unsafeHostProcRoot() then an attaker could change this after we + // did this check.) + if expectedMountId != gotMountId { + return fmt.Errorf("%w: symlink %s/%s has an overmount obscuring the real link (mount ids do not match %d != %d)", errUnsafeProcfs, dir.Name(), path, expectedMountId, gotMountId) + } + return nil +} + +func doRawProcSelfFdReadlink(procRoot *os.File, fd int) (string, error) { + fdPath := fmt.Sprintf("fd/%d", fd) + procFdLink, closer, err := procThreadSelf(procRoot, fdPath) + if err != nil { + return "", fmt.Errorf("get safe /proc/thread-self/%s handle: %w", fdPath, err) + } + defer procFdLink.Close() + defer closer() + + // Try to detect if there is a mount on top of the magic-link. Since we use the handle directly + // provide to the closure. If the closure uses the handle directly, this + // should be safe in general (a mount on top of the path afterwards would + // not affect the handle itself) and will definitely be safe if we are + // using privateProcRoot() (at least since Linux 5.12[1], when anonymous + // mount namespaces were completely isolated from external mounts including + // mount propagation events). + // + // [1]: Linux commit ee2e3f50629f ("mount: fix mounting of detached mounts + // onto targets that reside on shared mounts"). + if err := checkSymlinkOvermount(procRoot, procFdLink, ""); err != nil { + return "", fmt.Errorf("check safety of /proc/thread-self/fd/%d magiclink: %w", fd, err) + } + + // readlinkat implies AT_EMPTY_PATH since Linux 2.6.39. See Linux commit + // 65cfc6722361 ("readlinkat(), fchownat() and fstatat() with empty + // relative pathnames"). + return readlinkatFile(procFdLink, "") +} + +func rawProcSelfFdReadlink(fd int) (string, error) { + procRoot, err := getProcRoot() + if err != nil { + return "", err + } + return doRawProcSelfFdReadlink(procRoot, fd) +} + +func procSelfFdReadlink(f *os.File) (string, error) { + return rawProcSelfFdReadlink(int(f.Fd())) +} + +var ( + errPossibleBreakout = errors.New("possible breakout detected") + errInvalidDirectory = errors.New("wandered into deleted directory") + errDeletedInode = errors.New("cannot verify path of deleted inode") +) + +func isDeadInode(file *os.File) error { + // If the nlink of a file drops to 0, there is an attacker deleting + // directories during our walk, which could result in weird /proc values. + // It's better to error out in this case. + stat, err := fstat(file) + if err != nil { + return fmt.Errorf("check for dead inode: %w", err) + } + if stat.Nlink == 0 { + err := errDeletedInode + if stat.Mode&unix.S_IFMT == unix.S_IFDIR { + err = errInvalidDirectory + } + return fmt.Errorf("%w %q", err, file.Name()) + } + return nil +} + +func getUmask() int { + // umask is a per-thread property, but it is inherited by children, so we + // need to lock our OS thread to make sure that no other goroutine runs in + // this thread and no goroutines are spawned from this thread until we + // revert to the old umask. + // + // We could parse /proc/self/status to avoid this get-set problem, but + // /proc/thread-self requires LockOSThread anyway, so there's no real + // benefit over just using umask(2). + runtime.LockOSThread() + umask := unix.Umask(0) + unix.Umask(umask) + runtime.UnlockOSThread() + return umask +} + +func checkProcSelfFdPath(path string, file *os.File) error { + if err := isDeadInode(file); err != nil { + return err + } + actualPath, err := procSelfFdReadlink(file) + if err != nil { + return fmt.Errorf("get path of handle: %w", err) + } + if actualPath != path { + return fmt.Errorf("%w: handle path %q doesn't match expected path %q", errPossibleBreakout, actualPath, path) + } + return nil +} diff --git a/vendor/github.com/cyphar/filepath-securejoin/testing_mocks_linux.go b/vendor/github.com/cyphar/filepath-securejoin/testing_mocks_linux.go new file mode 100644 index 0000000000..a3aedf03d1 --- /dev/null +++ b/vendor/github.com/cyphar/filepath-securejoin/testing_mocks_linux.go @@ -0,0 +1,68 @@ +//go:build linux + +// Copyright (C) 2024 SUSE LLC. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package securejoin + +import ( + "os" + "testing" +) + +type forceGetProcRootLevel int + +const ( + forceGetProcRootDefault forceGetProcRootLevel = iota + forceGetProcRootOpenTree // force open_tree() + forceGetProcRootOpenTreeAtRecursive // force open_tree(AT_RECURSIVE) + forceGetProcRootUnsafe // force open() +) + +var testingForceGetProcRoot *forceGetProcRootLevel + +func testingCheckClose(check bool, f *os.File) bool { + if check { + if f != nil { + _ = f.Close() + } + return true + } + return false +} + +func testingForcePrivateProcRootOpenTree(f *os.File) bool { + return testing.Testing() && testingForceGetProcRoot != nil && + testingCheckClose(*testingForceGetProcRoot >= forceGetProcRootOpenTree, f) +} + +func testingForcePrivateProcRootOpenTreeAtRecursive(f *os.File) bool { + return testing.Testing() && testingForceGetProcRoot != nil && + testingCheckClose(*testingForceGetProcRoot >= forceGetProcRootOpenTreeAtRecursive, f) +} + +func testingForceGetProcRootUnsafe() bool { + return testing.Testing() && testingForceGetProcRoot != nil && + *testingForceGetProcRoot >= forceGetProcRootUnsafe +} + +type forceProcThreadSelfLevel int + +const ( + forceProcThreadSelfDefault forceProcThreadSelfLevel = iota + forceProcSelfTask + forceProcSelf +) + +var testingForceProcThreadSelf *forceProcThreadSelfLevel + +func testingForceProcSelfTask() bool { + return testing.Testing() && testingForceProcThreadSelf != nil && + *testingForceProcThreadSelf >= forceProcSelfTask +} + +func testingForceProcSelf() bool { + return testing.Testing() && testingForceProcThreadSelf != nil && + *testingForceProcThreadSelf >= forceProcSelf +} diff --git a/vendor/github.com/cyphar/filepath-securejoin/vfs.go b/vendor/github.com/cyphar/filepath-securejoin/vfs.go index a82a5eae11..6e27c7dd8e 100644 --- a/vendor/github.com/cyphar/filepath-securejoin/vfs.go +++ b/vendor/github.com/cyphar/filepath-securejoin/vfs.go @@ -1,4 +1,4 @@ -// Copyright (C) 2017 SUSE LLC. All rights reserved. +// Copyright (C) 2017-2024 SUSE LLC. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/vendor/github.com/klauspost/compress/flate/matchlen_amd64.s b/vendor/github.com/klauspost/compress/flate/matchlen_amd64.s index 9a7655c0f7..0782b86e3d 100644 --- a/vendor/github.com/klauspost/compress/flate/matchlen_amd64.s +++ b/vendor/github.com/klauspost/compress/flate/matchlen_amd64.s @@ -5,7 +5,6 @@ #include "textflag.h" // func matchLen(a []byte, b []byte) int -// Requires: BMI TEXT ·matchLen(SB), NOSPLIT, $0-56 MOVQ a_base+0(FP), AX MOVQ b_base+24(FP), CX @@ -17,17 +16,16 @@ TEXT ·matchLen(SB), NOSPLIT, $0-56 JB matchlen_match4_standalone matchlen_loopback_standalone: - MOVQ (AX)(SI*1), BX - XORQ (CX)(SI*1), BX - TESTQ BX, BX - JZ matchlen_loop_standalone + MOVQ (AX)(SI*1), BX + XORQ (CX)(SI*1), BX + JZ matchlen_loop_standalone #ifdef GOAMD64_v3 TZCNTQ BX, BX #else BSFQ BX, BX #endif - SARQ $0x03, BX + SHRL $0x03, BX LEAL (SI)(BX*1), SI JMP gen_match_len_end diff --git a/vendor/github.com/klauspost/compress/zstd/dict.go b/vendor/github.com/klauspost/compress/zstd/dict.go index 8d5567fe64..b7b83164bc 100644 --- a/vendor/github.com/klauspost/compress/zstd/dict.go +++ b/vendor/github.com/klauspost/compress/zstd/dict.go @@ -273,6 +273,9 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { enc.Encode(&block, b) addValues(&remain, block.literals) litTotal += len(block.literals) + if len(block.sequences) == 0 { + continue + } seqs += len(block.sequences) block.genCodes() addHist(&ll, block.coders.llEnc.Histogram()) @@ -286,6 +289,9 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { if offset == 0 { continue } + if int(offset) >= len(o.History) { + continue + } if offset > 3 { newOffsets[offset-3]++ } else { @@ -336,6 +342,9 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { if seqs/nUsed < 512 { // Use 512 as minimum. nUsed = seqs / 512 + if nUsed == 0 { + nUsed = 1 + } } copyHist := func(dst *fseEncoder, src *[256]int) ([]byte, error) { hist := dst.Histogram() @@ -358,6 +367,28 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { fakeLength += v hist[i] = uint32(v) } + + // Ensure we aren't trying to represent RLE. + if maxCount == fakeLength { + for i := range hist { + if uint8(i) == maxSym { + fakeLength++ + maxSym++ + hist[i+1] = 1 + if maxSym > 1 { + break + } + } + if hist[0] == 0 { + fakeLength++ + hist[i] = 1 + if maxSym > 1 { + break + } + } + } + } + dst.HistogramFinished(maxSym, maxCount) dst.reUsed = false dst.useRLE = false diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s index 17901e0804..ae7d4d3295 100644 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s @@ -162,12 +162,12 @@ finalize: MOVD h, ret+24(FP) RET -// func writeBlocks(d *Digest, b []byte) int +// func writeBlocks(s *Digest, b []byte) int TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40 LDP ·primes+0(SB), (prime1, prime2) // Load state. Assume v[1-4] are stored contiguously. - MOVD d+0(FP), digest + MOVD s+0(FP), digest LDP 0(digest), (v1, v2) LDP 16(digest), (v3, v4) diff --git a/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.s b/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.s index 9a7655c0f7..0782b86e3d 100644 --- a/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.s +++ b/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.s @@ -5,7 +5,6 @@ #include "textflag.h" // func matchLen(a []byte, b []byte) int -// Requires: BMI TEXT ·matchLen(SB), NOSPLIT, $0-56 MOVQ a_base+0(FP), AX MOVQ b_base+24(FP), CX @@ -17,17 +16,16 @@ TEXT ·matchLen(SB), NOSPLIT, $0-56 JB matchlen_match4_standalone matchlen_loopback_standalone: - MOVQ (AX)(SI*1), BX - XORQ (CX)(SI*1), BX - TESTQ BX, BX - JZ matchlen_loop_standalone + MOVQ (AX)(SI*1), BX + XORQ (CX)(SI*1), BX + JZ matchlen_loop_standalone #ifdef GOAMD64_v3 TZCNTQ BX, BX #else BSFQ BX, BX #endif - SARQ $0x03, BX + SHRL $0x03, BX LEAL (SI)(BX*1), SI JMP gen_match_len_end diff --git a/vendor/github.com/moby/sys/mountinfo/mounted_linux.go b/vendor/github.com/moby/sys/mountinfo/mounted_linux.go index e78e726196..58f13c269d 100644 --- a/vendor/github.com/moby/sys/mountinfo/mounted_linux.go +++ b/vendor/github.com/moby/sys/mountinfo/mounted_linux.go @@ -51,7 +51,7 @@ func mountedByOpenat2(path string) (bool, error) { Resolve: unix.RESOLVE_NO_XDEV, }) _ = unix.Close(dirfd) - switch err { //nolint:errorlint // unix errors are bare + switch err { case nil: // definitely not a mount _ = unix.Close(fd) return false, nil diff --git a/vendor/github.com/moby/sys/user/user.go b/vendor/github.com/moby/sys/user/user.go index 984466d1ab..198c493679 100644 --- a/vendor/github.com/moby/sys/user/user.go +++ b/vendor/github.com/moby/sys/user/user.go @@ -197,7 +197,6 @@ func ParseGroupFilter(r io.Reader, filter func(Group) bool) ([]Group, error) { for { var line []byte line, isPrefix, err = rd.ReadLine() - if err != nil { // We should return no error if EOF is reached // without a match. diff --git a/vendor/github.com/opencontainers/selinux/pkg/pwalk/README.md b/vendor/github.com/opencontainers/selinux/pkg/pwalk/README.md deleted file mode 100644 index 7e78dce015..0000000000 --- a/vendor/github.com/opencontainers/selinux/pkg/pwalk/README.md +++ /dev/null @@ -1,48 +0,0 @@ -## pwalk: parallel implementation of filepath.Walk - -This is a wrapper for [filepath.Walk](https://pkg.go.dev/path/filepath?tab=doc#Walk) -which may speed it up by calling multiple callback functions (WalkFunc) in parallel, -utilizing goroutines. - -By default, it utilizes 2\*runtime.NumCPU() goroutines for callbacks. -This can be changed by using WalkN function which has the additional -parameter, specifying the number of goroutines (concurrency). - -### pwalk vs pwalkdir - -This package is deprecated in favor of -[pwalkdir](https://pkg.go.dev/github.com/opencontainers/selinux/pkg/pwalkdir), -which is faster, but requires at least Go 1.16. - -### Caveats - -Please note the following limitations of this code: - -* Unlike filepath.Walk, the order of calls is non-deterministic; - -* Only primitive error handling is supported: - - * filepath.SkipDir is not supported; - - * no errors are ever passed to WalkFunc; - - * once any error is returned from any WalkFunc instance, no more new calls - to WalkFunc are made, and the error is returned to the caller of Walk; - - * if more than one walkFunc instance will return an error, only one - of such errors will be propagated and returned by Walk, others - will be silently discarded. - -### Documentation - -For the official documentation, see -https://pkg.go.dev/github.com/opencontainers/selinux/pkg/pwalk?tab=doc - -### Benchmarks - -For a WalkFunc that consists solely of the return statement, this -implementation is about 10% slower than the standard library's -filepath.Walk. - -Otherwise (if a WalkFunc is doing something) this is usually faster, -except when the WalkN(..., 1) is used. diff --git a/vendor/github.com/opencontainers/selinux/pkg/pwalk/pwalk.go b/vendor/github.com/opencontainers/selinux/pkg/pwalk/pwalk.go deleted file mode 100644 index a28b4c4bbb..0000000000 --- a/vendor/github.com/opencontainers/selinux/pkg/pwalk/pwalk.go +++ /dev/null @@ -1,123 +0,0 @@ -package pwalk - -import ( - "fmt" - "os" - "path/filepath" - "runtime" - "sync" -) - -// WalkFunc is the type of the function called by Walk to visit each -// file or directory. It is an alias for [filepath.WalkFunc]. -// -// Deprecated: use [github.com/opencontainers/selinux/pkg/pwalkdir] and [fs.WalkDirFunc]. -type WalkFunc = filepath.WalkFunc - -// Walk is a wrapper for filepath.Walk which can call multiple walkFn -// in parallel, allowing to handle each item concurrently. A maximum of -// twice the runtime.NumCPU() walkFn will be called at any one time. -// If you want to change the maximum, use WalkN instead. -// -// The order of calls is non-deterministic. -// -// Note that this implementation only supports primitive error handling: -// -// - no errors are ever passed to walkFn; -// -// - once a walkFn returns any error, all further processing stops -// and the error is returned to the caller of Walk; -// -// - filepath.SkipDir is not supported; -// -// - if more than one walkFn instance will return an error, only one -// of such errors will be propagated and returned by Walk, others -// will be silently discarded. -// -// Deprecated: use [github.com/opencontainers/selinux/pkg/pwalkdir.Walk] -func Walk(root string, walkFn WalkFunc) error { - return WalkN(root, walkFn, runtime.NumCPU()*2) -} - -// WalkN is a wrapper for filepath.Walk which can call multiple walkFn -// in parallel, allowing to handle each item concurrently. A maximum of -// num walkFn will be called at any one time. -// -// Please see Walk documentation for caveats of using this function. -// -// Deprecated: use [github.com/opencontainers/selinux/pkg/pwalkdir.WalkN] -func WalkN(root string, walkFn WalkFunc, num int) error { - // make sure limit is sensible - if num < 1 { - return fmt.Errorf("walk(%q): num must be > 0", root) - } - - files := make(chan *walkArgs, 2*num) - errCh := make(chan error, 1) // get the first error, ignore others - - // Start walking a tree asap - var ( - err error - wg sync.WaitGroup - - rootLen = len(root) - rootEntry *walkArgs - ) - wg.Add(1) - go func() { - err = filepath.Walk(root, func(p string, info os.FileInfo, err error) error { - if err != nil { - close(files) - return err - } - if len(p) == rootLen { - // Root entry is processed separately below. - rootEntry = &walkArgs{path: p, info: &info} - return nil - } - // add a file to the queue unless a callback sent an error - select { - case e := <-errCh: - close(files) - return e - default: - files <- &walkArgs{path: p, info: &info} - return nil - } - }) - if err == nil { - close(files) - } - wg.Done() - }() - - wg.Add(num) - for i := 0; i < num; i++ { - go func() { - for file := range files { - if e := walkFn(file.path, *file.info, nil); e != nil { - select { - case errCh <- e: // sent ok - default: // buffer full - } - } - } - wg.Done() - }() - } - - wg.Wait() - - if err == nil { - err = walkFn(rootEntry.path, *rootEntry.info, nil) - } - - return err -} - -// walkArgs holds the arguments that were passed to the Walk or WalkN -// functions. -type walkArgs struct { - info *os.FileInfo - path string -} diff --git a/vendor/modules.txt b/vendor/modules.txt index 98de683489..058499fcca 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,8 +1,8 @@ # dario.cat/mergo v1.0.0 ## explicit; go 1.13 dario.cat/mergo -# github.com/BurntSushi/toml v1.3.2 -## explicit; go 1.16 +# github.com/BurntSushi/toml v1.4.0 +## explicit; go 1.18 github.com/BurntSushi/toml github.com/BurntSushi/toml/internal # github.com/Masterminds/semver/v3 v3.2.1 @@ -17,7 +17,7 @@ github.com/Microsoft/go-winio/internal/socket github.com/Microsoft/go-winio/internal/stringbuffer github.com/Microsoft/go-winio/pkg/guid github.com/Microsoft/go-winio/vhd -# github.com/Microsoft/hcsshim v0.12.3 +# github.com/Microsoft/hcsshim v0.12.5 ## explicit; go 1.21 github.com/Microsoft/hcsshim github.com/Microsoft/hcsshim/computestorage @@ -165,7 +165,7 @@ github.com/containers/ocicrypt/keywrap/pkcs7 github.com/containers/ocicrypt/spec github.com/containers/ocicrypt/utils github.com/containers/ocicrypt/utils/keyprovider -# github.com/containers/storage v1.54.0 +# github.com/containers/storage v1.55.0 ## explicit; go 1.21 github.com/containers/storage github.com/containers/storage/drivers @@ -218,8 +218,8 @@ github.com/coreos/go-oidc/v3/oidc # github.com/cyberphone/json-canonicalization v0.0.0-20231217050601-ba74d44ecf5f ## explicit github.com/cyberphone/json-canonicalization/go/src/webpki.org/jsoncanonicalizer -# github.com/cyphar/filepath-securejoin v0.2.5 -## explicit; go 1.13 +# github.com/cyphar/filepath-securejoin v0.3.1 +## explicit; go 1.20 github.com/cyphar/filepath-securejoin # github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc ## explicit @@ -378,7 +378,7 @@ github.com/josharian/intern # github.com/json-iterator/go v1.1.12 ## explicit; go 1.12 github.com/json-iterator/go -# github.com/klauspost/compress v1.17.8 +# github.com/klauspost/compress v1.17.9 ## explicit; go 1.20 github.com/klauspost/compress github.com/klauspost/compress/flate @@ -422,11 +422,11 @@ github.com/mitchellh/mapstructure # github.com/moby/docker-image-spec v1.3.1 ## explicit; go 1.18 github.com/moby/docker-image-spec/specs-go/v1 -# github.com/moby/sys/mountinfo v0.7.1 -## explicit; go 1.16 -github.com/moby/sys/mountinfo -# github.com/moby/sys/user v0.1.0 +# github.com/moby/sys/mountinfo v0.7.2 ## explicit; go 1.17 +github.com/moby/sys/mountinfo +# github.com/moby/sys/user v0.2.0 +## explicit; go 1.21 github.com/moby/sys/user # github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd ## explicit @@ -455,7 +455,6 @@ github.com/opencontainers/runtime-spec/specs-go ## explicit; go 1.19 github.com/opencontainers/selinux/go-selinux github.com/opencontainers/selinux/go-selinux/label -github.com/opencontainers/selinux/pkg/pwalk github.com/opencontainers/selinux/pkg/pwalkdir # github.com/opentracing/opentracing-go v1.2.0 ## explicit; go 1.14