From be5fde385bcb709b8de771ad6733d86afc06adc1 Mon Sep 17 00:00:00 2001 From: Tosone Date: Thu, 8 Feb 2024 13:28:20 +0800 Subject: [PATCH] :sparkles: Add support oss (#298) * :sparkles: Remove offset in storage read api * :sparkles: Add support oss --- .github/workflows/gh-pages.yml | 2 +- .github/workflows/lint.yml | 4 +- .github/workflows/local-build.yml | 4 +- .github/workflows/test.yml | 42 ++- README.md | 2 +- build/Dockerfile | 8 +- build/Dockerfile.builder | 4 +- build/Dockerfile.debian | 8 +- conf/config-dev.yaml | 5 + go.mod | 44 +-- go.sum | 93 ++++--- pkg/builder/logger/obs/obs.go | 2 +- pkg/configs/configuration.go | 11 +- pkg/configs/default.go | 2 - pkg/handlers/caches/caches_get.go | 2 +- pkg/handlers/distribution/blob/blob_get.go | 2 +- .../distribution/manifest/manifest_put.go | 2 +- .../distribution/upload/upload_put.go | 2 +- pkg/storage/cos/cos.go | 7 +- pkg/storage/cos/cos_test.go | 4 +- pkg/storage/filesystem/filesystem.go | 10 +- pkg/storage/filesystem/filesystem_test.go | 14 +- pkg/storage/oss/oss.go | 251 +++++++++++++++++- pkg/storage/oss/oss_test.go | 157 +++++++++++ pkg/storage/s3/s3.go | 57 ++-- pkg/storage/s3/s3_test.go | 2 +- pkg/storage/storage.go | 3 +- 27 files changed, 584 insertions(+), 160 deletions(-) create mode 100644 pkg/storage/oss/oss_test.go diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 0b4ec50b..846fb28b 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - node: ["18"] + node: ["20"] steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 026a8efa..e8b3a5a7 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -15,8 +15,8 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - go: ["1.21"] - node: ["18"] + go: ["1.22"] + node: ["20"] steps: - name: Checkout branch uses: actions/checkout@v4 diff --git a/.github/workflows/local-build.yml b/.github/workflows/local-build.yml index 3082a02e..e73ac03e 100644 --- a/.github/workflows/local-build.yml +++ b/.github/workflows/local-build.yml @@ -15,8 +15,8 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - go: ["1.21"] - node: ["18"] + go: ["1.22"] + node: ["20"] steps: - name: Checkout branch uses: actions/checkout@v4 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a38ce5c5..7666148f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,8 +15,8 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - go: ["1.21"] - node: ["18"] + go: ["1.22"] + node: ["20"] services: mysql: image: mysql:8.0 @@ -76,6 +76,10 @@ jobs: COS_ENDPOINT: ${{ secrets.COS_ENDPOINT_SQLITE }} COS_AK: ${{ secrets.COS_AK }} COS_SK: ${{ secrets.COS_SK }} + OSS_ENDPOINT: ${{ secrets.OSS_ENDPOINT }} + OSS_AK: ${{ secrets.OSS_AK }} + OSS_SK: ${{ secrets.OSS_SK }} + OSS_BUCKET: ${{ secrets.OSS_BUCKET_SQLITE }} run: | CI_DATABASE_TYPE=sqlite3 go test -tags "timetzdata,exclude_graphdriver_devicemapper,exclude_graphdriver_btrfs,containers_image_openpgp" -timeout 30m -v -coverprofile=coverage.txt -covermode=atomic `go list ./... | grep -v "pkg/tests" | grep -v "pkg/dal/query" | grep -v "pkg/dal/cmd" | grep -v "pkg/types/enums" | grep -v "pkg/handlers/apidocs" | grep -v "pkg/utils/token/mocks" | grep -v "pkg/utils/password/mocks" | grep -v "pkg/handlers/distribution/clients/mocks"` - name: Run tests @@ -84,8 +88,12 @@ jobs: COS_ENDPOINT: ${{ secrets.COS_ENDPOINT_SQLITE }} COS_AK: ${{ secrets.COS_AK }} COS_SK: ${{ secrets.COS_SK }} + OSS_ENDPOINT: ${{ secrets.OSS_ENDPOINT }} + OSS_AK: ${{ secrets.OSS_AK }} + OSS_SK: ${{ secrets.OSS_SK }} + OSS_BUCKET: ${{ secrets.OSS_BUCKET_SQLITE }} run: | - CI_DATABASE_TYPE=sqlite3 go test -tags "timetzdata,exclude_graphdriver_devicemapper,exclude_graphdriver_btrfs,containers_image_openpgp" -timeout 30m -v -coverprofile=coverage.txt -covermode=atomic `go list ./... | grep -v "pkg/tests" | grep -v "pkg/dal/query" | grep -v "pkg/dal/cmd" | grep -v "pkg/types/enums" | grep -v "pkg/handlers/apidocs" | grep -v "pkg/utils/token/mocks" | grep -v "pkg/utils/password/mocks" | grep -v "pkg/handlers/distribution/clients/mocks" | grep -v "pkg/storage/cos"` + CI_DATABASE_TYPE=sqlite3 go test -tags "timetzdata,exclude_graphdriver_devicemapper,exclude_graphdriver_btrfs,containers_image_openpgp" -timeout 30m -v -coverprofile=coverage.txt -covermode=atomic `go list ./... | grep -v "pkg/tests" | grep -v "pkg/dal/query" | grep -v "pkg/dal/cmd" | grep -v "pkg/types/enums" | grep -v "pkg/handlers/apidocs" | grep -v "pkg/utils/token/mocks" | grep -v "pkg/utils/password/mocks" | grep -v "pkg/handlers/distribution/clients/mocks" | grep -v "pkg/storage/cos" | grep -v "pkg/storage/oss"` - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 with: @@ -96,8 +104,8 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - go: ["1.21"] - node: ["18"] + go: ["1.22"] + node: ["20"] services: mysql: image: mysql:8.0 @@ -157,6 +165,10 @@ jobs: COS_ENDPOINT: ${{ secrets.COS_ENDPOINT_POSTGRESQL }} COS_AK: ${{ secrets.COS_AK }} COS_SK: ${{ secrets.COS_SK }} + OSS_ENDPOINT: ${{ secrets.OSS_ENDPOINT }} + OSS_AK: ${{ secrets.OSS_AK }} + OSS_SK: ${{ secrets.OSS_SK }} + OSS_BUCKET: ${{ secrets.OSS_BUCKET_POSTGRESQL }} run: | CI_DATABASE_TYPE=postgresql go test -tags "timetzdata,exclude_graphdriver_devicemapper,exclude_graphdriver_btrfs,containers_image_openpgp" -timeout 30m -v -coverprofile=coverage.txt -covermode=atomic `go list ./... | grep -v "pkg/tests" | grep -v "pkg/dal/query" | grep -v "pkg/dal/cmd" | grep -v "pkg/types/enums" | grep -v "pkg/handlers/apidocs" | grep -v "pkg/utils/token/mocks" | grep -v "pkg/utils/password/mocks" | grep -v "pkg/handlers/distribution/clients/mocks"` - name: Run tests @@ -165,8 +177,12 @@ jobs: COS_ENDPOINT: ${{ secrets.COS_ENDPOINT_POSTGRESQL }} COS_AK: ${{ secrets.COS_AK }} COS_SK: ${{ secrets.COS_SK }} + OSS_ENDPOINT: ${{ secrets.OSS_ENDPOINT }} + OSS_AK: ${{ secrets.OSS_AK }} + OSS_SK: ${{ secrets.OSS_SK }} + OSS_BUCKET: ${{ secrets.OSS_BUCKET_POSTGRESQL }} run: | - CI_DATABASE_TYPE=postgresql go test -tags "timetzdata,exclude_graphdriver_devicemapper,exclude_graphdriver_btrfs,containers_image_openpgp" -timeout 30m -v -coverprofile=coverage.txt -covermode=atomic `go list ./... | grep -v "pkg/tests" | grep -v "pkg/dal/query" | grep -v "pkg/dal/cmd" | grep -v "pkg/types/enums" | grep -v "pkg/handlers/apidocs" | grep -v "pkg/utils/token/mocks" | grep -v "pkg/utils/password/mocks" | grep -v "pkg/handlers/distribution/clients/mocks" | grep -v "pkg/storage/cos"` + CI_DATABASE_TYPE=postgresql go test -tags "timetzdata,exclude_graphdriver_devicemapper,exclude_graphdriver_btrfs,containers_image_openpgp" -timeout 30m -v -coverprofile=coverage.txt -covermode=atomic `go list ./... | grep -v "pkg/tests" | grep -v "pkg/dal/query" | grep -v "pkg/dal/cmd" | grep -v "pkg/types/enums" | grep -v "pkg/handlers/apidocs" | grep -v "pkg/utils/token/mocks" | grep -v "pkg/utils/password/mocks" | grep -v "pkg/handlers/distribution/clients/mocks" | grep -v "pkg/storage/cos" | grep -v "pkg/storage/oss"` - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 with: @@ -178,8 +194,8 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - go: ["1.21"] - node: ["18"] + go: ["1.22"] + node: ["20"] services: mysql: image: mysql:8.0 @@ -239,6 +255,10 @@ jobs: COS_ENDPOINT: ${{ secrets.COS_ENDPOINT_MYSQL }} COS_AK: ${{ secrets.COS_AK }} COS_SK: ${{ secrets.COS_SK }} + OSS_ENDPOINT: ${{ secrets.OSS_ENDPOINT }} + OSS_AK: ${{ secrets.OSS_AK }} + OSS_SK: ${{ secrets.OSS_SK }} + OSS_BUCKET: ${{ secrets.OSS_BUCKET_MYSQL }} run: | CI_DATABASE_TYPE=mysql go test -tags "timetzdata,exclude_graphdriver_devicemapper,exclude_graphdriver_btrfs,containers_image_openpgp" -timeout 30m -v -coverprofile=coverage.txt -covermode=atomic `go list ./... | grep -v "pkg/tests" | grep -v "pkg/dal/query" | grep -v "pkg/dal/cmd" | grep -v "pkg/types/enums" | grep -v "pkg/handlers/apidocs" | grep -v "pkg/utils/token/mocks" | grep -v "pkg/utils/password/mocks" | grep -v "pkg/handlers/distribution/clients/mocks"` - name: Run tests @@ -247,8 +267,12 @@ jobs: COS_ENDPOINT: ${{ secrets.COS_ENDPOINT_MYSQL }} COS_AK: ${{ secrets.COS_AK }} COS_SK: ${{ secrets.COS_SK }} + OSS_ENDPOINT: ${{ secrets.OSS_ENDPOINT }} + OSS_AK: ${{ secrets.OSS_AK }} + OSS_SK: ${{ secrets.OSS_SK }} + OSS_BUCKET: ${{ secrets.OSS_BUCKET_MYSQL }} run: | - CI_DATABASE_TYPE=mysql go test -tags "timetzdata,exclude_graphdriver_devicemapper,exclude_graphdriver_btrfs,containers_image_openpgp" -timeout 30m -v -coverprofile=coverage.txt -covermode=atomic `go list ./... | grep -v "pkg/tests" | grep -v "pkg/dal/query" | grep -v "pkg/dal/cmd" | grep -v "pkg/types/enums" | grep -v "pkg/handlers/apidocs" | grep -v "pkg/utils/token/mocks" | grep -v "pkg/utils/password/mocks" | grep -v "pkg/handlers/distribution/clients/mocks" | grep -v "pkg/storage/cos"` + CI_DATABASE_TYPE=mysql go test -tags "timetzdata,exclude_graphdriver_devicemapper,exclude_graphdriver_btrfs,containers_image_openpgp" -timeout 30m -v -coverprofile=coverage.txt -covermode=atomic `go list ./... | grep -v "pkg/tests" | grep -v "pkg/dal/query" | grep -v "pkg/dal/cmd" | grep -v "pkg/types/enums" | grep -v "pkg/handlers/apidocs" | grep -v "pkg/utils/token/mocks" | grep -v "pkg/utils/password/mocks" | grep -v "pkg/handlers/distribution/clients/mocks" | grep -v "pkg/storage/cos" | grep -v "pkg/storage/oss"` - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 with: diff --git a/README.md b/README.md index 7e5fb0f9..f04ccc72 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ ![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/go-sigma/sigma/test.yml?style=for-the-badge) ![Codecov](https://img.shields.io/codecov/c/github/go-sigma/sigma?style=for-the-badge) ![GitHub repo size](https://img.shields.io/github/repo-size/go-sigma/sigma?style=for-the-badge) -Sigma is an image registry that is extremely easy to deploy and maintain. and it adheres to the interface standards defined by the [OCI Distribution Specification 1.1](https://github.com/opencontainers/distribution-spec/tree/v1.1.0-rc4). It can also support any other client programs that follow the interface definition of the OCI Distribution Specification, such as [oras](https://github.com/oras-project/oras), [apptainer](https://github.com/apptainer/apptainer), [helm](https://github.com/helm/helm), and [nerdctl](https://github.com/containerd/nerdctl). +Sigma is an image registry that is extremely easy to deploy and maintain, and it adheres to the interface standards defined by the [OCI Distribution Specification 1.1](https://github.com/opencontainers/distribution-spec/tree/v1.1.0-rc4), it can also support any other client programs that follow the interface definition of the OCI Distribution Specification, such as [oras](https://github.com/oras-project/oras), [apptainer](https://github.com/apptainer/apptainer), [helm](https://github.com/helm/helm), and [nerdctl](https://github.com/containerd/nerdctl). ## Demo Server diff --git a/build/Dockerfile b/build/Dockerfile index a1494a1c..e2e9a9dc 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -1,6 +1,6 @@ -ARG GOLANG_VERSION=1.21.6-alpine3.18 -ARG NODE_VERSION=18-alpine3.18 -ARG ALPINE_VERSION=3.18 +ARG GOLANG_VERSION=1.22.0-alpine3.19 +ARG NODE_VERSION=20-alpine3.19 +ARG ALPINE_VERSION=3.19 FROM --platform=$BUILDPLATFORM node:${NODE_VERSION} as web-builder @@ -33,7 +33,7 @@ RUN set -eux && \ FROM alpine:${ALPINE_VERSION} as trivy ARG USE_MIRROR=false -ARG TRIVY_VERSION=0.48.3 +ARG TRIVY_VERSION=0.49.0 ARG ORAS_VERSION=1.0.0 ARG TARGETOS TARGETARCH diff --git a/build/Dockerfile.builder b/build/Dockerfile.builder index 2a1389cd..69845ed1 100644 --- a/build/Dockerfile.builder +++ b/build/Dockerfile.builder @@ -1,6 +1,6 @@ -ARG GOLANG_VERSION=1.21.6-alpine3.18 +ARG GOLANG_VERSION=1.22.0-alpine3.19 ARG BUILDKIT_VERSION=v0.12.4-rootless -ARG ALPINE_VERSION=3.18 +ARG ALPINE_VERSION=3.19 FROM alpine:${ALPINE_VERSION} as cosign diff --git a/build/Dockerfile.debian b/build/Dockerfile.debian index a412d7b7..b3856ba6 100644 --- a/build/Dockerfile.debian +++ b/build/Dockerfile.debian @@ -1,6 +1,6 @@ -ARG GOLANG_VERSION=1.21.6-bookworm -ARG NODE_VERSION=18-alpine3.18 -ARG ALPINE_VERSION=3.18 +ARG GOLANG_VERSION=1.22.0-bookworm +ARG NODE_VERSION=20-alpine3.19 +ARG ALPINE_VERSION=3.19 ARG DEBIAN_VERSION=bookworm-slim FROM --platform=$BUILDPLATFORM node:${NODE_VERSION} as web-builder @@ -34,7 +34,7 @@ RUN set -eux && \ FROM alpine:${ALPINE_VERSION} as trivy ARG USE_MIRROR=false -ARG TRIVY_VERSION=0.48.3 +ARG TRIVY_VERSION=0.49.0 ARG ORAS_VERSION=1.0.0 ARG TARGETOS TARGETARCH diff --git a/conf/config-dev.yaml b/conf/config-dev.yaml index 163a9722..4c9516fa 100644 --- a/conf/config-dev.yaml +++ b/conf/config-dev.yaml @@ -96,6 +96,11 @@ storage: region: cn-north-1 bucket: sigma forcePathStyle: true + oss: + ak: sigma + sk: sigma-sigma + endpoint: http://127.0.0.1:9000 + forcePathStyle: true # Notice: the tag never update after the first pulled from remote registry, unless you delete the image and pull again. proxy: diff --git a/go.mod b/go.mod index 808505c3..422cc178 100644 --- a/go.mod +++ b/go.mod @@ -10,11 +10,11 @@ require ( github.com/alicebob/miniredis/v2 v2.31.1 github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/anchore/syft v0.101.0 - github.com/aquasecurity/trivy v0.48.3 - github.com/aws/aws-sdk-go v1.50.8 + github.com/aquasecurity/trivy v0.49.0 + github.com/aws/aws-sdk-go v1.50.13 github.com/bytedance/json v0.0.0-20190516032711-0d89175f1949 github.com/caarlos0/env/v9 v9.0.0 - github.com/casbin/casbin/v2 v2.81.0 + github.com/casbin/casbin/v2 v2.82.0 github.com/casbin/gorm-adapter/v3 v3.20.0 github.com/containers/podman/v4 v4.9.2 github.com/deckarep/golang-set/v2 v2.6.0 @@ -44,7 +44,7 @@ require ( github.com/labstack/echo/v4 v4.11.4 github.com/matoous/go-nanoid v1.5.0 github.com/matoous/go-nanoid/v2 v2.0.0 - github.com/mattn/go-sqlite3 v1.14.21 + github.com/mattn/go-sqlite3 v1.14.22 github.com/mholt/archiver/v3 v3.5.1 github.com/moby/buildkit v0.12.5 github.com/opencontainers/distribution-spec/specs-go v0.0.0-20240127010305-05cf62402d05 @@ -52,7 +52,7 @@ require ( github.com/opencontainers/image-spec v1.1.0-rc6 github.com/redis/go-redis/v9 v9.4.0 github.com/robfig/cron/v3 v3.0.1 - github.com/rs/zerolog v1.31.0 + github.com/rs/zerolog v1.32.0 github.com/sigstore/cosign/v2 v2.2.3 github.com/smartystreets/goconvey v1.8.1 github.com/spf13/cast v1.6.0 @@ -60,22 +60,22 @@ require ( github.com/spf13/viper v1.18.2 github.com/stretchr/testify v1.8.4 github.com/swaggo/echo-swagger v1.4.1 - github.com/swaggo/swag v1.16.2 + github.com/swaggo/swag v1.16.3 github.com/tencentyun/cos-go-sdk-v5 v0.7.46 github.com/tidwall/gjson v1.17.0 github.com/wagslane/go-password-validator v0.3.0 - github.com/xanzy/go-gitlab v0.96.0 + github.com/xanzy/go-gitlab v0.97.0 go.uber.org/mock v0.4.0 - golang.org/x/crypto v0.18.0 - golang.org/x/exp v0.0.0-20240119083558-1b970713d09a - golang.org/x/net v0.20.0 + golang.org/x/crypto v0.19.0 + golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3 + golang.org/x/net v0.21.0 golang.org/x/oauth2 v0.16.0 gopkg.in/yaml.v3 v3.0.1 - gorm.io/driver/mysql v1.5.2 - gorm.io/driver/postgres v1.5.4 - gorm.io/driver/sqlite v1.5.4 + gorm.io/driver/mysql v1.5.4 + gorm.io/driver/postgres v1.5.6 + gorm.io/driver/sqlite v1.5.5 gorm.io/gen v0.3.25 - gorm.io/gorm v1.25.6 + gorm.io/gorm v1.25.7 gorm.io/plugin/dbresolver v1.5.0 gorm.io/plugin/soft_delete v1.2.1 k8s.io/api v0.29.1 @@ -119,7 +119,7 @@ require ( github.com/clbanning/mxj v1.8.4 // indirect github.com/cloudflare/circl v1.3.7 // indirect github.com/containerd/cgroups/v3 v3.0.2 // indirect - github.com/containerd/containerd v1.7.11 // indirect + github.com/containerd/containerd v1.7.12 // indirect github.com/containerd/continuity v0.4.2 // indirect github.com/containerd/fifo v1.1.0 // indirect github.com/containerd/log v0.1.0 // indirect @@ -144,7 +144,7 @@ require ( github.com/disiqueira/gotree/v3 v3.0.2 // indirect github.com/docker/distribution v2.8.3+incompatible // indirect github.com/docker/docker-credential-helpers v0.8.0 // indirect - github.com/docker/go-connections v0.4.1-0.20231031175723-0b8c1f4e07a0 // indirect + github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect @@ -260,6 +260,7 @@ require ( github.com/moby/sys/mountinfo v0.7.1 // indirect github.com/moby/sys/sequential v0.5.0 // indirect github.com/moby/sys/signal v0.7.0 // indirect + github.com/moby/sys/user v0.1.0 // indirect github.com/moby/term v0.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -276,6 +277,7 @@ require ( github.com/opencontainers/selinux v1.11.0 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/ostreedev/ostree-go v0.0.0-20210805093236-719684c64e4f // indirect + github.com/package-url/packageurl-go v0.1.2 // indirect github.com/pborman/indent v1.2.1 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect @@ -291,7 +293,7 @@ require ( github.com/rivo/uniseg v0.4.4 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect - github.com/samber/lo v1.38.1 // indirect + github.com/samber/lo v1.39.0 // indirect github.com/sassoftware/relic v7.2.1+incompatible // indirect github.com/scylladb/go-set v1.0.3-0.20200225121959-cc7b2070d91e // indirect github.com/secure-systems-lab/go-securesystemslib v0.8.0 // indirect @@ -346,8 +348,8 @@ require ( go.uber.org/zap v1.26.0 // indirect golang.org/x/mod v0.14.0 // indirect golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.16.0 // indirect - golang.org/x/term v0.16.0 // indirect + golang.org/x/sys v0.17.0 // indirect + golang.org/x/term v0.17.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.17.0 // indirect @@ -367,9 +369,9 @@ require ( gorm.io/datatypes v1.2.0 // indirect gorm.io/driver/sqlserver v1.5.1 // indirect gorm.io/hints v1.1.2 // indirect - k8s.io/klog/v2 v2.110.1 // indirect + k8s.io/klog/v2 v2.120.0 // indirect k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect - k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect + k8s.io/utils v0.0.0-20231127182322-b307cd553661 // indirect modernc.org/libc v1.37.6 // indirect modernc.org/mathutil v1.6.0 // indirect modernc.org/memory v1.7.2 // indirect diff --git a/go.sum b/go.sum index b7c17ef0..469a5e0e 100644 --- a/go.sum +++ b/go.sum @@ -203,8 +203,8 @@ github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHG github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/aquasecurity/trivy v0.48.3 h1:DrN/ErD1QF20LvqtwuSdNUr91ALjp5q9qZhTk/1wlT0= -github.com/aquasecurity/trivy v0.48.3/go.mod h1:lIbs00AWZ0M2KBjAxA8DCDy95DqM1Gr9zysC4SfGvmI= +github.com/aquasecurity/trivy v0.49.0 h1:PXz1H+A2ss+8cqQhkXFvN7X21DAX43adtVoO73OreeU= +github.com/aquasecurity/trivy v0.49.0/go.mod h1:vuNV0olltU3ml6GPXZseCb8pbkLMyWEEYa9ZbYMxz4s= github.com/aquasecurity/trivy-db v0.0.0-20231005141211-4fc651f7ac8d h1:fjI9mkoTUAkbGqpzt9nJsO24RAdfG+ZSiLFj0G2jO8c= github.com/aquasecurity/trivy-db v0.0.0-20231005141211-4fc651f7ac8d/go.mod h1:cj9/QmD9N3OZnKQMp+/DvdV+ym3HyIkd4e+F0ZM3ZGs= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= @@ -219,8 +219,10 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:l github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= -github.com/aws/aws-sdk-go v1.50.8 h1:gY0WoOW+/Wz6XmYSgDH9ge3wnAevYDSQWPxxJvqAkP4= -github.com/aws/aws-sdk-go v1.50.8/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go v1.50.11 h1:BxUqMbkzKzzQ6FXLlZFNFkFjHnv6utbc4PItVs5SvaE= +github.com/aws/aws-sdk-go v1.50.11/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go v1.50.13 h1:yeXram2g7q8uKkQkAEeZyk9FmPzxI4UpGwAZGZtEGmM= +github.com/aws/aws-sdk-go v1.50.13/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aws/aws-sdk-go-v2 v1.24.1 h1:xAojnj+ktS95YZlDf0zxWBkbFtymPeDP+rvUQIH3uAU= github.com/aws/aws-sdk-go-v2 v1.24.1/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4= github.com/aws/aws-sdk-go-v2/config v1.26.6 h1:Z/7w9bUqlRI0FFQpetVuFYEsjzE3h7fpU6HuGmfPL/o= @@ -283,8 +285,8 @@ github.com/bytedance/json v0.0.0-20190516032711-0d89175f1949 h1:JG8x1j2Jvu4e1Jwz github.com/bytedance/json v0.0.0-20190516032711-0d89175f1949/go.mod h1:sP1wlZn6XebMh0e4IPCNmVKCwQIXK4CDjUsQUbi0Ewo= github.com/caarlos0/env/v9 v9.0.0 h1:SI6JNsOA+y5gj9njpgybykATIylrRMklbs5ch6wO6pc= github.com/caarlos0/env/v9 v9.0.0/go.mod h1:ye5mlCVMYh6tZ+vCgrs/B95sj88cg5Tlnc0XIzgZ020= -github.com/casbin/casbin/v2 v2.81.0 h1:vNwJXK7a+TJZElZ5saP+SFJvweZNtJ3MlVP6P4IuRqE= -github.com/casbin/casbin/v2 v2.81.0/go.mod h1:jX8uoN4veP85O/n2674r2qtfSXI6myvxW85f6TH50fw= +github.com/casbin/casbin/v2 v2.82.0 h1:2CgvunqQQoepcbGRnMc9vEcDhuqh3B5yWKoj+kKSxf8= +github.com/casbin/casbin/v2 v2.82.0/go.mod h1:jX8uoN4veP85O/n2674r2qtfSXI6myvxW85f6TH50fw= github.com/casbin/gorm-adapter/v3 v3.20.0 h1:VpGKTlL56xIkhNUOC07bnzwjA/xqfVOAbkt6sniVxMo= github.com/casbin/gorm-adapter/v3 v3.20.0/go.mod h1:pvTTuyP2Es8VPHLyUssGtvOb3ETYD2tG7TfT5K8X2Sg= github.com/casbin/govaluate v1.1.0 h1:6xdCWIpE9CwHdZhlVQW+froUrCsjb6/ZYNcXODfLT+E= @@ -379,8 +381,8 @@ github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09Zvgq github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= -github.com/containerd/containerd v1.7.11 h1:lfGKw3eU35sjV0aG2eYZTiwFEY1pCzxdzicHP3SZILw= -github.com/containerd/containerd v1.7.11/go.mod h1:5UluHxHTX2rdvYuZ5OJTC5m/KJNs0Zs9wVoJm9zf5ZE= +github.com/containerd/containerd v1.7.12 h1:+KQsnv4VnzyxWcfO9mlxxELaoztsDEjOuCMPAuPqgU0= +github.com/containerd/containerd v1.7.12/go.mod h1:/5OMpE1p0ylxtEUGY8kuCYkDRzJm9NO1TFMWjUpdevk= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= @@ -547,8 +549,8 @@ github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avu github.com/docker/docker-credential-helpers v0.8.0 h1:YQFtbBQb4VrpoPxhFuzEBPQ9E16qz5SpHLS+uswaCp8= github.com/docker/docker-credential-helpers v0.8.0/go.mod h1:UGFXcuoQ5TxPiB54nHOZ32AWRqQdECoh/Mg0AlEYb40= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-connections v0.4.1-0.20231031175723-0b8c1f4e07a0 h1:dPD5pdqsujF9jz2NQMQCDzrBSAF3M6kIxmfU98IOp9c= -github.com/docker/go-connections v0.4.1-0.20231031175723-0b8c1f4e07a0/go.mod h1:a6bNUGTbQBsY6VRHTr4h/rkOXjl244DyRD0tx3fgq4Q= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= @@ -664,7 +666,6 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -1206,8 +1207,8 @@ github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lL github.com/mattn/go-sqlite3 v1.14.3/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI= github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= -github.com/mattn/go-sqlite3 v1.14.21 h1:IXocQLOykluc3xPE0Lvy8FtggMz1G+U3mEjg+0zGizc= -github.com/mattn/go-sqlite3 v1.14.21/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= @@ -1264,6 +1265,8 @@ github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWK github.com/moby/sys/signal v0.7.0 h1:25RW3d5TnQEoKvRbEKUGay6DCQ46IxAVTT9CUMgmsSI= github.com/moby/sys/signal v0.7.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= +github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= +github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= @@ -1374,6 +1377,8 @@ github.com/ostreedev/ostree-go v0.0.0-20210805093236-719684c64e4f h1:/UDgs8FGMqw github.com/ostreedev/ostree-go v0.0.0-20210805093236-719684c64e4f/go.mod h1:J6OG6YJVEWopen4avK3VNQSnALmmjvniMmni/YFYAwc= github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= +github.com/package-url/packageurl-go v0.1.2 h1:0H2DQt6DHd/NeRlVwW4EZ4oEI6Bn40XlNPRqegcxuo4= +github.com/package-url/packageurl-go v0.1.2/go.mod h1:uQd4a7Rh3ZsVg5j0lNyAfyxIeGde9yrlhjF78GzeW0c= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/indent v1.2.1 h1:lFiviAbISHv3Rf0jcuh489bi06hj98JsVMtIDZQb9yM= @@ -1474,8 +1479,8 @@ github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= -github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= -github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0= +github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -1487,8 +1492,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= -github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM= -github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= +github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA= +github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= github.com/sassoftware/relic v7.2.1+incompatible h1:Pwyh1F3I0r4clFJXkSI8bOyJINGqpgjJU3DYAZeI05A= github.com/sassoftware/relic v7.2.1+incompatible/go.mod h1:CWfAxv73/iLZ17rbyhIEq3K9hs5w6FpNMdUT//qR+zk= github.com/sassoftware/relic/v7 v7.6.1 h1:O5s8ewCgq5QYNpv45dK4u6IpBmDM9RIcsbf/G1uXepQ= @@ -1622,8 +1627,8 @@ github.com/swaggo/echo-swagger v1.4.1 h1:Yf0uPaJWp1uRtDloZALyLnvdBeoEL5Kc7DtnjzO github.com/swaggo/echo-swagger v1.4.1/go.mod h1:C8bSi+9yH2FLZsnhqMZLIZddpUxZdBYuNHbtaS1Hljc= github.com/swaggo/files/v2 v2.0.0 h1:hmAt8Dkynw7Ssz46F6pn8ok6YmGZqHSVLZ+HQM7i0kw= github.com/swaggo/files/v2 v2.0.0/go.mod h1:24kk2Y9NYEJ5lHuCra6iVwkMjIekMCaFq/0JQj66kyM= -github.com/swaggo/swag v1.16.2 h1:28Pp+8DkQoV+HLzLx8RGJZXNGKbFqnuvSbAAtoxiY04= -github.com/swaggo/swag v1.16.2/go.mod h1:6YzXnDcpr0767iOejs318CwYkCQqyGer6BizOg03f+E= +github.com/swaggo/swag v1.16.3 h1:PnCYjPCah8FK4I26l2F/KQ4yz3sILcVUN3cTlBFA9Pg= +github.com/swaggo/swag v1.16.3/go.mod h1:DImHIuOFXKpMFAQjcC7FG4m3Dg4+QuUgUzJmKjI/gRk= github.com/sylabs/sif/v2 v2.15.0 h1:Nv0tzksFnoQiQ2eUwpAis9nVqEu4c3RcNSxX8P3Cecw= github.com/sylabs/sif/v2 v2.15.0/go.mod h1:X1H7eaPz6BAxA84POMESXoXfTqgAnLQkujyF/CQFWTc= github.com/sylabs/squashfs v0.6.1 h1:4hgvHnD9JGlYWwT0bPYNt9zaz23mAV3Js+VEgQoRGYQ= @@ -1694,8 +1699,8 @@ github.com/wagslane/go-password-validator v0.3.0 h1:vfxOPzGHkz5S146HDpavl0cw1DSV github.com/wagslane/go-password-validator v0.3.0/go.mod h1:TI1XJ6T5fRdRnHqHt14pvy1tNVnrwe7m3/f1f2fDphQ= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= -github.com/xanzy/go-gitlab v0.96.0 h1:LGkZ+wSNMRtHIBaYE4Hq3dZVjprwHv3Y1+rhKU3WETs= -github.com/xanzy/go-gitlab v0.96.0/go.mod h1:ETg8tcj4OhrB84UEgeE8dSuV/0h4BBL1uOV/qK0vlyI= +github.com/xanzy/go-gitlab v0.97.0 h1:StMqJ1Kvt00X43pYIBBjj52dFlghwSeBhRDRfzaZ7xY= +github.com/xanzy/go-gitlab v0.97.0/go.mod h1:ETg8tcj4OhrB84UEgeE8dSuV/0h4BBL1uOV/qK0vlyI= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= @@ -1825,8 +1830,8 @@ golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= -golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1837,8 +1842,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA= -golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= +golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3 h1:/RIbNt/Zr7rVhIkQhooTxCxFcdWLGIKnZA4IXNFSrvo= +golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1934,8 +1939,8 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2098,8 +2103,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -2110,8 +2115,8 @@ golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= -golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= -golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= +golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2438,14 +2443,14 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/datatypes v1.2.0 h1:5YT+eokWdIxhJgWHdrb2zYUimyk0+TaFth+7a0ybzco= gorm.io/datatypes v1.2.0/go.mod h1:o1dh0ZvjIjhH/bngTpypG6lVRJ5chTBxE09FH/71k04= gorm.io/driver/mysql v1.4.3/go.mod h1:sSIebwZAVPiT+27jK9HIwvsqOGKx3YMPmrA3mBJR10c= -gorm.io/driver/mysql v1.5.2 h1:QC2HRskSE75wBuOxe0+iCkyJZ+RqpudsQtqkp+IMuXs= -gorm.io/driver/mysql v1.5.2/go.mod h1:pQLhh1Ut/WUAySdTHwBpBv6+JKcj+ua4ZFx1QQTBzb8= -gorm.io/driver/postgres v1.5.4 h1:Iyrp9Meh3GmbSuyIAGyjkN+n9K+GHX9b9MqsTL4EJCo= -gorm.io/driver/postgres v1.5.4/go.mod h1:Bgo89+h0CRcdA33Y6frlaHHVuTdOf87pmyzwW9C/BH0= +gorm.io/driver/mysql v1.5.4 h1:igQmHfKcbaTVyAIHNhhB888vvxh8EdQ2uSUT0LPcBso= +gorm.io/driver/mysql v1.5.4/go.mod h1:9rYxJph/u9SWkWc9yY4XJ1F/+xO0S/ChOmbk3+Z5Tvs= +gorm.io/driver/postgres v1.5.6 h1:ydr9xEd5YAM0vxVDY0X139dyzNz10spDiDlC7+ibLeU= +gorm.io/driver/postgres v1.5.6/go.mod h1:3e019WlBaYI5o5LIdNV+LyxCMNtLOQETBXL2h4chKpA= gorm.io/driver/sqlite v1.1.3/go.mod h1:AKDgRWk8lcSQSw+9kxCJnX/yySj8G3rdwYlU57cB45c= gorm.io/driver/sqlite v1.5.0/go.mod h1:kDMDfntV9u/vuMmz8APHtHF0b4nyBB7sfCieC6G8k8I= -gorm.io/driver/sqlite v1.5.4 h1:IqXwXi8M/ZlPzH/947tn5uik3aYQslP9BVveoax0nV0= -gorm.io/driver/sqlite v1.5.4/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4= +gorm.io/driver/sqlite v1.5.5 h1:7MDMtUZhV065SilG62E0MquljeArQZNfJnjd9i9gx3E= +gorm.io/driver/sqlite v1.5.5/go.mod h1:6NgQ7sQWAIFsPrJJl1lSNSu2TABh0ZZ/zm5fosATavE= gorm.io/driver/sqlserver v1.5.1 h1:wpyW/pR26U94uaujltiFGXY7fd2Jw5hC9PB1ZF/Y5s4= gorm.io/driver/sqlserver v1.5.1/go.mod h1:AYHzzte2msKTmYBYsSIq8ZUsznLJwBdkB2wpI+kt0nM= gorm.io/gen v0.3.25 h1:uT/1YfvcnYUdike4XPYyi89FEnVHZF115GUXQm2Sfug= @@ -2456,10 +2461,10 @@ gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= gorm.io/gorm v1.24.7-0.20230306060331-85eaf9eeda11/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= gorm.io/gorm v1.25.0/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= gorm.io/gorm v1.25.1/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= -gorm.io/gorm v1.25.2-0.20230530020048-26663ab9bf55/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= gorm.io/gorm v1.25.2/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= -gorm.io/gorm v1.25.6 h1:V92+vVda1wEISSOMtodHVRcUIOPYa2tgQtyF+DfFx+A= -gorm.io/gorm v1.25.6/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +gorm.io/gorm v1.25.7 h1:VsD6acwRjz2zFxGO50gPO6AkNs7KKnvfzUjHQhZDz/A= +gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= gorm.io/hints v1.1.2 h1:b5j0kwk5p4+3BtDtYqqfY+ATSxjj+6ptPgVveuynn9o= gorm.io/hints v1.1.2/go.mod h1:/ARdpUHAtyEMCh5NNi3tI7FsGh+Cj/MIUlvNxCNCFWg= gorm.io/plugin/dbresolver v1.5.0 h1:XVHLxh775eP0CqVh3vcfJtYqja3uFl5Wr3cKlY8jgDY= @@ -2509,16 +2514,16 @@ k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAE k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= -k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= +k8s.io/klog/v2 v2.120.0 h1:z+q5mfovBj1fKFxiRzsa2DsJLPIVMk/KFL81LMOfK+8= +k8s.io/klog/v2 v2.120.0/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20231127182322-b307cd553661 h1:FepOBzJ0GXm8t0su67ln2wAZjbQ6RxQGZDnzuLcrUTI= +k8s.io/utils v0.0.0-20231127182322-b307cd553661/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= modernc.org/libc v1.37.6 h1:orZH3c5wmhIQFTXF+Nt+eeauyd+ZIt2BX6ARe+kD+aw= modernc.org/libc v1.37.6/go.mod h1:YAXkAZ8ktnkCKaN9sw/UDeUVkGYJ/YquGO4FTi5nmHE= modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= diff --git a/pkg/builder/logger/obs/obs.go b/pkg/builder/logger/obs/obs.go index 625f7085..372a7a32 100644 --- a/pkg/builder/logger/obs/obs.go +++ b/pkg/builder/logger/obs/obs.go @@ -88,5 +88,5 @@ func (w *writer) Close() error { // Read returns a reader for the given id func (o *obs) Read(ctx context.Context, id int64) (io.Reader, error) { - return o.storage.Reader(ctx, fmt.Sprintf("%s.log.gz", path.Join(consts.BuilderLogs, utils.DirWithSlash(strconv.FormatInt(id, 10)))), 0) + return o.storage.Reader(ctx, fmt.Sprintf("%s.log.gz", path.Join(consts.BuilderLogs, utils.DirWithSlash(strconv.FormatInt(id, 10))))) } diff --git a/pkg/configs/configuration.go b/pkg/configs/configuration.go index 72cb2ee8..701341b3 100644 --- a/pkg/configs/configuration.go +++ b/pkg/configs/configuration.go @@ -203,6 +203,15 @@ type ConfigurationStorageQiniu struct { UseHTTPS bool `yaml:"useHttps"` } +// ConfigurationStorageOss ... +type ConfigurationStorageOss struct { + Ak string `yaml:"ak"` + Sk string `yaml:"sk"` + Bucket string `yaml:"bucket"` + Endpoint string `yaml:"endpoint"` + ForcePathStyle bool `yaml:"forcePathStyle"` +} + // ConfigurationStorage ... type ConfigurationStorage struct { RootDirectory string `yaml:"rootDirectory"` @@ -211,7 +220,7 @@ type ConfigurationStorage struct { Filesystem ConfigurationStorageFilesystem `yaml:"filesystem"` S3 ConfigurationStorageS3 `yaml:"s3"` Cos ConfigurationStorageCos `yaml:"cos"` - Qiniu ConfigurationStorageQiniu `yaml:"qiniu"` + Oss ConfigurationStorageOss `yaml:"oss"` } // ConfigurationProxy ... diff --git a/pkg/configs/default.go b/pkg/configs/default.go index 426b5246..b384e8d3 100644 --- a/pkg/configs/default.go +++ b/pkg/configs/default.go @@ -27,8 +27,6 @@ func defaultSettings() { viper.SetDefault("auth.jwt.ttl", time.Hour) // the jwt token ttl viper.SetDefault("auth.jwt.refreshTtl", time.Hour*24) // the refresh token ttl - viper.SetDefault("storage.rootDirectory", "/var/lib/sigma") // the root directory for filesystem storage - viper.SetDefault("server.endpoint", "http://127.0.0.1:3000") viper.SetDefault("server.internalEndpoint", "http://127.0.0.1:3000") diff --git a/pkg/handlers/caches/caches_get.go b/pkg/handlers/caches/caches_get.go index 375d3372..c231af0a 100644 --- a/pkg/handlers/caches/caches_get.go +++ b/pkg/handlers/caches/caches_get.go @@ -52,7 +52,7 @@ func (h *handler) GetCache(c echo.Context) error { } var path = h.genPath(req.BuilderID) - reader, err := storage.Driver.Reader(ctx, path, 0) + reader, err := storage.Driver.Reader(ctx, path) if err != nil { if errors.Is(err, os.ErrNotExist) { log.Error().Err(err).Str("cache", path).Msg("Cache not found") diff --git a/pkg/handlers/distribution/blob/blob_get.go b/pkg/handlers/distribution/blob/blob_get.go index ea0ef785..f559516e 100644 --- a/pkg/handlers/distribution/blob/blob_get.go +++ b/pkg/handlers/distribution/blob/blob_get.go @@ -156,7 +156,7 @@ func (h *handler) GetBlob(c echo.Context) error { return c.Redirect(http.StatusPermanentRedirect, redirectUrl) } - reader, err := storage.Driver.Reader(ctx, path.Join(consts.Blobs, utils.GenPathByDigest(dgest)), 0) + reader, err := storage.Driver.Reader(ctx, path.Join(consts.Blobs, utils.GenPathByDigest(dgest))) if err != nil { log.Error().Err(err).Str("digest", dgest.String()).Msg("Get blob reader failed") return xerrors.NewDSError(c, xerrors.DSErrCodeUnknown) diff --git a/pkg/handlers/distribution/manifest/manifest_put.go b/pkg/handlers/distribution/manifest/manifest_put.go index 63d40c91..74df7689 100644 --- a/pkg/handlers/distribution/manifest/manifest_put.go +++ b/pkg/handlers/distribution/manifest/manifest_put.go @@ -196,7 +196,7 @@ func (h *handler) PutManifest(c echo.Context) error { } else { for _, reference := range manifest.References() { if reference.MediaType == "application/vnd.oci.image.config.v1+json" || reference.MediaType == "application/vnd.docker.container.image.v1+json" || reference.MediaType == "application/vnd.cncf.helm.config.v1+json" || reference.MediaType == "application/vnd.sylabs.sif.config.v1+json" { - configRawReader, err := storage.Driver.Reader(ctx, path.Join(consts.Blobs, utils.GenPathByDigest(reference.Digest)), 0) + configRawReader, err := storage.Driver.Reader(ctx, path.Join(consts.Blobs, utils.GenPathByDigest(reference.Digest))) if err != nil { log.Error().Err(err).Str("digest", reference.Digest.String()).Msg("Get image config raw layer failed") return xerrors.NewDSError(c, xerrors.DSErrCodeUnknown) diff --git a/pkg/handlers/distribution/upload/upload_put.go b/pkg/handlers/distribution/upload/upload_put.go index 55ddf753..74f3873c 100644 --- a/pkg/handlers/distribution/upload/upload_put.go +++ b/pkg/handlers/distribution/upload/upload_put.go @@ -170,7 +170,7 @@ func (h *handler) PutUpload(c echo.Context) error { return xerrors.NewDSError(c, xerrors.DSErrCodeUnknown) } - srcPathReader, err := storage.Driver.Reader(ctx, srcPath, 0) + srcPathReader, err := storage.Driver.Reader(ctx, srcPath) if err != nil { log.Error().Err(err).Str("srcPath", srcPath).Msg("Get blob upload failed") return xerrors.NewDSError(c, xerrors.DSErrCodeUnknown) diff --git a/pkg/storage/cos/cos.go b/pkg/storage/cos/cos.go index ccec19f5..6a137f4d 100644 --- a/pkg/storage/cos/cos.go +++ b/pkg/storage/cos/cos.go @@ -22,7 +22,6 @@ import ( "net/url" "path" "reflect" - "strconv" "strings" gonanoid "github.com/matoous/go-nanoid" @@ -192,10 +191,8 @@ func (t *tencentcos) Delete(ctx context.Context, path string) error { } // Reader returns a reader for the given path. -func (t *tencentcos) Reader(ctx context.Context, path string, offset int64) (io.ReadCloser, error) { - opt := &cos.ObjectGetOptions{ - Range: fmt.Sprintf("bytes=%s-", strconv.FormatInt(offset, 10)), - } +func (t *tencentcos) Reader(ctx context.Context, path string) (io.ReadCloser, error) { + opt := &cos.ObjectGetOptions{} resp, err := t.client.Object.Get(ctx, path, opt) if err != nil { return nil, fmt.Errorf("Get object failed: %v", err) diff --git a/pkg/storage/cos/cos_test.go b/pkg/storage/cos/cos_test.go index 24cdacb6..4ce67656 100644 --- a/pkg/storage/cos/cos_test.go +++ b/pkg/storage/cos/cos_test.go @@ -84,11 +84,11 @@ func TestDelete(t *testing.T) { err = driver.Upload(ctx, "dir/unit-test", strings.NewReader("test")) assert.NoError(t, err) - reader, err := driver.Reader(ctx, "dir/unit-test", 2) + reader, err := driver.Reader(ctx, "dir/unit-test") assert.NoError(t, err) data, err := io.ReadAll(reader) assert.NoError(t, err) - assert.Equal(t, "st", string(data)) + assert.Equal(t, "test", string(data)) err = driver.Move(ctx, "dir/unit-test", "dir/unit-test-to") assert.NoError(t, err) diff --git a/pkg/storage/filesystem/filesystem.go b/pkg/storage/filesystem/filesystem.go index 1fe2ac23..f7a6b141 100644 --- a/pkg/storage/filesystem/filesystem.go +++ b/pkg/storage/filesystem/filesystem.go @@ -98,19 +98,11 @@ func (f *fs) Delete(ctx context.Context, path string) error { } // Reader returns a reader for the file at the given path -func (f *fs) Reader(ctx context.Context, path string, offset int64) (io.ReadCloser, error) { +func (f *fs) Reader(ctx context.Context, path string) (io.ReadCloser, error) { fp, err := os.OpenFile(storage.SanitizePath(f.rootDirectory, path), os.O_RDONLY, 0644) if err != nil { return nil, err } - seekPos, err := fp.Seek(offset, io.SeekStart) - if err != nil { - fp.Close() // nolint: errcheck - return nil, err - } else if seekPos < offset { - fp.Close() // nolint: errcheck - return nil, err - } return fp, nil } diff --git a/pkg/storage/filesystem/filesystem_test.go b/pkg/storage/filesystem/filesystem_test.go index 7530f211..1692682a 100644 --- a/pkg/storage/filesystem/filesystem_test.go +++ b/pkg/storage/filesystem/filesystem_test.go @@ -50,23 +50,23 @@ func TestNew(t *testing.T) { err = os.WriteFile("test/storage/unit-test", []byte("test"), 0600) assert.NoError(t, err) - reader, err := driver.Reader(context.Background(), "unit-test", 0) + reader, err := driver.Reader(context.Background(), "unit-test") assert.NoError(t, err) assert.NotNil(t, reader) dataBytes, err := io.ReadAll(reader) assert.NoError(t, err) assert.Equal(t, "test", string(dataBytes)) - reader, err = driver.Reader(context.Background(), "unit-test", 1) + reader, err = driver.Reader(context.Background(), "unit-test") assert.NoError(t, err) assert.NotNil(t, reader) dataBytes, err = io.ReadAll(reader) assert.NoError(t, err) - assert.Equal(t, "est", string(dataBytes)) - reader, err = driver.Reader(context.Background(), "unit-test", 1000) + assert.Equal(t, "test", string(dataBytes)) + reader, err = driver.Reader(context.Background(), "unit-test") assert.NoError(t, err) dataBytes, err = io.ReadAll(reader) assert.NoError(t, err) - assert.Equal(t, "", string(dataBytes)) + assert.Equal(t, "test", string(dataBytes)) err = driver.Delete(context.Background(), "unit-test") assert.NoError(t, err) @@ -81,7 +81,7 @@ func TestNew(t *testing.T) { assert.NotEmpty(t, tag2) err = driver.CommitUpload(context.Background(), "unit-test-path", uploadID, []string{tag1, tag2}) assert.NoError(t, err) - reader, err = driver.Reader(context.Background(), "unit-test-path", 0) + reader, err = driver.Reader(context.Background(), "unit-test-path") assert.NoError(t, err) assert.NotNil(t, reader) dataBytes, err = io.ReadAll(reader) @@ -106,7 +106,7 @@ func TestNew(t *testing.T) { err = driver.Upload(context.Background(), "unit-test-path", strings.NewReader("test")) assert.NoError(t, err) - reader, err = driver.Reader(context.Background(), "unit-test-path", 0) + reader, err = driver.Reader(context.Background(), "unit-test-path") assert.NoError(t, err) assert.NotNil(t, reader) dataBytes, err = io.ReadAll(reader) diff --git a/pkg/storage/oss/oss.go b/pkg/storage/oss/oss.go index b7941a6d..371ae791 100644 --- a/pkg/storage/oss/oss.go +++ b/pkg/storage/oss/oss.go @@ -15,12 +15,22 @@ package oss import ( + "context" + "fmt" + "io" + "net/http" + "net/textproto" + "os" "path" "reflect" + "strconv" + "strings" "github.com/aliyun/aliyun-oss-go-sdk/oss" + "github.com/rs/zerolog/log" "github.com/go-sigma/sigma/pkg/configs" + "github.com/go-sigma/sigma/pkg/consts" "github.com/go-sigma/sigma/pkg/storage" "github.com/go-sigma/sigma/pkg/utils" ) @@ -35,6 +45,243 @@ var _ storage.Factory = factory{} // New ... func (f factory) New(config configs.Configuration) (storage.StorageDriver, error) { - _, _ = oss.New("endpoint", "accessID", "accessKey") - return nil, nil + client, err := oss.New(config.Storage.Oss.Endpoint, config.Storage.Oss.Ak, config.Storage.Oss.Sk, + oss.ForcePathStyle(config.Storage.Oss.ForcePathStyle)) + if err != nil { + return nil, err + } + bucket, err := client.Bucket(config.Storage.Oss.Bucket) + if err != nil { + return nil, err + } + + return &alioss{ + client: bucket, + rootDirectory: strings.TrimPrefix(config.Storage.RootDirectory, "/"), + bucket: config.Storage.Oss.Bucket, + }, nil +} + +type alioss struct { + client *oss.Bucket + rootDirectory string + bucket string +} + +func (a *alioss) sanitizePath(p string) string { + return strings.TrimPrefix(strings.TrimPrefix(path.Join(a.rootDirectory, p), "."), "/") +} + +// Move moves an object stored at sourcePath to destPath, removing the +// original object. +// Note: This may be no more efficient than a copy followed by a delete for +// many implementations. +func (a *alioss) Move(ctx context.Context, srcPath string, dstPath string) error { + srcPath = a.sanitizePath(srcPath) + dstPath = a.sanitizePath(dstPath) + + header, err := a.client.GetObjectMeta(srcPath) + if err != nil { + return err + } + + srcSize, err := strconv.ParseInt(header.Get(textproto.CanonicalMIMEHeaderKey("Content-Length")), 10, 64) + if err != nil { + log.Error().Err(err).Interface("Headers", header).Msg("Get object header failed") + return fmt.Errorf("Convert header content-length to int failed: %v", err) + } + + if srcSize <= storage.MultipartCopyThresholdSize { + _, err := a.client.CopyObject(srcPath, dstPath) + if err != nil { + return err + } + return nil + } + + createResp, err := a.client.InitiateMultipartUpload(dstPath, oss.EnableMd5()) + if err != nil { + return err + } + + numParts := (srcSize + storage.MultipartCopyChunkSize - 1) / storage.MultipartCopyChunkSize + completedParts := make([]oss.UploadPart, numParts) + errChan := make(chan error, numParts) + limiter := make(chan struct{}, storage.MultipartCopyMaxConcurrency) + + for i := range completedParts { + i := i + go func() { + limiter <- struct{}{} + firstByte := int64(i) * storage.MultipartCopyChunkSize + lastByte := firstByte + storage.MultipartCopyChunkSize + if lastByte >= srcSize { + lastByte = srcSize + } + partSize := lastByte - firstByte + uploadResp, err := a.client.UploadPartCopy(createResp, a.bucket, srcPath, firstByte, partSize, i+1) + if err == nil { + completedParts[i] = uploadResp + } + errChan <- err + <-limiter + }() + } + + for range completedParts { + err := <-errChan + if err != nil { + return err + } + } + + _, err = a.client.CompleteMultipartUpload(createResp, completedParts) + return err +} + +// Delete recursively deletes all objects stored at "path" and its subpaths. +func (a *alioss) Delete(ctx context.Context, path string) error { + objects := make([]string, 0, storage.MaxPaginationKeys) + for { + result, err := a.client.ListObjectsV2(oss.MaxKeys(storage.MaxPaginationKeys), + oss.Prefix(storage.SanitizePath(a.rootDirectory, path))) + if err != nil { + return err + } + + for _, r := range result.CommonPrefixes { + err := a.Delete(ctx, r) + if err != nil { + return err + } + } + + for _, r := range result.Objects { + objects = append(objects, r.Key) + } + + for _, o := range objects { + err = a.client.DeleteObject(o) + if err != nil { + return err + } + } + + objects = objects[:0] + + if !result.IsTruncated { + break + } + } + return nil +} + +// Reader retrieves an io.ReadCloser for the content stored at "path" +// with a given byte offset. +func (a *alioss) Reader(ctx context.Context, path string) (io.ReadCloser, error) { + return a.client.GetObject(a.sanitizePath(path)) +} + +// CreateUploadID creates a new multipart upload and returns an +// opaque upload ID. +func (a *alioss) CreateUploadID(ctx context.Context, path string) (string, error) { + result, err := a.client.InitiateMultipartUpload(a.sanitizePath(path)) + if err != nil { + return "", err + } + return result.UploadID, nil +} + +// WritePart writes a part of a multipart upload. +func (a *alioss) UploadPart(ctx context.Context, path, uploadID string, partNumber int64, body io.Reader) (string, error) { + file, err := os.CreateTemp("", "alioss") + if err != nil { + return "", err + } + _, err = io.Copy(file, body) + if err != nil { + return "", err + } + err = file.Sync() + if err != nil { + return "", err + } + stat, err := file.Stat() + if err != nil { + return "", err + } + err = file.Close() + if err != nil { + return "", err + } + defer func() { + err := os.Remove(file.Name()) + if err != nil { + log.Error().Err(err).Str("File", file.Name()).Msg("Remove upload temp file failed") + } + }() + + result, err := a.client.UploadPartFromFile(oss.InitiateMultipartUploadResult{ + Bucket: a.bucket, + Key: a.sanitizePath(path), + UploadID: uploadID, + }, file.Name(), 0, stat.Size(), int(partNumber)) + if err != nil { + return "", err + } + return result.ETag, nil +} + +// CommitUpload commits a multipart upload. +func (a *alioss) CommitUpload(ctx context.Context, path string, uploadID string, parts []string) error { + var ps = make([]oss.UploadPart, 0, len(parts)) + for index, p := range parts { + ps = append(ps, oss.UploadPart{ + PartNumber: index + 1, + ETag: p, + }) + } + _, err := a.client.CompleteMultipartUpload(oss.InitiateMultipartUploadResult{ + Bucket: a.bucket, + Key: a.sanitizePath(path), + UploadID: uploadID, + }, ps) + return err +} + +// AbortUpload aborts a multipart upload. +func (a *alioss) AbortUpload(ctx context.Context, path string, uploadID string) error { + return a.client.AbortMultipartUpload(oss.InitiateMultipartUploadResult{ + Bucket: a.bucket, + Key: a.sanitizePath(path), + UploadID: uploadID, + }) +} + +// Upload upload a file to the given path. +func (a *alioss) Upload(ctx context.Context, path string, body io.Reader) error { + file, err := os.CreateTemp("", "alioss") + if err != nil { + return err + } + _, err = io.Copy(file, body) + if err != nil { + return err + } + err = file.Close() + if err != nil { + return err + } + defer func() { + err := os.Remove(file.Name()) + if err != nil { + log.Error().Err(err).Str("File", file.Name()).Msg("Remove upload temp file failed") + } + }() + return a.client.UploadFile(a.sanitizePath(path), file.Name(), storage.MultipartCopyChunkSize, oss.EnableMd5()) +} + +// Redirect get a temporary link +func (a *alioss) Redirect(ctx context.Context, path string) (string, error) { + return a.client.SignURL(a.sanitizePath(path), http.MethodGet, int64(consts.ObsPresignMaxTtl.Seconds())) } diff --git a/pkg/storage/oss/oss_test.go b/pkg/storage/oss/oss_test.go new file mode 100644 index 00000000..221d4556 --- /dev/null +++ b/pkg/storage/oss/oss_test.go @@ -0,0 +1,157 @@ +// Copyright 2023 sigma +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package oss + +import ( + "bytes" + "context" + "crypto/rand" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/go-sigma/sigma/pkg/configs" +) + +func TestBigFileMove(t *testing.T) { + ctx := context.Background() + var config = configs.Configuration{ + Storage: configs.ConfigurationStorage{ + Oss: configs.ConfigurationStorageOss{ + Ak: os.Getenv("OSS_AK"), + Sk: os.Getenv("OSS_SK"), + Bucket: os.Getenv("OSS_BUCKET"), + Endpoint: os.Getenv("OSS_ENDPOINT"), + ForcePathStyle: false, + }, + RootDirectory: "sigma", + }, + } + f := factory{} + driver, err := f.New(config) + assert.NoError(t, err) + + var bigFile = "test-big-file.bin" + originalFile, _ := os.Create(bigFile) + for i := 0; i < 100; i++ { // generate 100MB + data := make([]byte, 1<<20) + _, _ = rand.Read(data) + _, _ = originalFile.Write(data) + } + err = originalFile.Close() + assert.NoError(t, err) + + defer os.Remove(bigFile) // nolint: errcheck + + bigFileBytes, err := os.ReadFile(bigFile) + assert.NoError(t, err) + + err = driver.Upload(ctx, "big-file", bytes.NewReader(bigFileBytes)) + assert.NoError(t, err) + err = driver.Move(ctx, "big-file", "big-file-move-to") + assert.NoError(t, err) + + err = driver.Delete(ctx, "big-file") + assert.NoError(t, err) + err = driver.Delete(ctx, "big-file-move-to") + assert.NoError(t, err) +} + +func TestDelete(t *testing.T) { + ctx := context.Background() + var config = configs.Configuration{ + Storage: configs.ConfigurationStorage{ + Oss: configs.ConfigurationStorageOss{ + Ak: os.Getenv("OSS_AK"), + Sk: os.Getenv("OSS_SK"), + Bucket: os.Getenv("OSS_BUCKET"), + Endpoint: os.Getenv("OSS_ENDPOINT"), + ForcePathStyle: false, + }, + RootDirectory: "sigma", + }, + } + f := factory{} + driver, err := f.New(config) + assert.NoError(t, err) + + err = driver.Upload(ctx, "dir/unit-test/unit-test/test.txt", strings.NewReader("test")) + assert.NoError(t, err) + err = driver.Upload(ctx, "dir/unit-test/test.txt", strings.NewReader("test")) + assert.NoError(t, err) + err = driver.Upload(ctx, "dir/test.txt", strings.NewReader("test")) + assert.NoError(t, err) + + err = driver.Delete(ctx, "dir") + assert.NoError(t, err) +} + +func TestMultiUpload(t *testing.T) { + ctx := context.Background() + var config = configs.Configuration{ + Storage: configs.ConfigurationStorage{ + Oss: configs.ConfigurationStorageOss{ + Ak: os.Getenv("OSS_AK"), + Sk: os.Getenv("OSS_SK"), + Bucket: os.Getenv("OSS_BUCKET"), + Endpoint: os.Getenv("OSS_ENDPOINT"), + ForcePathStyle: false, + }, + RootDirectory: "sigma", + }, + } + f := factory{} + driver, err := f.New(config) + assert.NoError(t, err) + + uploadID, err := driver.CreateUploadID(ctx, "upload-test") + assert.NoError(t, err) + var bigFile1 = "test-big-file1.bin" + originalFile1, _ := os.Create(bigFile1) + for i := 0; i < 1; i++ { // 1M + data := make([]byte, 1<<20) + _, _ = rand.Read(data) + _, _ = originalFile1.Write(data) + } + _ = originalFile1.Close() + file1Bytes, err := os.ReadFile(bigFile1) + assert.NoError(t, err) + defer os.RemoveAll(bigFile1) // nolint: errcheck + etag1, err := driver.UploadPart(ctx, "upload-test", uploadID, 1, bytes.NewReader(file1Bytes)) + assert.NoError(t, err) + var bigFile2 = "test-big-file2.bin" + originalFile2, _ := os.Create(bigFile2) + for i := 0; i < 1; i++ { // 1M + data := make([]byte, 1<<20) + _, _ = rand.Read(data) + _, _ = originalFile2.Write(data) + } + _ = originalFile1.Close() + file2, _ := os.Open(bigFile2) + defer file2.Close() // nolint: errcheck + defer os.RemoveAll(bigFile2) // nolint: errcheck + etag2, err := driver.UploadPart(ctx, "upload-test", uploadID, 2, file2) + assert.NoError(t, err) + err = driver.CommitUpload(ctx, "upload-test", uploadID, []string{etag1, etag2}) + assert.NoError(t, err) + err = driver.Move(ctx, "upload-test", "upload-test-move-to") + assert.NoError(t, err) + err = driver.Delete(ctx, "upload-test-move-to") + assert.NoError(t, err) + err = driver.Delete(ctx, "upload-test") + assert.NoError(t, err) +} diff --git a/pkg/storage/s3/s3.go b/pkg/storage/s3/s3.go index 1b639993..faa47c2c 100644 --- a/pkg/storage/s3/s3.go +++ b/pkg/storage/s3/s3.go @@ -22,7 +22,6 @@ import ( "os" "path" "reflect" - "strconv" "strings" "github.com/aws/aws-sdk-go/aws" @@ -31,7 +30,6 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" - "github.com/spf13/viper" "github.com/go-sigma/sigma/pkg/configs" "github.com/go-sigma/sigma/pkg/consts" @@ -41,7 +39,7 @@ import ( ) type awss3 struct { - S3 *s3.S3 + client *s3.S3 uploader *s3manager.Uploader rootDirectory string bucket string @@ -56,27 +54,20 @@ type factory struct{} var _ storage.Factory = factory{} func (f factory) New(config configs.Configuration) (storage.StorageDriver, error) { - endpoint := config.Storage.S3.Endpoint - region := config.Storage.S3.Region - ak := config.Storage.S3.Ak - sk := config.Storage.S3.Sk - bucket := config.Storage.S3.Bucket - forcePathStyle := config.Storage.S3.ForcePathStyle - sess, err := session.NewSession(&aws.Config{ - Endpoint: aws.String(endpoint), - Region: aws.String(region), - S3ForcePathStyle: aws.Bool(forcePathStyle), - Credentials: credentials.NewStaticCredentials(ak, sk, ""), + Endpoint: aws.String(config.Storage.S3.Endpoint), + Region: aws.String(config.Storage.S3.Region), + S3ForcePathStyle: aws.Bool(config.Storage.S3.ForcePathStyle), + Credentials: credentials.NewStaticCredentials(config.Storage.S3.Ak, config.Storage.S3.Sk, ""), }) if err != nil { return nil, fmt.Errorf("failed to create new session with aws config: %v", err) } return &awss3{ - S3: s3.New(sess), + client: s3.New(sess), uploader: s3manager.NewUploader(sess), - bucket: bucket, - rootDirectory: strings.TrimPrefix(viper.GetString("storage.rootDirectory"), "/"), + bucket: config.Storage.S3.Bucket, + rootDirectory: strings.TrimPrefix(config.Storage.RootDirectory, "/"), }, nil } @@ -89,7 +80,7 @@ func (a *awss3) Move(ctx context.Context, srcPath string, dstPath string) error srcPath = a.sanitizePath(srcPath) dstPath = a.sanitizePath(dstPath) - srcFile, err := a.S3.HeadObject(&s3.HeadObjectInput{ + srcFile, err := a.client.HeadObject(&s3.HeadObjectInput{ Bucket: aws.String(a.bucket), Key: aws.String(srcPath), }) @@ -99,7 +90,7 @@ func (a *awss3) Move(ctx context.Context, srcPath string, dstPath string) error srcSize := ptr.To(srcFile.ContentLength) if srcSize <= storage.MultipartCopyThresholdSize { - _, err := a.S3.CopyObject(&s3.CopyObjectInput{ + _, err := a.client.CopyObject(&s3.CopyObjectInput{ Bucket: aws.String(a.bucket), Key: aws.String(dstPath), CopySource: aws.String(path.Join(a.bucket, srcPath)), @@ -110,7 +101,7 @@ func (a *awss3) Move(ctx context.Context, srcPath string, dstPath string) error return nil } - createResp, err := a.S3.CreateMultipartUpload(&s3.CreateMultipartUploadInput{ + createResp, err := a.client.CreateMultipartUpload(&s3.CreateMultipartUploadInput{ Bucket: aws.String(a.bucket), Key: aws.String(dstPath), }) @@ -132,7 +123,7 @@ func (a *awss3) Move(ctx context.Context, srcPath string, dstPath string) error if lastByte >= srcSize { lastByte = srcSize - 1 } - uploadResp, err := a.S3.UploadPartCopy(&s3.UploadPartCopyInput{ + uploadResp, err := a.client.UploadPartCopy(&s3.UploadPartCopyInput{ Bucket: aws.String(a.bucket), CopySource: aws.String(path.Join(a.bucket, srcPath)), Key: aws.String(dstPath), @@ -158,7 +149,7 @@ func (a *awss3) Move(ctx context.Context, srcPath string, dstPath string) error } } - _, err = a.S3.CompleteMultipartUpload(&s3.CompleteMultipartUploadInput{ + _, err = a.client.CompleteMultipartUpload(&s3.CompleteMultipartUploadInput{ Bucket: aws.String(a.bucket), Key: aws.String(dstPath), UploadId: createResp.UploadId, @@ -179,12 +170,12 @@ func (a *awss3) Delete(ctx context.Context, path string) error { for { // list all the objects - resp, err := a.S3.ListObjectsV2(listObjectsInput) + resp, err := a.client.ListObjectsV2(listObjectsInput) // resp.Contents can only be empty on the first call // if there were no more results to return after the first call, resp.IsTruncated would have been false // and the loop would exit without recalling ListObjects - if err != nil || len(resp.Contents) == 0 { + if err != nil { return fmt.Errorf("failed to list objects: %w", err) } @@ -204,7 +195,7 @@ func (a *awss3) Delete(ctx context.Context, path string) error { // by default the response returns up to 1,000 key names. The response _might_ contain fewer keys but it will never contain more. // 10000 keys is coincidentally (?) also the max number of keys that can be deleted in a single Delete operation, so we'll just smack // Delete here straight away and reset the object slice when successful. - resp, err := a.S3.DeleteObjects(&s3.DeleteObjectsInput{ + resp, err := a.client.DeleteObjects(&s3.DeleteObjectsInput{ Bucket: aws.String(a.bucket), Delete: &s3.Delete{ Objects: s3Objects, @@ -243,16 +234,14 @@ func (a *awss3) Delete(ctx context.Context, path string) error { } // Reader returns a reader for the given path. -func (a *awss3) Reader(ctx context.Context, path string, offset int64) (io.ReadCloser, error) { - resp, err := a.S3.GetObject(&s3.GetObjectInput{ +func (a *awss3) Reader(ctx context.Context, path string) (io.ReadCloser, error) { + resp, err := a.client.GetObject(&s3.GetObjectInput{ Bucket: aws.String(a.bucket), Key: aws.String(a.sanitizePath(path)), - Range: aws.String("bytes=" + strconv.FormatInt(offset, 10) + "-"), }) if err != nil { if awsErr, ok := err.(awserr.Error); ok { if awsErr.Code() == s3.ErrCodeNoSuchKey { - fmt.Println(254, awsErr.Error()) return nil, os.ErrNotExist } return nil, awsErr @@ -264,7 +253,7 @@ func (a *awss3) Reader(ctx context.Context, path string, offset int64) (io.ReadC // CreateUploadID creates a new upload ID. func (a *awss3) CreateUploadID(ctx context.Context, path string) (string, error) { - resp, err := a.S3.CreateMultipartUploadWithContext(ctx, &s3.CreateMultipartUploadInput{ + resp, err := a.client.CreateMultipartUploadWithContext(ctx, &s3.CreateMultipartUploadInput{ Bucket: aws.String(a.bucket), Key: aws.String(a.sanitizePath(path)), }) @@ -284,7 +273,7 @@ func (a *awss3) UploadPart(ctx context.Context, path, uploadID string, partNumbe if err != nil { return "", err } - resp, err := a.S3.UploadPartCopyWithContext(ctx, &s3.UploadPartCopyInput{ + resp, err := a.client.UploadPartCopyWithContext(ctx, &s3.UploadPartCopyInput{ Bucket: aws.String(a.bucket), Key: aws.String(a.sanitizePath(path)), UploadId: aws.String(uploadID), @@ -306,7 +295,7 @@ func (a *awss3) CommitUpload(ctx context.Context, path, uploadID string, parts [ PartNumber: aws.Int64(int64(i + 1)), } } - _, err := a.S3.CompleteMultipartUploadWithContext(ctx, &s3.CompleteMultipartUploadInput{ + _, err := a.client.CompleteMultipartUploadWithContext(ctx, &s3.CompleteMultipartUploadInput{ Bucket: aws.String(a.bucket), Key: aws.String(a.sanitizePath(path)), UploadId: aws.String(uploadID), @@ -317,7 +306,7 @@ func (a *awss3) CommitUpload(ctx context.Context, path, uploadID string, parts [ // AbortUpload aborts an upload. func (a *awss3) AbortUpload(ctx context.Context, path string, uploadID string) error { - _, err := a.S3.AbortMultipartUploadWithContext(ctx, &s3.AbortMultipartUploadInput{ + _, err := a.client.AbortMultipartUploadWithContext(ctx, &s3.AbortMultipartUploadInput{ Bucket: aws.String(a.bucket), Key: aws.String(storage.SanitizePath(a.rootDirectory, path)), UploadId: aws.String(uploadID), @@ -337,7 +326,7 @@ func (a *awss3) Upload(ctx context.Context, path string, body io.Reader) error { // Redirect get a temporary link func (a *awss3) Redirect(ctx context.Context, path string) (string, error) { - req, _ := a.S3.GetObjectRequest(&s3.GetObjectInput{ + req, _ := a.client.GetObjectRequest(&s3.GetObjectInput{ Bucket: aws.String(a.bucket), Key: aws.String(a.sanitizePath(path)), }) diff --git a/pkg/storage/s3/s3_test.go b/pkg/storage/s3/s3_test.go index 15fa4e93..e8e5ce56 100644 --- a/pkg/storage/s3/s3_test.go +++ b/pkg/storage/s3/s3_test.go @@ -67,7 +67,7 @@ func TestNew(t *testing.T) { err = driver.Move(ctx, "dir/unit-test", "dir/unit-test-to") assert.NoError(t, err) - reader, err := driver.Reader(ctx, "dir/unit-test", 0) + reader, err := driver.Reader(ctx, "dir/unit-test") assert.NoError(t, err) contentBytes, err := io.ReadAll(reader) assert.NoError(t, err) diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 6fca6a6d..86317d38 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -53,8 +53,7 @@ type StorageDriver interface { // Reader retrieves an io.ReadCloser for the content stored at "path" // with a given byte offset. - // May be used to resume reading a stream by providing a nonzero offset. - Reader(ctx context.Context, path string, offset int64) (io.ReadCloser, error) + Reader(ctx context.Context, path string) (io.ReadCloser, error) // CreateUploadID creates a new multipart upload and returns an // opaque upload ID.