diff --git a/.github/workflows/e2e-v2-rate-limit.yml b/.github/workflows/e2e-v2-rate-limit.yml new file mode 100644 index 00000000000..72c518cbf57 --- /dev/null +++ b/.github/workflows/e2e-v2-rate-limit.yml @@ -0,0 +1,153 @@ +name: E2E Test With Rate Limit(API v2 - Rust Client) + +on: + push: + branches: [main, release-*] + paths-ignore: ["**.md", "**.png", "**.jpg", "**.svg", "**/docs/**"] + pull_request: + branches: [main, release-*] + paths-ignore: ["**.md", "**.png", "**.jpg", "**.svg", "**/docs/**"] + schedule: + - cron: '0 4 * * *' + +permissions: + contents: read + +env: + KIND_VERSION: v0.12.0 + CONTAINERD_VERSION: v1.5.2 + KIND_CONFIG_PATH: test/testdata/kind/config-v2.yaml + DRAGONFLY_CHARTS_PATH: deploy/helm-charts/charts/dragonfly + DRAGONFLY_FILE_SERVER_PATH: test/testdata/k8s/file-server.yaml + +jobs: + e2e_tests: + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + matrix: + module: + - "normal" + include: + - module: normal + charts-config: test/testdata/charts/config-v2-rate-limit.yaml + skip: "" + focus: "'Download With RateLimit'" + + steps: + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be + with: + tool-cache: false + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + + - name: Checkout code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + submodules: recursive + fetch-depth: 0 + + - name: Install Go + uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a + with: + go-version-file: go.mod + + - name: Get dependencies + run: | + go install github.com/onsi/ginkgo/v2/ginkgo@v2.12.0 + mkdir -p /tmp/artifact + + - name: Setup buildx + uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db + id: buildx + with: + install: true + + - name: Cache Docker layers + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Pull Rust Client Image + run: | + cd client-rs + CLIENT_TAG=$(git describe --tags $(git rev-parse HEAD)) + docker pull dragonflyoss/client:$CLIENT_TAG + docker tag dragonflyoss/client:$CLIENT_TAG dragonflyoss/client:latest + docker pull dragonflyoss/dfinit:$CLIENT_TAG + docker tag dragonflyoss/dfinit:$CLIENT_TAG dragonflyoss/dfinit:latest + + - name: Build Scheduler Image + uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355 + with: + context: . + file: build/images/scheduler/Dockerfile + push: false + load: true + tags: dragonflyoss/scheduler:latest + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new + + - name: Build Manager Image + uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355 + with: + context: . + file: build/images/manager/Dockerfile + push: false + load: true + tags: dragonflyoss/manager:latest + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new + + - name: Setup Kind + uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 + with: + version: ${{ env.KIND_VERSION }} + config: ${{ env.KIND_CONFIG_PATH }} + cluster_name: kind + + - name: Kind load images + run: | + kind load docker-image dragonflyoss/manager:latest + kind load docker-image dragonflyoss/scheduler:latest + kind load docker-image dragonflyoss/client:latest + kind load docker-image dragonflyoss/dfinit:latest + + - name: Setup dragonfly + run: | + helm install --wait --timeout 15m --dependency-update --create-namespace --namespace dragonfly-system -f ${{ matrix.charts-config }} dragonfly ${{ env.DRAGONFLY_CHARTS_PATH }} + kubectl apply -f ${{ env.DRAGONFLY_FILE_SERVER_PATH }} + kubectl wait po file-server-0 --namespace dragonfly-e2e --for=condition=ready --timeout=10m + + - name: Run E2E test + run: | + ginkgo -v -r --race --fail-fast --cover --trace --show-node-events --skip=${{ matrix.skip }} --focus=${{ matrix.focus }} test/e2e/v2 + cat coverprofile.out >> coverage.txt + + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@1e68e06f1dbfde0e4cefc87efeba9e4643565303 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage.txt + flags: e2etests + + - name: Upload Logs + uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b + if: always() + with: + name: ${{ matrix.module }}-e2e-tests-logs + path: | + /tmp/artifact/** diff --git a/test/e2e/v2/rate_limit_test.go b/test/e2e/v2/rate_limit_test.go new file mode 100644 index 00000000000..85cb2f9e521 --- /dev/null +++ b/test/e2e/v2/rate_limit_test.go @@ -0,0 +1,59 @@ +/* + * Copyright 2024 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package e2e + +import ( + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" //nolint + . "github.com/onsi/gomega" //nolint + + "d7y.io/dragonfly/v2/test/e2e/v2/util" +) + +var _ = Describe("Download With RateLimit", func() { + Context("/bin/kubectl file", func() { + It("should be ok", Label("download", "rateLimit"), func() { + clientPod, err := util.ClientExec() + fmt.Println(err) + Expect(err).NotTo(HaveOccurred()) + + start := time.Now() + out, err := clientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -H 'X-Dragonfly-Tag: download-rate-limit' %s --output %s", util.GetFileURL("/bin/kubectl"), util.GetOutputPath("kubectl-proxy"))).CombinedOutput() + elapsed := time.Since(start) + fmt.Println(err) + Expect(err).NotTo(HaveOccurred()) + // The download rate limit is 1MiB/s, and the file size is about 44MiB, so the download time should be greater than 44s. + Expect(elapsed).Should(BeNumerically(">", 44*time.Second)) + fmt.Println(string(out)) + + fileMetadata := util.FileMetadata{ + ID: "6b94aa169d11ba67c1852f14c4df39e5ebf84d1c0d7a6f6ad0ba2f3547883f4f", + Sha256: "327b4022d0bfd1d5e9c0701d4a3f989a536f7e6e865e102dcd77c7e7adb31f9a", + } + + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) + Expect(err).NotTo(HaveOccurred()) + Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, util.GetOutputPath("kubectl-proxy")) + Expect(err).NotTo(HaveOccurred()) + Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + }) + }) +}) diff --git a/test/testdata/charts/config-v2-rate-limit.yaml b/test/testdata/charts/config-v2-rate-limit.yaml new file mode 100644 index 00000000000..e42410ed08c --- /dev/null +++ b/test/testdata/charts/config-v2-rate-limit.yaml @@ -0,0 +1,183 @@ +manager: + image: + repository: dragonflyoss/manager + tag: latest + replicas: 1 + resources: + requests: + cpu: "0" + memory: "0" + limits: + cpu: "1" + memory: "2Gi" + extraVolumeMounts: + - name: logs + mountPath: "/var/log/" + - name: artifact + mountPath: /tmp/artifact + extraVolumes: + - name: logs + emptyDir: { } + - name: artifact + hostPath: + path: /tmp/artifact + metrics: + enable: true + config: + console: false + verbose: true + job: + rateLimit: + fillInterval: 1m + capacity: 100 + quantum: 100 + +scheduler: + image: + repository: dragonflyoss/scheduler + tag: latest + replicas: 3 + resources: + requests: + cpu: "0" + memory: "0" + limits: + cpu: "2" + memory: "4Gi" + service: + type: NodePort + nodePort: 30802 + extraVolumeMounts: + - name: logs + mountPath: "/var/log/" + - name: artifact + mountPath: /tmp/artifact + extraVolumes: + - name: logs + emptyDir: { } + - name: artifact + hostPath: + path: /tmp/artifact + metrics: + enable: true + enableHost: true + config: + console: false + verbose: true + scheduler: + gc: + hostGCInterval: 2m + +seedClient: + enable: true + replicas: 3 + image: + repository: dragonflyoss/client + tag: latest + resources: + requests: + cpu: "0" + memory: "0" + limits: + cpu: "2" + memory: "4Gi" + extraVolumeMounts: + - name: logs + mountPath: "/var/log/" + - name: artifact + mountPath: /tmp/artifact + extraVolumes: + - name: logs + emptyDir: { } + - name: artifact + hostPath: + path: /tmp/artifact + config: + download: + rateLimit: 1MiB + upload: + rateLimit: 1MiB + dynconfig: + refreshInterval: 1s + scheduler: + announceInterval: 1s + log: + level: info + proxy: + prefetch: true + prefetchRateLimit: 1MiB + registryMirror: + addr: https://index.docker.io + rules: + - regex: blobs/sha256.* + - regxe: file-server.* + +client: + enable: true + image: + repository: dragonflyoss/client + tag: latest + resources: + requests: + cpu: "0" + memory: "0" + limits: + cpu: "2" + memory: "4Gi" + # Allow client daemonSet to create a pod on master node for testing when the daemon goes offline. + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + extraVolumeMounts: + - name: logs + mountPath: "/var/log/" + - name: artifact + mountPath: /tmp/artifact + extraVolumes: + - name: logs + emptyDir: { } + - name: artifact + hostPath: + path: /tmp/artifact + dfinit: + enable: true + image: + repository: dragonflyoss/dfinit + tag: latest + config: + containerRuntime: + containerd: + configPath: /etc/containerd/config.toml + registries: + - hostNamespace: docker.io + serverAddr: https://index.docker.io + capabilities: ["pull", "resolve"] + - hostNamespace: ghcr.io + serverAddr: https://ghcr.io + capabilities: ["pull", "resolve"] + config: + download: + rateLimit: 1MiB + upload: + rateLimit: 1MiB + dynconfig: + refreshInterval: 1s + scheduler: + announceInterval: 1s + log: + level: info + proxy: + prefetch: false + prefetchRateLimit: 1MiB + registryMirror: + addr: https://index.docker.io + rules: + - regex: blobs/sha256.* + - regxe: file-server.* + +dfdaemon: + enable: false + +seedPeer: + enable: false