[WIP] e2e: add the e2e tests for rate limit of download, upload and prefetch

.github/workflows/e2e-v2-rate-limit.yml

@@ -0,0 +1,153 @@
+name: E2E Test With Rate Limit(API v2 - Rust Client)
+
+on:
+  push:
+    branches: [main, release-*]
+    paths-ignore: ["**.md", "**.png", "**.jpg", "**.svg", "**/docs/**"]
+  pull_request:
+    branches: [main, release-*]
+    paths-ignore: ["**.md", "**.png", "**.jpg", "**.svg", "**/docs/**"]
+  schedule:
+    - cron: '0 4 * * *'
+
+permissions:
+  contents: read
+
+env:
+  KIND_VERSION: v0.12.0
+  CONTAINERD_VERSION: v1.5.2
+  KIND_CONFIG_PATH: test/testdata/kind/config-v2.yaml
+  DRAGONFLY_CHARTS_PATH: deploy/helm-charts/charts/dragonfly
+  DRAGONFLY_FILE_SERVER_PATH: test/testdata/k8s/file-server.yaml
+
+jobs:
+  e2e_tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        module:
+          - "normal"
+        include:
+          - module: normal
+            charts-config: test/testdata/charts/config-v2-rate-limit.yaml
+            skip: ""
+            focus: "'Download With RateLimit'"
+
+    steps:
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be
+        with:
+          tool-cache: false
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          docker-images: true
+          swap-storage: true
+
+      - name: Checkout code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+        with:
+          submodules: recursive
+          fetch-depth: 0
+
+      - name: Install Go
+        uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a
+        with:
+          go-version-file: go.mod
+
+      - name: Get dependencies
+        run: |
+          go install github.com/onsi/ginkgo/v2/ginkgo@v2.12.0
+          mkdir -p /tmp/artifact
+
+      - name: Setup buildx
+        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db
+        id: buildx
+        with:
+          install: true
+
+      - name: Cache Docker layers
+        uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+      - name: Pull Rust Client Image
+        run: |
+          cd client-rs
+          CLIENT_TAG=$(git describe --tags $(git rev-parse HEAD))
+          docker pull dragonflyoss/client:$CLIENT_TAG
+          docker tag dragonflyoss/client:$CLIENT_TAG dragonflyoss/client:latest
+          docker pull dragonflyoss/dfinit:$CLIENT_TAG
+          docker tag dragonflyoss/dfinit:$CLIENT_TAG dragonflyoss/dfinit:latest
+
+      - name: Build Scheduler Image
+        uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355
+        with:
+          context: .
+          file: build/images/scheduler/Dockerfile
+          push: false
+          load: true
+          tags: dragonflyoss/scheduler:latest
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+
+      - name: Build Manager Image
+        uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355
+        with:
+          context: .
+          file: build/images/manager/Dockerfile
+          push: false
+          load: true
+          tags: dragonflyoss/manager:latest
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+
+      - name: Setup Kind
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3
+        with:
+          version: ${{ env.KIND_VERSION }}
+          config: ${{ env.KIND_CONFIG_PATH }}
+          cluster_name: kind
+
+      - name: Kind load images
+        run: |
+          kind load docker-image dragonflyoss/manager:latest
+          kind load docker-image dragonflyoss/scheduler:latest
+          kind load docker-image dragonflyoss/client:latest
+          kind load docker-image dragonflyoss/dfinit:latest
+
+      - name: Setup dragonfly
+        run: |
+          helm install --wait --timeout 15m --dependency-update --create-namespace --namespace dragonfly-system -f ${{ matrix.charts-config }} dragonfly ${{ env.DRAGONFLY_CHARTS_PATH }}
+          kubectl apply -f ${{ env.DRAGONFLY_FILE_SERVER_PATH }}
+          kubectl wait po file-server-0 --namespace dragonfly-e2e --for=condition=ready --timeout=10m
+
+      - name: Run E2E test
+        run: |
+          ginkgo -v -r --race --fail-fast --cover --trace --show-node-events --skip=${{ matrix.skip }} --focus=${{ matrix.focus }} test/e2e/v2
+          cat coverprofile.out >> coverage.txt
+
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@1e68e06f1dbfde0e4cefc87efeba9e4643565303
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./coverage.txt
+          flags: e2etests
+
+      - name: Upload Logs
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b
+        if: always()
+        with:
+          name: ${{ matrix.module }}-e2e-tests-logs
+          path: |
+            /tmp/artifact/**

test/e2e/v2/rate_limit_test.go

@@ -0,0 +1,59 @@
+/*
+ *     Copyright 2024 The Dragonfly Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package e2e
+
+import (
+	"fmt"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2" //nolint
+	. "github.com/onsi/gomega"    //nolint
+
+	"d7y.io/dragonfly/v2/test/e2e/v2/util"
+)
+
+var _ = Describe("Download With RateLimit", func() {
+	Context("/bin/kubectl file", func() {
+		It("should be ok", Label("download", "rateLimit"), func() {
+			clientPod, err := util.ClientExec()
+			fmt.Println(err)
+			Expect(err).NotTo(HaveOccurred())
+
+			start := time.Now()
+			out, err := clientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -H 'X-Dragonfly-Tag: download-rate-limit' %s --output %s", util.GetFileURL("/bin/kubectl"), util.GetOutputPath("kubectl-proxy"))).CombinedOutput()
+			elapsed := time.Since(start)
+			fmt.Println(err)
+			Expect(err).NotTo(HaveOccurred())
+			// The download rate limit is 1MiB/s, and the file size is about 44MiB, so the download time should be greater than 44s.
+			Expect(elapsed).Should(BeNumerically(">", 44*time.Second))
+			fmt.Println(string(out))
+
+			fileMetadata := util.FileMetadata{
+				ID:     "6b94aa169d11ba67c1852f14c4df39e5ebf84d1c0d7a6f6ad0ba2f3547883f4f",
+				Sha256: "327b4022d0bfd1d5e9c0701d4a3f989a536f7e6e865e102dcd77c7e7adb31f9a",
+			}
+
+			sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(fileMetadata.Sha256).To(Equal(sha256sum))
+
+			sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, util.GetOutputPath("kubectl-proxy"))
+			Expect(err).NotTo(HaveOccurred())
+			Expect(fileMetadata.Sha256).To(Equal(sha256sum))
+		})
+	})
+})

test/testdata/charts/config-v2-rate-limit.yaml

@@ -0,0 +1,183 @@
+manager:
+  image:
+    repository: dragonflyoss/manager
+    tag: latest
+  replicas: 1
+  resources:
+    requests:
+      cpu: "0"
+      memory: "0"
+    limits:
+      cpu: "1"
+      memory: "2Gi"
+  extraVolumeMounts:
+  - name: logs
+    mountPath: "/var/log/"
+  - name: artifact
+    mountPath: /tmp/artifact
+  extraVolumes:
+  - name: logs
+    emptyDir: { }
+  - name: artifact
+    hostPath:
+      path: /tmp/artifact
+  metrics:
+    enable: true
+  config:
+    console: false
+    verbose: true
+    job:
+      rateLimit:
+        fillInterval: 1m
+        capacity: 100
+        quantum: 100
+
+scheduler:
+  image:
+    repository: dragonflyoss/scheduler
+    tag: latest
+  replicas: 3
+  resources:
+    requests:
+      cpu: "0"
+      memory: "0"
+    limits:
+      cpu: "2"
+      memory: "4Gi"
+  service:
+    type: NodePort
+    nodePort: 30802
+  extraVolumeMounts:
+  - name: logs
+    mountPath: "/var/log/"
+  - name: artifact
+    mountPath: /tmp/artifact
+  extraVolumes:
+  - name: logs
+    emptyDir: { }
+  - name: artifact
+    hostPath:
+      path: /tmp/artifact
+  metrics:
+    enable: true
+    enableHost: true
+  config:
+    console: false
+    verbose: true
+    scheduler:
+      gc:
+        hostGCInterval: 2m
+
+seedClient:
+  enable: true
+  replicas: 3
+  image:
+    repository: dragonflyoss/client
+    tag: latest
+  resources:
+    requests:
+      cpu: "0"
+      memory: "0"
+    limits:
+      cpu: "2"
+      memory: "4Gi"
+  extraVolumeMounts:
+  - name: logs
+    mountPath: "/var/log/"
+  - name: artifact
+    mountPath: /tmp/artifact
+  extraVolumes:
+  - name: logs
+    emptyDir: { }
+  - name: artifact
+    hostPath:
+      path: /tmp/artifact
+  config:
+    download:
+      rateLimit: 1MiB
+    upload:
+      rateLimit: 1MiB
+    dynconfig:
+      refreshInterval: 1s
+    scheduler:
+      announceInterval: 1s
+    log:
+      level: info
+    proxy:
+      prefetch: true
+      prefetchRateLimit: 1MiB
+      registryMirror:
+        addr: https://index.docker.io
+      rules:
+      - regex: blobs/sha256.*
+      - regxe: file-server.*
+
+client:
+  enable: true
+  image:
+    repository: dragonflyoss/client
+    tag: latest
+  resources:
+    requests:
+      cpu: "0"
+      memory: "0"
+    limits:
+      cpu: "2"
+      memory: "4Gi"
+  # Allow client daemonSet to create a pod on master node for testing when the daemon goes offline.
+  tolerations:
+    - key: "node-role.kubernetes.io/master"
+      operator: "Exists"
+      effect: "NoSchedule"
+  extraVolumeMounts:
+  - name: logs
+    mountPath: "/var/log/"
+  - name: artifact
+    mountPath: /tmp/artifact
+  extraVolumes:
+  - name: logs
+    emptyDir: { }
+  - name: artifact
+    hostPath:
+      path: /tmp/artifact
+  dfinit:
+    enable: true
+    image:
+      repository: dragonflyoss/dfinit
+      tag: latest
+    config:
+      containerRuntime:
+        containerd:
+          configPath: /etc/containerd/config.toml
+          registries:
+            - hostNamespace: docker.io
+              serverAddr: https://index.docker.io
+              capabilities: ["pull", "resolve"]
+            - hostNamespace: ghcr.io
+              serverAddr: https://ghcr.io
+              capabilities: ["pull", "resolve"]
+  config:
+    download:
+      rateLimit: 1MiB
+    upload:
+      rateLimit: 1MiB
+    dynconfig:
+      refreshInterval: 1s
+    scheduler:
+      announceInterval: 1s
+    log:
+      level: info
+    proxy:
+      prefetch: false
+      prefetchRateLimit: 1MiB
+      registryMirror:
+        addr: https://index.docker.io
+      rules:
+      - regex: blobs/sha256.*
+      - regxe: file-server.*
+
+dfdaemon:
+  enable: false
+
+seedPeer:
+  enable: false