From bff9cd54c5a724af3b83beca427b52773ce8b6bb Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 6 Jun 2023 13:54:58 +0800 Subject: [PATCH] ci: Add integration tests for Cloudflare R2 (#2423) * ci: Add integration tests for Cloudflare R2 Signed-off-by: Xuanwo * Set env Signed-off-by: Xuanwo --------- Signed-off-by: Xuanwo --- .github/workflows/service_test_s3.yml | 33 +++++++++++++++++++++ core/src/services/s3/backend.rs | 6 +++- core/src/services/s3/compatible_services.md | 14 +++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/.github/workflows/service_test_s3.yml b/.github/workflows/service_test_s3.yml index 88266f81c82..648ac065780 100644 --- a/.github/workflows/service_test_s3.yml +++ b/.github/workflows/service_test_s3.yml @@ -178,3 +178,36 @@ jobs: OPENDAL_S3_BUCKET: test OPENDAL_S3_ENDPOINT: "http://127.0.0.1:9000" OPENDAL_S3_ALLOW_ANONYMOUS: on + + r2: + runs-on: ubuntu-latest + if: github.event_name == 'push' || !github.event.pull_request.head.repo.fork + steps: + - uses: actions/checkout@v3 + - name: Setup Rust toolchain + uses: ./.github/actions/setup + with: + need-nextest: true + + - name: Load secret + id: op-load-secret + uses: 1password/load-secrets-action@v1 + with: + export-env: true + env: + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + OPENDAL_S3_TEST: op://services/r2/test + OPENDAL_S3_BUCKET: op://services/r2/bucket + OPENDAL_S3_ENDPOINT: op://services/r2/endpoint + OPENDAL_S3_ACCESS_KEY_ID: op://services/r2/access_key_id + OPENDAL_S3_SECRET_ACCESS_KEY: op://services/r2/secret_access_key + + - name: Test + shell: bash + working-directory: core + run: cargo nextest run s3 + env: + OPENDAL_S3_REGION: auto + # This is the R2's limitation + # Refer to https://opendal.apache.org/docs/services/s3#compatible-services for more information + OPENDAL_S3_BATCH_MAX_OPERATIONS: 700 diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs index be720a8f961..056db3e59c6 100644 --- a/core/src/services/s3/backend.rs +++ b/core/src/services/s3/backend.rs @@ -560,7 +560,11 @@ impl Builder for S3Builder { .filter(|v| *v == "on" || *v == "true") .map(|_| builder.allow_anonymous()); map.get("default_storage_class") - .map(|v| builder.default_storage_class(v)); + .map(|v: &String| builder.default_storage_class(v)); + map.get("write_min_size") + .map(|v| builder.write_min_size(v.parse().expect("input must be a number"))); + map.get("batch_max_operations") + .map(|v| builder.batch_max_operations(v.parse().expect("input must be a number"))); builder } diff --git a/core/src/services/s3/compatible_services.md b/core/src/services/s3/compatible_services.md index 432ce7c892c..d5036cb3d51 100644 --- a/core/src/services/s3/compatible_services.md +++ b/core/src/services/s3/compatible_services.md @@ -97,3 +97,17 @@ To connect to wasabi, we need to set: - `bucket`: The bucket name of wasabi. > Refer to [What are the service URLs for Wasabi's different storage regions?](https://wasabi-support.zendesk.com/hc/en-us/articles/360015106031) for more details. + +### Cloudflare R2 + +[Cloudflare R2](https://developers.cloudflare.com/r2/) provides s3 compatible API. + +> Cloudflare R2 Storage allows developers to store large amounts of unstructured data without the costly egress bandwidth fees associated with typical cloud storage services. + + +To connect to r2, we need to set: + +- `endpoint`: The endpoint of r2, for example: `https://.r2.cloudflarestorage.com` +- `bucket`: The bucket name of r2. +- `region`: When you create a new bucket, the data location is set to Automatic by default. So please use `auto` for region. +- `batch_max_operations`: R2's delete objects will return `Internal Error` if the batch is larger than `700`. Please set this value `<= 700` to make sure batch delete work as expected.