Upgrade code to use aws sdk go v2 (#6486)

* oidc blocktype and rate limiter configuration * kvparams extract configuration params * remove blank lines * wip * kvparams extract configuration params * fix missing fmt * remove blank lines * wip * wip * wip: code compile, no test, no run * fix no such key * improve s3 blocker client cache * support non multipart upload api * Add missing generated files * gateway middleware test filename * fix err no such key and not found * fixing unit tests * dynamodb max conn per host 10 * fix s3 client cache behaviour * remove s3 adapter stream * remove streaming check size and timeout * configurable client log mode * Fix logging default test * gtw sig use request context * refactor sig interface and skip chunk test * default s3 endpoint with http scheme * fix test and apply put without seeker * fix test part number type * esti handle s3 endpoint with scheme * dont log key and authenticator * no need to log constants * without swap compute payload sha256 for unsigned payload * fix esti build code with wrong go version * Graveler test reduce branch update retry duration for specific test * use unsigned payload for put object and upload part * update aws-actions/configure-aws-credentials to v3 * comment and code style * fix upload helper uses separator as part of key prefix * use NoBody on zero size content * upload object update api swagger order revert client sdk changes * metadata client use web identity configuration * apply code review changes and additional lint issues * missing parts from previous commit * remove web identify from cloud metadata client * apply code review changes * remove inventory * Handle server value from put requests * move sse header extract function to a different file * limit upload to one attempt * fix controller after rebase * fix rebase compilation error * go mod tidy * put max retry based on seekable reader * Update docs and remove code of unuseed streaming_chunk_size and streaming_chunk_timeout * remove viper defaults * Presigned url expiry value consider credentials expiry * apply code review change * Use session expiry window as part of expiry adjust * Update comments based on code review
treeverse · Sep 12, 2023 · c8d8859 · c8d8859
1 parent e9a809b
commit c8d8859
Show file tree

Hide file tree

Showing 105 changed files with 2,468 additions and 3,706 deletions.
diff --git a/.github/workflows/docker-publish-lakefs-rclone-export.yaml b/.github/workflows/docker-publish-lakefs-rclone-export.yaml
@@ -32,7 +32,7 @@ jobs:
         id: version
 
       - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v2
+        uses: aws-actions/configure-aws-credentials@v3
         with:
           aws-region: us-east-1
           role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}

diff --git a/.github/workflows/docker-publish.yaml b/.github/workflows/docker-publish.yaml
@@ -52,7 +52,7 @@ jobs:
           GOLANGCI_LINT_FLAGS: --out-format github-actions
 
       - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v2
+        uses: aws-actions/configure-aws-credentials@v3
         with:
           aws-region: us-east-1
           role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}

diff --git a/.github/workflows/esti.yaml b/.github/workflows/esti.yaml
@@ -83,7 +83,7 @@ jobs:
         run: tar -xf /tmp/generated.tar.gz
 
       - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v2
+        uses: aws-actions/configure-aws-credentials@v3
         with:
           aws-region: us-east-1
           role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
@@ -116,7 +116,7 @@ jobs:
     if: needs.check-secrets.outputs.secretsavailable == 'true'
     steps:
       - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v2
+        uses: aws-actions/configure-aws-credentials@v3
         with:
           role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
           aws-region: us-east-1
@@ -186,7 +186,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19.2
+          go-version: "1.20.6"
         id: go
 
       - name: Generate uniquifying value
@@ -237,7 +237,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v2
+        uses: aws-actions/configure-aws-credentials@v3
         with:
           aws-region: us-east-1
           role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}

diff --git a/.github/workflows/goreleaser.yaml b/.github/workflows/goreleaser.yaml
@@ -103,7 +103,7 @@ jobs:
           go-version: "1.20.6"
 
       - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v2
+        uses: aws-actions/configure-aws-credentials@v3
         with:
           aws-region: us-east-1
           role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}

diff --git a/.github/workflows/publish-hadoop-lakefs.yaml b/.github/workflows/publish-hadoop-lakefs.yaml
@@ -49,7 +49,7 @@ jobs:
         run: sed -i.bak 's/<version>.*<\/version><!--MARKER.*/<version>'${{ steps.version.outputs.tag }}'<\/version>/' pom.xml
 
       - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v2
+        uses: aws-actions/configure-aws-credentials@v3
         with:
           aws-region: us-east-1
           role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}

diff --git a/.secretsignore b/.secretsignore
@@ -15,4 +15,4 @@ bPxRfiCYEXAMPLEKEY
 wJalrXUtnFEMI/K3MDENG/bPxRfiCYEXAMPLEKEY
 wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
 zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
-
+10a718b3f285d89c36e9864494cdd1507f3bc85b342df24736ea81f9a1134bcc
diff --git a/api/swagger.yml b/api/swagger.yml
@@ -3739,6 +3739,10 @@ paths:
                   description: Only a single file per upload which must be named "content".
                   type: string
                   format: binary
+          application/octet-stream:
+            schema:
+              type: string
+              format: binary
 
       parameters:
         - in: query

diff --git a/clients/java/README.md b/clients/java/README.md
diff --git a/clients/java/api/openapi.yaml b/clients/java/api/openapi.yaml
diff --git a/clients/java/docs/InlineObject.md b/clients/java/docs/InlineObject.md
diff --git a/clients/java/docs/ObjectsApi.md b/clients/java/docs/ObjectsApi.md
diff --git a/clients/java/src/main/java/io/lakefs/clients/api/ObjectsApi.java b/clients/java/src/main/java/io/lakefs/clients/api/ObjectsApi.java
diff --git a/clients/java/src/main/java/io/lakefs/clients/api/model/InlineObject.java b/clients/java/src/main/java/io/lakefs/clients/api/model/InlineObject.java
diff --git a/clients/java/src/test/java/io/lakefs/clients/api/model/InlineObjectTest.java b/clients/java/src/test/java/io/lakefs/clients/api/model/InlineObjectTest.java
diff --git a/clients/python/.openapi-generator/FILES b/clients/python/.openapi-generator/FILES
diff --git a/clients/python/README.md b/clients/python/README.md
diff --git a/clients/python/docs/InlineObject.md b/clients/python/docs/InlineObject.md
diff --git a/clients/python/docs/ObjectsApi.md b/clients/python/docs/ObjectsApi.md
diff --git a/clients/python/lakefs_client/api/objects_api.py b/clients/python/lakefs_client/api/objects_api.py