Merge pull request #764 from versity/test_cmdline_policy_delete_taggi…

…ng_two

Test cmdline policy delete tagging two

.github/workflows/docker-bats.yaml

@@ -0,0 +1,28 @@
+name: docker bats tests
+
+on: pull_request
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build Docker Image
+        run: |
+          mv tests/.env.docker.default tests/.env.docker
+          mv tests/.secrets.default tests/.secrets
+          docker build --build-arg="GO_LIBRARY=go1.21.7.linux-amd64.tar.gz" \
+            --build-arg="AWS_CLI=awscli-exe-linux-x86_64.zip" --build-arg="MC_FOLDER=linux-amd64" \
+            --progress=plain -f Dockerfile_test_bats -t bats_test .
+
+      - name: Set up Docker Compose
+        run: sudo apt-get install -y docker-compose
+
+      - name: Run Docker Container
+        run: docker-compose -f docker-compose-bats.yml up posix_backend

Dockerfile_test_bats

@@ -1,8 +1,11 @@
-FROM --platform=linux/arm64 ubuntu:latest
+FROM ubuntu:latest
 
 ARG DEBIAN_FRONTEND=noninteractive
 ARG SECRETS_FILE=tests/.secrets
 ARG CONFIG_FILE=tests/.env.docker
+ARG GO_LIBRARY=go1.21.7.linux-arm64.tar.gz
+ARG AWS_CLI=awscli-exe-linux-aarch64.zip
+ARG MC_FOLDER=linux-arm64
 
 ENV TZ=Etc/UTC
 RUN apt-get update && \
@@ -24,20 +27,20 @@ RUN apt-get update && \
 WORKDIR /tmp
 
 # Install AWS cli
-RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install
+RUN curl "https://awscli.amazonaws.com/${AWS_CLI}" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install
 
 # Install mc
-RUN curl https://dl.min.io/client/mc/release/linux-arm64/mc \
+RUN curl https://dl.min.io/client/mc/release/${MC_FOLDER}/mc \
   --create-dirs \
   -o /usr/local/minio-binaries/mc && \
 chmod -R 755 /usr/local/minio-binaries
 ENV PATH="/usr/local/minio-binaries":${PATH}
 
 # Download Go 1.21 (adjust the version and platform as needed)
-RUN wget https://golang.org/dl/go1.21.7.linux-arm64.tar.gz
+RUN wget https://golang.org/dl/${GO_LIBRARY}
 
 # Extract the downloaded archive
-RUN tar -xvf go1.21.7.linux-arm64.tar.gz -C /usr/local
+RUN tar -xvf $GO_LIBRARY -C /usr/local
 
 # Set Go environment variables
 ENV PATH="/usr/local/go/bin:${PATH}"
@@ -60,6 +63,10 @@ RUN git clone https://github.com/bats-core/bats-core.git && \
 USER tester
 COPY --chown=tester:tester . /home/tester
 
+# add bats support libraries
+RUN git clone https://github.com/bats-core/bats-support.git && rm -rf /home/tester/tests/bats-support && mv bats-support /home/tester/tests
+RUN git clone https://github.com/ztombol/bats-assert.git && rm -rf /home/tester/tests/bats-assert && mv bats-assert /home/tester/tests
+
 WORKDIR /home/tester
 RUN make
 

docker-compose-bats.yml

@@ -19,6 +19,7 @@ services:
       dockerfile: Dockerfile_test_bats
       args:
         - CONFIG_FILE=tests/.env.default
+    image: bats_test
   s3_backend:
     build:
       context: .

tests/.env.docker.default

@@ -0,0 +1,27 @@
+AWS_PROFILE=versity
+AWS_ENDPOINT_URL=https://127.0.0.1:7070
+VERSITY_EXE=./versitygw
+RUN_VERSITYGW=true
+BACKEND=posix
+LOCAL_FOLDER=/tmp/gw
+BUCKET_ONE_NAME=versity-gwtest-bucket-one
+BUCKET_TWO_NAME=versity-gwtest-bucket-two
+CERT=$PWD/cert-docker.pem
+KEY=$PWD/versitygw-docker.pem
+S3CMD_CONFIG=./tests/s3cfg.local.default
+SECRETS_FILE=./tests/.secrets
+MC_ALIAS=versity
+LOG_LEVEL=2
+USERS_FOLDER=$PWD/iam
+#TEST_LOG_FILE=test.log
+#VERSITY_LOG_FILE=versity.log
+IAM_TYPE=folder
+DIRECT=false
+#DIRECT_DISPLAY_NAME=
+#COVERAGE_DB=coverage.sql
+USERNAME_ONE=ABCDEFG
+PASSWORD_ONE=HIJKLMN
+USERNAME_TWO=HIJKLMN
+PASSWORD_TWO=OPQRSTU
+TEST_FILE_FOLDER=$PWD/versity-gwtest-files
+RECREATE_BUCKETS=true

tests/.secrets.default

@@ -0,0 +1,5 @@
+# change to your account attributes
+AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
+AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
+AWS_REGION=us-east-1
+AWS_PROFILE=versity

tests/README.md

@@ -58,8 +58,9 @@ To communicate directly with s3, in order to compare the gateway results to dire
 
 ## Instructions - Running With Docker
 
-1.  Create a `.secrets` file in the `tests` folder, and add the `AWS_PROFILE`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and the `AWS_PROFILE` fields, as well as the additional s3 fields explained in the **S3 Backend** section above if running with the s3 backend.
-2.  Build and run the `Dockerfile_test_bats` file.  Change the `SECRETS_FILE` and `CONFIG_FILE` parameters to point to your secrets and config file, respectively.  Example:  `docker build -t <tag> -f Dockerfile_test_bats --build-arg="SECRETS_FILE=<file>" --build-arg="CONFIG_FILE=<file>" .`.
+1.  Copy `.secrets.default` to `.secrets` in the `tests` folder and change the parameters and add the additional s3 fields explained in the **S3 Backend** section above if running with the s3 backend.
+2.  By default, the dockerfile uses the **arm** architecture (usually modern Mac).  If using **amd** (usually earlier Mac or Linux), you can either replace the corresponding `ARG` values directly, or with `arg="<param>=<amd library or folder>"`  Also, you can determine which is used by your OS with `uname -a`.
+3.  Build and run the `Dockerfile_test_bats` file.  Change the `SECRETS_FILE` and `CONFIG_FILE` parameters to point to your secrets and config file, respectively, if not using the defaults.  Example:  `docker build -t <tag> -f Dockerfile_test_bats --build-arg="SECRETS_FILE=<file>" --build-arg="CONFIG_FILE=<file>" .`.
 
 ## Instructions - Running with docker-compose
 
@@ -77,3 +78,7 @@ To run in insecure mode, comment out the `CERT` and `KEY` parameters in the `.en
 To use static buckets set the `RECREATE_BUCKETS` value to `false`.
 
 For the s3 backend, see the **S3 Backend** instructions above.
+
+If using AMD rather than ARM architecture, add the corresponding **args** values matching those in the Dockerfile for **amd** libraries.
+
+A single instance can be run with `docker-compose -f docker-compose-bats.yml up <service name>`

tests/commands/delete_bucket_tagging.sh

@@ -21,3 +21,17 @@ delete_bucket_tagging() {
   fi
   return 0
 }
+
+delete_bucket_tagging_with_user() {
+  log 6 "delete_bucket_tagging_with_user"
+  record_command "delete-bucket-tagging" "client:s3api"
+  if [ $# -ne 3 ]; then
+    log 2 "delete bucket tagging command missing username, password, bucket name"
+    return 1
+  fi
+  if ! error=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" aws --no-verify-ssl s3api delete-bucket-tagging --bucket "$3" 2>&1); then
+    log 2 "error deleting bucket tagging with user: $error"
+    return 1
+  fi
+  return 0
+}

tests/commands/list_objects.sh

@@ -64,4 +64,51 @@ list_objects_s3api() {
     IFS=$'\n' read -rd '' -a object_array <<<"$keys"
   fi
   return 0
-}
+}
+
+# list objects in bucket, v1
+# param:  bucket
+# export objects on success, return 1 for failure
+list_objects_s3api_v1() {
+  if [ $# -lt 1 ] || [ $# -gt 2 ]; then
+    echo "list objects command requires bucket, (optional) delimiter"
+    return 1
+  fi
+  if [ "$2" == "" ]; then
+    objects=$(aws --no-verify-ssl s3api list-objects --bucket "$1") || local result=$?
+  else
+    objects=$(aws --no-verify-ssl s3api list-objects --bucket "$1" --delimiter "$2") || local result=$?
+  fi
+  if [[ $result -ne 0 ]]; then
+    echo "error listing objects: $objects"
+    return 1
+  fi
+  export objects
+}
+
+list_objects_with_prefix() {
+  if [ $# -ne 3 ]; then
+    log 2 "'list_objects_with_prefix' command requires, client, bucket, prefix"
+    return 1
+  fi
+  local result=0
+  if [ "$1" == 's3' ]; then
+    objects=$(aws --no-verify-ssl s3 ls s3://"$2/$3" 2>&1) || result=$?
+  elif [ "$1" == 's3api' ]; then
+    objects=$(aws --no-verify-ssl s3api list-objects --bucket "$2" --prefix "$3" 2>&1) || result=$?
+  elif [ "$1" == 's3cmd' ]; then
+    objects=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2/$3" 2>&1) || result=$?
+  elif [[ "$1" == 'mc' ]]; then
+    objects=$(mc --insecure ls "$MC_ALIAS/$2/$3" 2>&1) || result=$?
+  else
+    log 2 "invalid command type '$1'"
+    return 1
+  fi
+  if [ $result -ne 0 ]; then
+    log 2 "error listing objects: $objects"
+    return 1
+  fi
+  log 5 "output: $objects"
+  export objects
+  return 0
+}

tests/commands/list_parts.sh

@@ -10,4 +10,16 @@ list_parts() {
     log 2 "Error listing multipart upload parts: $listed_parts"
     return 1
   fi
+}
+
+list_parts_with_user() {
+  if [ $# -ne 5 ]; then
+    log 2 "'list_parts_with_user' requires username, password, bucket, key, upload ID"
+    return 1
+  fi
+  record_command 'list-parts' 'client:s3api'
+  if ! listed_parts=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" aws --no-verify-ssl s3api list-parts --bucket "$3" --key "$4" --upload-id "$5" 2>&1); then
+    log 2 "Error listing multipart upload parts: $listed_parts"
+    return 1
+  fi
 }

tests/test_common.sh

@@ -611,3 +611,26 @@ EOF
 
   delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME"
 }
+
+test_common_ls_directory_object() {
+  test_file="a"
+
+  run create_test_files "$test_file"
+  assert_success "error creating file"
+
+  run setup_bucket "$1" "$BUCKET_ONE_NAME"
+  assert_success "error setting up bucket"
+
+  if [ "$1" == 's3cmd' ]; then
+    put_object_client="s3api"
+  else
+    put_object_client="$1"
+  fi
+  run put_object "$put_object_client" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file/"
+  assert_success "error putting test file folder"
+
+  run list_and_check_directory_obj "$1" "$test_file"
+  assert_success "error listing and checking directory object"
+
+  delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME"
+}

tests/test_mc.sh

@@ -122,4 +122,6 @@ export RUN_MC=true
   delete_bucket_or_contents "mc" "$BUCKET_ONE_NAME"
 }
 
-
+@test "test_ls_directory_object" {
+  test_common_ls_directory_object "mc"
+}

tests/test_s3.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bats
 
 source ./tests/test_common.sh
+source ./tests/util_file.sh
 
 # complete-multipart-upload
 @test "test_complete_multipart_upload" {
@@ -47,3 +48,7 @@ source ./tests/test_common.sh
   setup_bucket "s3" "$BUCKET_ONE_NAME"
   delete_bucket "s3" "$BUCKET_ONE_NAME" || fail "error deleting bucket"
 }
+
+@test "test_ls_directory_object" {
+  test_common_ls_directory_object "s3"
+}

tests/test_s3api.sh

@@ -433,6 +433,10 @@ export RUN_USERS=true
   test_s3api_policy_get_bucket_tagging
 }
 
+@test "test_policy_list_upload_parts" {
+  test_s3api_policy_list_upload_parts
+}
+
 @test "test_policy_put_acl" {
   test_s3api_policy_put_acl
 }
@@ -456,3 +460,7 @@ export RUN_USERS=true
   delete_bucket_or_contents "aws" "$bucket_name"
 }
 
+@test "test_ls_directory_object" {
+  test_common_ls_directory_object "s3api"
+}
+