Update Docker scripts (#33)

* Update Docker scripts

* Support GitLab CI vars
* Support local MinIO endpoint-url

* update branch var for gitlab

scripts/getminiodeps.sh

@@ -17,26 +17,38 @@ then
     exit 1
 fi
 
-# Check if GITHUB_REF_NAME is defined
-if [[ "$GITHUB_REF_NAME" == '' ]]
+# Check if GITHUB_REF_NAME or CI_COMMIT_BRANCH is defined
+if [[ "$GITHUB_REF_NAME" == '' &&  "$CI_COMMIT_BRANCH" == '' ]]
 then
-    echo "*** ERROR: GITHUB_REF_NAME var not defined"
+    echo "*** ERROR: GITHUB_REF_NAME or CI_COMMIT_BRANCH var not defined"
     exit 1
 fi
 
 # Derive the branch name for corresponding dss-sdk artifact
-# Use GITHUB_BASE_REF if PR
+# Use GITHUB_BASE_REF if GitHub PR
 if [[ "$GITHUB_REF_NAME" == *"/merge" ]]
 then
-    BRANCH_NAME="$GITHUB_BASE_REF"
-# Use GITHUB_REF_NAME if merge or push
+    BRANCH_NAME=$GITHUB_BASE_REF
+# Use GITHUB_REF_NAME if GitHub merge or push
+elif [[ "$GITHUB_REF_NAME" ]]
+then
+    BRANCH_NAME=$GITHUB_REF_NAME
+# Use CI_MERGE_REQUEST_TARGET_BRANCH_NAME if GitLab MR
+elif [[ "$CI_PIPELINE_SOURCE" == "merge_request_event" ]]
+then
+    BRANCH_NAME=$CI_MERGE_REQUEST_TARGET_BRANCH_NAME
+# Use CI_COMMIT_BRANCH if GitLab merge or push
+elif [[ "$CI_COMMIT_BRANCH" ]]
+then
+    BRANCH_NAME=$CI_COMMIT_BRANCH
 else
-    BRANCH_NAME="$GITHUB_REF_NAME"
+    echo "*** ERROR: Could not derive the branch name"
+    exit 1
 fi
 
 # Get latest dss-sdk main artifact
 set +e
-DSSSDKARTIFACT=$(aws s3 ls "$DSSS3URI/$BRANCH_NAME"/ | sort --reverse | grep -oP "${DSSSDKGLOB//\*/.*}" | head -n 1)
+DSSSDKARTIFACT=$(aws s3 ${MINIO_HOST_URL:+--endpoint-url $MINIO_HOST_URL} ls "$DSSS3URI/$BRANCH_NAME"/ | sort --reverse | grep -oP "${DSSSDKGLOB//\*/.*}" | head -n 1)
 set -e
 
 # Check if dss-sdk artifact found in bucket
@@ -49,7 +61,7 @@ fi
 # Download and extract dss-sdk artifact
 echo "Staging dss-sdk libs and includes from artifact: $DSSSDKARTIFACT from branch '$BRANCH_NAME'"
 mkdir -p ../dss-sdk/host ../dss-sdk/host_out
-aws s3 cp "$DSSS3URI/$BRANCH_NAME/$DSSSDKARTIFACT" - | tar xfz - --wildcards --directory=../dss-sdk/host/ nkv-sdk/include/* --directory=../host_out/ nkv-sdk/lib/* --strip=1
+aws s3 ${MINIO_HOST_URL:+--endpoint-url $MINIO_HOST_URL} cp "$DSSS3URI/$BRANCH_NAME/$DSSSDKARTIFACT" - | tar xfz - --wildcards --directory=../dss-sdk/host/ nkv-sdk/include/* --directory=../host_out/ nkv-sdk/lib/* --strip=1
 
 # Download s3-benchmark
 echo "Staging s3-benchmark from URL: $S3BENCHURL"

scripts/stagemergeartifacts.sh

@@ -6,8 +6,10 @@ set -e
 # This folder will match the branch name of the merge.
 # If artifacts exist in this folder matching a file glob, remove them first.
 #
-# This script will only execute when triggered by a GitHub merge event.
-# This is determined by the presence of the substring 'refs/heads/' in a $GITHUB_REF var.
+# This script will only execute when triggered by a GitHub / GitLab merge event.
+# More plainly, this is when a commit is pushed to a protected branch (after PR / MR).
+# This is determined by the presence of the substring 'refs/heads/' in a $GITHUB_REF var,
+#   or when $CI_PIPELINE_SOURCE == "push".
 #
 # This script must be passed the following vars:
 #   DSSS3URI:   The S3 URI pointing to the artifacts bucket
@@ -33,14 +35,23 @@ then
 fi
 
 # Check if build was triggered by merge. Otherwise skip (eg: skip PR or manual build)
-if [[ $GITHUB_REF != "refs/heads"* ]]
+if [[ $GITHUB_REF != "refs/heads"* && $CI_PIPELINE_SOURCE != "push" ]]
 then
-    echo "Not a GitHub merge. Skipping main artifacts rotation."
+    echo "Not a merge. Skipping artifact rotation."
     exit 0
 fi
 
 # Derive the branch name
-BRANCH_NAME=$(echo "$GITHUB_REF" | cut --complement -c 1-11)
+if [[ $GITHUB_REF ]]
+then
+    BRANCH_NAME=$(echo "$GITHUB_REF" | cut --complement -c 1-11)
+elif [[ $CI_COMMIT_BRANCH ]]
+then
+    BRANCH_NAME=$CI_COMMIT_BRANCH
+else
+    echo "*** ERROR - I could not derive the branch name."
+    exit 1
+fi
 
 # Find local artifacts
 LOCALARTIFACTS=()
@@ -61,7 +72,7 @@ done
 for GLOB in "${DSSGLOBLIST[@]}"
 do
     set +e
-    mapfile -t REMOTEARTIFACTS < <(aws s3 ls "$DSSS3URI/$BRANCH_NAME/" | grep -oP "${GLOB//\*/.*}")
+    mapfile -t REMOTEARTIFACTS < <(aws s3 ${MINIO_HOST_URL:+--endpoint-url $MINIO_HOST_URL} ls "$DSSS3URI/$BRANCH_NAME/" | grep -oP "${GLOB//\*/.*}")
     set -e
 
     if [[ "${REMOTEARTIFACTS[0]}" == '' ]]
@@ -71,7 +82,7 @@ do
         for artifact in "${REMOTEARTIFACTS[@]}"
         do
             echo "Deleting existing object from artifacts bucket: $artifact"
-            aws s3 rm "$DSSS3URI/$BRANCH_NAME/$artifact" --only-show-errors
+            aws s3 ${MINIO_HOST_URL:+--endpoint-url $MINIO_HOST_URL} rm "$DSSS3URI/$BRANCH_NAME/$artifact" --only-show-errors
         done
     fi
 done
@@ -80,5 +91,5 @@ done
 for file in "${LOCALARTIFACTS[@]}"
 do
     echo "Copying file to artifacts bucket: $file"
-    aws s3 cp "$file" "$DSSS3URI/$BRANCH_NAME/" --only-show-errors
+    aws s3 ${MINIO_HOST_URL:+--endpoint-url $MINIO_HOST_URL} cp "$file" "$DSSS3URI/$BRANCH_NAME/" --only-show-errors
 done