[tests-only][full-ci]Added script to report closed issues in expected failure file

tests/acceptance/lint-closed-issue.sh

@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+
+# This is a script to check (lint) if there are any closed issues in an expected failures file.
+# This script does not run in the drone pipeline but can be manually run often to find out if there are still closed issues in an expected failures file.
+# Pass GITHUB_ACCESS_TOKEN=(Your Access Token) for unlimited API request to GITHUB API
+# To run this script pass EXPECTED_FAILURES_FILE= (Path to expected to failure file)
+# Eg : GITHUB_ACCESS_TOKEN=  EXPECTED_FAILURES_FILE=  bash tests/acceptance/lint-closed-issue.sh
+
+log_error() {
+	echo -e "\e[31m$1\e[0m"
+}
+
+log_info() {
+	echo -e "\e[37m$1\e[0m"
+}
+
+log_success() {
+	echo -e "\e[32m$1\e[0m"
+}
+
+GITHUB_API_ENTRY_URL="https://api.github.com/repos/owncloud/"
+GITHUB_ISSUE_COMMON_URL="https://github.com/owncloud/"
+
+log_success "Pass Your GITHUB Access Token as GITHUB_ACCESS_TOKEN=(Your Access Token) through ENV for unlimited API Request..."
+
+if [ -n "${EXPECTED_FAILURES_FILE}" ]
+then
+	if [ -f "${EXPECTED_FAILURES_FILE}" ]
+	then
+		log_info "Expected failures file ${EXPECTED_FAILURES_FILE} found."
+	else
+		log_error "Expected failures file ${EXPECTED_FAILURES_FILE} not found"
+		log_error "Check the setting of EXPECTED_FAILURES_FILE environment variable"
+		exit 1
+	fi
+
+	log_info "Checking for closed issues in ${EXPECTED_FAILURES_FILE}."
+	echo "......."
+	LINE_NUMBER=0
+	NO_CLOSED_ISSUE=1
+	NO_INVALID_ISSUE=1
+	while read -r INPUT_LINE;
+	do
+		LINE_NUMBER=$(("$LINE_NUMBER" + 1))
+		# Ignore feature file lines (starting with -) in order to grab the GITHUB issue link only
+		if [[ "${INPUT_LINE}" =~ ^- ]]
+		then
+			continue
+		fi
+
+		# Report suspicious lines like ( https://github.com/owncloud/ocis/issues/111 )
+		# There should not be any space inside the brackets surrounding an issue link.
+		LINE_FORMAT_ERROR=0
+		if [[ "${INPUT_LINE}" =~ \([[:blank:]] ]];
+		then
+			log_error "Line ${LINE_NUMBER} contains space after the left-bracket"
+			LINE_FORMAT_ERROR=1
+			NO_INVALID_ISSUE=0
+		fi
+		if [[ "${INPUT_LINE}" =~ [[:blank:]]\) ]];
+		then
+			log_error "Line ${LINE_NUMBER} contains space before the right-bracket"
+			LINE_FORMAT_ERROR=1
+			NO_INVALID_ISSUE=0
+		fi
+		if [ ${LINE_FORMAT_ERROR} -eq 1 ];
+		then
+			continue
+		fi
+
+		# Match and find the GITHUB issue link like (https://github.com/owncloud/ocis/issues/1239) in description with regex in the expected failures file.
+		if [[ "${INPUT_LINE}" =~ \(([a-zA-Z0-9:/.#_-]+)\) ]];
+		then
+			# Get each issue link from the expected failures file like this (https://github.com/owncloud/ocis/issues/1239)
+			# Remove the small bracket () from the issue link
+			GITHUB_ISSUE_LINK=$(echo "${BASH_REMATCH[0]}" | tr -d "()")
+			# Change the issue link to GITHUB API URL to make request.
+			# With this url GITHUB_API_LINK we can make request to get the status
+			GITHUB_API_LINK=${GITHUB_ISSUE_LINK/${GITHUB_ISSUE_COMMON_URL}/${GITHUB_API_ENTRY_URL}}
+			# Make request with curl
+			# Check the state of the issue
+			ISSUE_STATE=$(curl -s -XGET -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${GITHUB_ACCESS_TOKEN}" "${GITHUB_API_LINK}" | jq -r ".state")
+			if [ "${ISSUE_STATE}" = "open" ];
+			then
+				continue
+			fi
+			if [ "${ISSUE_STATE}" = "closed" ];
+			then
+				NO_CLOSED_ISSUE=0
+				log_error "Issue \033[1;37m${GITHUB_ISSUE_LINK} \033[0;31mis closed but still in expected failures"
+				continue
+			fi
+			# The issue state is not "open" or "closed". Something is wrong.
+			# If the link goes to an issue number that does not exist, then the state is "null"
+			NO_INVALID_ISSUE=0
+			if [ "${ISSUE_STATE}" = "null" ];
+			then
+				log_error "Issue \033[1;37m${GITHUB_ISSUE_LINK} \033[0;31mdoes not exist"
+				continue
+			fi
+			# Otherwise, the link might be to a completely different place. The curl response probably has no "state".
+			log_error "Issue \033[1;37m${GITHUB_ISSUE_LINK} \033[0;31mhas unknown state - maybe the issue link is invalid"
+		fi
+	done < "${EXPECTED_FAILURES_FILE}"
+
+	# Last Check for if no closed issues in the expected failures file
+	if [ ${NO_CLOSED_ISSUE} -eq 1 ];
+	then
+		log_success "There are no closed issues in ${EXPECTED_FAILURES_FILE}."
+	fi
+
+	if [ ${NO_INVALID_ISSUE} -eq 0 ];
+	then
+		log_error "There are invalid issue links in ${EXPECTED_FAILURES_FILE}."
+		exit 1
+	fi
+else
+	log_error "Environment variable EXPECTED_FAILURES_FILE must be defined to be the file to check"
+	exit 1
+fi

tests/acceptance/lint-expected-failures.sh

@@ -1,15 +1,15 @@
 #!/usr/bin/env bash
 
 log_error() {
-    echo -e "\e[31m$1\e[0m"
+	echo -e "\e[31m$1\e[0m"
 }
 
 log_info() {
-    echo -e "\e[34m$1\e[0m"
+	echo -e "\e[34m$1\e[0m"
 }
 
 log_success() {
-    echo -e "\e[32m$1\e[0m"
+	echo -e "\e[32m$1\e[0m"
 }
 
 if [ -n "${EXPECTED_FAILURES_FILE}" ]
@@ -56,12 +56,12 @@ then
 			if [[ "${INPUT_LINE}" =~ features[[:blank:]]in[[:blank:]]the[[:blank:]]([a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+)[[:blank:]]repo ]]; then
 				FEATURE_FILE_REPO="${BASH_REMATCH[1]}"
 				log_info "Features are expected to be in the ${FEATURE_FILE_REPO} repo\n"
-  			FEATURE_FILE_SPEC_LINE_FOUND="true"
+				FEATURE_FILE_SPEC_LINE_FOUND="true"
 			fi
 			if [[ "${INPUT_LINE}" =~ repo[[:blank:]]in[[:blank:]]the[[:blank:]]([a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+)[[:blank:]]folder[[:blank:]]tree ]]; then
 				FEATURE_FILE_PATH="${BASH_REMATCH[1]}"
 				log_info "Features are expected to be in the ${FEATURE_FILE_PATH} folder tree\n"
-  			FEATURE_FILE_SPEC_LINE_FOUND="true"
+				FEATURE_FILE_SPEC_LINE_FOUND="true"
 			fi
 			if [[ $FEATURE_FILE_SPEC_LINE_FOUND == "true" ]]; then
 				continue
@@ -71,10 +71,10 @@ then
 			if [[ "${INPUT_LINE}" =~ ^-[[:space:]]\[([a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+\.feature:[0-9]+)] ]]; then
 				SUITE_SCENARIO_LINE="${BASH_REMATCH[1]}"
 			elif [[
-			  # report for lines like: " - someSuite/someName.feature:n"
-			  "${INPUT_LINE}" =~ ^[[:space:]]*-[[:space:]][a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+\.feature:[0-9]+[[:space:]]*$ ||
-			  # report for lines starting with: "[someSuite/someName.feature:n]"
-			  "${INPUT_LINE}" =~ ^[[:space:]]*\[([a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+\.feature:[0-9]+)]
+				# report for lines like: " - someSuite/someName.feature:n"
+				"${INPUT_LINE}" =~ ^[[:space:]]*-[[:space:]][a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+\.feature:[0-9]+[[:space:]]*$ ||
+				# report for lines starting with: "[someSuite/someName.feature:n]"
+				"${INPUT_LINE}" =~ ^[[:space:]]*\[([a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+\.feature:[0-9]+)]
 			]]; then
 				log_error "> Line ${LINE_NUMBER}: Not in the correct format."
 				log_error "  + Actual Line     : '${INPUT_LINE}'"

tests/acceptance/run.sh

@@ -2,8 +2,8 @@
 [[ "${DEBUG}" == "true" ]] && set -x
 
 # from http://stackoverflow.com/a/630387
-SCRIPT_PATH="`dirname \"$0\"`"              # relative
-SCRIPT_PATH="`( cd \"${SCRIPT_PATH}\" && pwd )`"  # absolutized and normalized
+SCRIPT_PATH="`dirname \"$0\"`" # relative
+SCRIPT_PATH="`( cd \"${SCRIPT_PATH}\" && pwd )`" # absolutized and normalized
 
 echo 'Script path: '${SCRIPT_PATH}
 
@@ -14,15 +14,15 @@ OCC=${OC_PATH}occ
 # This gives flexibility for callers that have installed their own behat
 if [ -z "${BEHAT_BIN}" ]
 then
-    BEHAT=${OC_PATH}vendor-bin/behat/vendor/bin/behat
+	BEHAT=${OC_PATH}vendor-bin/behat/vendor/bin/behat
 else
-    BEHAT=${BEHAT_BIN}
+	BEHAT=${BEHAT_BIN}
 fi
 BEHAT_TAGS_OPTION_FOUND=false
 
 if [ -n "${STEP_THROUGH}" ]
 then
-    STEP_THROUGH_OPTION="--step-through"
+	STEP_THROUGH_OPTION="--step-through"
 fi
 
 # The following environment variables can be specified:
@@ -98,7 +98,7 @@ fi
 #            testing app. We have to assume it is already enabled.
 # --show-oc-logs - tail the ownCloud log after the test run
 # --norerun - do not rerun failed webUI scenarios
-# --loop  - loop tests for given number of times. Only use it for debugging purposes
+# --loop - loop tests for given number of times. Only use it for debugging purposes
 # --part - run a subset of scenarios, need two numbers.
 #          first number: which part to run
 #          second number: in how many parts to divide the set of scenarios
@@ -404,7 +404,7 @@ function run_behat_tests() {
 		cat ${SCRIPT_PATH}/usernames.json
 	fi
 
-	${BEHAT} --colors --strict ${STEP_THROUGH_OPTION} -c ${BEHAT_YML} -f pretty ${BEHAT_SUITE_OPTION} --tags ${BEHAT_FILTER_TAGS} ${BEHAT_FEATURE} -v  2>&1 | tee -a ${TEST_LOG_FILE}
+	${BEHAT} --colors --strict ${STEP_THROUGH_OPTION} -c ${BEHAT_YML} -f pretty ${BEHAT_SUITE_OPTION} --tags ${BEHAT_FILTER_TAGS} ${BEHAT_FEATURE} -v 2>&1 | tee -a ${TEST_LOG_FILE}
 
 	BEHAT_EXIT_STATUS=${PIPESTATUS[0]}
 
@@ -518,9 +518,9 @@ function run_behat_tests() {
 				# brackets is the suite, feature and line number of the expected failure.
 				# Else ignore the line.
 				if [[ "${SUITE_SCENARIO}" =~ \[([a-zA-Z0-9-]+/[a-zA-Z0-9-]+\.feature:[0-9]+)] ]]; then
-				  SUITE_SCENARIO="${BASH_REMATCH[1]}"
+					SUITE_SCENARIO="${BASH_REMATCH[1]}"
 				else
-				  continue
+					continue
 				fi
 				if [ -n "${BEHAT_SUITE_TO_RUN}" ]
 				then
@@ -605,7 +605,7 @@ function run_behat_tests() {
 				fi
 
 				echo "Rerun failed scenario: ${FAILED_SCENARIO_PATH}"
-				${BEHAT} --colors --strict -c ${BEHAT_YML} -f pretty ${BEHAT_SUITE_OPTION} --tags ${BEHAT_FILTER_TAGS} ${FAILED_SCENARIO_PATH} -v  2>&1 | tee -a ${TEST_LOG_FILE}
+				${BEHAT} --colors --strict -c ${BEHAT_YML} -f pretty ${BEHAT_SUITE_OPTION} --tags ${BEHAT_FILTER_TAGS} ${FAILED_SCENARIO_PATH} -v 2>&1 | tee -a ${TEST_LOG_FILE}
 				BEHAT_EXIT_STATUS=${PIPESTATUS[0]}
 				if [ ${BEHAT_EXIT_STATUS} -eq 0 ]
 				then
@@ -987,7 +987,7 @@ SETTINGS+=("system;;skeletondirectory;;")
 # Set various settings
 for URL in ${OCC_URL} ${OCC_FED_URL}
 do
-  declare SETTINGS_CMDS='['
+	declare SETTINGS_CMDS='['
 	for i in "${!SETTINGS[@]}"
 	do
 		PREVIOUS_IFS=${IFS}
@@ -1099,7 +1099,7 @@ function version { echo "$@" | awk -F. '{ printf("%d%03d%03d\n", $1,$2,$3); }';
 
 # Skip tests for OC versions greater than 10.8.0
 if [ $(version $OWNCLOUD_VERSION_THREE_DIGIT) -gt $(version "10.8.0") ]; then
-    BEHAT_FILTER_TAGS='~@skipOnAllVersionsGreaterThanOcV10.8.0&&'${BEHAT_FILTER_TAGS}
+	BEHAT_FILTER_TAGS='~@skipOnAllVersionsGreaterThanOcV10.8.0&&'${BEHAT_FILTER_TAGS}
 fi
 
 if [ -n "${TEST_SERVER_FED_URL}" ]
@@ -1127,12 +1127,12 @@ fi
 # If the caller did not mention specific tags, skip the skipped tests by default
 if [ "${BEHAT_TAGS_OPTION_FOUND}" = false ]
 then
-  #  If the caller has already specified specifically to run "@skip" scenarios
-  #  then do not append "not @skip"
-  if [[ ! ${BEHAT_FILTER_TAGS} =~ "&&@skip&&" ]]
-  then
-	  BEHAT_FILTER_TAGS="${BEHAT_FILTER_TAGS}&&~@skip"
-  fi
+	# If the caller has already specified specifically to run "@skip" scenarios
+	# then do not append "not @skip"
+	if [[ ! ${BEHAT_FILTER_TAGS} =~ "&&@skip&&" ]]
+	then
+		BEHAT_FILTER_TAGS="${BEHAT_FILTER_TAGS}&&~@skip"
+	fi
 fi
 
 if [ -n "${BROWSER_VERSION}" ]
@@ -1297,33 +1297,33 @@ fi
 # Filter the UNEXPECTED_PASSED_SCENARIOS to remove scenarios that were not run.
 if [ "${UNEXPECTED_SUCCESS}" = true ]
 then
-  ACTUAL_UNEXPECTED_PASS=()
-  # if running a single feature or a single scenario
-  if [[ -n "${BEHAT_FEATURE}" ]]
-  then
-    for unexpected_passed_value in "${UNEXPECTED_PASSED_SCENARIOS[@]}"
-    do
-      # check only for the running feature
-      if [[ $BEHAT_FEATURE == *":"* ]]
-      then
-        BEHAT_FEATURE_WITH_LINE_NUM=$BEHAT_FEATURE
-      else
-        LINE_NUM=$(echo ${unexpected_passed_value} | cut -d":" -f2)
-        BEHAT_FEATURE_WITH_LINE_NUM=$BEHAT_FEATURE:$LINE_NUM
-      fi
-      if [[ $BEHAT_FEATURE_WITH_LINE_NUM == *"${unexpected_passed_value}" ]]
-      then
-        ACTUAL_UNEXPECTED_PASS+=("${unexpected_passed_value}")
-      fi
-    done
-  else
-    ACTUAL_UNEXPECTED_PASS=("${UNEXPECTED_PASSED_SCENARIOS[@]}")
-  fi
-
-  if [ ${#ACTUAL_UNEXPECTED_PASS[@]} -eq 0 ]
-  then
-    UNEXPECTED_SUCCESS=false
-  fi
+	ACTUAL_UNEXPECTED_PASS=()
+	# if running a single feature or a single scenario
+	if [[ -n "${BEHAT_FEATURE}" ]]
+	then
+		for unexpected_passed_value in "${UNEXPECTED_PASSED_SCENARIOS[@]}"
+		do
+			# check only for the running feature
+			if [[ $BEHAT_FEATURE == *":"* ]]
+			then
+				BEHAT_FEATURE_WITH_LINE_NUM=$BEHAT_FEATURE
+			else
+				LINE_NUM=$(echo ${unexpected_passed_value} | cut -d":" -f2)
+				BEHAT_FEATURE_WITH_LINE_NUM=$BEHAT_FEATURE:$LINE_NUM
+			fi
+			if [[ $BEHAT_FEATURE_WITH_LINE_NUM == *"${unexpected_passed_value}" ]]
+			then
+				ACTUAL_UNEXPECTED_PASS+=("${unexpected_passed_value}")
+			fi
+		done
+	else
+		ACTUAL_UNEXPECTED_PASS=("${UNEXPECTED_PASSED_SCENARIOS[@]}")
+	fi
+
+	if [ ${#ACTUAL_UNEXPECTED_PASS[@]} -eq 0 ]
+	then
+		UNEXPECTED_SUCCESS=false
+	fi
 fi
 
 if [ "${UNEXPECTED_FAILURE}" = false ] && [ "${UNEXPECTED_SUCCESS}" = false ] && [ "${UNEXPECTED_BEHAT_EXIT_STATUS}" = false ]
@@ -1340,24 +1340,24 @@ fi
 
 if [ "${UNEXPECTED_FAILURE}" = true ]
 then
-  tput setaf 3; echo "runsh: Total unexpected failed scenarios throughout the test run:"
-  tput setaf 1; printf "%s\n" "${UNEXPECTED_FAILED_SCENARIOS[@]}"
+	tput setaf 3; echo "runsh: Total unexpected failed scenarios throughout the test run:"
+	tput setaf 1; printf "%s\n" "${UNEXPECTED_FAILED_SCENARIOS[@]}"
 else
-  tput setaf 2; echo "runsh: There were no unexpected failures."
+	tput setaf 2; echo "runsh: There were no unexpected failures."
 fi
 
 if [ "${UNEXPECTED_SUCCESS}" = true ]
 then
-  tput setaf 3; echo "runsh: Total unexpected passed scenarios throughout the test run:"
-  tput setaf 1; printf "%s\n" "${ACTUAL_UNEXPECTED_PASS[@]}"
+	tput setaf 3; echo "runsh: Total unexpected passed scenarios throughout the test run:"
+	tput setaf 1; printf "%s\n" "${ACTUAL_UNEXPECTED_PASS[@]}"
 else
-  tput setaf 2; echo "runsh: There were no unexpected success."
+	tput setaf 2; echo "runsh: There were no unexpected success."
 fi
 
 if [ "${UNEXPECTED_BEHAT_EXIT_STATUS}" = true ]
 then
-  tput setaf 3; echo "runsh: The following Behat test runs exited with non-zero status:"
-  tput setaf 1; printf "%s\n" "${UNEXPECTED_BEHAT_EXIT_STATUSES[@]}"
+	tput setaf 3; echo "runsh: The following Behat test runs exited with non-zero status:"
+	tput setaf 1; printf "%s\n" "${UNEXPECTED_BEHAT_EXIT_STATUSES[@]}"
 fi
 
 # sync the file-system so all output will be flushed to storage.
@@ -1369,8 +1369,8 @@ sync
 # drone agent send all the output to the drone server.
 if [ -n "${CI_REPO}" ]
 then
-  echo "sleeping for 30 seconds at end of test run"
-  sleep 30
+	echo "sleeping for 30 seconds at end of test run"
+	sleep 30
 fi
 
 exit ${FINAL_EXIT_STATUS}