Further tweaks for CI workflow

1. Add support for optional severity/exemptions config in
   pytest_wrapper.sh
2. Add '--continue-on-collection-errors' to ci make targets
3. Default skip redshift sec groups test
4. Remove passing in --aws-profiles for make targets

In regards to #4 - within our CI workflow, it is significantly
cleaner to use AWS's env var flow than create an AWS credentials
file.

Makefile

@@ -5,21 +5,23 @@ TODAY := $(shell date '+%Y-%m-%d')
 
 AWS_PROFILE := default
 
+PYTEST_OPTS := ''
+
 all: check_venv
 	pytest
 
 awsci: check_venv
-	pytest aws/ --aws-profiles $(AWS_PROFILE) --json=results-$(AWS_PROFILE)-$(TODAY).json
+	pytest --continue-on-collection-errors aws/ --json=results-$(AWS_PROFILE)-$(TODAY).json $(PYTEST_OPTS)
 
 aws-sg: check_venv
-	pytest \
+	pytest --continue-on-collection-errors \
 		aws/ec2/test_ec2_security_group_in_use.py \
 		aws/ec2/test_ec2_security_group_opens_all_ports.py \
 		aws/ec2/test_ec2_security_group_opens_all_ports_to_all.py \
 		aws/ec2/test_ec2_security_group_opens_all_ports_to_self.py \
 		aws/rds/test_rds_db_security_group_does_not_grant_public_access.py \
 		aws/rds/test_rds_db_instance_not_publicly_accessible_by_vpc_sg.py \
-		--aws-profiles $(AWS_PROFILE) --json=results-sg-$(AWS_PROFILE)-$(TODAY).json
+		--json=results-sg-$(AWS_PROFILE)-$(TODAY).json $(PYTEST_OPTS)
 
 check_venv:
 ifeq ($(VIRTUAL_ENV),)

aws/redshift/test_redshift_security_group_does_not_allow_all_ips_access.py

@@ -10,6 +10,7 @@
 
 
 @pytest.mark.redshift
+@pytest.mark.xskip(reason="Most don't use Redshift security groups")
 @pytest.mark.parametrize(
     'security_group',
     redshift_cluster_security_groups(),

pytest_wrapper.sh

@@ -5,32 +5,40 @@
 set -ex
 
 if [ -z "$1" ]; then
-  echo "USAGE: ./pytest_wrapper.sh AWS_PROFILE"
+  echo "USAGE: ./pytest_wrapper.sh AWS_PROFILE [/path/to/severity.conf] [/path/to/exemptions.conf]"
   exit 1
 fi
 
-export AWS_PROFILE=$1
+export _AWS_PROFILE=$1
+
+pytestopts=""
+if [ ! -z "$2" ]; then
+  pytestopts="--severity-config ${2}"
+fi
+if [ ! -z "$3" ]; then
+  pytestopts="--exemptions-config ${3}"
+fi
 
 make install
 . venv/bin/activate
 
 # allow pytest commands to fail so we can report results
-make awsci AWS_PROFILE=$AWS_PROFILE || true
+make awsci AWS_PROFILE=$_AWS_PROFILE PYTEST_OPTS="${pytestopts}" || true
 
 date=$(date +%F)
 venv/bin/python3 service_report_generator.py \
-  --jo service-report-${AWS_PROFILE}-${date}.json \
-  --jm service-report-${AWS_PROFILE}-${date}.md \
-  results-${AWS_PROFILE}-${date}.json
+  --jo service-report-${_AWS_PROFILE}-${date}.json \
+  --mo service-report-${_AWS_PROFILE}-${date}.md \
+  results-${_AWS_PROFILE}-${date}.json
 
 # Check in the generated files
-mkdir -p /$RESULTS_DIR/aws-pytest/${AWS_PROFILE}/
-cp service-report-${AWS_PROFILE}-${date}.json /$RESULTS_DIR/aws-pytest/${AWS_PROFILE}/
-cp service-report-${AWS_PROFILE}-${date}.md /$RESULTS_DIR/aws-pytest/${AWS_PROFILE}/
+mkdir -p /$RESULTS_DIR/aws-pytest/${_AWS_PROFILE}/
+cp service-report-${_AWS_PROFILE}-${date}.json /$RESULTS_DIR/aws-pytest/${_AWS_PROFILE}/
+cp service-report-${_AWS_PROFILE}-${date}.md /$RESULTS_DIR/aws-pytest/${_AWS_PROFILE}/
 
 cd /$RESULTS_DIR/
 git pull
-git add aws-pytest/${AWS_PROFILE}/service-report-${AWS_PROFILE}-${date}.json
-git add aws-pytest/${AWS_PROFILE}/service-report-${AWS_PROFILE}-${date}.md
-git commit -m "Pytest Services Results - ${AWS_PROFILE} ${date}"
+git add aws-pytest/${_AWS_PROFILE}/service-report-${_AWS_PROFILE}-${date}.json
+git add aws-pytest/${_AWS_PROFILE}/service-report-${_AWS_PROFILE}-${date}.md
+git commit -m "Pytest Services Results - ${_AWS_PROFILE} ${date}"
 git push origin master:master