test

#!/usr/bin/env bash
set -euo pipefail

# Init lib.
# shellcheck source=SCRIPTDIR/../lib/vector-test-harness/init.sh
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)/../lib/vector-test-harness/init.sh"

#
# Usage
#

register_usage <<END
USAGE
   test -t test [-c configuration] [-s subjects] [-u user-id] [{-b|-r}]

DESCRIPTION
   Runs and profiles the provided case, uploading the results to S3.

OPTIONS
   -b, --bootstrap-only  Bootstrap the test only, do not test any subjects
   -r, --run-only        Don't run the bootstrap task, just run the test. This is helpful for repeated runs.
       --skip-terraform  Skip all terraform commands. This is helpful for repeated runs.
   -c, --configuration   The test configuration (default: default, env: VECTOR_TEST_CONFIGURATION)
   -s, --subject         The test subject, if not set all subjects are tested (ex: vector, env: VECTOR_TEST_SUBJECT)
   -t, --test            The test to run (ex: tcp_to_blackhole, env: VECTOR_TEST_NAME)
   -u, --user-id         Your user ID (ex: ben, env: VECTOR_TEST_USER_ID)
   -v, --vector-version  The Vector version to test (ex: v0.9.0, env: VECTOR_VERSION_TO_TEST)
   -h, --help            Print this message

EXAMPLE
   test -t tcp_to_blackhole -s vector
   test -t tcp_to_blackhole -s vector -v 0.9.0
   test -t tcp_to_blackhole -s vector -v nightly-2020-03-07
   test -t tcp_to_blackhole -s vector -v dev-branch-9-g4bc2c024
END

#
# Dependencies
#

ensure_commad_available "ansible-playbook" "sudo pip install ansible"
ensure_commad_available "terraform" "brew install terraform"
ensure_commad_available "aws" "pip install awscli"
ensure_echanced_getopt_available

#
# Variables
#

# Step toggling
BOOTSTRAP='true'
RUN='true'
SKIP_TERRAFORM='false'

# Required from the environment.
TEST_RESULTS_S3_BUCKET_NAME="${VECTOR_TEST_RESULTS_S3_BUCKET_NAME:-""}"
TEST_SSH_PUBLIC_KEY="${VECTOR_TEST_SSH_PUBLIC_KEY:-""}"
TEST_TIMESTAMP="${VECTOR_TEST_TIMESTAMP:-"$(date -u +%s)"}"
TEST_VECTOR_DEB_PATH="${VECTOR_TEST_VECTOR_DEB_PATH:-""}"
TEST_VECTOR_VERSION="${VECTOR_VERSION_TO_TEST:-"0.8.2"}" # TODO: improve default version detection

# Configurable via cli flags
TEST_CONFIGURATION="${VECTOR_TEST_CONFIGURATION:-"default"}"
TEST_NAME="${VECTOR_TEST_NAME:-""}"
TEST_SUBJECT="${VECTOR_TEST_SUBJECT:-""}"
TEST_USER_ID="${VECTOR_TEST_USER_ID:-""}"

#
# Flags
#

OPTIONS="bc:rs:t:u:v:h"
LONGOPTS="bootstrap-only,configuration:,run-only,skip-terraform,subjects:,test:,user-id:,vector-version:,help"
PARSED="$(parse_arguments "$OPTIONS" "$LONGOPTS" "$0" "$@")"
eval set -- "$PARSED"

while true; do
  case "$1" in
  -b | --bootstrap-only)
    BOOTSTRAP="true"
    RUN="false"
    shift
    ;;
  -r | --run-only)
    BOOTSTRAP="false"
    RUN="true"
    shift
    ;;
  --skip-terraform)
    SKIP_TERRAFORM="true"
    shift
    ;;
  -c | --configuration)
    TEST_CONFIGURATION="$2"
    shift 2
    ;;
  -s | --subject)
    TEST_SUBJECT="$2"
    shift 2
    ;;
  -t | --test)
    TEST_NAME="$2"
    shift 2
    ;;
  -u | --user-id)
    TEST_USER_ID="$2"
    shift 2
    ;;
  -v | --vector-version)
    TEST_VECTOR_VERSION="$2"
    shift 2
    ;;
  -h | --help)
    usage
    exit 0
    ;;
  --)
    shift
    break
    ;;
  *)
    fail_arg_invalid "Invalid option detected: $1"
    ;;
  esac
done

#
# Requirements
#

if [ -z "$TEST_CONFIGURATION" ]; then
  fail_arg_invalid "Error: you must supply a test configuration via the -c flag:"
fi

ensure_no_dashes "$TEST_CONFIGURATION" "Test configuration can't contain dashes"

if [ -z "$TEST_NAME" ]; then
  fail_arg_invalid "Error: you must supply a test name via the -t flag:"
fi

ensure_no_dashes "$TEST_NAME" "Test name can't contain dashes"

if [ -z "$TEST_USER_ID" ]; then
  fail_arg_invalid "Error: you must supply a user ID via the -u flag or VECTOR_TEST_USER_ID env var:"
fi

ensure_no_dashes "$TEST_USER_ID" "User ID can't contain dashes"

TEST_CASE_DIR="cases/$TEST_NAME"
if [[ ! -d "$TEST_CASE_DIR" ]]; then
  fail_arg_invalid "Error: test $TEST_NAME does not exist"
fi

if [[ "$TEST_VECTOR_VERSION" == *"latest"* ]]; then
  fail_arg_invalid "Error: Vector version $TEST_VECTOR_VERSION cannot contain 'latest'. Please specify the exact version instead. Dynamic versions cannot be used because it would otherwise be impossible to know which version the test data is associated with."
fi

IS_DEV_VECTOR_VERSION="false"
if [[ "$TEST_VECTOR_VERSION" == dev-* ]]; then
  IS_DEV_VECTOR_VERSION="true"
fi

if [[ "$IS_DEV_VECTOR_VERSION" == "true" ]] && [[ -z "$TEST_VECTOR_DEB_PATH" ]]; then
  fail_arg_invalid "Error: you must supply a \$VECTOR_TEST_VECTOR_DEB_PATH when specifying a development Vector version"
fi

#
# Header
#

print_vector_logo
print_divider
cat <<END
User ID: $TEST_USER_ID
Test name: $TEST_NAME
Test configuration: $TEST_CONFIGURATION
Test timestamp: $TEST_TIMESTAMP
Test results S3 bucket name: $TEST_RESULTS_S3_BUCKET_NAME
END
if [[ -n "$TEST_SUBJECT" ]]; then
  echo "Test subject: $TEST_SUBJECT"
else
  echo "Testing all subjects: ${TEST_SUBJECT_NAMES[*]}"
fi
if [[ -n "$TEST_VECTOR_VERSION" ]]; then
  echo "Vector version: $TEST_VECTOR_VERSION"
fi
if [[ -n "$TEST_VECTOR_DEB_PATH" ]]; then
  echo "Custom vector.deb path: $TEST_VECTOR_DEB_PATH"
fi
print_divider
print_spacer

prepare_terraform_params true
prepare_ansible_config true
prepare_ansible_extra_args_array true

#
# Execute
#

cd "$TEST_CASE_DIR"

#
# Terraform
#

if [ "$BOOTSTRAP" == 'true' ] && [ "$SKIP_TERRAFORM" != 'true' ]; then
  print_section_header "Provisioning test resources"

  cd terraform

  export TF_IN_AUTOMATION=true
  export TF_DATA_DIR=".terraform/$TEST_CONFIGURATION/$TEST_USER_ID"

  terraform init \
    "${TERRAFORM_COMMON_EXTRA_ARGS[@]}" \
    -backend-config="bucket=${VECTOR_TEST_STATE_S3_BUCKET_NAME}" \
    -backend-config="region=${AWS_DEFAULT_REGION}" \
    -backend-config="encrypt=true" \
    -backend-config="dynamodb_table=TerraformLocks" \
    -backend-config="key=vector-test-case/$TEST_NAME/$TEST_CONFIGURATION/$TEST_USER_ID.tfstate" \
    "${TERRAFORM_INIT_EXTRA_ARGS[@]}"

  terraform apply \
    "${TERRAFORM_COMMON_EXTRA_ARGS[@]}" \
    -auto-approve \
    -var pub_key="$TEST_SSH_PUBLIC_KEY" \
    -var test_name="$TEST_NAME" \
    -var test_configuration="$TEST_CONFIGURATION" \
    -var user_id="$TEST_USER_ID" \
    -var results_s3_bucket_name="$TEST_RESULTS_S3_BUCKET_NAME" \
    -var-file="../configurations/$TEST_CONFIGURATION/terraform.tfvars"

  cd ..
else
  echo "Bootstrap is disabled, skipping terraform bootstrap process"
fi

#
# Ansible
#

cd ansible
TEST_NAMESPACE="tag_TestName_$TEST_NAME:&tag_TestConfiguration_$TEST_CONFIGURATION:&tag_TestUserID_$TEST_USER_ID"
EXTRA_VARS="\
test_name=$TEST_NAME \
test_configuration=$TEST_CONFIGURATION \
test_namespace=$TEST_NAMESPACE \
test_timestamp=$TEST_TIMESTAMP \
user_id=$TEST_USER_ID \
results_s3_bucket_name=$TEST_RESULTS_S3_BUCKET_NAME"

ANSIBLE_COMMON_ARGS_ARRAY=(
  -u ubuntu
  --extra-vars "@../configurations/$TEST_CONFIGURATION/ansible.yml"
  --extra-vars "ansible_ssh_private_key_file=$VECTOR_TEST_SSH_PRIVATE_KEY"
  --extra-vars "$EXTRA_VARS"
)

if [[ -n "$TEST_VECTOR_VERSION" ]]; then
  ANSIBLE_COMMON_ARGS_ARRAY+=(
    --extra-vars "vector_version=$TEST_VECTOR_VERSION"
  )
fi

if [[ -n "$TEST_VECTOR_DEB_PATH" ]]; then
  ANSIBLE_COMMON_ARGS_ARRAY+=(
    --extra-vars "local_vector_deb_path=$TEST_VECTOR_DEB_PATH"
  )
fi

if [ "$BOOTSTRAP" == 'true' ]; then
  print_section_header "Bootstrapping test resources"

  set -x

  ansible-playbook \
    "${ANSIBLE_COMMON_ARGS_ARRAY[@]}" \
    "${ANSIBLE_EXTRA_ARGS_ARRAY[@]}" \
    bootstrap.yml

  set +x

else
  echo "Bootstrap is disabled, skipping ansible bootstrap process"
fi

run() {
  local RUN_SUBJECT="$1"
  if grep --quiet "role: $RUN_SUBJECT" bootstrap.yml; then
    print_section_header "Testing $RUN_SUBJECT"

    set -x

    ansible-playbook \
      "${ANSIBLE_COMMON_ARGS_ARRAY[@]}" \
      --extra-vars "test_subject=$RUN_SUBJECT" \
      "${ANSIBLE_EXTRA_ARGS_ARRAY[@]}" \
      run.yml

    set +x

  else
    echo "$RUN_SUBJECT is not installed for this test, skipping..."
  fi
}

if [ "$RUN" == 'true' ]; then
  if [ -z "$TEST_SUBJECT" ]; then
    echo "No test subject specified, running them all..."

    for SUBJECT in "${TEST_SUBJECT_NAMES[@]}"; do
      run "$SUBJECT"
    done
  else
    echo "Running $TEST_SUBJECT only..."
    run "$TEST_SUBJECT"
  fi
else
  echo "Run is disabled, skipping ansible run process"
fi

cd ..

#
# Footer
#

print_divider

cat <<END
Done!

To see results, run:

  bin/compare -t $TEST_NAME -c $TEST_CONFIGURATION

END