Initial commit of continuous integration components

Makefile

@@ -0,0 +1,237 @@
+################################################################################
+# Description:
+#  Executes testing and validation for python code and configuration files
+#  within a StackStorm pack.
+#
+# =============================================
+
+#PACK_DIR := $(dir $(realpath $(firstword $(MAKEFILE_LIST))))
+ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+PACK_DIR := $(ROOT_DIR)/../
+CI_DIR ?= $(ROOT_DIR)
+YAML_FILES := $(shell git ls-files '*.yaml' '*.yml')
+JSON_FILES := $(shell git ls-files '*.json')
+PY_FILES   := $(shell git ls-files '*.py')
+VIRTUALENV_DIR ?= $(ROOT_DIR)/virtualenv
+ST2_REPO_PATH ?= /tmp/st2
+ST2_REPO_BRANCH ?= master
+
+PACK_NAME ?= Caller_needs_to_set_variable_PACK_NAME
+
+export ST2_REPO_PATH ROOT_DIR
+
+# All components are prefixed by st2
+COMPONENTS := $(wildcard /tmp/st2/st2*)
+
+.PHONY: all
+# don't register right now (requires us to install stackstorm)
+#all: requirements lint packs-resource-register packs-tests
+all: requirements lint packs-tests
+
+.PHONY: pack-name
+pack-name:
+	@echo $(PACK_NAME)
+
+.PHONY: clean
+clean: .clean-st2-repo .clean-virtualenv .clean-pack
+
+.PHONY: lint
+lint: requirements flake8 pylint configs-check metadata-check
+
+.PHONY: flake8
+flake8: requirements .flake8
+
+.PHONY: pylint
+pylint: requirements .clone-st2-repo .pylint
+
+.PHONY: configs-check
+configs-check: requirements .clone-st2-repo .copy-pack-to-subdirectory .configs-check
+
+.PHONY: metadata-check
+metadata-check: requirements .metadata-check
+
+# Task which copies pack to temporary sub-directory so we can use old-style check scripts which
+# # require pack to be in a sub-directory
+.PHONY: .copy-pack-to-subdirectory
+.copy-pack-to-subdirectory:
+	mkdir -p /tmp/packs/$(PACK_NAME)
+	cd $(PACK_DIR);	find . -name 'ci' -prune -or -name '.git' -or -exec cp --parents '{}' '/tmp/packs/$(PACK_NAME)' ';'
+
+.PHONY: .clean-pack
+.clean-pack:
+	@echo "==================== cleaning packs ===================="
+	rm -rf /tmp/packs
+
+.PHONY: packs-resource-register
+packs-resource-register: requirements .clone-st2-repo .copy-pack-to-subdirectory .install-mongodb .packs-resource-register
+
+.PHONY: packs-missing-tests
+packs-missing-tests: requirements .packs-missing-tests
+
+.PHONY: packs-tests
+packs-tests: requirements .clone-st2-repo .packs-tests
+
+.PHONY: .flake8
+.flake8:
+	@echo
+	@echo "==================== flake8 ===================="
+	@echo
+	. $(VIRTUALENV_DIR)/bin/activate; \
+	for py in $(PY_FILES); do \
+		flake8 --config $(CI_DIR)/lint-configs/python/.flake8 $$py || exit 1; \
+	done
+
+
+.PHONY: .pylint
+.pylint:
+	@echo
+	@echo "==================== pylint ===================="
+	@echo
+	. $(VIRTUALENV_DIR)/bin/activate; \
+  REQUIREMENTS_DIR=$(CI_DIR)/ CONFIG_DIR=$(CI_DIR)/lint-configs/ st2-check-pylint-pack $(PACK_DIR) || exit 1;
+
+
+.PHONY: .configs-check
+.configs-check:
+	@echo
+	@echo "==================== configs-check ===================="
+	@echo
+	. $(VIRTUALENV_DIR)/bin/activate; \
+	for yaml in $(YAML_FILES); do \
+		st2-check-validate-yaml-file $$yaml || exit 1; \
+	done
+	. $(VIRTUALENV_DIR)/bin/activate; \
+	for json in $(JSON_FILES); do \
+		st2-check-validate-json-file $$json || exit 1; \
+	done
+	@echo
+	@echo "==================== example config check ===================="
+	@echo
+	. $(VIRTUALENV_DIR)/bin/activate; \
+	st2-check-validate-pack-example-config /tmp/packs/$(PACK_NAME) || exit 1;
+
+.PHONY: .metadata-check
+.metadata-check:
+	@echo
+	@echo "==================== metadata-check ===================="
+	@echo
+	. $(VIRTUALENV_DIR)/bin/activate; \
+  st2-check-validate-pack-metadata-exists $(PACK_DIR) || exit 1;
+
+.PHONY: .install-mongodb
+.install-monogodb:
+# @todo
+# install_mongodb() {
+#   ST2_MONGODB_PASSWORD=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 24 ; echo '')
+#   # Add key and repo for the latest stable MongoDB (3.2)
+#   sudo rpm --import https://www.mongodb.org/static/pgp/server-3.2.asc
+#   sudo sh -c "cat <<EOT > /etc/yum.repos.d/mongodb-org-3.2.repo
+# [mongodb-org-3.2]
+# name=MongoDB Repository
+# baseurl=https://repo.mongodb.org/yum/redhat/7/mongodb-org/3.2/x86_64/
+# gpgcheck=1
+# enabled=1
+# gpgkey=https://www.mongodb.org/static/pgp/server-3.2.asc
+# EOT"
+#   sudo yum -y install mongodb-org
+#   # Configure MongoDB to listen on localhost only
+#   sudo sed -i -e "s#bindIp:.*#bindIp: 127.0.0.1#g" /etc/mongod.conf
+#   sudo systemctl start mongod
+#   sudo systemctl enable mongod
+#   sleep 5
+#   # Create admin user and user used by StackStorm (MongoDB needs to be running)
+#   mongo <<EOF
+# use admin;
+# db.createUser({
+#     user: "admin",
+#     pwd: "${ST2_MONGODB_PASSWORD}",
+#     roles: [
+#         { role: "userAdminAnyDatabase", db: "admin" }
+#     ]
+# });
+# quit();
+# EOF
+#   mongo <<EOF
+# use st2;
+# db.createUser({
+#     user: "stackstorm",
+#     pwd: "${ST2_MONGODB_PASSWORD}",
+#     roles: [
+#         { role: "readWrite", db: "st2" }
+#     ]
+# });
+# quit();
+# EOF
+#   # Require authentication to be able to acccess the database
+#   sudo sh -c 'echo -e "security:\n  authorization: enabled" >> /etc/mongod.conf'
+#   # MongoDB needs to be restarted after enabling auth
+#   sudo systemctl restart mongod
+# }
+
+
+.PHONY: .packs-resource-register
+.packs-resource-register:
+	@echo
+	@echo "==================== packs-resource-register ===================="
+	@echo
+	. $(VIRTUALENV_DIR)/bin/activate; \
+	ST2_CONFIG_FILE=$(CI_DIR)/st2.tests.conf st2-check-register-pack-resources /tmp/packs/$(PACK_NAME) || exit 1;
+
+
+.PHONY: .packs-tests
+.packs-tests:
+	@echo
+	@echo "==================== packs-tests ===================="
+	@echo
+	. $(VIRTUALENV_DIR)/bin/activate; \
+	$(ST2_REPO_PATH)/st2common/bin/st2-run-pack-tests -x -p $(PACK_DIR) || exit 1;
+
+.PHONY: .packs-missing-tests
+.packs-missing-tests:
+	@echo
+	@echo "==================== pack-missing-tests ===================="
+	@echo
+	. $(VIRTUALENV_DIR)/bin/activate; \
+	st2-check-print-pack-tests-coverage $(PACK_DIR) || exit 1;
+
+.PHONY: .clone-st2-repo
+.clone-st2-repo:
+	@echo
+	@echo "==================== cloning st2 repo ===================="
+	@echo
+	if [ ! -d "$(ST2_REPO_PATH)" ]; then \
+		git clone https://github.com/StackStorm/st2.git --depth 1 --single-branch --branch $(ST2_REPO_BRANCH) $(ST2_REPO_PATH); \
+	else \
+		pushd $(ST2_REPO_PATH); \
+		git pull; \
+		popd; \
+	fi;
+
+.PHONY: .clean-st2-repo
+.clean-st2-repo:
+	@echo "==================== cleaning st2 repo ===================="
+	rm -rf $(ST2_REPO_PATH)
+
+.PHONY: requirements
+requirements: virtualenv
+	@echo
+	@echo "==================== requirements ===================="
+	@echo
+	. $(VIRTUALENV_DIR)/bin/activate; \
+	$(VIRTUALENV_DIR)/bin/pip install --upgrade pip; \
+	$(VIRTUALENV_DIR)/bin/pip install --cache-dir $(HOME)/.pip-cache -q -r $(CI_DIR)/requirements-dev.txt; \
+	$(VIRTUALENV_DIR)/bin/pip install --cache-dir $(HOME)/.pip-cache -q -r $(CI_DIR)/requirements-pack-tests.txt;
+
+.PHONY: virtualenv
+virtualenv: $(VIRTUALENV_DIR)/bin/activate
+$(VIRTUALENV_DIR)/bin/activate:
+	@echo
+	@echo "==================== virtualenv ===================="
+	@echo
+	test -d $(VIRTUALENV_DIR) || virtualenv --no-site-packages $(VIRTUALENV_DIR)
+
+
+.PHONY: .clean-virtualenv
+.clean-virtualenv:
+	@echo "==================== cleaning virtualenv ===================="
+	rm -rf $(VIRTUALENV_DIR)

lint-configs/python/.flake8

@@ -0,0 +1,4 @@
+[flake8]
+max-line-length = 100
+ignore = E128,E402
+exclude=*.egg/*

lint-configs/python/.pylintrc

@@ -0,0 +1,31 @@
+[MESSAGES CONTROL]
+# C0111 Missing docstring
+# I0011 Warning locally suppressed using disable-msg
+# I0012 Warning locally suppressed using disable-msg
+# W0704 Except doesn't do anything Used when an except clause does nothing but "pass" and there is no "else" clause
+# W0142 Used * or * magic* Used when a function or method is called using *args or **kwargs to dispatch arguments.
+# W0212 Access to a protected member %s of a client class
+# W0232 Class has no __init__ method Used when a class has no __init__ method, neither its parent classes.
+# W0613 Unused argument %r Used when a function or method argument is not used.
+# W0702 No exception's type specified Used when an except clause doesn't specify exceptions type to catch.
+# R0201 Method could be a function
+# W0614 Unused import XYZ from wildcard import
+# R0914 Too many local variables
+# R0912 Too many branches
+# R0915 Too many statements
+# R0913 Too many arguments
+# R0904 Too many public methods
+# E0211: Method has no argument
+# E1128: Assigning to function call which only returns None Used when an assignment is done on a function call but the inferred function returns nothing but None.
+# E1129: Context manager ‘%s’ doesn’t implement __enter__ and __exit__. Used when an instance in a with statement doesn’t implement the context manager protocol(__enter__/__exit__).
+disable=C0103,C0111,I0011,I0012,W0704,W0142,W0212,W0232,W0613,W0702,R0201,W0614,R0914,R0912,R0915,R0913,R0904,R0801,not-context-manager,assignment-from-none
+
+[TYPECHECK]
+# Note: This modules are manipulated during the runtime so we can't detect all the properties during
+# static analysis
+ignored-modules=distutils,eventlet.green.subprocess,six,six.moves
+
+[FORMAT]
+max-line-length=100
+max-module-lines=1000
+indent-string='    '

requirements-dev.txt

@@ -0,0 +1,7 @@
+# Needed for chek and lint scripts
+-e git+https://github.com/StackStorm/st2sdk.git@master#egg=st2sdk
+pyyaml
+pep8>=1.6.0,<1.7
+flake8>=2.3.0,<2.4
+astroid==1.3.8
+pylint==1.4.4

requirements-pack-tests.txt

@@ -0,0 +1,3 @@
+mock>=1.3.0,<2.0
+unittest2>=1.1.0,<2.0
+nose>=1.3.7

st2.tests.conf

@@ -0,0 +1,74 @@
+# Config file used by integration tests
+#
+[api]
+# Host and port to bind the API server.
+host = 0.0.0.0
+port = 9101
+logging = st2tests/conf/logging.api.conf
+mask_secrets = False
+# allow_origin is required for handling CORS in st2 web UI.
+# allow_origin = http://myhost1.example.com:3000,http://myhost2.example.com:3000
+
+[sensorcontainer]
+logging = st2tests/conf/logging.sensorcontainer.conf
+sensor_node_name = sensornode1
+partition_provider = name:default
+
+[rulesengine]
+logging = st2reactor/conf/logging.rulesengine.conf
+
+[actionrunner]
+logging = st2actions/conf/logging.conf
+
+[auth]
+host = 0.0.0.0
+port = 9100
+use_ssl = False
+debug = False
+enable = False
+logging = st2tests/conf/logging.auth.conf
+
+mode = standalone
+backend = flat_file
+backend_kwargs = {"file_path": "st2auth/conf/htpasswd_dev"}
+
+# Base URL to the API endpoint excluding the version (e.g. http://myhost.net:9101/)
+api_url = http://127.0.0.1:9101/
+
+[system]
+base_path = /opt/stackstorm
+admin_users = testu
+
+[content]
+system_packs_base_path =
+packs_base_paths = /tmp/packs/
+
+[syslog]
+host = localhost
+port = 514
+facility = local7
+protocol = udp
+
+[log]
+excludes = requests,paramiko
+redirect_stderr = False
+mask_secrets = False
+
+[system_user]
+user = stanley
+ssh_key_file = /home/vagrant/.ssh/stanley_rsa
+
+[messaging]
+url = amqp://guest:guest@localhost:5672/
+
+[ssh_runner]
+remote_dir = /tmp
+
+[resultstracker]
+logging = st2actions/conf/logging.resultstracker.conf
+
+[notifier]
+logging = st2actions/conf/logging.notifier.conf
+
+[exporter]
+logging = st2exporter/conf/logging.exporter.conf