finished adding code to do full coverage tests of the f5 modules usin…

…g ansible playbooks

Makefile

@@ -4,8 +4,16 @@ DOCTEST := python ../scripts/ansible-doc-test.py
 PYHOOK := 'import sys;sys.path.insert(1,".")'
 PYLINT := pylint --additional-builtins=_ --init-hook=$(PYHOOK)
 
+MODULE_TARGET = $(shell echo $@ | tr '-' '_')
+
 .PHONY: docs flake8
 
+all: clean-coverage
+	ansible-playbook -i inventory/hosts playbooks/toggle-coverage.yaml -e "f5_module=all toggle=on" --vault-password-file ./vault.txt
+	COVERAGE_PROCESS_START=${CURDIR}/.coveragerc ANSIBLE_KEEP_REMOTE_FILES=1 ansible-playbook -i inventory/hosts tests/bigip.yaml --vault-password-file ./vault.txt
+	ansible-playbook -i inventory/hosts playbooks/toggle-coverage.yaml -e "f5_module=all toggle=off" --vault-password-file ./vault.txt
+	flake8 library/*.py
+
 all-tests: flake8 ansible-doc
 
 all-tests-dev: flake8 ansible-doc ansible-doc-dev
@@ -52,97 +60,15 @@ ansible-doc-dev:
 		$(DOCTEST) -M . bigip_qkview_facts.py; \
 	)
 
-bigip-device-dns:
-	ansible-playbook -i inventory/hosts tests/bigip_device_dns.yaml -vvvv
-	flake8 library/bigip_device_dns.py
-
-bigip-device-ntp:
-	ansible-playbook -i inventory/hosts tests/bigip_device_ntp.yaml -vvvv
-	flake8 library/bigip_device_ntp.py
-
-bigip-device-sshd:
-	ansible-playbook -i inventory/hosts tests/bigip_device_sshd.yaml -vvvv
-	flake8 library/bigip_device_sshd.py
-
-bigip-facts:
-	ansible-playbook -i inventory/hosts tests/bigip_facts.yaml -vvvv
-	flake8 library/bigip_facts.py
-
-bigip-gtm-datacenter:
-	ansible-playbook -i inventory/hosts tests/bigip_gtm_datacenter.yaml -vvvv
-	flake8 library/bigip_gtm_datacenter.py
-
-bigip-gtm-facts:
-	ansible-playbook -i inventory/hosts tests/bigip_gtm_facts.yaml -vvvv
-	flake8 library/bigip_gtm_facts.py
-
-bigip-gtm-virtual-server:
-	ansible-playbook -i inventory/hosts tests/bigip_gtm_virtual_server.yaml -vvvv
-	flake8 library/bigip_gtm_virtual_server.py
-
-bigip-gtm-wide-ip:
-	ansible-playbook -i inventory/hosts tests/bigip_gtm_wide_ip.yaml -vvvv
-	flake8 library/bigip_gtm_wide_ip.py
-
-bigip-hostname:
-	ansible-playbook -i inventory/hosts tests/bigip_hostname.yaml -vvvv
-	flake8 library/bigip_hostname.py
-
-bigip-irule:
-	ansible-playbook -i inventory/hosts tests/bigip_irule.yaml -vvvv
-	flake8 library/bigip_irule.py
-
-bigip-monitor-http:
-	ansible-playbook -i inventory/hosts tests/bigip_monitor_http.yaml -vvvv
-	flake8 library/bigip_monitor_http.py
-
-bigip-monitor-tcp:
-	ansible-playbook -i inventory/hosts tests/bigip_monitor_tcp.yaml -vvvv
-	flake8 library/bigip_monitor_tcp.py
-
-bigip-node:
-	ansible-playbook -i inventory/hosts tests/bigip_node.yaml -vvvv
-	flake8 library/bigip_node.py
-
-bigip-pool:
-	ansible-playbook -i inventory/hosts tests/bigip_pool.yaml -vvvv
-	flake8 library/bigip_pool.py
-
-bigip-pool-member:
-	ansible-playbook -i inventory/hosts tests/bigip_pool_member.yaml -vvvv
-	flake8 library/bigip_pool_member.py
-
-bigip-routedomain:
-	ansible-playbook -i inventory/hosts tests/bigip_routedomain.yaml -vvvv
-	flake8 library/bigip_routedomain.py
-
-bigip-selfip:
-	ansible-playbook -i inventory/hosts tests/bigip_selfip.yaml -vvvv
-	flake8 library/bigip_selfip.py
-
-bigip-ssl-certificate:
-	ansible-playbook -i inventory/hosts tests/bigip_ssl_certificate.yaml -vvvv
-	flake8 library/bigip_ssl_certificate.py
-
-bigip-sys-db:
-	ansible-playbook -i inventory/hosts tests/bigip_sys_db.yaml -vvvv
-	flake8 library/bigip_sys_db.py
-
-bigip-sys-global:
-	ansible-playbook -i inventory/hosts tests/bigip_sys_global.yaml -vvvv
-	flake8 library/bigip_sys_global.py
-
-bigip-user-facts:
-	ansible-playbook -i inventory/hosts tests/bigip_user_facts.yaml -vvvv
-	flake8 library/bigip_user_facts.py
-
-bigip-virtual-server:
-	ansible-playbook -i inventory/hosts tests/bigip_virtual_server.yaml -vvvv
-	flake8 library/bigip_virtual_server.py
+clean-coverage:
+	$(shell rm cache/coverage/.coverage*)
+	$(shell rm .coverage)
 
-bigip-vlan:
-	ansible-playbook -i inventory/hosts tests/bigip_vlan.yaml -vvvv
-	flake8 library/bigip_vlan.py
+bigip-%: clean-coverage
+	ansible-playbook -i inventory/hosts playbooks/toggle-coverage.yaml -e "f5_module=${MODULE_TARGET} toggle=on" --vault-password-file ./vault.txt
+	COVERAGE_PROCESS_START=${CURDIR}/.coveragerc ANSIBLE_KEEP_REMOTE_FILES=1 ansible-playbook -i inventory/hosts tests/${MODULE_TARGET}.yaml --vault-password-file ./vault.txt
+	ansible-playbook -i inventory/hosts playbooks/toggle-coverage.yaml -e "f5_module=${MODULE_TARGET} toggle=off" --vault-password-file ./vault.txt
+	flake8 library/${MODULE_TARGET}.py
 
 fetch-upstream:
 	curl -o library/bigip_device_dns.py https://raw.githubusercontent.com/ansible/ansible-modules-extras/devel/network/f5/bigip_device_dns.py

ansible.cfg

@@ -5,3 +5,4 @@ roles_path=./roles
 library=./library
 callback_plugins = ./plugins/callback_plugins
 lookup_plugins = ./plugins/lookup_plugins
+filter_plugins = ./plugins/filter_plugins

roles/toggle_coverage/tasks/module-plural.yaml

@@ -0,0 +1,50 @@
+---
+
+- include: process-module.yaml
+  with_fileglob:
+      - "{{ base_dir }}/library/bigip_*.py"
+  loop_control:
+      loop_var: f5_module_filename
+
+# This cannot be inside of the files loop (which, as a consequence
+# makes the overall role slower) because of the way that coverage works
+#
+# For example, upon the first run, the 'coverage combine' command will
+# merge all of the .coverage files.
+#
+# On following runs, the tasks looking for filenames to replace in the
+# coverage file, causing the same index to exist multiple times. When
+# coverage tries to iterate over the keys, only the last key will be
+# evaluated, so the whole set of coverage will be off.
+#
+# For example, running in parallel produces keys that look like this
+#
+#    !coverage.py: This is a private format, don't read it directly!
+#    {
+#      "arcs": {
+#          "/p...e/var/f...s/jc/9d1...m45/T/ansible_rZUTVv/ansible_module_bigip_vlan.py":
+#              [[-136, 137],....
+#          ],
+#          "/p...e/var/f...s/jc/9d1...m45/T/ansible_jhhs8v/ansible_module_bigip_vlan.py":
+#              [[-143, 187],....
+#          ]
+#          ...
+#
+# The renaming task makes all of these look like this
+#
+#    !coverage.py: This is a private format, don't read it directly!
+#    {
+#      "arcs": {
+#          "/Users/trupp/src/f5-ansible/library/bigip_vlan.py":
+#              [[-136, 137],....
+#          ],
+#          "/Users/trupp/src/f5-ansible/library/bigip_vlan.py":
+#              [[-143, 187],....
+#          ]
+#          ...
+- name: Combine coverage files
+  shell: "coverage combine cache/coverage/"
+  when: toggle == 'off'
+  args:
+      chdir: "{{ base_dir|abspath }}"
+      executable: "/bin/bash"

roles/toggle_coverage/tasks/module-single.yaml

@@ -0,0 +1,40 @@
+---
+
+- name: Create some module shorthand variables
+  set_fact:
+      module_full_path: "{{ playbook_dir + '/../library/' + f5_module + '.py' }}"
+
+- name: Replace the name of the module in the coverage includes
+  lineinfile:
+      insertafter: "include ="
+      regexp: "(MODULE_NAME|{{ f5_module }}\\*)"
+      line: "    *ansible_module_{{ f5_module }}*"
+      state: "present"
+      dest: "{{ base_dir|abspath }}/.coveragerc"
+  when: toggle == 'on'
+  run_once: true
+
+- name: Replace cache name in coverage file with module location
+  replace:
+      dest: "{{ item|abspath }}"
+      regexp: '"([A-Za-z0-9_\./\\-]*{{ f5_module }}.py)"'
+      replace: '"{{ module_full_path|abspath }}"'
+  when: toggle == 'off'
+  with_fileglob:
+      - "{{ coverage_dir|abspath }}/.coverage.*"
+
+- name: Combine coverage files
+  shell: "coverage combine cache/coverage/"
+  when: toggle == 'off'
+  args:
+      chdir: "{{ base_dir|abspath }}"
+      executable: "/bin/bash"
+  run_once: true
+
+- name: Replace the name of the module to match original
+  lineinfile:
+      regexp: "\\*ansible_module_{{ f5_module }}\\*"
+      line: "    *{{ f5_module }}*"
+      state: "present"
+      dest: "{{ base_dir|abspath }}/.coveragerc"
+  when: toggle == 'off'

roles/toggle_coverage/tasks/process-module-off.yaml

@@ -0,0 +1,20 @@
+---
+
+- name: Specify which module we are handling
+  debug:
+      msg: "Disabling coverage for module '{{ module_current }}'"
+
+- name: Fix module location to no longer be temporary path
+  replace:
+      dest: "{{ item|abspath }}"
+      regexp: '"([A-Za-z0-9_\./\\-]*{{ module_current }}.py)"'
+      replace: '"{{ module_full_path|abspath }}"'
+  with_fileglob:
+      - "{{ coverage_dir|abspath }}/.coverage*"
+
+- name: Replace the name of the module in .coveragerc to match original
+  lineinfile:
+      regexp: "\\*ansible_module_{{ module_current }}\\*"
+      line: "    *{{ module_current }}*"
+      state: "present"
+      dest: "{{ base_dir|abspath }}/.coveragerc"

roles/toggle_coverage/tasks/process-module-on.yaml

@@ -0,0 +1,13 @@
+---
+
+- name: Specify which module we are handling
+  debug:
+      msg: "Enabling coverage for module '{{ module_current }}'"
+
+- name: Replace the name of the module in the coverage includes
+  lineinfile:
+      insertafter: "include ="
+      regexp: "(MODULE_NAME|{{ module_current }}\\*)"
+      line: "    *ansible_module_{{ module_current }}*"
+      state: "present"
+      dest: "{{ base_dir|abspath }}/.coveragerc"

roles/toggle_coverage/tasks/process-module.yaml

@@ -0,0 +1,17 @@
+---
+
+- name: Set some convenience facts for later usage
+  set_fact:
+      module_current: "{{ f5_module_filename|basename|splitext|first }}"
+
+- name: Create some module shorthand variables
+  set_fact:
+      module_full_path: "{{ playbook_dir + '/../library/' + module_current + '.py' }}"
+
+- name: Include tasks to enable coverage if requested
+  include: "process-module-on.yaml"
+  when: toggle == 'on'
+
+- name: Include tasks to disable coverage if requested
+  include: "process-module-off.yaml"
+  when: toggle == 'off'

roles/toggle_coverage/templates/coveragerc

@@ -0,0 +1,7 @@
+[run]
+branch = True
+cover_pylib = False
+concurrency = multiprocessing
+data_file = COVERAGE_PATH
+include =
+    MODULE_NAME

tests/bigip.yaml

@@ -0,0 +1,26 @@
+---
+
+# Test and measure coverage across all modules
+#
+# This playbook can be used to run all of the tests for the various modules.
+# Additionally, it will run the coverage tests of the modules so that that
+# can be represented on the Github page.
+#
+# Usage:
+#
+#    export COVERAGE_PROCESS_START=${CURDIR}/.coveragerc
+#    export ANSIBLE_KEEP_REMOTE_FILES=1
+#    ansible-playbook -i inventory/hosts tests/bigip.yaml [OPTIONS]
+#
+# Examples:
+#
+#    Run all tests on the modules
+#
+#    export COVERAGE_PROCESS_START=${CURDIR}/.coveragerc
+#    export ANSIBLE_KEEP_REMOTE_FILES=1
+#    ansible-playbook -i inventory/hosts tests/bigip.yaml --vaukt-password-file ./vault.txt
+#
+
+# Leave these in alphabetical order for readability
+- include: bigip_selfip.yaml
+- include: bigip_vlan.yaml