refactor: use black, flake8 and precommit

.flake8

@@ -0,0 +1,3 @@
+[flake8]
+max-line-length = 160
+ignore = W503,W504,E712

.github/workflows/pre-commit.yml

@@ -0,0 +1,22 @@
+name: Pre-Commit Check
+
+on:
+  push:
+    branches:
+      - "master"
+      - "stable"
+  pull_request:
+    branches:
+      - "master"
+      - "stable"
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Pre-Commit
+        uses: pre-commit/action@v3.0.0

.pre-commit-config.yaml

@@ -1,30 +1,19 @@
 repos:
-- repo: https://github.com/asottile/reorder_python_imports
-  rev: v1.3.5
-  hooks:
-  - id: reorder-python-imports
-    language_version: python3
-- repo: https://github.com/ambv/black
-  rev: 18.9b0
-  hooks:
-  - id: black
-    args: [--safe, --quiet, --line-length, "160"]
-    language_version: python3
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v2.1.0
+  rev: v4.4.0
   hooks:
   - id: trailing-whitespace
-    language_version: python3
   - id: end-of-file-fixer
-    language_version: python3
-  - id: check-yaml
-    language_version: python3
-  - id: debug-statements
-    language_version: python3
-  - id: flake8
-    language_version: python3
-- repo: https://github.com/asottile/pyupgrade
-  rev: v1.11.3
+  - id: check-ast
+  - id: check-docstring-first
+- repo: https://github.com/psf/black
+  rev: 23.3.0
   hooks:
-  - id: pyupgrade
-    language_version: python3
+  - id: black
+    args: [--quiet]
+    files: "grouper|listener|evaluator"
+- repo: https://github.com/PyCQA/flake8
+  rev: 6.0.0
+  hooks:
+  - id: flake8
+    files: "advisor_listener|common|database|listener|evaluator|manager|taskomatic|tests|vmaas_sync|notificator"

Jenkinsfile

@@ -51,4 +51,3 @@ pipeline {
         }
     }
 }
-

Pipfile

@@ -4,6 +4,7 @@ url = "https://pypi.org/simple"
 verify_ssl = true
 
 [dev-packages]
+pre-commit = "==3.3.3"
 
 [packages]
 aiodns = "==3.0.0"

README.md

@@ -17,6 +17,14 @@ This project uses semantic versioning https://semver.org/. This process is autom
   - Upload service, Inventory, Kafka message queue
   - Mocked Platform service is part of this repository (for development purposes)
 
+## Local setup
+Install a local python environment with pre-commit:
+```
+pipenv install --dev
+pipenv shell
+pre-commit install
+```
+
 ## Local testing
 Build images and start containers:
 ~~~bash
@@ -58,14 +66,12 @@ You can run all tests from scratch just after cloning repo using command:
 docker-compose -f docker-compose.test.yml up --build --exit-code-from test
 ~~~
 
-Or locally
-
+Or locally:
 ~~~bash
 # install postgresql and pyenv
-pyenv install 3.8
 cd tests
-pyenv shell
 pipenv install
+pipenv shell
 cd ..
 pytest -vvv tests/
 ~~~

database/README.md

@@ -26,7 +26,7 @@ This is a quick step-by-step of how to add a database schema upgrade.
 
  * add a CREATE USER command to database/schema/ve_db_user_create_postgresql.sql
  * add SELECT privileges for all tables for this user at the bottom of database/schema/ve_db_postgresql.sql
- * throughout the database/schema/ve_db_postgresql.sql file, for each table to which the user should have write privileges, add a GRANT INSERT, UPDATE, DELETE statement under the CREATE TABLE statement to give these privileges to the user.  If there is a sequence, also add a GRANT USAGE, SELECT, UPDATE on the sequence as well. 
+ * throughout the database/schema/ve_db_postgresql.sql file, for each table to which the user should have write privileges, add a GRANT INSERT, UPDATE, DELETE statement under the CREATE TABLE statement to give these privileges to the user.  If there is a sequence, also add a GRANT USAGE, SELECT, UPDATE on the sequence as well.
  * add a new envirnment variable to database/Dockerfile to hold the password for the new user.
  * in the database/schema/init_schema.sh file, add an addition psql command to alter the new user and provide the user's password using the environment variable defined in the previous step.
  * add an upgrade script (see section above) to create the new user and grant permissions to existing database instances.
@@ -55,5 +55,3 @@ This is a quick step-by-step of how to add a database schema upgrade.
 #### Command to open shell in container
 
 ```docker exec -it ve_db_ctr bash```
-
-

database/schema/upgrade_scripts/001-db-upgrade-support.sql

@@ -43,4 +43,3 @@ GRANT USAGE, SELECT ON db_upgrade_log_id_seq TO ve_db_user_listener;
 -- user for UI manager component
 GRANT SELECT ON db_version TO ve_db_user_manager;
 GRANT SELECT ON db_upgrade_log TO ve_db_user_manager;
-

database/schema/upgrade_scripts/002-satellite-managed-caching.sql

@@ -3,7 +3,7 @@
 DO $$
 BEGIN
     ALTER TABLE cve_affected_systems_cache ADD direct_systems_affected INT NOT NULL DEFAULT 0;
-EXCEPTION   
+EXCEPTION
     WHEN duplicate_column THEN
         RAISE NOTICE 'Column direct_systems_affected already exists.';
 END$$;
@@ -146,8 +146,8 @@ DO $$
 BEGIN
   CREATE TRIGGER system_platform_satellite_managed_cache
     AFTER UPDATE OF satellite_managed ON system_platform
-    FOR EACH ROW EXECUTE PROCEDURE satellite_managed_system_update_cache();  
-EXCEPTION   
+    FOR EACH ROW EXECUTE PROCEDURE satellite_managed_system_update_cache();
+EXCEPTION
   WHEN others THEN
     RAISE NOTICE 'Trigger system_platform_satellite_managed_cache already exists.';
 END$$;
@@ -156,4 +156,4 @@ END$$;
 GRANT UPDATE (direct_systems_affected) ON cve_affected_systems_cache TO ve_db_user_listener;
 
 -- refresh cached counts in order to populate direct_systems_affected
-SELECT refresh_all_cached_counts();
+SELECT refresh_all_cached_counts();

database/schema/upgrade_scripts/003-add-vmaas-sync-user.sql

@@ -5,4 +5,3 @@ ALTER USER ve_db_user_vmaas_sync WITH PASSWORD 've_db_user_vmaas_sync_pwd';
 
 GRANT SELECT ON ALL TABLES IN SCHEMA public TO ve_db_user_vmaas_sync;
 GRANT SELECT, INSERT, UPDATE, DELETE ON cve_metadata TO ve_db_user_vmaas_sync;
-

database/schema/upgrade_scripts/006-fix-refresh-caches.sql

@@ -30,4 +30,4 @@ $refresh_system_cached_counts$
     ) WHERE sp.inventory_id = inventory_id_in;
   END;
 $refresh_system_cached_counts$
-  LANGUAGE 'plpgsql';
+  LANGUAGE 'plpgsql';

database/schema/upgrade_scripts/013-satellite-managed-removal.sql

@@ -231,4 +231,4 @@ DROP FUNCTION IF EXISTS satellite_managed_system_update_cache;
 
 ALTER TABLE system_platform DROP COLUMN IF EXISTS satellite_managed;
 
-ALTER TABLE cve_affected_systems_cache DROP COLUMN IF EXISTS direct_systems_affected;
+ALTER TABLE cve_affected_systems_cache DROP COLUMN IF EXISTS direct_systems_affected;

database/schema/upgrade_scripts/025-migrate-affected-systems-to-account-data.sql

@@ -11,7 +11,7 @@ cad.rh_account_id = casc.rh_account_id;
 INSERT INTO cve_account_data (cve_id, rh_account_id, systems_affected)
 SELECT casc.cve_id, casc.rh_account_id, casc.systems_affected
 FROM cve_affected_systems_cache casc
-WHERE (casc.cve_id, casc.rh_account_id) NOT IN 
+WHERE (casc.cve_id, casc.rh_account_id) NOT IN
 (SELECT cad2.cve_id, cad2.rh_account_id FROM cve_account_data cad2);
 
 DROP TABLE cve_affected_systems_cache;

database/schema/upgrade_scripts/029-manager-inserting-accounts.sql

@@ -1,2 +1,2 @@
 GRANT INSERT, UPDATE ON rh_account TO ve_db_user_manager;
-GRANT USAGE, SELECT ON rh_account_id_seq TO ve_db_user_manager;
+GRANT USAGE, SELECT ON rh_account_id_seq TO ve_db_user_manager;

database/schema/upgrade_scripts/030-fqdn.sql

@@ -1 +1 @@
-ALTER TABLE system_platform ADD display_name TEXT;
+ALTER TABLE system_platform ADD display_name TEXT;

database/schema/upgrade_scripts/031-stale-dates.sql

@@ -1,3 +1,3 @@
 ALTER TABLE system_platform ADD stale_timestamp TIMESTAMP WITH TIME ZONE;
 ALTER TABLE system_platform ADD stale_warning_timestamp TIMESTAMP WITH TIME ZONE;
-ALTER TABLE system_platform ADD culled_timestamp TIMESTAMP WITH TIME ZONE;
+ALTER TABLE system_platform ADD culled_timestamp TIMESTAMP WITH TIME ZONE;

database/schema/upgrade_scripts/034-stale-flag.sql

@@ -90,4 +90,4 @@ $opt_out_system_update_cache$
     RETURN NEW;
   END;
 $opt_out_system_update_cache$
-  LANGUAGE 'plpgsql';
+  LANGUAGE 'plpgsql';

database/schema/upgrade_scripts/040-rules-table.sql

@@ -19,4 +19,4 @@ GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO ve_db_user_manager;
 GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO ve_db_user_vmaas_sync;
 GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO ve_db_user_metrics;
 GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO ve_db_user_taskomatic;
-GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO ve_db_user_advisor_listener;
+GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO ve_db_user_advisor_listener;

database/schema/upgrade_scripts/041-advisor-listener-write.sql

@@ -1,3 +1,3 @@
 GRANT SELECT, INSERT, UPDATE, DELETE ON rh_account TO ve_db_user_advisor_listener;
 GRANT SELECT, INSERT, UPDATE, DELETE ON system_platform TO ve_db_user_advisor_listener;
-GRANT SELECT, INSERT, UPDATE, DELETE ON cve_metadata TO ve_db_user_advisor_listener;
+GRANT SELECT, INSERT, UPDATE, DELETE ON cve_metadata TO ve_db_user_advisor_listener;

database/schema/upgrade_scripts/042-system-vulnerabilities.sql

@@ -1,3 +1,3 @@
 ALTER TABLE system_vulnerabilities ADD rule_id INT;
 
-GRANT SELECT, INSERT, UPDATE, DELETE ON system_vulnerabilities TO ve_db_user_advisor_listener;
+GRANT SELECT, INSERT, UPDATE, DELETE ON system_vulnerabilities TO ve_db_user_advisor_listener;

database/schema/upgrade_scripts/043-insights-rule-update.sql

@@ -1 +1 @@
-GRANT UPDATE ON insights_rule TO ve_db_user_advisor_listener;
+GRANT UPDATE ON insights_rule TO ve_db_user_advisor_listener;

database/schema/upgrade_scripts/044-rule_id-constraint.sql

@@ -1 +1 @@
-ALTER TABLE system_vulnerabilities ADD CONSTRAINT rule_id FOREIGN KEY (rule_id) REFERENCES insights_rule (id);
+ALTER TABLE system_vulnerabilities ADD CONSTRAINT rule_id FOREIGN KEY (rule_id) REFERENCES insights_rule (id);

database/schema/upgrade_scripts/045-cve_account_data-locking.sql

@@ -1 +1 @@
-GRANT SELECT, INSERT, UPDATE, DELETE ON cve_account_data TO ve_db_user_advisor_listener;
+GRANT SELECT, INSERT, UPDATE, DELETE ON cve_account_data TO ve_db_user_advisor_listener;

database/schema/upgrade_scripts/046-opt_out_system_cache.sql

@@ -86,4 +86,4 @@ $opt_out_system_update_cache$
     RETURN NEW;
   END;
 $opt_out_system_update_cache$
-  LANGUAGE 'plpgsql';
+  LANGUAGE 'plpgsql';

database/schema/upgrade_scripts/047-refresh_counts.sql

@@ -184,4 +184,4 @@ $refresh_system_cached_counts$
     ) WHERE sp.inventory_id = inventory_id_in;
   END;
 $refresh_system_cached_counts$
-  LANGUAGE 'plpgsql';
+  LANGUAGE 'plpgsql';

database/schema/upgrade_scripts/048-advisor_evaluated-column.sql

@@ -1 +1 @@
-ALTER TABLE system_platform ADD advisor_evaluated TIMESTAMP WITH TIME ZONE;
+ALTER TABLE system_platform ADD advisor_evaluated TIMESTAMP WITH TIME ZONE;

database/schema/upgrade_scripts/049-advisor_evaluated-usage.sql

@@ -264,4 +264,4 @@ $refresh_cve_account_cached_counts$
       AND rh_account_id = rh_account_id_in;
   END;
 $refresh_cve_account_cached_counts$
-  LANGUAGE 'plpgsql';
+  LANGUAGE 'plpgsql';

database/schema/upgrade_scripts/051-inisghts_rule-additional-data.sql

@@ -6,4 +6,4 @@ ALTER TABLE insights_rule ADD playbook_count INT;
 ALTER TABLE insights_rule ADD change_risk INT;
 ALTER TABLE insights_rule ADD kbase_node_id INT;
 
-GRANT UPDATE (description_text, summary_text, generic_text, reboot_required, playbook_count, change_risk, kbase_node_id) ON insights_rule TO ve_db_user_taskomatic;
+GRANT UPDATE (description_text, summary_text, generic_text, reboot_required, playbook_count, change_risk, kbase_node_id) ON insights_rule TO ve_db_user_taskomatic;

database/schema/upgrade_scripts/052-insights_rule-active.sql

@@ -1,3 +1,3 @@
 ALTER TABLE insights_rule ADD active BOOLEAN NOT NULL DEFAULT FALSE;
 
-GRANT UPDATE (active) ON insights_rule TO ve_db_user_taskomatic;
+GRANT UPDATE (active) ON insights_rule TO ve_db_user_taskomatic;

database/schema/upgrade_scripts/053-cve_rule_mapping-unique.sql

@@ -1 +1 @@
-ALTER TABLE cve_rule_mapping ADD CONSTRAINT cve_rule_mapping_cve_id_rule_id_key UNIQUE (cve_id, rule_id);
+ALTER TABLE cve_rule_mapping ADD CONSTRAINT cve_rule_mapping_cve_id_rule_id_key UNIQUE (cve_id, rule_id);

database/schema/upgrade_scripts/054-active-rules-procedures.sql

@@ -288,4 +288,4 @@ $refresh_system_cached_counts$
     ) WHERE sp.inventory_id = inventory_id_in;
   END;
 $refresh_system_cached_counts$
-  LANGUAGE 'plpgsql';
+  LANGUAGE 'plpgsql';

database/schema/upgrade_scripts/055-rule_hit_details.sql

@@ -1 +1 @@
-ALTER TABLE system_vulnerabilities ADD rule_hit_details TEXT;
+ALTER TABLE system_vulnerabilities ADD rule_hit_details TEXT;

database/schema/upgrade_scripts/056-dont-refresh-stale.sql

@@ -176,4 +176,4 @@ $refresh_cve_account_cached_counts$
       AND rh_account_id = rh_account_id_in;
   END;
 $refresh_cve_account_cached_counts$
-  LANGUAGE 'plpgsql';
+  LANGUAGE 'plpgsql';

database/schema/upgrade_scripts/057-rules_more_info.sql

@@ -2,4 +2,4 @@ ALTER TABLE insights_rule ADD reason_text TEXT;
 ALTER TABLE insights_rule ADD resolution_text TEXT;
 ALTER TABLE insights_rule ADD more_info_text TEXT;
 
-GRANT UPDATE (reason_text, resolution_text, more_info_text) ON insights_rule TO ve_db_user_taskomatic;
+GRANT UPDATE (reason_text, resolution_text, more_info_text) ON insights_rule TO ve_db_user_taskomatic;

database/schema/upgrade_scripts/058-system_platform-indexes.sql

@@ -2,4 +2,4 @@ DROP INDEX IF EXISTS system_platform_stale_timestamp_idx;
 
 CREATE INDEX ON system_platform(stale);
 
-CREATE INDEX ON system_platform(stale_warning_timestamp);
+CREATE INDEX ON system_platform(stale_warning_timestamp);

database/schema/upgrade_scripts/068-use_system_vulnerabilities_active.sql

@@ -298,4 +298,3 @@ $refresh_system_cached_counts$
   END;
 $refresh_system_cached_counts$
   LANGUAGE 'plpgsql';
-

database/schema/upgrade_scripts/077-use-account-id.sql

@@ -272,4 +272,4 @@ $delete_system$
     RETURN QUERY DELETE FROM system_platform WHERE id = system_id_in RETURNING inventory_id;
   END;
 $delete_system$
-  LANGUAGE 'plpgsql';
+  LANGUAGE 'plpgsql';

database/schema/upgrade_scripts/078-system-platform-uuid.sql

@@ -1,10 +1,10 @@
 -- Vanish non-uuid systems from DB
-CREATE OR REPLACE FUNCTION try_cast_uuid(uuid_in TEXT) 
+CREATE OR REPLACE FUNCTION try_cast_uuid(uuid_in TEXT)
   RETURNS uuid AS
 $$
   BEGIN
     RETURN uuid_in::UUID;
-  EXCEPTION WHEN invalid_text_representation THEN 
+  EXCEPTION WHEN invalid_text_representation THEN
     RETURN NULL;
   END;
 $$ LANGUAGE 'plpgsql';
@@ -30,10 +30,10 @@ BEGIN
       RETURN;
     END IF;
 
-    PERFORM * FROM system_platform 
+    PERFORM * FROM system_platform
     WHERE id BETWEEN low AND high FOR UPDATE;
 
-    UPDATE system_platform 
+    UPDATE system_platform
     SET inventory_id_new = try_cast_uuid(inventory_id)
     WHERE id BETWEEN low AND high;
 

database/schema/upgrade_scripts/085-content_version.sql

@@ -17,4 +17,4 @@ GRANT SELECT ON content_version TO ve_db_user_manager;
 GRANT SELECT ON content_version TO ve_db_user_vmaas_sync;
 GRANT SELECT ON content_version TO ve_db_user_metrics;
 GRANT SELECT ON content_version TO ve_db_user_taskomatic;
-GRANT SELECT ON content_version TO ve_db_user_advisor_listener;
+GRANT SELECT ON content_version TO ve_db_user_advisor_listener;

database/schema/upgrade_scripts/086-cve_name.sql

@@ -1 +1 @@
-ALTER TABLE cve_metadata ADD celebrity_name TEXT;
+ALTER TABLE cve_metadata ADD celebrity_name TEXT;

doc/schema.md

@@ -1,2 +1 @@
 ![engine](https://user-images.githubusercontent.com/6339153/120200721-eade9900-c224-11eb-85d7-0e4c4d765e43.jpg)
-

docker-compose-dbdocs.yml

@@ -13,7 +13,7 @@ services:
       - ./conf/database.env
     ports:
       - 5432:5432
-    
+
   schema_spy:
     container_name: schema-spy
     privileged: true

docker-compose.devel.yml

@@ -16,7 +16,7 @@ services:
       - label=disable
     working_dir: /git
     command: ["sleep", "infinity"]
-  
+
   ve_manager_admin:
     volumes:
       - .:/git

docker-compose.yml

@@ -36,7 +36,7 @@ services:
     depends_on:
       - ve_database
       - platform_mock
-  
+
   ve_manager_admin:
     command: /engine/entrypoint.sh manager-admin
     container_name: vulnerability-engine-manager-admin