feat: Policies (#1220)

* feat: Policies

* Add policies.go

* Ref to create

* Fix embed pattern

* Try to run a test

* psql: Use password

* Remove go:embed

* Two step policy run

* policy test: Generate tables without cq binary

* Run the correct file

* remove sql dir

Co-authored-by: Kemal Hadimli <disq@users.noreply.github.com>

.github/workflows/test_policy_sql.yml

@@ -0,0 +1,70 @@
+name: SQL Policy Validation Test
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+env: 
+  CGO_ENABLED: 0
+  CQ_NO_TELEMETRY: 1
+  PGPASSWORD: pass
+
+jobs:
+  SQLPolicyTest:
+    strategy:
+      matrix:
+        dbversion: [ "postgres:10" ]
+        go: [ "1.17" ]
+        platform: [ ubuntu-latest ] # can not run in macOS and widnowsOS
+    runs-on: ${{ matrix.platform }}
+    services:
+      postgres:
+        image: ${{ matrix.dbversion }}
+        env:
+          POSTGRES_PASSWORD: pass
+          POSTGRES_USER: postgres
+          POSTGRES_DB: postgres
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      - name: Check out code into the policy directory
+        uses: actions/checkout@v3
+
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ matrix.go }}
+
+      - uses: actions/cache@v3
+        with:
+          # In order:
+          # * Module download cache
+          # * Build cache (Linux)
+          # * Build cache (Mac)
+          # * Build cache (Windows)
+          path: |
+            ~/go/pkg/mod
+            ~/.cache/go-build
+            ~/Library/Caches/go-build
+            ~\AppData\Local\go-build
+          key: ${{ runner.os }}-go-${{ matrix.go }}-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-${{ matrix.go }}-
+
+      - name: Prepare for test - Create tables
+        run: |
+          go run ./test/gen-tables.go | psql -h localhost -p 5432 -U postgres -d postgres -w
+
+      - name: Run all policies
+        run: |
+          cd policies && psql -h localhost -p 5432 -U postgres -d postgres -w -f ./policy.sql

policies/README.md

@@ -0,0 +1,32 @@
+# CloudQuery Policies
+CloudQuery SQL Policies for AWS
+
+## Policies and Compliance Frameworks Available
+
+- [AWS CIS V1.2.0](./cis_v1.2.0/policy.sql)
+- [AWS PCI DSS v3.2.1](./pci_dss_v3.2.1/policy.sql)
+- [AWS Foundational Security Best Practices](./foundational_security/policy.sql)
+- [AWS Public Egress](./public_egress/policy.sql)
+- [AWS Publicly Available](./publicly_available/policy.sql)
+
+## Running
+
+You can execute policies with `psql`. For example:
+
+```bash
+# Execute the whole CIS Policy
+psql -U postgres -f  ./cis_v1.2.0/policy.sql
+```
+
+This will create all the results in `aws_policy_results` table which you can query directly, connect to any BI system (Grafana, Preset, AWS QuickSight, PowerBI, ...).
+
+You can also output it into CSV or HTML with the following built-in psql commands:
+
+```
+# default tabular output
+psql -U postgres -c "select * from aws_policy_results"
+# CSV output
+psql -U postgres -c "select * from aws_policy_results" --csv
+# HTML output
+psql -U postgres -c "select * from aws_policy_results" --html
+```

policies/cis_v1.2.0/policy.sql

@@ -0,0 +1,7 @@
+\set framework 'cis_v1.2.0'
+\set execution_time ''''`date '+%Y-%m-%d %H:%M:%S'`''''::timestamp
+\i create_aws_policy_results.sql
+\i cis_v1.2.0/section_1.sql
+\i cis_v1.2.0/section_2.sql
+\i cis_v1.2.0/section_3.sql
+\i cis_v1.2.0/section_4.sql

policies/cis_v1.2.0/section_1.sql

@@ -0,0 +1,46 @@
+\echo  "Executing CIS V1.2.0 Section 1"
+\set check_id "1.1"
+\echo "Executing check 1.1"
+\i queries/iam/avoid_root_usage.sql
+\set check_id "1.2"
+\echo "Executing check 1.2"
+\i queries/iam/mfa_enabled_for_console_access.sql
+\set check_id "1.3"
+\echo "Executing check 1.3"
+\i queries/iam/unused_creds_disabled.sql
+\set check_id "1.4"
+\echo "Executing check 1.4"
+\i queries/iam/old_access_keys.sql
+\set check_id "1.5"
+\echo "Executing check 1.5"
+\i queries/iam/password_policy_min_uppercase.sql
+\set check_id "1.6"
+\echo "Executing check 1.6"
+\i queries/iam/password_policy_min_lowercase.sql
+\set check_id "1.7"
+\echo "Executing check 1.7"
+\i queries/iam/password_policy_min_one_symbol.sql
+\set check_id "1.8"
+\echo "Executing check 1.8"
+\i queries/iam/password_policy_min_number.sql
+\set check_id "1.9"
+\echo "Executing check 1.9"
+\i queries/iam/password_policy_min_length.sql
+\set check_id "1.10"
+\echo "Executing check 1.10"
+\i queries/iam/password_policy_prevent_reuse.sql
+\set check_id "1.11"
+\echo "Executing check 1.11"
+\i queries/iam/password_policy_expire_old_passwords.sql
+\set check_id "1.12"
+\echo "Executing check 1.12"
+\i queries/iam/root_user_no_access_keys.sql
+\set check_id "1.13"
+\echo "Executing check 1.13"
+\i queries/iam/mfa_enabled_for_root.sql
+\set check_id "1.14"
+\echo "Executing check 1.14"
+\i queries/iam/hardware_mfa_enabled_for_root.sql
+\set check_id "1.16"
+\echo "Executing check 1.16"
+\i queries/iam/policies_attached_to_groups_roles.sql

policies/cis_v1.2.0/section_2.sql

@@ -0,0 +1,22 @@
+\echo  "Executing CIS V1.2.0 Section 2"
+\set check_id "2.1"
+\echo "Executing check 2.1"
+\i queries/cloudtrail/enabled_in_all_regions.sql
+\set check_id "2.2"
+\echo "Executing check 2.2"
+\i queries/cloudtrail/log_file_validation_enabled.sql
+\set check_id "2.4"
+\echo "Executing check 2.4"
+\i queries/cloudtrail/integrated_with_cloudwatch_logs.sql
+\set check_id "2.6"
+\echo "Executing check 2.6"
+\i queries/cloudtrail/bucket_access_logging.sql
+\set check_id "2.7"
+\echo "Executing check 2.7"
+\i queries/cloudtrail/logs_encrypted.sql
+\set check_id "2.8"
+\echo "Executing check 2.8"
+\i queries/kms/rotation_enabled_for_customer_key.sql
+\set check_id "2.9"
+\echo "Executing check 2.9"
+\i queries/ec2/flow_logs_enabled_in_all_vpcs.sql

policies/cis_v1.2.0/section_3.sql

@@ -0,0 +1,42 @@
+\echo  "Executing CIS V1.2.0 Section 3"
+\echo "Creating view_aws_log_metric_filter_and_alarm"
+\i views/log_metric_filter_and_alarm.sql
+\set check_id "3.1"
+\echo "Executing check 3.1"
+\i queries/cloudwatch/alarm_unauthorized_api.sql
+\set check_id "3.3"
+\echo "Executing check 3.3"
+\i queries/cloudwatch/alarm_root_account.sql
+\set check_id "3.4"
+\echo "Executing check 3.4"
+\i queries/cloudwatch/alarm_iam_policy_change.sql
+\set check_id "3.5"
+\echo "Executing check 3.5"
+\i queries/cloudwatch/alarm_cloudtrail_config_changes.sql
+\set check_id "3.6"
+\echo "Executing check 3.6"
+\i queries/cloudwatch/alarm_console_auth_failure.sql
+\set check_id "3.7"
+\echo "Executing check 3.7"
+\i queries/cloudwatch/alarm_delete_customer_cmk.sql
+\set check_id "3.8"
+\echo "Executing check 3.8"
+\i queries/cloudwatch/alarm_s3_bucket_policy_change.sql
+\set check_id "3.9"
+\echo "Executing check 3.9"
+\i queries/cloudwatch/alarm_aws_config_changes.sql
+\set check_id "3.10"
+\echo "Executing check 3.10"
+\i queries/cloudwatch/alarm_security_group_changes.sql
+\set check_id "3.11"
+\echo "Executing check 3.11"
+\i queries/cloudwatch/alarm_nacl_changes.sql
+\set check_id "3.12"
+\echo "Executing check 3.12"
+\i queries/cloudwatch/alarm_network_gateways.sql
+\set check_id "3.13"
+\echo "Executing check 3.13"
+\i queries/cloudwatch/alarm_route_table_changes.sql
+\set check_id "3.14"
+\echo "Executing check 3.14"
+\i queries/cloudwatch/alarm_vpc_changes.sql

policies/cis_v1.2.0/section_4.sql

@@ -0,0 +1,12 @@
+\echo  "Executing CIS V1.2.0 Section 4"
+\echo "Creating view_aws_security_group_ingress_rules"
+\i views/security_group_ingress_rules.sql
+\set check_id "4.1"
+\echo "Executing check 4.1"
+\i queries/ec2/no_broad_public_ingress_on_port_22.sql
+\set check_id "4.2"
+\echo "Executing check 4.2"
+\i queries/ec2/no_broad_public_ingress_on_port_3389.sql
+\set check_id "4.3"
+\echo "Executing check 4.3"
+\i queries/ec2/default_sg_no_access.sql

policies/create_aws_policy_results.sql

@@ -0,0 +1,9 @@
+create table if not exists aws_policy_results (
+    execution_time timestamp,
+    framework varchar(255),
+    check_id varchar(255),
+    title text,
+    account_id varchar(1024),
+    resource_id varchar(1024),
+    status varchar(16)
+)

policies/foundational_security/acm.sql

@@ -0,0 +1,3 @@
+\set check_id 'ACM.1'
+\echo "Executing check ACM.1"
+\i queries/acm/certificates_should_be_renewed.sql

policies/foundational_security/apigateway.sql

@@ -0,0 +1,22 @@
+\echo "Creating view_aws_apigateway_method_settings"
+\i views/api_gateway_method_settings.sql
+
+\set check_id 'ApiGateway.1'
+\echo "Executing check ApiGateway.1"
+\i queries/apigateway/api_gw_execution_logging_enabled.sql
+
+\set check_id 'ApiGateway.2'
+\echo "Executing check ApiGateway.2"
+\i queries/apigateway/api_gw_ssl_enabled.sql
+
+\set check_id 'ApiGateway.3'
+\echo "Executing check ApiGateway.3"
+\i queries/apigateway/api_gw_xray_enabled.sql
+
+\set check_id 'ApiGateway.4'
+\echo "Executing check ApiGateway.4"
+\i queries/apigateway/api_gw_associated_with_waf.sql
+
+\set check_id 'ApiGateway.5'
+\echo "Executing check ApiGateway.5"
+\i queries/apigateway/api_gw_cache_encrypted.sql

policies/foundational_security/autoscaling.sql

@@ -0,0 +1,3 @@
+\set check_id 'AutoScaling.1'
+\echo "Executing check AutoScaling.1"
+\i queries/autoscaling/autoscaling_groups_elb_check.sql

policies/foundational_security/awsconfig.sql

@@ -0,0 +1,3 @@
+\set check_id 'Config.1'
+\echo "Executing check Config.1"
+\i queries/config/enabled_all_regions.sql

policies/foundational_security/cloudfront.sql

@@ -0,0 +1,23 @@
+\set check_id 'Cloudfront.1'
+\echo "Executing check Cloudfront.1"
+\i queries/cloudfront/default_root_object_configured.sql
+
+\set check_id 'Cloudfront.2'
+\echo "Executing check Cloudfront.2"
+\i queries/cloudfront/origin_access_identity_enabled.sql
+
+\set check_id 'Cloudfront.3'
+\echo "Executing check Cloudfront.3"
+\i queries/cloudfront/viewer_policy_https.sql
+
+\set check_id 'Cloudfront.4'
+\echo "Executing check Cloudfront.4"
+\i queries/cloudfront/origin_failover_enabled.sql
+
+\set check_id 'Cloudfront.5'
+\echo "Executing check Cloudfront.5"
+\i queries/cloudfront/access_logs_enabled.sql
+
+\set check_id 'Cloudfront.6'
+\echo "Executing check Cloudfront.6"
+\i queries/cloudfront/associated_with_waf.sql

policies/foundational_security/cloudtrail.sql

@@ -0,0 +1,15 @@
+\set check_id 'CloudTrail.1'
+\echo "Executing check CloudTrail.1"
+\i queries/cloudtrail/enabled_in_all_regions.sql
+
+\set check_id 'CloudTrail.2'
+\echo "Executing check CloudTrail.2"
+\i queries/cloudtrail/logs_encrypted.sql
+
+\set check_id 'CloudTrail.4'
+\echo "Executing check CloudTrail.4"
+\i queries/cloudtrail/log_file_validation_enabled.sql
+
+\set check_id 'CloudTrail.5'
+\echo "Executing check CloudTrail.5"
+\i queries/cloudtrail/integrated_with_cloudwatch_logs.sql

policies/foundational_security/codebuild.sql

@@ -0,0 +1,7 @@
+\set check_id 'CodeBuild.1'
+\echo "Executing check CodeBuild.1"
+\i queries/codebuild/check_oauth_usage_for_sources.sql
+
+\set check_id 'CodeBuild.2'
+\echo "Executing check CodeBuild.2"
+\i queries/codebuild/check_environment_variables.sql

policies/foundational_security/dms.sql

@@ -0,0 +1,3 @@
+\set check_id 'DMS.1'
+\echo "Executing check DMS.1"
+\i queries/dms/replication_not_public.sql

policies/foundational_security/dynamodb.sql

@@ -0,0 +1,11 @@
+\set check_id 'DynamoDB.1'
+\echo "Executing check DynamoDB.1"
+\i queries/dynamodb/autoscale_or_ondemand.sql
+
+\set check_id 'DynamoDB.2'
+\echo "Executing check DynamoDB.2"
+\i queries/dynamodb/point_in_time_recovery.sql
+
+\set check_id 'DynamoDB.3'
+\echo "Executing check DynamoDB.3"
+\i queries/dynamodb/dax_encrypted_at_rest.sql