Feat: add sqlfluff for linting

.github/workflows/ci.yml

@@ -5,9 +5,9 @@ defaults:
     working-directory: integration_tests
 on: [push]
 jobs:
-  build:
+  test_lint:
     runs-on: ubuntu-latest
-    name: Build
+    name: Test & Lint
     services:
       postgres:
         image: postgres:14.0-alpine
@@ -27,11 +27,13 @@ jobs:
           python-version: '3.9'
       - name: Setup python venv
         uses: syphar/restore-virtualenv@v1
-      - name: Install dbt
+      - name: Install dependencies
         run: pip install -r requirements.txt
       - name: Run dbt deps
         run: dbt deps
       - name: Run dbt seed
         run: dbt seed --profiles-dir ci_profiles
       - name: Run dbt models & tests
         run: dbt build -s dbt_graph_theory_integration_tests --profiles-dir ci_profiles
+      - name: Run sqlfluff
+        run: sqlfluff lint -v

README.md

@@ -179,6 +179,16 @@ quit -- to end the server connection
 
 All CI models are required to run and pass tests for a merge to be allowed.
 
+### Running sqlfluff
+
+This package is linted by sqlfluff. To run this linter, simply run:
+
+```
+sqlfluff lint -v
+```
+
+from the integration_tests folder. sqlfluff does not currently support macros, meaning that only models in the integration_tests folder are linted.
+
 ----
 ## Contents
 

integration_tests/.sqlfluff

@@ -0,0 +1,17 @@
+[sqlfluff]
+templater = dbt
+dialect = postgres
+
+[sqlfluff:templater:dbt]
+profiles_dir = ci_profiles
+profile = integration_tests
+target = ci
+
+[sqlfluff:rules]
+tab_space_size = 4
+max_line_length = 100
+indent_unit = space
+comma_style = trailing
+
+[sqlfluff:rules:L010]
+capitalisation_policy = lower

integration_tests/.sqlfluffignore

@@ -0,0 +1,4 @@
+dbt_modules/
+dbt_packages/
+ci_venv/
+target/

integration_tests/macros/cte_difference.sql

@@ -2,10 +2,10 @@
 (
     select '{{cte_1}}' as _data_location, {{ fields|join(',') }} from {{ cte_1 }}
     except
-    select '{{cte_1}}' as _data_location,{{ fields|join(',') }} from {{ cte_2 }}
+    select '{{cte_1}}' as _data_location, {{ fields|join(',') }} from {{ cte_2 }}
     union
-    select '{{cte_2}}' as _data_location,{{ fields|join(',') }} from {{ cte_2 }}
+    select '{{cte_2}}' as _data_location, {{ fields|join(',') }} from {{ cte_2 }}
     except
-    select '{{cte_2}}' as _data_location,{{ fields|join(',') }} from {{ cte_1 }}
+    select '{{cte_2}}' as _data_location, {{ fields|join(',') }} from {{ cte_1 }}
 ) as diff
 {% endmacro %}

integration_tests/models/test_largest_connected_subgraph_identifier/test_largest_conn_subgraph_1_subgraph.sql

@@ -0,0 +1,26 @@
+with computed as (
+    {{ dbt_graph_theory.largest_connected_largest_conn_subgraph(
+        input=ref('test_largest_conn_subgraph_1_subgraph_data')
+    ) }}
+),
+
+subgraph_members as (
+    select v.* from (
+        values
+        (1, 'A', 'B', '1', array['A', 'B', 'C', 'D', 'E']),
+        (2, 'B', 'C', '1', array['A', 'B', 'C', 'D', 'E']),
+        (3, 'C', 'D', '1', array['A', 'B', 'C', 'D', 'E']),
+        (4, 'B', 'D', '1', array['A', 'B', 'C', 'D', 'E']),
+        (5, 'A', 'C', '1', array['A', 'B', 'C', 'D', 'E']),
+        (6, 'B', 'E', '1', array['A', 'B', 'C', 'D', 'E']),
+        (7, 'E', 'D', '1', array['A', 'B', 'C', 'D', 'E']),
+        (8, 'A', null, '1', array['A', 'B', 'C', 'D', 'E']),
+        (9, 'E', null, '1', array['A', 'B', 'C', 'D', 'E'])
+    ) as v (id, vertex_1, vertex_2, subgraph_id, subgraph_members)
+)
+
+select * from {{ cte_difference(
+    'computed',
+    'subgraph_members',
+    fields=["id", "vertex_1", "vertex_2", "subgraph_id", "subgraph_members"]
+) }}

integration_tests/models/test_subgraph_identifier/test_largest_conn_subgraph_2_subgraph_no_edge.sql → integration_tests/models/test_largest_connected_subgraph_identifier/test_largest_conn_subgraph_2_subgraph_no_edge.sql

@@ -1,7 +1,7 @@
 with computed as (
     {{ dbt_graph_theory.largest_connected_largest_conn_subgraph(
         input=ref('test_largest_conn_subgraph_2_subgraph_no_edge_data')
-    )}}
+    ) }}
 ),
 
 -- recast because vertex_2 is all null in seed data, interpreted as int dtype
@@ -17,11 +17,15 @@ recast_computed as (
 ),
 
 subgraph_members as (
-    select * from (
+    select v.* from (
         values
-            (1, 'A', null, '1', array['A']),
-            (2, 'B', null, '2', array['B'])
-    ) as t (id, vertex_1, vertex_2, subgraph_id, subgraph_members)
+        (1, 'A', null, '1', array['A']),
+        (2, 'B', null, '2', array['B'])
+    ) as v (id, vertex_1, vertex_2, subgraph_id, subgraph_members)
 )
 
-select * from {{ cte_difference('recast_computed', 'subgraph_members', fields=["id", "vertex_1", "vertex_2", "subgraph_id", "subgraph_members"]) }}
+select * from {{ cte_difference(
+    'recast_computed',
+    'subgraph_members',
+    fields=["id", "vertex_1", "vertex_2", "subgraph_id", "subgraph_members"]
+) }}

integration_tests/models/test_largest_connected_subgraph_identifier/test_largest_conn_subgraph_3_subgraph_no_edge.sql

@@ -0,0 +1,20 @@
+with computed as (
+    {{ dbt_graph_theory.largest_connected_largest_conn_subgraph(
+        input=ref('test_largest_conn_subgraph_3_subgraph_no_edge_data')
+    ) }}
+),
+
+subgraph_members as (
+    select v.* from (
+        values
+        (1, 'A', null, '1', array['A']),
+        (2, 'B', null, '2', array['B']),
+        (3, null, 'C', '3', array['C'])
+    ) as v (id, vertex_1, vertex_2, subgraph_id, subgraph_members)
+)
+
+select * from {{ cte_difference(
+    'computed',
+    'subgraph_members',
+    fields=["id", "vertex_1", "vertex_2", "subgraph_id", "subgraph_members"]
+) }}

integration_tests/models/test_largest_connected_subgraph_identifier/test_largest_conn_subgraph_4_subgraph.sql

@@ -0,0 +1,24 @@
+with computed as (
+    {{ dbt_graph_theory.largest_connected_largest_conn_subgraph(
+        input=ref('test_largest_conn_subgraph_4_subgraph_data')
+    ) }}
+),
+
+subgraph_members as (
+    select v.* from (
+        values
+        (1, 'A', 'B', '1', array['A', 'B', 'C', 'D']),
+        (2, 'B', 'C', '1', array['A', 'B', 'C', 'D']),
+        (3, 'C', 'D', '1', array['A', 'B', 'C', 'D']),
+        (4, 'E', null, '2', array['E', 'F']),
+        (5, 'E', 'F', '2', array['E', 'F']),
+        (6, 'G', null, '3', array['G']),
+        (7, 'H', 'I', '4', array['H', 'I'])
+    ) as v (id, vertex_1, vertex_2, subgraph_id, subgraph_members)
+)
+
+select * from {{ cte_difference(
+    'computed',
+    'subgraph_members',
+    fields=["id", "vertex_1", "vertex_2", "subgraph_id", "subgraph_members"]
+) }}

integration_tests/models/test_largest_connected_subgraph_identifier/test_largest_conn_subgraph_graph_id.sql

@@ -0,0 +1,25 @@
+with computed as (
+    {{ dbt_graph_theory.largest_connected_largest_conn_subgraph(
+        input=ref('test_largest_conn_subgraph_graph_id_data'),
+        graph_id='graph_id'
+    ) }}
+),
+
+subgraph_members as (
+    select v.* from (
+        values
+        (1, 1, 'A', 'B', '1__1', array['A', 'B', 'C', 'D']),
+        (1, 2, 'B', 'C', '1__1', array['A', 'B', 'C', 'D']),
+        (1, 3, 'C', 'D', '1__1', array['A', 'B', 'C', 'D']),
+        (1, 4, 'E', null, '1__2', array['E']),
+        (2, 1, 'A', 'B', '2__1', array['A', 'B', 'C', 'D']),
+        (2, 2, 'B', 'C', '2__1', array['A', 'B', 'C', 'D']),
+        (2, 3, 'C', 'D', '2__1', array['A', 'B', 'C', 'D'])
+    ) as v (graph_id, id, vertex_1, vertex_2, subgraph_id, subgraph_members)
+)
+
+select * from {{ cte_difference(
+    'computed',
+    'subgraph_members',
+    fields=["graph_id", "id", "vertex_1", "vertex_2", "subgraph_id", "subgraph_members"]
+) }}

integration_tests/models/test_largest_connected_subgraph_identifier/test_largest_conn_subgraph_graph_id_3_subgraph.sql

@@ -0,0 +1,35 @@
+with computed as (
+    {{ dbt_graph_theory.largest_connected_largest_conn_subgraph(
+        input=ref('test_largest_conn_subgraph_graph_id_3_subgraph_data'),
+        graph_id='graph_id'
+    ) }}
+),
+
+subgraph_members as (
+    select v.* from (
+        values
+        (1, 1, 'A', 'B', '1__1', array['A', 'B', 'C', 'D']),
+        (1, 2, 'B', 'A', '1__1', array['A', 'B', 'C', 'D']),
+        (1, 3, 'A', 'C', '1__1', array['A', 'B', 'C', 'D']),
+        (1, 4, 'C', 'D', '1__1', array['A', 'B', 'C', 'D']),
+        (1, 5, 'D', 'A', '1__1', array['A', 'B', 'C', 'D']),
+        (1, 6, 'E', 'F', '1__2', array['E', 'F', 'G', 'H']),
+        (1, 7, 'F', 'G', '1__2', array['E', 'F', 'G', 'H']),
+        (1, 8, 'G', 'E', '1__2', array['E', 'F', 'G', 'H']),
+        (1, 9, 'F', 'H', '1__2', array['E', 'F', 'G', 'H']),
+        (1, 10, 'I', 'J', '1__3', array['I', 'J']),
+        (2, 1, 'A', 'B', '2__1', array['A', 'B', 'C']),
+        (2, 2, 'B', 'C', '2__1', array['A', 'B', 'C']),
+        (2, 3, 'D', null, '2__2', array['D']),
+        (2, 4, 'E', 'F', '2__3', array['E', 'F', 'G', 'H']),
+        (2, 5, 'F', 'G', '2__3', array['E', 'F', 'G', 'H']),
+        (2, 6, 'G', 'E', '2__3', array['E', 'F', 'G', 'H']),
+        (2, 7, 'F', 'H', '2__3', array['E', 'F', 'G', 'H'])
+    ) as v (graph_id, id, vertex_1, vertex_2, subgraph_id, subgraph_members)
+)
+
+select * from {{ cte_difference(
+    'computed',
+    'subgraph_members',
+    fields=["graph_id", "id", "vertex_1", "vertex_2", "subgraph_id", "subgraph_members"]
+) }}

integration_tests/models/test_largest_connected_subgraph_identifier/test_largest_conn_subgraph_no_data.sql

@@ -0,0 +1,19 @@
+with computed as (
+    {{ dbt_graph_theory.largest_connected_largest_conn_subgraph(
+        input=ref('test_largest_conn_subgraph_no_data_data')
+    ) }}
+),
+
+subgraph_members as (
+    select v.* from (
+        values
+        (null::integer, null::integer, null::integer, null::text, array[null])
+    ) as v (id, vertex_1, vertex_2, subgraph_id, subgraph_members)
+    where false
+)
+
+select * from {{ cte_difference(
+    'computed',
+    'subgraph_members',
+    fields=["id", "vertex_1", "vertex_2", "subgraph_id", "subgraph_members"]
+) }}

integration_tests/models/test_largest_connected_subgraph_identifier/test_largest_conn_subgraph_no_data_graph_id.sql

@@ -0,0 +1,20 @@
+with computed as (
+    {{ dbt_graph_theory.largest_connected_largest_conn_subgraph(
+        input=ref('test_largest_conn_subgraph_no_data_graph_id_data'),
+        graph_id='graph_id'
+    ) }}
+),
+
+subgraph_members as (
+    select v.* from (
+        values
+        (null::integer, null::integer, null::integer, null::integer, null::text, array[null])
+    ) as v (graph_id, id, vertex_1, vertex_2, subgraph_id, subgraph_members)
+    where false
+)
+
+select * from {{ cte_difference(
+    'computed',
+    'subgraph_members',
+    fields=["graph_id", "id", "vertex_1", "vertex_2", "subgraph_id", "subgraph_members"]
+) }}