Skip to content

Commit

Permalink
Fix #9119: Get sources working again in dbt docs generate
Browse files Browse the repository at this point in the history
  • Loading branch information
aranke committed Nov 28, 2023
1 parent 32fde75 commit 5c16614
Show file tree
Hide file tree
Showing 3 changed files with 66 additions and 1 deletion.
6 changes: 6 additions & 0 deletions .changes/unreleased/Fixes-20231128-155225.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Fixes
body: ' Get sources working again in dbt docs generate'
time: 2023-11-28T15:52:25.738256Z
custom:
Author: aranke
Issue: "9119"
6 changes: 6 additions & 0 deletions core/dbt/task/generate.py
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,12 @@ def run(self) -> CatalogArtifact:
if self.job_queue is not None:
selected_node_ids = self.job_queue.get_selected_nodes()
selected_nodes = self._get_nodes_from_ids(self.manifest, selected_node_ids)

source_ids = self._get_nodes_from_ids(
self.manifest, self.manifest.sources.keys()
)
selected_nodes.extend(source_ids)

This comment has been minimized.

Copy link
@peterallenwebb

peterallenwebb Nov 28, 2023

Contributor

Saw your comment in the issue. I agree that it seems strange and surprising that the job_queue.get_selected_nodes() function doesn't include sources, and that we should investigate further.


relations = {
adapter.Relation.create_from(adapter.config, node_id)
for node_id in selected_nodes
Expand Down
55 changes: 54 additions & 1 deletion tests/functional/docs/test_generate.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,44 @@

from dbt.tests.util import run_dbt, get_manifest

sample_seed = """sample_num,sample_bool
1,true
2,false
3,true
"""

second_seed = """sample_num,sample_bool
4,true
5,false
6,true
"""

sample_config = """
sources:
- name: my_seed
schema: "{{ target.schema }}"
tables:
- name: sample_seed
- name: second_seed
- name: fake_seed
"""


class TestGenerate:
@pytest.fixture(scope="class")
def models(self):
return {"my_model.sql": "select 1 as fun", "alt_model.sql": "select 1 as notfun"}
return {
"my_model.sql": "select 1 as fun",
"alt_model.sql": "select 1 as notfun",
"sample_config.yml": sample_config,
}

@pytest.fixture(scope="class")
def seeds(self):
return {
"sample_seed.csv": sample_seed,
"second_seed.csv": sample_seed,
}

def test_manifest_not_compiled(self, project):
run_dbt(["docs", "generate", "--no-compile"])
Expand All @@ -33,3 +66,23 @@ def test_select_limits_no_match(self, project):
run_dbt(["run"])
catalog = run_dbt(["docs", "generate", "--select", "my_missing_model"])
assert len(catalog.nodes) == 0

def test_catalog_with_sources(self, project):

This comment has been minimized.

Copy link
@peterallenwebb

peterallenwebb Nov 28, 2023

Contributor

Don't recall if you were there, but at the last brown bag we decided to do one test per class is almost all cases, in order to avoid all potential concurrency/ordering issues.

run_dbt(["build"])
catalog = run_dbt(["docs", "generate"])

# 2 seeds + 2 models
assert len(catalog.nodes) == 4
# 2 sources (only ones that exist)
assert len(catalog.sources) == 2

def test_select_source(self, project):
run_dbt(["build"])
catalog = run_dbt(["docs", "generate", "--select", "source:test.my_seed.sample_seed"])

# 2 seeds
# TODO: Filtering doesn't work for seeds
assert len(catalog.nodes) == 2
# 2 sources
# TODO: Filtering doesn't work for sources
assert len(catalog.sources) == 2

This comment has been minimized.

Copy link
@peterallenwebb

peterallenwebb Nov 28, 2023

Contributor

It seems odd to assert behavior we regard as incorrect. Do you think it's valuable enough to leave in? If so, can you add some notes about what the correct values would be?

0 comments on commit 5c16614

Please sign in to comment.