Merge pull request #2160 from fishtown-analytics/feature/expanded-sch…

…ema-search-paths Allow search in macro and analysis paths (#2155)
dbt-labs · Feb 26, 2020 · 847bc09 · 847bc09
2 parents e486872 + 529b053
commit 847bc09
Show file tree

Hide file tree

Showing 8 changed files with 136 additions and 32 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,6 +8,7 @@
 - Add a "docs" field to models, with a "show" subfield ([#1671](https://github.com/fishtown-analytics/dbt/issues/1671), [#2107](https://github.com/fishtown-analytics/dbt/pull/2107))
 - Add a dbt-{dbt_version} user agent field to the bigquery connector ([#2121](https://github.com/fishtown-analytics/dbt/issues/2121), [#2146](https://github.com/fishtown-analytics/dbt/pull/2146))
 - Add support for `generate_database_name` macro ([#1695](https://github.com/fishtown-analytics/dbt/issues/1695), [#2143](https://github.com/fishtown-analytics/dbt/pull/2143))
+- Expand the search path for schema.yml (and by extension, the default docs path) to include macro-paths and analysis-paths (in addition to source-paths, data-paths, and snapshot-paths) ([#2155](https://github.com/fishtown-analytics/dbt/issues/2155), [#2160](https://github.com/fishtown-analytics/dbt/pull/2160))
 
 ### Fixes
 - Fix issue where dbt did not give an error in the presence of duplicate doc names ([#2054](https://github.com/fishtown-analytics/dbt/issues/2054), [#2080](https://github.com/fishtown-analytics/dbt/pull/2080))

diff --git a/core/dbt/config/project.py b/core/dbt/config/project.py
@@ -160,9 +160,14 @@ def _parse_versions(versions: Union[List[str], str]) -> List[VersionSpecifier]:
 
 
 def _all_source_paths(
-    source_paths: List[str], data_paths: List[str], snapshot_paths: List[str]
+    source_paths: List[str],
+    data_paths: List[str],
+    snapshot_paths: List[str],
+    analysis_paths: List[str],
+    macro_paths: List[str],
 ) -> List[str]:
-    return list(chain(source_paths, data_paths, snapshot_paths))
+    return list(chain(source_paths, data_paths, snapshot_paths, analysis_paths,
+                      macro_paths))
 
 
 T = TypeVar('T')
@@ -244,7 +249,8 @@ class Project:
     @property
     def all_source_paths(self) -> List[str]:
         return _all_source_paths(
-            self.source_paths, self.data_paths, self.snapshot_paths
+            self.source_paths, self.data_paths, self.snapshot_paths,
+            self.analysis_paths, self.macro_paths
         )
 
     @staticmethod
@@ -323,7 +329,8 @@ def from_project_config(
         snapshot_paths: List[str] = value_or(cfg.snapshot_paths, ['snapshots'])
 
         all_source_paths: List[str] = _all_source_paths(
-            source_paths, data_paths, snapshot_paths
+            source_paths, data_paths, snapshot_paths, analysis_paths,
+            macro_paths
         )
 
         docs_paths: List[str] = value_or(cfg.docs_paths, all_source_paths)

diff --git a/test/integration/029_docs_generate_tests/macros/macro.md b/test/integration/029_docs_generate_tests/macros/macro.md
@@ -0,0 +1,8 @@
+
+{% docs macro_info %}
+My custom test that I wrote that does nothing
+{% enddocs %}
+
+{% docs macro_arg_info %}
+The model for my custom test
+{% enddocs %}
diff --git a/test/integration/029_docs_generate_tests/macros/schema.yml b/test/integration/029_docs_generate_tests/macros/schema.yml
@@ -0,0 +1,10 @@
+version: 2
+macros:
+  - name: test_nothing
+    description: "{{ doc('macro_info') }}"
+    meta:
+      some_key: 100
+    arguments:
+      - name: model
+        type: Relation
+        description: "{{ doc('macro_arg_info') }}"
diff --git a/test/integration/029_docs_generate_tests/ref_models/docs.md b/test/integration/029_docs_generate_tests/ref_models/docs.md
@@ -25,11 +25,3 @@ My table
 {% docs column_info %}
 An ID field
 {% enddocs %}
-
-{% docs macro_info %}
-My custom test that I wrote that does nothing
-{% enddocs %}
-
-{% docs macro_arg_info %}
-The model for my custom test
-{% enddocs %}
diff --git a/test/integration/029_docs_generate_tests/ref_models/schema.yml b/test/integration/029_docs_generate_tests/ref_models/schema.yml
@@ -29,13 +29,3 @@ sources:
         columns:
           - name: id
             description: "{{ doc('column_info') }}"
-
-macros:
-  - name: test_nothing
-    description: "{{ doc('macro_info') }}"
-    meta:
-      some_key: 100
-    arguments:
-      - name: model
-        type: Relation
-        description: "{{ doc('macro_arg_info') }}"
diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py
@@ -1219,7 +1219,9 @@ def expected_seeded_manifest(self, model_database=None):
                 'test.test.unique_model_id': [],
             },
             'docs': {
-                'dbt.__overview__': ANY
+                'dbt.__overview__': ANY,
+                'test.macro_info': ANY,
+                'test.macro_arg_info': ANY,
             },
             'metadata': {
                 'project_id': '098f6bcd4621d373cade4e832627b4f6',
@@ -1239,6 +1241,29 @@ def expected_seeded_manifest(self, model_database=None):
                     'patches': [],
                     'macro_patches': [],
                 },
+                normalize('macros/macro.md'): {
+                    'checksum': self._checksum_file('macros/macro.md'),
+                    'docs': [
+                        'test.macro_info',
+                        'test.macro_arg_info',
+                    ],
+                    'macros': [],
+                    'nodes': [],
+                    'patches': [],
+                    'path': self._path_to('macros', 'macro.md'),
+                    'sources': [],
+                    'macro_patches': [],
+                },
+                normalize('macros/schema.yml'): {
+                    'path': self._path_to('macros', 'schema.yml'),
+                    'checksum': self._checksum_file('macros/schema.yml'),
+                    'docs': [],
+                    'macros': [],
+                    'nodes': [],
+                    'patches': [],
+                    'sources': [],
+                    'macro_patches': [['test', 'test_nothing']],
+                },
                 normalize('models/model.sql'): {
                     'path': self._path_to('models', 'model.sql'),
                     'checksum': self._checksum_file('models/model.sql'),
@@ -1704,19 +1729,19 @@ def expected_postgres_references_manifest(self, model_database=None):
                     'block_contents': 'My custom test that I wrote that does nothing',
                     'file_contents': macro_info,
                     'name': 'macro_info',
-                    'original_file_path': docs_path,
+                    'original_file_path': self.dir('macros/macro.md'),
                     'package_name': 'test',
-                    'path': 'docs.md',
+                    'path': 'macro.md',
                     'root_path': self.test_root_dir,
                     'unique_id': 'test.macro_info',
                 },
                 'test.macro_arg_info': {
                     'block_contents': 'The model for my custom test',
                     'file_contents': macro_arg_info,
                     'name': 'macro_arg_info',
-                    'original_file_path': docs_path,
+                    'original_file_path': self.dir('macros/macro.md'),
                     'package_name': 'test',
-                    'path': 'docs.md',
+                    'path': 'macro.md',
                     'root_path': self.test_root_dir,
                     'unique_id': 'test.macro_arg_info',
                 },
@@ -1806,13 +1831,24 @@ def expected_postgres_references_manifest(self, model_database=None):
                         'test.source_info',
                         'test.table_info',
                         'test.column_info',
+                    ],
+                    'macros': [],
+                    'nodes': [],
+                    'patches': [],
+                    'path': self._path_to('ref_models', 'docs.md'),
+                    'sources': [],
+                    'macro_patches': [],
+                },
+                normalize('macros/macro.md'): {
+                    'checksum': self._checksum_file('macros/macro.md'),
+                    'docs': [
                         'test.macro_info',
                         'test.macro_arg_info',
                     ],
                     'macros': [],
                     'nodes': [],
                     'patches': [],
-                    'path': self._path_to('ref_models', 'docs.md'),
+                    'path': self._path_to('macros', 'macro.md'),
                     'sources': [],
                     'macro_patches': [],
                 },
@@ -1824,6 +1860,16 @@ def expected_postgres_references_manifest(self, model_database=None):
                     'patches': ['ephemeral_summary', 'view_summary'],
                     'path': self._path_to('ref_models', 'schema.yml'),
                     'sources': ['source.test.my_source.my_table'],
+                    'macro_patches': [],
+                },
+                normalize('macros/schema.yml'): {
+                    'path': self._path_to('macros', 'schema.yml'),
+                    'checksum': self._checksum_file('macros/schema.yml'),
+                    'docs': [],
+                    'macros': [],
+                    'nodes': [],
+                    'patches': [],
+                    'sources': [],
                     'macro_patches': [['test', 'test_nothing']],
                 },
                 normalize('seed/schema.yml'): {
@@ -1850,7 +1896,7 @@ def expected_postgres_references_manifest(self, model_database=None):
                     'meta': {
                         'some_key': 100,
                     },
-                    'patch_path': self.dir('ref_models/schema.yml'),
+                    'patch_path': self.dir('macros/schema.yml'),
                     'resource_type': 'macro',
                     'unique_id': 'macro.test.test_nothing',
                     'tags': [],
@@ -2256,6 +2302,8 @@ def expected_bigquery_complex_manifest(self):
             },
             'docs': {
                 'dbt.__overview__': ANY,
+                'test.macro_info': ANY,
+                'test.macro_arg_info': ANY,
             },
             'metadata': {
                 'project_id': '098f6bcd4621d373cade4e832627b4f6',
@@ -2338,6 +2386,29 @@ def expected_bigquery_complex_manifest(self):
                     'sources': [],
                     'macro_patches': [],
                 },
+                normalize('macros/macro.md'): {
+                    'checksum': self._checksum_file('macros/macro.md'),
+                    'docs': [
+                        'test.macro_info',
+                        'test.macro_arg_info',
+                    ],
+                    'macros': [],
+                    'nodes': [],
+                    'patches': [],
+                    'path': self._path_to('macros', 'macro.md'),
+                    'sources': [],
+                    'macro_patches': [],
+                },
+                normalize('macros/schema.yml'): {
+                    'path': self._path_to('macros', 'schema.yml'),
+                    'checksum': self._checksum_file('macros/schema.yml'),
+                    'docs': [],
+                    'macros': [],
+                    'nodes': [],
+                    'patches': [],
+                    'sources': [],
+                    'macro_patches': [['test', 'test_nothing']],
+                },
                 normalize('seed/schema.yml'): {
                     'path': self._path_to('seed', 'schema.yml'),
                     'checksum': self._checksum_file('seed/schema.yml'),
@@ -2557,6 +2628,8 @@ def expected_redshift_incremental_view_manifest(self):
             },
             'docs': {
                 'dbt.__overview__': ANY,
+                'test.macro_info': ANY,
+                'test.macro_arg_info': ANY,
             },
             'metadata': {
                 'project_id': '098f6bcd4621d373cade4e832627b4f6',
@@ -2576,6 +2649,29 @@ def expected_redshift_incremental_view_manifest(self):
                     'sources': [],
                     'macro_patches': [],
                 },
+                normalize('macros/macro.md'): {
+                    'checksum': self._checksum_file('macros/macro.md'),
+                    'docs': [
+                        'test.macro_info',
+                        'test.macro_arg_info',
+                    ],
+                    'macros': [],
+                    'nodes': [],
+                    'patches': [],
+                    'path': self._path_to('macros', 'macro.md'),
+                    'sources': [],
+                    'macro_patches': [],
+                },
+                normalize('macros/schema.yml'): {
+                    'path': self._path_to('macros', 'schema.yml'),
+                    'checksum': self._checksum_file('macros/schema.yml'),
+                    'docs': [],
+                    'macros': [],
+                    'nodes': [],
+                    'patches': [],
+                    'sources': [],
+                    'macro_patches': [['test', 'test_nothing']],
+                },
                 normalize('rs_models/model.sql'): {
                     'checksum': self._checksum_file('rs_models/model.sql'),
                     'path': self._path_to('rs_models', 'model.sql'),

diff --git a/test/unit/test_config.py b/test/unit/test_config.py
@@ -551,7 +551,7 @@ def test_defaults(self):
         self.assertEqual(project.data_paths, ['data'])
         self.assertEqual(project.test_paths, ['test'])
         self.assertEqual(project.analysis_paths, [])
-        self.assertEqual(project.docs_paths, ['models', 'data', 'snapshots'])
+        self.assertEqual(project.docs_paths, ['models', 'data', 'snapshots', 'macros'])
         self.assertEqual(project.target_path, 'target')
         self.assertEqual(project.clean_targets, ['target'])
         self.assertEqual(project.log_path, 'logs')
@@ -591,7 +591,7 @@ def test_implicit_overrides(self):
         project = dbt.config.Project.from_project_config(
             self.default_project_data, None
         )
-        self.assertEqual(project.docs_paths, ['other-models', 'data', 'snapshots'])
+        self.assertEqual(project.docs_paths, ['other-models', 'data', 'snapshots', 'macros'])
         self.assertEqual(project.clean_targets, ['other-target'])
 
     def test_hashed_name(self):
@@ -1107,7 +1107,7 @@ def test_from_args(self):
         self.assertEqual(config.data_paths, ['data'])
         self.assertEqual(config.test_paths, ['test'])
         self.assertEqual(config.analysis_paths, [])
-        self.assertEqual(config.docs_paths, ['models', 'data', 'snapshots'])
+        self.assertEqual(config.docs_paths, ['models', 'data', 'snapshots', 'macros'])
         self.assertEqual(config.target_path, 'target')
         self.assertEqual(config.clean_targets, ['target'])
         self.assertEqual(config.log_path, 'logs')