Merge branch 'dev/octavius-catto' into fix/return_proper_msg_for_empt…

…y_profile

CHANGELOG.md

@@ -15,13 +15,14 @@
 - When a warn exception is not in a jinja do block, return an empty string instead of None ([#2222](https://github.com/fishtown-analytics/dbt/issues/2222), [#2259](https://github.com/fishtown-analytics/dbt/pull/2259))
 - Add dbt plugin versions to --version([#2272](https://github.com/fishtown-analytics/dbt/issues/2272), [#2279](https://github.com/fishtown-analytics/dbt/pull/2279))
 - Return error message when profile is empty in profiles.yml. ([#2292](https://github.com/fishtown-analytics/dbt/issues/2292), [#2297](https://github.com/fishtown-analytics/dbt/pull/2297))
+- Made file names lookups case-insensitve (.sql, .SQL, .yml, .YML) and if .yaml files are found, raise a warning indicating dbt will parse these files in future releases. ([#1681](https://github.com/fishtown-analytics/dbt/issues/1681), [#2263](https://github.com/fishtown-analytics/dbt/pull/2263))
 
 Contributors:
  - [@raalsky](https://github.com/Raalsky) ([#2224](https://github.com/fishtown-analytics/dbt/pull/2224), [#2228](https://github.com/fishtown-analytics/dbt/pull/2228))
  - [@ilkinulas](https://github.com/ilkinulas) [#2199](https://github.com/fishtown-analytics/dbt/pull/2199)
  - [@kyleabeauchamp](https://github.com/kyleabeauchamp) [#2262](https://github.com/fishtown-analytics/dbt/pull/2262)
  - [@jeremyyeo](https://github.com/jeremyyeo) [#2259](https://github.com/fishtown-analytics/dbt/pull/2259)
- - [@sumanau7](https://github.com/sumanau7) ([#2279](https://github.com/fishtown-analytics/dbt/pull/2279), [#2297](https://github.com/fishtown-analytics/dbt/pull/2297))
+ - [@sumanau7](https://github.com/sumanau7) ([#2279](https://github.com/fishtown-analytics/dbt/pull/2279), [#2297](https://github.com/fishtown-analytics/dbt/pull/2297), [#2263](https://github.com/fishtown-analytics/dbt/pull/2263))
 
 ## dbt 0.16.0 (March 23, 2020)
 

core/dbt/clients/system.py

@@ -3,6 +3,7 @@
 import json
 import os
 import os.path
+import re
 import shutil
 import subprocess
 import sys
@@ -40,6 +41,8 @@ def find_matching(
  """
  matching = []
  root_path = os.path.normpath(root_path)
+ regex = fnmatch.translate(file_pattern)
+ reobj = re.compile(regex, re.IGNORECASE)
 
  for relative_path_to_search in relative_paths_to_search:
  absolute_path_to_search = os.path.join(
@@ -52,8 +55,7 @@ def find_matching(
  relative_path = os.path.relpath(
  absolute_path, absolute_path_to_search
  )
-
- if fnmatch.fnmatch(local_file, file_pattern):
+ if reobj.match(local_file):
  matching.append({
  'searched_path': relative_path_to_search,
  'absolute_path': absolute_path,

core/dbt/parser/schemas.py

@@ -29,7 +29,8 @@
 )
 from dbt.exceptions import (
  validator_error_message, JSONValidationException,
- raise_invalid_schema_yml_version, ValidationException, CompilationException
+ raise_invalid_schema_yml_version, ValidationException,
+ CompilationException, warn_or_error
 )
 from dbt.node_types import NodeType
 from dbt.parser.base import SimpleParser
@@ -130,6 +131,20 @@ def resource_type(self) -> NodeType:
  return NodeType.Test
 
  def get_paths(self):
+ # TODO: In order to support this, make FilesystemSearcher accept a list
+ # of file patterns. eg: ['.yml', '.yaml']
+ yaml_files = list(FilesystemSearcher(
+ self.project, self.project.all_source_paths, '.yaml'
+ ))
+ if yaml_files:
+ warn_or_error(
+ 'A future version of dbt will parse files with both'
+ ' .yml and .yaml file extensions. dbt found'
+ f' {len(yaml_files)} files with .yaml extensions in'
+ ' your dbt project. To avoid errors when upgrading'
+ ' to a future release, either remove these files from'
+ ' your dbt project, or change their extensions.'
+ )
  return FilesystemSearcher(
  self.project, self.project.all_source_paths, '.yml'
  )

test/integration/008_schema_tests_test/case-sensitive-models/lowercase.sql

@@ -0,0 +1 @@
+select 1 as id

test/integration/008_schema_tests_test/case-sensitive-models/schema.yml

@@ -0,0 +1,15 @@
+version: 2
+
+models:
+ - name: lowercase
+ columns:
+ - name: id
+ quote: true
+ tests:
+ - unique
+ - name: uppercase
+ columns:
+ - name: id
+ quote: true
+ tests:
+ - unique

test/integration/008_schema_tests_test/case-sensitive-models/uppercase.SQL

@@ -0,0 +1 @@
+select 1 as id

test/integration/008_schema_tests_test/test_schema_v2_tests.py

@@ -361,3 +361,43 @@ def test_postgres_argument_rendering(self):
  results = self.run_dbt(['test', '--vars', '{myvar: foo}'])
  self.assertEqual(len(results), 1)
  self.run_dbt(['test'], expect_pass=False)
+
+
+class TestSchemaCaseInsensitive(DBTIntegrationTest):
+ @property
+ def schema(self):
+ return "schema_tests_008"
+
+ @property
+ def models(self):
+ return "case-sensitive-models"
+
+ @use_profile('postgres')
+ def test_postgres_schema_lowercase_sql(self):
+ results = self.run_dbt(strict=False)
+ self.assertEqual(len(results), 2)
+ results = self.run_dbt(['test', '-m', 'lowercase'], strict=False)
+ self.assertEqual(len(results), 1)
+
+ @use_profile('postgres')
+ def test_postgres_schema_uppercase_sql(self):
+ results = self.run_dbt(strict=False)
+ self.assertEqual(len(results), 2)
+ results = self.run_dbt(['test', '-m', 'uppercase'], strict=False)
+ self.assertEqual(len(results), 1)
+
+
+class TestSchemaYAMLExtension(DBTIntegrationTest):
+ @property
+ def schema(self):
+ return "schema_tests_008"
+
+ @property
+ def models(self):
+ return "case-sensitive-models"
+
+ @use_profile('postgres')
+ def test_postgres_yaml_extension(self):
+ with self.assertRaises(Exception) as exc:
+ self.run_dbt(["run"])
+ self.assertIn('yaml', str(exc.exception))

test/unit/test_system_client.py

@@ -2,7 +2,7 @@
 import shutil
 import stat
 import unittest
-from tempfile import mkdtemp
+from tempfile import mkdtemp, NamedTemporaryFile
 
 from dbt.exceptions import ExecutableError, WorkingDirectoryError
 import dbt.clients.system
@@ -133,3 +133,53 @@ def test__ok(self):
  out, err = dbt.clients.system.run_cmd(self.run_dir, self.exists_cmd)
  self.assertEqual(out.strip(), b'hello')
  self.assertEqual(err.strip(), b'')
+
+
+class TestFindMatching(unittest.TestCase):
+
+ def setUp(self):
+ self.base_dir = mkdtemp()
+ self.tempdir = mkdtemp(dir=self.base_dir)
+
+ def test_find_matching_lowercase_file_pattern(self):
+ with NamedTemporaryFile(
+ prefix='sql-files', suffix='.sql', dir=self.tempdir
+ ) as named_file:
+ file_path = os.path.dirname(named_file.name)
+ relative_path = os.path.basename(file_path)
+ out = dbt.clients.system.find_matching(
+ self.base_dir, [relative_path], '*.sql'
+ )
+ expected_output = [{
+ 'searched_path': relative_path,
+ 'absolute_path': named_file.name,
+ 'relative_path': os.path.basename(named_file.name)
+ }]
+ self.assertEqual(out, expected_output)
+
+ def test_find_matching_uppercase_file_pattern(self):
+ with NamedTemporaryFile(prefix='sql-files', suffix='.SQL', dir=self.tempdir) as named_file:
+ file_path = os.path.dirname(named_file.name)
+ relative_path = os.path.basename(file_path)
+ out = dbt.clients.system.find_matching(
+ self.base_dir, [relative_path], '*.sql'
+ )
+ expected_output = [{
+ 'searched_path': relative_path,
+ 'absolute_path': named_file.name,
+ 'relative_path': os.path.basename(named_file.name)
+ }]
+ self.assertEqual(out, expected_output)
+
+ def test_find_matching_file_pattern_not_found(self):
+ with NamedTemporaryFile(
+ prefix='sql-files', suffix='.SQLT', dir=self.tempdir
+ ):
+ out = dbt.clients.system.find_matching(self.tempdir, [''], '*.sql')
+ self.assertEqual(out, [])
+
+ def tearDown(self):
+ try:
+ shutil.rmtree(self.base_dir)
+ except:
+ pass