Validate require-dbt-version before validating dbt_project.ymls chema

CHANGELOG.md

@@ -5,6 +5,10 @@
 - Added 3 more adapter methods that the new dbt-adapter-test suite can use for testing. ([#2492](https://github.com/fishtown-analytics/dbt/issues/2492), [#2721](https://github.com/fishtown-analytics/dbt/pull/2721))
 
 
+### Fixes
+- dbt now validates the require-dbt-version field before it validates the dbt_project.yml schema ([#2638](https://github.com/fishtown-analytics/dbt/issues/2638), [#2726](https://github.com/fishtown-analytics/dbt/pull/2726))
+
+
 ## dbt 0.18.0rc1 (August 19, 2020)
 
 

core/dbt/config/project.py

@@ -215,6 +215,11 @@ class PartialProject:
  metadata=dict(description='The root directory of the project'),
  )
  project_dict: Dict[str, Any]
+ verify_version: bool = field(
+ metadata=dict(description=(
+ 'If True, verify the dbt version matches the required version'
+ ))
+ )
 
  def render(self, renderer):
  packages_dict = package_data_from_root(self.project_root)
@@ -225,6 +230,7 @@ def render(self, renderer):
  packages_dict,
  selectors_dict,
  renderer,
+ verify_version=self.verify_version,
  )
 
  def render_profile_name(self, renderer) -> Optional[str]:
@@ -292,6 +298,32 @@ def to_dict(self):
  return self.vars
 
 
+def validate_version(
+ required: List[VersionSpecifier],
+ project_name: str,
+) -> None:
+ """Ensure this package works with the installed version of dbt."""
+ installed = get_installed_version()
+ if not versions_compatible(*required):
+ msg = IMPOSSIBLE_VERSION_ERROR.format(
+ package=project_name,
+ version_spec=[
+ x.to_version_string() for x in required
+ ]
+ )
+ raise DbtProjectError(msg)
+
+ if not versions_compatible(installed, *required):
+ msg = INVALID_VERSION_ERROR.format(
+ package=project_name,
+ installed=installed.to_version_string(),
+ version_spec=[
+ x.to_version_string() for x in required
+ ]
+ )
+ raise DbtProjectError(msg)
+
+
 @dataclass
 class Project:
  project_name: str
@@ -363,6 +395,7 @@ def from_project_config(
  project_dict: Dict[str, Any],
  packages_dict: Optional[Dict[str, Any]] = None,
  selectors_dict: Optional[Dict[str, Any]] = None,
+ required_dbt_version: Optional[List[VersionSpecifier]] = None,
  ) -> 'Project':
  """Create a project from its project and package configuration, as read
  by yaml.safe_load().
@@ -374,6 +407,11 @@ def from_project_config(
  the packages file exists and is invalid.
  :returns: The project, with defaults populated.
  """
+ if required_dbt_version is None:
+ dbt_version = cls._get_required_version(project_dict)
+ else:
+ dbt_version = required_dbt_version
+
  try:
  project_dict = cls._preprocess(project_dict)
  except RecursionException:
@@ -460,18 +498,8 @@ def from_project_config(
  on_run_start: List[str] = value_or(cfg.on_run_start, [])
  on_run_end: List[str] = value_or(cfg.on_run_end, [])
 
- # weird type handling: no value_or use
- dbt_raw_version: Union[List[str], str] = '>=0.0.0'
- if cfg.require_dbt_version is not None:
- dbt_raw_version = cfg.require_dbt_version
-
  query_comment = _query_comment_from_cfg(cfg.query_comment)
 
- try:
- dbt_version = _parse_versions(dbt_raw_version)
- except SemverException as e:
- raise DbtProjectError(str(e)) from e
-
  try:
  packages = package_config_from_data(packages_dict)
  except ValidationError as e:
@@ -583,6 +611,30 @@ def validate(self):
  except ValidationError as e:
  raise DbtProjectError(validator_error_message(e)) from e
 
+ @classmethod
+ def _get_required_version(
+ cls, rendered_project: Dict[str, Any], verify_version: bool = False
+ ) -> List[VersionSpecifier]:
+ dbt_raw_version: Union[List[str], str] = '>=0.0.0'
+ required = rendered_project.get('require-dbt-version')
+ if required is not None:
+ dbt_raw_version = required
+
+ try:
+ dbt_version = _parse_versions(dbt_raw_version)
+ except SemverException as e:
+ raise DbtProjectError(str(e)) from e
+
+ if verify_version:
+ # no name is also an error that we want to raise
+ if 'name' not in rendered_project:
+ raise DbtProjectError(
+ 'Required "name" field not present in project',
+ )
+ validate_version(dbt_version, rendered_project['name'])
+
+ return dbt_version
+
  @classmethod
  def render_from_dict(
  cls,
@@ -591,18 +643,26 @@ def render_from_dict(
  packages_dict: Dict[str, Any],
  selectors_dict: Dict[str, Any],
  renderer: DbtProjectYamlRenderer,
+ *,
+ verify_version: bool = False
  ) -> 'Project':
  rendered_project = renderer.render_data(project_dict)
  rendered_project['project-root'] = project_root
  package_renderer = renderer.get_package_renderer()
  rendered_packages = package_renderer.render_data(packages_dict)
  selectors_renderer = renderer.get_selector_renderer()
  rendered_selectors = selectors_renderer.render_data(selectors_dict)
+
  try:
+ dbt_version = cls._get_required_version(
+ rendered_project, verify_version=verify_version
+ )
+
  return cls.from_project_config(
  rendered_project,
  rendered_packages,
  rendered_selectors,
+ dbt_version,
  )
  except DbtProjectError as exc:
  if exc.path is None:
@@ -611,7 +671,7 @@ def render_from_dict(
 
  @classmethod
  def partial_load(
- cls, project_root: str
+ cls, project_root: str, *, verify_version: bool = False
  ) -> PartialProject:
  project_root = os.path.normpath(project_root)
  project_dict = _raw_project_from(project_root)
@@ -626,41 +686,24 @@ def partial_load(
  project_name=project_name,
  project_root=project_root,
  project_dict=project_dict,
+ verify_version=verify_version,
  )
 
  @classmethod
  def from_project_root(
- cls, project_root: str, renderer: DbtProjectYamlRenderer
+ cls,
+ project_root: str,
+ renderer: DbtProjectYamlRenderer,
+ *,
+ verify_version: bool = False,
  ) -> 'Project':
- partial = cls.partial_load(project_root)
+ partial = cls.partial_load(project_root, verify_version=verify_version)
  renderer.version = partial.config_version
  return partial.render(renderer)
 
  def hashed_name(self):
  return hashlib.md5(self.project_name.encode('utf-8')).hexdigest()
 
- def validate_version(self):
- """Ensure this package works with the installed version of dbt."""
- installed = get_installed_version()
- if not versions_compatible(*self.dbt_version):
- msg = IMPOSSIBLE_VERSION_ERROR.format(
- package=self.project_name,
- version_spec=[
- x.to_version_string() for x in self.dbt_version
- ]
- )
- raise DbtProjectError(msg)
-
- if not versions_compatible(installed, *self.dbt_version):
- msg = INVALID_VERSION_ERROR.format(
- package=self.project_name,
- installed=installed.to_version_string(),
- version_spec=[
- x.to_version_string() for x in self.dbt_version
- ]
- )
- raise DbtProjectError(msg)
-
  def as_v1(self, all_projects: Iterable[str]):
  if self.config_version == 1:
  return self

core/dbt/config/runtime.py

@@ -138,7 +138,11 @@ def new_project(self, project_root: str) -> 'RuntimeConfig':
  # load the new project and its packages. Don't pass cli variables.
  renderer = DbtProjectYamlRenderer(generate_target_context(profile, {}))
 
- project = Project.from_project_root(project_root, renderer)
+ project = Project.from_project_root(
+ project_root,
+ renderer,
+ verify_version=getattr(self.args, 'version_check', False),
+ )
 
  cfg = self.from_parts(
  project=project,
@@ -173,9 +177,6 @@ def validate(self):
  except ValidationError as e:
  raise DbtProjectError(validator_error_message(e)) from e
 
- if getattr(self.args, 'version_check', False):
- self.validate_version()
-
  @classmethod
  def _get_rendered_profile(
  cls,
@@ -193,7 +194,11 @@ def collect_parts(
  ) -> Tuple[Project, Profile]:
  # profile_name from the project
  project_root = args.project_dir if args.project_dir else os.getcwd()
- partial = Project.partial_load(project_root)
+ version_check = getattr(args, 'version_check', False)
+ partial = Project.partial_load(
+ project_root,
+ verify_version=version_check
+ )
 
  # build the profile using the base renderer and the one fact we know
  cli_vars: Dict[str, Any] = parse_cli_vars(getattr(args, 'vars', '{}'))

core/dbt/main.py

@@ -406,6 +406,7 @@ def _build_debug_subparser(subparsers, base_subparser):
  If specified, DBT will show path information for this project
  '''
  )
+ _add_version_check(sub)
  sub.set_defaults(cls=debug_task.DebugTask, which='debug', rpc_method=None)
  return sub
 
@@ -597,6 +598,18 @@ def _add_table_mutability_arguments(*subparsers):
  )
 
 
+def _add_version_check(sub):
+ sub.add_argument(
+ '--no-version-check',
+ dest='version_check',
+ action='store_false',
+ help='''
+ If set, skip ensuring dbt's version matches the one specified in
+ the dbt_project.yml file ('require-dbt-version')
+ '''
+ )
+
+
 def _add_common_arguments(*subparsers):
  for sub in subparsers:
  sub.add_argument(
@@ -608,15 +621,7 @@ def _add_common_arguments(*subparsers):
  settings in profiles.yml.
  '''
  )
- sub.add_argument(
- '--no-version-check',
- dest='version_check',
- action='store_false',
- help='''
- If set, skip ensuring dbt's version matches the one specified in
- the dbt_project.yml file ('require-dbt-version')
- '''
- )
+ _add_version_check(sub)
 
 
 def _build_seed_subparser(subparsers, base_subparser):

core/dbt/task/debug.py

@@ -143,7 +143,9 @@ def _load_project(self):
 
  try:
  self.project = Project.from_project_root(
- self.project_dir, renderer
+ self.project_dir,
+ renderer,
+ verify_version=getattr(self.args, 'version_check', False),
  )
  except dbt.exceptions.DbtConfigError as exc:
  self.project_fail_details = str(exc)
@@ -181,7 +183,8 @@ def _choose_profile_names(self) -> Optional[List[str]]:
  if os.path.exists(self.project_path):
  try:
  partial = Project.partial_load(
- os.path.dirname(self.project_path)
+ os.path.dirname(self.project_path),
+ verify_version=getattr(self.args, 'version_check', False),
  )
  renderer = DbtProjectYamlRenderer(
  generate_base_context(self.cli_vars)