From 80f1694de44ba712362730ccd082c106f2291a3b Mon Sep 17 00:00:00 2001 From: Benoit Perigaud <8754100+b-per@users.noreply.github.com> Date: Tue, 29 Nov 2022 13:36:08 +0100 Subject: [PATCH 1/7] Add plan command to the CLI --- src/client/__init__.py | 2 +- src/main.py | 75 ++++++++++++++++++++++++++++++------------ 2 files changed, 55 insertions(+), 22 deletions(-) diff --git a/src/client/__init__.py b/src/client/__init__.py index b53f53d..f3f44ad 100644 --- a/src/client/__init__.py +++ b/src/client/__init__.py @@ -169,7 +169,7 @@ def get_env_vars( ), headers=self._headers, ) - logger.debug(response.json()) + # logger.debug(response.json()) variables = { name: CustomEnvironmentVariablePayload( diff --git a/src/main.py b/src/main.py index abedb2f..cad7f0c 100644 --- a/src/main.py +++ b/src/main.py @@ -9,15 +9,9 @@ from schemas import check_job_mapping_same -@click.group() -def cli(): - pass - - -@cli.command() -@click.argument("config", type=click.File("r")) -def sync(config): - """Synchronize a dbt Cloud job config file against dbt Cloud. +def compare_config_and_potentially_update(config, no_update): + """Compares the config of YML files versus dbt Cloud. + Depending on the value of no_update, it will either update the dbt Cloud config or not. CONFIG is the path to your jobs.yml config file. """ @@ -48,17 +42,26 @@ def sync(config): source_job=defined_jobs[identifier], dest_job=tracked_jobs[identifier] ): defined_jobs[identifier].id = tracked_jobs[identifier].id - dbt_cloud.update_job(job=defined_jobs[identifier]) + if no_update: + logger.warning("-- Plan -- The job {identifier} is different and would be updated.", identifier=identifier) + else: + dbt_cloud.update_job(job=defined_jobs[identifier]) # Create new jobs logger.info("Detected {count} new jobs.", count=len(created_jobs)) for identifier in created_jobs: - dbt_cloud.create_job(job=defined_jobs[identifier]) + if no_update: + logger.warning("-- Plan -- The job {identifier} is new and would be created.", identifier=identifier) + else: + dbt_cloud.create_job(job=defined_jobs[identifier]) # Remove Deleted Jobs logger.warning("Detected {count} deleted jobs.", count=len(deleted_jobs)) for identifier in deleted_jobs: - dbt_cloud.delete_job(job=tracked_jobs[identifier]) + if no_update: + logger.warning("-- Plan -- The job {identifier} is deleted and would be removed.", identifier=identifier) + else: + dbt_cloud.delete_job(job=tracked_jobs[identifier]) # -- ENV VARS -- # Now that we have replicated all jobs we can get their IDs for further API calls @@ -70,9 +73,12 @@ def sync(config): job_id = mapping_job_identifier_job_id[job.identifier] for env_var_yml in job.custom_environment_variables: env_var_yml.job_definition_id = job_id - updated_env_vars = dbt_cloud.update_env_var( - project_id=job.project_id, job_id=job_id, custom_env_var=env_var_yml - ) + if no_update: + logger.warning("-- Plan -- The env var {env_var} is new and would be created.", env_var=env_var_yml.name) + else: + updated_env_vars = dbt_cloud.update_env_var( + project_id=job.project_id, job_id=job_id, custom_env_var=env_var_yml + ) # Delete the env vars from dbt Cloud that are not in the yml for job in defined_jobs.values(): @@ -92,12 +98,39 @@ def sync(config): # If the env var is not in the YML but is defined at the "job" level in dbt Cloud, we delete it if env_var not in env_vars_for_job and "job" in env_var_val: logger.info(f"{env_var} not in the YML file but in the dbt Cloud job") - dbt_cloud.delete_env_var( - project_id=job.project_id, env_var_id=env_var_val["job"]["id"] - ) - logger.info( - f"Deleted the env_var {env_var} for the job {job.identifier}" - ) + if no_update: + logger.warning("-- Plan -- The env var {env_var} is deleted and would be removed.", env_var=env_var) + else: + dbt_cloud.delete_env_var( + project_id=job.project_id, env_var_id=env_var_val["job"]["id"] + ) + logger.info( + f"Deleted the env_var {env_var} for the job {job.identifier}" + ) + +@click.group() +def cli(): + pass + + +@cli.command() +@click.argument("config", type=click.File("r")) +def sync(config): + """Synchronize a dbt Cloud job config file against dbt Cloud. + + CONFIG is the path to your jobs.yml config file. + """ + compare_config_and_potentially_update(config, no_update=False) + + +@cli.command() +@click.argument("config", type=click.File("r")) +def plan(config): + """Check the difference betweeen a local file and dbt Cloud without updating dbt Cloud. + + CONFIG is the path to your jobs.yml config file. + """ + compare_config_and_potentially_update(config, no_update=True) @cli.command() From 737ac783af62afed2f1d2d2c28ed81bff93741ce Mon Sep 17 00:00:00 2001 From: Benoit Perigaud <8754100+b-per@users.noreply.github.com> Date: Sat, 3 Dec 2022 00:02:10 +0100 Subject: [PATCH 2/7] Move the check of matching env vars outside client --- src/client/__init__.py | 48 +++++++++++------------------------------- 1 file changed, 12 insertions(+), 36 deletions(-) diff --git a/src/client/__init__.py b/src/client/__init__.py index 085d02f..c124363 100644 --- a/src/client/__init__.py +++ b/src/client/__init__.py @@ -8,6 +8,7 @@ CustomEnvironmentVariablePayload, ) from schemas.job import JobDefinition +from schemas import check_env_var_same class DBTCloud: @@ -65,7 +66,7 @@ def update_job(self, job: JobDefinition) -> JobDefinition: if response.status_code >= 400: logger.error(response.json()) - logger.success("Updated successfully.") + logger.success("Job updated successfully.") return JobDefinition(**(response.json()["data"]), identifier=job.identifier) @@ -83,7 +84,7 @@ def create_job(self, job: JobDefinition) -> JobDefinition: if response.status_code >= 400: logger.error(response.json()) - logger.success("Created successfully.") + logger.success("Job created successfully.") return JobDefinition(**(response.json()["data"]), identifier=job.identifier) @@ -100,7 +101,7 @@ def delete_job(self, job: JobDefinition) -> None: if response.status_code >= 400: logger.error(response.json()) - logger.warning("Deleted successfully.") + logger.success("Job deleted successfully.") def get_jobs(self) -> List[JobDefinition]: """Return a list of Jobs for all the dbt Cloud jobs in an environment.""" @@ -206,41 +207,16 @@ def create_env_var( return response.json()["data"] def update_env_var( - self, custom_env_var: CustomEnvironmentVariable, project_id: int, job_id: int - ) -> Optional[CustomEnvironmentVariablePayload]: + self, custom_env_var: CustomEnvironmentVariable, project_id: int, job_id: int, env_var_id: int, yml_job_identifier: str = None) -> Optional[CustomEnvironmentVariablePayload]: """Update env vars job overwrite in dbt Cloud.""" self._check_for_creds() - all_env_vars = self.get_env_vars(project_id, job_id) - - if custom_env_var.name not in all_env_vars: - raise Exception( - f"Custom environment variable {custom_env_var.name} not found in dbt Cloud, " - f"you need to create it first." - ) - - env_var_id: Optional[int] - - # TODO: Move this logic out of the client layer, and move it into - # at least one layer higher up. We want the dbt Cloud client to be - # as naive as possible. - if custom_env_var.name not in all_env_vars: - return self.create_env_var( - CustomEnvironmentVariablePayload( - account_id=self.account_id, - project_id=project_id, - **custom_env_var.dict(), - ) - ) - - if all_env_vars[custom_env_var.name].value == custom_env_var.value: - logger.debug( - f"The env var {custom_env_var.name} is already up to date for the job {job_id}." - ) - return None - - env_var_id: int = all_env_vars[custom_env_var.name].id + # handle the case where the job was not created when we queued the function call + if yml_job_identifier and not job_id: + mapping_job_identifier_job_id = self.build_mapping_job_identifier_job_id() + job_id = mapping_job_identifier_job_id[yml_job_identifier] + custom_env_var.job_definition_id = job_id # the endpoint is different for updating an overwrite vs creating one if env_var_id: @@ -266,7 +242,7 @@ def update_env_var( self._clear_env_var_cache(job_definition_id=payload.job_definition_id) - logger.info(f"Updated the env_var {custom_env_var.name} for job {job_id}") + logger.success(f"Updated the env_var {custom_env_var.name} for job {job_id}") return CustomEnvironmentVariablePayload(**(response.json()["data"])) def delete_env_var(self, project_id: int, env_var_id: int) -> None: @@ -282,7 +258,7 @@ def delete_env_var(self, project_id: int, env_var_id: int) -> None: if response.status_code >= 400: logger.error(response.json()) - logger.warning("Deleted successfully.") + logger.success("Env Var Job Overwrite deleted successfully.") def get_environments(self) -> Dict: """Return a list of Environments for all the dbt Cloud jobs in an account""" From e6b9350adc3de59c0f740504a8bfb8aaf59f9744 Mon Sep 17 00:00:00 2001 From: Benoit Perigaud <8754100+b-per@users.noreply.github.com> Date: Sat, 3 Dec 2022 00:02:52 +0100 Subject: [PATCH 3/7] New function to check if a local config of env var exists in Cloud --- src/schemas/__init__.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/schemas/__init__.py b/src/schemas/__init__.py index a07dd0e..07916ab 100644 --- a/src/schemas/__init__.py +++ b/src/schemas/__init__.py @@ -1,9 +1,10 @@ -from typing import Any +from typing import Any, Tuple from deepdiff import DeepDiff from loguru import logger from schemas.job import JobDefinition +from schemas.custom_environment_variable import CustomEnvironmentVariable, CustomEnvironmentVariablePayload def _get_mismatched_dict_entries( @@ -25,7 +26,7 @@ def _job_to_dict(job: JobDefinition): def check_job_mapping_same(source_job: JobDefinition, dest_job: JobDefinition) -> bool: - """ " Checks if the source and destination jobs are the same""" + """Checks if the source and destination jobs are the same""" source_job_dict = _job_to_dict(source_job) dest_job_dict = _job_to_dict(dest_job) @@ -38,3 +39,24 @@ def check_job_mapping_same(source_job: JobDefinition, dest_job: JobDefinition) - else: logger.warning(f"❌ Jobs are different - Diff: {diffs}") return False + + +def check_env_var_same(source_env_var: CustomEnvironmentVariable, dest_env_vars: dict[str, CustomEnvironmentVariablePayload]) -> Tuple[bool, int]: + """Checks if the source env vars is the same in the destination env vars""" + + if source_env_var.name not in dest_env_vars: + raise Exception( + f"Custom environment variable {source_env_var.name} not found in dbt Cloud, " + f"you need to create it first." + ) + + env_var_id = dest_env_vars[source_env_var.name].id + + if dest_env_vars[source_env_var.name].value == source_env_var.value: + logger.debug( + f"The env var {source_env_var.name} is already up to date for the job {source_env_var.job_definition_id}." + ) + return (True, env_var_id) + else: + logger.warning(f"❌ The env var overwrite for {source_env_var.name} is different") + return (False, env_var_id) \ No newline at end of file From 63f8663a54686af74956b2ed31f8dbf4169d70ac Mon Sep 17 00:00:00 2001 From: Benoit Perigaud <8754100+b-per@users.noreply.github.com> Date: Sat, 3 Dec 2022 00:05:02 +0100 Subject: [PATCH 4/7] Create a ChangeSet Class This class stores the changes identified in 2 ways: - a descriptive way, with an action/type/id - a programmatic way, on how it should be handled (func and params) --- src/changeset/change_set.py | 38 +++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 src/changeset/change_set.py diff --git a/src/changeset/change_set.py b/src/changeset/change_set.py new file mode 100644 index 0000000..3666e9f --- /dev/null +++ b/src/changeset/change_set.py @@ -0,0 +1,38 @@ +from typing import Optional +from pydantic import BaseModel + +class Change(BaseModel): + """Describes what a given change is and hot to apply it.""" + identifier: str + type: str + action: str + sync_function: object + parameters: dict + + def __str__(self): + return f"{self.action.upper()} {self.type.capitalize()} {self.identifier}" + + def apply(self): + self.sync_function(**self.parameters) + + +class ChangeSet(BaseModel): + """Store the set of changes to be displayed or applied.""" + __root__: Optional[list[Change]] = [] + + def __iter__(self): + return iter(self.__root__) + + def append(self, change: Change): + self.__root__.append(change) + + def __str__(self): + list_str = [str(change) for change in self.__root__] + return "\n".join(list_str) + + def __len__(self): + return len(self.__root__) + + def apply(self): + for change in self.__root__: + change.apply() \ No newline at end of file From 5db033b1362f673002a1732d511b5a9bd9266950 Mon Sep 17 00:00:00 2001 From: Benoit Perigaud <8754100+b-per@users.noreply.github.com> Date: Sat, 3 Dec 2022 00:06:56 +0100 Subject: [PATCH 5/7] Build the ChangeSet first and then act on it The logic becomes a bit more complex to handle the case where we setup env var overwrites in a new job that doesn't exist in Cloud yet --- src/main.py | 163 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 117 insertions(+), 46 deletions(-) diff --git a/src/main.py b/src/main.py index f71df02..fda4ca1 100644 --- a/src/main.py +++ b/src/main.py @@ -7,9 +7,11 @@ from client import DBTCloud from loader.load import load_job_configuration from schemas import check_job_mapping_same +from changeset.change_set import Change, ChangeSet +from schemas import check_env_var_same -def compare_config_and_potentially_update(config, no_update): +def build_change_set(config): """Compares the config of YML files versus dbt Cloud. Depending on the value of no_update, it will either update the dbt Cloud config or not. @@ -29,6 +31,8 @@ def compare_config_and_potentially_update(config, no_update): job.identifier: job for job in cloud_jobs if job.identifier is not None } + dbt_cloud_change_set = ChangeSet() + # Use sets to find jobs for different operations shared_jobs = set(defined_jobs.keys()).intersection(set(tracked_jobs.keys())) created_jobs = set(defined_jobs.keys()) - set(tracked_jobs.keys()) @@ -41,27 +45,39 @@ def compare_config_and_potentially_update(config, no_update): if not check_job_mapping_same( source_job=defined_jobs[identifier], dest_job=tracked_jobs[identifier] ): + dbt_cloud_change = Change( + identifier=identifier, + type="job", + action="update", + sync_function=dbt_cloud.update_job, + parameters={"job": defined_jobs[identifier]}, + ) + dbt_cloud_change_set.append(dbt_cloud_change) defined_jobs[identifier].id = tracked_jobs[identifier].id - if no_update: - logger.warning("-- Plan -- The job {identifier} is different and would be updated.", identifier=identifier) - else: - dbt_cloud.update_job(job=defined_jobs[identifier]) # Create new jobs logger.info("Detected {count} new jobs.", count=len(created_jobs)) for identifier in created_jobs: - if no_update: - logger.warning("-- Plan -- The job {identifier} is new and would be created.", identifier=identifier) - else: - dbt_cloud.create_job(job=defined_jobs[identifier]) + dbt_cloud_change = Change( + identifier=identifier, + type="job", + action="create", + sync_function=dbt_cloud.create_job, + parameters={"job": defined_jobs[identifier]}, + ) + dbt_cloud_change_set.append(dbt_cloud_change) # Remove Deleted Jobs logger.warning("Detected {count} deleted jobs.", count=len(deleted_jobs)) for identifier in deleted_jobs: - if no_update: - logger.warning("-- Plan -- The job {identifier} is deleted and would be removed.", identifier=identifier) - else: - dbt_cloud.delete_job(job=tracked_jobs[identifier]) + dbt_cloud_change = Change( + identifier=identifier, + type="job", + action="delete", + sync_function=dbt_cloud.delete_job, + parameters={"job": tracked_jobs[identifier]}, + ) + dbt_cloud_change_set.append(dbt_cloud_change) # -- ENV VARS -- # Now that we have replicated all jobs we can get their IDs for further API calls @@ -70,43 +86,86 @@ def compare_config_and_potentially_update(config, no_update): # Replicate the env vars from the YML to dbt Cloud for job in defined_jobs.values(): - job_id = mapping_job_identifier_job_id[job.identifier] - for env_var_yml in job.custom_environment_variables: - env_var_yml.job_definition_id = job_id - if no_update: - logger.warning("-- Plan -- The env var {env_var} is new and would be created.", env_var=env_var_yml.name) - else: - updated_env_vars = dbt_cloud.update_env_var( - project_id=job.project_id, job_id=job_id, custom_env_var=env_var_yml + + if job.identifier in mapping_job_identifier_job_id: # the job already exists + job_id = mapping_job_identifier_job_id[job.identifier] + all_env_vars_for_job = dbt_cloud.get_env_vars( + project_id=job.project_id, job_id=job_id + ) + for env_var_yml in job.custom_environment_variables: + env_var_yml.job_definition_id = job_id + same_env_var, env_var_id = check_env_var_same( + source_env_var=env_var_yml, dest_env_vars=all_env_vars_for_job + ) + if not same_env_var: + dbt_cloud_change = Change( + identifier=f"{job.identifier}:{env_var_yml.name}", + type="env var overwrite", + action="update", + sync_function=dbt_cloud.update_env_var, + parameters={ + "project_id": job.project_id, + "job_id": job_id, + "custom_env_var": env_var_yml, + "env_var_id": env_var_id, + }, + ) + dbt_cloud_change_set.append(dbt_cloud_change) + + else: # the job doesn't exist yet so it doesn't have an ID + for env_var_yml in job.custom_environment_variables: + dbt_cloud_change = Change( + identifier=f"{job.identifier}:{env_var_yml.name}", + type="env var overwrite", + action="create", + sync_function=dbt_cloud.update_env_var, + parameters={ + "project_id": job.project_id, + "job_id": None, + "custom_env_var": env_var_yml, + "env_var_id": None, + "yml_job_identifier": job.identifier, + }, ) + dbt_cloud_change_set.append(dbt_cloud_change) # Delete the env vars from dbt Cloud that are not in the yml for job in defined_jobs.values(): - job_id = mapping_job_identifier_job_id[job.identifier] - # We get the env vars from dbt Cloud, now that the YML ones have been replicated - env_var_dbt_cloud = dbt_cloud.get_env_vars( - project_id=job.project_id, job_id=job_id - ) + # we only delete env var overwrite if the job already exists + if job.identifier in mapping_job_identifier_job_id: + job_id = mapping_job_identifier_job_id[job.identifier] - # And we get the list of env vars defined for a given job in the YML - env_vars_for_job = [ - env_var.name for env_var in job.custom_environment_variables - ] + # We get the env vars from dbt Cloud, now that the YML ones have been replicated + env_var_dbt_cloud = dbt_cloud.get_env_vars( + project_id=job.project_id, job_id=job_id + ) - for env_var, env_var_val in env_var_dbt_cloud.items(): - # If the env var is not in the YML but is defined at the "job" level in dbt Cloud, we delete it - if env_var not in env_vars_for_job and env_var_val.id: - logger.info(f"{env_var} not in the YML file but in the dbt Cloud job") - if no_update: - logger.warning("-- Plan -- The env var {env_var} is deleted and would be removed.", env_var=env_var) - else: - dbt_cloud.delete_env_var( - project_id=job.project_id, env_var_id=env_var_val.id - ) + # And we get the list of env vars defined for a given job in the YML + env_vars_for_job = [ + env_var.name for env_var in job.custom_environment_variables + ] + + for env_var, env_var_val in env_var_dbt_cloud.items(): + # If the env var is not in the YML but is defined at the "job" level in dbt Cloud, we delete it + if env_var not in env_vars_for_job and env_var_val.id: logger.info( - f"Deleted the env_var {env_var} for the job {job.identifier}" + f"{env_var} not in the YML file but in the dbt Cloud job" + ) + dbt_cloud_change = Change( + identifier=f"{job.identifier}:{env_var_yml.name}", + type="env var overwrite", + action="delete", + sync_function=dbt_cloud.delete_env_var, + parameters={ + "project_id": job.project_id, + "env_var_id": env_var_val.id, + }, ) + dbt_cloud_change_set.append(dbt_cloud_change) + + return dbt_cloud_change_set + @click.group() def cli(): @@ -120,17 +179,29 @@ def sync(config): CONFIG is the path to your jobs.yml config file. """ - compare_config_and_potentially_update(config, no_update=False) + change_set = build_change_set(config) + if len(change_set) == 0: + logger.success("-- PLAN -- No changes detected.") + else: + logger.warning("-- PLAN -- {count} changes detected.", count=len(change_set)) + print(change_set) + logger.info("-- SYNC --") + change_set.apply() @cli.command() @click.argument("config", type=click.File("r")) def plan(config): - """Check the difference betweeen a local file and dbt Cloud without updating dbt Cloud. + """Check the difference between a local file and dbt Cloud without updating dbt Cloud. CONFIG is the path to your jobs.yml config file. """ - compare_config_and_potentially_update(config, no_update=True) + change_set = build_change_set(config) + if len(change_set) == 0: + logger.success("-- PLAN -- No changes detected.") + else: + logger.warning("-- PLAN -- {count} changes detected.", count=len(change_set)) + print(change_set) @cli.command() @@ -154,11 +225,11 @@ def validate(config, online): if not online: return - # Retrive the list of Project IDs and Environment IDs from the config file + # Retrieve the list of Project IDs and Environment IDs from the config file config_project_ids = set([job.project_id for job in defined_jobs]) config_environment_ids = set([job.environment_id for job in defined_jobs]) - # Retrieve the list of Project IDs and Environment IDs from dbt Cloudby calling the environment API endpoint + # Retrieve the list of Project IDs and Environment IDs from dbt Cloud by calling the environment API endpoint dbt_cloud = DBTCloud( account_id=list(defined_jobs)[0].account_id, api_key=os.environ.get("API_KEY"), From b9b0a891be3d6d416c31132d5af2bb2f81aa3396 Mon Sep 17 00:00:00 2001 From: Benoit Perigaud <8754100+b-per@users.noreply.github.com> Date: Sat, 3 Dec 2022 00:12:33 +0100 Subject: [PATCH 6/7] Looks better with capwords --- src/changeset/change_set.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/changeset/change_set.py b/src/changeset/change_set.py index 3666e9f..69efe9e 100644 --- a/src/changeset/change_set.py +++ b/src/changeset/change_set.py @@ -1,5 +1,7 @@ from typing import Optional from pydantic import BaseModel +import string + class Change(BaseModel): """Describes what a given change is and hot to apply it.""" @@ -10,7 +12,7 @@ class Change(BaseModel): parameters: dict def __str__(self): - return f"{self.action.upper()} {self.type.capitalize()} {self.identifier}" + return f"{self.action.upper()} {string.capwords(self.type)} {self.identifier}" def apply(self): self.sync_function(**self.parameters) From 8c7bb9e69f7cff57257b711d8c601f2c9142a0cd Mon Sep 17 00:00:00 2001 From: Benoit Perigaud <8754100+b-per@users.noreply.github.com> Date: Mon, 5 Dec 2022 12:11:11 +0100 Subject: [PATCH 7/7] Leverage `rich` for showing tables in CLI --- requirements.txt | 3 ++- src/changeset/change_set.py | 27 +++++++++++++++++++++++---- src/main.py | 13 ++++++++----- src/schemas/__init__.py | 4 ++-- 4 files changed, 35 insertions(+), 12 deletions(-) diff --git a/requirements.txt b/requirements.txt index 327491d..ca96bda 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,4 +15,5 @@ requests==2.28.1 tomli==2.0.1 typing_extensions==4.4.0 urllib3==1.26.12 -croniter==1.3.7 \ No newline at end of file +croniter==1.3.7 +rich==12.6.0 \ No newline at end of file diff --git a/src/changeset/change_set.py b/src/changeset/change_set.py index 69efe9e..357c0cc 100644 --- a/src/changeset/change_set.py +++ b/src/changeset/change_set.py @@ -1,10 +1,12 @@ from typing import Optional from pydantic import BaseModel import string +from rich.table import Table class Change(BaseModel): """Describes what a given change is and hot to apply it.""" + identifier: str type: str action: str @@ -20,21 +22,38 @@ def apply(self): class ChangeSet(BaseModel): """Store the set of changes to be displayed or applied.""" + __root__: Optional[list[Change]] = [] def __iter__(self): return iter(self.__root__) - + def append(self, change: Change): self.__root__.append(change) - + def __str__(self): list_str = [str(change) for change in self.__root__] return "\n".join(list_str) - + + def to_table(self) -> Table: + """Return a table representation of the changeset.""" + + table = Table(title="Changes detected") + + table.add_column("Action", style="cyan", no_wrap=True) + table.add_column("Type", style="magenta") + table.add_column("ID", style="green") + + for change in self.__root__: + table.add_row( + change.action.upper(), string.capwords(change.type), change.identifier + ) + + return table + def __len__(self): return len(self.__root__) def apply(self): for change in self.__root__: - change.apply() \ No newline at end of file + change.apply() diff --git a/src/main.py b/src/main.py index fda4ca1..7c7eabf 100644 --- a/src/main.py +++ b/src/main.py @@ -9,6 +9,7 @@ from schemas import check_job_mapping_same from changeset.change_set import Change, ChangeSet from schemas import check_env_var_same +from rich.console import Console def build_change_set(config): @@ -68,7 +69,7 @@ def build_change_set(config): dbt_cloud_change_set.append(dbt_cloud_change) # Remove Deleted Jobs - logger.warning("Detected {count} deleted jobs.", count=len(deleted_jobs)) + logger.info("Detected {count} deleted jobs.", count=len(deleted_jobs)) for identifier in deleted_jobs: dbt_cloud_change = Change( identifier=identifier, @@ -183,8 +184,9 @@ def sync(config): if len(change_set) == 0: logger.success("-- PLAN -- No changes detected.") else: - logger.warning("-- PLAN -- {count} changes detected.", count=len(change_set)) - print(change_set) + logger.info("-- PLAN -- {count} changes detected.", count=len(change_set)) + console = Console() + console.log(change_set.to_table()) logger.info("-- SYNC --") change_set.apply() @@ -200,8 +202,9 @@ def plan(config): if len(change_set) == 0: logger.success("-- PLAN -- No changes detected.") else: - logger.warning("-- PLAN -- {count} changes detected.", count=len(change_set)) - print(change_set) + logger.info("-- PLAN -- {count} changes detected.", count=len(change_set)) + console = Console() + console.log(change_set.to_table()) @cli.command() diff --git a/src/schemas/__init__.py b/src/schemas/__init__.py index 07916ab..4701ec0 100644 --- a/src/schemas/__init__.py +++ b/src/schemas/__init__.py @@ -37,7 +37,7 @@ def check_job_mapping_same(source_job: JobDefinition, dest_job: JobDefinition) - logger.success("✅ Jobs identical") return True else: - logger.warning(f"❌ Jobs are different - Diff: {diffs}") + logger.info(f"❌ Jobs are different - Diff: {diffs}") return False @@ -58,5 +58,5 @@ def check_env_var_same(source_env_var: CustomEnvironmentVariable, dest_env_vars: ) return (True, env_var_id) else: - logger.warning(f"❌ The env var overwrite for {source_env_var.name} is different") + logger.info(f"❌ The env var overwrite for {source_env_var.name} is different") return (False, env_var_id) \ No newline at end of file