Skip to content

Commit

Permalink
Add project to tests (#81)
Browse files Browse the repository at this point in the history
  • Loading branch information
MaxHalford authored Jan 7, 2025
1 parent 4b9f27e commit 39e0be8
Show file tree
Hide file tree
Showing 5 changed files with 432 additions and 407 deletions.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
rebase:
fomo:
git fetch && git rebase origin/main
4 changes: 1 addition & 3 deletions lea/conductor.py
Original file line number Diff line number Diff line change
Expand Up @@ -216,9 +216,7 @@ def add_context_to_dependency(dependency: TableRef) -> TableRef:
),
)

if not script.is_test:
return script.replace_table_ref(self.add_write_context_to_table_ref(script.table_ref))
return script.replace_table_ref(script.table_ref.replace_dataset(self.write_dataset))
return script.replace_table_ref(self.add_write_context_to_table_ref(script.table_ref))

def run_script(self, script: Script):
# If the script is a test, we don't materialize it, we just query it. A test fails if it
Expand Down
16 changes: 1 addition & 15 deletions lea/databases.py
Original file line number Diff line number Diff line change
Expand Up @@ -306,23 +306,9 @@ def list_table_fields(self, dataset_name: str) -> dict[scripts.TableRef, set[scr
def make_job_config(self, script: scripts.SQLScript, **kwargs) -> bigquery.QueryJobConfig:
if self.print_mode:
rich.print(script)
job_config = bigquery.QueryJobConfig(
return bigquery.QueryJobConfig(
priority=bigquery.QueryPriority.INTERACTIVE,
use_query_cache=False,
dry_run=self.dry_run,
**kwargs,
)

# The approach we use works best if the tables are clustered by account_slug. This is
# because we only need to refresh the data for a subset of accounts, and clustering by
# account_slug allows BigQuery to only scan the data for the accounts that need to be
# refreshed. This is a good practice in general, but it's particularly important in this
# case.
if (
script is not None
and script.table_ref.name.endswith("audit")
and "account_slug" in {field.name for field in script.fields}
):
job_config.clustering_fields = ["account_slug"]

return job_config
Loading

0 comments on commit 39e0be8

Please sign in to comment.