From a36fb2dc25539c5721cef73b613bcbdfe0749a4d Mon Sep 17 00:00:00 2001 From: Martin Collignon <2604526+martincollignon@users.noreply.github.com> Date: Sun, 1 Dec 2024 14:05:04 +0100 Subject: [PATCH] fix: remove setup and requirements options from Dataflow job - Remove setup_file and requirements_file options - Use container dependencies exclusively - Simplify configuration --- dags/data_processing.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dags/data_processing.py b/dags/data_processing.py index bd50cff8..1fd7adc1 100644 --- a/dags/data_processing.py +++ b/dags/data_processing.py @@ -38,9 +38,7 @@ 'disk_size_gb': 100, 'region': 'europe-west1', 'temp_location': 'gs://landbrugsdata-processing/temp', - 'sdk_container_image': 'gcr.io/landbrugsdata-1/dataflow-processing:latest', - 'setup_file': 'gs://landbrugsdata-processing/dataflow/setup.py', - 'requirements_file': 'gs://landbrugsdata-processing/dataflow/requirements.txt' + 'sdk_container_image': 'gcr.io/landbrugsdata-1/dataflow-processing:latest' }, location='europe-west1', wait_until_finished=True,