Skip to content

Commit

Permalink
Merge pull request #153 from martincollignon/fix/dataflow-script-fixes
Browse files Browse the repository at this point in the history
fix: correct Dataflow script issues
  • Loading branch information
martincollignon authored Dec 1, 2024
2 parents 54eecd7 + dffc55d commit 2bac019
Showing 1 changed file with 6 additions and 6 deletions.
12 changes: 6 additions & 6 deletions backend/dataflow/validate_geometries.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,16 +94,16 @@ def process(self, element):

yield {'dataset': dataset, 'data': gdf, 'stats': stats}

def run(argv=None):
pipeline_options = ValidateGeometriesOptions(argv)
def run():
pipeline_options = PipelineOptions()
options = pipeline_options.view_as(ValidateGeometriesOptions)

with beam.Pipeline(options=pipeline_options) as p:
(p
| 'Create Dataset' >> beam.Create([options.dataset])
| 'Read Data' >> beam.Map(lambda dataset: read_dataset(dataset, options.input_bucket))
| 'Create Dataset' >> beam.Create([options.dataset.get()])
| 'Read Data' >> beam.Map(lambda dataset: read_dataset(dataset, options.input_bucket.get()))
| 'Validate and Optimize' >> beam.ParDo(ValidateAndOptimize())
| 'Write Results' >> beam.Map(lambda element: write_outputs(element, options.output_bucket))
| 'Write Results' >> beam.Map(lambda element: write_outputs(element, options.output_bucket.get()))
)

if __name__ == '__main__':
Expand Down

0 comments on commit 2bac019

Please sign in to comment.