diff --git a/sdks/python/apache_beam/yaml/examples/transforms/blueprint/gcs_text_to_bigquery.yaml b/sdks/python/apache_beam/yaml/examples/transforms/blueprint/gcs_text_to_bigquery.yaml index 304f5d2c100b..6b8c289402dc 100644 --- a/sdks/python/apache_beam/yaml/examples/transforms/blueprint/gcs_text_to_bigquery.yaml +++ b/sdks/python/apache_beam/yaml/examples/transforms/blueprint/gcs_text_to_bigquery.yaml @@ -36,6 +36,9 @@ pipeline: write_disposition: "WRITE_APPEND" num_streams: 1 +options: + temp_location: "gs://apache-beam-testing/temp" + # Expected: # Row(line='Fool\tThou shouldst not have been old till thou hadst') diff --git a/sdks/python/apache_beam/yaml/examples/transforms/blueprint/jdbc_to_bigquery.yaml b/sdks/python/apache_beam/yaml/examples/transforms/blueprint/jdbc_to_bigquery.yaml index 913f424ebc17..d75dce64f318 100644 --- a/sdks/python/apache_beam/yaml/examples/transforms/blueprint/jdbc_to_bigquery.yaml +++ b/sdks/python/apache_beam/yaml/examples/transforms/blueprint/jdbc_to_bigquery.yaml @@ -46,7 +46,8 @@ pipeline: config: path: "gs://my-bucket/yaml-123/writingToBigQueryErrors.json" - +options: + temp_location: "gs://apache-beam-testing/temp" # Expected: # Row(shipment_id='S1', customer_id='C1', shipment_date='2023-05-01', shipment_cost=150.0, customer_name='Alice', customer_email='alice@example.com') diff --git a/sdks/python/apache_beam/yaml/examples/transforms/blueprint/mysql_to_bigquery.yaml b/sdks/python/apache_beam/yaml/examples/transforms/blueprint/mysql_to_bigquery.yaml index b2c1e0fb86ec..a6938b7582f2 100644 --- a/sdks/python/apache_beam/yaml/examples/transforms/blueprint/mysql_to_bigquery.yaml +++ b/sdks/python/apache_beam/yaml/examples/transforms/blueprint/mysql_to_bigquery.yaml @@ -46,6 +46,9 @@ pipeline: config: path: "gs://my-bucket/yaml-123/writingToBigQueryErrors.json" +options: + temp_location: "gs://apache-beam-testing/temp" + # Expected: # Row(shipment_id='S1', customer_id='C1', shipment_date='2023-05-01', shipment_cost=150.0, customer_name='Alice', customer_email='alice@example.com') # Row(shipment_id='S2', customer_id='C2', shipment_date='2023-06-12', shipment_cost=300.0, customer_name='Bob', customer_email='bob@example.com') diff --git a/sdks/python/apache_beam/yaml/examples/transforms/blueprint/oracle_to_bigquery.yaml b/sdks/python/apache_beam/yaml/examples/transforms/blueprint/oracle_to_bigquery.yaml index 80e61fac53cf..18f87c13f1ce 100644 --- a/sdks/python/apache_beam/yaml/examples/transforms/blueprint/oracle_to_bigquery.yaml +++ b/sdks/python/apache_beam/yaml/examples/transforms/blueprint/oracle_to_bigquery.yaml @@ -46,6 +46,9 @@ pipeline: config: path: "gs://my-bucket/yaml-123/writingToBigQueryErrors.json" +options: + temp_location: "gs://apache-beam-testing/temp" + # Expected: # Row(shipment_id='S1', customer_id='C1', shipment_date='2023-05-01', shipment_cost=150.0, customer_name='Alice', customer_email='alice@example.com') # Row(shipment_id='S2', customer_id='C2', shipment_date='2023-06-12', shipment_cost=300.0, customer_name='Bob', customer_email='bob@example.com') diff --git a/sdks/python/apache_beam/yaml/examples/transforms/blueprint/postgres_to_bigquery.yaml b/sdks/python/apache_beam/yaml/examples/transforms/blueprint/postgres_to_bigquery.yaml index e0726186b279..b532636f46ee 100644 --- a/sdks/python/apache_beam/yaml/examples/transforms/blueprint/postgres_to_bigquery.yaml +++ b/sdks/python/apache_beam/yaml/examples/transforms/blueprint/postgres_to_bigquery.yaml @@ -46,6 +46,9 @@ pipeline: config: path: "gs://my-bucket/yaml-123/writingToBigQueryErrors.json" +options: + temp_location: "gs://apache-beam-testing/temp" + # Expected: # Row(shipment_id='S1', customer_id='C1', shipment_date='2023-05-01', shipment_cost=150.0, customer_name='Alice', customer_email='alice@example.com') # Row(shipment_id='S2', customer_id='C2', shipment_date='2023-06-12', shipment_cost=300.0, customer_name='Bob', customer_email='bob@example.com') diff --git a/sdks/python/apache_beam/yaml/examples/transforms/blueprint/spanner_to_bigquery.yaml b/sdks/python/apache_beam/yaml/examples/transforms/blueprint/spanner_to_bigquery.yaml index 0609a1a0dcfa..7da5058c3ad7 100644 --- a/sdks/python/apache_beam/yaml/examples/transforms/blueprint/spanner_to_bigquery.yaml +++ b/sdks/python/apache_beam/yaml/examples/transforms/blueprint/spanner_to_bigquery.yaml @@ -39,6 +39,9 @@ pipeline: write_disposition: "WRITE_APPEND" num_streams: 1 +options: + temp_location: "gs://apache-beam-testing/temp" + # Expected: # Row(shipment_id='S1', customer_id='C1', shipment_date='2023-05-01', shipment_cost=150.0, customer_name='Alice', customer_email='alice@example.com') # Row(shipment_id='S2', customer_id='C2', shipment_date='2023-06-12', shipment_cost=300.0, customer_name='Bob', customer_email='bob@example.com') diff --git a/sdks/python/apache_beam/yaml/examples/transforms/blueprint/sqlserver_to_bigquery.yaml b/sdks/python/apache_beam/yaml/examples/transforms/blueprint/sqlserver_to_bigquery.yaml index b7b9b75b76cf..d35f8ad5c44d 100644 --- a/sdks/python/apache_beam/yaml/examples/transforms/blueprint/sqlserver_to_bigquery.yaml +++ b/sdks/python/apache_beam/yaml/examples/transforms/blueprint/sqlserver_to_bigquery.yaml @@ -46,6 +46,9 @@ pipeline: config: path: "gs://my-bucket/yaml-123/writingToBigQueryErrors.json" +options: + temp_location: "gs://apache-beam-testing/temp" + # Expected: # Row(shipment_id='S1', customer_id='C1', shipment_date='2023-05-01', shipment_cost=150.0, customer_name='Alice', customer_email='alice@example.com') # Row(shipment_id='S2', customer_id='C2', shipment_date='2023-06-12', shipment_cost=300.0, customer_name='Bob', customer_email='bob@example.com') diff --git a/sdks/python/apache_beam/yaml/examples/transforms/ml/log_analysis/ml_preprocessing.yaml b/sdks/python/apache_beam/yaml/examples/transforms/ml/log_analysis/ml_preprocessing.yaml index c83eb19e6484..e567a46476be 100644 --- a/sdks/python/apache_beam/yaml/examples/transforms/ml/log_analysis/ml_preprocessing.yaml +++ b/sdks/python/apache_beam/yaml/examples/transforms/ml/log_analysis/ml_preprocessing.yaml @@ -117,6 +117,7 @@ pipeline: options: yaml_experimental_features: [ 'ML' ] + temp_location: "gs://apache-beam-testing/temp" # Expected: # Row(id=1, date='2024-10-01', time='12:00:00', level='INFO', process='Main', component='ComponentA', content='System started successfully', embedding=[0.13483997249264842, 0.26967994498529685, 0.40451991747794525, 0.5393598899705937, 0.674199862463242]) diff --git a/sdks/python/apache_beam/yaml/extended_tests/databases/bigquery.yaml b/sdks/python/apache_beam/yaml/extended_tests/databases/bigquery.yaml index d0357e098bf3..06224b51bcb6 100644 --- a/sdks/python/apache_beam/yaml/extended_tests/databases/bigquery.yaml +++ b/sdks/python/apache_beam/yaml/extended_tests/databases/bigquery.yaml @@ -100,7 +100,9 @@ pipelines: - type: WriteToBigQuery config: table: "{BQ_TABLE_1}" - + options: + project: "apache-beam-testing" + temp_location: "{TEMP_DIR_0}" # New read from BQ to verify row restriction with nullable field and filter # out nullable record - pipeline: