Skip to content

Commit

Permalink
chore(components): Bump image version for Structured Data pipelines
Browse files Browse the repository at this point in the history
Signed-off-by: Googler <nobody@google.com>
PiperOrigin-RevId: 699072053
  • Loading branch information
Googler committed Nov 26, 2024
1 parent 35793be commit 8136f76
Show file tree
Hide file tree
Showing 42 changed files with 448 additions and 447 deletions.
1 change: 1 addition & 0 deletions components/google-cloud/RELEASE.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
* Update Docker image.
* Bump supported KFP versions to `kfp>=2.6.0,<2.11.0`.
* Support Python versions 3.12 and 3.13.
* Bump image for Structured Data pipelines.

## Release 2.17.0
* Fix Gemini batch prediction support to `v1.model_evaluation.autosxs_pipeline` after output schema change.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ def automl_forecasting_ensemble(
# fmt: on
job_id = dsl.PIPELINE_JOB_ID_PLACEHOLDER
task_id = dsl.PIPELINE_TASK_ID_PLACEHOLDER
image_uri = 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625'
image_uri = 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20241121_0625'
display_name = f'automl-forecasting-ensemble-{job_id}-{task_id}'

error_file_path = f'{root_dir}/{job_id}/{task_id}/error.pb'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -99,14 +99,14 @@ def automl_forecasting_stage_1_tuner(
' 1, "machine_spec": {"machine_type": "n1-standard-8"},'
' "container_spec": {"image_uri":"'
),
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625',
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20241121_0625',
'", "args": ["forecasting_mp_l2l_stage_1_tuner',
'", "--region=',
location,
'", "--transform_output_path=',
transform_output.uri,
'", "--training_docker_uri=',
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625',
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20241121_0625',
'", "--reduce_search_space_mode=',
reduce_search_space_mode,
f'", "--component_id={dsl.PIPELINE_TASK_ID_PLACEHOLDER}',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -97,14 +97,14 @@ def automl_forecasting_stage_2_tuner(
' 1, "machine_spec": {"machine_type": "n1-standard-8"},'
' "container_spec": {"image_uri":"'
),
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625',
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20241121_0625',
'", "args": ["forecasting_mp_l2l_stage_2_tuner',
'", "--region=',
location,
'", "--transform_output_path=',
transform_output.uri,
'", "--training_docker_uri=',
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625',
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20241121_0625',
f'", "--component_id={dsl.PIPELINE_TASK_ID_PLACEHOLDER}',
'", "--training_base_dir=',
root_dir,
Expand Down

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ def automated_feature_engineering(
' 1, "machine_spec": {"machine_type": "n1-standard-16"},'
' "container_spec": {"image_uri":"'
),
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625',
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20241121_0625',
'", "args": ["feature_engineering", "--project=', project,
'", "--location=', location, '", "--data_source_bigquery_table_path=',
data_source_bigquery_table_path,
Expand Down

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ def distillation_stage_feature_transform_engine(
# fmt: on

return dsl.ContainerSpec(
image='us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625',
image='us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20241121_0625',
command=[],
args=[
'distillation_stage_feature_transform_engine',
Expand Down Expand Up @@ -185,7 +185,7 @@ def distillation_stage_feature_transform_engine(
dataflow_machine_type,
]
),
'--dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625',
'--dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20241121_0625',
dsl.ConcatPlaceholder(
items=[
'--dataflow_disk_size_gb=',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ def tabular_feature_ranking_and_selection(
' 1, "machine_spec": {"machine_type": "n1-standard-8"},'
' "container_spec": {"image_uri":"'
),
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625',
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20241121_0625',
'", "args": ["feature_selection", "--data_source=',
data_source.uri,
'", "--target_column=',
Expand Down Expand Up @@ -137,7 +137,7 @@ def tabular_feature_ranking_and_selection(
),
dataflow_max_num_workers,
'", "--dataflow_worker_container_image=',
'us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625',
'us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20241121_0625',
'", "--dataflow_machine_type=',
dataflow_machine_type,
'", "--dataflow_disk_size_gb=',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -983,8 +983,8 @@ deploymentSpec:
"/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}'
- '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}'
- '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}'
- --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625
- --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625
- --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20241121_0625
- --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20241121_0625
- '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}'
- '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}'
- '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}'
Expand All @@ -1001,7 +1001,7 @@ deploymentSpec:
- '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat":
["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}'
- '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}'
image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625
image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20241121_0625
resources:
cpuLimit: 8.0
memoryLimit: 30.0
Expand Down Expand Up @@ -1049,7 +1049,7 @@ deploymentSpec:
["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}'
- '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat":
["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}'
image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625
image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20241121_0625
pipelineInfo:
description: Defines pipeline for feature transform engine component.
name: feature-selection
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,7 @@ def feature_transform_engine(
# fmt: on

return dsl.ContainerSpec(
image='us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625',
image='us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20241121_0625',
command=[],
args=[
'feature_transform_engine',
Expand Down Expand Up @@ -637,8 +637,8 @@ def feature_transform_engine(
dsl.ConcatPlaceholder(
items=['--dataflow_machine_type=', dataflow_machine_type]
),
'--dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625',
'--feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625',
'--dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20241121_0625',
'--feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20241121_0625',
dsl.ConcatPlaceholder(
items=['--dataflow_disk_size_gb=', dataflow_disk_size_gb]
),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -158,15 +158,15 @@ def tabnet_hyperparameter_tuning_job(
', "disk_spec": ',
training_disk_spec,
', "container_spec": {"image_uri":"',
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/tabnet-training:20240808_0625',
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/tabnet-training:20241121_0625',
'", "args": ["--target_column=',
target_column,
'", "--weight_column=',
weight_column,
'", "--model_type=',
prediction_type,
'", "--prediction_docker_uri=',
'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625',
'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20241121_0625',
'", "--prediction_docker_uri_artifact_path=',
prediction_docker_uri_output,
'", "--baseline_path=',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2826,7 +2826,7 @@ deploymentSpec:
\"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}",
"\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\":
{\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"",
"us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\",
"us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20241121_0625", "\",
\"args\": [\"cancel_l2l_tuner\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}",
"/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--cleanup_lro_job_infos=",
"{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\"]}}]}}"]}'
Expand All @@ -2841,7 +2841,7 @@ deploymentSpec:
args:
- --executor_input
- '{{$}}'
image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625
image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20241121_0625
resources:
cpuLimit: 8.0
memoryLimit: 52.0
Expand All @@ -2866,7 +2866,7 @@ deploymentSpec:
\ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\
\ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\
\ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n"
image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625
image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20241121_0625
exec-feature-transform-engine:
container:
args:
Expand Down Expand Up @@ -2951,8 +2951,8 @@ deploymentSpec:
"/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}'
- '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}'
- '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}'
- --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625
- --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625
- --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20241121_0625
- --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20241121_0625
- '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}'
- '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}'
- '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}'
Expand All @@ -2969,7 +2969,7 @@ deploymentSpec:
- '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat":
["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}'
- '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}'
image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625
image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20241121_0625
resources:
cpuLimit: 8.0
memoryLimit: 30.0
Expand Down Expand Up @@ -3037,7 +3037,7 @@ deploymentSpec:
\ = {\n 'instanceSchemaUri': instance_schema_uri,\n 'predictionSchemaUri':\
\ prediction_schema_uri,\n }\n unmanaged_container_model.uri = os.path.join(\n\
\ trials_dir, 'trial_{}'.format(best_trial['id']), 'model'\n )\n\n"
image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625
image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20241121_0625
exec-get-model-display-name:
container:
args:
Expand All @@ -3064,7 +3064,7 @@ deploymentSpec:
\n return collections.namedtuple(\n 'Outputs',\n [\n \
\ 'model_display_name',\n ],\n )(\n model_display_name,\n )\n\
\n"
image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625
image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20241121_0625
exec-get-tabnet-study-spec-parameters:
container:
args:
Expand Down Expand Up @@ -3580,7 +3580,7 @@ deploymentSpec:
\ = ', '.join(extra_overrides)\n warnings.warn(\n f'The overrides\
\ {extra_override_str} were not found in the params and '\n 'will\
\ be ignored.'\n )\n\n return study_spec_parameters\n\n"
image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625
image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20241121_0625
exec-model-batch-predict:
container:
args:
Expand Down Expand Up @@ -3821,7 +3821,7 @@ deploymentSpec:
\ 'training_disk_spec',\n 'eval_machine_spec',\n 'eval_replica_count',\n\
\ ],\n )(\n training_machine_spec,\n training_disk_spec,\n\
\ eval_machine_spec,\n eval_replica_count,\n )\n\n"
image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625
image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20241121_0625
exec-set-optional-inputs:
container:
args:
Expand Down Expand Up @@ -3869,7 +3869,7 @@ deploymentSpec:
\ 'data_source_csv_filenames',\n 'data_source_bigquery_table_path',\n\
\ ],\n )(\n data_source_csv_filenames,\n data_source_bigquery_table_path,\n\
\ )\n\n"
image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625
image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20241121_0625
exec-split-materialized-data:
container:
args:
Expand Down Expand Up @@ -3915,7 +3915,7 @@ deploymentSpec:
\ 'w') as f:\n f.write(file_patterns[0])\n\n with tf.io.gfile.GFile(materialized_eval_split,\
\ 'w') as f:\n f.write(file_patterns[1])\n\n with tf.io.gfile.GFile(materialized_test_split,\
\ 'w') as f:\n f.write(file_patterns[2])\n\n"
image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625
image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20241121_0625
exec-tabnet-hyperparameter-tuning-job:
container:
args:
Expand Down Expand Up @@ -3943,11 +3943,11 @@ deploymentSpec:
", \"trial_job_spec\": {\"worker_pool_specs\": [{\"replica_count\":\"",
"1", "\", \"machine_spec\": ", "{{$.inputs.parameters[''training_machine_spec'']}}",
", \"disk_spec\": ", "{{$.inputs.parameters[''training_disk_spec'']}}",
", \"container_spec\": {\"image_uri\":\"", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/tabnet-training:20240808_0625",
", \"container_spec\": {\"image_uri\":\"", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/tabnet-training:20241121_0625",
"\", \"args\": [\"--target_column=", "{{$.inputs.parameters[''target_column'']}}",
"\", \"--weight_column=", "{{$.inputs.parameters[''weight_column'']}}",
"\", \"--model_type=", "{{$.inputs.parameters[''prediction_type'']}}", "\",
\"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625",
\"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20241121_0625",
"\", \"--prediction_docker_uri_artifact_path=", "{{$.outputs.parameters[''prediction_docker_uri_output''].output_file}}",
"\", \"--baseline_path=", "{{$.inputs.artifacts[''instance_baseline''].uri}}",
"\", \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\",
Expand Down Expand Up @@ -4016,7 +4016,7 @@ deploymentSpec:
["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}'
- '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat":
["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}'
image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625
image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20241121_0625
pipelineInfo:
description: The TabNet built-in algorithm HyperparameterTuningJob pipeline.
name: automl-tabular-tabnet-hyperparameter-tuning-job
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -165,15 +165,15 @@ def tabnet_trainer(
', "disk_spec": ',
training_disk_spec,
', "container_spec": {"image_uri":"',
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/tabnet-training:20240808_0625',
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/tabnet-training:20241121_0625',
'", "args": ["--target_column=',
target_column,
'", "--weight_column=',
weight_column,
'", "--model_type=',
prediction_type,
'", "--prediction_docker_uri=',
'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625',
'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20241121_0625',
'", "--baseline_path=',
instance_baseline.uri,
'", "--metadata_path=',
Expand Down
Loading

0 comments on commit 8136f76

Please sign in to comment.