From 3c19f0c307a225609cfc3af0cc428e6ced088d01 Mon Sep 17 00:00:00 2001 From: Travis Addair Date: Fri, 3 Mar 2023 14:47:30 -0800 Subject: [PATCH 1/5] Fixed learning_rate_scheduler params in automl --- .../automl/defaults/combiner/tabnet_config.yaml | 4 ++-- tests/integration_tests/test_automl.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/ludwig/automl/defaults/combiner/tabnet_config.yaml b/ludwig/automl/defaults/combiner/tabnet_config.yaml index 7fa67159356..9966e143135 100644 --- a/ludwig/automl/defaults/combiner/tabnet_config.yaml +++ b/ludwig/automl/defaults/combiner/tabnet_config.yaml @@ -16,10 +16,10 @@ hyperopt: trainer.learning_rate: space: choice categories: [0.005, 0.01, 0.02, 0.025] - trainer.decay_rate: + trainer.learning_rate_scheduler.decay_rate: space: choice categories: [0.8, 0.9, 0.95] - trainer.decay_steps: + trainer.learning_rate_scheduler.decay_steps: space: choice categories: [500, 2000, 8000, 10000, 20000] combiner.size: diff --git a/tests/integration_tests/test_automl.py b/tests/integration_tests/test_automl.py index 89089ffaec8..a05f8723804 100644 --- a/tests/integration_tests/test_automl.py +++ b/tests/integration_tests/test_automl.py @@ -8,6 +8,7 @@ import pytest from ludwig.api import LudwigModel +from ludwig.automl.automl import auto_train from ludwig.constants import COLUMN, ENCODER, INPUT_FEATURES, NAME, OUTPUT_FEATURES, PREPROCESSING, SPLIT, TYPE from ludwig.schema.model_types.base import ModelConfig from ludwig.types import FeatureConfigDict, ModelConfigDict @@ -27,6 +28,7 @@ ray = pytest.importorskip("ray") import dask.dataframe as dd # noqa +from ray.tune.experiment.trial import Trial # noqa from ludwig.automl import create_auto_config, create_auto_config_with_dataset_profile, train_with_config # noqa from ludwig.hyperopt.execution import RayTuneExecutor # noqa @@ -299,6 +301,21 @@ def test_train_with_config(time_budget, test_data_tabular_large, ray_cluster_2cp _run_train_with_config(time_budget, test_data_tabular_large, tmpdir) +@pytest.mark.distributed +def test_auto_train(test_data_tabular_large, ray_cluster_2cpu, tmpdir): + _, ofeatures, dataset_csv = test_data_tabular_large + results = auto_train( + dataset=dataset_csv, + target=ofeatures[0][NAME], + time_limit_s=120, + user_config={"hyperopt": {"executor": {"num_samples": 2}}}, + ) + + analysis = results.experiment_analysis + for trial in analysis.trials: + assert trial.status != Trial.ERROR, f"Error in trial {trial}" + + @pytest.mark.parametrize("fs_protocol,bucket", [private_param(("s3", "ludwig-tests"))], ids=["s3"]) def test_train_with_config_remote(fs_protocol, bucket, test_data_tabular_large, ray_cluster_2cpu): backend = { From 2e1f15678db376a9b790899634201070b7b56cd3 Mon Sep 17 00:00:00 2001 From: Travis Addair Date: Fri, 3 Mar 2023 14:49:36 -0800 Subject: [PATCH 2/5] Cleanup --- ludwig/benchmarking/examples/process_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ludwig/benchmarking/examples/process_config.py b/ludwig/benchmarking/examples/process_config.py index a137a63391e..8d4db9cd8ff 100644 --- a/ludwig/benchmarking/examples/process_config.py +++ b/ludwig/benchmarking/examples/process_config.py @@ -36,9 +36,9 @@ def process_config(ludwig_config: dict, experiment_dict: dict) -> dict: "categories": ["fill_with_const", "fill_with_mean"], }, "combiner.type": {"space": "choice", "categories": ["tabnet", "concat"]}, - "trainer.decay": {"space": "choice", "categories": [True, False]}, + "trainer.learning_rate_scheduler.decay": {"space": "choice", "categories": [True, False]}, "trainer.learning_rate": {"space": "loguniform", "lower": 0.0001, "upper": 0.1}, - "trainer.decay_rate": {"space": "uniform", "lower": 0.4, "upper": 0.96}, + "trainer.learning_rate_scheduler.decay_rate": {"space": "uniform", "lower": 0.4, "upper": 0.96}, "trainer.batch_size": {"space": "randint", "lower": 32, "upper": 2048}, }, "search_alg": {"type": "hyperopt"}, From 0b36d85481f0d6d70df0a556188a168d77c19af7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 3 Mar 2023 22:51:37 +0000 Subject: [PATCH 3/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .vscode/settings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index ddcb2bd76b4..fba148f3ce7 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -12,4 +12,4 @@ "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": false, "python.envFile": "${workspaceFolder}/.env" -} \ No newline at end of file +} From a4ec8eb212bfe33b4c5ce6eb4c91df71b8a01c36 Mon Sep 17 00:00:00 2001 From: Travis Addair Date: Fri, 3 Mar 2023 16:31:48 -0800 Subject: [PATCH 4/5] Fixed imports --- tests/integration_tests/test_automl.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/integration_tests/test_automl.py b/tests/integration_tests/test_automl.py index a05f8723804..9c62dbcbc25 100644 --- a/tests/integration_tests/test_automl.py +++ b/tests/integration_tests/test_automl.py @@ -8,7 +8,6 @@ import pytest from ludwig.api import LudwigModel -from ludwig.automl.automl import auto_train from ludwig.constants import COLUMN, ENCODER, INPUT_FEATURES, NAME, OUTPUT_FEATURES, PREPROCESSING, SPLIT, TYPE from ludwig.schema.model_types.base import ModelConfig from ludwig.types import FeatureConfigDict, ModelConfigDict @@ -30,7 +29,12 @@ import dask.dataframe as dd # noqa from ray.tune.experiment.trial import Trial # noqa -from ludwig.automl import create_auto_config, create_auto_config_with_dataset_profile, train_with_config # noqa +from ludwig.automl import ( # noqa + auto_train, + create_auto_config, + create_auto_config_with_dataset_profile, + train_with_config, +) from ludwig.hyperopt.execution import RayTuneExecutor # noqa pytestmark = pytest.mark.distributed From de1c7334191c92c5e1f1cebe174e375d77dad5e4 Mon Sep 17 00:00:00 2001 From: Travis Addair Date: Fri, 3 Mar 2023 20:54:19 -0800 Subject: [PATCH 5/5] Increase test timeout --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index c360420284d..305759027b5 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -59,7 +59,7 @@ jobs: ports: - 9000:9000 - timeout-minutes: 90 + timeout-minutes: 120 steps: - name: Setup ludwigai/ludwig-ray container for local testing with act. if: ${{ env.ACT }}