From f33dfd2a464906ad3cfde8590ec2a9c7ca0c0a79 Mon Sep 17 00:00:00 2001 From: Li Jiang Date: Fri, 2 Aug 2024 03:51:29 +0000 Subject: [PATCH 01/14] add test --- .github/workflows/python-package.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 3a4e2b6c69..fb505701aa 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -85,6 +85,7 @@ jobs: - name: Test with pytest if: matrix.python-version != '3.10' run: | + python test/nlp/test_autohf_classificationhead.py pytest test - name: Coverage if: matrix.python-version == '3.10' From 8d382ad5f6724ee94178656d7b12ffa11c7ef702 Mon Sep 17 00:00:00 2001 From: Li Jiang Date: Fri, 2 Aug 2024 05:14:36 +0000 Subject: [PATCH 02/14] Skip test_autohf_classificationhead.py for MacOS py311 --- .github/workflows/python-package.yml | 1 - test/nlp/test_autohf_classificationhead.py | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index fb505701aa..3a4e2b6c69 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -85,7 +85,6 @@ jobs: - name: Test with pytest if: matrix.python-version != '3.10' run: | - python test/nlp/test_autohf_classificationhead.py pytest test - name: Coverage if: matrix.python-version == '3.10' diff --git a/test/nlp/test_autohf_classificationhead.py b/test/nlp/test_autohf_classificationhead.py index 39d0b89b9f..9ce023f966 100644 --- a/test/nlp/test_autohf_classificationhead.py +++ b/test/nlp/test_autohf_classificationhead.py @@ -21,6 +21,9 @@ "textattack/bert-base-uncased-MNLI", ] +if sys.platform.startswith("darwin") and sys.version_info[0] == 3 and sys.version_info[1] == 11: + pytest.skip("skipping Python 3.11 on MacOS", allow_module_level=True) + def test_switch_1_1(): data_idx, model_path_idx = 0, 0 From 3fdae202fa8bbf46eafa8aec7127df620d65ee56 Mon Sep 17 00:00:00 2001 From: Li Jiang Date: Fri, 2 Aug 2024 05:59:22 +0000 Subject: [PATCH 03/14] Skip test/nlp/test_default.py for MacOS py311 --- test/nlp/test_default.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/nlp/test_default.py b/test/nlp/test_default.py index 7b8b252e25..b3c9016486 100644 --- a/test/nlp/test_default.py +++ b/test/nlp/test_default.py @@ -7,6 +7,9 @@ from flaml.default import portfolio +if sys.platform.startswith("darwin") and sys.version_info[0] == 3 and sys.version_info[1] == 11: + pytest.skip("skipping Python 3.11 on MacOS", allow_module_level=True) + def pop_args(fit_kwargs): fit_kwargs.pop("max_iter", None) From 3120636ff2b78c8e0c46d6ab477ff1001f1af6dd Mon Sep 17 00:00:00 2001 From: Li Jiang Date: Fri, 2 Aug 2024 07:59:41 +0000 Subject: [PATCH 04/14] Check test_tune --- .github/workflows/python-package.yml | 1 + test/tune/test_tune.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 3a4e2b6c69..42411f9ee8 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -85,6 +85,7 @@ jobs: - name: Test with pytest if: matrix.python-version != '3.10' run: | + python test/tune/test_tune.py pytest test - name: Coverage if: matrix.python-version == '3.10' diff --git a/test/tune/test_tune.py b/test/tune/test_tune.py index 658c5a387c..1155177523 100644 --- a/test/tune/test_tune.py +++ b/test/tune/test_tune.py @@ -496,4 +496,8 @@ def _test_xgboost_bohb(): if __name__ == "__main__": + test_nested_run() + test_nested_space() + test_run_training_function_return_value() + test_passing_search_alg() test_xgboost_bs() From 4bf42f353539dc68ec5d7aa9c2b2c85e727b06a0 Mon Sep 17 00:00:00 2001 From: Li Jiang Date: Mon, 5 Aug 2024 04:50:56 +0000 Subject: [PATCH 05/14] Check test_lexiflow --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 42411f9ee8..58c28fb6ad 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -85,7 +85,7 @@ jobs: - name: Test with pytest if: matrix.python-version != '3.10' run: | - python test/tune/test_tune.py + python test/tune/test_lexiflow.py pytest test - name: Coverage if: matrix.python-version == '3.10' From 99b145434e1431ce1e1fc7df9cc5041d27a89012 Mon Sep 17 00:00:00 2001 From: Li Jiang Date: Mon, 5 Aug 2024 05:58:53 +0000 Subject: [PATCH 06/14] Check test_tune --- .github/workflows/python-package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 58c28fb6ad..9c6b4deb35 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -85,8 +85,8 @@ jobs: - name: Test with pytest if: matrix.python-version != '3.10' run: | - python test/tune/test_lexiflow.py - pytest test + python test/tune/test_tune.py + pytest test/tune/test_tune.py - name: Coverage if: matrix.python-version == '3.10' run: | From 4f11b49cf0f985ac3efa5211c774f7945b030474 Mon Sep 17 00:00:00 2001 From: Li Jiang Date: Mon, 5 Aug 2024 06:12:22 +0000 Subject: [PATCH 07/14] Remove checks --- .github/workflows/python-package.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 9c6b4deb35..3a4e2b6c69 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -85,8 +85,7 @@ jobs: - name: Test with pytest if: matrix.python-version != '3.10' run: | - python test/tune/test_tune.py - pytest test/tune/test_tune.py + pytest test - name: Coverage if: matrix.python-version == '3.10' run: | From 9143e50a7b1eb9bd898ad543f367402b3243b936 Mon Sep 17 00:00:00 2001 From: Li Jiang Date: Mon, 5 Aug 2024 08:57:27 +0000 Subject: [PATCH 08/14] Skip test_nested_run for macos py311) --- test/tune/test_tune.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/tune/test_tune.py b/test/tune/test_tune.py index 1155177523..acbbb2b994 100644 --- a/test/tune/test_tune.py +++ b/test/tune/test_tune.py @@ -3,8 +3,10 @@ import logging import math import os +import sys import time +import pytest import sklearn.datasets import sklearn.metrics import xgboost as xgb @@ -17,6 +19,9 @@ except ImportError: print("skip test_xgboost because ray tune cannot be imported.") +# if sys.platform.startswith("darwin") and sys.version_info[0] == 3 and sys.version_info[1] == 11: +# pytest.skip("skipping Python 3.11 on MacOS", allow_module_level=True) + logger = logging.getLogger(__name__) os.makedirs("logs", exist_ok=True) logger.addHandler(logging.FileHandler("logs/tune.log")) @@ -49,6 +54,10 @@ def _easy_objective(config): return {"mean_loss": (0.1 + width * step / 100) ** (-1) + height * 0.1} +@pytest.mark.skipif( + sys.platform.startswith("darwin") and sys.version_info[0] == 3 and sys.version_info[1] == 11, + reason="skipping Python 3.11 on MacOS", +) def test_nested_run(): from flaml import AutoML, tune From e1a9f281c01d37a54c332c45bd364e149caf1daa Mon Sep 17 00:00:00 2001 From: Li Jiang Date: Mon, 5 Aug 2024 10:00:59 +0000 Subject: [PATCH 09/14] Skip test_nested_space for macos py311 --- test/tune/test_tune.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/tune/test_tune.py b/test/tune/test_tune.py index acbbb2b994..fa5db0478b 100644 --- a/test/tune/test_tune.py +++ b/test/tune/test_tune.py @@ -250,6 +250,10 @@ def _test_xgboost(method="BlendSearch"): logger.info(f"Best model parameters: {best_trial.config}") +@pytest.mark.skipif( + sys.platform.startswith("darwin") and sys.version_info[0] == 3 and sys.version_info[1] == 11, + reason="skipping Python 3.11 on MacOS", +) def test_nested_space(): from flaml import CFO, tune From 226926aaa907a2d8e9169f8a7bf3e11379074d61 Mon Sep 17 00:00:00 2001 From: Li Jiang Date: Mon, 5 Aug 2024 19:21:29 +0800 Subject: [PATCH 10/14] Test tune on MacOS Python 3.11 w/o pytest --- .github/workflows/python-package.yml | 4 ++++ test/tune/test_tune.py | 13 +++---------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 3a4e2b6c69..6ee79df248 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -86,6 +86,10 @@ jobs: if: matrix.python-version != '3.10' run: | pytest test + - name: Test tune for MacOS Python 3.11 + if: matrix.python-version == '3.11' && matrix.os == 'macOS-latest' + run: | + python test/tune/test_tune.py - name: Coverage if: matrix.python-version == '3.10' run: | diff --git a/test/tune/test_tune.py b/test/tune/test_tune.py index fa5db0478b..412b39cadd 100644 --- a/test/tune/test_tune.py +++ b/test/tune/test_tune.py @@ -19,8 +19,9 @@ except ImportError: print("skip test_xgboost because ray tune cannot be imported.") -# if sys.platform.startswith("darwin") and sys.version_info[0] == 3 and sys.version_info[1] == 11: -# pytest.skip("skipping Python 3.11 on MacOS", allow_module_level=True) +# TODO: Skip Python 3.11 on MacOS due to pytest segfault in github actions +if sys.platform.startswith("darwin") and sys.version_info[0] == 3 and sys.version_info[1] == 11: + pytest.skip("skipping Python 3.11 on MacOS", allow_module_level=True) logger = logging.getLogger(__name__) os.makedirs("logs", exist_ok=True) @@ -54,10 +55,6 @@ def _easy_objective(config): return {"mean_loss": (0.1 + width * step / 100) ** (-1) + height * 0.1} -@pytest.mark.skipif( - sys.platform.startswith("darwin") and sys.version_info[0] == 3 and sys.version_info[1] == 11, - reason="skipping Python 3.11 on MacOS", -) def test_nested_run(): from flaml import AutoML, tune @@ -250,10 +247,6 @@ def _test_xgboost(method="BlendSearch"): logger.info(f"Best model parameters: {best_trial.config}") -@pytest.mark.skipif( - sys.platform.startswith("darwin") and sys.version_info[0] == 3 and sys.version_info[1] == 11, - reason="skipping Python 3.11 on MacOS", -) def test_nested_space(): from flaml import CFO, tune From 122423a12e972b86dd3c953491368a89a2db6c0e Mon Sep 17 00:00:00 2001 From: Li Jiang Date: Mon, 5 Aug 2024 21:27:19 +0800 Subject: [PATCH 11/14] Split tests by folder --- .github/workflows/python-package.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 6ee79df248..2f6781b946 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -30,7 +30,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-2019] - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.11"] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} @@ -85,7 +85,12 @@ jobs: - name: Test with pytest if: matrix.python-version != '3.10' run: | - pytest test + pytest test/autogen + pytest test/automl + pytest test/default + pytest test/nlp + pytest test/spark + pytest test/tune - name: Test tune for MacOS Python 3.11 if: matrix.python-version == '3.11' && matrix.os == 'macOS-latest' run: | From a5689280137b9bbe6533f7abe2b37085d0dee551 Mon Sep 17 00:00:00 2001 From: Li Jiang Date: Tue, 6 Aug 2024 10:34:24 +0800 Subject: [PATCH 12/14] Skip test lexiflow for MacOS py311 --- test/tune/test_lexiflow.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/tune/test_lexiflow.py b/test/tune/test_lexiflow.py index dda6b924cb..9ee46dc48b 100644 --- a/test/tune/test_lexiflow.py +++ b/test/tune/test_lexiflow.py @@ -1,7 +1,9 @@ import math +import sys from collections import defaultdict import numpy as np +import pytest import thop import torch import torch.nn as nn @@ -19,6 +21,9 @@ N_TRAIN_EXAMPLES = BATCHSIZE * 30 N_VALID_EXAMPLES = BATCHSIZE * 10 +if sys.platform.startswith("darwin") and sys.version_info[0] == 3 and sys.version_info[1] == 11: + pytest.skip("skipping Python 3.11 on MacOS", allow_module_level=True) + def _BraninCurrin(config): # Rescale brain From b5ec1fcd6fc4ecd23dbd8e8ffdb4d032e900b77a Mon Sep 17 00:00:00 2001 From: Li Jiang Date: Tue, 6 Aug 2024 11:06:04 +0800 Subject: [PATCH 13/14] Enable test_tune for MacOS py311 --- test/tune/test_tune.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/tune/test_tune.py b/test/tune/test_tune.py index 412b39cadd..38481ccd00 100644 --- a/test/tune/test_tune.py +++ b/test/tune/test_tune.py @@ -20,8 +20,8 @@ print("skip test_xgboost because ray tune cannot be imported.") # TODO: Skip Python 3.11 on MacOS due to pytest segfault in github actions -if sys.platform.startswith("darwin") and sys.version_info[0] == 3 and sys.version_info[1] == 11: - pytest.skip("skipping Python 3.11 on MacOS", allow_module_level=True) +# if sys.platform.startswith("darwin") and sys.version_info[0] == 3 and sys.version_info[1] == 11: +# pytest.skip("skipping Python 3.11 on MacOS", allow_module_level=True) logger = logging.getLogger(__name__) os.makedirs("logs", exist_ok=True) From e2ebaa30f62db98fdb3bf0fcabaef581deeede0f Mon Sep 17 00:00:00 2001 From: Li Jiang Date: Tue, 6 Aug 2024 11:47:03 +0800 Subject: [PATCH 14/14] Clean up --- .github/workflows/python-package.yml | 13 ++----------- test/tune/test_tune.py | 3 --- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 2f6781b946..3a4e2b6c69 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -30,7 +30,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-2019] - python-version: ["3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} @@ -85,16 +85,7 @@ jobs: - name: Test with pytest if: matrix.python-version != '3.10' run: | - pytest test/autogen - pytest test/automl - pytest test/default - pytest test/nlp - pytest test/spark - pytest test/tune - - name: Test tune for MacOS Python 3.11 - if: matrix.python-version == '3.11' && matrix.os == 'macOS-latest' - run: | - python test/tune/test_tune.py + pytest test - name: Coverage if: matrix.python-version == '3.10' run: | diff --git a/test/tune/test_tune.py b/test/tune/test_tune.py index 38481ccd00..5f17272dc5 100644 --- a/test/tune/test_tune.py +++ b/test/tune/test_tune.py @@ -19,9 +19,6 @@ except ImportError: print("skip test_xgboost because ray tune cannot be imported.") -# TODO: Skip Python 3.11 on MacOS due to pytest segfault in github actions -# if sys.platform.startswith("darwin") and sys.version_info[0] == 3 and sys.version_info[1] == 11: -# pytest.skip("skipping Python 3.11 on MacOS", allow_module_level=True) logger = logging.getLogger(__name__) os.makedirs("logs", exist_ok=True)