diff --git a/tests/hyfi/dotenv/test_dotenv.py b/tests/hyfi/dotenv/test_dotenv.py index 5813eb26..5de7ce4e 100644 --- a/tests/hyfi/dotenv/test_dotenv.py +++ b/tests/hyfi/dotenv/test_dotenv.py @@ -8,7 +8,8 @@ def test_dotenv_config(): os.environ["HYFI_PROJECT_NAME"] = "hyfi" os.environ["HYFI_GLOBAL_ROOT"] = ENVs.expand_posix_vars("$WORKSPACE_ROOT") config = DotEnvConfig() - pprint(config.dict()) + print(type(config)) + pprint(config.model_dump()) # Test that the default values are set correctly assert config._config_name_ == "__init__" assert config.HYFI_PROJECT_NAME == "hyfi" diff --git a/tests/hyfi/main/test_main.py b/tests/hyfi/main/test_main.py index 2dac0681..cec55672 100644 --- a/tests/hyfi/main/test_main.py +++ b/tests/hyfi/main/test_main.py @@ -6,7 +6,7 @@ def test_about(): print(HyFI.about()) - assert HyFI.config.__version__ == __version__ + assert HyFI.config._version_ == __version__ def test_version(): @@ -16,7 +16,7 @@ def test_version(): def test_envs(): os.environ["HYFI_PROJECT_NAME"] = "hyfi" envs = HyFI.dotenv() - pprint(envs.dict()) + pprint(envs.model_dump()) assert envs.HYFI_PROJECT_NAME == "hyfi" @@ -29,7 +29,7 @@ def test_init_project(): log_level="WARNING", verbose=True, ) - pprint(prj.dict()) + pprint(prj.model_dump()) assert prj.path.global_workspace_name == "testspace" @@ -37,7 +37,7 @@ def test_compose(): os.environ["HYFI_PROJECT_ROOT"] = "." cfg = HyFI.compose("path=__task__") print(cfg["task_root"]) - assert cfg["task_root"] == "workspace/tasks/task1" + assert cfg["task_root"] == "workspace/tasks" if __name__ == "__main__": diff --git a/tests/hyfi/path/test_batch_path.py b/tests/hyfi/path/test_batch_path.py index 0d575531..089009e2 100644 --- a/tests/hyfi/path/test_batch_path.py +++ b/tests/hyfi/path/test_batch_path.py @@ -4,13 +4,18 @@ def test_path_config(): - config = BatchPathConfig(task_root="workspace/test_task") - pprint(config.dict()) + config = BatchPathConfig( + task_root="workspace/tasks", + task_name="test-task", + ) + pprint(config.model_dump()) # Test that the default values are set correctly assert config._config_name_ == "__batch__" + config.batch_name = "test-batch" print(config.batch_dir) assert ( - config.batch_dir == Path("workspace/test_task/outputs/batch-outputs").absolute() + config.batch_dir + == Path("workspace/tasks/test-task/outputs/test-batch") ) # Test that the log_dir is created diff --git a/tests/hyfi/path/test_path.py b/tests/hyfi/path/test_path.py index 860944b9..816581c1 100644 --- a/tests/hyfi/path/test_path.py +++ b/tests/hyfi/path/test_path.py @@ -6,19 +6,20 @@ def test_path_config(): config = PathConfig( + _config_name_="__init__", project_root="workspace/tmp", global_hyfi_root=ENVs.expand_posix_vars("$HOME/.hyfi"), project_workspace_name="testspace", ) - pprint(config.dict()) + pprint(config.model_dump()) # Test that the default values are set correctly assert config._config_name_ == "__init__" assert config.home == ENVs.expand_posix_vars("$HOME") - assert Path(config.project_root).absolute() == (Path.cwd() / "workspace/tmp").absolute() assert ( - Path(config.project_workspace_root).absolute() - == Path.cwd().absolute() / "workspace/tmp/testspace" + Path(config.project_root).absolute() + == (Path.cwd() / "workspace/tmp").absolute() ) + assert config.workspace_dir == Path("workspace/tmp/testspace") # Test that the log_dir and cache_dir properties return the correct values assert Path(config.log_dir).is_dir() diff --git a/tests/hyfi/pipeline/test_pipe.py b/tests/hyfi/pipeline/test_pipe.py index 5b21241d..846750c5 100644 --- a/tests/hyfi/pipeline/test_pipe.py +++ b/tests/hyfi/pipeline/test_pipe.py @@ -27,14 +27,14 @@ def test_pipe(): print(df3) config = HyFI.compose("pipe=__dataframe_instance_methods__") pipe_config = DataframePipeConfig(**config) - pipe_config._run_ = "filter" - pipe_config._with_ = {"items": ["id"]} + pipe_config.run = "filter" + pipe_config.run_with = {"items": ["id"]} print(pipe_config) df4 = HyFI.run_pipe(df, pipe_config) print(df4) config = HyFI.compose("pipe=__dataframe_external_funcs__") pipe_config = DataframePipeConfig(**config) - pipe_config._run_ = "lambda x: x.replace('Economic', 'ECON')" + pipe_config.run = "lambda x: x.replace('Economic', 'ECON')" pipe_config.columns_to_apply = "text" pipe_config.num_workers = 5 pipe_config.verbose = True diff --git a/tests/hyfi/pipeline/test_pipeline.py b/tests/hyfi/pipeline/test_pipeline.py index 7405dbfe..c3ea247e 100644 --- a/tests/hyfi/pipeline/test_pipeline.py +++ b/tests/hyfi/pipeline/test_pipeline.py @@ -1,26 +1,33 @@ import pandas as pd +from hyfi.composer import Composer from hyfi.main import HyFI from hyfi.pipeline import PipelineConfig from hyfi.pipeline.configs import PipeConfig, RunningConfig def test_running_config(): + assert Composer.generate_alias_for_special_keys("_run_") == "run" + assert Composer.generate_alias_for_special_keys("_pipe_") == "pipe_target" + assert Composer.generate_alias_for_special_keys("_with_") == "run_with" + assert Composer.generate_alias_for_special_keys("with") == "run_with" + config = RunningConfig(**{"with": {"a": 1, "b": 2}}) - print(config.dict()) + print(config.model_dump()) + print(config.kwargs) assert config.kwargs == {"a": 1, "b": 2} def test_pipe(): data_path = "https://assets.entelecheia.ai/datasets/bok_minutes/meta-bok_minutes-train.parquet" config = HyFI.compose("pipe=load_dataframes") - config._with_ = {"data_files": data_path} + config.run_with = {"data_files": data_path} config.verbose = True HyFI.print(config) pipe = PipeConfig(**config) - HyFI.print(pipe.dict()) + HyFI.print(pipe.model_dump()) print(pipe.kwargs) - assert pipe._run_ == "hyfi.main.HyFI.load_dataframes" + assert pipe.run == "hyfi.main.HyFI.load_dataframes" def test_pipeline(): @@ -28,9 +35,10 @@ def test_pipeline(): "https://assets.entelecheia.ai/datasets/esg_coverage/ESG_ratings_raw.csv" ) config = HyFI.compose("pipeline=__test_dataframe__") - config.pipe1._with_ = {"data_files": data_path} + config.pipe1.run_with = {"data_files": data_path} + # HyFI.print(config) config = PipelineConfig(**config) - HyFI.print(config.dict(exclude_none=True)) + HyFI.print(config.model_dump(exclude_none=True)) data = HyFI.run_pipeline(config) assert type(data) == pd.DataFrame print(data[data.code == "A005930"]) diff --git a/tests/hyfi/pipeline/test_task_pipelines.py b/tests/hyfi/pipeline/test_task_pipelines.py index ad77bf57..784bee6a 100644 --- a/tests/hyfi/pipeline/test_task_pipelines.py +++ b/tests/hyfi/pipeline/test_task_pipelines.py @@ -9,9 +9,9 @@ def test_task_config(): ) task = TaskConfig(_config_name_="__test__") - HyFI.print(task.dict()) + HyFI.print(task.model_dump()) HyFI.run_task(task, project=project) - HyFI.print(task.dict()) + HyFI.print(task.model_dump()) if __name__ == "__main__": diff --git a/tests/hyfi/project/test_project.py b/tests/hyfi/project/test_project.py index 5595d694..7ea52aca 100644 --- a/tests/hyfi/project/test_project.py +++ b/tests/hyfi/project/test_project.py @@ -5,14 +5,17 @@ def test_project_config(): - os.environ["HYFI_PROJECT_NAME"] = "hyfi" - config = ProjectConfig(project_name="test", project_root="./workspace/tmp", num_workers=2) + os.environ["HYFI_PROJECT_NAME"] = "hyfi-test" + config = ProjectConfig( + project_root="./workspace/tmp", num_workers=2 + ) # config.init_project() - pprint(config.dict()) - assert config.project_name == "test" - assert Path(config.project_root).absolute() == (Path.cwd() / "workspace/tmp").absolute() + pprint(config.model_dump()) + assert config.project_name == "hyfi-test" + print(config.workspace_dir) + assert config.workspace_dir == Path("workspace/tmp/workspace") assert config.num_workers == 2 - # pprint(config.path.dict()) + # pprint(config.path.model_dump()) if __name__ == "__main__": diff --git a/tests/hyfi/task/test_batch_task.py b/tests/hyfi/task/test_batch_task.py index 5ac945f8..9d82ea52 100644 --- a/tests/hyfi/task/test_batch_task.py +++ b/tests/hyfi/task/test_batch_task.py @@ -10,7 +10,7 @@ def test_batch_task_config(): task_name="demo2", batch_name="batch11", ) - pprint(config.batch.dict()) + pprint(config.batch.model_dump()) # Test that the default values are set correctly assert config.batch.batch_name == "batch11" diff --git a/tests/hyfi/task/test_task.py b/tests/hyfi/task/test_task.py index 96f15c04..4b1c6be1 100644 --- a/tests/hyfi/task/test_task.py +++ b/tests/hyfi/task/test_task.py @@ -7,7 +7,7 @@ def test_task_config(): config = TaskConfig( task_name="demo2", ) - pprint(config.dict()) + pprint(config.model_dump()) # Test that the default values are set correctly assert config.task_name == "demo2" @@ -15,6 +15,16 @@ def test_task_config(): assert Path(config.log_dir).is_dir() assert Path(config.cache_dir).is_dir() + config.task_name = "demo3" + pprint(config.model_dump()) + # Test that the default values are set correctly + print(config.task_dir) + assert config.task_dir == Path("workspace/tasks/demo3") + config.task_root = "workspace/tmp" + pprint(config.model_dump()) + print(config.root_dir) + assert config.task_dir == Path("workspace/tmp/demo3") + if __name__ == "__main__": test_task_config() diff --git a/tests/notebooks/test_HyFI.ipynb b/tests/notebooks/test_HyFI.ipynb index ddb50545..9ecd7c98 100644 --- a/tests/notebooks/test_HyFI.ipynb +++ b/tests/notebooks/test_HyFI.ipynb @@ -14,22 +14,46 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 12, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "time: 365 µs (started: 2023-07-03 04:13:21 +09:00)\n" + ] + } + ], "source": [ "from hyfi.main import HyFI" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ + "DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): assets.entelecheia.ai:443\n", + "DEBUG:urllib3.connectionpool:https://assets.entelecheia.ai:443 \"GET /logo-square-512.png HTTP/1.1\" 200 28078\n", + "DEBUG:PIL.PngImagePlugin:STREAM b'IHDR' 16 13\n", + "DEBUG:PIL.PngImagePlugin:STREAM b'sRGB' 41 1\n", + "DEBUG:PIL.PngImagePlugin:STREAM b'eXIf' 54 56\n", + "DEBUG:PIL.PngImagePlugin:STREAM b'iDOT' 122 28\n", + "DEBUG:PIL.PngImagePlugin:b'iDOT' 122 28 (unknown)\n", + "DEBUG:PIL.PngImagePlugin:STREAM b'IDAT' 162 13602\n", + "DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): assets.entelecheia.ai:443\n", + "DEBUG:urllib3.connectionpool:https://assets.entelecheia.ai:443 \"GET /logo-circle-512.png HTTP/1.1\" 200 33897\n", + "DEBUG:PIL.PngImagePlugin:STREAM b'IHDR' 16 13\n", + "DEBUG:PIL.PngImagePlugin:STREAM b'sRGB' 41 1\n", + "DEBUG:PIL.PngImagePlugin:STREAM b'eXIf' 54 56\n", + "DEBUG:PIL.PngImagePlugin:STREAM b'iDOT' 122 28\n", + "DEBUG:PIL.PngImagePlugin:b'iDOT' 122 28 (unknown)\n", + "DEBUG:PIL.PngImagePlugin:STREAM b'IDAT' 162 16384\n", "INFO:hyfi.graphics.collage:Creating collage of 2 images with 2 columns from 2 images\n" ] }, @@ -40,9 +64,16 @@ "" ] }, - "execution_count": 2, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "time: 183 ms (started: 2023-07-03 04:13:21 +09:00)\n" + ] } ], "source": [ @@ -58,7 +89,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -86,9 +117,16 @@ "'test.gif'" ] }, - "execution_count": 3, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "time: 5.06 ms (started: 2023-07-03 04:13:21 +09:00)\n" + ] } ], "source": [ @@ -100,7 +138,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 15, "metadata": {}, "outputs": [ { @@ -113,9 +151,10 @@ "description : Hydra Fast Interface (Hydra and Pydantic based interface framework)\n", "homepage : https://hyfi.entelecheia.ai\n", "license : MIT\n", - "version : 0.14.2\n", + "version : 0.16.3\n", "\n", - "Execute `hyfi --help` to see what you can do with HyFI\n" + "Execute `hyfi --help` to see what you can do with HyFI\n", + "time: 22.4 ms (started: 2023-07-03 04:13:21 +09:00)\n" ] } ], @@ -135,7 +174,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 16, "metadata": { "id": "BzxPwsOVPWef" }, @@ -144,7 +183,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "HyFI version: 0.14.2\n" + "HyFI version: 0.16.3\n", + "time: 1.82 ms (started: 2023-07-03 04:13:21 +09:00)\n" ] } ], @@ -155,58 +195,65 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'config_name': '__init__',\n", - " 'DOTENV_FILENAME': '.env',\n", - " 'DOTENV_DIR': '/mnt/nvme1n1p2/home/yj.lee/workspace/projects/hyfi',\n", - " 'DOTENV_PATH': '/mnt/nvme1n1p2/home/yj.lee/workspace/projects/hyfi/.env',\n", + "{'DOTENV_FILENAME': '.env',\n", + " 'DOTENV_DIR': '',\n", + " 'DOTENV_PATH': '',\n", " 'HYFI_RESOURCE_DIR': '',\n", - " 'HYFI_GLOBAL_ROOT': '',\n", + " 'HYFI_GLOBAL_ROOT': '/home/yj.lee/.hyfi',\n", " 'HYFI_GLOBAL_WORKSPACE_NAME': 'workspace',\n", - " 'HYFI_PROJECT_NAME': 'test-project',\n", + " 'HYFI_PROJECT_NAME': 'hyfi',\n", " 'HYFI_PROJECT_DESC': '',\n", - " 'HYFI_PROJECT_ROOT': '',\n", + " 'HYFI_PROJECT_ROOT': '/home/yj.lee/.hyfi/workspace/projects/hyfi',\n", " 'HYFI_PROJECT_WORKSPACE_NAME': 'workspace',\n", - " 'HYFI_LOG_LEVEL': 'INFO',\n", - " 'HYFI_VERBOSE': False,\n", - " 'HYFI_NUM_WORKERS': 1,\n", - " 'CACHED_PATH_CACHE_ROOT': '',\n", + " 'HYFI_LOG_LEVEL': 'DEBUG',\n", + " 'HYFI_VERBOSE': 'False',\n", + " 'HYFI_NUM_WORKERS': -1,\n", + " 'CACHED_PATH_CACHE_ROOT': '/home/yj.lee/.hyfi/workspace/projects/hyfi/workspace/.cache/cached_path',\n", " 'CUDA_DEVICE_ORDER': 'PCI_BUS_ID',\n", " 'CUDA_VISIBLE_DEVICES': '',\n", - " 'WANDB_PROJECT': '',\n", + " 'WANDB_PROJECT': 'hyfi',\n", " 'WANDB_DISABLED': '',\n", - " 'WANDB_DIR': '',\n", - " 'WANDB_NOTEBOOK_NAME': '',\n", - " 'WANDB_SILENT': False,\n", + " 'WANDB_DIR': '/home/yj.lee/.hyfi/workspace/projects/hyfi/workspace/logs',\n", + " 'WANDB_NOTEBOOK_NAME': '/home/yj.lee/.hyfi/workspace/projects/hyfi/workspace/logs/hyfi-nb',\n", + " 'WANDB_SILENT': 'True',\n", " 'LABEL_STUDIO_SERVER': '',\n", " 'KMP_DUPLICATE_LIB_OK': 'True',\n", - " 'TOKENIZERS_PARALLELISM': False}" + " 'TOKENIZERS_PARALLELISM': 'False'}" ] }, - "execution_count": 6, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "time: 15.3 ms (started: 2023-07-03 04:13:21 +09:00)\n" + ] } ], "source": [ - "HyFI.dotenv().dict()" + "HyFI.dotenv().model_dump()" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 18, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "The system workspace root is /mnt/nvme1n1p2/home/yj.lee/workspace and the user is yj.lee.\n" + "The system workspace root is /mnt/nvme1n1p2/home/yj.lee/workspace and the user is yj.lee.\n", + "time: 10.4 ms (started: 2023-07-03 04:13:21 +09:00)\n" ] } ], @@ -244,7 +291,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -252,25 +299,20 @@ "output_type": "stream", "text": [ "INFO:hyfi.utils.notebooks:Google Colab not detected.\n", - "DEBUG:matplotlib.pyplot:Loaded backend module://matplotlib_inline.backend_inline version unknown.\n", "DEBUG:hyfi.__global__.config:HyFiConfig initialized with hyfi_config_module=hyfi.conf, hyfi_config_path=pkg://hyfi.conf, hyfi_user_config_path=\n", - "DEBUG:hydra.core.utils:Setting JobRuntime:name=UNKNOWN_NAME\n", + "INFO:hyfi.composer:Composing `ProjectConfig` class with `__init__` config in `project` group.\n", + "DEBUG:hydra.core.utils:Setting JobRuntime:name=app\n", "DEBUG:hydra.core.utils:Setting JobRuntime:name=app\n", "DEBUG:hydra.core.utils:Setting JobRuntime:name=app\n", - "DEBUG:hydra.core.utils:Setting JobRuntime:name=app\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ "DEBUG:hydra.core.utils:Setting JobRuntime:name=app\n", + "INFO:hyfi.composer:Composing `JobLibConfig` class with `__init__` config in `joblib` group.\n", "DEBUG:hydra.core.utils:Setting JobRuntime:name=app\n", "DEBUG:hydra.core.utils:Setting JobRuntime:name=app\n", "DEBUG:hydra.core.utils:Setting JobRuntime:name=app\n", "DEBUG:hydra.core.utils:Setting JobRuntime:name=app\n", "DEBUG:hydra.core.utils:Setting JobRuntime:name=app\n", "DEBUG:hydra.core.utils:Setting JobRuntime:name=app\n", + "INFO:hyfi.composer:Composing `PathConfig` class with `__init__` config in `path` group.\n", "DEBUG:hydra.core.utils:Setting JobRuntime:name=app\n", "DEBUG:hydra.core.utils:Setting JobRuntime:name=app\n", "DEBUG:hydra.core.utils:Setting JobRuntime:name=app\n", @@ -283,9 +325,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "project_dir: /home/yj.lee/.hyfi/workspace/projects/hyfi\n", - "project_workspace_dir: /home/yj.lee/.hyfi/workspace/projects/hyfi/workspace\n", - "time: 1.21 s (started: 2023-06-27 10:21:40 +09:00)\n" + "Project directory: /home/yj.lee/.hyfi/workspace/projects/hyfi\n", + "Workspace directory: /home/yj.lee/.hyfi/workspace/projects/hyfi/workspace\n", + "time: 898 ms (started: 2023-07-03 04:13:21 +09:00)\n" ] } ], @@ -299,22 +341,21 @@ " verbose=True,\n", ")\n", "\n", - "print(\"project_dir:\", prj.project_dir)\n", - "print(\"project_workspace_dir:\", prj.project_workspace_dir)" + "print(\"Project directory:\", prj.root_dir)\n", + "print(\"Workspace directory:\", prj.workspace_dir)" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 20, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'config_name': '__init__',\n", - " 'DOTENV_FILENAME': '.env',\n", - " 'DOTENV_DIR': '/mnt/nvme1n1p2/home/yj.lee/workspace/projects/hyfi',\n", - " 'DOTENV_PATH': '/mnt/nvme1n1p2/home/yj.lee/workspace/projects/hyfi/.env',\n", + "{'DOTENV_FILENAME': '.env',\n", + " 'DOTENV_DIR': '',\n", + " 'DOTENV_PATH': '',\n", " 'HYFI_RESOURCE_DIR': '',\n", " 'HYFI_GLOBAL_ROOT': '/home/yj.lee/.hyfi',\n", " 'HYFI_GLOBAL_WORKSPACE_NAME': 'workspace',\n", @@ -325,20 +366,20 @@ " 'HYFI_LOG_LEVEL': 'DEBUG',\n", " 'HYFI_VERBOSE': False,\n", " 'HYFI_NUM_WORKERS': -1,\n", - " 'CACHED_PATH_CACHE_ROOT': '/home/yj.lee/.hyfi/workspace/.cache/cached_path',\n", + " 'CACHED_PATH_CACHE_ROOT': '/home/yj.lee/.hyfi/workspace/projects/hyfi/workspace/.cache/cached_path',\n", " 'CUDA_DEVICE_ORDER': 'PCI_BUS_ID',\n", " 'CUDA_VISIBLE_DEVICES': '',\n", " 'WANDB_PROJECT': 'hyfi',\n", " 'WANDB_DISABLED': '',\n", " 'WANDB_DIR': '/home/yj.lee/.hyfi/workspace/projects/hyfi/workspace/logs',\n", " 'WANDB_NOTEBOOK_NAME': '/home/yj.lee/.hyfi/workspace/projects/hyfi/workspace/logs/hyfi-nb',\n", - " 'WANDB_SILENT': False,\n", + " 'WANDB_SILENT': 'True',\n", " 'LABEL_STUDIO_SERVER': '',\n", " 'KMP_DUPLICATE_LIB_OK': 'True',\n", - " 'TOKENIZERS_PARALLELISM': False}" + " 'TOKENIZERS_PARALLELISM': 'False'}" ] }, - "execution_count": 9, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" }, @@ -346,26 +387,25 @@ "name": "stdout", "output_type": "stream", "text": [ - "time: 2.23 ms (started: 2023-06-27 10:21:41 +09:00)\n" + "time: 1.84 ms (started: 2023-07-03 04:13:22 +09:00)\n" ] } ], "source": [ - "prj.dotenv.dict()" + "prj.dotenv.model_dump()" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 21, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'config_name': '__init__',\n", - " 'DOTENV_FILENAME': '.env',\n", - " 'DOTENV_DIR': '/mnt/nvme1n1p2/home/yj.lee/workspace/projects/hyfi',\n", - " 'DOTENV_PATH': '/mnt/nvme1n1p2/home/yj.lee/workspace/projects/hyfi/.env',\n", + "{'DOTENV_FILENAME': '.env',\n", + " 'DOTENV_DIR': '',\n", + " 'DOTENV_PATH': '',\n", " 'HYFI_RESOURCE_DIR': '',\n", " 'HYFI_GLOBAL_ROOT': '/home/yj.lee/.hyfi',\n", " 'HYFI_GLOBAL_WORKSPACE_NAME': 'workspace',\n", @@ -374,22 +414,22 @@ " 'HYFI_PROJECT_ROOT': '/home/yj.lee/.hyfi/workspace/projects/hyfi',\n", " 'HYFI_PROJECT_WORKSPACE_NAME': 'workspace',\n", " 'HYFI_LOG_LEVEL': 'DEBUG',\n", - " 'HYFI_VERBOSE': False,\n", + " 'HYFI_VERBOSE': 'False',\n", " 'HYFI_NUM_WORKERS': -1,\n", - " 'CACHED_PATH_CACHE_ROOT': '/home/yj.lee/.hyfi/workspace/.cache/cached_path',\n", + " 'CACHED_PATH_CACHE_ROOT': '/home/yj.lee/.hyfi/workspace/projects/hyfi/workspace/.cache/cached_path',\n", " 'CUDA_DEVICE_ORDER': 'PCI_BUS_ID',\n", " 'CUDA_VISIBLE_DEVICES': '',\n", " 'WANDB_PROJECT': 'hyfi',\n", " 'WANDB_DISABLED': '',\n", " 'WANDB_DIR': '/home/yj.lee/.hyfi/workspace/projects/hyfi/workspace/logs',\n", " 'WANDB_NOTEBOOK_NAME': '/home/yj.lee/.hyfi/workspace/projects/hyfi/workspace/logs/hyfi-nb',\n", - " 'WANDB_SILENT': False,\n", + " 'WANDB_SILENT': 'True',\n", " 'LABEL_STUDIO_SERVER': '',\n", " 'KMP_DUPLICATE_LIB_OK': 'True',\n", - " 'TOKENIZERS_PARALLELISM': False}" + " 'TOKENIZERS_PARALLELISM': 'False'}" ] }, - "execution_count": 10, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" }, @@ -397,12 +437,12 @@ "name": "stdout", "output_type": "stream", "text": [ - "time: 13.3 ms (started: 2023-06-27 10:21:41 +09:00)\n" + "time: 18.5 ms (started: 2023-07-03 04:13:22 +09:00)\n" ] } ], "source": [ - "HyFI.dotenv().dict()" + "HyFI.dotenv().model_dump()" ] }, { @@ -417,7 +457,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 22, "metadata": {}, "outputs": [ { @@ -434,7 +474,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "time: 183 ms (started: 2023-06-27 10:21:41 +09:00)\n" + "time: 200 ms (started: 2023-07-03 04:13:22 +09:00)\n" ] } ], @@ -454,22 +494,26 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 23, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "{'task_cache': 'workspace/tasks/task1/cache',\n", - " 'task_datasets': 'workspace/tasks/task1/datasets',\n", - " 'task_library': 'workspace/tasks/task1/libs',\n", - " 'task_log': 'workspace/tasks/task1/logs',\n", - " 'task_models': 'workspace/tasks/task1/models',\n", - " 'task_outputs': 'workspace/tasks/task1/outputs',\n", - " 'task_root': 'workspace/tasks/task1',\n", - " 'task_tmp': 'workspace/tasks/task1/tmp'}\n", - "time: 22.2 ms (started: 2023-06-27 10:21:41 +09:00)\n" + "{'dirnames': {'archive': 'archive',\n", + " 'cache': '.cache',\n", + " 'datasets': 'datasets',\n", + " 'inputs': 'inputs',\n", + " 'library': 'libs',\n", + " 'logs': 'logs',\n", + " 'models': 'models',\n", + " 'modules': 'modules',\n", + " 'outputs': 'outputs',\n", + " 'tmp': 'tmp'},\n", + " 'task_name': 'demo-task',\n", + " 'task_root': 'workspace/tasks'}\n", + "time: 21.1 ms (started: 2023-07-03 04:13:22 +09:00)\n" ] } ], diff --git a/tests/notebooks/test_widgets.ipynb b/tests/notebooks/test_widgets.ipynb index 4c5554a0..1f984018 100644 --- a/tests/notebooks/test_widgets.ipynb +++ b/tests/notebooks/test_widgets.ipynb @@ -14,16 +14,16 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ - "from hyfi.main import HyFI" + "from hyfi.main import HyFI\n" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -44,18 +44,18 @@ " style={\"description_width\": \"50px\"},\n", " layout={\"width\": \"95%\", \"height\": \"100px\"},\n", ")\n", - "translate_button = HyFI.create_button(\"Translate\", layout={\"width\": \"95%\"})" + "translate_button = HyFI.create_button(\"Translate\", layout={\"width\": \"95%\"})\n" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "1f68393daed54e118c01bb7e42ee5d19", + "model_id": "104f7f3de553467c982d73f51c24e771", "version_major": 2, "version_minor": 0 }, @@ -63,7 +63,7 @@ "GridspecLayout(children=(Dropdown(description='From Language', layout=Layout(grid_area='widget001', width='aut…" ] }, - "execution_count": 4, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -77,12 +77,12 @@ "grid[1, :] = input_prompt\n", "grid[2, :] = generated_txt\n", "grid[3, :] = translate_button\n", - "grid" + "grid\n" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -105,18 +105,18 @@ " style={\"description_width\": \"50px\"},\n", " layout={\"width\": \"95%\", \"height\": \"150px\"},\n", ")\n", - "generate_button = HyFI.create_button(\"Generate SQL\", layout={\"width\": \"95%\"})" + "generate_button = HyFI.create_button(\"Generate SQL\", layout={\"width\": \"95%\"})\n" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "b098901c2c26407da7d6491384a332d7", + "model_id": "2b563554b923423ca21bec26ecc53e23", "version_major": 2, "version_minor": 0 }, @@ -124,7 +124,7 @@ "GridspecLayout(children=(Textarea(value=\"Instruction: Given an input question, respond with syntactically corr…" ] }, - "execution_count": 6, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -136,7 +136,7 @@ "grid[0, 0] = input_prompt\n", "grid[1, 0] = generated_txt\n", "grid[2, 0] = generate_button\n", - "grid" + "grid\n" ] }, { diff --git a/tests/notebooks/test_workflows.ipynb b/tests/notebooks/test_workflows.ipynb index c04d2d7e..b076b86e 100644 --- a/tests/notebooks/test_workflows.ipynb +++ b/tests/notebooks/test_workflows.ipynb @@ -6,38 +6,38 @@ "metadata": {}, "outputs": [], "source": [ - "from hyfi.main import HyFI" + "from hyfi.main import HyFI\n" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "{'job1': {'step1': {'name': None, 'uses': None, 'with': None},\n", - " 'step2': {'name': None, 'uses': None, 'with': None}}}\n" + "job1\n" ] } ], "source": [ "config = HyFI.compose(\"jobs=job1\")\n", - "HyFI.print(config)" + "HyFI.print(config)\n" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "{'project': {'dotenv': {'CACHED_PATH_CACHE_ROOT': None,\n", + "{'project': {'_config_name_': '__init__',\n", + " 'dotenv': {'CACHED_PATH_CACHE_ROOT': None,\n", " 'CUDA_DEVICE_ORDER': None,\n", " 'CUDA_VISIBLE_DEVICES': None,\n", " 'DOTENV_DIR': '/mnt/nvme1n1p2/home/yj.lee/workspace/projects/hyfi/tests/notebooks',\n", @@ -56,7 +56,6 @@ " 'HYFI_PROJECT_ROOT': None,\n", " 'HYFI_PROJECT_WORKSPACE_NAME': None,\n", " 'HYFI_RESOURCE_DIR': None,\n", - " 'HYFI_TASK_NAME': None,\n", " 'HYFI_VERBOSE': False,\n", " 'KMP_DUPLICATE_LIB_OK': None,\n", " 'LABEL_STUDIO_SERVER': None,\n", @@ -82,129 +81,32 @@ " 'num_workers': '1'},\n", " 'num_workers': '1'},\n", " 'num_workers': '1',\n", - " 'path': {'global_archive': '/home/yj.lee/.hyfi/workspace/archive',\n", - " 'global_cache': '/home/yj.lee/.hyfi/workspace/.cache',\n", - " 'global_datasets': '/home/yj.lee/.hyfi/workspace/datasets',\n", + " 'path': {'dirnames': {'archive': 'archive',\n", + " 'cache': '.cache',\n", + " 'datasets': 'datasets',\n", + " 'inputs': 'inputs',\n", + " 'library': 'libs',\n", + " 'logs': 'logs',\n", + " 'models': 'models',\n", + " 'modules': 'modules',\n", + " 'outputs': 'outputs',\n", + " 'tmp': 'tmp'},\n", " 'global_hyfi_root': '/home/yj.lee/.hyfi',\n", - " 'global_library': '/home/yj.lee/.hyfi/workspace/libs',\n", - " 'global_logs': '/home/yj.lee/.hyfi/workspace/logs',\n", - " 'global_models': '/home/yj.lee/.hyfi/workspace/models',\n", - " 'global_modules': '/home/yj.lee/.hyfi/workspace/modules',\n", - " 'global_tmp': '/home/yj.lee/.hyfi/workspace/.tmp',\n", " 'global_workspace_name': 'workspace',\n", - " 'global_workspace_root': '/home/yj.lee/.hyfi/workspace',\n", " 'home': '/home/yj.lee',\n", " 'hyfi': '/mnt/nvme1n1p2/home/yj.lee/workspace/projects/hyfi/src/hyfi',\n", - " 'project_archive': '/home/yj.lee/.hyfi/workspace/projects/test-project/workspace/archive',\n", - " 'project_cache': '/home/yj.lee/.hyfi/workspace/projects/test-project/workspace/.cache',\n", - " 'project_datasets': '/home/yj.lee/.hyfi/workspace/projects/test-project/workspace/datasets',\n", - " 'project_library': '/home/yj.lee/.hyfi/workspace/projects/test-project/workspace/libs',\n", - " 'project_logs': '/home/yj.lee/.hyfi/workspace/projects/test-project/workspace/logs',\n", - " 'project_models': '/home/yj.lee/.hyfi/workspace/projects/test-project/workspace/models',\n", - " 'project_modules': '/home/yj.lee/.hyfi/workspace/projects/test-project/workspace/modules',\n", - " 'project_outputs': '/home/yj.lee/.hyfi/workspace/projects/test-project/workspace/outputs',\n", " 'project_root': '/home/yj.lee/.hyfi/workspace/projects/test-project',\n", - " 'project_tmp': '/home/yj.lee/.hyfi/workspace/projects/test-project/workspace/.tmp',\n", " 'project_workspace_name': 'workspace',\n", - " 'project_workspace_root': '/home/yj.lee/.hyfi/workspace/projects/test-project/workspace',\n", " 'resources': '/mnt/nvme1n1p2/home/yj.lee/workspace/projects/hyfi/src/hyfi/resources',\n", " 'runtime': '/mnt/nvme1n1p2/home/yj.lee/workspace/projects/hyfi/tests/notebooks'},\n", " 'project_description': '',\n", " 'project_name': 'test-project',\n", " 'project_root': '/home/yj.lee/.hyfi/workspace/projects/test-project',\n", " 'project_workspace_name': 'workspace',\n", - " 'task_name': 'test-task',\n", " 'use_huggingface_hub': False,\n", " 'use_wandb': False,\n", " 'verbose': False},\n", - " 'workflows': {'task': {'autoload': False,\n", - " 'module': {'library_dir': 'libs', 'modules': None},\n", - " 'project': {'dotenv': {'CACHED_PATH_CACHE_ROOT': None,\n", - " 'CUDA_DEVICE_ORDER': None,\n", - " 'CUDA_VISIBLE_DEVICES': None,\n", - " 'DOTENV_DIR': '/mnt/nvme1n1p2/home/yj.lee/workspace/projects/hyfi/tests/notebooks',\n", - " 'DOTENV_FILENAME': '.env',\n", - " 'DOTENV_PATH': '/mnt/nvme1n1p2/home/yj.lee/workspace/projects/hyfi/tests/notebooks/.env',\n", - " 'ECOS_API_KEY': None,\n", - " 'FRED_API_KEY': None,\n", - " 'HF_USER_ACCESS_TOKEN': None,\n", - " 'HUGGING_FACE_HUB_TOKEN': None,\n", - " 'HYFI_GLOBAL_ROOT': None,\n", - " 'HYFI_GLOBAL_WORKSPACE_NAME': None,\n", - " 'HYFI_LOG_LEVEL': 'WARNING',\n", - " 'HYFI_NUM_WORKERS': None,\n", - " 'HYFI_PROJECT_DESC': None,\n", - " 'HYFI_PROJECT_NAME': None,\n", - " 'HYFI_PROJECT_ROOT': None,\n", - " 'HYFI_PROJECT_WORKSPACE_NAME': None,\n", - " 'HYFI_RESOURCE_DIR': None,\n", - " 'HYFI_TASK_NAME': None,\n", - " 'HYFI_VERBOSE': False,\n", - " 'KMP_DUPLICATE_LIB_OK': None,\n", - " 'LABEL_STUDIO_SERVER': None,\n", - " 'NASDAQ_API_KEY': None,\n", - " 'OPENAI_API_KEY': None,\n", - " 'TOKENIZERS_PARALLELISM': None,\n", - " 'WANDB_API_KEY': None,\n", - " 'WANDB_DIR': None,\n", - " 'WANDB_DISABLED': None,\n", - " 'WANDB_NOTEBOOK_NAME': None,\n", - " 'WANDB_PROJECT': None,\n", - " 'WANDB_SILENT': None},\n", - " 'global_hyfi_root': '/home/yj.lee/.hyfi',\n", - " 'global_workspace_name': 'workspace',\n", - " 'joblib': {'batcher': {'backend': 'joblib',\n", - " 'minibatch_size': 1000,\n", - " 'procs': '1',\n", - " 'task_num_cpus': 1,\n", - " 'task_num_gpus': 0,\n", - " 'verbose': 10},\n", - " 'distributed_framework': {'backend': 'joblib',\n", - " 'initialize': True,\n", - " 'num_workers': '1'},\n", - " 'num_workers': '1'},\n", - " 'num_workers': '1',\n", - " 'path': {'global_archive': '/home/yj.lee/.hyfi/workspace/archive',\n", - " 'global_cache': '/home/yj.lee/.hyfi/workspace/.cache',\n", - " 'global_datasets': '/home/yj.lee/.hyfi/workspace/datasets',\n", - " 'global_hyfi_root': '/home/yj.lee/.hyfi',\n", - " 'global_library': '/home/yj.lee/.hyfi/workspace/libs',\n", - " 'global_logs': '/home/yj.lee/.hyfi/workspace/logs',\n", - " 'global_models': '/home/yj.lee/.hyfi/workspace/models',\n", - " 'global_modules': '/home/yj.lee/.hyfi/workspace/modules',\n", - " 'global_tmp': '/home/yj.lee/.hyfi/workspace/.tmp',\n", - " 'global_workspace_name': 'workspace',\n", - " 'global_workspace_root': '/home/yj.lee/.hyfi/workspace',\n", - " 'home': '/home/yj.lee',\n", - " 'hyfi': '/mnt/nvme1n1p2/home/yj.lee/workspace/projects/hyfi/src/hyfi',\n", - " 'project_archive': '/home/yj.lee/.hyfi/workspace/projects/test-project/workspace/archive',\n", - " 'project_cache': '/home/yj.lee/.hyfi/workspace/projects/test-project/workspace/.cache',\n", - " 'project_datasets': '/home/yj.lee/.hyfi/workspace/projects/test-project/workspace/datasets',\n", - " 'project_library': '/home/yj.lee/.hyfi/workspace/projects/test-project/workspace/libs',\n", - " 'project_logs': '/home/yj.lee/.hyfi/workspace/projects/test-project/workspace/logs',\n", - " 'project_models': '/home/yj.lee/.hyfi/workspace/projects/test-project/workspace/models',\n", - " 'project_modules': '/home/yj.lee/.hyfi/workspace/projects/test-project/workspace/modules',\n", - " 'project_outputs': '/home/yj.lee/.hyfi/workspace/projects/test-project/workspace/outputs',\n", - " 'project_root': '/home/yj.lee/.hyfi/workspace/projects/test-project',\n", - " 'project_tmp': '/home/yj.lee/.hyfi/workspace/projects/test-project/workspace/.tmp',\n", - " 'project_workspace_name': 'workspace',\n", - " 'project_workspace_root': '/home/yj.lee/.hyfi/workspace/projects/test-project/workspace',\n", - " 'resources': '/mnt/nvme1n1p2/home/yj.lee/workspace/projects/hyfi/src/hyfi/resources',\n", - " 'runtime': '/mnt/nvme1n1p2/home/yj.lee/workspace/projects/hyfi/tests/notebooks'},\n", - " 'project_description': '',\n", - " 'project_name': 'test-project',\n", - " 'project_root': '/home/yj.lee/.hyfi/workspace/projects/test-project',\n", - " 'project_workspace_name': 'workspace',\n", - " 'task_name': 'test-task',\n", - " 'use_huggingface_hub': False,\n", - " 'use_wandb': False,\n", - " 'verbose': False},\n", - " 'steps': {'step': {'name': None,\n", - " 'uses': None,\n", - " 'with': None}},\n", - " 'task_name': 'demo-task',\n", - " 'task_root': '/home/yj.lee/.hyfi/workspace/projects/test-project/workspace/demo-task',\n", - " 'version': '0.0.0'}}}\n" + " 'workflows': '__init__'}\n" ] } ], @@ -212,7 +114,7 @@ "config = HyFI.compose(\n", " \"workflows=__init__\", overrides=[\"+project=__init__\"], global_package=True\n", ")\n", - "HyFI.print(config)" + "HyFI.print(config)\n" ] }, {