From f520539a305834e372c295f4d56e04b6ba9a8798 Mon Sep 17 00:00:00 2001 From: Wenzhe Xue Date: Thu, 3 Oct 2024 21:19:22 -0700 Subject: [PATCH 1/5] Integrate pytest_log10_managed_evaluation into main package - Move plugin files to src/pytest_log10_managed_evaluation/ - Update pyproject.toml to include plugin - Remove separate plugin pyproject.toml --- pyproject.toml | 12 +++--- .../__init__.py | 0 .../plugin.py | 0 .../pyproject.toml | 42 ------------------- .../serialize.py | 0 .../utils.py | 0 6 files changed, 6 insertions(+), 48 deletions(-) rename src/pytest_log10_managed_evaluation/{pytest_log10_managed_evaluation => }/__init__.py (100%) rename src/pytest_log10_managed_evaluation/{pytest_log10_managed_evaluation => }/plugin.py (100%) delete mode 100644 src/pytest_log10_managed_evaluation/pyproject.toml rename src/pytest_log10_managed_evaluation/{pytest_log10_managed_evaluation => }/serialize.py (100%) rename src/pytest_log10_managed_evaluation/{pytest_log10_managed_evaluation => }/utils.py (100%) diff --git a/pyproject.toml b/pyproject.toml index 5cf4e959..aa7fd033 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ classifiers = [ ] packages = [ { include = "log10", from = "src" }, + { include = "pytest_log10_managed_evaluation", from = "src" }, ] [tool.poetry.scripts] @@ -55,8 +56,8 @@ click = {version = "^8.1.7", optional = true} rich = {version = "^13.7.1", optional = true} tabulate = {version = "^0.9.0", optional = true} pandas = {version = ">=2", optional = true} -# pytest = {version = ">=8.0.0", optional = true} -# pytest-metadata = {version = ">=1.0.0", optional = true} +pytest = {version = ">=8.0.0", optional = true} +pytest-metadata = {version = ">=1.0.0", optional = true} [tool.poetry.extras] autofeedback_icl = ["magentic"] @@ -69,11 +70,10 @@ mosaicml = ["mosaicml-cli"] google-generativeai = ["google-generativeai"] lamini = ["lamini"] cli = ["click", "rich", "tabulate", "pandas"] -pytest = ["pytest_log10_managed_evaluation"] +pytest = ["pytest", "pytest-metadata"] -[tool.poetry.dependencies.pytest_log10_managed_evaluation] -path = "src/pytest_log10_managed_evaluation" -optional = true +[tool.poetry.plugin.pytest11] +log10_managed_evaluation = "pytest_log10_managed_evaluation.plugin" [tool.ruff] # Never enforce `E501` (line length violations). diff --git a/src/pytest_log10_managed_evaluation/pytest_log10_managed_evaluation/__init__.py b/src/pytest_log10_managed_evaluation/__init__.py similarity index 100% rename from src/pytest_log10_managed_evaluation/pytest_log10_managed_evaluation/__init__.py rename to src/pytest_log10_managed_evaluation/__init__.py diff --git a/src/pytest_log10_managed_evaluation/pytest_log10_managed_evaluation/plugin.py b/src/pytest_log10_managed_evaluation/plugin.py similarity index 100% rename from src/pytest_log10_managed_evaluation/pytest_log10_managed_evaluation/plugin.py rename to src/pytest_log10_managed_evaluation/plugin.py diff --git a/src/pytest_log10_managed_evaluation/pyproject.toml b/src/pytest_log10_managed_evaluation/pyproject.toml deleted file mode 100644 index 076a1dba..00000000 --- a/src/pytest_log10_managed_evaluation/pyproject.toml +++ /dev/null @@ -1,42 +0,0 @@ -[build-system] -requires = ["poetry-core>=1.0.0"] -build-backend = "poetry.core.masonry.api" - -[tool.poetry] -name = "pytest_log10_managed_evaluation" -description = "pytest plugin for log10 managed evaluation" - -version = "0.14.0rc1" # Match this with log10-io version -authors = ["log10 team"] -license = "MIT" -classifiers = [ - "Framework :: Pytest", - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "Topic :: Software Development :: Testing", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "License :: OSI Approved :: MIT License", -] - -packages = [ - { include = "pytest_log10_managed_evaluation", from = "." } -] - -[tool.poetry.dependencies] -python = ">=3.10,<4.0" -pytest = ">=6.2.0" -pytest-metadata = ">=1.0.0" - -[project.urls] -"Homepage" = "https://github.com/log10-io/log10" -"Bug Tracker" = "https://github.com/log10-io/log10/issues" - -[tool.poetry.plugins.pytest11] -log10_managed_evaluation = "pytest_log10_managed_evaluation.plugin" diff --git a/src/pytest_log10_managed_evaluation/pytest_log10_managed_evaluation/serialize.py b/src/pytest_log10_managed_evaluation/serialize.py similarity index 100% rename from src/pytest_log10_managed_evaluation/pytest_log10_managed_evaluation/serialize.py rename to src/pytest_log10_managed_evaluation/serialize.py diff --git a/src/pytest_log10_managed_evaluation/pytest_log10_managed_evaluation/utils.py b/src/pytest_log10_managed_evaluation/utils.py similarity index 100% rename from src/pytest_log10_managed_evaluation/pytest_log10_managed_evaluation/utils.py rename to src/pytest_log10_managed_evaluation/utils.py From 6a35334f2f1066ad2ac0ba7339169a322a3d54d2 Mon Sep 17 00:00:00 2001 From: Wenzhe Xue Date: Thu, 3 Oct 2024 21:36:45 -0700 Subject: [PATCH 2/5] add option to enable log10 plugin --- src/pytest_log10_managed_evaluation/plugin.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/pytest_log10_managed_evaluation/plugin.py b/src/pytest_log10_managed_evaluation/plugin.py index 6f8fde75..235fea27 100644 --- a/src/pytest_log10_managed_evaluation/plugin.py +++ b/src/pytest_log10_managed_evaluation/plugin.py @@ -416,6 +416,12 @@ def json_metadata(request): def pytest_addoption(parser): group = parser.getgroup("jsonreport", "reporting test results as JSON") + group.addoption( + "--log10", + action="store_true", + default=False, + help="Enable Log10 managed evaluation reporting", + ) group.addoption( "--pretty-print", action="store_true", @@ -432,12 +438,22 @@ def pytest_addoption(parser): "--eval-session-name", action="store", default=None, help="Name for the evaluation session in JSON report" ) parser.addini("eval_session_name", "Name for the evaluation session in JSON report", default=None) + parser.addini( + "log10", + "Enable Log10 managed evaluation reporting", + type="bool", + default=False, + ) def pytest_configure(config): if config.option.collectonly: return + log10_enabled = config.getoption("log10") or config.getini("log10") + if not log10_enabled: + return + if hasattr(config, "workerinput"): Plugin = JSONReportWorker else: From c82294950aad8534fb591ba07b14697b1709f8bf Mon Sep 17 00:00:00 2001 From: Wenzhe Xue Date: Thu, 3 Oct 2024 21:41:52 -0700 Subject: [PATCH 3/5] Update README for opt-in Log10 reporting - Add --log10 option and INI setting - Update usage instructions and examples - Clarify new opt-in behavior for Log10 reporting --- src/pytest_log10_managed_evaluation/README.md | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/src/pytest_log10_managed_evaluation/README.md b/src/pytest_log10_managed_evaluation/README.md index 1860e522..aff503f0 100644 --- a/src/pytest_log10_managed_evaluation/README.md +++ b/src/pytest_log10_managed_evaluation/README.md @@ -5,33 +5,47 @@ A pytest plugin for managing evaluation in Log10 platform. ## Installation After [configuring the Log10 environment variables](https://docs.log10.io/observability/advanced/logging#configuration), ```bash -pip install 'log10-io[pytest]' +pip install log10-io ``` ## Options | Option | Type | Description | |--------|------|-------------| +| `--log10` | Command-line | Enable Log10 managed evaluation reporting | | `--eval-session-name` | Command-line | Set name for the evaluation session | | `--local` | Command-line | Run pytest locally without showing the session in Log10. A JSON report file will be saved locally. | | `--pretty-print` | Command-line | Pretty-print the JSON report with indentation. | +| `log10` | INI | Enable Log10 managed evaluation reporting. Set in `pytest.ini` or `setup.cfg`. | | `eval_session_name` | INI | Set name for the evaluation session. Set in `pytest.ini` or `setup.cfg`. | ## Usage -Once installed, the plugin is automatically enabled when you run pytest. Simply execute your tests as you normally would, e.g.: +To enable the Log10 managed evaluation reporting, you need to use the `--log10` option or set `log10 = true` in your pytest configuration file. Once enabled, execute your tests as you normally would: + ```bash -pytest tests +pytest tests --log10 ``` This will run your tests and upload the results to the Log10 platform for managed evaluation. +### Enabling Log10 Reporting in Configuration + +You can enable Log10 reporting by default in your pytest configuration file: + +Example in `pytest.ini`: + +```ini +[pytest] +log10 = true +``` + ### Running Tests Locally If you prefer to run tests locally without uploading results to Log10, use the `--local` option: ```bash -pytest tests --local +pytest tests --log10 --local ``` When using the `--local` option, a JSON report will be generated and saved in the `.pytest_log10_eval_reports` folder in your project directory. This is useful for: @@ -45,7 +59,7 @@ This helps in organizing and identifying specific test runs in the Log10 platfor To assign a custom name to your evaluation session, use the `--eval-session-name` option: ```bash -pytest tests --eval-session-name +pytest tests --log10 --eval-session-name ``` Use `eval_session_name` in INI file @@ -64,7 +78,7 @@ eval_session_name = For improved readability of local JSON reports, use the `--pretty-print` option: ```bash -pytest tests --local --pretty-print +pytest tests --log10 --local --pretty-print ``` This will format the JSON report with proper indentation, making it easier to read and analyze. From ddc282d9969dc84290636ceaeda8abdd417d2734 Mon Sep 17 00:00:00 2001 From: Wenzhe Xue Date: Thu, 3 Oct 2024 22:26:16 -0700 Subject: [PATCH 4/5] restructure the dir to src/log10/pytest_log10_managed_evaluation --- pyproject.toml | 10 ++++------ .../pytest_log10_managed_evaluation/README.md | 0 .../pytest_log10_managed_evaluation/__init__.py | 0 .../pytest_log10_managed_evaluation/plugin.py | 0 .../pytest_log10_managed_evaluation/serialize.py | 0 .../pytest_log10_managed_evaluation/utils.py | 0 6 files changed, 4 insertions(+), 6 deletions(-) rename src/{ => log10}/pytest_log10_managed_evaluation/README.md (100%) rename src/{ => log10}/pytest_log10_managed_evaluation/__init__.py (100%) rename src/{ => log10}/pytest_log10_managed_evaluation/plugin.py (100%) rename src/{ => log10}/pytest_log10_managed_evaluation/serialize.py (100%) rename src/{ => log10}/pytest_log10_managed_evaluation/utils.py (100%) diff --git a/pyproject.toml b/pyproject.toml index aa7fd033..8143576e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,6 @@ classifiers = [ ] packages = [ { include = "log10", from = "src" }, - { include = "pytest_log10_managed_evaluation", from = "src" }, ] [tool.poetry.scripts] @@ -42,6 +41,8 @@ anthropic = "<1" requests = "^2.31.0" python-dotenv = "^1.0.0" backoff = "^2.2.1" +pytest = ">=8.0.0" +pytest-metadata = ">=1.0.0" langchain = {version = "<0.2.0", optional = true} magentic = {version = ">=0.17.0", optional = true, markers = "python_version >= '3.10'"} litellm = {version = "^1.41.12", optional = true} @@ -56,8 +57,6 @@ click = {version = "^8.1.7", optional = true} rich = {version = "^13.7.1", optional = true} tabulate = {version = "^0.9.0", optional = true} pandas = {version = ">=2", optional = true} -pytest = {version = ">=8.0.0", optional = true} -pytest-metadata = {version = ">=1.0.0", optional = true} [tool.poetry.extras] autofeedback_icl = ["magentic"] @@ -70,10 +69,9 @@ mosaicml = ["mosaicml-cli"] google-generativeai = ["google-generativeai"] lamini = ["lamini"] cli = ["click", "rich", "tabulate", "pandas"] -pytest = ["pytest", "pytest-metadata"] -[tool.poetry.plugin.pytest11] -log10_managed_evaluation = "pytest_log10_managed_evaluation.plugin" +[tool.poetry.plugins."pytest11"] +log10_managed_evaluation = "log10.pytest_log10_managed_evaluation.plugin" [tool.ruff] # Never enforce `E501` (line length violations). diff --git a/src/pytest_log10_managed_evaluation/README.md b/src/log10/pytest_log10_managed_evaluation/README.md similarity index 100% rename from src/pytest_log10_managed_evaluation/README.md rename to src/log10/pytest_log10_managed_evaluation/README.md diff --git a/src/pytest_log10_managed_evaluation/__init__.py b/src/log10/pytest_log10_managed_evaluation/__init__.py similarity index 100% rename from src/pytest_log10_managed_evaluation/__init__.py rename to src/log10/pytest_log10_managed_evaluation/__init__.py diff --git a/src/pytest_log10_managed_evaluation/plugin.py b/src/log10/pytest_log10_managed_evaluation/plugin.py similarity index 100% rename from src/pytest_log10_managed_evaluation/plugin.py rename to src/log10/pytest_log10_managed_evaluation/plugin.py diff --git a/src/pytest_log10_managed_evaluation/serialize.py b/src/log10/pytest_log10_managed_evaluation/serialize.py similarity index 100% rename from src/pytest_log10_managed_evaluation/serialize.py rename to src/log10/pytest_log10_managed_evaluation/serialize.py diff --git a/src/pytest_log10_managed_evaluation/utils.py b/src/log10/pytest_log10_managed_evaluation/utils.py similarity index 100% rename from src/pytest_log10_managed_evaluation/utils.py rename to src/log10/pytest_log10_managed_evaluation/utils.py From 098c74f9f21fd4a0d72219c9e70b994e0eba9fae Mon Sep 17 00:00:00 2001 From: Wenzhe Xue Date: Thu, 3 Oct 2024 22:39:52 -0700 Subject: [PATCH 5/5] minor update: - update poetry.lock - revert gh workflow test.yml to use --all-extras --- .github/workflows/test.yml | 2 +- poetry.lock | 22 ++-------------------- 2 files changed, 3 insertions(+), 21 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 074daf24..91933a6f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -63,7 +63,7 @@ jobs: cache: "poetry" architecture: 'x64' - name: Install dependencies - run: poetry install --extras "autofeedback_icl litellm langchain gemini mistralai together mosaicml google-generativeai lamini cli" + run: poetry install --all-extras - name: Run cli tests run: poetry run pytest -vv tests/test_cli.py diff --git a/poetry.lock b/poetry.lock index 20fb52b8..a844ce8a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2747,28 +2747,11 @@ pytest = ">=7.0.0,<9" docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1.0)"] testing = ["coverage (>=6.2)", "hypothesis (>=5.7.1)"] -[[package]] -name = "pytest-log10-managed-evaluation" -version = "0.13.0" -description = "pytest plugin for log10 managed evaluation" -optional = true -python-versions = ">=3.10,<4.0" -files = [] -develop = false - -[package.dependencies] -pytest = ">=6.2.0" -pytest-metadata = ">=1.0.0" - -[package.source] -type = "directory" -url = "src/pytest_log10_managed_evaluation" - [[package]] name = "pytest-metadata" version = "3.1.1" description = "pytest plugin for test session metadata" -optional = true +optional = false python-versions = ">=3.8" files = [ {file = "pytest_metadata-3.1.1-py3-none-any.whl", hash = "sha256:c8e0844db684ee1c798cfa38908d20d67d0463ecb6137c72e91f418558dd5f4b"}, @@ -4144,10 +4127,9 @@ langchain = ["langchain"] litellm = ["litellm"] mistralai = ["mistralai"] mosaicml = ["mosaicml-cli"] -pytest = ["pytest_log10_managed_evaluation"] together = ["together"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<4.0" -content-hash = "2f906897683b29de65c10ff297aece98c77e85efe3fd00a84b7a33bf6f617d86" +content-hash = "63ad0505baa103e0f410ef9c86e31c729a3bb0d6e88e41bb4c74bc1a8618bd4e"