-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add CI test to ensure recommender runs directly and with serverless o…
…ffline (#31) Adds PyTest tests to CI, and uses pytest to implement a version of the smoketest with both direct Python API calls and using `serverless offline start` to make sure that we don't break the local dev workflow in addition to the Docker image workflow. It also uses the `actions/cache` action to cache the model data so we don't hit our S3 bucket so often.
- Loading branch information
1 parent
2091506
commit ee9ad3a
Showing
8 changed files
with
245 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| name: Tests | ||
|
|
||
| on: | ||
| push: | ||
| branches: | ||
| - main | ||
| pull_request: | ||
|
|
||
| # override default shell for mamba activation | ||
| defaults: | ||
| run: | ||
| shell: bash -el {0} | ||
|
|
||
| jobs: | ||
| run-tests: | ||
| name: Run the PyTest tests | ||
| runs-on: ubuntu-latest | ||
|
|
||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| fetch-depth: 0 | ||
|
|
||
| - name: Install environment | ||
| uses: mamba-org/setup-micromamba@v1 | ||
| with: | ||
| environment-file: conda-lock.yml | ||
| environment-name: poprox | ||
| create-args: --category main --category dev --category test | ||
|
|
||
| - name: Install Node dependencies | ||
| run: | | ||
| npm ci | ||
| - name: Install recommender package | ||
| run: | | ||
| pip install --no-deps -e . | ||
| - name: Cache model data | ||
| uses: actions/cache@v4 | ||
| with: | ||
| path: .dvc/cache | ||
| key: test-dvc-cache-${{ hashFiles('src/models/**.dvc') }} | ||
|
|
||
| - name: Fetch model data | ||
| run: | | ||
| dvc pull -R src/models | ||
| env: | ||
| AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}} | ||
| AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}} | ||
|
|
||
| - name: Run tests | ||
| run: | | ||
| python -m pytest -v |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,64 @@ | ||
| """ | ||
| Test the POPROX endpoint running under Serverless Offline. | ||
| """ | ||
|
|
||
| import logging | ||
| import sys | ||
| from pathlib import Path | ||
| from threading import Condition, Lock, Thread | ||
|
|
||
| import requests | ||
| from pexpect import EOF, spawn | ||
| from pytest import fail, fixture, mark | ||
|
|
||
| logger = logging.getLogger(__name__) | ||
|
|
||
|
|
||
| @fixture(scope="module") | ||
| def sl_listener(): | ||
| """ | ||
| Fixture that starts and stops serverless offline to test endpoint responses. | ||
| """ | ||
|
|
||
| thread = ServerlessBackground() | ||
| thread.start() | ||
| try: | ||
| with thread.lock: | ||
| if thread.ready.wait(10): | ||
| logger.info("ready for tests") | ||
| yield | ||
| else: | ||
| fail("serverless timed out") | ||
| finally: | ||
| thread.proc.sendintr() | ||
|
|
||
|
|
||
| class ServerlessBackground(Thread): | ||
| def __init__(self, *args, **kwargs): | ||
| super().__init__(*args, **kwargs) | ||
| self.lock = Lock() | ||
| self.ready = Condition(self.lock) | ||
|
|
||
| def run(self): | ||
| logger.info("starting serverless") | ||
| self.proc = spawn("npx serverless offline start", logfile=sys.stdout.buffer) | ||
| self.proc.expect(r"Server ready:") | ||
| logger.info("server ready") | ||
| with self.lock: | ||
| self.ready.notify_all() | ||
| self.proc.expect(EOF) | ||
|
|
||
|
|
||
| @mark.serverless | ||
| def test_basic_request(sl_listener): | ||
| test_dir = Path(__file__) | ||
| req_f = test_dir.parent / "basic-request.json" | ||
| req_body = req_f.read_text() | ||
|
|
||
| logger.info("sending request") | ||
| res = requests.post("http://localhost:3000", req_body) | ||
| assert res.status_code == 200 | ||
| logger.info("response: %s", res.text) | ||
| body = res.json() | ||
| assert "recommendations" in body | ||
| assert len(body["recommendations"]) > 0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| """ | ||
| Test the POPROX API through direct call. | ||
| """ | ||
|
|
||
| import logging | ||
| from pathlib import Path | ||
|
|
||
| from poprox_concepts.api.recommendations import RecommendationRequest | ||
| from poprox_recommender.default import select_articles | ||
|
|
||
| logger = logging.getLogger(__name__) | ||
|
|
||
|
|
||
| def test_direct_basic_request(): | ||
| test_dir = Path(__file__) | ||
| req_f = test_dir.parent / "basic-request.json" | ||
| req = RecommendationRequest.model_validate_json(req_f.read_text()) | ||
|
|
||
| logger.info("generating recommendations") | ||
| recs = select_articles( | ||
| req.todays_articles, | ||
| req.past_articles, | ||
| req.click_histories, | ||
| req.num_recs, | ||
| ) | ||
| # do we get recommendations? | ||
| assert len(recs) > 0 |