Merge branch 'main' into javiermtorres/issue-706-organize-creation-re…

…cords

docker-compose.yaml

@@ -42,14 +42,30 @@ services:
     profiles:
       - local
 
+  redis:
+    image: redis:8.0-M03-alpine
+    command: redis-server --save 60 1 --loglevel warning
+    profiles:
+      - local
+    volumes:
+      - redis-data:/data
+    healthcheck:
+      test: ["CMD-SHELL", "redis-cli ping"]
+      interval: 1s
+      timeout: 3s
+      retries: 5
+
   ray:
     image: rayproject/ray:2.30.0-py311${COMPUTE_TYPE}${RAY_ARCH_SUFFIX}
+    depends_on:
+      redis:
+        condition: service_healthy
     ports:
       - "6379:6379"
       - "8265:8265"
       - "10001:10001"
     # https://docs.ray.io/en/releases-2.30.0/cluster/cli.html#ray-start for more info about the command
-    command: bash -c "ray start --head --dashboard-port=8265 --port=6379 --dashboard-host=0.0.0.0 --ray-client-server-port 10001 --block" # pragma: allowlist secret
+    command: bash -c "RAY_REDIS_ADDRESS=redis:6379 ray start --head --dashboard-port=8265 --port=6379 --dashboard-host=0.0.0.0 --ray-client-server-port 10001 --block" # pragma: allowlist secret
     shm_size: 2g
     volumes:
       - ${HOME}/.cache/huggingface:/home/ray/.cache/huggingface
@@ -155,3 +171,4 @@ services:
 volumes:
     minio-data:
     database_volume:
+    redis-data:

lumigator/python/mzai/backend/backend/services/experiments.py

@@ -62,14 +62,32 @@ def get_experiment(self, experiment_id: UUID) -> ExperimentResponse:
         loguru.logger.info(f"Obtaining info for experiment {experiment_id}: {record}")
 
         all_succeeded = True
+        any_running = False
+        any_failed = False
+        # Any running: running
+        # Any failed: failed
+        # All succeeded: succeeded
         for job in self._get_all_owned_jobs(experiment_id):
             loguru.logger.info(f"Checking sub job: {job}")
             if self._job_service.get_job(job.id).status != JobStatus.SUCCEEDED:
                 all_succeeded = False
-                break
+                if self._job_service.get_job(job.id).status == JobStatus.FAILED:
+                    any_failed = True
+                    # Any failed job makes further inspection unnecessary
+                    break
+                if self._job_service.get_job(job.id).status == JobStatus.RUNNING:
+                    any_running = True
+                    # Any running job will move status to running, but
+                    # searching for failures is still necessary
 
         if all_succeeded:
             record = self._experiment_repo.update(experiment_id, status=JobStatus.SUCCEEDED)
+        elif any_failed:
+            record = self._experiment_repo.update(experiment_id, status=JobStatus.FAILED)
+        elif any_running:
+            record = self._experiment_repo.update(experiment_id, status=JobStatus.RUNNING)
+        else:
+            record = self._experiment_repo.update(experiment_id, status=JobStatus.PENDING)
 
         return ExperimentResponse.model_validate(record)
 

lumigator/python/mzai/backend/backend/tests/README.md

@@ -6,30 +6,68 @@ down into two distinct categories:
 - `unit` tests: Involve isolated functions/classes.
 - `integration` tests: Depend on some external service to run.
 
-The external services that the application depends on are the database, Ray cluster, and S3 storage.
-Currently, we are using the [`TestContainers`](https://testcontainers-python.readthedocs.io/en/latest/)
-library to provide some of these dependencies for testing.
 
-`TestContainers` provides a simple interface for spinning up a Docker container running some service
-as part of the testing lifecycle. This is configured in the `conftest.py` file that contains
-fixtures for the entire test suite.
+## Running Tests
+
+We currently test unit and integration tests via the `Makefile`:
+- `test-sdk`
+- `test-backend`
+- `test-jobs-unit`
+
+To run the backend tests, for example, you can use the configured commands in the `Makefile`:
+
+```
+- make test-backend-integration
+- make test-backend-unit
+- make test-backend # runs both
+```
+
+### Running Individual Tests
+To run individual tests, you can change the pytest path by specifying the `-k` flag in pytest, which runs tests [based on string expression](https://docs.pytest.org/en/6.2.x/usage.html#specifying-tests-selecting-tests)
+
+```sh
+uv run $(DEBUGPY_ARGS) -m pytest -s -o \
+python_files="backend/tests/integration/*/test_*.py" \
+-k 'test_full_experiment_launch'
+```
+
+As an example, editing this runs only the `test_full_experiment_launch` method in `test-backend-integration`:
+
+```sh
+test-backend-integration:
+	cd lumigator/python/mzai/backend/; \
+	docker container list --all; \
+	S3_BUCKET=lumigator-storage \
+	RAY_HEAD_NODE_HOST=localhost \
+	RAY_DASHBOARD_PORT=8265 \
+	SQLALCHEMY_DATABASE_URL=sqlite:////tmp/local.db \
+	RAY_WORKER_GPUS="0.0" \
+	RAY_WORKER_GPUS_FRACTION="0.0" \
+	INFERENCE_PIP_REQS=../jobs/inference/requirements_cpu.txt \
+	INFERENCE_WORK_DIR=../jobs/inference \
+	EVALUATOR_PIP_REQS=../jobs/evaluator/requirements.txt \
+	EVALUATOR_WORK_DIR=../jobs/evaluator \
+	EVALUATOR_LITE_PIP_REQS=../jobs/evaluator_lite/requirements.txt \
+	EVALUATOR_LITE_WORK_DIR=../jobs/evaluator_lite \
+	PYTHONPATH=../jobs:$$PYTHONPATH \
+	uv run $(DEBUGPY_ARGS) -m pytest -s -o \
+	python_files="backend/tests/integration/*/test_*.py" \
+	-k 'test_full_experiment_launch'
+```
+
+Additionally, you can run this directly on the commandline so as to not change the Makefile.
 
 ## Test Dependencies
 
+The external services that the application depends on are the database, Ray cluster, and S3-compatible storage.
+
 The backend tests offer two sets of service dependencies as fixtures:
 
 * `dependency_overrides_fakes`, to be used in unit tests, mocks all external services.
 * `dependency_overrides_services`, to be used in integration tests, provides real clients for external services (except the DB).
 
 By default, an embedded SQLite database is used in both cases.
 
-## Running Tests
-
-To run the tests, you can use the following command:
-
-```
-SQLALCHEMY_DATABASE_URL=sqlite:///local.db uv run pytest
-```
 
 ## Test Settings
 

lumigator/python/mzai/backend/pyproject.toml

@@ -31,7 +31,6 @@ dependencies = [
 dev-dependencies = [
     "pytest>=8.3.3",
     "requests-mock>=1.12.1",
-    "testcontainers>=4.8.2",
     "moto[s3]>=5.0,<6",
     "debugpy>=1.8.11"
 ]