Build and tests fixes

- Fix poetry config, clean requirements files
- Revert to docker-compose
- Tune poetry build, adjust project naming/versioning
- Make tests run in own container
- Fix pytest runtime
- Setup newman(postman) tests in container
- Fix main/environmental tests issues, make use of profiles
- Cleanup Makefile
- Github actions/workflow initial fix

Author: mtykhenko

.github/workflows/ci.yml

@@ -2,9 +2,9 @@ name: CI/CD Pipeline
 
 on:
   push:
-    branches: [ main, dev, test ]
+    branches: [main, dev, test]
   pull_request:
-    branches: [ main, dev, test ]
+    branches: [main, dev, test]
 
 jobs:
   format:
@@ -14,7 +14,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.11
+          python-version: 3.12
       - name: Setup Poetry
         run: |
           pip install poetry
@@ -24,10 +24,11 @@ jobs:
         uses: actions/cache@v2
         id: poetry-cache
         with:
-          path: .venv
+          path: ./backend/.venv
           key: poetry-${{ hashFiles('**/poetry.lock') }}
       - if: steps.poetry-cache.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction --no-root
+        run: |
+          poetry install --only main --no-root -C ./backend
       - name: Check Poetry configuration
         run: make check-toml
       - name: Run formatting
@@ -41,7 +42,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.11
+          python-version: 3.12
       - name: Setup Poetry
         run: |
           pip install poetry
@@ -68,9 +69,9 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.11
+          python-version: 3.12
       - name: Setup Poetry
-        run: | 
+        run: |
           pip install poetry
           poetry config virtualenvs.create true
           poetry config virtualenvs.in-project true
@@ -93,9 +94,9 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.11
+          python-version: 3.12
       - name: Setup Poetry
-        run: | 
+        run: |
           pip install poetry
           poetry config virtualenvs.create true
           poetry config virtualenvs.in-project true
@@ -127,4 +128,4 @@ jobs:
       - name: Deploy to production
         run: |
           # Add your deployment steps here
-          echo "Deploying to production"
+          echo "Deploying to production"

Makefile

@@ -6,72 +6,73 @@ export $(shell sed 's/=.*//' .env)
 export PYTHONPATH=$(pwd):$(pwd)/vectordbs:$(pwd)/rag_solution
 
 # Directories
-SOURCE_DIR := rag_solution
-TEST_DIR := tests
+SOURCE_DIR := ./backend/rag_solution
+TEST_DIR := ./backend/tests
 PROJECT_DIRS := $(SOURCE_DIR) $(TEST_DIR)
 
 # Project info
-PROJECT_NAME ?= rag_modulo
+PROJECT_NAME ?= rag-modulo
 PYTHON_VERSION ?= 3.11
-PROJECT_VERSION ?= v$(shell poetry version -s)
+PROJECT_VERSION ?= 1.0.0
 
 # Tools
-DOCKER_COMPOSE := podman-compose
+DOCKER_COMPOSE := docker-compose
 
 # Set a default value for VECTOR_DB if not already set
 VECTOR_DB ?= milvus
 
 .DEFAULT_GOAL := help
 
-.PHONY: init-env init check-toml format lint audit test run-services build-app run-app clean all info help api-test install-deps newman-test run-backend run-frontend
+.PHONY: init-env build-frontend build-backend build-tests build-all test api-test newman-test all-test run-app run-backend run-frontend run-services stop-containers clean create-volumes logs info help
 
+# Init
 init-env:
 	@touch .env
 	@echo "PROJECT_NAME=${PROJECT_NAME}" >> .env
 	@echo "PYTHON_VERSION=${PYTHON_VERSION}" >> .env
 	@echo "VECTOR_DB=${VECTOR_DB}" >> .env
 
-install-deps:
-	pip install -r requirements.txt
+# Build
+build-frontend:
+	$(DOCKER_COMPOSE) build frontend
 
-init: init-env install-deps
+build-backend:
+	$(DOCKER_COMPOSE) build backend
 
-check-toml:
-	# No equivalent for pip, so this can be left empty or removed
+build-tests:
+	$(DOCKER_COMPOSE) build test
 
-format:
-	ruff check $(SOURCE_DIR) && black $(PROJECT_DIRS) && isort $(PROJECT_DIRS)
+build-all:
+	@echo "Building application containers..."
+	$(DOCKER_COMPOSE) --profile default --profile test build
 
-lint:
-	ruff check $(SOURCE_DIR)
-	mypy --install-types --show-error-codes --non-interactive $(SOURCE_DIR)
+# Test
+test: build-all run-app
+	$(DOCKER_COMPOSE) run test pytest -v -s -m "not (chromadb or elasticsearch or pinecone or weaviate)" || { echo "Tests failed"; $(MAKE) stop-containers; exit 1; }
 
-audit:
-	bandit -r $(SOURCE_DIR) -x $(TEST_DIR)
+api-test: build-all run-app
+	$(DOCKER_COMPOSE) run test pytest -v -s -m "api and not (chromadb or elasticsearch or pinecone or weaviate)" || { echo "API Tests failed"; $(MAKE) stop-containers; exit 1; }
 
-test: run-services
-	pytest $(TEST_DIR) || { echo "Tests failed"; exit 1; }
-	@trap '$(DOCKER_COMPOSE) down' EXIT; \
-	echo "Waiting for Docker containers to stop..."
-	@while docker ps | grep -q "milvus-standalone"; do sleep 1; done
+newman-test: build-all run-app
+	$(DOCKER_COMPOSE) run test newman run tests/postman/rag_modulo_api_collection.json --env-var "backend_base_url=${REACT_APP_API_URL}" || { echo "Postman Tests failed"; $(MAKE) stop-containers; exit 1; }
 
-api-test: run-services
-	pytest $(TEST_DIR)/api -v -m api || { echo "API Tests failed"; exit 1; }
-	@trap '$(DOCKER_COMPOSE) down' EXIT; \
-	echo "Waiting for Docker containers to stop..."
-	@while docker ps | grep -q "milvus-standalone"; do sleep 1; done
+all-test: build-all run-app
+	$(DOCKER_COMPOSE) run test pytest -v -s -m "not (chromadb or elasticsearch or pinecone or weaviate)" || { echo "Tests failed"; $(MAKE) stop-containers; exit 1; }
+	$(DOCKER_COMPOSE) run test newman run tests/postman/rag_modulo_api_collection.json --env-var "backend_base_url=${REACT_APP_API_URL}" || { echo "Postman Tests failed"; $(MAKE) stop-containers; exit 1; }
 
-newman-test: run-services
-	newman run postman/rag_modulo_api_collection.json
-	@trap '$(DOCKER_COMPOSE) down' EXIT; \
-	echo "Waiting for Docker containers to stop..."
-	@while docker ps | grep -q "milvus-standalone"; do sleep 1; done
+# Run
+run-app: build-all run-backend run-frontend
+	@echo "All application containers are now running."
 
-create-volumes:
-	@echo "Creating volume directories with correct permissions..."
-	@mkdir -p ./volumes/postgres ./volumes/etcd ./volumes/minio ./volumes/milvus
-	@chmod -R 777 ./volumes
-	@echo "Volume directories created and permissions set."
+run-backend: run-services
+	@echo "Starting backend..."
+	$(DOCKER_COMPOSE) up -d backend
+	@echo "Backend is now running."
+
+run-frontend: run-services
+	@echo "Starting frontend..."
+	$(DOCKER_COMPOSE) up -d frontend
+	@echo "Frontend is now running."
 
 run-services: create-volumes
 	@if [ -z "$(VECTOR_DB)" ]; then \
@@ -128,34 +129,10 @@ run-services: create-volumes
 	@echo "Milvus logs saved to milvus.log:"
 	@$(DOCKER_COMPOSE) logs milvus-standalone > milvus.log
 
-run-tests:
-	$(DOCKER_COMPOSE) run --rm test pytest $(ARGS)
-
-build-frontend:
-	docker build -t $(PROJECT_NAME)-frontend -f webui/Dockerfile.frontend ./webui
-
-build-backend:
-	docker build --build-arg CACHEBUST=$(date +%s) -t $(PROJECT_NAME)-backend --no-cache -f Dockerfile.backend .
-
-build-app:
-	@echo "Building application containers..."
-	$(DOCKER_COMPOSE) build
-
-run-backend: run-services
-	@echo "Starting backend..."
-	$(DOCKER_COMPOSE) up -d backend
-	@echo "Backend is now running."
-
-run-frontend: run-services
-	@echo "Starting frontend..."
-	$(DOCKER_COMPOSE) up -d frontend
-	@echo "Frontend is now running."
-
-run-app: build-app run-backend run-frontend
-	@echo "All application containers are now running."
-
-logs:
-	$(DOCKER_COMPOSE) logs -f
+# Stop / clean
+stop-containers:
+	echo "Stopping containers and removing volumes ..."	
+	$(DOCKER_COMPOSE) down -v
 
 clean:
 	@echo "Cleaning up Docker Compose resources..."
@@ -164,9 +141,18 @@ clean:
 	-@podman pod rm -f $(podman pod ps -q) || true
 	-@podman rm -f $(podman ps -a -q) || true
 	-@podman volume prune -f || true
+	-@podman container prune -f || true
 	rm -rf .pytest_cache .mypy_cache data volumes my_chroma_data tests
 
-all: format lint audit test
+# Service / reusable targets
+create-volumes:
+	@echo "Creating volume directories with correct permissions..."
+	@mkdir -p ./volumes/postgres ./volumes/etcd ./volumes/minio ./volumes/milvus
+	@chmod -R 777 ./volumes
+	@echo "Volume directories created and permissions set."
+
+logs:
+	$(DOCKER_COMPOSE) logs -f
 
 info:
 	@echo "Project name: ${PROJECT_NAME}"
@@ -178,43 +164,22 @@ help:
 	@echo "Usage: make [target]"
 	@echo ""
 	@echo "Targets:"
-	@echo "  init-env      Initialize .env file with default values"
-	@echo "  init          Initialize environment and install dependencies"
-	@echo "  install-deps  Install project dependencies"
-	@echo "  check-toml    Check TOML files for syntax errors"
-	@echo "  format        Format code using ruff, black, and isort"
-	@echo "  lint          Lint code using ruff and mypy"
-	@echo "  audit         Audit code using bandit"
-	@echo "  test          Run all tests using pytest"
-	@echo "  api-test      Run API tests using pytest"
-	@echo "  newman-test   Run API tests using Newman"
-	@echo "  run-services  Start services using Docker Compose"
-	@echo "  build-app     Build app using Docker Compose"
-	@echo "  run-backend   Run backend using Docker Compose"
-	@echo "  run-frontend  Run frontend using Docker Compose"
-	@echo "  run-app       Run both backend and frontend using Docker Compose"
-	@echo "  logs          View logs of running containers"
-	@echo "  clean         Clean up Docker Compose volumes and cache"
-	@echo "  all           Format, lint, audit, and test"
+	@echo "  init-env      		Initialize .env file with default values"
+	@echo "  build-frontend  	Build frontend code/container"
+	@echo "  build-backend   	Build backend code/container"
+	@echo "  build-tests   		Build test code/container"
+	@echo "  build-all   		Build frontend/backend/test code/container"
+	@echo "  test          		Run all tests using pytest"
+	@echo "  api-test      		Run API tests using pytest"
+	@echo "  newman-test   		Run API tests using newman"
+	@echo "  all-test   		Run all tests using pytest and newman"
+	@echo "  run-app       		Run both backend and frontend using Docker Compose"
+	@echo "  run-backend   		Run backend using Docker Compose"
+	@echo "  run-frontend  		Run frontend using Docker Compose"
+	@echo "  run-services  		Run services using Docker Compose"
+	@echo "  stop-containers  	Stop all containers using Docker Compose"	
+	@echo "  clean         		Clean up Docker Compose volumes and cache"
+	@echo "  create-volumes     Create folders for container volumes"	
+	@echo "  logs          		View logs of running containers"
 	@echo "  info          Display project information"
-	@echo "  help          Display this help message"
-	@echo "  dev           Start development environment"
-	@echo "  dev-build     Build development environment"
-	@echo "  dev-down      Stop development environment"
-	@echo "  dev-backend   Start backend development server"
-	@echo "  dev-frontend  Start frontend development server"
-
-dev:
-	docker-compose -f docker-compose.dev.yml up
-
-dev-build:
-	docker-compose -f docker-compose.dev.yml build
-
-dev-down:
-	docker-compose -f docker-compose.dev.yml down
-
-dev-backend:
-	uvicorn main:app --reload --host 0.0.0.0 --port 8000
-
-dev-frontend:
-	cd webui && npm start
+	@echo "  help          Display this help message"