Skip to content

Build and tests fixes #14

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mtykhenko merged 6 commits into from
Oct 7, 2024
Merged

Build and tests fixes #14

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
09a22cf
Fix poetry config, clean requirements files
mtykhenko Sep 30, 2024
c83afb0
Merge branch 'main' into poetry-adjustments
mtykhenko Sep 30, 2024
6acdb08
Revert to docker-compose
mtykhenko Sep 30, 2024
275c9b7
Merge branch 'main' into poetry-adjustments
mtykhenko Oct 1, 2024
daad789
Tune poetry build, adjust project naming/versioning
mtykhenko Oct 2, 2024
e71055e
Make tests run in own container
mtykhenko Oct 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 12 additions & 11 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@ name: CI/CD Pipeline

on:
push:
branches: [ main, dev, test ]
branches: [main, dev, test]
pull_request:
branches: [ main, dev, test ]
branches: [main, dev, test]

jobs:
format:
Expand All @@ -14,7 +14,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.11
python-version: 3.12
- name: Setup Poetry
run: |
pip install poetry
Expand All @@ -24,10 +24,11 @@ jobs:
uses: actions/cache@v2
id: poetry-cache
with:
path: .venv
path: ./backend/.venv
key: poetry-${{ hashFiles('**/poetry.lock') }}
- if: steps.poetry-cache.outputs.cache-hit != 'true'
run: poetry install --no-interaction --no-root
run: |
poetry install --only main --no-root -C ./backend
- name: Check Poetry configuration
run: make check-toml
- name: Run formatting
Expand All @@ -41,7 +42,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.11
python-version: 3.12
- name: Setup Poetry
run: |
pip install poetry
Expand All @@ -68,9 +69,9 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.11
python-version: 3.12
- name: Setup Poetry
run: |
run: |
pip install poetry
poetry config virtualenvs.create true
poetry config virtualenvs.in-project true
Expand All @@ -93,9 +94,9 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.11
python-version: 3.12
- name: Setup Poetry
run: |
run: |
pip install poetry
poetry config virtualenvs.create true
poetry config virtualenvs.in-project true
Expand Down Expand Up @@ -127,4 +128,4 @@ jobs:
- name: Deploy to production
run: |
# Add your deployment steps here
echo "Deploying to production"
echo "Deploying to production"
177 changes: 71 additions & 106 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,72 +6,73 @@ export $(shell sed 's/=.*//' .env)
export PYTHONPATH=$(pwd):$(pwd)/vectordbs:$(pwd)/rag_solution

# Directories
SOURCE_DIR := rag_solution
TEST_DIR := tests
SOURCE_DIR := ./backend/rag_solution
TEST_DIR := ./backend/tests
PROJECT_DIRS := $(SOURCE_DIR) $(TEST_DIR)

# Project info
PROJECT_NAME ?= rag_modulo
PROJECT_NAME ?= rag-modulo
PYTHON_VERSION ?= 3.11
PROJECT_VERSION ?= v$(shell poetry version -s)
PROJECT_VERSION ?= 1.0.0

# Tools
DOCKER_COMPOSE := podman-compose
DOCKER_COMPOSE := docker-compose

# Set a default value for VECTOR_DB if not already set
VECTOR_DB ?= milvus

.DEFAULT_GOAL := help

.PHONY: init-env init check-toml format lint audit test run-services build-app run-app clean all info help api-test install-deps newman-test run-backend run-frontend
.PHONY: init-env build-frontend build-backend build-tests build-all test api-test newman-test all-test run-app run-backend run-frontend run-services stop-containers clean create-volumes logs info help

# Init
init-env:
@touch .env
@echo "PROJECT_NAME=${PROJECT_NAME}" >> .env
@echo "PYTHON_VERSION=${PYTHON_VERSION}" >> .env
@echo "VECTOR_DB=${VECTOR_DB}" >> .env

install-deps:
pip install -r requirements.txt
# Build
build-frontend:
$(DOCKER_COMPOSE) build frontend

init: init-env install-deps
build-backend:
$(DOCKER_COMPOSE) build backend

check-toml:
# No equivalent for pip, so this can be left empty or removed
build-tests:
$(DOCKER_COMPOSE) build test

format:
ruff check $(SOURCE_DIR) && black $(PROJECT_DIRS) && isort $(PROJECT_DIRS)
build-all:
@echo "Building application containers..."
$(DOCKER_COMPOSE) --profile default --profile test build

lint:
ruff check $(SOURCE_DIR)
mypy --install-types --show-error-codes --non-interactive $(SOURCE_DIR)
# Test
test: build-all run-app
$(DOCKER_COMPOSE) run test pytest -v -s -m "not (chromadb or elasticsearch or pinecone or weaviate)" || { echo "Tests failed"; $(MAKE) stop-containers; exit 1; }

audit:
bandit -r $(SOURCE_DIR) -x $(TEST_DIR)
api-test: build-all run-app
$(DOCKER_COMPOSE) run test pytest -v -s -m "api and not (chromadb or elasticsearch or pinecone or weaviate)" || { echo "API Tests failed"; $(MAKE) stop-containers; exit 1; }

test: run-services
pytest $(TEST_DIR) || { echo "Tests failed"; exit 1; }
@trap '$(DOCKER_COMPOSE) down' EXIT; \
echo "Waiting for Docker containers to stop..."
@while docker ps | grep -q "milvus-standalone"; do sleep 1; done
newman-test: build-all run-app
$(DOCKER_COMPOSE) run test newman run tests/postman/rag_modulo_api_collection.json --env-var "backend_base_url=${REACT_APP_API_URL}" || { echo "Postman Tests failed"; $(MAKE) stop-containers; exit 1; }

api-test: run-services
pytest $(TEST_DIR)/api -v -m api || { echo "API Tests failed"; exit 1; }
@trap '$(DOCKER_COMPOSE) down' EXIT; \
echo "Waiting for Docker containers to stop..."
@while docker ps | grep -q "milvus-standalone"; do sleep 1; done
all-test: build-all run-app
$(DOCKER_COMPOSE) run test pytest -v -s -m "not (chromadb or elasticsearch or pinecone or weaviate)" || { echo "Tests failed"; $(MAKE) stop-containers; exit 1; }
$(DOCKER_COMPOSE) run test newman run tests/postman/rag_modulo_api_collection.json --env-var "backend_base_url=${REACT_APP_API_URL}" || { echo "Postman Tests failed"; $(MAKE) stop-containers; exit 1; }

newman-test: run-services
newman run postman/rag_modulo_api_collection.json
@trap '$(DOCKER_COMPOSE) down' EXIT; \
echo "Waiting for Docker containers to stop..."
@while docker ps | grep -q "milvus-standalone"; do sleep 1; done
# Run
run-app: build-all run-backend run-frontend
@echo "All application containers are now running."

create-volumes:
@echo "Creating volume directories with correct permissions..."
@mkdir -p ./volumes/postgres ./volumes/etcd ./volumes/minio ./volumes/milvus
@chmod -R 777 ./volumes
@echo "Volume directories created and permissions set."
run-backend: run-services
@echo "Starting backend..."
$(DOCKER_COMPOSE) up -d backend
@echo "Backend is now running."

run-frontend: run-services
@echo "Starting frontend..."
$(DOCKER_COMPOSE) up -d frontend
@echo "Frontend is now running."

run-services: create-volumes
@if [ -z "$(VECTOR_DB)" ]; then \
Expand Down Expand Up @@ -128,34 +129,10 @@ run-services: create-volumes
@echo "Milvus logs saved to milvus.log:"
@$(DOCKER_COMPOSE) logs milvus-standalone > milvus.log

run-tests:
$(DOCKER_COMPOSE) run --rm test pytest $(ARGS)

build-frontend:
docker build -t $(PROJECT_NAME)-frontend -f webui/Dockerfile.frontend ./webui

build-backend:
docker build --build-arg CACHEBUST=$(date +%s) -t $(PROJECT_NAME)-backend --no-cache -f Dockerfile.backend .

build-app:
@echo "Building application containers..."
$(DOCKER_COMPOSE) build

run-backend: run-services
@echo "Starting backend..."
$(DOCKER_COMPOSE) up -d backend
@echo "Backend is now running."

run-frontend: run-services
@echo "Starting frontend..."
$(DOCKER_COMPOSE) up -d frontend
@echo "Frontend is now running."

run-app: build-app run-backend run-frontend
@echo "All application containers are now running."

logs:
$(DOCKER_COMPOSE) logs -f
# Stop / clean
stop-containers:
echo "Stopping containers and removing volumes ..."
$(DOCKER_COMPOSE) down -v

clean:
@echo "Cleaning up Docker Compose resources..."
Expand All @@ -164,9 +141,18 @@ clean:
-@podman pod rm -f $(podman pod ps -q) || true
-@podman rm -f $(podman ps -a -q) || true
-@podman volume prune -f || true
-@podman container prune -f || true
rm -rf .pytest_cache .mypy_cache data volumes my_chroma_data tests

all: format lint audit test
# Service / reusable targets
create-volumes:
@echo "Creating volume directories with correct permissions..."
@mkdir -p ./volumes/postgres ./volumes/etcd ./volumes/minio ./volumes/milvus
@chmod -R 777 ./volumes
@echo "Volume directories created and permissions set."

logs:
$(DOCKER_COMPOSE) logs -f

info:
@echo "Project name: ${PROJECT_NAME}"
Expand All @@ -178,43 +164,22 @@ help:
@echo "Usage: make [target]"
@echo ""
@echo "Targets:"
@echo " init-env Initialize .env file with default values"
@echo " init Initialize environment and install dependencies"
@echo " install-deps Install project dependencies"
@echo " check-toml Check TOML files for syntax errors"
@echo " format Format code using ruff, black, and isort"
@echo " lint Lint code using ruff and mypy"
@echo " audit Audit code using bandit"
@echo " test Run all tests using pytest"
@echo " api-test Run API tests using pytest"
@echo " newman-test Run API tests using Newman"
@echo " run-services Start services using Docker Compose"
@echo " build-app Build app using Docker Compose"
@echo " run-backend Run backend using Docker Compose"
@echo " run-frontend Run frontend using Docker Compose"
@echo " run-app Run both backend and frontend using Docker Compose"
@echo " logs View logs of running containers"
@echo " clean Clean up Docker Compose volumes and cache"
@echo " all Format, lint, audit, and test"
@echo " init-env Initialize .env file with default values"
@echo " build-frontend Build frontend code/container"
@echo " build-backend Build backend code/container"
@echo " build-tests Build test code/container"
@echo " build-all Build frontend/backend/test code/container"
@echo " test Run all tests using pytest"
@echo " api-test Run API tests using pytest"
@echo " newman-test Run API tests using newman"
@echo " all-test Run all tests using pytest and newman"
@echo " run-app Run both backend and frontend using Docker Compose"
@echo " run-backend Run backend using Docker Compose"
@echo " run-frontend Run frontend using Docker Compose"
@echo " run-services Run services using Docker Compose"
@echo " stop-containers Stop all containers using Docker Compose"
@echo " clean Clean up Docker Compose volumes and cache"
@echo " create-volumes Create folders for container volumes"
@echo " logs View logs of running containers"
@echo " info Display project information"
@echo " help Display this help message"
@echo " dev Start development environment"
@echo " dev-build Build development environment"
@echo " dev-down Stop development environment"
@echo " dev-backend Start backend development server"
@echo " dev-frontend Start frontend development server"

dev:
docker-compose -f docker-compose.dev.yml up

dev-build:
docker-compose -f docker-compose.dev.yml build

dev-down:
docker-compose -f docker-compose.dev.yml down

dev-backend:
uvicorn main:app --reload --host 0.0.0.0 --port 8000

dev-frontend:
cd webui && npm start
@echo " help Display this help message"
Loading