feat: Optimize CI/CD pipeline for faster PR feedback

.github/workflows/01-lint.yml

@@ -10,6 +10,12 @@ on:
   push:
     branches: [main]
 
+# 🔄 Concurrency Control
+# Cancel outdated workflow runs when new commits are pushed
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 permissions:
   contents: read
 

.github/workflows/security.yml → .github/workflows/02-security.yml

@@ -1,60 +1,78 @@
 name: Security Scan
 
+# 📋 Purpose
+# Scan codebase for secrets and credentials using Gitleaks and TruffleHog
+# Runs on every PR and push to main to catch leaked secrets early
+
 on:
   pull_request:
     branches: [ main ]
+  push:
+    branches: [ main ]
   workflow_dispatch:
 
+# 🔄 Concurrency Control
+# Cancel outdated workflow runs when new commits are pushed
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 permissions:
   contents: read
   security-events: write
 
 jobs:
   gitleaks:
-    name: gitleaks
+    name: 🔐 Gitleaks Secret Scanning
     runs-on: ubuntu-latest
     continue-on-error: true  # Don't block PR on false positives
     steps:
-      - name: Checkout code
+      # 0️⃣ Checkout source code with full history for secret scanning
+      - name: 📥 Checkout code
         uses: actions/checkout@v4
         with:
-          fetch-depth: 0
+          fetch-depth: 0  # Need full history to scan all commits
 
-      - name: Run gitleaks
+      # 1️⃣ Run Gitleaks to detect secrets in git history
+      - name: 🔍 Run Gitleaks
         uses: gitleaks/gitleaks-action@v2
         env:
           GITLEAKS_CONFIG: .gitleaks.toml
           GITLEAKS_ENABLE_UPLOAD: false
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         continue-on-error: true
 
-      - name: Report gitleaks status
+      # 2️⃣ Report findings if secrets detected
+      - name: 📊 Report Gitleaks status
         if: failure()
         run: |
           echo "⚠️ Gitleaks detected potential secrets. Please review the findings above."
           echo "If these are false positives, update .gitleaks.toml allowlist."
           exit 0
 
   trufflehog:
-    name: trufflehog
+    name: 🐷 TruffleHog Secret Scanning
     runs-on: ubuntu-latest
     continue-on-error: true  # Don't block PR on false positives
     steps:
-      - name: Checkout code
+      # 0️⃣ Checkout source code with full history
+      - name: 📥 Checkout code
         uses: actions/checkout@v4
         with:
-          fetch-depth: 0
+          fetch-depth: 0  # Need full history for differential scanning
 
-      - name: Run trufflehog
+      # 1️⃣ Run TruffleHog to detect verified secrets
+      - name: 🔍 Run TruffleHog
         uses: trufflesecurity/trufflehog@v3.63.4
         with:
           path: ./
           base: ${{ github.event.pull_request.base.sha }}
           head: ${{ github.event.pull_request.head.sha }}
-          extra_args: --only-verified
+          extra_args: --only-verified  # Only flag verified secrets
         continue-on-error: true
 
-      - name: Report trufflehog status
+      # 2️⃣ Report findings if verified secrets detected
+      - name: 📊 Report TruffleHog status
         if: failure()
         run: |
           echo "⚠️ TruffleHog detected verified secrets. Please review the findings above."

.github/workflows/03-build-secure.yml

@@ -5,13 +5,13 @@ on:
     branches: [main]
     paths:
       # Only run on PR if Dockerfiles or dependencies change
+      # NOTE: Do NOT include workflow file itself to avoid expensive builds on CI/CD changes
       - 'backend/Dockerfile.backend'
       - 'backend/pyproject.toml'
       - 'backend/poetry.lock'
       - 'frontend/Dockerfile.frontend'
       - 'frontend/package*.json'
       - 'docker-compose*.yml'
-      - '.github/workflows/03-build-secure.yml'
   push:
     branches: [main]
     # Always scan on merge to main
@@ -21,6 +21,11 @@ on:
   workflow_dispatch:
     # Manual trigger option
 
+# Cancel outdated workflow runs for the same PR
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 permissions:
   contents: read
   security-events: write  # For SARIF uploads

.github/workflows/04-pytest.yml

@@ -0,0 +1,92 @@
+name: Unit Tests
+
+# 📋 Purpose
+# Run Python unit tests with pytest
+# Validates atomic/unit tests without requiring infrastructure
+
+on:
+  pull_request:
+    branches: [main]
+  push:
+    branches: [main]
+
+# 🔄 Concurrency Control
+# Cancel outdated workflow runs when new commits are pushed
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+env:
+  # CI Environment Variables
+  TESTING: true
+  SKIP_AUTH: true
+  DEVELOPMENT_MODE: true
+  # Test environment variables
+  JWT_SECRET_KEY: test-secret-key-for-ci
+  RAG_LLM: openai
+  WATSONX_INSTANCE_ID: test-instance-id
+  WATSONX_APIKEY: test-api-key
+  WATSONX_URL: https://test.watsonx.com
+  VECTOR_DB: milvus
+  MILVUS_HOST: milvus-standalone
+  MILVUS_PORT: 19530
+  EMBEDDING_MODEL: sentence-transformers/all-minilm-l6-v2
+  DATA_DIR: /tmp/test-data
+
+jobs:
+  unit-tests:
+    name: 🧪 Unit Tests
+    runs-on: ubuntu-latest
+    steps:
+      # 0️⃣ Checkout source code
+      - name: 📥 Checkout code
+        uses: actions/checkout@v4
+
+      # 1️⃣ Setup Python environment
+      - name: 🐍 Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.12'
+
+      # 2️⃣ Install Poetry package manager
+      - name: 📦 Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: latest
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+
+      # 3️⃣ Cache Poetry dependencies for faster builds
+      - name: 📚 Cache Poetry dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/pypoetry
+            backend/.venv
+          key: ${{ runner.os }}-poetry-${{ hashFiles('backend/poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-
+
+      # 4️⃣ Install Python dependencies
+      - name: 📥 Install dependencies
+        run: cd backend && poetry install --with dev,test
+
+      # 5️⃣ Run unit/atomic tests
+      - name: 🧪 Run unit tests
+        run: |
+          cd backend
+          poetry run pytest tests/ -m "unit or atomic" --tb=short -v --maxfail=5
+
+      # 6️⃣ Generate coverage report (optional)
+      - name: 📊 Run tests with coverage
+        if: success()
+        run: |
+          cd backend
+          poetry run pytest tests/ -m "unit or atomic" \
+            --cov=rag_solution \
+            --cov-report=term-missing \
+            --cov-report=html \
+            --tb=short

.github/workflows/ci.yml → .github/workflows/05-ci.yml

@@ -1,10 +1,12 @@
-name: CI/CD Pipeline
+name: Main - Integration Tests & Build
+
+# This workflow runs full integration tests and builds containers
+# For fast PR checks, see pr-fast-check.yml
 
 on:
-  pull_request:
-    branches: [main]
   push:
-    branches: [main]
+    branches: [main]  # Only on merge to main
+  workflow_dispatch:  # Manual trigger
 
 permissions:
   contents: read