fix: address all critical issues in PR #411

- Fix IBM Code Engine volume mounting (remove non-functional persistent volumes)
- Add comprehensive warnings about ephemeral storage limitations
- Remove hardcoded credentials and add password validation
- Fix Terraform backend DynamoDB issue for IBM Cloud
- Complete Ansible requirements.yml file
- Add missing inventory files for all cloud providers
- Add health checks for all infrastructure services
- Create CI validation workflow for Terraform/Ansible
- Make backup scripts executable
- Update documentation with storage limitations

Resolves critical issues identified in PR review:
- Missing data persistence (now documented as limitation)
- Hardcoded credentials (removed, added validation)
- Terraform backend issues (fixed for IBM Cloud)
- Missing files (added inventory files and CI validation)
- Health checks (added for all infrastructure services)

Author: manavgup

.github/workflows/terraform-ansible-validation.yml

@@ -0,0 +1,277 @@
+# =============================================================================
+# GitHub Actions Workflow: Terraform and Ansible Validation
+# =============================================================================
+# Description: Validates Terraform configurations and Ansible playbooks
+#              for syntax, security, and best practices.
+#
+# Author: RAG Modulo Team
+# Last Updated: 2025-01-13
+# Version: 1.0
+#
+# Triggers:
+#   - Pull requests to main branch
+#   - Push to main branch
+#   - Manual trigger
+#
+# =============================================================================
+
+name: "Terraform & Ansible Validation"
+
+on:
+  push:
+    branches: [ main ]
+    paths:
+      - 'deployment/**'
+      - '.github/workflows/terraform-ansible-validation.yml'
+  pull_request:
+    branches: [ main ]
+    paths:
+      - 'deployment/**'
+      - '.github/workflows/terraform-ansible-validation.yml'
+  workflow_dispatch:
+
+env:
+  TF_VERSION: "1.6.0"
+  ANSIBLE_VERSION: "8.0.0"
+
+jobs:
+  # ==========================================================================
+  # TERRAFORM VALIDATION
+  # ==========================================================================
+
+  terraform-validate:
+    name: "🔍 Terraform Validation"
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./deployment/terraform
+
+    steps:
+      - name: "📥 Checkout code"
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: "📦 Setup Terraform"
+        uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: ${{ env.TF_VERSION }}
+
+      - name: "🔍 Terraform Format Check"
+        run: |
+          echo "Checking Terraform formatting..."
+          terraform fmt -check -recursive
+          echo "✅ Terraform formatting is correct"
+
+      - name: "🔍 Terraform Init"
+        run: |
+          echo "Initializing Terraform..."
+          terraform init -backend=false
+          echo "✅ Terraform initialized successfully"
+
+      - name: "🔍 Terraform Validate"
+        run: |
+          echo "Validating Terraform configurations..."
+          terraform validate
+          echo "✅ Terraform validation passed"
+
+      - name: "🔍 Terraform Plan (Dry Run)"
+        run: |
+          echo "Running Terraform plan (dry run)..."
+          terraform plan -var-file="environments/ibm/dev.tfvars" -out=tfplan
+          echo "✅ Terraform plan completed successfully"
+        env:
+          TF_VAR_ibmcloud_api_key: "dummy-key-for-validation"
+          TF_VAR_container_registry_password: "dummy-password-for-validation"
+          TF_VAR_database_password: "dummy-database-password-for-validation"
+          TF_VAR_minio_password: "dummy-minio-password-for-validation"
+
+      - name: "🔍 Security Scan (tfsec)"
+        uses: aquasecurity/tfsec-action@v1.0.3
+        with:
+          working_directory: ./deployment/terraform
+          soft_fail: true
+
+      - name: "🔍 Security Scan (checkov)"
+        uses: bridgecrewio/checkov-action@master
+        with:
+          directory: ./deployment/terraform
+          framework: terraform
+          soft_fail: true
+
+  # ==========================================================================
+  # ANSIBLE VALIDATION
+  # ==========================================================================
+
+  ansible-validate:
+    name: "🔍 Ansible Validation"
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./deployment/ansible
+
+    steps:
+      - name: "📥 Checkout code"
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: "📦 Setup Python"
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: "📦 Install Ansible"
+        run: |
+          echo "Installing Ansible..."
+          pip install ansible==${{ env.ANSIBLE_VERSION }}
+          ansible --version
+          echo "✅ Ansible installed successfully"
+
+      - name: "📦 Install Ansible Collections"
+        run: |
+          echo "Installing Ansible collections..."
+          ansible-galaxy collection install -r requirements.yml
+          echo "✅ Ansible collections installed successfully"
+
+      - name: "🔍 Ansible Syntax Check"
+        run: |
+          echo "Checking Ansible playbook syntax..."
+          ansible-playbook --syntax-check playbooks/deploy-rag-modulo.yml
+          ansible-playbook --syntax-check playbooks/backup-rag-modulo.yml
+          echo "✅ Ansible syntax check passed"
+
+      - name: "🔍 Ansible Lint"
+        run: |
+          echo "Installing ansible-lint..."
+          pip install ansible-lint
+
+          echo "Running ansible-lint..."
+          ansible-lint playbooks/deploy-rag-modulo.yml
+          ansible-lint playbooks/backup-rag-modulo.yml
+          echo "✅ Ansible lint check passed"
+
+      - name: "🔍 Ansible Dry Run"
+        run: |
+          echo "Running Ansible dry run..."
+          ansible-playbook --check --diff playbooks/deploy-rag-modulo.yml -i inventories/ibm/hosts.yml
+          echo "✅ Ansible dry run completed successfully"
+
+  # ==========================================================================
+  # DOCUMENTATION VALIDATION
+  # ==========================================================================
+
+  docs-validate:
+    name: "📚 Documentation Validation"
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: "📥 Checkout code"
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: "📦 Setup Python"
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: "📦 Install MkDocs"
+        run: |
+          echo "Installing MkDocs and plugins..."
+          pip install mkdocs mkdocs-material mkdocs-mermaid2-plugin
+          echo "✅ MkDocs installed successfully"
+
+      - name: "🔍 MkDocs Build"
+        run: |
+          echo "Building documentation..."
+          mkdocs build --strict
+          echo "✅ Documentation build completed successfully"
+
+      - name: "🔍 MkDocs Lint"
+        run: |
+          echo "Installing mkdocs-lint..."
+          pip install mkdocs-lint
+
+          echo "Running mkdocs-lint..."
+          mkdocs-lint
+          echo "✅ MkDocs lint check passed"
+
+  # ==========================================================================
+  # INTEGRATION VALIDATION
+  # ==========================================================================
+
+  integration-validate:
+    name: "🔗 Integration Validation"
+    runs-on: ubuntu-latest
+    needs: [terraform-validate, ansible-validate, docs-validate]
+
+    steps:
+      - name: "📥 Checkout code"
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: "🔍 Validate File Structure"
+        run: |
+          echo "Validating deployment file structure..."
+
+          # Check required directories exist
+          test -d "deployment/terraform/modules/ibm-cloud/code-engine" || exit 1
+          test -d "deployment/terraform/environments/ibm" || exit 1
+          test -d "deployment/ansible/playbooks" || exit 1
+          test -d "deployment/ansible/inventories" || exit 1
+          test -d "deployment/scripts" || exit 1
+
+          # Check required files exist
+          test -f "deployment/terraform/modules/ibm-cloud/code-engine/main.tf" || exit 1
+          test -f "deployment/terraform/modules/ibm-cloud/code-engine/variables.tf" || exit 1
+          test -f "deployment/terraform/environments/ibm/main.tf" || exit 1
+          test -f "deployment/terraform/environments/ibm/dev.tfvars" || exit 1
+          test -f "deployment/ansible/playbooks/deploy-rag-modulo.yml" || exit 1
+          test -f "deployment/ansible/playbooks/backup-rag-modulo.yml" || exit 1
+          test -f "deployment/ansible/requirements.yml" || exit 1
+          test -f "deployment/scripts/backup-rag-modulo.sh" || exit 1
+          test -f "deployment/scripts/restore-rag-modulo.sh" || exit 1
+
+          # Check inventory files exist
+          test -f "deployment/ansible/inventories/ibm/hosts.yml" || exit 1
+          test -f "deployment/ansible/inventories/aws/hosts.yml" || exit 1
+          test -f "deployment/ansible/inventories/azure/hosts.yml" || exit 1
+          test -f "deployment/ansible/inventories/gcp/hosts.yml" || exit 1
+
+          echo "✅ File structure validation passed"
+
+      - name: "🔍 Validate Script Permissions"
+        run: |
+          echo "Validating script permissions..."
+
+          # Check scripts are executable
+          test -x "deployment/scripts/backup-rag-modulo.sh" || exit 1
+          test -x "deployment/scripts/restore-rag-modulo.sh" || exit 1
+
+          echo "✅ Script permissions validation passed"
+
+      - name: "🔍 Validate Documentation Links"
+        run: |
+          echo "Validating documentation links..."
+
+          # Check if referenced documentation exists
+          if [ -f "docs/deployment/terraform-ansible-architecture.md" ]; then
+            echo "✅ Main documentation exists"
+          else
+            echo "⚠️  Main documentation missing"
+          fi
+
+          echo "✅ Documentation validation completed"
+
+      - name: "📊 Validation Summary"
+        run: |
+          echo "🎉 All validation checks completed successfully!"
+          echo ""
+          echo "✅ Terraform validation passed"
+          echo "✅ Ansible validation passed"
+          echo "✅ Documentation validation passed"
+          echo "✅ Integration validation passed"
+          echo ""
+          echo "🚀 Deployment configuration is ready for use!"

deployment/ansible/inventories/aws/hosts.yml

@@ -0,0 +1,60 @@
+# =============================================================================
+# Ansible Inventory: AWS Environment
+# =============================================================================
+# Description: Inventory file for AWS EKS deployment
+#
+# Author: RAG Modulo Team
+# Last Updated: 2025-01-13
+# Version: 1.0
+#
+# Usage:
+#   ansible-playbook -i inventories/aws/hosts.yml playbooks/deploy-rag-modulo.yml
+#
+# =============================================================================
+
+all:
+  children:
+    aws_cloud:
+      hosts:
+        localhost:
+          ansible_connection: local
+          ansible_python_interpreter: "{{ ansible_playbook_python }}"
+
+          # AWS Configuration
+          cloud_provider: aws
+          region: us-east-1
+          availability_zones: ["us-east-1a", "us-east-1b", "us-east-1c"]
+
+          # EKS Configuration
+          cluster_name: rag-modulo-cluster
+          node_group_name: rag-modulo-nodes
+          node_instance_type: t3.medium
+          node_desired_size: 2
+          node_max_size: 5
+          node_min_size: 1
+
+          # Application Configuration
+          app_name: rag-modulo
+          environment: dev
+          namespace: rag-modulo
+
+          # Deployment Configuration
+          terraform_state_path: "deployment/terraform/environments/aws/terraform.tfstate"
+          deployment_scripts_path: "deployment/scripts"
+
+          # Health Check URLs (will be set dynamically)
+          backend_health_url: ""
+          frontend_health_url: ""
+          backend_url: ""
+          frontend_url: ""
+
+          # Backup Configuration
+          backup_enabled: true
+          backup_retention_days: 30
+          backup_compression: gzip
+          s3_backup_bucket: rag-modulo-backups
+
+          # Notification Configuration
+          notification_enabled: false
+          slack_webhook: ""
+          email_notifications: ""

deployment/ansible/inventories/azure/hosts.yml

@@ -0,0 +1,59 @@
+# =============================================================================
+# Ansible Inventory: Azure Environment
+# =============================================================================
+# Description: Inventory file for Azure AKS deployment
+#
+# Author: RAG Modulo Team
+# Last Updated: 2025-01-13
+# Version: 1.0
+#
+# Usage:
+#   ansible-playbook -i inventories/azure/hosts.yml playbooks/deploy-rag-modulo.yml
+#
+# =============================================================================
+
+all:
+  children:
+    azure_cloud:
+      hosts:
+        localhost:
+          ansible_connection: local
+          ansible_python_interpreter: "{{ ansible_playbook_python }}"
+
+          # Azure Configuration
+          cloud_provider: azure
+          location: East US
+          resource_group: rag-modulo-rg
+
+          # AKS Configuration
+          cluster_name: rag-modulo-aks
+          node_count: 2
+          vm_size: Standard_B2s
+          kubernetes_version: "1.28"
+
+          # Application Configuration
+          app_name: rag-modulo
+          environment: dev
+          namespace: rag-modulo
+
+          # Deployment Configuration
+          terraform_state_path: "deployment/terraform/environments/azure/terraform.tfstate"
+          deployment_scripts_path: "deployment/scripts"
+
+          # Health Check URLs (will be set dynamically)
+          backend_health_url: ""
+          frontend_health_url: ""
+          backend_url: ""
+          frontend_url: ""
+
+          # Backup Configuration
+          backup_enabled: true
+          backup_retention_days: 30
+          backup_compression: gzip
+          storage_account: ragmodulobackups
+          container_name: rag-modulo-backups
+
+          # Notification Configuration
+          notification_enabled: false
+          slack_webhook: ""
+          email_notifications: ""