deploy-sync-jobs.yml

name: Deploy Sync Jobs

on:
  push:
    branches: [ main ]
    paths:
      - 'backend/**'
  workflow_dispatch:

env:
  PROJECT_ID: landbrugsdata-1
  REGION: europe-west1

jobs:
  deploy:
    runs-on: ubuntu-latest
    permissions:
      contents: 'read'
      id-token: 'write'

    steps:
    - uses: actions/checkout@v4

    - id: 'auth'
      name: 'Authenticate to Google Cloud'
      uses: 'google-github-actions/auth@v2'
      with:
        credentials_json: '${{ secrets.GCP_SA_KEY }}'

    - name: Set up Cloud SDK
      uses: google-github-actions/setup-gcloud@v2

    - name: Build and Push Containers
      run: |
        cd backend
        # Run builds in parallel and capture outputs
        (gcloud builds submit --config cloudbuild.yaml --substitutions=_SERVICE_NAME=data-sync,_DOCKERFILE=Dockerfile.sync . --quiet) &
        PID1=$!
        (gcloud builds submit --config cloudbuild.yaml --substitutions=_SERVICE_NAME=dataflow-processing,_DOCKERFILE=Dockerfile.processing . --quiet) &
        PID2=$!
        
        # Wait for both builds and check exit status
        wait $PID1 || { echo "data-sync build failed"; exit 1; }
        wait $PID2 || { echo "dataflow-processing build failed"; exit 1; }

    - name: Deploy Cloud Run Jobs
      run: |
        # Deploy sync jobs
        declare -A jobs=(
          ["cadastral-sync"]="4|8Gi|DB_PASSWORD=db-password:latest,DATAFORDELER_USERNAME=datafordeler-username:latest,DATAFORDELER_PASSWORD=datafordeler-password:latest|cadastral"
          ["wetlands-sync"]="4|16Gi|DB_PASSWORD=db-password:latest|wetlands"
          ["water-projects-sync"]="4|8Gi|DB_PASSWORD=db-password:latest|water_projects"
          ["agricultural-fields-sync"]="8|16Gi|DB_PASSWORD=db-password:latest|agricultural_fields"
          ["herd-data-sync"]="4|8Gi|DB_PASSWORD=db-password:latest,FVM_USERNAME=fvm_username:latest,FVM_PASSWORD=fvm_password:latest|herd_data"
          ["bnbo-status-sync"]="4|8Gi|DB_PASSWORD=db-password:latest|bnbo_status"
        )

        for job in "${!jobs[@]}"; do
          IFS='|' read -r cpu memory secrets sync_type <<< "${jobs[$job]}"
          
          echo "Deploying $job..."
          
          ENV_VARS="DB_HOST=/cloudsql/$PROJECT_ID:$REGION:landbrugsdata-db,DB_NAME=landbrugsdata,DB_USER=landbrugsdata,SYNC_TYPE=$sync_type"
          
          if ! gcloud run jobs update $job \
            --image gcr.io/$PROJECT_ID/data-sync \
            --region $REGION \
            --service-account="cadastral-sync-sa@$PROJECT_ID.iam.gserviceaccount.com" \
            --set-secrets="$secrets" \
            --set-env-vars="$ENV_VARS" \
            --cpu="$cpu" \
            --memory="$memory" \
            --max-retries=3 \
            --task-timeout=6h \
            --set-cloudsql-instances=$PROJECT_ID:$REGION:landbrugsdata-db; then
            
            echo "Job doesn't exist, creating new job..."
            gcloud run jobs create $job \
              --image gcr.io/$PROJECT_ID/data-sync \
              --region $REGION \
              --service-account="cadastral-sync-sa@$PROJECT_ID.iam.gserviceaccount.com" \
              --set-secrets="$secrets" \
              --set-env-vars="$ENV_VARS" \
              --cpu="$cpu" \
              --memory="$memory" \
              --max-retries=3 \
              --task-timeout=6h \
              --set-cloudsql-instances=$PROJECT_ID:$REGION:landbrugsdata-db
          fi
          
          echo "Deployment of $job completed successfully"
        done

        # Remove dataflow processing job as it's no longer needed
        echo "Note: dataflow-processing job is no longer needed for dissolution operations"