Merge pull request #209 from martincollignon/fix/add-parsers-cleanup #164
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy Sync Jobs | |
| on: | |
| push: | |
| branches: [ main ] | |
| paths: | |
| - 'backend/**' | |
| workflow_dispatch: | |
| env: | |
| PROJECT_ID: landbrugsdata-1 | |
| REGION: europe-west1 | |
| jobs: | |
| deploy: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - id: 'auth' | |
| name: 'Authenticate to Google Cloud' | |
| uses: 'google-github-actions/auth@v2' | |
| with: | |
| credentials_json: '${{ secrets.GCP_SA_KEY }}' | |
| - name: Set up Cloud SDK | |
| uses: google-github-actions/setup-gcloud@v2 | |
| - name: Build and Push Containers | |
| run: | | |
| cd backend | |
| # Run builds in parallel and capture outputs | |
| (gcloud builds submit --config cloudbuild.yaml --substitutions=_SERVICE_NAME=data-sync,_DOCKERFILE=Dockerfile.sync . --quiet) & | |
| PID1=$! | |
| (gcloud builds submit --config cloudbuild.yaml --substitutions=_SERVICE_NAME=dataflow-processing,_DOCKERFILE=Dockerfile.processing . --quiet) & | |
| PID2=$! | |
| # Wait for both builds and check exit status | |
| wait $PID1 || { echo "data-sync build failed"; exit 1; } | |
| wait $PID2 || { echo "dataflow-processing build failed"; exit 1; } | |
| - name: Deploy Cloud Run Jobs | |
| run: | | |
| # Deploy sync jobs | |
| declare -A jobs=( | |
| ["cadastral-sync"]="4|8Gi|DB_PASSWORD=db-password:latest,DATAFORDELER_USERNAME=datafordeler-username:latest,DATAFORDELER_PASSWORD=datafordeler-password:latest|cadastral" | |
| ["wetlands-sync"]="4|16Gi|DB_PASSWORD=db-password:latest|wetlands" | |
| ["water-projects-sync"]="4|8Gi|DB_PASSWORD=db-password:latest|water_projects" | |
| ["agricultural-fields-sync"]="4|8Gi|DB_PASSWORD=db-password:latest|agricultural_fields" | |
| ["property-owners-sync"]="4|8Gi|SSH_KEY=ssh-brugerdatafordeleren:latest|property_owners" | |
| ) | |
| for job in "${!jobs[@]}"; do | |
| IFS='|' read -r cpu memory secrets sync_type <<< "${jobs[$job]}" | |
| echo "Deploying $job..." | |
| ENV_VARS="DB_HOST=/cloudsql/$PROJECT_ID:$REGION:landbrugsdata-db,DB_NAME=landbrugsdata,DB_USER=landbrugsdata,SYNC_TYPE=$sync_type" | |
| if ! gcloud run jobs update $job \ | |
| --image gcr.io/$PROJECT_ID/data-sync \ | |
| --region $REGION \ | |
| --service-account="cadastral-sync-sa@$PROJECT_ID.iam.gserviceaccount.com" \ | |
| --set-secrets="$secrets" \ | |
| --set-env-vars="$ENV_VARS" \ | |
| --cpu="$cpu" \ | |
| --memory="$memory" \ | |
| --max-retries=3 \ | |
| --task-timeout=6h \ | |
| --set-cloudsql-instances=$PROJECT_ID:$REGION:landbrugsdata-db; then | |
| echo "Job doesn't exist, creating new job..." | |
| gcloud run jobs create $job \ | |
| --image gcr.io/$PROJECT_ID/data-sync \ | |
| --region $REGION \ | |
| --service-account="cadastral-sync-sa@$PROJECT_ID.iam.gserviceaccount.com" \ | |
| --set-secrets="$secrets" \ | |
| --set-env-vars="$ENV_VARS" \ | |
| --cpu="$cpu" \ | |
| --memory="$memory" \ | |
| --max-retries=3 \ | |
| --task-timeout=6h \ | |
| --set-cloudsql-instances=$PROJECT_ID:$REGION:landbrugsdata-db | |
| fi | |
| echo "Deployment of $job completed successfully" | |
| done | |
| # Remove dataflow processing job as it's no longer needed | |
| echo "Note: dataflow-processing job is no longer needed for dissolution operations" |