feat: add recoverer for failed tasks

Makefile

@@ -28,6 +28,9 @@ importer-tests:
 alias-tests:
 	cd gcp/workers/alias && ./run_tests.sh
 
+recoverer-tests:
+	cd gcp/workers/recoverer && ./run_tests.sh
+
 website-tests:
 	cd gcp/website && ./run_tests.sh
 
@@ -73,4 +76,4 @@ run-api-server-test:
 	cd gcp/api && $(install-cmd) && GOOGLE_CLOUD_PROJECT=oss-vdb-test OSV_VULNERABILITIES_BUCKET=osv-test-vulnerabilities $(run-cmd) python test_server.py $(HOME)/.config/gcloud/application_default_credentials.json $(ARGS)
 
 # TODO: API integration tests.
-all-tests: lib-tests worker-tests importer-tests alias-tests website-tests vulnfeed-tests
+all-tests: lib-tests worker-tests importer-tests alias-tests recoverer-tests website-tests vulnfeed-tests

deployment/build-and-stage.yaml

@@ -98,6 +98,15 @@ steps:
   args: ['push', '--all-tags', 'gcr.io/oss-vdb/alias-computation']
   waitFor: ['build-alias-computation', 'cloud-build-queue']
 
+- name: gcr.io/cloud-builders/docker
+  args: ['build', '-t', 'gcr.io/oss-vdb/recoverer:latest', '-t', 'gcr.io/oss-vdb/recoverer:$COMMIT_SHA', '.']
+  dir: 'gcp/workers/recoverer'
+  id: 'build-recoverer'
+  waitFor: ['build-worker']
+- name: gcr.io/cloud-builders/docker
+  args: ['push', '--all-tags', 'gcr.io/oss-vdb/recoverer']
+  waitFor: ['build-recoverer', 'cloud-build-queue']
+
 # Build/push staging-api-test images to gcr.io/oss-vdb-test.
 - name: gcr.io/cloud-builders/docker
   args: ['build', '-t', 'gcr.io/oss-vdb-test/staging-api-test:latest', '-t', 'gcr.io/oss-vdb-test/staging-api-test:$COMMIT_SHA', '.']
@@ -291,7 +300,8 @@ steps:
      debian-copyright-mirror=gcr.io/oss-vdb/debian-copyright-mirror:$COMMIT_SHA,\
      cpe-repo-gen=gcr.io/oss-vdb/cpe-repo-gen:$COMMIT_SHA,\
      nvd-cve-osv=gcr.io/oss-vdb/nvd-cve-osv:$COMMIT_SHA,\
-     nvd-mirror=gcr.io/oss-vdb/nvd-mirror:$COMMIT_SHA"
+     nvd-mirror=gcr.io/oss-vdb/nvd-mirror:$COMMIT_SHA,\
+     recoverer=gcr.io/oss-vdb/recoverer:$COMMIT_SHA"
   ]
   dir: deployment/clouddeploy/gke-workers
 
@@ -347,3 +357,4 @@ images:
 - 'gcr.io/oss-vdb/nvd-mirror:$COMMIT_SHA'
 - 'gcr.io/oss-vdb-test/staging-api-test:$COMMIT_SHA'
 - 'gcr.io/oss-vdb-test/osv-linter:$COMMIT_SHA'
+- 'gcr.io/oss-vdb/recoverer:$COMMIT_SHA'

deployment/clouddeploy/gke-workers/base/kustomization.yaml

@@ -24,3 +24,4 @@ resources:
 - ksm_service_account.yaml
 - ksm_service.yaml
 - ksm_stateful_set.yaml
+- recoverer.yaml

deployment/clouddeploy/gke-workers/base/recoverer.yaml

@@ -0,0 +1,32 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: recoverer
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      name: recoverer
+  template:
+    metadata:
+      labels:
+        name: recoverer
+    spec:
+      containers:
+      - name: recoverer
+        image: recoverer
+        imagePullPolicy: Always

deployment/clouddeploy/gke-workers/environments/oss-vdb-test/kustomization.yaml

@@ -20,3 +20,4 @@ patches:
 - path: alias-computation.yaml
 - path: backup.yaml
 - path: generate-sitemap.yaml
+- path: recoverer.yaml

deployment/clouddeploy/gke-workers/environments/oss-vdb-test/recoverer.yaml

@@ -0,0 +1,14 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: recoverer
+spec:
+  template:
+    spec:
+      containers:
+      - name: recoverer
+        env:
+          - name: GOOGLE_CLOUD_PROJECT
+            value: oss-vdb-test
+          - name: OSV_VULNERABILITIES_BUCKET
+            value: osv-test-vulnerabilities

deployment/clouddeploy/gke-workers/environments/oss-vdb/kustomization.yaml

@@ -19,3 +19,4 @@ patches:
 - path: alias-computation.yaml
 - path: backup.yaml
 - path: generate-sitemap.yaml
+- path: recoverer.yaml

deployment/clouddeploy/gke-workers/environments/oss-vdb/recoverer.yaml

@@ -0,0 +1,13 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: recoverer
+spec:
+  template:
+    spec:
+      containers:
+      - name: recoverer
+        env:
+          - name: GOOGLE_CLOUD_PROJECT
+            value: oss-vdb
+

deployment/deploy-prod.yaml

@@ -50,6 +50,8 @@ steps:
   args: ['container', 'images', 'add-tag', '--quiet', 'gcr.io/oss-vdb/osv-server:$COMMIT_SHA', 'gcr.io/oss-vdb/osv-server:$TAG_NAME']
 - name: gcr.io/cloud-builders/gcloud
   args: ['container', 'images', 'add-tag', '--quiet', 'gcr.io/oss-vdb/osv-website:$COMMIT_SHA', 'gcr.io/oss-vdb/osv-website:$TAG_NAME']
+- name: gcr.io/cloud-builders/gcloud
+  args: ['container', 'images', 'add-tag', '--quiet', 'gcr.io/oss-vdb/recoverer:$COMMIT_SHA', 'gcr.io/oss-vdb/recoverer:$TAG_NAME']
 
 serviceAccount: 'projects/oss-vdb/serviceAccounts/deployment@oss-vdb.iam.gserviceaccount.com'
 options:

deployment/terraform/modules/osv/pubsub_tasks.tf

@@ -64,3 +64,22 @@ resource "google_pubsub_topic_iam_member" "failed_tasks_service_publisher" {
   role    = "roles/pubsub.publisher"
   member  = "serviceAccount:${google_project_service_identity.pubsub.email}"
 }
+
+resource "google_pubsub_subscription" "recovery" {
+  project                    = var.project_id
+  name                       = "recovery"
+  topic                      = google_pubsub_topic.failed_tasks.id
+  message_retention_duration = "604800s" # 7 days
+  ack_deadline_seconds       = 600
+
+  expiration_policy {
+    ttl = "" # never expires
+  }
+}
+
+resource "google_pubsub_subscription_iam_member" "recovery_service_subscriber" {
+  project      = var.project_id
+  subscription = google_pubsub_subscription.recovery.name
+  role         = "roles/pubsub.subscriber"
+  member       = "serviceAccount:${google_project_service_identity.pubsub.email}"
+}