Fix broken cloud deploys by using depot ephemeral registry, skip the registry proxy

.changeset/pink-mice-battle.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Fix broken cloud deploys by using depot ephemeral registry

apps/webapp/app/env.server.ts

@@ -147,7 +147,10 @@ const EnvironmentSchema = z.object({
   CONTAINER_REGISTRY_ORIGIN: z.string().optional(),
   CONTAINER_REGISTRY_USERNAME: z.string().optional(),
   CONTAINER_REGISTRY_PASSWORD: z.string().optional(),
+  ENABLE_REGISTRY_PROXY: z.string().optional(),
   DEPLOY_REGISTRY_HOST: z.string().optional(),
+  DEPLOY_REGISTRY_USERNAME: z.string().optional(),
+  DEPLOY_REGISTRY_PASSWORD: z.string().optional(),
   DEPLOY_REGISTRY_NAMESPACE: z.string().default("trigger"),
   DEPLOY_TIMEOUT_MS: z.coerce
     .number()

apps/webapp/app/routes/api.v1.deployments.$deploymentId.start-indexing.ts → apps/webapp/app/routes/api.v2.deployments.$deploymentId.finalize.ts

@@ -1,9 +1,10 @@
 import { ActionFunctionArgs, json } from "@remix-run/server-runtime";
-import { StartDeploymentIndexingRequestBody } from "@trigger.dev/core/v3";
+import { FinalizeDeploymentRequestBody } from "@trigger.dev/core/v3";
 import { z } from "zod";
 import { authenticateApiRequest } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
-import { StartDeploymentIndexing } from "~/v3/services/startDeploymentIndexing.server";
+import { ServiceValidationError } from "~/v3/services/baseService.server";
+import { FinalizeDeploymentV2Service } from "~/v3/services/finalizeDeploymentV2";
 
 const ParamsSchema = z.object({
   deploymentId: z.string(),
@@ -34,21 +35,31 @@ export async function action({ request, params }: ActionFunctionArgs) {
   const { deploymentId } = parsedParams.data;
 
   const rawBody = await request.json();
-  const body = StartDeploymentIndexingRequestBody.safeParse(rawBody);
+  const body = FinalizeDeploymentRequestBody.safeParse(rawBody);
 
   if (!body.success) {
     return json({ error: "Invalid body", issues: body.error.issues }, { status: 400 });
   }
 
-  const service = new StartDeploymentIndexing();
+  try {
+    const service = new FinalizeDeploymentV2Service();
+    await service.call(authenticatedEnv, deploymentId, body.data);
 
-  const deployment = await service.call(authenticatedEnv, deploymentId, body.data);
-
-  return json(
-    {
-      id: deployment.friendlyId,
-      contentHash: deployment.contentHash,
-    },
-    { status: 200 }
-  );
+    return json(
+      {
+        id: deploymentId,
+      },
+      { status: 200 }
+    );
+  } catch (error) {
+    if (error instanceof ServiceValidationError) {
+      return json({ error: error.message }, { status: 400 });
+    } else if (error instanceof Error) {
+      logger.error("Error finalizing deployment", { error: error.message });
+      return json({ error: `Internal server error: ${error.message}` }, { status: 500 });
+    } else {
+      logger.error("Error finalizing deployment", { error: String(error) });
+      return json({ error: "Internal server error" }, { status: 500 });
+    }
+  }
 }

apps/webapp/app/v3/registryProxy.server.ts

@@ -435,6 +435,11 @@ function initializeProxy() {
     return;
   }
 
+  if (!env.ENABLE_REGISTRY_PROXY || env.ENABLE_REGISTRY_PROXY === "false") {
+    logger.info("Registry proxy is disabled");
+    return;
+  }
+
   return new RegistryProxy({
     origin: env.CONTAINER_REGISTRY_ORIGIN,
     auth: {

apps/webapp/app/v3/services/finalizeDeployment.server.ts

@@ -16,7 +16,7 @@ export class FinalizeDeploymentService extends BaseService {
     id: string,
     body: FinalizeDeploymentRequestBody
   ) {
-    const deployment = await this._prisma.workerDeployment.findUnique({
+    const deployment = await this._prisma.workerDeployment.findFirst({
       where: {
         friendlyId: id,
         environmentId: authenticatedEnv.id,
@@ -48,6 +48,12 @@ export class FinalizeDeploymentService extends BaseService {
       throw new ServiceValidationError("Worker deployment is not in DEPLOYING status");
     }
 
+    let imageReference = body.imageReference;
+
+    if (registryProxy && body.selfHosted !== true && body.skipRegistryProxy !== true) {
+      imageReference = registryProxy.rewriteImageReference(body.imageReference);
+    }
+
     // Link the deployment with the background worker
     const finalizedDeployment = await this._prisma.workerDeployment.update({
       where: {
@@ -56,10 +62,7 @@ export class FinalizeDeploymentService extends BaseService {
       data: {
         status: "DEPLOYED",
         deployedAt: new Date(),
-        imageReference:
-          registryProxy && body.selfHosted !== true
-            ? registryProxy.rewriteImageReference(body.imageReference)
-            : body.imageReference,
+        imageReference: imageReference,
       },
     });
 

apps/webapp/app/v3/services/finalizeDeploymentV2.ts

@@ -0,0 +1,259 @@
+import { ExternalBuildData, FinalizeDeploymentRequestBody } from "@trigger.dev/core/v3/schemas";
+import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
+import { logger } from "~/services/logger.server";
+import { BaseService, ServiceValidationError } from "./baseService.server";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { mkdtemp, writeFile } from "node:fs/promises";
+import { env } from "~/env.server";
+import { depot as execDepot } from "@depot/cli";
+import { FinalizeDeploymentService } from "./finalizeDeployment.server";
+
+export class FinalizeDeploymentV2Service extends BaseService {
+  public async call(
+    authenticatedEnv: AuthenticatedEnvironment,
+    id: string,
+    body: FinalizeDeploymentRequestBody
+  ) {
+    // if it's self hosted, lets just use the v1 finalize deployment service
+    if (body.selfHosted) {
+      const finalizeService = new FinalizeDeploymentService();
+
+      return finalizeService.call(authenticatedEnv, id, body);
+    }
+
+    const deployment = await this._prisma.workerDeployment.findFirst({
+      where: {
+        friendlyId: id,
+        environmentId: authenticatedEnv.id,
+      },
+      include: {
+        environment: true,
+        worker: {
+          include: {
+            tasks: true,
+            project: true,
+          },
+        },
+      },
+    });
+
+    if (!deployment) {
+      logger.error("Worker deployment not found", { id });
+      return;
+    }
+
+    if (!deployment.worker) {
+      logger.error("Worker deployment does not have a worker", { id });
+      throw new ServiceValidationError("Worker deployment does not have a worker");
+    }
+
+    if (deployment.status !== "DEPLOYING") {
+      logger.error("Worker deployment is not in DEPLOYING status", { id });
+      throw new ServiceValidationError("Worker deployment is not in DEPLOYING status");
+    }
+
+    const externalBuildData = deployment.externalBuildData
+      ? ExternalBuildData.safeParse(deployment.externalBuildData)
+      : undefined;
+
+    if (!externalBuildData) {
+      throw new ServiceValidationError("External build data is missing");
+    }
+
+    if (!externalBuildData.success) {
+      throw new ServiceValidationError("External build data is invalid");
+    }
+
+    if (
+      !env.DEPLOY_REGISTRY_HOST ||
+      !env.DEPLOY_REGISTRY_USERNAME ||
+      !env.DEPLOY_REGISTRY_PASSWORD
+    ) {
+      throw new ServiceValidationError("Missing deployment registry credentials");
+    }
+
+    if (!env.DEPOT_TOKEN) {
+      throw new ServiceValidationError("Missing depot token");
+    }
+
+    const pushResult = await executePushToRegistry({
+      depot: {
+        buildId: externalBuildData.data.buildId,
+        orgToken: env.DEPOT_TOKEN,
+        projectId: externalBuildData.data.projectId,
+      },
+      registry: {
+        host: env.DEPLOY_REGISTRY_HOST,
+        namespace: env.DEPLOY_REGISTRY_NAMESPACE,
+        username: env.DEPLOY_REGISTRY_USERNAME,
+        password: env.DEPLOY_REGISTRY_PASSWORD,
+      },
+      deployment: {
+        version: deployment.version,
+        environmentSlug: deployment.environment.slug,
+        projectExternalRef: deployment.worker.project.externalRef,
+      },
+    });
+
+    if (!pushResult.ok) {
+      throw new ServiceValidationError(pushResult.error);
+    }
+
+    const finalizeService = new FinalizeDeploymentService();
+
+    const finalizedDeployment = await finalizeService.call(authenticatedEnv, id, {
+      imageReference: pushResult.image,
+      skipRegistryProxy: true,
+    });
+
+    return finalizedDeployment;
+  }
+}
+
+type ExecutePushToRegistryOptions = {
+  depot: {
+    buildId: string;
+    orgToken: string;
+    projectId: string;
+  };
+  registry: {
+    host: string;
+    namespace: string;
+    username: string;
+    password: string;
+  };
+  deployment: {
+    version: string;
+    environmentSlug: string;
+    projectExternalRef: string;
+  };
+};
+
+type ExecutePushResult =
+  | {
+      ok: true;
+      image: string;
+      logs: string;
+    }
+  | {
+      ok: false;
+      error: string;
+      logs: string;
+    };
+
+async function executePushToRegistry({
+  depot,
+  registry,
+  deployment,
+}: ExecutePushToRegistryOptions): Promise<ExecutePushResult> {
+  // Step 1: We need to "login" to the digital ocean registry
+  const configDir = await ensureLoggedIntoDockerRegistry(registry.host, {
+    username: registry.username,
+    password: registry.password,
+  });
+
+  const imageTag = `${registry.host}/${registry.namespace}/${deployment.projectExternalRef}:${deployment.version}.${deployment.environmentSlug}`;
+
+  // Step 2: We need to run the depot push command
+  // DEPOT_TOKEN="<org token>" DEPOT_PROJECT_ID="<project id>" depot push <build id> -t registry.digitalocean.com/trigger-failover/proj_bzhdaqhlymtuhlrcgbqy:20250124.54.prod
+  // Step 4: Build and push the image
+  const childProcess = execDepot(["push", depot.buildId, "-t", imageTag, "--progress", "plain"], {
+    env: {
+      NODE_ENV: process.env.NODE_ENV,
+      DEPOT_TOKEN: depot.orgToken,
+      DEPOT_PROJECT_ID: depot.projectId,
+      DEPOT_NO_SUMMARY_LINK: "1",
+      DEPOT_NO_UPDATE_NOTIFIER: "1",
+      DOCKER_CONFIG: configDir,
+    },
+  });
+
+  const errors: string[] = [];
+
+  try {
+    const processCode = await new Promise<number | null>((res, rej) => {
+      // For some reason everything is output on stderr, not stdout
+      childProcess.stderr?.on("data", (data: Buffer) => {
+        const text = data.toString();
+
+        // Emitted data chunks can contain multiple lines. Remove empty lines.
+        const lines = text.split("\n").filter(Boolean);
+
+        errors.push(...lines);
+        logger.debug(text, {
+          imageTag,
+          deployment,
+        });
+      });
+
+      childProcess.on("error", (e) => rej(e));
+      childProcess.on("close", (code) => res(code));
+    });
+
+    const logs = extractLogs(errors);
+
+    if (processCode !== 0) {
+      return {
+        ok: false as const,
+        error: `Error pushing image`,
+        logs,
+      };
+    }
+
+    return {
+      ok: true as const,
+      image: imageTag,
+      logs,
+    };
+  } catch (e) {
+    return {
+      ok: false as const,
+      error: e instanceof Error ? e.message : JSON.stringify(e),
+      logs: extractLogs(errors),
+    };
+  }
+}
+
+async function ensureLoggedIntoDockerRegistry(
+  registryHost: string,
+  auth: { username: string; password: string }
+) {
+  const tmpDir = await createTempDir();
+  // Read the current docker config
+  const dockerConfigPath = join(tmpDir, "config.json");
+
+  await writeJSONFile(dockerConfigPath, {
+    auths: {
+      [registryHost]: {
+        auth: Buffer.from(`${auth.username}:${auth.password}`).toString("base64"),
+      },
+    },
+  });
+
+  logger.debug(`Writing docker config to ${dockerConfigPath}`);
+
+  return tmpDir;
+}
+
+// Create a temporary directory within the OS's temp directory
+async function createTempDir(): Promise<string> {
+  // Generate a unique temp directory path
+  const tempDirPath: string = join(tmpdir(), "trigger-");
+
+  // Create the temp directory synchronously and return the path
+  const directory = await mkdtemp(tempDirPath);
+
+  return directory;
+}
+
+async function writeJSONFile(path: string, json: any, pretty = false) {
+  await writeFile(path, JSON.stringify(json, undefined, pretty ? 2 : undefined), "utf8");
+}
+
+function extractLogs(outputs: string[]) {
+  // Remove empty lines
+  const cleanedOutputs = outputs.map((line) => line.trim()).filter((line) => line !== "");
+
+  return cleanedOutputs.map((line) => line.trim()).join("\n");
+}