feat: redis-based external imports

backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts

@@ -1,9 +1,15 @@
 import { ForbiddenError, subject } from "@casl/ability";
 
-import { TableName, TSecretTagJunctionInsert, TSecretV2TagJunctionInsert } from "@app/db/schemas";
+import {
+  TableName,
+  TSecretSnapshotFolders,
+  TSecretSnapshotSecretsV2,
+  TSecretTagJunctionInsert,
+  TSecretV2TagJunctionInsert
+} from "@app/db/schemas";
 import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
 import { InternalServerError, NotFoundError } from "@app/lib/errors";
-import { groupBy } from "@app/lib/fn";
+import { chunkArray, groupBy } from "@app/lib/fn";
 import { logger } from "@app/lib/logger";
 import { TKmsServiceFactory } from "@app/services/kms/kms-service";
 import { KmsDataKey } from "@app/services/kms/kms-types";
@@ -240,22 +246,37 @@ export const secretSnapshotServiceFactory = ({
             },
             tx
           );
-          const snapshotSecrets = await snapshotSecretV2BridgeDAL.insertMany(
-            secretVersions.map(({ id }) => ({
-              secretVersionId: id,
-              envId: folder.environment.envId,
-              snapshotId: newSnapshot.id
-            })),
-            tx
-          );
-          const snapshotFolders = await snapshotFolderDAL.insertMany(
-            folderVersions.map(({ id }) => ({
-              folderVersionId: id,
-              envId: folder.environment.envId,
-              snapshotId: newSnapshot.id
-            })),
-            tx
-          );
+
+          const chunkedSnapshotSecrets = chunkArray(secretVersions, 2500);
+          const chunkedSnapshotFolders = chunkArray(folderVersions, 2500);
+          const snapshotSecrets: TSecretSnapshotSecretsV2[] = [];
+          const snapshotFolders: TSecretSnapshotFolders[] = [];
+
+          for await (const chunk of chunkedSnapshotSecrets) {
+            const result = await snapshotSecretV2BridgeDAL.insertMany(
+              chunk.map(({ id }) => ({
+                secretVersionId: id,
+                envId: folder.environment.envId,
+                snapshotId: newSnapshot.id
+              })),
+              tx
+            );
+
+            snapshotSecrets.push(...result);
+          }
+
+          for await (const chunk of chunkedSnapshotFolders) {
+            const result = await snapshotFolderDAL.insertMany(
+              chunk.map(({ id }) => ({
+                folderVersionId: id,
+                envId: folder.environment.envId,
+                snapshotId: newSnapshot.id
+              })),
+              tx
+            );
+
+            snapshotFolders.push(...result);
+          }
 
           return { ...newSnapshot, secrets: snapshotSecrets, folder: snapshotFolders };
         });

backend/src/lib/fn/array.ts

@@ -70,3 +70,14 @@ export const objectify = <T, Key extends string | number | symbol, Value = T>(
     {} as Record<Key, Value>
   );
 };
+
+/**
+ * Chunks an array into smaller arrays of the given size.
+ */
+export const chunkArray = <T>(array: T[], chunkSize: number): T[][] => {
+  const chunks: T[][] = [];
+  for (let i = 0; i < array.length; i += chunkSize) {
+    chunks.push(array.slice(i, i + chunkSize));
+  }
+  return chunks;
+};

backend/src/queue/queue-service.ts

@@ -1,7 +1,7 @@
 import { Job, JobsOptions, Queue, QueueOptions, RepeatOptions, Worker, WorkerListener } from "bullmq";
 import Redis from "ioredis";
 
-import { SecretKeyEncoding } from "@app/db/schemas";
+import { SecretEncryptionAlgo, SecretKeyEncoding } from "@app/db/schemas";
 import { TCreateAuditLogDTO } from "@app/ee/services/audit-log/audit-log-types";
 import {
   TScanFullRepoEventPayload,
@@ -32,7 +32,8 @@ export enum QueueName {
   SecretReplication = "secret-replication",
   SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication
   ProjectV3Migration = "project-v3-migration",
-  AccessTokenStatusUpdate = "access-token-status-update"
+  AccessTokenStatusUpdate = "access-token-status-update",
+  ImportSecretsFromExternalSource = "import-secrets-from-external-source"
 }
 
 export enum QueueJobs {
@@ -56,7 +57,8 @@ export enum QueueJobs {
   SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication
   ProjectV3Migration = "project-v3-migration",
   IdentityAccessTokenStatusUpdate = "identity-access-token-status-update",
-  ServiceTokenStatusUpdate = "service-token-status-update"
+  ServiceTokenStatusUpdate = "service-token-status-update",
+  ImportSecretsFromExternalSource = "import-secrets-from-external-source"
 }
 
 export type TQueueJobTypes = {
@@ -166,6 +168,19 @@ export type TQueueJobTypes = {
     name: QueueJobs.ProjectV3Migration;
     payload: { projectId: string };
   };
+  [QueueName.ImportSecretsFromExternalSource]: {
+    name: QueueJobs.ImportSecretsFromExternalSource;
+    payload: {
+      actorEmail: string;
+      data: {
+        iv: string;
+        tag: string;
+        ciphertext: string;
+        algorithm: SecretEncryptionAlgo;
+        encoding: SecretKeyEncoding;
+      };
+    };
+  };
 };
 
 export type TQueueServiceFactory = ReturnType<typeof queueServiceFactory>;

backend/src/server/routes/index.ts

@@ -97,6 +97,7 @@ import { certificateTemplateDALFactory } from "@app/services/certificate-templat
 import { certificateTemplateEstConfigDALFactory } from "@app/services/certificate-template/certificate-template-est-config-dal";
 import { certificateTemplateServiceFactory } from "@app/services/certificate-template/certificate-template-service";
 import { cmekServiceFactory } from "@app/services/cmek/cmek-service";
+import { externalMigrationQueueFactory } from "@app/services/external-migration/external-migration-queue";
 import { externalMigrationServiceFactory } from "@app/services/external-migration/external-migration-service";
 import { groupProjectDALFactory } from "@app/services/group-project/group-project-dal";
 import { groupProjectMembershipRoleDALFactory } from "@app/services/group-project/group-project-membership-role-dal";
@@ -1202,12 +1203,19 @@ export const registerRoutes = async (
     permissionService
   });
 
-  const migrationService = externalMigrationServiceFactory({
-    projectService,
+  const externalMigrationQueue = externalMigrationQueueFactory({
     orgService,
     projectEnvService,
-    permissionService,
-    secretService
+    projectService,
+    smtpService,
+    queueService,
+    secretV2BridgeService
+  });
+
+  const migrationService = externalMigrationServiceFactory({
+    externalMigrationQueue,
+    userDAL,
+    permissionService
   });
 
   await superAdminService.initServerCfg();

backend/src/services/external-migration/external-migration-fns.ts

@@ -4,22 +4,23 @@ import sjcl from "sjcl";
 import tweetnacl from "tweetnacl";
 import tweetnaclUtil from "tweetnacl-util";
 
-import { OrgMembershipRole, ProjectMembershipRole, SecretType } from "@app/db/schemas";
+import { OrgMembershipRole, ProjectMembershipRole } from "@app/db/schemas";
 import { BadRequestError } from "@app/lib/errors";
+import { chunkArray } from "@app/lib/fn";
 import { logger } from "@app/lib/logger";
 import { alphaNumericNanoId } from "@app/lib/nanoid";
 
 import { TOrgServiceFactory } from "../org/org-service";
 import { TProjectServiceFactory } from "../project/project-service";
 import { TProjectEnvServiceFactory } from "../project-env/project-env-service";
-import { TSecretServiceFactory } from "../secret/secret-service";
+import type { TSecretV2BridgeServiceFactory } from "../secret-v2-bridge/secret-v2-bridge-service";
 import { InfisicalImportData, TEnvKeyExportJSON, TImportInfisicalDataCreate } from "./external-migration-types";
 
 export type TImportDataIntoInfisicalDTO = {
-  projectService: TProjectServiceFactory;
-  orgService: TOrgServiceFactory;
-  projectEnvService: TProjectEnvServiceFactory;
-  secretService: TSecretServiceFactory;
+  projectService: Pick<TProjectServiceFactory, "createProject">;
+  orgService: Pick<TOrgServiceFactory, "inviteUserToOrganization">;
+  projectEnvService: Pick<TProjectEnvServiceFactory, "createEnvironment">;
+  secretV2BridgeService: Pick<TSecretV2BridgeServiceFactory, "createManySecret">;
 
   input: TImportInfisicalDataCreate;
 };
@@ -46,13 +47,13 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise<Infisica
   const parsedJson: TEnvKeyExportJSON = JSON.parse(decryptedJson) as TEnvKeyExportJSON;
 
   const infisicalImportData: InfisicalImportData = {
-    projects: new Map<string, { name: string; id: string }>(),
-    environments: new Map<string, { name: string; id: string; projectId: string }>(),
-    secrets: new Map<string, { name: string; id: string; projectId: string; environmentId: string; value: string }>()
+    projects: [],
+    environments: [],
+    secrets: []
   };
 
   parsedJson.apps.forEach((app: { name: string; id: string }) => {
-    infisicalImportData.projects.set(app.id, { name: app.name, id: app.id });
+    infisicalImportData.projects.push({ name: app.name, id: app.id });
   });
 
   // string to string map for env templates
@@ -63,7 +64,7 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise<Infisica
 
   // environments
   for (const env of parsedJson.baseEnvironments) {
-    infisicalImportData.environments?.set(env.id, {
+    infisicalImportData.environments.push({
       id: env.id,
       name: envTemplates.get(env.environmentRoleId)!,
       projectId: env.envParentId
@@ -75,9 +76,8 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise<Infisica
     if (!env.includes("|")) {
       const envData = parsedJson.envs[env];
       for (const secret of Object.keys(envData.variables)) {
-        const id = randomUUID();
-        infisicalImportData.secrets?.set(id, {
-          id,
+        infisicalImportData.secrets.push({
+          id: randomUUID(),
           name: secret,
           environmentId: env,
           value: envData.variables[secret].val
@@ -93,7 +93,7 @@ export const importDataIntoInfisicalFn = async ({
   projectService,
   orgService,
   projectEnvService,
-  secretService,
+  secretV2BridgeService,
   input: { data, actor, actorId, actorOrgId, actorAuthMethod }
 }: TImportDataIntoInfisicalDTO) => {
   // Import data to infisical
@@ -104,7 +104,7 @@ export const importDataIntoInfisicalFn = async ({
   const originalToNewProjectId = new Map<string, string>();
   const originalToNewEnvironmentId = new Map<string, string>();
 
-  for await (const [id, project] of data.projects) {
+  for await (const project of data.projects) {
     const newProject = await projectService
       .createProject({
         actor,
@@ -115,7 +115,7 @@ export const importDataIntoInfisicalFn = async ({
         createDefaultEnvs: false
       })
       .catch(() => {
-        throw new BadRequestError({ message: `Failed to import to project [name:${project.name}] [id:${id}]` });
+        throw new BadRequestError({ message: `Failed to import to project [name:${project.name}` });
       });
 
     originalToNewProjectId.set(project.id, newProject.id);
@@ -141,7 +141,7 @@ export const importDataIntoInfisicalFn = async ({
 
   // Import environments
   if (data.environments) {
-    for await (const [id, environment] of data.environments) {
+    for await (const environment of data.environments) {
       try {
         const newEnvironment = await projectEnvService.createEnvironment({
           actor,
@@ -154,12 +154,12 @@ export const importDataIntoInfisicalFn = async ({
         });
 
         if (!newEnvironment) {
-          logger.error(`Failed to import environment: [name:${environment.name}] [id:${id}]`);
+          logger.error(`Failed to import environment: [name:${environment.name}]`);
           throw new BadRequestError({
-            message: `Failed to import environment: [name:${environment.name}] [id:${id}]`
+            message: `Failed to import environment: [name:${environment.name}]`
           });
         }
-        originalToNewEnvironmentId.set(id, newEnvironment.slug);
+        originalToNewEnvironmentId.set(environment.id, newEnvironment.slug);
       } catch (error) {
         throw new BadRequestError({
           message: `Failed to import environment: ${environment.name}]`,
@@ -169,28 +169,47 @@ export const importDataIntoInfisicalFn = async ({
     }
   }
 
-  // Import secrets
-  if (data.secrets) {
-    for await (const [id, secret] of data.secrets) {
-      const dataProjectId = data.environments?.get(secret.environmentId)?.projectId;
-      if (!dataProjectId) {
-        throw new BadRequestError({ message: `Failed to import secret "${secret.name}", project not found` });
+  if (data.secrets && data.secrets.length > 0) {
+    const mappedToEnvironmentId = new Map<
+      string,
+      {
+        secretKey: string;
+        secretValue: string;
+      }[]
+    >();
+
+    for (const secret of data.secrets) {
+      if (!mappedToEnvironmentId.has(secret.environmentId)) {
+        mappedToEnvironmentId.set(secret.environmentId, []);
       }
-      const projectId = originalToNewProjectId.get(dataProjectId);
-      const newSecret = await secretService.createSecretRaw({
-        actorId,
-        actor,
-        actorOrgId,
-        environment: originalToNewEnvironmentId.get(secret.environmentId)!,
-        actorAuthMethod,
-        projectId: projectId!,
-        secretPath: "/",
-        secretName: secret.name,
-        type: SecretType.Shared,
+      mappedToEnvironmentId.get(secret.environmentId)!.push({
+        secretKey: secret.name,
         secretValue: secret.value || ""
       });
-      if (!newSecret) {
-        throw new BadRequestError({ message: `Failed to import secret: [name:${secret.name}] [id:${id}]` });
+    }
+
+    // for each of the mappedEnvironmentId
+    for await (const [envId, secrets] of mappedToEnvironmentId) {
+      const environment = data.environments.find((env) => env.id === envId);
+      const projectId = environment?.projectId;
+
+      if (!projectId) {
+        throw new BadRequestError({ message: `Failed to import secret, project not found` });
+      }
+
+      const secretBatches = chunkArray(secrets, 2500);
+
+      for await (const secretBatch of secretBatches) {
+        await secretV2BridgeService.createManySecret({
+          actorId,
+          actor,
+          actorOrgId,
+          environment: originalToNewEnvironmentId.get(envId)!,
+          actorAuthMethod,
+          projectId: originalToNewProjectId.get(projectId)!,
+          secretPath: "/",
+          secrets: secretBatch
+        });
       }
     }
   }