Merge pull request #2573 from Infisical/daniel/envkey-import-bug

feat: Process Envkey import in queue
2026-09-22 13:39:35 +00:00 · 2024-10-13 22:48:59 -07:00
parent 6a89e3527c 865db5a9b3
commit 7fdcb29bab
20 changed files with 488 additions and 190 deletions
--- a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts
+++ b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts
@@ -240,7 +240,8 @@ export const secretSnapshotServiceFactory = ({
            },
            tx
          );
-          const snapshotSecrets = await snapshotSecretV2BridgeDAL.insertMany(
+
+          const snapshotSecrets = await snapshotSecretV2BridgeDAL.batchInsert(
            secretVersions.map(({ id }) => ({
              secretVersionId: id,
              envId: folder.environment.envId,
@@ -248,7 +249,8 @@ export const secretSnapshotServiceFactory = ({
            })),
            tx
          );
-          const snapshotFolders = await snapshotFolderDAL.insertMany(
+
+          const snapshotFolders = await snapshotFolderDAL.batchInsert(
            folderVersions.map(({ id }) => ({
              folderVersionId: id,
              envId: folder.environment.envId,
--- a/backend/src/lib/fn/array.ts
+++ b/backend/src/lib/fn/array.ts
@@ -70,3 +70,14 @@ export const objectify = <T, Key extends string | number | symbol, Value = T>(
    {} as Record<Key, Value>
  );
 };
+
+/**
+ * Chunks an array into smaller arrays of the given size.
+ */
+export const chunkArray = <T>(array: T[], chunkSize: number): T[][] => {
+  const chunks: T[][] = [];
+  for (let i = 0; i < array.length; i += chunkSize) {
+    chunks.push(array.slice(i, i + chunkSize));
+  }
+  return chunks;
+};
--- a/backend/src/queue/queue-service.ts
+++ b/backend/src/queue/queue-service.ts
@@ -1,7 +1,7 @@
 import { Job, JobsOptions, Queue, QueueOptions, RepeatOptions, Worker, WorkerListener } from "bullmq";
 import Redis from "ioredis";

-import { SecretKeyEncoding } from "@app/db/schemas";
+import { SecretEncryptionAlgo, SecretKeyEncoding } from "@app/db/schemas";
 import { TCreateAuditLogDTO } from "@app/ee/services/audit-log/audit-log-types";
 import {
  TScanFullRepoEventPayload,
@@ -32,7 +32,8 @@ export enum QueueName {
  SecretReplication = "secret-replication",
  SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication
  ProjectV3Migration = "project-v3-migration",
-  AccessTokenStatusUpdate = "access-token-status-update"
+  AccessTokenStatusUpdate = "access-token-status-update",
+  ImportSecretsFromExternalSource = "import-secrets-from-external-source"
 }

 export enum QueueJobs {
@@ -56,7 +57,8 @@ export enum QueueJobs {
  SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication
  ProjectV3Migration = "project-v3-migration",
  IdentityAccessTokenStatusUpdate = "identity-access-token-status-update",
-  ServiceTokenStatusUpdate = "service-token-status-update"
+  ServiceTokenStatusUpdate = "service-token-status-update",
+  ImportSecretsFromExternalSource = "import-secrets-from-external-source"
 }

 export type TQueueJobTypes = {
@@ -166,6 +168,19 @@ export type TQueueJobTypes = {
    name: QueueJobs.ProjectV3Migration;
    payload: { projectId: string };
  };
+  [QueueName.ImportSecretsFromExternalSource]: {
+    name: QueueJobs.ImportSecretsFromExternalSource;
+    payload: {
+      actorEmail: string;
+      data: {
+        iv: string;
+        tag: string;
+        ciphertext: string;
+        algorithm: SecretEncryptionAlgo;
+        encoding: SecretKeyEncoding;
+      };
+    };
+  };
 };

 export type TQueueServiceFactory = ReturnType<typeof queueServiceFactory>;
--- a/backend/src/server/routes/index.ts
+++ b/backend/src/server/routes/index.ts
@@ -97,6 +97,7 @@ import { certificateTemplateDALFactory } from "@app/services/certificate-templat
 import { certificateTemplateEstConfigDALFactory } from "@app/services/certificate-template/certificate-template-est-config-dal";
 import { certificateTemplateServiceFactory } from "@app/services/certificate-template/certificate-template-service";
 import { cmekServiceFactory } from "@app/services/cmek/cmek-service";
+import { externalMigrationQueueFactory } from "@app/services/external-migration/external-migration-queue";
 import { externalMigrationServiceFactory } from "@app/services/external-migration/external-migration-service";
 import { groupProjectDALFactory } from "@app/services/group-project/group-project-dal";
 import { groupProjectMembershipRoleDALFactory } from "@app/services/group-project/group-project-membership-role-dal";
@@ -1202,12 +1203,26 @@ export const registerRoutes = async (
    permissionService
  });

-  const migrationService = externalMigrationServiceFactory({
-    projectService,
-    orgService,
+  const externalMigrationQueue = externalMigrationQueueFactory({
    projectEnvService,
-    permissionService,
-    secretService
+    projectDAL,
+    projectService,
+    smtpService,
+    kmsService,
+    projectEnvDAL,
+    secretVersionDAL: secretVersionV2BridgeDAL,
+    secretTagDAL,
+    secretVersionTagDAL: secretVersionTagV2BridgeDAL,
+    folderDAL,
+    secretDAL: secretV2BridgeDAL,
+    queueService,
+    secretV2BridgeService
+  });
+
+  const migrationService = externalMigrationServiceFactory({
+    externalMigrationQueue,
+    userDAL,
+    permissionService
  });

  await superAdminService.initServerCfg();
--- a/backend/src/services/external-migration/external-migration-fns.ts
+++ b/backend/src/services/external-migration/external-migration-fns.ts
@@ -4,22 +4,41 @@ import sjcl from "sjcl";
 import tweetnacl from "tweetnacl";
 import tweetnaclUtil from "tweetnacl-util";

-import { OrgMembershipRole, ProjectMembershipRole, SecretType } from "@app/db/schemas";
-import { BadRequestError } from "@app/lib/errors";
+import { SecretType } from "@app/db/schemas";
+import { BadRequestError, NotFoundError } from "@app/lib/errors";
+import { chunkArray } from "@app/lib/fn";
 import { logger } from "@app/lib/logger";
 import { alphaNumericNanoId } from "@app/lib/nanoid";

-import { TOrgServiceFactory } from "../org/org-service";
+import { TKmsServiceFactory } from "../kms/kms-service";
+import { KmsDataKey } from "../kms/kms-types";
+import { TProjectDALFactory } from "../project/project-dal";
 import { TProjectServiceFactory } from "../project/project-service";
+import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
 import { TProjectEnvServiceFactory } from "../project-env/project-env-service";
-import { TSecretServiceFactory } from "../secret/secret-service";
+import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
+import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal";
+import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";
+import { fnSecretBulkInsert, getAllNestedSecretReferences } from "../secret-v2-bridge/secret-v2-bridge-fns";
+import type { TSecretV2BridgeServiceFactory } from "../secret-v2-bridge/secret-v2-bridge-service";
+import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal";
+import { TSecretVersionV2TagDALFactory } from "../secret-v2-bridge/secret-version-tag-dal";
 import { InfisicalImportData, TEnvKeyExportJSON, TImportInfisicalDataCreate } from "./external-migration-types";

 export type TImportDataIntoInfisicalDTO = {
-  projectService: TProjectServiceFactory;
-  orgService: TOrgServiceFactory;
-  projectEnvService: TProjectEnvServiceFactory;
-  secretService: TSecretServiceFactory;
+  projectDAL: Pick<TProjectDALFactory, "transaction">;
+  projectEnvDAL: Pick<TProjectEnvDALFactory, "find" | "findLastEnvPosition" | "create" | "findOne">;
+  kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
+
+  secretDAL: Pick<TSecretV2BridgeDALFactory, "insertMany" | "upsertSecretReferences" | "findBySecretKeys">;
+  secretVersionDAL: Pick<TSecretVersionV2DALFactory, "insertMany" | "create">;
+  secretTagDAL: Pick<TSecretTagDALFactory, "saveTagsToSecretV2" | "create">;
+  secretVersionTagDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany" | "create">;
+
+  folderDAL: Pick<TSecretFolderDALFactory, "create" | "findBySecretPath">;
+  projectService: Pick<TProjectServiceFactory, "createProject">;
+  projectEnvService: Pick<TProjectEnvServiceFactory, "createEnvironment">;
+  secretV2BridgeService: Pick<TSecretV2BridgeServiceFactory, "createManySecret">;

  input: TImportInfisicalDataCreate;
 };
@@ -46,13 +65,13 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise<Infisica
  const parsedJson: TEnvKeyExportJSON = JSON.parse(decryptedJson) as TEnvKeyExportJSON;

  const infisicalImportData: InfisicalImportData = {
-    projects: new Map<string, { name: string; id: string }>(),
-    environments: new Map<string, { name: string; id: string; projectId: string }>(),
-    secrets: new Map<string, { name: string; id: string; projectId: string; environmentId: string; value: string }>()
+    projects: [],
+    environments: [],
+    secrets: []
  };

  parsedJson.apps.forEach((app: { name: string; id: string }) => {
-    infisicalImportData.projects.set(app.id, { name: app.name, id: app.id });
+    infisicalImportData.projects.push({ name: app.name, id: app.id });
  });

  // string to string map for env templates
@@ -63,7 +82,7 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise<Infisica

  // environments
  for (const env of parsedJson.baseEnvironments) {
-    infisicalImportData.environments?.set(env.id, {
+    infisicalImportData.environments.push({
      id: env.id,
      name: envTemplates.get(env.environmentRoleId)!,
      projectId: env.envParentId
@@ -75,9 +94,8 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise<Infisica
    if (!env.includes("|")) {
      const envData = parsedJson.envs[env];
      for (const secret of Object.keys(envData.variables)) {
-        const id = randomUUID();
-        infisicalImportData.secrets?.set(id, {
-          id,
+        infisicalImportData.secrets.push({
+          id: randomUUID(),
          name: secret,
          environmentId: env,
          value: envData.variables[secret].val
@@ -91,9 +109,14 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise<Infisica

 export const importDataIntoInfisicalFn = async ({
  projectService,
-  orgService,
-  projectEnvService,
-  secretService,
+  projectEnvDAL,
+  projectDAL,
+  secretDAL,
+  kmsService,
+  secretVersionDAL,
+  secretTagDAL,
+  secretVersionTagDAL,
+  folderDAL,
  input: { data, actor, actorId, actorOrgId, actorAuthMethod }
 }: TImportDataIntoInfisicalDTO) => {
  // Import data to infisical
@@ -104,94 +127,132 @@ export const importDataIntoInfisicalFn = async ({
  const originalToNewProjectId = new Map<string, string>();
  const originalToNewEnvironmentId = new Map<string, string>();

-  for await (const [id, project] of data.projects) {
-    const newProject = await projectService
-      .createProject({
-        actor,
-        actorId,
-        actorOrgId,
-        actorAuthMethod,
-        workspaceName: project.name,
-        createDefaultEnvs: false
-      })
-      .catch(() => {
-        throw new BadRequestError({ message: `Failed to import to project [name:${project.name}] [id:${id}]` });
-      });
-
-    originalToNewProjectId.set(project.id, newProject.id);
-  }
-
-  // Invite user importing projects
-  const invites = await orgService.inviteUserToOrganization({
-    actorAuthMethod,
-    actorId,
-    actorOrgId,
-    actor,
-    inviteeEmails: [],
-    orgId: actorOrgId,
-    organizationRoleSlug: OrgMembershipRole.NoAccess,
-    projects: Array.from(originalToNewProjectId.values()).map((project) => ({
-      id: project,
-      projectRoleSlug: [ProjectMembershipRole.Member]
-    }))
-  });
-  if (!invites) {
-    throw new BadRequestError({ message: `Failed to invite user to projects: [userId:${actorId}]` });
-  }
-
-  // Import environments
-  if (data.environments) {
-    for await (const [id, environment] of data.environments) {
-      try {
-        const newEnvironment = await projectEnvService.createEnvironment({
+  await projectDAL.transaction(async (tx) => {
+    for await (const project of data.projects) {
+      const newProject = await projectService
+        .createProject({
          actor,
          actorId,
          actorOrgId,
          actorAuthMethod,
-          name: environment.name,
-          projectId: originalToNewProjectId.get(environment.projectId)!,
-          slug: slugify(`${environment.name}-${alphaNumericNanoId(4)}`)
+          workspaceName: project.name,
+          createDefaultEnvs: false,
+          tx
+        })
+        .catch((e) => {
+          logger.error(e, `Failed to import to project [name:${project.name}]`);
+          throw new BadRequestError({ message: `Failed to import to project [name:${project.name}]` });
        });

-        if (!newEnvironment) {
-          logger.error(`Failed to import environment: [name:${environment.name}] [id:${id}]`);
+      originalToNewProjectId.set(project.id, newProject.id);
+    }
+
+    // Import environments
+    if (data.environments) {
+      for await (const environment of data.environments) {
+        const projectId = originalToNewProjectId.get(environment.projectId)!;
+        const slug = slugify(`${environment.name}-${alphaNumericNanoId(4)}`);
+
+        const existingEnv = await projectEnvDAL.findOne({ projectId, slug }, tx);
+
+        if (existingEnv) {
          throw new BadRequestError({
-            message: `Failed to import environment: [name:${environment.name}] [id:${id}]`
+            message: `Environment with slug '${slug}' already exist`,
+            name: "CreateEnvironment"
          });
        }
-        originalToNewEnvironmentId.set(id, newEnvironment.slug);
-      } catch (error) {
-        throw new BadRequestError({
-          message: `Failed to import environment: ${environment.name}]`,
-          name: "EnvKeyMigrationImportEnvironment"
+
+        const lastPos = await projectEnvDAL.findLastEnvPosition(projectId, tx);
+        const doc = await projectEnvDAL.create({ slug, name: environment.name, projectId, position: lastPos + 1 }, tx);
+        await folderDAL.create({ name: "root", parentId: null, envId: doc.id, version: 1 }, tx);
+
+        originalToNewEnvironmentId.set(environment.id, doc.slug);
+      }
+    }
+
+    if (data.secrets && data.secrets.length > 0) {
+      const mappedToEnvironmentId = new Map<
+        string,
+        {
+          secretKey: string;
+          secretValue: string;
+        }[]
+      >();
+
+      for (const secret of data.secrets) {
+        if (!mappedToEnvironmentId.has(secret.environmentId)) {
+          mappedToEnvironmentId.set(secret.environmentId, []);
+        }
+        mappedToEnvironmentId.get(secret.environmentId)!.push({
+          secretKey: secret.name,
+          secretValue: secret.value || ""
        });
      }
-    }
-  }

-  // Import secrets
-  if (data.secrets) {
-    for await (const [id, secret] of data.secrets) {
-      const dataProjectId = data.environments?.get(secret.environmentId)?.projectId;
-      if (!dataProjectId) {
-        throw new BadRequestError({ message: `Failed to import secret "${secret.name}", project not found` });
-      }
-      const projectId = originalToNewProjectId.get(dataProjectId);
-      const newSecret = await secretService.createSecretRaw({
-        actorId,
-        actor,
-        actorOrgId,
-        environment: originalToNewEnvironmentId.get(secret.environmentId)!,
-        actorAuthMethod,
-        projectId: projectId!,
-        secretPath: "/",
-        secretName: secret.name,
-        type: SecretType.Shared,
-        secretValue: secret.value || ""
-      });
-      if (!newSecret) {
-        throw new BadRequestError({ message: `Failed to import secret: [name:${secret.name}] [id:${id}]` });
+      // for each of the mappedEnvironmentId
+      for await (const [envId, secrets] of mappedToEnvironmentId) {
+        const environment = data.environments.find((env) => env.id === envId);
+        const projectId = originalToNewProjectId.get(environment?.projectId as string)!;
+
+        if (!projectId) {
+          throw new BadRequestError({ message: `Failed to import secret, project not found` });
+        }
+
+        const { encryptor: secretManagerEncrypt } = await kmsService.createCipherPairWithDataKey(
+          {
+            type: KmsDataKey.SecretManager,
+            projectId
+          },
+          tx
+        );
+
+        const envSlug = originalToNewEnvironmentId.get(envId)!;
+        const folder = await folderDAL.findBySecretPath(projectId, envSlug, "/", tx);
+        if (!folder)
+          throw new NotFoundError({
+            message: `Folder not found for the given environment slug (${envSlug}) & secret path (/)`,
+            name: "Create secret"
+          });
+
+        const secretsByKeys = await secretDAL.findBySecretKeys(
+          folder.id,
+          secrets.map((el) => ({
+            key: el.secretKey,
+            type: SecretType.Shared
+          })),
+          tx
+        );
+        if (secretsByKeys.length) {
+          throw new BadRequestError({
+            message: `Secret already exist: ${secretsByKeys.map((el) => el.key).join(",")}`
+          });
+        }
+
+        const secretBatches = chunkArray(secrets, 2500);
+        for await (const secretBatch of secretBatches) {
+          await fnSecretBulkInsert({
+            inputSecrets: secretBatch.map((el) => {
+              const references = getAllNestedSecretReferences(el.secretValue);
+
+              return {
+                version: 1,
+                encryptedValue: el.secretValue
+                  ? secretManagerEncrypt({ plainText: Buffer.from(el.secretValue) }).cipherTextBlob
+                  : undefined,
+                key: el.secretKey,
+                references,
+                type: SecretType.Shared
+              };
+            }),
+            folderId: folder.id,
+            secretDAL,
+            secretVersionDAL,
+            secretTagDAL,
+            secretVersionTagDAL,
+            tx
+          });
+        }
      }
    }
-  }
+  });
 };
--- a/backend/src/services/external-migration/external-migration-queue.ts
+++ b/backend/src/services/external-migration/external-migration-queue.ts
@@ -0,0 +1,141 @@
+import { SecretEncryptionAlgo, SecretKeyEncoding } from "@app/db/schemas";
+import { infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption";
+import { logger } from "@app/lib/logger";
+import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
+
+import { TKmsServiceFactory } from "../kms/kms-service";
+import { TProjectDALFactory } from "../project/project-dal";
+import { TProjectServiceFactory } from "../project/project-service";
+import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
+import { TProjectEnvServiceFactory } from "../project-env/project-env-service";
+import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
+import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal";
+import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";
+import { TSecretV2BridgeServiceFactory } from "../secret-v2-bridge/secret-v2-bridge-service";
+import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal";
+import { TSecretVersionV2TagDALFactory } from "../secret-v2-bridge/secret-version-tag-dal";
+import { SmtpTemplates, TSmtpService } from "../smtp/smtp-service";
+import { importDataIntoInfisicalFn } from "./external-migration-fns";
+import { ExternalPlatforms, TImportInfisicalDataCreate } from "./external-migration-types";
+
+export type TExternalMigrationQueueFactoryDep = {
+  smtpService: TSmtpService;
+  queueService: TQueueServiceFactory;
+
+  projectDAL: Pick<TProjectDALFactory, "transaction">;
+  projectEnvDAL: Pick<TProjectEnvDALFactory, "find" | "findLastEnvPosition" | "create" | "findOne">;
+  kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
+
+  secretDAL: Pick<TSecretV2BridgeDALFactory, "insertMany" | "upsertSecretReferences" | "findBySecretKeys">;
+  secretVersionDAL: Pick<TSecretVersionV2DALFactory, "insertMany" | "create">;
+  secretTagDAL: Pick<TSecretTagDALFactory, "saveTagsToSecretV2" | "create">;
+  secretVersionTagDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany" | "create">;
+
+  folderDAL: Pick<TSecretFolderDALFactory, "create" | "findBySecretPath">;
+  projectService: Pick<TProjectServiceFactory, "createProject">;
+  projectEnvService: Pick<TProjectEnvServiceFactory, "createEnvironment">;
+  secretV2BridgeService: Pick<TSecretV2BridgeServiceFactory, "createManySecret">;
+};
+
+export type TExternalMigrationQueueFactory = ReturnType<typeof externalMigrationQueueFactory>;
+
+export const externalMigrationQueueFactory = ({
+  queueService,
+  projectService,
+  smtpService,
+  projectDAL,
+  projectEnvService,
+  secretV2BridgeService,
+  kmsService,
+  projectEnvDAL,
+  secretDAL,
+  secretVersionDAL,
+  secretTagDAL,
+  secretVersionTagDAL,
+  folderDAL
+}: TExternalMigrationQueueFactoryDep) => {
+  const startImport = async (dto: {
+    actorEmail: string;
+    data: {
+      iv: string;
+      tag: string;
+      ciphertext: string;
+      algorithm: SecretEncryptionAlgo;
+      encoding: SecretKeyEncoding;
+    };
+  }) => {
+    await queueService.queue(
+      QueueName.ImportSecretsFromExternalSource,
+      QueueJobs.ImportSecretsFromExternalSource,
+      dto,
+      {
+        removeOnComplete: true,
+        removeOnFail: true
+      }
+    );
+  };
+
+  queueService.start(QueueName.ImportSecretsFromExternalSource, async (job) => {
+    try {
+      const { data, actorEmail } = job.data;
+
+      await smtpService.sendMail({
+        recipients: [actorEmail],
+        subjectLine: "Infisical import started",
+        substitutions: {
+          provider: ExternalPlatforms.EnvKey
+        },
+        template: SmtpTemplates.ExternalImportStarted
+      });
+
+      const decrypted = infisicalSymmetricDecrypt({
+        ciphertext: data.ciphertext,
+        iv: data.iv,
+        keyEncoding: data.encoding,
+        tag: data.tag
+      });
+
+      const decryptedJson = JSON.parse(decrypted) as TImportInfisicalDataCreate;
+
+      await importDataIntoInfisicalFn({
+        input: decryptedJson,
+        projectDAL,
+        projectEnvDAL,
+        secretDAL,
+        secretVersionDAL,
+        secretTagDAL,
+        secretVersionTagDAL,
+        folderDAL,
+        kmsService,
+        projectService,
+        projectEnvService,
+        secretV2BridgeService
+      });
+
+      await smtpService.sendMail({
+        recipients: [actorEmail],
+        subjectLine: "Infisical import successful",
+        substitutions: {
+          provider: ExternalPlatforms.EnvKey
+        },
+        template: SmtpTemplates.ExternalImportSuccessful
+      });
+    } catch (err) {
+      await smtpService.sendMail({
+        recipients: [job.data.actorEmail],
+        subjectLine: "Infisical import failed",
+        substitutions: {
+          provider: ExternalPlatforms.EnvKey,
+          // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-assignment
+          error: (err as any)?.message || "Unknown error"
+        },
+        template: SmtpTemplates.ExternalImportFailed
+      });
+
+      logger.error(err, "Failed to import data from external source");
+    }
+  });
+  return {
+    startImport
+  };
+};
--- a/backend/src/services/external-migration/external-migration-service.ts
+++ b/backend/src/services/external-migration/external-migration-service.ts
@@ -1,30 +1,25 @@
 import { OrgMembershipRole } from "@app/db/schemas";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
+import { infisicalSymmetricEncypt } from "@app/lib/crypto/encryption";
 import { ForbiddenRequestError } from "@app/lib/errors";

-import { TOrgServiceFactory } from "../org/org-service";
-import { TProjectServiceFactory } from "../project/project-service";
-import { TProjectEnvServiceFactory } from "../project-env/project-env-service";
-import { TSecretServiceFactory } from "../secret/secret-service";
-import { decryptEnvKeyDataFn, importDataIntoInfisicalFn, parseEnvKeyDataFn } from "./external-migration-fns";
+import { TUserDALFactory } from "../user/user-dal";
+import { decryptEnvKeyDataFn, parseEnvKeyDataFn } from "./external-migration-fns";
+import { TExternalMigrationQueueFactory } from "./external-migration-queue";
 import { TImportEnvKeyDataCreate } from "./external-migration-types";

 type TExternalMigrationServiceFactoryDep = {
-  projectService: TProjectServiceFactory;
-  orgService: TOrgServiceFactory;
-  projectEnvService: TProjectEnvServiceFactory;
-  secretService: TSecretServiceFactory;
  permissionService: TPermissionServiceFactory;
+  externalMigrationQueue: TExternalMigrationQueueFactory;
+  userDAL: Pick<TUserDALFactory, "findById">;
 };

 export type TExternalMigrationServiceFactory = ReturnType<typeof externalMigrationServiceFactory>;

 export const externalMigrationServiceFactory = ({
-  projectService,
-  orgService,
-  projectEnvService,
  permissionService,
-  secretService
+  externalMigrationQueue,
+  userDAL
 }: TExternalMigrationServiceFactoryDep) => {
  const importEnvKeyData = async ({
    decryptionKey,
@@ -41,21 +36,28 @@ export const externalMigrationServiceFactory = ({
      actorAuthMethod,
      actorOrgId
    );
-
    if (membership.role !== OrgMembershipRole.Admin) {
      throw new ForbiddenRequestError({ message: "Only admins can import data" });
    }

+    const user = await userDAL.findById(actorId);
    const json = await decryptEnvKeyDataFn(decryptionKey, encryptedJson);
    const envKeyData = await parseEnvKeyDataFn(json);
-    const response = await importDataIntoInfisicalFn({
-      input: { data: envKeyData, actor, actorId, actorOrgId, actorAuthMethod },
-      projectService,
-      orgService,
-      projectEnvService,
-      secretService
+
+    const stringifiedJson = JSON.stringify({
+      data: envKeyData,
+      actor,
+      actorId,
+      actorOrgId,
+      actorAuthMethod
+    });
+
+    const encrypted = infisicalSymmetricEncypt(stringifiedJson);
+
+    await externalMigrationQueue.startImport({
+      actorEmail: user.email!,
+      data: encrypted
    });
-    return response;
  };

  return {
--- a/backend/src/services/external-migration/external-migration-types.ts
+++ b/backend/src/services/external-migration/external-migration-types.ts
@@ -1,26 +1,9 @@
 import { ActorAuthMethod, ActorType } from "../auth/auth-type";

 export type InfisicalImportData = {
-  projects: Map<string, { name: string; id: string }>;
-
-  environments?: Map<
-    string,
-    {
-      name: string;
-      id: string;
-      projectId: string;
-    }
-  >;
-
-  secrets?: Map<
-    string,
-    {
-      name: string;
-      id: string;
-      environmentId: string;
-      value?: string;
-    }
-  >;
+  projects: Array<{ name: string; id: string }>;
+  environments: Array<{ name: string; id: string; projectId: string }>;
+  secrets: Array<{ name: string; id: string; environmentId: string; value: string }>;
 };

 export type TImportEnvKeyDataCreate = {
@@ -104,3 +87,7 @@ export type TEnvKeyExportJSON = {
    }
  >;
 };
+
+export enum ExternalPlatforms {
+  EnvKey = "EnvKey"
+}
--- a/backend/src/services/kms/kms-service.ts
+++ b/backend/src/services/kms/kms-service.ts
@@ -160,8 +160,8 @@ export const kmsServiceFactory = ({
   * In mean time the rest of the request will wait until creation is finished followed by getting the created on
   * In real time this would be milliseconds
   */
-  const getOrgKmsKeyId = async (orgId: string) => {
-    let org = await orgDAL.findById(orgId);
+  const getOrgKmsKeyId = async (orgId: string, trx?: Knex) => {
+    let org = await orgDAL.findById(orgId, trx);

    if (!org) {
      throw new NotFoundError({ message: "Org not found" });
@@ -180,9 +180,9 @@ export const kmsServiceFactory = ({
            waitingCb: () => logger.info("KMS. Waiting for org key to be created")
          });

-          org = await orgDAL.findById(orgId);
+          org = await orgDAL.findById(orgId, trx);
        } else {
-          const keyId = await orgDAL.transaction(async (tx) => {
+          const keyId = await (trx || orgDAL).transaction(async (tx) => {
            org = await orgDAL.findById(orgId, tx);
            if (org.kmsDefaultKeyId) {
              return org.kmsDefaultKeyId;
@@ -240,11 +240,12 @@ export const kmsServiceFactory = ({

  const decryptWithKmsKey = async ({
    kmsId,
-    depth = 0
-  }: Omit<TDecryptWithKmsDTO, "cipherTextBlob"> & { depth?: number }) => {
+    depth = 0,
+    tx
+  }: Omit<TDecryptWithKmsDTO, "cipherTextBlob"> & { depth?: number; tx?: Knex }) => {
    if (depth > 2) throw new BadRequestError({ message: "KMS depth max limit" });

-    const kmsDoc = await kmsDAL.findByIdWithAssociatedKms(kmsId);
+    const kmsDoc = await kmsDAL.findByIdWithAssociatedKms(kmsId, tx);
    if (!kmsDoc) {
      throw new NotFoundError({ message: "KMS ID not found" });
    }
@@ -261,7 +262,8 @@ export const kmsServiceFactory = ({
      // we put a limit of depth to avoid too many cycles
      const orgKmsDecryptor = await decryptWithKmsKey({
        kmsId: kmsDoc.orgKms.id,
-        depth: depth + 1
+        depth: depth + 1,
+        tx
      });

      const orgKmsDataKey = await orgKmsDecryptor({
@@ -375,9 +377,9 @@ export const kmsServiceFactory = ({
    };
  };

-  const $getOrgKmsDataKey = async (orgId: string) => {
-    const kmsKeyId = await getOrgKmsKeyId(orgId);
-    let org = await orgDAL.findById(orgId);
+  const $getOrgKmsDataKey = async (orgId: string, trx?: Knex) => {
+    const kmsKeyId = await getOrgKmsKeyId(orgId, trx);
+    let org = await orgDAL.findById(orgId, trx);

    if (!org) {
      throw new NotFoundError({ message: "Org not found" });
@@ -396,9 +398,9 @@ export const kmsServiceFactory = ({
            waitingCb: () => logger.info("KMS. Waiting for org data key to be created")
          });

-          org = await orgDAL.findById(orgId);
+          org = await orgDAL.findById(orgId, trx);
        } else {
-          const orgDataKey = await orgDAL.transaction(async (tx) => {
+          const orgDataKey = await (trx || orgDAL).transaction(async (tx) => {
            org = await orgDAL.findById(orgId, tx);
            if (org.kmsEncryptedDataKey) {
              return;
@@ -455,8 +457,8 @@ export const kmsServiceFactory = ({
    });
  };

-  const getProjectSecretManagerKmsKeyId = async (projectId: string) => {
-    let project = await projectDAL.findById(projectId);
+  const getProjectSecretManagerKmsKeyId = async (projectId: string, trx?: Knex) => {
+    let project = await projectDAL.findById(projectId, trx);
    if (!project) {
      throw new NotFoundError({ message: "Project not found" });
    }
@@ -477,7 +479,7 @@ export const kmsServiceFactory = ({

          project = await projectDAL.findById(projectId);
        } else {
-          const kmsKeyId = await projectDAL.transaction(async (tx) => {
+          const kmsKeyId = await (trx || projectDAL).transaction(async (tx) => {
            project = await projectDAL.findById(projectId, tx);
            if (project.kmsSecretManagerKeyId) {
              return project.kmsSecretManagerKeyId;
@@ -520,9 +522,9 @@ export const kmsServiceFactory = ({
    return project.kmsSecretManagerKeyId;
  };

-  const $getProjectSecretManagerKmsDataKey = async (projectId: string) => {
-    const kmsKeyId = await getProjectSecretManagerKmsKeyId(projectId);
-    let project = await projectDAL.findById(projectId);
+  const $getProjectSecretManagerKmsDataKey = async (projectId: string, trx?: Knex) => {
+    const kmsKeyId = await getProjectSecretManagerKmsKeyId(projectId, trx);
+    let project = await projectDAL.findById(projectId, trx);

    if (!project.kmsSecretManagerEncryptedDataKey) {
      const lock = await keyStore
@@ -538,18 +540,21 @@ export const kmsServiceFactory = ({
            delay: 500
          });

-          project = await projectDAL.findById(projectId);
+          project = await projectDAL.findById(projectId, trx);
        } else {
-          const projectDataKey = await projectDAL.transaction(async (tx) => {
+          const projectDataKey = await (trx || projectDAL).transaction(async (tx) => {
            project = await projectDAL.findById(projectId, tx);
            if (project.kmsSecretManagerEncryptedDataKey) {
              return;
            }

            const dataKey = randomSecureBytes();
-            const kmsEncryptor = await encryptWithKmsKey({
-              kmsId: kmsKeyId
-            });
+            const kmsEncryptor = await encryptWithKmsKey(
+              {
+                kmsId: kmsKeyId
+              },
+              tx
+            );

            const { cipherTextBlob } = await kmsEncryptor({
              plainText: dataKey
@@ -585,7 +590,8 @@ export const kmsServiceFactory = ({
    }

    const kmsDecryptor = await decryptWithKmsKey({
-      kmsId: kmsKeyId
+      kmsId: kmsKeyId,
+      tx: trx
    });

    return kmsDecryptor({
@@ -593,13 +599,13 @@ export const kmsServiceFactory = ({
    });
  };

-  const $getDataKey = async (dto: TEncryptWithKmsDataKeyDTO) => {
+  const $getDataKey = async (dto: TEncryptWithKmsDataKeyDTO, trx?: Knex) => {
    switch (dto.type) {
      case KmsDataKey.SecretManager: {
-        return $getProjectSecretManagerKmsDataKey(dto.projectId);
+        return $getProjectSecretManagerKmsDataKey(dto.projectId, trx);
      }
      default: {
-        return $getOrgKmsDataKey(dto.orgId);
+        return $getOrgKmsDataKey(dto.orgId, trx);
      }
    }
  };
@@ -607,8 +613,9 @@ export const kmsServiceFactory = ({
  // by keeping the decrypted data key in inner scope
  // none of the entities outside can interact directly or expose the data key
  // NOTICE: If changing here update migrations/utils/kms
-  const createCipherPairWithDataKey = async (encryptionContext: TEncryptWithKmsDataKeyDTO) => {
-    const dataKey = await $getDataKey(encryptionContext);
+  const createCipherPairWithDataKey = async (encryptionContext: TEncryptWithKmsDataKeyDTO, trx?: Knex) => {
+    const dataKey = await $getDataKey(encryptionContext, trx);
+
    const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);

    return {
--- a/backend/src/services/org/org-types.ts
+++ b/backend/src/services/org/org-types.ts
@@ -26,18 +26,13 @@ export type TDeleteOrgMembershipDTO = {
 };

 export type TInviteUserToOrgDTO = {
-  actorId: string;
-  actor: ActorType;
-  orgId: string;
-  actorOrgId: string | undefined;
-  actorAuthMethod: ActorAuthMethod;
  inviteeEmails: string[];
  organizationRoleSlug: string;
  projects?: {
    id: string;
    projectRoleSlug?: string[];
  }[];
-};
+} & TOrgPermission;

 export type TVerifyUserToOrgDTO = {
  email: string;
--- a/backend/src/services/project/project-service.ts
+++ b/backend/src/services/project/project-service.ts
@@ -147,6 +147,7 @@ export const projectServiceFactory = ({
    workspaceName,
    slug: projectSlug,
    kmsKeyId,
+    tx: trx,
    createDefaultEnvs = true
  }: TCreateProjectDTO) => {
    const organization = await orgDAL.findOne({ id: actorOrgId });
@@ -169,7 +170,7 @@ export const projectServiceFactory = ({
      });
    }

-    const results = await projectDAL.transaction(async (tx) => {
+    const results = await (trx || projectDAL).transaction(async (tx) => {
      const ghostUser = await orgService.addGhostUser(organization.id, tx);

      if (kmsKeyId) {
--- a/backend/src/services/project/project-types.ts
+++ b/backend/src/services/project/project-types.ts
@@ -1,3 +1,5 @@
+import { Knex } from "knex";
+
 import { TProjectKeys } from "@app/db/schemas";
 import { TProjectPermission } from "@app/lib/types";

@@ -30,6 +32,7 @@ export type TCreateProjectDTO = {
  slug?: string;
  kmsKeyId?: string;
  createDefaultEnvs?: boolean;
+  tx?: Knex;
 };

 export type TDeleteProjectBySlugDTO = {
--- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts
+++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts
@@ -82,7 +82,10 @@ export const fnSecretBulkInsert = async ({
    })
  );

-  const newSecrets = await secretDAL.insertMany(sanitizedInputSecrets.map((el) => ({ ...el, folderId })));
+  const newSecrets = await secretDAL.insertMany(
+    sanitizedInputSecrets.map((el) => ({ ...el, folderId })),
+    tx
+  );
  const newSecretGroupedByKeyName = groupBy(newSecrets, (item) => item.key);
  const newSecretTags = inputSecrets.flatMap(({ tagIds: secretTags = [], key }) =>
    secretTags.map((tag) => ({
--- a/backend/src/services/secret/secret-queue.ts
+++ b/backend/src/services/secret/secret-queue.ts
@@ -85,7 +85,7 @@ type TSecretQueueFactoryDep = {
  secretTagDAL: TSecretTagDALFactory;
  userDAL: Pick<TUserDALFactory, "findById">;
  secretVersionTagDAL: TSecretVersionTagDALFactory;
-  kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
+  kmsService: TKmsServiceFactory;
  secretV2BridgeDAL: TSecretV2BridgeDALFactory;
  secretVersionV2BridgeDAL: Pick<TSecretVersionV2DALFactory, "batchInsert" | "insertMany" | "findLatestVersionMany">;
  secretVersionTagV2BridgeDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany" | "batchInsert">;
--- a/backend/src/services/smtp/smtp-service.ts
+++ b/backend/src/services/smtp/smtp-service.ts
@@ -34,7 +34,10 @@ export enum SmtpTemplates {
  WorkspaceInvite = "workspaceInvitation.handlebars",
  ScimUserProvisioned = "scimUserProvisioned.handlebars",
  PkiExpirationAlert = "pkiExpirationAlert.handlebars",
-  IntegrationSyncFailed = "integrationSyncFailed.handlebars"
+  IntegrationSyncFailed = "integrationSyncFailed.handlebars",
+  ExternalImportSuccessful = "externalImportSuccessful.handlebars",
+  ExternalImportFailed = "externalImportFailed.handlebars",
+  ExternalImportStarted = "externalImportStarted.handlebars"
 }

 export enum SmtpHost {
--- a/backend/src/services/smtp/templates/externalImportFailed.handlebars
+++ b/backend/src/services/smtp/templates/externalImportFailed.handlebars
@@ -0,0 +1,21 @@
+<html>
+
+  <head>
+    <meta charset="utf-8" />
+    <meta http-equiv="x-ua-compatible" content="ie=edge" />
+    <title>Import failed</title>
+  </head>
+
+  <body>
+    <h2>An import from {{provider}} to Infisical has failed</h2>
+    <p>An import from
+      {{provider}}
+      to Infisical has failed due to unforeseen circumstances. Please re-try your import, and if the issue persists, you
+      can contact the Infisical team at team@infisical.com.
+    </p>
+
+    <p>Error: {{error}}</p>
+
+  </body>
+
+</html>
--- a/backend/src/services/smtp/templates/externalImportStarted.handlebars
+++ b/backend/src/services/smtp/templates/externalImportStarted.handlebars
@@ -0,0 +1,17 @@
+<html>
+
+  <head>
+    <meta charset="utf-8" />
+    <meta http-equiv="x-ua-compatible" content="ie=edge" />
+    <title>Import in progress</title>
+  </head>
+
+  <body>
+    <h2>An import from {{provider}} to Infisical is in progress</h2>
+    <p>An import from
+      {{provider}}
+      to Infisical is in progress. The import process may take up to 30 minutes, and you will receive once the import
+      has finished or if it fails.</p>
+  </body>
+
+</html>
--- a/backend/src/services/smtp/templates/externalImportSuccessful.handlebars
+++ b/backend/src/services/smtp/templates/externalImportSuccessful.handlebars
@@ -0,0 +1,14 @@
+<html>
+
+  <head>
+    <meta charset="utf-8" />
+    <meta http-equiv="x-ua-compatible" content="ie=edge" />
+    <title>Import successful</title>
+  </head>
+
+  <body>
+    <h2>An import from {{provider}} to Infisical was successful</h2>
+    <p>An import from {{provider}} was successful. Your data is now available in Infisical.</p>
+  </body>
+
+</html>
--- a/frontend/src/hooks/api/migration/mutations.tsx
+++ b/frontend/src/hooks/api/migration/mutations.tsx
@@ -18,11 +18,10 @@ export const useImportEnvKey = () => {
      };
      decryptionKey: string;
    }) => {
-      const { data } = await apiRequest.post("/api/v3/migrate/env-key/", {
+      await apiRequest.post("/api/v3/migrate/env-key/", {
        encryptedJson,
        decryptionKey
      });
-      return data;
    },
    onSuccess: () => {
      queryClient.invalidateQueries(workspaceKeys.getAllUserWorkspace);
--- a/frontend/src/views/Settings/OrgSettingsPage/components/ImportTab/components/EnvKeyPlatformModal.tsx
+++ b/frontend/src/views/Settings/OrgSettingsPage/components/ImportTab/components/EnvKeyPlatformModal.tsx
@@ -54,8 +54,9 @@ export const EnvKeyPlatformModal = ({ onClose }: Props) => {
        decryptionKey: data.encryptionKey
      });
      createNotification({
-        text: "Data imported successfully.",
-        type: "success"
+        title: "Import started",
+        text: "Your data is being imported. You will receive an email when the import is complete or if the import fails. This may take up to 10 minutes.",
+        type: "info"
      });

      onClose();