diff --git a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts index a1efb25ed..6e3d53e58 100644 --- a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts +++ b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts @@ -1,9 +1,15 @@ import { ForbiddenError, subject } from "@casl/ability"; -import { TableName, TSecretTagJunctionInsert, TSecretV2TagJunctionInsert } from "@app/db/schemas"; +import { + TableName, + TSecretSnapshotFolders, + TSecretSnapshotSecretsV2, + TSecretTagJunctionInsert, + TSecretV2TagJunctionInsert +} from "@app/db/schemas"; import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; import { InternalServerError, NotFoundError } from "@app/lib/errors"; -import { groupBy } from "@app/lib/fn"; +import { chunkArray, groupBy } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; @@ -240,22 +246,37 @@ export const secretSnapshotServiceFactory = ({ }, tx ); - const snapshotSecrets = await snapshotSecretV2BridgeDAL.insertMany( - secretVersions.map(({ id }) => ({ - secretVersionId: id, - envId: folder.environment.envId, - snapshotId: newSnapshot.id - })), - tx - ); - const snapshotFolders = await snapshotFolderDAL.insertMany( - folderVersions.map(({ id }) => ({ - folderVersionId: id, - envId: folder.environment.envId, - snapshotId: newSnapshot.id - })), - tx - ); + + const chunkedSnapshotSecrets = chunkArray(secretVersions, 2500); + const chunkedSnapshotFolders = chunkArray(folderVersions, 2500); + const snapshotSecrets: TSecretSnapshotSecretsV2[] = []; + const snapshotFolders: TSecretSnapshotFolders[] = []; + + for await (const chunk of chunkedSnapshotSecrets) { + const result = await snapshotSecretV2BridgeDAL.insertMany( + chunk.map(({ id }) => ({ + secretVersionId: id, + envId: folder.environment.envId, + snapshotId: newSnapshot.id + })), + tx + ); + + snapshotSecrets.push(...result); + } + + for await (const chunk of chunkedSnapshotFolders) { + const result = await snapshotFolderDAL.insertMany( + chunk.map(({ id }) => ({ + folderVersionId: id, + envId: folder.environment.envId, + snapshotId: newSnapshot.id + })), + tx + ); + + snapshotFolders.push(...result); + } return { ...newSnapshot, secrets: snapshotSecrets, folder: snapshotFolders }; }); diff --git a/backend/src/lib/fn/array.ts b/backend/src/lib/fn/array.ts index 959d01aef..e7db061f3 100644 --- a/backend/src/lib/fn/array.ts +++ b/backend/src/lib/fn/array.ts @@ -70,3 +70,14 @@ export const objectify = ( {} as Record ); }; + +/** + * Chunks an array into smaller arrays of the given size. + */ +export const chunkArray = (array: T[], chunkSize: number): T[][] => { + const chunks: T[][] = []; + for (let i = 0; i < array.length; i += chunkSize) { + chunks.push(array.slice(i, i + chunkSize)); + } + return chunks; +}; diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index 0606f9dba..457eebcc1 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -1,7 +1,7 @@ import { Job, JobsOptions, Queue, QueueOptions, RepeatOptions, Worker, WorkerListener } from "bullmq"; import Redis from "ioredis"; -import { SecretKeyEncoding } from "@app/db/schemas"; +import { SecretEncryptionAlgo, SecretKeyEncoding } from "@app/db/schemas"; import { TCreateAuditLogDTO } from "@app/ee/services/audit-log/audit-log-types"; import { TScanFullRepoEventPayload, @@ -32,7 +32,8 @@ export enum QueueName { SecretReplication = "secret-replication", SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication ProjectV3Migration = "project-v3-migration", - AccessTokenStatusUpdate = "access-token-status-update" + AccessTokenStatusUpdate = "access-token-status-update", + ImportSecretsFromExternalSource = "import-secrets-from-external-source" } export enum QueueJobs { @@ -56,7 +57,8 @@ export enum QueueJobs { SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication ProjectV3Migration = "project-v3-migration", IdentityAccessTokenStatusUpdate = "identity-access-token-status-update", - ServiceTokenStatusUpdate = "service-token-status-update" + ServiceTokenStatusUpdate = "service-token-status-update", + ImportSecretsFromExternalSource = "import-secrets-from-external-source" } export type TQueueJobTypes = { @@ -166,6 +168,19 @@ export type TQueueJobTypes = { name: QueueJobs.ProjectV3Migration; payload: { projectId: string }; }; + [QueueName.ImportSecretsFromExternalSource]: { + name: QueueJobs.ImportSecretsFromExternalSource; + payload: { + actorEmail: string; + data: { + iv: string; + tag: string; + ciphertext: string; + algorithm: SecretEncryptionAlgo; + encoding: SecretKeyEncoding; + }; + }; + }; }; export type TQueueServiceFactory = ReturnType; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index e9af5043b..3d649c469 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -97,6 +97,7 @@ import { certificateTemplateDALFactory } from "@app/services/certificate-templat import { certificateTemplateEstConfigDALFactory } from "@app/services/certificate-template/certificate-template-est-config-dal"; import { certificateTemplateServiceFactory } from "@app/services/certificate-template/certificate-template-service"; import { cmekServiceFactory } from "@app/services/cmek/cmek-service"; +import { externalMigrationQueueFactory } from "@app/services/external-migration/external-migration-queue"; import { externalMigrationServiceFactory } from "@app/services/external-migration/external-migration-service"; import { groupProjectDALFactory } from "@app/services/group-project/group-project-dal"; import { groupProjectMembershipRoleDALFactory } from "@app/services/group-project/group-project-membership-role-dal"; @@ -1202,12 +1203,19 @@ export const registerRoutes = async ( permissionService }); - const migrationService = externalMigrationServiceFactory({ - projectService, + const externalMigrationQueue = externalMigrationQueueFactory({ orgService, projectEnvService, - permissionService, - secretService + projectService, + smtpService, + queueService, + secretV2BridgeService + }); + + const migrationService = externalMigrationServiceFactory({ + externalMigrationQueue, + userDAL, + permissionService }); await superAdminService.initServerCfg(); diff --git a/backend/src/services/external-migration/external-migration-fns.ts b/backend/src/services/external-migration/external-migration-fns.ts index 3af44c368..bae98da2f 100644 --- a/backend/src/services/external-migration/external-migration-fns.ts +++ b/backend/src/services/external-migration/external-migration-fns.ts @@ -4,22 +4,23 @@ import sjcl from "sjcl"; import tweetnacl from "tweetnacl"; import tweetnaclUtil from "tweetnacl-util"; -import { OrgMembershipRole, ProjectMembershipRole, SecretType } from "@app/db/schemas"; +import { OrgMembershipRole, ProjectMembershipRole } from "@app/db/schemas"; import { BadRequestError } from "@app/lib/errors"; +import { chunkArray } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { TOrgServiceFactory } from "../org/org-service"; import { TProjectServiceFactory } from "../project/project-service"; import { TProjectEnvServiceFactory } from "../project-env/project-env-service"; -import { TSecretServiceFactory } from "../secret/secret-service"; +import type { TSecretV2BridgeServiceFactory } from "../secret-v2-bridge/secret-v2-bridge-service"; import { InfisicalImportData, TEnvKeyExportJSON, TImportInfisicalDataCreate } from "./external-migration-types"; export type TImportDataIntoInfisicalDTO = { - projectService: TProjectServiceFactory; - orgService: TOrgServiceFactory; - projectEnvService: TProjectEnvServiceFactory; - secretService: TSecretServiceFactory; + projectService: Pick; + orgService: Pick; + projectEnvService: Pick; + secretV2BridgeService: Pick; input: TImportInfisicalDataCreate; }; @@ -46,13 +47,13 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise(), - environments: new Map(), - secrets: new Map() + projects: [], + environments: [], + secrets: [] }; parsedJson.apps.forEach((app: { name: string; id: string }) => { - infisicalImportData.projects.set(app.id, { name: app.name, id: app.id }); + infisicalImportData.projects.push({ name: app.name, id: app.id }); }); // string to string map for env templates @@ -63,7 +64,7 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise { // Import data to infisical @@ -104,7 +104,7 @@ export const importDataIntoInfisicalFn = async ({ const originalToNewProjectId = new Map(); const originalToNewEnvironmentId = new Map(); - for await (const [id, project] of data.projects) { + for await (const project of data.projects) { const newProject = await projectService .createProject({ actor, @@ -115,7 +115,7 @@ export const importDataIntoInfisicalFn = async ({ createDefaultEnvs: false }) .catch(() => { - throw new BadRequestError({ message: `Failed to import to project [name:${project.name}] [id:${id}]` }); + throw new BadRequestError({ message: `Failed to import to project [name:${project.name}` }); }); originalToNewProjectId.set(project.id, newProject.id); @@ -141,7 +141,7 @@ export const importDataIntoInfisicalFn = async ({ // Import environments if (data.environments) { - for await (const [id, environment] of data.environments) { + for await (const environment of data.environments) { try { const newEnvironment = await projectEnvService.createEnvironment({ actor, @@ -154,12 +154,12 @@ export const importDataIntoInfisicalFn = async ({ }); if (!newEnvironment) { - logger.error(`Failed to import environment: [name:${environment.name}] [id:${id}]`); + logger.error(`Failed to import environment: [name:${environment.name}]`); throw new BadRequestError({ - message: `Failed to import environment: [name:${environment.name}] [id:${id}]` + message: `Failed to import environment: [name:${environment.name}]` }); } - originalToNewEnvironmentId.set(id, newEnvironment.slug); + originalToNewEnvironmentId.set(environment.id, newEnvironment.slug); } catch (error) { throw new BadRequestError({ message: `Failed to import environment: ${environment.name}]`, @@ -169,28 +169,47 @@ export const importDataIntoInfisicalFn = async ({ } } - // Import secrets - if (data.secrets) { - for await (const [id, secret] of data.secrets) { - const dataProjectId = data.environments?.get(secret.environmentId)?.projectId; - if (!dataProjectId) { - throw new BadRequestError({ message: `Failed to import secret "${secret.name}", project not found` }); + if (data.secrets && data.secrets.length > 0) { + const mappedToEnvironmentId = new Map< + string, + { + secretKey: string; + secretValue: string; + }[] + >(); + + for (const secret of data.secrets) { + if (!mappedToEnvironmentId.has(secret.environmentId)) { + mappedToEnvironmentId.set(secret.environmentId, []); } - const projectId = originalToNewProjectId.get(dataProjectId); - const newSecret = await secretService.createSecretRaw({ - actorId, - actor, - actorOrgId, - environment: originalToNewEnvironmentId.get(secret.environmentId)!, - actorAuthMethod, - projectId: projectId!, - secretPath: "/", - secretName: secret.name, - type: SecretType.Shared, + mappedToEnvironmentId.get(secret.environmentId)!.push({ + secretKey: secret.name, secretValue: secret.value || "" }); - if (!newSecret) { - throw new BadRequestError({ message: `Failed to import secret: [name:${secret.name}] [id:${id}]` }); + } + + // for each of the mappedEnvironmentId + for await (const [envId, secrets] of mappedToEnvironmentId) { + const environment = data.environments.find((env) => env.id === envId); + const projectId = environment?.projectId; + + if (!projectId) { + throw new BadRequestError({ message: `Failed to import secret, project not found` }); + } + + const secretBatches = chunkArray(secrets, 2500); + + for await (const secretBatch of secretBatches) { + await secretV2BridgeService.createManySecret({ + actorId, + actor, + actorOrgId, + environment: originalToNewEnvironmentId.get(envId)!, + actorAuthMethod, + projectId: originalToNewProjectId.get(projectId)!, + secretPath: "/", + secrets: secretBatch + }); } } } diff --git a/backend/src/services/external-migration/external-migration-queue.ts b/backend/src/services/external-migration/external-migration-queue.ts new file mode 100644 index 000000000..4bea10814 --- /dev/null +++ b/backend/src/services/external-migration/external-migration-queue.ts @@ -0,0 +1,111 @@ +import { SecretEncryptionAlgo, SecretKeyEncoding } from "@app/db/schemas"; +import { infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption"; +import { logger } from "@app/lib/logger"; +import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; + +import { TOrgServiceFactory } from "../org/org-service"; +import { TProjectServiceFactory } from "../project/project-service"; +import { TProjectEnvServiceFactory } from "../project-env/project-env-service"; +import { TSecretV2BridgeServiceFactory } from "../secret-v2-bridge/secret-v2-bridge-service"; +import { SmtpTemplates, TSmtpService } from "../smtp/smtp-service"; +import { importDataIntoInfisicalFn } from "./external-migration-fns"; +import { ExternalPlatforms, TImportInfisicalDataCreate } from "./external-migration-types"; + +export type TExternalMigrationQueueFactoryDep = { + projectService: Pick; + orgService: Pick; + projectEnvService: Pick; + secretV2BridgeService: Pick; + + smtpService: TSmtpService; + queueService: TQueueServiceFactory; +}; + +export type TExternalMigrationQueueFactory = ReturnType; + +export const externalMigrationQueueFactory = ({ + queueService, + projectService, + orgService, + smtpService, + projectEnvService, + secretV2BridgeService +}: TExternalMigrationQueueFactoryDep) => { + const startImport = async (dto: { + actorEmail: string; + data: { + iv: string; + tag: string; + ciphertext: string; + algorithm: SecretEncryptionAlgo; + encoding: SecretKeyEncoding; + }; + }) => { + await queueService.queue( + QueueName.ImportSecretsFromExternalSource, + QueueJobs.ImportSecretsFromExternalSource, + dto, + { + removeOnComplete: true, + removeOnFail: true + } + ); + }; + + queueService.start(QueueName.ImportSecretsFromExternalSource, async (job) => { + try { + const { data, actorEmail } = job.data; + + await smtpService.sendMail({ + recipients: [actorEmail], + subjectLine: "Infisical import started", + substitutions: { + provider: ExternalPlatforms.EnvKey + }, + template: SmtpTemplates.ExternalImportStarted + }); + + const decrypted = infisicalSymmetricDecrypt({ + ciphertext: data.ciphertext, + iv: data.iv, + keyEncoding: data.encoding, + tag: data.tag + }); + + const decryptedJson = JSON.parse(decrypted) as TImportInfisicalDataCreate; + + await importDataIntoInfisicalFn({ + input: decryptedJson, + projectService, + orgService, + projectEnvService, + secretV2BridgeService + }); + + await smtpService.sendMail({ + recipients: [actorEmail], + subjectLine: "Infisical import successful", + substitutions: { + provider: ExternalPlatforms.EnvKey + }, + template: SmtpTemplates.ExternalImportSuccessful + }); + } catch (err) { + await smtpService.sendMail({ + recipients: [job.data.actorEmail], + subjectLine: "Infisical import failed", + substitutions: { + provider: ExternalPlatforms.EnvKey, + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-assignment + error: (err as any)?.message || "Unknown error" + }, + template: SmtpTemplates.ExternalImportFailed + }); + + logger.error(err, "Failed to import data from external source"); + } + }); + return { + startImport + }; +}; diff --git a/backend/src/services/external-migration/external-migration-service.ts b/backend/src/services/external-migration/external-migration-service.ts index a65a278f5..700819022 100644 --- a/backend/src/services/external-migration/external-migration-service.ts +++ b/backend/src/services/external-migration/external-migration-service.ts @@ -1,30 +1,25 @@ import { OrgMembershipRole } from "@app/db/schemas"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; +import { infisicalSymmetricEncypt } from "@app/lib/crypto/encryption"; import { ForbiddenRequestError } from "@app/lib/errors"; -import { TOrgServiceFactory } from "../org/org-service"; -import { TProjectServiceFactory } from "../project/project-service"; -import { TProjectEnvServiceFactory } from "../project-env/project-env-service"; -import { TSecretServiceFactory } from "../secret/secret-service"; -import { decryptEnvKeyDataFn, importDataIntoInfisicalFn, parseEnvKeyDataFn } from "./external-migration-fns"; +import { TUserDALFactory } from "../user/user-dal"; +import { decryptEnvKeyDataFn, parseEnvKeyDataFn } from "./external-migration-fns"; +import { TExternalMigrationQueueFactory } from "./external-migration-queue"; import { TImportEnvKeyDataCreate } from "./external-migration-types"; type TExternalMigrationServiceFactoryDep = { - projectService: TProjectServiceFactory; - orgService: TOrgServiceFactory; - projectEnvService: TProjectEnvServiceFactory; - secretService: TSecretServiceFactory; permissionService: TPermissionServiceFactory; + externalMigrationQueue: TExternalMigrationQueueFactory; + userDAL: Pick; }; export type TExternalMigrationServiceFactory = ReturnType; export const externalMigrationServiceFactory = ({ - projectService, - orgService, - projectEnvService, permissionService, - secretService + externalMigrationQueue, + userDAL }: TExternalMigrationServiceFactoryDep) => { const importEnvKeyData = async ({ decryptionKey, @@ -41,21 +36,28 @@ export const externalMigrationServiceFactory = ({ actorAuthMethod, actorOrgId ); - if (membership.role !== OrgMembershipRole.Admin) { throw new ForbiddenRequestError({ message: "Only admins can import data" }); } + const user = await userDAL.findById(actorId); const json = await decryptEnvKeyDataFn(decryptionKey, encryptedJson); const envKeyData = await parseEnvKeyDataFn(json); - const response = await importDataIntoInfisicalFn({ - input: { data: envKeyData, actor, actorId, actorOrgId, actorAuthMethod }, - projectService, - orgService, - projectEnvService, - secretService + + const stringifiedJson = JSON.stringify({ + data: envKeyData, + actor, + actorId, + actorOrgId, + actorAuthMethod + }); + + const encrypted = infisicalSymmetricEncypt(stringifiedJson); + + await externalMigrationQueue.startImport({ + actorEmail: user.email!, + data: encrypted }); - return response; }; return { diff --git a/backend/src/services/external-migration/external-migration-types.ts b/backend/src/services/external-migration/external-migration-types.ts index 4139b0cf5..53c954bf9 100644 --- a/backend/src/services/external-migration/external-migration-types.ts +++ b/backend/src/services/external-migration/external-migration-types.ts @@ -1,26 +1,9 @@ import { ActorAuthMethod, ActorType } from "../auth/auth-type"; export type InfisicalImportData = { - projects: Map; - - environments?: Map< - string, - { - name: string; - id: string; - projectId: string; - } - >; - - secrets?: Map< - string, - { - name: string; - id: string; - environmentId: string; - value?: string; - } - >; + projects: Array<{ name: string; id: string }>; + environments: Array<{ name: string; id: string; projectId: string }>; + secrets: Array<{ name: string; id: string; environmentId: string; value: string }>; }; export type TImportEnvKeyDataCreate = { @@ -104,3 +87,7 @@ export type TEnvKeyExportJSON = { } >; }; + +export enum ExternalPlatforms { + EnvKey = "EnvKey" +} diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index 4b3f7dfbf..15fd33027 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -85,7 +85,7 @@ type TSecretQueueFactoryDep = { secretTagDAL: TSecretTagDALFactory; userDAL: Pick; secretVersionTagDAL: TSecretVersionTagDALFactory; - kmsService: Pick; + kmsService: TKmsServiceFactory; secretV2BridgeDAL: TSecretV2BridgeDALFactory; secretVersionV2BridgeDAL: Pick; secretVersionTagV2BridgeDAL: Pick; diff --git a/backend/src/services/smtp/smtp-service.ts b/backend/src/services/smtp/smtp-service.ts index 823da4cca..1f38babb3 100644 --- a/backend/src/services/smtp/smtp-service.ts +++ b/backend/src/services/smtp/smtp-service.ts @@ -34,7 +34,10 @@ export enum SmtpTemplates { WorkspaceInvite = "workspaceInvitation.handlebars", ScimUserProvisioned = "scimUserProvisioned.handlebars", PkiExpirationAlert = "pkiExpirationAlert.handlebars", - IntegrationSyncFailed = "integrationSyncFailed.handlebars" + IntegrationSyncFailed = "integrationSyncFailed.handlebars", + ExternalImportSuccessful = "externalImportSuccessful.handlebars", + ExternalImportFailed = "externalImportFailed.handlebars", + ExternalImportStarted = "externalImportStarted.handlebars" } export enum SmtpHost { diff --git a/backend/src/services/smtp/templates/externalImportFailed.handlebars b/backend/src/services/smtp/templates/externalImportFailed.handlebars new file mode 100644 index 000000000..c7869af27 --- /dev/null +++ b/backend/src/services/smtp/templates/externalImportFailed.handlebars @@ -0,0 +1,21 @@ + + + + + + Import failed + + + +

An import from {{provider}} to Infisical has failed

+

An import from + {{provider}} + to Infisical has failed due to unforeseen circumstances. Please re-try your import, and if the issue persists, you + can contact the Infisical team at team@infisical.com. +

+ +

Error: {{error}}

+ + + + \ No newline at end of file diff --git a/backend/src/services/smtp/templates/externalImportStarted.handlebars b/backend/src/services/smtp/templates/externalImportStarted.handlebars new file mode 100644 index 000000000..551f972cc --- /dev/null +++ b/backend/src/services/smtp/templates/externalImportStarted.handlebars @@ -0,0 +1,17 @@ + + + + + + Import in progress + + + +

An import from {{provider}} to Infisical is in progress

+

An import from + {{provider}} + to Infisical is in progress. The import process may take up to 30 minutes, and you will receive once the import + has finished or if it fails.

+ + + \ No newline at end of file diff --git a/backend/src/services/smtp/templates/externalImportSuccessful.handlebars b/backend/src/services/smtp/templates/externalImportSuccessful.handlebars new file mode 100644 index 000000000..51a1c465e --- /dev/null +++ b/backend/src/services/smtp/templates/externalImportSuccessful.handlebars @@ -0,0 +1,14 @@ + + + + + + Import successful + + + +

An import from {{provider}} to Infisical was successful

+

An import from {{provider}} was successful. Your data is now available in Infisical.

+ + + \ No newline at end of file diff --git a/frontend/src/hooks/api/migration/mutations.tsx b/frontend/src/hooks/api/migration/mutations.tsx index fb9bebeff..7d434ddbe 100644 --- a/frontend/src/hooks/api/migration/mutations.tsx +++ b/frontend/src/hooks/api/migration/mutations.tsx @@ -18,11 +18,10 @@ export const useImportEnvKey = () => { }; decryptionKey: string; }) => { - const { data } = await apiRequest.post("/api/v3/migrate/env-key/", { + await apiRequest.post("/api/v3/migrate/env-key/", { encryptedJson, decryptionKey }); - return data; }, onSuccess: () => { queryClient.invalidateQueries(workspaceKeys.getAllUserWorkspace);