diff --git a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts index a1efb25ed..de285a0f2 100644 --- a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts +++ b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts @@ -240,7 +240,8 @@ export const secretSnapshotServiceFactory = ({ }, tx ); - const snapshotSecrets = await snapshotSecretV2BridgeDAL.insertMany( + + const snapshotSecrets = await snapshotSecretV2BridgeDAL.batchInsert( secretVersions.map(({ id }) => ({ secretVersionId: id, envId: folder.environment.envId, @@ -248,7 +249,8 @@ export const secretSnapshotServiceFactory = ({ })), tx ); - const snapshotFolders = await snapshotFolderDAL.insertMany( + + const snapshotFolders = await snapshotFolderDAL.batchInsert( folderVersions.map(({ id }) => ({ folderVersionId: id, envId: folder.environment.envId, diff --git a/backend/src/lib/fn/array.ts b/backend/src/lib/fn/array.ts index 959d01aef..e7db061f3 100644 --- a/backend/src/lib/fn/array.ts +++ b/backend/src/lib/fn/array.ts @@ -70,3 +70,14 @@ export const objectify = ( {} as Record ); }; + +/** + * Chunks an array into smaller arrays of the given size. + */ +export const chunkArray = (array: T[], chunkSize: number): T[][] => { + const chunks: T[][] = []; + for (let i = 0; i < array.length; i += chunkSize) { + chunks.push(array.slice(i, i + chunkSize)); + } + return chunks; +}; diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index 0606f9dba..457eebcc1 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -1,7 +1,7 @@ import { Job, JobsOptions, Queue, QueueOptions, RepeatOptions, Worker, WorkerListener } from "bullmq"; import Redis from "ioredis"; -import { SecretKeyEncoding } from "@app/db/schemas"; +import { SecretEncryptionAlgo, SecretKeyEncoding } from "@app/db/schemas"; import { TCreateAuditLogDTO } from "@app/ee/services/audit-log/audit-log-types"; import { TScanFullRepoEventPayload, @@ -32,7 +32,8 @@ export enum QueueName { SecretReplication = "secret-replication", SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication ProjectV3Migration = "project-v3-migration", - AccessTokenStatusUpdate = "access-token-status-update" + AccessTokenStatusUpdate = "access-token-status-update", + ImportSecretsFromExternalSource = "import-secrets-from-external-source" } export enum QueueJobs { @@ -56,7 +57,8 @@ export enum QueueJobs { SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication ProjectV3Migration = "project-v3-migration", IdentityAccessTokenStatusUpdate = "identity-access-token-status-update", - ServiceTokenStatusUpdate = "service-token-status-update" + ServiceTokenStatusUpdate = "service-token-status-update", + ImportSecretsFromExternalSource = "import-secrets-from-external-source" } export type TQueueJobTypes = { @@ -166,6 +168,19 @@ export type TQueueJobTypes = { name: QueueJobs.ProjectV3Migration; payload: { projectId: string }; }; + [QueueName.ImportSecretsFromExternalSource]: { + name: QueueJobs.ImportSecretsFromExternalSource; + payload: { + actorEmail: string; + data: { + iv: string; + tag: string; + ciphertext: string; + algorithm: SecretEncryptionAlgo; + encoding: SecretKeyEncoding; + }; + }; + }; }; export type TQueueServiceFactory = ReturnType; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index e9af5043b..c195c92d7 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -97,6 +97,7 @@ import { certificateTemplateDALFactory } from "@app/services/certificate-templat import { certificateTemplateEstConfigDALFactory } from "@app/services/certificate-template/certificate-template-est-config-dal"; import { certificateTemplateServiceFactory } from "@app/services/certificate-template/certificate-template-service"; import { cmekServiceFactory } from "@app/services/cmek/cmek-service"; +import { externalMigrationQueueFactory } from "@app/services/external-migration/external-migration-queue"; import { externalMigrationServiceFactory } from "@app/services/external-migration/external-migration-service"; import { groupProjectDALFactory } from "@app/services/group-project/group-project-dal"; import { groupProjectMembershipRoleDALFactory } from "@app/services/group-project/group-project-membership-role-dal"; @@ -1202,12 +1203,26 @@ export const registerRoutes = async ( permissionService }); - const migrationService = externalMigrationServiceFactory({ - projectService, - orgService, + const externalMigrationQueue = externalMigrationQueueFactory({ projectEnvService, - permissionService, - secretService + projectDAL, + projectService, + smtpService, + kmsService, + projectEnvDAL, + secretVersionDAL: secretVersionV2BridgeDAL, + secretTagDAL, + secretVersionTagDAL: secretVersionTagV2BridgeDAL, + folderDAL, + secretDAL: secretV2BridgeDAL, + queueService, + secretV2BridgeService + }); + + const migrationService = externalMigrationServiceFactory({ + externalMigrationQueue, + userDAL, + permissionService }); await superAdminService.initServerCfg(); diff --git a/backend/src/services/external-migration/external-migration-fns.ts b/backend/src/services/external-migration/external-migration-fns.ts index 3af44c368..a69b5ef7c 100644 --- a/backend/src/services/external-migration/external-migration-fns.ts +++ b/backend/src/services/external-migration/external-migration-fns.ts @@ -4,22 +4,41 @@ import sjcl from "sjcl"; import tweetnacl from "tweetnacl"; import tweetnaclUtil from "tweetnacl-util"; -import { OrgMembershipRole, ProjectMembershipRole, SecretType } from "@app/db/schemas"; -import { BadRequestError } from "@app/lib/errors"; +import { SecretType } from "@app/db/schemas"; +import { BadRequestError, NotFoundError } from "@app/lib/errors"; +import { chunkArray } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; -import { TOrgServiceFactory } from "../org/org-service"; +import { TKmsServiceFactory } from "../kms/kms-service"; +import { KmsDataKey } from "../kms/kms-types"; +import { TProjectDALFactory } from "../project/project-dal"; import { TProjectServiceFactory } from "../project/project-service"; +import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TProjectEnvServiceFactory } from "../project-env/project-env-service"; -import { TSecretServiceFactory } from "../secret/secret-service"; +import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; +import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; +import { fnSecretBulkInsert, getAllNestedSecretReferences } from "../secret-v2-bridge/secret-v2-bridge-fns"; +import type { TSecretV2BridgeServiceFactory } from "../secret-v2-bridge/secret-v2-bridge-service"; +import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal"; +import { TSecretVersionV2TagDALFactory } from "../secret-v2-bridge/secret-version-tag-dal"; import { InfisicalImportData, TEnvKeyExportJSON, TImportInfisicalDataCreate } from "./external-migration-types"; export type TImportDataIntoInfisicalDTO = { - projectService: TProjectServiceFactory; - orgService: TOrgServiceFactory; - projectEnvService: TProjectEnvServiceFactory; - secretService: TSecretServiceFactory; + projectDAL: Pick; + projectEnvDAL: Pick; + kmsService: Pick; + + secretDAL: Pick; + secretVersionDAL: Pick; + secretTagDAL: Pick; + secretVersionTagDAL: Pick; + + folderDAL: Pick; + projectService: Pick; + projectEnvService: Pick; + secretV2BridgeService: Pick; input: TImportInfisicalDataCreate; }; @@ -46,13 +65,13 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise(), - environments: new Map(), - secrets: new Map() + projects: [], + environments: [], + secrets: [] }; parsedJson.apps.forEach((app: { name: string; id: string }) => { - infisicalImportData.projects.set(app.id, { name: app.name, id: app.id }); + infisicalImportData.projects.push({ name: app.name, id: app.id }); }); // string to string map for env templates @@ -63,7 +82,7 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise { // Import data to infisical @@ -104,94 +127,132 @@ export const importDataIntoInfisicalFn = async ({ const originalToNewProjectId = new Map(); const originalToNewEnvironmentId = new Map(); - for await (const [id, project] of data.projects) { - const newProject = await projectService - .createProject({ - actor, - actorId, - actorOrgId, - actorAuthMethod, - workspaceName: project.name, - createDefaultEnvs: false - }) - .catch(() => { - throw new BadRequestError({ message: `Failed to import to project [name:${project.name}] [id:${id}]` }); - }); - - originalToNewProjectId.set(project.id, newProject.id); - } - - // Invite user importing projects - const invites = await orgService.inviteUserToOrganization({ - actorAuthMethod, - actorId, - actorOrgId, - actor, - inviteeEmails: [], - orgId: actorOrgId, - organizationRoleSlug: OrgMembershipRole.NoAccess, - projects: Array.from(originalToNewProjectId.values()).map((project) => ({ - id: project, - projectRoleSlug: [ProjectMembershipRole.Member] - })) - }); - if (!invites) { - throw new BadRequestError({ message: `Failed to invite user to projects: [userId:${actorId}]` }); - } - - // Import environments - if (data.environments) { - for await (const [id, environment] of data.environments) { - try { - const newEnvironment = await projectEnvService.createEnvironment({ + await projectDAL.transaction(async (tx) => { + for await (const project of data.projects) { + const newProject = await projectService + .createProject({ actor, actorId, actorOrgId, actorAuthMethod, - name: environment.name, - projectId: originalToNewProjectId.get(environment.projectId)!, - slug: slugify(`${environment.name}-${alphaNumericNanoId(4)}`) + workspaceName: project.name, + createDefaultEnvs: false, + tx + }) + .catch((e) => { + logger.error(e, `Failed to import to project [name:${project.name}]`); + throw new BadRequestError({ message: `Failed to import to project [name:${project.name}]` }); }); - if (!newEnvironment) { - logger.error(`Failed to import environment: [name:${environment.name}] [id:${id}]`); + originalToNewProjectId.set(project.id, newProject.id); + } + + // Import environments + if (data.environments) { + for await (const environment of data.environments) { + const projectId = originalToNewProjectId.get(environment.projectId)!; + const slug = slugify(`${environment.name}-${alphaNumericNanoId(4)}`); + + const existingEnv = await projectEnvDAL.findOne({ projectId, slug }, tx); + + if (existingEnv) { throw new BadRequestError({ - message: `Failed to import environment: [name:${environment.name}] [id:${id}]` + message: `Environment with slug '${slug}' already exist`, + name: "CreateEnvironment" }); } - originalToNewEnvironmentId.set(id, newEnvironment.slug); - } catch (error) { - throw new BadRequestError({ - message: `Failed to import environment: ${environment.name}]`, - name: "EnvKeyMigrationImportEnvironment" + + const lastPos = await projectEnvDAL.findLastEnvPosition(projectId, tx); + const doc = await projectEnvDAL.create({ slug, name: environment.name, projectId, position: lastPos + 1 }, tx); + await folderDAL.create({ name: "root", parentId: null, envId: doc.id, version: 1 }, tx); + + originalToNewEnvironmentId.set(environment.id, doc.slug); + } + } + + if (data.secrets && data.secrets.length > 0) { + const mappedToEnvironmentId = new Map< + string, + { + secretKey: string; + secretValue: string; + }[] + >(); + + for (const secret of data.secrets) { + if (!mappedToEnvironmentId.has(secret.environmentId)) { + mappedToEnvironmentId.set(secret.environmentId, []); + } + mappedToEnvironmentId.get(secret.environmentId)!.push({ + secretKey: secret.name, + secretValue: secret.value || "" }); } - } - } - // Import secrets - if (data.secrets) { - for await (const [id, secret] of data.secrets) { - const dataProjectId = data.environments?.get(secret.environmentId)?.projectId; - if (!dataProjectId) { - throw new BadRequestError({ message: `Failed to import secret "${secret.name}", project not found` }); - } - const projectId = originalToNewProjectId.get(dataProjectId); - const newSecret = await secretService.createSecretRaw({ - actorId, - actor, - actorOrgId, - environment: originalToNewEnvironmentId.get(secret.environmentId)!, - actorAuthMethod, - projectId: projectId!, - secretPath: "/", - secretName: secret.name, - type: SecretType.Shared, - secretValue: secret.value || "" - }); - if (!newSecret) { - throw new BadRequestError({ message: `Failed to import secret: [name:${secret.name}] [id:${id}]` }); + // for each of the mappedEnvironmentId + for await (const [envId, secrets] of mappedToEnvironmentId) { + const environment = data.environments.find((env) => env.id === envId); + const projectId = originalToNewProjectId.get(environment?.projectId as string)!; + + if (!projectId) { + throw new BadRequestError({ message: `Failed to import secret, project not found` }); + } + + const { encryptor: secretManagerEncrypt } = await kmsService.createCipherPairWithDataKey( + { + type: KmsDataKey.SecretManager, + projectId + }, + tx + ); + + const envSlug = originalToNewEnvironmentId.get(envId)!; + const folder = await folderDAL.findBySecretPath(projectId, envSlug, "/", tx); + if (!folder) + throw new NotFoundError({ + message: `Folder not found for the given environment slug (${envSlug}) & secret path (/)`, + name: "Create secret" + }); + + const secretsByKeys = await secretDAL.findBySecretKeys( + folder.id, + secrets.map((el) => ({ + key: el.secretKey, + type: SecretType.Shared + })), + tx + ); + if (secretsByKeys.length) { + throw new BadRequestError({ + message: `Secret already exist: ${secretsByKeys.map((el) => el.key).join(",")}` + }); + } + + const secretBatches = chunkArray(secrets, 2500); + for await (const secretBatch of secretBatches) { + await fnSecretBulkInsert({ + inputSecrets: secretBatch.map((el) => { + const references = getAllNestedSecretReferences(el.secretValue); + + return { + version: 1, + encryptedValue: el.secretValue + ? secretManagerEncrypt({ plainText: Buffer.from(el.secretValue) }).cipherTextBlob + : undefined, + key: el.secretKey, + references, + type: SecretType.Shared + }; + }), + folderId: folder.id, + secretDAL, + secretVersionDAL, + secretTagDAL, + secretVersionTagDAL, + tx + }); + } } } - } + }); }; diff --git a/backend/src/services/external-migration/external-migration-queue.ts b/backend/src/services/external-migration/external-migration-queue.ts new file mode 100644 index 000000000..0ee51c5fa --- /dev/null +++ b/backend/src/services/external-migration/external-migration-queue.ts @@ -0,0 +1,141 @@ +import { SecretEncryptionAlgo, SecretKeyEncoding } from "@app/db/schemas"; +import { infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption"; +import { logger } from "@app/lib/logger"; +import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; + +import { TKmsServiceFactory } from "../kms/kms-service"; +import { TProjectDALFactory } from "../project/project-dal"; +import { TProjectServiceFactory } from "../project/project-service"; +import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; +import { TProjectEnvServiceFactory } from "../project-env/project-env-service"; +import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; +import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; +import { TSecretV2BridgeServiceFactory } from "../secret-v2-bridge/secret-v2-bridge-service"; +import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal"; +import { TSecretVersionV2TagDALFactory } from "../secret-v2-bridge/secret-version-tag-dal"; +import { SmtpTemplates, TSmtpService } from "../smtp/smtp-service"; +import { importDataIntoInfisicalFn } from "./external-migration-fns"; +import { ExternalPlatforms, TImportInfisicalDataCreate } from "./external-migration-types"; + +export type TExternalMigrationQueueFactoryDep = { + smtpService: TSmtpService; + queueService: TQueueServiceFactory; + + projectDAL: Pick; + projectEnvDAL: Pick; + kmsService: Pick; + + secretDAL: Pick; + secretVersionDAL: Pick; + secretTagDAL: Pick; + secretVersionTagDAL: Pick; + + folderDAL: Pick; + projectService: Pick; + projectEnvService: Pick; + secretV2BridgeService: Pick; +}; + +export type TExternalMigrationQueueFactory = ReturnType; + +export const externalMigrationQueueFactory = ({ + queueService, + projectService, + smtpService, + projectDAL, + projectEnvService, + secretV2BridgeService, + kmsService, + projectEnvDAL, + secretDAL, + secretVersionDAL, + secretTagDAL, + secretVersionTagDAL, + folderDAL +}: TExternalMigrationQueueFactoryDep) => { + const startImport = async (dto: { + actorEmail: string; + data: { + iv: string; + tag: string; + ciphertext: string; + algorithm: SecretEncryptionAlgo; + encoding: SecretKeyEncoding; + }; + }) => { + await queueService.queue( + QueueName.ImportSecretsFromExternalSource, + QueueJobs.ImportSecretsFromExternalSource, + dto, + { + removeOnComplete: true, + removeOnFail: true + } + ); + }; + + queueService.start(QueueName.ImportSecretsFromExternalSource, async (job) => { + try { + const { data, actorEmail } = job.data; + + await smtpService.sendMail({ + recipients: [actorEmail], + subjectLine: "Infisical import started", + substitutions: { + provider: ExternalPlatforms.EnvKey + }, + template: SmtpTemplates.ExternalImportStarted + }); + + const decrypted = infisicalSymmetricDecrypt({ + ciphertext: data.ciphertext, + iv: data.iv, + keyEncoding: data.encoding, + tag: data.tag + }); + + const decryptedJson = JSON.parse(decrypted) as TImportInfisicalDataCreate; + + await importDataIntoInfisicalFn({ + input: decryptedJson, + projectDAL, + projectEnvDAL, + secretDAL, + secretVersionDAL, + secretTagDAL, + secretVersionTagDAL, + folderDAL, + kmsService, + projectService, + projectEnvService, + secretV2BridgeService + }); + + await smtpService.sendMail({ + recipients: [actorEmail], + subjectLine: "Infisical import successful", + substitutions: { + provider: ExternalPlatforms.EnvKey + }, + template: SmtpTemplates.ExternalImportSuccessful + }); + } catch (err) { + await smtpService.sendMail({ + recipients: [job.data.actorEmail], + subjectLine: "Infisical import failed", + substitutions: { + provider: ExternalPlatforms.EnvKey, + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-assignment + error: (err as any)?.message || "Unknown error" + }, + template: SmtpTemplates.ExternalImportFailed + }); + + logger.error(err, "Failed to import data from external source"); + } + }); + return { + startImport + }; +}; diff --git a/backend/src/services/external-migration/external-migration-service.ts b/backend/src/services/external-migration/external-migration-service.ts index a65a278f5..700819022 100644 --- a/backend/src/services/external-migration/external-migration-service.ts +++ b/backend/src/services/external-migration/external-migration-service.ts @@ -1,30 +1,25 @@ import { OrgMembershipRole } from "@app/db/schemas"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; +import { infisicalSymmetricEncypt } from "@app/lib/crypto/encryption"; import { ForbiddenRequestError } from "@app/lib/errors"; -import { TOrgServiceFactory } from "../org/org-service"; -import { TProjectServiceFactory } from "../project/project-service"; -import { TProjectEnvServiceFactory } from "../project-env/project-env-service"; -import { TSecretServiceFactory } from "../secret/secret-service"; -import { decryptEnvKeyDataFn, importDataIntoInfisicalFn, parseEnvKeyDataFn } from "./external-migration-fns"; +import { TUserDALFactory } from "../user/user-dal"; +import { decryptEnvKeyDataFn, parseEnvKeyDataFn } from "./external-migration-fns"; +import { TExternalMigrationQueueFactory } from "./external-migration-queue"; import { TImportEnvKeyDataCreate } from "./external-migration-types"; type TExternalMigrationServiceFactoryDep = { - projectService: TProjectServiceFactory; - orgService: TOrgServiceFactory; - projectEnvService: TProjectEnvServiceFactory; - secretService: TSecretServiceFactory; permissionService: TPermissionServiceFactory; + externalMigrationQueue: TExternalMigrationQueueFactory; + userDAL: Pick; }; export type TExternalMigrationServiceFactory = ReturnType; export const externalMigrationServiceFactory = ({ - projectService, - orgService, - projectEnvService, permissionService, - secretService + externalMigrationQueue, + userDAL }: TExternalMigrationServiceFactoryDep) => { const importEnvKeyData = async ({ decryptionKey, @@ -41,21 +36,28 @@ export const externalMigrationServiceFactory = ({ actorAuthMethod, actorOrgId ); - if (membership.role !== OrgMembershipRole.Admin) { throw new ForbiddenRequestError({ message: "Only admins can import data" }); } + const user = await userDAL.findById(actorId); const json = await decryptEnvKeyDataFn(decryptionKey, encryptedJson); const envKeyData = await parseEnvKeyDataFn(json); - const response = await importDataIntoInfisicalFn({ - input: { data: envKeyData, actor, actorId, actorOrgId, actorAuthMethod }, - projectService, - orgService, - projectEnvService, - secretService + + const stringifiedJson = JSON.stringify({ + data: envKeyData, + actor, + actorId, + actorOrgId, + actorAuthMethod + }); + + const encrypted = infisicalSymmetricEncypt(stringifiedJson); + + await externalMigrationQueue.startImport({ + actorEmail: user.email!, + data: encrypted }); - return response; }; return { diff --git a/backend/src/services/external-migration/external-migration-types.ts b/backend/src/services/external-migration/external-migration-types.ts index 4139b0cf5..53c954bf9 100644 --- a/backend/src/services/external-migration/external-migration-types.ts +++ b/backend/src/services/external-migration/external-migration-types.ts @@ -1,26 +1,9 @@ import { ActorAuthMethod, ActorType } from "../auth/auth-type"; export type InfisicalImportData = { - projects: Map; - - environments?: Map< - string, - { - name: string; - id: string; - projectId: string; - } - >; - - secrets?: Map< - string, - { - name: string; - id: string; - environmentId: string; - value?: string; - } - >; + projects: Array<{ name: string; id: string }>; + environments: Array<{ name: string; id: string; projectId: string }>; + secrets: Array<{ name: string; id: string; environmentId: string; value: string }>; }; export type TImportEnvKeyDataCreate = { @@ -104,3 +87,7 @@ export type TEnvKeyExportJSON = { } >; }; + +export enum ExternalPlatforms { + EnvKey = "EnvKey" +} diff --git a/backend/src/services/kms/kms-service.ts b/backend/src/services/kms/kms-service.ts index 50dc0bd08..e1166d8c0 100644 --- a/backend/src/services/kms/kms-service.ts +++ b/backend/src/services/kms/kms-service.ts @@ -160,8 +160,8 @@ export const kmsServiceFactory = ({ * In mean time the rest of the request will wait until creation is finished followed by getting the created on * In real time this would be milliseconds */ - const getOrgKmsKeyId = async (orgId: string) => { - let org = await orgDAL.findById(orgId); + const getOrgKmsKeyId = async (orgId: string, trx?: Knex) => { + let org = await orgDAL.findById(orgId, trx); if (!org) { throw new NotFoundError({ message: "Org not found" }); @@ -180,9 +180,9 @@ export const kmsServiceFactory = ({ waitingCb: () => logger.info("KMS. Waiting for org key to be created") }); - org = await orgDAL.findById(orgId); + org = await orgDAL.findById(orgId, trx); } else { - const keyId = await orgDAL.transaction(async (tx) => { + const keyId = await (trx || orgDAL).transaction(async (tx) => { org = await orgDAL.findById(orgId, tx); if (org.kmsDefaultKeyId) { return org.kmsDefaultKeyId; @@ -240,11 +240,12 @@ export const kmsServiceFactory = ({ const decryptWithKmsKey = async ({ kmsId, - depth = 0 - }: Omit & { depth?: number }) => { + depth = 0, + tx + }: Omit & { depth?: number; tx?: Knex }) => { if (depth > 2) throw new BadRequestError({ message: "KMS depth max limit" }); - const kmsDoc = await kmsDAL.findByIdWithAssociatedKms(kmsId); + const kmsDoc = await kmsDAL.findByIdWithAssociatedKms(kmsId, tx); if (!kmsDoc) { throw new NotFoundError({ message: "KMS ID not found" }); } @@ -261,7 +262,8 @@ export const kmsServiceFactory = ({ // we put a limit of depth to avoid too many cycles const orgKmsDecryptor = await decryptWithKmsKey({ kmsId: kmsDoc.orgKms.id, - depth: depth + 1 + depth: depth + 1, + tx }); const orgKmsDataKey = await orgKmsDecryptor({ @@ -375,9 +377,9 @@ export const kmsServiceFactory = ({ }; }; - const $getOrgKmsDataKey = async (orgId: string) => { - const kmsKeyId = await getOrgKmsKeyId(orgId); - let org = await orgDAL.findById(orgId); + const $getOrgKmsDataKey = async (orgId: string, trx?: Knex) => { + const kmsKeyId = await getOrgKmsKeyId(orgId, trx); + let org = await orgDAL.findById(orgId, trx); if (!org) { throw new NotFoundError({ message: "Org not found" }); @@ -396,9 +398,9 @@ export const kmsServiceFactory = ({ waitingCb: () => logger.info("KMS. Waiting for org data key to be created") }); - org = await orgDAL.findById(orgId); + org = await orgDAL.findById(orgId, trx); } else { - const orgDataKey = await orgDAL.transaction(async (tx) => { + const orgDataKey = await (trx || orgDAL).transaction(async (tx) => { org = await orgDAL.findById(orgId, tx); if (org.kmsEncryptedDataKey) { return; @@ -455,8 +457,8 @@ export const kmsServiceFactory = ({ }); }; - const getProjectSecretManagerKmsKeyId = async (projectId: string) => { - let project = await projectDAL.findById(projectId); + const getProjectSecretManagerKmsKeyId = async (projectId: string, trx?: Knex) => { + let project = await projectDAL.findById(projectId, trx); if (!project) { throw new NotFoundError({ message: "Project not found" }); } @@ -477,7 +479,7 @@ export const kmsServiceFactory = ({ project = await projectDAL.findById(projectId); } else { - const kmsKeyId = await projectDAL.transaction(async (tx) => { + const kmsKeyId = await (trx || projectDAL).transaction(async (tx) => { project = await projectDAL.findById(projectId, tx); if (project.kmsSecretManagerKeyId) { return project.kmsSecretManagerKeyId; @@ -520,9 +522,9 @@ export const kmsServiceFactory = ({ return project.kmsSecretManagerKeyId; }; - const $getProjectSecretManagerKmsDataKey = async (projectId: string) => { - const kmsKeyId = await getProjectSecretManagerKmsKeyId(projectId); - let project = await projectDAL.findById(projectId); + const $getProjectSecretManagerKmsDataKey = async (projectId: string, trx?: Knex) => { + const kmsKeyId = await getProjectSecretManagerKmsKeyId(projectId, trx); + let project = await projectDAL.findById(projectId, trx); if (!project.kmsSecretManagerEncryptedDataKey) { const lock = await keyStore @@ -538,18 +540,21 @@ export const kmsServiceFactory = ({ delay: 500 }); - project = await projectDAL.findById(projectId); + project = await projectDAL.findById(projectId, trx); } else { - const projectDataKey = await projectDAL.transaction(async (tx) => { + const projectDataKey = await (trx || projectDAL).transaction(async (tx) => { project = await projectDAL.findById(projectId, tx); if (project.kmsSecretManagerEncryptedDataKey) { return; } const dataKey = randomSecureBytes(); - const kmsEncryptor = await encryptWithKmsKey({ - kmsId: kmsKeyId - }); + const kmsEncryptor = await encryptWithKmsKey( + { + kmsId: kmsKeyId + }, + tx + ); const { cipherTextBlob } = await kmsEncryptor({ plainText: dataKey @@ -585,7 +590,8 @@ export const kmsServiceFactory = ({ } const kmsDecryptor = await decryptWithKmsKey({ - kmsId: kmsKeyId + kmsId: kmsKeyId, + tx: trx }); return kmsDecryptor({ @@ -593,13 +599,13 @@ export const kmsServiceFactory = ({ }); }; - const $getDataKey = async (dto: TEncryptWithKmsDataKeyDTO) => { + const $getDataKey = async (dto: TEncryptWithKmsDataKeyDTO, trx?: Knex) => { switch (dto.type) { case KmsDataKey.SecretManager: { - return $getProjectSecretManagerKmsDataKey(dto.projectId); + return $getProjectSecretManagerKmsDataKey(dto.projectId, trx); } default: { - return $getOrgKmsDataKey(dto.orgId); + return $getOrgKmsDataKey(dto.orgId, trx); } } }; @@ -607,8 +613,9 @@ export const kmsServiceFactory = ({ // by keeping the decrypted data key in inner scope // none of the entities outside can interact directly or expose the data key // NOTICE: If changing here update migrations/utils/kms - const createCipherPairWithDataKey = async (encryptionContext: TEncryptWithKmsDataKeyDTO) => { - const dataKey = await $getDataKey(encryptionContext); + const createCipherPairWithDataKey = async (encryptionContext: TEncryptWithKmsDataKeyDTO, trx?: Knex) => { + const dataKey = await $getDataKey(encryptionContext, trx); + const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256); return { diff --git a/backend/src/services/org/org-types.ts b/backend/src/services/org/org-types.ts index 9f1973349..d62a2c25b 100644 --- a/backend/src/services/org/org-types.ts +++ b/backend/src/services/org/org-types.ts @@ -26,18 +26,13 @@ export type TDeleteOrgMembershipDTO = { }; export type TInviteUserToOrgDTO = { - actorId: string; - actor: ActorType; - orgId: string; - actorOrgId: string | undefined; - actorAuthMethod: ActorAuthMethod; inviteeEmails: string[]; organizationRoleSlug: string; projects?: { id: string; projectRoleSlug?: string[]; }[]; -}; +} & TOrgPermission; export type TVerifyUserToOrgDTO = { email: string; diff --git a/backend/src/services/project/project-service.ts b/backend/src/services/project/project-service.ts index ec6c54dcc..d99212744 100644 --- a/backend/src/services/project/project-service.ts +++ b/backend/src/services/project/project-service.ts @@ -147,6 +147,7 @@ export const projectServiceFactory = ({ workspaceName, slug: projectSlug, kmsKeyId, + tx: trx, createDefaultEnvs = true }: TCreateProjectDTO) => { const organization = await orgDAL.findOne({ id: actorOrgId }); @@ -169,7 +170,7 @@ export const projectServiceFactory = ({ }); } - const results = await projectDAL.transaction(async (tx) => { + const results = await (trx || projectDAL).transaction(async (tx) => { const ghostUser = await orgService.addGhostUser(organization.id, tx); if (kmsKeyId) { diff --git a/backend/src/services/project/project-types.ts b/backend/src/services/project/project-types.ts index 7193f1121..d35fcb24f 100644 --- a/backend/src/services/project/project-types.ts +++ b/backend/src/services/project/project-types.ts @@ -1,3 +1,5 @@ +import { Knex } from "knex"; + import { TProjectKeys } from "@app/db/schemas"; import { TProjectPermission } from "@app/lib/types"; @@ -30,6 +32,7 @@ export type TCreateProjectDTO = { slug?: string; kmsKeyId?: string; createDefaultEnvs?: boolean; + tx?: Knex; }; export type TDeleteProjectBySlugDTO = { diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts index 78e21b579..2ee32a7d7 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts @@ -82,7 +82,10 @@ export const fnSecretBulkInsert = async ({ }) ); - const newSecrets = await secretDAL.insertMany(sanitizedInputSecrets.map((el) => ({ ...el, folderId }))); + const newSecrets = await secretDAL.insertMany( + sanitizedInputSecrets.map((el) => ({ ...el, folderId })), + tx + ); const newSecretGroupedByKeyName = groupBy(newSecrets, (item) => item.key); const newSecretTags = inputSecrets.flatMap(({ tagIds: secretTags = [], key }) => secretTags.map((tag) => ({ diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index 4b3f7dfbf..15fd33027 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -85,7 +85,7 @@ type TSecretQueueFactoryDep = { secretTagDAL: TSecretTagDALFactory; userDAL: Pick; secretVersionTagDAL: TSecretVersionTagDALFactory; - kmsService: Pick; + kmsService: TKmsServiceFactory; secretV2BridgeDAL: TSecretV2BridgeDALFactory; secretVersionV2BridgeDAL: Pick; secretVersionTagV2BridgeDAL: Pick; diff --git a/backend/src/services/smtp/smtp-service.ts b/backend/src/services/smtp/smtp-service.ts index 823da4cca..1f38babb3 100644 --- a/backend/src/services/smtp/smtp-service.ts +++ b/backend/src/services/smtp/smtp-service.ts @@ -34,7 +34,10 @@ export enum SmtpTemplates { WorkspaceInvite = "workspaceInvitation.handlebars", ScimUserProvisioned = "scimUserProvisioned.handlebars", PkiExpirationAlert = "pkiExpirationAlert.handlebars", - IntegrationSyncFailed = "integrationSyncFailed.handlebars" + IntegrationSyncFailed = "integrationSyncFailed.handlebars", + ExternalImportSuccessful = "externalImportSuccessful.handlebars", + ExternalImportFailed = "externalImportFailed.handlebars", + ExternalImportStarted = "externalImportStarted.handlebars" } export enum SmtpHost { diff --git a/backend/src/services/smtp/templates/externalImportFailed.handlebars b/backend/src/services/smtp/templates/externalImportFailed.handlebars new file mode 100644 index 000000000..c7869af27 --- /dev/null +++ b/backend/src/services/smtp/templates/externalImportFailed.handlebars @@ -0,0 +1,21 @@ + + + + + + Import failed + + + +

An import from {{provider}} to Infisical has failed

+

An import from + {{provider}} + to Infisical has failed due to unforeseen circumstances. Please re-try your import, and if the issue persists, you + can contact the Infisical team at team@infisical.com. +

+ +

Error: {{error}}

+ + + + \ No newline at end of file diff --git a/backend/src/services/smtp/templates/externalImportStarted.handlebars b/backend/src/services/smtp/templates/externalImportStarted.handlebars new file mode 100644 index 000000000..551f972cc --- /dev/null +++ b/backend/src/services/smtp/templates/externalImportStarted.handlebars @@ -0,0 +1,17 @@ + + + + + + Import in progress + + + +

An import from {{provider}} to Infisical is in progress

+

An import from + {{provider}} + to Infisical is in progress. The import process may take up to 30 minutes, and you will receive once the import + has finished or if it fails.

+ + + \ No newline at end of file diff --git a/backend/src/services/smtp/templates/externalImportSuccessful.handlebars b/backend/src/services/smtp/templates/externalImportSuccessful.handlebars new file mode 100644 index 000000000..51a1c465e --- /dev/null +++ b/backend/src/services/smtp/templates/externalImportSuccessful.handlebars @@ -0,0 +1,14 @@ + + + + + + Import successful + + + +

An import from {{provider}} to Infisical was successful

+

An import from {{provider}} was successful. Your data is now available in Infisical.

+ + + \ No newline at end of file diff --git a/frontend/src/hooks/api/migration/mutations.tsx b/frontend/src/hooks/api/migration/mutations.tsx index fb9bebeff..7d434ddbe 100644 --- a/frontend/src/hooks/api/migration/mutations.tsx +++ b/frontend/src/hooks/api/migration/mutations.tsx @@ -18,11 +18,10 @@ export const useImportEnvKey = () => { }; decryptionKey: string; }) => { - const { data } = await apiRequest.post("/api/v3/migrate/env-key/", { + await apiRequest.post("/api/v3/migrate/env-key/", { encryptedJson, decryptionKey }); - return data; }, onSuccess: () => { queryClient.invalidateQueries(workspaceKeys.getAllUserWorkspace); diff --git a/frontend/src/views/Settings/OrgSettingsPage/components/ImportTab/components/EnvKeyPlatformModal.tsx b/frontend/src/views/Settings/OrgSettingsPage/components/ImportTab/components/EnvKeyPlatformModal.tsx index 1bfabf434..8b09ba5c7 100644 --- a/frontend/src/views/Settings/OrgSettingsPage/components/ImportTab/components/EnvKeyPlatformModal.tsx +++ b/frontend/src/views/Settings/OrgSettingsPage/components/ImportTab/components/EnvKeyPlatformModal.tsx @@ -54,8 +54,9 @@ export const EnvKeyPlatformModal = ({ onClose }: Props) => { decryptionKey: data.encryptionKey }); createNotification({ - text: "Data imported successfully.", - type: "success" + title: "Import started", + text: "Your data is being imported. You will receive an email when the import is complete or if the import fails. This may take up to 10 minutes.", + type: "info" }); onClose();