From 162005d72f65e52ac67f23ed357b8b03b6b120b3 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Fri, 11 Oct 2024 11:11:34 +0400 Subject: [PATCH 1/3] feat: redis-based external imports --- .../secret-snapshot-service.ts | 57 ++++++--- backend/src/lib/fn/array.ts | 11 ++ backend/src/queue/queue-service.ts | 21 +++- backend/src/server/routes/index.ts | 16 ++- .../external-migration-fns.ts | 99 +++++++++------- .../external-migration-queue.ts | 111 ++++++++++++++++++ .../external-migration-service.ts | 44 +++---- .../external-migration-types.ts | 27 ++--- backend/src/services/secret/secret-queue.ts | 2 +- backend/src/services/smtp/smtp-service.ts | 5 +- .../templates/externalImportFailed.handlebars | 21 ++++ .../externalImportStarted.handlebars | 17 +++ .../externalImportSuccessful.handlebars | 14 +++ .../src/hooks/api/migration/mutations.tsx | 3 +- 14 files changed, 338 insertions(+), 110 deletions(-) create mode 100644 backend/src/services/external-migration/external-migration-queue.ts create mode 100644 backend/src/services/smtp/templates/externalImportFailed.handlebars create mode 100644 backend/src/services/smtp/templates/externalImportStarted.handlebars create mode 100644 backend/src/services/smtp/templates/externalImportSuccessful.handlebars diff --git a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts index a1efb25ed..6e3d53e58 100644 --- a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts +++ b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts @@ -1,9 +1,15 @@ import { ForbiddenError, subject } from "@casl/ability"; -import { TableName, TSecretTagJunctionInsert, TSecretV2TagJunctionInsert } from "@app/db/schemas"; +import { + TableName, + TSecretSnapshotFolders, + TSecretSnapshotSecretsV2, + TSecretTagJunctionInsert, + TSecretV2TagJunctionInsert +} from "@app/db/schemas"; import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; import { InternalServerError, NotFoundError } from "@app/lib/errors"; -import { groupBy } from "@app/lib/fn"; +import { chunkArray, groupBy } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; @@ -240,22 +246,37 @@ export const secretSnapshotServiceFactory = ({ }, tx ); - const snapshotSecrets = await snapshotSecretV2BridgeDAL.insertMany( - secretVersions.map(({ id }) => ({ - secretVersionId: id, - envId: folder.environment.envId, - snapshotId: newSnapshot.id - })), - tx - ); - const snapshotFolders = await snapshotFolderDAL.insertMany( - folderVersions.map(({ id }) => ({ - folderVersionId: id, - envId: folder.environment.envId, - snapshotId: newSnapshot.id - })), - tx - ); + + const chunkedSnapshotSecrets = chunkArray(secretVersions, 2500); + const chunkedSnapshotFolders = chunkArray(folderVersions, 2500); + const snapshotSecrets: TSecretSnapshotSecretsV2[] = []; + const snapshotFolders: TSecretSnapshotFolders[] = []; + + for await (const chunk of chunkedSnapshotSecrets) { + const result = await snapshotSecretV2BridgeDAL.insertMany( + chunk.map(({ id }) => ({ + secretVersionId: id, + envId: folder.environment.envId, + snapshotId: newSnapshot.id + })), + tx + ); + + snapshotSecrets.push(...result); + } + + for await (const chunk of chunkedSnapshotFolders) { + const result = await snapshotFolderDAL.insertMany( + chunk.map(({ id }) => ({ + folderVersionId: id, + envId: folder.environment.envId, + snapshotId: newSnapshot.id + })), + tx + ); + + snapshotFolders.push(...result); + } return { ...newSnapshot, secrets: snapshotSecrets, folder: snapshotFolders }; }); diff --git a/backend/src/lib/fn/array.ts b/backend/src/lib/fn/array.ts index 959d01aef..e7db061f3 100644 --- a/backend/src/lib/fn/array.ts +++ b/backend/src/lib/fn/array.ts @@ -70,3 +70,14 @@ export const objectify = ( {} as Record ); }; + +/** + * Chunks an array into smaller arrays of the given size. + */ +export const chunkArray = (array: T[], chunkSize: number): T[][] => { + const chunks: T[][] = []; + for (let i = 0; i < array.length; i += chunkSize) { + chunks.push(array.slice(i, i + chunkSize)); + } + return chunks; +}; diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index 0606f9dba..457eebcc1 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -1,7 +1,7 @@ import { Job, JobsOptions, Queue, QueueOptions, RepeatOptions, Worker, WorkerListener } from "bullmq"; import Redis from "ioredis"; -import { SecretKeyEncoding } from "@app/db/schemas"; +import { SecretEncryptionAlgo, SecretKeyEncoding } from "@app/db/schemas"; import { TCreateAuditLogDTO } from "@app/ee/services/audit-log/audit-log-types"; import { TScanFullRepoEventPayload, @@ -32,7 +32,8 @@ export enum QueueName { SecretReplication = "secret-replication", SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication ProjectV3Migration = "project-v3-migration", - AccessTokenStatusUpdate = "access-token-status-update" + AccessTokenStatusUpdate = "access-token-status-update", + ImportSecretsFromExternalSource = "import-secrets-from-external-source" } export enum QueueJobs { @@ -56,7 +57,8 @@ export enum QueueJobs { SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication ProjectV3Migration = "project-v3-migration", IdentityAccessTokenStatusUpdate = "identity-access-token-status-update", - ServiceTokenStatusUpdate = "service-token-status-update" + ServiceTokenStatusUpdate = "service-token-status-update", + ImportSecretsFromExternalSource = "import-secrets-from-external-source" } export type TQueueJobTypes = { @@ -166,6 +168,19 @@ export type TQueueJobTypes = { name: QueueJobs.ProjectV3Migration; payload: { projectId: string }; }; + [QueueName.ImportSecretsFromExternalSource]: { + name: QueueJobs.ImportSecretsFromExternalSource; + payload: { + actorEmail: string; + data: { + iv: string; + tag: string; + ciphertext: string; + algorithm: SecretEncryptionAlgo; + encoding: SecretKeyEncoding; + }; + }; + }; }; export type TQueueServiceFactory = ReturnType; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index e9af5043b..3d649c469 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -97,6 +97,7 @@ import { certificateTemplateDALFactory } from "@app/services/certificate-templat import { certificateTemplateEstConfigDALFactory } from "@app/services/certificate-template/certificate-template-est-config-dal"; import { certificateTemplateServiceFactory } from "@app/services/certificate-template/certificate-template-service"; import { cmekServiceFactory } from "@app/services/cmek/cmek-service"; +import { externalMigrationQueueFactory } from "@app/services/external-migration/external-migration-queue"; import { externalMigrationServiceFactory } from "@app/services/external-migration/external-migration-service"; import { groupProjectDALFactory } from "@app/services/group-project/group-project-dal"; import { groupProjectMembershipRoleDALFactory } from "@app/services/group-project/group-project-membership-role-dal"; @@ -1202,12 +1203,19 @@ export const registerRoutes = async ( permissionService }); - const migrationService = externalMigrationServiceFactory({ - projectService, + const externalMigrationQueue = externalMigrationQueueFactory({ orgService, projectEnvService, - permissionService, - secretService + projectService, + smtpService, + queueService, + secretV2BridgeService + }); + + const migrationService = externalMigrationServiceFactory({ + externalMigrationQueue, + userDAL, + permissionService }); await superAdminService.initServerCfg(); diff --git a/backend/src/services/external-migration/external-migration-fns.ts b/backend/src/services/external-migration/external-migration-fns.ts index 3af44c368..bae98da2f 100644 --- a/backend/src/services/external-migration/external-migration-fns.ts +++ b/backend/src/services/external-migration/external-migration-fns.ts @@ -4,22 +4,23 @@ import sjcl from "sjcl"; import tweetnacl from "tweetnacl"; import tweetnaclUtil from "tweetnacl-util"; -import { OrgMembershipRole, ProjectMembershipRole, SecretType } from "@app/db/schemas"; +import { OrgMembershipRole, ProjectMembershipRole } from "@app/db/schemas"; import { BadRequestError } from "@app/lib/errors"; +import { chunkArray } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { TOrgServiceFactory } from "../org/org-service"; import { TProjectServiceFactory } from "../project/project-service"; import { TProjectEnvServiceFactory } from "../project-env/project-env-service"; -import { TSecretServiceFactory } from "../secret/secret-service"; +import type { TSecretV2BridgeServiceFactory } from "../secret-v2-bridge/secret-v2-bridge-service"; import { InfisicalImportData, TEnvKeyExportJSON, TImportInfisicalDataCreate } from "./external-migration-types"; export type TImportDataIntoInfisicalDTO = { - projectService: TProjectServiceFactory; - orgService: TOrgServiceFactory; - projectEnvService: TProjectEnvServiceFactory; - secretService: TSecretServiceFactory; + projectService: Pick; + orgService: Pick; + projectEnvService: Pick; + secretV2BridgeService: Pick; input: TImportInfisicalDataCreate; }; @@ -46,13 +47,13 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise(), - environments: new Map(), - secrets: new Map() + projects: [], + environments: [], + secrets: [] }; parsedJson.apps.forEach((app: { name: string; id: string }) => { - infisicalImportData.projects.set(app.id, { name: app.name, id: app.id }); + infisicalImportData.projects.push({ name: app.name, id: app.id }); }); // string to string map for env templates @@ -63,7 +64,7 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise { // Import data to infisical @@ -104,7 +104,7 @@ export const importDataIntoInfisicalFn = async ({ const originalToNewProjectId = new Map(); const originalToNewEnvironmentId = new Map(); - for await (const [id, project] of data.projects) { + for await (const project of data.projects) { const newProject = await projectService .createProject({ actor, @@ -115,7 +115,7 @@ export const importDataIntoInfisicalFn = async ({ createDefaultEnvs: false }) .catch(() => { - throw new BadRequestError({ message: `Failed to import to project [name:${project.name}] [id:${id}]` }); + throw new BadRequestError({ message: `Failed to import to project [name:${project.name}` }); }); originalToNewProjectId.set(project.id, newProject.id); @@ -141,7 +141,7 @@ export const importDataIntoInfisicalFn = async ({ // Import environments if (data.environments) { - for await (const [id, environment] of data.environments) { + for await (const environment of data.environments) { try { const newEnvironment = await projectEnvService.createEnvironment({ actor, @@ -154,12 +154,12 @@ export const importDataIntoInfisicalFn = async ({ }); if (!newEnvironment) { - logger.error(`Failed to import environment: [name:${environment.name}] [id:${id}]`); + logger.error(`Failed to import environment: [name:${environment.name}]`); throw new BadRequestError({ - message: `Failed to import environment: [name:${environment.name}] [id:${id}]` + message: `Failed to import environment: [name:${environment.name}]` }); } - originalToNewEnvironmentId.set(id, newEnvironment.slug); + originalToNewEnvironmentId.set(environment.id, newEnvironment.slug); } catch (error) { throw new BadRequestError({ message: `Failed to import environment: ${environment.name}]`, @@ -169,28 +169,47 @@ export const importDataIntoInfisicalFn = async ({ } } - // Import secrets - if (data.secrets) { - for await (const [id, secret] of data.secrets) { - const dataProjectId = data.environments?.get(secret.environmentId)?.projectId; - if (!dataProjectId) { - throw new BadRequestError({ message: `Failed to import secret "${secret.name}", project not found` }); + if (data.secrets && data.secrets.length > 0) { + const mappedToEnvironmentId = new Map< + string, + { + secretKey: string; + secretValue: string; + }[] + >(); + + for (const secret of data.secrets) { + if (!mappedToEnvironmentId.has(secret.environmentId)) { + mappedToEnvironmentId.set(secret.environmentId, []); } - const projectId = originalToNewProjectId.get(dataProjectId); - const newSecret = await secretService.createSecretRaw({ - actorId, - actor, - actorOrgId, - environment: originalToNewEnvironmentId.get(secret.environmentId)!, - actorAuthMethod, - projectId: projectId!, - secretPath: "/", - secretName: secret.name, - type: SecretType.Shared, + mappedToEnvironmentId.get(secret.environmentId)!.push({ + secretKey: secret.name, secretValue: secret.value || "" }); - if (!newSecret) { - throw new BadRequestError({ message: `Failed to import secret: [name:${secret.name}] [id:${id}]` }); + } + + // for each of the mappedEnvironmentId + for await (const [envId, secrets] of mappedToEnvironmentId) { + const environment = data.environments.find((env) => env.id === envId); + const projectId = environment?.projectId; + + if (!projectId) { + throw new BadRequestError({ message: `Failed to import secret, project not found` }); + } + + const secretBatches = chunkArray(secrets, 2500); + + for await (const secretBatch of secretBatches) { + await secretV2BridgeService.createManySecret({ + actorId, + actor, + actorOrgId, + environment: originalToNewEnvironmentId.get(envId)!, + actorAuthMethod, + projectId: originalToNewProjectId.get(projectId)!, + secretPath: "/", + secrets: secretBatch + }); } } } diff --git a/backend/src/services/external-migration/external-migration-queue.ts b/backend/src/services/external-migration/external-migration-queue.ts new file mode 100644 index 000000000..4bea10814 --- /dev/null +++ b/backend/src/services/external-migration/external-migration-queue.ts @@ -0,0 +1,111 @@ +import { SecretEncryptionAlgo, SecretKeyEncoding } from "@app/db/schemas"; +import { infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption"; +import { logger } from "@app/lib/logger"; +import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; + +import { TOrgServiceFactory } from "../org/org-service"; +import { TProjectServiceFactory } from "../project/project-service"; +import { TProjectEnvServiceFactory } from "../project-env/project-env-service"; +import { TSecretV2BridgeServiceFactory } from "../secret-v2-bridge/secret-v2-bridge-service"; +import { SmtpTemplates, TSmtpService } from "../smtp/smtp-service"; +import { importDataIntoInfisicalFn } from "./external-migration-fns"; +import { ExternalPlatforms, TImportInfisicalDataCreate } from "./external-migration-types"; + +export type TExternalMigrationQueueFactoryDep = { + projectService: Pick; + orgService: Pick; + projectEnvService: Pick; + secretV2BridgeService: Pick; + + smtpService: TSmtpService; + queueService: TQueueServiceFactory; +}; + +export type TExternalMigrationQueueFactory = ReturnType; + +export const externalMigrationQueueFactory = ({ + queueService, + projectService, + orgService, + smtpService, + projectEnvService, + secretV2BridgeService +}: TExternalMigrationQueueFactoryDep) => { + const startImport = async (dto: { + actorEmail: string; + data: { + iv: string; + tag: string; + ciphertext: string; + algorithm: SecretEncryptionAlgo; + encoding: SecretKeyEncoding; + }; + }) => { + await queueService.queue( + QueueName.ImportSecretsFromExternalSource, + QueueJobs.ImportSecretsFromExternalSource, + dto, + { + removeOnComplete: true, + removeOnFail: true + } + ); + }; + + queueService.start(QueueName.ImportSecretsFromExternalSource, async (job) => { + try { + const { data, actorEmail } = job.data; + + await smtpService.sendMail({ + recipients: [actorEmail], + subjectLine: "Infisical import started", + substitutions: { + provider: ExternalPlatforms.EnvKey + }, + template: SmtpTemplates.ExternalImportStarted + }); + + const decrypted = infisicalSymmetricDecrypt({ + ciphertext: data.ciphertext, + iv: data.iv, + keyEncoding: data.encoding, + tag: data.tag + }); + + const decryptedJson = JSON.parse(decrypted) as TImportInfisicalDataCreate; + + await importDataIntoInfisicalFn({ + input: decryptedJson, + projectService, + orgService, + projectEnvService, + secretV2BridgeService + }); + + await smtpService.sendMail({ + recipients: [actorEmail], + subjectLine: "Infisical import successful", + substitutions: { + provider: ExternalPlatforms.EnvKey + }, + template: SmtpTemplates.ExternalImportSuccessful + }); + } catch (err) { + await smtpService.sendMail({ + recipients: [job.data.actorEmail], + subjectLine: "Infisical import failed", + substitutions: { + provider: ExternalPlatforms.EnvKey, + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-assignment + error: (err as any)?.message || "Unknown error" + }, + template: SmtpTemplates.ExternalImportFailed + }); + + logger.error(err, "Failed to import data from external source"); + } + }); + return { + startImport + }; +}; diff --git a/backend/src/services/external-migration/external-migration-service.ts b/backend/src/services/external-migration/external-migration-service.ts index a65a278f5..700819022 100644 --- a/backend/src/services/external-migration/external-migration-service.ts +++ b/backend/src/services/external-migration/external-migration-service.ts @@ -1,30 +1,25 @@ import { OrgMembershipRole } from "@app/db/schemas"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; +import { infisicalSymmetricEncypt } from "@app/lib/crypto/encryption"; import { ForbiddenRequestError } from "@app/lib/errors"; -import { TOrgServiceFactory } from "../org/org-service"; -import { TProjectServiceFactory } from "../project/project-service"; -import { TProjectEnvServiceFactory } from "../project-env/project-env-service"; -import { TSecretServiceFactory } from "../secret/secret-service"; -import { decryptEnvKeyDataFn, importDataIntoInfisicalFn, parseEnvKeyDataFn } from "./external-migration-fns"; +import { TUserDALFactory } from "../user/user-dal"; +import { decryptEnvKeyDataFn, parseEnvKeyDataFn } from "./external-migration-fns"; +import { TExternalMigrationQueueFactory } from "./external-migration-queue"; import { TImportEnvKeyDataCreate } from "./external-migration-types"; type TExternalMigrationServiceFactoryDep = { - projectService: TProjectServiceFactory; - orgService: TOrgServiceFactory; - projectEnvService: TProjectEnvServiceFactory; - secretService: TSecretServiceFactory; permissionService: TPermissionServiceFactory; + externalMigrationQueue: TExternalMigrationQueueFactory; + userDAL: Pick; }; export type TExternalMigrationServiceFactory = ReturnType; export const externalMigrationServiceFactory = ({ - projectService, - orgService, - projectEnvService, permissionService, - secretService + externalMigrationQueue, + userDAL }: TExternalMigrationServiceFactoryDep) => { const importEnvKeyData = async ({ decryptionKey, @@ -41,21 +36,28 @@ export const externalMigrationServiceFactory = ({ actorAuthMethod, actorOrgId ); - if (membership.role !== OrgMembershipRole.Admin) { throw new ForbiddenRequestError({ message: "Only admins can import data" }); } + const user = await userDAL.findById(actorId); const json = await decryptEnvKeyDataFn(decryptionKey, encryptedJson); const envKeyData = await parseEnvKeyDataFn(json); - const response = await importDataIntoInfisicalFn({ - input: { data: envKeyData, actor, actorId, actorOrgId, actorAuthMethod }, - projectService, - orgService, - projectEnvService, - secretService + + const stringifiedJson = JSON.stringify({ + data: envKeyData, + actor, + actorId, + actorOrgId, + actorAuthMethod + }); + + const encrypted = infisicalSymmetricEncypt(stringifiedJson); + + await externalMigrationQueue.startImport({ + actorEmail: user.email!, + data: encrypted }); - return response; }; return { diff --git a/backend/src/services/external-migration/external-migration-types.ts b/backend/src/services/external-migration/external-migration-types.ts index 4139b0cf5..53c954bf9 100644 --- a/backend/src/services/external-migration/external-migration-types.ts +++ b/backend/src/services/external-migration/external-migration-types.ts @@ -1,26 +1,9 @@ import { ActorAuthMethod, ActorType } from "../auth/auth-type"; export type InfisicalImportData = { - projects: Map; - - environments?: Map< - string, - { - name: string; - id: string; - projectId: string; - } - >; - - secrets?: Map< - string, - { - name: string; - id: string; - environmentId: string; - value?: string; - } - >; + projects: Array<{ name: string; id: string }>; + environments: Array<{ name: string; id: string; projectId: string }>; + secrets: Array<{ name: string; id: string; environmentId: string; value: string }>; }; export type TImportEnvKeyDataCreate = { @@ -104,3 +87,7 @@ export type TEnvKeyExportJSON = { } >; }; + +export enum ExternalPlatforms { + EnvKey = "EnvKey" +} diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index 4b3f7dfbf..15fd33027 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -85,7 +85,7 @@ type TSecretQueueFactoryDep = { secretTagDAL: TSecretTagDALFactory; userDAL: Pick; secretVersionTagDAL: TSecretVersionTagDALFactory; - kmsService: Pick; + kmsService: TKmsServiceFactory; secretV2BridgeDAL: TSecretV2BridgeDALFactory; secretVersionV2BridgeDAL: Pick; secretVersionTagV2BridgeDAL: Pick; diff --git a/backend/src/services/smtp/smtp-service.ts b/backend/src/services/smtp/smtp-service.ts index 823da4cca..1f38babb3 100644 --- a/backend/src/services/smtp/smtp-service.ts +++ b/backend/src/services/smtp/smtp-service.ts @@ -34,7 +34,10 @@ export enum SmtpTemplates { WorkspaceInvite = "workspaceInvitation.handlebars", ScimUserProvisioned = "scimUserProvisioned.handlebars", PkiExpirationAlert = "pkiExpirationAlert.handlebars", - IntegrationSyncFailed = "integrationSyncFailed.handlebars" + IntegrationSyncFailed = "integrationSyncFailed.handlebars", + ExternalImportSuccessful = "externalImportSuccessful.handlebars", + ExternalImportFailed = "externalImportFailed.handlebars", + ExternalImportStarted = "externalImportStarted.handlebars" } export enum SmtpHost { diff --git a/backend/src/services/smtp/templates/externalImportFailed.handlebars b/backend/src/services/smtp/templates/externalImportFailed.handlebars new file mode 100644 index 000000000..c7869af27 --- /dev/null +++ b/backend/src/services/smtp/templates/externalImportFailed.handlebars @@ -0,0 +1,21 @@ + + + + + + Import failed + + + +

An import from {{provider}} to Infisical has failed

+

An import from + {{provider}} + to Infisical has failed due to unforeseen circumstances. Please re-try your import, and if the issue persists, you + can contact the Infisical team at team@infisical.com. +

+ +

Error: {{error}}

+ + + + \ No newline at end of file diff --git a/backend/src/services/smtp/templates/externalImportStarted.handlebars b/backend/src/services/smtp/templates/externalImportStarted.handlebars new file mode 100644 index 000000000..551f972cc --- /dev/null +++ b/backend/src/services/smtp/templates/externalImportStarted.handlebars @@ -0,0 +1,17 @@ + + + + + + Import in progress + + + +

An import from {{provider}} to Infisical is in progress

+

An import from + {{provider}} + to Infisical is in progress. The import process may take up to 30 minutes, and you will receive once the import + has finished or if it fails.

+ + + \ No newline at end of file diff --git a/backend/src/services/smtp/templates/externalImportSuccessful.handlebars b/backend/src/services/smtp/templates/externalImportSuccessful.handlebars new file mode 100644 index 000000000..51a1c465e --- /dev/null +++ b/backend/src/services/smtp/templates/externalImportSuccessful.handlebars @@ -0,0 +1,14 @@ + + + + + + Import successful + + + +

An import from {{provider}} to Infisical was successful

+

An import from {{provider}} was successful. Your data is now available in Infisical.

+ + + \ No newline at end of file diff --git a/frontend/src/hooks/api/migration/mutations.tsx b/frontend/src/hooks/api/migration/mutations.tsx index fb9bebeff..7d434ddbe 100644 --- a/frontend/src/hooks/api/migration/mutations.tsx +++ b/frontend/src/hooks/api/migration/mutations.tsx @@ -18,11 +18,10 @@ export const useImportEnvKey = () => { }; decryptionKey: string; }) => { - const { data } = await apiRequest.post("/api/v3/migrate/env-key/", { + await apiRequest.post("/api/v3/migrate/env-key/", { encryptedJson, decryptionKey }); - return data; }, onSuccess: () => { queryClient.invalidateQueries(workspaceKeys.getAllUserWorkspace); From ad2f19658bc590c73866f42f013d339213b3cb7d Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Sat, 12 Oct 2024 07:40:14 +0400 Subject: [PATCH 2/3] requested changes --- .../secret-snapshot-service.ts | 55 ++-- backend/src/server/routes/index.ts | 8 + .../external-migration-fns.ts | 253 +++++++++++------- .../external-migration-queue.ts | 42 ++- backend/src/services/kms/kms-service.ts | 67 ++--- backend/src/services/org/org-service.ts | 72 +++-- backend/src/services/org/org-types.ts | 11 +- .../project-env/project-env-service.ts | 5 +- .../services/project-env/project-env-types.ts | 3 + .../src/services/project/project-service.ts | 3 +- backend/src/services/project/project-types.ts | 3 + .../secret-v2-bridge/secret-v2-bridge-fns.ts | 5 +- .../secret-v2-bridge-service.ts | 5 +- .../secret-v2-bridge-types.ts | 1 + .../components/EnvKeyPlatformModal.tsx | 5 +- 15 files changed, 335 insertions(+), 203 deletions(-) diff --git a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts index 6e3d53e58..de285a0f2 100644 --- a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts +++ b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts @@ -1,15 +1,9 @@ import { ForbiddenError, subject } from "@casl/ability"; -import { - TableName, - TSecretSnapshotFolders, - TSecretSnapshotSecretsV2, - TSecretTagJunctionInsert, - TSecretV2TagJunctionInsert -} from "@app/db/schemas"; +import { TableName, TSecretTagJunctionInsert, TSecretV2TagJunctionInsert } from "@app/db/schemas"; import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; import { InternalServerError, NotFoundError } from "@app/lib/errors"; -import { chunkArray, groupBy } from "@app/lib/fn"; +import { groupBy } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; @@ -247,36 +241,23 @@ export const secretSnapshotServiceFactory = ({ tx ); - const chunkedSnapshotSecrets = chunkArray(secretVersions, 2500); - const chunkedSnapshotFolders = chunkArray(folderVersions, 2500); - const snapshotSecrets: TSecretSnapshotSecretsV2[] = []; - const snapshotFolders: TSecretSnapshotFolders[] = []; + const snapshotSecrets = await snapshotSecretV2BridgeDAL.batchInsert( + secretVersions.map(({ id }) => ({ + secretVersionId: id, + envId: folder.environment.envId, + snapshotId: newSnapshot.id + })), + tx + ); - for await (const chunk of chunkedSnapshotSecrets) { - const result = await snapshotSecretV2BridgeDAL.insertMany( - chunk.map(({ id }) => ({ - secretVersionId: id, - envId: folder.environment.envId, - snapshotId: newSnapshot.id - })), - tx - ); - - snapshotSecrets.push(...result); - } - - for await (const chunk of chunkedSnapshotFolders) { - const result = await snapshotFolderDAL.insertMany( - chunk.map(({ id }) => ({ - folderVersionId: id, - envId: folder.environment.envId, - snapshotId: newSnapshot.id - })), - tx - ); - - snapshotFolders.push(...result); - } + const snapshotFolders = await snapshotFolderDAL.batchInsert( + folderVersions.map(({ id }) => ({ + folderVersionId: id, + envId: folder.environment.envId, + snapshotId: newSnapshot.id + })), + tx + ); return { ...newSnapshot, secrets: snapshotSecrets, folder: snapshotFolders }; }); diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 3d649c469..973872d71 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -1206,8 +1206,16 @@ export const registerRoutes = async ( const externalMigrationQueue = externalMigrationQueueFactory({ orgService, projectEnvService, + projectDAL, projectService, smtpService, + kmsService, + projectEnvDAL, + secretVersionDAL: secretVersionV2BridgeDAL, + secretTagDAL, + secretVersionTagDAL: secretVersionTagV2BridgeDAL, + folderDAL, + secretDAL: secretV2BridgeDAL, queueService, secretV2BridgeService }); diff --git a/backend/src/services/external-migration/external-migration-fns.ts b/backend/src/services/external-migration/external-migration-fns.ts index bae98da2f..e2707c0b2 100644 --- a/backend/src/services/external-migration/external-migration-fns.ts +++ b/backend/src/services/external-migration/external-migration-fns.ts @@ -4,19 +4,39 @@ import sjcl from "sjcl"; import tweetnacl from "tweetnacl"; import tweetnaclUtil from "tweetnacl-util"; -import { OrgMembershipRole, ProjectMembershipRole } from "@app/db/schemas"; -import { BadRequestError } from "@app/lib/errors"; +import { OrgMembershipRole, ProjectMembershipRole, SecretType } from "@app/db/schemas"; +import { BadRequestError, NotFoundError } from "@app/lib/errors"; import { chunkArray } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; +import { TKmsServiceFactory } from "../kms/kms-service"; +import { KmsDataKey } from "../kms/kms-types"; import { TOrgServiceFactory } from "../org/org-service"; +import { TProjectDALFactory } from "../project/project-dal"; import { TProjectServiceFactory } from "../project/project-service"; +import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TProjectEnvServiceFactory } from "../project-env/project-env-service"; +import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; +import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; +import { fnSecretBulkInsert, getAllNestedSecretReferences } from "../secret-v2-bridge/secret-v2-bridge-fns"; import type { TSecretV2BridgeServiceFactory } from "../secret-v2-bridge/secret-v2-bridge-service"; +import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal"; +import { TSecretVersionV2TagDALFactory } from "../secret-v2-bridge/secret-version-tag-dal"; import { InfisicalImportData, TEnvKeyExportJSON, TImportInfisicalDataCreate } from "./external-migration-types"; export type TImportDataIntoInfisicalDTO = { + projectDAL: Pick; + projectEnvDAL: Pick; + kmsService: Pick; + + secretDAL: Pick; + secretVersionDAL: Pick; + secretTagDAL: Pick; + secretVersionTagDAL: Pick; + + folderDAL: Pick; projectService: Pick; orgService: Pick; projectEnvService: Pick; @@ -91,9 +111,15 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise { // Import data to infisical @@ -104,113 +130,152 @@ export const importDataIntoInfisicalFn = async ({ const originalToNewProjectId = new Map(); const originalToNewEnvironmentId = new Map(); - for await (const project of data.projects) { - const newProject = await projectService - .createProject({ - actor, - actorId, - actorOrgId, - actorAuthMethod, - workspaceName: project.name, - createDefaultEnvs: false - }) - .catch(() => { - throw new BadRequestError({ message: `Failed to import to project [name:${project.name}` }); - }); - - originalToNewProjectId.set(project.id, newProject.id); - } - - // Invite user importing projects - const invites = await orgService.inviteUserToOrganization({ - actorAuthMethod, - actorId, - actorOrgId, - actor, - inviteeEmails: [], - orgId: actorOrgId, - organizationRoleSlug: OrgMembershipRole.NoAccess, - projects: Array.from(originalToNewProjectId.values()).map((project) => ({ - id: project, - projectRoleSlug: [ProjectMembershipRole.Member] - })) - }); - if (!invites) { - throw new BadRequestError({ message: `Failed to invite user to projects: [userId:${actorId}]` }); - } - - // Import environments - if (data.environments) { - for await (const environment of data.environments) { - try { - const newEnvironment = await projectEnvService.createEnvironment({ + await projectDAL.transaction(async (tx) => { + for await (const project of data.projects) { + const newProject = await projectService + .createProject({ actor, actorId, actorOrgId, actorAuthMethod, - name: environment.name, - projectId: originalToNewProjectId.get(environment.projectId)!, - slug: slugify(`${environment.name}-${alphaNumericNanoId(4)}`) + workspaceName: project.name, + createDefaultEnvs: false, + tx + }) + .catch((e) => { + logger.error(e, `Failed to import to project [name:${project.name}]`); + throw new BadRequestError({ message: `Failed to import to project [name:${project.name}]` }); }); - if (!newEnvironment) { - logger.error(`Failed to import environment: [name:${environment.name}]`); + originalToNewProjectId.set(project.id, newProject.id); + } + + // Invite user importing projects + const invites = await orgService.inviteUserToOrganization({ + verifyPermissions: false, + actorAuthMethod, + actorId, + actorOrgId, + actor, + inviteeEmails: [], + orgId: actorOrgId, + organizationRoleSlug: OrgMembershipRole.NoAccess, + projects: Array.from(originalToNewProjectId.values()).map((project) => ({ + id: project, + projectRoleSlug: [ProjectMembershipRole.Member] + })), + tx + }); + if (!invites) { + throw new BadRequestError({ message: `Failed to invite user to projects: [userId:${actorId}]` }); + } + + // Import environments + if (data.environments) { + for await (const environment of data.environments) { + const projectId = originalToNewProjectId.get(environment.projectId)!; + const slug = slugify(`${environment.name}-${alphaNumericNanoId(4)}`); + + const existingEnv = await projectEnvDAL.findOne({ projectId, slug }, tx); + + if (existingEnv) { throw new BadRequestError({ - message: `Failed to import environment: [name:${environment.name}]` + message: `Environment with slug '${slug}' already exist`, + name: "CreateEnvironment" }); } - originalToNewEnvironmentId.set(environment.id, newEnvironment.slug); - } catch (error) { - throw new BadRequestError({ - message: `Failed to import environment: ${environment.name}]`, - name: "EnvKeyMigrationImportEnvironment" + + const lastPos = await projectEnvDAL.findLastEnvPosition(projectId, tx); + const doc = await projectEnvDAL.create({ slug, name: environment.name, projectId, position: lastPos + 1 }, tx); + await folderDAL.create({ name: "root", parentId: null, envId: doc.id, version: 1 }, tx); + + originalToNewEnvironmentId.set(environment.id, doc.slug); + } + } + + if (data.secrets && data.secrets.length > 0) { + const mappedToEnvironmentId = new Map< + string, + { + secretKey: string; + secretValue: string; + }[] + >(); + + for (const secret of data.secrets) { + if (!mappedToEnvironmentId.has(secret.environmentId)) { + mappedToEnvironmentId.set(secret.environmentId, []); + } + mappedToEnvironmentId.get(secret.environmentId)!.push({ + secretKey: secret.name, + secretValue: secret.value || "" }); } - } - } - if (data.secrets && data.secrets.length > 0) { - const mappedToEnvironmentId = new Map< - string, - { - secretKey: string; - secretValue: string; - }[] - >(); + // for each of the mappedEnvironmentId + for await (const [envId, secrets] of mappedToEnvironmentId) { + const environment = data.environments.find((env) => env.id === envId); + const projectId = originalToNewProjectId.get(environment?.projectId as string)!; - for (const secret of data.secrets) { - if (!mappedToEnvironmentId.has(secret.environmentId)) { - mappedToEnvironmentId.set(secret.environmentId, []); - } - mappedToEnvironmentId.get(secret.environmentId)!.push({ - secretKey: secret.name, - secretValue: secret.value || "" - }); - } + if (!projectId) { + throw new BadRequestError({ message: `Failed to import secret, project not found` }); + } - // for each of the mappedEnvironmentId - for await (const [envId, secrets] of mappedToEnvironmentId) { - const environment = data.environments.find((env) => env.id === envId); - const projectId = environment?.projectId; + const { encryptor: secretManagerEncrypt } = await kmsService.createCipherPairWithDataKey( + { + type: KmsDataKey.SecretManager, + projectId + }, + tx + ); - if (!projectId) { - throw new BadRequestError({ message: `Failed to import secret, project not found` }); - } + const envSlug = originalToNewEnvironmentId.get(envId)!; + const folder = await folderDAL.findBySecretPath(projectId, envSlug, "/", tx); + if (!folder) + throw new NotFoundError({ + message: `Folder not found for the given environment slug (${envSlug}) & secret path (/)`, + name: "Create secret" + }); - const secretBatches = chunkArray(secrets, 2500); + const secretsByKeys = await secretDAL.findBySecretKeys( + folder.id, + secrets.map((el) => ({ + key: el.secretKey, + type: SecretType.Shared + })), + tx + ); + if (secretsByKeys.length) { + throw new BadRequestError({ + message: `Secret already exist: ${secretsByKeys.map((el) => el.key).join(",")}` + }); + } - for await (const secretBatch of secretBatches) { - await secretV2BridgeService.createManySecret({ - actorId, - actor, - actorOrgId, - environment: originalToNewEnvironmentId.get(envId)!, - actorAuthMethod, - projectId: originalToNewProjectId.get(projectId)!, - secretPath: "/", - secrets: secretBatch - }); + const secretBatches = chunkArray(secrets, 2500); + for await (const secretBatch of secretBatches) { + await fnSecretBulkInsert({ + inputSecrets: secretBatch.map((el) => { + const references = getAllNestedSecretReferences(el.secretValue); + + return { + version: 1, + encryptedValue: el.secretValue + ? secretManagerEncrypt({ plainText: Buffer.from(el.secretValue) }).cipherTextBlob + : undefined, + key: el.secretKey, + references, + type: SecretType.Shared + }; + }), + folderId: folder.id, + secretDAL, + secretVersionDAL, + secretTagDAL, + secretVersionTagDAL, + tx + }); + } } } - } + }); }; diff --git a/backend/src/services/external-migration/external-migration-queue.ts b/backend/src/services/external-migration/external-migration-queue.ts index 4bea10814..879899d49 100644 --- a/backend/src/services/external-migration/external-migration-queue.ts +++ b/backend/src/services/external-migration/external-migration-queue.ts @@ -3,22 +3,40 @@ import { infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption"; import { logger } from "@app/lib/logger"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; +import { TKmsServiceFactory } from "../kms/kms-service"; import { TOrgServiceFactory } from "../org/org-service"; +import { TProjectDALFactory } from "../project/project-dal"; import { TProjectServiceFactory } from "../project/project-service"; +import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TProjectEnvServiceFactory } from "../project-env/project-env-service"; +import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; +import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; import { TSecretV2BridgeServiceFactory } from "../secret-v2-bridge/secret-v2-bridge-service"; +import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal"; +import { TSecretVersionV2TagDALFactory } from "../secret-v2-bridge/secret-version-tag-dal"; import { SmtpTemplates, TSmtpService } from "../smtp/smtp-service"; import { importDataIntoInfisicalFn } from "./external-migration-fns"; import { ExternalPlatforms, TImportInfisicalDataCreate } from "./external-migration-types"; export type TExternalMigrationQueueFactoryDep = { + smtpService: TSmtpService; + queueService: TQueueServiceFactory; + + projectDAL: Pick; + projectEnvDAL: Pick; + kmsService: Pick; + + secretDAL: Pick; + secretVersionDAL: Pick; + secretTagDAL: Pick; + secretVersionTagDAL: Pick; + + folderDAL: Pick; projectService: Pick; orgService: Pick; projectEnvService: Pick; secretV2BridgeService: Pick; - - smtpService: TSmtpService; - queueService: TQueueServiceFactory; }; export type TExternalMigrationQueueFactory = ReturnType; @@ -28,8 +46,16 @@ export const externalMigrationQueueFactory = ({ projectService, orgService, smtpService, + projectDAL, projectEnvService, - secretV2BridgeService + secretV2BridgeService, + kmsService, + projectEnvDAL, + secretDAL, + secretVersionDAL, + secretTagDAL, + secretVersionTagDAL, + folderDAL }: TExternalMigrationQueueFactoryDep) => { const startImport = async (dto: { actorEmail: string; @@ -76,6 +102,14 @@ export const externalMigrationQueueFactory = ({ await importDataIntoInfisicalFn({ input: decryptedJson, + projectDAL, + projectEnvDAL, + secretDAL, + secretVersionDAL, + secretTagDAL, + secretVersionTagDAL, + folderDAL, + kmsService, projectService, orgService, projectEnvService, diff --git a/backend/src/services/kms/kms-service.ts b/backend/src/services/kms/kms-service.ts index 50dc0bd08..e1166d8c0 100644 --- a/backend/src/services/kms/kms-service.ts +++ b/backend/src/services/kms/kms-service.ts @@ -160,8 +160,8 @@ export const kmsServiceFactory = ({ * In mean time the rest of the request will wait until creation is finished followed by getting the created on * In real time this would be milliseconds */ - const getOrgKmsKeyId = async (orgId: string) => { - let org = await orgDAL.findById(orgId); + const getOrgKmsKeyId = async (orgId: string, trx?: Knex) => { + let org = await orgDAL.findById(orgId, trx); if (!org) { throw new NotFoundError({ message: "Org not found" }); @@ -180,9 +180,9 @@ export const kmsServiceFactory = ({ waitingCb: () => logger.info("KMS. Waiting for org key to be created") }); - org = await orgDAL.findById(orgId); + org = await orgDAL.findById(orgId, trx); } else { - const keyId = await orgDAL.transaction(async (tx) => { + const keyId = await (trx || orgDAL).transaction(async (tx) => { org = await orgDAL.findById(orgId, tx); if (org.kmsDefaultKeyId) { return org.kmsDefaultKeyId; @@ -240,11 +240,12 @@ export const kmsServiceFactory = ({ const decryptWithKmsKey = async ({ kmsId, - depth = 0 - }: Omit & { depth?: number }) => { + depth = 0, + tx + }: Omit & { depth?: number; tx?: Knex }) => { if (depth > 2) throw new BadRequestError({ message: "KMS depth max limit" }); - const kmsDoc = await kmsDAL.findByIdWithAssociatedKms(kmsId); + const kmsDoc = await kmsDAL.findByIdWithAssociatedKms(kmsId, tx); if (!kmsDoc) { throw new NotFoundError({ message: "KMS ID not found" }); } @@ -261,7 +262,8 @@ export const kmsServiceFactory = ({ // we put a limit of depth to avoid too many cycles const orgKmsDecryptor = await decryptWithKmsKey({ kmsId: kmsDoc.orgKms.id, - depth: depth + 1 + depth: depth + 1, + tx }); const orgKmsDataKey = await orgKmsDecryptor({ @@ -375,9 +377,9 @@ export const kmsServiceFactory = ({ }; }; - const $getOrgKmsDataKey = async (orgId: string) => { - const kmsKeyId = await getOrgKmsKeyId(orgId); - let org = await orgDAL.findById(orgId); + const $getOrgKmsDataKey = async (orgId: string, trx?: Knex) => { + const kmsKeyId = await getOrgKmsKeyId(orgId, trx); + let org = await orgDAL.findById(orgId, trx); if (!org) { throw new NotFoundError({ message: "Org not found" }); @@ -396,9 +398,9 @@ export const kmsServiceFactory = ({ waitingCb: () => logger.info("KMS. Waiting for org data key to be created") }); - org = await orgDAL.findById(orgId); + org = await orgDAL.findById(orgId, trx); } else { - const orgDataKey = await orgDAL.transaction(async (tx) => { + const orgDataKey = await (trx || orgDAL).transaction(async (tx) => { org = await orgDAL.findById(orgId, tx); if (org.kmsEncryptedDataKey) { return; @@ -455,8 +457,8 @@ export const kmsServiceFactory = ({ }); }; - const getProjectSecretManagerKmsKeyId = async (projectId: string) => { - let project = await projectDAL.findById(projectId); + const getProjectSecretManagerKmsKeyId = async (projectId: string, trx?: Knex) => { + let project = await projectDAL.findById(projectId, trx); if (!project) { throw new NotFoundError({ message: "Project not found" }); } @@ -477,7 +479,7 @@ export const kmsServiceFactory = ({ project = await projectDAL.findById(projectId); } else { - const kmsKeyId = await projectDAL.transaction(async (tx) => { + const kmsKeyId = await (trx || projectDAL).transaction(async (tx) => { project = await projectDAL.findById(projectId, tx); if (project.kmsSecretManagerKeyId) { return project.kmsSecretManagerKeyId; @@ -520,9 +522,9 @@ export const kmsServiceFactory = ({ return project.kmsSecretManagerKeyId; }; - const $getProjectSecretManagerKmsDataKey = async (projectId: string) => { - const kmsKeyId = await getProjectSecretManagerKmsKeyId(projectId); - let project = await projectDAL.findById(projectId); + const $getProjectSecretManagerKmsDataKey = async (projectId: string, trx?: Knex) => { + const kmsKeyId = await getProjectSecretManagerKmsKeyId(projectId, trx); + let project = await projectDAL.findById(projectId, trx); if (!project.kmsSecretManagerEncryptedDataKey) { const lock = await keyStore @@ -538,18 +540,21 @@ export const kmsServiceFactory = ({ delay: 500 }); - project = await projectDAL.findById(projectId); + project = await projectDAL.findById(projectId, trx); } else { - const projectDataKey = await projectDAL.transaction(async (tx) => { + const projectDataKey = await (trx || projectDAL).transaction(async (tx) => { project = await projectDAL.findById(projectId, tx); if (project.kmsSecretManagerEncryptedDataKey) { return; } const dataKey = randomSecureBytes(); - const kmsEncryptor = await encryptWithKmsKey({ - kmsId: kmsKeyId - }); + const kmsEncryptor = await encryptWithKmsKey( + { + kmsId: kmsKeyId + }, + tx + ); const { cipherTextBlob } = await kmsEncryptor({ plainText: dataKey @@ -585,7 +590,8 @@ export const kmsServiceFactory = ({ } const kmsDecryptor = await decryptWithKmsKey({ - kmsId: kmsKeyId + kmsId: kmsKeyId, + tx: trx }); return kmsDecryptor({ @@ -593,13 +599,13 @@ export const kmsServiceFactory = ({ }); }; - const $getDataKey = async (dto: TEncryptWithKmsDataKeyDTO) => { + const $getDataKey = async (dto: TEncryptWithKmsDataKeyDTO, trx?: Knex) => { switch (dto.type) { case KmsDataKey.SecretManager: { - return $getProjectSecretManagerKmsDataKey(dto.projectId); + return $getProjectSecretManagerKmsDataKey(dto.projectId, trx); } default: { - return $getOrgKmsDataKey(dto.orgId); + return $getOrgKmsDataKey(dto.orgId, trx); } } }; @@ -607,8 +613,9 @@ export const kmsServiceFactory = ({ // by keeping the decrypted data key in inner scope // none of the entities outside can interact directly or expose the data key // NOTICE: If changing here update migrations/utils/kms - const createCipherPairWithDataKey = async (encryptionContext: TEncryptWithKmsDataKeyDTO) => { - const dataKey = await $getDataKey(encryptionContext); + const createCipherPairWithDataKey = async (encryptionContext: TEncryptWithKmsDataKeyDTO, trx?: Knex) => { + const dataKey = await $getDataKey(encryptionContext, trx); + const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256); return { diff --git a/backend/src/services/org/org-service.ts b/backend/src/services/org/org-service.ts index fd9b8106b..34488ed34 100644 --- a/backend/src/services/org/org-service.ts +++ b/backend/src/services/org/org-service.ts @@ -1,5 +1,6 @@ -import { ForbiddenError } from "@casl/ability"; +import { ForbiddenError, MongoAbility } from "@casl/ability"; import slugify from "@sindresorhus/slugify"; +import { MongoQuery } from "@ucast/mongo2js"; import crypto from "crypto"; import jwt from "jsonwebtoken"; import { Knex } from "knex"; @@ -19,7 +20,11 @@ import { TProjects } from "@app/db/schemas/projects"; import { TGroupDALFactory } from "@app/ee/services/group/group-dal"; import { TLicenseServiceFactory } from "@app/ee/services/license/license-service"; import { TOidcConfigDALFactory } from "@app/ee/services/oidc/oidc-config-dal"; -import { OrgPermissionActions, OrgPermissionSubjects } from "@app/ee/services/permission/org-permission"; +import { + OrgPermissionActions, + OrgPermissionSet, + OrgPermissionSubjects +} from "@app/ee/services/permission/org-permission"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; import { TProjectUserAdditionalPrivilegeDALFactory } from "@app/ee/services/project-user-additional-privilege/project-user-additional-privilege-dal"; @@ -487,11 +492,17 @@ export const orgServiceFactory = ({ organizationRoleSlug, projects: invitedProjects, actorAuthMethod, - actorOrgId + actorOrgId, + tx: trx, + verifyPermissions = true }: TInviteUserToOrgDTO) => { const appCfg = getConfig(); - const { permission } = await permissionService.getOrgPermission(actor, actorId, orgId, actorAuthMethod, actorOrgId); + let permission: MongoAbility | undefined; + if (verifyPermissions) { + permission = (await permissionService.getOrgPermission(actor, actorId, orgId, actorAuthMethod, actorOrgId)) + .permission; + } const org = await orgDAL.findOrgById(orgId); @@ -512,12 +523,15 @@ export const orgServiceFactory = ({ } const projectsToInvite = invitedProjects?.length - ? await projectDAL.find({ - orgId, - $in: { - id: invitedProjects?.map(({ id }) => id) - } - }) + ? await projectDAL.find( + { + orgId, + $in: { + id: invitedProjects?.map(({ id }) => id) + } + }, + { tx: trx } + ) : []; if (projectsToInvite.length !== invitedProjects?.length) { throw new ForbiddenRequestError({ @@ -534,7 +548,7 @@ export const orgServiceFactory = ({ const mailsForOrgInvitation: { email: string; userId: string; firstName: string; lastName: string }[] = []; const mailsForProjectInvitation: { email: string[]; projectName: string }[] = []; const newProjectMemberships: TProjectMemberships[] = []; - await orgDAL.transaction(async (tx) => { + await (trx || orgDAL).transaction(async (tx) => { const users: Pick[] = []; for await (const inviteeEmail of inviteeEmails) { @@ -620,8 +634,16 @@ export const orgServiceFactory = ({ }); } - // as its used by project invite also - ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Create, OrgPermissionSubjects.Member); + if (verifyPermissions) { + if (!permission) { + throw new ForbiddenRequestError({ + name: "InviteUser", + message: "Failed to invite user because no permission was found" + }); + } + // as its used by project invite also + ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Create, OrgPermissionSubjects.Member); + } let roleId; const orgRole = isCustomOrgRole ? OrgMembershipRole.Custom : organizationRoleSlug; if (isCustomOrgRole) { @@ -662,17 +684,19 @@ export const orgServiceFactory = ({ // if there exist no project membership we set is as given by the request for await (const project of projectsToInvite) { const projectId = project.id; - const { permission: projectPermission } = await permissionService.getProjectPermission( - actor, - actorId, - projectId, - actorAuthMethod, - actorOrgId - ); - ForbiddenError.from(projectPermission).throwUnlessCan( - ProjectPermissionActions.Create, - ProjectPermissionSub.Member - ); + if (verifyPermissions) { + const { permission: projectPermission } = await permissionService.getProjectPermission( + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + ); + ForbiddenError.from(projectPermission).throwUnlessCan( + ProjectPermissionActions.Create, + ProjectPermissionSub.Member + ); + } const existingMembers = await projectMembershipDAL.find( { projectId: project.id, diff --git a/backend/src/services/org/org-types.ts b/backend/src/services/org/org-types.ts index 9f1973349..5bda46e5f 100644 --- a/backend/src/services/org/org-types.ts +++ b/backend/src/services/org/org-types.ts @@ -1,3 +1,5 @@ +import { Knex } from "knex"; + import { TOrgPermission } from "@app/lib/types"; import { ActorAuthMethod, ActorType } from "../auth/auth-type"; @@ -26,18 +28,15 @@ export type TDeleteOrgMembershipDTO = { }; export type TInviteUserToOrgDTO = { - actorId: string; - actor: ActorType; - orgId: string; - actorOrgId: string | undefined; - actorAuthMethod: ActorAuthMethod; inviteeEmails: string[]; organizationRoleSlug: string; + tx?: Knex; + verifyPermissions?: boolean; projects?: { id: string; projectRoleSlug?: string[]; }[]; -}; +} & TOrgPermission; export type TVerifyUserToOrgDTO = { email: string; diff --git a/backend/src/services/project-env/project-env-service.ts b/backend/src/services/project-env/project-env-service.ts index 67bdd867a..ecbdd97b7 100644 --- a/backend/src/services/project-env/project-env-service.ts +++ b/backend/src/services/project-env/project-env-service.ts @@ -39,7 +39,8 @@ export const projectEnvServiceFactory = ({ actorAuthMethod, position, name, - slug + slug, + tx: trx }: TCreateEnvDTO) => { const { permission } = await permissionService.getProjectPermission( actor, @@ -83,7 +84,7 @@ export const projectEnvServiceFactory = ({ }); } - const env = await projectEnvDAL.transaction(async (tx) => { + const env = await (trx || projectEnvDAL).transaction(async (tx) => { if (position !== undefined) { // Check if there's an environment at the specified position const existingEnvWithPosition = await projectEnvDAL.findOne({ projectId, position }, tx); diff --git a/backend/src/services/project-env/project-env-types.ts b/backend/src/services/project-env/project-env-types.ts index a87c76d4d..427086cb7 100644 --- a/backend/src/services/project-env/project-env-types.ts +++ b/backend/src/services/project-env/project-env-types.ts @@ -1,9 +1,12 @@ +import { Knex } from "knex"; + import { TProjectPermission } from "@app/lib/types"; export type TCreateEnvDTO = { name: string; slug: string; position?: number; + tx?: Knex; } & TProjectPermission; export type TUpdateEnvDTO = { diff --git a/backend/src/services/project/project-service.ts b/backend/src/services/project/project-service.ts index ec6c54dcc..d99212744 100644 --- a/backend/src/services/project/project-service.ts +++ b/backend/src/services/project/project-service.ts @@ -147,6 +147,7 @@ export const projectServiceFactory = ({ workspaceName, slug: projectSlug, kmsKeyId, + tx: trx, createDefaultEnvs = true }: TCreateProjectDTO) => { const organization = await orgDAL.findOne({ id: actorOrgId }); @@ -169,7 +170,7 @@ export const projectServiceFactory = ({ }); } - const results = await projectDAL.transaction(async (tx) => { + const results = await (trx || projectDAL).transaction(async (tx) => { const ghostUser = await orgService.addGhostUser(organization.id, tx); if (kmsKeyId) { diff --git a/backend/src/services/project/project-types.ts b/backend/src/services/project/project-types.ts index 7193f1121..d35fcb24f 100644 --- a/backend/src/services/project/project-types.ts +++ b/backend/src/services/project/project-types.ts @@ -1,3 +1,5 @@ +import { Knex } from "knex"; + import { TProjectKeys } from "@app/db/schemas"; import { TProjectPermission } from "@app/lib/types"; @@ -30,6 +32,7 @@ export type TCreateProjectDTO = { slug?: string; kmsKeyId?: string; createDefaultEnvs?: boolean; + tx?: Knex; }; export type TDeleteProjectBySlugDTO = { diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts index 78e21b579..2ee32a7d7 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts @@ -82,7 +82,10 @@ export const fnSecretBulkInsert = async ({ }) ); - const newSecrets = await secretDAL.insertMany(sanitizedInputSecrets.map((el) => ({ ...el, folderId }))); + const newSecrets = await secretDAL.insertMany( + sanitizedInputSecrets.map((el) => ({ ...el, folderId })), + tx + ); const newSecretGroupedByKeyName = groupBy(newSecrets, (item) => item.key); const newSecretTags = inputSecrets.flatMap(({ tagIds: secretTags = [], key }) => secretTags.map((tag) => ({ diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts index 6e8d95e1c..a520c5439 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts @@ -911,7 +911,8 @@ export const secretV2BridgeServiceFactory = ({ actorOrgId, environment, projectId, - secrets: inputSecrets + secrets: inputSecrets, + tx: trx }: TCreateManySecretDTO) => { const { permission } = await permissionService.getProjectPermission( actor, @@ -951,7 +952,7 @@ export const secretV2BridgeServiceFactory = ({ const { encryptor: secretManagerEncryptor, decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ type: KmsDataKey.SecretManager, projectId }); - const newSecrets = await secretDAL.transaction(async (tx) => + const newSecrets = await (trx || secretDAL).transaction(async (tx) => fnSecretBulkInsert({ inputSecrets: inputSecrets.map((el) => { const references = getAllNestedSecretReferences(el.secretValue); diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts index a76c57561..02bf5da26 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts @@ -81,6 +81,7 @@ export type TCreateManySecretDTO = Omit & { secretPath: string; projectId: string; environment: string; + tx?: Knex; secrets: { secretKey: string; secretValue: string; diff --git a/frontend/src/views/Settings/OrgSettingsPage/components/ImportTab/components/EnvKeyPlatformModal.tsx b/frontend/src/views/Settings/OrgSettingsPage/components/ImportTab/components/EnvKeyPlatformModal.tsx index 1bfabf434..8b09ba5c7 100644 --- a/frontend/src/views/Settings/OrgSettingsPage/components/ImportTab/components/EnvKeyPlatformModal.tsx +++ b/frontend/src/views/Settings/OrgSettingsPage/components/ImportTab/components/EnvKeyPlatformModal.tsx @@ -54,8 +54,9 @@ export const EnvKeyPlatformModal = ({ onClose }: Props) => { decryptionKey: data.encryptionKey }); createNotification({ - text: "Data imported successfully.", - type: "success" + title: "Import started", + text: "Your data is being imported. You will receive an email when the import is complete or if the import fails. This may take up to 10 minutes.", + type: "info" }); onClose(); From 865db5a9b3bbeaa73c78354e5aeca60d885d294e Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Sat, 12 Oct 2024 07:54:21 +0400 Subject: [PATCH 3/3] removed redundancies --- backend/src/server/routes/index.ts | 1 - .../external-migration-fns.ts | 25 +------ .../external-migration-queue.ts | 4 -- backend/src/services/org/org-service.ts | 72 +++++++------------ backend/src/services/org/org-types.ts | 4 -- .../project-env/project-env-service.ts | 5 +- .../services/project-env/project-env-types.ts | 3 - .../secret-v2-bridge-service.ts | 5 +- .../secret-v2-bridge-types.ts | 1 - 9 files changed, 29 insertions(+), 91 deletions(-) diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 973872d71..c195c92d7 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -1204,7 +1204,6 @@ export const registerRoutes = async ( }); const externalMigrationQueue = externalMigrationQueueFactory({ - orgService, projectEnvService, projectDAL, projectService, diff --git a/backend/src/services/external-migration/external-migration-fns.ts b/backend/src/services/external-migration/external-migration-fns.ts index e2707c0b2..a69b5ef7c 100644 --- a/backend/src/services/external-migration/external-migration-fns.ts +++ b/backend/src/services/external-migration/external-migration-fns.ts @@ -4,7 +4,7 @@ import sjcl from "sjcl"; import tweetnacl from "tweetnacl"; import tweetnaclUtil from "tweetnacl-util"; -import { OrgMembershipRole, ProjectMembershipRole, SecretType } from "@app/db/schemas"; +import { SecretType } from "@app/db/schemas"; import { BadRequestError, NotFoundError } from "@app/lib/errors"; import { chunkArray } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; @@ -12,7 +12,6 @@ import { alphaNumericNanoId } from "@app/lib/nanoid"; import { TKmsServiceFactory } from "../kms/kms-service"; import { KmsDataKey } from "../kms/kms-types"; -import { TOrgServiceFactory } from "../org/org-service"; import { TProjectDALFactory } from "../project/project-dal"; import { TProjectServiceFactory } from "../project/project-service"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; @@ -38,7 +37,6 @@ export type TImportDataIntoInfisicalDTO = { folderDAL: Pick; projectService: Pick; - orgService: Pick; projectEnvService: Pick; secretV2BridgeService: Pick; @@ -113,7 +111,6 @@ export const importDataIntoInfisicalFn = async ({ projectService, projectEnvDAL, projectDAL, - orgService, secretDAL, kmsService, secretVersionDAL, @@ -150,26 +147,6 @@ export const importDataIntoInfisicalFn = async ({ originalToNewProjectId.set(project.id, newProject.id); } - // Invite user importing projects - const invites = await orgService.inviteUserToOrganization({ - verifyPermissions: false, - actorAuthMethod, - actorId, - actorOrgId, - actor, - inviteeEmails: [], - orgId: actorOrgId, - organizationRoleSlug: OrgMembershipRole.NoAccess, - projects: Array.from(originalToNewProjectId.values()).map((project) => ({ - id: project, - projectRoleSlug: [ProjectMembershipRole.Member] - })), - tx - }); - if (!invites) { - throw new BadRequestError({ message: `Failed to invite user to projects: [userId:${actorId}]` }); - } - // Import environments if (data.environments) { for await (const environment of data.environments) { diff --git a/backend/src/services/external-migration/external-migration-queue.ts b/backend/src/services/external-migration/external-migration-queue.ts index 879899d49..0ee51c5fa 100644 --- a/backend/src/services/external-migration/external-migration-queue.ts +++ b/backend/src/services/external-migration/external-migration-queue.ts @@ -4,7 +4,6 @@ import { logger } from "@app/lib/logger"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; import { TKmsServiceFactory } from "../kms/kms-service"; -import { TOrgServiceFactory } from "../org/org-service"; import { TProjectDALFactory } from "../project/project-dal"; import { TProjectServiceFactory } from "../project/project-service"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; @@ -34,7 +33,6 @@ export type TExternalMigrationQueueFactoryDep = { folderDAL: Pick; projectService: Pick; - orgService: Pick; projectEnvService: Pick; secretV2BridgeService: Pick; }; @@ -44,7 +42,6 @@ export type TExternalMigrationQueueFactory = ReturnType { const appCfg = getConfig(); - let permission: MongoAbility | undefined; - if (verifyPermissions) { - permission = (await permissionService.getOrgPermission(actor, actorId, orgId, actorAuthMethod, actorOrgId)) - .permission; - } + const { permission } = await permissionService.getOrgPermission(actor, actorId, orgId, actorAuthMethod, actorOrgId); const org = await orgDAL.findOrgById(orgId); @@ -523,15 +512,12 @@ export const orgServiceFactory = ({ } const projectsToInvite = invitedProjects?.length - ? await projectDAL.find( - { - orgId, - $in: { - id: invitedProjects?.map(({ id }) => id) - } - }, - { tx: trx } - ) + ? await projectDAL.find({ + orgId, + $in: { + id: invitedProjects?.map(({ id }) => id) + } + }) : []; if (projectsToInvite.length !== invitedProjects?.length) { throw new ForbiddenRequestError({ @@ -548,7 +534,7 @@ export const orgServiceFactory = ({ const mailsForOrgInvitation: { email: string; userId: string; firstName: string; lastName: string }[] = []; const mailsForProjectInvitation: { email: string[]; projectName: string }[] = []; const newProjectMemberships: TProjectMemberships[] = []; - await (trx || orgDAL).transaction(async (tx) => { + await orgDAL.transaction(async (tx) => { const users: Pick[] = []; for await (const inviteeEmail of inviteeEmails) { @@ -634,16 +620,8 @@ export const orgServiceFactory = ({ }); } - if (verifyPermissions) { - if (!permission) { - throw new ForbiddenRequestError({ - name: "InviteUser", - message: "Failed to invite user because no permission was found" - }); - } - // as its used by project invite also - ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Create, OrgPermissionSubjects.Member); - } + // as its used by project invite also + ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Create, OrgPermissionSubjects.Member); let roleId; const orgRole = isCustomOrgRole ? OrgMembershipRole.Custom : organizationRoleSlug; if (isCustomOrgRole) { @@ -684,19 +662,17 @@ export const orgServiceFactory = ({ // if there exist no project membership we set is as given by the request for await (const project of projectsToInvite) { const projectId = project.id; - if (verifyPermissions) { - const { permission: projectPermission } = await permissionService.getProjectPermission( - actor, - actorId, - projectId, - actorAuthMethod, - actorOrgId - ); - ForbiddenError.from(projectPermission).throwUnlessCan( - ProjectPermissionActions.Create, - ProjectPermissionSub.Member - ); - } + const { permission: projectPermission } = await permissionService.getProjectPermission( + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + ); + ForbiddenError.from(projectPermission).throwUnlessCan( + ProjectPermissionActions.Create, + ProjectPermissionSub.Member + ); const existingMembers = await projectMembershipDAL.find( { projectId: project.id, diff --git a/backend/src/services/org/org-types.ts b/backend/src/services/org/org-types.ts index 5bda46e5f..d62a2c25b 100644 --- a/backend/src/services/org/org-types.ts +++ b/backend/src/services/org/org-types.ts @@ -1,5 +1,3 @@ -import { Knex } from "knex"; - import { TOrgPermission } from "@app/lib/types"; import { ActorAuthMethod, ActorType } from "../auth/auth-type"; @@ -30,8 +28,6 @@ export type TDeleteOrgMembershipDTO = { export type TInviteUserToOrgDTO = { inviteeEmails: string[]; organizationRoleSlug: string; - tx?: Knex; - verifyPermissions?: boolean; projects?: { id: string; projectRoleSlug?: string[]; diff --git a/backend/src/services/project-env/project-env-service.ts b/backend/src/services/project-env/project-env-service.ts index ecbdd97b7..67bdd867a 100644 --- a/backend/src/services/project-env/project-env-service.ts +++ b/backend/src/services/project-env/project-env-service.ts @@ -39,8 +39,7 @@ export const projectEnvServiceFactory = ({ actorAuthMethod, position, name, - slug, - tx: trx + slug }: TCreateEnvDTO) => { const { permission } = await permissionService.getProjectPermission( actor, @@ -84,7 +83,7 @@ export const projectEnvServiceFactory = ({ }); } - const env = await (trx || projectEnvDAL).transaction(async (tx) => { + const env = await projectEnvDAL.transaction(async (tx) => { if (position !== undefined) { // Check if there's an environment at the specified position const existingEnvWithPosition = await projectEnvDAL.findOne({ projectId, position }, tx); diff --git a/backend/src/services/project-env/project-env-types.ts b/backend/src/services/project-env/project-env-types.ts index 427086cb7..a87c76d4d 100644 --- a/backend/src/services/project-env/project-env-types.ts +++ b/backend/src/services/project-env/project-env-types.ts @@ -1,12 +1,9 @@ -import { Knex } from "knex"; - import { TProjectPermission } from "@app/lib/types"; export type TCreateEnvDTO = { name: string; slug: string; position?: number; - tx?: Knex; } & TProjectPermission; export type TUpdateEnvDTO = { diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts index a520c5439..6e8d95e1c 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts @@ -911,8 +911,7 @@ export const secretV2BridgeServiceFactory = ({ actorOrgId, environment, projectId, - secrets: inputSecrets, - tx: trx + secrets: inputSecrets }: TCreateManySecretDTO) => { const { permission } = await permissionService.getProjectPermission( actor, @@ -952,7 +951,7 @@ export const secretV2BridgeServiceFactory = ({ const { encryptor: secretManagerEncryptor, decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ type: KmsDataKey.SecretManager, projectId }); - const newSecrets = await (trx || secretDAL).transaction(async (tx) => + const newSecrets = await secretDAL.transaction(async (tx) => fnSecretBulkInsert({ inputSecrets: inputSecrets.map((el) => { const references = getAllNestedSecretReferences(el.secretValue); diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts index 02bf5da26..a76c57561 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts @@ -81,7 +81,6 @@ export type TCreateManySecretDTO = Omit & { secretPath: string; projectId: string; environment: string; - tx?: Knex; secrets: { secretKey: string; secretValue: string;