diff --git a/backend/src/services/external-migration/external-migration-fns.ts b/backend/src/services/external-migration/external-migration-fns.ts index 934c1c8e1..70498e16e 100644 --- a/backend/src/services/external-migration/external-migration-fns.ts +++ b/backend/src/services/external-migration/external-migration-fns.ts @@ -4,7 +4,7 @@ import sjcl from "sjcl"; import tweetnacl from "tweetnacl"; import tweetnaclUtil from "tweetnacl-util"; -import { SecretType } from "@app/db/schemas"; +import { SecretType, TSecretFolders } from "@app/db/schemas"; import { BadRequestError, NotFoundError } from "@app/lib/errors"; import { chunkArray } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; @@ -35,7 +35,7 @@ export type TImportDataIntoInfisicalDTO = { secretTagDAL: Pick; secretVersionTagDAL: Pick; - folderDAL: Pick; + folderDAL: Pick; projectService: Pick; projectEnvService: Pick; secretV2BridgeService: Pick; @@ -67,6 +67,7 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise a.id === env.envParentId)?.id; + + // If we find the app from the envParentId, we know this is a root-level environment. + if (appId) { + infisicalImportData.environments.push({ + id: env.id, + name: envTemplates.get(env.environmentRoleId)!, + projectId: appId + }); + } + } + + const findRootInheritedSecret = ( + secret: { val?: string; inheritsEnvironmentId?: string }, + secretName: string, + envs: typeof parsedJson.envs + ): { val?: string } => { + if (!secret) { + return { + val: "" + }; + } + + // If we have a direct value, return it + if (secret.val !== undefined) { + return secret; + } + + // If there's no inheritance, return the secret as is + if (!secret.inheritsEnvironmentId) { + return secret; + } + + const inheritedEnv = envs[secret.inheritsEnvironmentId]; + if (!inheritedEnv) return secret; + return findRootInheritedSecret(inheritedEnv.variables[secretName], secretName, envs); + }; + + const processBranches = () => { + for (const subEnv of parsedJson.subEnvironments) { + const app = parsedJson.apps.find((a) => a.id === subEnv.envParentId); + const block = parsedJson.blocks.find((b) => b.id === subEnv.envParentId); + + if (app) { + // Handle regular app branches + const branchEnvironment = infisicalImportData.environments.find((e) => e.id === subEnv.parentEnvironmentId); + + infisicalImportData.folders.push({ + name: subEnv.subName, + parentFolderId: subEnv.parentEnvironmentId, + environmentId: branchEnvironment!.id, + id: subEnv.id + }); + } + + if (block) { + // Handle block branches + // 1. Find all apps that use this block + const appsUsingBlock = parsedJson.appBlocks.filter((ab) => ab.blockId === block.id); + + for (const { appId, orderIndex } of appsUsingBlock) { + // 2. Find the matching environment in the app based on the environment role + const blockBaseEnv = parsedJson.baseEnvironments.find((be) => be.id === subEnv.parentEnvironmentId); + + // eslint-disable-next-line no-continue + if (!blockBaseEnv) continue; + + const matchingAppEnv = parsedJson.baseEnvironments.find( + (be) => be.envParentId === appId && be.environmentRoleId === blockBaseEnv.environmentRoleId + ); + + // eslint-disable-next-line no-continue + if (!matchingAppEnv) continue; + + // 3. Create a folder in the matching app environment + infisicalImportData.folders.push({ + name: subEnv.subName, + parentFolderId: matchingAppEnv.id, + environmentId: matchingAppEnv.id, + id: `${subEnv.id}-${appId}` // Create unique ID for each app's copy of the branch + }); + + // 4. Process secrets in the block branch for this app + const branchSecrets = parsedJson.envs[subEnv.id]?.variables || {}; + for (const [secretName, secretData] of Object.entries(branchSecrets)) { + if (secretData.inheritsEnvironmentId) { + const resolvedSecret = findRootInheritedSecret(secretData, secretName, parsedJson.envs); + + // If the secret already exists in the environment, we need to check the orderIndex of the appBlock. The appBlock with the highest orderIndex should take precedence. + const preExistingSecretIndex = infisicalImportData.secrets.findIndex( + (s) => s.name === secretName && s.environmentId === matchingAppEnv.id + ); + + if (preExistingSecretIndex !== -1) { + const preExistingSecret = infisicalImportData.secrets[preExistingSecretIndex]; + + if ( + preExistingSecret.appBlockOrderIndex !== undefined && + orderIndex > preExistingSecret.appBlockOrderIndex + ) { + // if the existing secret has a lower orderIndex, we should replace it + infisicalImportData.secrets[preExistingSecretIndex] = { + ...preExistingSecret, + value: resolvedSecret.val || "", + appBlockOrderIndex: orderIndex + }; + } + + // eslint-disable-next-line no-continue + continue; + } + + infisicalImportData.secrets.push({ + id: randomUUID(), + name: secretName, + environmentId: matchingAppEnv.id, + value: resolvedSecret.val || "", + folderId: `${subEnv.id}-${appId}`, + appBlockOrderIndex: orderIndex + }); + } else { + // If the secret already exists in the environment, we need to check the orderIndex of the appBlock. The appBlock with the highest orderIndex should take precedence. + const preExistingSecretIndex = infisicalImportData.secrets.findIndex( + (s) => s.name === secretName && s.environmentId === matchingAppEnv.id + ); + + if (preExistingSecretIndex !== -1) { + const preExistingSecret = infisicalImportData.secrets[preExistingSecretIndex]; + + if ( + preExistingSecret.appBlockOrderIndex !== undefined && + orderIndex > preExistingSecret.appBlockOrderIndex + ) { + // if the existing secret has a lower orderIndex, we should replace it + infisicalImportData.secrets[preExistingSecretIndex] = { + ...preExistingSecret, + value: secretData.val || "", + appBlockOrderIndex: orderIndex + }; + } + + // eslint-disable-next-line no-continue + continue; + } + + infisicalImportData.secrets.push({ + id: randomUUID(), + name: secretName, + environmentId: matchingAppEnv.id, + value: secretData.val || "", + folderId: `${subEnv.id}-${appId}`, + appBlockOrderIndex: orderIndex + }); + } + } + } + } + } + }; + + const processBlocksForApp = (appIds: string[]) => { + for (const appId of appIds) { + const blocksInApp = parsedJson.appBlocks.filter((ab) => ab.appId === appId); + logger.info( + { + blocksInApp + }, + "[processBlocksForApp]: Processing blocks for app" + ); + + for (const appBlock of blocksInApp) { + // 1. find all base environments for this block + const blockBaseEnvironments = parsedJson.baseEnvironments.filter((env) => env.envParentId === appBlock.blockId); + logger.info( + { + blockBaseEnvironments + }, + "[processBlocksForApp]: Processing block base environments" + ); + + for (const blockBaseEnvironment of blockBaseEnvironments) { + // 2. find the corresponding environment that is not from the block + const matchingEnv = parsedJson.baseEnvironments.find( + (be) => + be.environmentRoleId === blockBaseEnvironment.environmentRoleId && be.envParentId !== appBlock.blockId + ); + + if (!matchingEnv) { + throw new Error(`Could not find environment for block ${appBlock.blockId}`); + } + + // 3. find all the secrets for this environment block + const blockSecrets = parsedJson.envs[blockBaseEnvironment.id].variables; + + logger.info( + { + blockSecretsLength: Object.keys(blockSecrets).length + }, + "[processBlocksForApp]: Processing block secrets" + ); + + // 4. process each secret + for (const secret of Object.keys(blockSecrets)) { + const selectedSecret = blockSecrets[secret]; + + if (selectedSecret.inheritsEnvironmentId) { + const resolvedSecret = findRootInheritedSecret(selectedSecret, secret, parsedJson.envs); + + // If the secret already exists in the environment, we need to check the orderIndex of the appBlock. The appBlock with the highest orderIndex should take precedence. + const preExistingSecretIndex = infisicalImportData.secrets.findIndex( + (s) => s.name === secret && s.environmentId === matchingEnv.id + ); + + if (preExistingSecretIndex !== -1) { + const preExistingSecret = infisicalImportData.secrets[preExistingSecretIndex]; + + if ( + preExistingSecret.appBlockOrderIndex !== undefined && + appBlock.orderIndex > preExistingSecret.appBlockOrderIndex + ) { + // if the existing secret has a lower orderIndex, we should replace it + infisicalImportData.secrets[preExistingSecretIndex] = { + ...preExistingSecret, + value: selectedSecret.val || "", + appBlockOrderIndex: appBlock.orderIndex + }; + } + + // eslint-disable-next-line no-continue + continue; + } + + infisicalImportData.secrets.push({ + id: randomUUID(), + name: secret, + environmentId: matchingEnv.id, + value: resolvedSecret.val || "", + appBlockOrderIndex: appBlock.orderIndex + }); + } else { + // If the secret already exists in the environment, we need to check the orderIndex of the appBlock. The appBlock with the highest orderIndex should take precedence. + const preExistingSecretIndex = infisicalImportData.secrets.findIndex( + (s) => s.name === secret && s.environmentId === matchingEnv.id + ); + + if (preExistingSecretIndex !== -1) { + const preExistingSecret = infisicalImportData.secrets[preExistingSecretIndex]; + + if ( + preExistingSecret.appBlockOrderIndex !== undefined && + appBlock.orderIndex > preExistingSecret.appBlockOrderIndex + ) { + // if the existing secret has a lower orderIndex, we should replace it + infisicalImportData.secrets[preExistingSecretIndex] = { + ...preExistingSecret, + value: selectedSecret.val || "", + appBlockOrderIndex: appBlock.orderIndex + }; + } + + // eslint-disable-next-line no-continue + continue; + } + + infisicalImportData.secrets.push({ + id: randomUUID(), + name: secret, + environmentId: matchingEnv.id, + value: selectedSecret.val || "", + appBlockOrderIndex: appBlock.orderIndex + }); + } + } + } + } + } + }; + + processBranches(); + processBlocksForApp(infisicalImportData.projects.map((app) => app.id)); + for (const env of Object.keys(parsedJson.envs)) { - if (!env.includes("|")) { - const envData = parsedJson.envs[env]; - for (const secret of Object.keys(envData.variables)) { + // Skip user-specific environments + // eslint-disable-next-line no-continue + if (env.includes("|")) continue; + + const envData = parsedJson.envs[env]; + const baseEnv = parsedJson.baseEnvironments.find((be) => be.id === env); + const subEnv = parsedJson.subEnvironments.find((se) => se.id === env); + + // Skip if we can't find either a base environment or sub-environment + if (!baseEnv && !subEnv) { + logger.info( + { + envId: env + }, + "[parseEnvKeyDataFn]: Could not find base or sub environment for env, skipping" + ); + // eslint-disable-next-line no-continue + continue; + } + + // If this is a base environment of a block, skip it (handled by processBlocksForApp) + if (baseEnv) { + const isBlock = parsedJson.appBlocks.some((block) => block.blockId === baseEnv.envParentId); + if (isBlock) { + logger.info( + { + envId: env, + baseEnv + }, + "[parseEnvKeyDataFn]: Skipping block environment (handled separately)" + ); + // eslint-disable-next-line no-continue + continue; + } + } + + // Process each secret in this environment or branch + for (const [secretName, secretData] of Object.entries(envData.variables)) { + const environmentId = subEnv ? subEnv.parentEnvironmentId : env; + const indexOfExistingSecret = infisicalImportData.secrets.findIndex( + (s) => s.name === secretName && s.environmentId === environmentId + ); + + if (secretData.inheritsEnvironmentId) { + const resolvedSecret = findRootInheritedSecret(secretData, secretName, parsedJson.envs); + + // Check if there's already a secret with this name in the environment, if there is, we should override it. Because if there's already one, we know its coming from a block. + // Variables from the normal environment should take precedence over variables from the block. + + if (indexOfExistingSecret !== -1) { + // if a existing secret is found, we should replace it directly + const newSecret: (typeof infisicalImportData.secrets)[number] = { + ...infisicalImportData.secrets[indexOfExistingSecret], + value: resolvedSecret.val || "" + }; + + infisicalImportData.secrets[indexOfExistingSecret] = newSecret; + + // eslint-disable-next-line no-continue + continue; + } + infisicalImportData.secrets.push({ id: randomUUID(), - name: secret, - environmentId: env, - value: envData.variables[secret].val + name: secretName, + environmentId: subEnv ? subEnv.parentEnvironmentId : env, + value: resolvedSecret.val || "", + ...(subEnv && { folderId: subEnv.id }) // Add folderId if this is a branch secret + }); + } else { + // Check if there's already a secret with this name in the environment, if there is, we should override it. Because if there's already one, we know its coming from a block. + // Variables from the normal environment should take precedence over variables from the block. + + if (indexOfExistingSecret !== -1) { + // if a existing secret is found, we should replace it directly + const newSecret: (typeof infisicalImportData.secrets)[number] = { + ...infisicalImportData.secrets[indexOfExistingSecret], + value: secretData.val || "" + }; + + infisicalImportData.secrets[indexOfExistingSecret] = newSecret; + + // eslint-disable-next-line no-continue + continue; + } + + infisicalImportData.secrets.push({ + id: randomUUID(), + name: secretName, + environmentId: subEnv ? subEnv.parentEnvironmentId : env, + value: secretData.val || "", + ...(subEnv && { folderId: subEnv.id }) // Add folderId if this is a branch secret }); } } @@ -125,7 +488,17 @@ export const importDataIntoInfisicalFn = async ({ } const originalToNewProjectId = new Map(); - const originalToNewEnvironmentId = new Map(); + const originalToNewEnvironmentId = new Map< + string, + { envId: string; envSlug: string; rootFolderId: string; projectId: string } + >(); + const originalToNewFolderId = new Map< + string, + { + folderId: string; + projectId: string; + } + >(); const projectsNotImported: string[] = []; await projectDAL.transaction(async (tx) => { @@ -170,65 +543,161 @@ export const importDataIntoInfisicalFn = async ({ const lastPos = await projectEnvDAL.findLastEnvPosition(projectId, tx); const doc = await projectEnvDAL.create({ slug, name: environment.name, projectId, position: lastPos + 1 }, tx); - await folderDAL.create({ name: "root", parentId: null, envId: doc.id, version: 1 }, tx); + const folder = await folderDAL.create({ name: "root", parentId: null, envId: doc.id, version: 1 }, tx); - originalToNewEnvironmentId.set(environment.id, doc.slug); + originalToNewEnvironmentId.set(environment.id, { + envSlug: doc.slug, + envId: doc.id, + rootFolderId: folder.id, + projectId + }); } } + if (data.folders) { + for await (const folder of data.folders) { + const parentEnv = originalToNewEnvironmentId.get(folder.parentFolderId as string); + + if (!parentEnv) { + // eslint-disable-next-line no-continue + continue; + } + + const newFolder = await folderDAL.create( + { + name: folder.name, + envId: parentEnv.envId, + parentId: parentEnv.rootFolderId + }, + tx + ); + + originalToNewFolderId.set(folder.id, { + folderId: newFolder.id, + projectId: parentEnv.projectId + }); + } + } + + // Useful for debugging: + // console.log("data.secrets", data.secrets); + // console.log("data.folders", data.folders); + // console.log("data.environment", data.environments); + if (data.secrets && data.secrets.length > 0) { const mappedToEnvironmentId = new Map< string, { secretKey: string; secretValue: string; + folderId?: string; }[] >(); for (const secret of data.secrets) { - if (!originalToNewEnvironmentId.get(secret.environmentId)) { + const targetId = secret.folderId || secret.environmentId; + + // Skip if we can't find either an environment or folder mapping for this secret + if (!originalToNewEnvironmentId.get(secret.environmentId) && !originalToNewFolderId.get(targetId)) { // eslint-disable-next-line no-continue continue; } - if (!mappedToEnvironmentId.has(secret.environmentId)) { - mappedToEnvironmentId.set(secret.environmentId, []); + if (!mappedToEnvironmentId.has(targetId)) { + mappedToEnvironmentId.set(targetId, []); } - mappedToEnvironmentId.get(secret.environmentId)!.push({ + mappedToEnvironmentId.get(targetId)!.push({ secretKey: secret.name, - secretValue: secret.value || "" + secretValue: secret.value || "", + folderId: secret.folderId }); } // for each of the mappedEnvironmentId - for await (const [envId, secrets] of mappedToEnvironmentId) { - const environment = data.environments.find((env) => env.id === envId); - const projectId = originalToNewProjectId.get(environment?.projectId as string)!; + for await (const [targetId, secrets] of mappedToEnvironmentId) { + logger.info("[importDataIntoInfisicalFn]: Processing secrets for targetId", targetId); - if (!projectId) { - throw new BadRequestError({ message: `Failed to import secret, project not found` }); + let selectedFolder: TSecretFolders | undefined; + let selectedProjectId: string | undefined; + + // Case 1: Secret belongs to a folder / branch / branch of a block + const foundFolder = originalToNewFolderId.get(targetId); + if (foundFolder) { + logger.info("[importDataIntoInfisicalFn]: Processing secrets for folder"); + selectedFolder = await folderDAL.findById(foundFolder.folderId, tx); + selectedProjectId = foundFolder.projectId; + } else { + logger.info("[importDataIntoInfisicalFn]: Processing secrets for normal environment"); + const environment = data.environments.find((env) => env.id === targetId); + if (!environment) { + logger.info( + { + targetId + }, + "[importDataIntoInfisicalFn]: Could not find environment for secret" + ); + // eslint-disable-next-line no-continue + continue; + } + + const projectId = originalToNewProjectId.get(environment.projectId)!; + + if (!projectId) { + throw new BadRequestError({ message: `Failed to import secret, project not found` }); + } + + const env = originalToNewEnvironmentId.get(targetId); + if (!env) { + logger.info( + { + targetId + }, + "[importDataIntoInfisicalFn]: Could not find environment for secret" + ); + + // eslint-disable-next-line no-continue + continue; + } + + const folder = await folderDAL.findBySecretPath(projectId, env.envSlug, "/", tx); + + if (!folder) { + throw new NotFoundError({ + message: `Folder not found for the given environment slug (${env.envSlug}) & secret path (/)`, + name: "Create secret" + }); + } + + selectedFolder = folder; + selectedProjectId = projectId; + } + + if (!selectedFolder) { + throw new NotFoundError({ + message: `Folder not found for the given environment slug & secret path`, + name: "CreateSecret" + }); + } + + if (!selectedProjectId) { + throw new NotFoundError({ + message: `Project not found for the given environment slug & secret path`, + name: "CreateSecret" + }); } const { encryptor: secretManagerEncrypt } = await kmsService.createCipherPairWithDataKey( { type: KmsDataKey.SecretManager, - projectId + projectId: selectedProjectId }, tx ); - const envSlug = originalToNewEnvironmentId.get(envId)!; - const folder = await folderDAL.findBySecretPath(projectId, envSlug, "/", tx); - if (!folder) - throw new NotFoundError({ - message: `Folder not found for the given environment slug (${envSlug}) & secret path (/)`, - name: "Create secret" - }); - const secretBatches = chunkArray(secrets, 2500); for await (const secretBatch of secretBatches) { const secretsByKeys = await secretDAL.findBySecretKeys( - folder.id, + selectedFolder.id, secretBatch.map((el) => ({ key: el.secretKey, type: SecretType.Shared @@ -254,7 +723,7 @@ export const importDataIntoInfisicalFn = async ({ type: SecretType.Shared }; }), - folderId: folder.id, + folderId: selectedFolder.id, secretDAL, secretVersionDAL, secretTagDAL, diff --git a/backend/src/services/external-migration/external-migration-queue.ts b/backend/src/services/external-migration/external-migration-queue.ts index 76c7ceb1e..3cbe8b616 100644 --- a/backend/src/services/external-migration/external-migration-queue.ts +++ b/backend/src/services/external-migration/external-migration-queue.ts @@ -31,7 +31,7 @@ export type TExternalMigrationQueueFactoryDep = { secretTagDAL: Pick; secretVersionTagDAL: Pick; - folderDAL: Pick; + folderDAL: Pick; projectService: Pick; projectEnvService: Pick; secretV2BridgeService: Pick; diff --git a/backend/src/services/external-migration/external-migration-types.ts b/backend/src/services/external-migration/external-migration-types.ts index 53c954bf9..32c70a688 100644 --- a/backend/src/services/external-migration/external-migration-types.ts +++ b/backend/src/services/external-migration/external-migration-types.ts @@ -2,8 +2,16 @@ import { ActorAuthMethod, ActorType } from "../auth/auth-type"; export type InfisicalImportData = { projects: Array<{ name: string; id: string }>; - environments: Array<{ name: string; id: string; projectId: string }>; - secrets: Array<{ name: string; id: string; environmentId: string; value: string }>; + environments: Array<{ name: string; id: string; projectId: string; envParentId?: string }>; + folders: Array<{ id: string; name: string; environmentId: string; parentFolderId?: string }>; + secrets: Array<{ + id: string; + name: string; + environmentId: string; + value: string; + folderId?: string; + appBlockOrderIndex?: number; // Not used for infisical import, only used for building the import structure to determine which block(s) take precedence. + }>; }; export type TImportEnvKeyDataCreate = { @@ -28,62 +36,62 @@ export type TEnvKeyExportJSON = { org: { id: string; name: string; - settings: { - auth: { - inviteExpirationMs: number; - deviceGrantExpirationMs: number; - tokenExpirationMs: number; - }; - crypto: { - requiresPassphrase: boolean; - requiresLockout: boolean; - }; - envs: { - autoCaps: boolean; - autoCommitLocals: boolean; - }; - }; }; + + // Apps are projects apps: { id: string; name: string; - settings: Record; }[]; - defaultOrgRoles: { + // Blocks are basically global projects that can be imported in other projects + blocks: { id: string; - defaultName: string; + name: string; }[]; - defaultAppRoles: { - id: string; - defaultName: string; + + appBlocks: { + appId: string; + blockId: string; + orderIndex: number; }[]; + defaultEnvironmentRoles: { id: string; defaultName: string; - settings: { - autoCommit: boolean; - }; }[]; + + nonDefaultEnvironmentRoles: { + id: string; + name: string; + }[]; + baseEnvironments: { id: string; envParentId: string; environmentRoleId: string; - settings: Record; }[]; - orgUsers: { + + // Branches for both blocks and apps + subEnvironments: { id: string; - firstName: string; - lastName: string; - email: string; - provider: string; - orgRoleId: string; - uid: string; + envParentId: string; + environmentRoleId: string; + parentEnvironmentId: string; + subName: string; }[]; + envs: Record< string, { - variables: Record; - inherits: Record; + variables: Record< + string, + { + val?: string; + inheritsEnvironmentId?: string; + } + >; + + inherits: Record; } >; };