From f4ba441ec39fccb5a6d7ea008583db9694462817 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Mon, 21 Oct 2024 20:39:08 +0400 Subject: [PATCH 1/6] feat: envkey data migration refactor --- .../external-migration-fns.ts | 211 +++++++++++++++--- .../external-migration-types.ts | 72 +++--- 2 files changed, 218 insertions(+), 65 deletions(-) diff --git a/backend/src/services/external-migration/external-migration-fns.ts b/backend/src/services/external-migration/external-migration-fns.ts index 6d996022a..44e901ab3 100644 --- a/backend/src/services/external-migration/external-migration-fns.ts +++ b/backend/src/services/external-migration/external-migration-fns.ts @@ -4,7 +4,7 @@ import sjcl from "sjcl"; import tweetnacl from "tweetnacl"; import tweetnaclUtil from "tweetnacl-util"; -import { SecretType } from "@app/db/schemas"; +import { SecretType, TSecretFolders } from "@app/db/schemas"; import { BadRequestError, NotFoundError } from "@app/lib/errors"; import { chunkArray } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; @@ -35,7 +35,7 @@ export type TImportDataIntoInfisicalDTO = { secretTagDAL: Pick; secretVersionTagDAL: Pick; - folderDAL: Pick; + folderDAL: Pick; projectService: Pick; projectEnvService: Pick; secretV2BridgeService: Pick; @@ -67,6 +67,7 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise a.id === env.envParentId); + + // If we find the app from the envParentId, we know this is a root-level environment. + if (app) { + infisicalImportData.environments.push({ + id: env.id, + name: envTemplates.get(env.environmentRoleId)!, + projectId: app.id + }); + } else { + // const parentBlock = parsedJson.blocks.find((b) => b.id === env.envParentId); + // // If this is found, then we know this is a sub-environment. The `parentEnvironment` is the sub environment. + // const subEnvironment = parsedJson.subEnvironments.find( + // (s) => s.parentEnvironmentId === env.id && parsedJson.apps.find((a) => a.id === s.envParentId) + // ); + // if (subEnvironment) { + // infisicalImportData.folders.push({ + // name: subEnvironment.subName, + // parentFolderId: subEnvironment.parentEnvironmentId, + // environmentId: env.id, + // id: subEnvironment.id + // }); + // } else if (parentBlock) { + // // TODO(daniel): Find a way to get the secrets from the parent block, so we can later insert it + // } + } + } + + for (const subEnv of parsedJson.subEnvironments) { + // this will only find the app if the subEnv is a branch, not a block. + const app = parsedJson.apps.find((a) => a.id === subEnv.envParentId); + + const parentEnvironment = infisicalImportData.environments.find((e) => e.id === subEnv.parentEnvironmentId); + + if (app) { + infisicalImportData.folders.push({ + name: subEnv.subName, + parentFolderId: subEnv.parentEnvironmentId, + environmentId: parentEnvironment!.id, + id: subEnv.id + }); + } + } + + // secrets with/without inheritance for (const env of Object.keys(parsedJson.envs)) { if (!env.includes("|")) { const envData = parsedJson.envs[env]; for (const secret of Object.keys(envData.variables)) { + const selectedSecret = envData.variables[secret]; + + if (selectedSecret.inheritsEnvironmentId) { + const findRootInheritedSecret = (currentSecret: { val?: string; inheritsEnvironmentId?: string }) => { + if (currentSecret.inheritsEnvironmentId) { + const inheritedSecret = parsedJson.envs[currentSecret.inheritsEnvironmentId].variables[secret]; + if (inheritedSecret) { + // eslint-disable-next-line no-param-reassign + currentSecret.val = inheritedSecret.val; + } + + findRootInheritedSecret(inheritedSecret); + } + return currentSecret; + }; + + const sec = findRootInheritedSecret(selectedSecret); + + infisicalImportData.secrets.push({ + id: randomUUID(), + name: secret, + environmentId: env, + value: sec.val || "???" + }); + + // eslint-disable-next-line no-continue + continue; + } + infisicalImportData.secrets.push({ id: randomUUID(), name: secret, environmentId: env, - value: envData.variables[secret].val + value: selectedSecret.val || "???_???" }); } } @@ -125,7 +197,17 @@ export const importDataIntoInfisicalFn = async ({ } const originalToNewProjectId = new Map(); - const originalToNewEnvironmentId = new Map(); + const originalToNewEnvironmentId = new Map< + string, + { envId: string; envSlug: string; rootFolderId: string; projectId: string } + >(); + const originalToNewFolderId = new Map< + string, + { + folderId: string; + projectId: string; + } + >(); const projectsNotImported: string[] = []; await projectDAL.transaction(async (tx) => { @@ -170,12 +252,46 @@ export const importDataIntoInfisicalFn = async ({ const lastPos = await projectEnvDAL.findLastEnvPosition(projectId, tx); const doc = await projectEnvDAL.create({ slug, name: environment.name, projectId, position: lastPos + 1 }, tx); - await folderDAL.create({ name: "root", parentId: null, envId: doc.id, version: 1 }, tx); + const folder = await folderDAL.create({ name: "root", parentId: null, envId: doc.id, version: 1 }, tx); - originalToNewEnvironmentId.set(environment.id, doc.slug); + originalToNewEnvironmentId.set(environment.id, { + envSlug: doc.slug, + envId: doc.id, + rootFolderId: folder.id, + projectId + }); } } + if (data.folders) { + for await (const folder of data.folders) { + const parentEnv = originalToNewEnvironmentId.get(folder.parentFolderId as string); + + if (!parentEnv) { + // eslint-disable-next-line no-continue + continue; + } + + const newFolder = await folderDAL.create( + { + name: folder.name, + envId: parentEnv.envId, + parentId: parentEnv.rootFolderId + }, + tx + ); + + originalToNewFolderId.set(folder.id, { + folderId: newFolder.id, + projectId: parentEnv.projectId + }); + } + } + + console.log("data.folders", data.folders); + + console.log("data.secrets", data.secrets); + if (data.secrets && data.secrets.length > 0) { const mappedToEnvironmentId = new Map< string, @@ -186,7 +302,7 @@ export const importDataIntoInfisicalFn = async ({ >(); for (const secret of data.secrets) { - if (!originalToNewEnvironmentId.get(secret.environmentId)) { + if (!originalToNewEnvironmentId.get(secret.environmentId) && !originalToNewFolderId.get(secret.environmentId)) { // eslint-disable-next-line no-continue continue; } @@ -202,33 +318,68 @@ export const importDataIntoInfisicalFn = async ({ // for each of the mappedEnvironmentId for await (const [envId, secrets] of mappedToEnvironmentId) { - const environment = data.environments.find((env) => env.id === envId); - const projectId = originalToNewProjectId.get(environment?.projectId as string)!; + console.log(`envId ${envId} secrets:`, secrets); - if (!projectId) { - throw new BadRequestError({ message: `Failed to import secret, project not found` }); + const environment = data.environments.find((env) => env.id === envId); + const foundFolder = originalToNewFolderId.get(envId); + + console.log(`FOUND FOLDER BY ENV.ID ${envId}`, foundFolder); + + let selectedFolder: TSecretFolders | undefined; + let selectedProjectId: string | undefined; + if (foundFolder) { + console.log("RUNNING FOLDER HANDLER"); + + selectedFolder = await folderDAL.findById(foundFolder.folderId, tx); + selectedProjectId = foundFolder.projectId; + } else if (environment) { + console.log("RUNNING ENVIRONMENT HANDLER"); + const projectId = originalToNewProjectId.get(environment.projectId)!; + + if (!projectId) { + throw new BadRequestError({ message: `Failed to import secret, project not found` }); + } + + const env = originalToNewEnvironmentId.get(envId)!; + const folder = await folderDAL.findBySecretPath(projectId, env.envSlug, "/", tx); + + if (!folder) { + throw new NotFoundError({ + message: `Folder not found for the given environment slug (${env.envSlug}) & secret path (/)`, + name: "Create secret" + }); + } + + selectedFolder = folder; + selectedProjectId = projectId; + } + + if (!selectedFolder) { + throw new NotFoundError({ + message: `Folder not found for the given environment slug & secret path`, + name: "CreateSecret" + }); + } + + if (!selectedProjectId) { + throw new NotFoundError({ + message: `Project not found for the given environment slug & secret path`, + name: "CreateSecret" + }); } const { encryptor: secretManagerEncrypt } = await kmsService.createCipherPairWithDataKey( { type: KmsDataKey.SecretManager, - projectId + projectId: selectedProjectId }, tx ); - const envSlug = originalToNewEnvironmentId.get(envId)!; - const folder = await folderDAL.findBySecretPath(projectId, envSlug, "/", tx); - if (!folder) - throw new NotFoundError({ - message: `Folder not found for the given environment slug (${envSlug}) & secret path (/)`, - name: "Create secret" - }); - const secretBatches = chunkArray(secrets, 2500); for await (const secretBatch of secretBatches) { const secretsByKeys = await secretDAL.findBySecretKeys( - folder.id, + selectedFolder.id, secretBatch.map((el) => ({ key: el.secretKey, type: SecretType.Shared @@ -254,7 +405,7 @@ export const importDataIntoInfisicalFn = async ({ type: SecretType.Shared }; }), - folderId: folder.id, + folderId: selectedFolder.id, secretDAL, secretVersionDAL, secretTagDAL, diff --git a/backend/src/services/external-migration/external-migration-types.ts b/backend/src/services/external-migration/external-migration-types.ts index 53c954bf9..cdf1ef6ac 100644 --- a/backend/src/services/external-migration/external-migration-types.ts +++ b/backend/src/services/external-migration/external-migration-types.ts @@ -3,7 +3,8 @@ import { ActorAuthMethod, ActorType } from "../auth/auth-type"; export type InfisicalImportData = { projects: Array<{ name: string; id: string }>; environments: Array<{ name: string; id: string; projectId: string }>; - secrets: Array<{ name: string; id: string; environmentId: string; value: string }>; + folders: Array<{ id: string; name: string; environmentId: string; parentFolderId?: string }>; + secrets: Array<{ id: string; name: string; environmentId: string; value: string; folderId?: string }>; }; export type TImportEnvKeyDataCreate = { @@ -28,62 +29,63 @@ export type TEnvKeyExportJSON = { org: { id: string; name: string; - settings: { - auth: { - inviteExpirationMs: number; - deviceGrantExpirationMs: number; - tokenExpirationMs: number; - }; - crypto: { - requiresPassphrase: boolean; - requiresLockout: boolean; - }; - envs: { - autoCaps: boolean; - autoCommitLocals: boolean; - }; - }; + // settings, which we dont care about }; + + // Apps are projects apps: { id: string; name: string; - settings: Record; }[]; - defaultOrgRoles: { + // Blocks are basically global projects that can be imported in other projects + blocks: { id: string; - defaultName: string; + name: string; }[]; - defaultAppRoles: { - id: string; - defaultName: string; + + appBlocks: { + appId: string; + blockId: string; + orderIndex: number; }[]; + defaultEnvironmentRoles: { id: string; defaultName: string; - settings: { - autoCommit: boolean; - }; }[]; + + nonDefaultEnvironmentRoles: { + id: string; + name: string; + }[]; + baseEnvironments: { id: string; envParentId: string; environmentRoleId: string; - settings: Record; }[]; - orgUsers: { + + // Branches for both blocks and apps + subEnvironments: { id: string; - firstName: string; - lastName: string; - email: string; - provider: string; - orgRoleId: string; - uid: string; + envParentId: string; + environmentRoleId: string; + parentEnvironmentId: string; + subName: string; }[]; + envs: Record< string, { - variables: Record; - inherits: Record; + variables: Record< + string, + { + val?: string; + inheritsEnvironmentId?: string; + } + >; + + inherits: Record; } >; }; From b6955d0e9be7b20e2f571e5cbd3d5c5869921feb Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Mon, 21 Oct 2024 20:39:46 +0400 Subject: [PATCH 2/6] Update external-migration-queue.ts --- .../src/services/external-migration/external-migration-queue.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/services/external-migration/external-migration-queue.ts b/backend/src/services/external-migration/external-migration-queue.ts index 76c7ceb1e..3cbe8b616 100644 --- a/backend/src/services/external-migration/external-migration-queue.ts +++ b/backend/src/services/external-migration/external-migration-queue.ts @@ -31,7 +31,7 @@ export type TExternalMigrationQueueFactoryDep = { secretTagDAL: Pick; secretVersionTagDAL: Pick; - folderDAL: Pick; + folderDAL: Pick; projectService: Pick; projectEnvService: Pick; secretV2BridgeService: Pick; From c183ef2b4f2a1d25a72c30a4aec1a9e4a77e19b1 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Tue, 22 Oct 2024 22:09:33 +0400 Subject: [PATCH 3/6] feat: envkey import refactor --- .../external-migration-fns.ts | 333 +++++++++++++----- .../external-migration-types.ts | 2 +- 2 files changed, 249 insertions(+), 86 deletions(-) diff --git a/backend/src/services/external-migration/external-migration-fns.ts b/backend/src/services/external-migration/external-migration-fns.ts index 44e901ab3..90991c33a 100644 --- a/backend/src/services/external-migration/external-migration-fns.ts +++ b/backend/src/services/external-migration/external-migration-fns.ts @@ -88,89 +88,225 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise a.id === env.envParentId); + const appId = parsedJson.apps.find((a) => a.id === env.envParentId)?.id; // If we find the app from the envParentId, we know this is a root-level environment. - if (app) { + if (appId) { infisicalImportData.environments.push({ id: env.id, name: envTemplates.get(env.environmentRoleId)!, - projectId: app.id - }); - } else { - // const parentBlock = parsedJson.blocks.find((b) => b.id === env.envParentId); - // // If this is found, then we know this is a sub-environment. The `parentEnvironment` is the sub environment. - // const subEnvironment = parsedJson.subEnvironments.find( - // (s) => s.parentEnvironmentId === env.id && parsedJson.apps.find((a) => a.id === s.envParentId) - // ); - // if (subEnvironment) { - // infisicalImportData.folders.push({ - // name: subEnvironment.subName, - // parentFolderId: subEnvironment.parentEnvironmentId, - // environmentId: env.id, - // id: subEnvironment.id - // }); - // } else if (parentBlock) { - // // TODO(daniel): Find a way to get the secrets from the parent block, so we can later insert it - // } - } - } - - for (const subEnv of parsedJson.subEnvironments) { - // this will only find the app if the subEnv is a branch, not a block. - const app = parsedJson.apps.find((a) => a.id === subEnv.envParentId); - - const parentEnvironment = infisicalImportData.environments.find((e) => e.id === subEnv.parentEnvironmentId); - - if (app) { - infisicalImportData.folders.push({ - name: subEnv.subName, - parentFolderId: subEnv.parentEnvironmentId, - environmentId: parentEnvironment!.id, - id: subEnv.id + projectId: appId }); } } - // secrets with/without inheritance - for (const env of Object.keys(parsedJson.envs)) { - if (!env.includes("|")) { - const envData = parsedJson.envs[env]; - for (const secret of Object.keys(envData.variables)) { - const selectedSecret = envData.variables[secret]; + const findRootInheritedSecret = ( + secret: { val?: string; inheritsEnvironmentId?: string }, + secretName: string, + envs: typeof parsedJson.envs + ): { val?: string } => { + if (!secret.inheritsEnvironmentId) return secret; + const inheritedEnv = envs[secret.inheritsEnvironmentId]; + if (!inheritedEnv) return secret; + return findRootInheritedSecret(inheritedEnv.variables[secretName], secretName, envs); + }; - if (selectedSecret.inheritsEnvironmentId) { - const findRootInheritedSecret = (currentSecret: { val?: string; inheritsEnvironmentId?: string }) => { - if (currentSecret.inheritsEnvironmentId) { - const inheritedSecret = parsedJson.envs[currentSecret.inheritsEnvironmentId].variables[secret]; - if (inheritedSecret) { - // eslint-disable-next-line no-param-reassign - currentSecret.val = inheritedSecret.val; - } + const processBranches = () => { + for (const subEnv of parsedJson.subEnvironments) { + const app = parsedJson.apps.find((a) => a.id === subEnv.envParentId); + const block = parsedJson.blocks.find((b) => b.id === subEnv.envParentId); - findRootInheritedSecret(inheritedSecret); - } - return currentSecret; - }; + if (app) { + // Handle regular app branches + const branchEnvironment = infisicalImportData.environments.find((e) => e.id === subEnv.parentEnvironmentId); - const sec = findRootInheritedSecret(selectedSecret); + infisicalImportData.folders.push({ + name: subEnv.subName, + parentFolderId: subEnv.parentEnvironmentId, + environmentId: branchEnvironment!.id, + id: subEnv.id + }); + } - infisicalImportData.secrets.push({ - id: randomUUID(), - name: secret, - environmentId: env, - value: sec.val || "???" - }); + if (block) { + // Handle block branches + // 1. Find all apps that use this block + const appsUsingBlock = parsedJson.appBlocks.filter((ab) => ab.blockId === block.id).map((ab) => ab.appId); + + for (const appId of appsUsingBlock) { + // 2. Find the matching environment in the app based on the environment role + const blockBaseEnv = parsedJson.baseEnvironments.find((be) => be.id === subEnv.parentEnvironmentId); // eslint-disable-next-line no-continue - continue; - } + if (!blockBaseEnv) continue; + const matchingAppEnv = parsedJson.baseEnvironments.find( + (be) => be.envParentId === appId && be.environmentRoleId === blockBaseEnv.environmentRoleId + ); + + // eslint-disable-next-line no-continue + if (!matchingAppEnv) continue; + + // 3. Create a folder in the matching app environment + infisicalImportData.folders.push({ + name: subEnv.subName, + parentFolderId: matchingAppEnv.id, + environmentId: matchingAppEnv.id, + id: `${subEnv.id}-${appId}` // Create unique ID for each app's copy of the branch + }); + + // 4. Process secrets in the block branch for this app + const branchSecrets = parsedJson.envs[subEnv.id]?.variables || {}; + for (const [secretName, secretData] of Object.entries(branchSecrets)) { + if (secretData.inheritsEnvironmentId) { + const resolvedSecret = findRootInheritedSecret(secretData, secretName, parsedJson.envs); + infisicalImportData.secrets.push({ + id: randomUUID(), + name: secretName, + environmentId: matchingAppEnv.id, + value: resolvedSecret.val || "", + folderId: `${subEnv.id}-${appId}` + }); + } else { + infisicalImportData.secrets.push({ + id: randomUUID(), + name: secretName, + environmentId: matchingAppEnv.id, + value: secretData.val || "", + folderId: `${subEnv.id}-${appId}` + }); + } + } + } + } + } + }; + + const processBlocksForApp = (appIds: string[]) => { + for (const appId of appIds) { + const blocksInApp = parsedJson.appBlocks.filter((ab) => ab.appId === appId); + logger.info( + { + blocksInApp + }, + "[processBlocksForApp]: Processing blocks for app" + ); + + for (const appBlock of blocksInApp) { + // 1. find all base environments for this block + const blockBaseEnvironments = parsedJson.baseEnvironments.filter((env) => env.envParentId === appBlock.blockId); + logger.info( + { + blockBaseEnvironments + }, + "[processBlocksForApp]: Processing block base environments" + ); + + for (const blockBaseEnvironment of blockBaseEnvironments) { + // 2. find the corresponding environment that is not from the block + const matchingEnv = parsedJson.baseEnvironments.find( + (be) => + be.environmentRoleId === blockBaseEnvironment.environmentRoleId && be.envParentId !== appBlock.blockId + ); + + if (!matchingEnv) { + throw new Error(`Could not find environment for block ${appBlock.blockId}`); + } + + // 3. find all the secrets for this environment block + const blockSecrets = parsedJson.envs[blockBaseEnvironment.id].variables; + + logger.info( + { + blockSecretsLength: Object.keys(blockSecrets).length + }, + "[processBlocksForApp]: Processing block secrets" + ); + + // 4. process each secret + for (const secret of Object.keys(blockSecrets)) { + const selectedSecret = blockSecrets[secret]; + + if (selectedSecret.inheritsEnvironmentId) { + const resolvedSecret = findRootInheritedSecret(selectedSecret, secret, parsedJson.envs); + infisicalImportData.secrets.push({ + id: randomUUID(), + name: secret, + environmentId: matchingEnv.id, + value: resolvedSecret.val || "" + }); + } else { + infisicalImportData.secrets.push({ + id: randomUUID(), + name: secret, + environmentId: matchingEnv.id, + value: selectedSecret.val || "" + }); + } + } + } + } + } + }; + + processBranches(); + processBlocksForApp(infisicalImportData.projects.map((app) => app.id)); + + for (const env of Object.keys(parsedJson.envs)) { + // Skip user-specific environments + // eslint-disable-next-line no-continue + if (env.includes("|")) continue; + + const envData = parsedJson.envs[env]; + const baseEnv = parsedJson.baseEnvironments.find((be) => be.id === env); + const subEnv = parsedJson.subEnvironments.find((se) => se.id === env); + + // Skip if we can't find either a base environment or sub-environment + if (!baseEnv && !subEnv) { + logger.info( + { + envId: env + }, + "[parseEnvKeyDataFn]: Could not find base or sub environment for env, skipping" + ); + // eslint-disable-next-line no-continue + continue; + } + + // If this is a base environment of a block, skip it (handled by processBlocksForApp) + if (baseEnv) { + const isBlock = parsedJson.appBlocks.some((block) => block.blockId === baseEnv.envParentId); + if (isBlock) { + logger.info( + { + envId: env, + baseEnv + }, + "[parseEnvKeyDataFn]: Skipping block environment (handled separately)" + ); + // eslint-disable-next-line no-continue + continue; + } + } + + // Process each secret in this environment or branch + for (const [secretName, secretData] of Object.entries(envData.variables)) { + if (secretData.inheritsEnvironmentId) { + const resolvedSecret = findRootInheritedSecret(secretData, secretName, parsedJson.envs); infisicalImportData.secrets.push({ id: randomUUID(), - name: secret, - environmentId: env, - value: selectedSecret.val || "???_???" + name: secretName, + environmentId: subEnv ? subEnv.parentEnvironmentId : env, + value: resolvedSecret.val || "", + ...(subEnv && { folderId: subEnv.id }) // Add folderId if this is a branch secret + }); + } else { + infisicalImportData.secrets.push({ + id: randomUUID(), + name: secretName, + environmentId: subEnv ? subEnv.parentEnvironmentId : env, + value: secretData.val || "", + ...(subEnv && { folderId: subEnv.id }) // Add folderId if this is a branch secret }); } } @@ -288,9 +424,10 @@ export const importDataIntoInfisicalFn = async ({ } } - console.log("data.folders", data.folders); - - console.log("data.secrets", data.secrets); + // Useful for debugging: + // console.log("data.secrets", data.secrets); + // console.log("data.folders", data.folders); + // console.log("data.environment", data.environments); if (data.secrets && data.secrets.length > 0) { const mappedToEnvironmentId = new Map< @@ -298,49 +435,75 @@ export const importDataIntoInfisicalFn = async ({ { secretKey: string; secretValue: string; + folderId?: string; }[] >(); for (const secret of data.secrets) { - if (!originalToNewEnvironmentId.get(secret.environmentId) && !originalToNewFolderId.get(secret.environmentId)) { + const targetId = secret.folderId || secret.environmentId; + + // Skip if we can't find either an environment or folder mapping for this secret + if (!originalToNewEnvironmentId.get(secret.environmentId) && !originalToNewFolderId.get(targetId)) { // eslint-disable-next-line no-continue continue; } - if (!mappedToEnvironmentId.has(secret.environmentId)) { - mappedToEnvironmentId.set(secret.environmentId, []); + if (!mappedToEnvironmentId.has(targetId)) { + mappedToEnvironmentId.set(targetId, []); } - mappedToEnvironmentId.get(secret.environmentId)!.push({ + mappedToEnvironmentId.get(targetId)!.push({ secretKey: secret.name, - secretValue: secret.value || "" + secretValue: secret.value || "", + folderId: secret.folderId }); } // for each of the mappedEnvironmentId - for await (const [envId, secrets] of mappedToEnvironmentId) { - console.log(`envId ${envId} secrets:`, secrets); - - const environment = data.environments.find((env) => env.id === envId); - const foundFolder = originalToNewFolderId.get(envId); - - console.log(`FOUND FOLDER BY ENV.ID ${envId}`, foundFolder); + for await (const [targetId, secrets] of mappedToEnvironmentId) { + logger.info("[importDataIntoInfisicalFn]: Processing secrets for targetId", targetId); let selectedFolder: TSecretFolders | undefined; let selectedProjectId: string | undefined; - if (foundFolder) { - console.log("RUNNING FOLDER HANDLER"); + // Case 1: Secret belongs to a folder / branch / branch of a block + const foundFolder = originalToNewFolderId.get(targetId); + if (foundFolder) { + logger.info("[importDataIntoInfisicalFn]: Processing secrets for folder"); selectedFolder = await folderDAL.findById(foundFolder.folderId, tx); selectedProjectId = foundFolder.projectId; - } else if (environment) { - console.log("RUNNING ENVIRONMENT HANDLER"); + } else { + logger.info("[importDataIntoInfisicalFn]: Processing secrets for normal environment"); + const environment = data.environments.find((env) => env.id === targetId); + if (!environment) { + logger.info( + { + targetId + }, + "[importDataIntoInfisicalFn]: Could not find environment for secret" + ); + // eslint-disable-next-line no-continue + continue; + } + const projectId = originalToNewProjectId.get(environment.projectId)!; if (!projectId) { throw new BadRequestError({ message: `Failed to import secret, project not found` }); } - const env = originalToNewEnvironmentId.get(envId)!; + const env = originalToNewEnvironmentId.get(targetId); + if (!env) { + logger.info( + { + targetId + }, + "[importDataIntoInfisicalFn]: Could not find environment for secret" + ); + + // eslint-disable-next-line no-continue + continue; + } + const folder = await folderDAL.findBySecretPath(projectId, env.envSlug, "/", tx); if (!folder) { diff --git a/backend/src/services/external-migration/external-migration-types.ts b/backend/src/services/external-migration/external-migration-types.ts index cdf1ef6ac..9aa96f9b7 100644 --- a/backend/src/services/external-migration/external-migration-types.ts +++ b/backend/src/services/external-migration/external-migration-types.ts @@ -2,7 +2,7 @@ import { ActorAuthMethod, ActorType } from "../auth/auth-type"; export type InfisicalImportData = { projects: Array<{ name: string; id: string }>; - environments: Array<{ name: string; id: string; projectId: string }>; + environments: Array<{ name: string; id: string; projectId: string; envParentId?: string }>; folders: Array<{ id: string; name: string; environmentId: string; parentFolderId?: string }>; secrets: Array<{ id: string; name: string; environmentId: string; value: string; folderId?: string }>; }; From 1c32dd5d8a303a2b43c8343deef856fd4be723d8 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Tue, 22 Oct 2024 22:37:33 +0400 Subject: [PATCH 4/6] Update external-migration-types.ts --- .../src/services/external-migration/external-migration-types.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/src/services/external-migration/external-migration-types.ts b/backend/src/services/external-migration/external-migration-types.ts index 9aa96f9b7..f6a67e69a 100644 --- a/backend/src/services/external-migration/external-migration-types.ts +++ b/backend/src/services/external-migration/external-migration-types.ts @@ -29,7 +29,6 @@ export type TEnvKeyExportJSON = { org: { id: string; name: string; - // settings, which we dont care about }; // Apps are projects From d6ffd4fa5fc35163fa05681e4e5294e2a7685a4c Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Wed, 23 Oct 2024 17:23:17 +0400 Subject: [PATCH 5/6] fix: block precedence & root env priority --- .../external-migration-fns.ts | 152 +++++++++++++++++- .../external-migration-types.ts | 9 +- 2 files changed, 154 insertions(+), 7 deletions(-) diff --git a/backend/src/services/external-migration/external-migration-fns.ts b/backend/src/services/external-migration/external-migration-fns.ts index 90991c33a..c86482168 100644 --- a/backend/src/services/external-migration/external-migration-fns.ts +++ b/backend/src/services/external-migration/external-migration-fns.ts @@ -131,9 +131,9 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise ab.blockId === block.id).map((ab) => ab.appId); + const appsUsingBlock = parsedJson.appBlocks.filter((ab) => ab.blockId === block.id); - for (const appId of appsUsingBlock) { + for (const { appId, orderIndex } of appsUsingBlock) { // 2. Find the matching environment in the app based on the environment role const blockBaseEnv = parsedJson.baseEnvironments.find((be) => be.id === subEnv.parentEnvironmentId); @@ -160,20 +160,71 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise s.name === secretName && s.environmentId === matchingAppEnv.id + ); + + if (preExistingSecretIndex !== -1) { + const preExistingSecret = infisicalImportData.secrets[preExistingSecretIndex]; + + if ( + preExistingSecret.appBlockOrderIndex !== undefined && + orderIndex > preExistingSecret.appBlockOrderIndex + ) { + // if the existing secret has a lower orderIndex, we should replace it + infisicalImportData.secrets[preExistingSecretIndex] = { + ...preExistingSecret, + value: resolvedSecret.val || "", + appBlockOrderIndex: orderIndex + }; + } + + // eslint-disable-next-line no-continue + continue; + } + infisicalImportData.secrets.push({ id: randomUUID(), name: secretName, environmentId: matchingAppEnv.id, value: resolvedSecret.val || "", - folderId: `${subEnv.id}-${appId}` + folderId: `${subEnv.id}-${appId}`, + appBlockOrderIndex: orderIndex }); } else { + // If the secret already exists in the environment, we need to check the orderIndex of the appBlock. The appBlock with the highest orderIndex should take precedence. + const preExistingSecretIndex = infisicalImportData.secrets.findIndex( + (s) => s.name === secretName && s.environmentId === matchingAppEnv.id + ); + + if (preExistingSecretIndex !== -1) { + const preExistingSecret = infisicalImportData.secrets[preExistingSecretIndex]; + + if ( + preExistingSecret.appBlockOrderIndex !== undefined && + orderIndex > preExistingSecret.appBlockOrderIndex + ) { + // if the existing secret has a lower orderIndex, we should replace it + infisicalImportData.secrets[preExistingSecretIndex] = { + ...preExistingSecret, + value: secretData.val || "", + appBlockOrderIndex: orderIndex + }; + } + + // eslint-disable-next-line no-continue + continue; + } + infisicalImportData.secrets.push({ id: randomUUID(), name: secretName, environmentId: matchingAppEnv.id, value: secretData.val || "", - folderId: `${subEnv.id}-${appId}` + folderId: `${subEnv.id}-${appId}`, + appBlockOrderIndex: orderIndex }); } } @@ -229,18 +280,69 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise s.name === secret && s.environmentId === matchingEnv.id + ); + + if (preExistingSecretIndex !== -1) { + const preExistingSecret = infisicalImportData.secrets[preExistingSecretIndex]; + + if ( + preExistingSecret.appBlockOrderIndex !== undefined && + appBlock.orderIndex > preExistingSecret.appBlockOrderIndex + ) { + // if the existing secret has a lower orderIndex, we should replace it + infisicalImportData.secrets[preExistingSecretIndex] = { + ...preExistingSecret, + value: selectedSecret.val || "", + appBlockOrderIndex: appBlock.orderIndex + }; + } + + // eslint-disable-next-line no-continue + continue; + } + infisicalImportData.secrets.push({ id: randomUUID(), name: secret, environmentId: matchingEnv.id, - value: resolvedSecret.val || "" + value: resolvedSecret.val || "", + appBlockOrderIndex: appBlock.orderIndex }); } else { + // If the secret already exists in the environment, we need to check the orderIndex of the appBlock. The appBlock with the highest orderIndex should take precedence. + const preExistingSecretIndex = infisicalImportData.secrets.findIndex( + (s) => s.name === secret && s.environmentId === matchingEnv.id + ); + + if (preExistingSecretIndex !== -1) { + const preExistingSecret = infisicalImportData.secrets[preExistingSecretIndex]; + + if ( + preExistingSecret.appBlockOrderIndex !== undefined && + appBlock.orderIndex > preExistingSecret.appBlockOrderIndex + ) { + // if the existing secret has a lower orderIndex, we should replace it + infisicalImportData.secrets[preExistingSecretIndex] = { + ...preExistingSecret, + value: selectedSecret.val || "", + appBlockOrderIndex: appBlock.orderIndex + }; + } + + // eslint-disable-next-line no-continue + continue; + } + infisicalImportData.secrets.push({ id: randomUUID(), name: secret, environmentId: matchingEnv.id, - value: selectedSecret.val || "" + value: selectedSecret.val || "", + appBlockOrderIndex: appBlock.orderIndex }); } } @@ -291,8 +393,30 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise s.name === secretName && s.environmentId === environmentId + ); + if (secretData.inheritsEnvironmentId) { const resolvedSecret = findRootInheritedSecret(secretData, secretName, parsedJson.envs); + + // Check if there's already a secret with this name in the environment, if there is, we should override it. Because if there's already one, we know its coming from a block. + // Variables from the normal environment should take precedence over variables from the block. + + if (indexOfExistingSecret !== -1) { + // if a existing secret is found, we should replace it directly + const newSecret: (typeof infisicalImportData.secrets)[number] = { + ...infisicalImportData.secrets[indexOfExistingSecret], + value: resolvedSecret.val || "" + }; + + infisicalImportData.secrets[indexOfExistingSecret] = newSecret; + + // eslint-disable-next-line no-continue + continue; + } + infisicalImportData.secrets.push({ id: randomUUID(), name: secretName, @@ -301,6 +425,22 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise; environments: Array<{ name: string; id: string; projectId: string; envParentId?: string }>; folders: Array<{ id: string; name: string; environmentId: string; parentFolderId?: string }>; - secrets: Array<{ id: string; name: string; environmentId: string; value: string; folderId?: string }>; + secrets: Array<{ + id: string; + name: string; + environmentId: string; + value: string; + folderId?: string; + appBlockOrderIndex?: number; // Not used for infisical import, only used for building the import structure to determine which block(s) take precedence. + }>; }; export type TImportEnvKeyDataCreate = { From b1ba770a71c79ff7933f903b4434e40ac97a30bd Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Fri, 25 Oct 2024 20:29:21 +0400 Subject: [PATCH 6/6] Update external-migration-fns.ts --- .../external-migration-fns.ts | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/backend/src/services/external-migration/external-migration-fns.ts b/backend/src/services/external-migration/external-migration-fns.ts index c86482168..ad1c4d8ed 100644 --- a/backend/src/services/external-migration/external-migration-fns.ts +++ b/backend/src/services/external-migration/external-migration-fns.ts @@ -105,7 +105,22 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise { - if (!secret.inheritsEnvironmentId) return secret; + if (!secret) { + return { + val: "" + }; + } + + // If we have a direct value, return it + if (secret.val !== undefined) { + return secret; + } + + // If there's no inheritance, return the secret as is + if (!secret.inheritsEnvironmentId) { + return secret; + } + const inheritedEnv = envs[secret.inheritsEnvironmentId]; if (!inheritedEnv) return secret; return findRootInheritedSecret(inheritedEnv.variables[secretName], secretName, envs);