From f4ba441ec39fccb5a6d7ea008583db9694462817 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Mon, 21 Oct 2024 20:39:08 +0400 Subject: [PATCH] feat: envkey data migration refactor --- .../external-migration-fns.ts | 211 +++++++++++++++--- .../external-migration-types.ts | 72 +++--- 2 files changed, 218 insertions(+), 65 deletions(-) diff --git a/backend/src/services/external-migration/external-migration-fns.ts b/backend/src/services/external-migration/external-migration-fns.ts index 6d996022a..44e901ab3 100644 --- a/backend/src/services/external-migration/external-migration-fns.ts +++ b/backend/src/services/external-migration/external-migration-fns.ts @@ -4,7 +4,7 @@ import sjcl from "sjcl"; import tweetnacl from "tweetnacl"; import tweetnaclUtil from "tweetnacl-util"; -import { SecretType } from "@app/db/schemas"; +import { SecretType, TSecretFolders } from "@app/db/schemas"; import { BadRequestError, NotFoundError } from "@app/lib/errors"; import { chunkArray } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; @@ -35,7 +35,7 @@ export type TImportDataIntoInfisicalDTO = { secretTagDAL: Pick; secretVersionTagDAL: Pick; - folderDAL: Pick; + folderDAL: Pick; projectService: Pick; projectEnvService: Pick; secretV2BridgeService: Pick; @@ -67,6 +67,7 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise a.id === env.envParentId); + + // If we find the app from the envParentId, we know this is a root-level environment. + if (app) { + infisicalImportData.environments.push({ + id: env.id, + name: envTemplates.get(env.environmentRoleId)!, + projectId: app.id + }); + } else { + // const parentBlock = parsedJson.blocks.find((b) => b.id === env.envParentId); + // // If this is found, then we know this is a sub-environment. The `parentEnvironment` is the sub environment. + // const subEnvironment = parsedJson.subEnvironments.find( + // (s) => s.parentEnvironmentId === env.id && parsedJson.apps.find((a) => a.id === s.envParentId) + // ); + // if (subEnvironment) { + // infisicalImportData.folders.push({ + // name: subEnvironment.subName, + // parentFolderId: subEnvironment.parentEnvironmentId, + // environmentId: env.id, + // id: subEnvironment.id + // }); + // } else if (parentBlock) { + // // TODO(daniel): Find a way to get the secrets from the parent block, so we can later insert it + // } + } + } + + for (const subEnv of parsedJson.subEnvironments) { + // this will only find the app if the subEnv is a branch, not a block. + const app = parsedJson.apps.find((a) => a.id === subEnv.envParentId); + + const parentEnvironment = infisicalImportData.environments.find((e) => e.id === subEnv.parentEnvironmentId); + + if (app) { + infisicalImportData.folders.push({ + name: subEnv.subName, + parentFolderId: subEnv.parentEnvironmentId, + environmentId: parentEnvironment!.id, + id: subEnv.id + }); + } + } + + // secrets with/without inheritance for (const env of Object.keys(parsedJson.envs)) { if (!env.includes("|")) { const envData = parsedJson.envs[env]; for (const secret of Object.keys(envData.variables)) { + const selectedSecret = envData.variables[secret]; + + if (selectedSecret.inheritsEnvironmentId) { + const findRootInheritedSecret = (currentSecret: { val?: string; inheritsEnvironmentId?: string }) => { + if (currentSecret.inheritsEnvironmentId) { + const inheritedSecret = parsedJson.envs[currentSecret.inheritsEnvironmentId].variables[secret]; + if (inheritedSecret) { + // eslint-disable-next-line no-param-reassign + currentSecret.val = inheritedSecret.val; + } + + findRootInheritedSecret(inheritedSecret); + } + return currentSecret; + }; + + const sec = findRootInheritedSecret(selectedSecret); + + infisicalImportData.secrets.push({ + id: randomUUID(), + name: secret, + environmentId: env, + value: sec.val || "???" + }); + + // eslint-disable-next-line no-continue + continue; + } + infisicalImportData.secrets.push({ id: randomUUID(), name: secret, environmentId: env, - value: envData.variables[secret].val + value: selectedSecret.val || "???_???" }); } } @@ -125,7 +197,17 @@ export const importDataIntoInfisicalFn = async ({ } const originalToNewProjectId = new Map(); - const originalToNewEnvironmentId = new Map(); + const originalToNewEnvironmentId = new Map< + string, + { envId: string; envSlug: string; rootFolderId: string; projectId: string } + >(); + const originalToNewFolderId = new Map< + string, + { + folderId: string; + projectId: string; + } + >(); const projectsNotImported: string[] = []; await projectDAL.transaction(async (tx) => { @@ -170,12 +252,46 @@ export const importDataIntoInfisicalFn = async ({ const lastPos = await projectEnvDAL.findLastEnvPosition(projectId, tx); const doc = await projectEnvDAL.create({ slug, name: environment.name, projectId, position: lastPos + 1 }, tx); - await folderDAL.create({ name: "root", parentId: null, envId: doc.id, version: 1 }, tx); + const folder = await folderDAL.create({ name: "root", parentId: null, envId: doc.id, version: 1 }, tx); - originalToNewEnvironmentId.set(environment.id, doc.slug); + originalToNewEnvironmentId.set(environment.id, { + envSlug: doc.slug, + envId: doc.id, + rootFolderId: folder.id, + projectId + }); } } + if (data.folders) { + for await (const folder of data.folders) { + const parentEnv = originalToNewEnvironmentId.get(folder.parentFolderId as string); + + if (!parentEnv) { + // eslint-disable-next-line no-continue + continue; + } + + const newFolder = await folderDAL.create( + { + name: folder.name, + envId: parentEnv.envId, + parentId: parentEnv.rootFolderId + }, + tx + ); + + originalToNewFolderId.set(folder.id, { + folderId: newFolder.id, + projectId: parentEnv.projectId + }); + } + } + + console.log("data.folders", data.folders); + + console.log("data.secrets", data.secrets); + if (data.secrets && data.secrets.length > 0) { const mappedToEnvironmentId = new Map< string, @@ -186,7 +302,7 @@ export const importDataIntoInfisicalFn = async ({ >(); for (const secret of data.secrets) { - if (!originalToNewEnvironmentId.get(secret.environmentId)) { + if (!originalToNewEnvironmentId.get(secret.environmentId) && !originalToNewFolderId.get(secret.environmentId)) { // eslint-disable-next-line no-continue continue; } @@ -202,33 +318,68 @@ export const importDataIntoInfisicalFn = async ({ // for each of the mappedEnvironmentId for await (const [envId, secrets] of mappedToEnvironmentId) { - const environment = data.environments.find((env) => env.id === envId); - const projectId = originalToNewProjectId.get(environment?.projectId as string)!; + console.log(`envId ${envId} secrets:`, secrets); - if (!projectId) { - throw new BadRequestError({ message: `Failed to import secret, project not found` }); + const environment = data.environments.find((env) => env.id === envId); + const foundFolder = originalToNewFolderId.get(envId); + + console.log(`FOUND FOLDER BY ENV.ID ${envId}`, foundFolder); + + let selectedFolder: TSecretFolders | undefined; + let selectedProjectId: string | undefined; + if (foundFolder) { + console.log("RUNNING FOLDER HANDLER"); + + selectedFolder = await folderDAL.findById(foundFolder.folderId, tx); + selectedProjectId = foundFolder.projectId; + } else if (environment) { + console.log("RUNNING ENVIRONMENT HANDLER"); + const projectId = originalToNewProjectId.get(environment.projectId)!; + + if (!projectId) { + throw new BadRequestError({ message: `Failed to import secret, project not found` }); + } + + const env = originalToNewEnvironmentId.get(envId)!; + const folder = await folderDAL.findBySecretPath(projectId, env.envSlug, "/", tx); + + if (!folder) { + throw new NotFoundError({ + message: `Folder not found for the given environment slug (${env.envSlug}) & secret path (/)`, + name: "Create secret" + }); + } + + selectedFolder = folder; + selectedProjectId = projectId; + } + + if (!selectedFolder) { + throw new NotFoundError({ + message: `Folder not found for the given environment slug & secret path`, + name: "CreateSecret" + }); + } + + if (!selectedProjectId) { + throw new NotFoundError({ + message: `Project not found for the given environment slug & secret path`, + name: "CreateSecret" + }); } const { encryptor: secretManagerEncrypt } = await kmsService.createCipherPairWithDataKey( { type: KmsDataKey.SecretManager, - projectId + projectId: selectedProjectId }, tx ); - const envSlug = originalToNewEnvironmentId.get(envId)!; - const folder = await folderDAL.findBySecretPath(projectId, envSlug, "/", tx); - if (!folder) - throw new NotFoundError({ - message: `Folder not found for the given environment slug (${envSlug}) & secret path (/)`, - name: "Create secret" - }); - const secretBatches = chunkArray(secrets, 2500); for await (const secretBatch of secretBatches) { const secretsByKeys = await secretDAL.findBySecretKeys( - folder.id, + selectedFolder.id, secretBatch.map((el) => ({ key: el.secretKey, type: SecretType.Shared @@ -254,7 +405,7 @@ export const importDataIntoInfisicalFn = async ({ type: SecretType.Shared }; }), - folderId: folder.id, + folderId: selectedFolder.id, secretDAL, secretVersionDAL, secretTagDAL, diff --git a/backend/src/services/external-migration/external-migration-types.ts b/backend/src/services/external-migration/external-migration-types.ts index 53c954bf9..cdf1ef6ac 100644 --- a/backend/src/services/external-migration/external-migration-types.ts +++ b/backend/src/services/external-migration/external-migration-types.ts @@ -3,7 +3,8 @@ import { ActorAuthMethod, ActorType } from "../auth/auth-type"; export type InfisicalImportData = { projects: Array<{ name: string; id: string }>; environments: Array<{ name: string; id: string; projectId: string }>; - secrets: Array<{ name: string; id: string; environmentId: string; value: string }>; + folders: Array<{ id: string; name: string; environmentId: string; parentFolderId?: string }>; + secrets: Array<{ id: string; name: string; environmentId: string; value: string; folderId?: string }>; }; export type TImportEnvKeyDataCreate = { @@ -28,62 +29,63 @@ export type TEnvKeyExportJSON = { org: { id: string; name: string; - settings: { - auth: { - inviteExpirationMs: number; - deviceGrantExpirationMs: number; - tokenExpirationMs: number; - }; - crypto: { - requiresPassphrase: boolean; - requiresLockout: boolean; - }; - envs: { - autoCaps: boolean; - autoCommitLocals: boolean; - }; - }; + // settings, which we dont care about }; + + // Apps are projects apps: { id: string; name: string; - settings: Record; }[]; - defaultOrgRoles: { + // Blocks are basically global projects that can be imported in other projects + blocks: { id: string; - defaultName: string; + name: string; }[]; - defaultAppRoles: { - id: string; - defaultName: string; + + appBlocks: { + appId: string; + blockId: string; + orderIndex: number; }[]; + defaultEnvironmentRoles: { id: string; defaultName: string; - settings: { - autoCommit: boolean; - }; }[]; + + nonDefaultEnvironmentRoles: { + id: string; + name: string; + }[]; + baseEnvironments: { id: string; envParentId: string; environmentRoleId: string; - settings: Record; }[]; - orgUsers: { + + // Branches for both blocks and apps + subEnvironments: { id: string; - firstName: string; - lastName: string; - email: string; - provider: string; - orgRoleId: string; - uid: string; + envParentId: string; + environmentRoleId: string; + parentEnvironmentId: string; + subName: string; }[]; + envs: Record< string, { - variables: Record; - inherits: Record; + variables: Record< + string, + { + val?: string; + inheritsEnvironmentId?: string; + } + >; + + inherits: Record; } >; };