From 0485b56e8d94b87e2a25425dd0070a8b49067ab3 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Tue, 1 Oct 2024 03:51:55 +0400 Subject: [PATCH] fix: improvements --- .../routes/v3/external-migration-router.ts | 7 +- .../external-migration-fns.ts | 130 ++++++++++++++- .../external-migration-service.ts | 148 ++---------------- .../src/services/project/project-service.ts | 12 +- .../components/ImportTab/ImportTab.tsx | 63 ++++---- 5 files changed, 180 insertions(+), 180 deletions(-) diff --git a/backend/src/server/routes/v3/external-migration-router.ts b/backend/src/server/routes/v3/external-migration-router.ts index 4577da97a..71861bf4c 100644 --- a/backend/src/server/routes/v3/external-migration-router.ts +++ b/backend/src/server/routes/v3/external-migration-router.ts @@ -18,14 +18,11 @@ export const registerExternalMigrationRouter = async (server: FastifyZodProvider nonce: z.string().trim().min(1), data: z.string().trim().min(1) }) - }), - response: { - 200: z.object({}) - } + }) }, onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), handler: async (req) => { - await server.services.migration.importEnvnKeyData({ + await server.services.migration.importEnvKeyData({ decryptionKey: req.body.decryptionKey, encryptedJson: req.body.encryptedJson, actorId: req.permission.id, diff --git a/backend/src/services/external-migration/external-migration-fns.ts b/backend/src/services/external-migration/external-migration-fns.ts index de2c8a390..0e3cc576f 100644 --- a/backend/src/services/external-migration/external-migration-fns.ts +++ b/backend/src/services/external-migration/external-migration-fns.ts @@ -1,16 +1,33 @@ +import slugify from "@sindresorhus/slugify"; import { randomUUID } from "crypto"; import sjcl from "sjcl"; import tweetnacl from "tweetnacl"; import tweetnaclUtil from "tweetnacl-util"; +import { OrgMembershipRole, ProjectMembershipRole, SecretType } from "@app/db/schemas"; import { BadRequestError } from "@app/lib/errors"; +import { logger } from "@app/lib/logger"; +import { alphaNumericNanoId } from "@app/lib/nanoid"; -import { InfisicalImportData, TEnvKeyExportJSON } from "./external-migration-types"; +import { TOrgServiceFactory } from "../org/org-service"; +import { TProjectServiceFactory } from "../project/project-service"; +import { TProjectEnvServiceFactory } from "../project-env/project-env-service"; +import { TSecretServiceFactory } from "../secret/secret-service"; +import { InfisicalImportData, TEnvKeyExportJSON, TImportInfisicalDataCreate } from "./external-migration-types"; + +export type TImportDataIntoInfisicalDTO = { + projectService: TProjectServiceFactory; + orgService: TOrgServiceFactory; + projectEnvService: TProjectEnvServiceFactory; + secretService: TSecretServiceFactory; + + input: TImportInfisicalDataCreate; +}; const { codec, hash } = sjcl; const { secretbox } = tweetnacl; -export const decryptEnvKeyData = async (decryptionKey: string, encryptedJson: { nonce: string; data: string }) => { +export const decryptEnvKeyDataFn = async (decryptionKey: string, encryptedJson: { nonce: string; data: string }) => { const key = tweetnaclUtil.decodeBase64(codec.base64.fromBits(hash.sha256.hash(decryptionKey))); const nonce = tweetnaclUtil.decodeBase64(encryptedJson.nonce); const encryptedData = tweetnaclUtil.decodeBase64(encryptedJson.data); @@ -25,7 +42,7 @@ export const decryptEnvKeyData = async (decryptionKey: string, encryptedJson: { return decryptedJson; }; -export const parseEnvKeyData = async (decryptedJson: string): Promise => { +export const parseEnvKeyDataFn = async (decryptedJson: string): Promise => { const parsedJson: TEnvKeyExportJSON = JSON.parse(decryptedJson) as TEnvKeyExportJSON; const infisicalImportData: InfisicalImportData = { @@ -71,3 +88,110 @@ export const parseEnvKeyData = async (decryptedJson: string): Promise { + // Import data to infisical + if (!data || !data.projects) { + throw new BadRequestError({ message: "No projects found in data" }); + } + + const originalToNewProjectId = new Map(); + const originalToNewEnvironmentId = new Map(); + + for await (const [id, project] of data.projects) { + const newProject = await projectService + .createProject({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + workspaceName: project.name, + createDefaultEnvs: false + }) + .catch(() => { + throw new BadRequestError({ message: `Failed to import to project [name:${project.name}] [id:${id}]` }); + }); + + originalToNewProjectId.set(project.id, newProject.id); + } + + // Invite user importing projects + const invites = await orgService.inviteUserToOrganization({ + actorAuthMethod, + actorId, + actorOrgId, + actor, + inviteeEmails: [], + orgId: actorOrgId, + organizationRoleSlug: OrgMembershipRole.NoAccess, + projects: Array.from(originalToNewProjectId.values()).map((project) => ({ + id: project, + projectRoleSlug: [ProjectMembershipRole.Member] + })) + }); + if (!invites) { + throw new BadRequestError({ message: `Failed to invite user to projects: [userId:${actorId}]` }); + } + + // Import environments + if (data.environments) { + for await (const [id, environment] of data.environments) { + try { + const newEnvironment = await projectEnvService.createEnvironment({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + name: environment.name, + projectId: originalToNewProjectId.get(environment.projectId)!, + slug: slugify(`${environment.name}-${alphaNumericNanoId(4)}`) + }); + + if (!newEnvironment) { + logger.error(`Failed to import environment: [name:${environment.name}] [id:${id}]`); + throw new BadRequestError({ + message: `Failed to import environment: [name:${environment.name}] [id:${id}]` + }); + } + originalToNewEnvironmentId.set(id, newEnvironment.slug); + } catch (error) { + throw new BadRequestError({ + message: `Failed to import environment: ${environment.name}]`, + name: "EnvKeyMigrationImportEnvironment" + }); + } + } + } + + // Import secrets + if (data.secrets) { + for await (const [id, secret] of data.secrets) { + const dataProjectId = data.environments?.get(secret.environmentId)?.projectId; + if (!dataProjectId) { + throw new BadRequestError({ message: `Failed to import secret "${secret.name}", project not found` }); + } + const projectId = originalToNewProjectId.get(dataProjectId); + const newSecret = await secretService.createSecretRaw({ + actorId, + actor, + actorOrgId, + environment: originalToNewEnvironmentId.get(secret.environmentId)!, + actorAuthMethod, + projectId: projectId!, + secretPath: "/", + secretName: secret.name, + type: SecretType.Shared, + secretValue: secret.value + }); + if (!newSecret) { + throw new BadRequestError({ message: `Failed to import secret: [name:${secret.name}] [id:${id}]` }); + } + } + } +}; diff --git a/backend/src/services/external-migration/external-migration-service.ts b/backend/src/services/external-migration/external-migration-service.ts index bbb464064..95fd9a496 100644 --- a/backend/src/services/external-migration/external-migration-service.ts +++ b/backend/src/services/external-migration/external-migration-service.ts @@ -1,16 +1,9 @@ -import slugify from "@sindresorhus/slugify"; - -import { OrgMembershipRole, ProjectMembershipRole, SecretType } from "@app/db/schemas"; -import { BadRequestError } from "@app/lib/errors"; -import { logger } from "@app/lib/logger"; -import { alphaNumericNanoId } from "@app/lib/nanoid"; - import { TOrgServiceFactory } from "../org/org-service"; import { TProjectServiceFactory } from "../project/project-service"; import { TProjectEnvServiceFactory } from "../project-env/project-env-service"; import { TSecretServiceFactory } from "../secret/secret-service"; -import { decryptEnvKeyData, parseEnvKeyData } from "./external-migration-fns"; -import { TImportEnvKeyDataCreate, TImportInfisicalDataCreate } from "./external-migration-types"; +import { decryptEnvKeyDataFn, importDataIntoInfisicalFn, parseEnvKeyDataFn } from "./external-migration-fns"; +import { TImportEnvKeyDataCreate } from "./external-migration-types"; type TExternalMigrationServiceFactoryDep = { projectService: TProjectServiceFactory; @@ -27,128 +20,7 @@ export const externalMigrationServiceFactory = ({ projectEnvService, secretService }: TExternalMigrationServiceFactoryDep) => { - const importInfisicalData = async ({ - data, - actor, - actorId, - actorOrgId, - actorAuthMethod - }: TImportInfisicalDataCreate) => { - // Import data to infisical - if (!data || !data.projects) { - throw new BadRequestError({ message: "No projects found in data" }); - } - - const orginalToNewProjectId = new Map(); - const orginalToNewEnvironmentId = new Map(); - - // Import projects - const projectPromises = []; - for (const [id, project] of data.projects) { - const projectPromise = projectService - .createProject({ - actor, - actorId, - actorOrgId, - actorAuthMethod, - workspaceName: project.name, - createDefaultEnvs: false - }) - .then((projectResponse) => { - if (!projectResponse) { - throw new BadRequestError({ message: `Failed to import to project [name:${project.name}] [id:${id}]` }); - } - orginalToNewProjectId.set(project.id, projectResponse.id); - }); - projectPromises.push(projectPromise); - } - await Promise.all(projectPromises); - - // Invite user importing projects - const response = await orgService.inviteUserToOrganization({ - actorAuthMethod, - actorId, - actorOrgId, - actor, - inviteeEmails: [], - orgId: actorOrgId, - organizationRoleSlug: OrgMembershipRole.NoAccess, - projects: Array.from(orginalToNewProjectId.values()).map((project) => { - return { - id: project, - projectRoleSlug: [ProjectMembershipRole.Member] - }; - }) - }); - if (!response) { - throw new BadRequestError({ message: `Failed to invite user to projects: [userId:${actorId}]` }); - } - - // Import environments - if (data.environments) { - for await (const [id, environment] of data.environments) { - try { - // TODO: we can create envs parallely once the position constraint is handled differently - const newEnvironment = await projectEnvService.createEnvironment({ - actor, - actorId, - actorOrgId, - actorAuthMethod, - name: environment.name, - projectId: orginalToNewProjectId.get(environment.projectId)!, - slug: slugify(`${environment.name}-${alphaNumericNanoId(4)}`) - }); - - if (!newEnvironment) { - logger.error(`Failed to import environment: [name:${environment.name}] [id:${id}]`); - throw new BadRequestError({ - message: `Failed to import environment: [name:${environment.name}] [id:${id}]` - }); - } - orginalToNewEnvironmentId.set(id, newEnvironment.slug); - } catch (error) { - return { - success: false - }; - } - } - } - - // Import secrets - if (data.secrets) { - for await (const [id, secret] of data.secrets) { - const dataProjectId = data.environments?.get(secret.environmentId)?.projectId; - if (!dataProjectId) { - logger.error(`Failed to import secret: [name:${secret.name}] [id:${id}], project not found`); - return { - success: false, - message: `Failed to import secret: [name:${secret.name}] [id:${id}], project not found` - }; - } - const projectId = orginalToNewProjectId.get(dataProjectId); - // TODO: we can create secrets parallely once the KMS ID bug on create is fixed - const newSecret = await secretService.createSecretRaw({ - actorId, - actor, - actorOrgId, - environment: orginalToNewEnvironmentId.get(secret.environmentId)!, - actorAuthMethod, - projectId: projectId!, - secretPath: "/", - secretName: secret.name, - type: SecretType.Shared, - secretValue: secret.value - }); - if (!newSecret) { - throw new BadRequestError({ message: `Failed to import secret: [name:${secret.name}] [id:${id}]` }); - } - } - } - - return true; - }; - - const importEnvnKeyData = async ({ + const importEnvKeyData = async ({ decryptionKey, encryptedJson, actor, @@ -156,13 +28,19 @@ export const externalMigrationServiceFactory = ({ actorOrgId, actorAuthMethod }: TImportEnvKeyDataCreate) => { - const json = await decryptEnvKeyData(decryptionKey, encryptedJson); - const envKeyData = await parseEnvKeyData(json); - const response = await importInfisicalData({ data: envKeyData, actor, actorId, actorOrgId, actorAuthMethod }); + const json = await decryptEnvKeyDataFn(decryptionKey, encryptedJson); + const envKeyData = await parseEnvKeyDataFn(json); + const response = await importDataIntoInfisicalFn({ + input: { data: envKeyData, actor, actorId, actorOrgId, actorAuthMethod }, + projectService, + orgService, + projectEnvService, + secretService + }); return response; }; return { - importEnvnKeyData + importEnvKeyData }; }; diff --git a/backend/src/services/project/project-service.ts b/backend/src/services/project/project-service.ts index f3868d0be..ec6c54dcc 100644 --- a/backend/src/services/project/project-service.ts +++ b/backend/src/services/project/project-service.ts @@ -1,7 +1,7 @@ import { ForbiddenError } from "@casl/ability"; import slugify from "@sindresorhus/slugify"; -import { OrgMembershipRole, ProjectMembershipRole, ProjectVersion } from "@app/db/schemas"; +import { OrgMembershipRole, ProjectMembershipRole, ProjectVersion, TProjectEnvironments } from "@app/db/schemas"; import { TLicenseServiceFactory } from "@app/ee/services/license/license-service"; import { OrgPermissionActions, OrgPermissionSubjects } from "@app/ee/services/permission/org-permission"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; @@ -208,15 +208,7 @@ export const projectServiceFactory = ({ ); // set default environments and root folder for provided environments - let envs: { - id: string; - createdAt: Date; - updatedAt: Date; - projectId: string; - name: string; - slug: string; - position: number; - }[] = []; + let envs: TProjectEnvironments[] = []; if (createDefaultEnvs) { envs = await projectEnvDAL.insertMany( DEFAULT_PROJECT_ENVS.map((el, i) => ({ ...el, projectId: project.id, position: i + 1 })), diff --git a/frontend/src/views/Settings/OrgSettingsPage/components/ImportTab/ImportTab.tsx b/frontend/src/views/Settings/OrgSettingsPage/components/ImportTab/ImportTab.tsx index 0c444750c..443ab6db8 100644 --- a/frontend/src/views/Settings/OrgSettingsPage/components/ImportTab/ImportTab.tsx +++ b/frontend/src/views/Settings/OrgSettingsPage/components/ImportTab/ImportTab.tsx @@ -24,8 +24,7 @@ type TForm = z.infer; export const ImportTab = () => { const fileUploadRef = useRef(null); - const { mutateAsync: importEnvKey - } = useImportEnvKey(); + const { mutateAsync: importEnvKey } = useImportEnvKey(); const { handleSubmit, @@ -50,11 +49,14 @@ export const ImportTab = () => { const parseJson = (src: ArrayBuffer) => { const file = src.toString(); - const formatedData: Record = JSON.parse(file); - if (Object.keys(formatedData).includes("nonce") && Object.keys(formatedData).includes("data")) { + const formattedData: Record = JSON.parse(file); + if ( + Object.keys(formattedData).includes("nonce") && + Object.keys(formattedData).includes("data") + ) { const data = { - nonce: formatedData.nonce, - data: formatedData.data + nonce: formattedData.nonce, + data: formattedData.data }; setValue("encryptedJson", data); trigger("encryptedJson"); @@ -86,9 +88,9 @@ export const ImportTab = () => { if (!event?.target?.result) return; // parse function's argument looks like to be ArrayBuffer parseJson(event.target.result as ArrayBuffer); - } + }; reader.readAsText(file); - } + }; const submitExport = async (data: TForm) => { if (!data.encryptedJson) { @@ -99,7 +101,7 @@ export const ImportTab = () => { return; } - try{ + try { await importEnvKey({ encryptedJson: data.encryptedJson, decryptionKey: data.encryptionKey }); createNotification({ text: "Data imported successfully.", @@ -109,11 +111,10 @@ export const ImportTab = () => { if (fileUploadRef.current) { fileUploadRef.current.value = ""; } - } catch (error) { + } catch { reset(); } - - } + }; const watchEncryptedJsonFile: any = watch("file"); useEffect(() => { @@ -124,13 +125,13 @@ export const ImportTab = () => { return (