diff --git a/backend/package-lock.json b/backend/package-lock.json index 6ea34f692..7cf7e40b8 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -37,6 +37,7 @@ "@slack/oauth": "^3.0.1", "@slack/web-api": "^7.3.4", "@team-plain/typescript-sdk": "^4.6.1", + "@types/sjcl": "^1.0.34", "@ucast/mongo2js": "^1.3.4", "ajv": "^8.12.0", "argon2": "^0.31.2", @@ -85,6 +86,7 @@ "safe-regex": "^2.1.1", "scim-patch": "^0.8.3", "scim2-parse-filter": "^0.2.10", + "sjcl": "^1.0.8", "smee-client": "^2.0.0", "tedious": "^18.2.1", "tweetnacl": "^1.0.3", @@ -7296,6 +7298,12 @@ "@types/node": "*" } }, + "node_modules/@types/sjcl": { + "version": "1.0.34", + "resolved": "https://registry.npmjs.org/@types/sjcl/-/sjcl-1.0.34.tgz", + "integrity": "sha512-bQHEeK5DTQRunIfQeUMgtpPsNNCcZyQ9MJuAfW1I7iN0LDunTc78Fu17STbLMd7KiEY/g2zHVApippa70h6HoQ==", + "license": "MIT" + }, "node_modules/@types/uuid": { "version": "9.0.7", "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.7.tgz", @@ -16397,6 +16405,15 @@ "node": ">=10" } }, + "node_modules/sjcl": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/sjcl/-/sjcl-1.0.8.tgz", + "integrity": "sha512-LzIjEQ0S0DpIgnxMEayM1rq9aGwGRG4OnZhCdjx7glTaJtf4zRfpg87ImfjSJjoW9vKpagd82McDOwbRT5kQKQ==", + "license": "(BSD-2-Clause OR GPL-2.0-only)", + "engines": { + "node": "*" + } + }, "node_modules/slash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", @@ -17874,12 +17891,14 @@ "node_modules/tweetnacl": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-1.0.3.tgz", - "integrity": "sha512-6rt+RN7aOi1nGMyC4Xa5DdYiukl2UWCbcJft7YhxReBGQD7OAM8Pbxw6YMo4r2diNEA8FEmu32YOn9rhaiE5yw==" + "integrity": "sha512-6rt+RN7aOi1nGMyC4Xa5DdYiukl2UWCbcJft7YhxReBGQD7OAM8Pbxw6YMo4r2diNEA8FEmu32YOn9rhaiE5yw==", + "license": "Unlicense" }, "node_modules/tweetnacl-util": { "version": "0.15.1", "resolved": "https://registry.npmjs.org/tweetnacl-util/-/tweetnacl-util-0.15.1.tgz", - "integrity": "sha512-RKJBIj8lySrShN4w6i/BonWp2Z/uxwC3h4y7xsRrpP59ZboCd0GpEVsOnMDYLMmKBpYhb5TgHzZXy7wTfYFBRw==" + "integrity": "sha512-RKJBIj8lySrShN4w6i/BonWp2Z/uxwC3h4y7xsRrpP59ZboCd0GpEVsOnMDYLMmKBpYhb5TgHzZXy7wTfYFBRw==", + "license": "Unlicense" }, "node_modules/type-check": { "version": "0.4.0", diff --git a/backend/package.json b/backend/package.json index 21f841da7..74b210c05 100644 --- a/backend/package.json +++ b/backend/package.json @@ -134,6 +134,7 @@ "@slack/oauth": "^3.0.1", "@slack/web-api": "^7.3.4", "@team-plain/typescript-sdk": "^4.6.1", + "@types/sjcl": "^1.0.34", "@ucast/mongo2js": "^1.3.4", "ajv": "^8.12.0", "argon2": "^0.31.2", @@ -182,6 +183,7 @@ "safe-regex": "^2.1.1", "scim-patch": "^0.8.3", "scim2-parse-filter": "^0.2.10", + "sjcl": "^1.0.8", "smee-client": "^2.0.0", "tedious": "^18.2.1", "tweetnacl": "^1.0.3", diff --git a/backend/src/@types/fastify.d.ts b/backend/src/@types/fastify.d.ts index 1b487ac1f..10be1c4c9 100644 --- a/backend/src/@types/fastify.d.ts +++ b/backend/src/@types/fastify.d.ts @@ -38,6 +38,7 @@ import { TAuthTokenServiceFactory } from "@app/services/auth-token/auth-token-se import { TCertificateServiceFactory } from "@app/services/certificate/certificate-service"; import { TCertificateAuthorityServiceFactory } from "@app/services/certificate-authority/certificate-authority-service"; import { TCertificateTemplateServiceFactory } from "@app/services/certificate-template/certificate-template-service"; +import { TExternalMigrationServiceFactory } from "@app/services/external-migration/external-migration-service"; import { TGroupProjectServiceFactory } from "@app/services/group-project/group-project-service"; import { TIdentityServiceFactory } from "@app/services/identity/identity-service"; import { TIdentityAccessTokenServiceFactory } from "@app/services/identity-access-token/identity-access-token-service"; @@ -181,6 +182,7 @@ declare module "fastify" { orgAdmin: TOrgAdminServiceFactory; slack: TSlackServiceFactory; workflowIntegration: TWorkflowIntegrationServiceFactory; + migration: TExternalMigrationServiceFactory; }; // this is exclusive use for middlewares in which we need to inject data // everywhere else access using service layer diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 5d9632215..3648f7862 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -96,6 +96,7 @@ import { certificateAuthorityServiceFactory } from "@app/services/certificate-au import { certificateTemplateDALFactory } from "@app/services/certificate-template/certificate-template-dal"; import { certificateTemplateEstConfigDALFactory } from "@app/services/certificate-template/certificate-template-est-config-dal"; import { certificateTemplateServiceFactory } from "@app/services/certificate-template/certificate-template-service"; +import { externalMigrationServiceFactory } from "@app/services/external-migration/external-migration-service"; import { groupProjectDALFactory } from "@app/services/group-project/group-project-dal"; import { groupProjectMembershipRoleDALFactory } from "@app/services/group-project/group-project-membership-role-dal"; import { groupProjectServiceFactory } from "@app/services/group-project/group-project-service"; @@ -1185,6 +1186,13 @@ export const registerRoutes = async ( workflowIntegrationDAL }); + const migrationService = externalMigrationServiceFactory({ + projectService, + orgService, + projectEnvService, + secretService + }); + await superAdminService.initServerCfg(); // // setup the communication with license key server @@ -1268,7 +1276,8 @@ export const registerRoutes = async ( externalKms: externalKmsService, orgAdmin: orgAdminService, slack: slackService, - workflowIntegration: workflowIntegrationService + workflowIntegration: workflowIntegrationService, + migration: migrationService }); const cronJobs: CronJob[] = []; diff --git a/backend/src/server/routes/v3/external-migration-router.ts b/backend/src/server/routes/v3/external-migration-router.ts new file mode 100644 index 000000000..f58b18d2b --- /dev/null +++ b/backend/src/server/routes/v3/external-migration-router.ts @@ -0,0 +1,42 @@ +import { z } from "zod"; + +import { readLimit } from "@app/server/config/rateLimiter"; +import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; +import { AuthMode } from "@app/services/auth/auth-type"; + +export const registerExternalMigrationRouter = async (server: FastifyZodProvider) => { + server.route({ + method: "POST", + url: "/envkey", + config: { + rateLimit: readLimit + }, + schema: { + body: z.object({ + decryptionKey: z.string().trim().min(1), + encryptedJson: z.object({ + nonce: z.string().trim().min(1), + data: z.string().trim().min(1) + }) + }), + response: { + 200: z.object({ + success: z.boolean(), + error: z.string().optional() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const result = await server.services.migration.importEnvnKeyData({ + decryptionKey: req.body.decryptionKey, + encryptedJson: req.body.encryptedJson, + actorId: req.permission.id, + actor: req.permission.type, + actorOrgId: req.permission.orgId, + actorAuthMethod: req.permission.authMethod + }); + return result; + } + }); +}; diff --git a/backend/src/server/routes/v3/index.ts b/backend/src/server/routes/v3/index.ts index 6a061a635..ed8401560 100644 --- a/backend/src/server/routes/v3/index.ts +++ b/backend/src/server/routes/v3/index.ts @@ -1,3 +1,4 @@ +import { registerExternalMigrationRouter } from "./external-migration-router"; import { registerLoginRouter } from "./login-router"; import { registerSecretBlindIndexRouter } from "./secret-blind-index-router"; import { registerSecretRouter } from "./secret-router"; @@ -10,4 +11,5 @@ export const registerV3Routes = async (server: FastifyZodProvider) => { await server.register(registerUserRouter, { prefix: "/users" }); await server.register(registerSecretRouter, { prefix: "/secrets" }); await server.register(registerSecretBlindIndexRouter, { prefix: "/workspaces" }); + await server.register(registerExternalMigrationRouter, { prefix: "/migrate" }); }; diff --git a/backend/src/services/external-migration/external-migration-fns.ts b/backend/src/services/external-migration/external-migration-fns.ts new file mode 100644 index 000000000..4da9278e3 --- /dev/null +++ b/backend/src/services/external-migration/external-migration-fns.ts @@ -0,0 +1,68 @@ +import { randomUUID } from "crypto"; +import { codec, hash } from "sjcl"; +import { secretbox } from "tweetnacl"; +import { decodeBase64, encodeUTF8 } from "tweetnacl-util"; + +import { InfisicalImportData, TEnvKeyExportJSON } from "./external-migration-types"; + +export const decryptEnvKeyData = async (decryptionKey: string, encryptedJson: { nonce: string; data: string }) => { + const key = decodeBase64(codec.base64.fromBits(hash.sha256.hash(decryptionKey))); + const nonce = decodeBase64(encryptedJson.nonce); + const encryptedData = decodeBase64(encryptedJson.data); + + const decrypted = secretbox.open(encryptedData, nonce, key); + + if (!decrypted) { + throw new Error("Decryption failed"); + } + + const decryptedJson = encodeUTF8(decrypted); + return decryptedJson; +}; + +export const parseEnvKeyData = async (decryptedJson: string): Promise => { + const parsedJson: TEnvKeyExportJSON = JSON.parse(decryptedJson) as TEnvKeyExportJSON; + + const infisicalImportData: InfisicalImportData = { + projects: new Map(), + environments: new Map(), + secrets: new Map() + }; + + parsedJson.apps.forEach((app: { name: string; id: string }) => { + infisicalImportData.projects?.set(app.id, { name: app.name, id: app.id }); + }); + + // string to string map for env templates + const envTemplates = new Map(); + for (const env of parsedJson.defaultEnvironmentRoles) { + envTemplates.set(env.id, env.defaultName); + } + + // environments + for (const env of parsedJson.baseEnvironments) { + infisicalImportData.environments?.set(env.id, { + id: env.id, + name: envTemplates.get(env.environmentRoleId)!, + projectId: env.envParentId + }); + } + + // secrets + for (const env of Object.keys(parsedJson.envs)) { + if (!env.includes("|")) { + const envData = parsedJson.envs[env]; + for (const secret of Object.keys(envData.variables)) { + const id = randomUUID(); + infisicalImportData.secrets?.set(id, { + id, + name: secret, + environmentId: env, + value: envData.variables[secret].val + }); + } + } + } + + return infisicalImportData; +}; diff --git a/backend/src/services/external-migration/external-migration-service.ts b/backend/src/services/external-migration/external-migration-service.ts new file mode 100644 index 000000000..e3da0b9eb --- /dev/null +++ b/backend/src/services/external-migration/external-migration-service.ts @@ -0,0 +1,185 @@ +import slugify from "@sindresorhus/slugify"; + +import { OrgMembershipRole, ProjectMembershipRole, SecretType } from "@app/db/schemas"; +import { BadRequestError } from "@app/lib/errors"; +import { logger } from "@app/lib/logger"; +import { alphaNumericNanoId } from "@app/lib/nanoid"; + +import { TOrgServiceFactory } from "../org/org-service"; +import { TProjectServiceFactory } from "../project/project-service"; +import { TProjectEnvServiceFactory } from "../project-env/project-env-service"; +import { TSecretServiceFactory } from "../secret/secret-service"; +import { decryptEnvKeyData, parseEnvKeyData } from "./external-migration-fns"; +import { TImportEnvKeyDataCreate, TImportInfisicalDataCreate } from "./external-migration-types"; + +type TExternalMigrationServiceFactoryDep = { + projectService: TProjectServiceFactory; + orgService: TOrgServiceFactory; + projectEnvService: TProjectEnvServiceFactory; + secretService: TSecretServiceFactory; +}; + +export type TExternalMigrationServiceFactory = ReturnType; + +export const externalMigrationServiceFactory = ({ + projectService, + orgService, + projectEnvService, + secretService +}: TExternalMigrationServiceFactoryDep) => { + const importInfisicalData = async ({ + data, + actor, + actorId, + actorOrgId, + actorAuthMethod + }: TImportInfisicalDataCreate) => { + // Import data to infisical + if (!data || !data.projects) { + logger.error("No projects found in data"); + return { + success: false, + message: "No projects found in data" + }; + } + + const orginalToNewProjectId = new Map(); + const orginalToNewEnvironmentId = new Map(); + + // Import projects + const projectPromises = []; + for (const [id, project] of data.projects) { + const projectPromise = projectService + .createProject({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + workspaceName: project?.name, + createDefaultEnvs: false + }) + .then((projectResponse) => { + if (!projectResponse) { + logger.error(`Failed to import project: [name:${project.name}] [id:${id}]`); + throw new Error(`Failed to import project: [name:${project.name}] [id:${id}]`); + } + orginalToNewProjectId.set(project.id, projectResponse.id); + }); + projectPromises.push(projectPromise); + } + await Promise.all(projectPromises); + + // Invite user importing projects + const response = await orgService.inviteUserToOrganization({ + actorAuthMethod, + actorId, + actorOrgId, + actor, + inviteeEmails: [], + orgId: actorOrgId, + organizationRoleSlug: OrgMembershipRole.NoAccess, + projects: Array.from(orginalToNewProjectId.values()).map((project) => { + return { + id: project, + projectRoleSlug: [ProjectMembershipRole.Member] + }; + }) + }); + if (!response) { + logger.error(`Failed to invite user to projects: [userId:${actorId}]`); + return { + success: false, + message: `Failed to invite user to project: [userId:${actorId}]` + }; + } + + // Import environments + if (data.environments) { + for (const [id, environment] of data.environments) { + try { + // TODO: we can create envs parallely once the position constraint is handled differently + // eslint-disable-next-line + const newEnvironment = await projectEnvService.createEnvironment({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + name: environment.name, + projectId: orginalToNewProjectId.get(environment.projectId)!, + slug: slugify(`${environment.name}-${alphaNumericNanoId(4)}`) + }); + + if (!newEnvironment) { + logger.error(`Failed to import environment: [name:${environment.name}] [id:${id}]`); + throw new BadRequestError({ + message: `Failed to import environment: [name:${environment.name}] [id:${id}]` + }); + } + orginalToNewEnvironmentId.set(id, newEnvironment.slug); + } catch (error) { + return { + success: false + }; + } + } + } + + // Import secrets + if (data.secrets) { + for (const [id, secret] of data.secrets) { + const dataProjectId = data.environments?.get(secret.environmentId)?.projectId; + if (!dataProjectId) { + logger.error(`Failed to import secret: [name:${secret.name}] [id:${id}], project not found`); + return { + success: false, + message: `Failed to import secret: [name:${secret.name}] [id:${id}], project not found` + }; + } + const projectId = orginalToNewProjectId.get(dataProjectId); + // TODO: we can create secrets parallely once the KMS ID bug on create is fixed + // eslint-disable-next-line + const newSecret = await secretService.createSecretRaw({ + actorId, + actor, + actorOrgId, + environment: orginalToNewEnvironmentId.get(secret.environmentId)!, + actorAuthMethod, + projectId: projectId!, + secretPath: "/", + secretName: secret.name, + type: SecretType.Shared, + secretValue: secret.value + }); + if (!newSecret) { + logger.error(`Failed to import secret: [name:${secret.name}] [id:${id}]`); + return { + success: false, + message: `Failed to import secret: [name:${secret.name}] [id:${id}]` + }; + } + } + } + + return { + success: true + }; + }; + + const importEnvnKeyData = async ({ + decryptionKey, + encryptedJson, + actor, + actorId, + actorOrgId, + actorAuthMethod + }: TImportEnvKeyDataCreate) => { + const json = await decryptEnvKeyData(decryptionKey, encryptedJson); + const envKeyData = await parseEnvKeyData(json); + const response = await importInfisicalData({ data: envKeyData, actor, actorId, actorOrgId, actorAuthMethod }); + return response; + }; + + return { + importEnvnKeyData + }; +}; diff --git a/backend/src/services/external-migration/external-migration-types.ts b/backend/src/services/external-migration/external-migration-types.ts new file mode 100644 index 000000000..ee8a1453e --- /dev/null +++ b/backend/src/services/external-migration/external-migration-types.ts @@ -0,0 +1,106 @@ +import { ActorAuthMethod, ActorType } from "../auth/auth-type"; + +export type InfisicalImportData = { + projects?: Map; + + environments?: Map< + string, + { + name: string; + id: string; + projectId: string; + } + >; + + secrets?: Map< + string, + { + name: string; + id: string; + environmentId: string; + value: string; + } + >; +}; + +export type TImportEnvKeyDataCreate = { + decryptionKey: string; + encryptedJson: { nonce: string; data: string }; + actor: ActorType; + actorId: string; + actorOrgId: string; + actorAuthMethod: ActorAuthMethod; +}; + +export type TImportInfisicalDataCreate = { + data: InfisicalImportData; + actor: ActorType; + actorId: string; + actorOrgId: string; + actorAuthMethod: ActorAuthMethod; +}; + +export type TEnvKeyExportJSON = { + schemaVersion: string; + org: { + id: string; + name: string; + settings: { + auth: { + inviteExpirationMs: number; + deviceGrantExpirationMs: number; + tokenExpirationMs: number; + }; + crypto: { + requiresPassphrase: boolean; + requiresLockout: boolean; + }; + envs: { + autoCaps: boolean; + autoCommitLocals: boolean; + }; + }; + }; + apps: { + id: string; + name: string; + settings: Record; + }[]; + defaultOrgRoles: { + id: string; + defaultName: string; + }[]; + defaultAppRoles: { + id: string; + defaultName: string; + }[]; + defaultEnvironmentRoles: { + id: string; + defaultName: string; + settings: { + autoCommit: boolean; + }; + }[]; + baseEnvironments: { + id: string; + envParentId: string; + environmentRoleId: string; + settings: Record; + }[]; + orgUsers: { + id: string; + firstName: string; + lastName: string; + email: string; + provider: string; + orgRoleId: string; + uid: string; + }[]; + envs: Record< + string, + { + variables: Record; + inherits: Record; + } + >; +}; diff --git a/backend/src/services/project/project-service.ts b/backend/src/services/project/project-service.ts index 863cf24c2..efb958e31 100644 --- a/backend/src/services/project/project-service.ts +++ b/backend/src/services/project/project-service.ts @@ -146,7 +146,8 @@ export const projectServiceFactory = ({ actorAuthMethod, workspaceName, slug: projectSlug, - kmsKeyId + kmsKeyId, + createDefaultEnvs = true }: TCreateProjectDTO) => { const organization = await orgDAL.findOne({ id: actorOrgId }); @@ -207,14 +208,17 @@ export const projectServiceFactory = ({ ); // set default environments and root folder for provided environments - const envs = await projectEnvDAL.insertMany( - DEFAULT_PROJECT_ENVS.map((el, i) => ({ ...el, projectId: project.id, position: i + 1 })), - tx - ); - await folderDAL.insertMany( - envs.map(({ id }) => ({ name: ROOT_FOLDER_NAME, envId: id, version: 1 })), - tx - ); + let envs; + if (createDefaultEnvs) { + envs = await projectEnvDAL.insertMany( + DEFAULT_PROJECT_ENVS.map((el, i) => ({ ...el, projectId: project.id, position: i + 1 })), + tx + ); + await folderDAL.insertMany( + envs.map(({ id }) => ({ name: ROOT_FOLDER_NAME, envId: id, version: 1 })), + tx + ); + } // 3. Create a random key that we'll use as the project key. const { key: encryptedProjectKey, iv: encryptedProjectKeyIv } = createProjectKey({ @@ -360,7 +364,7 @@ export const projectServiceFactory = ({ return { ...project, - environments: envs, + environments: envs || [], _id: project.id }; }); diff --git a/backend/src/services/project/project-types.ts b/backend/src/services/project/project-types.ts index 3bfdae709..7193f1121 100644 --- a/backend/src/services/project/project-types.ts +++ b/backend/src/services/project/project-types.ts @@ -29,6 +29,7 @@ export type TCreateProjectDTO = { workspaceName: string; slug?: string; kmsKeyId?: string; + createDefaultEnvs?: boolean; }; export type TDeleteProjectBySlugDTO = {