feat: add migration service to import from envkey

backend/package-lock.json

@@ -37,6 +37,7 @@
         "@slack/oauth": "^3.0.1",
         "@slack/web-api": "^7.3.4",
         "@team-plain/typescript-sdk": "^4.6.1",
+        "@types/sjcl": "^1.0.34",
         "@ucast/mongo2js": "^1.3.4",
         "ajv": "^8.12.0",
         "argon2": "^0.31.2",
@@ -85,6 +86,7 @@
         "safe-regex": "^2.1.1",
         "scim-patch": "^0.8.3",
         "scim2-parse-filter": "^0.2.10",
+        "sjcl": "^1.0.8",
         "smee-client": "^2.0.0",
         "tedious": "^18.2.1",
         "tweetnacl": "^1.0.3",
@@ -7296,6 +7298,12 @@
         "@types/node": "*"
       }
     },
+    "node_modules/@types/sjcl": {
+      "version": "1.0.34",
+      "resolved": "https://registry.npmjs.org/@types/sjcl/-/sjcl-1.0.34.tgz",
+      "integrity": "sha512-bQHEeK5DTQRunIfQeUMgtpPsNNCcZyQ9MJuAfW1I7iN0LDunTc78Fu17STbLMd7KiEY/g2zHVApippa70h6HoQ==",
+      "license": "MIT"
+    },
     "node_modules/@types/uuid": {
       "version": "9.0.7",
       "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.7.tgz",
@@ -16397,6 +16405,15 @@
         "node": ">=10"
       }
     },
+    "node_modules/sjcl": {
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/sjcl/-/sjcl-1.0.8.tgz",
+      "integrity": "sha512-LzIjEQ0S0DpIgnxMEayM1rq9aGwGRG4OnZhCdjx7glTaJtf4zRfpg87ImfjSJjoW9vKpagd82McDOwbRT5kQKQ==",
+      "license": "(BSD-2-Clause OR GPL-2.0-only)",
+      "engines": {
+        "node": "*"
+      }
+    },
     "node_modules/slash": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz",
@@ -17874,12 +17891,14 @@
     "node_modules/tweetnacl": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-1.0.3.tgz",
-      "integrity": "sha512-6rt+RN7aOi1nGMyC4Xa5DdYiukl2UWCbcJft7YhxReBGQD7OAM8Pbxw6YMo4r2diNEA8FEmu32YOn9rhaiE5yw=="
+      "integrity": "sha512-6rt+RN7aOi1nGMyC4Xa5DdYiukl2UWCbcJft7YhxReBGQD7OAM8Pbxw6YMo4r2diNEA8FEmu32YOn9rhaiE5yw==",
+      "license": "Unlicense"
     },
     "node_modules/tweetnacl-util": {
       "version": "0.15.1",
       "resolved": "https://registry.npmjs.org/tweetnacl-util/-/tweetnacl-util-0.15.1.tgz",
-      "integrity": "sha512-RKJBIj8lySrShN4w6i/BonWp2Z/uxwC3h4y7xsRrpP59ZboCd0GpEVsOnMDYLMmKBpYhb5TgHzZXy7wTfYFBRw=="
+      "integrity": "sha512-RKJBIj8lySrShN4w6i/BonWp2Z/uxwC3h4y7xsRrpP59ZboCd0GpEVsOnMDYLMmKBpYhb5TgHzZXy7wTfYFBRw==",
+      "license": "Unlicense"
     },
     "node_modules/type-check": {
       "version": "0.4.0",

backend/package.json

@@ -134,6 +134,7 @@
     "@slack/oauth": "^3.0.1",
     "@slack/web-api": "^7.3.4",
     "@team-plain/typescript-sdk": "^4.6.1",
+    "@types/sjcl": "^1.0.34",
     "@ucast/mongo2js": "^1.3.4",
     "ajv": "^8.12.0",
     "argon2": "^0.31.2",
@@ -182,6 +183,7 @@
     "safe-regex": "^2.1.1",
     "scim-patch": "^0.8.3",
     "scim2-parse-filter": "^0.2.10",
+    "sjcl": "^1.0.8",
     "smee-client": "^2.0.0",
     "tedious": "^18.2.1",
     "tweetnacl": "^1.0.3",

backend/src/@types/fastify.d.ts

@@ -38,6 +38,7 @@ import { TAuthTokenServiceFactory } from "@app/services/auth-token/auth-token-se
 import { TCertificateServiceFactory } from "@app/services/certificate/certificate-service";
 import { TCertificateAuthorityServiceFactory } from "@app/services/certificate-authority/certificate-authority-service";
 import { TCertificateTemplateServiceFactory } from "@app/services/certificate-template/certificate-template-service";
+import { TExternalMigrationServiceFactory } from "@app/services/external-migration/external-migration-service";
 import { TGroupProjectServiceFactory } from "@app/services/group-project/group-project-service";
 import { TIdentityServiceFactory } from "@app/services/identity/identity-service";
 import { TIdentityAccessTokenServiceFactory } from "@app/services/identity-access-token/identity-access-token-service";
@@ -181,6 +182,7 @@ declare module "fastify" {
       orgAdmin: TOrgAdminServiceFactory;
       slack: TSlackServiceFactory;
       workflowIntegration: TWorkflowIntegrationServiceFactory;
+      migration: TExternalMigrationServiceFactory;
     };
     // this is exclusive use for middlewares in which we need to inject data
     // everywhere else access using service layer

backend/src/server/routes/index.ts

@@ -96,6 +96,7 @@ import { certificateAuthorityServiceFactory } from "@app/services/certificate-au
 import { certificateTemplateDALFactory } from "@app/services/certificate-template/certificate-template-dal";
 import { certificateTemplateEstConfigDALFactory } from "@app/services/certificate-template/certificate-template-est-config-dal";
 import { certificateTemplateServiceFactory } from "@app/services/certificate-template/certificate-template-service";
+import { externalMigrationServiceFactory } from "@app/services/external-migration/external-migration-service";
 import { groupProjectDALFactory } from "@app/services/group-project/group-project-dal";
 import { groupProjectMembershipRoleDALFactory } from "@app/services/group-project/group-project-membership-role-dal";
 import { groupProjectServiceFactory } from "@app/services/group-project/group-project-service";
@@ -1185,6 +1186,13 @@ export const registerRoutes = async (
     workflowIntegrationDAL
   });
 
+  const migrationService = externalMigrationServiceFactory({
+    projectService,
+    orgService,
+    projectEnvService,
+    secretService
+  });
+
   await superAdminService.initServerCfg();
   //
   // setup the communication with license key server
@@ -1268,7 +1276,8 @@ export const registerRoutes = async (
     externalKms: externalKmsService,
     orgAdmin: orgAdminService,
     slack: slackService,
-    workflowIntegration: workflowIntegrationService
+    workflowIntegration: workflowIntegrationService,
+    migration: migrationService
   });
 
   const cronJobs: CronJob[] = [];

backend/src/server/routes/v3/external-migration-router.ts

@@ -0,0 +1,42 @@
+import { z } from "zod";
+
+import { readLimit } from "@app/server/config/rateLimiter";
+import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
+import { AuthMode } from "@app/services/auth/auth-type";
+
+export const registerExternalMigrationRouter = async (server: FastifyZodProvider) => {
+  server.route({
+    method: "POST",
+    url: "/envkey",
+    config: {
+      rateLimit: readLimit
+    },
+    schema: {
+      body: z.object({
+        decryptionKey: z.string().trim().min(1),
+        encryptedJson: z.object({
+          nonce: z.string().trim().min(1),
+          data: z.string().trim().min(1)
+        })
+      }),
+      response: {
+        200: z.object({
+          success: z.boolean(),
+          error: z.string().optional()
+        })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
+    handler: async (req) => {
+      const result = await server.services.migration.importEnvnKeyData({
+        decryptionKey: req.body.decryptionKey,
+        encryptedJson: req.body.encryptedJson,
+        actorId: req.permission.id,
+        actor: req.permission.type,
+        actorOrgId: req.permission.orgId,
+        actorAuthMethod: req.permission.authMethod
+      });
+      return result;
+    }
+  });
+};

backend/src/server/routes/v3/index.ts

@@ -1,3 +1,4 @@
+import { registerExternalMigrationRouter } from "./external-migration-router";
 import { registerLoginRouter } from "./login-router";
 import { registerSecretBlindIndexRouter } from "./secret-blind-index-router";
 import { registerSecretRouter } from "./secret-router";
@@ -10,4 +11,5 @@ export const registerV3Routes = async (server: FastifyZodProvider) => {
   await server.register(registerUserRouter, { prefix: "/users" });
   await server.register(registerSecretRouter, { prefix: "/secrets" });
   await server.register(registerSecretBlindIndexRouter, { prefix: "/workspaces" });
+  await server.register(registerExternalMigrationRouter, { prefix: "/migrate" });
 };

backend/src/services/external-migration/external-migration-fns.ts

@@ -0,0 +1,68 @@
+import { randomUUID } from "crypto";
+import { codec, hash } from "sjcl";
+import { secretbox } from "tweetnacl";
+import { decodeBase64, encodeUTF8 } from "tweetnacl-util";
+
+import { InfisicalImportData, TEnvKeyExportJSON } from "./external-migration-types";
+
+export const decryptEnvKeyData = async (decryptionKey: string, encryptedJson: { nonce: string; data: string }) => {
+  const key = decodeBase64(codec.base64.fromBits(hash.sha256.hash(decryptionKey)));
+  const nonce = decodeBase64(encryptedJson.nonce);
+  const encryptedData = decodeBase64(encryptedJson.data);
+
+  const decrypted = secretbox.open(encryptedData, nonce, key);
+
+  if (!decrypted) {
+    throw new Error("Decryption failed");
+  }
+
+  const decryptedJson = encodeUTF8(decrypted);
+  return decryptedJson;
+};
+
+export const parseEnvKeyData = async (decryptedJson: string): Promise<InfisicalImportData> => {
+  const parsedJson: TEnvKeyExportJSON = JSON.parse(decryptedJson) as TEnvKeyExportJSON;
+
+  const infisicalImportData: InfisicalImportData = {
+    projects: new Map<string, { name: string; id: string }>(),
+    environments: new Map<string, { name: string; id: string; projectId: string }>(),
+    secrets: new Map<string, { name: string; id: string; projectId: string; environmentId: string; value: string }>()
+  };
+
+  parsedJson.apps.forEach((app: { name: string; id: string }) => {
+    infisicalImportData.projects?.set(app.id, { name: app.name, id: app.id });
+  });
+
+  // string to string map for env templates
+  const envTemplates = new Map<string, string>();
+  for (const env of parsedJson.defaultEnvironmentRoles) {
+    envTemplates.set(env.id, env.defaultName);
+  }
+
+  // environments
+  for (const env of parsedJson.baseEnvironments) {
+    infisicalImportData.environments?.set(env.id, {
+      id: env.id,
+      name: envTemplates.get(env.environmentRoleId)!,
+      projectId: env.envParentId
+    });
+  }
+
+  // secrets
+  for (const env of Object.keys(parsedJson.envs)) {
+    if (!env.includes("|")) {
+      const envData = parsedJson.envs[env];
+      for (const secret of Object.keys(envData.variables)) {
+        const id = randomUUID();
+        infisicalImportData.secrets?.set(id, {
+          id,
+          name: secret,
+          environmentId: env,
+          value: envData.variables[secret].val
+        });
+      }
+    }
+  }
+
+  return infisicalImportData;
+};

backend/src/services/external-migration/external-migration-service.ts

@@ -0,0 +1,185 @@
+import slugify from "@sindresorhus/slugify";
+
+import { OrgMembershipRole, ProjectMembershipRole, SecretType } from "@app/db/schemas";
+import { BadRequestError } from "@app/lib/errors";
+import { logger } from "@app/lib/logger";
+import { alphaNumericNanoId } from "@app/lib/nanoid";
+
+import { TOrgServiceFactory } from "../org/org-service";
+import { TProjectServiceFactory } from "../project/project-service";
+import { TProjectEnvServiceFactory } from "../project-env/project-env-service";
+import { TSecretServiceFactory } from "../secret/secret-service";
+import { decryptEnvKeyData, parseEnvKeyData } from "./external-migration-fns";
+import { TImportEnvKeyDataCreate, TImportInfisicalDataCreate } from "./external-migration-types";
+
+type TExternalMigrationServiceFactoryDep = {
+  projectService: TProjectServiceFactory;
+  orgService: TOrgServiceFactory;
+  projectEnvService: TProjectEnvServiceFactory;
+  secretService: TSecretServiceFactory;
+};
+
+export type TExternalMigrationServiceFactory = ReturnType<typeof externalMigrationServiceFactory>;
+
+export const externalMigrationServiceFactory = ({
+  projectService,
+  orgService,
+  projectEnvService,
+  secretService
+}: TExternalMigrationServiceFactoryDep) => {
+  const importInfisicalData = async ({
+    data,
+    actor,
+    actorId,
+    actorOrgId,
+    actorAuthMethod
+  }: TImportInfisicalDataCreate) => {
+    // Import data to infisical
+    if (!data || !data.projects) {
+      logger.error("No projects found in data");
+      return {
+        success: false,
+        message: "No projects found in data"
+      };
+    }
+
+    const orginalToNewProjectId = new Map<string, string>();
+    const orginalToNewEnvironmentId = new Map<string, string>();
+
+    // Import projects
+    const projectPromises = [];
+    for (const [id, project] of data.projects) {
+      const projectPromise = projectService
+        .createProject({
+          actor,
+          actorId,
+          actorOrgId,
+          actorAuthMethod,
+          workspaceName: project?.name,
+          createDefaultEnvs: false
+        })
+        .then((projectResponse) => {
+          if (!projectResponse) {
+            logger.error(`Failed to import project: [name:${project.name}] [id:${id}]`);
+            throw new Error(`Failed to import project: [name:${project.name}] [id:${id}]`);
+          }
+          orginalToNewProjectId.set(project.id, projectResponse.id);
+        });
+      projectPromises.push(projectPromise);
+    }
+    await Promise.all(projectPromises);
+
+    // Invite user importing projects
+    const response = await orgService.inviteUserToOrganization({
+      actorAuthMethod,
+      actorId,
+      actorOrgId,
+      actor,
+      inviteeEmails: [],
+      orgId: actorOrgId,
+      organizationRoleSlug: OrgMembershipRole.NoAccess,
+      projects: Array.from(orginalToNewProjectId.values()).map((project) => {
+        return {
+          id: project,
+          projectRoleSlug: [ProjectMembershipRole.Member]
+        };
+      })
+    });
+    if (!response) {
+      logger.error(`Failed to invite user to projects: [userId:${actorId}]`);
+      return {
+        success: false,
+        message: `Failed to invite user to project: [userId:${actorId}]`
+      };
+    }
+
+    // Import environments
+    if (data.environments) {
+      for (const [id, environment] of data.environments) {
+        try {
+          // TODO: we can create envs parallely once the position constraint is handled differently
+          // eslint-disable-next-line
+          const newEnvironment = await projectEnvService.createEnvironment({
+            actor,
+            actorId,
+            actorOrgId,
+            actorAuthMethod,
+            name: environment.name,
+            projectId: orginalToNewProjectId.get(environment.projectId)!,
+            slug: slugify(`${environment.name}-${alphaNumericNanoId(4)}`)
+          });
+
+          if (!newEnvironment) {
+            logger.error(`Failed to import environment: [name:${environment.name}] [id:${id}]`);
+            throw new BadRequestError({
+              message: `Failed to import environment: [name:${environment.name}] [id:${id}]`
+            });
+          }
+          orginalToNewEnvironmentId.set(id, newEnvironment.slug);
+        } catch (error) {
+          return {
+            success: false
+          };
+        }
+      }
+    }
+
+    // Import secrets
+    if (data.secrets) {
+      for (const [id, secret] of data.secrets) {
+        const dataProjectId = data.environments?.get(secret.environmentId)?.projectId;
+        if (!dataProjectId) {
+          logger.error(`Failed to import secret: [name:${secret.name}] [id:${id}], project not found`);
+          return {
+            success: false,
+            message: `Failed to import secret: [name:${secret.name}] [id:${id}], project not found`
+          };
+        }
+        const projectId = orginalToNewProjectId.get(dataProjectId);
+        // TODO: we can create secrets parallely once the KMS ID bug on create is fixed
+        // eslint-disable-next-line
+        const newSecret = await secretService.createSecretRaw({
+          actorId,
+          actor,
+          actorOrgId,
+          environment: orginalToNewEnvironmentId.get(secret.environmentId)!,
+          actorAuthMethod,
+          projectId: projectId!,
+          secretPath: "/",
+          secretName: secret.name,
+          type: SecretType.Shared,
+          secretValue: secret.value
+        });
+        if (!newSecret) {
+          logger.error(`Failed to import secret: [name:${secret.name}] [id:${id}]`);
+          return {
+            success: false,
+            message: `Failed to import secret: [name:${secret.name}] [id:${id}]`
+          };
+        }
+      }
+    }
+
+    return {
+      success: true
+    };
+  };
+
+  const importEnvnKeyData = async ({
+    decryptionKey,
+    encryptedJson,
+    actor,
+    actorId,
+    actorOrgId,
+    actorAuthMethod
+  }: TImportEnvKeyDataCreate) => {
+    const json = await decryptEnvKeyData(decryptionKey, encryptedJson);
+    const envKeyData = await parseEnvKeyData(json);
+    const response = await importInfisicalData({ data: envKeyData, actor, actorId, actorOrgId, actorAuthMethod });
+    return response;
+  };
+
+  return {
+    importEnvnKeyData
+  };
+};

backend/src/services/external-migration/external-migration-types.ts

@@ -0,0 +1,106 @@
+import { ActorAuthMethod, ActorType } from "../auth/auth-type";
+
+export type InfisicalImportData = {
+  projects?: Map<string, { name: string; id: string }>;
+
+  environments?: Map<
+    string,
+    {
+      name: string;
+      id: string;
+      projectId: string;
+    }
+  >;
+
+  secrets?: Map<
+    string,
+    {
+      name: string;
+      id: string;
+      environmentId: string;
+      value: string;
+    }
+  >;
+};
+
+export type TImportEnvKeyDataCreate = {
+  decryptionKey: string;
+  encryptedJson: { nonce: string; data: string };
+  actor: ActorType;
+  actorId: string;
+  actorOrgId: string;
+  actorAuthMethod: ActorAuthMethod;
+};
+
+export type TImportInfisicalDataCreate = {
+  data: InfisicalImportData;
+  actor: ActorType;
+  actorId: string;
+  actorOrgId: string;
+  actorAuthMethod: ActorAuthMethod;
+};
+
+export type TEnvKeyExportJSON = {
+  schemaVersion: string;
+  org: {
+    id: string;
+    name: string;
+    settings: {
+      auth: {
+        inviteExpirationMs: number;
+        deviceGrantExpirationMs: number;
+        tokenExpirationMs: number;
+      };
+      crypto: {
+        requiresPassphrase: boolean;
+        requiresLockout: boolean;
+      };
+      envs: {
+        autoCaps: boolean;
+        autoCommitLocals: boolean;
+      };
+    };
+  };
+  apps: {
+    id: string;
+    name: string;
+    settings: Record<string, unknown>;
+  }[];
+  defaultOrgRoles: {
+    id: string;
+    defaultName: string;
+  }[];
+  defaultAppRoles: {
+    id: string;
+    defaultName: string;
+  }[];
+  defaultEnvironmentRoles: {
+    id: string;
+    defaultName: string;
+    settings: {
+      autoCommit: boolean;
+    };
+  }[];
+  baseEnvironments: {
+    id: string;
+    envParentId: string;
+    environmentRoleId: string;
+    settings: Record<string, unknown>;
+  }[];
+  orgUsers: {
+    id: string;
+    firstName: string;
+    lastName: string;
+    email: string;
+    provider: string;
+    orgRoleId: string;
+    uid: string;
+  }[];
+  envs: Record<
+    string,
+    {
+      variables: Record<string, { val: string }>;
+      inherits: Record<string, unknown>;
+    }
+  >;
+};

backend/src/services/project/project-service.ts

@@ -146,7 +146,8 @@ export const projectServiceFactory = ({
     actorAuthMethod,
     workspaceName,
     slug: projectSlug,
-    kmsKeyId
+    kmsKeyId,
+    createDefaultEnvs = true
   }: TCreateProjectDTO) => {
     const organization = await orgDAL.findOne({ id: actorOrgId });
 
@@ -207,14 +208,17 @@ export const projectServiceFactory = ({
       );
 
       // set default environments and root folder for provided environments
-      const envs = await projectEnvDAL.insertMany(
-        DEFAULT_PROJECT_ENVS.map((el, i) => ({ ...el, projectId: project.id, position: i + 1 })),
-        tx
-      );
-      await folderDAL.insertMany(
-        envs.map(({ id }) => ({ name: ROOT_FOLDER_NAME, envId: id, version: 1 })),
-        tx
-      );
+      let envs;
+      if (createDefaultEnvs) {
+        envs = await projectEnvDAL.insertMany(
+          DEFAULT_PROJECT_ENVS.map((el, i) => ({ ...el, projectId: project.id, position: i + 1 })),
+          tx
+        );
+        await folderDAL.insertMany(
+          envs.map(({ id }) => ({ name: ROOT_FOLDER_NAME, envId: id, version: 1 })),
+          tx
+        );
+      }
 
       // 3. Create a random key that we'll use as the project key.
       const { key: encryptedProjectKey, iv: encryptedProjectKeyIv } = createProjectKey({
@@ -360,7 +364,7 @@ export const projectServiceFactory = ({
 
       return {
         ...project,
-        environments: envs,
+        environments: envs || [],
         _id: project.id
       };
     });

backend/src/services/project/project-types.ts

@@ -29,6 +29,7 @@ export type TCreateProjectDTO = {
   workspaceName: string;
   slug?: string;
   kmsKeyId?: string;
+  createDefaultEnvs?: boolean;
 };
 
 export type TDeleteProjectBySlugDTO = {