feat: add migration service to import from envkey

backend/src/services/external-migration/external-migration-fns.ts

@@ -0,0 +1,68 @@
+import { randomUUID } from "crypto";
+import { codec, hash } from "sjcl";
+import { secretbox } from "tweetnacl";
+import { decodeBase64, encodeUTF8 } from "tweetnacl-util";
+
+import { InfisicalImportData, TEnvKeyExportJSON } from "./external-migration-types";
+
+export const decryptEnvKeyData = async (decryptionKey: string, encryptedJson: { nonce: string; data: string }) => {
+  const key = decodeBase64(codec.base64.fromBits(hash.sha256.hash(decryptionKey)));
+  const nonce = decodeBase64(encryptedJson.nonce);
+  const encryptedData = decodeBase64(encryptedJson.data);
+
+  const decrypted = secretbox.open(encryptedData, nonce, key);
+
+  if (!decrypted) {
+    throw new Error("Decryption failed");
+  }
+
+  const decryptedJson = encodeUTF8(decrypted);
+  return decryptedJson;
+};
+
+export const parseEnvKeyData = async (decryptedJson: string): Promise<InfisicalImportData> => {
+  const parsedJson: TEnvKeyExportJSON = JSON.parse(decryptedJson) as TEnvKeyExportJSON;
+
+  const infisicalImportData: InfisicalImportData = {
+    projects: new Map<string, { name: string; id: string }>(),
+    environments: new Map<string, { name: string; id: string; projectId: string }>(),
+    secrets: new Map<string, { name: string; id: string; projectId: string; environmentId: string; value: string }>()
+  };
+
+  parsedJson.apps.forEach((app: { name: string; id: string }) => {
+    infisicalImportData.projects?.set(app.id, { name: app.name, id: app.id });
+  });
+
+  // string to string map for env templates
+  const envTemplates = new Map<string, string>();
+  for (const env of parsedJson.defaultEnvironmentRoles) {
+    envTemplates.set(env.id, env.defaultName);
+  }
+
+  // environments
+  for (const env of parsedJson.baseEnvironments) {
+    infisicalImportData.environments?.set(env.id, {
+      id: env.id,
+      name: envTemplates.get(env.environmentRoleId)!,
+      projectId: env.envParentId
+    });
+  }
+
+  // secrets
+  for (const env of Object.keys(parsedJson.envs)) {
+    if (!env.includes("|")) {
+      const envData = parsedJson.envs[env];
+      for (const secret of Object.keys(envData.variables)) {
+        const id = randomUUID();
+        infisicalImportData.secrets?.set(id, {
+          id,
+          name: secret,
+          environmentId: env,
+          value: envData.variables[secret].val
+        });
+      }
+    }
+  }
+
+  return infisicalImportData;
+};

backend/src/services/external-migration/external-migration-service.ts

@@ -0,0 +1,185 @@
+import slugify from "@sindresorhus/slugify";
+
+import { OrgMembershipRole, ProjectMembershipRole, SecretType } from "@app/db/schemas";
+import { BadRequestError } from "@app/lib/errors";
+import { logger } from "@app/lib/logger";
+import { alphaNumericNanoId } from "@app/lib/nanoid";
+
+import { TOrgServiceFactory } from "../org/org-service";
+import { TProjectServiceFactory } from "../project/project-service";
+import { TProjectEnvServiceFactory } from "../project-env/project-env-service";
+import { TSecretServiceFactory } from "../secret/secret-service";
+import { decryptEnvKeyData, parseEnvKeyData } from "./external-migration-fns";
+import { TImportEnvKeyDataCreate, TImportInfisicalDataCreate } from "./external-migration-types";
+
+type TExternalMigrationServiceFactoryDep = {
+  projectService: TProjectServiceFactory;
+  orgService: TOrgServiceFactory;
+  projectEnvService: TProjectEnvServiceFactory;
+  secretService: TSecretServiceFactory;
+};
+
+export type TExternalMigrationServiceFactory = ReturnType<typeof externalMigrationServiceFactory>;
+
+export const externalMigrationServiceFactory = ({
+  projectService,
+  orgService,
+  projectEnvService,
+  secretService
+}: TExternalMigrationServiceFactoryDep) => {
+  const importInfisicalData = async ({
+    data,
+    actor,
+    actorId,
+    actorOrgId,
+    actorAuthMethod
+  }: TImportInfisicalDataCreate) => {
+    // Import data to infisical
+    if (!data || !data.projects) {
+      logger.error("No projects found in data");
+      return {
+        success: false,
+        message: "No projects found in data"
+      };
+    }
+
+    const orginalToNewProjectId = new Map<string, string>();
+    const orginalToNewEnvironmentId = new Map<string, string>();
+
+    // Import projects
+    const projectPromises = [];
+    for (const [id, project] of data.projects) {
+      const projectPromise = projectService
+        .createProject({
+          actor,
+          actorId,
+          actorOrgId,
+          actorAuthMethod,
+          workspaceName: project?.name,
+          createDefaultEnvs: false
+        })
+        .then((projectResponse) => {
+          if (!projectResponse) {
+            logger.error(`Failed to import project: [name:${project.name}] [id:${id}]`);
+            throw new Error(`Failed to import project: [name:${project.name}] [id:${id}]`);
+          }
+          orginalToNewProjectId.set(project.id, projectResponse.id);
+        });
+      projectPromises.push(projectPromise);
+    }
+    await Promise.all(projectPromises);
+
+    // Invite user importing projects
+    const response = await orgService.inviteUserToOrganization({
+      actorAuthMethod,
+      actorId,
+      actorOrgId,
+      actor,
+      inviteeEmails: [],
+      orgId: actorOrgId,
+      organizationRoleSlug: OrgMembershipRole.NoAccess,
+      projects: Array.from(orginalToNewProjectId.values()).map((project) => {
+        return {
+          id: project,
+          projectRoleSlug: [ProjectMembershipRole.Member]
+        };
+      })
+    });
+    if (!response) {
+      logger.error(`Failed to invite user to projects: [userId:${actorId}]`);
+      return {
+        success: false,
+        message: `Failed to invite user to project: [userId:${actorId}]`
+      };
+    }
+
+    // Import environments
+    if (data.environments) {
+      for (const [id, environment] of data.environments) {
+        try {
+          // TODO: we can create envs parallely once the position constraint is handled differently
+          // eslint-disable-next-line
+          const newEnvironment = await projectEnvService.createEnvironment({
+            actor,
+            actorId,
+            actorOrgId,
+            actorAuthMethod,
+            name: environment.name,
+            projectId: orginalToNewProjectId.get(environment.projectId)!,
+            slug: slugify(`${environment.name}-${alphaNumericNanoId(4)}`)
+          });
+
+          if (!newEnvironment) {
+            logger.error(`Failed to import environment: [name:${environment.name}] [id:${id}]`);
+            throw new BadRequestError({
+              message: `Failed to import environment: [name:${environment.name}] [id:${id}]`
+            });
+          }
+          orginalToNewEnvironmentId.set(id, newEnvironment.slug);
+        } catch (error) {
+          return {
+            success: false
+          };
+        }
+      }
+    }
+
+    // Import secrets
+    if (data.secrets) {
+      for (const [id, secret] of data.secrets) {
+        const dataProjectId = data.environments?.get(secret.environmentId)?.projectId;
+        if (!dataProjectId) {
+          logger.error(`Failed to import secret: [name:${secret.name}] [id:${id}], project not found`);
+          return {
+            success: false,
+            message: `Failed to import secret: [name:${secret.name}] [id:${id}], project not found`
+          };
+        }
+        const projectId = orginalToNewProjectId.get(dataProjectId);
+        // TODO: we can create secrets parallely once the KMS ID bug on create is fixed
+        // eslint-disable-next-line
+        const newSecret = await secretService.createSecretRaw({
+          actorId,
+          actor,
+          actorOrgId,
+          environment: orginalToNewEnvironmentId.get(secret.environmentId)!,
+          actorAuthMethod,
+          projectId: projectId!,
+          secretPath: "/",
+          secretName: secret.name,
+          type: SecretType.Shared,
+          secretValue: secret.value
+        });
+        if (!newSecret) {
+          logger.error(`Failed to import secret: [name:${secret.name}] [id:${id}]`);
+          return {
+            success: false,
+            message: `Failed to import secret: [name:${secret.name}] [id:${id}]`
+          };
+        }
+      }
+    }
+
+    return {
+      success: true
+    };
+  };
+
+  const importEnvnKeyData = async ({
+    decryptionKey,
+    encryptedJson,
+    actor,
+    actorId,
+    actorOrgId,
+    actorAuthMethod
+  }: TImportEnvKeyDataCreate) => {
+    const json = await decryptEnvKeyData(decryptionKey, encryptedJson);
+    const envKeyData = await parseEnvKeyData(json);
+    const response = await importInfisicalData({ data: envKeyData, actor, actorId, actorOrgId, actorAuthMethod });
+    return response;
+  };
+
+  return {
+    importEnvnKeyData
+  };
+};

backend/src/services/external-migration/external-migration-types.ts

@@ -0,0 +1,106 @@
+import { ActorAuthMethod, ActorType } from "../auth/auth-type";
+
+export type InfisicalImportData = {
+  projects?: Map<string, { name: string; id: string }>;
+
+  environments?: Map<
+    string,
+    {
+      name: string;
+      id: string;
+      projectId: string;
+    }
+  >;
+
+  secrets?: Map<
+    string,
+    {
+      name: string;
+      id: string;
+      environmentId: string;
+      value: string;
+    }
+  >;
+};
+
+export type TImportEnvKeyDataCreate = {
+  decryptionKey: string;
+  encryptedJson: { nonce: string; data: string };
+  actor: ActorType;
+  actorId: string;
+  actorOrgId: string;
+  actorAuthMethod: ActorAuthMethod;
+};
+
+export type TImportInfisicalDataCreate = {
+  data: InfisicalImportData;
+  actor: ActorType;
+  actorId: string;
+  actorOrgId: string;
+  actorAuthMethod: ActorAuthMethod;
+};
+
+export type TEnvKeyExportJSON = {
+  schemaVersion: string;
+  org: {
+    id: string;
+    name: string;
+    settings: {
+      auth: {
+        inviteExpirationMs: number;
+        deviceGrantExpirationMs: number;
+        tokenExpirationMs: number;
+      };
+      crypto: {
+        requiresPassphrase: boolean;
+        requiresLockout: boolean;
+      };
+      envs: {
+        autoCaps: boolean;
+        autoCommitLocals: boolean;
+      };
+    };
+  };
+  apps: {
+    id: string;
+    name: string;
+    settings: Record<string, unknown>;
+  }[];
+  defaultOrgRoles: {
+    id: string;
+    defaultName: string;
+  }[];
+  defaultAppRoles: {
+    id: string;
+    defaultName: string;
+  }[];
+  defaultEnvironmentRoles: {
+    id: string;
+    defaultName: string;
+    settings: {
+      autoCommit: boolean;
+    };
+  }[];
+  baseEnvironments: {
+    id: string;
+    envParentId: string;
+    environmentRoleId: string;
+    settings: Record<string, unknown>;
+  }[];
+  orgUsers: {
+    id: string;
+    firstName: string;
+    lastName: string;
+    email: string;
+    provider: string;
+    orgRoleId: string;
+    uid: string;
+  }[];
+  envs: Record<
+    string,
+    {
+      variables: Record<string, { val: string }>;
+      inherits: Record<string, unknown>;
+    }
+  >;
+};

backend/src/services/project/project-service.ts

@@ -146,7 +146,8 @@ export const projectServiceFactory = ({
     actorAuthMethod,
     workspaceName,
     slug: projectSlug,
-    kmsKeyId
+    kmsKeyId,
+    createDefaultEnvs = true
   }: TCreateProjectDTO) => {
     const organization = await orgDAL.findOne({ id: actorOrgId });
 
@@ -207,14 +208,17 @@ export const projectServiceFactory = ({
       );
 
       // set default environments and root folder for provided environments
-      const envs = await projectEnvDAL.insertMany(
-        DEFAULT_PROJECT_ENVS.map((el, i) => ({ ...el, projectId: project.id, position: i + 1 })),
-        tx
-      );
-      await folderDAL.insertMany(
-        envs.map(({ id }) => ({ name: ROOT_FOLDER_NAME, envId: id, version: 1 })),
-        tx
-      );
+      let envs;
+      if (createDefaultEnvs) {
+        envs = await projectEnvDAL.insertMany(
+          DEFAULT_PROJECT_ENVS.map((el, i) => ({ ...el, projectId: project.id, position: i + 1 })),
+          tx
+        );
+        await folderDAL.insertMany(
+          envs.map(({ id }) => ({ name: ROOT_FOLDER_NAME, envId: id, version: 1 })),
+          tx
+        );
+      }
 
       // 3. Create a random key that we'll use as the project key.
       const { key: encryptedProjectKey, iv: encryptedProjectKeyIv } = createProjectKey({
@@ -360,7 +364,7 @@ export const projectServiceFactory = ({
 
       return {
         ...project,
-        environments: envs,
+        environments: envs || [],
         _id: project.id
       };
     });

backend/src/services/project/project-types.ts

@@ -29,6 +29,7 @@ export type TCreateProjectDTO = {
   workspaceName: string;
   slug?: string;
   kmsKeyId?: string;
+  createDefaultEnvs?: boolean;
 };
 
 export type TDeleteProjectBySlugDTO = {