fix: improvements

backend/src/server/routes/v3/external-migration-router.ts

@@ -18,14 +18,11 @@ export const registerExternalMigrationRouter = async (server: FastifyZodProvider
           nonce: z.string().trim().min(1),
           data: z.string().trim().min(1)
         })
-      }),
-      response: {
-        200: z.object({})
-      }
+      })
     },
     onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
     handler: async (req) => {
-      await server.services.migration.importEnvnKeyData({
+      await server.services.migration.importEnvKeyData({
         decryptionKey: req.body.decryptionKey,
         encryptedJson: req.body.encryptedJson,
         actorId: req.permission.id,

backend/src/services/external-migration/external-migration-fns.ts

@@ -1,16 +1,33 @@
+import slugify from "@sindresorhus/slugify";
 import { randomUUID } from "crypto";
 import sjcl from "sjcl";
 import tweetnacl from "tweetnacl";
 import tweetnaclUtil from "tweetnacl-util";
 
+import { OrgMembershipRole, ProjectMembershipRole, SecretType } from "@app/db/schemas";
 import { BadRequestError } from "@app/lib/errors";
+import { logger } from "@app/lib/logger";
+import { alphaNumericNanoId } from "@app/lib/nanoid";
 
-import { InfisicalImportData, TEnvKeyExportJSON } from "./external-migration-types";
+import { TOrgServiceFactory } from "../org/org-service";
+import { TProjectServiceFactory } from "../project/project-service";
+import { TProjectEnvServiceFactory } from "../project-env/project-env-service";
+import { TSecretServiceFactory } from "../secret/secret-service";
+import { InfisicalImportData, TEnvKeyExportJSON, TImportInfisicalDataCreate } from "./external-migration-types";
+
+export type TImportDataIntoInfisicalDTO = {
+  projectService: TProjectServiceFactory;
+  orgService: TOrgServiceFactory;
+  projectEnvService: TProjectEnvServiceFactory;
+  secretService: TSecretServiceFactory;
+
+  input: TImportInfisicalDataCreate;
+};
 
 const { codec, hash } = sjcl;
 const { secretbox } = tweetnacl;
 
-export const decryptEnvKeyData = async (decryptionKey: string, encryptedJson: { nonce: string; data: string }) => {
+export const decryptEnvKeyDataFn = async (decryptionKey: string, encryptedJson: { nonce: string; data: string }) => {
   const key = tweetnaclUtil.decodeBase64(codec.base64.fromBits(hash.sha256.hash(decryptionKey)));
   const nonce = tweetnaclUtil.decodeBase64(encryptedJson.nonce);
   const encryptedData = tweetnaclUtil.decodeBase64(encryptedJson.data);
@@ -25,7 +42,7 @@ export const decryptEnvKeyData = async (decryptionKey: string, encryptedJson: {
   return decryptedJson;
 };
 
-export const parseEnvKeyData = async (decryptedJson: string): Promise<InfisicalImportData> => {
+export const parseEnvKeyDataFn = async (decryptedJson: string): Promise<InfisicalImportData> => {
   const parsedJson: TEnvKeyExportJSON = JSON.parse(decryptedJson) as TEnvKeyExportJSON;
 
   const infisicalImportData: InfisicalImportData = {
@@ -71,3 +88,110 @@ export const parseEnvKeyData = async (decryptedJson: string): Promise<InfisicalI
 
   return infisicalImportData;
 };
+
+export const importDataIntoInfisicalFn = async ({
+  projectService,
+  orgService,
+  projectEnvService,
+  secretService,
+  input: { data, actor, actorId, actorOrgId, actorAuthMethod }
+}: TImportDataIntoInfisicalDTO) => {
+  // Import data to infisical
+  if (!data || !data.projects) {
+    throw new BadRequestError({ message: "No projects found in data" });
+  }
+
+  const originalToNewProjectId = new Map<string, string>();
+  const originalToNewEnvironmentId = new Map<string, string>();
+
+  for await (const [id, project] of data.projects) {
+    const newProject = await projectService
+      .createProject({
+        actor,
+        actorId,
+        actorOrgId,
+        actorAuthMethod,
+        workspaceName: project.name,
+        createDefaultEnvs: false
+      })
+      .catch(() => {
+        throw new BadRequestError({ message: `Failed to import to project [name:${project.name}] [id:${id}]` });
+      });
+
+    originalToNewProjectId.set(project.id, newProject.id);
+  }
+
+  // Invite user importing projects
+  const invites = await orgService.inviteUserToOrganization({
+    actorAuthMethod,
+    actorId,
+    actorOrgId,
+    actor,
+    inviteeEmails: [],
+    orgId: actorOrgId,
+    organizationRoleSlug: OrgMembershipRole.NoAccess,
+    projects: Array.from(originalToNewProjectId.values()).map((project) => ({
+      id: project,
+      projectRoleSlug: [ProjectMembershipRole.Member]
+    }))
+  });
+  if (!invites) {
+    throw new BadRequestError({ message: `Failed to invite user to projects: [userId:${actorId}]` });
+  }
+
+  // Import environments
+  if (data.environments) {
+    for await (const [id, environment] of data.environments) {
+      try {
+        const newEnvironment = await projectEnvService.createEnvironment({
+          actor,
+          actorId,
+          actorOrgId,
+          actorAuthMethod,
+          name: environment.name,
+          projectId: originalToNewProjectId.get(environment.projectId)!,
+          slug: slugify(`${environment.name}-${alphaNumericNanoId(4)}`)
+        });
+
+        if (!newEnvironment) {
+          logger.error(`Failed to import environment: [name:${environment.name}] [id:${id}]`);
+          throw new BadRequestError({
+            message: `Failed to import environment: [name:${environment.name}] [id:${id}]`
+          });
+        }
+        originalToNewEnvironmentId.set(id, newEnvironment.slug);
+      } catch (error) {
+        throw new BadRequestError({
+          message: `Failed to import environment: ${environment.name}]`,
+          name: "EnvKeyMigrationImportEnvironment"
+        });
+      }
+    }
+  }
+
+  // Import secrets
+  if (data.secrets) {
+    for await (const [id, secret] of data.secrets) {
+      const dataProjectId = data.environments?.get(secret.environmentId)?.projectId;
+      if (!dataProjectId) {
+        throw new BadRequestError({ message: `Failed to import secret "${secret.name}", project not found` });
+      }
+      const projectId = originalToNewProjectId.get(dataProjectId);
+      const newSecret = await secretService.createSecretRaw({
+        actorId,
+        actor,
+        actorOrgId,
+        environment: originalToNewEnvironmentId.get(secret.environmentId)!,
+        actorAuthMethod,
+        projectId: projectId!,
+        secretPath: "/",
+        secretName: secret.name,
+        type: SecretType.Shared,
+        secretValue: secret.value
+      });
+      if (!newSecret) {
+        throw new BadRequestError({ message: `Failed to import secret: [name:${secret.name}] [id:${id}]` });
+      }
+    }
+  }
+};

backend/src/services/external-migration/external-migration-service.ts

@@ -1,16 +1,9 @@
-import slugify from "@sindresorhus/slugify";
-
-import { OrgMembershipRole, ProjectMembershipRole, SecretType } from "@app/db/schemas";
-import { BadRequestError } from "@app/lib/errors";
-import { logger } from "@app/lib/logger";
-import { alphaNumericNanoId } from "@app/lib/nanoid";
-
 import { TOrgServiceFactory } from "../org/org-service";
 import { TProjectServiceFactory } from "../project/project-service";
 import { TProjectEnvServiceFactory } from "../project-env/project-env-service";
 import { TSecretServiceFactory } from "../secret/secret-service";
-import { decryptEnvKeyData, parseEnvKeyData } from "./external-migration-fns";
-import { TImportEnvKeyDataCreate, TImportInfisicalDataCreate } from "./external-migration-types";
+import { decryptEnvKeyDataFn, importDataIntoInfisicalFn, parseEnvKeyDataFn } from "./external-migration-fns";
+import { TImportEnvKeyDataCreate } from "./external-migration-types";
 
 type TExternalMigrationServiceFactoryDep = {
   projectService: TProjectServiceFactory;
@@ -27,128 +20,7 @@ export const externalMigrationServiceFactory = ({
   projectEnvService,
   secretService
 }: TExternalMigrationServiceFactoryDep) => {
-  const importInfisicalData = async ({
-    data,
-    actor,
-    actorId,
-    actorOrgId,
-    actorAuthMethod
-  }: TImportInfisicalDataCreate) => {
-    // Import data to infisical
-    if (!data || !data.projects) {
-      throw new BadRequestError({ message: "No projects found in data" });
-    }
-
-    const orginalToNewProjectId = new Map<string, string>();
-    const orginalToNewEnvironmentId = new Map<string, string>();
-
-    // Import projects
-    const projectPromises = [];
-    for (const [id, project] of data.projects) {
-      const projectPromise = projectService
-        .createProject({
-          actor,
-          actorId,
-          actorOrgId,
-          actorAuthMethod,
-          workspaceName: project.name,
-          createDefaultEnvs: false
-        })
-        .then((projectResponse) => {
-          if (!projectResponse) {
-            throw new BadRequestError({ message: `Failed to import to project [name:${project.name}] [id:${id}]` });
-          }
-          orginalToNewProjectId.set(project.id, projectResponse.id);
-        });
-      projectPromises.push(projectPromise);
-    }
-    await Promise.all(projectPromises);
-
-    // Invite user importing projects
-    const response = await orgService.inviteUserToOrganization({
-      actorAuthMethod,
-      actorId,
-      actorOrgId,
-      actor,
-      inviteeEmails: [],
-      orgId: actorOrgId,
-      organizationRoleSlug: OrgMembershipRole.NoAccess,
-      projects: Array.from(orginalToNewProjectId.values()).map((project) => {
-        return {
-          id: project,
-          projectRoleSlug: [ProjectMembershipRole.Member]
-        };
-      })
-    });
-    if (!response) {
-      throw new BadRequestError({ message: `Failed to invite user to projects: [userId:${actorId}]` });
-    }
-
-    // Import environments
-    if (data.environments) {
-      for await (const [id, environment] of data.environments) {
-        try {
-          // TODO: we can create envs parallely once the position constraint is handled differently
-          const newEnvironment = await projectEnvService.createEnvironment({
-            actor,
-            actorId,
-            actorOrgId,
-            actorAuthMethod,
-            name: environment.name,
-            projectId: orginalToNewProjectId.get(environment.projectId)!,
-            slug: slugify(`${environment.name}-${alphaNumericNanoId(4)}`)
-          });
-
-          if (!newEnvironment) {
-            logger.error(`Failed to import environment: [name:${environment.name}] [id:${id}]`);
-            throw new BadRequestError({
-              message: `Failed to import environment: [name:${environment.name}] [id:${id}]`
-            });
-          }
-          orginalToNewEnvironmentId.set(id, newEnvironment.slug);
-        } catch (error) {
-          return {
-            success: false
-          };
-        }
-      }
-    }
-
-    // Import secrets
-    if (data.secrets) {
-      for await (const [id, secret] of data.secrets) {
-        const dataProjectId = data.environments?.get(secret.environmentId)?.projectId;
-        if (!dataProjectId) {
-          logger.error(`Failed to import secret: [name:${secret.name}] [id:${id}], project not found`);
-          return {
-            success: false,
-            message: `Failed to import secret: [name:${secret.name}] [id:${id}], project not found`
-          };
-        }
-        const projectId = orginalToNewProjectId.get(dataProjectId);
-        // TODO: we can create secrets parallely once the KMS ID bug on create is fixed
-        const newSecret = await secretService.createSecretRaw({
-          actorId,
-          actor,
-          actorOrgId,
-          environment: orginalToNewEnvironmentId.get(secret.environmentId)!,
-          actorAuthMethod,
-          projectId: projectId!,
-          secretPath: "/",
-          secretName: secret.name,
-          type: SecretType.Shared,
-          secretValue: secret.value
-        });
-        if (!newSecret) {
-          throw new BadRequestError({ message: `Failed to import secret: [name:${secret.name}] [id:${id}]` });
-        }
-      }
-    }
-
-    return true;
-  };
-
-  const importEnvnKeyData = async ({
+  const importEnvKeyData = async ({
     decryptionKey,
     encryptedJson,
     actor,
@@ -156,13 +28,19 @@ export const externalMigrationServiceFactory = ({
     actorOrgId,
     actorAuthMethod
   }: TImportEnvKeyDataCreate) => {
-    const json = await decryptEnvKeyData(decryptionKey, encryptedJson);
-    const envKeyData = await parseEnvKeyData(json);
-    const response = await importInfisicalData({ data: envKeyData, actor, actorId, actorOrgId, actorAuthMethod });
+    const json = await decryptEnvKeyDataFn(decryptionKey, encryptedJson);
+    const envKeyData = await parseEnvKeyDataFn(json);
+    const response = await importDataIntoInfisicalFn({
+      input: { data: envKeyData, actor, actorId, actorOrgId, actorAuthMethod },
+      projectService,
+      orgService,
+      projectEnvService,
+      secretService
+    });
     return response;
   };
 
   return {
-    importEnvnKeyData
+    importEnvKeyData
   };
 };

backend/src/services/project/project-service.ts

@@ -1,7 +1,7 @@
 import { ForbiddenError } from "@casl/ability";
 import slugify from "@sindresorhus/slugify";
 
-import { OrgMembershipRole, ProjectMembershipRole, ProjectVersion } from "@app/db/schemas";
+import { OrgMembershipRole, ProjectMembershipRole, ProjectVersion, TProjectEnvironments } from "@app/db/schemas";
 import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
 import { OrgPermissionActions, OrgPermissionSubjects } from "@app/ee/services/permission/org-permission";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
@@ -208,15 +208,7 @@ export const projectServiceFactory = ({
       );
 
       // set default environments and root folder for provided environments
-      let envs: {
-        id: string;
-        createdAt: Date;
-        updatedAt: Date;
-        projectId: string;
-        name: string;
-        slug: string;
-        position: number;
-      }[] = [];
+      let envs: TProjectEnvironments[] = [];
       if (createDefaultEnvs) {
         envs = await projectEnvDAL.insertMany(
           DEFAULT_PROJECT_ENVS.map((el, i) => ({ ...el, projectId: project.id, position: i + 1 })),