feat: completed migration

backend/src/db/migrations/20241205160300_project-split-to-products.ts

@@ -1,13 +1,248 @@
 import { Knex } from "knex";
+import { v4 as uuidV4 } from "uuid";
+import slugify from "@sindresorhus/slugify";
 
-import { TableName } from "../schemas";
+import { ProjectType, TableName } from "../schemas";
+import { alphaNumericNanoId } from "@app/lib/nanoid";
 
+/* eslint-disable no-await-in-loop,no-param-reassign,@typescript-eslint/ban-ts-comment */
+const newProject = async (knex: Knex, projectId: string, projectType: ProjectType) => {
+  const newProjectId = uuidV4();
+  const project = await knex(TableName.Project).where("id", projectId).first();
+  await knex(TableName.Project).insert({
+    ...project,
+    type: projectType,
+    // @ts-ignore id is required
+    id: newProjectId,
+    slug: slugify(`${project?.name}-${alphaNumericNanoId(4)}`)
+  });
+
+  const customRoleMapping: Record<string, string> = {};
+  const projectCustomRoles = await knex(TableName.ProjectRoles).where("projectId", projectId);
+  if (projectCustomRoles.length) {
+    await knex(TableName.ProjectRoles).insert(
+      projectCustomRoles.map((el) => {
+        const id = uuidV4();
+        customRoleMapping[el.id] = id;
+        el.id = id;
+        el.projectId = newProjectId;
+        el.permissions = el.permissions ? JSON.stringify(el.permissions) : el.permissions;
+        return el;
+      })
+    );
+  }
+  const groupMembershipMapping: Record<string, string> = {};
+  const groupMemberships = await knex(TableName.GroupProjectMembership).where("projectId", projectId);
+  if (groupMemberships.length) {
+    await knex(TableName.GroupProjectMembership).insert(
+      groupMemberships.map((el) => {
+        const id = uuidV4();
+        groupMembershipMapping[el.id] = id;
+        el.id = id;
+        el.projectId = newProjectId;
+        return el;
+      })
+    );
+  }
+
+  const groupMembershipRoles = await knex(TableName.GroupProjectMembershipRole).whereIn(
+    "projectMembershipId",
+    groupMemberships.map((el) => el.id)
+  );
+  if (groupMembershipRoles.length) {
+    await knex(TableName.GroupProjectMembershipRole).insert(
+      groupMembershipRoles.map((el) => {
+        const id = uuidV4();
+        el.id = id;
+        el.projectMembershipId = groupMembershipMapping[el.id];
+        el.customRoleId = el.customRoleId ? customRoleMapping[el.customRoleId] : el.customRoleId;
+        return el;
+      })
+    );
+  }
+
+  const identityProjectMembershipMapping: Record<string, string> = {};
+  const identities = await knex(TableName.IdentityProjectMembership).where("projectId", projectId);
+  if (identities.length) {
+    await knex(TableName.IdentityProjectMembership).insert(
+      identities.map((el) => {
+        const id = uuidV4();
+        identityProjectMembershipMapping[el.id] = id;
+        el.id = id;
+        el.projectId = newProjectId;
+        return el;
+      })
+    );
+  }
+
+  const identitiesRoles = await knex(TableName.IdentityProjectMembershipRole).whereIn(
+    "projectMembershipId",
+    identities.map((el) => el.id)
+  );
+  if (identitiesRoles.length) {
+    await knex(TableName.IdentityProjectMembershipRole).insert(
+      identitiesRoles.map((el) => {
+        const id = uuidV4();
+        el.id = id;
+        el.projectMembershipId = identityProjectMembershipMapping[el.projectMembershipId];
+        el.customRoleId = el.customRoleId ? customRoleMapping[el.customRoleId] : el.customRoleId;
+        return el;
+      })
+    );
+  }
+
+  const projectMembershipMapping: Record<string, string> = {};
+  const projectUserMembers = await knex(TableName.ProjectMembership).where("projectId", projectId);
+  if (projectUserMembers.length) {
+    await knex(TableName.ProjectMembership).insert(
+      projectUserMembers.map((el) => {
+        const id = uuidV4();
+        projectMembershipMapping[el.id] = id;
+        el.id = id;
+        el.projectId = newProjectId;
+        return el;
+      })
+    );
+  }
+  const membershipRoles = await knex(TableName.ProjectUserMembershipRole).whereIn(
+    "projectMembershipId",
+    projectUserMembers.map((el) => el.id)
+  );
+  if (membershipRoles.length) {
+    await knex(TableName.ProjectUserMembershipRole).insert(
+      membershipRoles.map((el) => {
+        const id = uuidV4();
+        el.id = id;
+        el.projectMembershipId = projectMembershipMapping[el.projectMembershipId];
+        el.customRoleId = el.customRoleId ? customRoleMapping[el.customRoleId] : el.customRoleId;
+        return el;
+      })
+    );
+  }
+
+  const kmsKeys = await knex(TableName.KmsKey).where("projectId", projectId).andWhere("isReserved", true);
+  if (kmsKeys.length) {
+    await knex(TableName.KmsKey).insert(
+      kmsKeys.map((el) => {
+        const id = uuidV4();
+        el.id = id;
+        el.projectId = newProjectId;
+        el.slug = slugify(alphaNumericNanoId(8).toLowerCase());
+        return el;
+      })
+    );
+  }
+  const projectBot = await knex(TableName.ProjectBot).where("projectId", projectId).first();
+  if (projectBot) {
+    const newProjectBot = { ...projectBot, id: uuidV4(), projectId: newProjectId };
+    await knex(TableName.ProjectBot).insert(newProjectBot);
+  }
+
+  const projectKeys = await knex(TableName.ProjectKeys).where("projectId", projectId);
+  if (projectKeys.length) {
+    await knex(TableName.ProjectKeys).insert(
+      projectKeys.map((el) => {
+        const id = uuidV4();
+        el.id = id;
+        el.projectId = newProjectId;
+        return el;
+      })
+    );
+  }
+
+  const serviceTokens = await knex(TableName.ServiceToken).where("projectId", projectId);
+  if (serviceTokens.length) {
+    await knex(TableName.ServiceToken).insert(
+      serviceTokens.map((el) => {
+        el.id = uuidV4();
+        el.projectId = projectId;
+        el.scopes = el.scopes ? JSON.stringify(el.scopes) : el.scopes;
+        return el;
+      })
+    );
+  }
+  return newProjectId;
+};
+/* eslint-enable */
+
+const BATCH_SIZE = 500;
 export async function up(knex: Knex): Promise<void> {
+  const hasSplitMappingTable = await knex.schema.hasTable(TableName.ProjectSplitBackfillIds);
+  if (!hasSplitMappingTable) {
+    await knex.schema.createTable(TableName.ProjectSplitBackfillIds, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.string("sourceProjectId", 36).notNullable();
+      t.foreign("sourceProjectId").references("id").inTable(TableName.Project).onDelete("CASCADE");
+      t.string("destinationProjectType").notNullable();
+      t.string("destinationProjectId", 36).notNullable();
+      t.foreign("destinationProjectId").references("id").inTable(TableName.Project).onDelete("CASCADE");
+    });
+  }
+
   const hasTypeColumn = await knex.schema.hasColumn(TableName.Project, "type");
   if (!hasTypeColumn) {
     await knex.schema.alterTable(TableName.Project, (t) => {
       t.string("type");
     });
+
+    let projectsToBeTyped;
+    do {
+      // eslint-disable-next-line no-await-in-loop
+      projectsToBeTyped = await knex(TableName.Project).whereNull("type").limit(BATCH_SIZE).select("id");
+      if (projectsToBeTyped.length) {
+        // eslint-disable-next-line no-await-in-loop
+        await knex(TableName.Project)
+          .whereIn(
+            "id",
+            projectsToBeTyped.map((el) => el.id)
+          )
+          .update({ type: ProjectType.SecretManager });
+      }
+    } while (projectsToBeTyped.length > 0);
+
+    const projectsWithCertificates = await knex(TableName.CertificateAuthority)
+      .distinct("projectId")
+      .select("projectId");
+    /* eslint-disable no-await-in-loop,no-param-reassign */
+    for (const { projectId } of projectsWithCertificates) {
+      const newProjectId = await newProject(knex, projectId, ProjectType.CertificateManager);
+      await knex(TableName.CertificateAuthority).where("projectId", projectId).update({ projectId: newProjectId });
+      await knex(TableName.PkiAlert).where("projectId", projectId).update({ projectId: newProjectId });
+      await knex(TableName.PkiCollection).where("projectId", projectId).update({ projectId: newProjectId });
+      await knex(TableName.ProjectSplitBackfillIds).insert({
+        sourceProjectId: projectId,
+        destinationProjectType: ProjectType.CertificateManager,
+        destinationProjectId: newProjectId
+      });
+    }
+
+    const projectsWithCmek = await knex(TableName.KmsKey)
+      .where("isReserved", false)
+      .whereNotNull("projectId")
+      .distinct("projectId")
+      .select("projectId");
+    for (const { projectId } of projectsWithCmek) {
+      if (projectId) {
+        const newProjectId = await newProject(knex, projectId, ProjectType.Cmek);
+        await knex(TableName.KmsKey)
+          .where({
+            isReserved: false,
+            projectId
+          })
+          .update({ projectId: newProjectId });
+        await knex(TableName.ProjectSplitBackfillIds).insert({
+          sourceProjectId: projectId,
+          destinationProjectType: ProjectType.Cmek,
+          destinationProjectId: newProjectId
+        });
+      }
+    }
+
+    /* eslint-enable */
+
+    await knex.schema.alterTable(TableName.Project, (t) => {
+      t.string("type").notNullable().alter();
+    });
   }
 }
 
@@ -18,4 +253,9 @@ export async function down(knex: Knex): Promise<void> {
       t.dropColumn("type");
     });
   }
+
+  const hasSplitMappingTable = await knex.schema.hasTable(TableName.ProjectSplitBackfillIds);
+  if (hasSplitMappingTable) {
+    await knex.schema.dropTableIfExists(TableName.ProjectSplitBackfillIds);
+  }
 }

backend/src/db/schemas/models.ts

@@ -106,6 +106,7 @@ export enum TableName {
   SecretApprovalRequestSecretV2 = "secret_approval_requests_secrets_v2",
   SecretApprovalRequestSecretTagV2 = "secret_approval_request_secret_tags_v2",
   SnapshotSecretV2 = "secret_snapshot_secrets_v2",
+  ProjectSplitBackfillIds = "project_split_backfill_ids",
   // junction tables with tags
   SecretV2JnTag = "secret_v2_tag_junction",
   JnSecretTag = "secret_tag_junction",

backend/src/lib/api-docs/constants.ts

@@ -428,7 +428,8 @@ export const ORGANIZATIONS = {
     search: "The text string that identity membership names will be filtered by."
   },
   GET_PROJECTS: {
-    organizationId: "The ID of the organization to get projects from."
+    organizationId: "The ID of the organization to get projects from.",
+    type: "The type of project to filter by."
   },
   LIST_GROUPS: {
     organizationId: "The ID of the organization to list groups for."

backend/src/server/routes/sanitizedSchemas.ts

@@ -220,6 +220,7 @@ export const SanitizedProjectSchema = ProjectsSchema.pick({
   id: true,
   name: true,
   description: true,
+  type: true,
   slug: true,
   autoCapitalization: true,
   orgId: true,

backend/src/server/routes/v1/project-router.ts

@@ -5,6 +5,7 @@ import {
   ProjectMembershipsSchema,
   ProjectRolesSchema,
   ProjectSlackConfigsSchema,
+  ProjectType,
   UserEncryptionKeysSchema,
   UsersSchema
 } from "@app/db/schemas";
@@ -135,7 +136,8 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => {
         includeRoles: z
           .enum(["true", "false"])
           .default("false")
-          .transform((value) => value === "true")
+          .transform((value) => value === "true"),
+        type: z.nativeEnum(ProjectType).optional()
       }),
       response: {
         200: z.object({
@@ -154,7 +156,8 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => {
         actorId: req.permission.id,
         actorAuthMethod: req.permission.authMethod,
         actor: req.permission.type,
-        actorOrgId: req.permission.orgId
+        actorOrgId: req.permission.orgId,
+        type: req.query.type
       });
       return { workspaces };
     }

backend/src/server/routes/v2/organization-router.ts

@@ -5,6 +5,7 @@ import {
   OrgMembershipsSchema,
   ProjectMembershipsSchema,
   ProjectsSchema,
+  ProjectType,
   UserEncryptionKeysSchema,
   UsersSchema
 } from "@app/db/schemas";
@@ -76,7 +77,8 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => {
         }
       ],
       params: z.object({
-        organizationId: z.string().trim().describe(ORGANIZATIONS.GET_PROJECTS.organizationId)
+        organizationId: z.string().trim().describe(ORGANIZATIONS.GET_PROJECTS.organizationId),
+        type: z.nativeEnum(ProjectType).optional().describe(ORGANIZATIONS.GET_PROJECTS.type)
       }),
       response: {
         200: z.object({

backend/src/services/org/org-service.ts

@@ -15,7 +15,6 @@ import {
   TProjectUserMembershipRolesInsert,
   TUsers
 } from "@app/db/schemas";
-import { TProjects } from "@app/db/schemas/projects";
 import { TGroupDALFactory } from "@app/ee/services/group/group-dal";
 import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
 import { TOidcConfigDALFactory } from "@app/ee/services/oidc/oidc-config-dal";
@@ -196,26 +195,18 @@ export const orgServiceFactory = ({
     return org;
   };
 
-  const findAllWorkspaces = async ({ actor, actorId, orgId }: TFindAllWorkspacesDTO) => {
-    const organizationWorkspaceIds = new Set((await projectDAL.find({ orgId })).map((workspace) => workspace.id));
-
-    let workspaces: (TProjects & { organization: string } & {
-      environments: {
-        id: string;
-        slug: string;
-        name: string;
-      }[];
-    })[];
-
+  const findAllWorkspaces = async ({ actor, actorId, orgId, type }: TFindAllWorkspacesDTO) => {
     if (actor === ActorType.USER) {
-      workspaces = await projectDAL.findAllProjects(actorId);
-    } else if (actor === ActorType.IDENTITY) {
-      workspaces = await projectDAL.findAllProjectsByIdentity(actorId);
-    } else {
-      throw new BadRequestError({ message: "Invalid actor type" });
+      const workspaces = await projectDAL.findAllProjects(actorId, orgId, type);
+      return workspaces;
     }
 
-    return workspaces.filter((workspace) => organizationWorkspaceIds.has(workspace.id));
+    if (actor === ActorType.IDENTITY) {
+      const workspaces = await projectDAL.findAllProjectsByIdentity(actorId, type);
+      return workspaces;
+    }
+
+    throw new BadRequestError({ message: "Invalid actor type" });
   };
 
   const addGhostUser = async (orgId: string, tx?: Knex) => {

backend/src/services/org/org-types.ts

@@ -1,6 +1,7 @@
 import { TOrgPermission } from "@app/lib/types";
 
 import { ActorAuthMethod, ActorType, MfaMethod } from "../auth/auth-type";
+import { ProjectType } from "@app/db/schemas";
 
 export type TUpdateOrgMembershipDTO = {
   userId: string;
@@ -55,6 +56,7 @@ export type TFindAllWorkspacesDTO = {
   actorOrgId: string | undefined;
   actorAuthMethod: ActorAuthMethod;
   orgId: string;
+  type?: ProjectType;
 };
 
 export type TUpdateOrgDTO = {

backend/src/services/project/project-dal.ts

@@ -1,7 +1,14 @@
 import { Knex } from "knex";
 
 import { TDbClient } from "@app/db";
-import { ProjectsSchema, ProjectUpgradeStatus, ProjectVersion, TableName, TProjectsUpdate } from "@app/db/schemas";
+import {
+  ProjectsSchema,
+  ProjectType,
+  ProjectUpgradeStatus,
+  ProjectVersion,
+  TableName,
+  TProjectsUpdate
+} from "@app/db/schemas";
 import { BadRequestError, DatabaseError, NotFoundError, UnauthorizedError } from "@app/lib/errors";
 import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex";
 
@@ -12,12 +19,18 @@ export type TProjectDALFactory = ReturnType<typeof projectDALFactory>;
 export const projectDALFactory = (db: TDbClient) => {
   const projectOrm = ormify(db, TableName.Project);
 
-  const findAllProjects = async (userId: string) => {
+  const findAllProjects = async (userId: string, orgId: string, projectType?: ProjectType | null) => {
     try {
       const workspaces = await db
         .replicaNode()(TableName.ProjectMembership)
         .where({ userId })
         .join(TableName.Project, `${TableName.ProjectMembership}.projectId`, `${TableName.Project}.id`)
+        .where(`${TableName.Project}.orgId`, orgId)
+        .andWhere((qb) => {
+          if (projectType) {
+            void qb.where(`${TableName.Project}.type`, projectType);
+          }
+        })
         .leftJoin(TableName.Environment, `${TableName.Environment}.projectId`, `${TableName.Project}.id`)
         .select(
           selectAllTableCols(TableName.Project),
@@ -31,14 +44,17 @@ export const projectDALFactory = (db: TDbClient) => {
           { column: `${TableName.Environment}.position`, order: "asc" }
         ]);
 
-      const groups: string[] = await db(TableName.UserGroupMembership)
-        .where({ userId })
-        .select(selectAllTableCols(TableName.UserGroupMembership))
-        .pluck("groupId");
+      const groups = db(TableName.UserGroupMembership).where({ userId }).select("groupId");
 
       const groupWorkspaces = await db(TableName.GroupProjectMembership)
         .whereIn("groupId", groups)
         .join(TableName.Project, `${TableName.GroupProjectMembership}.projectId`, `${TableName.Project}.id`)
+        .where(`${TableName.Project}.orgId`, orgId)
+        .andWhere((qb) => {
+          if (projectType) {
+            void qb.where(`${TableName.Project}.type`, projectType);
+          }
+        })
         .whereNotIn(
           `${TableName.Project}.id`,
           workspaces.map(({ id }) => id)
@@ -108,12 +124,13 @@ export const projectDALFactory = (db: TDbClient) => {
     }
   };
 
-  const findAllProjectsByIdentity = async (identityId: string) => {
+  const findAllProjectsByIdentity = async (identityId: string, projectType?: ProjectType) => {
     try {
       const workspaces = await db
         .replicaNode()(TableName.IdentityProjectMembership)
         .where({ identityId })
         .join(TableName.Project, `${TableName.IdentityProjectMembership}.projectId`, `${TableName.Project}.id`)
+        .where(`${TableName.Project}.type`, projectType)
         .leftJoin(TableName.Environment, `${TableName.Environment}.projectId`, `${TableName.Project}.id`)
         .select(
           selectAllTableCols(TableName.Project),

backend/src/services/project/project-service.ts

@@ -157,7 +157,6 @@ export const projectServiceFactory = ({
     type = ProjectType.SecretManager
   }: TCreateProjectDTO) => {
     const organization = await orgDAL.findOne({ id: actorOrgId });
-
     const { permission, membership: orgMembership } = await permissionService.getOrgPermission(
       actor,
       actorId,
@@ -432,8 +431,8 @@ export const projectServiceFactory = ({
     return deletedProject;
   };
 
-  const getProjects = async ({ actorId, includeRoles, actorAuthMethod, actorOrgId }: TListProjectsDTO) => {
-    const workspaces = await projectDAL.findAllProjects(actorId);
+  const getProjects = async ({ actorId, includeRoles, actorAuthMethod, actorOrgId, type }: TListProjectsDTO) => {
+    const workspaces = await projectDAL.findAllProjects(actorId, actorOrgId, type);
 
     if (includeRoles) {
       const { permission } = await permissionService.getUserOrgPermission(actorId, actorOrgId, actorAuthMethod);

backend/src/services/project/project-types.ts

@@ -85,6 +85,7 @@ export type TDeleteProjectDTO = {
 
 export type TListProjectsDTO = {
   includeRoles: boolean;
+  type?: ProjectType | null;
 } & Omit<TProjectPermission, "projectId">;
 
 export type TUpgradeProjectDTO = {