checkpoint

backend/src/ee/routes/v1/project-router.ts

@@ -349,4 +349,35 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => {
       return backup;
     }
   });
+
+  server.route({
+    method: "POST",
+    url: "/:workspaceId/migrate-v3",
+    config: {
+      rateLimit: writeLimit
+    },
+    schema: {
+      params: z.object({
+        workspaceId: z.string().trim()
+      }),
+
+      response: {
+        200: z.object({
+          message: z.string()
+        })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT]),
+    handler: async (req) => {
+      const migration = await server.services.secret.startSecretV2Migration({
+        actor: req.permission.type,
+        actorId: req.permission.id,
+        actorAuthMethod: req.permission.authMethod,
+        actorOrgId: req.permission.orgId,
+        projectId: req.params.workspaceId
+      });
+
+      return migration;
+    }
+  });
 };

backend/src/queue/queue-service.ts

@@ -25,7 +25,8 @@ export enum QueueName {
   DynamicSecretRevocation = "dynamic-secret-revocation",
   CaCrlRotation = "ca-crl-rotation",
   SecretReplication = "secret-replication",
-  SecretSync = "secret-sync" // parent queue to push integration sync, webhook, and secret replication
+  SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication
+  ProjectV3Migration = "project-v3-migration"
 }
 
 export enum QueueJobs {
@@ -44,7 +45,8 @@ export enum QueueJobs {
   DynamicSecretPruning = "dynamic-secret-pruning",
   CaCrlRotation = "ca-crl-rotation-job",
   SecretReplication = "secret-replication",
-  SecretSync = "secret-sync" // parent queue to push integration sync, webhook, and secret replication
+  SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication
+  ProjectV3Migration = "project-v3-migration"
 }
 
 export type TQueueJobTypes = {
@@ -136,6 +138,10 @@ export type TQueueJobTypes = {
     name: QueueJobs.SecretSync;
     payload: TSyncSecretsDTO;
   };
+  [QueueName.ProjectV3Migration]: {
+    name: QueueJobs.ProjectV3Migration;
+    payload: { projectId: string };
+  };
 };
 
 export type TQueueServiceFactory = ReturnType<typeof queueServiceFactory>;

backend/src/services/secret-folder/secret-folder-dal.ts

@@ -331,6 +331,27 @@ export const secretFolderDALFactory = (db: TDbClient) => {
     }
   };
 
+  // special query for project migration
+  const findByProjectId = async (projectId: string, tx?: Knex) => {
+    try {
+      const folders = await (tx || db.replicaNode())(TableName.SecretFolder)
+        .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`)
+        .join(TableName.Project, `${TableName.Environment}.projectId`, `${TableName.Project}.id`)
+        .select(selectAllTableCols(TableName.SecretFolder))
+        .where({ projectId })
+        .select(
+          db.ref("id").withSchema(TableName.Environment).as("envId"),
+          db.ref("slug").withSchema(TableName.Environment).as("envSlug"),
+          db.ref("name").withSchema(TableName.Environment).as("envName"),
+          db.ref("projectId").withSchema(TableName.Environment),
+          db.ref("version").withSchema(TableName.Project).as("projectVersion")
+        );
+      return folders;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "Find by id" });
+    }
+  };
+
   return {
     ...secretFolderOrm,
     update,
@@ -338,6 +359,7 @@ export const secretFolderDALFactory = (db: TDbClient) => {
     findById,
     findByManySecretPath,
     findSecretPathByFolderIds,
-    findClosestFolder
+    findClosestFolder,
+    findByProjectId
   };
 };

backend/src/services/secret/secret-queue.ts

@@ -41,6 +41,7 @@ import {
   TRemoveSecretReminderDTO,
   TSyncSecretsDTO
 } from "./secret-types";
+import { ProjectUpgradeStatus, ProjectVersion } from "@app/db/schemas";
 
 export type TSecretQueueFactory = ReturnType<typeof secretQueueFactory>;
 type TSecretQueueFactoryDep = {
@@ -52,7 +53,7 @@ type TSecretQueueFactoryDep = {
   secretDAL: TSecretDALFactory;
   secretImportDAL: Pick<TSecretImportDALFactory, "find">;
   webhookDAL: Pick<TWebhookDALFactory, "findAllWebhooks" | "transaction" | "update" | "bulkUpdate">;
-  projectEnvDAL: Pick<TProjectEnvDALFactory, "findOne">;
+  projectEnvDAL: Pick<TProjectEnvDALFactory, "findOne" | "find">;
   projectDAL: TProjectDALFactory;
   projectBotDAL: TProjectBotDALFactory;
   projectMembershipDAL: Pick<TProjectMembershipDALFactory, "findAllProjectMembers">;
@@ -761,6 +762,108 @@ export const secretQueueFactory = ({
     });
   });
 
+  const startSecretV2Migration = async (projectId: string) => {
+    await queueService.queue(
+      QueueName.ProjectV3Migration,
+      QueueJobs.ProjectV3Migration,
+      { projectId },
+      {
+        attempts: 2,
+        backoff: {
+          type: "exponential",
+          delay: 3000
+        },
+        removeOnComplete: true,
+        removeOnFail: true
+      }
+    );
+  };
+
+  const MIGRATION_BATCH_SIZE = 10000;
+  queueService.start(QueueName.ProjectV3Migration, async (job) => {
+    const { projectId } = job.data;
+    const { botKey, shouldUseSecretV2Bridge: isProjectUpgradedToV3 } = await projectBotService.getBotKey(projectId);
+    if (isProjectUpgradedToV3) {
+      return;
+    }
+    if (!botKey) throw new BadRequestError({ message: "Bot not found" });
+    await projectDAL.updateById(projectId, { upgradeStatus: ProjectUpgradeStatus.InProgress });
+    const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({
+      projectId,
+      type: KmsDataKey.SecretManager
+    });
+
+    const folders = await folderDAL.findByProjectId(projectId);
+    // except secret version and snapshot migrate rest of everything first in a transaction
+    await secretDAL.transaction(async (tx) => {
+      for (const folder of folders) {
+        const folderId = folder.id;
+        let projectV1Secrets;
+        do {
+          // eslint-disable-next-line no-await-in-loop
+          projectV1Secrets = await secretDAL.find({ folderId }, { limit: MIGRATION_BATCH_SIZE, tx });
+          await secretV2BridgeDAL.insertMany(
+            projectV1Secrets.map((el) => {
+              const key = decryptSymmetric128BitHexKeyUTF8({
+                ciphertext: el.secretKeyCiphertext,
+                iv: el.secretKeyIV,
+                tag: el.secretKeyTag,
+                key: botKey
+              });
+              const value = decryptSymmetric128BitHexKeyUTF8({
+                ciphertext: el.secretValueCiphertext,
+                iv: el.secretValueIV,
+                tag: el.secretValueTag,
+                key: botKey
+              });
+              const comment =
+                el.secretCommentCiphertext && el.secretCommentTag && el.secretCommentIV
+                  ? decryptSymmetric128BitHexKeyUTF8({
+                      ciphertext: el.secretCommentCiphertext,
+                      iv: el.secretCommentIV,
+                      tag: el.secretCommentTag,
+                      key: botKey
+                    })
+                  : "";
+              const encryptedValue = secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob;
+              const encryptedComment = comment
+                ? secretManagerEncryptor({ plainText: Buffer.from(comment) }).cipherTextBlob
+                : null;
+              return {
+                id: el.id,
+                createdAt: el.createdAt,
+                updatedAt: el.updatedAt,
+                skipMultilineEncoding: el.skipMultilineEncoding,
+                encryptedComment,
+                encryptedValue,
+                key,
+                version: el.version,
+                type: el.type,
+                userId: el.userId,
+                folderId: el.folderId,
+                metadata: el.metadata,
+                reminderNote: el.secretReminderNote,
+                reminderRepeatDays: el.secretReminderRepeatDays
+              };
+            }),
+            tx
+          );
+          projectV1Secrets = await secretDAL.delete({ folderId, $in: { id: projectV1Secrets.map((el) => el.id) } }, tx);
+        } while (projectV1Secrets.length > 0);
+      }
+      await projectDAL.updateById(projectId, { upgradeStatus: null, version: ProjectVersion.V3 }, tx);
+    });
+  });
+
+  // eslint-disable-next-line
+  queueService.listen(QueueName.ProjectV3Migration, "failed", async (job, err) => {
+    if (job?.data) {
+      const { projectId } = job.data;
+      await projectDAL.updateById(projectId, { upgradeStatus: ProjectUpgradeStatus.Failed });
+      logger.error(err, `Failed to migrate project to v3: ${projectId}`);
+    }
+  });
+
   queueService.listen(QueueName.IntegrationSync, "failed", (job, err) => {
     logger.error(err, "Failed to sync integration %s", job?.id);
   });
@@ -772,6 +875,7 @@ export const secretQueueFactory = ({
   return {
     // depth is internal only field thus no need to make it available outside
     syncSecrets,
+    startSecretV2Migration,
     syncIntegrations,
     addSecretReminder,
     removeSecretReminder,

backend/src/services/secret/secret-service.ts

@@ -68,6 +68,7 @@ import {
   TGetSecretsRawDTO,
   TGetSecretVersionsDTO,
   TMoveSecretsDTO,
+  TStartSecretsV2MigrationDTO,
   TUpdateBulkSecretDTO,
   TUpdateManySecretRawDTO,
   TUpdateSecretDTO,
@@ -90,7 +91,10 @@ type TSecretServiceFactoryDep = {
   secretBlindIndexDAL: TSecretBlindIndexDALFactory;
   permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
   snapshotService: Pick<TSecretSnapshotServiceFactory, "performSnapshot">;
-  secretQueueService: Pick<TSecretQueueFactory, "syncSecrets" | "handleSecretReminder" | "removeSecretReminder">;
+  secretQueueService: Pick<
+    TSecretQueueFactory,
+    "syncSecrets" | "handleSecretReminder" | "removeSecretReminder" | "startSecretV2Migration"
+  >;
   projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
   secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds">;
   secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
@@ -2638,6 +2642,30 @@ export const secretServiceFactory = ({
     };
   };
 
+  const startSecretV2Migration = async ({
+    projectId,
+    actor,
+    actorId,
+    actorOrgId,
+    actorAuthMethod
+  }: TStartSecretsV2MigrationDTO) => {
+    const { hasRole } = await permissionService.getProjectPermission(
+      actor,
+      actorId,
+      projectId,
+      actorAuthMethod,
+      actorOrgId
+    );
+
+    if (!hasRole(ProjectMembershipRole.Admin))
+      throw new BadRequestError({ message: "Only admins are allowed to take this action" });
+
+    const { shouldUseSecretV2Bridge: isProjectV3 } = await projectBotService.getBotKey(projectId);
+    if (isProjectV3) throw new BadRequestError({ message: "project is already in v3" });
+    await secretQueueService.startSecretV2Migration(projectId);
+    return { message: "Migrating project to new KMS architecture" };
+  };
+
   return {
     attachTags,
     detachTags,
@@ -2659,6 +2687,7 @@ export const secretServiceFactory = ({
     deleteManySecretsRaw,
     getSecretVersions,
     backfillSecretReferences,
-    moveSecrets
+    moveSecrets,
+    startSecretV2Migration
   };
 };

backend/src/services/secret/secret-types.ts

@@ -449,3 +449,5 @@ export enum SecretProtectionType {
   Approval = "approval",
   Direct = "direct"
 }
+
+export type TStartSecretsV2MigrationDTO = TProjectPermission;