checkpoint

backend/src/services/secret-folder/secret-folder-dal.ts

@@ -331,6 +331,27 @@ export const secretFolderDALFactory = (db: TDbClient) => {
     }
   };
 
+  // special query for project migration
+  const findByProjectId = async (projectId: string, tx?: Knex) => {
+    try {
+      const folders = await (tx || db.replicaNode())(TableName.SecretFolder)
+        .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`)
+        .join(TableName.Project, `${TableName.Environment}.projectId`, `${TableName.Project}.id`)
+        .select(selectAllTableCols(TableName.SecretFolder))
+        .where({ projectId })
+        .select(
+          db.ref("id").withSchema(TableName.Environment).as("envId"),
+          db.ref("slug").withSchema(TableName.Environment).as("envSlug"),
+          db.ref("name").withSchema(TableName.Environment).as("envName"),
+          db.ref("projectId").withSchema(TableName.Environment),
+          db.ref("version").withSchema(TableName.Project).as("projectVersion")
+        );
+      return folders;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "Find by id" });
+    }
+  };
+
   return {
     ...secretFolderOrm,
     update,
@@ -338,6 +359,7 @@ export const secretFolderDALFactory = (db: TDbClient) => {
     findById,
     findByManySecretPath,
     findSecretPathByFolderIds,
-    findClosestFolder
+    findClosestFolder,
+    findByProjectId
   };
 };

backend/src/services/secret/secret-queue.ts

@@ -41,6 +41,7 @@ import {
   TRemoveSecretReminderDTO,
   TSyncSecretsDTO
 } from "./secret-types";
+import { ProjectUpgradeStatus, ProjectVersion } from "@app/db/schemas";
 
 export type TSecretQueueFactory = ReturnType<typeof secretQueueFactory>;
 type TSecretQueueFactoryDep = {
@@ -52,7 +53,7 @@ type TSecretQueueFactoryDep = {
   secretDAL: TSecretDALFactory;
   secretImportDAL: Pick<TSecretImportDALFactory, "find">;
   webhookDAL: Pick<TWebhookDALFactory, "findAllWebhooks" | "transaction" | "update" | "bulkUpdate">;
-  projectEnvDAL: Pick<TProjectEnvDALFactory, "findOne">;
+  projectEnvDAL: Pick<TProjectEnvDALFactory, "findOne" | "find">;
   projectDAL: TProjectDALFactory;
   projectBotDAL: TProjectBotDALFactory;
   projectMembershipDAL: Pick<TProjectMembershipDALFactory, "findAllProjectMembers">;
@@ -761,6 +762,108 @@ export const secretQueueFactory = ({
     });
   });
 
+  const startSecretV2Migration = async (projectId: string) => {
+    await queueService.queue(
+      QueueName.ProjectV3Migration,
+      QueueJobs.ProjectV3Migration,
+      { projectId },
+      {
+        attempts: 2,
+        backoff: {
+          type: "exponential",
+          delay: 3000
+        },
+        removeOnComplete: true,
+        removeOnFail: true
+      }
+    );
+  };
+
+  const MIGRATION_BATCH_SIZE = 10000;
+  queueService.start(QueueName.ProjectV3Migration, async (job) => {
+    const { projectId } = job.data;
+    const { botKey, shouldUseSecretV2Bridge: isProjectUpgradedToV3 } = await projectBotService.getBotKey(projectId);
+    if (isProjectUpgradedToV3) {
+      return;
+    }
+    if (!botKey) throw new BadRequestError({ message: "Bot not found" });
+    await projectDAL.updateById(projectId, { upgradeStatus: ProjectUpgradeStatus.InProgress });
+    const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({
+      projectId,
+      type: KmsDataKey.SecretManager
+    });
+
+    const folders = await folderDAL.findByProjectId(projectId);
+    // except secret version and snapshot migrate rest of everything first in a transaction
+    await secretDAL.transaction(async (tx) => {
+      for (const folder of folders) {
+        const folderId = folder.id;
+        let projectV1Secrets;
+        do {
+          // eslint-disable-next-line no-await-in-loop
+          projectV1Secrets = await secretDAL.find({ folderId }, { limit: MIGRATION_BATCH_SIZE, tx });
+          await secretV2BridgeDAL.insertMany(
+            projectV1Secrets.map((el) => {
+              const key = decryptSymmetric128BitHexKeyUTF8({
+                ciphertext: el.secretKeyCiphertext,
+                iv: el.secretKeyIV,
+                tag: el.secretKeyTag,
+                key: botKey
+              });
+              const value = decryptSymmetric128BitHexKeyUTF8({
+                ciphertext: el.secretValueCiphertext,
+                iv: el.secretValueIV,
+                tag: el.secretValueTag,
+                key: botKey
+              });
+              const comment =
+                el.secretCommentCiphertext && el.secretCommentTag && el.secretCommentIV
+                  ? decryptSymmetric128BitHexKeyUTF8({
+                      ciphertext: el.secretCommentCiphertext,
+                      iv: el.secretCommentIV,
+                      tag: el.secretCommentTag,
+                      key: botKey
+                    })
+                  : "";
+              const encryptedValue = secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob;
+              const encryptedComment = comment
+                ? secretManagerEncryptor({ plainText: Buffer.from(comment) }).cipherTextBlob
+                : null;
+              return {
+                id: el.id,
+                createdAt: el.createdAt,
+                updatedAt: el.updatedAt,
+                skipMultilineEncoding: el.skipMultilineEncoding,
+                encryptedComment,
+                encryptedValue,
+                key,
+                version: el.version,
+                type: el.type,
+                userId: el.userId,
+                folderId: el.folderId,
+                metadata: el.metadata,
+                reminderNote: el.secretReminderNote,
+                reminderRepeatDays: el.secretReminderRepeatDays
+              };
+            }),
+            tx
+          );
+          projectV1Secrets = await secretDAL.delete({ folderId, $in: { id: projectV1Secrets.map((el) => el.id) } }, tx);
+        } while (projectV1Secrets.length > 0);
+      }
+      await projectDAL.updateById(projectId, { upgradeStatus: null, version: ProjectVersion.V3 }, tx);
+    });
+  });
+
+  // eslint-disable-next-line
+  queueService.listen(QueueName.ProjectV3Migration, "failed", async (job, err) => {
+    if (job?.data) {
+      const { projectId } = job.data;
+      await projectDAL.updateById(projectId, { upgradeStatus: ProjectUpgradeStatus.Failed });
+      logger.error(err, `Failed to migrate project to v3: ${projectId}`);
+    }
+  });
+
   queueService.listen(QueueName.IntegrationSync, "failed", (job, err) => {
     logger.error(err, "Failed to sync integration %s", job?.id);
   });
@@ -772,6 +875,7 @@ export const secretQueueFactory = ({
   return {
     // depth is internal only field thus no need to make it available outside
     syncSecrets,
+    startSecretV2Migration,
     syncIntegrations,
     addSecretReminder,
     removeSecretReminder,

backend/src/services/secret/secret-service.ts

@@ -68,6 +68,7 @@ import {
   TGetSecretsRawDTO,
   TGetSecretVersionsDTO,
   TMoveSecretsDTO,
+  TStartSecretsV2MigrationDTO,
   TUpdateBulkSecretDTO,
   TUpdateManySecretRawDTO,
   TUpdateSecretDTO,
@@ -90,7 +91,10 @@ type TSecretServiceFactoryDep = {
   secretBlindIndexDAL: TSecretBlindIndexDALFactory;
   permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
   snapshotService: Pick<TSecretSnapshotServiceFactory, "performSnapshot">;
-  secretQueueService: Pick<TSecretQueueFactory, "syncSecrets" | "handleSecretReminder" | "removeSecretReminder">;
+  secretQueueService: Pick<
+    TSecretQueueFactory,
+    "syncSecrets" | "handleSecretReminder" | "removeSecretReminder" | "startSecretV2Migration"
+  >;
   projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
   secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds">;
   secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
@@ -2638,6 +2642,30 @@ export const secretServiceFactory = ({
     };
   };
 
+  const startSecretV2Migration = async ({
+    projectId,
+    actor,
+    actorId,
+    actorOrgId,
+    actorAuthMethod
+  }: TStartSecretsV2MigrationDTO) => {
+    const { hasRole } = await permissionService.getProjectPermission(
+      actor,
+      actorId,
+      projectId,
+      actorAuthMethod,
+      actorOrgId
+    );
+
+    if (!hasRole(ProjectMembershipRole.Admin))
+      throw new BadRequestError({ message: "Only admins are allowed to take this action" });
+
+    const { shouldUseSecretV2Bridge: isProjectV3 } = await projectBotService.getBotKey(projectId);
+    if (isProjectV3) throw new BadRequestError({ message: "project is already in v3" });
+    await secretQueueService.startSecretV2Migration(projectId);
+    return { message: "Migrating project to new KMS architecture" };
+  };
+
   return {
     attachTags,
     detachTags,
@@ -2659,6 +2687,7 @@ export const secretServiceFactory = ({
     deleteManySecretsRaw,
     getSecretVersions,
     backfillSecretReferences,
-    moveSecrets
+    moveSecrets,
+    startSecretV2Migration
   };
 };

backend/src/services/secret/secret-types.ts

@@ -449,3 +449,5 @@ export enum SecretProtectionType {
   Approval = "approval",
   Direct = "direct"
 }
+
+export type TStartSecretsV2MigrationDTO = TProjectPermission;