From 6bb41913bf01d5a1d6b7b1ed084352d8b82b4b44 Mon Sep 17 00:00:00 2001 From: = Date: Fri, 26 Jul 2024 01:01:04 +0530 Subject: [PATCH] feat: completed migration backend logic --- .../secret-approval-request-secret-dal.ts | 54 ++++++ .../services/secret-snapshot/snapshot-dal.ts | 69 +++++++- backend/src/server/routes/index.ts | 5 +- backend/src/services/secret/secret-queue.ts | 156 +++++++++++++++++- 4 files changed, 278 insertions(+), 6 deletions(-) diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts index ab0ad7ed3..b827f6a21 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts @@ -354,12 +354,66 @@ export const secretApprovalRequestSecretDALFactory = (db: TDbClient) => { throw new DatabaseError({ error, name: "FindByRequestId" }); } }; + // special query for migration to v2 secret + const findByProjectId = async (projectId: string, tx?: Knex) => { + try { + const docs = await (tx || db)(TableName.SecretApprovalRequestSecret) + .join( + TableName.SecretApprovalRequest, + `${TableName.SecretApprovalRequest}.id`, + `${TableName.SecretApprovalRequestSecret}.requestId` + ) + .join(TableName.SecretFolder, `${TableName.SecretApprovalRequest}.folderId`, `${TableName.SecretFolder}.id`) + .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) + .leftJoin( + TableName.SecretApprovalRequestSecretTag, + `${TableName.SecretApprovalRequestSecret}.id`, + `${TableName.SecretApprovalRequestSecretTag}.secretId` + ) + .where({ projectId }) + .select(selectAllTableCols(TableName.SecretApprovalRequestSecret)) + .select( + db.ref("id").withSchema(TableName.SecretApprovalRequestSecretTag).as("secretApprovalTagId"), + db.ref("secretId").withSchema(TableName.SecretApprovalRequestSecretTag).as("secretApprovalTagSecretId"), + db.ref("tagId").withSchema(TableName.SecretApprovalRequestSecretTag).as("secretApprovalTagSecretTagId"), + db.ref("createdAt").withSchema(TableName.SecretApprovalRequestSecretTag).as("secretApprovalTagCreatedAt"), + db.ref("updatedAt").withSchema(TableName.SecretApprovalRequestSecretTag).as("secretApprovalTagUpdatedAt") + ); + const formatedDoc = sqlNestRelationships({ + data: docs, + key: "id", + parentMapper: (data) => SecretApprovalRequestsSecretsSchema.parse(data), + childrenMapper: [ + { + key: "secretApprovalTagId", + label: "tags" as const, + mapper: ({ + secretApprovalTagSecretId, + secretApprovalTagId, + secretApprovalTagUpdatedAt, + secretApprovalTagCreatedAt + }) => ({ + secretApprovalTagSecretId, + secretApprovalTagId, + secretApprovalTagUpdatedAt, + secretApprovalTagCreatedAt + }) + } + ] + }); + return formatedDoc; + } catch (error) { + throw new DatabaseError({ error, name: "FindByRequestId" }); + } + }; + return { ...secretApprovalRequestSecretOrm, insertV2Bridge: secretApprovalRequestSecretV2Orm.insertMany, findByRequestId, findByRequestIdBridgeSecretV2, bulkUpdateNoVersionIncrement, + findByProjectId, insertApprovalSecretTags: secretApprovalRequestSecretTagOrm.insertMany, insertApprovalSecretV2Tags: secretApprovalRequestSecretV2TagOrm.insertMany }; diff --git a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts index 399aec44c..658b6026f 100644 --- a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts +++ b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts @@ -1,5 +1,6 @@ /* eslint-disable no-await-in-loop */ import { Knex } from "knex"; +import { z } from "zod"; import { TDbClient } from "@app/db"; import { @@ -719,6 +720,71 @@ export const snapshotDALFactory = (db: TDbClient) => { } }; + // special query for migration for secret v2 + const findNSecretV1SnapshotByFolderId = async (folderId: string, n = 15, tx?: Knex) => { + try { + const data = await (tx || db.replicaNode())(TableName.Snapshot) + .leftJoin(TableName.SnapshotSecret, `${TableName.Snapshot}.id`, `${TableName.SnapshotSecret}.snapshotId`) + .leftJoin( + TableName.SecretVersion, + `${TableName.SnapshotSecret}.secretVersionId`, + `${TableName.SecretVersion}.id` + ) + .leftJoin( + TableName.SecretVersionTag, + `${TableName.SecretVersionTag}.${TableName.SecretVersion}Id`, + `${TableName.SecretVersion}.id` + ) + .select(selectAllTableCols(TableName.SecretVersion)) + .select( + db.ref("id").withSchema(TableName.Snapshot).as("snapshotId"), + db.ref("createdAt").withSchema(TableName.Snapshot).as("snapshotCreatedAt"), + db.ref("updatedAt").withSchema(TableName.Snapshot).as("snapshotUpdatedAt"), + db.ref("envId").withSchema(TableName.SnapshotSecret).as("snapshotEnvId"), + db.ref("id").withSchema(TableName.SecretVersionTag).as("secretVersionTagId"), + db.ref("secret_versionsId").withSchema(TableName.SecretVersionTag).as("secretVersionTagSecretId"), + db.ref("secret_versionsId").withSchema(TableName.SecretVersionTag).as("secretVersionTagSecretTagId"), + db.raw( + `DENSE_RANK() OVER (partition by ${TableName.Snapshot}."id" ORDER BY ${TableName.SecretVersion}."createdAt") as rank` + ) + ) + .orderBy(`${TableName.Snapshot}.createdAt`, "desc") + .where(`${TableName.Snapshot}.folderId`, folderId) + .andWhere("rank", "<", n); + + return sqlNestRelationships({ + data, + key: "snapshotId", + parentMapper: ({ snapshotId: id, snapshotCreatedAt: createdAt, snapshotUpdatedAt: updatedAt }) => ({ + id, + folderId, + createdAt, + updatedAt + }), + childrenMapper: [ + { + key: "id", + label: "secretVersions" as const, + mapper: (el) => SecretVersionsSchema.extend({ snapshotEnvId: z.string() }).parse(el), + childrenMapper: [ + { + key: "secretVersionTagId", + label: "tags" as const, + mapper: ({ secretVersionTagId, secretVersionTagSecretId, secretVersionTagSecretTagId }) => ({ + id: secretVersionTagId, + secretVersionId: secretVersionTagSecretId, + secretTagId: secretVersionTagSecretTagId + }) + } + ] + } + ] + }); + } catch (error) { + throw new DatabaseError({ error, name: "FindSecretSnapshotDataById" }); + } + }; + return { ...secretSnapshotOrm, findById, @@ -728,6 +794,7 @@ export const snapshotDALFactory = (db: TDbClient) => { countOfSnapshotsByFolderId, findSecretSnapshotDataById, findSecretSnapshotV2DataById, - pruneExcessSnapshots + pruneExcessSnapshots, + findNSecretV1SnapshotByFolderId }; }; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 01aebc1eb..8bd6bec50 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -721,7 +721,10 @@ export const registerRoutes = async ( secretV2BridgeDAL, secretVersionTagV2BridgeDAL, secretRotationDAL, - integrationAuthDAL + integrationAuthDAL, + snapshotDAL, + secretApprovalRequestSecretDAL, + snapshotSecretV2BridgeDAL }); const secretImportService = secretImportServiceFactory({ licenseService, diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index ca9d73386..f50da6a8d 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -1,8 +1,11 @@ /* eslint-disable no-await-in-loop */ import { AxiosError } from "axios"; -import { ProjectUpgradeStatus, ProjectVersion } from "@app/db/schemas"; +import { ProjectUpgradeStatus, ProjectVersion, TSecretSnapshotSecretsV2, TSecretVersionsV2 } from "@app/db/schemas"; +import { TSecretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal"; import { TSecretRotationDALFactory } from "@app/ee/services/secret-rotation/secret-rotation-dal"; +import { TSnapshotDALFactory } from "@app/ee/services/secret-snapshot/snapshot-dal"; +import { TSnapshotSecretV2DALFactory } from "@app/ee/services/secret-snapshot/snapshot-secret-v2-dal"; import { getConfig } from "@app/lib/config/env"; import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; import { daysToMillisecond, secondsToMillis } from "@app/lib/dates"; @@ -73,6 +76,12 @@ type TSecretQueueFactoryDep = { secretVersionV2BridgeDAL: Pick; secretVersionTagV2BridgeDAL: Pick; secretRotationDAL: Pick; + secretApprovalRequestSecretDAL: Pick< + TSecretApprovalRequestSecretDALFactory, + "findByProjectId" | "insertV2Bridge" | "insertApprovalSecretV2Tags" + >; + snapshotDAL: Pick; + snapshotSecretV2BridgeDAL: Pick; }; export type TGetSecrets = { @@ -113,7 +122,10 @@ export const secretQueueFactory = ({ secretVersionV2BridgeDAL, kmsService, secretVersionTagV2BridgeDAL, - secretRotationDAL + secretRotationDAL, + secretApprovalRequestSecretDAL, + snapshotDAL, + snapshotSecretV2BridgeDAL }: TSecretQueueFactoryDep) => { const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => { const appCfg = getConfig(); @@ -869,7 +881,81 @@ export const secretQueueFactory = ({ }), tx ); - await secretV2BridgeDAL.upsertSecretReferences(secretReferences); + await secretV2BridgeDAL.upsertSecretReferences(secretReferences, tx); + } + + const snapshots = await snapshotDAL.findNSecretV1SnapshotByFolderId(folderId, 10, tx); + const projectV3SecretVersions: Record = {}; + const projectV3SecretVersionTags: { secret_versions_v2Id: string; secret_tagsId: string }[] = []; + const projectV3SnapshotSecrets: Omit[] = []; + snapshots.forEach(({ secretVersions = [], ...snapshot }) => { + secretVersions.forEach((el) => { + projectV3SnapshotSecrets.push({ + secretVersionId: el.id, + snapshotId: snapshot.id, + createdAt: snapshot.createdAt, + updatedAt: snapshot.updatedAt, + envId: el.snapshotEnvId + }); + if (projectV3SecretVersions[el.id]) return; + + const key = decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.secretKeyCiphertext, + iv: el.secretKeyIV, + tag: el.secretKeyTag, + key: botKey + }); + const value = decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.secretValueCiphertext, + iv: el.secretValueIV, + tag: el.secretValueTag, + key: botKey + }); + const comment = + el.secretCommentCiphertext && el.secretCommentTag && el.secretCommentIV + ? decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.secretCommentCiphertext, + iv: el.secretCommentIV, + tag: el.secretCommentTag, + key: botKey + }) + : ""; + const encryptedValue = secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob; + + const encryptedComment = comment + ? secretManagerEncryptor({ plainText: Buffer.from(comment) }).cipherTextBlob + : null; + projectV3SecretVersions[el.id] = { + id: el.id, + createdAt: el.createdAt, + updatedAt: el.updatedAt, + skipMultilineEncoding: el.skipMultilineEncoding, + encryptedComment, + encryptedValue, + key, + version: el.version, + type: el.type, + userId: el.userId, + folderId: el.folderId, + metadata: el.metadata, + reminderNote: el.secretReminderNote, + reminderRepeatDays: el.secretReminderRepeatDays, + secretId: el.secretId, + envId: el.envId + }; + el.tags.forEach(({ secretTagId }) => { + projectV3SecretVersionTags.push({ secret_tagsId: secretTagId, secret_versions_v2Id: el.id }); + }); + }); + }); + if (projectV3SecretVersionTags.length) { + await secretVersionV2BridgeDAL.insertMany(Object.values(projectV3SecretVersions), tx); + } + if (projectV3SecretVersionTags.length) { + await secretVersionTagV2BridgeDAL.insertMany(projectV3SecretVersionTags, tx); + } + if (projectV3SnapshotSecrets.length) { + await snapshotSecretV2BridgeDAL.insertMany(projectV3SnapshotSecrets, tx); } } /* @@ -959,7 +1045,6 @@ export const secretQueueFactory = ({ * Secret Rotation Secret Migration * Saving the new encrypted colum * */ - const projectV1SecretRotations = await secretRotationDAL.find({ projectId }, tx); await secretRotationDAL.secretOutputV2InsertMany( projectV1SecretRotations.flatMap((el) => @@ -967,6 +1052,69 @@ export const secretQueueFactory = ({ ), tx ); + + /* + * approvals + * */ + const projectV1ApprovalSecrets = await secretApprovalRequestSecretDAL.findByProjectId(projectId); + if (projectV1ApprovalSecrets.length) { + await secretApprovalRequestSecretDAL.insertV2Bridge( + projectV1ApprovalSecrets.map((el) => { + const key = decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.secretKeyCiphertext, + iv: el.secretKeyIV, + tag: el.secretKeyTag, + key: botKey + }); + const value = decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.secretValueCiphertext, + iv: el.secretValueIV, + tag: el.secretValueTag, + key: botKey + }); + const comment = + el.secretCommentCiphertext && el.secretCommentTag && el.secretCommentIV + ? decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.secretCommentCiphertext, + iv: el.secretCommentIV, + tag: el.secretCommentTag, + key: botKey + }) + : ""; + const encryptedValue = secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob; + const encryptedComment = comment + ? secretManagerEncryptor({ plainText: Buffer.from(comment) }).cipherTextBlob + : null; + return { + id: el.id, + createdAt: el.createdAt, + updatedAt: el.updatedAt, + skipMultilineEncoding: el.skipMultilineEncoding, + encryptedComment, + encryptedValue, + key, + version: el.version, + metadata: el.metadata, + reminderNote: el.secretReminderNote, + reminderRepeatDays: el.secretReminderRepeatDays, + requestId: el.requestId, + op: el.op, + secretId: el.secretId, + secretVersion: el.secretVersion + }; + }), + tx + ); + } + const projectV1SecretApprovalSecretTags = projectV1ApprovalSecrets.flatMap((el) => + el.tags.map((tag) => ({ + secretId: tag.secretApprovalTagSecretId, + tagId: tag.secretApprovalTagId + })) + ); + if (projectV1SecretApprovalSecretTags.length) { + await secretApprovalRequestSecretDAL.insertApprovalSecretV2Tags(projectV1SecretApprovalSecretTags, tx); + } await projectDAL.updateById(projectId, { upgradeStatus: null, version: ProjectVersion.V3 }, tx); }); });