From e2caa98c74f61e6393d730207834b43d4610865e Mon Sep 17 00:00:00 2001 From: = Date: Fri, 26 Jul 2024 01:27:44 +0530 Subject: [PATCH] feat: finished migrator logic --- .../services/secret-snapshot/snapshot-dal.ts | 10 +++++--- backend/src/server/routes/index.ts | 3 ++- .../resource-cleanup-queue.ts | 6 ++++- backend/src/services/secret/secret-queue.ts | 12 ++++++---- backend/src/services/secret/secret-service.ts | 24 ++++++++++++------- 5 files changed, 36 insertions(+), 19 deletions(-) diff --git a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts index 658b6026f..b0dec99fb 100644 --- a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts +++ b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts @@ -723,7 +723,7 @@ export const snapshotDALFactory = (db: TDbClient) => { // special query for migration for secret v2 const findNSecretV1SnapshotByFolderId = async (folderId: string, n = 15, tx?: Knex) => { try { - const data = await (tx || db.replicaNode())(TableName.Snapshot) + const query = (tx || db.replicaNode())(TableName.Snapshot) .leftJoin(TableName.SnapshotSecret, `${TableName.Snapshot}.id`, `${TableName.SnapshotSecret}.snapshotId`) .leftJoin( TableName.SecretVersion, @@ -749,8 +749,12 @@ export const snapshotDALFactory = (db: TDbClient) => { ) ) .orderBy(`${TableName.Snapshot}.createdAt`, "desc") - .where(`${TableName.Snapshot}.folderId`, folderId) - .andWhere("rank", "<", n); + .where(`${TableName.Snapshot}.folderId`, folderId); + const data = await (tx || db) + .with("w", query) + .select("*") + .from[number]>("w") + .andWhere("w.rank", "<", n); return sqlNestRelationships({ data, diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 8bd6bec50..fded5cb57 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -1016,7 +1016,8 @@ export const registerRoutes = async ( secretFolderVersionDAL: folderVersionDAL, snapshotDAL, identityAccessTokenDAL, - secretSharingDAL + secretSharingDAL, + secretVersionV2DAL: secretVersionV2BridgeDAL }); const oidcService = oidcConfigServiceFactory({ diff --git a/backend/src/services/resource-cleanup/resource-cleanup-queue.ts b/backend/src/services/resource-cleanup/resource-cleanup-queue.ts index 2e01e3549..cb9756dd4 100644 --- a/backend/src/services/resource-cleanup/resource-cleanup-queue.ts +++ b/backend/src/services/resource-cleanup/resource-cleanup-queue.ts @@ -7,11 +7,13 @@ import { TIdentityAccessTokenDALFactory } from "../identity-access-token/identit import { TSecretVersionDALFactory } from "../secret/secret-version-dal"; import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal"; import { TSecretSharingDALFactory } from "../secret-sharing/secret-sharing-dal"; +import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal"; type TDailyResourceCleanUpQueueServiceFactoryDep = { auditLogDAL: Pick; identityAccessTokenDAL: Pick; secretVersionDAL: Pick; + secretVersionV2DAL: Pick; secretFolderVersionDAL: Pick; snapshotDAL: Pick; secretSharingDAL: Pick; @@ -27,7 +29,8 @@ export const dailyResourceCleanUpQueueServiceFactory = ({ secretVersionDAL, secretFolderVersionDAL, identityAccessTokenDAL, - secretSharingDAL + secretSharingDAL, + secretVersionV2DAL }: TDailyResourceCleanUpQueueServiceFactoryDep) => { queueService.start(QueueName.DailyResourceCleanUp, async () => { logger.info(`${QueueName.DailyResourceCleanUp}: queue task started`); @@ -36,6 +39,7 @@ export const dailyResourceCleanUpQueueServiceFactory = ({ await secretSharingDAL.pruneExpiredSharedSecrets(); await snapshotDAL.pruneExcessSnapshots(); await secretVersionDAL.pruneExcessVersions(); + await secretVersionV2DAL.pruneExcessVersions(); await secretFolderVersionDAL.pruneExcessVersions(); logger.info(`${QueueName.DailyResourceCleanUp}: queue task completed`); }); diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index f50da6a8d..87a24fdf1 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -885,7 +885,7 @@ export const secretQueueFactory = ({ } const snapshots = await snapshotDAL.findNSecretV1SnapshotByFolderId(folderId, 10, tx); - const projectV3SecretVersions: Record = {}; + const projectV3SecretVersionsGroupById: Record = {}; const projectV3SecretVersionTags: { secret_versions_v2Id: string; secret_tagsId: string }[] = []; const projectV3SnapshotSecrets: Omit[] = []; snapshots.forEach(({ secretVersions = [], ...snapshot }) => { @@ -897,7 +897,7 @@ export const secretQueueFactory = ({ updatedAt: snapshot.updatedAt, envId: el.snapshotEnvId }); - if (projectV3SecretVersions[el.id]) return; + if (projectV3SecretVersionsGroupById[el.id]) return; const key = decryptSymmetric128BitHexKeyUTF8({ ciphertext: el.secretKeyCiphertext, @@ -925,7 +925,7 @@ export const secretQueueFactory = ({ const encryptedComment = comment ? secretManagerEncryptor({ plainText: Buffer.from(comment) }).cipherTextBlob : null; - projectV3SecretVersions[el.id] = { + projectV3SecretVersionsGroupById[el.id] = { id: el.id, createdAt: el.createdAt, updatedAt: el.updatedAt, @@ -948,12 +948,14 @@ export const secretQueueFactory = ({ }); }); }); - if (projectV3SecretVersionTags.length) { - await secretVersionV2BridgeDAL.insertMany(Object.values(projectV3SecretVersions), tx); + const projectV3SecretVersions = Object.values(projectV3SecretVersionsGroupById); + if (projectV3SecretVersions.length) { + await secretVersionV2BridgeDAL.insertMany(projectV3SecretVersions, tx); } if (projectV3SecretVersionTags.length) { await secretVersionTagV2BridgeDAL.insertMany(projectV3SecretVersionTags, tx); } + if (projectV3SnapshotSecrets.length) { await snapshotSecretV2BridgeDAL.insertMany(projectV3SnapshotSecrets, tx); } diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 18fd7ff8d..36e8db64a 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -1935,15 +1935,21 @@ export const secretServiceFactory = ({ offset = 0, secretId }: TGetSecretVersionsDTO) => { - const secretVersionV2 = await secretV2BridgeService.getSecretVersions({ - actorId, - actor, - actorOrgId, - actorAuthMethod, - limit, - offset, - secretId - }); + const secretVersionV2 = await secretV2BridgeService + .getSecretVersions({ + actorId, + actor, + actorOrgId, + actorAuthMethod, + limit, + offset, + secretId + }) + .catch((err) => { + if ((err as Error).message === "BadRequest: Failed to find secret") { + return null; + } + }); if (secretVersionV2) return secretVersionV2; const secret = await secretDAL.findById(secretId);