From b0e7304bffed246ad159599cb5e6e0f28c83a4d3 Mon Sep 17 00:00:00 2001 From: Maidul Islam Date: Mon, 5 Jun 2023 20:19:15 -0700 Subject: [PATCH 1/4] Patch backfill data --- backend/src/utils/setup/backfillData.ts | 30 +++++++++++++++++++------ 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/backend/src/utils/setup/backfillData.ts b/backend/src/utils/setup/backfillData.ts index ba56e266b..f78ec08fc 100644 --- a/backend/src/utils/setup/backfillData.ts +++ b/backend/src/utils/setup/backfillData.ts @@ -335,6 +335,19 @@ export const backfillSecretFolders = async () => { } ); + await SecretVersion.updateMany( + { + folder: { + $exists: false, + }, + }, + { + $set: { + folder: "root", + }, + } + ); + let secretSnapshots = await SecretSnapshot.find({ environment: { $exists: false, @@ -352,12 +365,15 @@ export const backfillSecretFolders = async () => { groupSnapByEnv[secVer.environment].push(secVer); }); - const newSnapshots = Object.keys(groupSnapByEnv).map((snapEnv) => ({ - ...secSnapshot.toObject({ virtuals: false }), - _id: new Types.ObjectId(), - environment: snapEnv, - secretVersions: groupSnapByEnv[snapEnv], - })); + const newSnapshots = Object.keys(groupSnapByEnv).map((snapEnv) => { + const secretIdsOfEnvGroup = groupSnapByEnv[snapEnv] ? groupSnapByEnv[snapEnv].map(secretVersion => secretVersion._id) : [] + return { + ...secSnapshot.toObject({ virtuals: false }), + _id: new Types.ObjectId(), + environment: snapEnv, + secretVersions: secretIdsOfEnvGroup, + } + }); await SecretSnapshot.insertMany(newSnapshots); await secSnapshot.delete(); @@ -369,7 +385,7 @@ export const backfillSecretFolders = async () => { }, }) .populate<{ secretVersions: ISecretVersion[] }>("secretVersions") - .limit(50); + .limit(5000); } console.log("Migration: Folder migration v1 complete") From c0fbe82ecb7095db7fcb30b8876734f43f1ec4bf Mon Sep 17 00:00:00 2001 From: Maidul Islam Date: Mon, 5 Jun 2023 20:21:09 -0700 Subject: [PATCH 2/4] update populate number --- backend/src/utils/setup/backfillData.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/utils/setup/backfillData.ts b/backend/src/utils/setup/backfillData.ts index f78ec08fc..3015de9aa 100644 --- a/backend/src/utils/setup/backfillData.ts +++ b/backend/src/utils/setup/backfillData.ts @@ -385,7 +385,7 @@ export const backfillSecretFolders = async () => { }, }) .populate<{ secretVersions: ISecretVersion[] }>("secretVersions") - .limit(5000); + .limit(50); } console.log("Migration: Folder migration v1 complete") From e28d0cbace3d3a8ff1cd5a02c5a9225208965b9d Mon Sep 17 00:00:00 2001 From: Maidul Islam Date: Mon, 5 Jun 2023 23:12:48 -0700 Subject: [PATCH 3/4] bring back tags to secret version --- .../ee/controllers/v1/secretSnapshotController.ts | 8 +++++++- backend/src/ee/models/secretVersion.ts | 6 ++++++ backend/src/utils/setup/backfillData.ts | 14 ++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/backend/src/ee/controllers/v1/secretSnapshotController.ts b/backend/src/ee/controllers/v1/secretSnapshotController.ts index a6ee2ce4d..ace016afe 100644 --- a/backend/src/ee/controllers/v1/secretSnapshotController.ts +++ b/backend/src/ee/controllers/v1/secretSnapshotController.ts @@ -19,7 +19,13 @@ export const getSecretSnapshot = async (req: Request, res: Response) => { secretSnapshot = await SecretSnapshot.findById(secretSnapshotId) .lean() - .populate<{ secretVersions: ISecretVersion[] }>("secretVersions") + .populate<{ secretVersions: ISecretVersion[] }>({ + path: 'secretVersions', + populate: { + path: 'tags', + model: 'Tag' + } + }) .populate<{ folderVersion: TFolderRootVersionSchema }>("folderVersion"); if (!secretSnapshot) throw new Error("Failed to find secret snapshot"); diff --git a/backend/src/ee/models/secretVersion.ts b/backend/src/ee/models/secretVersion.ts index ea2a9fd2a..b915e640e 100644 --- a/backend/src/ee/models/secretVersion.ts +++ b/backend/src/ee/models/secretVersion.ts @@ -27,6 +27,7 @@ export interface ISecretVersion { keyEncoding: "utf8" | "base64"; createdAt: string; folder?: string; + tags?: string[]; } const secretVersionSchema = new Schema( @@ -112,6 +113,11 @@ const secretVersionSchema = new Schema( type: String, required: true, }, + tags: { + ref: 'Tag', + type: [Schema.Types.ObjectId], + default: [] + }, }, { timestamps: true, diff --git a/backend/src/utils/setup/backfillData.ts b/backend/src/utils/setup/backfillData.ts index 3015de9aa..5886ff776 100644 --- a/backend/src/utils/setup/backfillData.ts +++ b/backend/src/utils/setup/backfillData.ts @@ -348,6 +348,20 @@ export const backfillSecretFolders = async () => { } ); + // Back fill because tags were missing in secret versions + await SecretVersion.updateMany( + { + tags: { + $exists: false, + }, + }, + { + $set: { + tags: [], + }, + } + ); + let secretSnapshots = await SecretSnapshot.find({ environment: { $exists: false, From aaca66e5a46d9d7a084be475419eaed9ad906cf2 Mon Sep 17 00:00:00 2001 From: Tuan Dang Date: Tue, 6 Jun 2023 14:24:06 +0100 Subject: [PATCH 4/4] Patch support for ENCRYPTION_KEY and ROOT_ENCRYPTION_KEY in generateSecretBlindIndexHelper --- backend/src/helpers/secrets.ts | 56 ++++++++++++++++++++++------- backend/src/validation/workspace.ts | 1 - 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/backend/src/helpers/secrets.ts b/backend/src/helpers/secrets.ts index 193c82698..69ca78e7e 100644 --- a/backend/src/helpers/secrets.ts +++ b/backend/src/helpers/secrets.ts @@ -185,26 +185,56 @@ const generateSecretBlindIndexHelper = async ({ workspaceId: Types.ObjectId; }) => { // check if workspace blind index data exists + const encryptionKey = await getEncryptionKey(); + const rootEncryptionKey = await getRootEncryptionKey(); + const secretBlindIndexData = await SecretBlindIndexData.findOne({ workspace: workspaceId, - }); + }).select('+algorithm +keyEncoding'); if (!secretBlindIndexData) throw SecretBlindIndexDataNotFoundError(); - // decrypt workspace salt - const salt = decryptSymmetric128BitHexKeyUTF8({ - ciphertext: secretBlindIndexData.encryptedSaltCiphertext, - iv: secretBlindIndexData.saltIV, - tag: secretBlindIndexData.saltTag, - key: await getEncryptionKey(), - }); + let salt; + if ( + rootEncryptionKey && + secretBlindIndexData.keyEncoding === ENCODING_SCHEME_BASE64 + ) { + salt = client.decryptSymmetric( + secretBlindIndexData.encryptedSaltCiphertext, + rootEncryptionKey, + secretBlindIndexData.saltIV, + secretBlindIndexData.saltTag + ); - const secretBlindIndex = await generateSecretBlindIndexWithSaltHelper({ - secretName, - salt, - }); + const secretBlindIndex = await generateSecretBlindIndexWithSaltHelper({ + secretName, + salt, + }); - return secretBlindIndex; + return secretBlindIndex; + } else if ( + encryptionKey && + secretBlindIndexData.keyEncoding === ENCODING_SCHEME_UTF8 + ) { + // decrypt workspace salt + salt = decryptSymmetric128BitHexKeyUTF8({ + ciphertext: secretBlindIndexData.encryptedSaltCiphertext, + iv: secretBlindIndexData.saltIV, + tag: secretBlindIndexData.saltTag, + key: encryptionKey, + }); + + const secretBlindIndex = await generateSecretBlindIndexWithSaltHelper({ + secretName, + salt, + }); + + return secretBlindIndex; + } + + throw InternalServerError({ + message: 'Failed to generate secret blind index' + }); }; /** diff --git a/backend/src/validation/workspace.ts b/backend/src/validation/workspace.ts index 1cbdca420..b7a04634f 100644 --- a/backend/src/validation/workspace.ts +++ b/backend/src/validation/workspace.ts @@ -51,7 +51,6 @@ export const validateClientForWorkspace = async ({ requiredPermissions?: string[]; requireBlindIndicesEnabled: boolean; }) => { - const workspace = await Workspace.findById(workspaceId); if (!workspace) throw WorkspaceNotFoundError({