From 65bec23292d98cbebc26ed08d9fd537d57f6a32c Mon Sep 17 00:00:00 2001 From: Tuan Dang Date: Thu, 16 Feb 2023 22:43:53 +0700 Subject: [PATCH] Begin rewiring frontend to use batch route for CRUD secret ops --- .../src/controllers/v2/secretsController.ts | 251 +++++++++++++++++- .../src/ee/controllers/v1/secretController.ts | 8 +- backend/src/ee/models/secretVersion.ts | 11 +- backend/src/routes/v2/secrets.ts | 18 ++ backend/src/types/secret/index.d.ts | 38 +++ frontend/src/pages/api/files/batchSecrets.ts | 38 +++ frontend/src/pages/dashboard/[id].tsx | 77 +++++- 7 files changed, 413 insertions(+), 28 deletions(-) create mode 100644 frontend/src/pages/api/files/batchSecrets.ts diff --git a/backend/src/controllers/v2/secretsController.ts b/backend/src/controllers/v2/secretsController.ts index 9a011bc5f..8f516b2dd 100644 --- a/backend/src/controllers/v2/secretsController.ts +++ b/backend/src/controllers/v2/secretsController.ts @@ -1,7 +1,8 @@ import to from 'await-to-js'; import { Types } from 'mongoose'; import { Request, Response } from 'express'; -import { ISecret, Membership, Secret, Workspace } from '../../models'; +import { ISecret, Secret } from '../../models'; +import { IAction } from '../../ee/models'; import { SECRET_PERSONAL, SECRET_SHARED, @@ -20,6 +21,250 @@ import { ABILITY_READ, ABILITY_WRITE } from '../../variables/organization'; import { userHasNoAbility, userHasWorkspaceAccess, userHasWriteOnlyAbility } from '../../ee/helpers/checkMembershipPermissions'; import Tag from '../../models/tag'; import _ from 'lodash'; +import { + BatchSecretRequest, + BatchSecret +} from '../../types/secret'; + +/** + * Peform a batch of any specified CUD secret operations + * @param req + * @param res + */ +export const batchSecrets = async (req: Request, res: Response) => { + const channel = getChannelFromUserAgent(req.headers['user-agent']); + const { + workspaceId, + environment, + requests + }: { + workspaceId: string; + environment: string; + requests: BatchSecretRequest[]; + }= req.body; + + // construct object containing all secrets + // listed across requests + const listedSecretsObj: { + [key: string]: { + version: number; + type: string; + } + } = (await Secret.find({ + _id: { + $in: requests + .map((request) => request.secret._id) + .filter((secretId) => secretId !== undefined) + } + }).select('version type')).reduce((obj: any, secret: ISecret) => ({ + ...obj, + [secret._id.toString()]: secret + }), {}); + + + const createSecrets: BatchSecret[] = []; + const updateSecrets: BatchSecret[] = []; + const deleteSecrets: Types.ObjectId[] = []; + const actions: IAction[] = []; + + requests.forEach((request) => { + switch (request.method) { + case 'POST': + createSecrets.push({ + ...request.secret, + version: 1, + user: request.secret.type === SECRET_PERSONAL ? req.user : undefined, + environment, + workspace: new Types.ObjectId(workspaceId) + }); + break; + case 'PATCH': + updateSecrets.push({ + ...request.secret, + _id: new Types.ObjectId(request.secret._id) + }); + break; + case 'DELETE': + deleteSecrets.push(new Types.ObjectId(request.secret._id)); + break; + } + }); + + // handle create secrets + let createdSecrets: ISecret[] = []; + if (createSecrets.length > 0) { + createdSecrets = await Secret.insertMany(createSecrets); + // (EE) add secret versions for new secrets + await EESecretService.addSecretVersions({ + secretVersions: createdSecrets.map((n: any) => { + return ({ + ...n._doc, + _id: new Types.ObjectId(), + secret: n._id, + isDeleted: false + }); + }) + }); + + const addAction = await EELogService.createAction({ + name: ACTION_ADD_SECRETS, + userId: req.user._id, + workspaceId: new Types.ObjectId(workspaceId), + secretIds: createdSecrets.map((n) => n._id) + }) as IAction; + actions.push(addAction); + + if (postHogClient) { + postHogClient.capture({ + event: 'secrets added', + distinctId: req.user.email, + properties: { + numberOfSecrets: createdSecrets.length, + environment, + workspaceId, + channel, + userAgent: req.headers?.['user-agent'] + } + }); + } + } + + // handle update secrets + let updatedSecrets: ISecret[] = []; + if (updateSecrets.length > 0) { + const updateOperations = updateSecrets.map((u) => ({ + updateOne: { + filter: { _id: new Types.ObjectId(u._id) }, + update: { + $inc: { + version: 1 + }, + ...u, + _id: new Types.ObjectId(u._id) + } + } + })); + + await Secret.bulkWrite(updateOperations); + + const secretVersions = updateSecrets.map((u) => ({ + secret: new Types.ObjectId(u._id), + version: listedSecretsObj[u._id.toString()].version, + workspace: new Types.ObjectId(workspaceId), + type: listedSecretsObj[u._id.toString()].type, + environment, + isDeleted: false, + secretKeyCiphertext: u.secretKeyCiphertext, + secretKeyIV: u.secretKeyIV, + secretKeyTag: u.secretKeyTag, + secretValueCiphertext: u.secretValueCiphertext, + secretValueIV: u.secretValueIV, + secretValueTag: u.secretValueTag, + secretCommentCiphertext: u.secretCommentCiphertext, + secretCommentIV: u.secretCommentIV, + secretCommentTag: u.secretCommentTag, + tags: u.tags + })); + + await EESecretService.addSecretVersions({ + secretVersions + }); + + updatedSecrets = await Secret.find({ + _id: { + $in: updateSecrets.map((u) => new Types.ObjectId(u._id)) + } + }); + + if (postHogClient) { + postHogClient.capture({ + event: 'secrets modified', + distinctId: req.user.email, + properties: { + numberOfSecrets: updateSecrets.length, + environment, + workspaceId, + channel, + userAgent: req.headers?.['user-agent'] + } + }); + } + } + + // handle delete secrets + if (deleteSecrets.length > 0) { + await Secret.deleteMany({ + _id: { + $in: deleteSecrets + } + }); + + await EESecretService.markDeletedSecretVersions({ + secretIds: deleteSecrets + }); + + const deleteAction = await EELogService.createAction({ + name: ACTION_DELETE_SECRETS, + userId: req.user._id, + workspaceId: new Types.ObjectId(workspaceId), + secretIds: deleteSecrets + }) as IAction; + actions.push(deleteAction); + + if (postHogClient) { + postHogClient.capture({ + event: 'secrets deleted', + distinctId: req.user.email, + properties: { + numberOfSecrets: deleteSecrets.length, + environment, + workspaceId, + channel: channel, + userAgent: req.headers?.['user-agent'] + } + }); + } + } + + if (actions.length > 1) { + // (EE) create (audit) log + await EELogService.createLog({ + userId: req.user._id.toString(), + workspaceId: new Types.ObjectId(workspaceId), + actions, + channel, + ipAddress: req.ip + }); + } + + // // trigger event - push secrets + await EventService.handleEvent({ + event: eventPushSecrets({ + workspaceId + }) + }); + + // (EE) take a secret snapshot + await EESecretService.takeSecretSnapshot({ + workspaceId + }); + + const resObj: { [key: string]: ISecret[] | string[] } = {} + + if (createSecrets.length > 0) { + resObj['createdSecrets'] = createdSecrets; + } + + if (updateSecrets.length > 0) { + resObj['updatedSecrets'] = updatedSecrets; + } + + if (deleteSecrets.length > 0) { + resObj['deletedSecrets'] = deleteSecrets.map((d) => d.toString()); + } + + return res.status(200).send(resObj); +} /** * Create secret(s) for workspace with id [workspaceId] and environment [environment] @@ -166,11 +411,9 @@ export const createSecrets = async (req: Request, res: Response) => { secretKeyCiphertext, secretKeyIV, secretKeyTag, - secretKeyHash, secretValueCiphertext, secretValueIV, secretValueTag, - secretValueHash, secretCommentCiphertext, secretCommentIV, secretCommentTag, @@ -187,11 +430,9 @@ export const createSecrets = async (req: Request, res: Response) => { secretKeyCiphertext, secretKeyIV, secretKeyTag, - secretKeyHash, secretValueCiphertext, secretValueIV, secretValueTag, - secretValueHash, secretCommentCiphertext, secretCommentIV, secretCommentTag, diff --git a/backend/src/ee/controllers/v1/secretController.ts b/backend/src/ee/controllers/v1/secretController.ts index 562c8aa88..e1aca670f 100644 --- a/backend/src/ee/controllers/v1/secretController.ts +++ b/backend/src/ee/controllers/v1/secretController.ts @@ -158,11 +158,9 @@ export const rollbackSecretVersion = async (req: Request, res: Response) => { secretKeyCiphertext, secretKeyIV, secretKeyTag, - secretKeyHash, secretValueCiphertext, secretValueIV, secretValueTag, - secretValueHash } = oldSecretVersion; // update secret @@ -179,11 +177,9 @@ export const rollbackSecretVersion = async (req: Request, res: Response) => { secretKeyCiphertext, secretKeyIV, secretKeyTag, - secretKeyHash, secretValueCiphertext, secretValueIV, secretValueTag, - secretValueHash }, { new: true @@ -204,11 +200,9 @@ export const rollbackSecretVersion = async (req: Request, res: Response) => { secretKeyCiphertext, secretKeyIV, secretKeyTag, - secretKeyHash, secretValueCiphertext, secretValueIV, - secretValueTag, - secretValueHash + secretValueTag }).save(); // take secret snapshot diff --git a/backend/src/ee/models/secretVersion.ts b/backend/src/ee/models/secretVersion.ts index efa042765..3095b52fb 100644 --- a/backend/src/ee/models/secretVersion.ts +++ b/backend/src/ee/models/secretVersion.ts @@ -5,22 +5,19 @@ import { } from '../../variables'; export interface ISecretVersion { - _id: Types.ObjectId; secret: Types.ObjectId; version: number; workspace: Types.ObjectId; // new type: string; // new - user: Types.ObjectId; // new + user?: Types.ObjectId; // new environment: string; // new isDeleted: boolean; secretKeyCiphertext: string; secretKeyIV: string; secretKeyTag: string; - secretKeyHash: string; secretValueCiphertext: string; secretValueIV: string; secretValueTag: string; - secretValueHash: string; tags?: string[]; } @@ -72,9 +69,6 @@ const secretVersionSchema = new Schema( type: String, // symmetric required: true }, - secretKeyHash: { - type: String - }, secretValueCiphertext: { type: String, required: true @@ -87,9 +81,6 @@ const secretVersionSchema = new Schema( type: String, // symmetric required: true }, - secretValueHash: { - type: String - }, tags: { ref: 'Tag', type: [Schema.Types.ObjectId], diff --git a/backend/src/routes/v2/secrets.ts b/backend/src/routes/v2/secrets.ts index 9c5577d2f..81a288283 100644 --- a/backend/src/routes/v2/secrets.ts +++ b/backend/src/routes/v2/secrets.ts @@ -15,6 +15,24 @@ import { SECRET_SHARED } from '../../variables'; +// TODO: create batch update endpoint + +router.post( + '/batch', + body('workspaceId').exists().isString().trim(), + body('environment').exists().isString().trim(), + body('requests').exists(), // perform validation for batch requests + validateRequest, + requireAuth({ + acceptedAuthModes: ['jwt', 'apiKey'] + }), + requireWorkspaceAuth({ + acceptedRoles: [ADMIN, MEMBER], + location: 'body' + }), + secretsController.batchSecrets +) + router.post( '/', body('workspaceId').exists().isString().trim(), diff --git a/backend/src/types/secret/index.d.ts b/backend/src/types/secret/index.d.ts index 177df8c0f..257038f60 100644 --- a/backend/src/types/secret/index.d.ts +++ b/backend/src/types/secret/index.d.ts @@ -1,5 +1,7 @@ +import { Types } from 'mongoose'; import { Assign, Omit } from 'utility-types'; import { ISecret } from '../../models'; +import { mongo } from 'mongoose'; // Everything is required, except the omitted types export type CreateSecretRequestBody = Omit; @@ -12,3 +14,39 @@ export type SanitizedSecretModify = Partial; + +export interface BatchSecretRequest { + id: string; + method: 'POST' | 'PATCH' | 'DELETE'; + secret: Secret; +} + +export interface BatchSecret { + _id: string; + type: 'shared' | 'personal', + secretKeyCiphertext: string; + secretKeyIV: string; + secretKeyTag: string; + secretValueCiphertext: string; + secretValueIV: string; + secretValueTag: string; + secretCommentCiphertext: string; + secretCommentIV: string; + secretCommentTag: string; + tags: string[]; +} + +export interface BatchSecret { + _id: string; + type: 'shared' | 'personal', + secretKeyCiphertext: string; + secretKeyIV: string; + secretKeyTag: string; + secretValueCiphertext: string; + secretValueIV: string; + secretValueTag: string; + secretCommentCiphertext: string; + secretCommentIV: string; + secretCommentTag: string; + tags: string[]; +} \ No newline at end of file diff --git a/frontend/src/pages/api/files/batchSecrets.ts b/frontend/src/pages/api/files/batchSecrets.ts new file mode 100644 index 000000000..77060b9a9 --- /dev/null +++ b/frontend/src/pages/api/files/batchSecrets.ts @@ -0,0 +1,38 @@ +import { apiRequest } from "@app/config/request"; + +interface RequestType { + method: string; + secret: { + type: string; + secretKeyCiphertext: string; + secretKeyIV: string; + secretKeyTag: string; + secretValueCiphertext: string; + secretValueIV: string; + secretValueTag: string; + secretCommentCiphertext: string; + secretCommentIV: string; + secretCommentTag: string; + tags: string[]; + } +} + +const batchSecrets = async ({ + workspaceId, + environment, + requests +}: { + workspaceId: string; + environment: string; + requests: RequestType[]; +}) => { + const { data } = await apiRequest.post('/api/v2/secrets/batch', { + workspaceId, + environment, + requests + }); + + return data; +} + +export default batchSecrets; \ No newline at end of file diff --git a/frontend/src/pages/dashboard/[id].tsx b/frontend/src/pages/dashboard/[id].tsx index d0dda0240..ecd360fb0 100644 --- a/frontend/src/pages/dashboard/[id].tsx +++ b/frontend/src/pages/dashboard/[id].tsx @@ -41,9 +41,10 @@ import performSecretRollback from '@app/ee/api/secrets/PerformSecretRollback'; import PITRecoverySidebar from '@app/ee/components/PITRecoverySidebar'; import { useLeaveConfirm } from '@app/hooks'; -import addSecrets from '../api/files/AddSecrets'; -import deleteSecrets from '../api/files/DeleteSecrets'; -import updateSecrets from '../api/files/UpdateSecrets'; +// import addSecrets from '../api/files/AddSecrets'; +// import deleteSecrets from '../api/files/DeleteSecrets'; +// import updateSecrets from '../api/files/UpdateSecrets'; +import batchSecrets from '../api/files/batchSecrets'; import getUser from '../api/user/getUser'; import checkUserAction from '../api/userActions/checkUserAction'; import registerUserAction from '../api/userActions/registerUserAction'; @@ -490,8 +491,18 @@ export default function Dashboard() { })); console.log('override update', overridesToBeUpdated.length); + const requests: any = []; // TODO: fix any if (secretsToBeDeleted.concat(overridesToBeDeleted).length > 0) { - await deleteSecrets({ secretIds: secretsToBeDeleted.concat(overridesToBeDeleted) }); + console.log('DELETE: ', secretsToBeDeleted.concat(overridesToBeDeleted)); + // await deleteSecrets({ secretIds: secretsToBeDeleted.concat(overridesToBeDeleted) }); + secretsToBeDeleted.concat(overridesToBeDeleted).forEach((_id: string) => { + requests.push({ + method: 'DELETE', + secret: { + _id + } + }); + }); } if (selectedEnv && secretsToBeAdded.concat(overridesToBeAdded).length > 0) { const secrets = await encryptSecrets({ @@ -499,7 +510,28 @@ export default function Dashboard() { workspaceId, env: selectedEnv.slug }); - if (secrets) await addSecrets({ secrets, env: selectedEnv.slug, workspaceId }); + if (secrets) { + console.log('ADD: ', secrets); + // await addSecrets({ secrets, env: selectedEnv.slug, workspaceId }); + secrets.forEach((secret) => { + requests.push({ + method: 'POST', + secret: { + type: secret.type, + secretKeyCiphertext: secret.secretKeyCiphertext, + secretKeyIV: secret.secretKeyIV, + secretKeyTag: secret.secretKeyTag, + secretValueCiphertext: secret.secretValueCiphertext, + secretValueIV: secret.secretValueIV, + secretValueTag: secret.secretValueTag, + secretCommentCiphertext: secret.secretCommentCiphertext, + secretCommentIV: secret.secretCommentIV, + secretCommentTag: secret.secretCommentTag, + tags: secret.tags + } + }) + }); + } } if (selectedEnv && !selectedEnv.isReadDenied && secretsToBeUpdated.concat(overridesToBeUpdated).length > 0) { const secrets = await encryptSecrets({ @@ -507,7 +539,40 @@ export default function Dashboard() { workspaceId, env: selectedEnv.slug }); - if (secrets) await updateSecrets({ secrets }); + if (secrets) { + console.log('UPDATE: ', secrets); + // await updateSecrets({ secrets }); + secrets.forEach((secret) => { + requests.push({ + method: 'PATCH', + secret: { + _id: secret.id, + type: secret.type, + secretKeyCiphertext: secret.secretKeyCiphertext, + secretKeyIV: secret.secretKeyIV, + secretKeyTag: secret.secretKeyTag, + secretValueCiphertext: secret.secretValueCiphertext, + secretValueIV: secret.secretValueIV, + secretValueTag: secret.secretValueTag, + secretCommentCiphertext: secret.secretCommentCiphertext, + secretCommentIV: secret.secretCommentIV, + secretCommentTag: secret.secretCommentTag, + tags: secret.tags + } + }); + }); + } + } + + if (selectedEnv && requests.length > 0) { + console.log('make batch secret request: '); + const result = await batchSecrets({ + workspaceId, + environment: selectedEnv.slug, + requests + }); + + console.log('result of batchSecrets', result); } setInitialData(structuredClone(newData));