Begin rewiring frontend to use batch route for CRUD secret ops

backend/src/controllers/v2/secretsController.ts

@@ -1,7 +1,8 @@
 import to from 'await-to-js';
 import { Types } from 'mongoose';
 import { Request, Response } from 'express';
-import { ISecret, Membership, Secret, Workspace } from '../../models';
+import { ISecret, Secret } from '../../models';
+import { IAction } from '../../ee/models';
 import {
     SECRET_PERSONAL,
     SECRET_SHARED,
@@ -20,6 +21,250 @@ import { ABILITY_READ, ABILITY_WRITE } from '../../variables/organization';
 import { userHasNoAbility, userHasWorkspaceAccess, userHasWriteOnlyAbility } from '../../ee/helpers/checkMembershipPermissions';
 import Tag from '../../models/tag';
 import _ from 'lodash';
+import {
+    BatchSecretRequest, 
+    BatchSecret
+} from '../../types/secret';
+
+/**
+ * Peform a batch of any specified CUD secret operations
+ * @param req 
+ * @param res 
+ */
+export const batchSecrets = async (req: Request, res: Response) => {
+    const channel = getChannelFromUserAgent(req.headers['user-agent']);
+    const {
+        workspaceId,
+        environment,
+        requests
+    }: {
+        workspaceId: string;
+        environment: string;
+        requests: BatchSecretRequest[];
+    }= req.body;
+    
+    // construct object containing all secrets
+    // listed across requests
+    const listedSecretsObj: {
+        [key: string]: { 
+            version: number;
+            type: string;
+        }
+    } = (await Secret.find({
+        _id: {
+            $in: requests
+                .map((request) => request.secret._id)
+                .filter((secretId) => secretId !== undefined)
+        }
+    }).select('version type')).reduce((obj: any, secret: ISecret) => ({
+        ...obj,
+        [secret._id.toString()]: secret
+    }), {});
+    
+
+    const createSecrets: BatchSecret[] = [];
+    const updateSecrets: BatchSecret[] = [];
+    const deleteSecrets: Types.ObjectId[] = [];
+    const actions: IAction[] = [];
+    
+    requests.forEach((request) => {
+        switch (request.method) {
+            case 'POST':
+                createSecrets.push({
+                    ...request.secret,
+                    version: 1,
+                    user: request.secret.type === SECRET_PERSONAL ? req.user : undefined,
+                    environment,
+                    workspace: new Types.ObjectId(workspaceId)
+                });
+                break;
+            case 'PATCH':
+                updateSecrets.push({
+                    ...request.secret,
+                    _id: new Types.ObjectId(request.secret._id)
+                });
+                break;
+            case 'DELETE':
+                deleteSecrets.push(new Types.ObjectId(request.secret._id));
+                break;
+        }
+    });
+    
+    // handle create secrets
+    let createdSecrets: ISecret[] = [];
+    if (createSecrets.length > 0) {
+        createdSecrets = await Secret.insertMany(createSecrets);
+        // (EE) add secret versions for new secrets
+        await EESecretService.addSecretVersions({
+            secretVersions: createdSecrets.map((n: any) => {
+                return ({
+                    ...n._doc,
+                    _id: new Types.ObjectId(),
+                    secret: n._id,
+                    isDeleted: false
+                });
+            })
+        });
+
+        const addAction = await EELogService.createAction({
+            name: ACTION_ADD_SECRETS,
+            userId: req.user._id,
+            workspaceId: new Types.ObjectId(workspaceId),
+            secretIds: createdSecrets.map((n) => n._id)
+        }) as IAction;
+        actions.push(addAction);
+
+        if (postHogClient) {
+            postHogClient.capture({
+                event: 'secrets added',
+                distinctId: req.user.email,
+                properties: {
+                    numberOfSecrets: createdSecrets.length,
+                    environment,
+                    workspaceId,
+                    channel,
+                    userAgent: req.headers?.['user-agent']
+                }
+            });
+        }
+    }
+    
+    // handle update secrets
+    let updatedSecrets: ISecret[] = [];
+    if (updateSecrets.length > 0) {
+        const updateOperations = updateSecrets.map((u) => ({
+            updateOne: {
+                filter: { _id: new Types.ObjectId(u._id) },
+                update: {
+                    $inc: {
+                        version: 1
+                    },
+                    ...u,
+                    _id: new Types.ObjectId(u._id)
+                }
+            }
+        }));
+
+        await Secret.bulkWrite(updateOperations);
+        
+        const secretVersions = updateSecrets.map((u) => ({
+            secret: new Types.ObjectId(u._id),
+            version: listedSecretsObj[u._id.toString()].version,
+            workspace: new Types.ObjectId(workspaceId),
+            type: listedSecretsObj[u._id.toString()].type,
+            environment,
+            isDeleted: false,
+            secretKeyCiphertext: u.secretKeyCiphertext,
+            secretKeyIV: u.secretKeyIV,
+            secretKeyTag: u.secretKeyTag,
+            secretValueCiphertext: u.secretValueCiphertext,
+            secretValueIV: u.secretValueIV,
+            secretValueTag: u.secretValueTag,
+            secretCommentCiphertext: u.secretCommentCiphertext,
+            secretCommentIV: u.secretCommentIV,
+            secretCommentTag: u.secretCommentTag,
+            tags: u.tags
+        }));
+
+        await EESecretService.addSecretVersions({
+            secretVersions
+        });
+
+        updatedSecrets = await Secret.find({
+            _id: {
+                $in: updateSecrets.map((u) => new Types.ObjectId(u._id))
+            }
+        });
+
+        if (postHogClient) {
+            postHogClient.capture({
+                event: 'secrets modified',
+                distinctId: req.user.email,
+                properties: {
+                    numberOfSecrets: updateSecrets.length,
+                    environment,
+                    workspaceId,
+                    channel,
+                    userAgent: req.headers?.['user-agent']
+                }
+            });
+        }
+    }
+
+    // handle delete secrets
+    if (deleteSecrets.length > 0) {
+        await Secret.deleteMany({
+            _id: {
+                $in: deleteSecrets
+            }
+        });
+
+        await EESecretService.markDeletedSecretVersions({
+            secretIds: deleteSecrets
+        });
+
+        const deleteAction = await EELogService.createAction({
+            name: ACTION_DELETE_SECRETS,
+            userId: req.user._id,
+            workspaceId: new Types.ObjectId(workspaceId),
+            secretIds: deleteSecrets
+        }) as IAction;
+        actions.push(deleteAction);
+
+        if (postHogClient) {
+            postHogClient.capture({
+                event: 'secrets deleted',
+                distinctId: req.user.email,
+                properties: {
+                    numberOfSecrets: deleteSecrets.length,
+                    environment,
+                    workspaceId,
+                    channel: channel,
+                    userAgent: req.headers?.['user-agent']
+                }
+            });
+        }
+    }
+    
+    if (actions.length > 1) {
+        // (EE) create (audit) log
+        await EELogService.createLog({
+            userId: req.user._id.toString(),
+            workspaceId: new Types.ObjectId(workspaceId),
+            actions,
+            channel,
+            ipAddress: req.ip
+        });
+    }
+
+    // // trigger event - push secrets
+    await EventService.handleEvent({
+        event: eventPushSecrets({
+            workspaceId
+        })
+    });
+
+    // (EE) take a secret snapshot
+    await EESecretService.takeSecretSnapshot({
+        workspaceId
+    });
+    
+    const resObj: { [key: string]: ISecret[] | string[] } = {}
+
+    if (createSecrets.length > 0) {
+        resObj['createdSecrets'] = createdSecrets;
+    }
+
+    if (updateSecrets.length > 0) {
+        resObj['updatedSecrets'] = updatedSecrets;
+    }
+    
+    if (deleteSecrets.length > 0) {
+        resObj['deletedSecrets'] = deleteSecrets.map((d) => d.toString());
+    }
+    
+    return res.status(200).send(resObj);
+}
 
 /**
  * Create secret(s) for workspace with id [workspaceId] and environment [environment]
@@ -166,11 +411,9 @@ export const createSecrets = async (req: Request, res: Response) => {
             secretKeyCiphertext,
             secretKeyIV,
             secretKeyTag,
-            secretKeyHash,
             secretValueCiphertext,
             secretValueIV,
             secretValueTag,
-            secretValueHash,
             secretCommentCiphertext,
             secretCommentIV,
             secretCommentTag,
@@ -187,11 +430,9 @@ export const createSecrets = async (req: Request, res: Response) => {
             secretKeyCiphertext,
             secretKeyIV,
             secretKeyTag,
-            secretKeyHash,
             secretValueCiphertext,
             secretValueIV,
             secretValueTag,
-            secretValueHash,
             secretCommentCiphertext,
             secretCommentIV,
             secretCommentTag,

backend/src/ee/controllers/v1/secretController.ts

@@ -158,11 +158,9 @@ export const rollbackSecretVersion = async (req: Request, res: Response) => {
 			secretKeyCiphertext,
 			secretKeyIV,
 			secretKeyTag,
-			secretKeyHash,
 			secretValueCiphertext,
 			secretValueIV,
 			secretValueTag,
-			secretValueHash
 		} = oldSecretVersion;
 		
 		// update secret
@@ -179,11 +177,9 @@ export const rollbackSecretVersion = async (req: Request, res: Response) => {
 				secretKeyCiphertext,
 				secretKeyIV,
 				secretKeyTag,
-				secretKeyHash,
 				secretValueCiphertext,
 				secretValueIV,
 				secretValueTag,
-				secretValueHash
 			},
 			{
 				new: true
@@ -204,11 +200,9 @@ export const rollbackSecretVersion = async (req: Request, res: Response) => {
 			secretKeyCiphertext,
 			secretKeyIV,
 			secretKeyTag,
-			secretKeyHash,
 			secretValueCiphertext,
 			secretValueIV,
-			secretValueTag,
+			secretValueTag
-			secretValueHash	
 		}).save();
 		
 		// take secret snapshot

backend/src/ee/models/secretVersion.ts

@@ -5,22 +5,19 @@ import {
 } from '../../variables';
 
 export interface ISecretVersion {
-	_id: Types.ObjectId;
 	secret: Types.ObjectId;
 	version: number;
 	workspace: Types.ObjectId; // new
 	type: string; // new
-	user: Types.ObjectId; // new
+	user?: Types.ObjectId; // new
 	environment: string; // new
 	isDeleted: boolean;
 	secretKeyCiphertext: string;
 	secretKeyIV: string;
 	secretKeyTag: string;
-	secretKeyHash: string;
 	secretValueCiphertext: string;
 	secretValueIV: string;
 	secretValueTag: string;
-	secretValueHash: string;
 	tags?: string[];
 }
 
@@ -72,9 +69,6 @@ const secretVersionSchema = new Schema<ISecretVersion>(
 			type: String, // symmetric
 			required: true
 		},
-		secretKeyHash: {
-			type: String
-		},
 		secretValueCiphertext: {
 			type: String,
 			required: true
@@ -87,9 +81,6 @@ const secretVersionSchema = new Schema<ISecretVersion>(
 			type: String, // symmetric
 			required: true
 		},
-		secretValueHash: {
-			type: String
-		},
 		tags: {
 			ref: 'Tag',
 			type: [Schema.Types.ObjectId],

backend/src/routes/v2/secrets.ts

@@ -15,6 +15,24 @@ import {
     SECRET_SHARED
 } from '../../variables';
 
+// TODO: create batch update endpoint
+
+router.post(
+    '/batch',
+    body('workspaceId').exists().isString().trim(),
+    body('environment').exists().isString().trim(),
+    body('requests').exists(), // perform validation for batch requests
+    validateRequest,
+    requireAuth({
+        acceptedAuthModes: ['jwt', 'apiKey']
+    }),
+    requireWorkspaceAuth({
+        acceptedRoles: [ADMIN, MEMBER],
+        location: 'body'
+    }),
+    secretsController.batchSecrets
+)
+
 router.post(
     '/',
     body('workspaceId').exists().isString().trim(),

backend/src/types/secret/index.d.ts

@@ -1,5 +1,7 @@
+import { Types } from 'mongoose';
 import { Assign, Omit } from 'utility-types';
 import { ISecret } from '../../models';
+import { mongo } from 'mongoose';
 
 // Everything is required, except the omitted types
 export type CreateSecretRequestBody = Omit<ISecret, "user" | "version" | "environment" | "workspace">;
@@ -12,3 +14,39 @@ export type SanitizedSecretModify = Partial<Omit<ISecret, "user" | "version" | "
 
 // Everything is required, except the omitted types
 export type SanitizedSecretForCreate = Omit<ISecret, "version" | "_id">;
+
+export interface BatchSecretRequest {
+    id: string;
+    method: 'POST' | 'PATCH' | 'DELETE';
+    secret: Secret;
+}
+
+export interface BatchSecret {
+    _id: string;
+    type: 'shared' | 'personal',
+    secretKeyCiphertext: string;
+    secretKeyIV: string;
+    secretKeyTag: string;
+    secretValueCiphertext: string;
+    secretValueIV: string;
+    secretValueTag: string;
+    secretCommentCiphertext: string;
+    secretCommentIV: string;
+    secretCommentTag: string;
+    tags: string[];
+}
+
+export interface BatchSecret {
+    _id: string;
+    type: 'shared' | 'personal',
+    secretKeyCiphertext: string;
+    secretKeyIV: string;
+    secretKeyTag: string;
+    secretValueCiphertext: string;
+    secretValueIV: string;
+    secretValueTag: string;
+    secretCommentCiphertext: string;
+    secretCommentIV: string;
+    secretCommentTag: string;
+    tags: string[];
+}

frontend/src/pages/api/files/batchSecrets.ts

@@ -0,0 +1,38 @@
+import { apiRequest } from "@app/config/request";
+
+interface RequestType {
+    method: string;
+    secret: {
+        type: string;
+        secretKeyCiphertext: string;
+        secretKeyIV: string;
+        secretKeyTag: string;
+        secretValueCiphertext: string;
+        secretValueIV: string;
+        secretValueTag: string;
+        secretCommentCiphertext: string;
+        secretCommentIV: string;
+        secretCommentTag: string;
+        tags: string[];
+    }
+}
+
+const batchSecrets = async ({
+    workspaceId,
+    environment,
+    requests
+}: {
+    workspaceId: string;
+    environment: string;
+    requests: RequestType[];
+}) => {
+    const { data } = await apiRequest.post('/api/v2/secrets/batch', {
+        workspaceId,
+        environment,
+        requests
+    });
+    
+    return data;
+}
+
+export default batchSecrets;

frontend/src/pages/dashboard/[id].tsx

@@ -41,9 +41,10 @@ import performSecretRollback from '@app/ee/api/secrets/PerformSecretRollback';
 import PITRecoverySidebar from '@app/ee/components/PITRecoverySidebar';
 import { useLeaveConfirm } from '@app/hooks';
 
-import addSecrets from '../api/files/AddSecrets';
+// import addSecrets from '../api/files/AddSecrets';
-import deleteSecrets from '../api/files/DeleteSecrets';
+// import deleteSecrets from '../api/files/DeleteSecrets';
-import updateSecrets from '../api/files/UpdateSecrets';
+// import updateSecrets from '../api/files/UpdateSecrets';
+import batchSecrets from '../api/files/batchSecrets';
 import getUser from '../api/user/getUser';
 import checkUserAction from '../api/userActions/checkUserAction';
 import registerUserAction from '../api/userActions/registerUserAction';
@@ -490,8 +491,18 @@ export default function Dashboard() {
       }));
     console.log('override update', overridesToBeUpdated.length);
 
+    const requests: any = []; // TODO: fix any
     if (secretsToBeDeleted.concat(overridesToBeDeleted).length > 0) {
-      await deleteSecrets({ secretIds: secretsToBeDeleted.concat(overridesToBeDeleted) });
+      console.log('DELETE: ', secretsToBeDeleted.concat(overridesToBeDeleted));
+      // await deleteSecrets({ secretIds: secretsToBeDeleted.concat(overridesToBeDeleted) });
+      secretsToBeDeleted.concat(overridesToBeDeleted).forEach((_id: string) => {
+        requests.push({
+          method: 'DELETE',
+          secret: {
+            _id
+          }
+        });
+      });
     }
     if (selectedEnv && secretsToBeAdded.concat(overridesToBeAdded).length > 0) {
       const secrets = await encryptSecrets({
@@ -499,7 +510,28 @@ export default function Dashboard() {
         workspaceId,
         env: selectedEnv.slug
       });
-      if (secrets) await addSecrets({ secrets, env: selectedEnv.slug, workspaceId });
+      if (secrets) {
+        console.log('ADD: ', secrets);
+        // await addSecrets({ secrets, env: selectedEnv.slug, workspaceId });
+        secrets.forEach((secret) => {
+          requests.push({
+            method: 'POST',
+            secret: {
+              type: secret.type,
+              secretKeyCiphertext: secret.secretKeyCiphertext,
+              secretKeyIV: secret.secretKeyIV,
+              secretKeyTag: secret.secretKeyTag,
+              secretValueCiphertext: secret.secretValueCiphertext,
+              secretValueIV: secret.secretValueIV,
+              secretValueTag: secret.secretValueTag,
+              secretCommentCiphertext: secret.secretCommentCiphertext,
+              secretCommentIV: secret.secretCommentIV,
+              secretCommentTag: secret.secretCommentTag,
+              tags: secret.tags
+            }
+          })
+        });
+      }
     }
     if (selectedEnv && !selectedEnv.isReadDenied && secretsToBeUpdated.concat(overridesToBeUpdated).length > 0) {
       const secrets = await encryptSecrets({
@@ -507,7 +539,40 @@ export default function Dashboard() {
         workspaceId,
         env: selectedEnv.slug
       });
-      if (secrets) await updateSecrets({ secrets });
+      if (secrets) {
+        console.log('UPDATE: ', secrets);
+        // await updateSecrets({ secrets });
+        secrets.forEach((secret) => {
+          requests.push({
+            method: 'PATCH',
+            secret: {
+              _id: secret.id,
+              type: secret.type,
+              secretKeyCiphertext: secret.secretKeyCiphertext,
+              secretKeyIV: secret.secretKeyIV,
+              secretKeyTag: secret.secretKeyTag,
+              secretValueCiphertext: secret.secretValueCiphertext,
+              secretValueIV: secret.secretValueIV,
+              secretValueTag: secret.secretValueTag,
+              secretCommentCiphertext: secret.secretCommentCiphertext,
+              secretCommentIV: secret.secretCommentIV,
+              secretCommentTag: secret.secretCommentTag,
+              tags: secret.tags 
+            }
+          });
+        });
+      }
+    }
+    
+    if (selectedEnv && requests.length > 0) {
+      console.log('make batch secret request: ');
+      const result = await batchSecrets({
+        workspaceId,
+        environment: selectedEnv.slug,
+        requests
+      });
+
+      console.log('result of batchSecrets', result);
     }
 
     setInitialData(structuredClone(newData));