Merge crud cli api and cli changes

backend/package-lock.json

@@ -15,6 +15,7 @@
         "@sentry/tracing": "^7.19.0",
         "@types/crypto-js": "^4.1.1",
         "@types/libsodium-wrappers": "^0.7.10",
+        "await-to-js": "^3.0.0",
         "axios": "^1.1.3",
         "bcrypt": "^5.1.0",
         "bigint-conversion": "^2.2.2",
@@ -39,6 +40,7 @@
         "tweetnacl": "^1.0.3",
         "tweetnacl-util": "^0.15.1",
         "typescript": "^4.9.3",
+        "utility-types": "^3.10.0",
         "winston": "^3.8.2",
         "winston-loki": "^6.0.6"
       },
@@ -3744,6 +3746,14 @@
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
       "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
     },
+    "node_modules/await-to-js": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/await-to-js/-/await-to-js-3.0.0.tgz",
+      "integrity": "sha512-zJAaP9zxTcvTHRlejau3ZOY4V7SRpiByf3/dxx2uyKxxor19tpmpV2QRsTKikckwhaPmr2dVpxxMr7jOCYVp5g==",
+      "engines": {
+        "node": ">=6.0.0"
+      }
+    },
     "node_modules/axios": {
       "version": "1.1.3",
       "resolved": "https://registry.npmjs.org/axios/-/axios-1.1.3.tgz",
@@ -11521,6 +11531,14 @@
       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
       "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="
     },
+    "node_modules/utility-types": {
+      "version": "3.10.0",
+      "resolved": "https://registry.npmjs.org/utility-types/-/utility-types-3.10.0.tgz",
+      "integrity": "sha512-O11mqxmi7wMKCo6HKFt5AhO4BwY3VV68YU07tgxfz8zJTIxr4BpsezN49Ffwy9j3ZpwwJp4fkRwjRzq3uWE6Rg==",
+      "engines": {
+        "node": ">= 4"
+      }
+    },
     "node_modules/utils-merge": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",
@@ -14962,6 +14980,11 @@
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
       "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
     },
+    "await-to-js": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/await-to-js/-/await-to-js-3.0.0.tgz",
+      "integrity": "sha512-zJAaP9zxTcvTHRlejau3ZOY4V7SRpiByf3/dxx2uyKxxor19tpmpV2QRsTKikckwhaPmr2dVpxxMr7jOCYVp5g=="
+    },
     "axios": {
       "version": "1.1.3",
       "resolved": "https://registry.npmjs.org/axios/-/axios-1.1.3.tgz",
@@ -17175,9 +17198,9 @@
       "dev": true
     },
     "json5": {
-      "version": "2.2.2",
-      "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.2.tgz",
-      "integrity": "sha512-46Tk9JiOL2z7ytNQWFLpj99RZkVgeHf87yGQKsIkaPz1qSH9UczKH1rO7K3wgRselo0tYMUNfecYpm/p1vC7tQ==",
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.1.tgz",
+      "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==",
       "dev": true
     },
     "jsonwebtoken": {
@@ -20632,6 +20655,11 @@
       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
       "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="
     },
+    "utility-types": {
+      "version": "3.10.0",
+      "resolved": "https://registry.npmjs.org/utility-types/-/utility-types-3.10.0.tgz",
+      "integrity": "sha512-O11mqxmi7wMKCo6HKFt5AhO4BwY3VV68YU07tgxfz8zJTIxr4BpsezN49Ffwy9j3ZpwwJp4fkRwjRzq3uWE6Rg=="
+    },
     "utils-merge": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",

backend/package.json

@@ -6,6 +6,7 @@
     "@sentry/tracing": "^7.19.0",
     "@types/crypto-js": "^4.1.1",
     "@types/libsodium-wrappers": "^0.7.10",
+    "await-to-js": "^3.0.0",
     "axios": "^1.1.3",
     "bcrypt": "^5.1.0",
     "bigint-conversion": "^2.2.2",
@@ -30,6 +31,7 @@
     "tweetnacl": "^1.0.3",
     "tweetnacl-util": "^0.15.1",
     "typescript": "^4.9.3",
+    "utility-types": "^3.10.0",
     "winston": "^3.8.2",
     "winston-loki": "^6.0.6"
   },

backend/src/controllers/v2/index.ts

@@ -1,9 +1,11 @@
 import * as workspaceController from './workspaceController';
 import * as serviceTokenDataController from './serviceTokenDataController';
 import * as apiKeyDataController from './apiKeyDataController';
+import * as secretController from './secretController';
 
-export { 
+export {
     workspaceController,
     serviceTokenDataController,
-    apiKeyDataController
+    apiKeyDataController,
+    secretController
 }

backend/src/controllers/v2/secretController.ts

@@ -0,0 +1,138 @@
+import to from "await-to-js";
+import { Request, Response } from "express";
+import mongoose, { Types } from "mongoose";
+import Secret, { ISecret } from "../../models/secret";
+import { CreateSecretRequestBody, ModifySecretRequestBody, SanitizedSecretForCreate, SanitizedSecretModify } from "../../types/secret/types";
+const { ValidationError } = mongoose.Error;
+import { BadRequestError, InternalServerError, UnauthorizedRequestError, ValidationError as RouteValidationError } from '../../utils/errors';
+import { AnyBulkWriteOperation } from 'mongodb';
+
+export const batchCreateSecrets = async (req: Request, res: Response) => {
+  const secretsToCreate: CreateSecretRequestBody[] = req.body.secrets;
+  const { workspaceId, environmentName } = req.params
+  const sanitizedSecretesToCreate: SanitizedSecretForCreate[] = []
+
+  secretsToCreate.forEach(rawSecret => {
+    const safeUpdateFields: SanitizedSecretForCreate = {
+      secretKeyCiphertext: rawSecret.secretKeyCiphertext,
+      secretKeyIV: rawSecret.secretKeyIV,
+      secretKeyTag: rawSecret.secretKeyTag,
+      secretKeyHash: rawSecret.secretKeyHash,
+      secretValueCiphertext: rawSecret.secretValueCiphertext,
+      secretValueIV: rawSecret.secretValueIV,
+      secretValueTag: rawSecret.secretValueTag,
+      secretValueHash: rawSecret.secretValueHash,
+      secretCommentCiphertext: rawSecret.secretCommentCiphertext,
+      secretCommentIV: rawSecret.secretCommentIV,
+      secretCommentTag: rawSecret.secretCommentTag,
+      secretCommentHash: rawSecret.secretCommentHash,
+      workspace: new Types.ObjectId(workspaceId),
+      environment: environmentName,
+      type: rawSecret.type,
+      user: new Types.ObjectId(req.user._id)
+    }
+
+    sanitizedSecretesToCreate.push(safeUpdateFields)
+  })
+
+  const [bulkCreateError, newlyCreatedSecrets] = await to(Secret.insertMany(sanitizedSecretesToCreate).then())
+  if (bulkCreateError) {
+    if (bulkCreateError instanceof ValidationError) {
+      throw RouteValidationError({ message: bulkCreateError.message, stack: bulkCreateError.stack })
+    }
+
+    throw InternalServerError({ message: "Unable to process your batch create request. Please try again", stack: bulkCreateError.stack })
+  }
+
+  res.status(200).send()
+}
+
+
+export const createSingleSecret = async (req: Request, res: Response) => {
+  try {
+    const secretFromDB = await Secret.findById(req.params.secretId)
+    return res.status(200).send(secretFromDB);
+  } catch (e) {
+    throw BadRequestError({ message: "Unable to find the requested secret" })
+  }
+}
+
+export const batchDeleteSecrets = async (req: Request, res: Response) => {
+  const { workspaceId, environmentName } = req.params
+  const secretIdsToDelete: string[] = req.body.secretIds
+
+  const [secretIdsUserCanDeleteError, secretIdsUserCanDelete] = await to(Secret.find({ workspace: workspaceId, environment: environmentName }, { _id: 1 }).then())
+  if (secretIdsUserCanDeleteError) {
+    throw InternalServerError({ message: `Unable to fetch secrets you own: [error=${secretIdsUserCanDeleteError.message}]` })
+  }
+
+  const secretsUserCanDeleteSet: Set<string> = new Set(secretIdsUserCanDelete.map(objectId => objectId._id.toString()));
+  const deleteOperationsToPerform: AnyBulkWriteOperation<ISecret>[] = []
+
+  secretIdsToDelete.forEach(secretIdToDelete => {
+    if (secretsUserCanDeleteSet.has(secretIdToDelete)) {
+      const deleteOperation = { deleteOne: { filter: { _id: new Types.ObjectId(secretIdToDelete) } } }
+      deleteOperationsToPerform.push(deleteOperation)
+    } else {
+      throw RouteValidationError({ message: "You cannot delete secrets that you do not have access to" })
+    }
+  })
+
+  const [bulkDeleteError, bulkDelete] = await to(Secret.bulkWrite(deleteOperationsToPerform).then())
+  if (bulkDeleteError) {
+    if (bulkDeleteError instanceof ValidationError) {
+      throw RouteValidationError({ message: "Unable to apply modifications, please try again", stack: bulkDeleteError.stack })
+    }
+    throw InternalServerError()
+  }
+
+  res.status(200).send()
+}
+
+export const batchModifySecrets = async (req: Request, res: Response) => {
+  const { workspaceId, environmentName } = req.params
+  const secretsModificationsRequested: ModifySecretRequestBody[] = req.body.secrets;
+  const [secretIdsUserCanModifyError, secretIdsUserCanModify] = await to(Secret.find({ workspace: workspaceId, environment: environmentName }, { _id: 1 }).then())
+  if (secretIdsUserCanModifyError) {
+    throw InternalServerError({ message: "Unable to fetch secrets you own" })
+  }
+
+  const secretsUserCanModifySet: Set<string> = new Set(secretIdsUserCanModify.map(objectId => objectId._id.toString()));
+  const updateOperationsToPerform: any = []
+
+
+  secretsModificationsRequested.forEach(userModifiedSecret => {
+    if (secretsUserCanModifySet.has(userModifiedSecret._id.toString())) {
+      const sanitizedSecret: SanitizedSecretModify = {
+        secretKeyCiphertext: userModifiedSecret.secretKeyCiphertext,
+        secretKeyIV: userModifiedSecret.secretKeyIV,
+        secretKeyTag: userModifiedSecret.secretKeyTag,
+        secretKeyHash: userModifiedSecret.secretKeyHash,
+        secretValueCiphertext: userModifiedSecret.secretValueCiphertext,
+        secretValueIV: userModifiedSecret.secretValueIV,
+        secretValueTag: userModifiedSecret.secretValueTag,
+        secretValueHash: userModifiedSecret.secretValueHash,
+        secretCommentCiphertext: userModifiedSecret.secretCommentCiphertext,
+        secretCommentIV: userModifiedSecret.secretCommentIV,
+        secretCommentTag: userModifiedSecret.secretCommentTag,
+        secretCommentHash: userModifiedSecret.secretCommentHash,
+      }
+
+      const updateOperation = { updateOne: { filter: { _id: userModifiedSecret._id, workspace: workspaceId }, update: { $inc: { version: 1 }, $set: sanitizedSecret } } }
+      updateOperationsToPerform.push(updateOperation)
+    } else {
+      throw UnauthorizedRequestError({ message: "You do not have permission to modify one or more of the requested secrets" })
+    }
+  })
+
+  const [bulkModificationInfoError, bulkModificationInfo] = await to(Secret.bulkWrite(updateOperationsToPerform).then())
+  if (bulkModificationInfoError) {
+    if (bulkModificationInfoError instanceof ValidationError) {
+      throw RouteValidationError({ message: "Unable to apply modifications, please try again", stack: bulkModificationInfoError.stack })
+    }
+
+    throw InternalServerError()
+  }
+
+  return res.status(200).send()
+}

backend/src/middleware/requireAuth.ts

@@ -25,28 +25,28 @@ declare module 'jsonwebtoken' {
  * @param {String[]} obj.acceptedAuthModes - accepted modes of authentication (jwt/st)
  * @returns
  */
-const requireAuth  = ({
+const requireAuth = ({
 	acceptedAuthModes = ['jwt']
 }: {
 	acceptedAuthModes: string[];
 }) => {
 	return async (req: Request, res: Response, next: NextFunction) => {
-		const [ AUTH_TOKEN_TYPE, AUTH_TOKEN_VALUE ] = <[string, string]>req.headers['authorization']?.split(' ', 2) ?? [null, null]
-		if(AUTH_TOKEN_TYPE === null) 
-			return next(BadRequestError({message: `Missing Authorization Header in the request header.`}))
-		if(AUTH_TOKEN_TYPE.toLowerCase() !== 'bearer') 
-			return next(BadRequestError({message: `The provided authentication type '${AUTH_TOKEN_TYPE}' is not supported.`}))
-		if(AUTH_TOKEN_VALUE === null) 
-			return next(BadRequestError({message: 'Missing Authorization Body in the request header'}))
-		
+		const [AUTH_TOKEN_TYPE, AUTH_TOKEN_VALUE] = <[string, string]>req.headers['authorization']?.split(' ', 2) ?? [null, null]
+		if (AUTH_TOKEN_TYPE === null)
+			return next(BadRequestError({ message: `Missing Authorization Header in the request header.` }))
+		if (AUTH_TOKEN_TYPE.toLowerCase() !== 'bearer')
+			return next(BadRequestError({ message: `The provided authentication type '${AUTH_TOKEN_TYPE}' is not supported.` }))
+		if (AUTH_TOKEN_VALUE === null)
+			return next(BadRequestError({ message: 'Missing Authorization Body in the request header' }))
+
 		// validate auth token against 
 		const authMode = validateAuthMode({
 			authTokenValue: AUTH_TOKEN_VALUE,
 			acceptedAuthModes
 		});
-		
+
 		if (!acceptedAuthModes.includes(authMode)) throw new Error('Failed to validate auth mode');
-		
+
 		// attach auth payloads
 		switch (authMode) {
 			case 'serviceToken':

backend/src/middleware/validateRequest.ts

@@ -1,6 +1,6 @@
 import { Request, Response, NextFunction } from 'express';
 import { validationResult } from 'express-validator';
-import { BadRequestError, UnauthorizedRequestError } from '../utils/errors';
+import { BadRequestError, UnauthorizedRequestError, ValidationError } from '../utils/errors';
 
 /**
  * Validate intended inputs on [req] via express-validator
@@ -15,12 +15,12 @@ const validate = (req: Request, res: Response, next: NextFunction) => {
 	try {
 		const errors = validationResult(req);
 		if (!errors.isEmpty()) {
-			return next(BadRequestError({context: {errors: errors.array}}))
+			return next(ValidationError({ context: { errors: `One or more of your parameters are invalid [error(s)=${(JSON.stringify(errors))}]` } }))
 		}
 
 		return next();
 	} catch (err) {
-		return next(UnauthorizedRequestError({message: 'Unauthenticated requests are not allowed. Try logging in'}))
+		return next(UnauthorizedRequestError({ message: 'Unauthenticated requests are not allowed. Try logging in' }))
 	}
 };
 

backend/src/models/secret.ts

@@ -33,7 +33,8 @@ const secretSchema = new Schema<ISecret>(
 	{
 		version: {
 			type: Number,
-			required: true
+			required: true,
+			default: 1
 		},
 		workspace: {
 			type: Schema.Types.ObjectId,

backend/src/routes/v2/secret.ts

@@ -1,4 +1,75 @@
 import express from 'express';
+import { requireAuth, requireWorkspaceAuth, validateRequest } from '../../middleware';
+import { body, param } from 'express-validator';
+import { ADMIN, MEMBER, COMPLETED, GRANTED } from '../../variables';
+import { CreateSecretRequestBody, ModifySecretRequestBody } from '../../types/secret/types';
+import { secretController } from '../../controllers/v2';
+
 const router = express.Router();
 
+/**
+ * Create many secrets for a given workspace and environmentName
+ */
+router.post(
+  '/batch-create/workspace/:workspaceId/environment/:environmentName',
+  requireAuth,
+  requireWorkspaceAuth({
+    acceptedRoles: [ADMIN, MEMBER],
+    acceptedStatuses: [COMPLETED, GRANTED]
+  }),
+  param('workspaceId').exists().isMongoId().trim(),
+  param('environmentName').exists().trim(),
+  body('secrets').exists().isArray().custom((value) => value.every((item: CreateSecretRequestBody) => typeof item === 'object')),
+  validateRequest,
+  secretController.batchCreateSecrets
+);
+
+/**
+ * Get a single secret by secret id
+ */
+router.get(
+  '/:secretId', requireAuth, param('secretId').exists().trim(),
+  requireWorkspaceAuth({
+    acceptedRoles: [ADMIN, MEMBER],
+    acceptedStatuses: [COMPLETED, GRANTED]
+  }),
+  validateRequest,
+  secretController.createSingleSecret
+);
+
+/**
+ * Batch delete secrets in a given workspace and environment name
+ */
+router.delete(
+  '/batch/workspace/:workspaceId/environment/:environmentName',
+  requireAuth,
+  param('workspaceId').exists().isMongoId().trim(),
+  param('environmentName').exists().trim(),
+  body('secretIds').exists().isArray().custom(array => array.length > 0),
+  requireWorkspaceAuth({
+    acceptedRoles: [ADMIN, MEMBER],
+    acceptedStatuses: [COMPLETED, GRANTED]
+  }),
+  validateRequest,
+  secretController.batchDeleteSecrets
+
+);
+
+/**
+ * Apply modifications to many existing secrets in a given workspace and environment
+ */
+router.patch(
+  '/batch-modify/workspace/:workspaceId/environment/:environmentName',
+  requireAuth,
+  body('secrets').exists().isArray().custom((secrets: ModifySecretRequestBody[]) => secrets.length > 0),
+  param('workspaceId').exists().isMongoId().trim(),
+  param('environmentName').exists().trim(),
+  requireWorkspaceAuth({
+    acceptedRoles: [ADMIN, MEMBER],
+    acceptedStatuses: [COMPLETED, GRANTED]
+  }),
+  validateRequest,
+  secretController.batchModifySecrets
+);
+
 export default router;

backend/src/types/secret/types.ts

@@ -0,0 +1,14 @@
+import { Assign, Omit } from 'utility-types';
+import { ISecret } from '../../models';
+
+// Everything is required, except the omitted types
+export type CreateSecretRequestBody = Omit<ISecret, "user" | "version" | "environment" | "workspace">;
+
+// Omit the listed properties, then make everything optional and then make _id required 
+export type ModifySecretRequestBody = Assign<Partial<Omit<ISecret, "user" | "version" | "environment" | "workspace">>, { _id: string }>;
+
+// Used for modeling sanitized secrets before uplaod. To be used for converting user input for uploading
+export type SanitizedSecretModify = Partial<Omit<ISecret, "user" | "version" | "environment" | "workspace">>;
+
+// Everything is required, except the omitted types
+export type SanitizedSecretForCreate = Omit<ISecret, "version" | "_id">;

backend/tsconfig.json

@@ -1,7 +1,9 @@
 {
   "compilerOptions": {
     "target": "es2016",
-    "lib": ["es6"],
+    "lib": [
+      "es6"
+    ],
     "module": "commonjs",
     "rootDir": "src",
     "resolveJsonModule": true,
@@ -13,8 +15,15 @@
     "strict": true,
     "noImplicitAny": true,
     "skipLibCheck": true,
-    "typeRoots": ["./src/types", "./node_modules/@types"]
+    "typeRoots": [
+      "./src/types",
+      "./node_modules/@types"
+    ]
   },
-  "include": ["src/**/*"],
-  "exclude": ["node_modules"]
-}
+  "include": [
+    "src/**/*"
+  ],
+  "exclude": [
+    "node_modules"
+  ]
+}