From f93594b62faa45ca67c07f6937bc303fe392ac1e Mon Sep 17 00:00:00 2001 From: Tuan Dang Date: Mon, 26 Dec 2022 22:50:59 -0500 Subject: [PATCH] Migrate POST /v1/secret/:workspaceId to /v2/workspace/:workspaceId/secrets and cleared room for /v2 secret routes --- backend/src/app.ts | 8 +- .../src/controllers/v1/secretController.ts | 2 +- .../src/controllers/v2/workspaceController.ts | 35 +- backend/src/helpers/bot.ts | 1 - backend/src/helpers/secret.ts | 338 ++++++++++++++---- backend/src/routes/v2/workspace.ts | 4 +- .../components/utilities/secrets/pushKeys.ts | 42 +-- frontend/pages/api/files/UploadSecrets.ts | 2 +- 8 files changed, 315 insertions(+), 117 deletions(-) diff --git a/backend/src/app.ts b/backend/src/app.ts index f4271a34b..8320a7d76 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -66,7 +66,7 @@ if (NODE_ENV === 'production') { app.use(helmet()); } -// (EE) routers +// (EE) routes app.use('/api/v1/secret', eeSecretRouter); app.use('/api/v1/workspace', eeWorkspaceRouter); @@ -89,9 +89,9 @@ app.use('/api/v1/stripe', v1StripeRouter); app.use('/api/v1/integration', v1IntegrationRouter); app.use('/api/v1/integration-auth', v1IntegrationAuthRouter); -// v2 routes (new) -app.use('/api/v1/workspace', v2WorkspaceRouter); -app.use('/api/v1/secret', v2SecretRouter); +// v2 routes +app.use('/api/v2/workspace', v2WorkspaceRouter); +app.use('/api/v2/secret', v2SecretRouter); //* Handle unrouted requests and respond with proper error message as well as status code diff --git a/backend/src/controllers/v1/secretController.ts b/backend/src/controllers/v1/secretController.ts index 5d4b4da73..238b38ced 100644 --- a/backend/src/controllers/v1/secretController.ts +++ b/backend/src/controllers/v1/secretController.ts @@ -2,7 +2,7 @@ import { Request, Response } from 'express'; import * as Sentry from '@sentry/node'; import { Key, Secret } from '../../models'; import { - pushSecrets as push, + v1PushSecrets as push, pullSecrets as pull, reformatPullSecrets } from '../../helpers/secret'; diff --git a/backend/src/controllers/v2/workspaceController.ts b/backend/src/controllers/v2/workspaceController.ts index 3e784f22c..86693b6c4 100644 --- a/backend/src/controllers/v2/workspaceController.ts +++ b/backend/src/controllers/v2/workspaceController.ts @@ -15,7 +15,7 @@ import { deleteWorkspace as deleteWork } from '../../helpers/workspace'; import { - pushSecrets as push, + v2PushSecrets as push, pullSecrets as pull, reformatPullSecrets } from '../../helpers/secret'; @@ -24,17 +24,20 @@ import { addMemberships } from '../../helpers/membership'; import { postHogClient, EventService } from '../../services'; import { eventPushSecrets } from '../../events'; import { ADMIN, COMPLETED, GRANTED, ENV_SET } from '../../variables'; - -interface PushSecret { - ciphertextKey: string; - ivKey: string; - tagKey: string; - hashKey: string; - ciphertextValue: string; - ivValue: string; - tagValue: string; - hashValue: string; - type: 'shared' | 'personal'; +interface V2PushSecret { + type: string; // personal or shared + secretKeyCiphertext: string; + secretKeyIV: string; + secretKeyTag: string; + secretKeyHash: string; + secretValueCiphertext: string; + secretValueIV: string; + secretValueTag: string; + secretValueHash: string; + secretCommentCiphertext?: string; + secretCommentIV?: string; + secretCommentTag?: string; + secretCommentHash?: string; } /** @@ -364,11 +367,11 @@ export const getWorkspaceServiceTokens = async ( * @param res * @returns */ -export const pushSecrets = async (req: Request, res: Response) => { +export const pushWorkspaceSecrets = async (req: Request, res: Response) => { // upload (encrypted) secrets to workspace with id [workspaceId] try { - let { secrets }: { secrets: PushSecret[] } = req.body; + let { secrets }: { secrets: V2PushSecret[] } = req.body; const { keys, environment, channel } = req.body; const { workspaceId } = req.params; @@ -379,7 +382,7 @@ export const pushSecrets = async (req: Request, res: Response) => { // sanitize secrets secrets = secrets.filter( - (s: PushSecret) => s.ciphertextKey !== '' && s.ciphertextValue !== '' + (s: V2PushSecret) => s.secretKeyCiphertext !== '' && s.secretValueCiphertext !== '' ); await push({ @@ -437,6 +440,8 @@ export const pushSecrets = async (req: Request, res: Response) => { * @returns */ export const pullSecrets = async (req: Request, res: Response) => { + // TODO: only return secrets, do not return workspace key + let secrets; let key; try { diff --git a/backend/src/helpers/bot.ts b/backend/src/helpers/bot.ts index abaf73af4..b3f276b53 100644 --- a/backend/src/helpers/bot.ts +++ b/backend/src/helpers/bot.ts @@ -12,7 +12,6 @@ import { decryptSymmetric, decryptAsymmetric } from '../utils/crypto'; -import { decryptSecrets } from '../helpers/secret'; import { ENCRYPTION_KEY } from '../config'; import { SECRET_SHARED } from '../variables'; diff --git a/backend/src/helpers/secret.ts b/backend/src/helpers/secret.ts index 0e3d82749..f055971ae 100644 --- a/backend/src/helpers/secret.ts +++ b/backend/src/helpers/secret.ts @@ -14,9 +14,8 @@ import { } from '../ee/helpers/secret'; import { decryptSymmetric } from '../utils/crypto'; import { SECRET_SHARED, SECRET_PERSONAL } from '../variables'; -import { LICENSE_KEY } from '../config'; -interface PushSecret { +interface V1PushSecret { ciphertextKey: string; ivKey: string; tagKey: string; @@ -32,6 +31,22 @@ interface PushSecret { type: 'shared' | 'personal'; } +interface V2PushSecret { + type: string; // personal or shared + secretKeyCiphertext: string; + secretKeyIV: string; + secretKeyTag: string; + secretKeyHash: string; + secretValueCiphertext: string; + secretValueIV: string; + secretValueTag: string; + secretValueHash: string; + secretCommentCiphertext?: string; + secretCommentIV?: string; + secretCommentTag?: string; + secretCommentHash?: string; +} + interface Update { [index: string]: any; } @@ -49,7 +64,7 @@ type DecryptSecretType = 'text' | 'object' | 'expanded'; * @param {String} obj.environment - environment for secrets * @param {Object[]} obj.secrets - secrets to push */ -const pushSecrets = async ({ +const v1PushSecrets = async ({ userId, workspaceId, environment, @@ -58,7 +73,7 @@ const pushSecrets = async ({ userId: string; workspaceId: string; environment: string; - secrets: PushSecret[]; + secrets: V1PushSecret[]; }): Promise => { // TODO: clean up function and fix up types try { @@ -99,7 +114,7 @@ const pushSecrets = async ({ if (`${s.type}-${s.secretKeyHash}` in newSecretsObj) { if (s.secretValueHash !== newSecretsObj[`${s.type}-${s.secretKeyHash}`].hashValue || s.secretCommentHash !== newSecretsObj[`${s.type}-${s.secretKeyHash}`].hashComment) { - // case: filter secrets where value changed + // case: filter secrets where value or comment changed return true; } @@ -259,6 +274,249 @@ const pushSecrets = async ({ } }; +/** + * Push secrets for user with id [userId] to workspace + * with id [workspaceId] with environment [environment]. Follow steps: + * 1. Handle shared secrets (insert, delete) + * 2. handle personal secrets (insert, delete) + * @param {Object} obj + * @param {String} obj.userId - id of user to push secrets for + * @param {String} obj.workspaceId - id of workspace to push to + * @param {String} obj.environment - environment for secrets + * @param {Object[]} obj.secrets - secrets to push + */ + const v2PushSecrets = async ({ + userId, + workspaceId, + environment, + secrets +}: { + userId: string; + workspaceId: string; + environment: string; + secrets: V2PushSecret[]; +}): Promise => { + // TODO: clean up function and fix up types + try { + // construct useful data structures + const oldSecrets = await pullSecrets({ + userId, + workspaceId, + environment + }); + + const oldSecretsObj: any = oldSecrets.reduce((accumulator, s: any) => + ({ ...accumulator, [`${s.type}-${s.secretKeyHash}`]: s }) + , {}); + const newSecretsObj: any = secrets.reduce((accumulator, s) => + ({ ...accumulator, [`${s.type}-${s.secretKeyHash}`]: s }) + , {}); + + // handle deleting secrets + const toDelete = oldSecrets + .filter( + (s: ISecret) => !(`${s.type}-${s.secretKeyHash}` in newSecretsObj) + ) + .map((s) => s._id); + if (toDelete.length > 0) { + await Secret.deleteMany({ + _id: { $in: toDelete } + }); + + await SecretVersion.updateMany({ + secret: { $in: toDelete } + }, { + isDeleted: true + }); + } + + const toUpdate = oldSecrets + .filter((s) => { + if (`${s.type}-${s.secretKeyHash}` in newSecretsObj) { + if (s.secretValueHash !== newSecretsObj[`${s.type}-${s.secretKeyHash}`].secretValueHash + || s.secretCommentHash !== newSecretsObj[`${s.type}-${s.secretKeyHash}`].secretCommentHash) { + // case: filter secrets where value or comment changed + return true; + } + + if (!s.version) { + // case: filter (legacy) secrets that were not versioned + return true; + } + } + + return false; + }); + + const operations = toUpdate + .map((s) => { + const { + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretValueHash, + secretCommentCiphertext, + secretCommentIV, + secretCommentTag, + secretCommentHash, + } = newSecretsObj[`${s.type}-${s.secretKeyHash}`]; + + const update: Update = { + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretValueHash, + secretCommentCiphertext, + secretCommentIV, + secretCommentTag, + secretCommentHash, + } + + if (!s.version) { + // case: (legacy) secret was not versioned + update.version = 1; + } else { + update['$inc'] = { + version: 1 + } + } + + if (s.type === SECRET_PERSONAL) { + // attach user associated with the personal secret + update['user'] = userId; + } + + return { + updateOne: { + filter: { + _id: oldSecretsObj[`${s.type}-${s.secretKeyHash}`]._id + }, + update + } + }; + }); + await Secret.bulkWrite(operations as any); + + // (EE) add secret versions for updated secrets + await EESecretService.addSecretVersions({ + secretVersions: toUpdate.map((s) => { + const { + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretKeyHash, + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretValueHash, + secretCommentCiphertext, + secretCommentIV, + secretCommentTag, + secretCommentHash, + } = newSecretsObj[`${s.type}-${s.secretKeyHash}`]; + + return ({ + secret: s._id, + version: s.version ? s.version + 1 : 1, + isDeleted: false, + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretKeyHash, + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretValueHash + }) + }) + }); + + // handle adding new secrets + const toAdd = secrets.filter((s) => !(`${s.type}-${s.secretKeyHash}` in oldSecretsObj)); + + if (toAdd.length > 0) { + // add secrets + const newSecrets = await Secret.insertMany( + toAdd.map(({ + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretKeyHash, + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretValueHash, + secretCommentCiphertext, + secretCommentIV, + secretCommentTag, + secretCommentHash, + }, idx) => { + const obj: any = { + version: 1, + workspace: workspaceId, + type: toAdd[idx].type, + environment, + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretKeyHash, + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretValueHash, + secretCommentCiphertext, + secretCommentIV, + secretCommentTag, + secretCommentHash + }; + + if (toAdd[idx].type === 'personal') { + obj['user' as keyof typeof obj] = userId; + } + + return obj; + }) + ); + + // (EE) add secret versions for new secrets + EESecretService.addSecretVersions({ + secretVersions: newSecrets.map(({ + _id, + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretKeyHash, + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretValueHash + }) => ({ + secret: _id, + version: 1, + isDeleted: false, + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretKeyHash, + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretValueHash + })) + }); + } + + // (EE) take a secret snapshot + await EESecretService.takeSecretSnapshot({ + workspaceId + }) + } catch (err) { + Sentry.setUser(null); + Sentry.captureException(err); + throw new Error('Failed to push shared and personal secrets'); + } +}; + /** * Pull secrets for user with id [userId] for workspace * with id [workspaceId] with environment [environment] @@ -350,73 +608,9 @@ const reformatPullSecrets = ({ secrets }: { secrets: ISecret[] }) => { return reformatedSecrets; }; -/** - * Return decrypted secrets in format [format] - * @param {Object} obj - * @param {Object[]} obj.secrets - array of (encrypted) secret key-value pair objects - * @param {String} obj.key - symmetric key to decrypt secret key-value pairs - * @param {String} obj.format - desired return format that is either "text," "object," or "expanded" - * @return {String|Object} (decrypted) secrets also called the content - */ -const decryptSecrets = ({ - secrets, - key, - format -}: { - secrets: PushSecret[]; - key: string; - format: DecryptSecretType; -}) => { - // init content - let content: any = format === 'text' ? '' : {}; - - // decrypt secrets - secrets.forEach((s, idx) => { - const secretKey = decryptSymmetric({ - ciphertext: s.ciphertextKey, - iv: s.ivKey, - tag: s.tagKey, - key - }); - - const secretValue = decryptSymmetric({ - ciphertext: s.ciphertextValue, - iv: s.ivValue, - tag: s.tagValue, - key - }); - - switch (format) { - case 'text': - content += secretKey; - content += '='; - content += secretValue; - - if (idx < secrets.length) { - content += '\n'; - } - break; - case 'object': - content[secretKey] = secretValue; - break; - case 'expanded': - content[secretKey] = { - ...s, - plaintextKey: secretKey, - plaintextValue: secretValue - }; - break; - } - }); - - return content; -}; - - - export { - pushSecrets, + v1PushSecrets, + v2PushSecrets, pullSecrets, - reformatPullSecrets, - decryptSecrets + reformatPullSecrets }; diff --git a/backend/src/routes/v2/workspace.ts b/backend/src/routes/v2/workspace.ts index 4e954c75d..98eacd9ea 100644 --- a/backend/src/routes/v2/workspace.ts +++ b/backend/src/routes/v2/workspace.ts @@ -120,7 +120,7 @@ router.get( workspaceController.getWorkspaceIntegrationAuthorizations ); -router.get( +router.get( // TODO: modify '/:workspaceId/service-tokens', requireAuth, requireWorkspaceAuth({ @@ -145,7 +145,7 @@ router.post( body('channel'), param('workspaceId').exists().trim(), validateRequest, - workspaceController.pushSecrets + workspaceController.pushWorkspaceSecrets ); router.get( diff --git a/frontend/components/utilities/secrets/pushKeys.ts b/frontend/components/utilities/secrets/pushKeys.ts index e8a303992..1bedf357c 100644 --- a/frontend/components/utilities/secrets/pushKeys.ts +++ b/frontend/components/utilities/secrets/pushKeys.ts @@ -47,9 +47,9 @@ const pushKeys = async({ obj, workspaceId, env }: { obj: object; workspaceId: st const secrets = Object.keys(obj).map((key) => { // encrypt key const { - ciphertext: ciphertextKey, - iv: ivKey, - tag: tagKey, + ciphertext: secretKeyCiphertext, + iv: secretKeyIV, + tag: secretKeyTag, } = encryptSymmetric({ plaintext: key.slice(1), key: randomBytes, @@ -57,9 +57,9 @@ const pushKeys = async({ obj, workspaceId, env }: { obj: object; workspaceId: st // encrypt value const { - ciphertext: ciphertextValue, - iv: ivValue, - tag: tagValue, + ciphertext: secretValueCiphertext, + iv: secretValueIV, + tag: secretValueTag, } = encryptSymmetric({ plaintext: obj[key as keyof typeof obj][0], key: randomBytes, @@ -67,9 +67,9 @@ const pushKeys = async({ obj, workspaceId, env }: { obj: object; workspaceId: st // encrypt comment const { - ciphertext: ciphertextComment, - iv: ivComment, - tag: tagComment, + ciphertext: secretCommentCiphertext, + iv: secretCommentIV, + tag: secretCommentTag, } = encryptSymmetric({ plaintext: obj[key as keyof typeof obj][1], key: randomBytes, @@ -78,18 +78,18 @@ const pushKeys = async({ obj, workspaceId, env }: { obj: object; workspaceId: st const visibility = key.charAt(0) == "p" ? "personal" : "shared"; return { - ciphertextKey, - ivKey, - tagKey, - hashKey: crypto.createHash("sha256").update(key.slice(1)).digest("hex"), - ciphertextValue, - ivValue, - tagValue, - hashValue: crypto.createHash("sha256").update(obj[key as keyof typeof obj][0]).digest("hex"), - ciphertextComment, - ivComment, - tagComment, - hashComment: crypto.createHash("sha256").update(obj[key as keyof typeof obj][1]).digest("hex"), + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretKeyHash: crypto.createHash("sha256").update(key.slice(1)).digest("hex"), + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretValueHash: crypto.createHash("sha256").update(obj[key as keyof typeof obj][0]).digest("hex"), + secretCommentCiphertext, + secretCommentIV, + secretCommentTag, + secretCommentHash: crypto.createHash("sha256").update(obj[key as keyof typeof obj][1]).digest("hex"), type: visibility, }; }); diff --git a/frontend/pages/api/files/UploadSecrets.ts b/frontend/pages/api/files/UploadSecrets.ts index 04fcb78b5..03a98d483 100644 --- a/frontend/pages/api/files/UploadSecrets.ts +++ b/frontend/pages/api/files/UploadSecrets.ts @@ -22,7 +22,7 @@ const uploadSecrets = async ({ keys, environment }: Props) => { - return SecurityClient.fetchCall('/api/v1/secret/' + workspaceId, { + return SecurityClient.fetchCall('/api/v2/workspace/' + workspaceId + '/secrets', { method: 'POST', headers: { 'Content-Type': 'application/json'