From 7e534629ff56d651095d13fa2ae153273e3e4909 Mon Sep 17 00:00:00 2001 From: Vladyslav Matsiiako Date: Fri, 6 Jan 2023 17:32:19 -0800 Subject: [PATCH 01/13] Started adding telemetry to v2 routes --- .../src/controllers/v2/secretController.ts | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+) diff --git a/backend/src/controllers/v2/secretController.ts b/backend/src/controllers/v2/secretController.ts index 3bb9f1720..f291dd1e7 100644 --- a/backend/src/controllers/v2/secretController.ts +++ b/backend/src/controllers/v2/secretController.ts @@ -9,6 +9,7 @@ import { AnyBulkWriteOperation } from 'mongodb'; import { SECRET_PERSONAL, SECRET_SHARED } from "../../variables"; import { validateMembership } from "../../helpers/membership"; import { ADMIN, MEMBER } from '../../variables'; +import { postHogClient } from '../../services'; export const createSingleSecret = async (req: Request, res: Response) => { const secretToCreate: CreateSecretRequestBody = req.body.secret; @@ -38,6 +39,20 @@ export const createSingleSecret = async (req: Request, res: Response) => { throw RouteValidationError({ message: error.message, stack: error.stack }) } + if (postHogClient) { + postHogClient.capture({ + event: 'secrets added', + distinctId: req.user.email, + properties: { + numberOfSecrets: 1, + environment: environmentName, + workspaceId, + // #TODO: why does this route have no channel? + // channel: channel ? channel : 'cli' + } + }); + } + res.status(200).send() } @@ -78,6 +93,20 @@ export const batchCreateSecrets = async (req: Request, res: Response) => { throw InternalServerError({ message: "Unable to process your batch create request. Please try again", stack: bulkCreateError.stack }) } + if (postHogClient) { + postHogClient.capture({ + event: 'secrets added', + distinctId: req.user.email, + properties: { + numberOfSecrets: (secretsToCreate ?? []).length, + environment: environmentName, + workspaceId, + // #TODO: why does this route have no channel? + // channel: channel ? channel : 'cli' + } + }); + } + res.status(200).send() } @@ -93,10 +122,12 @@ export const batchDeleteSecrets = async (req: Request, res: Response) => { const secretsUserCanDeleteSet: Set = new Set(secretIdsUserCanDelete.map(objectId => objectId._id.toString())); const deleteOperationsToPerform: AnyBulkWriteOperation[] = [] + let numSecretsDeleted = 0; secretIdsToDelete.forEach(secretIdToDelete => { if (secretsUserCanDeleteSet.has(secretIdToDelete)) { const deleteOperation = { deleteOne: { filter: { _id: new Types.ObjectId(secretIdToDelete) } } } deleteOperationsToPerform.push(deleteOperation) + numSecretsDeleted++; } else { throw RouteValidationError({ message: "You cannot delete secrets that you do not have access to" }) } @@ -110,6 +141,20 @@ export const batchDeleteSecrets = async (req: Request, res: Response) => { throw InternalServerError() } + if (postHogClient) { + postHogClient.capture({ + event: 'secrets deleted', + distinctId: req.user.email, + properties: { + numberOfSecrets: numSecretsDeleted, + environment: environmentName, + workspaceId, + // #TODO: why does this route have no channel? + // channel: channel ? channel : 'cli' + } + }); + } + res.status(200).send() } @@ -134,6 +179,21 @@ export const deleteSingleSecret = async (req: Request, res: Response) => { await Secret.findByIdAndDelete(secretId) + if (postHogClient) { + postHogClient.capture({ + event: 'secrets deleted', + distinctId: req.user.email, + properties: { + numberOfSecrets: 1, + // #TODO: how do we get env name ans project id? + // environment: environmentName, + // workspaceId, + // #TODO: why does this route have no channel? + // channel: channel ? channel : 'cli' + } + }); + } + res.status(200).send() } else { throw BadRequestError() @@ -184,9 +244,24 @@ export const batchModifySecrets = async (req: Request, res: Response) => { throw InternalServerError() } + if (postHogClient) { + postHogClient.capture({ + event: 'secrets modified', + distinctId: req.user.email, + properties: { + numberOfSecrets: (secretsModificationsRequested ?? []).length, + environment: environmentName, + workspaceId, + // #TODO: why does this route have no channel? + // channel: channel ? channel : 'cli' + } + }); + } + return res.status(200).send() } +// #TODO: I assume this should be '...Secret'? export const modifySingleSecrets = async (req: Request, res: Response) => { const { workspaceId, environmentName } = req.params const secretModificationsRequested: ModifySecretRequestBody = req.body.secret; @@ -216,6 +291,20 @@ export const modifySingleSecrets = async (req: Request, res: Response) => { throw RouteValidationError({ message: "Unable to apply modifications, please try again", stack: error.stack }) } + if (postHogClient) { + postHogClient.capture({ + event: 'secrets modified', + distinctId: req.user.email, + properties: { + numberOfSecrets: 1, + environment: environmentName, + workspaceId, + // #TODO: why does this route have no channel? + // channel: channel ? channel : 'cli' + } + }); + } + return res.status(200).send(singleModificationUpdate) } @@ -245,6 +334,20 @@ export const fetchAllSecrets = async (req: Request, res: Response) => { throw RouteValidationError({ message: "Unable to get secrets, please try again", stack: retriveAllSecretsError.stack }) } + if (postHogClient) { + postHogClient.capture({ + event: 'secrets pulled', + distinctId: req.user.email, + properties: { + numberOfSecrets: (allSecrets ?? []).length, + environment, + workspaceId, + // #TODO: why does this route have no channel? + // channel: channel ? channel : 'cli' + } + }); + } + return res.json(allSecrets) } @@ -270,6 +373,21 @@ export const fetchSingleSecret = async (req: Request, res: Response) => { res.json(singleSecretRetrieved) + if (postHogClient) { + postHogClient.capture({ + event: 'secrets pulled', + distinctId: req.user.email, + properties: { + numberOfSecrets: 1, + // #TODO: how do we get environment and workspace here? Do we need that? When is this route used? + // environment, + // workspaceId, + // #TODO: why does this route have no channel? + // channel: channel ? channel : 'cli' + } + }); + } + } else { throw BadRequestError() } From 47ab0b4a0fbda441e93496df074376c0c1944a8b Mon Sep 17 00:00:00 2001 From: Tuan Dang Date: Sat, 7 Jan 2023 20:12:53 +0700 Subject: [PATCH 02/13] Add endpoints for rolling back a workspace to a secret snapshot and rolling back a secret to a version --- .../controllers/v2/apiKeyDataController.ts | 1 - .../src/ee/controllers/v1/secretController.ts | 101 +++++++++++ .../v1/secretSnapshotController.ts | 6 + .../ee/controllers/v1/workspaceController.ts | 165 +++++++++++++++++- backend/src/ee/helpers/action.ts | 75 ++------ backend/src/ee/helpers/secretVersion.ts | 110 ++++++++++++ backend/src/ee/models/secretVersion.ts | 13 +- backend/src/ee/routes/v1/secret.ts | 15 +- backend/src/ee/routes/v1/secretSnapshot.ts | 2 +- backend/src/ee/routes/v1/workspace.ts | 16 +- backend/src/helpers/secret.ts | 2 + 11 files changed, 432 insertions(+), 74 deletions(-) create mode 100644 backend/src/ee/helpers/secretVersion.ts diff --git a/backend/src/controllers/v2/apiKeyDataController.ts b/backend/src/controllers/v2/apiKeyDataController.ts index 3aacde8af..cafbacb5b 100644 --- a/backend/src/controllers/v2/apiKeyDataController.ts +++ b/backend/src/controllers/v2/apiKeyDataController.ts @@ -65,7 +65,6 @@ export const createAPIKeyData = async (req: Request, res: Response) => { apiKey = `ak.${apiKeyData._id.toString()}.${secret}`; } catch (err) { - console.error(err); Sentry.setUser({ email: req.user.email }); Sentry.captureException(err); return res.status(400).send({ diff --git a/backend/src/ee/controllers/v1/secretController.ts b/backend/src/ee/controllers/v1/secretController.ts index a2d68ca96..751f21611 100644 --- a/backend/src/ee/controllers/v1/secretController.ts +++ b/backend/src/ee/controllers/v1/secretController.ts @@ -1,6 +1,8 @@ import { Request, Response } from 'express'; import * as Sentry from '@sentry/node'; +import { Secret } from '../../../models'; import { SecretVersion } from '../../models'; +import { EESecretService } from '../../services'; /** * Return secret versions for secret with id [secretId] @@ -33,4 +35,103 @@ import { SecretVersion } from '../../models'; return res.status(200).send({ secretVersions }); +} + +/** + * Roll back secret with id [secretId] to version [version] + * @param req + * @param res + * @returns + */ +export const rollbackSecretVersion = async (req: Request, res: Response) => { + let secret; + try { + const { secretId } = req.params; + const { version } = req.body; + + // validate secret version + const oldSecretVersion = await SecretVersion.findOne({ + secret: secretId, + version + }); + + if (!oldSecretVersion) throw new Error('Failed to find secret version'); + + const { + workspace, + type, + user, + environment, + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretKeyHash, + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretValueHash + } = oldSecretVersion; + + // update secret + secret = await Secret.findByIdAndUpdate( + secretId, + { + $inc: { + version: 1 + }, + workspace, + type, + user, + environment, + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretKeyHash, + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretValueHash + }, + { + new: true + } + ); + + if (!secret) throw new Error('Failed to find and update secret'); + + // add new secret version + await new SecretVersion({ + secret: secretId, + version: secret.version, + workspace, + type, + user, + environment, + isDeleted: false, + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretKeyHash, + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretValueHash + }).save(); + + // take secret snapshot + await EESecretService.takeSecretSnapshot({ + workspaceId: secret.workspace.toString() + }); + + } catch (err) { + Sentry.setUser({ email: req.user.email }); + Sentry.captureException(err); + return res.status(400).send({ + message: 'Failed to roll back secret version' + }); + } + + return res.status(200).send({ + secret + }); } \ No newline at end of file diff --git a/backend/src/ee/controllers/v1/secretSnapshotController.ts b/backend/src/ee/controllers/v1/secretSnapshotController.ts index 40e1a74a6..6e8605c2f 100644 --- a/backend/src/ee/controllers/v1/secretSnapshotController.ts +++ b/backend/src/ee/controllers/v1/secretSnapshotController.ts @@ -2,6 +2,12 @@ import { Request, Response } from 'express'; import * as Sentry from '@sentry/node'; import { SecretSnapshot } from '../../models'; +/** + * Return secret snapshot with id [secretSnapshotId] + * @param req + * @param res + * @returns + */ export const getSecretSnapshot = async (req: Request, res: Response) => { let secretSnapshot; try { diff --git a/backend/src/ee/controllers/v1/workspaceController.ts b/backend/src/ee/controllers/v1/workspaceController.ts index 88c31b8e1..8fd7c8746 100644 --- a/backend/src/ee/controllers/v1/workspaceController.ts +++ b/backend/src/ee/controllers/v1/workspaceController.ts @@ -1,9 +1,17 @@ -import e, { Request, Response } from 'express'; +import { Request, Response } from 'express'; import * as Sentry from '@sentry/node'; +import { Types } from 'mongoose'; +import { + Secret +} from '../../../models'; import { SecretSnapshot, - Log + Log, + SecretVersion, + ISecretVersion } from '../../models'; +import { EESecretService } from '../../services'; +import { getLatestSecretVersionIds } from '../../helpers/secretVersion'; /** * Return secret snapshots for workspace with id [workspaceId] @@ -63,6 +71,159 @@ export const getWorkspaceSecretSnapshotsCount = async (req: Request, res: Respon }); } +/** + * Rollback secret snapshot with id [secretSnapshotId] to version [version] + * @param req + * @param res + * @returns + */ +export const rollbackWorkspaceSecretSnapshot = async (req: Request, res: Response) => { + let secrets; + try { + const { workspaceId } = req.params; + const { version } = req.body; + + // validate secret snapshot + const secretSnapshot = await SecretSnapshot.findOne({ + workspace: workspaceId, + version + }).populate<{ secretVersions: ISecretVersion[]}>('secretVersions'); + + if (!secretSnapshot) throw new Error('Failed to find secret snapshot'); + + // TODO: fix any + const oldSecretVersionsObj: any = secretSnapshot.secretVersions + .reduce((accumulator, s) => ({ + ...accumulator, + [`${s.secret.toString()}`]: s + }), {}); + + const latestSecretVersionIds = await getLatestSecretVersionIds({ + secretIds: secretSnapshot.secretVersions.map((sv) => sv.secret) + }); + + // TODO: fix any + const latestSecretVersions: any = (await SecretVersion.find({ + _id: { + $in: latestSecretVersionIds.map((s) => s.versionId) + } + }, 'secret version')) + .reduce((accumulator, s) => ({ + ...accumulator, + [`${s.secret.toString()}`]: s + }), {}); + + // delete existing secrets + await Secret.deleteMany({ + workspace: workspaceId + }); + + // add secrets + secrets = await Secret.insertMany( + secretSnapshot.secretVersions.map((sv) => { + const secretId = sv.secret; + const { + workspace, + type, + user, + environment, + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretKeyHash, + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretValueHash, + createdAt + } = oldSecretVersionsObj[secretId.toString()]; + + return ({ + _id: secretId, + version: latestSecretVersions[secretId.toString()].version + 1, + workspace, + type, + user, + environment, + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretKeyHash, + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretValueHash, + secretCommentCiphertext: '', + secretCommentIV: '', + secretCommentTag: '', + createdAt + }); + }) + ); + + // add secret versions + await SecretVersion.insertMany( + secrets.map(({ + _id, + version, + workspace, + type, + user, + environment, + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretKeyHash, + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretValueHash + }) => ({ + _id: new Types.ObjectId(), + secret: _id, + version, + workspace, + type, + user, + environment, + isDeleted: false, + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretKeyHash, + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretValueHash + })) + ); + + // update secret versions of restored secrets as not deleted + await SecretVersion.updateMany({ + secret: { + $in: secretSnapshot.secretVersions.map((sv) => sv.secret) + } + }, { + isDeleted: false + }); + + // take secret snapshot + await EESecretService.takeSecretSnapshot({ + workspaceId + }); + } catch (err) { + Sentry.setUser({ email: req.user.email }); + Sentry.captureException(err); + return res.status(400).send({ + message: 'Failed to roll back secret snapshot' + }); + } + + return res.status(200).send({ + secrets + }); +} + /** * Return (audit) logs for workspace with id [workspaceId] * @param req diff --git a/backend/src/ee/helpers/action.ts b/backend/src/ee/helpers/action.ts index 2971e3f96..389cfedbf 100644 --- a/backend/src/ee/helpers/action.ts +++ b/backend/src/ee/helpers/action.ts @@ -1,7 +1,10 @@ import * as Sentry from '@sentry/node'; import { Types } from 'mongoose'; -import { Secret } from '../../models'; import { SecretVersion, Action } from '../models'; +import { + getLatestSecretVersionIds, + getLatestNSecretSecretVersionIds +} from '../helpers/secretVersion'; import { ACTION_UPDATE_SECRETS } from '../../variables'; /** @@ -30,65 +33,23 @@ const createActionSecretHelper = async ({ if (name === ACTION_UPDATE_SECRETS) { // case: action is updating secrets // -> add old and new secret versions - - // TODO: make query more efficient - latestSecretVersions = (await SecretVersion.aggregate([ - { - $match: { - secret: { - $in: secretIds, - }, - }, - }, - { - $sort: { version: -1 }, - }, - { - $group: { - _id: "$secret", - versions: { $push: "$$ROOT" }, - }, - }, - { - $project: { - _id: 0, - secret: "$_id", - versions: { $slice: ["$versions", 2] }, - }, - } - ])) - .map((s) => ({ - oldSecretVersion: s.versions[0]._id, - newSecretVersion: s.versions[1]._id - })); - - + latestSecretVersions = (await getLatestNSecretSecretVersionIds({ + secretIds, + n: 2 + })) + .map((s) => ({ + oldSecretVersion: s.versions[0]._id, + newSecretVersion: s.versions[1]._id + })); } else { // case: action is adding, deleting, or reading secrets // -> add new secret versions - latestSecretVersions = (await SecretVersion.aggregate([ - { - $match: { - secret: { - $in: secretIds - } - } - }, - { - $group: { - _id: '$secret', - version: { $max: '$version' }, - versionId: { $max: '$_id' } // secret version id - } - }, - { - $sort: { version: -1 } - } - ]) - .exec()) - .map((s) => ({ - newSecretVersion: s.versionId - })); + latestSecretVersions = (await getLatestSecretVersionIds({ + secretIds + })) + .map((s) => ({ + newSecretVersion: s.versionId + })); } action = await new Action({ diff --git a/backend/src/ee/helpers/secretVersion.ts b/backend/src/ee/helpers/secretVersion.ts new file mode 100644 index 000000000..d5859e183 --- /dev/null +++ b/backend/src/ee/helpers/secretVersion.ts @@ -0,0 +1,110 @@ +import * as Sentry from '@sentry/node'; +import { Types } from 'mongoose'; +import { SecretVersion } from '../models'; + +/** + * Return latest secret versions for secrets with ids [secretIds] + * @param {Object} obj + * @param {Object} obj.secretIds = ids of secrets to get latest versions for + * @returns + */ +const getLatestSecretVersionIds = async ({ + secretIds +}: { + secretIds: Types.ObjectId[]; +}) => { + + interface LatestSecretVersionId { + _id: Types.ObjectId; + version: number; + versionId: Types.ObjectId; + } + + let latestSecretVersionIds: LatestSecretVersionId[]; + try { + latestSecretVersionIds = (await SecretVersion.aggregate([ + { + $match: { + secret: { + $in: secretIds + } + } + }, + { + $group: { + _id: '$secret', + version: { $max: '$version' }, + versionId: { $max: '$_id' } // id of latest secret version + } + }, + { + $sort: { version: -1 } + } + ]) + .exec()); + + } catch (err) { + Sentry.setUser(null); + Sentry.captureException(err); + throw new Error('Failed to get latest secret versions'); + } + + return latestSecretVersionIds; +} + +/** + * Return latest [n] secret versions for secrets with ids [secretIds] + * @param {Object} obj + * @param {Object} obj.secretIds = ids of secrets to get latest versions for + * @param {Number} obj.n - number of latest secret versions to return for each secret + * @returns + */ +const getLatestNSecretSecretVersionIds = async ({ + secretIds, + n +}: { + secretIds: Types.ObjectId[]; + n: number; +}) => { + + // TODO: optimize query + let latestNSecretVersions; + try { + latestNSecretVersions = (await SecretVersion.aggregate([ + { + $match: { + secret: { + $in: secretIds, + }, + }, + }, + { + $sort: { version: -1 }, + }, + { + $group: { + _id: "$secret", + versions: { $push: "$$ROOT" }, + }, + }, + { + $project: { + _id: 0, + secret: "$_id", + versions: { $slice: ["$versions", n] }, + }, + } + ])); + } catch (err) { + Sentry.setUser(null); + Sentry.captureException(err); + throw new Error('Failed to get latest n secret versions'); + } + + return latestNSecretVersions; +} + +export { + getLatestSecretVersionIds, + getLatestNSecretSecretVersionIds +} diff --git a/backend/src/ee/models/secretVersion.ts b/backend/src/ee/models/secretVersion.ts index 0197c3a25..71283e93f 100644 --- a/backend/src/ee/models/secretVersion.ts +++ b/backend/src/ee/models/secretVersion.ts @@ -8,17 +8,8 @@ import { ENV_PROD } from '../../variables'; -/** - * TODO: - * 1. Modify SecretVersion to also contain XX - * - type - * - user - * - environment - * 2. Modify SecretSnapshot to point to arrays of SecretVersion - */ - export interface ISecretVersion { - _id?: Types.ObjectId; + _id: Types.ObjectId; secret: Types.ObjectId; version: number; workspace: Types.ObjectId; // new @@ -68,7 +59,7 @@ const secretVersionSchema = new Schema( enum: [ENV_DEV, ENV_TESTING, ENV_STAGING, ENV_PROD], required: true }, - isDeleted: { + isDeleted: { // consider removing field type: Boolean, default: false, required: true diff --git a/backend/src/ee/routes/v1/secret.ts b/backend/src/ee/routes/v1/secret.ts index 43cc8bafc..ac3089e33 100644 --- a/backend/src/ee/routes/v1/secret.ts +++ b/backend/src/ee/routes/v1/secret.ts @@ -5,7 +5,7 @@ import { requireSecretAuth, validateRequest } from '../../../middleware'; -import { query, param } from 'express-validator'; +import { query, param, body } from 'express-validator'; import { secretController } from '../../controllers/v1'; import { ADMIN, MEMBER } from '../../../variables'; @@ -24,4 +24,17 @@ router.get( secretController.getSecretVersions ); +router.post( + '/:secretId/secret-versions/rollback', + requireAuth({ + acceptedAuthModes: ['jwt'] + }), + requireSecretAuth({ + acceptedRoles: [ADMIN, MEMBER] + }), + param('secretId').exists().trim(), + body('version').exists().isInt(), + secretController.rollbackSecretVersion +); + export default router; \ No newline at end of file diff --git a/backend/src/ee/routes/v1/secretSnapshot.ts b/backend/src/ee/routes/v1/secretSnapshot.ts index 80aa7d1ee..d10da4456 100644 --- a/backend/src/ee/routes/v1/secretSnapshot.ts +++ b/backend/src/ee/routes/v1/secretSnapshot.ts @@ -7,7 +7,7 @@ import { requireAuth, validateRequest } from '../../../middleware'; -import { param } from 'express-validator'; +import { param, body } from 'express-validator'; import { ADMIN, MEMBER } from '../../../variables'; import { secretSnapshotController } from '../../controllers/v1'; diff --git a/backend/src/ee/routes/v1/workspace.ts b/backend/src/ee/routes/v1/workspace.ts index 4b2e839eb..c9da58261 100644 --- a/backend/src/ee/routes/v1/workspace.ts +++ b/backend/src/ee/routes/v1/workspace.ts @@ -5,7 +5,7 @@ import { requireWorkspaceAuth, validateRequest } from '../../../middleware'; -import { param, query } from 'express-validator'; +import { param, query, body } from 'express-validator'; import { ADMIN, MEMBER } from '../../../variables'; import { workspaceController } from '../../controllers/v1'; @@ -37,6 +37,20 @@ router.get( workspaceController.getWorkspaceSecretSnapshotsCount ); +router.post( + '/:workspaceId/secret-snapshots/rollback', + requireAuth({ + acceptedAuthModes: ['jwt'] + }), + requireWorkspaceAuth({ + acceptedRoles: [ADMIN, MEMBER] + }), + param('workspaceId').exists().trim(), + body('version').exists().isInt(), + validateRequest, + workspaceController.rollbackWorkspaceSecretSnapshot +); + router.get( '/:workspaceId/logs', requireAuth({ diff --git a/backend/src/helpers/secret.ts b/backend/src/helpers/secret.ts index 920e8dc1d..e4df6d971 100644 --- a/backend/src/helpers/secret.ts +++ b/backend/src/helpers/secret.ts @@ -187,6 +187,7 @@ const v1PushSecrets = async ({ }) => { const newSecret = newSecretsObj[`${type}-${secretKeyHash}`]; return ({ + _id: new Types.ObjectId(), secret: _id, version: version ? version + 1 : 1, workspace: new Types.ObjectId(workspaceId), @@ -258,6 +259,7 @@ const v1PushSecrets = async ({ secretValueTag, secretValueHash }) => ({ + _id: new Types.ObjectId(), secret: _id, version, workspace, From 35d23cf55c197b27b43966e25db242021f27d844 Mon Sep 17 00:00:00 2001 From: Tuan Dang Date: Mon, 9 Jan 2023 01:03:40 +0700 Subject: [PATCH 03/13] Finish preliminary /v2/secrets routes for batch/single CRUD secrets endpoints --- backend/src/app.ts | 10 +- backend/src/controllers/v2/index.ts | 4 +- .../src/controllers/v2/secretController.ts | 232 ++++----- .../src/controllers/v2/secretsController.ts | 472 ++++++++++++++++++ backend/src/ee/models/secretVersion.ts | 6 +- backend/src/helpers/secret.ts | 47 +- backend/src/middleware/index.ts | 2 + backend/src/middleware/requireSecretAuth.ts | 5 +- backend/src/middleware/requireSecretsAuth.ts | 49 ++ backend/src/models/secret.ts | 6 +- backend/src/routes/v2/index.ts | 4 +- backend/src/routes/v2/secret.ts | 71 ++- backend/src/routes/v2/secrets.ts | 177 +++++++ backend/src/types/express/index.d.ts | 4 +- 14 files changed, 918 insertions(+), 171 deletions(-) create mode 100644 backend/src/controllers/v2/secretsController.ts create mode 100644 backend/src/middleware/requireSecretsAuth.ts create mode 100644 backend/src/routes/v2/secrets.ts diff --git a/backend/src/app.ts b/backend/src/app.ts index 000fa647d..06f0c7a0b 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -38,6 +38,7 @@ import { } from './routes/v1'; import { secret as v2SecretRouter, + secrets as v2SecretsRouter, workspace as v2WorkspaceRouter, serviceTokenData as v2ServiceTokenDataRouter, apiKeyData as v2APIKeyDataRouter, @@ -91,16 +92,17 @@ app.use('/api/v1/membership', v1MembershipRouter); app.use('/api/v1/key', v1KeyRouter); app.use('/api/v1/invite-org', v1InviteOrgRouter); app.use('/api/v1/secret', v1SecretRouter); -app.use('/api/v1/service-token', v1ServiceTokenRouter); // deprecate +app.use('/api/v1/service-token', v1ServiceTokenRouter); // stop supporting app.use('/api/v1/password', v1PasswordRouter); app.use('/api/v1/stripe', v1StripeRouter); app.use('/api/v1/integration', v1IntegrationRouter); app.use('/api/v1/integration-auth', v1IntegrationAuthRouter); // v2 routes -app.use('/api/v2/workspace', v2WorkspaceRouter); -app.use('/api/v2/secret', v2SecretRouter); -app.use('/api/v2/service-token', v2ServiceTokenDataRouter); +app.use('/api/v2/workspace', v2WorkspaceRouter); // TODO: turn into plural route +app.use('/api/v2/secret', v2SecretRouter); // stop supporting, TODO: revise +app.use('/api/v2/secrets', v2SecretsRouter); +app.use('/api/v2/service-token', v2ServiceTokenDataRouter); // TODO: turn into plural route app.use('/api/v2/api-key-data', v2APIKeyDataRouter); diff --git a/backend/src/controllers/v2/index.ts b/backend/src/controllers/v2/index.ts index 2c5cce660..1651c09ee 100644 --- a/backend/src/controllers/v2/index.ts +++ b/backend/src/controllers/v2/index.ts @@ -2,10 +2,12 @@ import * as workspaceController from './workspaceController'; import * as serviceTokenDataController from './serviceTokenDataController'; import * as apiKeyDataController from './apiKeyDataController'; import * as secretController from './secretController'; +import * as secretsController from './secretsController'; export { workspaceController, serviceTokenDataController, apiKeyDataController, - secretController + secretController, + secretsController } diff --git a/backend/src/controllers/v2/secretController.ts b/backend/src/controllers/v2/secretController.ts index f291dd1e7..b993d3811 100644 --- a/backend/src/controllers/v2/secretController.ts +++ b/backend/src/controllers/v2/secretController.ts @@ -7,13 +7,16 @@ const { ValidationError } = mongoose.Error; import { BadRequestError, InternalServerError, UnauthorizedRequestError, ValidationError as RouteValidationError } from '../../utils/errors'; import { AnyBulkWriteOperation } from 'mongodb'; import { SECRET_PERSONAL, SECRET_SHARED } from "../../variables"; -import { validateMembership } from "../../helpers/membership"; -import { ADMIN, MEMBER } from '../../variables'; import { postHogClient } from '../../services'; -export const createSingleSecret = async (req: Request, res: Response) => { +/** + * Create secret for workspace with id [workspaceId] and environment [environment] + * @param req + * @param res + */ +export const createSecret = async (req: Request, res: Response) => { const secretToCreate: CreateSecretRequestBody = req.body.secret; - const { workspaceId, environmentName } = req.params + const { workspaceId, environment } = req.params const sanitizedSecret: SanitizedSecretForCreate = { secretKeyCiphertext: secretToCreate.secretKeyCiphertext, secretKeyIV: secretToCreate.secretKeyIV, @@ -28,13 +31,13 @@ export const createSingleSecret = async (req: Request, res: Response) => { secretCommentTag: secretToCreate.secretCommentTag, secretCommentHash: secretToCreate.secretCommentHash, workspace: new Types.ObjectId(workspaceId), - environment: environmentName, + environment, type: secretToCreate.type, user: new Types.ObjectId(req.user._id) } - const [error, newlyCreatedSecret] = await to(Secret.create(sanitizedSecret).then()) + const [error, secret] = await to(Secret.create(sanitizedSecret).then()) if (error instanceof ValidationError) { throw RouteValidationError({ message: error.message, stack: error.stack }) } @@ -45,20 +48,27 @@ export const createSingleSecret = async (req: Request, res: Response) => { distinctId: req.user.email, properties: { numberOfSecrets: 1, - environment: environmentName, workspaceId, - // #TODO: why does this route have no channel? - // channel: channel ? channel : 'cli' + environment, + channel: req.headers?.['user-agent']?.toLowerCase().includes('mozilla') ? 'web' : 'cli', + userAgent: req.headers?.['user-agent'] } }); - } + } - res.status(200).send() + res.status(200).send({ + secret + }) } -export const batchCreateSecrets = async (req: Request, res: Response) => { +/** + * Create many secrets for workspace wiht id [workspaceId] and environment [environment] + * @param req + * @param res + */ +export const createSecrets = async (req: Request, res: Response) => { const secretsToCreate: CreateSecretRequestBody[] = req.body.secrets; - const { workspaceId, environmentName } = req.params + const { workspaceId, environment } = req.params const sanitizedSecretesToCreate: SanitizedSecretForCreate[] = [] secretsToCreate.forEach(rawSecret => { @@ -76,7 +86,7 @@ export const batchCreateSecrets = async (req: Request, res: Response) => { secretCommentTag: rawSecret.secretCommentTag, secretCommentHash: rawSecret.secretCommentHash, workspace: new Types.ObjectId(workspaceId), - environment: environmentName, + environment, type: rawSecret.type, user: new Types.ObjectId(req.user._id) } @@ -84,7 +94,7 @@ export const batchCreateSecrets = async (req: Request, res: Response) => { sanitizedSecretesToCreate.push(safeUpdateFields) }) - const [bulkCreateError, newlyCreatedSecrets] = await to(Secret.insertMany(sanitizedSecretesToCreate).then()) + const [bulkCreateError, secrets] = await to(Secret.insertMany(sanitizedSecretesToCreate).then()) if (bulkCreateError) { if (bulkCreateError instanceof ValidationError) { throw RouteValidationError({ message: bulkCreateError.message, stack: bulkCreateError.stack }) @@ -99,18 +109,25 @@ export const batchCreateSecrets = async (req: Request, res: Response) => { distinctId: req.user.email, properties: { numberOfSecrets: (secretsToCreate ?? []).length, - environment: environmentName, workspaceId, - // #TODO: why does this route have no channel? - // channel: channel ? channel : 'cli' + environment, + channel: req.headers?.['user-agent']?.toLowerCase().includes('mozilla') ? 'web' : 'cli', + userAgent: req.headers?.['user-agent'] } }); } - res.status(200).send() + res.status(200).send({ + secrets + }) } -export const batchDeleteSecrets = async (req: Request, res: Response) => { +/** + * Delete secrets in workspace with id [workspaceId] and environment [environment] + * @param req + * @param res + */ +export const deleteSecrets = async (req: Request, res: Response) => { const { workspaceId, environmentName } = req.params const secretIdsToDelete: string[] = req.body.secretIds @@ -149,8 +166,8 @@ export const batchDeleteSecrets = async (req: Request, res: Response) => { numberOfSecrets: numSecretsDeleted, environment: environmentName, workspaceId, - // #TODO: why does this route have no channel? - // channel: channel ? channel : 'cli' + channel: req.headers?.['user-agent']?.toLowerCase().includes('mozilla') ? 'web' : 'cli', + userAgent: req.headers?.['user-agent'] } }); } @@ -158,49 +175,40 @@ export const batchDeleteSecrets = async (req: Request, res: Response) => { res.status(200).send() } -export const deleteSingleSecret = async (req: Request, res: Response) => { - const { secretId } = req.params; +/** + * Delete secret with id [secretId] + * @param req + * @param res + */ +export const deleteSecret = async (req: Request, res: Response) => { + await Secret.findByIdAndDelete(req._secret._id) - const [error, singleSecretRetrieved] = await to(Secret.findById(secretId).then()) - if (error instanceof ValidationError) { - throw RouteValidationError({ message: "Unable to get secret, please try again", stack: error.stack }) + if (postHogClient) { + postHogClient.capture({ + event: 'secrets deleted', + distinctId: req.user.email, + properties: { + numberOfSecrets: 1, + workspaceId: req._secret.workspace.toString(), + environment: req._secret.environment, + channel: req.headers?.['user-agent']?.toLowerCase().includes('mozilla') ? 'web' : 'cli', + userAgent: req.headers?.['user-agent'] + } + }); } - if (singleSecretRetrieved) { - const [membershipValidationError, membership] = await to(validateMembership({ - userId: req.user._id, - workspaceId: singleSecretRetrieved.workspace._id.toString(), - acceptedRoles: [ADMIN, MEMBER] - })) - - if (membershipValidationError || !membership) { - throw UnauthorizedRequestError() - } - - await Secret.findByIdAndDelete(secretId) - - if (postHogClient) { - postHogClient.capture({ - event: 'secrets deleted', - distinctId: req.user.email, - properties: { - numberOfSecrets: 1, - // #TODO: how do we get env name ans project id? - // environment: environmentName, - // workspaceId, - // #TODO: why does this route have no channel? - // channel: channel ? channel : 'cli' - } - }); - } - - res.status(200).send() - } else { - throw BadRequestError() - } + res.status(200).send({ + secret: req._secret + }) } -export const batchModifySecrets = async (req: Request, res: Response) => { +/** + * Update secrets for workspace with id [workspaceId] and environment [environment] + * @param req + * @param res + * @returns + */ +export const updateSecrets = async (req: Request, res: Response) => { const { workspaceId, environmentName } = req.params const secretsModificationsRequested: ModifySecretRequestBody[] = req.body.secrets; const [secretIdsUserCanModifyError, secretIdsUserCanModify] = await to(Secret.find({ workspace: workspaceId, environment: environmentName }, { _id: 1 }).then()) @@ -252,8 +260,8 @@ export const batchModifySecrets = async (req: Request, res: Response) => { numberOfSecrets: (secretsModificationsRequested ?? []).length, environment: environmentName, workspaceId, - // #TODO: why does this route have no channel? - // channel: channel ? channel : 'cli' + channel: req.headers?.['user-agent']?.toLowerCase().includes('mozilla') ? 'web' : 'cli', + userAgent: req.headers?.['user-agent'] } }); } @@ -261,8 +269,13 @@ export const batchModifySecrets = async (req: Request, res: Response) => { return res.status(200).send() } -// #TODO: I assume this should be '...Secret'? -export const modifySingleSecrets = async (req: Request, res: Response) => { +/** + * Update a secret within workspace with id [workspaceId] and environment [environment] + * @param req + * @param res + * @returns + */ +export const updateSecret = async (req: Request, res: Response) => { const { workspaceId, environmentName } = req.params const secretModificationsRequested: ModifySecretRequestBody = req.body.secret; @@ -299,8 +312,8 @@ export const modifySingleSecrets = async (req: Request, res: Response) => { numberOfSecrets: 1, environment: environmentName, workspaceId, - // #TODO: why does this route have no channel? - // channel: channel ? channel : 'cli' + channel: req.headers?.['user-agent']?.toLowerCase().includes('mozilla') ? 'web' : 'cli', + userAgent: req.headers?.['user-agent'] } }); } @@ -308,11 +321,18 @@ export const modifySingleSecrets = async (req: Request, res: Response) => { return res.status(200).send(singleModificationUpdate) } -export const fetchAllSecrets = async (req: Request, res: Response) => { +/** + * Return secrets for workspace with id [workspaceId], environment [environment] and user + * with id [req.user._id] + * @param req + * @param res + * @returns + */ +export const getSecrets = async (req: Request, res: Response) => { const { environment } = req.query; const { workspaceId } = req.params; - let userId: string | undefined = undefined // Used for choosing the personal secrets to fetch in + let userId: string | undefined = undefined // used for getting personal secrets for user if (req.user) { userId = req.user._id.toString(); } @@ -321,7 +341,7 @@ export const fetchAllSecrets = async (req: Request, res: Response) => { userId = req.serviceTokenData.user._id } - const [retriveAllSecretsError, allSecrets] = await to(Secret.find( + const [err, secrets] = await to(Secret.find( { workspace: workspaceId, environment, @@ -330,8 +350,8 @@ export const fetchAllSecrets = async (req: Request, res: Response) => { } ).then()) - if (retriveAllSecretsError instanceof ValidationError) { - throw RouteValidationError({ message: "Unable to get secrets, please try again", stack: retriveAllSecretsError.stack }) + if (err) { + throw RouteValidationError({ message: "Failed to get secrets, please try again", stack: err.stack }) } if (postHogClient) { @@ -339,56 +359,40 @@ export const fetchAllSecrets = async (req: Request, res: Response) => { event: 'secrets pulled', distinctId: req.user.email, properties: { - numberOfSecrets: (allSecrets ?? []).length, + numberOfSecrets: (secrets ?? []).length, environment, workspaceId, - // #TODO: why does this route have no channel? - // channel: channel ? channel : 'cli' + channel: req.headers?.['user-agent']?.toLowerCase().includes('mozilla') ? 'web' : 'cli', + userAgent: req.headers?.['user-agent'] } }); } - return res.json(allSecrets) + return res.json(secrets) } -export const fetchSingleSecret = async (req: Request, res: Response) => { - const { secretId } = req.params; - - const [error, singleSecretRetrieved] = await to(Secret.findById(secretId).then()) - - if (error instanceof ValidationError) { - throw RouteValidationError({ message: "Unable to get secret, please try again", stack: error.stack }) - } - - if (singleSecretRetrieved) { - const [membershipValidationError, membership] = await to(validateMembership({ - userId: req.user._id, - workspaceId: singleSecretRetrieved.workspace._id.toString(), - acceptedRoles: [ADMIN, MEMBER] - })) - - if (membershipValidationError || !membership) { - throw UnauthorizedRequestError() - } - - res.json(singleSecretRetrieved) - - if (postHogClient) { - postHogClient.capture({ - event: 'secrets pulled', - distinctId: req.user.email, - properties: { - numberOfSecrets: 1, - // #TODO: how do we get environment and workspace here? Do we need that? When is this route used? - // environment, - // workspaceId, - // #TODO: why does this route have no channel? - // channel: channel ? channel : 'cli' - } - }); - } - - } else { - throw BadRequestError() +/** + * Return secret with id [secretId] + * @param req + * @param res + * @returns + */ +export const getSecret = async (req: Request, res: Response) => { + if (postHogClient) { + postHogClient.capture({ + event: 'secrets pulled', + distinctId: req.user.email, + properties: { + numberOfSecrets: 1, + workspaceId: req._secret.workspace.toString(), + environment: req._secret.environment, + channel: req.headers?.['user-agent']?.toLowerCase().includes('mozilla') ? 'web' : 'cli', + userAgent: req.headers?.['user-agent'] + } + }); } + + return res.status(200).send({ + secret: req._secret + }); } \ No newline at end of file diff --git a/backend/src/controllers/v2/secretsController.ts b/backend/src/controllers/v2/secretsController.ts new file mode 100644 index 000000000..ac8301570 --- /dev/null +++ b/backend/src/controllers/v2/secretsController.ts @@ -0,0 +1,472 @@ +import to from 'await-to-js'; +import { Types } from 'mongoose'; +import { Request, Response } from 'express'; +import { ISecret, Secret } from '../../models'; +import { + SECRET_PERSONAL, + SECRET_SHARED, + ACTION_ADD_SECRETS, + ACTION_READ_SECRETS, + ACTION_UPDATE_SECRETS, + ACTION_DELETE_SECRETS +} from '../../variables'; +import { ValidationError } from '../../utils/errors'; +import { EESecretService, EELogService } from '../../ee/services'; +import { postHogClient } from '../../services'; + +/** + * Create secret(s) for workspace with id [workspaceId] and environment [environment] + * @param req + * @param res + */ +export const createSecrets = async (req: Request, res: Response) => { + const channel = req.headers?.['user-agent']?.toLowerCase().includes('mozilla') ? 'web' : 'cli'; + const { workspaceId, environment } = req.body; + + let toAdd; + if (Array.isArray(req.body.secrets)) { + // case: create multiple secrets + toAdd = req.body.secrets; + } else if (typeof req.body.secrets === 'object') { + // case: create 1 secret + toAdd = [req.body.secrets]; + } + + const newSecrets = await Secret.insertMany( + toAdd.map(({ + type, + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretValueCiphertext, + secretValueIV, + secretValueTag, + }: { + type: string; + secretKeyCiphertext: string; + secretKeyIV: string; + secretKeyTag: string; + secretValueCiphertext: string; + secretValueIV: string; + secretValueTag: string; + }) => ({ + version: 1, + workspace: new Types.ObjectId(workspaceId), + type, + user: type === SECRET_PERSONAL ? req.user : undefined, + environment, + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretValueCiphertext, + secretValueIV, + secretValueTag + })) + ); + + // (EE) add secret versions for new secrets + EESecretService.addSecretVersions({ + secretVersions: newSecrets.map(({ + _id, + version, + workspace, + type, + user, + environment, + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretKeyHash, + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretValueHash + }) => ({ + _id: new Types.ObjectId(), + secret: _id, + version, + workspace, + type, + user, + environment, + isDeleted: false, + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretKeyHash, + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretValueHash + })) + }); + + const addAction = await EELogService.createActionSecret({ + name: ACTION_ADD_SECRETS, + userId: req.user._id.toString(), + workspaceId, + secretIds: newSecrets.map((n) => n._id) + }); + + // (EE) create (audit) log + addAction && await EELogService.createLog({ + userId: req.user._id.toString(), + workspaceId, + actions: [addAction], + channel, + ipAddress: req.ip + }); + + // (EE) take a secret snapshot + await EESecretService.takeSecretSnapshot({ + workspaceId + }); + + if (postHogClient) { + postHogClient.capture({ + event: 'secrets deleted', + distinctId: req.user.email, + properties: { + numberOfSecrets: toAdd.length, + environment, + workspaceId, + channel: req.headers?.['user-agent']?.toLowerCase().includes('mozilla') ? 'web' : 'cli', + userAgent: req.headers?.['user-agent'] + } + }); + } + + return res.status(200).send({ + secrets: newSecrets + }); +} + +/** + * Return secret(s) for workspace with id [workspaceId], environment [environment] and user + * with id [req.user._id] + * @param req + * @param res + * @returns + */ +export const getSecrets = async (req: Request, res: Response) => { + const { workspaceId, environment } = req.query; + + let userId: Types.ObjectId | undefined = undefined // used for getting personal secrets for user + if (req.user) { + userId = req.user._id; + } + + if (req.serviceTokenData) { + userId = req.serviceTokenData.user._id + } + + const [err, secrets] = await to(Secret.find( + { + workspace: workspaceId, + environment, + $or: [ + { user: userId }, + { user: { $exists: false } } + ], + type: { $in: [SECRET_SHARED, SECRET_PERSONAL] } + } + ).then()) + + if (err) throw ValidationError({ message: 'Failed to get secrets', stack: err.stack }); + + const channel = req.headers?.['user-agent']?.toLowerCase().includes('mozilla') ? 'web' : 'cli'; + + const readAction = await EELogService.createActionSecret({ + name: ACTION_READ_SECRETS, + userId: req.user._id.toString(), + workspaceId: workspaceId as string, + secretIds: secrets.map((n: any) => n._id) + }); + + readAction && await EELogService.createLog({ + userId: req.user._id.toString(), + workspaceId: workspaceId as string, + actions: [readAction], + channel, + ipAddress: req.ip + }); + + return res.status(200).send({ + secrets + }); +} + +/** + * Update secret(s) in workspace with id [workspaceId] and environment [environment] + * @param req + * @param res + */ +export const updateSecrets = async (req: Request, res: Response) => { + const channel = req.headers?.['user-agent']?.toLowerCase().includes('mozilla') ? 'web' : 'cli'; + const { workspaceId, environment } = req.body; + + // TODO: move type + interface PatchSecret { + id: string; + secretKeyCiphertext: string; + secretKeyIV: string; + secretKeyTag: string; + secretValueCiphertext: string; + secretValueIV: string; + secretValueTag: string; + secretCommentCiphertext: string; + secretCommentIV: string; + secretCommentTag: string; + } + + const ops = req.body.secrets.map((secret: PatchSecret) => { + const { + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretCommentCiphertext, + secretCommentIV, + secretCommentTag + } = secret; + return ({ + updateOne: { + filter: { _id: new Types.ObjectId(secret.id) }, + update: { + $inc: { + version: 1 + }, + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretValueCiphertext, + secretValueIV, + secretValueTag, + ...(( + secretCommentCiphertext && + secretCommentIV && + secretCommentTag + ) ? { + secretCommentCiphertext, + secretCommentIV, + secretCommentTag + } : {}), + } + } + }); + }); + const b = await Secret.bulkWrite(ops); + + let newSecretsObj: { [key: string]: PatchSecret } = {}; + req.body.secrets.forEach((secret: PatchSecret) => { + newSecretsObj[secret.id] = secret; + }); + + await EESecretService.addSecretVersions({ + secretVersions: req.secrets.map((secret: ISecret) => { + const { + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretCommentCiphertext, + secretCommentIV, + secretCommentTag + } = newSecretsObj[secret._id.toString()] + return ({ + secret: secret._id, + version: secret.version + 1, + workspace: secret.workspace, + type: secret.type, + environment: secret.environment, + isDeleted: false, + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretValueCiphertext, + secretValueIV, + secretValueTag, + ...(( + secretCommentCiphertext && + secretCommentIV && + secretCommentTag + ) ? { + secretCommentCiphertext, + secretCommentIV, + secretCommentTag + } : { + secretCommentCiphertext: '', + secretCommentIV: '', + secretCommentTag: '' + }) + }); + }) + }); + + // group secrets into workspaces so updated secrets can + // be logged and snapshotted separately for each workspace + let workspaceSecretObj: any = {}; + req.secrets.forEach((s: any) => { + if (s.workspace.toString() in workspaceSecretObj) { + workspaceSecretObj[s.workspace.toString()].push(s); + } else { + workspaceSecretObj[s.workspace.toString()] = [s] + } + }); + + Object.keys(workspaceSecretObj).forEach(async (key) => { + const updateAction = await EELogService.createActionSecret({ + name: ACTION_DELETE_SECRETS, + userId: req.user._id.toString(), + workspaceId: key, + secretIds: workspaceSecretObj[key].map((secret: ISecret) => secret._id) + }); + + // (EE) create (audit) log + updateAction && await EELogService.createLog({ + userId: req.user._id.toString(), + workspaceId: key, + actions: [updateAction], + channel, + ipAddress: req.ip + }); + + // (EE) take a secret snapshot + await EESecretService.takeSecretSnapshot({ + workspaceId: key + }) + + if (postHogClient) { + postHogClient.capture({ + event: 'secrets deleted', + distinctId: req.user.email, + properties: { + numberOfSecrets: workspaceSecretObj[key].length, + environment: workspaceSecretObj[key][0].environment, + workspaceId: key, + channel: req.headers?.['user-agent']?.toLowerCase().includes('mozilla') ? 'web' : 'cli', + userAgent: req.headers?.['user-agent'] + } + }); + } + }); + + const updateAction = await EELogService.createActionSecret({ + name: ACTION_UPDATE_SECRETS, + userId: req.user._id.toString(), + workspaceId, + secretIds: req.secrets.map((secret: ISecret) => secret._id) + }); + + // (EE) create (audit) log + updateAction && await EELogService.createLog({ + userId: req.user._id.toString(), + workspaceId, + actions: [updateAction], + channel, + ipAddress: req.ip + }); + + // (EE) take a secret snapshot + await EESecretService.takeSecretSnapshot({ + workspaceId + }); + + if (postHogClient) { + postHogClient.capture({ + event: 'secrets modified', + distinctId: req.user.email, + properties: { + numberOfSecrets: req.secrets.length, + environment, + workspaceId, + channel: req.headers?.['user-agent']?.toLowerCase().includes('mozilla') ? 'web' : 'cli', + userAgent: req.headers?.['user-agent'] + } + }); + } + + return res.status(200).send({ + secrets: await Secret.find({ + _id: { + $in: req.secrets.map((secret: ISecret) => secret._id) + } + }) + }); +} + +/** + * Delete secret(s) in workspace with id [workspaceId] and environment [environment] + * @param req + * @param res + */ +export const deleteSecrets = async (req: Request, res: Response) => { + const channel = req.headers?.['user-agent']?.toLowerCase().includes('mozilla') ? 'web' : 'cli'; + const toDelete = req.secrets.map((s: any) => s._id); + + await Secret.deleteMany({ + _id: { + $in: toDelete + } + }); + + await EESecretService.markDeletedSecretVersions({ + secretIds: toDelete + }); + + // group secrets into workspaces so deleted secrets can + // be logged and snapshotted separately for each workspace + let workspaceSecretObj: any = {}; + req.secrets.forEach((s: any) => { + if (s.workspace.toString() in workspaceSecretObj) { + workspaceSecretObj[s.workspace.toString()].push(s); + } else { + workspaceSecretObj[s.workspace.toString()] = [s] + } + }); + + Object.keys(workspaceSecretObj).forEach(async (key) => { + const deleteAction = await EELogService.createActionSecret({ + name: ACTION_DELETE_SECRETS, + userId: req.user._id.toString(), + workspaceId: key, + secretIds: workspaceSecretObj[key].map((secret: ISecret) => secret._id) + }); + + // (EE) create (audit) log + deleteAction && await EELogService.createLog({ + userId: req.user._id.toString(), + workspaceId: key, + actions: [deleteAction], + channel, + ipAddress: req.ip + }); + + // (EE) take a secret snapshot + await EESecretService.takeSecretSnapshot({ + workspaceId: key + }) + + if (postHogClient) { + postHogClient.capture({ + event: 'secrets deleted', + distinctId: req.user.email, + properties: { + numberOfSecrets: workspaceSecretObj[key].length, + environment: workspaceSecretObj[key][0].environment, + workspaceId: key, + channel: req.headers?.['user-agent']?.toLowerCase().includes('mozilla') ? 'web' : 'cli', + userAgent: req.headers?.['user-agent'] + } + }); + } + }); + + return res.status(400).send({ + secrets: req.secrets + }); +} \ No newline at end of file diff --git a/backend/src/ee/models/secretVersion.ts b/backend/src/ee/models/secretVersion.ts index 0197c3a25..eb5bdd7e6 100644 --- a/backend/src/ee/models/secretVersion.ts +++ b/backend/src/ee/models/secretVersion.ts @@ -86,8 +86,7 @@ const secretVersionSchema = new Schema( required: true }, secretKeyHash: { - type: String, - required: true + type: String }, secretValueCiphertext: { type: String, @@ -102,8 +101,7 @@ const secretVersionSchema = new Schema( required: true }, secretValueHash: { - type: String, - required: true + type: String } }, { diff --git a/backend/src/helpers/secret.ts b/backend/src/helpers/secret.ts index 920e8dc1d..b8f9ffc9b 100644 --- a/backend/src/helpers/secret.ts +++ b/backend/src/helpers/secret.ts @@ -3,6 +3,7 @@ import { Types } from 'mongoose'; import { Secret, ISecret, + Membership } from '../models'; import { EESecretService, @@ -20,6 +21,47 @@ import { ACTION_READ_SECRETS } from '../variables'; +/** + * Validate that user with id [userId] can modify secrets with ids [secretIds] + * @param {Object} obj + * @param {Object} obj.userId - id of user to validate + * @param {Object} obj.secretIds - secret ids + * @returns {Secret[]} secrets + */ +const validateSecrets = async ({ + userId, + secretIds +}: { + userId: string; + secretIds: string[]; +}) =>{ + let secrets; + try { + secrets = await Secret.find({ + _id: { + $in: secretIds + } + }); + + const workspaceIdsSet = new Set((await Membership.find({ + user: userId + }, 'workspace')) + .map((m) => m.workspace.toString())); + + secrets.forEach((secret: ISecret) => { + if (!workspaceIdsSet.has(secret.workspace.toString())) { + throw new Error('Failed to validate secret'); + } + }); + + } catch (err) { + console.error(err); + throw new Error('Failed to validate secrets'); + } + + return secrets; +} + interface V1PushSecret { ciphertextKey: string; ivKey: string; @@ -258,6 +300,7 @@ const v1PushSecrets = async ({ secretValueTag, secretValueHash }) => ({ + _id: new Types.ObjectId(), secret: _id, version, workspace, @@ -280,7 +323,7 @@ const v1PushSecrets = async ({ // (EE) take a secret snapshot await EESecretService.takeSecretSnapshot({ workspaceId - }) + }); } catch (err) { Sentry.setUser(null); Sentry.captureException(err); @@ -527,6 +570,7 @@ const v1PushSecrets = async ({ environment: string; }): Promise => { let secrets: any; // TODO: FIX any + try { // get shared workspace secrets const sharedSecrets = await Secret.find({ @@ -655,6 +699,7 @@ const reformatPullSecrets = ({ secrets }: { secrets: ISecret[] }) => { }; export { + validateSecrets, v1PushSecrets, v2PushSecrets, pullSecrets, diff --git a/backend/src/middleware/index.ts b/backend/src/middleware/index.ts index 119a86abe..6a3537076 100644 --- a/backend/src/middleware/index.ts +++ b/backend/src/middleware/index.ts @@ -8,6 +8,7 @@ import requireIntegrationAuthorizationAuth from './requireIntegrationAuthorizati import requireServiceTokenAuth from './requireServiceTokenAuth'; import requireServiceTokenDataAuth from './requireServiceTokenDataAuth'; import requireSecretAuth from './requireSecretAuth'; +import requireSecretsAuth from './requireSecretsAuth'; import validateRequest from './validateRequest'; export { @@ -21,5 +22,6 @@ export { requireServiceTokenAuth, requireServiceTokenDataAuth, requireSecretAuth, + requireSecretsAuth, validateRequest }; diff --git a/backend/src/middleware/requireSecretAuth.ts b/backend/src/middleware/requireSecretAuth.ts index c6a291200..36e47247e 100644 --- a/backend/src/middleware/requireSecretAuth.ts +++ b/backend/src/middleware/requireSecretAuth.ts @@ -5,6 +5,9 @@ import { validateMembership } from '../helpers/membership'; +// note: used for old /v1/secret and /v2/secret routes. +// newer /v2/secrets routes use [requireSecretsAuth] middleware + /** * Validate if user on request has proper membership to modify secret. * @param {Object} obj @@ -34,7 +37,7 @@ const requireSecretAuth = ({ acceptedRoles }); - req.secret = secret as any; + req._secret = secret; next(); } catch (err) { diff --git a/backend/src/middleware/requireSecretsAuth.ts b/backend/src/middleware/requireSecretsAuth.ts new file mode 100644 index 000000000..c8b89a74c --- /dev/null +++ b/backend/src/middleware/requireSecretsAuth.ts @@ -0,0 +1,49 @@ +import { Request, Response, NextFunction } from 'express'; +import { UnauthorizedRequestError } from '../utils/errors'; +import { Secret, Membership } from '../models'; +import { validateSecrets } from '../helpers/secret'; + +// TODO: make this work for delete route + +const requireSecretsAuth = ({ + acceptedRoles +}: { + acceptedRoles: string[]; +}) => { + return async (req: Request, res: Response, next: NextFunction) => { + let secrets; + try { + if (Array.isArray(req.body.secrets)) { + // case: validate multiple secrets + secrets = await validateSecrets({ + userId: req.user._id.toString(), + secretIds: req.body.secrets.map((s: any) => s.id) + }); + } else if (typeof req.body.secrets === 'object') { // change this to check for object + // case: validate 1 secret + secrets = await validateSecrets({ + userId: req.user._id.toString(), + secretIds: req.body.secrets.id + }); + } else if (Array.isArray(req.body.secretIds)) { + secrets = await validateSecrets({ + userId: req.user._id.toString(), + secretIds: req.body.secretIds + }); + } else if (typeof req.body.secretIds === 'string') { + // case: validate secretIds + secrets = await validateSecrets({ + userId: req.user._id.toString(), + secretIds: [req.body.secretIds] + }); + } + + req.secrets = secrets; + return next(); + } catch (err) { + return next(UnauthorizedRequestError({ message: 'Unable to authenticate secret(s)' })); + } + } +} + +export default requireSecretsAuth; \ No newline at end of file diff --git a/backend/src/models/secret.ts b/backend/src/models/secret.ts index bbaaff8c3..a01b92d80 100644 --- a/backend/src/models/secret.ts +++ b/backend/src/models/secret.ts @@ -69,8 +69,7 @@ const secretSchema = new Schema( required: true }, secretKeyHash: { - type: String, - required: true + type: String }, secretValueCiphertext: { type: String, @@ -85,8 +84,7 @@ const secretSchema = new Schema( required: true }, secretValueHash: { - type: String, - required: true + type: String }, secretCommentCiphertext: { type: String, diff --git a/backend/src/routes/v2/index.ts b/backend/src/routes/v2/index.ts index d0f3833ba..8bea42620 100644 --- a/backend/src/routes/v2/index.ts +++ b/backend/src/routes/v2/index.ts @@ -1,10 +1,12 @@ -import secret from './secret'; +import secret from './secret'; // stop-supporting +import secrets from './secrets'; import workspace from './workspace'; import serviceTokenData from './serviceTokenData'; import apiKeyData from './apiKeyData'; export { secret, + secrets, workspace, serviceTokenData, apiKeyData diff --git a/backend/src/routes/v2/secret.ts b/backend/src/routes/v2/secret.ts index 893af8683..b0c0f8f36 100644 --- a/backend/src/routes/v2/secret.ts +++ b/backend/src/routes/v2/secret.ts @@ -1,18 +1,21 @@ -import express, { Request, Response } from 'express'; -import { requireAuth, requireWorkspaceAuth, validateRequest } from '../../middleware'; +import express from 'express'; +import { + requireAuth, + requireWorkspaceAuth, + requireSecretAuth, + validateRequest +} from '../../middleware'; import { body, param, query } from 'express-validator'; import { ADMIN, MEMBER } from '../../variables'; import { CreateSecretRequestBody, ModifySecretRequestBody } from '../../types/secret'; import { secretController } from '../../controllers/v2'; -import { fetchAllSecrets, fetchSingleSecret } from '../../controllers/v2/secretController'; + +// note to devs: stop supporting const router = express.Router(); -/** - * Create many secrets for a given workspace and environmentName - */ router.post( - '/batch-create/workspace/:workspaceId/environment/:environmentName', + '/batch-create/workspace/:workspaceId/environment/:environment', requireAuth({ acceptedAuthModes: ['jwt'] }), @@ -20,17 +23,15 @@ router.post( acceptedRoles: [ADMIN, MEMBER] }), param('workspaceId').exists().isMongoId().trim(), - param('environmentName').exists().trim(), + param('environment').exists().trim(), body('secrets').exists().isArray().custom((value) => value.every((item: CreateSecretRequestBody) => typeof item === 'object')), + body('channel'), validateRequest, - secretController.batchCreateSecrets + secretController.createSecrets ); -/** - * Create single secret for a given workspace and environmentName - */ router.post( - '/workspace/:workspaceId/environment/:environmentName', + '/workspace/:workspaceId/environment/:environment', requireAuth({ acceptedAuthModes: ['jwt'] }), @@ -38,15 +39,13 @@ router.post( acceptedRoles: [ADMIN, MEMBER] }), param('workspaceId').exists().isMongoId().trim(), - param('environmentName').exists().trim(), + param('environment').exists().trim(), body('secret').exists().isObject(), + body('channel'), validateRequest, - secretController.createSingleSecret + secretController.createSecret ); -/** - * Get all secrets for a given environment and workspace id - */ router.get( '/workspace/:workspaceId', param('workspaceId').exists().trim(), @@ -57,25 +56,23 @@ router.get( requireWorkspaceAuth({ acceptedRoles: [ADMIN, MEMBER] }), + query('channel'), validateRequest, - fetchAllSecrets + secretController.getSecrets ); -/** - * Get single secret by id - */ router.get( '/:secretId', requireAuth({ acceptedAuthModes: ['jwt', 'serviceToken'] }), + requireSecretAuth({ + acceptedRoles: [ADMIN, MEMBER] + }), validateRequest, - fetchSingleSecret + secretController.getSecret ); -/** - * Batch delete secrets in a given workspace and environment name - */ router.delete( '/batch/workspace/:workspaceId/environment/:environmentName', requireAuth({ @@ -88,26 +85,22 @@ router.delete( acceptedRoles: [ADMIN, MEMBER] }), validateRequest, - secretController.batchDeleteSecrets - + secretController.deleteSecrets ); -/** - * delete single secret by id - */ router.delete( '/:secretId', requireAuth({ acceptedAuthModes: ['jwt'] }), + requireSecretAuth({ + acceptedRoles: [ADMIN, MEMBER] + }), param('secretId').isMongoId(), validateRequest, - secretController.deleteSingleSecret + secretController.deleteSecret ); -/** - * Apply modifications to many existing secrets in a given workspace and environment - */ router.patch( '/batch-modify/workspace/:workspaceId/environment/:environmentName', requireAuth({ @@ -120,12 +113,10 @@ router.patch( acceptedRoles: [ADMIN, MEMBER] }), validateRequest, - secretController.batchModifySecrets + secretController.updateSecrets ); -/** - * Apply modifications to single existing secret in a given workspace and environment - */ + router.patch( '/workspace/:workspaceId/environment/:environmentName', requireAuth({ @@ -138,7 +129,7 @@ router.patch( acceptedRoles: [ADMIN, MEMBER] }), validateRequest, - secretController.modifySingleSecrets + secretController.updateSecret ); export default router; diff --git a/backend/src/routes/v2/secrets.ts b/backend/src/routes/v2/secrets.ts new file mode 100644 index 000000000..a4c62bbc1 --- /dev/null +++ b/backend/src/routes/v2/secrets.ts @@ -0,0 +1,177 @@ +import express from 'express'; +const router = express.Router(); +import { + requireAuth, + requireWorkspaceAuth, + requireSecretsAuth, + validateRequest +} from '../../middleware'; +import { query, check, body } from 'express-validator'; +import { secretsController } from '../../controllers/v2'; +import { + ADMIN, + MEMBER, + SECRET_PERSONAL, + SECRET_SHARED +} from '../../variables'; + +router.post( + '/', + body('workspaceId').exists().isString().trim(), + body('environment').exists().isString().trim().isIn(['dev', 'staging', 'prod', 'test']), + body('secrets') + .exists() + .custom((value) => { + if (Array.isArray(value)) { + // case: create multiple secrets + if (value.length === 0) throw new Error('secrets cannot be an empty array') + for (const secret of value) { + if ( + !secret.type || + !(secret.type === SECRET_PERSONAL || secret.type === SECRET_SHARED) || + !secret.secretKeyCiphertext || + !secret.secretKeyIV || + !secret.secretKeyTag || + !secret.secretValueCiphertext || + !secret.secretValueIV || + !secret.secretValueTag + ) { + throw new Error('secrets array must contain objects that conform to the Secret interface'); + } + } + } else if (typeof value === 'object') { + // case: update 1 secret + if ( + !value.type || + !(value.type === SECRET_PERSONAL || value.type === SECRET_SHARED) || + !value.secretKeyCiphertext || + !value.secretKeyIV || + !value.secretKeyTag || + !value.secretValueCiphertext || + !value.secretValueIV || + !value.secretValueTag + ) { + throw new Error('secrets array must contain objects that conform to the Secret interface'); + } + } else { + throw new Error('secrets must be an object or an array of objects') + } + + return true; + }), + validateRequest, + requireAuth({ + acceptedAuthModes: ['jwt'] + }), + requireWorkspaceAuth({ + acceptedRoles: [ADMIN, MEMBER], + location: 'body' + }), + secretsController.createSecrets +); + +router.get( + '/', + query('workspaceId').exists().trim(), + query('environment').exists().trim().isIn(['dev', 'staging', 'prod', 'test']), + validateRequest, + requireAuth({ + acceptedAuthModes: ['jwt', 'serviceToken'] + }), + requireWorkspaceAuth({ + acceptedRoles: [ADMIN, MEMBER], + location: 'query' + }), + secretsController.getSecrets +); + +router.patch( + '/', + body('workspaceId').exists().trim(), + body('environment').exists().trim().isIn(['dev', 'staging', 'prod', 'test']), + body('secrets') + .exists() + .custom((value) => { + if (Array.isArray(value)) { + // case: update multiple secrets + if (value.length === 0) throw new Error('secrets cannot be an empty array') + for (const secret of value) { + if ( + !secret.id || + !secret.secretKeyCiphertext || + !secret.secretKeyIV || + !secret.secretKeyTag || + !secret.secretValueCiphertext || + !secret.secretValueIV || + !secret.secretValueTag + ) { + throw new Error('secrets array must contain objects that conform to the Secret interface'); + } + } + } else if (typeof value === 'object') { + // case: update 1 secret + if ( + !value.id || + !value.secretKeyCiphertext || + !value.secretKeyIV || + !value.secretKeyTag || + !value.secretValueCiphertext || + !value.secretValueIV || + !value.secretValueTag + ) { + throw new Error('secrets array must contain objects that conform to the Secret interface'); + } + } else { + throw new Error('secrets must be an object or an array of objects') + } + + return true; + }), + validateRequest, + requireAuth({ + acceptedAuthModes: ['jwt'] + }), + requireWorkspaceAuth({ + acceptedRoles: [ADMIN, MEMBER], + location: 'body' + }), + requireSecretsAuth({ + acceptedRoles: [ADMIN, MEMBER] + }), + secretsController.updateSecrets +); + +router.delete( + '/', + [ + check('secretIds') + .exists() + .custom((value) => { + // case: delete 1 secret + if (typeof value === 'string') return true; + + if (Array.isArray(value)) { + // case: delete multiple secrets + if (value.length === 0) throw new Error('secrets cannot be an empty array'); + return value.every((id: string) => typeof id === 'string') + } + + throw new Error('secretIds must be a string or an array of strings'); + }) + .not() + .isEmpty() + ], + validateRequest, + requireAuth({ + acceptedAuthModes: ['jwt'] + }), + requireSecretsAuth({ + acceptedRoles: [ADMIN, MEMBER] + }), + secretsController.deleteSecrets +); + +export default router; + + + diff --git a/backend/src/types/express/index.d.ts b/backend/src/types/express/index.d.ts index f43b5fa79..ae9edb4c5 100644 --- a/backend/src/types/express/index.d.ts +++ b/backend/src/types/express/index.d.ts @@ -1,4 +1,5 @@ import * as express from 'express'; +import { ISecret } from '../../models'; // TODO: fix (any) types declare global { @@ -12,7 +13,8 @@ declare global { integration: any; integrationAuth: any; bot: any; - secret: any; + _secret: any; + secrets: any; secretSnapshot: any; serviceToken: any; accessToken: any; From bd5dad71d43d781c13edefa3c3c8b799d6206944 Mon Sep 17 00:00:00 2001 From: Tuan Dang Date: Mon, 9 Jan 2023 10:27:36 +0700 Subject: [PATCH 04/13] Correct logging references --- .../src/controllers/v2/secretsController.ts | 63 +++++++------------ backend/src/ee/models/secretVersion.ts | 9 --- backend/src/routes/v2/secrets.ts | 14 ++--- 3 files changed, 25 insertions(+), 61 deletions(-) diff --git a/backend/src/controllers/v2/secretsController.ts b/backend/src/controllers/v2/secretsController.ts index ac8301570..30748cb3f 100644 --- a/backend/src/controllers/v2/secretsController.ts +++ b/backend/src/controllers/v2/secretsController.ts @@ -13,6 +13,7 @@ import { import { ValidationError } from '../../utils/errors'; import { EESecretService, EELogService } from '../../ee/services'; import { postHogClient } from '../../services'; +import { BadRequestError } from '../../utils/errors'; /** * Create secret(s) for workspace with id [workspaceId] and environment [environment] @@ -124,7 +125,7 @@ export const createSecrets = async (req: Request, res: Response) => { if (postHogClient) { postHogClient.capture({ - event: 'secrets deleted', + event: 'secrets added', distinctId: req.user.email, properties: { numberOfSecrets: toAdd.length, @@ -190,6 +191,20 @@ export const getSecrets = async (req: Request, res: Response) => { channel, ipAddress: req.ip }); + + if (postHogClient) { + postHogClient.capture({ + event: 'secrets deleted', + distinctId: req.user.email, + properties: { + numberOfSecrets: secrets.length, + environment, + workspaceId, + channel, + userAgent: req.headers?.['user-agent'] + } + }); + } return res.status(200).send({ secrets @@ -197,13 +212,12 @@ export const getSecrets = async (req: Request, res: Response) => { } /** - * Update secret(s) in workspace with id [workspaceId] and environment [environment] + * Update secret(s) * @param req * @param res */ export const updateSecrets = async (req: Request, res: Response) => { const channel = req.headers?.['user-agent']?.toLowerCase().includes('mozilla') ? 'web' : 'cli'; - const { workspaceId, environment } = req.body; // TODO: move type interface PatchSecret { @@ -257,7 +271,7 @@ export const updateSecrets = async (req: Request, res: Response) => { } }); }); - const b = await Secret.bulkWrite(ops); + await Secret.bulkWrite(ops); let newSecretsObj: { [key: string]: PatchSecret } = {}; req.body.secrets.forEach((secret: PatchSecret) => { @@ -320,7 +334,7 @@ export const updateSecrets = async (req: Request, res: Response) => { Object.keys(workspaceSecretObj).forEach(async (key) => { const updateAction = await EELogService.createActionSecret({ - name: ACTION_DELETE_SECRETS, + name: ACTION_UPDATE_SECRETS, userId: req.user._id.toString(), workspaceId: key, secretIds: workspaceSecretObj[key].map((secret: ISecret) => secret._id) @@ -342,7 +356,7 @@ export const updateSecrets = async (req: Request, res: Response) => { if (postHogClient) { postHogClient.capture({ - event: 'secrets deleted', + event: 'secrets modified', distinctId: req.user.email, properties: { numberOfSecrets: workspaceSecretObj[key].length, @@ -354,41 +368,6 @@ export const updateSecrets = async (req: Request, res: Response) => { }); } }); - - const updateAction = await EELogService.createActionSecret({ - name: ACTION_UPDATE_SECRETS, - userId: req.user._id.toString(), - workspaceId, - secretIds: req.secrets.map((secret: ISecret) => secret._id) - }); - - // (EE) create (audit) log - updateAction && await EELogService.createLog({ - userId: req.user._id.toString(), - workspaceId, - actions: [updateAction], - channel, - ipAddress: req.ip - }); - - // (EE) take a secret snapshot - await EESecretService.takeSecretSnapshot({ - workspaceId - }); - - if (postHogClient) { - postHogClient.capture({ - event: 'secrets modified', - distinctId: req.user.email, - properties: { - numberOfSecrets: req.secrets.length, - environment, - workspaceId, - channel: req.headers?.['user-agent']?.toLowerCase().includes('mozilla') ? 'web' : 'cli', - userAgent: req.headers?.['user-agent'] - } - }); - } return res.status(200).send({ secrets: await Secret.find({ @@ -400,7 +379,7 @@ export const updateSecrets = async (req: Request, res: Response) => { } /** - * Delete secret(s) in workspace with id [workspaceId] and environment [environment] + * Delete secret(s) with id [workspaceId] and environment [environment] * @param req * @param res */ diff --git a/backend/src/ee/models/secretVersion.ts b/backend/src/ee/models/secretVersion.ts index eb5bdd7e6..87a94e9b5 100644 --- a/backend/src/ee/models/secretVersion.ts +++ b/backend/src/ee/models/secretVersion.ts @@ -8,15 +8,6 @@ import { ENV_PROD } from '../../variables'; -/** - * TODO: - * 1. Modify SecretVersion to also contain XX - * - type - * - user - * - environment - * 2. Modify SecretSnapshot to point to arrays of SecretVersion - */ - export interface ISecretVersion { _id?: Types.ObjectId; secret: Types.ObjectId; diff --git a/backend/src/routes/v2/secrets.ts b/backend/src/routes/v2/secrets.ts index a4c62bbc1..73461d9cd 100644 --- a/backend/src/routes/v2/secrets.ts +++ b/backend/src/routes/v2/secrets.ts @@ -36,7 +36,7 @@ router.post( !secret.secretValueIV || !secret.secretValueTag ) { - throw new Error('secrets array must contain objects that conform to the Secret interface'); + throw new Error('secrets array must contain objects that have required secret properties'); } } } else if (typeof value === 'object') { @@ -51,7 +51,7 @@ router.post( !value.secretValueIV || !value.secretValueTag ) { - throw new Error('secrets array must contain objects that conform to the Secret interface'); + throw new Error('secrets object is missing required secret properties'); } } else { throw new Error('secrets must be an object or an array of objects') @@ -87,8 +87,6 @@ router.get( router.patch( '/', - body('workspaceId').exists().trim(), - body('environment').exists().trim().isIn(['dev', 'staging', 'prod', 'test']), body('secrets') .exists() .custom((value) => { @@ -105,7 +103,7 @@ router.patch( !secret.secretValueIV || !secret.secretValueTag ) { - throw new Error('secrets array must contain objects that conform to the Secret interface'); + throw new Error('secrets array must contain objects that have required secret properties'); } } } else if (typeof value === 'object') { @@ -119,7 +117,7 @@ router.patch( !value.secretValueIV || !value.secretValueTag ) { - throw new Error('secrets array must contain objects that conform to the Secret interface'); + throw new Error('secrets object is missing required secret properties'); } } else { throw new Error('secrets must be an object or an array of objects') @@ -131,10 +129,6 @@ router.patch( requireAuth({ acceptedAuthModes: ['jwt'] }), - requireWorkspaceAuth({ - acceptedRoles: [ADMIN, MEMBER], - location: 'body' - }), requireSecretsAuth({ acceptedRoles: [ADMIN, MEMBER] }), From d2c77d998523a810deca6cedf65d2b09d7c444af Mon Sep 17 00:00:00 2001 From: Tuan Dang Date: Mon, 9 Jan 2023 17:11:33 +0700 Subject: [PATCH 05/13] Patch integrations Secret querying by workspaceId --- backend/src/controllers/v2/apiKeyDataController.ts | 1 - backend/src/helpers/bot.ts | 4 ++-- backend/src/helpers/secret.ts | 1 - backend/src/variables/integration.ts | 6 +++--- frontend/components/integrations/Integration.tsx | 13 +++++++++---- 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/backend/src/controllers/v2/apiKeyDataController.ts b/backend/src/controllers/v2/apiKeyDataController.ts index 3aacde8af..cafbacb5b 100644 --- a/backend/src/controllers/v2/apiKeyDataController.ts +++ b/backend/src/controllers/v2/apiKeyDataController.ts @@ -65,7 +65,6 @@ export const createAPIKeyData = async (req: Request, res: Response) => { apiKey = `ak.${apiKeyData._id.toString()}.${secret}`; } catch (err) { - console.error(err); Sentry.setUser({ email: req.user.email }); Sentry.captureException(err); return res.status(400).send({ diff --git a/backend/src/helpers/bot.ts b/backend/src/helpers/bot.ts index b3f276b53..7519ef18b 100644 --- a/backend/src/helpers/bot.ts +++ b/backend/src/helpers/bot.ts @@ -72,7 +72,7 @@ const getSecretsHelper = async ({ try { const key = await getKey({ workspaceId }); const secrets = await Secret.find({ - workspaceId, + workspace: workspaceId, environment, type: SECRET_SHARED }); @@ -84,7 +84,7 @@ const getSecretsHelper = async ({ tag: secret.secretKeyTag, key }); - + const secretValue = decryptSymmetric({ ciphertext: secret.secretValueCiphertext, iv: secret.secretValueIV, diff --git a/backend/src/helpers/secret.ts b/backend/src/helpers/secret.ts index b8f9ffc9b..d00c62964 100644 --- a/backend/src/helpers/secret.ts +++ b/backend/src/helpers/secret.ts @@ -55,7 +55,6 @@ const validateSecrets = async ({ }); } catch (err) { - console.error(err); throw new Error('Failed to validate secrets'); } diff --git a/backend/src/variables/integration.ts b/backend/src/variables/integration.ts index ed18c5a2a..00e817c57 100644 --- a/backend/src/variables/integration.ts +++ b/backend/src/variables/integration.ts @@ -48,7 +48,7 @@ const INTEGRATION_OPTIONS = [ name: 'Vercel', slug: 'vercel', image: 'Vercel', - isAvailable: false, + isAvailable: true, type: 'vercel', clientId: '', clientSlug: CLIENT_SLUG_VERCEL, @@ -58,7 +58,7 @@ const INTEGRATION_OPTIONS = [ name: 'Netlify', slug: 'netlify', image: 'Netlify', - isAvailable: false, + isAvailable: true, type: 'oauth2', clientId: CLIENT_ID_NETLIFY, docsLink: '' @@ -67,7 +67,7 @@ const INTEGRATION_OPTIONS = [ name: 'GitHub', slug: 'github', image: 'GitHub', - isAvailable: false, + isAvailable: true, type: 'oauth2', clientId: CLIENT_ID_GITHUB, docsLink: '' diff --git a/frontend/components/integrations/Integration.tsx b/frontend/components/integrations/Integration.tsx index ae9a1a613..11edf7cb7 100644 --- a/frontend/components/integrations/Integration.tsx +++ b/frontend/components/integrations/Integration.tsx @@ -23,6 +23,7 @@ import { interface Integration { _id: string; app?: string; + target?: string; environment: string; integration: string; integrationAuth: string; @@ -69,7 +70,11 @@ const Integration = ({ switch (integration.integration) { case "vercel": - setIntegrationTarget("Development"); + setIntegrationTarget( + integration?.target + ? integration.target.charAt(0).toUpperCase() + integration.target.substring(1) + : "Development" + ); break; case "netlify": setIntegrationContext(integration?.context ? contextNetlifyMapping[integration.context] : "Local development"); @@ -93,11 +98,11 @@ const Integration = ({ From 486aa139c2e900027bc00cde81478e454449f9d1 Mon Sep 17 00:00:00 2001 From: Vladyslav Matsiiako Date: Mon, 9 Jan 2023 13:14:07 -0800 Subject: [PATCH 06/13] Changed frontend to use the new secrets routes --- .../src/controllers/v2/secretsController.ts | 8 +- backend/src/routes/v2/secrets.ts | 2 +- backend/src/services/PostHogClient.ts | 8 +- frontend/components/basic/Toggle.tsx | 2 +- .../components/dashboard/CommentField.tsx | 6 +- .../dashboard/DownloadSecretsMenu.tsx | 75 +++++++++ frontend/components/dashboard/SideBar.tsx | 2 +- .../{attemptLogin.js => attemptLogin.ts} | 61 ++++---- .../utilities/secrets/checkOverrides.ts | 33 ++++ .../utilities/secrets/downloadDotEnv.ts | 46 ++++++ .../utilities/secrets/downloadYaml.ts | 52 +++++++ .../utilities/secrets/encryptSecrets.ts | 116 ++++++++++++++ .../utilities/secrets/getSecretsForProject.ts | 113 ++++++++------ .../components/utilities/secrets/pushKeys.ts | 126 --------------- .../utilities/secrets/pushKeysIntegration.ts | 79 ---------- frontend/ee/api/secrets/GetActionData.ts | 1 - .../ee/api/secrets/PerformSecretRollback.ts | 30 ++++ frontend/ee/components/PITRecoverySidebar.tsx | 2 +- frontend/pages/api/files/AddSecrets.ts | 48 ++++++ frontend/pages/api/files/DeleteSecrets.ts | 27 ++++ frontend/pages/api/files/GetSecrets.ts | 10 +- frontend/pages/api/files/UpdateSecrets.ts | 44 ++++++ frontend/pages/dashboard/[id].tsx | 147 ++++++++---------- 23 files changed, 648 insertions(+), 390 deletions(-) create mode 100644 frontend/components/dashboard/DownloadSecretsMenu.tsx rename frontend/components/utilities/{attemptLogin.js => attemptLogin.ts} (68%) create mode 100644 frontend/components/utilities/secrets/checkOverrides.ts create mode 100644 frontend/components/utilities/secrets/downloadDotEnv.ts create mode 100644 frontend/components/utilities/secrets/downloadYaml.ts create mode 100644 frontend/components/utilities/secrets/encryptSecrets.ts delete mode 100644 frontend/components/utilities/secrets/pushKeys.ts delete mode 100644 frontend/components/utilities/secrets/pushKeysIntegration.ts create mode 100644 frontend/ee/api/secrets/PerformSecretRollback.ts create mode 100644 frontend/pages/api/files/AddSecrets.ts create mode 100644 frontend/pages/api/files/DeleteSecrets.ts create mode 100644 frontend/pages/api/files/UpdateSecrets.ts diff --git a/backend/src/controllers/v2/secretsController.ts b/backend/src/controllers/v2/secretsController.ts index 30748cb3f..69cacd75c 100644 --- a/backend/src/controllers/v2/secretsController.ts +++ b/backend/src/controllers/v2/secretsController.ts @@ -273,7 +273,7 @@ export const updateSecrets = async (req: Request, res: Response) => { }); await Secret.bulkWrite(ops); - let newSecretsObj: { [key: string]: PatchSecret } = {}; + const newSecretsObj: { [key: string]: PatchSecret } = {}; req.body.secrets.forEach((secret: PatchSecret) => { newSecretsObj[secret.id] = secret; }); @@ -323,7 +323,7 @@ export const updateSecrets = async (req: Request, res: Response) => { // group secrets into workspaces so updated secrets can // be logged and snapshotted separately for each workspace - let workspaceSecretObj: any = {}; + const workspaceSecretObj: any = {}; req.secrets.forEach((s: any) => { if (s.workspace.toString() in workspaceSecretObj) { workspaceSecretObj[s.workspace.toString()].push(s); @@ -399,7 +399,7 @@ export const deleteSecrets = async (req: Request, res: Response) => { // group secrets into workspaces so deleted secrets can // be logged and snapshotted separately for each workspace - let workspaceSecretObj: any = {}; + const workspaceSecretObj: any = {}; req.secrets.forEach((s: any) => { if (s.workspace.toString() in workspaceSecretObj) { workspaceSecretObj[s.workspace.toString()].push(s); @@ -445,7 +445,7 @@ export const deleteSecrets = async (req: Request, res: Response) => { } }); - return res.status(400).send({ + return res.status(200).send({ secrets: req.secrets }); } \ No newline at end of file diff --git a/backend/src/routes/v2/secrets.ts b/backend/src/routes/v2/secrets.ts index 73461d9cd..1f68d4420 100644 --- a/backend/src/routes/v2/secrets.ts +++ b/backend/src/routes/v2/secrets.ts @@ -138,7 +138,7 @@ router.patch( router.delete( '/', [ - check('secretIds') + body('secretIds') .exists() .custom((value) => { // case: delete 1 secret diff --git a/backend/src/services/PostHogClient.ts b/backend/src/services/PostHogClient.ts index 4ce0117f0..0d91a1c13 100644 --- a/backend/src/services/PostHogClient.ts +++ b/backend/src/services/PostHogClient.ts @@ -7,12 +7,12 @@ import { } from '../config'; import { getLogger } from '../utils/logger'; -if(TELEMETRY_ENABLED){ +if(!TELEMETRY_ENABLED){ getLogger("backend-main").info([ "", - "Infisical collects telemetry data about general usage.", - "The data helps us understand how the product is doing and guide our product development to create the best possible platform; it also helps us demonstrate growth for investors as we support Infisical as open-source software.", - "To opt out of telemetry, you can set `TELEMETRY_ENABLED=false` within the environment variables", + "To improve, Infisical collects telemetry data about general usage.", + "This helps us understand how the product is doing and guide our product development to create the best possible platform; it also helps us demonstrate growth as we support Infisical as open-source software.", + "To opt into telemetry, you can set `TELEMETRY_ENABLED=true` within the environment variables.", ].join('\n')) } diff --git a/frontend/components/basic/Toggle.tsx b/frontend/components/basic/Toggle.tsx index d15aed622..c9957cc3c 100644 --- a/frontend/components/basic/Toggle.tsx +++ b/frontend/components/basic/Toggle.tsx @@ -34,7 +34,7 @@ interface ToggleProps { * @param {string} obj.value - value of a certain secret * @param {number} obj.pos - position of a certain secret #TODO: make the secret id persistent? - * @param {string} obj.id - id of a certain secret + * @param {string} obj.id - id of a certain secret (NOTE: THIS IS THE ID OF THE MAIN SECRET - NOT OF AN OVERRIDE) * @param {function} obj.deleteOverride - a function that deleted an override for a certain secret * @param {string[]} obj.sharedToHide - an array of shared secrets that we want to hide visually because they are overriden. * @param {function} obj.setSharedToHide - a function that updates the array of secrets that we want to hide visually diff --git a/frontend/components/dashboard/CommentField.tsx b/frontend/components/dashboard/CommentField.tsx index ea29aa73c..62ff0c2f1 100644 --- a/frontend/components/dashboard/CommentField.tsx +++ b/frontend/components/dashboard/CommentField.tsx @@ -6,10 +6,10 @@ import { useTranslation } from "next-i18next"; const CommentField = ({ comment, modifyComment, position }: { comment: string; modifyComment: (value: string, posistion: number) => void; position: number;}) => { const { t } = useTranslation(); - return
-

{t("dashboard:sidebar.comments")}

+ return
+

{t("dashboard:sidebar.comments")}