From 47ab0b4a0fbda441e93496df074376c0c1944a8b Mon Sep 17 00:00:00 2001 From: Tuan Dang Date: Sat, 7 Jan 2023 20:12:53 +0700 Subject: [PATCH] Add endpoints for rolling back a workspace to a secret snapshot and rolling back a secret to a version --- .../controllers/v2/apiKeyDataController.ts | 1 - .../src/ee/controllers/v1/secretController.ts | 101 +++++++++++ .../v1/secretSnapshotController.ts | 6 + .../ee/controllers/v1/workspaceController.ts | 165 +++++++++++++++++- backend/src/ee/helpers/action.ts | 75 ++------ backend/src/ee/helpers/secretVersion.ts | 110 ++++++++++++ backend/src/ee/models/secretVersion.ts | 13 +- backend/src/ee/routes/v1/secret.ts | 15 +- backend/src/ee/routes/v1/secretSnapshot.ts | 2 +- backend/src/ee/routes/v1/workspace.ts | 16 +- backend/src/helpers/secret.ts | 2 + 11 files changed, 432 insertions(+), 74 deletions(-) create mode 100644 backend/src/ee/helpers/secretVersion.ts diff --git a/backend/src/controllers/v2/apiKeyDataController.ts b/backend/src/controllers/v2/apiKeyDataController.ts index 3aacde8af..cafbacb5b 100644 --- a/backend/src/controllers/v2/apiKeyDataController.ts +++ b/backend/src/controllers/v2/apiKeyDataController.ts @@ -65,7 +65,6 @@ export const createAPIKeyData = async (req: Request, res: Response) => { apiKey = `ak.${apiKeyData._id.toString()}.${secret}`; } catch (err) { - console.error(err); Sentry.setUser({ email: req.user.email }); Sentry.captureException(err); return res.status(400).send({ diff --git a/backend/src/ee/controllers/v1/secretController.ts b/backend/src/ee/controllers/v1/secretController.ts index a2d68ca96..751f21611 100644 --- a/backend/src/ee/controllers/v1/secretController.ts +++ b/backend/src/ee/controllers/v1/secretController.ts @@ -1,6 +1,8 @@ import { Request, Response } from 'express'; import * as Sentry from '@sentry/node'; +import { Secret } from '../../../models'; import { SecretVersion } from '../../models'; +import { EESecretService } from '../../services'; /** * Return secret versions for secret with id [secretId] @@ -33,4 +35,103 @@ import { SecretVersion } from '../../models'; return res.status(200).send({ secretVersions }); +} + +/** + * Roll back secret with id [secretId] to version [version] + * @param req + * @param res + * @returns + */ +export const rollbackSecretVersion = async (req: Request, res: Response) => { + let secret; + try { + const { secretId } = req.params; + const { version } = req.body; + + // validate secret version + const oldSecretVersion = await SecretVersion.findOne({ + secret: secretId, + version + }); + + if (!oldSecretVersion) throw new Error('Failed to find secret version'); + + const { + workspace, + type, + user, + environment, + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretKeyHash, + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretValueHash + } = oldSecretVersion; + + // update secret + secret = await Secret.findByIdAndUpdate( + secretId, + { + $inc: { + version: 1 + }, + workspace, + type, + user, + environment, + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretKeyHash, + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretValueHash + }, + { + new: true + } + ); + + if (!secret) throw new Error('Failed to find and update secret'); + + // add new secret version + await new SecretVersion({ + secret: secretId, + version: secret.version, + workspace, + type, + user, + environment, + isDeleted: false, + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretKeyHash, + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretValueHash + }).save(); + + // take secret snapshot + await EESecretService.takeSecretSnapshot({ + workspaceId: secret.workspace.toString() + }); + + } catch (err) { + Sentry.setUser({ email: req.user.email }); + Sentry.captureException(err); + return res.status(400).send({ + message: 'Failed to roll back secret version' + }); + } + + return res.status(200).send({ + secret + }); } \ No newline at end of file diff --git a/backend/src/ee/controllers/v1/secretSnapshotController.ts b/backend/src/ee/controllers/v1/secretSnapshotController.ts index 40e1a74a6..6e8605c2f 100644 --- a/backend/src/ee/controllers/v1/secretSnapshotController.ts +++ b/backend/src/ee/controllers/v1/secretSnapshotController.ts @@ -2,6 +2,12 @@ import { Request, Response } from 'express'; import * as Sentry from '@sentry/node'; import { SecretSnapshot } from '../../models'; +/** + * Return secret snapshot with id [secretSnapshotId] + * @param req + * @param res + * @returns + */ export const getSecretSnapshot = async (req: Request, res: Response) => { let secretSnapshot; try { diff --git a/backend/src/ee/controllers/v1/workspaceController.ts b/backend/src/ee/controllers/v1/workspaceController.ts index 88c31b8e1..8fd7c8746 100644 --- a/backend/src/ee/controllers/v1/workspaceController.ts +++ b/backend/src/ee/controllers/v1/workspaceController.ts @@ -1,9 +1,17 @@ -import e, { Request, Response } from 'express'; +import { Request, Response } from 'express'; import * as Sentry from '@sentry/node'; +import { Types } from 'mongoose'; +import { + Secret +} from '../../../models'; import { SecretSnapshot, - Log + Log, + SecretVersion, + ISecretVersion } from '../../models'; +import { EESecretService } from '../../services'; +import { getLatestSecretVersionIds } from '../../helpers/secretVersion'; /** * Return secret snapshots for workspace with id [workspaceId] @@ -63,6 +71,159 @@ export const getWorkspaceSecretSnapshotsCount = async (req: Request, res: Respon }); } +/** + * Rollback secret snapshot with id [secretSnapshotId] to version [version] + * @param req + * @param res + * @returns + */ +export const rollbackWorkspaceSecretSnapshot = async (req: Request, res: Response) => { + let secrets; + try { + const { workspaceId } = req.params; + const { version } = req.body; + + // validate secret snapshot + const secretSnapshot = await SecretSnapshot.findOne({ + workspace: workspaceId, + version + }).populate<{ secretVersions: ISecretVersion[]}>('secretVersions'); + + if (!secretSnapshot) throw new Error('Failed to find secret snapshot'); + + // TODO: fix any + const oldSecretVersionsObj: any = secretSnapshot.secretVersions + .reduce((accumulator, s) => ({ + ...accumulator, + [`${s.secret.toString()}`]: s + }), {}); + + const latestSecretVersionIds = await getLatestSecretVersionIds({ + secretIds: secretSnapshot.secretVersions.map((sv) => sv.secret) + }); + + // TODO: fix any + const latestSecretVersions: any = (await SecretVersion.find({ + _id: { + $in: latestSecretVersionIds.map((s) => s.versionId) + } + }, 'secret version')) + .reduce((accumulator, s) => ({ + ...accumulator, + [`${s.secret.toString()}`]: s + }), {}); + + // delete existing secrets + await Secret.deleteMany({ + workspace: workspaceId + }); + + // add secrets + secrets = await Secret.insertMany( + secretSnapshot.secretVersions.map((sv) => { + const secretId = sv.secret; + const { + workspace, + type, + user, + environment, + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretKeyHash, + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretValueHash, + createdAt + } = oldSecretVersionsObj[secretId.toString()]; + + return ({ + _id: secretId, + version: latestSecretVersions[secretId.toString()].version + 1, + workspace, + type, + user, + environment, + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretKeyHash, + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretValueHash, + secretCommentCiphertext: '', + secretCommentIV: '', + secretCommentTag: '', + createdAt + }); + }) + ); + + // add secret versions + await SecretVersion.insertMany( + secrets.map(({ + _id, + version, + workspace, + type, + user, + environment, + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretKeyHash, + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretValueHash + }) => ({ + _id: new Types.ObjectId(), + secret: _id, + version, + workspace, + type, + user, + environment, + isDeleted: false, + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretKeyHash, + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretValueHash + })) + ); + + // update secret versions of restored secrets as not deleted + await SecretVersion.updateMany({ + secret: { + $in: secretSnapshot.secretVersions.map((sv) => sv.secret) + } + }, { + isDeleted: false + }); + + // take secret snapshot + await EESecretService.takeSecretSnapshot({ + workspaceId + }); + } catch (err) { + Sentry.setUser({ email: req.user.email }); + Sentry.captureException(err); + return res.status(400).send({ + message: 'Failed to roll back secret snapshot' + }); + } + + return res.status(200).send({ + secrets + }); +} + /** * Return (audit) logs for workspace with id [workspaceId] * @param req diff --git a/backend/src/ee/helpers/action.ts b/backend/src/ee/helpers/action.ts index 2971e3f96..389cfedbf 100644 --- a/backend/src/ee/helpers/action.ts +++ b/backend/src/ee/helpers/action.ts @@ -1,7 +1,10 @@ import * as Sentry from '@sentry/node'; import { Types } from 'mongoose'; -import { Secret } from '../../models'; import { SecretVersion, Action } from '../models'; +import { + getLatestSecretVersionIds, + getLatestNSecretSecretVersionIds +} from '../helpers/secretVersion'; import { ACTION_UPDATE_SECRETS } from '../../variables'; /** @@ -30,65 +33,23 @@ const createActionSecretHelper = async ({ if (name === ACTION_UPDATE_SECRETS) { // case: action is updating secrets // -> add old and new secret versions - - // TODO: make query more efficient - latestSecretVersions = (await SecretVersion.aggregate([ - { - $match: { - secret: { - $in: secretIds, - }, - }, - }, - { - $sort: { version: -1 }, - }, - { - $group: { - _id: "$secret", - versions: { $push: "$$ROOT" }, - }, - }, - { - $project: { - _id: 0, - secret: "$_id", - versions: { $slice: ["$versions", 2] }, - }, - } - ])) - .map((s) => ({ - oldSecretVersion: s.versions[0]._id, - newSecretVersion: s.versions[1]._id - })); - - + latestSecretVersions = (await getLatestNSecretSecretVersionIds({ + secretIds, + n: 2 + })) + .map((s) => ({ + oldSecretVersion: s.versions[0]._id, + newSecretVersion: s.versions[1]._id + })); } else { // case: action is adding, deleting, or reading secrets // -> add new secret versions - latestSecretVersions = (await SecretVersion.aggregate([ - { - $match: { - secret: { - $in: secretIds - } - } - }, - { - $group: { - _id: '$secret', - version: { $max: '$version' }, - versionId: { $max: '$_id' } // secret version id - } - }, - { - $sort: { version: -1 } - } - ]) - .exec()) - .map((s) => ({ - newSecretVersion: s.versionId - })); + latestSecretVersions = (await getLatestSecretVersionIds({ + secretIds + })) + .map((s) => ({ + newSecretVersion: s.versionId + })); } action = await new Action({ diff --git a/backend/src/ee/helpers/secretVersion.ts b/backend/src/ee/helpers/secretVersion.ts new file mode 100644 index 000000000..d5859e183 --- /dev/null +++ b/backend/src/ee/helpers/secretVersion.ts @@ -0,0 +1,110 @@ +import * as Sentry from '@sentry/node'; +import { Types } from 'mongoose'; +import { SecretVersion } from '../models'; + +/** + * Return latest secret versions for secrets with ids [secretIds] + * @param {Object} obj + * @param {Object} obj.secretIds = ids of secrets to get latest versions for + * @returns + */ +const getLatestSecretVersionIds = async ({ + secretIds +}: { + secretIds: Types.ObjectId[]; +}) => { + + interface LatestSecretVersionId { + _id: Types.ObjectId; + version: number; + versionId: Types.ObjectId; + } + + let latestSecretVersionIds: LatestSecretVersionId[]; + try { + latestSecretVersionIds = (await SecretVersion.aggregate([ + { + $match: { + secret: { + $in: secretIds + } + } + }, + { + $group: { + _id: '$secret', + version: { $max: '$version' }, + versionId: { $max: '$_id' } // id of latest secret version + } + }, + { + $sort: { version: -1 } + } + ]) + .exec()); + + } catch (err) { + Sentry.setUser(null); + Sentry.captureException(err); + throw new Error('Failed to get latest secret versions'); + } + + return latestSecretVersionIds; +} + +/** + * Return latest [n] secret versions for secrets with ids [secretIds] + * @param {Object} obj + * @param {Object} obj.secretIds = ids of secrets to get latest versions for + * @param {Number} obj.n - number of latest secret versions to return for each secret + * @returns + */ +const getLatestNSecretSecretVersionIds = async ({ + secretIds, + n +}: { + secretIds: Types.ObjectId[]; + n: number; +}) => { + + // TODO: optimize query + let latestNSecretVersions; + try { + latestNSecretVersions = (await SecretVersion.aggregate([ + { + $match: { + secret: { + $in: secretIds, + }, + }, + }, + { + $sort: { version: -1 }, + }, + { + $group: { + _id: "$secret", + versions: { $push: "$$ROOT" }, + }, + }, + { + $project: { + _id: 0, + secret: "$_id", + versions: { $slice: ["$versions", n] }, + }, + } + ])); + } catch (err) { + Sentry.setUser(null); + Sentry.captureException(err); + throw new Error('Failed to get latest n secret versions'); + } + + return latestNSecretVersions; +} + +export { + getLatestSecretVersionIds, + getLatestNSecretSecretVersionIds +} diff --git a/backend/src/ee/models/secretVersion.ts b/backend/src/ee/models/secretVersion.ts index 0197c3a25..71283e93f 100644 --- a/backend/src/ee/models/secretVersion.ts +++ b/backend/src/ee/models/secretVersion.ts @@ -8,17 +8,8 @@ import { ENV_PROD } from '../../variables'; -/** - * TODO: - * 1. Modify SecretVersion to also contain XX - * - type - * - user - * - environment - * 2. Modify SecretSnapshot to point to arrays of SecretVersion - */ - export interface ISecretVersion { - _id?: Types.ObjectId; + _id: Types.ObjectId; secret: Types.ObjectId; version: number; workspace: Types.ObjectId; // new @@ -68,7 +59,7 @@ const secretVersionSchema = new Schema( enum: [ENV_DEV, ENV_TESTING, ENV_STAGING, ENV_PROD], required: true }, - isDeleted: { + isDeleted: { // consider removing field type: Boolean, default: false, required: true diff --git a/backend/src/ee/routes/v1/secret.ts b/backend/src/ee/routes/v1/secret.ts index 43cc8bafc..ac3089e33 100644 --- a/backend/src/ee/routes/v1/secret.ts +++ b/backend/src/ee/routes/v1/secret.ts @@ -5,7 +5,7 @@ import { requireSecretAuth, validateRequest } from '../../../middleware'; -import { query, param } from 'express-validator'; +import { query, param, body } from 'express-validator'; import { secretController } from '../../controllers/v1'; import { ADMIN, MEMBER } from '../../../variables'; @@ -24,4 +24,17 @@ router.get( secretController.getSecretVersions ); +router.post( + '/:secretId/secret-versions/rollback', + requireAuth({ + acceptedAuthModes: ['jwt'] + }), + requireSecretAuth({ + acceptedRoles: [ADMIN, MEMBER] + }), + param('secretId').exists().trim(), + body('version').exists().isInt(), + secretController.rollbackSecretVersion +); + export default router; \ No newline at end of file diff --git a/backend/src/ee/routes/v1/secretSnapshot.ts b/backend/src/ee/routes/v1/secretSnapshot.ts index 80aa7d1ee..d10da4456 100644 --- a/backend/src/ee/routes/v1/secretSnapshot.ts +++ b/backend/src/ee/routes/v1/secretSnapshot.ts @@ -7,7 +7,7 @@ import { requireAuth, validateRequest } from '../../../middleware'; -import { param } from 'express-validator'; +import { param, body } from 'express-validator'; import { ADMIN, MEMBER } from '../../../variables'; import { secretSnapshotController } from '../../controllers/v1'; diff --git a/backend/src/ee/routes/v1/workspace.ts b/backend/src/ee/routes/v1/workspace.ts index 4b2e839eb..c9da58261 100644 --- a/backend/src/ee/routes/v1/workspace.ts +++ b/backend/src/ee/routes/v1/workspace.ts @@ -5,7 +5,7 @@ import { requireWorkspaceAuth, validateRequest } from '../../../middleware'; -import { param, query } from 'express-validator'; +import { param, query, body } from 'express-validator'; import { ADMIN, MEMBER } from '../../../variables'; import { workspaceController } from '../../controllers/v1'; @@ -37,6 +37,20 @@ router.get( workspaceController.getWorkspaceSecretSnapshotsCount ); +router.post( + '/:workspaceId/secret-snapshots/rollback', + requireAuth({ + acceptedAuthModes: ['jwt'] + }), + requireWorkspaceAuth({ + acceptedRoles: [ADMIN, MEMBER] + }), + param('workspaceId').exists().trim(), + body('version').exists().isInt(), + validateRequest, + workspaceController.rollbackWorkspaceSecretSnapshot +); + router.get( '/:workspaceId/logs', requireAuth({ diff --git a/backend/src/helpers/secret.ts b/backend/src/helpers/secret.ts index 920e8dc1d..e4df6d971 100644 --- a/backend/src/helpers/secret.ts +++ b/backend/src/helpers/secret.ts @@ -187,6 +187,7 @@ const v1PushSecrets = async ({ }) => { const newSecret = newSecretsObj[`${type}-${secretKeyHash}`]; return ({ + _id: new Types.ObjectId(), secret: _id, version: version ? version + 1 : 1, workspace: new Types.ObjectId(workspaceId), @@ -258,6 +259,7 @@ const v1PushSecrets = async ({ secretValueTag, secretValueHash }) => ({ + _id: new Types.ObjectId(), secret: _id, version, workspace,