From dca3bd4fbb0fbd089891ce3869838e664611fd92 Mon Sep 17 00:00:00 2001 From: Tuan Dang Date: Fri, 23 Dec 2022 10:06:37 -0500 Subject: [PATCH] Complete v1 secret versioning and project secret snapshots --- backend/src/helpers/secret.ts | 151 ++++++++++++++++++++++++--- backend/src/models/index.ts | 6 ++ backend/src/models/secret.ts | 6 ++ backend/src/models/secretSnapshot.ts | 109 +++++++++++++++++++ backend/src/models/secretVersion.ts | 75 +++++++++++++ 5 files changed, 332 insertions(+), 15 deletions(-) create mode 100644 backend/src/models/secretSnapshot.ts create mode 100644 backend/src/models/secretVersion.ts diff --git a/backend/src/helpers/secret.ts b/backend/src/helpers/secret.ts index 042aba4fa..b82b64bfc 100644 --- a/backend/src/helpers/secret.ts +++ b/backend/src/helpers/secret.ts @@ -1,7 +1,11 @@ import * as Sentry from '@sentry/node'; import { Secret, - ISecret + ISecret, + SecretVersion, + ISecretVersion, + SecretSnapshot, + ISecretSnapshot } from '../models'; import { decryptSymmetric } from '../utils/crypto'; import { SECRET_SHARED, SECRET_PERSONAL } from '../variables'; @@ -19,7 +23,7 @@ interface PushSecret { } interface Update { - [index: string]: string; + [index: string]: any; } type DecryptSecretType = 'text' | 'object' | 'expanded'; @@ -61,17 +65,27 @@ const pushSecrets = async ({ }, {}); // handle deleting secrets - const toDelete = oldSecrets.filter( - (s: ISecret) => !(s.secretKeyHash in newSecretsObj) - ); + const toDelete = oldSecrets + .filter( + (s: ISecret) => !(s.secretKeyHash in newSecretsObj) + ) + .map((s) => s._id); if (toDelete.length > 0) { await Secret.deleteMany({ - _id: { $in: toDelete.map((s) => s._id) } + _id: { $in: toDelete } + }, { + rawResult: true + }); + + await SecretVersion.updateMany({ + secret: { $in: toDelete } + }, { + isDeleted: true }); } // handle modifying secrets where type or value changed - const operations = secrets + const toUpdate = secrets .filter((s) => { if (s.hashKey in oldSecretsObj) { if (s.hashValue !== oldSecretsObj[s.hashKey].secretValueHash) { @@ -86,18 +100,22 @@ const pushSecrets = async ({ } return false; - }) + }); + + const operations = toUpdate .map((s) => { const update: Update = { - type: s.type, secretValueCiphertext: s.ciphertextValue, secretValueIV: s.ivValue, secretValueTag: s.tagValue, - secretValueHash: s.hashValue + secretValueHash: s.hashValue, + $inc: { + version: 1 + } }; if (s.type === SECRET_PERSONAL) { - // attach user assocaited with the personal secret + // attach user associated with the personal secret update['user'] = userId; } @@ -111,16 +129,40 @@ const pushSecrets = async ({ } }; }); - const a = await Secret.bulkWrite(operations as any); + await Secret.bulkWrite(operations as any); + await SecretVersion.insertMany( + toUpdate.map(({ + ciphertextKey, + ivKey, + tagKey, + hashKey, + ciphertextValue, + ivValue, + tagValue, + hashValue + }) => ({ + secret: oldSecretsObj[hashKey]._id, + version: oldSecretsObj[hashKey].version + 1, + isDeleted: false, + secretKeyCiphertext: ciphertextKey, + secretKeyIV: ivKey, + secretKeyTag: tagKey, + secretKeyHash: hashKey, + secretValueCiphertext: ciphertextValue, + secretValueIV: ivValue, + secretValueTag: tagValue, + secretValueHash: hashValue + })) + ); // handle adding new secrets const toAdd = secrets.filter((s) => !(s.hashKey in oldSecretsObj)); if (toAdd.length > 0) { // add secrets - await Secret.insertMany( + const newSecrets = await Secret.insertMany( toAdd.map((s, idx) => { - let obj: any = { + const obj: any = { workspace: workspaceId, type: toAdd[idx].type, environment, @@ -141,7 +183,39 @@ const pushSecrets = async ({ return obj; }) ); + + await SecretVersion.insertMany( + newSecrets.map(({ + _id, + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretKeyHash, + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretValueHash + }) => ({ + secret: _id, + version: 1, + isDeleted: false, + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretKeyHash, + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretValueHash + })) + ); } + + await takeSecretSnapshotHelper({ + workspaceId + }); + // TODO: in the future add secret snapshot to capture entire + // state of project at this point in time } catch (err) { Sentry.setUser(null); Sentry.captureException(err); @@ -295,9 +369,56 @@ const decryptSecrets = ({ return content; }; +/** + * Saves a copy of the current state of secrets in workspace with id + * [workspaceId] under a new snapshot with incremented version under the + * secretsnapshots collection. + * @param {Object} obj + * @param {String} obj.workspaceId + */ +const takeSecretSnapshotHelper = async ({ + workspaceId +}: { + workspaceId: string; +}) => { + try { + const secrets = await Secret.find({ + workspace: workspaceId + }); + + const latestSecretSnapshot = await SecretSnapshot.findOne({ + workspace: workspaceId + }).sort({ version: -1 }); + + if (!latestSecretSnapshot) { + // case: no snapshots exist for workspace -> create first snapshot + await new SecretSnapshot({ + workspace: workspaceId, + version: 1, + secrets + }).save(); + + return; + } + + // case: snapshots exist for workspace + await new SecretSnapshot({ + workspace: workspaceId, + version: latestSecretSnapshot.version + 1, + secrets + }).save(); + + } catch (err) { + Sentry.setUser(null); + Sentry.captureException(err); + throw new Error('Failed to take a secret snapshot'); + } +} + export { pushSecrets, pullSecrets, reformatPullSecrets, - decryptSecrets + decryptSecrets, + takeSecretSnapshotHelper }; diff --git a/backend/src/models/index.ts b/backend/src/models/index.ts index 9b07f6766..f43e4309f 100644 --- a/backend/src/models/index.ts +++ b/backend/src/models/index.ts @@ -7,6 +7,8 @@ import Membership, { IMembership } from './membership'; import MembershipOrg, { IMembershipOrg } from './membershipOrg'; import Organization, { IOrganization } from './organization'; import Secret, { ISecret } from './secret'; +import SecretVersion, { ISecretVersion } from './secretVersion'; +import SecretSnapshot, { ISecretSnapshot } from './secretSnapshot'; import ServiceToken, { IServiceToken } from './serviceToken'; import Token, { IToken } from './token'; import User, { IUser } from './user'; @@ -32,6 +34,10 @@ export { IOrganization, Secret, ISecret, + SecretVersion, + ISecretVersion, + SecretSnapshot, + ISecretSnapshot, ServiceToken, IServiceToken, Token, diff --git a/backend/src/models/secret.ts b/backend/src/models/secret.ts index b83ef728d..ee879de30 100644 --- a/backend/src/models/secret.ts +++ b/backend/src/models/secret.ts @@ -10,6 +10,7 @@ import { export interface ISecret { _id: Types.ObjectId; + version: number; workspace: Types.ObjectId; type: string; user: Types.ObjectId; @@ -26,6 +27,11 @@ export interface ISecret { const secretSchema = new Schema( { + version: { + type: Number, + default: 1, + required: true + }, workspace: { type: Schema.Types.ObjectId, ref: 'Workspace', diff --git a/backend/src/models/secretSnapshot.ts b/backend/src/models/secretSnapshot.ts new file mode 100644 index 000000000..376115308 --- /dev/null +++ b/backend/src/models/secretSnapshot.ts @@ -0,0 +1,109 @@ +import { Schema, model, Types } from 'mongoose'; +import { + SECRET_SHARED, + SECRET_PERSONAL, + ENV_DEV, + ENV_TESTING, + ENV_STAGING, + ENV_PROD +} from '../variables'; + +export interface ISecretSnapshot { + workspace: Types.ObjectId; + version: number; + secrets: { + version: number; + workspace: Types.ObjectId; + type: string; + user: Types.ObjectId; + environment: string; + secretKeyCiphertext: string; + secretKeyIV: string; + secretKeyTag: string; + secretKeyHash: string; + secretValueCiphertext: string; + secretValueIV: string; + secretValueTag: string; + secretValueHash: string; + }[] +} + +const secretSnapshotSchema = new Schema( + { + workspace: { + type: Schema.Types.ObjectId, + ref: 'Workspace', + required: true + }, + version: { + type: Number, + required: true + }, + secrets: [{ + version: { + type: Number, + default: 1, + required: true + }, + workspace: { + type: Schema.Types.ObjectId, + ref: 'Workspace', + required: true + }, + type: { + type: String, + enum: [SECRET_SHARED, SECRET_PERSONAL], + required: true + }, + user: { + // user associated with the personal secret + type: Schema.Types.ObjectId, + ref: 'User' + }, + environment: { + type: String, + enum: [ENV_DEV, ENV_TESTING, ENV_STAGING, ENV_PROD], + required: true + }, + secretKeyCiphertext: { + type: String, + required: true + }, + secretKeyIV: { + type: String, // symmetric + required: true + }, + secretKeyTag: { + type: String, // symmetric + required: true + }, + secretKeyHash: { + type: String, + required: true + }, + secretValueCiphertext: { + type: String, + required: true + }, + secretValueIV: { + type: String, // symmetric + required: true + }, + secretValueTag: { + type: String, // symmetric + required: true + }, + secretValueHash: { + type: String, + required: true + } + }] + }, + { + timestamps: true + } +); + +const SecretSnapshot = model('SecretSnapshot', secretSnapshotSchema); + +export default SecretSnapshot; \ No newline at end of file diff --git a/backend/src/models/secretVersion.ts b/backend/src/models/secretVersion.ts new file mode 100644 index 000000000..97c8ba585 --- /dev/null +++ b/backend/src/models/secretVersion.ts @@ -0,0 +1,75 @@ +import { Schema, model, Types } from 'mongoose'; + +export interface ISecretVersion { + _id: Types.ObjectId; + secret: Types.ObjectId; + version: number; + isDeleted: boolean; + secretKeyCiphertext: string; + secretKeyIV: string; + secretKeyTag: string; + secretKeyHash: string; + secretValueCiphertext: string; + secretValueIV: string; + secretValueTag: string; + secretValueHash: string; +} + +const secretVersionSchema = new Schema( + { + secret: { // could be deleted + type: Schema.Types.ObjectId, + ref: 'Secret', + required: true + }, + version: { + type: Number, + default: 1, + required: true + }, + isDeleted: { + type: Boolean, + default: false, + required: true + }, + secretKeyCiphertext: { + type: String, + required: true + }, + secretKeyIV: { + type: String, // symmetric + required: true + }, + secretKeyTag: { + type: String, // symmetric + required: true + }, + secretKeyHash: { + type: String, + required: true + }, + secretValueCiphertext: { + type: String, + required: true + }, + secretValueIV: { + type: String, // symmetric + required: true + }, + secretValueTag: { + type: String, // symmetric + required: true + }, + secretValueHash: { + type: String, + required: true + } + }, + { + timestamps: true + } +); + +const SecretVersion = model('SecretVersion', secretVersionSchema); + +export default SecretVersion; \ No newline at end of file