From e4e0370dad184a0c92414078b9b11be141cd8169 Mon Sep 17 00:00:00 2001 From: Tuan Dang Date: Fri, 23 Dec 2022 10:06:37 -0500 Subject: [PATCH 1/8] Complete v1 secret versioning and project secret snapshots --- backend/src/helpers/secret.ts | 151 ++++++++++++++++++++++++--- backend/src/models/index.ts | 6 ++ backend/src/models/secret.ts | 6 ++ backend/src/models/secretSnapshot.ts | 109 +++++++++++++++++++ backend/src/models/secretVersion.ts | 75 +++++++++++++ 5 files changed, 332 insertions(+), 15 deletions(-) create mode 100644 backend/src/models/secretSnapshot.ts create mode 100644 backend/src/models/secretVersion.ts diff --git a/backend/src/helpers/secret.ts b/backend/src/helpers/secret.ts index 042aba4fa..b82b64bfc 100644 --- a/backend/src/helpers/secret.ts +++ b/backend/src/helpers/secret.ts @@ -1,7 +1,11 @@ import * as Sentry from '@sentry/node'; import { Secret, - ISecret + ISecret, + SecretVersion, + ISecretVersion, + SecretSnapshot, + ISecretSnapshot } from '../models'; import { decryptSymmetric } from '../utils/crypto'; import { SECRET_SHARED, SECRET_PERSONAL } from '../variables'; @@ -19,7 +23,7 @@ interface PushSecret { } interface Update { - [index: string]: string; + [index: string]: any; } type DecryptSecretType = 'text' | 'object' | 'expanded'; @@ -61,17 +65,27 @@ const pushSecrets = async ({ }, {}); // handle deleting secrets - const toDelete = oldSecrets.filter( - (s: ISecret) => !(s.secretKeyHash in newSecretsObj) - ); + const toDelete = oldSecrets + .filter( + (s: ISecret) => !(s.secretKeyHash in newSecretsObj) + ) + .map((s) => s._id); if (toDelete.length > 0) { await Secret.deleteMany({ - _id: { $in: toDelete.map((s) => s._id) } + _id: { $in: toDelete } + }, { + rawResult: true + }); + + await SecretVersion.updateMany({ + secret: { $in: toDelete } + }, { + isDeleted: true }); } // handle modifying secrets where type or value changed - const operations = secrets + const toUpdate = secrets .filter((s) => { if (s.hashKey in oldSecretsObj) { if (s.hashValue !== oldSecretsObj[s.hashKey].secretValueHash) { @@ -86,18 +100,22 @@ const pushSecrets = async ({ } return false; - }) + }); + + const operations = toUpdate .map((s) => { const update: Update = { - type: s.type, secretValueCiphertext: s.ciphertextValue, secretValueIV: s.ivValue, secretValueTag: s.tagValue, - secretValueHash: s.hashValue + secretValueHash: s.hashValue, + $inc: { + version: 1 + } }; if (s.type === SECRET_PERSONAL) { - // attach user assocaited with the personal secret + // attach user associated with the personal secret update['user'] = userId; } @@ -111,16 +129,40 @@ const pushSecrets = async ({ } }; }); - const a = await Secret.bulkWrite(operations as any); + await Secret.bulkWrite(operations as any); + await SecretVersion.insertMany( + toUpdate.map(({ + ciphertextKey, + ivKey, + tagKey, + hashKey, + ciphertextValue, + ivValue, + tagValue, + hashValue + }) => ({ + secret: oldSecretsObj[hashKey]._id, + version: oldSecretsObj[hashKey].version + 1, + isDeleted: false, + secretKeyCiphertext: ciphertextKey, + secretKeyIV: ivKey, + secretKeyTag: tagKey, + secretKeyHash: hashKey, + secretValueCiphertext: ciphertextValue, + secretValueIV: ivValue, + secretValueTag: tagValue, + secretValueHash: hashValue + })) + ); // handle adding new secrets const toAdd = secrets.filter((s) => !(s.hashKey in oldSecretsObj)); if (toAdd.length > 0) { // add secrets - await Secret.insertMany( + const newSecrets = await Secret.insertMany( toAdd.map((s, idx) => { - let obj: any = { + const obj: any = { workspace: workspaceId, type: toAdd[idx].type, environment, @@ -141,7 +183,39 @@ const pushSecrets = async ({ return obj; }) ); + + await SecretVersion.insertMany( + newSecrets.map(({ + _id, + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretKeyHash, + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretValueHash + }) => ({ + secret: _id, + version: 1, + isDeleted: false, + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretKeyHash, + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretValueHash + })) + ); } + + await takeSecretSnapshotHelper({ + workspaceId + }); + // TODO: in the future add secret snapshot to capture entire + // state of project at this point in time } catch (err) { Sentry.setUser(null); Sentry.captureException(err); @@ -295,9 +369,56 @@ const decryptSecrets = ({ return content; }; +/** + * Saves a copy of the current state of secrets in workspace with id + * [workspaceId] under a new snapshot with incremented version under the + * secretsnapshots collection. + * @param {Object} obj + * @param {String} obj.workspaceId + */ +const takeSecretSnapshotHelper = async ({ + workspaceId +}: { + workspaceId: string; +}) => { + try { + const secrets = await Secret.find({ + workspace: workspaceId + }); + + const latestSecretSnapshot = await SecretSnapshot.findOne({ + workspace: workspaceId + }).sort({ version: -1 }); + + if (!latestSecretSnapshot) { + // case: no snapshots exist for workspace -> create first snapshot + await new SecretSnapshot({ + workspace: workspaceId, + version: 1, + secrets + }).save(); + + return; + } + + // case: snapshots exist for workspace + await new SecretSnapshot({ + workspace: workspaceId, + version: latestSecretSnapshot.version + 1, + secrets + }).save(); + + } catch (err) { + Sentry.setUser(null); + Sentry.captureException(err); + throw new Error('Failed to take a secret snapshot'); + } +} + export { pushSecrets, pullSecrets, reformatPullSecrets, - decryptSecrets + decryptSecrets, + takeSecretSnapshotHelper }; diff --git a/backend/src/models/index.ts b/backend/src/models/index.ts index 78c38060b..daab77b2a 100644 --- a/backend/src/models/index.ts +++ b/backend/src/models/index.ts @@ -9,6 +9,8 @@ import Membership, { IMembership } from './membership'; import MembershipOrg, { IMembershipOrg } from './membershipOrg'; import Organization, { IOrganization } from './organization'; import Secret, { ISecret } from './secret'; +import SecretVersion, { ISecretVersion } from './secretVersion'; +import SecretSnapshot, { ISecretSnapshot } from './secretSnapshot'; import ServiceToken, { IServiceToken } from './serviceToken'; import Token, { IToken } from './token'; import User, { IUser } from './user'; @@ -38,6 +40,10 @@ export { IOrganization, Secret, ISecret, + SecretVersion, + ISecretVersion, + SecretSnapshot, + ISecretSnapshot, ServiceToken, IServiceToken, Token, diff --git a/backend/src/models/secret.ts b/backend/src/models/secret.ts index b83ef728d..ee879de30 100644 --- a/backend/src/models/secret.ts +++ b/backend/src/models/secret.ts @@ -10,6 +10,7 @@ import { export interface ISecret { _id: Types.ObjectId; + version: number; workspace: Types.ObjectId; type: string; user: Types.ObjectId; @@ -26,6 +27,11 @@ export interface ISecret { const secretSchema = new Schema( { + version: { + type: Number, + default: 1, + required: true + }, workspace: { type: Schema.Types.ObjectId, ref: 'Workspace', diff --git a/backend/src/models/secretSnapshot.ts b/backend/src/models/secretSnapshot.ts new file mode 100644 index 000000000..376115308 --- /dev/null +++ b/backend/src/models/secretSnapshot.ts @@ -0,0 +1,109 @@ +import { Schema, model, Types } from 'mongoose'; +import { + SECRET_SHARED, + SECRET_PERSONAL, + ENV_DEV, + ENV_TESTING, + ENV_STAGING, + ENV_PROD +} from '../variables'; + +export interface ISecretSnapshot { + workspace: Types.ObjectId; + version: number; + secrets: { + version: number; + workspace: Types.ObjectId; + type: string; + user: Types.ObjectId; + environment: string; + secretKeyCiphertext: string; + secretKeyIV: string; + secretKeyTag: string; + secretKeyHash: string; + secretValueCiphertext: string; + secretValueIV: string; + secretValueTag: string; + secretValueHash: string; + }[] +} + +const secretSnapshotSchema = new Schema( + { + workspace: { + type: Schema.Types.ObjectId, + ref: 'Workspace', + required: true + }, + version: { + type: Number, + required: true + }, + secrets: [{ + version: { + type: Number, + default: 1, + required: true + }, + workspace: { + type: Schema.Types.ObjectId, + ref: 'Workspace', + required: true + }, + type: { + type: String, + enum: [SECRET_SHARED, SECRET_PERSONAL], + required: true + }, + user: { + // user associated with the personal secret + type: Schema.Types.ObjectId, + ref: 'User' + }, + environment: { + type: String, + enum: [ENV_DEV, ENV_TESTING, ENV_STAGING, ENV_PROD], + required: true + }, + secretKeyCiphertext: { + type: String, + required: true + }, + secretKeyIV: { + type: String, // symmetric + required: true + }, + secretKeyTag: { + type: String, // symmetric + required: true + }, + secretKeyHash: { + type: String, + required: true + }, + secretValueCiphertext: { + type: String, + required: true + }, + secretValueIV: { + type: String, // symmetric + required: true + }, + secretValueTag: { + type: String, // symmetric + required: true + }, + secretValueHash: { + type: String, + required: true + } + }] + }, + { + timestamps: true + } +); + +const SecretSnapshot = model('SecretSnapshot', secretSnapshotSchema); + +export default SecretSnapshot; \ No newline at end of file diff --git a/backend/src/models/secretVersion.ts b/backend/src/models/secretVersion.ts new file mode 100644 index 000000000..97c8ba585 --- /dev/null +++ b/backend/src/models/secretVersion.ts @@ -0,0 +1,75 @@ +import { Schema, model, Types } from 'mongoose'; + +export interface ISecretVersion { + _id: Types.ObjectId; + secret: Types.ObjectId; + version: number; + isDeleted: boolean; + secretKeyCiphertext: string; + secretKeyIV: string; + secretKeyTag: string; + secretKeyHash: string; + secretValueCiphertext: string; + secretValueIV: string; + secretValueTag: string; + secretValueHash: string; +} + +const secretVersionSchema = new Schema( + { + secret: { // could be deleted + type: Schema.Types.ObjectId, + ref: 'Secret', + required: true + }, + version: { + type: Number, + default: 1, + required: true + }, + isDeleted: { + type: Boolean, + default: false, + required: true + }, + secretKeyCiphertext: { + type: String, + required: true + }, + secretKeyIV: { + type: String, // symmetric + required: true + }, + secretKeyTag: { + type: String, // symmetric + required: true + }, + secretKeyHash: { + type: String, + required: true + }, + secretValueCiphertext: { + type: String, + required: true + }, + secretValueIV: { + type: String, // symmetric + required: true + }, + secretValueTag: { + type: String, // symmetric + required: true + }, + secretValueHash: { + type: String, + required: true + } + }, + { + timestamps: true + } +); + +const SecretVersion = model('SecretVersion', secretVersionSchema); + +export default SecretVersion; \ No newline at end of file From dca3bd4fbb0fbd089891ce3869838e664611fd92 Mon Sep 17 00:00:00 2001 From: Tuan Dang Date: Fri, 23 Dec 2022 10:06:37 -0500 Subject: [PATCH 2/8] Complete v1 secret versioning and project secret snapshots --- backend/src/helpers/secret.ts | 151 ++++++++++++++++++++++++--- backend/src/models/index.ts | 6 ++ backend/src/models/secret.ts | 6 ++ backend/src/models/secretSnapshot.ts | 109 +++++++++++++++++++ backend/src/models/secretVersion.ts | 75 +++++++++++++ 5 files changed, 332 insertions(+), 15 deletions(-) create mode 100644 backend/src/models/secretSnapshot.ts create mode 100644 backend/src/models/secretVersion.ts diff --git a/backend/src/helpers/secret.ts b/backend/src/helpers/secret.ts index 042aba4fa..b82b64bfc 100644 --- a/backend/src/helpers/secret.ts +++ b/backend/src/helpers/secret.ts @@ -1,7 +1,11 @@ import * as Sentry from '@sentry/node'; import { Secret, - ISecret + ISecret, + SecretVersion, + ISecretVersion, + SecretSnapshot, + ISecretSnapshot } from '../models'; import { decryptSymmetric } from '../utils/crypto'; import { SECRET_SHARED, SECRET_PERSONAL } from '../variables'; @@ -19,7 +23,7 @@ interface PushSecret { } interface Update { - [index: string]: string; + [index: string]: any; } type DecryptSecretType = 'text' | 'object' | 'expanded'; @@ -61,17 +65,27 @@ const pushSecrets = async ({ }, {}); // handle deleting secrets - const toDelete = oldSecrets.filter( - (s: ISecret) => !(s.secretKeyHash in newSecretsObj) - ); + const toDelete = oldSecrets + .filter( + (s: ISecret) => !(s.secretKeyHash in newSecretsObj) + ) + .map((s) => s._id); if (toDelete.length > 0) { await Secret.deleteMany({ - _id: { $in: toDelete.map((s) => s._id) } + _id: { $in: toDelete } + }, { + rawResult: true + }); + + await SecretVersion.updateMany({ + secret: { $in: toDelete } + }, { + isDeleted: true }); } // handle modifying secrets where type or value changed - const operations = secrets + const toUpdate = secrets .filter((s) => { if (s.hashKey in oldSecretsObj) { if (s.hashValue !== oldSecretsObj[s.hashKey].secretValueHash) { @@ -86,18 +100,22 @@ const pushSecrets = async ({ } return false; - }) + }); + + const operations = toUpdate .map((s) => { const update: Update = { - type: s.type, secretValueCiphertext: s.ciphertextValue, secretValueIV: s.ivValue, secretValueTag: s.tagValue, - secretValueHash: s.hashValue + secretValueHash: s.hashValue, + $inc: { + version: 1 + } }; if (s.type === SECRET_PERSONAL) { - // attach user assocaited with the personal secret + // attach user associated with the personal secret update['user'] = userId; } @@ -111,16 +129,40 @@ const pushSecrets = async ({ } }; }); - const a = await Secret.bulkWrite(operations as any); + await Secret.bulkWrite(operations as any); + await SecretVersion.insertMany( + toUpdate.map(({ + ciphertextKey, + ivKey, + tagKey, + hashKey, + ciphertextValue, + ivValue, + tagValue, + hashValue + }) => ({ + secret: oldSecretsObj[hashKey]._id, + version: oldSecretsObj[hashKey].version + 1, + isDeleted: false, + secretKeyCiphertext: ciphertextKey, + secretKeyIV: ivKey, + secretKeyTag: tagKey, + secretKeyHash: hashKey, + secretValueCiphertext: ciphertextValue, + secretValueIV: ivValue, + secretValueTag: tagValue, + secretValueHash: hashValue + })) + ); // handle adding new secrets const toAdd = secrets.filter((s) => !(s.hashKey in oldSecretsObj)); if (toAdd.length > 0) { // add secrets - await Secret.insertMany( + const newSecrets = await Secret.insertMany( toAdd.map((s, idx) => { - let obj: any = { + const obj: any = { workspace: workspaceId, type: toAdd[idx].type, environment, @@ -141,7 +183,39 @@ const pushSecrets = async ({ return obj; }) ); + + await SecretVersion.insertMany( + newSecrets.map(({ + _id, + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretKeyHash, + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretValueHash + }) => ({ + secret: _id, + version: 1, + isDeleted: false, + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretKeyHash, + secretValueCiphertext, + secretValueIV, + secretValueTag, + secretValueHash + })) + ); } + + await takeSecretSnapshotHelper({ + workspaceId + }); + // TODO: in the future add secret snapshot to capture entire + // state of project at this point in time } catch (err) { Sentry.setUser(null); Sentry.captureException(err); @@ -295,9 +369,56 @@ const decryptSecrets = ({ return content; }; +/** + * Saves a copy of the current state of secrets in workspace with id + * [workspaceId] under a new snapshot with incremented version under the + * secretsnapshots collection. + * @param {Object} obj + * @param {String} obj.workspaceId + */ +const takeSecretSnapshotHelper = async ({ + workspaceId +}: { + workspaceId: string; +}) => { + try { + const secrets = await Secret.find({ + workspace: workspaceId + }); + + const latestSecretSnapshot = await SecretSnapshot.findOne({ + workspace: workspaceId + }).sort({ version: -1 }); + + if (!latestSecretSnapshot) { + // case: no snapshots exist for workspace -> create first snapshot + await new SecretSnapshot({ + workspace: workspaceId, + version: 1, + secrets + }).save(); + + return; + } + + // case: snapshots exist for workspace + await new SecretSnapshot({ + workspace: workspaceId, + version: latestSecretSnapshot.version + 1, + secrets + }).save(); + + } catch (err) { + Sentry.setUser(null); + Sentry.captureException(err); + throw new Error('Failed to take a secret snapshot'); + } +} + export { pushSecrets, pullSecrets, reformatPullSecrets, - decryptSecrets + decryptSecrets, + takeSecretSnapshotHelper }; diff --git a/backend/src/models/index.ts b/backend/src/models/index.ts index 9b07f6766..f43e4309f 100644 --- a/backend/src/models/index.ts +++ b/backend/src/models/index.ts @@ -7,6 +7,8 @@ import Membership, { IMembership } from './membership'; import MembershipOrg, { IMembershipOrg } from './membershipOrg'; import Organization, { IOrganization } from './organization'; import Secret, { ISecret } from './secret'; +import SecretVersion, { ISecretVersion } from './secretVersion'; +import SecretSnapshot, { ISecretSnapshot } from './secretSnapshot'; import ServiceToken, { IServiceToken } from './serviceToken'; import Token, { IToken } from './token'; import User, { IUser } from './user'; @@ -32,6 +34,10 @@ export { IOrganization, Secret, ISecret, + SecretVersion, + ISecretVersion, + SecretSnapshot, + ISecretSnapshot, ServiceToken, IServiceToken, Token, diff --git a/backend/src/models/secret.ts b/backend/src/models/secret.ts index b83ef728d..ee879de30 100644 --- a/backend/src/models/secret.ts +++ b/backend/src/models/secret.ts @@ -10,6 +10,7 @@ import { export interface ISecret { _id: Types.ObjectId; + version: number; workspace: Types.ObjectId; type: string; user: Types.ObjectId; @@ -26,6 +27,11 @@ export interface ISecret { const secretSchema = new Schema( { + version: { + type: Number, + default: 1, + required: true + }, workspace: { type: Schema.Types.ObjectId, ref: 'Workspace', diff --git a/backend/src/models/secretSnapshot.ts b/backend/src/models/secretSnapshot.ts new file mode 100644 index 000000000..376115308 --- /dev/null +++ b/backend/src/models/secretSnapshot.ts @@ -0,0 +1,109 @@ +import { Schema, model, Types } from 'mongoose'; +import { + SECRET_SHARED, + SECRET_PERSONAL, + ENV_DEV, + ENV_TESTING, + ENV_STAGING, + ENV_PROD +} from '../variables'; + +export interface ISecretSnapshot { + workspace: Types.ObjectId; + version: number; + secrets: { + version: number; + workspace: Types.ObjectId; + type: string; + user: Types.ObjectId; + environment: string; + secretKeyCiphertext: string; + secretKeyIV: string; + secretKeyTag: string; + secretKeyHash: string; + secretValueCiphertext: string; + secretValueIV: string; + secretValueTag: string; + secretValueHash: string; + }[] +} + +const secretSnapshotSchema = new Schema( + { + workspace: { + type: Schema.Types.ObjectId, + ref: 'Workspace', + required: true + }, + version: { + type: Number, + required: true + }, + secrets: [{ + version: { + type: Number, + default: 1, + required: true + }, + workspace: { + type: Schema.Types.ObjectId, + ref: 'Workspace', + required: true + }, + type: { + type: String, + enum: [SECRET_SHARED, SECRET_PERSONAL], + required: true + }, + user: { + // user associated with the personal secret + type: Schema.Types.ObjectId, + ref: 'User' + }, + environment: { + type: String, + enum: [ENV_DEV, ENV_TESTING, ENV_STAGING, ENV_PROD], + required: true + }, + secretKeyCiphertext: { + type: String, + required: true + }, + secretKeyIV: { + type: String, // symmetric + required: true + }, + secretKeyTag: { + type: String, // symmetric + required: true + }, + secretKeyHash: { + type: String, + required: true + }, + secretValueCiphertext: { + type: String, + required: true + }, + secretValueIV: { + type: String, // symmetric + required: true + }, + secretValueTag: { + type: String, // symmetric + required: true + }, + secretValueHash: { + type: String, + required: true + } + }] + }, + { + timestamps: true + } +); + +const SecretSnapshot = model('SecretSnapshot', secretSnapshotSchema); + +export default SecretSnapshot; \ No newline at end of file diff --git a/backend/src/models/secretVersion.ts b/backend/src/models/secretVersion.ts new file mode 100644 index 000000000..97c8ba585 --- /dev/null +++ b/backend/src/models/secretVersion.ts @@ -0,0 +1,75 @@ +import { Schema, model, Types } from 'mongoose'; + +export interface ISecretVersion { + _id: Types.ObjectId; + secret: Types.ObjectId; + version: number; + isDeleted: boolean; + secretKeyCiphertext: string; + secretKeyIV: string; + secretKeyTag: string; + secretKeyHash: string; + secretValueCiphertext: string; + secretValueIV: string; + secretValueTag: string; + secretValueHash: string; +} + +const secretVersionSchema = new Schema( + { + secret: { // could be deleted + type: Schema.Types.ObjectId, + ref: 'Secret', + required: true + }, + version: { + type: Number, + default: 1, + required: true + }, + isDeleted: { + type: Boolean, + default: false, + required: true + }, + secretKeyCiphertext: { + type: String, + required: true + }, + secretKeyIV: { + type: String, // symmetric + required: true + }, + secretKeyTag: { + type: String, // symmetric + required: true + }, + secretKeyHash: { + type: String, + required: true + }, + secretValueCiphertext: { + type: String, + required: true + }, + secretValueIV: { + type: String, // symmetric + required: true + }, + secretValueTag: { + type: String, // symmetric + required: true + }, + secretValueHash: { + type: String, + required: true + } + }, + { + timestamps: true + } +); + +const SecretVersion = model('SecretVersion', secretVersionSchema); + +export default SecretVersion; \ No newline at end of file From c4ebea74224b9388cfe5f536a2c8e840b1e479b7 Mon Sep 17 00:00:00 2001 From: Tuan Dang Date: Fri, 23 Dec 2022 17:47:42 -0500 Subject: [PATCH 3/8] Finish get secret versions route --- backend/src/controllers/secretController.ts | 37 ++++++++++++++++++--- backend/src/routes/secret.ts | 14 ++++++++ 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/backend/src/controllers/secretController.ts b/backend/src/controllers/secretController.ts index d1cf5f65d..350375e28 100644 --- a/backend/src/controllers/secretController.ts +++ b/backend/src/controllers/secretController.ts @@ -1,6 +1,6 @@ import { Request, Response } from 'express'; import * as Sentry from '@sentry/node'; -import { Key } from '../models'; +import { Key, Secret, SecretVersion } from '../models'; import { pushSecrets as push, pullSecrets as pull, @@ -160,9 +160,6 @@ export const pullSecrets = async (req: Request, res: Response) => { * @returns */ export const pullSecretsServiceToken = async (req: Request, res: Response) => { - // get (encrypted) secrets from workspace with id [workspaceId] - // service token route - let secrets; let key; try { @@ -217,3 +214,35 @@ export const pullSecretsServiceToken = async (req: Request, res: Response) => { key }); }; + +/** + * Return secret versions for secret with id [secretId] + * @param req + * @param res + */ +export const getSecretVersions = async (req: Request, res: Response) => { + let secretVersions; + try { + const { secretId } = req.params; + + const offset: number = parseInt(req.query.offset as string); + const limit: number = parseInt(req.query.limit as string); + + secretVersions = await SecretVersion.find({ + secret: secretId + }) + .skip(offset) + .limit(limit); + + } catch (err) { + Sentry.setUser({ email: req.serviceToken.user.email }); + Sentry.captureException(err); + return res.status(400).send({ + message: 'Failed to get secret versions' + }); + } + + return res.status(200).send({ + secretVersions + }); +} \ No newline at end of file diff --git a/backend/src/routes/secret.ts b/backend/src/routes/secret.ts index 98b3009de..073384129 100644 --- a/backend/src/routes/secret.ts +++ b/backend/src/routes/secret.ts @@ -50,4 +50,18 @@ router.get( secretController.pullSecretsServiceToken ); +router.get( + '/:secretId/secret-versions', + requireAuth, + requireWorkspaceAuth({ + acceptedRoles: [ADMIN, MEMBER], + acceptedStatuses: [COMPLETED, GRANTED] + }), + param('secretId').exists().trim(), + query('offset').exists().isInt(), + query('limit').exists().isInt(), + validateRequest, + secretController.getSecretVersions +); + export default router; From 9c769853b4a002b72b18b40e2cbfd3a9afda2114 Mon Sep 17 00:00:00 2001 From: Tuan Dang Date: Sat, 24 Dec 2022 20:01:33 -0500 Subject: [PATCH 4/8] Patch secret-override mechanism with versioning/snapshots --- backend/src/helpers/secret.ts | 35 ++++++++++++++--------------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/backend/src/helpers/secret.ts b/backend/src/helpers/secret.ts index b82b64bfc..5c8edddf7 100644 --- a/backend/src/helpers/secret.ts +++ b/backend/src/helpers/secret.ts @@ -57,17 +57,17 @@ const pushSecrets = async ({ workspaceId, environment }); - const oldSecretsObj: any = oldSecrets.reduce((accumulator, s: any) => { - return { ...accumulator, [s.secretKeyHash]: s }; - }, {}); - const newSecretsObj = secrets.reduce((accumulator, s) => { - return { ...accumulator, [s.hashKey]: s }; - }, {}); + const oldSecretsObj: any = oldSecrets.reduce((accumulator, s: any) => + ({ ...accumulator, [`${s.type}-${s.secretKeyHash}`]: s }) + , {}); + const newSecretsObj = secrets.reduce((accumulator, s) => + ({ ...accumulator, [`${s.type}-${s.hashKey}`]: s }) + , {}); // handle deleting secrets const toDelete = oldSecrets .filter( - (s: ISecret) => !(s.secretKeyHash in newSecretsObj) + (s: ISecret) => !(`${s.type}-${s.secretKeyHash}` in newSecretsObj) ) .map((s) => s._id); if (toDelete.length > 0) { @@ -87,16 +87,11 @@ const pushSecrets = async ({ // handle modifying secrets where type or value changed const toUpdate = secrets .filter((s) => { - if (s.hashKey in oldSecretsObj) { - if (s.hashValue !== oldSecretsObj[s.hashKey].secretValueHash) { + if (`${s.type}-${s.hashKey}` in oldSecretsObj) { + if (s.hashValue !== oldSecretsObj[`${s.type}-${s.hashKey}`].secretValueHash) { // case: filter secrets where value changed return true; } - - if (s.type !== oldSecretsObj[s.hashKey].type) { - // case: filter secrets where type changed - return true; - } } return false; @@ -122,8 +117,7 @@ const pushSecrets = async ({ return { updateOne: { filter: { - workspace: workspaceId, - _id: oldSecretsObj[s.hashKey]._id + _id: oldSecretsObj[`${s.type}-${s.hashKey}`]._id }, update } @@ -132,6 +126,7 @@ const pushSecrets = async ({ await Secret.bulkWrite(operations as any); await SecretVersion.insertMany( toUpdate.map(({ + type, ciphertextKey, ivKey, tagKey, @@ -141,8 +136,8 @@ const pushSecrets = async ({ tagValue, hashValue }) => ({ - secret: oldSecretsObj[hashKey]._id, - version: oldSecretsObj[hashKey].version + 1, + secret: oldSecretsObj[`${type}-${hashKey}`]._id, + version: oldSecretsObj[`${type}-${hashKey}`].version + 1, isDeleted: false, secretKeyCiphertext: ciphertextKey, secretKeyIV: ivKey, @@ -156,7 +151,7 @@ const pushSecrets = async ({ ); // handle adding new secrets - const toAdd = secrets.filter((s) => !(s.hashKey in oldSecretsObj)); + const toAdd = secrets.filter((s) => !(`${s.type}-${s.hashKey}` in oldSecretsObj)); if (toAdd.length > 0) { // add secrets @@ -214,8 +209,6 @@ const pushSecrets = async ({ await takeSecretSnapshotHelper({ workspaceId }); - // TODO: in the future add secret snapshot to capture entire - // state of project at this point in time } catch (err) { Sentry.setUser(null); Sentry.captureException(err); From f37fc9c59d13b26af7c519652bc4bbd12166f802 Mon Sep 17 00:00:00 2001 From: Tuan Dang Date: Sun, 25 Dec 2022 14:30:02 -0500 Subject: [PATCH 5/8] Small modifications to secret versioning/snapshot --- backend/src/controllers/secretController.ts | 2 +- .../src/controllers/workspaceController.ts | 35 ++++++++++++++++++- backend/src/routes/workspace.ts | 16 ++++++++- 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/backend/src/controllers/secretController.ts b/backend/src/controllers/secretController.ts index 2ee34e959..73f7d0606 100644 --- a/backend/src/controllers/secretController.ts +++ b/backend/src/controllers/secretController.ts @@ -244,7 +244,7 @@ export const getSecretVersions = async (req: Request, res: Response) => { .limit(limit); } catch (err) { - Sentry.setUser({ email: req.serviceToken.user.email }); + Sentry.setUser({ email: req.user.email }); Sentry.captureException(err); return res.status(400).send({ message: 'Failed to get secret versions' diff --git a/backend/src/controllers/workspaceController.ts b/backend/src/controllers/workspaceController.ts index a402834d7..7f901d0b2 100644 --- a/backend/src/controllers/workspaceController.ts +++ b/backend/src/controllers/workspaceController.ts @@ -7,7 +7,8 @@ import { Integration, IntegrationAuth, IUser, - ServiceToken + ServiceToken, + SecretSnapshot } from '../models'; import { createWorkspace as create, @@ -334,4 +335,36 @@ export const getWorkspaceServiceTokens = async ( return res.status(200).send({ serviceTokens }); +} + +/** + * Return secret snapshots for workspace with id [workspaceId] + * @param req + * @param res + */ + export const getWorkspaceSecretSnapshots = async (req: Request, res: Response) => { + let secretSnapshots; + try { + const { workspaceId } = req.params; + + const offset: number = parseInt(req.query.offset as string); + const limit: number = parseInt(req.query.limit as string); + + secretSnapshots = await SecretSnapshot.find({ + workspace: workspaceId + }) + .skip(offset) + .limit(limit); + + } catch (err) { + Sentry.setUser({ email: req.user.email }); + Sentry.captureException(err); + return res.status(400).send({ + message: 'Failed to get secret snapshots' + }); + } + + return res.status(200).send({ + secretSnapshots + }); } \ No newline at end of file diff --git a/backend/src/routes/workspace.ts b/backend/src/routes/workspace.ts index 1d20c102a..c5ffd739e 100644 --- a/backend/src/routes/workspace.ts +++ b/backend/src/routes/workspace.ts @@ -1,6 +1,6 @@ import express from 'express'; const router = express.Router(); -import { body, param } from 'express-validator'; +import { body, param, query } from 'express-validator'; import { requireAuth, requireWorkspaceAuth, @@ -130,4 +130,18 @@ router.get( workspaceController.getWorkspaceServiceTokens ); +router.get( + '/:workspaceId/secret-snapshots', + requireAuth, + requireWorkspaceAuth({ + acceptedRoles: [ADMIN, MEMBER], + acceptedStatuses: [GRANTED] + }), + param('workspaceId').exists().trim(), + query('offset').exists().isInt(), + query('limit').exists().isInt(), + validateRequest, + workspaceController.getWorkspaceSecretSnapshots +); + export default router; From 26fe1dd8216dac850eeadfdc456b68d68544a9a9 Mon Sep 17 00:00:00 2001 From: Tuan Dang Date: Sun, 25 Dec 2022 17:08:21 -0500 Subject: [PATCH 6/8] Move secret versioning and snapshot functionality into ee and begin license scoping --- backend/src/app.ts | 10 +++ backend/src/config/index.ts | 4 +- backend/src/controllers/secretController.ts | 36 +--------- .../src/controllers/workspaceController.ts | 33 --------- backend/src/ee/controllers/index.ts | 6 +- .../src/ee/controllers/secretController.ts | 35 ++++++++++ .../src/ee/controllers/workspaceController.ts | 35 ++++++++++ backend/src/ee/helpers/license.ts | 21 ------ backend/src/ee/helpers/secret.ts | 57 ++++++++++++++++ backend/src/ee/models/index.ts | 7 ++ backend/src/{ => ee}/models/secretSnapshot.ts | 2 +- backend/src/{ => ee}/models/secretVersion.ts | 0 backend/src/ee/routes/index.ts | 7 ++ backend/src/ee/routes/secret.ts | 26 +++++++ backend/src/ee/routes/workspace.ts | 27 ++++++++ backend/src/ee/services/EELicenseService.ts | 22 ++++++ backend/src/ee/services/EESecretService.ts | 29 ++++++++ backend/src/ee/services/index.ts | 7 ++ backend/src/helpers/secret.ts | 67 +++++-------------- backend/src/models/index.ts | 6 -- backend/src/routes/secret.ts | 16 +---- backend/src/routes/workspace.ts | 14 ---- docs/self-hosting/configuration/envars.mdx | 1 + 23 files changed, 290 insertions(+), 178 deletions(-) create mode 100644 backend/src/ee/controllers/secretController.ts create mode 100644 backend/src/ee/controllers/workspaceController.ts delete mode 100644 backend/src/ee/helpers/license.ts create mode 100644 backend/src/ee/helpers/secret.ts create mode 100644 backend/src/ee/models/index.ts rename backend/src/{ => ee}/models/secretSnapshot.ts (99%) rename backend/src/{ => ee}/models/secretVersion.ts (100%) create mode 100644 backend/src/ee/routes/index.ts create mode 100644 backend/src/ee/routes/secret.ts create mode 100644 backend/src/ee/routes/workspace.ts create mode 100644 backend/src/ee/services/EELicenseService.ts create mode 100644 backend/src/ee/services/EESecretService.ts create mode 100644 backend/src/ee/services/index.ts diff --git a/backend/src/app.ts b/backend/src/app.ts index e49551a91..2178e91e8 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -10,6 +10,11 @@ dotenv.config(); import { PORT, NODE_ENV, SITE_URL } from './config'; import { apiLimiter } from './helpers/rateLimiter'; +import { + workspace as eeWorkspaceRouter, + secret as eeSecretRouter +} from './ee/routes'; + import { signup as signupRouter, auth as authRouter, @@ -29,6 +34,7 @@ import { integration as integrationRouter, integrationAuth as integrationAuthRouter } from './routes'; + import { getLogger } from './utils/logger'; import { RouteNotFoundError } from './utils/errors'; import { requestErrorHandler } from './middleware/requestErrorHandler'; @@ -56,6 +62,10 @@ if (NODE_ENV === 'production') { app.use(helmet()); } +// /ee routers +app.use('/api/v1/secret', eeSecretRouter); +app.use('/api/v1/workspace', eeWorkspaceRouter); + // routers app.use('/api/v1/signup', signupRouter); app.use('/api/v1/auth', authRouter); diff --git a/backend/src/config/index.ts b/backend/src/config/index.ts index dfbc2111c..3fb475099 100644 --- a/backend/src/config/index.ts +++ b/backend/src/config/index.ts @@ -41,6 +41,7 @@ const STRIPE_PUBLISHABLE_KEY = process.env.STRIPE_PUBLISHABLE_KEY!; const STRIPE_SECRET_KEY = process.env.STRIPE_SECRET_KEY!; const STRIPE_WEBHOOK_SECRET = process.env.STRIPE_WEBHOOK_SECRET!; const TELEMETRY_ENABLED = process.env.TELEMETRY_ENABLED! !== 'false' && true; +const LICENSE_KEY = process.env.LICENSE_KEY!; export { PORT, @@ -83,5 +84,6 @@ export { STRIPE_PUBLISHABLE_KEY, STRIPE_SECRET_KEY, STRIPE_WEBHOOK_SECRET, - TELEMETRY_ENABLED + TELEMETRY_ENABLED, + LICENSE_KEY }; diff --git a/backend/src/controllers/secretController.ts b/backend/src/controllers/secretController.ts index 73f7d0606..6672c4b49 100644 --- a/backend/src/controllers/secretController.ts +++ b/backend/src/controllers/secretController.ts @@ -1,6 +1,6 @@ import { Request, Response } from 'express'; import * as Sentry from '@sentry/node'; -import { Key, Secret, SecretVersion } from '../models'; +import { Key, Secret } from '../models'; import { pushSecrets as push, pullSecrets as pull, @@ -222,36 +222,4 @@ export const pullSecretsServiceToken = async (req: Request, res: Response) => { secrets: reformatPullSecrets({ secrets }), key }); -}; - -/** - * Return secret versions for secret with id [secretId] - * @param req - * @param res - */ -export const getSecretVersions = async (req: Request, res: Response) => { - let secretVersions; - try { - const { secretId } = req.params; - - const offset: number = parseInt(req.query.offset as string); - const limit: number = parseInt(req.query.limit as string); - - secretVersions = await SecretVersion.find({ - secret: secretId - }) - .skip(offset) - .limit(limit); - - } catch (err) { - Sentry.setUser({ email: req.user.email }); - Sentry.captureException(err); - return res.status(400).send({ - message: 'Failed to get secret versions' - }); - } - - return res.status(200).send({ - secretVersions - }); -} \ No newline at end of file +}; \ No newline at end of file diff --git a/backend/src/controllers/workspaceController.ts b/backend/src/controllers/workspaceController.ts index 7f901d0b2..6f3e4bd11 100644 --- a/backend/src/controllers/workspaceController.ts +++ b/backend/src/controllers/workspaceController.ts @@ -8,7 +8,6 @@ import { IntegrationAuth, IUser, ServiceToken, - SecretSnapshot } from '../models'; import { createWorkspace as create, @@ -335,36 +334,4 @@ export const getWorkspaceServiceTokens = async ( return res.status(200).send({ serviceTokens }); -} - -/** - * Return secret snapshots for workspace with id [workspaceId] - * @param req - * @param res - */ - export const getWorkspaceSecretSnapshots = async (req: Request, res: Response) => { - let secretSnapshots; - try { - const { workspaceId } = req.params; - - const offset: number = parseInt(req.query.offset as string); - const limit: number = parseInt(req.query.limit as string); - - secretSnapshots = await SecretSnapshot.find({ - workspace: workspaceId - }) - .skip(offset) - .limit(limit); - - } catch (err) { - Sentry.setUser({ email: req.user.email }); - Sentry.captureException(err); - return res.status(400).send({ - message: 'Failed to get secret snapshots' - }); - } - - return res.status(200).send({ - secretSnapshots - }); } \ No newline at end of file diff --git a/backend/src/ee/controllers/index.ts b/backend/src/ee/controllers/index.ts index e4fb89a8e..23880070d 100644 --- a/backend/src/ee/controllers/index.ts +++ b/backend/src/ee/controllers/index.ts @@ -1,5 +1,9 @@ import * as stripeController from './stripeController'; +import * as secretController from './secretController'; +import * as workspaceController from './workspaceController'; export { - stripeController + stripeController, + secretController, + workspaceController } \ No newline at end of file diff --git a/backend/src/ee/controllers/secretController.ts b/backend/src/ee/controllers/secretController.ts new file mode 100644 index 000000000..503a81c51 --- /dev/null +++ b/backend/src/ee/controllers/secretController.ts @@ -0,0 +1,35 @@ +import { Request, Response } from 'express'; +import * as Sentry from '@sentry/node'; +import { SecretVersion } from '../models'; + +/** + * Return secret versions for secret with id [secretId] + * @param req + * @param res + */ + export const getSecretVersions = async (req: Request, res: Response) => { + let secretVersions; + try { + const { secretId } = req.params; + + const offset: number = parseInt(req.query.offset as string); + const limit: number = parseInt(req.query.limit as string); + + secretVersions = await SecretVersion.find({ + secret: secretId + }) + .skip(offset) + .limit(limit); + + } catch (err) { + Sentry.setUser({ email: req.user.email }); + Sentry.captureException(err); + return res.status(400).send({ + message: 'Failed to get secret versions' + }); + } + + return res.status(200).send({ + secretVersions + }); +} \ No newline at end of file diff --git a/backend/src/ee/controllers/workspaceController.ts b/backend/src/ee/controllers/workspaceController.ts new file mode 100644 index 000000000..423e71793 --- /dev/null +++ b/backend/src/ee/controllers/workspaceController.ts @@ -0,0 +1,35 @@ +import { Request, Response } from 'express'; +import * as Sentry from '@sentry/node'; +import { SecretSnapshot } from '../models'; + +/** + * Return secret snapshots for workspace with id [workspaceId] + * @param req + * @param res + */ + export const getWorkspaceSecretSnapshots = async (req: Request, res: Response) => { + let secretSnapshots; + try { + const { workspaceId } = req.params; + + const offset: number = parseInt(req.query.offset as string); + const limit: number = parseInt(req.query.limit as string); + + secretSnapshots = await SecretSnapshot.find({ + workspace: workspaceId + }) + .skip(offset) + .limit(limit); + + } catch (err) { + Sentry.setUser({ email: req.user.email }); + Sentry.captureException(err); + return res.status(400).send({ + message: 'Failed to get secret snapshots' + }); + } + + return res.status(200).send({ + secretSnapshots + }); +} \ No newline at end of file diff --git a/backend/src/ee/helpers/license.ts b/backend/src/ee/helpers/license.ts deleted file mode 100644 index 256bdc23a..000000000 --- a/backend/src/ee/helpers/license.ts +++ /dev/null @@ -1,21 +0,0 @@ - -/** - * @param {Object} obj - * @param {Object} obj.licenseKey - Infisical license key - */ -const checkLicenseKey = ({ - licenseKey -}: { - licenseKey: string -}) => { - try { - // TODO - - } catch (err) { - - } -} - -export { - checkLicenseKey -} \ No newline at end of file diff --git a/backend/src/ee/helpers/secret.ts b/backend/src/ee/helpers/secret.ts new file mode 100644 index 000000000..57fc178ea --- /dev/null +++ b/backend/src/ee/helpers/secret.ts @@ -0,0 +1,57 @@ +import * as Sentry from '@sentry/node'; +import { + Secret +} from '../../models'; +import { + SecretSnapshot +} from '../models'; + +/** + * Save a copy of the current state of secrets in workspace with id + * [workspaceId] under a new snapshot with incremented version under the + * secretsnapshots collection. + * @param {Object} obj + * @param {String} obj.workspaceId + */ + const takeSecretSnapshotHelper = async ({ + workspaceId +}: { + workspaceId: string; +}) => { + try { + const secrets = await Secret.find({ + workspace: workspaceId + }); + + const latestSecretSnapshot = await SecretSnapshot.findOne({ + workspace: workspaceId + }).sort({ version: -1 }); + + if (!latestSecretSnapshot) { + // case: no snapshots exist for workspace -> create first snapshot + await new SecretSnapshot({ + workspace: workspaceId, + version: 1, + secrets + }).save(); + + return; + } + + // case: snapshots exist for workspace + await new SecretSnapshot({ + workspace: workspaceId, + version: latestSecretSnapshot.version + 1, + secrets + }).save(); + + } catch (err) { + Sentry.setUser(null); + Sentry.captureException(err); + throw new Error('Failed to take a secret snapshot'); + } +} + +export { + takeSecretSnapshotHelper +} \ No newline at end of file diff --git a/backend/src/ee/models/index.ts b/backend/src/ee/models/index.ts new file mode 100644 index 000000000..930c01c4f --- /dev/null +++ b/backend/src/ee/models/index.ts @@ -0,0 +1,7 @@ +import SecretSnapshot, { ISecretSnapshot } from "./secretSnapshot"; +import SecretVersion, { ISecretVersion } from "./secretVersion"; + +export { + SecretSnapshot, + SecretVersion +} \ No newline at end of file diff --git a/backend/src/models/secretSnapshot.ts b/backend/src/ee/models/secretSnapshot.ts similarity index 99% rename from backend/src/models/secretSnapshot.ts rename to backend/src/ee/models/secretSnapshot.ts index 376115308..69633a92e 100644 --- a/backend/src/models/secretSnapshot.ts +++ b/backend/src/ee/models/secretSnapshot.ts @@ -6,7 +6,7 @@ import { ENV_TESTING, ENV_STAGING, ENV_PROD -} from '../variables'; +} from '../../variables'; export interface ISecretSnapshot { workspace: Types.ObjectId; diff --git a/backend/src/models/secretVersion.ts b/backend/src/ee/models/secretVersion.ts similarity index 100% rename from backend/src/models/secretVersion.ts rename to backend/src/ee/models/secretVersion.ts diff --git a/backend/src/ee/routes/index.ts b/backend/src/ee/routes/index.ts new file mode 100644 index 000000000..960665f4a --- /dev/null +++ b/backend/src/ee/routes/index.ts @@ -0,0 +1,7 @@ +import secret from './secret'; +import workspace from './workspace'; + +export { + secret, + workspace +} \ No newline at end of file diff --git a/backend/src/ee/routes/secret.ts b/backend/src/ee/routes/secret.ts new file mode 100644 index 000000000..d8f1cb05b --- /dev/null +++ b/backend/src/ee/routes/secret.ts @@ -0,0 +1,26 @@ +import express from 'express'; +const router = express.Router(); +import { + requireAuth, + requireWorkspaceAuth, + validateRequest +} from '../../middleware'; +import { body, query, param } from 'express-validator'; +import { secretController } from '../controllers'; +import { ADMIN, MEMBER, COMPLETED, GRANTED } from '../../variables'; + +router.get( + '/:secretId/secret-versions', + requireAuth, + requireWorkspaceAuth({ + acceptedRoles: [ADMIN, MEMBER], + acceptedStatuses: [COMPLETED, GRANTED] + }), + param('secretId').exists().trim(), + query('offset').exists().isInt(), + query('limit').exists().isInt(), + validateRequest, + secretController.getSecretVersions +); + +export default router; \ No newline at end of file diff --git a/backend/src/ee/routes/workspace.ts b/backend/src/ee/routes/workspace.ts new file mode 100644 index 000000000..e27300eb3 --- /dev/null +++ b/backend/src/ee/routes/workspace.ts @@ -0,0 +1,27 @@ +import express from 'express'; +const router = express.Router(); +import { + requireAuth, + requireWorkspaceAuth, + validateRequest +} from '../../middleware'; +import { param, query } from 'express-validator'; +import { ADMIN, MEMBER, GRANTED } from '../../variables'; +import { workspaceController } from '../controllers'; + +router.get( + '/:workspaceId/secret-snapshots', + requireAuth, + requireWorkspaceAuth({ + acceptedRoles: [ADMIN, MEMBER], + acceptedStatuses: [GRANTED] + }), + param('workspaceId').exists().trim(), + query('offset').exists().isInt(), + query('limit').exists().isInt(), + validateRequest, + workspaceController.getWorkspaceSecretSnapshots +); + + +export default router; \ No newline at end of file diff --git a/backend/src/ee/services/EELicenseService.ts b/backend/src/ee/services/EELicenseService.ts new file mode 100644 index 000000000..2c0228ef5 --- /dev/null +++ b/backend/src/ee/services/EELicenseService.ts @@ -0,0 +1,22 @@ +/** + * Class to handle Enterprise Edition license actions + */ +class EELicenseService { + /** + * Check if license key [licenseKey] corresponds to a + * valid Infisical Enterprise Edition license. + * @param {Object} obj + * @param {Object} obj.licenseKey + * @returns {Boolean} + */ + static async checkLicense({ + licenseKey + }: { + licenseKey: string; + }) { + // TODO + return true; + } +} + +export default EELicenseService; \ No newline at end of file diff --git a/backend/src/ee/services/EESecretService.ts b/backend/src/ee/services/EESecretService.ts new file mode 100644 index 000000000..dca8bdfe4 --- /dev/null +++ b/backend/src/ee/services/EESecretService.ts @@ -0,0 +1,29 @@ +import { takeSecretSnapshotHelper } from '../helpers/secret'; +import EELicenseService from './EELicenseService'; + +/** + * Class to handle Enterprise Edition secret actions + */ +class EESecretService { + + /** + * Save a copy of the current state of secrets in workspace with id + * [workspaceId] under a new snapshot with incremented version under the + * SecretSnapshot collection. + * Requires a valid license key [licenseKey] + * @param {Object} obj + * @param {String} obj.workspaceId + */ + static async takeSecretSnapshot({ + licenseKey, + workspaceId + }: { + licenseKey: string; + workspaceId: string; + }) { + EELicenseService.checkLicense({ licenseKey }); + await takeSecretSnapshotHelper({ workspaceId }); + } +} + +export default EESecretService; \ No newline at end of file diff --git a/backend/src/ee/services/index.ts b/backend/src/ee/services/index.ts new file mode 100644 index 000000000..3cec256bb --- /dev/null +++ b/backend/src/ee/services/index.ts @@ -0,0 +1,7 @@ +import EELicenseService from "./EELicenseService"; +import EESecretService from "./EESecretService"; + +export { + EELicenseService, + EESecretService +} \ No newline at end of file diff --git a/backend/src/helpers/secret.ts b/backend/src/helpers/secret.ts index 5c8edddf7..630bff458 100644 --- a/backend/src/helpers/secret.ts +++ b/backend/src/helpers/secret.ts @@ -2,13 +2,19 @@ import * as Sentry from '@sentry/node'; import { Secret, ISecret, - SecretVersion, - ISecretVersion, - SecretSnapshot, - ISecretSnapshot } from '../models'; +import { + EESecretService +} from '../ee/services'; +import { + SecretVersion +} from '../ee/models'; +import { + takeSecretSnapshotHelper +} from '../ee/helpers/secret'; import { decryptSymmetric } from '../utils/crypto'; import { SECRET_SHARED, SECRET_PERSONAL } from '../variables'; +import { LICENSE_KEY } from '../config'; interface PushSecret { ciphertextKey: string; @@ -206,9 +212,11 @@ const pushSecrets = async ({ ); } - await takeSecretSnapshotHelper({ + // (EE) take a secret snapshot + await EESecretService.takeSecretSnapshot({ + licenseKey: LICENSE_KEY, workspaceId - }); + }) } catch (err) { Sentry.setUser(null); Sentry.captureException(err); @@ -362,56 +370,11 @@ const decryptSecrets = ({ return content; }; -/** - * Saves a copy of the current state of secrets in workspace with id - * [workspaceId] under a new snapshot with incremented version under the - * secretsnapshots collection. - * @param {Object} obj - * @param {String} obj.workspaceId - */ -const takeSecretSnapshotHelper = async ({ - workspaceId -}: { - workspaceId: string; -}) => { - try { - const secrets = await Secret.find({ - workspace: workspaceId - }); - - const latestSecretSnapshot = await SecretSnapshot.findOne({ - workspace: workspaceId - }).sort({ version: -1 }); - - if (!latestSecretSnapshot) { - // case: no snapshots exist for workspace -> create first snapshot - await new SecretSnapshot({ - workspace: workspaceId, - version: 1, - secrets - }).save(); - return; - } - - // case: snapshots exist for workspace - await new SecretSnapshot({ - workspace: workspaceId, - version: latestSecretSnapshot.version + 1, - secrets - }).save(); - - } catch (err) { - Sentry.setUser(null); - Sentry.captureException(err); - throw new Error('Failed to take a secret snapshot'); - } -} export { pushSecrets, pullSecrets, reformatPullSecrets, - decryptSecrets, - takeSecretSnapshotHelper + decryptSecrets }; diff --git a/backend/src/models/index.ts b/backend/src/models/index.ts index daab77b2a..78c38060b 100644 --- a/backend/src/models/index.ts +++ b/backend/src/models/index.ts @@ -9,8 +9,6 @@ import Membership, { IMembership } from './membership'; import MembershipOrg, { IMembershipOrg } from './membershipOrg'; import Organization, { IOrganization } from './organization'; import Secret, { ISecret } from './secret'; -import SecretVersion, { ISecretVersion } from './secretVersion'; -import SecretSnapshot, { ISecretSnapshot } from './secretSnapshot'; import ServiceToken, { IServiceToken } from './serviceToken'; import Token, { IToken } from './token'; import User, { IUser } from './user'; @@ -40,10 +38,6 @@ export { IOrganization, Secret, ISecret, - SecretVersion, - ISecretVersion, - SecretSnapshot, - ISecretSnapshot, ServiceToken, IServiceToken, Token, diff --git a/backend/src/routes/secret.ts b/backend/src/routes/secret.ts index 073384129..26224fd87 100644 --- a/backend/src/routes/secret.ts +++ b/backend/src/routes/secret.ts @@ -7,8 +7,8 @@ import { validateRequest } from '../middleware'; import { body, query, param } from 'express-validator'; -import { ADMIN, MEMBER, COMPLETED, GRANTED } from '../variables'; import { secretController } from '../controllers'; +import { ADMIN, MEMBER, COMPLETED, GRANTED } from '../variables'; router.post( '/:workspaceId', @@ -50,18 +50,4 @@ router.get( secretController.pullSecretsServiceToken ); -router.get( - '/:secretId/secret-versions', - requireAuth, - requireWorkspaceAuth({ - acceptedRoles: [ADMIN, MEMBER], - acceptedStatuses: [COMPLETED, GRANTED] - }), - param('secretId').exists().trim(), - query('offset').exists().isInt(), - query('limit').exists().isInt(), - validateRequest, - secretController.getSecretVersions -); - export default router; diff --git a/backend/src/routes/workspace.ts b/backend/src/routes/workspace.ts index c5ffd739e..acd2aaf8b 100644 --- a/backend/src/routes/workspace.ts +++ b/backend/src/routes/workspace.ts @@ -130,18 +130,4 @@ router.get( workspaceController.getWorkspaceServiceTokens ); -router.get( - '/:workspaceId/secret-snapshots', - requireAuth, - requireWorkspaceAuth({ - acceptedRoles: [ADMIN, MEMBER], - acceptedStatuses: [GRANTED] - }), - param('workspaceId').exists().trim(), - query('offset').exists().isInt(), - query('limit').exists().isInt(), - validateRequest, - workspaceController.getWorkspaceSecretSnapshots -); - export default router; diff --git a/docs/self-hosting/configuration/envars.mdx b/docs/self-hosting/configuration/envars.mdx index 0b9fd5e71..f598336b6 100644 --- a/docs/self-hosting/configuration/envars.mdx +++ b/docs/self-hosting/configuration/envars.mdx @@ -28,6 +28,7 @@ Configuring Infisical requires setting some environment variables. There is a fi | `SMTP_FROM_ADDRESS` | ❗️ Email address to be used for sending emails (e.g. `team@infisical.com`) | `None` | | `SMTP_FROM_NAME` | Name label to be used in From field (e.g. `Team`) | `Infisical` | | `TELEMETRY_ENABLED` | `true` or `false`. [More](../overview). | `true` | +| `LICENSE_KEY` | License key if using Infisical Enterprise Edition | `true` | | `CLIENT_ID_HEROKU` | OAuth2 client ID for Heroku integration | `None` | | `CLIENT_ID_VERCEL` | OAuth2 client ID for Vercel integration | `None` | | `CLIENT_ID_NETLIFY` | OAuth2 client ID for Netlify integration | `None` | From 9f724b5eded34722e040add598e7fcbd962da656 Mon Sep 17 00:00:00 2001 From: Tuan Dang Date: Sun, 25 Dec 2022 20:04:27 -0500 Subject: [PATCH 7/8] Refactor EE secret versioning/snapshot access --- backend/src/app.ts | 3 +-- backend/src/ee/helpers/secret.ts | 21 ++++++++++++++-- backend/src/ee/models/index.ts | 4 ++- backend/src/ee/models/secretVersion.ts | 2 +- backend/src/ee/services/EELicenseService.ts | 27 +++++++++------------ backend/src/ee/services/EESecretService.ts | 24 ++++++++++++++++-- backend/src/helpers/secret.ts | 17 +++++++------ 7 files changed, 68 insertions(+), 30 deletions(-) diff --git a/backend/src/app.ts b/backend/src/app.ts index 2178e91e8..211a53dab 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -1,4 +1,3 @@ - import { patchRouterParam } from './utils/patchAsyncRoutes'; import express from 'express'; import helmet from 'helmet'; @@ -7,7 +6,7 @@ import cookieParser from 'cookie-parser'; import dotenv from 'dotenv'; dotenv.config(); -import { PORT, NODE_ENV, SITE_URL } from './config'; +import { PORT, NODE_ENV, SITE_URL, LICENSE_KEY } from './config'; import { apiLimiter } from './helpers/rateLimiter'; import { diff --git a/backend/src/ee/helpers/secret.ts b/backend/src/ee/helpers/secret.ts index 57fc178ea..a688a108f 100644 --- a/backend/src/ee/helpers/secret.ts +++ b/backend/src/ee/helpers/secret.ts @@ -3,7 +3,9 @@ import { Secret } from '../../models'; import { - SecretSnapshot + SecretSnapshot, + SecretVersion, + ISecretVersion } from '../models'; /** @@ -52,6 +54,21 @@ import { } } +const addSecretVersionsHelper = async ({ + secretVersions +}: { + secretVersions: ISecretVersion[] +}) => { + try { + await SecretVersion.insertMany(secretVersions); + } catch (err) { + Sentry.setUser(null); + Sentry.captureException(err); + throw new Error('Failed to add secret versions'); + } +} + export { - takeSecretSnapshotHelper + takeSecretSnapshotHelper, + addSecretVersionsHelper } \ No newline at end of file diff --git a/backend/src/ee/models/index.ts b/backend/src/ee/models/index.ts index 930c01c4f..35d41c19a 100644 --- a/backend/src/ee/models/index.ts +++ b/backend/src/ee/models/index.ts @@ -3,5 +3,7 @@ import SecretVersion, { ISecretVersion } from "./secretVersion"; export { SecretSnapshot, - SecretVersion + ISecretSnapshot, + SecretVersion, + ISecretVersion } \ No newline at end of file diff --git a/backend/src/ee/models/secretVersion.ts b/backend/src/ee/models/secretVersion.ts index 97c8ba585..a93a037f6 100644 --- a/backend/src/ee/models/secretVersion.ts +++ b/backend/src/ee/models/secretVersion.ts @@ -1,7 +1,7 @@ import { Schema, model, Types } from 'mongoose'; export interface ISecretVersion { - _id: Types.ObjectId; + _id?: Types.ObjectId; secret: Types.ObjectId; version: number; isDeleted: boolean; diff --git a/backend/src/ee/services/EELicenseService.ts b/backend/src/ee/services/EELicenseService.ts index 2c0228ef5..f31482dde 100644 --- a/backend/src/ee/services/EELicenseService.ts +++ b/backend/src/ee/services/EELicenseService.ts @@ -1,22 +1,19 @@ +import { LICENSE_KEY } from '../../config'; + /** * Class to handle Enterprise Edition license actions */ class EELicenseService { - /** - * Check if license key [licenseKey] corresponds to a - * valid Infisical Enterprise Edition license. - * @param {Object} obj - * @param {Object} obj.licenseKey - * @returns {Boolean} - */ - static async checkLicense({ - licenseKey - }: { - licenseKey: string; - }) { - // TODO - return true; + + private readonly _isLicenseValid: boolean; + + constructor(licenseKey: string) { + this._isLicenseValid = true; + } + + public get isLicenseValid(): boolean { + return this._isLicenseValid; } } -export default EELicenseService; \ No newline at end of file +export default new EELicenseService(LICENSE_KEY); \ No newline at end of file diff --git a/backend/src/ee/services/EESecretService.ts b/backend/src/ee/services/EESecretService.ts index dca8bdfe4..9cc5fba2f 100644 --- a/backend/src/ee/services/EESecretService.ts +++ b/backend/src/ee/services/EESecretService.ts @@ -1,4 +1,8 @@ -import { takeSecretSnapshotHelper } from '../helpers/secret'; +import { ISecretVersion } from '../models'; +import { + takeSecretSnapshotHelper, + addSecretVersionsHelper +} from '../helpers/secret'; import EELicenseService from './EELicenseService'; /** @@ -21,9 +25,25 @@ class EESecretService { licenseKey: string; workspaceId: string; }) { - EELicenseService.checkLicense({ licenseKey }); + if (!EELicenseService.isLicenseValid) return; await takeSecretSnapshotHelper({ workspaceId }); } + + /** + * Adds secret versions [secretVersions] to the SecretVersion collection. + * @param {Object} obj + * @param {SecretVersion} obj.secretVersions + */ + static async addSecretVersions({ + secretVersions + }: { + secretVersions: ISecretVersion[]; + }) { + if (!EELicenseService.isLicenseValid) return; + await addSecretVersionsHelper({ + secretVersions + }); + } } export default EESecretService; \ No newline at end of file diff --git a/backend/src/helpers/secret.ts b/backend/src/helpers/secret.ts index 630bff458..c01503402 100644 --- a/backend/src/helpers/secret.ts +++ b/backend/src/helpers/secret.ts @@ -130,8 +130,10 @@ const pushSecrets = async ({ }; }); await Secret.bulkWrite(operations as any); - await SecretVersion.insertMany( - toUpdate.map(({ + + // (EE) add secret versions for updated secrets + await EESecretService.addSecretVersions({ + secretVersions: toUpdate.map(({ type, ciphertextKey, ivKey, @@ -153,8 +155,8 @@ const pushSecrets = async ({ secretValueIV: ivValue, secretValueTag: tagValue, secretValueHash: hashValue - })) - ); + })) + }); // handle adding new secrets const toAdd = secrets.filter((s) => !(`${s.type}-${s.hashKey}` in oldSecretsObj)); @@ -185,8 +187,9 @@ const pushSecrets = async ({ }) ); - await SecretVersion.insertMany( - newSecrets.map(({ + // (EE) add secret versions for new secrets + EESecretService.addSecretVersions({ + secretVersions: newSecrets.map(({ _id, secretKeyCiphertext, secretKeyIV, @@ -209,7 +212,7 @@ const pushSecrets = async ({ secretValueTag, secretValueHash })) - ); + }); } // (EE) take a secret snapshot From 0c6dfbe4b4fd652143625f031168c55ead41c4f1 Mon Sep 17 00:00:00 2001 From: Tuan Dang Date: Mon, 26 Dec 2022 17:52:13 -0500 Subject: [PATCH 8/8] Fix commonjs import/export for patchRouterParam and make secret versioning/snapshots compatible with prev unversioned secrets --- backend/src/app.ts | 10 ++- backend/src/ee/services/EESecretService.ts | 2 - backend/src/helpers/secret.ts | 93 +++++++++++++--------- backend/src/models/secret.ts | 3 +- backend/src/utils/patchAsyncRoutes.js | 8 +- 5 files changed, 67 insertions(+), 49 deletions(-) diff --git a/backend/src/app.ts b/backend/src/app.ts index 211a53dab..0fe39ae80 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -1,4 +1,6 @@ -import { patchRouterParam } from './utils/patchAsyncRoutes'; +// eslint-disable-next-line @typescript-eslint/no-var-requires +const { patchRouterParam } = require('./utils/patchAsyncRoutes'); + import express from 'express'; import helmet from 'helmet'; import cors from 'cors'; @@ -6,7 +8,7 @@ import cookieParser from 'cookie-parser'; import dotenv from 'dotenv'; dotenv.config(); -import { PORT, NODE_ENV, SITE_URL, LICENSE_KEY } from './config'; +import { PORT, NODE_ENV, SITE_URL } from './config'; import { apiLimiter } from './helpers/rateLimiter'; import { @@ -38,8 +40,8 @@ import { getLogger } from './utils/logger'; import { RouteNotFoundError } from './utils/errors'; import { requestErrorHandler } from './middleware/requestErrorHandler'; -//* Patch Async route params to handle Promise Rejections -patchRouterParam() +// patch async route params to handle Promise Rejections +patchRouterParam(); export const app = express(); diff --git a/backend/src/ee/services/EESecretService.ts b/backend/src/ee/services/EESecretService.ts index 9cc5fba2f..643f763f1 100644 --- a/backend/src/ee/services/EESecretService.ts +++ b/backend/src/ee/services/EESecretService.ts @@ -19,10 +19,8 @@ class EESecretService { * @param {String} obj.workspaceId */ static async takeSecretSnapshot({ - licenseKey, workspaceId }: { - licenseKey: string; workspaceId: string; }) { if (!EELicenseService.isLicenseValid) return; diff --git a/backend/src/helpers/secret.ts b/backend/src/helpers/secret.ts index c01503402..4b3585c40 100644 --- a/backend/src/helpers/secret.ts +++ b/backend/src/helpers/secret.ts @@ -56,6 +56,7 @@ const pushSecrets = async ({ environment: string; secrets: PushSecret[]; }): Promise => { + // TODO: clean up function and fix up types try { // construct useful data structures const oldSecrets = await pullSecrets({ @@ -63,10 +64,11 @@ const pushSecrets = async ({ workspaceId, environment }); + const oldSecretsObj: any = oldSecrets.reduce((accumulator, s: any) => ({ ...accumulator, [`${s.type}-${s.secretKeyHash}`]: s }) , {}); - const newSecretsObj = secrets.reduce((accumulator, s) => + const newSecretsObj: any = secrets.reduce((accumulator, s) => ({ ...accumulator, [`${s.type}-${s.hashKey}`]: s }) , {}); @@ -79,8 +81,6 @@ const pushSecrets = async ({ if (toDelete.length > 0) { await Secret.deleteMany({ _id: { $in: toDelete } - }, { - rawResult: true }); await SecretVersion.updateMany({ @@ -89,31 +89,48 @@ const pushSecrets = async ({ isDeleted: true }); } - - // handle modifying secrets where type or value changed - const toUpdate = secrets + + const toUpdate = oldSecrets .filter((s) => { - if (`${s.type}-${s.hashKey}` in oldSecretsObj) { - if (s.hashValue !== oldSecretsObj[`${s.type}-${s.hashKey}`].secretValueHash) { + if (`${s.type}-${s.secretKeyHash}` in newSecretsObj) { + if (s.secretValueHash !== newSecretsObj[`${s.type}-${s.secretKeyHash}`].hashValue) { // case: filter secrets where value changed return true; } - } + if (!s.version) { + // case: filter (legacy) secrets that were not versioned + return true; + } + } + return false; }); - + const operations = toUpdate .map((s) => { + const { + ciphertextValue, + ivValue, + tagValue, + hashValue + } = newSecretsObj[`${s.type}-${s.secretKeyHash}`]; + const update: Update = { - secretValueCiphertext: s.ciphertextValue, - secretValueIV: s.ivValue, - secretValueTag: s.tagValue, - secretValueHash: s.hashValue, - $inc: { + secretValueCiphertext: ciphertextValue, + secretValueIV: ivValue, + secretValueTag: tagValue, + secretValueHash: hashValue + } + + if (!s.version) { + // case: (legacy) secret was not versioned + update.version = 1; + } else { + update['$inc'] = { version: 1 } - }; + } if (s.type === SECRET_PERSONAL) { // attach user associated with the personal secret @@ -123,7 +140,7 @@ const pushSecrets = async ({ return { updateOne: { filter: { - _id: oldSecretsObj[`${s.type}-${s.hashKey}`]._id + _id: oldSecretsObj[`${s.type}-${s.secretKeyHash}`]._id }, update } @@ -134,28 +151,26 @@ const pushSecrets = async ({ // (EE) add secret versions for updated secrets await EESecretService.addSecretVersions({ secretVersions: toUpdate.map(({ + _id, + version, type, - ciphertextKey, - ivKey, - tagKey, - hashKey, - ciphertextValue, - ivValue, - tagValue, - hashValue - }) => ({ - secret: oldSecretsObj[`${type}-${hashKey}`]._id, - version: oldSecretsObj[`${type}-${hashKey}`].version + 1, - isDeleted: false, - secretKeyCiphertext: ciphertextKey, - secretKeyIV: ivKey, - secretKeyTag: tagKey, - secretKeyHash: hashKey, - secretValueCiphertext: ciphertextValue, - secretValueIV: ivValue, - secretValueTag: tagValue, - secretValueHash: hashValue - })) + secretKeyHash, + }) => { + const newSecret = newSecretsObj[`${type}-${secretKeyHash}`]; + return ({ + secret: _id, + version: version ? version + 1 : 1, + isDeleted: false, + secretKeyCiphertext: newSecret.ciphertextKey, + secretKeyIV: newSecret.ivKey, + secretKeyTag: newSecret.tagKey, + secretKeyHash: newSecret.hashKey, + secretValueCiphertext: newSecret.ciphertextValue, + secretValueIV: newSecret.ivValue, + secretValueTag: newSecret.tagValue, + secretValueHash: newSecret.hashValue + }) + }) }); // handle adding new secrets @@ -166,6 +181,7 @@ const pushSecrets = async ({ const newSecrets = await Secret.insertMany( toAdd.map((s, idx) => { const obj: any = { + version: 1, workspace: workspaceId, type: toAdd[idx].type, environment, @@ -217,7 +233,6 @@ const pushSecrets = async ({ // (EE) take a secret snapshot await EESecretService.takeSecretSnapshot({ - licenseKey: LICENSE_KEY, workspaceId }) } catch (err) { diff --git a/backend/src/models/secret.ts b/backend/src/models/secret.ts index ee879de30..d34139ecb 100644 --- a/backend/src/models/secret.ts +++ b/backend/src/models/secret.ts @@ -29,8 +29,7 @@ const secretSchema = new Schema( { version: { type: Number, - default: 1, - required: true + required: true }, workspace: { type: Schema.Types.ObjectId, diff --git a/backend/src/utils/patchAsyncRoutes.js b/backend/src/utils/patchAsyncRoutes.js index 6f6d2367f..24fe007f9 100644 --- a/backend/src/utils/patchAsyncRoutes.js +++ b/backend/src/utils/patchAsyncRoutes.js @@ -45,7 +45,7 @@ function wrap(fn) { return copyFnProps(fn, newFn); } -export function patchRouterParam() { +function patchRouterParam() { const originalParam = Router.prototype.constructor.param; Router.prototype.constructor.param = function param(name, fn) { fn = wrap(fn); @@ -62,4 +62,8 @@ Object.defineProperty(Layer.prototype, 'handle', { fn = wrap(fn); this.__handle = fn; }, -}); \ No newline at end of file +}); + +module.exports = { + patchRouterParam +};