Add v1 audit log backend models and wiring to push secrets

This commit is contained in:
Tuan Dang
2022-12-27 12:12:39 -05:00
parent 019e90dc77
commit 9497a26eb2
26 changed files with 459 additions and 272 deletions
+3 -1
View File
@@ -13,7 +13,8 @@ import { apiLimiter } from './helpers/rateLimiter';
import { import {
workspace as eeWorkspaceRouter, workspace as eeWorkspaceRouter,
secret as eeSecretRouter secret as eeSecretRouter,
log as eeLogRouter,
} from './ee/routes/v1'; } from './ee/routes/v1';
import { import {
signup as v1SignupRouter, signup as v1SignupRouter,
@@ -69,6 +70,7 @@ if (NODE_ENV === 'production') {
// (EE) routes // (EE) routes
app.use('/api/v1/secret', eeSecretRouter); app.use('/api/v1/secret', eeSecretRouter);
app.use('/api/v1/workspace', eeWorkspaceRouter); app.use('/api/v1/workspace', eeWorkspaceRouter);
app.use('/api/v1/log', eeLogRouter);
// v1 routes // v1 routes
app.use('/api/v1/signup', v1SignupRouter); app.use('/api/v1/signup', v1SignupRouter);
-30
View File
@@ -1,30 +0,0 @@
import { Request, Response } from 'express';
import * as Sentry from '@sentry/node';
import {
Log
} from '../models';
export const getLogs = async (req: Request, res: Response) => {
// get logs
console.log('getLogs');
let logs;
try {
const { workspaceId } = req.params;
logs = await Log.find({
workspace: workspaceId
});
} catch (err) {
Sentry.setUser({ email: req.user.email });
Sentry.captureException(err);
return res.status(400).send({
message: 'Failed to get audit logs'
});
}
return res.status(200).send({
logs
});
}
+1 -3
View File
@@ -14,7 +14,6 @@ import * as stripeController from './stripeController';
import * as userActionController from './userActionController'; import * as userActionController from './userActionController';
import * as userController from './userController'; import * as userController from './userController';
import * as workspaceController from './workspaceController'; import * as workspaceController from './workspaceController';
import * as logController from './logController';
export { export {
authController, authController,
@@ -32,6 +31,5 @@ export {
stripeController, stripeController,
userActionController, userActionController,
userController, userController,
workspaceController, workspaceController
logController
}; };
@@ -389,7 +389,9 @@ export const pushWorkspaceSecrets = async (req: Request, res: Response) => {
userId: req.user._id, userId: req.user._id,
workspaceId, workspaceId,
environment, environment,
secrets secrets,
channel: channel ? channel : 'cli',
ipAddress: req.ip
}); });
await pushKeys({ await pushKeys({
@@ -1,6 +1,9 @@
import { Request, Response } from 'express'; import { Request, Response } from 'express';
import * as Sentry from '@sentry/node'; import * as Sentry from '@sentry/node';
import { SecretSnapshot } from '../../models'; import {
SecretSnapshot,
Log
} from '../../models';
/** /**
* Return secret snapshots for workspace with id [workspaceId] * Return secret snapshots for workspace with id [workspaceId]
@@ -32,4 +35,33 @@ import { SecretSnapshot } from '../../models';
return res.status(200).send({ return res.status(200).send({
secretSnapshots secretSnapshots
}); });
}
export const getWorkspaceLogs = async (req: Request, res: Response) => {
let logs
try {
const { workspaceId } = req.params;
const offset: number = parseInt(req.query.offset as string);
const limit: number = parseInt(req.query.limit as string);
const filters: any = req.query.filters || {};
filters.workspace = workspaceId;
logs = await Log.find(filters)
.skip(offset)
.limit(limit)
.populate('actions');
} catch (err) {
Sentry.setUser({ email: req.user.email });
Sentry.captureException(err);
return res.status(400).send({
message: 'Failed to get workspace logs'
});
}
return res.status(200).send({
logs
});
} }
+40
View File
@@ -0,0 +1,40 @@
import * as Sentry from '@sentry/node';
import {
Log,
IAction
} from '../models';
const createLogHelper = async ({
userId,
workspaceId,
actions,
channel,
ipAddress
}: {
userId: string;
workspaceId: string;
actions: IAction[];
channel: string;
ipAddress: string;
}) => {
let log;
try {
log = await new Log({
user: userId,
workspace: workspaceId,
actions,
channel,
ipAddress
}).save();
} catch (err) {
Sentry.setUser(null);
Sentry.captureException(err);
throw new Error('Failed to create log');
}
return log;
}
export {
createLogHelper
}
+40
View File
@@ -0,0 +1,40 @@
import { Schema, model, Types } from 'mongoose';
export interface IAction {
name: string;
user?: Types.ObjectId,
workspace?: Types.ObjectId,
payload: {
secretVersions?: Types.ObjectId[]
}
}
const actionSchema = new Schema<IAction>(
{
name: {
type: String,
required: true
},
user: {
type: Schema.Types.ObjectId,
ref: 'User',
required: true
},
workspace: {
type: Schema.Types.ObjectId,
ref: 'Workspace'
},
payload: {
secretVersions: [{
type: Schema.Types.ObjectId,
ref: 'SecretVersion'
}]
}
}, {
timestamps: true
}
);
const Action = model<IAction>('Action', actionSchema);
export default Action;
+9 -3
View File
@@ -1,9 +1,15 @@
import SecretSnapshot, { ISecretSnapshot } from "./secretSnapshot"; import SecretSnapshot, { ISecretSnapshot } from './secretSnapshot';
import SecretVersion, { ISecretVersion } from "./secretVersion"; import SecretVersion, { ISecretVersion } from './secretVersion';
import Log, { ILog } from './log';
import Action, { IAction } from './action';
export { export {
SecretSnapshot, SecretSnapshot,
ISecretSnapshot, ISecretSnapshot,
SecretVersion, SecretVersion,
ISecretVersion ISecretVersion,
Log,
ILog,
Action,
IAction
} }
+41
View File
@@ -0,0 +1,41 @@
import { Schema, model, Types } from 'mongoose';
export interface ILog {
_id: Types.ObjectId;
user?: Types.ObjectId;
workspace?: Types.ObjectId;
actions: Types.ObjectId[];
channel: string;
ipAddress?: string;
}
const logSchema = new Schema<ILog>(
{
user: {
type: Schema.Types.ObjectId,
ref: 'User'
},
workspace: {
type: Schema.Types.ObjectId,
ref: 'Workspace'
},
actions: [{
type: Schema.Types.ObjectId,
ref: 'Action'
}],
channel: {
type: String,
enum: ['web', 'cli', 'auto'],
required: true
},
ipAddress: {
type: String
}
}, {
timestamps: true
}
);
const Log = model<ILog>('Log', logSchema);
export default Log;
+3 -1
View File
@@ -1,7 +1,9 @@
import secret from './secret'; import secret from './secret';
import workspace from './workspace'; import workspace from './workspace';
import log from './log';
export { export {
secret, secret,
workspace workspace,
log
} }
+4
View File
@@ -0,0 +1,4 @@
import express from 'express';
const router = express.Router();
export default router;
+14
View File
@@ -23,5 +23,19 @@ router.get(
workspaceController.getWorkspaceSecretSnapshots workspaceController.getWorkspaceSecretSnapshots
); );
router.get(
'/:workspaceId/logs',
requireAuth,
requireWorkspaceAuth({
acceptedRoles: [ADMIN, MEMBER],
acceptedStatuses: [GRANTED]
}),
param('workspaceId').exists().trim(),
query('offset').exists().isInt(),
query('limit').exists().isInt(),
query('filters').exists(),
validateRequest,
workspaceController.getWorkspaceLogs
);
export default router; export default router;
+47
View File
@@ -0,0 +1,47 @@
import {
Action,
IAction
} from '../models';
import {
createLogHelper
} from '../helpers/log';
import EELicenseService from './EELicenseService';
/**
* Class to handle Enterprise Edition log actions
*/
class EELogService {
/**
* Create an (audit) log
* @param {Object} obj
* @param {String} obj.userId - id of user associated with the log
* @param {String} obj.workspaceId - id of workspace associated with the log
* @param {Action} obj.actions - actions to include in log
* @param {String} obj.channel - channel (web/cli/auto) associated with the log
* @param {String} obj.ipAddress - ip address associated with the log
*/
static async createLog({
userId,
workspaceId,
actions,
channel,
ipAddress
}: {
userId: string;
workspaceId: string;
actions: IAction[];
channel: string;
ipAddress: string;
}) {
if (!EELicenseService.isLicenseValid) return;
return await createLogHelper({
userId,
workspaceId,
actions,
channel,
ipAddress
})
}
}
export default EELogService;
+3 -1
View File
@@ -1,7 +1,9 @@
import EELicenseService from "./EELicenseService"; import EELicenseService from "./EELicenseService";
import EESecretService from "./EESecretService"; import EESecretService from "./EESecretService";
import EELogService from "./EELogService";
export { export {
EELicenseService, EELicenseService,
EESecretService EESecretService,
EELogService
} }
View File
+198 -80
View File
@@ -1,19 +1,29 @@
import * as Sentry from '@sentry/node'; import * as Sentry from '@sentry/node';
import { Types } from 'mongoose';
import { import {
Secret, Secret,
ISecret, ISecret,
} from '../models'; } from '../models';
import { import {
EESecretService EESecretService,
EELogService
} from '../ee/services'; } from '../ee/services';
import { import {
SecretVersion SecretVersion,
Action,
IAction
} from '../ee/models'; } from '../ee/models';
import { import {
takeSecretSnapshotHelper takeSecretSnapshotHelper
} from '../ee/helpers/secret'; } from '../ee/helpers/secret';
import { decryptSymmetric } from '../utils/crypto'; import { decryptSymmetric } from '../utils/crypto';
import { SECRET_SHARED, SECRET_PERSONAL } from '../variables'; import {
SECRET_SHARED,
SECRET_PERSONAL,
ACTION_ADD_SECRETS,
ACTION_UPDATE_SECRETS,
ACTION_DELETE_SECRETS
} from '../variables';
interface V1PushSecret { interface V1PushSecret {
ciphertextKey: string; ciphertextKey: string;
@@ -284,20 +294,28 @@ const v1PushSecrets = async ({
* @param {String} obj.workspaceId - id of workspace to push to * @param {String} obj.workspaceId - id of workspace to push to
* @param {String} obj.environment - environment for secrets * @param {String} obj.environment - environment for secrets
* @param {Object[]} obj.secrets - secrets to push * @param {Object[]} obj.secrets - secrets to push
* @param {String} obj.channel - channel (web/cli/auto)
* @param {String} obj.ipAddress - ip address of request to push secrets
*/ */
const v2PushSecrets = async ({ const v2PushSecrets = async ({
userId, userId,
workspaceId, workspaceId,
environment, environment,
secrets secrets,
channel,
ipAddress
}: { }: {
userId: string; userId: string;
workspaceId: string; workspaceId: string;
environment: string; environment: string;
secrets: V2PushSecret[]; secrets: V2PushSecret[];
channel: string;
ipAddress: string;
}): Promise<void> => { }): Promise<void> => {
// TODO: clean up function and fix up types // TODO: clean up function and fix up types
try { try {
const actions: IAction[] = [];
// construct useful data structures // construct useful data structures
const oldSecrets = await pullSecrets({ const oldSecrets = await pullSecrets({
userId, userId,
@@ -327,7 +345,37 @@ const v1PushSecrets = async ({
secret: { $in: toDelete } secret: { $in: toDelete }
}, { }, {
isDeleted: true isDeleted: true
}, {
new: true
}); });
// add audit log for deleted secrets
const deletedLatestSecretVersions = (await SecretVersion.aggregate([
{
$match: { secret: { $in: toDelete } }
},
{
$group: {
_id: '$secret',
version: { $max: '$version' }
}
},
{
$sort: { version: -1 }
}
])
.exec())
.map((s) => s._id);
const deleteAction = await new Action({
name: ACTION_DELETE_SECRETS,
user: new Types.ObjectId(userId),
workspace: new Types.ObjectId(workspaceId),
payload: {
secretVersions: deletedLatestSecretVersions
}
}).save();
actions.push(deleteAction);
} }
const toUpdate = oldSecrets const toUpdate = oldSecrets
@@ -348,88 +396,119 @@ const v1PushSecrets = async ({
return false; return false;
}); });
const operations = toUpdate if (toUpdate.length > 0) {
.map((s) => { const operations = toUpdate
const { .map((s) => {
secretValueCiphertext, const {
secretValueIV, secretValueCiphertext,
secretValueTag, secretValueIV,
secretValueHash, secretValueTag,
secretCommentCiphertext, secretValueHash,
secretCommentIV, secretCommentCiphertext,
secretCommentTag, secretCommentIV,
secretCommentHash, secretCommentTag,
} = newSecretsObj[`${s.type}-${s.secretKeyHash}`]; secretCommentHash,
} = newSecretsObj[`${s.type}-${s.secretKeyHash}`];
const update: Update = { const update: Update = {
secretValueCiphertext, secretValueCiphertext,
secretValueIV, secretValueIV,
secretValueTag, secretValueTag,
secretValueHash, secretValueHash,
secretCommentCiphertext, secretCommentCiphertext,
secretCommentIV, secretCommentIV,
secretCommentTag, secretCommentTag,
secretCommentHash, secretCommentHash,
}
if (!s.version) {
// case: (legacy) secret was not versioned
update.version = 1;
} else {
update['$inc'] = {
version: 1
} }
}
if (s.type === SECRET_PERSONAL) { if (!s.version) {
// attach user associated with the personal secret // case: (legacy) secret was not versioned
update['user'] = userId; update.version = 1;
} } else {
update['$inc'] = {
return { version: 1
updateOne: { }
filter: {
_id: oldSecretsObj[`${s.type}-${s.secretKeyHash}`]._id
},
update
} }
};
if (s.type === SECRET_PERSONAL) {
// attach user associated with the personal secret
update['user'] = userId;
}
return {
updateOne: {
filter: {
_id: oldSecretsObj[`${s.type}-${s.secretKeyHash}`]._id
},
update
}
};
});
await Secret.bulkWrite(operations as any);
// (EE) add secret versions for updated secrets
await EESecretService.addSecretVersions({
secretVersions: toUpdate.map((s) => {
const {
secretKeyCiphertext,
secretKeyIV,
secretKeyTag,
secretKeyHash,
secretValueCiphertext,
secretValueIV,
secretValueTag,
secretValueHash,
secretCommentCiphertext,
secretCommentIV,
secretCommentTag,
secretCommentHash,
} = newSecretsObj[`${s.type}-${s.secretKeyHash}`];
return ({
secret: s._id,
version: s.version ? s.version + 1 : 1,
isDeleted: false,
secretKeyCiphertext,
secretKeyIV,
secretKeyTag,
secretKeyHash,
secretValueCiphertext,
secretValueIV,
secretValueTag,
secretValueHash
})
})
}); });
await Secret.bulkWrite(operations as any);
// (EE) add secret versions for updated secrets
await EESecretService.addSecretVersions({
secretVersions: toUpdate.map((s) => {
const {
secretKeyCiphertext,
secretKeyIV,
secretKeyTag,
secretKeyHash,
secretValueCiphertext,
secretValueIV,
secretValueTag,
secretValueHash,
secretCommentCiphertext,
secretCommentIV,
secretCommentTag,
secretCommentHash,
} = newSecretsObj[`${s.type}-${s.secretKeyHash}`];
return ({ // add audit log for updated secrets
secret: s._id, const updatedLatestSecretVersions = (await SecretVersion.aggregate([
version: s.version ? s.version + 1 : 1, {
isDeleted: false, $match: { secret: { $in: toUpdate.map((u) => u._id) } }
secretKeyCiphertext, },
secretKeyIV, {
secretKeyTag, $group: {
secretKeyHash, _id: '$secret',
secretValueCiphertext, version: { $max: '$version' }
secretValueIV, }
secretValueTag, },
secretValueHash {
}) $sort: { version: -1 }
}) }
}); ])
.exec())
.map((s) => s._id);
const updateAction = await new Action({
name: ACTION_UPDATE_SECRETS,
user: new Types.ObjectId(userId),
workspace: new Types.ObjectId(workspaceId),
payload: {
secretVersions: updatedLatestSecretVersions
}
}).save();
actions.push(updateAction);
}
// handle adding new secrets // handle adding new secrets
const toAdd = secrets.filter((s) => !(`${s.type}-${s.secretKeyHash}` in oldSecretsObj)); const toAdd = secrets.filter((s) => !(`${s.type}-${s.secretKeyHash}` in oldSecretsObj));
@@ -504,12 +583,51 @@ const v1PushSecrets = async ({
secretValueHash secretValueHash
})) }))
}); });
// add audit log for new secrets
const newLatestSecretVersions = (await SecretVersion.aggregate([
{
$match: { secret: { $in: newSecrets.map((n) => n._id) } }
},
{
$group: {
_id: '$secret',
version: { $max: '$version' }
}
},
{
$sort: { version: -1 }
}
])
.exec())
.map((s) => s._id);
const addAction = await new Action({
name: ACTION_ADD_SECRETS,
user: new Types.ObjectId(userId),
workspace: new Types.ObjectId(workspaceId),
payload: {
secretVersions: newLatestSecretVersions
}
}).save();
actions.push(addAction);
} }
// (EE) take a secret snapshot // (EE) take a secret snapshot
await EESecretService.takeSecretSnapshot({ await EESecretService.takeSecretSnapshot({
workspaceId workspaceId
}) })
if (actions.length > 0) {
await EELogService.createLog({
userId,
workspaceId,
actions,
channel,
ipAddress
});
}
} catch (err) { } catch (err) {
Sentry.setUser(null); Sentry.setUser(null);
Sentry.captureException(err); Sentry.captureException(err);
View File
View File
+1 -4
View File
@@ -14,7 +14,6 @@ import Token, { IToken } from './token';
import User, { IUser } from './user'; import User, { IUser } from './user';
import UserAction, { IUserAction } from './userAction'; import UserAction, { IUserAction } from './userAction';
import Workspace, { IWorkspace } from './workspace'; import Workspace, { IWorkspace } from './workspace';
import Log, { ILog } from './log';
export { export {
BackupPrivateKey, BackupPrivateKey,
@@ -48,7 +47,5 @@ export {
UserAction, UserAction,
IUserAction, IUserAction,
Workspace, Workspace,
IWorkspace, IWorkspace
Log,
ILog
}; };
-102
View File
@@ -1,102 +0,0 @@
import { Schema, model, Types } from 'mongoose';
export interface ILog {
_id: Types.ObjectId;
user?: Types.ObjectId;
workspace: Types.ObjectId;
event: string;
groupId: string;
payload: {
numberofSecrets?: number;
environment?: string;
},
channel: string;
ipAddress?: string;
}
// log group consists of logs (each log is associated with 1 event)
// scenario:
// do we in the future record old and new values for secrets? (when you log update secret,
// do you want to know what the old secret value was changed to?)
// Option 1:
// action 1: pushed secrets (top-level event)
// - log 1 (groupId: ABC): modified 10 secrets (sub-level event)
// ---- array of secret ids that were modified
// - log 2 (groupId: ABC): deleted 5 secrets
// ---- array of secret ids that were deleted
// - log 3 (groupId: ABC): created 10 secrets
// ---- array of secret ids that were created
// action 2: pull secrets
// - log 4 (groupId: DEF): read 20 secrets
// ---- array of secret ids that were read
// Option 2 (many logs):
// action 1: pushed secrets (top-level event)
// - log 1 (groupId: ABC): modified secret abc
// - log 2 (groupId: ABC): modified secret def
// - log 3 (groupId: ABC): modified secret ghi
// - log 4 (groupId: ABC): created secret jkl
// - log 5 (groupId: ABC): created secret mno
// - log 6 (groupId: ABC): deleted secret pqr
// action 2: pull secrets (pulling 100 secrets = 100 logs; 10 times per day, 5 people => 5000 logs)
// - log 7 (groupId: DEF): read secret abc
// - log 8 (groupId: DEF): read secret def
// - log 9 (groupId: DEF): read secret ghi
// - log 10 (groupId: DEF): read secret jkl
// - log 11 (groupId: DEF): read secret mno
// logGroup
// ---- log (query for log groups by person and by secret etc.)
/**
* Action: save secrets
* -
*
*/
const logSchema = new Schema<ILog>(
{
user: {
type: Schema.Types.ObjectId,
ref: 'User'
},
workspace: {
type: Schema.Types.ObjectId,
ref: 'Workspace'
},
event: { // CRUD secrets
type: String,
required: true
},
groupId: {
type: String,
required: true,
},
payload: {
secrets: [{
type: Schema.Types.ObjectId,
ref: 'Secret'
}]
},
channel: {
type: String,
enum: ['web', 'cli', 'auto'],
required: true
},
ipAddress: { // store in bytes?
type: String
}
}, {
timestamps: true
}
);
const Log = model<ILog>('Log', logSchema);
export default Log;
-25
View File
@@ -1,25 +0,0 @@
import { Schema, model, Types } from 'mongoose';
export interface ILogGroup {
workspace: Types.ObjectId,
logs: [Types.ObjectId]
}
const logGroupSchema = new Schema<ILogGroup>(
{
workspace: {
type: Schema.Types.ObjectId,
ref: 'Workspace'
},
logs: [{
type: Schema.Types.ObjectId,
ref: 'Log'
}]
}, {
timestamps: true
}
);
const LogGroup = model<ILogGroup>('LogGroup', logGroupSchema);
export default LogGroup;
-17
View File
@@ -1,17 +0,0 @@
import express from 'express';
const router = express.Router();
import {
requireAuth,
validateRequest
} from '../middleware';
import { logController } from '../controllers';
// TODO: workspaceId validation
router.get(
'/:workspaceId',
requireAuth,
validateRequest,
logController.getLogs
);
export default router;
+1 -3
View File
@@ -15,7 +15,6 @@ import password from './password';
import stripe from './stripe'; import stripe from './stripe';
import integration from './integration'; import integration from './integration';
import integrationAuth from './integrationAuth'; import integrationAuth from './integrationAuth';
import log from './log';
export { export {
signup, signup,
@@ -34,6 +33,5 @@ export {
password, password,
stripe, stripe,
integration, integration,
integrationAuth, integrationAuth
log
}; };
+1
View File
@@ -4,6 +4,7 @@ import { requireAuth, validateRequest } from '../../middleware';
import { body, query } from 'express-validator'; import { body, query } from 'express-validator';
import { userActionController } from '../../controllers/v1'; import { userActionController } from '../../controllers/v1';
// note: [userAction] will be deprecated in /v2 in favor of [action]
router.post( router.post(
'/', '/',
requireAuth, requireAuth,
+9
View File
@@ -0,0 +1,9 @@
const ACTION_ADD_SECRETS = 'addSecrets';
const ACTION_DELETE_SECRETS = 'deleteSecrets';
const ACTION_UPDATE_SECRETS = 'updateSecrets';
export {
ACTION_ADD_SECRETS,
ACTION_DELETE_SECRETS,
ACTION_UPDATE_SECRETS
}
+8
View File
@@ -33,6 +33,11 @@ import {
} from './organization'; } from './organization';
import { SECRET_SHARED, SECRET_PERSONAL } from './secret'; import { SECRET_SHARED, SECRET_PERSONAL } from './secret';
import { EVENT_PUSH_SECRETS, EVENT_PULL_SECRETS } from './event'; import { EVENT_PUSH_SECRETS, EVENT_PULL_SECRETS } from './event';
import {
ACTION_ADD_SECRETS,
ACTION_UPDATE_SECRETS,
ACTION_DELETE_SECRETS
} from './action';
import { SMTP_HOST_SENDGRID, SMTP_HOST_MAILGUN } from './smtp'; import { SMTP_HOST_SENDGRID, SMTP_HOST_MAILGUN } from './smtp';
import { PLAN_STARTER, PLAN_PRO } from './stripe'; import { PLAN_STARTER, PLAN_PRO } from './stripe';
@@ -67,6 +72,9 @@ export {
INTEGRATION_GITHUB_API_URL, INTEGRATION_GITHUB_API_URL,
EVENT_PUSH_SECRETS, EVENT_PUSH_SECRETS,
EVENT_PULL_SECRETS, EVENT_PULL_SECRETS,
ACTION_ADD_SECRETS,
ACTION_UPDATE_SECRETS,
ACTION_DELETE_SECRETS,
INTEGRATION_OPTIONS, INTEGRATION_OPTIONS,
SMTP_HOST_SENDGRID, SMTP_HOST_SENDGRID,
SMTP_HOST_MAILGUN, SMTP_HOST_MAILGUN,