Add v1 audit log backend models and wiring to push secrets

This commit is contained in:
Tuan Dang
2022-12-27 12:12:39 -05:00
parent 019e90dc77
commit 9497a26eb2
26 changed files with 459 additions and 272 deletions

View File

@@ -13,7 +13,8 @@ import { apiLimiter } from './helpers/rateLimiter';
import {
workspace as eeWorkspaceRouter,
secret as eeSecretRouter
secret as eeSecretRouter,
log as eeLogRouter,
} from './ee/routes/v1';
import {
signup as v1SignupRouter,
@@ -69,6 +70,7 @@ if (NODE_ENV === 'production') {
// (EE) routes
app.use('/api/v1/secret', eeSecretRouter);
app.use('/api/v1/workspace', eeWorkspaceRouter);
app.use('/api/v1/log', eeLogRouter);
// v1 routes
app.use('/api/v1/signup', v1SignupRouter);

View File

@@ -1,30 +0,0 @@
import { Request, Response } from 'express';
import * as Sentry from '@sentry/node';
import {
Log
} from '../models';
export const getLogs = async (req: Request, res: Response) => {
// get logs
console.log('getLogs');
let logs;
try {
const { workspaceId } = req.params;
logs = await Log.find({
workspace: workspaceId
});
} catch (err) {
Sentry.setUser({ email: req.user.email });
Sentry.captureException(err);
return res.status(400).send({
message: 'Failed to get audit logs'
});
}
return res.status(200).send({
logs
});
}

View File

@@ -14,7 +14,6 @@ import * as stripeController from './stripeController';
import * as userActionController from './userActionController';
import * as userController from './userController';
import * as workspaceController from './workspaceController';
import * as logController from './logController';
export {
authController,
@@ -32,6 +31,5 @@ export {
stripeController,
userActionController,
userController,
workspaceController,
logController
workspaceController
};

View File

@@ -389,7 +389,9 @@ export const pushWorkspaceSecrets = async (req: Request, res: Response) => {
userId: req.user._id,
workspaceId,
environment,
secrets
secrets,
channel: channel ? channel : 'cli',
ipAddress: req.ip
});
await pushKeys({

View File

@@ -1,6 +1,9 @@
import { Request, Response } from 'express';
import * as Sentry from '@sentry/node';
import { SecretSnapshot } from '../../models';
import {
SecretSnapshot,
Log
} from '../../models';
/**
* Return secret snapshots for workspace with id [workspaceId]
@@ -32,4 +35,33 @@ import { SecretSnapshot } from '../../models';
return res.status(200).send({
secretSnapshots
});
}
export const getWorkspaceLogs = async (req: Request, res: Response) => {
let logs
try {
const { workspaceId } = req.params;
const offset: number = parseInt(req.query.offset as string);
const limit: number = parseInt(req.query.limit as string);
const filters: any = req.query.filters || {};
filters.workspace = workspaceId;
logs = await Log.find(filters)
.skip(offset)
.limit(limit)
.populate('actions');
} catch (err) {
Sentry.setUser({ email: req.user.email });
Sentry.captureException(err);
return res.status(400).send({
message: 'Failed to get workspace logs'
});
}
return res.status(200).send({
logs
});
}

View File

@@ -0,0 +1,40 @@
import * as Sentry from '@sentry/node';
import {
Log,
IAction
} from '../models';
const createLogHelper = async ({
userId,
workspaceId,
actions,
channel,
ipAddress
}: {
userId: string;
workspaceId: string;
actions: IAction[];
channel: string;
ipAddress: string;
}) => {
let log;
try {
log = await new Log({
user: userId,
workspace: workspaceId,
actions,
channel,
ipAddress
}).save();
} catch (err) {
Sentry.setUser(null);
Sentry.captureException(err);
throw new Error('Failed to create log');
}
return log;
}
export {
createLogHelper
}

View File

@@ -0,0 +1,40 @@
import { Schema, model, Types } from 'mongoose';
export interface IAction {
name: string;
user?: Types.ObjectId,
workspace?: Types.ObjectId,
payload: {
secretVersions?: Types.ObjectId[]
}
}
const actionSchema = new Schema<IAction>(
{
name: {
type: String,
required: true
},
user: {
type: Schema.Types.ObjectId,
ref: 'User',
required: true
},
workspace: {
type: Schema.Types.ObjectId,
ref: 'Workspace'
},
payload: {
secretVersions: [{
type: Schema.Types.ObjectId,
ref: 'SecretVersion'
}]
}
}, {
timestamps: true
}
);
const Action = model<IAction>('Action', actionSchema);
export default Action;

View File

@@ -1,9 +1,15 @@
import SecretSnapshot, { ISecretSnapshot } from "./secretSnapshot";
import SecretVersion, { ISecretVersion } from "./secretVersion";
import SecretSnapshot, { ISecretSnapshot } from './secretSnapshot';
import SecretVersion, { ISecretVersion } from './secretVersion';
import Log, { ILog } from './log';
import Action, { IAction } from './action';
export {
SecretSnapshot,
ISecretSnapshot,
SecretVersion,
ISecretVersion
ISecretVersion,
Log,
ILog,
Action,
IAction
}

View File

@@ -0,0 +1,41 @@
import { Schema, model, Types } from 'mongoose';
export interface ILog {
_id: Types.ObjectId;
user?: Types.ObjectId;
workspace?: Types.ObjectId;
actions: Types.ObjectId[];
channel: string;
ipAddress?: string;
}
const logSchema = new Schema<ILog>(
{
user: {
type: Schema.Types.ObjectId,
ref: 'User'
},
workspace: {
type: Schema.Types.ObjectId,
ref: 'Workspace'
},
actions: [{
type: Schema.Types.ObjectId,
ref: 'Action'
}],
channel: {
type: String,
enum: ['web', 'cli', 'auto'],
required: true
},
ipAddress: {
type: String
}
}, {
timestamps: true
}
);
const Log = model<ILog>('Log', logSchema);
export default Log;

View File

@@ -1,7 +1,9 @@
import secret from './secret';
import workspace from './workspace';
import log from './log';
export {
secret,
workspace
workspace,
log
}

View File

@@ -0,0 +1,4 @@
import express from 'express';
const router = express.Router();
export default router;

View File

@@ -23,5 +23,19 @@ router.get(
workspaceController.getWorkspaceSecretSnapshots
);
router.get(
'/:workspaceId/logs',
requireAuth,
requireWorkspaceAuth({
acceptedRoles: [ADMIN, MEMBER],
acceptedStatuses: [GRANTED]
}),
param('workspaceId').exists().trim(),
query('offset').exists().isInt(),
query('limit').exists().isInt(),
query('filters').exists(),
validateRequest,
workspaceController.getWorkspaceLogs
);
export default router;

View File

@@ -0,0 +1,47 @@
import {
Action,
IAction
} from '../models';
import {
createLogHelper
} from '../helpers/log';
import EELicenseService from './EELicenseService';
/**
* Class to handle Enterprise Edition log actions
*/
class EELogService {
/**
* Create an (audit) log
* @param {Object} obj
* @param {String} obj.userId - id of user associated with the log
* @param {String} obj.workspaceId - id of workspace associated with the log
* @param {Action} obj.actions - actions to include in log
* @param {String} obj.channel - channel (web/cli/auto) associated with the log
* @param {String} obj.ipAddress - ip address associated with the log
*/
static async createLog({
userId,
workspaceId,
actions,
channel,
ipAddress
}: {
userId: string;
workspaceId: string;
actions: IAction[];
channel: string;
ipAddress: string;
}) {
if (!EELicenseService.isLicenseValid) return;
return await createLogHelper({
userId,
workspaceId,
actions,
channel,
ipAddress
})
}
}
export default EELogService;

View File

@@ -1,7 +1,9 @@
import EELicenseService from "./EELicenseService";
import EESecretService from "./EESecretService";
import EELogService from "./EELogService";
export {
EELicenseService,
EESecretService
EESecretService,
EELogService
}

View File

@@ -1,19 +1,29 @@
import * as Sentry from '@sentry/node';
import { Types } from 'mongoose';
import {
Secret,
ISecret,
} from '../models';
import {
EESecretService
EESecretService,
EELogService
} from '../ee/services';
import {
SecretVersion
SecretVersion,
Action,
IAction
} from '../ee/models';
import {
takeSecretSnapshotHelper
} from '../ee/helpers/secret';
import { decryptSymmetric } from '../utils/crypto';
import { SECRET_SHARED, SECRET_PERSONAL } from '../variables';
import {
SECRET_SHARED,
SECRET_PERSONAL,
ACTION_ADD_SECRETS,
ACTION_UPDATE_SECRETS,
ACTION_DELETE_SECRETS
} from '../variables';
interface V1PushSecret {
ciphertextKey: string;
@@ -284,20 +294,28 @@ const v1PushSecrets = async ({
* @param {String} obj.workspaceId - id of workspace to push to
* @param {String} obj.environment - environment for secrets
* @param {Object[]} obj.secrets - secrets to push
* @param {String} obj.channel - channel (web/cli/auto)
* @param {String} obj.ipAddress - ip address of request to push secrets
*/
const v2PushSecrets = async ({
userId,
workspaceId,
environment,
secrets
secrets,
channel,
ipAddress
}: {
userId: string;
workspaceId: string;
environment: string;
secrets: V2PushSecret[];
channel: string;
ipAddress: string;
}): Promise<void> => {
// TODO: clean up function and fix up types
try {
const actions: IAction[] = [];
// construct useful data structures
const oldSecrets = await pullSecrets({
userId,
@@ -327,7 +345,37 @@ const v1PushSecrets = async ({
secret: { $in: toDelete }
}, {
isDeleted: true
}, {
new: true
});
// add audit log for deleted secrets
const deletedLatestSecretVersions = (await SecretVersion.aggregate([
{
$match: { secret: { $in: toDelete } }
},
{
$group: {
_id: '$secret',
version: { $max: '$version' }
}
},
{
$sort: { version: -1 }
}
])
.exec())
.map((s) => s._id);
const deleteAction = await new Action({
name: ACTION_DELETE_SECRETS,
user: new Types.ObjectId(userId),
workspace: new Types.ObjectId(workspaceId),
payload: {
secretVersions: deletedLatestSecretVersions
}
}).save();
actions.push(deleteAction);
}
const toUpdate = oldSecrets
@@ -348,88 +396,119 @@ const v1PushSecrets = async ({
return false;
});
const operations = toUpdate
.map((s) => {
const {
secretValueCiphertext,
secretValueIV,
secretValueTag,
secretValueHash,
secretCommentCiphertext,
secretCommentIV,
secretCommentTag,
secretCommentHash,
} = newSecretsObj[`${s.type}-${s.secretKeyHash}`];
if (toUpdate.length > 0) {
const operations = toUpdate
.map((s) => {
const {
secretValueCiphertext,
secretValueIV,
secretValueTag,
secretValueHash,
secretCommentCiphertext,
secretCommentIV,
secretCommentTag,
secretCommentHash,
} = newSecretsObj[`${s.type}-${s.secretKeyHash}`];
const update: Update = {
secretValueCiphertext,
secretValueIV,
secretValueTag,
secretValueHash,
secretCommentCiphertext,
secretCommentIV,
secretCommentTag,
secretCommentHash,
}
if (!s.version) {
// case: (legacy) secret was not versioned
update.version = 1;
} else {
update['$inc'] = {
version: 1
const update: Update = {
secretValueCiphertext,
secretValueIV,
secretValueTag,
secretValueHash,
secretCommentCiphertext,
secretCommentIV,
secretCommentTag,
secretCommentHash,
}
}
if (s.type === SECRET_PERSONAL) {
// attach user associated with the personal secret
update['user'] = userId;
}
return {
updateOne: {
filter: {
_id: oldSecretsObj[`${s.type}-${s.secretKeyHash}`]._id
},
update
if (!s.version) {
// case: (legacy) secret was not versioned
update.version = 1;
} else {
update['$inc'] = {
version: 1
}
}
};
if (s.type === SECRET_PERSONAL) {
// attach user associated with the personal secret
update['user'] = userId;
}
return {
updateOne: {
filter: {
_id: oldSecretsObj[`${s.type}-${s.secretKeyHash}`]._id
},
update
}
};
});
await Secret.bulkWrite(operations as any);
// (EE) add secret versions for updated secrets
await EESecretService.addSecretVersions({
secretVersions: toUpdate.map((s) => {
const {
secretKeyCiphertext,
secretKeyIV,
secretKeyTag,
secretKeyHash,
secretValueCiphertext,
secretValueIV,
secretValueTag,
secretValueHash,
secretCommentCiphertext,
secretCommentIV,
secretCommentTag,
secretCommentHash,
} = newSecretsObj[`${s.type}-${s.secretKeyHash}`];
return ({
secret: s._id,
version: s.version ? s.version + 1 : 1,
isDeleted: false,
secretKeyCiphertext,
secretKeyIV,
secretKeyTag,
secretKeyHash,
secretValueCiphertext,
secretValueIV,
secretValueTag,
secretValueHash
})
})
});
await Secret.bulkWrite(operations as any);
// (EE) add secret versions for updated secrets
await EESecretService.addSecretVersions({
secretVersions: toUpdate.map((s) => {
const {
secretKeyCiphertext,
secretKeyIV,
secretKeyTag,
secretKeyHash,
secretValueCiphertext,
secretValueIV,
secretValueTag,
secretValueHash,
secretCommentCiphertext,
secretCommentIV,
secretCommentTag,
secretCommentHash,
} = newSecretsObj[`${s.type}-${s.secretKeyHash}`];
return ({
secret: s._id,
version: s.version ? s.version + 1 : 1,
isDeleted: false,
secretKeyCiphertext,
secretKeyIV,
secretKeyTag,
secretKeyHash,
secretValueCiphertext,
secretValueIV,
secretValueTag,
secretValueHash
})
})
});
// add audit log for updated secrets
const updatedLatestSecretVersions = (await SecretVersion.aggregate([
{
$match: { secret: { $in: toUpdate.map((u) => u._id) } }
},
{
$group: {
_id: '$secret',
version: { $max: '$version' }
}
},
{
$sort: { version: -1 }
}
])
.exec())
.map((s) => s._id);
const updateAction = await new Action({
name: ACTION_UPDATE_SECRETS,
user: new Types.ObjectId(userId),
workspace: new Types.ObjectId(workspaceId),
payload: {
secretVersions: updatedLatestSecretVersions
}
}).save();
actions.push(updateAction);
}
// handle adding new secrets
const toAdd = secrets.filter((s) => !(`${s.type}-${s.secretKeyHash}` in oldSecretsObj));
@@ -504,12 +583,51 @@ const v1PushSecrets = async ({
secretValueHash
}))
});
// add audit log for new secrets
const newLatestSecretVersions = (await SecretVersion.aggregate([
{
$match: { secret: { $in: newSecrets.map((n) => n._id) } }
},
{
$group: {
_id: '$secret',
version: { $max: '$version' }
}
},
{
$sort: { version: -1 }
}
])
.exec())
.map((s) => s._id);
const addAction = await new Action({
name: ACTION_ADD_SECRETS,
user: new Types.ObjectId(userId),
workspace: new Types.ObjectId(workspaceId),
payload: {
secretVersions: newLatestSecretVersions
}
}).save();
actions.push(addAction);
}
// (EE) take a secret snapshot
await EESecretService.takeSecretSnapshot({
workspaceId
})
if (actions.length > 0) {
await EELogService.createLog({
userId,
workspaceId,
actions,
channel,
ipAddress
});
}
} catch (err) {
Sentry.setUser(null);
Sentry.captureException(err);

View File

@@ -14,7 +14,6 @@ import Token, { IToken } from './token';
import User, { IUser } from './user';
import UserAction, { IUserAction } from './userAction';
import Workspace, { IWorkspace } from './workspace';
import Log, { ILog } from './log';
export {
BackupPrivateKey,
@@ -48,7 +47,5 @@ export {
UserAction,
IUserAction,
Workspace,
IWorkspace,
Log,
ILog
IWorkspace
};

View File

@@ -1,102 +0,0 @@
import { Schema, model, Types } from 'mongoose';
export interface ILog {
_id: Types.ObjectId;
user?: Types.ObjectId;
workspace: Types.ObjectId;
event: string;
groupId: string;
payload: {
numberofSecrets?: number;
environment?: string;
},
channel: string;
ipAddress?: string;
}
// log group consists of logs (each log is associated with 1 event)
// scenario:
// do we in the future record old and new values for secrets? (when you log update secret,
// do you want to know what the old secret value was changed to?)
// Option 1:
// action 1: pushed secrets (top-level event)
// - log 1 (groupId: ABC): modified 10 secrets (sub-level event)
// ---- array of secret ids that were modified
// - log 2 (groupId: ABC): deleted 5 secrets
// ---- array of secret ids that were deleted
// - log 3 (groupId: ABC): created 10 secrets
// ---- array of secret ids that were created
// action 2: pull secrets
// - log 4 (groupId: DEF): read 20 secrets
// ---- array of secret ids that were read
// Option 2 (many logs):
// action 1: pushed secrets (top-level event)
// - log 1 (groupId: ABC): modified secret abc
// - log 2 (groupId: ABC): modified secret def
// - log 3 (groupId: ABC): modified secret ghi
// - log 4 (groupId: ABC): created secret jkl
// - log 5 (groupId: ABC): created secret mno
// - log 6 (groupId: ABC): deleted secret pqr
// action 2: pull secrets (pulling 100 secrets = 100 logs; 10 times per day, 5 people => 5000 logs)
// - log 7 (groupId: DEF): read secret abc
// - log 8 (groupId: DEF): read secret def
// - log 9 (groupId: DEF): read secret ghi
// - log 10 (groupId: DEF): read secret jkl
// - log 11 (groupId: DEF): read secret mno
// logGroup
// ---- log (query for log groups by person and by secret etc.)
/**
* Action: save secrets
* -
*
*/
const logSchema = new Schema<ILog>(
{
user: {
type: Schema.Types.ObjectId,
ref: 'User'
},
workspace: {
type: Schema.Types.ObjectId,
ref: 'Workspace'
},
event: { // CRUD secrets
type: String,
required: true
},
groupId: {
type: String,
required: true,
},
payload: {
secrets: [{
type: Schema.Types.ObjectId,
ref: 'Secret'
}]
},
channel: {
type: String,
enum: ['web', 'cli', 'auto'],
required: true
},
ipAddress: { // store in bytes?
type: String
}
}, {
timestamps: true
}
);
const Log = model<ILog>('Log', logSchema);
export default Log;

View File

@@ -1,25 +0,0 @@
import { Schema, model, Types } from 'mongoose';
export interface ILogGroup {
workspace: Types.ObjectId,
logs: [Types.ObjectId]
}
const logGroupSchema = new Schema<ILogGroup>(
{
workspace: {
type: Schema.Types.ObjectId,
ref: 'Workspace'
},
logs: [{
type: Schema.Types.ObjectId,
ref: 'Log'
}]
}, {
timestamps: true
}
);
const LogGroup = model<ILogGroup>('LogGroup', logGroupSchema);
export default LogGroup;

View File

@@ -1,17 +0,0 @@
import express from 'express';
const router = express.Router();
import {
requireAuth,
validateRequest
} from '../middleware';
import { logController } from '../controllers';
// TODO: workspaceId validation
router.get(
'/:workspaceId',
requireAuth,
validateRequest,
logController.getLogs
);
export default router;

View File

@@ -15,7 +15,6 @@ import password from './password';
import stripe from './stripe';
import integration from './integration';
import integrationAuth from './integrationAuth';
import log from './log';
export {
signup,
@@ -34,6 +33,5 @@ export {
password,
stripe,
integration,
integrationAuth,
log
integrationAuth
};

View File

@@ -4,6 +4,7 @@ import { requireAuth, validateRequest } from '../../middleware';
import { body, query } from 'express-validator';
import { userActionController } from '../../controllers/v1';
// note: [userAction] will be deprecated in /v2 in favor of [action]
router.post(
'/',
requireAuth,

View File

@@ -0,0 +1,9 @@
const ACTION_ADD_SECRETS = 'addSecrets';
const ACTION_DELETE_SECRETS = 'deleteSecrets';
const ACTION_UPDATE_SECRETS = 'updateSecrets';
export {
ACTION_ADD_SECRETS,
ACTION_DELETE_SECRETS,
ACTION_UPDATE_SECRETS
}

View File

@@ -33,6 +33,11 @@ import {
} from './organization';
import { SECRET_SHARED, SECRET_PERSONAL } from './secret';
import { EVENT_PUSH_SECRETS, EVENT_PULL_SECRETS } from './event';
import {
ACTION_ADD_SECRETS,
ACTION_UPDATE_SECRETS,
ACTION_DELETE_SECRETS
} from './action';
import { SMTP_HOST_SENDGRID, SMTP_HOST_MAILGUN } from './smtp';
import { PLAN_STARTER, PLAN_PRO } from './stripe';
@@ -67,6 +72,9 @@ export {
INTEGRATION_GITHUB_API_URL,
EVENT_PUSH_SECRETS,
EVENT_PULL_SECRETS,
ACTION_ADD_SECRETS,
ACTION_UPDATE_SECRETS,
ACTION_DELETE_SECRETS,
INTEGRATION_OPTIONS,
SMTP_HOST_SENDGRID,
SMTP_HOST_MAILGUN,