Finish v1 audit logs, secret versioning, version all unversioned secrets

This commit is contained in:
Tuan Dang
2023-01-02 14:18:49 +07:00
parent 4576e8f6a7
commit a8f0c391bc
19 changed files with 401 additions and 195 deletions

View File

@@ -13,7 +13,8 @@ import { apiLimiter } from './helpers/rateLimiter';
import {
workspace as eeWorkspaceRouter,
secret as eeSecretRouter
secret as eeSecretRouter,
action as eeActionRouter
} from './ee/routes/v1';
import {
signup as v1SignupRouter,
@@ -69,6 +70,7 @@ if (NODE_ENV === 'production') {
// (EE) routes
app.use('/api/v1/secret', eeSecretRouter);
app.use('/api/v1/workspace', eeWorkspaceRouter);
app.use('/api/v1/action', eeActionRouter);
// v1 routes
app.use('/api/v1/signup', v1SignupRouter);

View File

@@ -369,7 +369,6 @@ export const getWorkspaceServiceTokens = async (
*/
export const pushWorkspaceSecrets = async (req: Request, res: Response) => {
// upload (encrypted) secrets to workspace with id [workspaceId]
try {
let { secrets }: { secrets: V2PushSecret[] } = req.body;
const { keys, environment, channel } = req.body;
@@ -400,7 +399,6 @@ export const pushWorkspaceSecrets = async (req: Request, res: Response) => {
keys
});
if (postHogClient) {
postHogClient.capture({
event: 'secrets pushed',

View File

@@ -1,18 +1,29 @@
import { Request, Response } from 'express';
import * as Sentry from '@sentry/node';
import { Action } from '../../models';
import { Action, SecretVersion } from '../../models';
import { ActionNotFoundError } from '../../../utils/errors';
export const getAction = (req: Request, res: Response) => {
export const getAction = async (req: Request, res: Response) => {
let action;
// try {
// const { actionId } = req.params;
try {
const { actionId } = req.params;
// action = await Action.findById(actionId);
action = await Action
.findById(actionId)
.populate([
'payload.secretVersions.oldSecretVersion',
'payload.secretVersions.newSecretVersion'
]);
// } catch (err) {
if (!action) throw ActionNotFoundError({
message: 'Failed to find action'
});
// }
} catch (err) {
throw ActionNotFoundError({
message: 'Failed to find action'
});
}
return res.status(200).send({
action

View File

@@ -41,14 +41,23 @@ export const getWorkspaceLogs = async (req: Request, res: Response) => {
let logs
try {
const { workspaceId } = req.params;
const { userId, actionNames } = req.query;
const offset: number = parseInt(req.query.offset as string);
const limit: number = parseInt(req.query.limit as string);
const filters: any = req.query.filters || {};
filters.workspace = workspaceId;
logs = await Log.find(filters)
logs = await Log.find({
workspace: workspaceId,
...( userId ? { user: userId } : {}),
...(
actionNames
? {
actionNames: {
$in: actionNames
}
} : {}
)
})
.skip(offset)
.limit(limit)
.populate('actions')

View File

@@ -0,0 +1,112 @@
import * as Sentry from '@sentry/node';
import { Types } from 'mongoose';
import { Secret } from '../../models';
import { SecretVersion, Action } from '../models';
import { ACTION_UPDATE_SECRETS } from '../../variables';
/**
* Create an (audit) action for secrets including
* add, delete, update, and read actions.
* @param {Object} obj
* @param {String} obj.name - name of action
* @param {ObjectId[]} obj.secretIds - ids of relevant secrets
* @returns {Action} action - new action
*/
const createActionSecretHelper = async ({
name,
userId,
workspaceId,
secretIds
}: {
name: string;
userId: string;
workspaceId: string;
secretIds: Types.ObjectId[];
}) => {
let action;
let latestSecretVersions;
try {
if (name === ACTION_UPDATE_SECRETS) {
// case: action is updating secrets
// -> add old and new secret versions
// TODO: make query more efficient
latestSecretVersions = (await SecretVersion.aggregate([
{
$match: {
secret: {
$in: secretIds,
},
},
},
{
$sort: { version: -1 },
},
{
$group: {
_id: "$secret",
versions: { $push: "$$ROOT" },
},
},
{
$project: {
_id: 0,
secret: "$_id",
versions: { $slice: ["$versions", 2] },
},
}
]))
.map((s) => ({
oldSecretVersion: s.versions[0]._id,
newSecretVersion: s.versions[1]._id
}));
} else {
// case: action is adding, deleting, or reading secrets
// -> add new secret versions
latestSecretVersions = (await SecretVersion.aggregate([
{
$match: {
secret: {
$in: secretIds
}
}
},
{
$group: {
_id: '$secret',
version: { $max: '$version' },
versionId: { $max: '$_id' } // secret version id
}
},
{
$sort: { version: -1 }
}
])
.exec())
.map((s) => ({
newSecretVersion: s.versionId
}));
}
action = await new Action({
name,
user: userId,
workspace: workspaceId,
payload: {
secretVersions: latestSecretVersions
}
}).save();
} catch (err) {
Sentry.setUser(null);
Sentry.captureException(err);
throw new Error('Failed to create action');
}
return action;
}
export { createActionSecretHelper };

View File

@@ -22,6 +22,7 @@ const createLogHelper = async ({
log = await new Log({
user: userId,
workspace: workspaceId,
actionNames: actions.map((a) => a.name),
actions,
channel,
ipAddress

View File

@@ -1,3 +1,4 @@
import { Types } from 'mongoose';
import * as Sentry from '@sentry/node';
import {
Secret
@@ -59,16 +60,40 @@ const addSecretVersionsHelper = async ({
}: {
secretVersions: ISecretVersion[]
}) => {
let newSecretVersions;
try {
await SecretVersion.insertMany(secretVersions);
newSecretVersions = await SecretVersion.insertMany(secretVersions);
} catch (err) {
Sentry.setUser(null);
Sentry.captureException(err);
throw new Error('Failed to add secret versions');
}
return newSecretVersions;
}
const markDeletedSecretVersionsHelper = async ({
secretIds
}: {
secretIds: Types.ObjectId[];
}) => {
try {
await SecretVersion.updateMany({
secret: { $in: secretIds }
}, {
isDeleted: true
}, {
new: true
});
} catch (err) {
Sentry.setUser(null);
Sentry.captureException(err);
throw new Error('Failed to mark secret versions as deleted');
}
}
export {
takeSecretSnapshotHelper,
addSecretVersionsHelper
addSecretVersionsHelper,
markDeletedSecretVersionsHelper
}

View File

@@ -26,8 +26,14 @@ const actionSchema = new Schema<IAction>(
},
payload: {
secretVersions: [{
type: Schema.Types.ObjectId,
ref: 'SecretVersion'
oldSecretVersion: {
type: Schema.Types.ObjectId,
ref: 'SecretVersion'
},
newSecretVersion: {
type: Schema.Types.ObjectId,
ref: 'SecretVersion'
}
}]
}
}, {

View File

@@ -1,9 +1,16 @@
import { Schema, model, Types } from 'mongoose';
import {
ACTION_ADD_SECRETS,
ACTION_UPDATE_SECRETS,
ACTION_READ_SECRETS,
ACTION_DELETE_SECRETS
} from '../../variables';
export interface ILog {
_id: Types.ObjectId;
user?: Types.ObjectId;
workspace?: Types.ObjectId;
actionNames: string[];
actions: Types.ObjectId[];
channel: string;
ipAddress?: string;
@@ -19,9 +26,20 @@ const logSchema = new Schema<ILog>(
type: Schema.Types.ObjectId,
ref: 'Workspace'
},
actionNames: {
type: [String],
enum: [
ACTION_ADD_SECRETS,
ACTION_UPDATE_SECRETS,
ACTION_READ_SECRETS,
ACTION_DELETE_SECRETS
],
required: true
},
actions: [{
type: Schema.Types.ObjectId,
ref: 'Action'
ref: 'Action',
required: true
}],
channel: {
type: String,

View File

@@ -33,7 +33,6 @@ router.get(
param('workspaceId').exists().trim(),
query('offset').exists().isInt(),
query('limit').exists().isInt(),
query('filters').exists(),
validateRequest,
workspaceController.getWorkspaceLogs
);

View File

@@ -1,10 +1,15 @@
import { Types } from 'mongoose';
import {
Log,
Action,
IAction
} from '../models';
import {
createLogHelper
} from '../helpers/log';
import {
createActionSecretHelper
} from '../helpers/action';
import EELicenseService from './EELicenseService';
/**
@@ -19,6 +24,7 @@ class EELogService {
* @param {Action} obj.actions - actions to include in log
* @param {String} obj.channel - channel (web/cli/auto) associated with the log
* @param {String} obj.ipAddress - ip address associated with the log
* @returns {Log} log - new audit log
*/
static async createLog({
userId,
@@ -33,7 +39,7 @@ class EELogService {
channel: string;
ipAddress: string;
}) {
if (!EELicenseService.isLicenseValid) return;
if (!EELicenseService.isLicenseValid) return null;
return await createLogHelper({
userId,
workspaceId,
@@ -42,6 +48,34 @@ class EELogService {
ipAddress
})
}
/**
* Create an (audit) action for secrets including
* add, delete, update, and read actions.
* @param {Object} obj
* @param {String} obj.name - name of action
* @param {ObjectId[]} obj.secretIds - secret ids
* @returns {Action} action - new action
*/
static async createActionSecret({
name,
userId,
workspaceId,
secretIds
}: {
name: string;
userId: string;
workspaceId: string;
secretIds: Types.ObjectId[];
}) {
if (!EELicenseService.isLicenseValid) return null;
return await createActionSecretHelper({
name,
userId,
workspaceId,
secretIds
});
}
}
export default EELogService;

View File

@@ -1,7 +1,9 @@
import { Types } from 'mongoose';
import { ISecretVersion } from '../models';
import {
takeSecretSnapshotHelper,
addSecretVersionsHelper
addSecretVersionsHelper,
markDeletedSecretVersionsHelper
} from '../helpers/secret';
import EELicenseService from './EELicenseService';
@@ -28,7 +30,7 @@ class EESecretService {
}
/**
* Adds secret versions [secretVersions] to the SecretVersion collection.
* Add secret versions [secretVersions] to the SecretVersion collection.
* @param {Object} obj
* @param {SecretVersion} obj.secretVersions
*/
@@ -38,10 +40,27 @@ class EESecretService {
secretVersions: ISecretVersion[];
}) {
if (!EELicenseService.isLicenseValid) return;
await addSecretVersionsHelper({
return await addSecretVersionsHelper({
secretVersions
});
}
/**
* Mark secret versions associated with secrets with ids [secretIds]
* as deleted.
* @param {Object} obj
* @param {ObjectId[]} obj.secretIds - secret ids
*/
static async markDeletedSecretVersions({
secretIds
}: {
secretIds: Types.ObjectId[];
}) {
if (!EELicenseService.isLicenseValid) return;
await markDeletedSecretVersionsHelper({
secretIds
});
}
}
export default EESecretService;

View File

@@ -0,0 +1,78 @@
import mongoose from 'mongoose';
import { ISecret, Secret } from '../models';
import { EESecretService } from '../ee/services';
import { getLogger } from '../utils/logger';
/**
* Initialize database connection
* @param {Object} obj
* @param {String} obj.mongoURL - mongo connection string
* @returns
*/
const initDatabaseHelper = async ({
mongoURL
}: {
mongoURL: string;
}) => {
try {
await mongoose.connect(mongoURL);
getLogger("database").info("Database connection established");
await prepareDatabase();
} catch (err) {
getLogger("database").error(`Unable to establish Database connection due to the error.\n${err}`);
}
return mongoose.connection;
}
/**
* Prepare database by:
* - Setting unversioned secrets to version 1
* - Initializing secret versions for unversioned secrets
*/
const prepareDatabase = async () => {
try {
// set previously unversioned secrets in Secret to version 1
await Secret.updateMany(
{ version: { $exists: false } },
{ $set: { version: 1 } }
);
// initialize secret versions for unversioned secrets
const unversionedSecrets: ISecret[] = await Secret.aggregate([
{
$lookup: {
from: 'secretversions',
localField: '_id',
foreignField: 'secret',
as: 'versions',
},
},
{
$match: {
versions: { $size: 0 },
},
},
]);
if (unversionedSecrets.length > 0) {
await EESecretService.addSecretVersions({
secretVersions: unversionedSecrets.map((s, idx) => ({
...s,
secret: s._id,
version: s.version ? s.version : 1,
isDeleted: false,
workspace: s.workspace,
environment: s.environment
}))
});
}
} catch (err) {
getLogger('database').error('Failed to prepare database');
}
}
export {
initDatabaseHelper
}

View File

@@ -285,6 +285,9 @@ const v1PushSecrets = async ({
}
};
// TODO: optimize this route.
// TODO: ensure that it's possible to query for and filter logs
/**
* Push secrets for user with id [userId] to workspace
* with id [workspaceId] with environment [environment]. Follow steps:
@@ -341,42 +344,19 @@ const v1PushSecrets = async ({
await Secret.deleteMany({
_id: { $in: toDelete }
});
await EESecretService.markDeletedSecretVersions({
secretIds: toDelete
});
await SecretVersion.updateMany({
secret: { $in: toDelete }
}, {
isDeleted: true
}, {
new: true
const deleteAction = await EELogService.createActionSecret({
name: ACTION_DELETE_SECRETS,
userId,
workspaceId,
secretIds: toDelete
});
// add audit log for deleted secrets
const deletedLatestSecretVersions = (await SecretVersion.aggregate([
{
$match: { secret: { $in: toDelete } }
},
{
$group: {
_id: '$secret',
version: { $max: '$version' }
}
},
{
$sort: { version: -1 }
}
])
.exec())
.map((s) => s._id);
const deleteAction = await new Action({
name: ACTION_DELETE_SECRETS,
user: new Types.ObjectId(userId),
workspace: new Types.ObjectId(workspaceId),
payload: {
secretVersions: deletedLatestSecretVersions
}
}).save();
actions.push(deleteAction);
deleteAction && actions.push(deleteAction);
}
const toUpdate = oldSecrets
@@ -459,10 +439,6 @@ const v1PushSecrets = async ({
secretValueIV,
secretValueTag,
secretValueHash,
secretCommentCiphertext,
secretCommentIV,
secretCommentTag,
secretCommentHash,
} = newSecretsObj[`${s.type}-${s.secretKeyHash}`];
return ({
@@ -481,34 +457,14 @@ const v1PushSecrets = async ({
})
});
// add audit log for updated secrets
const updatedLatestSecretVersions = (await SecretVersion.aggregate([
{
$match: { secret: { $in: toUpdate.map((u) => u._id) } }
},
{
$group: {
_id: '$secret',
version: { $max: '$version' }
}
},
{
$sort: { version: -1 }
}
])
.exec())
.map((s) => s._id);
const updateAction = await new Action({
const updateAction = await EELogService.createActionSecret({
name: ACTION_UPDATE_SECRETS,
user: new Types.ObjectId(userId),
workspace: new Types.ObjectId(workspaceId),
payload: {
secretVersions: updatedLatestSecretVersions
}
}).save();
userId,
workspaceId,
secretIds: toUpdate.map((u) => u._id)
});
actions.push(updateAction);
updateAction && actions.push(updateAction);
}
// handle adding new secrets
@@ -517,45 +473,14 @@ const v1PushSecrets = async ({
if (toAdd.length > 0) {
// add secrets
const newSecrets = await Secret.insertMany(
toAdd.map(({
secretKeyCiphertext,
secretKeyIV,
secretKeyTag,
secretKeyHash,
secretValueCiphertext,
secretValueIV,
secretValueTag,
secretValueHash,
secretCommentCiphertext,
secretCommentIV,
secretCommentTag,
secretCommentHash,
}, idx) => {
const obj: any = {
version: 1,
workspace: workspaceId,
type: toAdd[idx].type,
environment,
secretKeyCiphertext,
secretKeyIV,
secretKeyTag,
secretKeyHash,
secretValueCiphertext,
secretValueIV,
secretValueTag,
secretValueHash,
secretCommentCiphertext,
secretCommentIV,
secretCommentTag,
secretCommentHash
};
if (toAdd[idx].type === 'personal') {
obj['user' as keyof typeof obj] = userId;
}
return obj;
})
toAdd.map((s, idx) => ({
...s,
version: 1,
workspace: workspaceId,
type: toAdd[idx].type,
environment,
...( toAdd[idx].type === 'personal' ? { user: userId } : {})
}))
);
// (EE) add secret versions for new secrets
@@ -584,35 +509,14 @@ const v1PushSecrets = async ({
secretValueHash
}))
});
// add audit log for new secrets
const newLatestSecretVersions = (await SecretVersion.aggregate([
{
$match: { secret: { $in: newSecrets.map((n) => n._id) } }
},
{
$group: {
_id: '$secret',
version: { $max: '$version' }
}
},
{
$sort: { version: -1 }
}
])
.exec())
.map((s) => s._id);
const addAction = await new Action({
const addAction = await EELogService.createActionSecret({
name: ACTION_ADD_SECRETS,
user: new Types.ObjectId(userId),
workspace: new Types.ObjectId(workspaceId),
payload: {
secretVersions: newLatestSecretVersions
}
}).save();
actions.push(addAction);
userId,
workspaceId,
secretIds: newSecrets.map((n) => n._id)
});
addAction && actions.push(addAction);
}
// (EE) take a secret snapshot
@@ -620,6 +524,7 @@ const v1PushSecrets = async ({
workspaceId
})
// (EE) create (audit) log
if (actions.length > 0) {
await EELogService.createLog({
userId,
@@ -637,7 +542,7 @@ const v1PushSecrets = async ({
};
/**
* Pull secrets for user with id [userId] for workspace
* Get secrets for user with id [userId] for workspace
* with id [workspaceId] with environment [environment]
* @param {Object} obj
* @param {String} obj.userId -id of user to pull secrets for
@@ -704,7 +609,7 @@ const pullSecrets = async ({
channel: string;
ipAddress: string;
}): Promise<ISecret[]> => {
let secrets: any; // TODO: FIX any
let secrets: any;
try {
secrets = await getSecrets({
@@ -712,35 +617,15 @@ const pullSecrets = async ({
workspaceId,
environment
})
// add audit log for new secrets
const readLatestSecretVersions = (await SecretVersion.aggregate([
{
$match: { secret: { $in: secrets.map((n: any) => n._id) } }
},
{
$group: {
_id: '$secret',
version: { $max: '$version' }
}
},
{
$sort: { version: -1 }
}
])
.exec())
.map((s) => s._id);
const readAction = await new Action({
const readAction = await EELogService.createActionSecret({
name: ACTION_READ_SECRETS,
user: new Types.ObjectId(userId),
workspace: new Types.ObjectId(workspaceId),
payload: {
secretVersions: readLatestSecretVersions
}
}).save();
userId,
workspaceId,
secretIds: secrets.map((n: any) => n._id)
});
await EELogService.createLog({
readAction && await EELogService.createLog({
userId,
workspaceId,
actions: [readAction],

View File

@@ -4,12 +4,12 @@ dotenv.config();
import * as Sentry from '@sentry/node';
import { SENTRY_DSN, NODE_ENV, MONGO_URL } from './config';
import { server } from './app';
import { initDatabase } from './services/database';
import { DatabaseService } from './services';
import { setUpHealthEndpoint } from './services/health';
import { initSmtp } from './services/smtp';
import { setTransporter } from './helpers/nodemailer';
initDatabase(MONGO_URL);
DatabaseService.initDatabase(MONGO_URL);
setUpHealthEndpoint(server);

View File

@@ -52,7 +52,8 @@ const userSchema = new Schema<IUser>(
},
refreshVersion: {
type: Number,
default: 0
default: 0,
select: false
}
},
{

View File

@@ -0,0 +1,16 @@
import mongoose from 'mongoose';
import { getLogger } from '../utils/logger';
import { initDatabaseHelper } from '../helpers/database';
/**
* Class to handle database actions
*/
class DatabaseService {
static async initDatabase(MONGO_URL: string) {
return await initDatabaseHelper({
mongoURL: MONGO_URL
});
}
}
export default DatabaseService;

View File

@@ -1,10 +0,0 @@
import mongoose from 'mongoose';
import { getLogger } from '../utils/logger';
export const initDatabase = (MONGO_URL: string) => {
mongoose
.connect(MONGO_URL)
.then(() => getLogger("database").info("Database connection established"))
.catch((e) => getLogger("database").error(`Unable to establish Database connection due to the error.\n${e}`));
return mongoose.connection;
};

View File

@@ -1,9 +1,11 @@
import DatabaseService from './DatabaseService';
import postHogClient from './PostHogClient';
import BotService from './BotService';
import EventService from './EventService';
import IntegrationService from './IntegrationService';
export {
DatabaseService,
postHogClient,
BotService,
EventService,