awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-10-09 21:28:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(folders): added support for snapshot by env and folder

Browse Source
This commit is contained in:
akhilmhdh
2023-06-04 13:18:05 +05:30
parent 08a4404fed
commit deaa85cbe7
32 changed files with 4059 additions and 3290 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/package-lock.json
Generated
+23 -15
View File
@@ -29,7 +29,6 @@
"crypto-js": "^4.1.1",
"dotenv": "^16.0.1",
"express": "^4.18.1",
"express-async-errors": "^3.1.1",
"express-rate-limit": "^6.7.0",
"express-validator": "^6.14.2",
"handlebars": "^4.7.7",
@@ -42,6 +41,7 @@
"lodash": "^4.17.21",
"mongoose": "^6.10.5",
"node-cache": "^5.1.2",
"nanoid": "^3.3.6",
"nodemailer": "^6.8.0",
"passport": "^0.6.0",
"passport-google-oauth20": "^2.0.0",
@@ -5146,14 +5146,6 @@
"node": ">= 0.10.0"
}
},
"node_modules/express-async-errors": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/express-async-errors/-/express-async-errors-3.1.1.tgz",
"integrity": "sha512-h6aK1da4tpqWSbyCa3FxB/V6Ehd4EEB15zyQq9qe75OZBp0krinNKuH4rAY+S/U/2I36vdLAUFSjQJ+TFmODng==",
"peerDependencies": {
"express": "^4.16.2"
}
},
"node_modules/express-rate-limit": {
"version": "6.7.0",
"resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-6.7.0.tgz",
@@ -7388,6 +7380,23 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"node_modules/nanoid": {
"version": "3.3.6",
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz",
"integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/ai"
}
],
"bin": {
"nanoid": "bin/nanoid.cjs"
},
"engines": {
"node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1"
}
},
"node_modules/natural-compare": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz",
@@ -16230,12 +16239,6 @@
}
}
},
"express-async-errors": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/express-async-errors/-/express-async-errors-3.1.1.tgz",
"integrity": "sha512-h6aK1da4tpqWSbyCa3FxB/V6Ehd4EEB15zyQq9qe75OZBp0krinNKuH4rAY+S/U/2I36vdLAUFSjQJ+TFmODng==",
"requires": {}
},
"express-rate-limit": {
"version": "6.7.0",
"resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-6.7.0.tgz",
@@ -17909,6 +17912,11 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"nanoid": {
"version": "3.3.6",
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz",
"integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA=="
},
"natural-compare": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz",
backend/package.json
+1 -1
View File
@@ -20,7 +20,6 @@
"crypto-js": "^4.1.1",
"dotenv": "^16.0.1",
"express": "^4.18.1",
"express-async-errors": "^3.1.1",
"express-rate-limit": "^6.7.0",
"express-validator": "^6.14.2",
"handlebars": "^4.7.7",
@@ -33,6 +32,7 @@
"lodash": "^4.17.21",
"mongoose": "^6.10.5",
"node-cache": "^5.1.2",
"nanoid": "^3.3.6",
"nodemailer": "^6.8.0",
"passport": "^0.6.0",
"passport-google-oauth20": "^2.0.0",
backend/src/controllers/v1/secretsFolderController.ts
+190 -64
View File
@@ -1,107 +1,233 @@
import { Request, Response } from 'express';
import { Secret } from '../../models';
import Folder from '../../models/folder';
import { BadRequestError } from '../../utils/errors';
import { ROOT_FOLDER_PATH, getFolderPath, getParentPath, normalizePath, validateFolderName } from '../../utils/folder';
import { ADMIN, MEMBER } from '../../variables';
import { validateMembership } from '../../helpers/membership';
import { Request, Response } from "express";
import { Secret } from "../../models";
import Folder from "../../models/folder";
import { BadRequestError } from "../../utils/errors";
import {
appendFolder,
deleteFolderById,
getAllFolderIds,
searchByFolderIdWithDir,
searchByFolderId,
validateFolderName,
generateFolderId,
getParentFromFolderId,
} from "../../services/FolderService";
import { ADMIN, MEMBER } from "../../variables";
import { validateMembership } from "../../helpers/membership";
import { FolderVersion } from "../../ee/models";
import { EESecretService } from "../../ee/services";
// TODO
// verify workspace id/environment
export const createFolder = async (req: Request, res: Response) => {
const { workspaceId, environment, folderName, parentFolderId } = req.body
const { workspaceId, environment, folderName, parentFolderId } = req.body;
if (!validateFolderName(folderName)) {
throw BadRequestError({ message: "Folder name cannot contain spaces. Only underscore and dashes" })
throw BadRequestError({
message: "Folder name cannot contain spaces. Only underscore and dashes",
});
}
if (parentFolderId) {
const parentFolder = await Folder.find({ environment: environment, workspace: workspaceId, id: parentFolderId });
const folders = await Folder.findOne({
workspace: workspaceId,
environment,
}).lean();
// space has no folders initialized
if (!folders) {
const id = generateFolderId();
const folder = new Folder({
workspace: workspaceId,
environment,
nodes: {
id: "root",
name: "root",
version: 1,
children: [{ id, name: folderName, children: [], version: 1 }],
},
});
await folder.save();
const folderVersion = new FolderVersion({
workspace: workspaceId,
environment,
nodes: folder.nodes,
});
await folderVersion.save();
return res.json({ folder: { id, name: folderName } });
}
const folder = appendFolder(folders.nodes, { folderName, parentFolderId });
await Folder.findByIdAndUpdate(folders._id, folders);
const parentFolder = searchByFolderId(folders.nodes, parentFolderId);
const folderVersion = new FolderVersion({
workspace: workspaceId,
environment,
// root condition
nodes: parentFolder || folders.nodes,
});
await folderVersion.save();
return res.json({ folder });
};
export const updateFolderById = async (req: Request, res: Response) => {
const { folderId } = req.params;
const { name, workspaceId, environment } = req.body;
const folders = await Folder.findOne({ workspace: workspaceId, environment });
if (!folders) {
throw BadRequestError({ message: "The folder doesn't exist" });
}
// check that user is a member of the workspace
await validateMembership({
userId: req.user._id.toString(),
workspaceId,
acceptedRoles: [ADMIN, MEMBER],
});
const parentFolder = getParentFromFolderId(folders.nodes, folderId);
if (!parentFolder) {
throw BadRequestError({ message: "The parent folder doesn't exist" })
throw BadRequestError({ message: "The folder doesn't exist" });
}
const folder = parentFolder.children.find(({ id }) => id === folderId);
if (!folder) {
throw BadRequestError({ message: "The folder doesn't exist" });
}
let completePath = await getFolderPath(parentFolderId)
if (completePath == ROOT_FOLDER_PATH) {
completePath = ""
}
parentFolder.version += 1;
folder.name = name;
folder.version += 1;
const currentFolderPath = completePath + "/" + folderName // construct new path with current folder to be created
const normalizedCurrentPath = normalizePath(currentFolderPath)
const normalizedParentPath = getParentPath(normalizedCurrentPath)
const existingFolder = await Folder.findOne({
name: folderName,
await Folder.findByIdAndUpdate(folders._id, folders);
const folderVersion = new FolderVersion({
workspace: workspaceId,
environment: environment,
parent: parentFolderId,
path: normalizedCurrentPath
environment,
nodes: parentFolder,
});
await folderVersion.save();
if (existingFolder) {
return res.json(existingFolder)
}
const newFolder = new Folder({
name: folderName,
// decision to take snapshot ot not
const secretCount = await Secret.count({
workspace: workspaceId,
environment: environment,
parent: parentFolderId,
path: normalizedCurrentPath,
parentPath: normalizedParentPath
environment,
folder: folderId,
});
if (secretCount || folder.children.length) {
await EESecretService.takeSecretSnapshot({
workspaceId,
environment,
folderId: parentFolder.name === "root" ? undefined : parentFolder.id,
});
await newFolder.save();
return res.json(newFolder)
}
return res.json({
message: "Successfully updated folder",
folder: { name: folder.name, id: folder.id },
});
};
export const deleteFolder = async (req: Request, res: Response) => {
const { folderId } = req.params
const queue: any[] = [folderId];
const { folderId } = req.params;
const { workspaceId, environment } = req.body;
const folder = await Folder.findById(folderId);
if (!folder) {
throw BadRequestError({ message: "The folder doesn't exist" })
const folders = await Folder.findOne({ workspace: workspaceId, environment });
if (!folders) {
throw BadRequestError({ message: "The folder doesn't exist" });
}
// check that user is a member of the workspace
await validateMembership({
userId: req.user._id.toString(),
workspaceId: folder.workspace as any,
acceptedRoles: [ADMIN, MEMBER]
workspaceId,
acceptedRoles: [ADMIN, MEMBER],
});
while (queue.length > 0) {
const currentFolderId = queue.shift();
const delOp = deleteFolderById(folders.nodes, folderId);
if (!delOp) {
throw BadRequestError({ message: "The folder doesn't exist" });
}
const { deletedNode: delFolder, parent: parentFolder } = delOp;
const childFolders = await Folder.find({ parent: currentFolderId });
for (const childFolder of childFolders) {
queue.push(childFolder._id);
const delFolderIds = getAllFolderIds(delFolder);
// take a pre-delete snapshot if there are secrets inside the folders
const secretCount = await Secret.count({
workspace: workspaceId,
environment,
folder: folderId,
});
if (secretCount || delFolder.children.length) {
await EESecretService.takeSecretSnapshot({
workspaceId,
environment,
folderId: parentFolder.id,
});
}
await Secret.deleteMany({ folder: currentFolderId });
await Folder.deleteOne({ _id: currentFolderId });
await Folder.findByIdAndUpdate(folders._id, folders);
const folderVersion = new FolderVersion({
workspace: workspaceId,
environment,
nodes: parentFolder,
});
await folderVersion.save();
if (delFolderIds.length) {
await Secret.deleteMany({
folder: { $in: delFolderIds.map(({ id }) => id) },
workspace: workspaceId,
environment,
});
}
res.send()
}
// post delete snapshot
await EESecretService.takeSecretSnapshot({
workspaceId,
environment,
folderId: parentFolder.id,
});
res.send({ message: "successfully deleted folders", folders: delFolderIds });
};
// TODO: validate workspace
export const getFolderById = async (req: Request, res: Response) => {
const { folderId } = req.params
export const getFolders = async (req: Request, res: Response) => {
const { workspaceId, environment, parentFolderId } = req.query as {
workspaceId: string;
environment: string;
parentFolderId?: string;
};
const folder = await Folder.findById(folderId);
if (!folder) {
throw BadRequestError({ message: "The folder doesn't exist" })
const folders = await Folder.findOne({ workspace: workspaceId, environment });
if (!folders) {
res.send({ folders: [], dir: [] });
return;
}
// check that user is a member of the workspace
await validateMembership({
userId: req.user._id.toString(),
workspaceId: folder.workspace as any,
acceptedRoles: [ADMIN, MEMBER]
workspaceId,
acceptedRoles: [ADMIN, MEMBER],
});
res.send({ folder })
if (!parentFolderId) {
const rootFolders = folders.nodes.children.map(({ id, name }) => ({
id,
name,
}));
res.send({ folders: rootFolders });
return;
}
const folderBySearch = searchByFolderIdWithDir(folders.nodes, parentFolderId);
if (!folderBySearch) {
throw BadRequestError({ message: "The folder doesn't exist" });
}
const { folder, dir } = folderBySearch;
res.send({
folders: folder.children.map(({ id, name }) => ({ id, name })),
dir,
});
};
backend/src/controllers/v2/secretsController.ts
+318 -290
View File
File diff suppressed because it is too large Load Diff
backend/src/ee/controllers/v1/secretController.ts
+31 -25
View File
@@ -1,8 +1,8 @@
import { Request, Response } from 'express';
import * as Sentry from '@sentry/node';
import { Secret } from '../../../models';
import { SecretVersion } from '../../models';
import { EESecretService } from '../../services';
import { Request, Response } from "express";
import * as Sentry from "@sentry/node";
import { Secret } from "../../../models";
import { SecretVersion } from "../../models";
import { EESecretService } from "../../services";
/**
* Return secret versions for secret with id [secretId]
@@ -57,30 +57,32 @@ import { EESecretService } from '../../services';
*/
let secretVersions;
try {
const { secretId } = req.params;
const { secretId, workspaceId, environment, folderId } = req.params;
const offset: number = parseInt(req.query.offset as string);
const limit: number = parseInt(req.query.limit as string);
secretVersions = await SecretVersion.find({
secret: secretId
secret: secretId,
workspace: workspaceId,
environment,
folder: folderId,
})
.sort({ createdAt: -1 })
.skip(offset)
.limit(limit);
} catch (err) {
Sentry.setUser({ email: req.user.email });
Sentry.captureException(err);
return res.status(400).send({
message: 'Failed to get secret versions'
message: "Failed to get secret versions",
});
}
return res.status(200).send({
secretVersions
secretVersions,
});
}
};
/**
* Roll back secret with id [secretId] to version [version]
@@ -145,10 +147,10 @@ export const rollbackSecretVersion = async (req: Request, res: Response) => {
// validate secret version
const oldSecretVersion = await SecretVersion.findOne({
secret: secretId,
version
}).select('+secretBlindIndex')
version,
}).select("+secretBlindIndex");
if (!oldSecretVersion) throw new Error('Failed to find secret version');
if (!oldSecretVersion) throw new Error("Failed to find secret version");
const {
workspace,
@@ -162,8 +164,9 @@ export const rollbackSecretVersion = async (req: Request, res: Response) => {
secretValueCiphertext,
secretValueIV,
secretValueTag,
folder,
algorithm,
keyEncoding
keyEncoding,
} = oldSecretVersion;
// update secret
@@ -171,7 +174,7 @@ export const rollbackSecretVersion = async (req: Request, res: Response) => {
secretId,
{
$inc: {
version: 1
version: 1,
},
workspace,
type,
@@ -184,15 +187,16 @@ export const rollbackSecretVersion = async (req: Request, res: Response) => {
secretValueCiphertext,
secretValueIV,
secretValueTag,
folderId: folder,
algorithm,
keyEncoding
keyEncoding,
},
{
new: true
new: true,
}
);
if (!secret) throw new Error('Failed to find and update secret');
if (!secret) throw new Error("Failed to find and update secret");
// add new secret version
await new SecretVersion({
@@ -210,24 +214,26 @@ export const rollbackSecretVersion = async (req: Request, res: Response) => {
secretValueCiphertext,
secretValueIV,
secretValueTag,
folder,
algorithm,
keyEncoding
keyEncoding,
}).save();
// take secret snapshot
await EESecretService.takeSecretSnapshot({
workspaceId: secret.workspace
workspaceId: secret.workspace,
environment,
folderId: folder,
});
} catch (err) {
Sentry.setUser({ email: req.user.email });
Sentry.captureException(err);
return res.status(400).send({
message: 'Failed to roll back secret version'
message: "Failed to roll back secret version",
});
}
return res.status(200).send({
secret
secret,
});
}
};
backend/src/ee/controllers/v1/secretSnapshotController.ts
+26 -11
View File
@@ -1,6 +1,10 @@
import { Request, Response } from 'express';
import * as Sentry from '@sentry/node';
import { SecretSnapshot } from '../../models';
import { Request, Response } from "express";
import * as Sentry from "@sentry/node";
import {
ISecretVersion,
SecretSnapshot,
TFolderRootVersionSchema,
} from "../../models";
/**
* Return secret snapshot with id [secretSnapshotId]
@@ -13,21 +17,32 @@ export const getSecretSnapshot = async (req: Request, res: Response) => {
try {
const { secretSnapshotId } = req.params;
secretSnapshot = await SecretSnapshot
.findById(secretSnapshotId)
.populate('secretVersions');
if (!secretSnapshot) throw new Error('Failed to find secret snapshot');
secretSnapshot = await SecretSnapshot.findById(secretSnapshotId)
.lean()
.populate<{ secretVersions: ISecretVersion[] }>("secretVersions")
.populate<{ folderVersion: TFolderRootVersionSchema }>("folderVersion");
if (!secretSnapshot) throw new Error("Failed to find secret snapshot");
} catch (err) {
Sentry.setUser({ email: req.user.email });
Sentry.captureException(err);
return res.status(400).send({
message: 'Failed to get secret snapshot'
message: "Failed to get secret snapshot",
});
}
const folderId = secretSnapshot.folderId;
// to show only the folder required secrets
secretSnapshot.secretVersions = secretSnapshot.secretVersions.filter(
({ folder }) => folder === folderId
);
secretSnapshot.folderVersion =
secretSnapshot?.folderVersion?.nodes?.children?.map(({ id, name }) => ({
id,
name,
})) as any;
return res.status(200).send({
secretSnapshot
secretSnapshot,
});
}
};
backend/src/ee/controllers/v1/workspaceController.ts
+163 -78
View File
@@ -1,24 +1,32 @@
import { Request, Response } from 'express';
import * as Sentry from '@sentry/node';
import { Types } from 'mongoose';
import {
Secret
} from '../../../models';
import { Request, Response } from "express";
import * as Sentry from "@sentry/node";
import { Types } from "mongoose";
import { Secret } from "../../../models";
import {
SecretSnapshot,
Log,
SecretVersion,
ISecretVersion
} from '../../models';
import { EESecretService } from '../../services';
import { getLatestSecretVersionIds } from '../../helpers/secretVersion';
ISecretVersion,
FolderVersion,
TFolderRootVersionSchema,
} from "../../models";
import { EESecretService } from "../../services";
import { getLatestSecretVersionIds } from "../../helpers/secretVersion";
import Folder from "../../../models/folder";
import {
getAllFolderIds,
searchByFolderId,
} from "../../../services/FolderService";
/**
* Return secret snapshots for workspace with id [workspaceId]
* @param req
* @param res
*/
export const getWorkspaceSecretSnapshots = async (req: Request, res: Response) => {
export const getWorkspaceSecretSnapshots = async (
req: Request,
res: Response
) => {
/*
#swagger.summary = 'Return project secret snapshot ids'
#swagger.description = 'Return project secret snapshots ids'
@@ -67,54 +75,63 @@ import { getLatestSecretVersionIds } from '../../helpers/secretVersion';
let secretSnapshots;
try {
const { workspaceId } = req.params;
const { environment, folderId } = req.query;
const offset: number = parseInt(req.query.offset as string);
const limit: number = parseInt(req.query.limit as string);
secretSnapshots = await SecretSnapshot.find({
workspace: workspaceId
workspace: workspaceId,
environment,
folderId: folderId || "root",
})
.sort({ createdAt: -1 })
.skip(offset)
.limit(limit);
} catch (err) {
Sentry.setUser({ email: req.user.email });
Sentry.captureException(err);
return res.status(400).send({
message: 'Failed to get secret snapshots'
message: "Failed to get secret snapshots",
});
}
return res.status(200).send({
secretSnapshots
secretSnapshots,
});
}
};
/**
* Return count of secret snapshots for workspace with id [workspaceId]
* @param req
* @param res
*/
export const getWorkspaceSecretSnapshotsCount = async (req: Request, res: Response) => {
export const getWorkspaceSecretSnapshotsCount = async (
req: Request,
res: Response
) => {
let count;
try {
const { workspaceId } = req.params;
const { environment, folderId } = req.query;
count = await SecretSnapshot.countDocuments({
workspace: workspaceId
workspace: workspaceId,
environment,
folderId: folderId || "root",
});
} catch (err) {
Sentry.setUser({ email: req.user.email });
Sentry.captureException(err);
return res.status(400).send({
message: 'Failed to count number of secret snapshots'
message: "Failed to count number of secret snapshots",
});
}
return res.status(200).send({
count
count,
});
}
};
/**
* Rollback secret snapshot with id [secretSnapshotId] to version [version]
@@ -122,7 +139,10 @@ export const getWorkspaceSecretSnapshotsCount = async (req: Request, res: Respon
* @param res
* @returns
*/
export const rollbackWorkspaceSecretSnapshot = async (req: Request, res: Response) => {
export const rollbackWorkspaceSecretSnapshot = async (
req: Request,
res: Response
) => {
/*
#swagger.summary = 'Roll back project secrets to those captured in a secret snapshot version.'
#swagger.description = 'Roll back project secrets to those captured in a secret snapshot version.'
@@ -177,44 +197,88 @@ export const rollbackWorkspaceSecretSnapshot = async (req: Request, res: Respons
let secrets;
try {
const { workspaceId } = req.params;
const { version } = req.body;
const { version, environment, folderId = "root" } = req.body;
// validate secret snapshot
const secretSnapshot = await SecretSnapshot.findOne({
workspace: workspaceId,
version
}).populate<{ secretVersions: ISecretVersion[]}>({
path: 'secretVersions',
select: '+secretBlindIndex'
});
version,
environment,
folderId: folderId,
})
.populate<{ secretVersions: ISecretVersion[] }>({
path: "secretVersions",
select: "+secretBlindIndex",
})
.populate<{ folderVersion: TFolderRootVersionSchema }>("folderVersion");
if (!secretSnapshot) throw new Error('Failed to find secret snapshot');
if (!secretSnapshot) throw new Error("Failed to find secret snapshot");
// TODO: fix any
const oldSecretVersionsObj: any = secretSnapshot.secretVersions
.reduce((accumulator, s) => ({
const oldSecretVersionsObj: any = secretSnapshot.secretVersions.reduce(
(accumulator, s) => ({
...accumulator,
[`${s.secret.toString()}`]: s
}), {});
[`${s.secret.toString()}`]: s,
}),
{}
);
const latestSecretVersionIds = await getLatestSecretVersionIds({
secretIds: secretSnapshot.secretVersions.map((sv) => sv.secret)
secretIds: secretSnapshot.secretVersions.map((sv) => sv.secret),
});
// TODO: fix any
const latestSecretVersions: any = (await SecretVersion.find({
const latestSecretVersions: any = (
await SecretVersion.find(
{
_id: {
$in: latestSecretVersionIds.map((s) => s.versionId)
}
}, 'secret version'))
.reduce((accumulator, s) => ({
$in: latestSecretVersionIds.map((s) => s.versionId),
},
},
"secret version"
)
).reduce(
(accumulator, s) => ({
...accumulator,
[`${s.secret.toString()}`]: s
}), {});
[`${s.secret.toString()}`]: s,
}),
{}
);
let folderIds: string[] = [];
const folders = await Folder.findOne({
workspace: workspaceId,
environment,
}).lean();
const latestFolderVersion = await FolderVersion.findOne({
environment,
workspace: workspaceId,
"nodes.id": folderId,
}).sort({ "nodes.version": -1 });
if (folders && folderId) {
const folder = searchByFolderId(folders.nodes, folderId);
if (folder) {
folderIds = getAllFolderIds(folder).map(({ id }) => id);
folder.children = secretSnapshot?.folderVersion?.nodes?.children || [];
folder.version = (latestFolderVersion?.nodes?.version || 0) + 1;
}
}
const secDelQuery: Record<string, unknown> = {
workspace: workspaceId,
environment,
// undefined means root thus collect all secrets
};
if (folderId !== "root" && folderIds.length)
secDelQuery.folder = { $in: folderIds };
// delete existing secrets
await Secret.deleteMany({
workspace: workspaceId
await Secret.deleteMany(secDelQuery);
await Folder.deleteOne({
workspace: workspaceId,
environment,
});
// add secrets
@@ -235,10 +299,11 @@ export const rollbackWorkspaceSecretSnapshot = async (req: Request, res: Respons
secretValueIV,
secretValueTag,
secretValueHash,
createdAt
createdAt,
folder: secFolderId,
} = oldSecretVersionsObj[secretId.toString()];
return ({
return {
_id: secretId,
version: latestSecretVersions[secretId.toString()].version + 1,
workspace,
@@ -254,17 +319,19 @@ export const rollbackWorkspaceSecretSnapshot = async (req: Request, res: Respons
secretValueIV,
secretValueTag,
secretValueHash,
secretCommentCiphertext: '',
secretCommentIV: '',
secretCommentTag: '',
createdAt
});
secretCommentCiphertext: "",
secretCommentIV: "",
secretCommentTag: "",
createdAt,
folder: secFolderId,
};
})
);
// add secret versions
const secretV = await SecretVersion.insertMany(
secrets.map(({
secrets.map(
({
_id,
version,
workspace,
@@ -279,7 +346,8 @@ export const rollbackWorkspaceSecretSnapshot = async (req: Request, res: Respons
secretValueCiphertext,
secretValueIV,
secretValueTag,
secretValueHash
secretValueHash,
folder: secFolderId,
}) => ({
_id: new Types.ObjectId(),
secret: _id,
@@ -297,35 +365,54 @@ export const rollbackWorkspaceSecretSnapshot = async (req: Request, res: Respons
secretValueCiphertext,
secretValueIV,
secretValueTag,
secretValueHash
}))
secretValueHash,
folder: secFolderId,
})
)
);
// update secret versions of restored secrets as not deleted
await SecretVersion.updateMany({
secret: {
$in: secretSnapshot.secretVersions.map((sv) => sv.secret)
}
}, {
isDeleted: false
const newFolder = new Folder(folders);
newFolder._id = new Types.ObjectId();
newFolder.isNew = true;
await newFolder.save();
// create new folder version
const newFolderVersion = new FolderVersion({
workspace: workspaceId,
environment,
nodes: newFolder.nodes,
});
await newFolderVersion.save();
// update secret versions of restored secrets as not deleted
await SecretVersion.updateMany(
{
secret: {
$in: secretSnapshot.secretVersions.map((sv) => sv.secret),
},
},
{
isDeleted: false,
}
);
// take secret snapshot
await EESecretService.takeSecretSnapshot({
workspaceId: new Types.ObjectId(workspaceId)
workspaceId: new Types.ObjectId(workspaceId),
environment,
folderId,
});
} catch (err) {
Sentry.setUser({ email: req.user.email });
Sentry.captureException(err);
return res.status(400).send({
message: 'Failed to roll back secret snapshot'
message: "Failed to roll back secret snapshot",
});
}
return res.status(200).send({
secrets
secrets,
});
}
};
/**
* Return (audit) logs for workspace with id [workspaceId]
@@ -400,7 +487,7 @@ export const getWorkspaceLogs = async (req: Request, res: Response) => {
}
}
*/
let logs
let logs;
try {
const { workspaceId } = req.params;
@@ -413,30 +500,28 @@ export const getWorkspaceLogs = async (req: Request, res: Response) => {
logs = await Log.find({
workspace: workspaceId,
...(userId ? { user: userId } : {}),
...(
actionNames
...(actionNames
? {
actionNames: {
$in: actionNames.split(',')
$in: actionNames.split(","),
},
}
} : {}
)
: {}),
})
.sort({ createdAt: sortBy === 'recent' ? -1 : 1 })
.sort({ createdAt: sortBy === "recent" ? -1 : 1 })
.skip(offset)
.limit(limit)
.populate('actions')
.populate('user serviceAccount serviceTokenData');
.populate("actions")
.populate("user serviceAccount serviceTokenData");
} catch (err) {
Sentry.setUser({ email: req.user.email });
Sentry.captureException(err);
return res.status(400).send({
message: 'Failed to get workspace logs'
message: "Failed to get workspace logs",
});
}
return res.status(200).send({
logs
logs,
});
}
};
backend/src/ee/helpers/secret.ts
+44 -8
View File
@@ -1,6 +1,16 @@
import { Types } from "mongoose";
import { Secret, ISecret } from "../../models";
import { SecretSnapshot, SecretVersion, ISecretVersion } from "../models";
import Folder from "../../models/folder";
import {
getAllFolderIds,
searchByFolderId,
} from "../../services/FolderService";
import {
SecretSnapshot,
SecretVersion,
ISecretVersion,
FolderVersion,
} from "../models";
/**
* Save a secret snapshot that is a copy of the current state of secrets in workspace with id
@@ -12,22 +22,40 @@ import { SecretSnapshot, SecretVersion, ISecretVersion } from "../models";
*/
const takeSecretSnapshotHelper = async ({
workspaceId,
environment,
folderId = "root",
}: {
workspaceId: Types.ObjectId;
environment: string;
folderId?: string;
}) => {
const secretIds = (
await Secret.find(
{
// get all folder ids
let folderIds: string[] = [];
const folders = await Folder.findOne({
workspace: workspaceId,
},
"_id"
)
).map((s) => s._id);
environment,
}).lean();
if (folders && folderId) {
const folder = searchByFolderId(folders.nodes, folderId);
if (folder) folderIds = getAllFolderIds(folder).map(({ id }) => id);
}
const secQuery: Record<string, unknown> = {
workspace: workspaceId,
environment,
// undefined means root thus collect all secrets
};
if (folderId !== "root") secQuery.folder = { $in: folderIds };
const secretIds = (await Secret.find(secQuery, "_id").lean()).map(
(s) => s._id
);
const latestSecretVersions = (
await SecretVersion.aggregate([
{
$match: {
environment,
workspace: new Types.ObjectId(workspaceId),
secret: {
$in: secretIds,
},
@@ -45,6 +73,11 @@ const takeSecretSnapshotHelper = async ({
},
]).exec()
).map((s) => s.versionId);
const latestFolderVersion = await FolderVersion.findOne({
environment,
workspace: workspaceId,
"nodes.id": folderId || "root",
}).sort({ "nodes.version": -1 });
const latestSecretSnapshot = await SecretSnapshot.findOne({
workspace: workspaceId,
@@ -52,8 +85,11 @@ const takeSecretSnapshotHelper = async ({
const secretSnapshot = await new SecretSnapshot({
workspace: workspaceId,
environment,
version: latestSecretSnapshot ? latestSecretSnapshot.version + 1 : 1,
secretVersions: latestSecretVersions,
folderId,
folderVersion: latestFolderVersion,
}).save();
return secretSnapshot;
backend/src/ee/helpers/secretVersion.ts
+21 -31
View File
@@ -1,5 +1,5 @@
import { Types } from 'mongoose';
import { SecretVersion } from '../models';
import { Types } from "mongoose";
import { SecretVersion } from "../models";
/**
* Return latest secret versions for secrets with ids [secretIds]
@@ -8,39 +8,32 @@ import { SecretVersion } from '../models';
* @returns
*/
const getLatestSecretVersionIds = async ({
secretIds
secretIds,
}: {
secretIds: Types.ObjectId[];
}) => {
interface LatestSecretVersionId {
_id: Types.ObjectId;
version: number;
versionId: Types.ObjectId;
}
const latestSecretVersionIds = (await SecretVersion.aggregate([
const latestSecretVersionIds = await SecretVersion.aggregate([
{
$match: {
secret: {
$in: secretIds
}
}
$in: secretIds,
},
},
},
{
$group: {
_id: '$secret',
version: { $max: '$version' },
versionId: { $max: '$_id' } // id of latest secret version
}
_id: "$secret",
version: { $max: "$version" },
versionId: { $max: "$_id" }, // id of latest secret version
},
},
{
$sort: { version: -1 }
}
])
.exec());
$sort: { version: -1 },
},
]).exec();
return latestSecretVersionIds;
}
};
/**
* Return latest [n] secret versions for secrets with ids [secretIds]
@@ -51,13 +44,13 @@ const getLatestSecretVersionIds = async ({
*/
const getLatestNSecretSecretVersionIds = async ({
secretIds,
n
n,
}: {
secretIds: Types.ObjectId[];
n: number;
}) => {
// TODO: optimize query
const latestNSecretVersions = (await SecretVersion.aggregate([
const latestNSecretVersions = await SecretVersion.aggregate([
{
$match: {
secret: {
@@ -80,13 +73,10 @@ const getLatestNSecretSecretVersionIds = async ({
secret: "$_id",
versions: { $slice: ["$versions", n] },
},
}
]));
},
]);
return latestNSecretVersions;
}
};
export {
getLatestSecretVersionIds,
getLatestNSecretSecretVersionIds
}
export { getLatestSecretVersionIds, getLatestNSecretSecretVersionIds };
backend/src/ee/models/folderVersion.ts
+60
View File
@@ -0,0 +1,60 @@
import { model, Schema, Types } from "mongoose";
export type TFolderRootVersionSchema = {
_id: Types.ObjectId;
workspace: Types.ObjectId;
environment: string;
nodes: TFolderVersionSchema;
};
export type TFolderVersionSchema = {
id: string;
name: string;
version: number;
children: TFolderVersionSchema[];
};
const folderVersionSchema = new Schema<TFolderVersionSchema>({
id: {
required: true,
type: String,
default: "root",
},
name: {
required: true,
type: String,
default: "root",
},
version: {
required: true,
type: Number,
default: 1,
},
});
folderVersionSchema.add({ children: [folderVersionSchema] });
const folderRootVersionSchema = new Schema<TFolderRootVersionSchema>(
{
workspace: {
type: Schema.Types.ObjectId,
ref: "Workspace",
required: true,
},
environment: {
type: String,
required: true,
},
nodes: folderVersionSchema,
},
{
timestamps: true,
}
);
const FolderVersion = model<TFolderRootVersionSchema>(
"FolderVersion",
folderRootVersionSchema
);
export default FolderVersion;
backend/src/ee/models/index.ts
+9 -6
View File
@@ -1,15 +1,18 @@
import SecretSnapshot, { ISecretSnapshot } from './secretSnapshot';
import SecretVersion, { ISecretVersion } from './secretVersion';
import Log, { ILog } from './log';
import Action, { IAction } from './action';
import SecretSnapshot, { ISecretSnapshot } from "./secretSnapshot";
import SecretVersion, { ISecretVersion } from "./secretVersion";
import FolderVersion, { TFolderRootVersionSchema } from "./folderVersion";
import Log, { ILog } from "./log";
import Action, { IAction } from "./action";
export {
SecretSnapshot,
ISecretSnapshot,
SecretVersion,
ISecretVersion,
FolderVersion,
TFolderRootVersionSchema,
Log,
ILog,
Action,
IAction
}
IAction,
};
backend/src/ee/models/secretSnapshot.ts
+31 -10
View File
@@ -1,33 +1,54 @@
import { Schema, model, Types } from 'mongoose';
import { Schema, model, Types } from "mongoose";
export interface ISecretSnapshot {
workspace: Types.ObjectId;
environment: string;
folderId: string | "root";
version: number;
secretVersions: Types.ObjectId[];
folderVersion: Types.ObjectId;
}
const secretSnapshotSchema = new Schema<ISecretSnapshot>(
{
workspace: {
type: Schema.Types.ObjectId,
ref: 'Workspace',
required: true
ref: "Workspace",
required: true,
},
environment: {
type: String,
required: true,
},
folderId: {
type: String,
default: "root",
},
version: {
type: Number,
required: true
default: 1,
required: true,
},
secretVersions: [{
secretVersions: [
{
type: Schema.Types.ObjectId,
ref: 'SecretVersion',
required: true
}]
ref: "SecretVersion",
required: true,
},
],
folderVersion: {
type: Schema.Types.ObjectId,
ref: "FolderVersion",
},
},
{
timestamps: true
timestamps: true,
}
);
const SecretSnapshot = model<ISecretSnapshot>('SecretSnapshot', secretSnapshotSchema);
const SecretSnapshot = model<ISecretSnapshot>(
"SecretSnapshot",
secretSnapshotSchema
);
export default SecretSnapshot;
backend/src/ee/models/secretVersion.ts
+40 -32
View File
@@ -1,11 +1,11 @@
import { Schema, model, Types } from 'mongoose';
import { Schema, model, Types } from "mongoose";
import {
SECRET_SHARED,
SECRET_PERSONAL,
ALGORITHM_AES_256_GCM,
ENCODING_SCHEME_UTF8,
ENCODING_SCHEME_BASE64
} from '../../variables';
ENCODING_SCHEME_BASE64,
} from "../../variables";
export interface ISecretVersion {
_id: Types.ObjectId;
@@ -23,93 +23,101 @@ export interface ISecretVersion {
secretValueCiphertext: string;
secretValueIV: string;
secretValueTag: string;
algorithm: 'aes-256-gcm';
keyEncoding: 'utf8' | 'base64';
algorithm: "aes-256-gcm";
keyEncoding: "utf8" | "base64";
folder?: string;
}
const secretVersionSchema = new Schema<ISecretVersion>(
{
secret: { // could be deleted
secret: {
// could be deleted
type: Schema.Types.ObjectId,
ref: 'Secret',
required: true
ref: "Secret",
required: true,
},
version: {
type: Number,
default: 1,
required: true
required: true,
},
workspace: {
type: Schema.Types.ObjectId,
ref: 'Workspace',
required: true
ref: "Workspace",
required: true,
},
type: {
type: String,
enum: [SECRET_SHARED, SECRET_PERSONAL],
required: true
required: true,
},
user: {
// user associated with the personal secret
type: Schema.Types.ObjectId,
ref: 'User'
ref: "User",
},
environment: {
type: String,
required: true
required: true,
},
isDeleted: { // consider removing field
isDeleted: {
// consider removing field
type: Boolean,
default: false,
required: true
required: true,
},
secretBlindIndex: {
type: String,
select: false
select: false,
},
secretKeyCiphertext: {
type: String,
required: true
required: true,
},
secretKeyIV: {
type: String, // symmetric
required: true
required: true,
},
secretKeyTag: {
type: String, // symmetric
required: true
required: true,
},
secretValueCiphertext: {
type: String,
required: true
required: true,
},
secretValueIV: {
type: String, // symmetric
required: true
required: true,
},
secretValueTag: {
type: String, // symmetric
required: true
required: true,
},
algorithm: { // the encryption algorithm used
algorithm: {
// the encryption algorithm used
type: String,
enum: [ALGORITHM_AES_256_GCM],
required: true
required: true,
},
keyEncoding: {
type: String,
enum: [
ENCODING_SCHEME_UTF8,
ENCODING_SCHEME_BASE64
],
required: true
enum: [ENCODING_SCHEME_UTF8, ENCODING_SCHEME_BASE64],
required: true,
},
folder: {
type: String,
required: false,
},
},
{
timestamps: true
timestamps: true,
}
);
const SecretVersion = model<ISecretVersion>('SecretVersion', secretVersionSchema);
const SecretVersion = model<ISecretVersion>(
"SecretVersion",
secretVersionSchema
);
export default SecretVersion;
backend/src/ee/routes/v1/workspace.ts
+36 -30
View File
@@ -1,74 +1,80 @@
import express from 'express';
import express from "express";
const router = express.Router();
import {
requireAuth,
requireWorkspaceAuth,
validateRequest
} from '../../../middleware';
import { param, query, body } from 'express-validator';
import { ADMIN, MEMBER } from '../../../variables';
import { workspaceController } from '../../controllers/v1';
validateRequest,
} from "../../../middleware";
import { param, query, body } from "express-validator";
import { ADMIN, MEMBER } from "../../../variables";
import { workspaceController } from "../../controllers/v1";
router.get(
'/:workspaceId/secret-snapshots',
"/:workspaceId/secret-snapshots",
requireAuth({
acceptedAuthModes: ['jwt', 'apiKey']
acceptedAuthModes: ["jwt", "apiKey"],
}),
requireWorkspaceAuth({
acceptedRoles: [ADMIN, MEMBER],
locationWorkspaceId: 'params'
locationWorkspaceId: "params",
}),
param('workspaceId').exists().trim(),
query('offset').exists().isInt(),
query('limit').exists().isInt(),
param("workspaceId").exists().trim(),
query("environment").isString().exists().trim(),
query("folderId").default("root").isString().trim(),
query("offset").exists().isInt(),
query("limit").exists().isInt(),
validateRequest,
workspaceController.getWorkspaceSecretSnapshots
);
router.get(
'/:workspaceId/secret-snapshots/count',
"/:workspaceId/secret-snapshots/count",
requireAuth({
acceptedAuthModes: ['jwt']
acceptedAuthModes: ["jwt"],
}),
requireWorkspaceAuth({
acceptedRoles: [ADMIN, MEMBER],
locationWorkspaceId: 'params'
locationWorkspaceId: "params",
}),
param('workspaceId').exists().trim(),
param("workspaceId").exists().trim(),
query("environment").isString().exists().trim(),
query("folderId").default("root").isString().trim(),
validateRequest,
workspaceController.getWorkspaceSecretSnapshotsCount
);
router.post(
'/:workspaceId/secret-snapshots/rollback',
"/:workspaceId/secret-snapshots/rollback",
requireAuth({
acceptedAuthModes: ['jwt', 'apiKey']
acceptedAuthModes: ["jwt", "apiKey"],
}),
requireWorkspaceAuth({
acceptedRoles: [ADMIN, MEMBER],
locationWorkspaceId: 'params'
locationWorkspaceId: "params",
}),
param('workspaceId').exists().trim(),
body('version').exists().isInt(),
param("workspaceId").exists().trim(),
body("environment").isString().exists().trim(),
query("folderId").default("root").isString().exists().trim(),
body("version").exists().isInt(),
validateRequest,
workspaceController.rollbackWorkspaceSecretSnapshot
);
router.get(
'/:workspaceId/logs',
"/:workspaceId/logs",
requireAuth({
acceptedAuthModes: ['jwt', 'apiKey']
acceptedAuthModes: ["jwt", "apiKey"],
}),
requireWorkspaceAuth({
acceptedRoles: [ADMIN, MEMBER],
locationWorkspaceId: 'params'
locationWorkspaceId: "params",
}),
param('workspaceId').exists().trim(),
query('offset').exists().isInt(),
query('limit').exists().isInt(),
query('sortBy'),
query('userId'),
query('actionNames'),
param("workspaceId").exists().trim(),
query("offset").exists().isInt(),
query("limit").exists().isInt(),
query("sortBy"),
query("userId"),
query("actionNames"),
validateRequest,
workspaceController.getWorkspaceLogs
);
backend/src/ee/services/EESecretService.ts
+15 -8
View File
@@ -3,7 +3,7 @@ import { ISecretVersion } from '../models';
import {
takeSecretSnapshotHelper,
addSecretVersionsHelper,
markDeletedSecretVersionsHelper
markDeletedSecretVersionsHelper,
} from '../helpers/secret';
import EELicenseService from './EELicenseService';
@@ -11,7 +11,6 @@ import EELicenseService from './EELicenseService';
* Class to handle Enterprise Edition secret actions
*/
class EESecretService {
/**
* Save a secret snapshot that is a copy of the current state of secrets in workspace with id
* [workspaceId] under a new snapshot with incremented version under the
@@ -22,12 +21,20 @@ class EESecretService {
* @returns {SecretSnapshot} secretSnapshot - new secret snpashot
*/
static async takeSecretSnapshot({
workspaceId
workspaceId,
environment,
folderId,
}: {
workspaceId: Types.ObjectId;
environment: string;
folderId?: string;
}) {
if (!EELicenseService.isLicenseValid) return;
return await takeSecretSnapshotHelper({ workspaceId });
return await takeSecretSnapshotHelper({
workspaceId,
environment,
folderId,
});
}
/**
@@ -37,13 +44,13 @@ class EESecretService {
* @returns {SecretVersion[]} newSecretVersions - new secret versions
*/
static async addSecretVersions({
secretVersions
secretVersions,
}: {
secretVersions: ISecretVersion[];
}) {
if (!EELicenseService.isLicenseValid) return;
return await addSecretVersionsHelper({
secretVersions
secretVersions,
});
}
@@ -54,13 +61,13 @@ class EESecretService {
* @param {ObjectId[]} obj.secretIds - secret ids
*/
static async markDeletedSecretVersions({
secretIds
secretIds,
}: {
secretIds: Types.ObjectId[];
}) {
if (!EELicenseService.isLicenseValid) return;
await markDeletedSecretVersionsHelper({
secretIds
secretIds,
});
}
}
backend/src/helpers/membership.ts
+21 -29
View File
@@ -1,13 +1,7 @@
import * as Sentry from '@sentry/node';
import { Types } from 'mongoose';
import {
Membership,
Key
} from '../models';
import {
MembershipNotFoundError,
BadRequestError
} from '../utils/errors';
import { Membership, Key } from '../models';
import { MembershipNotFoundError, BadRequestError } from '../utils/errors';
/**
* Validate that user with id [userId] is a member of workspace with id [workspaceId]
@@ -22,28 +16,31 @@ const validateMembership = async ({
workspaceId,
acceptedRoles,
}: {
userId: Types.ObjectId;
workspaceId: Types.ObjectId;
userId: Types.ObjectId | string;
workspaceId: Types.ObjectId | string;
acceptedRoles?: Array<'admin' | 'member'>;
}) => {
const membership = await Membership.findOne({
user: userId,
workspace: workspaceId
}).populate("workspace");
workspace: workspaceId,
}).populate('workspace');
if (!membership) {
throw MembershipNotFoundError({ message: 'Failed to find workspace membership' });
throw MembershipNotFoundError({
message: 'Failed to find workspace membership',
});
}
if (acceptedRoles) {
if (!acceptedRoles.includes(membership.role)) {
throw BadRequestError({ message: 'Failed authorization for membership role' });
throw BadRequestError({
message: 'Failed authorization for membership role',
});
}
}
return membership;
}
};
/**
* Return membership matching criteria specified in query [queryObj]
@@ -74,7 +71,7 @@ const findMembership = async (queryObj: any) => {
const addMemberships = async ({
userIds,
workspaceId,
roles
roles,
}: {
userIds: string[];
workspaceId: string;
@@ -87,15 +84,15 @@ const addMemberships = async ({
filter: {
user: userId,
workspace: workspaceId,
role: roles[idx]
role: roles[idx],
},
update: {
user: userId,
workspace: workspaceId,
role: roles[idx]
role: roles[idx],
},
upsert: true,
},
upsert: true
}
};
});
@@ -116,7 +113,7 @@ const deleteMembership = async ({ membershipId }: { membershipId: string }) => {
let deletedMembership;
try {
deletedMembership = await Membership.findOneAndDelete({
_id: membershipId
_id: membershipId,
});
// delete keys associated with the membership
@@ -124,7 +121,7 @@ const deleteMembership = async ({ membershipId }: { membershipId: string }) => {
// case: membership had a registered user
await Key.deleteMany({
receiver: deletedMembership.user,
workspace: deletedMembership.workspace
workspace: deletedMembership.workspace,
});
}
} catch (err) {
@@ -136,9 +133,4 @@ const deleteMembership = async ({ membershipId }: { membershipId: string }) => {
return deletedMembership;
};
export {
validateMembership,
addMemberships,
findMembership,
deleteMembership
};
export { validateMembership, addMemberships, findMembership, deleteMembership };
backend/src/helpers/secret.ts
+8 -8
View File
@@ -1,5 +1,5 @@
import { Types } from "mongoose";
import { Secret, ISecret, Membership } from "../models";
import { Secret, ISecret } from "../models";
import { EESecretService, EELogService } from "../ee/services";
import { IAction, SecretVersion } from "../ee/models";
import {
@@ -12,8 +12,6 @@ import {
ALGORITHM_AES_256_GCM,
ENCODING_SCHEME_UTF8,
} from "../variables";
import _ from "lodash";
import { BadRequestError, UnauthorizedRequestError } from "../utils/errors";
interface V1PushSecret {
ciphertextKey: string;
@@ -197,7 +195,7 @@ const v1PushSecrets = async ({
secretValueTag: newSecret.tagValue,
secretValueHash: newSecret.hashValue,
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_UTF8
keyEncoding: ENCODING_SCHEME_UTF8,
});
}),
});
@@ -230,7 +228,7 @@ const v1PushSecrets = async ({
secretCommentTag: s.tagComment,
secretCommentHash: s.hashComment,
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_UTF8
keyEncoding: ENCODING_SCHEME_UTF8,
};
if (toAdd[idx].type === "personal") {
@@ -261,7 +259,7 @@ const v1PushSecrets = async ({
secretValueTag,
secretValueHash,
algorithm,
keyEncoding
keyEncoding,
}) =>
new SecretVersion({
secret: _id,
@@ -280,7 +278,7 @@ const v1PushSecrets = async ({
secretValueTag,
secretValueHash,
algorithm,
keyEncoding
keyEncoding,
})
),
});
@@ -289,6 +287,7 @@ const v1PushSecrets = async ({
// (EE) take a secret snapshot
await EESecretService.takeSecretSnapshot({
workspaceId: new Types.ObjectId(workspaceId),
environment,
});
};
@@ -491,7 +490,7 @@ const v2PushSecrets = async ({
secret: secretDocument._id,
isDeleted: false,
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_UTF8
keyEncoding: ENCODING_SCHEME_UTF8,
});
}),
});
@@ -508,6 +507,7 @@ const v2PushSecrets = async ({
// (EE) take a secret snapshot
await EESecretService.takeSecretSnapshot({
workspaceId: new Types.ObjectId(workspaceId),
environment,
});
// (EE) create (audit) log
backend/src/helpers/secrets.ts
+144 -129
View File
@@ -4,19 +4,15 @@ import {
GetSecretsParams,
GetSecretParams,
UpdateSecretParams,
DeleteSecretParams
DeleteSecretParams,
} from '../interfaces/services/SecretService';
import {
Secret,
ISecret,
SecretBlindIndexData,
} from '../models';
import { Secret, ISecret, SecretBlindIndexData } from '../models';
import { SecretVersion } from '../ee/models';
import {
BadRequestError,
SecretNotFoundError,
SecretBlindIndexDataNotFoundError,
InternalServerError
InternalServerError,
} from '../utils/errors';
import {
SECRET_PERSONAL,
@@ -27,23 +23,20 @@ import {
ACTION_DELETE_SECRETS,
ALGORITHM_AES_256_GCM,
ENCODING_SCHEME_UTF8,
ENCODING_SCHEME_BASE64
ENCODING_SCHEME_BASE64,
} from '../variables';
import crypto from 'crypto';
import * as argon2 from 'argon2';
import {
encryptSymmetric128BitHexKeyUTF8,
decryptSymmetric128BitHexKeyUTF8
decryptSymmetric128BitHexKeyUTF8,
} from '../utils/crypto';
import { getEncryptionKey, client, getRootEncryptionKey } from '../config';
import { TelemetryService } from '../services';
import {
EESecretService,
EELogService
} from '../ee/services';
import { EESecretService, EELogService } from '../ee/services';
import {
getAuthDataPayloadIdObj,
getAuthDataPayloadUserObj
getAuthDataPayloadUserObj,
} from '../utils/auth';
/**
@@ -53,11 +46,10 @@ import {
* @param {Types.ObjectId} obj.workspaceId
*/
const createSecretBlindIndexDataHelper = async ({
workspaceId
workspaceId,
}: {
workspaceId: Types.ObjectId;
}) => {
// initialize random blind index salt for workspace
const salt = crypto.randomBytes(16).toString('base64');
@@ -68,7 +60,7 @@ const createSecretBlindIndexDataHelper = async ({
const {
ciphertext: encryptedSaltCiphertext,
iv: saltIV,
tag: saltTag
tag: saltTag,
} = client.encryptSymmetric(salt, rootEncryptionKey);
return await new SecretBlindIndexData({
@@ -77,16 +69,16 @@ const createSecretBlindIndexDataHelper = async ({
saltIV,
saltTag,
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_BASE64
keyEncoding: ENCODING_SCHEME_BASE64,
}).save();
} else {
const {
ciphertext: encryptedSaltCiphertext,
iv: saltIV,
tag: saltTag
tag: saltTag,
} = encryptSymmetric128BitHexKeyUTF8({
plaintext: salt,
key: encryptionKey
key: encryptionKey,
});
return await new SecretBlindIndexData({
@@ -95,10 +87,10 @@ const createSecretBlindIndexDataHelper = async ({
saltIV,
saltTag,
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_UTF8
keyEncoding: ENCODING_SCHEME_UTF8,
}).save();
}
}
};
/**
* Get secret blind index salt for workspace with id [workspaceId]
@@ -107,41 +99,46 @@ const createSecretBlindIndexDataHelper = async ({
* @returns
*/
const getSecretBlindIndexSaltHelper = async ({
workspaceId
workspaceId,
}: {
workspaceId: Types.ObjectId;
}) => {
const encryptionKey = await getEncryptionKey();
const rootEncryptionKey = await getRootEncryptionKey();
const secretBlindIndexData = await SecretBlindIndexData.findOne({
workspace: workspaceId
workspace: workspaceId,
}).select('+algorithm +keyEncoding');
if (!secretBlindIndexData) throw SecretBlindIndexDataNotFoundError();
if (rootEncryptionKey && secretBlindIndexData.keyEncoding === ENCODING_SCHEME_BASE64) {
if (
rootEncryptionKey &&
secretBlindIndexData.keyEncoding === ENCODING_SCHEME_BASE64
) {
return client.decryptSymmetric(
secretBlindIndexData.encryptedSaltCiphertext,
rootEncryptionKey,
secretBlindIndexData.saltIV,
secretBlindIndexData.saltTag
);
} else if (encryptionKey && secretBlindIndexData.keyEncoding === ENCODING_SCHEME_UTF8) {
} else if (
encryptionKey &&
secretBlindIndexData.keyEncoding === ENCODING_SCHEME_UTF8
) {
// decrypt workspace salt
return decryptSymmetric128BitHexKeyUTF8({
ciphertext: secretBlindIndexData.encryptedSaltCiphertext,
iv: secretBlindIndexData.saltIV,
tag: secretBlindIndexData.saltTag,
key: encryptionKey
key: encryptionKey,
});
}
throw InternalServerError({
message: 'Failed to obtain workspace salt needed for secret blind indexing'
message: 'Failed to obtain workspace salt needed for secret blind indexing',
});
}
};
/**
* Generate blind index for secret with name [secretName]
@@ -152,25 +149,26 @@ const getSecretBlindIndexSaltHelper = async ({
*/
const generateSecretBlindIndexWithSaltHelper = async ({
secretName,
salt
salt,
}: {
secretName: string;
salt: string;
}) => {
// generate secret blind index
const secretBlindIndex = (await argon2.hash(secretName, {
const secretBlindIndex = (
await argon2.hash(secretName, {
type: argon2.argon2id,
salt: Buffer.from(salt, 'base64'),
saltLength: 16, // default 16 bytes
memoryCost: 65536, // default pool of 64 MiB per thread.
hashLength: 32,
parallelism: 1,
raw: true
})).toString('base64');
raw: true,
})
).toString('base64');
return secretBlindIndex;
}
};
/**
* Generate blind index for secret with name [secretName]
@@ -181,15 +179,14 @@ const getSecretBlindIndexSaltHelper = async ({
*/
const generateSecretBlindIndexHelper = async ({
secretName,
workspaceId
workspaceId,
}: {
secretName: string;
workspaceId: Types.ObjectId;
}) => {
// check if workspace blind index data exists
const secretBlindIndexData = await SecretBlindIndexData.findOne({
workspace: workspaceId
workspace: workspaceId,
});
if (!secretBlindIndexData) throw SecretBlindIndexDataNotFoundError();
@@ -199,16 +196,16 @@ const generateSecretBlindIndexHelper = async ({
ciphertext: secretBlindIndexData.encryptedSaltCiphertext,
iv: secretBlindIndexData.saltIV,
tag: secretBlindIndexData.saltTag,
key: await getEncryptionKey()
key: await getEncryptionKey(),
});
const secretBlindIndex = await generateSecretBlindIndexWithSaltHelper({
secretName,
salt
salt,
});
return secretBlindIndex;
}
};
/**
* Create secret with name [secretName]
@@ -234,22 +231,24 @@ const createSecretHelper = async ({
secretValueTag,
secretCommentCiphertext,
secretCommentIV,
secretCommentTag
secretCommentTag,
folderId,
}: CreateSecretParams) => {
const secretBlindIndex = await generateSecretBlindIndexHelper({
secretName,
workspaceId: new Types.ObjectId(workspaceId)
workspaceId: new Types.ObjectId(workspaceId),
});
const exists = await Secret.exists({
secretBlindIndex,
workspace: new Types.ObjectId(workspaceId),
type,
...(type === SECRET_PERSONAL ? getAuthDataPayloadUserObj(authData) : {})
...(type === SECRET_PERSONAL ? getAuthDataPayloadUserObj(authData) : {}),
});
if (exists) throw BadRequestError({
message: 'Failed to create secret that already exists'
if (exists)
throw BadRequestError({
message: 'Failed to create secret that already exists',
});
if (type === SECRET_PERSONAL) {
@@ -259,11 +258,13 @@ const createSecretHelper = async ({
const exists = await Secret.exists({
secretBlindIndex,
workspace: new Types.ObjectId(workspaceId),
type: SECRET_SHARED
type: SECRET_SHARED,
});
if (!exists) throw BadRequestError({
message: 'Failed to create personal secret override for no corresponding shared secret'
if (!exists)
throw BadRequestError({
message:
'Failed to create personal secret override for no corresponding shared secret',
});
}
@@ -284,8 +285,9 @@ const createSecretHelper = async ({
secretCommentCiphertext,
secretCommentIV,
secretCommentTag,
folder: folderId,
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_UTF8
keyEncoding: ENCODING_SCHEME_UTF8,
}).save();
const secretVersion = new SecretVersion({
@@ -304,12 +306,12 @@ const createSecretHelper = async ({
secretValueIV,
secretValueTag,
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_UTF8
keyEncoding: ENCODING_SCHEME_UTF8,
});
// // (EE) add version for new secret
// (EE) add version for new secret
await EESecretService.addSecretVersions({
secretVersions: [secretVersion]
secretVersions: [secretVersion],
});
// (EE) create (audit) log
@@ -317,20 +319,23 @@ const createSecretHelper = async ({
name: ACTION_ADD_SECRETS,
...getAuthDataPayloadIdObj(authData),
workspaceId,
secretIds: [secret._id]
secretIds: [secret._id],
});
action && await EELogService.createLog({
action &&
(await EELogService.createLog({
...getAuthDataPayloadIdObj(authData),
workspaceId,
actions: [action],
channel: authData.authChannel,
ipAddress: authData.authIP
});
ipAddress: authData.authIP,
}));
// (EE) take a secret snapshot
await EESecretService.takeSecretSnapshot({
workspaceId
workspaceId,
environment,
folderId,
});
const postHogClient = await TelemetryService.getPostHogClient();
@@ -339,20 +344,20 @@ const createSecretHelper = async ({
postHogClient.capture({
event: 'secrets added',
distinctId: await TelemetryService.getDistinctId({
authData
authData,
}),
properties: {
numberOfSecrets: 1,
environment,
workspaceId,
channel: authData.authChannel,
userAgent: authData.authUserAgent
}
userAgent: authData.authUserAgent,
},
});
}
return secret;
}
};
/**
* Get secrets for workspace with id [workspaceId] and environment [environment]
@@ -365,7 +370,7 @@ const createSecretHelper = async ({
const getSecretsHelper = async ({
workspaceId,
environment,
authData
authData,
}: GetSecretsParams) => {
let secrets: ISecret[] = [];
@@ -374,34 +379,37 @@ const getSecretsHelper = async ({
workspace: new Types.ObjectId(workspaceId),
environment,
type: SECRET_PERSONAL,
...getAuthDataPayloadUserObj(authData)
...getAuthDataPayloadUserObj(authData),
});
// concat with shared secrets
secrets = secrets.concat(await Secret.find({
secrets = secrets.concat(
await Secret.find({
workspace: new Types.ObjectId(workspaceId),
environment,
type: SECRET_SHARED,
secretBlindIndex: {
$nin: secrets.map((secret) => secret.secretBlindIndex)
}
}));
$nin: secrets.map((secret) => secret.secretBlindIndex),
},
})
);
// (EE) create (audit) log
const action = await EELogService.createAction({
name: ACTION_READ_SECRETS,
...getAuthDataPayloadIdObj(authData),
workspaceId,
secretIds: secrets.map((secret) => secret._id)
secretIds: secrets.map((secret) => secret._id),
});
action && await EELogService.createLog({
action &&
(await EELogService.createLog({
...getAuthDataPayloadIdObj(authData),
workspaceId,
actions: [action],
channel: authData.authChannel,
ipAddress: authData.authIP
});
ipAddress: authData.authIP,
}));
const postHogClient = await TelemetryService.getPostHogClient();
@@ -409,20 +417,20 @@ const getSecretsHelper = async ({
postHogClient.capture({
event: 'secrets pulled',
distinctId: await TelemetryService.getDistinctId({
authData
authData,
}),
properties: {
numberOfSecrets: secrets.length,
environment,
workspaceId,
channel: authData.authChannel,
userAgent: authData.authUserAgent
}
userAgent: authData.authUserAgent,
},
});
}
return secrets;
}
};
/**
* Get secret with name [secretName]
@@ -439,11 +447,11 @@ const getSecretHelper = async ({
workspaceId,
environment,
type,
authData
authData,
}: GetSecretParams) => {
const secretBlindIndex = await generateSecretBlindIndexHelper({
secretName,
workspaceId: new Types.ObjectId(workspaceId)
workspaceId: new Types.ObjectId(workspaceId),
});
let secret: ISecret | null = null;
@@ -453,7 +461,7 @@ const getSecretHelper = async ({
workspace: new Types.ObjectId(workspaceId),
environment,
type: type ?? SECRET_PERSONAL,
...(type === SECRET_PERSONAL ? getAuthDataPayloadUserObj(authData) : {})
...(type === SECRET_PERSONAL ? getAuthDataPayloadUserObj(authData) : {}),
});
if (!secret) {
@@ -463,7 +471,7 @@ const getSecretHelper = async ({
secretBlindIndex,
workspace: new Types.ObjectId(workspaceId),
environment,
type: SECRET_SHARED
type: SECRET_SHARED,
});
}
@@ -474,16 +482,17 @@ const getSecretHelper = async ({
name: ACTION_READ_SECRETS,
...getAuthDataPayloadIdObj(authData),
workspaceId,
secretIds: [secret._id]
secretIds: [secret._id],
});
action && await EELogService.createLog({
action &&
(await EELogService.createLog({
...getAuthDataPayloadIdObj(authData),
workspaceId,
actions: [action],
channel: authData.authChannel,
ipAddress: authData.authIP
});
ipAddress: authData.authIP,
}));
const postHogClient = await TelemetryService.getPostHogClient();
@@ -491,20 +500,20 @@ const getSecretHelper = async ({
postHogClient.capture({
event: 'secrets pull',
distinctId: await TelemetryService.getDistinctId({
authData
authData,
}),
properties: {
numberOfSecrets: 1,
environment,
workspaceId,
channel: authData.authChannel,
userAgent: authData.authUserAgent
}
userAgent: authData.authUserAgent,
},
});
}
return secret;
}
};
/**
* Update secret with name [secretName]
@@ -527,11 +536,11 @@ const updateSecretHelper = async ({
authData,
secretValueCiphertext,
secretValueIV,
secretValueTag
secretValueTag,
}: UpdateSecretParams) => {
const secretBlindIndex = await generateSecretBlindIndexHelper({
secretName,
workspaceId: new Types.ObjectId(workspaceId)
workspaceId: new Types.ObjectId(workspaceId),
});
let secret: ISecret | null = null;
@@ -543,16 +552,16 @@ const updateSecretHelper = async ({
secretBlindIndex,
workspace: new Types.ObjectId(workspaceId),
environment,
type
type,
},
{
secretValueCiphertext,
secretValueIV,
secretValueTag,
$inc: { version: 1 }
$inc: { version: 1 },
},
{
new: true
new: true,
}
);
} else {
@@ -564,16 +573,16 @@ const updateSecretHelper = async ({
workspace: new Types.ObjectId(workspaceId),
environment,
type,
...getAuthDataPayloadUserObj(authData)
...getAuthDataPayloadUserObj(authData),
},
{
secretValueCiphertext,
secretValueIV,
secretValueTag,
$inc: { version: 1 }
$inc: { version: 1 },
},
{
new: true
new: true,
}
);
}
@@ -596,12 +605,12 @@ const updateSecretHelper = async ({
secretValueIV,
secretValueTag,
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_UTF8
keyEncoding: ENCODING_SCHEME_UTF8,
});
// (EE) add version for new secret
await EESecretService.addSecretVersions({
secretVersions: [secretVersion]
secretVersions: [secretVersion],
});
// (EE) create (audit) log
@@ -609,20 +618,23 @@ const updateSecretHelper = async ({
name: ACTION_UPDATE_SECRETS,
...getAuthDataPayloadIdObj(authData),
workspaceId,
secretIds: [secret._id]
secretIds: [secret._id],
});
action && await EELogService.createLog({
action &&
(await EELogService.createLog({
...getAuthDataPayloadIdObj(authData),
workspaceId,
actions: [action],
channel: authData.authChannel,
ipAddress: authData.authIP
});
ipAddress: authData.authIP,
}));
// (EE) take a secret snapshot
await EESecretService.takeSecretSnapshot({
workspaceId
workspaceId,
environment,
folderId: secret?.folder,
});
const postHogClient = await TelemetryService.getPostHogClient();
@@ -631,20 +643,20 @@ const updateSecretHelper = async ({
postHogClient.capture({
event: 'secrets modified',
distinctId: await TelemetryService.getDistinctId({
authData
authData,
}),
properties: {
numberOfSecrets: 1,
environment,
workspaceId,
channel: authData.authChannel,
userAgent: authData.authUserAgent
}
userAgent: authData.authUserAgent,
},
});
}
return secret;
}
};
/**
* Delete secret with name [secretName]
@@ -661,11 +673,11 @@ const deleteSecretHelper = async ({
workspaceId,
environment,
type,
authData
authData,
}: DeleteSecretParams) => {
const secretBlindIndex = await generateSecretBlindIndexHelper({
secretName,
workspaceId: new Types.ObjectId(workspaceId)
workspaceId: new Types.ObjectId(workspaceId),
});
let secrets: ISecret[] = [];
@@ -675,20 +687,20 @@ const deleteSecretHelper = async ({
secrets = await Secret.find({
secretBlindIndex,
workspaceId: new Types.ObjectId(workspaceId),
environment
environment,
});
secret = await Secret.findOneAndDelete({
secretBlindIndex,
workspaceId: new Types.ObjectId(workspaceId),
environment,
type
type,
});
await Secret.deleteMany({
secretBlindIndex,
workspaceId: new Types.ObjectId(workspaceId),
environment
environment,
});
} else {
secret = await Secret.findOneAndDelete({
@@ -696,7 +708,7 @@ const deleteSecretHelper = async ({
workspaceId: new Types.ObjectId(workspaceId),
environment,
type,
...getAuthDataPayloadUserObj(authData)
...getAuthDataPayloadUserObj(authData),
});
if (secret) {
@@ -707,7 +719,7 @@ const deleteSecretHelper = async ({
if (!secret) throw SecretNotFoundError();
await EESecretService.markDeletedSecretVersions({
secretIds: secrets.map((secret) => secret._id)
secretIds: secrets.map((secret) => secret._id),
});
// (EE) create (audit) log
@@ -715,21 +727,24 @@ const deleteSecretHelper = async ({
name: ACTION_DELETE_SECRETS,
...getAuthDataPayloadIdObj(authData),
workspaceId,
secretIds: secrets.map((secret) => secret._id)
secretIds: secrets.map((secret) => secret._id),
});
// (EE) take a secret snapshot
action && await EELogService.createLog({
action &&
(await EELogService.createLog({
...getAuthDataPayloadIdObj(authData),
workspaceId,
actions: [action],
channel: authData.authChannel,
ipAddress: authData.authIP
});
ipAddress: authData.authIP,
}));
// (EE) take a secret snapshot
await EESecretService.takeSecretSnapshot({
workspaceId
workspaceId,
environment,
folderId: secret?.folder,
});
const postHogClient = await TelemetryService.getPostHogClient();
@@ -738,23 +753,23 @@ const deleteSecretHelper = async ({
postHogClient.capture({
event: 'secrets deleted',
distinctId: await TelemetryService.getDistinctId({
authData
authData,
}),
properties: {
numberOfSecrets: secrets.length,
environment,
workspaceId,
channel: authData.authChannel,
userAgent: authData.authUserAgent
}
userAgent: authData.authUserAgent,
},
});
}
return ({
return {
secrets,
secret
});
}
secret,
};
};
export {
createSecretBlindIndexDataHelper,
@@ -765,5 +780,5 @@ export {
getSecretsHelper,
getSecretHelper,
updateSecretHelper,
deleteSecretHelper
}
deleteSecretHelper,
};
backend/src/index.ts
+17 -20
View File
@@ -1,8 +1,6 @@
import dotenv from "dotenv";
import dotenv from 'dotenv';
dotenv.config();
import express from 'express';
// eslint-disable-next-line @typescript-eslint/no-var-requires
require('express-async-errors');
import helmet from 'helmet';
import cors from 'cors';
import { DatabaseService } from './services';
@@ -21,7 +19,7 @@ import {
secretSnapshot as eeSecretSnapshotRouter,
action as eeActionRouter,
organizations as eeOrganizationsRouter,
cloudProducts as eeCloudProductsRouter
cloudProducts as eeCloudProductsRouter,
} from './ee/routes/v1';
import {
signup as v1SignupRouter,
@@ -41,7 +39,7 @@ import {
stripe as v1StripeRouter,
integration as v1IntegrationRouter,
integrationAuth as v1IntegrationAuthRouter,
secretsFolder as v1SecretsFolder
secretsFolder as v1SecretsFolder,
} from './routes/v1';
import {
signup as v2SignupRouter,
@@ -61,17 +59,13 @@ import {
auth as v3AuthRouter,
secrets as v3SecretsRouter,
signup as v3SignupRouter,
workspaces as v3WorkspacesRouter
workspaces as v3WorkspacesRouter,
} from './routes/v3';
import { healthCheck } from './routes/status';
import { getLogger } from './utils/logger';
import { RouteNotFoundError } from './utils/errors';
import { requestErrorHandler } from './middleware/requestErrorHandler';
import {
getNodeEnv,
getPort,
getSiteURL
} from './config';
import { getNodeEnv, getPort, getSiteURL } from './config';
import { setup } from './utils/setup';
const main = async () => {
@@ -86,7 +80,7 @@ const main = async () => {
app.use(
cors({
credentials: true,
origin: await getSiteURL()
origin: await getSiteURL(),
})
);
@@ -126,7 +120,7 @@ const main = async () => {
app.use('/api/v1/stripe', v1StripeRouter);
app.use('/api/v1/integration', v1IntegrationRouter);
app.use('/api/v1/integration-auth', v1IntegrationAuthRouter);
app.use('/api/v1/folder', v1SecretsFolder)
app.use('/api/v1/folders', v1SecretsFolder);
// v2 routes (improvements)
app.use('/api/v2/signup', v2SignupRouter);
@@ -143,34 +137,37 @@ const main = async () => {
app.use('/api/v2/api-key', v2APIKeyDataRouter);
// v3 routes (experimental)
app.use("/api/v3/auth", v3AuthRouter);
app.use('/api/v3/secrets', v3SecretsRouter);
app.use('/api/v3/workspaces', v3WorkspacesRouter);
app.use("/api/v3/signup", v3SignupRouter);
// api docs
app.use('/api-docs', swaggerUi.serve, swaggerUi.setup(swaggerFile))
app.use('/api-docs', swaggerUi.serve, swaggerUi.setup(swaggerFile));
// server status
app.use('/api', healthCheck)
app.use('/api', healthCheck);
//* Handle unrouted requests and respond with proper error message as well as status code
app.use((req, res, next) => {
if (res.headersSent) return next();
next(RouteNotFoundError({ message: `The requested source '(${req.method})${req.url}' was not found` }))
next(
RouteNotFoundError({
message: `The requested source '(${req.method})${req.url}' was not found`,
})
);
});
app.use(requestErrorHandler);
const server = app.listen(await getPort(), async () => {
(await getLogger("backend-main")).info(
(await getLogger('backend-main')).info(
`Server started listening at port ${await getPort()}`
);
});
// await createTestUserForDevelopment();
setUpHealthEndpoint(server);
server.on("close", async () => {
server.on('close', async () => {
await DatabaseService.closeDatabase();
});
backend/src/interfaces/services/SecretService/index.ts
+9 -7
View File
@@ -1,11 +1,12 @@
import { Types } from 'mongoose';
import { AuthData } from '../../middleware';
import { Types } from "mongoose";
import { AuthData } from "../../middleware";
export interface CreateSecretParams {
secretName: string;
workspaceId: Types.ObjectId;
environment: string;
type: 'shared' | 'personal';
folderId?: string;
type: "shared" | "personal";
authData: AuthData;
secretKeyCiphertext: string;
secretKeyIV: string;
@@ -28,7 +29,7 @@ export interface GetSecretParams {
secretName: string;
workspaceId: Types.ObjectId;
environment: string;
type?: 'shared' | 'personal';
type?: "shared" | "personal";
authData: AuthData;
}
@@ -36,17 +37,18 @@ export interface UpdateSecretParams {
secretName: string;
workspaceId: Types.ObjectId;
environment: string;
type: 'shared' | 'personal',
authData: AuthData
type: "shared" | "personal";
authData: AuthData;
secretValueCiphertext: string;
secretValueIV: string;
secretValueTag: string;
folderId?: string;
}
export interface DeleteSecretParams {
secretName: string;
workspaceId: Types.ObjectId;
environment: string;
type: 'shared' | 'personal';
type: "shared" | "personal";
authData: AuthData;
}
backend/src/middleware/requestErrorHandler.ts
+5
View File
@@ -7,6 +7,11 @@ import { getNodeEnv } from '../config';
export const requestErrorHandler: ErrorRequestHandler = async (error: RequestError | Error, req, res, next) => {
if (res.headersSent) return next();
if ((await getNodeEnv()) !== "production") {
/* eslint-disable no-console */
console.log(error)
/* eslint-enable no-console */
}
//TODO: Find better way to type check for error. In current setting you need to cast type to get the functions and variables from RequestError
if (!(error instanceof RequestError)) {
backend/src/models/folder.ts
+41 -21
View File
@@ -1,36 +1,56 @@
import { Schema, Types, model } from 'mongoose';
import { Schema, model, Types } from "mongoose";
const folderSchema = new Schema({
name: {
type: String,
export type TFolderRootSchema = {
_id: Types.ObjectId;
workspace: Types.ObjectId;
environment: string;
nodes: TFolderSchema;
};
export type TFolderSchema = {
id: string;
name: string;
version: number;
children: TFolderSchema[];
};
const folderSchema = new Schema<TFolderSchema>({
id: {
required: true,
type: String,
},
version: {
required: true,
type: Number,
default: 1,
},
name: {
required: true,
type: String,
default: "root",
},
});
folderSchema.add({ children: [folderSchema] });
const folderRootSchema = new Schema<TFolderRootSchema>(
{
workspace: {
type: Schema.Types.ObjectId,
ref: 'Workspace',
ref: "Workspace",
required: true,
},
environment: {
type: String,
required: true,
},
parent: {
type: Schema.Types.ObjectId,
ref: 'Folder',
required: false, // optional for root folders
nodes: folderSchema,
},
path: {
type: String,
required: true
},
parentPath: {
type: String,
required: true,
},
}, {
timestamps: true
});
{
timestamps: true,
}
);
const Folder = model('Folder', folderSchema);
const Folder = model<TFolderRootSchema>("Folder", folderRootSchema);
export default Folder;
backend/src/models/secret.ts
+39 -48
View File
@@ -1,12 +1,11 @@
import { Schema, model, Types } from 'mongoose';
import { Schema, model, Types } from "mongoose";
import {
SECRET_SHARED,
SECRET_PERSONAL,
ALGORITHM_AES_256_GCM,
ENCODING_SCHEME_UTF8,
ENCODING_SCHEME_BASE64
} from '../variables';
import { ROOT_FOLDER_PATH } from '../utils/folder';
ENCODING_SCHEME_BASE64,
} from "../variables";
export interface ISecret {
_id: Types.ObjectId;
@@ -28,11 +27,10 @@ export interface ISecret {
secretCommentIV?: string;
secretCommentTag?: string;
secretCommentHash?: string;
algorithm: 'aes-256-gcm';
keyEncoding: 'utf8' | 'base64';
algorithm: "aes-256-gcm";
keyEncoding: "utf8" | "base64";
tags?: string[];
path?: string;
folder?: Types.ObjectId;
folder?: string;
}
const secretSchema = new Schema<ISecret>(
@@ -40,115 +38,108 @@ const secretSchema = new Schema<ISecret>(
version: {
type: Number,
required: true,
default: 1
default: 1,
},
workspace: {
type: Schema.Types.ObjectId,
ref: 'Workspace',
required: true
ref: "Workspace",
required: true,
},
type: {
type: String,
enum: [SECRET_SHARED, SECRET_PERSONAL],
required: true
required: true,
},
user: {
// user associated with the personal secret
type: Schema.Types.ObjectId,
ref: 'User'
ref: "User",
},
tags: {
ref: 'Tag',
ref: "Tag",
type: [Schema.Types.ObjectId],
default: []
default: [],
},
environment: {
type: String,
required: true
required: true,
},
secretBlindIndex: {
type: String,
select: false
select: false,
},
secretKeyCiphertext: {
type: String,
required: true
required: true,
},
secretKeyIV: {
type: String, // symmetric
required: true
required: true,
},
secretKeyTag: {
type: String, // symmetric
required: true
required: true,
},
secretKeyHash: {
type: String
type: String,
},
secretValueCiphertext: {
type: String,
required: true
required: true,
},
secretValueIV: {
type: String, // symmetric
required: true
required: true,
},
secretValueTag: {
type: String, // symmetric
required: true
required: true,
},
secretValueHash: {
type: String
type: String,
},
secretCommentCiphertext: {
type: String,
required: false
required: false,
},
secretCommentIV: {
type: String, // symmetric
required: false
required: false,
},
secretCommentTag: {
type: String, // symmetric
required: false
required: false,
},
secretCommentHash: {
type: String,
required: false
required: false,
},
algorithm: { // the encryption algorithm used
algorithm: {
// the encryption algorithm used
type: String,
enum: [ALGORITHM_AES_256_GCM],
required: true
required: true,
default: ALGORITHM_AES_256_GCM,
},
keyEncoding: {
type: String,
enum: [
ENCODING_SCHEME_UTF8,
ENCODING_SCHEME_BASE64
],
required: true
enum: [ENCODING_SCHEME_UTF8, ENCODING_SCHEME_BASE64],
required: true,
default: ENCODING_SCHEME_UTF8,
},
// the full path to the secret in relation to folders
path: {
type: String,
required: false,
default: ROOT_FOLDER_PATH
},
folder: {
type: Schema.Types.ObjectId,
ref: 'Folder',
required: false,
type: String,
default: "root",
},
},
{
timestamps: true
timestamps: true,
}
);
secretSchema.index({ tags: 1 }, { background: true });
secretSchema.index({ tags: 1 }, { background: true })
const Secret = model<ISecret>('Secret', secretSchema);
const Secret = model<ISecret>("Secret", secretSchema);
export default Secret;
backend/src/routes/v1/secretsFolder.ts
+42 -22
View File
@@ -1,50 +1,70 @@
import express, { Request, Response } from 'express';
import express from "express";
const router = express.Router();
import {
requireAuth,
requireWorkspaceAuth,
validateRequest
} from '../../middleware';
import { body, param } from 'express-validator';
import { createFolder, deleteFolder, getFolderById } from '../../controllers/v1/secretsFolderController';
import { ADMIN, MEMBER } from '../../variables';
validateRequest,
} from "../../middleware";
import { body, param, query } from "express-validator";
import {
createFolder,
deleteFolder,
getFolders,
updateFolderById,
} from "../../controllers/v1/secretsFolderController";
import { ADMIN, MEMBER } from "../../variables";
router.post(
'/',
"/",
requireAuth({
acceptedAuthModes: ['jwt']
acceptedAuthModes: ["jwt"],
}),
requireWorkspaceAuth({
acceptedRoles: [ADMIN, MEMBER],
locationWorkspaceId: 'body'
locationWorkspaceId: "body",
}),
body('workspaceId').exists(),
body('environment').exists(),
body('folderName').exists(),
body('parentFolderId'),
body("workspaceId").exists(),
body("environment").exists(),
body("folderName").exists(),
body("parentFolderId"),
validateRequest,
createFolder
);
router.delete(
'/:folderId',
router.patch(
"/:folderId",
requireAuth({
acceptedAuthModes: ['jwt']
acceptedAuthModes: ["jwt"],
}),
param('folderId').exists(),
body("workspaceId").exists(),
body("environment").exists(),
param("folderId").not().isIn(["root"]).exists(),
validateRequest,
updateFolderById
);
router.delete(
"/:folderId",
requireAuth({
acceptedAuthModes: ["jwt"],
}),
body("workspaceId").exists(),
body("environment").exists(),
param("folderId").not().isIn(["root"]).exists(),
validateRequest,
deleteFolder
);
router.get(
'/:folderId',
"/",
requireAuth({
acceptedAuthModes: ['jwt']
acceptedAuthModes: ["jwt"],
}),
param('folderId').exists(),
query("workspaceId").exists().isString().trim(),
query("environment").exists().isString().trim(),
query("parentFolderId").optional().isString().trim(),
validateRequest,
getFolderById
getFolders
);
export default router;
backend/src/routes/v2/secrets.ts
+63 -34
View File
@@ -5,7 +5,7 @@ import {
requireAuth,
requireWorkspaceAuth,
requireSecretsAuth,
validateRequest
validateRequest,
} from '../../middleware';
import { validateClientForSecrets } from '../../validation';
import { query, body } from 'express-validator';
@@ -20,22 +20,25 @@ import {
AUTH_MODE_JWT,
AUTH_MODE_SERVICE_ACCOUNT,
AUTH_MODE_SERVICE_TOKEN,
AUTH_MODE_API_KEY
AUTH_MODE_API_KEY,
} from '../../variables';
import {
BatchSecretRequest
} from '../../types/secret';
import { BatchSecretRequest } from '../../types/secret';
router.post(
'/batch',
requireAuth({
acceptedAuthModes: [AUTH_MODE_JWT, AUTH_MODE_API_KEY, AUTH_MODE_SERVICE_TOKEN]
acceptedAuthModes: [
AUTH_MODE_JWT,
AUTH_MODE_API_KEY,
AUTH_MODE_SERVICE_TOKEN,
],
}),
requireWorkspaceAuth({
acceptedRoles: [ADMIN, MEMBER],
locationWorkspaceId: 'body'
locationWorkspaceId: 'body',
}),
body('workspaceId').exists().isString().trim(),
body('folderId').default('root').isString().trim(),
body('environment').exists().isString().trim(),
body('requests')
.exists()
@@ -43,13 +46,15 @@ router.post(
if (Array.isArray(requests)) {
const secretIds = requests
.map((request) => request.secret._id)
.filter((secretId) => secretId !== undefined)
.filter((secretId) => secretId !== undefined);
if (secretIds.length > 0) {
req.secrets = await validateClientForSecrets({
authData: req.authData,
secretIds: secretIds.map((secretId: string) => new Types.ObjectId(secretId)),
requiredPermissions: []
secretIds: secretIds.map(
(secretId: string) => new Types.ObjectId(secretId)
),
requiredPermissions: [],
});
}
}
@@ -63,24 +68,30 @@ router.post(
'/',
body('workspaceId').exists().isString().trim(),
body('environment').exists().isString().trim(),
body('folderId').default('root').isString().trim(),
body('secrets')
.exists()
.custom((value) => {
if (Array.isArray(value)) {
// case: create multiple secrets
if (value.length === 0) throw new Error('secrets cannot be an empty array')
if (value.length === 0)
throw new Error('secrets cannot be an empty array');
for (const secret of value) {
if (
!secret.type ||
!(secret.type === SECRET_PERSONAL || secret.type === SECRET_SHARED) ||
!(
secret.type === SECRET_PERSONAL || secret.type === SECRET_SHARED
) ||
!secret.secretKeyCiphertext ||
!secret.secretKeyIV ||
!secret.secretKeyTag ||
(typeof secret.secretValueCiphertext !== 'string') ||
typeof secret.secretValueCiphertext !== 'string' ||
!secret.secretValueIV ||
!secret.secretValueTag
) {
throw new Error('secrets array must contain objects that have required secret properties');
throw new Error(
'secrets array must contain objects that have required secret properties'
);
}
}
} else if (typeof value === 'object') {
@@ -95,23 +106,29 @@ router.post(
!value.secretValueIV ||
!value.secretValueTag
) {
throw new Error('secrets object is missing required secret properties');
throw new Error(
'secrets object is missing required secret properties'
);
}
} else {
throw new Error('secrets must be an object or an array of objects')
throw new Error('secrets must be an object or an array of objects');
}
return true;
}),
validateRequest,
requireAuth({
acceptedAuthModes: [AUTH_MODE_JWT, AUTH_MODE_API_KEY, AUTH_MODE_SERVICE_TOKEN]
acceptedAuthModes: [
AUTH_MODE_JWT,
AUTH_MODE_API_KEY,
AUTH_MODE_SERVICE_TOKEN,
],
}),
requireWorkspaceAuth({
acceptedRoles: [ADMIN, MEMBER],
locationWorkspaceId: 'body',
locationEnvironment: 'body',
requiredPermissions: [PERMISSION_WRITE_SECRETS]
requiredPermissions: [PERMISSION_WRITE_SECRETS],
}),
secretsController.createSecrets
);
@@ -121,15 +138,21 @@ router.get(
query('workspaceId').exists().trim(),
query('environment').exists().trim(),
query('tagSlugs'),
query('folderId').default('root').isString().trim(),
validateRequest,
requireAuth({
acceptedAuthModes: [AUTH_MODE_JWT, AUTH_MODE_API_KEY, AUTH_MODE_SERVICE_TOKEN, AUTH_MODE_SERVICE_ACCOUNT]
acceptedAuthModes: [
AUTH_MODE_JWT,
AUTH_MODE_API_KEY,
AUTH_MODE_SERVICE_TOKEN,
AUTH_MODE_SERVICE_ACCOUNT,
],
}),
requireWorkspaceAuth({
acceptedRoles: [ADMIN, MEMBER],
locationWorkspaceId: 'query',
locationEnvironment: 'query',
requiredPermissions: [PERMISSION_READ_SECRETS]
requiredPermissions: [PERMISSION_READ_SECRETS],
}),
secretsController.getSecrets
);
@@ -141,34 +164,35 @@ router.patch(
.custom((value) => {
if (Array.isArray(value)) {
// case: update multiple secrets
if (value.length === 0) throw new Error('secrets cannot be an empty array')
if (value.length === 0)
throw new Error('secrets cannot be an empty array');
for (const secret of value) {
if (
!secret.id
) {
if (!secret.id) {
throw new Error('Each secret must contain a ID property');
}
}
} else if (typeof value === 'object') {
// case: update 1 secret
if (
!value.id
) {
if (!value.id) {
throw new Error('secret must contain a ID property');
}
} else {
throw new Error('secrets must be an object or an array of objects')
throw new Error('secrets must be an object or an array of objects');
}
return true;
}),
validateRequest,
requireAuth({
acceptedAuthModes: [AUTH_MODE_JWT, AUTH_MODE_API_KEY, AUTH_MODE_SERVICE_TOKEN]
acceptedAuthModes: [
AUTH_MODE_JWT,
AUTH_MODE_API_KEY,
AUTH_MODE_SERVICE_TOKEN,
],
}),
requireSecretsAuth({
acceptedRoles: [ADMIN, MEMBER],
requiredPermissions: [PERMISSION_WRITE_SECRETS]
requiredPermissions: [PERMISSION_WRITE_SECRETS],
}),
secretsController.updateSecrets
);
@@ -183,8 +207,9 @@ router.delete(
if (Array.isArray(value)) {
// case: delete multiple secrets
if (value.length === 0) throw new Error('secrets cannot be an empty array');
return value.every((id: string) => typeof id === 'string')
if (value.length === 0)
throw new Error('secrets cannot be an empty array');
return value.every((id: string) => typeof id === 'string');
}
throw new Error('secretIds must be a string or an array of strings');
@@ -193,11 +218,15 @@ router.delete(
.isEmpty(),
validateRequest,
requireAuth({
acceptedAuthModes: [AUTH_MODE_JWT, AUTH_MODE_API_KEY, AUTH_MODE_SERVICE_TOKEN]
acceptedAuthModes: [
AUTH_MODE_JWT,
AUTH_MODE_API_KEY,
AUTH_MODE_SERVICE_TOKEN,
],
}),
requireSecretsAuth({
acceptedRoles: [ADMIN, MEMBER],
requiredPermissions: [PERMISSION_WRITE_SECRETS]
requiredPermissions: [PERMISSION_WRITE_SECRETS],
}),
secretsController.deleteSecrets
);
backend/src/services/FolderService.ts
+178
View File
@@ -0,0 +1,178 @@
import { nanoid } from "nanoid";
import { TFolderSchema } from "../models/folder";
type TAppendFolderDTO = {
folderName: string;
parentFolderId?: string;
};
type TRenameFolderDTO = {
folderName: string;
folderId: string;
};
export const validateFolderName = (folderName: string) => {
const validNameRegex = /^[a-zA-Z0-9-_]+$/;
return validNameRegex.test(folderName);
};
export const generateFolderId = (): string => nanoid(12);
// simple bfs search
export const searchByFolderId = (
root: TFolderSchema,
folderId: string
): TFolderSchema | undefined => {
const queue = [root];
while (queue.length) {
const folder = queue.pop() as TFolderSchema;
if (folder.id === folderId) {
return folder;
}
queue.push(...folder.children);
}
};
// bfs and then append to the folder
const appendChild = (folders: TFolderSchema, folderName: string) => {
const folder = folders.children.find(({ name }) => name === folderName);
if (folder) {
throw new Error("Folder already exists");
}
const id = generateFolderId();
folders.version += 1;
folders.children.push({
id,
name: folderName,
children: [],
version: 1,
});
return { id, name: folderName };
};
// root of append child wrapper
export const appendFolder = (
folders: TFolderSchema,
{ folderName, parentFolderId }: TAppendFolderDTO
) => {
const isRoot = !parentFolderId;
if (isRoot) {
return appendChild(folders, folderName);
}
const folder = searchByFolderId(folders, parentFolderId);
if (!folder) {
throw new Error("Parent Folder not found");
}
return appendChild(folder, folderName);
};
export const renameFolder = (
folders: TFolderSchema,
{ folderName, folderId }: TRenameFolderDTO
) => {
const folder = searchByFolderId(folders, folderId);
if (!folder) {
throw new Error("Folder doesn't exist");
}
folder.name = folderName;
};
// bfs but stops on parent folder
// Then unmount the required child and then return both
export const deleteFolderById = (folders: TFolderSchema, folderId: string) => {
const queue = [folders];
while (queue.length) {
const folder = queue.pop() as TFolderSchema;
folder.version += 1;
const index = folder.children.findIndex(({ id }) => folderId === id);
if (index !== -1) {
const deletedFolder = folder.children.splice(index, 1);
return { deletedNode: deletedFolder[0], parent: folder };
}
queue.push(...folder.children);
}
};
// bfs but return parent of the folderID
export const getParentFromFolderId = (
folders: TFolderSchema,
folderId: string
) => {
const queue = [folders];
while (queue.length) {
const folder = queue.pop() as TFolderSchema;
const index = folder.children.findIndex(({ id }) => folderId === id);
if (index !== -1) return folder;
queue.push(...folder.children);
}
};
// to get all folders ids from everything from below nodes
export const getAllFolderIds = (folders: TFolderSchema) => {
const folderIds: Array<{ id: string; name: string }> = [];
const queue = [folders];
while (queue.length) {
const folder = queue.pop() as TFolderSchema;
folderIds.push({ id: folder.id, name: folder.name });
queue.push(...folder.children);
}
return folderIds;
};
// To get the path of a folder from the root. Used for breadcrumbs
// LOGIC: We do dfs instead if bfs
// Each time we go down we record the current node
// We then record the number of childs of each root node
// When we reach leaf node or when all childs of a root node are visited
// We remove it from path recorded by using the total child record
export const searchByFolderIdWithDir = (
folders: TFolderSchema,
folderId: string
) => {
const stack = [folders];
const dir: Array<{ id: string; name: string }> = [];
const hits: Record<string, number> = {};
while (stack.length) {
const folder = stack.shift() as TFolderSchema;
// score the hit
hits[folder.id] = folder.children.length;
const parent = dir[dir.length - 1];
if (parent) hits[parent.id] -= 1;
if (folder.id === folderId) {
dir.push({ name: folder.name, id: folder.id });
return { folder, dir };
}
if (folder.children.length) {
dir.push({ name: folder.name, id: folder.id });
stack.unshift(...folder.children);
} else {
if (!hits[parent.id]) {
dir.pop();
}
}
}
return;
};
// to get folder of a path given
// Like /frontend/folder#1
export const getFolderByPath = (folders: TFolderSchema, searchPath: string) => {
const path = searchPath.split("/").filter(Boolean);
const queue = [folders];
let segment: TFolderSchema | undefined;
while (queue.length && path.length) {
const folder = queue.pop();
const segmentPath = path.shift();
segment = folder?.children.find(({ name }) => name === segmentPath);
if (!segment) return;
queue.push(segment);
}
return segment;
};
backend/src/types/secret/index.d.ts
Vendored
+17 -24
View File
@@ -1,29 +1,37 @@
import { Types } from 'mongoose';
import { Assign, Omit } from 'utility-types';
import { ISecret } from '../../models';
import { mongo } from 'mongoose';
import { Types } from "mongoose";
import { Assign, Omit } from "utility-types";
import { ISecret } from "../../models";
import { mongo } from "mongoose";
// Everything is required, except the omitted types
export type CreateSecretRequestBody = Omit<ISecret, "user" | "version" | "environment" | "workspace">;
export type CreateSecretRequestBody = Omit<
ISecret,
"user" | "version" | "environment" | "workspace"
>;
// Omit the listed properties, then make everything optional and then make _id required
export type ModifySecretRequestBody = Assign<Partial<Omit<ISecret, "user" | "version" | "environment" | "workspace">>, { _id: string }>;
export type ModifySecretRequestBody = Assign<
Partial<Omit<ISecret, "user" | "version" | "environment" | "workspace">>,
{ _id: string }
>;
// Used for modeling sanitized secrets before uplaod. To be used for converting user input for uploading
export type SanitizedSecretModify = Partial<Omit<ISecret, "user" | "version" | "environment" | "workspace">>;
export type SanitizedSecretModify = Partial<
Omit<ISecret, "user" | "version" | "environment" | "workspace">
>;
// Everything is required, except the omitted types
export type SanitizedSecretForCreate = Omit<ISecret, "version" | "_id">;
export interface BatchSecretRequest {
id: string;
method: 'POST' | 'PATCH' | 'DELETE';
method: "POST" | "PATCH" | "DELETE";
secret: Secret;
}
export interface BatchSecret {
_id: string;
type: 'shared' | 'personal',
type: "shared" | "personal";
secretBlindIndex: string;
secretKeyCiphertext: string;
secretKeyIV: string;
@@ -36,18 +44,3 @@ export interface BatchSecret {
secretCommentTag: string;
tags: string[];
}
export interface BatchSecret {
_id: string;
type: 'shared' | 'personal',
secretKeyCiphertext: string;
secretKeyIV: string;
secretKeyTag: string;
secretValueCiphertext: string;
secretValueIV: string;
secretValueTag: string;
secretCommentCiphertext: string;
secretCommentIV: string;
secretCommentTag: string;
tags: string[];
}
backend/src/utils/folder.ts
+71 -71
View File
@@ -1,87 +1,87 @@
import Folder from "../models/folder";
// import Folder from "../models/folder";
export const ROOT_FOLDER_PATH = "/"
// export const ROOT_FOLDER_PATH = "/"
export const getFolderPath = async (folderId: string) => {
let currentFolder = await Folder.findById(folderId);
const pathSegments = [];
// export const getFolderPath = async (folderId: string) => {
// let currentFolder = await Folder.findById(folderId);
// const pathSegments = [];
while (currentFolder) {
pathSegments.unshift(currentFolder.name);
currentFolder = currentFolder.parent ? await Folder.findById(currentFolder.parent) : null;
}
// while (currentFolder) {
// pathSegments.unshift(currentFolder.name);
// currentFolder = currentFolder.parent ? await Folder.findById(currentFolder.parent) : null;
// }
return '/' + pathSegments.join('/');
};
// return '/' + pathSegments.join('/');
// };
/**
Returns the folder ID associated with the specified secret path in the given workspace and environment.
@param workspaceId - The ID of the workspace to search in.
@param environment - The environment to search in.
@param secretPath - The secret path to search for.
@returns The folder ID associated with the specified secret path, or undefined if the path is at the root folder level.
@throws Error if the specified secret path is not found.
*/
export const getFolderIdFromPath = async (workspaceId: string, environment: string, secretPath: string) => {
const secretPathParts = secretPath.split("/").filter(path => path != "")
if (secretPathParts.length <= 1) {
return undefined // root folder, so no folder id
}
// /**
// Returns the folder ID associated with the specified secret path in the given workspace and environment.
// @param workspaceId - The ID of the workspace to search in.
// @param environment - The environment to search in.
// @param secretPath - The secret path to search for.
// @returns The folder ID associated with the specified secret path, or undefined if the path is at the root folder level.
// @throws Error if the specified secret path is not found.
// */
// export const getFolderIdFromPath = async (workspaceId: string, environment: string, secretPath: string) => {
// const secretPathParts = secretPath.split("/").filter(path => path != "")
// if (secretPathParts.length <= 1) {
// return undefined // root folder, so no folder id
// }
const folderId = await Folder.find({ path: secretPath, workspace: workspaceId, environment: environment })
if (!folderId) {
throw Error("Secret path not found")
}
// const folderId = await Folder.find({ path: secretPath, workspace: workspaceId, environment: environment })
// if (!folderId) {
// throw Error("Secret path not found")
// }
return folderId
}
// return folderId
// }
/**
* Cleans up a path by removing empty parts, duplicate slashes,
* and ensuring it starts with ROOT_FOLDER_PATH.
* @param path - The input path to clean up.
* @returns The cleaned-up path string.
*/
export const normalizePath = (path: string) => {
if (path == undefined || path == "" || path == ROOT_FOLDER_PATH) {
return ROOT_FOLDER_PATH
}
// /**
// * Cleans up a path by removing empty parts, duplicate slashes,
// * and ensuring it starts with ROOT_FOLDER_PATH.
// * @param path - The input path to clean up.
// * @returns The cleaned-up path string.
// */
// export const normalizePath = (path: string) => {
// if (path == undefined || path == "" || path == ROOT_FOLDER_PATH) {
// return ROOT_FOLDER_PATH
// }
const pathParts = path.split("/").filter(part => part != "")
const cleanPathString = ROOT_FOLDER_PATH + pathParts.join("/")
// const pathParts = path.split("/").filter(part => part != "")
// const cleanPathString = ROOT_FOLDER_PATH + pathParts.join("/")
return cleanPathString
}
// return cleanPathString
// }
export const getFoldersInDirectory = async (workspaceId: string, environment: string, pathString: string) => {
const normalizedPath = normalizePath(pathString)
const foldersInDirectory = await Folder.find({
workspace: workspaceId,
environment: environment,
parentPath: normalizedPath,
});
// export const getFoldersInDirectory = async (workspaceId: string, environment: string, pathString: string) => {
// const normalizedPath = normalizePath(pathString)
// const foldersInDirectory = await Folder.find({
// workspace: workspaceId,
// environment: environment,
// parentPath: normalizedPath,
// });
return foldersInDirectory;
}
// return foldersInDirectory;
// }
/**
* Returns the parent path of the given path.
* @param path - The input path.
* @returns The parent path string.
*/
export const getParentPath = (path: string) => {
const normalizedPath = normalizePath(path);
const folderParts = normalizedPath.split('/').filter(part => part !== '');
// /**
// * Returns the parent path of the given path.
// * @param path - The input path.
// * @returns The parent path string.
// */
// export const getParentPath = (path: string) => {
// const normalizedPath = normalizePath(path);
// const folderParts = normalizedPath.split('/').filter(part => part !== '');
let folderParent = ROOT_FOLDER_PATH;
if (folderParts.length > 1) {
folderParent = ROOT_FOLDER_PATH + folderParts.slice(0, folderParts.length - 1).join('/');
}
// let folderParent = ROOT_FOLDER_PATH;
// if (folderParts.length > 1) {
// folderParent = ROOT_FOLDER_PATH + folderParts.slice(0, folderParts.length - 1).join('/');
// }
return folderParent;
}
// return folderParent;
// }
export const validateFolderName = (folderName: string) => {
const validNameRegex = /^[a-zA-Z0-9-_]+$/;
return validNameRegex.test(folderName);
}
// export const validateFolderName = (folderName: string) => {
// const validNameRegex = /^[a-zA-Z0-9-_]+$/;
// return validNameRegex.test(folderName);
// }
backend/src/utils/patchAsyncRoutes.js
+69
View File
@@ -0,0 +1,69 @@
/*
Original work Copyright (c) 2016, Nikolay Nemshilov <[email protected]>
Modified work Copyright (c) 2016, David Banham <[email protected]>
Permission to use, copy, modify, and/or distribute this software for any purpose
with or without fee is hereby granted, provided that the above copyright notice
and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF
THIS SOFTWARE.
*/
/* eslint-disable @typescript-eslint/no-var-requires */
/* eslint-env node */
const Layer = require('express/lib/router/layer');
const Router = require('express/lib/router');
const last = (arr = []) => arr[arr.length - 1];
const noop = Function.prototype;
function copyFnProps(oldFn, newFn) {
Object.keys(oldFn).forEach((key) => {
newFn[key] = oldFn[key];
});
return newFn;
}
function wrap(fn) {
const newFn = function newFn(...args) {
const ret = fn.apply(this, args);
const next = (args.length === 5 ? args[2] : last(args)) || noop;
if (ret && ret.catch) ret.catch(err => next(err));
return ret;
};
Object.defineProperty(newFn, 'length', {
value: fn.length,
writable: false,
});
return copyFnProps(fn, newFn);
}
function patchRouterParam() {
const originalParam = Router.prototype.constructor.param;
Router.prototype.constructor.param = function param(name, fn) {
fn = wrap(fn);
return originalParam.call(this, name, fn);
};
}
Object.defineProperty(Layer.prototype, 'handle', {
enumerable: true,
get() {
return this.__handle;
},
set(fn) {
fn = wrap(fn);
this.__handle = fn;
},
});
module.exports = {
patchRouterParam
};
backend/src/utils/setup/backfillData.ts
+125 -76
View File
@@ -1,7 +1,8 @@
import crypto from 'crypto';
import { encryptSymmetric128BitHexKeyUTF8 } from '../crypto';
import { EESecretService } from '../../ee/services';
import { SecretVersion } from '../../ee/models';
import crypto from "crypto";
import { Types } from "mongoose";
import { encryptSymmetric128BitHexKeyUTF8 } from "../crypto";
import { EESecretService } from "../../ee/services";
import { ISecretVersion, SecretSnapshot, SecretVersion } from "../../ee/models";
import {
Secret,
ISecret,
@@ -10,21 +11,15 @@ import {
Bot,
BackupPrivateKey,
IntegrationAuth,
} from '../../models';
import {
generateKeyPair
} from '../../utils/crypto';
import {
client,
getEncryptionKey,
getRootEncryptionKey
} from '../../config';
} from "../../models";
import { generateKeyPair } from "../../utils/crypto";
import { client, getEncryptionKey, getRootEncryptionKey } from "../../config";
import {
ALGORITHM_AES_256_GCM,
ENCODING_SCHEME_UTF8,
ENCODING_SCHEME_BASE64
} from '../../variables';
import { InternalServerError } from '../errors';
ENCODING_SCHEME_BASE64,
} from "../../variables";
import { InternalServerError } from "../errors";
/**
* Backfill secrets to ensure that they're all versioned and have
@@ -64,12 +59,12 @@ export const backfillSecretVersions = async () => {
workspace: s.workspace,
environment: s.environment,
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_UTF8
keyEncoding: ENCODING_SCHEME_UTF8,
})
),
});
}
}
};
/**
* Backfill workspace bots to ensure that every workspace has a bot
@@ -78,11 +73,11 @@ export const backfillBots = async () => {
const encryptionKey = await getEncryptionKey();
const rootEncryptionKey = await getRootEncryptionKey();
const workspaceIdsWithBot = await Bot.distinct('workspace');
const workspaceIdsToAddBot = await Workspace.distinct('_id', {
const workspaceIdsWithBot = await Bot.distinct("workspace");
const workspaceIdsToAddBot = await Workspace.distinct("_id", {
_id: {
$nin: workspaceIdsWithBot
}
$nin: workspaceIdsWithBot,
},
});
if (workspaceIdsToAddBot.length === 0) return;
@@ -95,11 +90,11 @@ export const backfillBots = async () => {
const {
ciphertext: encryptedPrivateKey,
iv,
tag
tag,
} = client.encryptSymmetric(privateKey, rootEncryptionKey);
return new Bot({
name: 'Infisical Bot',
name: "Infisical Bot",
workspace: workspaceToAddBot,
isActive: false,
publicKey,
@@ -107,20 +102,20 @@ export const backfillBots = async () => {
iv,
tag,
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_BASE64
keyEncoding: ENCODING_SCHEME_BASE64,
});
} else if (encryptionKey) {
const {
ciphertext: encryptedPrivateKey,
iv,
tag
tag,
} = encryptSymmetric128BitHexKeyUTF8({
plaintext: privateKey,
key: encryptionKey
key: encryptionKey,
});
return new Bot({
name: 'Infisical Bot',
name: "Infisical Bot",
workspace: workspaceToAddBot,
isActive: false,
publicKey,
@@ -128,47 +123,49 @@ export const backfillBots = async () => {
iv,
tag,
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_UTF8
keyEncoding: ENCODING_SCHEME_UTF8,
});
}
throw InternalServerError({
message: 'Failed to backfill workspace bots due to missing encryption key'
message:
"Failed to backfill workspace bots due to missing encryption key",
});
})
);
await Bot.insertMany(botsToInsert);
}
};
/**
* Backfill secret blind index data to ensure that every workspace
* has a secret blind index data
*/
export const backfillSecretBlindIndexData = async () => {
const encryptionKey = await getEncryptionKey();
const rootEncryptionKey = await getRootEncryptionKey();
const workspaceIdsBlindIndexed = await SecretBlindIndexData.distinct('workspace');
const workspaceIdsToBlindIndex = await Workspace.distinct('_id', {
const workspaceIdsBlindIndexed = await SecretBlindIndexData.distinct(
"workspace"
);
const workspaceIdsToBlindIndex = await Workspace.distinct("_id", {
_id: {
$nin: workspaceIdsBlindIndexed
}
$nin: workspaceIdsBlindIndexed,
},
});
if (workspaceIdsToBlindIndex.length === 0) return;
const secretBlindIndexDataToInsert = await Promise.all(
workspaceIdsToBlindIndex.map(async (workspaceToBlindIndex) => {
const salt = crypto.randomBytes(16).toString('base64');
const salt = crypto.randomBytes(16).toString("base64");
if (rootEncryptionKey) {
const {
ciphertext: encryptedSaltCiphertext,
iv: saltIV,
tag: saltTag
} = client.encryptSymmetric(salt, rootEncryptionKey)
tag: saltTag,
} = client.encryptSymmetric(salt, rootEncryptionKey);
return new SecretBlindIndexData({
workspace: workspaceToBlindIndex,
@@ -176,16 +173,16 @@ export const backfillSecretBlindIndexData = async () => {
saltIV,
saltTag,
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_BASE64
keyEncoding: ENCODING_SCHEME_BASE64,
});
} else if (encryptionKey) {
const {
ciphertext: encryptedSaltCiphertext,
iv: saltIV,
tag: saltTag
tag: saltTag,
} = encryptSymmetric128BitHexKeyUTF8({
plaintext: salt,
key: encryptionKey
key: encryptionKey,
});
return new SecretBlindIndexData({
@@ -194,18 +191,19 @@ export const backfillSecretBlindIndexData = async () => {
saltIV,
saltTag,
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_UTF8
keyEncoding: ENCODING_SCHEME_UTF8,
});
}
throw InternalServerError({
message: 'Failed to backfill secret blind index data due to missing encryption key'
message:
"Failed to backfill secret blind index data due to missing encryption key",
});
})
);
SecretBlindIndexData.insertMany(secretBlindIndexDataToInsert);
}
};
/**
* Backfill Secret, SecretVersion, SecretBlindIndexData, Bot,
@@ -213,22 +211,21 @@ export const backfillSecretBlindIndexData = async () => {
* they all have encryption metadata documented
*/
export const backfillEncryptionMetadata = async () => {
// backfill secret encryption metadata
await Secret.updateMany(
{
algorithm: {
$exists: false
$exists: false,
},
keyEncoding: {
$exists: false
}
$exists: false,
},
},
{
$set: {
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_UTF8
}
keyEncoding: ENCODING_SCHEME_UTF8,
},
}
);
@@ -236,17 +233,17 @@ export const backfillEncryptionMetadata = async () => {
await SecretVersion.updateMany(
{
algorithm: {
$exists: false
$exists: false,
},
keyEncoding: {
$exists: false
}
$exists: false,
},
},
{
$set: {
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_UTF8
}
keyEncoding: ENCODING_SCHEME_UTF8,
},
}
);
@@ -254,17 +251,17 @@ export const backfillEncryptionMetadata = async () => {
await SecretBlindIndexData.updateMany(
{
algorithm: {
$exists: false
$exists: false,
},
keyEncoding: {
$exists: false
}
$exists: false,
},
},
{
$set: {
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_UTF8
}
keyEncoding: ENCODING_SCHEME_UTF8,
},
}
);
@@ -272,17 +269,17 @@ export const backfillEncryptionMetadata = async () => {
await Bot.updateMany(
{
algorithm: {
$exists: false
$exists: false,
},
keyEncoding: {
$exists: false
}
$exists: false,
},
},
{
$set: {
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_UTF8
}
keyEncoding: ENCODING_SCHEME_UTF8,
},
}
);
@@ -290,17 +287,17 @@ export const backfillEncryptionMetadata = async () => {
await BackupPrivateKey.updateMany(
{
algorithm: {
$exists: false
$exists: false,
},
keyEncoding: {
$exists: false
}
$exists: false,
},
},
{
$set: {
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_UTF8
}
keyEncoding: ENCODING_SCHEME_UTF8,
},
}
);
@@ -308,17 +305,69 @@ export const backfillEncryptionMetadata = async () => {
await IntegrationAuth.updateMany(
{
algorithm: {
$exists: false
$exists: false,
},
keyEncoding: {
$exists: false
}
$exists: false,
},
},
{
$set: {
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_UTF8
}
keyEncoding: ENCODING_SCHEME_UTF8,
},
}
);
};
export const backfillSecretFolders = async () => {
await Secret.updateMany(
{
folder: {
$exists: false,
},
},
{
$set: {
folder: "root",
},
}
);
let secretSnapshots = await SecretSnapshot.find({
environment: {
$exists: false,
},
})
.populate<{ secretVersions: ISecretVersion[] }>("secretVersions")
.limit(50);
while (secretSnapshots.length > 0) {
for (const secSnapshot of secretSnapshots) {
const groupSnapByEnv: Record<string, Array<ISecretVersion>> = {};
secSnapshot.secretVersions.forEach((secVer) => {
if (!groupSnapByEnv?.[secVer.environment])
groupSnapByEnv[secVer.environment] = [];
groupSnapByEnv[secVer.environment].push(secVer);
});
const newSnapshots = Object.keys(groupSnapByEnv).map((snapEnv) => ({
...secSnapshot.toObject({ virtuals: false }),
_id: new Types.ObjectId(),
environment: snapEnv,
secretVersions: groupSnapByEnv[snapEnv],
}));
await SecretSnapshot.insertMany(newSnapshots);
await secSnapshot.delete();
}
secretSnapshots = await SecretSnapshot.find({
environment: {
$exists: false,
},
})
.populate<{ secretVersions: ISecretVersion[] }>("secretVersions")
.limit(50);
}
};
backend/src/utils/setup/index.ts
+22 -12
View File
@@ -3,26 +3,27 @@ import { DatabaseService, TelemetryService } from '../../services';
import { setTransporter } from '../../helpers/nodemailer';
import { EELicenseService } from '../../ee/services';
import { initSmtp } from '../../services/smtp';
import { createTestUserForDevelopment } from '../addDevelopmentUser'
import {
validateEncryptionKeysConfig
} from './validateConfig';
import { createTestUserForDevelopment } from '../addDevelopmentUser';
// eslint-disable-next-line @typescript-eslint/no-var-requires
const { patchRouterParam } = require('../patchAsyncRoutes');
import { validateEncryptionKeysConfig } from './validateConfig';
import {
backfillSecretVersions,
backfillBots,
backfillSecretBlindIndexData,
backfillEncryptionMetadata
backfillEncryptionMetadata,
backfillSecretFolders,
} from './backfillData';
import {
reencryptBotPrivateKeys,
reencryptSecretBlindIndexDataSalts
reencryptSecretBlindIndexDataSalts,
} from './reencryptData';
import {
getNodeEnv,
getMongoURL,
getSentryDSN,
getClientSecretGoogle,
getClientIdGoogle
getClientIdGoogle,
} from '../../config';
import { initializePassport } from '../auth';
@@ -37,6 +38,7 @@ import { initializePassport } from '../auth';
* - Re-encrypting data
*/
export const setup = async () => {
patchRouterParam();
await validateEncryptionKeysConfig();
await TelemetryService.logTelemetryMessage();
@@ -56,6 +58,14 @@ export const setup = async () => {
await initializePassport();
}
// re-encrypt any data previously encrypted under server hex 128-bit ENCRYPTION_KEY
// to base64 256-bit ROOT_ENCRYPTION_KEY
// await reencryptBotPrivateKeys();
// await reencryptSecretBlindIndexDataSalts();
// initializing the database connection
await DatabaseService.initDatabase(await getMongoURL());
/**
* NOTE: the order in this setup function is critical.
* It is important to backfill data before performing any re-encryption functionality.
@@ -66,20 +76,20 @@ export const setup = async () => {
await backfillBots();
await backfillSecretBlindIndexData();
await backfillEncryptionMetadata();
await backfillSecretFolders();
// re-encrypt any data previously encrypted under server hex 128-bit ENCRYPTION_KEY
// to base64 256-bit ROOT_ENCRYPTION_KEY
// await reencryptBotPrivateKeys();
// await reencryptSecretBlindIndexDataSalts();
await reencryptBotPrivateKeys();
await reencryptSecretBlindIndexDataSalts();
// initializing Sentry
Sentry.init({
dsn: await getSentryDSN(),
tracesSampleRate: 1.0,
debug: (await getNodeEnv()) === 'production' ? false : true,
environment: (await getNodeEnv())
environment: await getNodeEnv(),
});
await createTestUserForDevelopment();
}
};