feat(secret-import): implemented api for secret import

This commit is contained in:
akhilmhdh
2023-07-17 23:08:42 +05:30
parent 38b6a48bee
commit 202900a7a3
12 changed files with 438 additions and 50 deletions

View File

@@ -15,6 +15,7 @@ import * as userController from "./userController";
import * as workspaceController from "./workspaceController";
import * as secretScanningController from "./secretScanningController";
import * as webhookController from "./webhookController";
import * as secretImportController from "./secretImportController";
export {
authController,
@@ -33,5 +34,6 @@ export {
userController,
workspaceController,
secretScanningController,
webhookController
webhookController,
secretImportController
};

View File

@@ -0,0 +1,117 @@
import { Request, Response } from "express";
import { validateMembership } from "../../helpers";
import SecretImport from "../../models/secretImports";
import { getAllImportedSecrets } from "../../services/SecretImportService";
import { BadRequestError } from "../../utils/errors";
import { ADMIN, MEMBER } from "../../variables";
export const createSecretImport = async (req: Request, res: Response) => {
const { workspaceId, environment, folderId, secretImport } = req.body;
const importSecDoc = await SecretImport.findOne({
workspace: workspaceId,
environment,
folderId
});
if (!importSecDoc) {
const doc = new SecretImport({
workspace: workspaceId,
environment,
folderId,
imports: [{ environment: secretImport.environment, secretPath: secretImport.secretPath }]
});
await doc.save();
return res.status(200).json({ message: "successfully created secret import" });
}
const doesImportExist = importSecDoc.imports.find(
(el) => el.environment === secretImport.environment && el.secretPath === secretImport.secretPath
);
if (doesImportExist) {
throw BadRequestError({ message: "Secret import already exist" });
}
importSecDoc.imports.push({
environment: secretImport.environment,
secretPath: secretImport.secretPath
});
await importSecDoc.save();
return res.status(200).json({ message: "successfully created secret import" });
};
// to keep the ordering, you must pass all the imports in here not the only updated one
// this is because the order decide which import gets overriden
export const updateSecretImport = async (req: Request, res: Response) => {
const { id } = req.params;
const { secretImports } = req.body;
const importSecDoc = await SecretImport.findById(id);
if (!importSecDoc) {
throw BadRequestError({ message: "Import not found" });
}
await validateMembership({
userId: req.user._id.toString(),
workspaceId: importSecDoc.workspace,
acceptedRoles: [ADMIN, MEMBER]
});
importSecDoc.imports = secretImports;
await importSecDoc.save();
return res.status(200).json({ message: "successfully updated secret import" });
};
export const deleteSecretImport = async (req: Request, res: Response) => {
const { id } = req.params;
const { secretImportEnv, secretImportPath } = req.body;
const importSecDoc = await SecretImport.findById(id);
if (!importSecDoc) {
throw BadRequestError({ message: "Import not found" });
}
await validateMembership({
userId: req.user._id.toString(),
workspaceId: importSecDoc.workspace,
acceptedRoles: [ADMIN, MEMBER]
});
importSecDoc.imports = importSecDoc.imports.filter(
({ environment, secretPath }) =>
!(environment === secretImportEnv && secretPath === secretImportPath)
);
await importSecDoc.save();
return res.status(200).json({ message: "successfully delete secret import" });
};
export const getSecretImports = async (req: Request, res: Response) => {
const { workspaceId, environment, folderId } = req.query;
const importSecDoc = await SecretImport.findOne({
workspace: workspaceId,
environment,
folderId
});
if (!importSecDoc) {
return res.status(200).json({ secretImport: {} });
}
return res.status(200).json({ secretImport: importSecDoc });
};
export const getAllSecretsFromImport = async (req: Request, res: Response) => {
const { workspaceId, environment, folderId } = req.query as {
workspaceId: string;
environment: string;
folderId: string;
};
const importSecDoc = await SecretImport.findOne({
workspace: workspaceId,
environment,
folderId
});
if (!importSecDoc) {
return res.status(200).json({ secrets: {} });
}
const secrets = await getAllImportedSecrets(workspaceId, environment, folderId);
return res.status(200).json({ secrets });
};

View File

@@ -35,6 +35,7 @@ import {
} from "../../services/FolderService";
import { isValidScope } from "../../helpers/secrets";
import path from "path";
import { getAllImportedSecrets } from "../../services/SecretImportService";
/**
* Peform a batch of any specified CUD secret operations
@@ -690,7 +691,7 @@ export const getSecrets = async (req: Request, res: Response) => {
}
*/
const { tagSlugs, secretPath } = req.query;
const { tagSlugs, secretPath, include_imports } = req.query;
let { folderId } = req.query;
const workspaceId = req.query.workspaceId as string;
const environment = req.query.environment as string;
@@ -827,6 +828,12 @@ export const getSecrets = async (req: Request, res: Response) => {
secrets = await Secret.find(secretQuery).populate("tags");
}
// TODO(akhilmhdh) - secret-imp change this to org type
let importedSecrets: any[] = [];
if (include_imports) {
importedSecrets = await getAllImportedSecrets(workspaceId, environment, folderId as string);
}
const channel = getChannelFromUserAgent(req.headers["user-agent"]);
const readAction = await EELogService.createAction({
@@ -868,7 +875,8 @@ export const getSecrets = async (req: Request, res: Response) => {
}
return res.status(200).send({
secrets
secrets,
...(include_imports && { imports: importedSecrets })
});
};

View File

@@ -5,6 +5,10 @@ import { eventPushSecrets } from "../../events";
import { BotService } from "../../services";
import { repackageSecretToRaw } from "../../helpers/secrets";
import { encryptSymmetric128BitHexKeyUTF8 } from "../../utils/crypto";
import { getAllImportedSecrets } from "../../services/SecretImportService";
import Folder from "../../models/folder";
import { getFolderByPath } from "../../services/FolderService";
import { BadRequestError } from "../../utils/errors";
/**
* Return secrets for workspace with id [workspaceId] and environment
@@ -16,6 +20,7 @@ export const getSecretsRaw = async (req: Request, res: Response) => {
const workspaceId = req.query.workspaceId as string;
const environment = req.query.environment as string;
const secretPath = req.query.secretPath as string;
const includeImports = req.query.include_imports as string;
const secrets = await SecretService.getSecrets({
workspaceId: new Types.ObjectId(workspaceId),
@@ -28,13 +33,38 @@ export const getSecretsRaw = async (req: Request, res: Response) => {
workspaceId: new Types.ObjectId(workspaceId)
});
if (includeImports) {
const folders = await Folder.findOne({ workspace: workspaceId, environment });
let folderId = "root";
// if folder exist get it and replace folderid with new one
if (folders) {
const folder = getFolderByPath(folders.nodes, secretPath as string);
if (!folder) {
throw BadRequestError({ message: "Folder not found" });
}
folderId = folder.id;
}
const importedSecrets = await getAllImportedSecrets(workspaceId, environment, folderId);
return res.status(200).send({
secrets: secrets.map((secret) =>
repackageSecretToRaw({
secret,
key
})
),
imports: importedSecrets.map((el) => ({
...el,
secrets: el.secrets.map((secret) => repackageSecretToRaw({ secret, key }))
}))
});
}
return res.status(200).send({
secrets: secrets.map((secret) => {
const rep = repackageSecretToRaw({
secret,
key
});
return rep;
})
});
@@ -232,6 +262,7 @@ export const getSecrets = async (req: Request, res: Response) => {
const workspaceId = req.query.workspaceId as string;
const environment = req.query.environment as string;
const secretPath = req.query.secretPath as string;
const includeImports = req.query.include_imports as string;
const secrets = await SecretService.getSecrets({
workspaceId: new Types.ObjectId(workspaceId),
@@ -240,6 +271,24 @@ export const getSecrets = async (req: Request, res: Response) => {
authData: req.authData
});
if (includeImports) {
const folders = await Folder.findOne({ workspace: workspaceId, environment });
let folderId = "root";
// if folder exist get it and replace folderid with new one
if (folders) {
const folder = getFolderByPath(folders.nodes, secretPath as string);
if (!folder) {
throw BadRequestError({ message: "Folder not found" });
}
folderId = folder.id;
}
const importedSecrets = await getAllImportedSecrets(workspaceId, environment, folderId);
return res.status(200).send({
secrets,
imports: importedSecrets
});
}
return res.status(200).send({
secrets
});

View File

@@ -41,7 +41,8 @@ import {
userAction as v1UserActionRouter,
user as v1UserRouter,
workspace as v1WorkspaceRouter,
webhooks as v1WebhooksRouter
webhooks as v1WebhooksRouter,
secretImport as v1SecretImportRouter
} from "./routes/v1";
import {
auth as v2AuthRouter,
@@ -128,6 +129,7 @@ const main = async () => {
app.use("/api/v1/folders", v1SecretsFolder);
app.use("/api/v1/secret-scanning", v1SecretScanningRouter);
app.use("/api/v1/webhooks", v1WebhooksRouter);
app.use("/api/v1/secret-imports", v1SecretImportRouter);
// v2 routes (improvements)
app.use("/api/v2/signup", v2SignupRouter);

View File

@@ -0,0 +1,52 @@
import { Schema, Types, model } from "mongoose";
export interface ISecretImports {
_id: Types.ObjectId;
workspace: Types.ObjectId;
environment: string;
folderId: string;
imports: Array<{
environment: string;
secretPath: string;
}>;
}
const secretImportSchema = new Schema<ISecretImports>(
{
workspace: {
type: Schema.Types.ObjectId,
ref: "Workspace",
required: true
},
environment: {
type: String,
required: true
},
folderId: {
type: String,
required: true,
default: "root"
},
imports: {
type: [
{
environment: {
type: String,
required: true
},
secretPath: {
type: String,
required: true
}
}
],
default: []
}
},
{
timestamps: true
}
);
const SecretImport = model<ISecretImports>("SecretImports", secretImportSchema);
export default SecretImport;

View File

@@ -17,6 +17,7 @@ import integrationAuth from "./integrationAuth";
import secretsFolder from "./secretsFolder";
import secretScanning from "./secretScanning";
import webhooks from "./webhook";
import secretImport from "./secretImport";
export {
signup,
@@ -37,5 +38,6 @@ export {
integrationAuth,
secretsFolder,
secretScanning,
webhooks
webhooks,
secretImport
};

View File

@@ -0,0 +1,76 @@
import express from "express";
const router = express.Router();
import { body, param, query } from "express-validator";
import { secretImportController } from "../../controllers/v1";
import { requireAuth, requireWorkspaceAuth, validateRequest } from "../../middleware";
import { ADMIN, AUTH_MODE_JWT, MEMBER } from "../../variables";
router.post(
"/",
requireAuth({
acceptedAuthModes: [AUTH_MODE_JWT]
}),
requireWorkspaceAuth({
acceptedRoles: [ADMIN, MEMBER],
locationWorkspaceId: "body"
}),
body("workspaceId").exists().isString().trim().notEmpty(),
body("environment").exists().isString().trim().notEmpty(),
body("folderId").default("root").isString().trim(),
body("secretImport").exists().isObject(),
body("secretImport.environment").isString().exists().trim(),
body("secretImport.secretPath").isString().exists().trim(),
validateRequest,
secretImportController.createSecretImport
);
router.put(
"/:id",
requireAuth({
acceptedAuthModes: [AUTH_MODE_JWT]
}),
param("id").exists().isString().trim(),
body("secretImports").exists().isArray(),
body("secretImports.*.environment").isString().exists().trim(),
body("secretImports.*.secretPath").isString().exists().trim(),
validateRequest,
secretImportController.updateSecretImport
);
router.delete(
"/:id",
requireAuth({
acceptedAuthModes: [AUTH_MODE_JWT]
}),
param("id").exists().isString().trim(),
body("secretImportPath").isString().exists().trim(),
body("secretImportEnv").isString().exists().trim(),
validateRequest,
secretImportController.updateSecretImport
);
router.get(
"/",
requireAuth({
acceptedAuthModes: [AUTH_MODE_JWT]
}),
query("workspaceId").exists().isString().trim().notEmpty(),
query("environment").exists().isString().trim().notEmpty(),
query("folderId").default("root").isString().trim(),
validateRequest,
secretImportController.getSecretImports
);
router.get(
"/secrets",
requireAuth({
acceptedAuthModes: [AUTH_MODE_JWT]
}),
query("workspaceId").exists().isString().trim().notEmpty(),
query("environment").exists().isString().trim().notEmpty(),
query("folderId").default("root").isString().trim(),
validateRequest,
secretImportController.getAllSecretsFromImport
);
export default router;

View File

@@ -127,6 +127,7 @@ router.get(
query("tagSlugs"),
query("folderId").default("root").isString().trim(),
query("secretPath").optional().isString().trim(),
query("include_imports").optional().default(false).isBoolean(),
validateRequest,
requireAuth({
acceptedAuthModes: [

View File

@@ -1,10 +1,6 @@
import express from "express";
const router = express.Router();
import {
requireAuth,
requireWorkspaceAuth,
validateRequest,
} from "../../middleware";
import { requireAuth, requireWorkspaceAuth, validateRequest } from "../../middleware";
import { body, param, query } from "express-validator";
import { secretsController } from "../../controllers/v3";
import {
@@ -17,7 +13,7 @@ import {
PERMISSION_READ_SECRETS,
PERMISSION_WRITE_SECRETS,
SECRET_PERSONAL,
SECRET_SHARED,
SECRET_SHARED
} from "../../variables";
router.get(
@@ -25,14 +21,15 @@ router.get(
query("workspaceId").exists().isString().trim(),
query("environment").exists().isString().trim(),
query("secretPath").default("/").isString().trim(),
query("include_imports").isBoolean().default(false),
validateRequest,
requireAuth({
acceptedAuthModes: [
AUTH_MODE_JWT,
AUTH_MODE_API_KEY,
AUTH_MODE_SERVICE_TOKEN,
AUTH_MODE_SERVICE_ACCOUNT,
],
AUTH_MODE_SERVICE_ACCOUNT
]
}),
requireWorkspaceAuth({
acceptedRoles: [ADMIN, MEMBER],
@@ -40,7 +37,7 @@ router.get(
locationEnvironment: "query",
requiredPermissions: [PERMISSION_READ_SECRETS],
requireBlindIndicesEnabled: true,
requireE2EEOff: true,
requireE2EEOff: true
}),
secretsController.getSecretsRaw
);
@@ -58,8 +55,8 @@ router.get(
AUTH_MODE_JWT,
AUTH_MODE_API_KEY,
AUTH_MODE_SERVICE_TOKEN,
AUTH_MODE_SERVICE_ACCOUNT,
],
AUTH_MODE_SERVICE_ACCOUNT
]
}),
requireWorkspaceAuth({
acceptedRoles: [ADMIN, MEMBER],
@@ -67,7 +64,7 @@ router.get(
locationEnvironment: "query",
requiredPermissions: [PERMISSION_READ_SECRETS],
requireBlindIndicesEnabled: true,
requireE2EEOff: true,
requireE2EEOff: true
}),
secretsController.getSecretByNameRaw
);
@@ -86,8 +83,8 @@ router.post(
AUTH_MODE_JWT,
AUTH_MODE_API_KEY,
AUTH_MODE_SERVICE_TOKEN,
AUTH_MODE_SERVICE_ACCOUNT,
],
AUTH_MODE_SERVICE_ACCOUNT
]
}),
requireWorkspaceAuth({
acceptedRoles: [ADMIN, MEMBER],
@@ -95,7 +92,7 @@ router.post(
locationEnvironment: "body",
requiredPermissions: [PERMISSION_WRITE_SECRETS],
requireBlindIndicesEnabled: true,
requireE2EEOff: true,
requireE2EEOff: true
}),
secretsController.createSecretRaw
);
@@ -114,8 +111,8 @@ router.patch(
AUTH_MODE_JWT,
AUTH_MODE_API_KEY,
AUTH_MODE_SERVICE_TOKEN,
AUTH_MODE_SERVICE_ACCOUNT,
],
AUTH_MODE_SERVICE_ACCOUNT
]
}),
requireWorkspaceAuth({
acceptedRoles: [ADMIN, MEMBER],
@@ -123,7 +120,7 @@ router.patch(
locationEnvironment: "body",
requiredPermissions: [PERMISSION_WRITE_SECRETS],
requireBlindIndicesEnabled: true,
requireE2EEOff: true,
requireE2EEOff: true
}),
secretsController.updateSecretByNameRaw
);
@@ -141,8 +138,8 @@ router.delete(
AUTH_MODE_JWT,
AUTH_MODE_API_KEY,
AUTH_MODE_SERVICE_TOKEN,
AUTH_MODE_SERVICE_ACCOUNT,
],
AUTH_MODE_SERVICE_ACCOUNT
]
}),
requireWorkspaceAuth({
acceptedRoles: [ADMIN, MEMBER],
@@ -150,7 +147,7 @@ router.delete(
locationEnvironment: "body",
requiredPermissions: [PERMISSION_WRITE_SECRETS],
requireBlindIndicesEnabled: true,
requireE2EEOff: true,
requireE2EEOff: true
}),
secretsController.deleteSecretByNameRaw
);
@@ -166,8 +163,8 @@ router.get(
AUTH_MODE_JWT,
AUTH_MODE_API_KEY,
AUTH_MODE_SERVICE_TOKEN,
AUTH_MODE_SERVICE_ACCOUNT,
],
AUTH_MODE_SERVICE_ACCOUNT
]
}),
requireWorkspaceAuth({
acceptedRoles: [ADMIN, MEMBER],
@@ -175,7 +172,7 @@ router.get(
locationEnvironment: "query",
requiredPermissions: [PERMISSION_READ_SECRETS],
requireBlindIndicesEnabled: true,
requireE2EEOff: false,
requireE2EEOff: false
}),
secretsController.getSecrets
);
@@ -201,8 +198,8 @@ router.post(
AUTH_MODE_JWT,
AUTH_MODE_API_KEY,
AUTH_MODE_SERVICE_TOKEN,
AUTH_MODE_SERVICE_ACCOUNT,
],
AUTH_MODE_SERVICE_ACCOUNT
]
}),
requireWorkspaceAuth({
acceptedRoles: [ADMIN, MEMBER],
@@ -210,7 +207,7 @@ router.post(
locationEnvironment: "body",
requiredPermissions: [PERMISSION_WRITE_SECRETS],
requireBlindIndicesEnabled: true,
requireE2EEOff: false,
requireE2EEOff: false
}),
secretsController.createSecret
);
@@ -228,15 +225,15 @@ router.get(
AUTH_MODE_JWT,
AUTH_MODE_API_KEY,
AUTH_MODE_SERVICE_TOKEN,
AUTH_MODE_SERVICE_ACCOUNT,
],
AUTH_MODE_SERVICE_ACCOUNT
]
}),
requireWorkspaceAuth({
acceptedRoles: [ADMIN, MEMBER],
locationWorkspaceId: "query",
locationEnvironment: "query",
requiredPermissions: [PERMISSION_READ_SECRETS],
requireBlindIndicesEnabled: true,
requireBlindIndicesEnabled: true
}),
secretsController.getSecretByName
);
@@ -257,8 +254,8 @@ router.patch(
AUTH_MODE_JWT,
AUTH_MODE_API_KEY,
AUTH_MODE_SERVICE_TOKEN,
AUTH_MODE_SERVICE_ACCOUNT,
],
AUTH_MODE_SERVICE_ACCOUNT
]
}),
requireWorkspaceAuth({
acceptedRoles: [ADMIN, MEMBER],
@@ -266,7 +263,7 @@ router.patch(
locationEnvironment: "body",
requiredPermissions: [PERMISSION_WRITE_SECRETS],
requireBlindIndicesEnabled: true,
requireE2EEOff: false,
requireE2EEOff: false
}),
secretsController.updateSecretByName
);
@@ -284,8 +281,8 @@ router.delete(
AUTH_MODE_JWT,
AUTH_MODE_API_KEY,
AUTH_MODE_SERVICE_TOKEN,
AUTH_MODE_SERVICE_ACCOUNT,
],
AUTH_MODE_SERVICE_ACCOUNT
]
}),
requireWorkspaceAuth({
acceptedRoles: [ADMIN, MEMBER],
@@ -293,7 +290,7 @@ router.delete(
locationEnvironment: "body",
requiredPermissions: [PERMISSION_WRITE_SECRETS],
requireBlindIndicesEnabled: true,
requireE2EEOff: false,
requireE2EEOff: false
}),
secretsController.deleteSecretByName
);

View File

@@ -0,0 +1,82 @@
import { Types } from "mongoose";
import Folder from "../models/folder";
import Secret, { ISecret } from "../models/secret";
import SecretImport from "../models/secretImports";
import { getFolderByPath } from "./FolderService";
type TSecretImportFid = { environment: string; folderId: string; secretPath: string };
export const getAllImportedSecrets = async (
workspaceId: string,
environment: string,
folderId = "root"
) => {
const secImports = await SecretImport.findOne({
workspace: workspaceId,
environment,
folderId
});
if (!secImports) return [];
if (secImports.imports.length === 0) return [];
const importedEnv: Record<string, boolean> = {}; // to get folders from all environment
secImports.imports.forEach((el) => (importedEnv[el.environment] = true));
const folders = await Folder.find({
workspace: workspaceId,
environment: { $in: Object.keys(importedEnv) }
});
const importedSecByFid: TSecretImportFid[] = [];
secImports.imports.forEach((el) => {
const folder = folders.find((fl) => fl.environment === el.environment);
if (folder) {
const secPathFolder = getFolderByPath(folder.nodes, el.secretPath);
if (secPathFolder)
importedSecByFid.push({
environment: el.environment,
folderId: secPathFolder.id,
secretPath: el.secretPath
});
}
});
if (importedSecByFid.length === 0) return [];
const secsGroupedByRef = await Secret.aggregate([
{
$match: {
workspace: new Types.ObjectId(workspaceId),
type: "shared"
}
},
{
$group: {
_id: {
environment: "$environment",
folderId: "$folder"
},
secrets: { $push: "$$ROOT" }
}
},
{
$match: {
$or: importedSecByFid.map(({ environment, folderId: fid }) => ({
"_id.environment": environment,
"_id.folderId": fid
}))
}
}
]);
// now let stitch together secrets.
const importedSecrets: Array<TSecretImportFid & { secrets: ISecret[] }> = [];
importedSecByFid.forEach(({ environment, folderId, secretPath }) => {
const secretsGrouped = secsGroupedByRef.find(
(el) => el._id.environment === environment && el._id.folderId === folderId
);
if (secretsGrouped) {
importedSecrets.push({ secretPath, folderId, environment, secrets: secretsGrouped.secrets });
}
});
return importedSecrets;
};

View File

@@ -8,11 +8,11 @@ import TokenService from "./TokenService";
import SecretService from "./SecretService";
export {
TelemetryService,
DatabaseService,
BotService,
EventService,
IntegrationService,
TokenService,
SecretService,
}
TelemetryService,
DatabaseService,
BotService,
EventService,
IntegrationService,
TokenService,
SecretService
};