Merge remote-tracking branch 'origin' into stv3-org-roles

This commit is contained in:
Tuan Dang
2023-12-04 16:14:31 +07:00
40 changed files with 2029 additions and 331 deletions
+112 -29
View File
@@ -6,13 +6,7 @@ import { BotService } from "../../services";
import { containsGlobPatterns, repackageSecretToRaw } from "../../helpers/secrets";
import { encryptSymmetric128BitHexKeyUTF8 } from "../../utils/crypto";
import { getAllImportedSecrets } from "../../services/SecretImportService";
import {
Folder,
IServiceTokenData,
Membership,
ServiceTokenData,
User
} from "../../models";
import { Folder, IServiceTokenData, Membership, ServiceTokenData, User } from "../../models";
import { getFolderByPath } from "../../services/FolderService";
import { BadRequestError } from "../../utils/errors";
import { validateRequest } from "../../helpers/validation";
@@ -34,6 +28,7 @@ import {
} from "../../ee/services/SecretApprovalService";
import { CommitType } from "../../ee/models/secretApprovalRequest";
import { logger } from "../../utils/logging";
import { createReminder, deleteReminder } from "../../helpers/reminder";
const checkSecretsPermission = async ({
authData,
@@ -197,10 +192,11 @@ export const getSecretsRaw = async (req: Request, res: Response) => {
query: { include_imports: includeImports }
} = validatedData;
logger.info(`getSecretsRaw: fetch raw secrets [environment=${environment}] [workspaceId=${workspaceId}] [secretPath=${secretPath}] [includeImports=${includeImports}]`)
logger.info(
`getSecretsRaw: fetch raw secrets [environment=${environment}] [workspaceId=${workspaceId}] [secretPath=${secretPath}] [includeImports=${includeImports}]`
);
if (req.authData.authPayload instanceof ServiceTokenData) {
// if the service token has single scope, it will get all secrets for that scope by default
const serviceTokenDetails: IServiceTokenData = req?.serviceTokenData;
if (
@@ -356,7 +352,9 @@ export const getSecretByNameRaw = async (req: Request, res: Response) => {
params: { secretName }
} = await validateRequest(reqValidator.GetSecretByNameRawV3, req);
logger.info(`getSecretByNameRaw: fetch raw secret by name [environment=${environment}] [workspaceId=${workspaceId}] [secretPath=${secretPath}] [type=${type}] [include_imports=${include_imports}]`)
logger.info(
`getSecretByNameRaw: fetch raw secret by name [environment=${environment}] [workspaceId=${workspaceId}] [secretPath=${secretPath}] [type=${type}] [include_imports=${include_imports}]`
);
await checkSecretsPermission({
authData: req.authData,
@@ -481,7 +479,9 @@ export const createSecretRaw = async (req: Request, res: Response) => {
}
} = await validateRequest(reqValidator.CreateSecretRawV3, req);
logger.info(`createSecretRaw: create a secret raw by name and value [environment=${environment}] [workspaceId=${workspaceId}] [secretPath=${secretPath}] [type=${type}] [skipMultilineEncoding=${skipMultilineEncoding}]`)
logger.info(
`createSecretRaw: create a secret raw by name and value [environment=${environment}] [workspaceId=${workspaceId}] [secretPath=${secretPath}] [type=${type}] [skipMultilineEncoding=${skipMultilineEncoding}]`
);
await checkSecretsPermission({
authData: req.authData,
@@ -628,7 +628,9 @@ export const updateSecretByNameRaw = async (req: Request, res: Response) => {
body: { workspaceId, environment, secretValue, secretPath, type, skipMultilineEncoding }
} = await validateRequest(reqValidator.UpdateSecretByNameRawV3, req);
logger.info(`updateSecretByNameRaw: update raw secret by name [environment=${environment}] [workspaceId=${workspaceId}] [secretPath=${secretPath}] [type=${type}] [skipMultilineEncoding=${skipMultilineEncoding}]`)
logger.info(
`updateSecretByNameRaw: update raw secret by name [environment=${environment}] [workspaceId=${workspaceId}] [secretPath=${secretPath}] [type=${type}] [skipMultilineEncoding=${skipMultilineEncoding}]`
);
await checkSecretsPermission({
authData: req.authData,
@@ -752,7 +754,9 @@ export const deleteSecretByNameRaw = async (req: Request, res: Response) => {
body: { environment, secretPath, type, workspaceId }
} = await validateRequest(reqValidator.DeleteSecretByNameRawV3, req);
logger.info(`deleteSecretByNameRaw: delete a secret by name [environment=${environment}] [workspaceId=${workspaceId}] [secretPath=${secretPath}] [type=${type}]`)
logger.info(
`deleteSecretByNameRaw: delete a secret by name [environment=${environment}] [workspaceId=${workspaceId}] [secretPath=${secretPath}] [type=${type}]`
);
await checkSecretsPermission({
authData: req.authData,
@@ -807,7 +811,9 @@ export const getSecrets = async (req: Request, res: Response) => {
query: { secretPath }
} = validatedData;
logger.info(`getSecrets: fetch encrypted secrets [environment=${environment}] [workspaceId=${workspaceId}] [includeImports=${includeImports}]`)
logger.info(
`getSecrets: fetch encrypted secrets [environment=${environment}] [workspaceId=${workspaceId}] [includeImports=${includeImports}]`
);
const { authVerifier: permissionCheckFn } = await checkSecretsPermission({
authData: req.authData,
@@ -863,7 +869,9 @@ export const getSecretByName = async (req: Request, res: Response) => {
params: { secretName }
} = await validateRequest(reqValidator.GetSecretByNameV3, req);
logger.info(`getSecretByName: get a single secret by name [environment=${environment}] [workspaceId=${workspaceId}] [include_imports=${include_imports}] [type=${type}]`)
logger.info(
`getSecretByName: get a single secret by name [environment=${environment}] [workspaceId=${workspaceId}] [include_imports=${include_imports}] [type=${type}]`
);
await checkSecretsPermission({
authData: req.authData,
@@ -915,7 +923,9 @@ export const createSecret = async (req: Request, res: Response) => {
params: { secretName }
} = await validateRequest(reqValidator.CreateSecretV3, req);
logger.info(`createSecret: create an encrypted secret [environment=${environment}] [workspaceId=${workspaceId}] [skipMultilineEncoding=${skipMultilineEncoding}] [type=${type}]`)
logger.info(
`createSecret: create an encrypted secret [environment=${environment}] [workspaceId=${workspaceId}] [skipMultilineEncoding=${skipMultilineEncoding}] [type=${type}]`
);
await checkSecretsPermission({
authData: req.authData,
@@ -932,7 +942,11 @@ export const createSecret = async (req: Request, res: Response) => {
});
if (membership && type !== "personal") {
const secretApprovalPolicy = await getSecretPolicyOfBoard(workspaceId, environment, secretPath);
const secretApprovalPolicy = await getSecretPolicyOfBoard(
workspaceId,
environment,
secretPath
);
if (secretApprovalPolicy) {
const secretApprovalRequest = await generateSecretApprovalRequest({
workspaceId,
@@ -1024,12 +1038,16 @@ export const updateSecretByName = async (req: Request, res: Response) => {
secretKeyIV,
secretKeyTag,
secretKeyCiphertext,
skipMultilineEncoding
skipMultilineEncoding,
secretReminderRepeatDays,
secretReminderNote
},
params: { secretName }
} = await validateRequest(reqValidator.UpdateSecretByNameV3, req);
logger.info(`updateSecretByName: update a encrypted secret by name [environment=${environment}] [workspaceId=${workspaceId}] [skipMultilineEncoding=${skipMultilineEncoding}] [type=${type}]`)
logger.info(
`updateSecretByName: update a encrypted secret by name [environment=${environment}] [workspaceId=${workspaceId}] [skipMultilineEncoding=${skipMultilineEncoding}] [type=${type}]`
);
if (newSecretName && (!secretKeyIV || !secretKeyTag || !secretKeyCiphertext)) {
throw BadRequestError({ message: "Missing encrypted key" });
@@ -1050,7 +1068,11 @@ export const updateSecretByName = async (req: Request, res: Response) => {
});
if (membership && type !== "personal") {
const secretApprovalPolicy = await getSecretPolicyOfBoard(workspaceId, environment, secretPath);
const secretApprovalPolicy = await getSecretPolicyOfBoard(
workspaceId,
environment,
secretPath
);
if (secretApprovalPolicy) {
const secretApprovalRequest = await generateSecretApprovalRequest({
workspaceId,
@@ -1084,6 +1106,41 @@ export const updateSecretByName = async (req: Request, res: Response) => {
}
}
if (type !== "personal") {
const existingSecret = await SecretService.getSecret({
secretName,
workspaceId: new Types.ObjectId(workspaceId),
environment,
type,
secretPath,
authData: req.authData
});
if (secretReminderRepeatDays !== undefined) {
if (
(secretReminderRepeatDays &&
existingSecret.secretReminderRepeatDays !== secretReminderRepeatDays) ||
(secretReminderNote && existingSecret.secretReminderNote !== secretReminderNote)
) {
await createReminder(existingSecret, {
_id: existingSecret._id,
secretReminderRepeatDays,
secretReminderNote,
workspace: existingSecret.workspace
});
} else if (
secretReminderRepeatDays === null &&
secretReminderNote === null &&
existingSecret.secretReminderRepeatDays
) {
await deleteReminder({
_id: existingSecret._id,
secretReminderRepeatDays: existingSecret.secretReminderRepeatDays
});
}
}
}
const secret = await SecretService.updateSecret({
secretName,
workspaceId: new Types.ObjectId(workspaceId),
@@ -1094,6 +1151,8 @@ export const updateSecretByName = async (req: Request, res: Response) => {
newSecretName,
secretValueCiphertext,
secretValueIV,
secretReminderRepeatDays,
secretReminderNote,
secretValueTag,
secretPath,
tags,
@@ -1130,7 +1189,9 @@ export const deleteSecretByName = async (req: Request, res: Response) => {
params: { secretName }
} = await validateRequest(reqValidator.DeleteSecretByNameV3, req);
logger.info(`deleteSecretByName: delete a encrypted secret by name [environment=${environment}] [workspaceId=${workspaceId}] [type=${type}]`)
logger.info(
`deleteSecretByName: delete a encrypted secret by name [environment=${environment}] [workspaceId=${workspaceId}] [type=${type}]`
);
await checkSecretsPermission({
authData: req.authData,
@@ -1147,7 +1208,11 @@ export const deleteSecretByName = async (req: Request, res: Response) => {
});
if (membership && type !== "personal") {
const secretApprovalPolicy = await getSecretPolicyOfBoard(workspaceId, environment, secretPath);
const secretApprovalPolicy = await getSecretPolicyOfBoard(
workspaceId,
environment,
secretPath
);
if (secretApprovalPolicy) {
const secretApprovalRequest = await generateSecretApprovalRequest({
workspaceId,
@@ -1197,7 +1262,9 @@ export const createSecretByNameBatch = async (req: Request, res: Response) => {
body: { secrets, secretPath, environment, workspaceId }
} = await validateRequest(reqValidator.CreateSecretByNameBatchV3, req);
logger.info(`createSecretByNameBatch: create a list of secrets by their names [environment=${environment}] [workspaceId=${workspaceId}] [secretsLength=${secrets?.length}]`)
logger.info(
`createSecretByNameBatch: create a list of secrets by their names [environment=${environment}] [workspaceId=${workspaceId}] [secretsLength=${secrets?.length}]`
);
await checkSecretsPermission({
authData: req.authData,
@@ -1214,7 +1281,11 @@ export const createSecretByNameBatch = async (req: Request, res: Response) => {
});
if (membership) {
const secretApprovalPolicy = await getSecretPolicyOfBoard(workspaceId, environment, secretPath);
const secretApprovalPolicy = await getSecretPolicyOfBoard(
workspaceId,
environment,
secretPath
);
if (secretApprovalPolicy) {
const secretApprovalRequest = await generateSecretApprovalRequest({
workspaceId,
@@ -1258,7 +1329,9 @@ export const updateSecretByNameBatch = async (req: Request, res: Response) => {
body: { secrets, secretPath, environment, workspaceId }
} = await validateRequest(reqValidator.UpdateSecretByNameBatchV3, req);
logger.info(`updateSecretByNameBatch: update a list of secrets by their names [environment=${environment}] [workspaceId=${workspaceId}] [secretsLength=${secrets?.length}]`)
logger.info(
`updateSecretByNameBatch: update a list of secrets by their names [environment=${environment}] [workspaceId=${workspaceId}] [secretsLength=${secrets?.length}]`
);
await checkSecretsPermission({
authData: req.authData,
@@ -1275,7 +1348,11 @@ export const updateSecretByNameBatch = async (req: Request, res: Response) => {
});
if (membership) {
const secretApprovalPolicy = await getSecretPolicyOfBoard(workspaceId, environment, secretPath);
const secretApprovalPolicy = await getSecretPolicyOfBoard(
workspaceId,
environment,
secretPath
);
if (secretApprovalPolicy) {
const secretApprovalRequest = await generateSecretApprovalRequest({
workspaceId,
@@ -1319,7 +1396,9 @@ export const deleteSecretByNameBatch = async (req: Request, res: Response) => {
body: { secrets, secretPath, environment, workspaceId }
} = await validateRequest(reqValidator.DeleteSecretByNameBatchV3, req);
logger.info(`deleteSecretByNameBatch: delete a list of secrets by their names [environment=${environment}] [workspaceId=${workspaceId}] [secretsLength=${secrets?.length}]`)
logger.info(
`deleteSecretByNameBatch: delete a list of secrets by their names [environment=${environment}] [workspaceId=${workspaceId}] [secretsLength=${secrets?.length}]`
);
await checkSecretsPermission({
authData: req.authData,
@@ -1336,7 +1415,11 @@ export const deleteSecretByNameBatch = async (req: Request, res: Response) => {
});
if (membership) {
const secretApprovalPolicy = await getSecretPolicyOfBoard(workspaceId, environment, secretPath);
const secretApprovalPolicy = await getSecretPolicyOfBoard(
workspaceId,
environment,
secretPath
);
if (secretApprovalPolicy) {
const secretApprovalRequest = await generateSecretApprovalRequest({
workspaceId,
@@ -1373,4 +1456,4 @@ export const deleteSecretByNameBatch = async (req: Request, res: Response) => {
return res.status(200).send({
secrets: deletedSecrets
});
};
};
+58
View File
@@ -0,0 +1,58 @@
import { ISecret } from "../models";
import {
createRecurringSecretReminder,
deleteRecurringSecretReminder,
updateRecurringSecretReminder
} from "../queues/reminders/sendSecretReminders";
type TPartialSecret = Pick<
ISecret,
"_id" | "secretReminderRepeatDays" | "secretReminderNote" | "workspace"
>;
type TPartialSecretDeleteReminder = Pick<ISecret, "_id" | "secretReminderRepeatDays">;
export const createReminder = async (oldSecret: TPartialSecret, newSecret: TPartialSecret) => {
if (oldSecret._id !== newSecret._id) {
throw new Error("Secret id's don't match");
}
if (!newSecret.secretReminderRepeatDays) {
throw new Error("No repeat days provided");
}
const secretId = oldSecret._id.toString();
const workspaceId = oldSecret.workspace.toString();
if (oldSecret.secretReminderRepeatDays) {
// This will first delete the existing recurring job, and then create a new one.
await updateRecurringSecretReminder({
workspaceId,
secretId,
repeatDays: newSecret.secretReminderRepeatDays,
note: newSecret.secretReminderNote
});
} else {
// This will create a new recurring job.
await createRecurringSecretReminder({
workspaceId,
secretId,
repeatDays: newSecret.secretReminderRepeatDays,
note: newSecret.secretReminderNote
});
}
};
export const deleteReminder = async (secret: TPartialSecretDeleteReminder) => {
if (!secret._id) {
throw new Error("No secret id provided");
}
if (!secret.secretReminderRepeatDays) {
throw new Error("No repeat days provided");
}
await deleteRecurringSecretReminder({
secretId: secret._id.toString(),
repeatDays: secret.secretReminderRepeatDays
});
};
+20 -12
View File
@@ -574,18 +574,20 @@ export const getSecretsHelper = async ({
const approximateForNoneCapturedEvents = secrets.length * 10;
if (shouldCapture) {
postHogClient.capture({
event: "secrets pulled",
distinctId: await TelemetryService.getDistinctId({ authData }),
properties: {
numberOfSecrets: shouldRecordK8Event ? approximateForNoneCapturedEvents : secrets.length,
environment,
workspaceId,
folderId,
channel: authData.userAgentType,
userAgent: authData.userAgent
}
});
if (workspaceId.toString() != "650e71fbae3e6c8572f436d4") {
postHogClient.capture({
event: "secrets pulled",
distinctId: await TelemetryService.getDistinctId({ authData }),
properties: {
numberOfSecrets: shouldRecordK8Event ? approximateForNoneCapturedEvents : secrets.length,
environment,
workspaceId,
folderId,
channel: authData.userAgentType,
userAgent: authData.userAgent
}
});
}
}
}
@@ -717,6 +719,8 @@ export const updateSecretHelper = async ({
secretValueIV,
secretValueTag,
secretPath,
secretReminderRepeatDays,
secretReminderNote,
tags,
secretCommentCiphertext,
secretCommentIV,
@@ -781,6 +785,10 @@ export const updateSecretHelper = async ({
secretCommentIV,
secretCommentTag,
secretCommentCiphertext,
secretReminderRepeatDays,
secretReminderNote,
skipMultilineEncoding,
secretBlindIndex: newSecretNameBlindIndex,
secretKeyIV,
@@ -58,6 +58,10 @@ export interface UpdateSecretParams {
secretCommentCiphertext?: string;
secretCommentIV?: string;
secretCommentTag?: string;
secretReminderRepeatDays?: number | null;
secretReminderNote?: string | null;
skipMultilineEncoding?: boolean;
tags?: string[];
}
+19
View File
@@ -27,6 +27,12 @@ export interface ISecret {
secretCommentIV?: string;
secretCommentTag?: string;
secretCommentHash?: string;
// ? NOTE: This works great for workspace-level reminders.
// ? If we want to do it on a user-basis, we should ideally have a seperate model for reminders.
secretReminderRepeatDays?: number | null;
secretReminderNote?: string | null;
skipMultilineEncoding?: boolean;
algorithm: "aes-256-gcm";
keyEncoding: "utf8" | "base64";
@@ -118,10 +124,23 @@ const secretSchema = new Schema<ISecret>(
type: String,
required: false
},
secretReminderRepeatDays: {
type: Number,
required: false,
default: null
},
secretReminderNote: {
type: String,
required: false,
default: null
},
skipMultilineEncoding: {
type: Boolean,
required: false
},
algorithm: {
// the encryption algorithm used
type: String,
@@ -0,0 +1,83 @@
import Queue, { Job } from "bull";
import { IUser, Membership, Organization, Workspace } from "../../models";
import { Types } from "mongoose";
import { sendMail } from "../../helpers";
type TSendSecretReminders = {
workspaceId: string;
secretId: string;
repeatDays: number;
note: string | undefined | null;
};
type TDeleteSecretReminder = {
secretId: string;
repeatDays: number;
};
const DAY_IN_MS = 86400000;
export const sendSecretReminders = new Queue(
"send-secret-reminders",
process.env.REDIS_URL as string
);
sendSecretReminders.process(async (job: Job<TSendSecretReminders>) => {
const { workspaceId }: TSendSecretReminders = job.data;
const workspace = await Workspace.findById(new Types.ObjectId(workspaceId));
const organization = await Organization.findById(new Types.ObjectId(workspace?.organization));
if (!workspace) {
throw new Error("Workspace for reminder not found");
}
if (!organization) {
throw new Error("Organization for reminder not found");
}
const memberships = await Membership.find({
workspace: workspaceId
}).populate<{ user: IUser }>("user");
await sendMail({
template: "secretReminder.handlebars",
subjectLine: "Infisical secret reminder",
recipients: [...memberships.map((membership) => membership.user.email)],
substitutions: {
reminderNote: job.data.note, // May not be present.
workspaceName: workspace.name,
organizationName: organization.name
}
});
});
export const createRecurringSecretReminder = (jobDetails: TSendSecretReminders) => {
const repeat = jobDetails.repeatDays * DAY_IN_MS;
return sendSecretReminders.add(jobDetails, {
delay: repeat,
repeat: {
every: repeat
},
jobId: `reminder-${jobDetails.secretId}`,
removeOnComplete: true,
removeOnFail: {
count: 20
}
});
};
export const deleteRecurringSecretReminder = (jobDetails: TDeleteSecretReminder) => {
const repeat = jobDetails.repeatDays * DAY_IN_MS;
return sendSecretReminders.removeRepeatable({
every: repeat,
jobId: `reminder-${jobDetails.secretId}`
});
};
export const updateRecurringSecretReminder = async (jobDetails: TSendSecretReminders) => {
// We need to delete the potentially existing reminder job first, or the new one won't be created.
await deleteRecurringSecretReminder(jobDetails);
await createRecurringSecretReminder(jobDetails);
};
@@ -0,0 +1,19 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta http-equiv="x-ua-compatible" content="ie=edge">
<title>Secret Reminder</title>
</head>
<body>
<h2>Infisical</h2>
<h2>You have a new secret reminder!</h2>
<p>You have a new secret reminder from workspace "{{workspaceName}}", in {{organizationName}}</p>
{{#if reminderNote}}
<p>Here's the note included with the reminder: {{reminderNote}}</p>
{{/if}}
</body>
</html>
+6 -1
View File
@@ -10,7 +10,6 @@ import { AuthData } from "../interfaces/middleware";
import { ActorType } from "../ee/models";
import { z } from "zod";
import { SECRET_PERSONAL, SECRET_SHARED } from "../variables";
/**
* Validate authenticated clients for secrets with id [secretId] based
* on any known permissions.
@@ -260,6 +259,7 @@ export const CreateSecretRawV3 = z.object({
.string()
.transform((val) => (val.at(-1) === "\n" ? `${val.trim()}\n` : val.trim())),
secretComment: z.string().trim().optional().default(""),
skipMultilineEncoding: z.boolean().optional(),
type: z.enum([SECRET_SHARED, SECRET_PERSONAL])
}),
@@ -275,6 +275,7 @@ export const UpdateSecretByNameRawV3 = z.object({
body: z.object({
workspaceId: z.string().trim(),
environment: z.string().trim(),
secretValue: z
.string()
.transform((val) => (val.at(-1) === "\n" ? `${val.trim()}\n` : val.trim())),
@@ -360,6 +361,10 @@ export const UpdateSecretByNameV3 = z.object({
secretCommentCiphertext: z.string().trim().optional(),
secretCommentIV: z.string().trim().optional(),
secretCommentTag: z.string().trim().optional(),
secretReminderRepeatDays: z.number().min(1).max(365).optional().nullable(),
secretReminderNote: z.string().trim().nullable().optional(),
tags: z.string().array().optional(),
skipMultilineEncoding: z.boolean().optional(),
// to update secret name