awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-10-14 09:30:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(secret-reminders): rework secret reminders logic

Browse Source
This commit is contained in:
Carlos Monastyrski
2025-07-04 09:47:36 -03:00
parent f6002d81b3
commit 0efc314f33
34 changed files with 1709 additions and 535 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/services/org/org-service.ts
+5 -5
View File
@@ -46,7 +46,6 @@ import {
import { groupBy } from "@app/lib/fn";
import { alphaNumericNanoId } from "@app/lib/nanoid";
import { isDisposableEmail } from "@app/lib/validator";
import { TQueueServiceFactory } from "@app/queue";
import { getDefaultOrgMembershipRoleForUpdateOrg } from "@app/services/org/org-role-fns";
import { TOrgMembershipDALFactory } from "@app/services/org-membership/org-membership-dal";
import { TUserAliasDALFactory } from "@app/services/user-alias/user-alias-dal";
@@ -64,6 +63,7 @@ import { TProjectKeyDALFactory } from "../project-key/project-key-dal";
import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal";
import { TProjectUserMembershipRoleDALFactory } from "../project-membership/project-user-membership-role-dal";
import { TProjectRoleDALFactory } from "../project-role/project-role-dal";
import { TReminderServiceFactory } from "../reminder/reminder-service";
import { TSecretDALFactory } from "../secret/secret-dal";
import { fnDeleteProjectSecretReminders } from "../secret/secret-fns";
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
@@ -123,8 +123,8 @@ type TOrgServiceFactoryDep = {
projectBotDAL: Pick<TProjectBotDALFactory, "findOne" | "updateById">;
projectUserMembershipRoleDAL: Pick<TProjectUserMembershipRoleDALFactory, "insertMany" | "create">;
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
queueService: Pick<TQueueServiceFactory, "stopRepeatableJob">;
loginService: Pick<TAuthLoginFactory, "generateUserTokens">;
reminderService: Pick<TReminderServiceFactory, "deleteReminderBySecretId">;
};
export type TOrgServiceFactory = ReturnType<typeof orgServiceFactory>;
@@ -156,8 +156,8 @@ export const orgServiceFactory = ({
projectUserMembershipRoleDAL,
identityMetadataDAL,
projectBotService,
queueService,
loginService
loginService,
reminderService
}: TOrgServiceFactoryDep) => {
/*
* Get organization details by the organization id
@@ -599,7 +599,7 @@ export const orgServiceFactory = ({
await fnDeleteProjectSecretReminders(project.id, {
secretDAL,
secretV2BridgeDAL,
queueService,
reminderService,
projectBotService,
folderDAL
});
backend/src/services/project/project-service.ts
+5 -5
View File
@@ -32,7 +32,6 @@ import { BadRequestError, ForbiddenRequestError, NotFoundError } from "@app/lib/
import { groupBy } from "@app/lib/fn";
import { alphaNumericNanoId } from "@app/lib/nanoid";
import { TProjectPermission } from "@app/lib/types";
import { TQueueServiceFactory } from "@app/queue";
import { TPkiSubscriberDALFactory } from "@app/services/pki-subscriber/pki-subscriber-dal";
import { ActorAuthMethod, ActorType } from "../auth/auth-type";
@@ -60,6 +59,7 @@ import { TProjectMembershipDALFactory } from "../project-membership/project-memb
import { TProjectUserMembershipRoleDALFactory } from "../project-membership/project-user-membership-role-dal";
import { TProjectRoleDALFactory } from "../project-role/project-role-dal";
import { getPredefinedRoles } from "../project-role/project-role-fns";
import { TReminderServiceFactory } from "../reminder/reminder-service";
import { TSecretDALFactory } from "../secret/secret-dal";
import { fnDeleteProjectSecretReminders } from "../secret/secret-fns";
import { ROOT_FOLDER_NAME, TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
@@ -162,7 +162,6 @@ type TProjectServiceFactoryDep = {
permissionService: TPermissionServiceFactory;
orgService: Pick<TOrgServiceFactory, "addGhostUser">;
licenseService: Pick<TLicenseServiceFactory, "getPlan" | "invalidateGetPlan">;
queueService: Pick<TQueueServiceFactory, "stopRepeatableJob">;
smtpService: Pick<TSmtpService, "sendMail">;
orgDAL: Pick<TOrgDALFactory, "findOne">;
keyStore: Pick<TKeyStoreFactory, "deleteItem">;
@@ -179,6 +178,7 @@ type TProjectServiceFactoryDep = {
| "createCipherPairWithDataKey"
>;
projectTemplateService: TProjectTemplateServiceFactory;
reminderService: Pick<TReminderServiceFactory, "deleteReminderBySecretId">;
};
export type TProjectServiceFactory = ReturnType<typeof projectServiceFactory>;
@@ -191,7 +191,6 @@ export const projectServiceFactory = ({
projectQueue,
projectKeyDAL,
permissionService,
queueService,
projectBotService,
orgDAL,
userDAL,
@@ -226,7 +225,8 @@ export const projectServiceFactory = ({
microsoftTeamsIntegrationDAL,
projectTemplateService,
groupProjectDAL,
smtpService
smtpService,
reminderService
}: TProjectServiceFactoryDep) => {
/*
* Create workspace. Make user the admin
@@ -555,7 +555,7 @@ export const projectServiceFactory = ({
await fnDeleteProjectSecretReminders(project.id, {
secretDAL,
secretV2BridgeDAL,
queueService,
reminderService,
projectBotService,
folderDAL
});
backend/src/services/reminder-recipients/reminder-dal.ts
+11
View File
@@ -0,0 +1,11 @@
import { TDbClient } from "@app/db";
import { TableName } from "@app/db/schemas";
import { ormify } from "@app/lib/knex";
export type TReminderRecipientDALFactory = ReturnType<typeof reminderRecipientDALFactory>;
export const reminderRecipientDALFactory = (db: TDbClient) => {
const reminderRecipientOrm = ormify(db, TableName.RemindersRecipients);
return { ...reminderRecipientOrm };
};
backend/src/services/reminder-recipients/reminder-types.ts
+6
View File
@@ -0,0 +1,6 @@
export type TReminderRecipient = {
id: string;
reminderId: string;
userId: string;
createdAt: Date;
};
backend/src/services/reminder/reminder-dal.ts
+135
View File
@@ -0,0 +1,135 @@
import { Knex } from "knex";
import { TDbClient } from "@app/db";
import {
TableName,
TOrganizations,
TProjectEnvironments,
TProjects,
TSecretFolders,
TSecretsV2,
TUsers
} from "@app/db/schemas";
import { RemindersSchema } from "@app/db/schemas/reminders";
import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex";
export type TReminderDALFactory = ReturnType<typeof reminderDALFactory>;
export const reminderDALFactory = (db: TDbClient) => {
const reminderOrm = ormify(db, TableName.Reminders);
const getTodayDateRange = () => {
const now = new Date();
const startOfDay = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate(), 0, 0, 0, 0));
const endOfDay = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate(), 23, 59, 59, 999));
return { startOfDay, endOfDay };
};
const findSecretDailyReminders = async (tx?: Knex) => {
const { startOfDay, endOfDay } = getTodayDateRange();
const rawReminders = await (tx || db)(TableName.Reminders)
.whereBetween("nextReminderDate", [startOfDay, endOfDay])
.leftJoin(
TableName.RemindersRecipients,
`${TableName.Reminders}.id`,
`${TableName.RemindersRecipients}.reminderId`
)
.leftJoin<TUsers>(TableName.Users, `${TableName.RemindersRecipients}.userId`, `${TableName.Users}.id`)
.leftJoin<TSecretsV2>(TableName.SecretV2, `${TableName.Reminders}.secretId`, `${TableName.SecretV2}.id`)
.leftJoin<TSecretFolders>(
TableName.SecretFolder,
`${TableName.SecretV2}.folderId`,
`${TableName.SecretFolder}.id`
)
.leftJoin<TProjectEnvironments>(
TableName.Environment,
`${TableName.SecretFolder}.envId`,
`${TableName.Environment}.id`
)
.leftJoin<TProjects>(TableName.Project, `${TableName.Environment}.projectId`, `${TableName.Project}.id`)
.leftJoin<TOrganizations>(TableName.Organization, `${TableName.Project}.orgId`, `${TableName.Organization}.id`)
.select(selectAllTableCols(TableName.Reminders))
.select(db.ref("email").withSchema(TableName.Users))
.select(db.ref("name").withSchema(TableName.Project).as("projectName"))
.select(db.ref("id").withSchema(TableName.Project).as("projectId"))
.select(db.ref("name").withSchema(TableName.Organization).as("organizationName"));
const reminders = sqlNestRelationships({
data: rawReminders,
key: "id",
parentMapper: (el) => ({
_id: el.id,
...RemindersSchema.parse(el),
projectName: el.projectName,
projectId: el.projectId,
organizationName: el.organizationName
}),
childrenMapper: [
{
key: "email",
label: "recipients" as const,
mapper: ({ email }) => ({
email
})
}
]
});
return reminders;
};
const findUpcomingReminders = async (daysAhead: number = 7, tx?: Knex) => {
const { startOfDay } = getTodayDateRange();
const futureDate = new Date(startOfDay);
futureDate.setDate(futureDate.getDate() + daysAhead);
const reminders = await (tx || db)(TableName.Reminders)
.where("nextReminderDate", ">=", startOfDay)
.where("nextReminderDate", "<=", futureDate)
.orderBy("nextReminderDate", "asc")
.leftJoin(
TableName.RemindersRecipients,
`${TableName.Reminders}.id`,
`${TableName.RemindersRecipients}.reminderId`
)
.select(selectAllTableCols(TableName.Reminders))
.select(db.ref("userId").withSchema(TableName.RemindersRecipients));
return reminders;
};
const findSecretReminder = async (secretId: string, tx?: Knex) => {
const rawReminders = await (tx || db)(TableName.Reminders)
.where(`${TableName.Reminders}.secretId`, secretId)
.leftJoin(
TableName.RemindersRecipients,
`${TableName.Reminders}.id`,
`${TableName.RemindersRecipients}.reminderId`
)
.select(selectAllTableCols(TableName.Reminders))
.select(db.ref("userId").withSchema(TableName.RemindersRecipients));
const reminders = sqlNestRelationships({
data: rawReminders,
key: "id",
parentMapper: (el) => ({
_id: el.id,
...RemindersSchema.parse(el)
}),
childrenMapper: [
{
key: "userId",
label: "recipients" as const,
mapper: ({ userId }) => userId
}
]
});
return reminders[0] || null;
};
return {
...reminderOrm,
findSecretDailyReminders,
findUpcomingReminders,
findSecretReminder
};
};
backend/src/services/reminder/reminder-queue.ts
+190
View File
@@ -0,0 +1,190 @@
/* eslint-disable no-await-in-loop */
import RE2 from "re2";
import { logger } from "@app/lib/logger";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import { TSecretReminderRecipientsDALFactory } from "../secret-reminder-recipients/secret-reminder-recipients-dal";
import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";
import { TReminderServiceFactory } from "./reminder-service";
type TDailyReminderQueueServiceFactoryDep = {
reminderService: TReminderServiceFactory;
queueService: TQueueServiceFactory;
secretDAL: Pick<TSecretV2BridgeDALFactory, "findSecretsWithReminderRecipients">;
secretReminderRecipientsDAL: Pick<TSecretReminderRecipientsDALFactory, "delete">;
};
export type TDailyReminderQueueServiceFactory = ReturnType<typeof dailyReminderQueueServiceFactory>;
const uuidRegex = new RE2(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i);
export const dailyReminderQueueServiceFactory = ({
reminderService,
queueService,
secretDAL,
secretReminderRecipientsDAL
}: TDailyReminderQueueServiceFactoryDep) => {
queueService.start(QueueName.DailyReminders, async () => {
logger.info(`${QueueName.DailyReminders}: queue task started`);
await reminderService.sendDailyReminders();
logger.info(`${QueueName.DailyReminders}: queue task completed`);
});
queueService.start(QueueName.SecretReminderMigration, async () => {
const REMINDER_PRUNE_BATCH_SIZE = 5_000;
const MAX_RETRY_ON_FAILURE = 3;
let numberOfRetryOnFailure = 0;
let deletedReminderCount = 0;
logger.info(`${QueueName.SecretReminderMigration}: queue task started`);
try {
const repeatableJobs = await queueService.getRepeatableJobs(QueueName.SecretReminder);
const delayedJobs = await queueService.getDelayedJobs(QueueName.SecretReminder);
logger.info(`${QueueName.SecretReminderMigration}: found ${repeatableJobs.length} secret reminder jobs`);
const reminderJobs = repeatableJobs
.map((job) => ({ secretId: job.id?.replace("reminder-", "") as string, jobKey: job.key }))
.filter(Boolean);
const reminderDelayedJobs = delayedJobs.reduce((map, job) => {
const match = uuidRegex.exec(job.repeatJobKey || "");
if (match) {
map.set(match[0], {
timestamp: job.timestamp,
delay: job.delay
});
}
return map;
}, new Map<string, { timestamp: number; delay: number }>());
if (reminderJobs.length === 0) {
logger.info(`${QueueName.SecretReminderMigration}: no reminder jobs found`);
return;
}
for (let offset = 0; offset < reminderJobs.length; offset += REMINDER_PRUNE_BATCH_SIZE) {
try {
const batch = reminderJobs.slice(offset, offset + REMINDER_PRUNE_BATCH_SIZE);
const batchIds = batch.map((job) => job.secretId);
// Find existing secrets with pagination
// eslint-disable-next-line no-await-in-loop
const secrets = await secretDAL.findSecretsWithReminderRecipients(batchIds, REMINDER_PRUNE_BATCH_SIZE);
const secretsWithReminder = secrets.filter((secret) => secret.reminderRepeatDays);
const foundSecretIds = new Set(secretsWithReminder.map((secret) => secret.id));
// Find IDs that don't exist in either table
const secretIdsNotFound = batchIds.filter((secretId) => !foundSecretIds.has(secretId));
// Delete reminders for non-existent secrets
for (const secretId of secretIdsNotFound) {
const jobKey = reminderJobs.find((r) => r.secretId === secretId)?.jobKey;
if (jobKey) {
// eslint-disable-next-line no-await-in-loop
await queueService.stopRepeatableJobByKey(QueueName.SecretReminder, jobKey);
deletedReminderCount += 1;
}
}
for (const secretId of foundSecretIds) {
const jobKey = reminderJobs.find((r) => r.secretId === secretId)?.jobKey;
if (jobKey) {
await queueService.stopRepeatableJobByKey(QueueName.SecretReminder, jobKey);
deletedReminderCount += 1;
}
}
await reminderService.batchCreateReminders(
secretsWithReminder.map((secret) => {
const delayedJob = reminderDelayedJobs.get(secret.id);
const nextDate = delayedJob ? new Date(delayedJob.timestamp + delayedJob.delay) : undefined;
return {
secretId: secret.id,
message: secret.reminderNote,
repeatDays: secret.reminderRepeatDays,
nextReminderDate: nextDate,
recipients: secret.recipients || []
};
})
);
await secretReminderRecipientsDAL.delete({ $in: { secretId: secretsWithReminder.map((s) => s.id) } });
numberOfRetryOnFailure = 0;
} catch (error) {
numberOfRetryOnFailure += 1;
logger.error(error, `Failed to process batch at offset ${offset}`);
if (numberOfRetryOnFailure >= MAX_RETRY_ON_FAILURE) {
break;
}
// Retry the current batch
offset -= REMINDER_PRUNE_BATCH_SIZE;
// eslint-disable-next-line no-promise-executor-return, @typescript-eslint/no-loop-func, no-await-in-loop
await new Promise((resolve) => setTimeout(resolve, 500 * numberOfRetryOnFailure));
}
// Small delay between batches
// eslint-disable-next-line no-promise-executor-return, @typescript-eslint/no-loop-func, no-await-in-loop
await new Promise((resolve) => setTimeout(resolve, 10));
}
} catch (error) {
logger.error(error, "Failed to complete secret reminder pruning");
} finally {
logger.info(
`${QueueName.SecretReminderMigration}: secret reminders completed. Deleted ${deletedReminderCount} reminders`
);
}
});
// we do a repeat cron job in utc timezone at 12 Midnight each day
const startDailyRemindersJob = async () => {
// clear previous job
await queueService.stopRepeatableJob(
QueueName.DailyReminders,
QueueJobs.DailyReminders,
{ pattern: "0 0 * * *", utc: true },
QueueName.DailyReminders // just a job id
);
await queueService.queue(QueueName.DailyReminders, QueueJobs.DailyReminders, undefined, {
delay: 5000,
jobId: QueueName.DailyReminders,
repeat: { pattern: "0 0 * * *", utc: true }
});
};
// TODO: remove once all the old reminders in queues are migrated
const startSecretReminderMigrationJob = async () => {
// clear previous job
await queueService.stopRepeatableJob(
QueueName.SecretReminderMigration,
QueueJobs.SecretReminderMigration,
{ pattern: "0 */1 * * *", utc: true },
QueueName.SecretReminderMigration // just a job id
);
await queueService.queue(QueueName.SecretReminderMigration, QueueJobs.SecretReminderMigration, undefined, {
delay: 5000,
jobId: QueueName.SecretReminderMigration,
repeat: { pattern: "0 */1 * * *", utc: true }
});
};
queueService.listen(QueueName.DailyReminders, "failed", (_, err) => {
logger.error(err, `${QueueName.DailyReminders}: resource cleanup failed`);
});
queueService.listen(QueueName.SecretReminderMigration, "failed", (_, err) => {
logger.error(err, `${QueueName.SecretReminderMigration}: resource cleanup failed`);
});
return {
startDailyRemindersJob,
startSecretReminderMigrationJob
};
};
backend/src/services/reminder/reminder-service.ts
+332
View File
@@ -0,0 +1,332 @@
/* eslint-disable no-await-in-loop */
import { ForbiddenError } from "@casl/ability";
import { Knex } from "knex";
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service-types";
import { ProjectPermissionSecretActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
import { BadRequestError } from "@app/lib/errors";
import { logger } from "@app/lib/logger";
import { ActorAuthMethod, ActorType } from "../auth/auth-type";
import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal";
import { TReminderRecipientDALFactory } from "../reminder-recipients/reminder-dal";
import { SmtpTemplates, TSmtpService } from "../smtp/smtp-service";
import { TReminderDALFactory } from "./reminder-dal";
import { TBatchCreateReminderDTO, TCreateReminderDTO } from "./reminder-types";
type TReminderServiceFactoryDep = {
reminderDAL: TReminderDALFactory;
reminderRecipientDAL: TReminderRecipientDALFactory;
smtpService: TSmtpService;
projectMembershipDAL: Pick<TProjectMembershipDALFactory, "findAllProjectMembers">;
permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
};
export type TReminderServiceFactory = ReturnType<typeof reminderServiceFactory>;
export const reminderServiceFactory = ({
reminderDAL,
reminderRecipientDAL,
smtpService,
projectMembershipDAL,
permissionService
}: TReminderServiceFactoryDep) => {
const addDays = (days: number, fromDate: Date = new Date()): Date => {
const result = new Date(fromDate);
result.setDate(result.getDate() + days);
return result;
};
const $manageReminderRecipients = async (reminderId: string, newRecipients?: string[] | null): Promise<void> => {
if (!newRecipients || newRecipients.length === 0) {
// If no recipients provided, remove all existing recipients
await reminderRecipientDAL.deleteById(reminderId);
return;
}
// Remove duplicates from input
const uniqueRecipients = [...new Set(newRecipients)];
// Get existing recipients
const existingRecipients = await reminderRecipientDAL.find({ reminderId });
const existingUserIds = new Set(existingRecipients.map((r) => r.userId));
const newUserIds = new Set(uniqueRecipients);
// Find recipients to add and remove
const recipientsToAdd = uniqueRecipients.filter((userId) => !existingUserIds.has(userId));
const recipientsToRemove = existingRecipients.filter((r) => !newUserIds.has(r.userId));
// Perform database operations
if (recipientsToRemove.length > 0) {
await reminderRecipientDAL.delete({ $in: { id: recipientsToRemove.map((r) => r.id) } });
}
if (recipientsToAdd.length > 0) {
await reminderRecipientDAL.insertMany(
recipientsToAdd.map((userId) => ({
reminderId,
userId
}))
);
}
};
const createReminderInternal = async ({
secretId,
message,
repeatDays,
nextReminderDate: nextReminderDateInput,
recipients
}: {
secretId?: string;
message?: string | null;
repeatDays?: number | null;
nextReminderDate?: string | null;
recipients?: string[] | null;
}) => {
if (!secretId) {
throw new BadRequestError({ message: "secretId is required" });
}
let nextReminderDate;
if (nextReminderDateInput) {
nextReminderDate = new Date(nextReminderDateInput);
}
if (repeatDays && repeatDays > 0) {
nextReminderDate = addDays(repeatDays);
}
if (!nextReminderDate) {
throw new BadRequestError({ message: "repeatDays must be a positive number" });
}
const existingReminder = await reminderDAL.findOne({ secretId });
let reminderId: string;
if (existingReminder) {
// Update existing reminder
await reminderDAL.updateById(existingReminder.id, {
message,
repeatDays,
nextReminderDate
});
reminderId = existingReminder.id;
} else {
// Create new reminder
const newReminder = await reminderDAL.create({
secretId,
message,
repeatDays,
nextReminderDate
});
reminderId = newReminder.id;
}
// Manage recipients (add/update/delete as needed)
await $manageReminderRecipients(reminderId, recipients);
return { id: reminderId, created: !existingReminder };
};
const createReminder = async ({
actor,
actorId,
actorOrgId,
actorAuthMethod,
projectId,
reminder
}: TCreateReminderDTO) => {
const { permission } = await permissionService.getProjectPermission({
actor,
actorId,
projectId,
actorAuthMethod,
actorOrgId
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionSecretActions.Edit, ProjectPermissionSub.Secrets);
const response = await createReminderInternal(reminder);
return response;
};
const getReminder = async ({
secretId,
projectId,
actor,
actorId,
actorOrgId,
actorAuthMethod
}: {
secretId: string;
projectId: string;
actor: ActorType;
actorId: string;
actorOrgId: string;
actorAuthMethod: ActorAuthMethod;
}) => {
const { permission } = await permissionService.getProjectPermission({
actor,
actorId,
projectId,
actorAuthMethod,
actorOrgId
});
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretActions.DescribeSecret,
ProjectPermissionSub.Secrets
);
const reminder = await reminderDAL.findSecretReminder(secretId);
return reminder;
};
const sendDailyReminders = async () => {
const remindersToSend = await reminderDAL.findSecretDailyReminders();
for (const reminder of remindersToSend) {
try {
const recipients: string[] = reminder.recipients
.map((r) => r.email)
.filter((email): email is string => Boolean(email));
if (recipients.length === 0) {
const members = await projectMembershipDAL.findAllProjectMembers(reminder.projectId);
recipients.push(...members.map((m) => m.user.email).filter((email): email is string => Boolean(email)));
}
await smtpService.sendMail({
template: SmtpTemplates.SecretReminder,
subjectLine: "Infisical secret reminder",
recipients,
substitutions: {
reminderNote: reminder.message || "",
projectName: reminder.projectName || "",
organizationName: reminder.organizationName || ""
}
});
if (reminder.repeatDays) {
await reminderDAL.updateById(reminder.id, { nextReminderDate: addDays(reminder.repeatDays) });
} else {
await reminderDAL.deleteById(reminder.id);
}
} catch (error) {
logger.error(
error,
`Failed to send reminder to recipients ${reminder.recipients.map((r) => r.email).join(", ")}`
);
}
}
};
const deleteReminder = async ({
actor,
actorId,
actorOrgId,
actorAuthMethod,
secretId,
projectId
}: {
actor: ActorType;
actorId: string;
actorOrgId: string;
actorAuthMethod: ActorAuthMethod;
secretId: string;
projectId: string;
}) => {
const { permission } = await permissionService.getProjectPermission({
actor,
actorId,
projectId,
actorAuthMethod,
actorOrgId
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionSecretActions.Edit, ProjectPermissionSub.Secrets);
await reminderDAL.delete({ secretId });
};
const removeReminderRecipients = async (secretId: string, tx?: Knex) => {
const reminder = await reminderDAL.findOne({ secretId }, tx);
if (!reminder) {
return;
}
await reminderRecipientDAL.delete({ reminderId: reminder.id }, tx);
};
const deleteReminderBySecretId = async (secretId: string, tx?: Knex) => {
await reminderDAL.delete({ secretId }, tx);
};
const batchCreateReminders = async (remindersData: TBatchCreateReminderDTO) => {
if (!remindersData || remindersData.length === 0) {
return { created: 0, reminderIds: [] };
}
const processedReminders = remindersData.map(
({ secretId, message, repeatDays, nextReminderDate: nextReminderDateInput, recipients }) => {
let nextReminderDate;
if (nextReminderDateInput) {
nextReminderDate = new Date(nextReminderDateInput);
}
if (repeatDays && repeatDays > 0 && !nextReminderDate) {
nextReminderDate = addDays(repeatDays);
}
if (!nextReminderDate) {
throw new BadRequestError({
message: `repeatDays must be a positive number for secretId: ${secretId}`
});
}
return {
secretId,
message,
repeatDays,
nextReminderDate,
recipients: recipients ? [...new Set(recipients)] : []
};
}
);
const newReminders = await reminderDAL.insertMany(
processedReminders.map(({ secretId, message, repeatDays, nextReminderDate }) => ({
secretId,
message,
repeatDays,
nextReminderDate
}))
);
const allRecipientInserts: Array<{ reminderId: string; userId: string }> = [];
newReminders.forEach((reminder, index) => {
const { recipients } = processedReminders[index];
if (recipients && recipients.length > 0) {
recipients.forEach((userId) => {
allRecipientInserts.push({
reminderId: reminder.id,
userId
});
});
}
});
if (allRecipientInserts.length > 0) {
await reminderRecipientDAL.insertMany(allRecipientInserts);
}
return {
created: newReminders.length,
reminderIds: newReminders.map((r) => r.id)
};
};
return {
createReminder,
getReminder,
sendDailyReminders,
deleteReminder,
removeReminderRecipients,
deleteReminderBySecretId,
batchCreateReminders,
createReminderInternal
};
};
backend/src/services/reminder/reminder-types.ts
+44
View File
@@ -0,0 +1,44 @@
import { ActorAuthMethod, ActorType } from "../auth/auth-type";
export type TReminder = {
id: string;
secretId?: string;
message?: string;
repeatDays?: number;
nextReminderDate?: Date;
createdAt: Date;
};
export type TCreateReminderDTO = {
actor: ActorType;
actorId: string;
actorOrgId: string;
actorAuthMethod: ActorAuthMethod;
projectId: string;
reminder: {
secretId?: string;
message?: string | null;
repeatDays?: number | null;
nextReminderDate?: string | null;
recipients?: string[] | null;
};
};
export type TCreateSecretReminderDTO = {
secretName: string;
projectId: string;
environment: string;
secretPath: string;
message?: string | null;
repeatDays?: number | null;
nextReminderDate?: string | null;
recipients?: string[] | null;
};
export type TBatchCreateReminderDTO = {
secretId: string;
message?: string | null;
repeatDays?: number | null;
nextReminderDate?: string | Date | null;
recipients?: string[] | null;
}[];
backend/src/services/resource-cleanup/resource-cleanup-queue.ts
-4
View File
@@ -5,7 +5,6 @@ import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import { TIdentityAccessTokenDALFactory } from "../identity-access-token/identity-access-token-dal";
import { TIdentityUaClientSecretDALFactory } from "../identity-ua/identity-ua-client-secret-dal";
import { TSecretDALFactory } from "../secret/secret-dal";
import { TSecretVersionDALFactory } from "../secret/secret-version-dal";
import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal";
import { TSecretSharingDALFactory } from "../secret-sharing/secret-sharing-dal";
@@ -18,7 +17,6 @@ type TDailyResourceCleanUpQueueServiceFactoryDep = {
identityUniversalAuthClientSecretDAL: Pick<TIdentityUaClientSecretDALFactory, "removeExpiredClientSecrets">;
secretVersionDAL: Pick<TSecretVersionDALFactory, "pruneExcessVersions">;
secretVersionV2DAL: Pick<TSecretVersionV2DALFactory, "pruneExcessVersions">;
secretDAL: Pick<TSecretDALFactory, "pruneSecretReminders">;
secretFolderVersionDAL: Pick<TSecretFolderVersionDALFactory, "pruneExcessVersions">;
snapshotDAL: Pick<TSnapshotDALFactory, "pruneExcessSnapshots">;
secretSharingDAL: Pick<TSecretSharingDALFactory, "pruneExpiredSharedSecrets" | "pruneExpiredSecretRequests">;
@@ -34,7 +32,6 @@ export const dailyResourceCleanUpQueueServiceFactory = ({
snapshotDAL,
secretVersionDAL,
secretFolderVersionDAL,
secretDAL,
identityAccessTokenDAL,
secretSharingDAL,
secretVersionV2DAL,
@@ -43,7 +40,6 @@ export const dailyResourceCleanUpQueueServiceFactory = ({
}: TDailyResourceCleanUpQueueServiceFactoryDep) => {
queueService.start(QueueName.DailyResourceCleanUp, async () => {
logger.info(`${QueueName.DailyResourceCleanUp}: queue task started`);
await secretDAL.pruneSecretReminders(queueService);
await auditLogDAL.pruneAuditLog();
await identityAccessTokenDAL.removeExpiredTokens();
await identityUniversalAuthClientSecretDAL.removeExpiredClientSecrets();
backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts
+55 -8
View File
@@ -513,18 +513,19 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
`${TableName.SecretV2JnTag}.${TableName.SecretTag}Id`,
`${TableName.SecretTag}.id`
)
.leftJoin(
TableName.SecretReminderRecipients,
`${TableName.SecretV2}.id`,
`${TableName.SecretReminderRecipients}.secretId`
)
.leftJoin(TableName.Users, `${TableName.SecretReminderRecipients}.userId`, `${TableName.Users}.id`)
.leftJoin(TableName.ResourceMetadata, `${TableName.SecretV2}.id`, `${TableName.ResourceMetadata}.secretId`)
.leftJoin(
TableName.SecretRotationV2SecretMapping,
`${TableName.SecretV2}.id`,
`${TableName.SecretRotationV2SecretMapping}.secretId`
)
.leftJoin(TableName.Reminders, `${TableName.SecretV2}.id`, `${TableName.Reminders}.secretId`)
.leftJoin(
TableName.RemindersRecipients,
`${TableName.Reminders}.id`,
`${TableName.RemindersRecipients}.reminderId`
)
.leftJoin(TableName.Users, `${TableName.RemindersRecipients}.userId`, `${TableName.Users}.id`)
.where((qb) => {
if (filters?.metadataFilter && filters.metadataFilter.length > 0) {
filters.metadataFilter.forEach((meta) => {
@@ -547,7 +548,11 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
}) as rank`
)
)
.select(db.ref("id").withSchema(TableName.SecretReminderRecipients).as("reminderRecipientId"))
.select(db.ref("id").withSchema(TableName.Reminders).as("reminderId"))
.select(db.ref("message").withSchema(TableName.Reminders).as("reminderNote"))
.select(db.ref("repeatDays").withSchema(TableName.Reminders).as("reminderRepeatDays"))
.select(db.ref("nextReminderDate").withSchema(TableName.Reminders).as("nextReminderDate"))
.select(db.ref("id").withSchema(TableName.RemindersRecipients).as("reminderRecipientId"))
.select(db.ref("username").withSchema(TableName.Users).as("reminderRecipientUsername"))
.select(db.ref("email").withSchema(TableName.Users).as("reminderRecipientEmail"))
.select(db.ref("id").withSchema(TableName.Users).as("reminderRecipientUserId"))
@@ -809,6 +814,47 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
}
};
const findSecretsWithReminderRecipients = async (ids: string[], limit: number, tx?: Knex) => {
try {
// Create a subquery to get limited secret IDs
const limitedSecretIds = (tx || db)(TableName.SecretV2)
.whereIn(`${TableName.SecretV2}.id`, ids)
.limit(limit)
.select("id");
// Join with all recipients for the limited secrets
const docs = await (tx || db)(TableName.SecretV2)
.whereIn(`${TableName.SecretV2}.id`, limitedSecretIds)
.leftJoin(
TableName.SecretReminderRecipients,
`${TableName.SecretV2}.id`,
`${TableName.SecretReminderRecipients}.secretId`
)
.select(selectAllTableCols(TableName.SecretV2))
.select(db.ref("userId").withSchema(TableName.SecretReminderRecipients).as("reminderRecipientUserId"));
const data = sqlNestRelationships({
data: docs,
key: "id",
parentMapper: (el) => ({
_id: el.id,
...SecretsV2Schema.parse(el)
}),
childrenMapper: [
{
key: "reminderRecipientUserId",
label: "recipients" as const,
mapper: ({ reminderRecipientUserId }) => reminderRecipientUserId
}
]
});
return data;
} catch (error) {
throw new DatabaseError({ error, name: "FindSecretsWithReminderRecipients" });
}
};
return {
...secretOrm,
update,
@@ -826,6 +872,7 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
countByFolderIds,
findOne,
find,
invalidateSecretCacheByProjectId
invalidateSecretCacheByProjectId,
findSecretsWithReminderRecipients
};
};
backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts
+2 -21
View File
@@ -223,20 +223,7 @@ export const fnSecretBulkUpdate = async ({
const actorType = actor?.type || ActorType.PLATFORM;
const sanitizedInputSecrets = inputSecrets.map(
({
filter,
data: {
skipMultilineEncoding,
type,
key,
encryptedValue,
userId,
encryptedComment,
metadata,
reminderNote,
reminderRepeatDays
}
}) => ({
({ filter, data: { skipMultilineEncoding, type, key, encryptedValue, userId, encryptedComment, metadata } }) => ({
filter: { ...filter, folderId },
data: {
skipMultilineEncoding,
@@ -245,9 +232,7 @@ export const fnSecretBulkUpdate = async ({
userId,
encryptedComment,
metadata,
reminderNote,
encryptedValue,
reminderRepeatDays
encryptedValue
}
})
);
@@ -263,9 +248,7 @@ export const fnSecretBulkUpdate = async ({
encryptedComment,
version,
metadata,
reminderNote,
encryptedValue,
reminderRepeatDays,
id: secretId
}) => ({
skipMultilineEncoding,
@@ -275,9 +258,7 @@ export const fnSecretBulkUpdate = async ({
encryptedComment,
version,
metadata: metadata ? JSON.stringify(metadata) : [],
reminderNote,
encryptedValue,
reminderRepeatDays,
folderId,
secretId,
userActorId,
backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts
+41 -23
View File
@@ -32,6 +32,7 @@ import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-serv
import { TKmsServiceFactory } from "../kms/kms-service";
import { KmsDataKey } from "../kms/kms-types";
import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
import { TReminderServiceFactory } from "../reminder/reminder-service";
import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal";
import { TSecretQueueFactory } from "../secret/secret-queue";
import { TGetASecretByIdDTO } from "../secret/secret-types";
@@ -108,6 +109,7 @@ type TSecretV2BridgeServiceFactoryDep = {
snapshotService: Pick<TSecretSnapshotServiceFactory, "performSnapshot">;
resourceMetadataDAL: Pick<TResourceMetadataDALFactory, "insertMany" | "delete">;
keyStore: Pick<TKeyStoreFactory, "getItem" | "setExpiry" | "setItemWithExpiry" | "deleteItem">;
reminderService: Pick<TReminderServiceFactory, "createReminder" | "getReminder">;
};
export type TSecretV2BridgeServiceFactory = ReturnType<typeof secretV2BridgeServiceFactory>;
@@ -132,7 +134,8 @@ export const secretV2BridgeServiceFactory = ({
secretApprovalRequestSecretDAL,
kmsService,
resourceMetadataDAL,
keyStore
keyStore,
reminderService
}: TSecretV2BridgeServiceFactoryDep) => {
const $validateSecretReferences = async (
projectId: string,
@@ -303,7 +306,6 @@ export const secretV2BridgeServiceFactory = ({
{
version: 1,
type,
reminderRepeatDays: inputSecretData.secretReminderRepeatDays,
encryptedComment: setKnexStringValue(
inputSecretData.secretComment,
(value) => secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob
@@ -311,7 +313,6 @@ export const secretV2BridgeServiceFactory = ({
encryptedValue: inputSecretData.secretValue
? secretManagerEncryptor({ plainText: Buffer.from(inputSecretData.secretValue) }).cipherTextBlob
: undefined,
reminderNote: inputSecretData.secretReminderNote,
skipMultilineEncoding: inputSecretData.skipMultilineEncoding,
key: secretName,
userId: inputSecret.type === SecretType.Personal ? actorId : null,
@@ -337,6 +338,21 @@ export const secretV2BridgeServiceFactory = ({
return createdSecret;
});
if (inputSecret.secretReminderRepeatDays) {
await reminderService.createReminder({
actor,
actorId,
actorOrgId,
actorAuthMethod,
projectId,
reminder: {
secretId: secret.id,
message: inputSecret.secretReminderNote,
repeatDays: inputSecret.secretReminderRepeatDays
}
});
}
await secretDAL.invalidateSecretCacheByProjectId(projectId);
if (inputSecret.type === SecretType.Shared) {
await snapshotService.performSnapshot(folderId);
@@ -512,12 +528,10 @@ export const secretV2BridgeServiceFactory = ({
{
filter: { id: secretId },
data: {
reminderRepeatDays: inputSecret.secretReminderRepeatDays,
encryptedComment: setKnexStringValue(
inputSecret.secretComment,
(value) => secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob
),
reminderNote: inputSecret.secretReminderNote,
skipMultilineEncoding: inputSecret.skipMultilineEncoding,
key: inputSecret.newSecretName || secretName,
tags: inputSecret.tagIds,
@@ -538,19 +552,21 @@ export const secretV2BridgeServiceFactory = ({
tx
})
);
await secretQueueService.handleSecretReminder({
newSecret: {
id: updatedSecret[0].id,
...inputSecret
},
oldSecret: {
id: secret.id,
secretReminderNote: secret.reminderNote,
secretReminderRepeatDays: secret.reminderRepeatDays,
secretReminderRecipients: secret.secretReminderRecipients?.map((el) => el.user.id)
},
projectId
});
if (inputSecret.secretReminderRepeatDays) {
await reminderService.createReminder({
actor,
actorId,
actorOrgId,
actorAuthMethod,
projectId,
reminder: {
secretId: secret.id,
message: inputSecret.secretReminderNote,
repeatDays: inputSecret.secretReminderRepeatDays,
recipients: inputSecret.secretReminderRecipients
}
});
}
await secretDAL.invalidateSecretCacheByProjectId(projectId);
if (inputSecret.type === SecretType.Shared) {
@@ -1868,12 +1884,10 @@ export const secretV2BridgeServiceFactory = ({
return {
filter: { id: originalSecret.id, type: SecretType.Shared },
data: {
reminderRepeatDays: el.secretReminderRepeatDays,
encryptedComment: setKnexStringValue(
el.secretComment,
(value) => secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob
),
reminderNote: el.secretReminderNote,
skipMultilineEncoding: el.skipMultilineEncoding,
key: el.newSecretName || el.secretKey,
tags: el.tagIds,
@@ -2522,9 +2536,7 @@ export const secretV2BridgeServiceFactory = ({
key: doc.key,
encryptedComment: doc.encryptedComment,
skipMultilineEncoding: doc.skipMultilineEncoding,
reminderNote: doc.reminderNote,
secretMetadata: doc.secretMetadata,
reminderRepeatDays: doc.reminderRepeatDays,
...(doc.encryptedValue
? {
encryptedValue: doc.encryptedValue,
@@ -2914,6 +2926,11 @@ export const secretV2BridgeServiceFactory = ({
});
};
const findSecretIdsByFolderIdAndKeys = async ({ folderId, keys }: { folderId: string; keys: string[] }) => {
const secrets = await secretDAL.find({ folderId, $in: { [`${TableName.SecretV2}.key` as "key"]: keys } });
return secrets.map((el) => ({ id: el.id, key: el.key }));
};
return {
createSecret,
deleteSecret,
@@ -2933,6 +2950,7 @@ export const secretV2BridgeServiceFactory = ({
getSecretsByFolderMappings,
getSecretById,
getAccessibleSecrets,
getSecretVersionsByIds
getSecretVersionsByIds,
findSecretIdsByFolderIdAndKeys
};
};
backend/src/services/secret/secret-dal.ts
-91
View File
@@ -5,8 +5,6 @@ import { TDbClient } from "@app/db";
import { SecretsSchema, SecretType, TableName, TSecrets, TSecretsUpdate } from "@app/db/schemas";
import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors";
import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex";
import { logger } from "@app/lib/logger";
import { QueueName, TQueueServiceFactory } from "@app/queue";
export type TSecretDALFactory = ReturnType<typeof secretDALFactory>;
@@ -383,94 +381,6 @@ export const secretDALFactory = (db: TDbClient) => {
}
};
const pruneSecretReminders = async (queueService: TQueueServiceFactory) => {
const REMINDER_PRUNE_BATCH_SIZE = 5_000;
const MAX_RETRY_ON_FAILURE = 3;
let numberOfRetryOnFailure = 0;
let deletedReminderCount = 0;
logger.info(`${QueueName.DailyResourceCleanUp}: secret reminders started`);
try {
const repeatableJobs = await queueService.getRepeatableJobs(QueueName.SecretReminder);
const reminderJobs = repeatableJobs
.map((job) => ({ secretId: job.id?.replace("reminder-", "") as string, jobKey: job.key }))
.filter(Boolean);
if (reminderJobs.length === 0) {
logger.info(`${QueueName.DailyResourceCleanUp}: no reminder jobs found`);
return;
}
for (let offset = 0; offset < reminderJobs.length; offset += REMINDER_PRUNE_BATCH_SIZE) {
try {
const batchIds = reminderJobs.slice(offset, offset + REMINDER_PRUNE_BATCH_SIZE).map((r) => r.secretId);
const payload = {
$in: {
id: batchIds
}
};
const opts = {
limit: REMINDER_PRUNE_BATCH_SIZE
};
// Find existing secrets with pagination
// eslint-disable-next-line no-await-in-loop
const [secrets, secretsV2] = await Promise.all([
ormify(db, TableName.Secret).find(payload, opts),
ormify(db, TableName.SecretV2).find(payload, opts)
]);
const foundSecretIds = new Set([
...secrets.map((secret) => secret.id),
...secretsV2.map((secret) => secret.id)
]);
// Find IDs that don't exist in either table
const secretIdsNotFound = batchIds.filter((secretId) => !foundSecretIds.has(secretId));
// Delete reminders for non-existent secrets
for (const secretId of secretIdsNotFound) {
const jobKey = reminderJobs.find((r) => r.secretId === secretId)?.jobKey;
if (jobKey) {
// eslint-disable-next-line no-await-in-loop
await queueService.stopRepeatableJobByKey(QueueName.SecretReminder, jobKey);
deletedReminderCount += 1;
}
}
numberOfRetryOnFailure = 0;
} catch (error) {
numberOfRetryOnFailure += 1;
logger.error(error, `Failed to process batch at offset ${offset}`);
if (numberOfRetryOnFailure >= MAX_RETRY_ON_FAILURE) {
break;
}
// Retry the current batch
offset -= REMINDER_PRUNE_BATCH_SIZE;
// eslint-disable-next-line no-promise-executor-return, @typescript-eslint/no-loop-func, no-await-in-loop
await new Promise((resolve) => setTimeout(resolve, 500 * numberOfRetryOnFailure));
}
// Small delay between batches
// eslint-disable-next-line no-promise-executor-return, @typescript-eslint/no-loop-func, no-await-in-loop
await new Promise((resolve) => setTimeout(resolve, 10));
}
} catch (error) {
logger.error(error, "Failed to complete secret reminder pruning");
} finally {
logger.info(
`${QueueName.DailyResourceCleanUp}: secret reminders completed. Deleted ${deletedReminderCount} reminders`
);
}
};
return {
...secretOrm,
update,
@@ -485,7 +395,6 @@ export const secretDALFactory = (db: TDbClient) => {
upsertSecretReferences,
findReferencedSecretReferences,
findAllProjectSecretValues,
pruneSecretReminders,
findManySecretsWithTags
};
};
backend/src/services/secret/secret-fns.ts
+4 -15
View File
@@ -20,11 +20,9 @@ import {
decryptSymmetric128BitHexKeyUTF8,
encryptSymmetric128BitHexKeyUTF8
} from "@app/lib/crypto";
import { daysToMillisecond, secondsToMillis } from "@app/lib/dates";
import { BadRequestError, NotFoundError } from "@app/lib/errors";
import { groupBy, unique } from "@app/lib/fn";
import { logger } from "@app/lib/logger";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import {
fnSecretBulkInsert as fnSecretV2BridgeBulkInsert,
fnSecretBulkUpdate as fnSecretV2BridgeBulkUpdate,
@@ -36,6 +34,7 @@ import { KmsDataKey } from "../kms/kms-types";
import { getBotKeyFnFactory } from "../project-bot/project-bot-fns";
import { TProjectBotServiceFactory } from "../project-bot/project-bot-service";
import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
import { TReminderServiceFactory } from "../reminder/reminder-service";
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";
import { TSecretDALFactory } from "./secret-dal";
@@ -1184,14 +1183,14 @@ export const decryptSecretWithBot = (
type TFnDeleteProjectSecretReminders = {
secretDAL: Pick<TSecretDALFactory, "find">;
secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "find">;
queueService: Pick<TQueueServiceFactory, "stopRepeatableJob">;
reminderService: Pick<TReminderServiceFactory, "deleteReminderBySecretId">;
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
folderDAL: Pick<TSecretFolderDALFactory, "findByProjectId">;
};
export const fnDeleteProjectSecretReminders = async (
projectId: string,
{ secretDAL, secretV2BridgeDAL, queueService, projectBotService, folderDAL }: TFnDeleteProjectSecretReminders
{ secretDAL, secretV2BridgeDAL, reminderService, projectBotService, folderDAL }: TFnDeleteProjectSecretReminders
) => {
const projectFolders = await folderDAL.findByProjectId(projectId);
const { shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId, false);
@@ -1206,23 +1205,13 @@ export const fnDeleteProjectSecretReminders = async (
$notNull: ["secretReminderRepeatDays"]
});
const appCfg = getConfig();
for await (const secret of projectSecrets) {
const repeatDays = shouldUseSecretV2Bridge
? (secret as { reminderRepeatDays: number }).reminderRepeatDays
: (secret as { secretReminderRepeatDays: number }).secretReminderRepeatDays;
// We're using the queue service directly to get around conflicting imports.
if (repeatDays) {
await queueService.stopRepeatableJob(
QueueName.SecretReminder,
QueueJobs.SecretReminder,
{
// on prod it this will be in days, in development this will be second
every: appCfg.NODE_ENV === "development" ? secondsToMillis(repeatDays) : daysToMillisecond(repeatDays)
},
`reminder-${secret.id}`
);
await reminderService.deleteReminderBySecretId(secret.id);
}
}
};
backend/src/services/secret/secret-queue.ts
+39 -167
View File
@@ -20,7 +20,6 @@ import { KeyStorePrefixes, KeyStoreTtls, TKeyStoreFactory } from "@app/keystore/
import { getConfig } from "@app/lib/config/env";
import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
import { infisicalSymmetricEncypt } from "@app/lib/crypto/encryption";
import { daysToMillisecond, secondsToMillis } from "@app/lib/dates";
import { BadRequestError, NotFoundError } from "@app/lib/errors";
import { getTimeDifferenceInSeconds, groupBy, isSamePath, unique } from "@app/lib/fn";
import { logger } from "@app/lib/logger";
@@ -41,7 +40,6 @@ import { TIntegrationAuthServiceFactory } from "../integration-auth/integration-
import { syncIntegrationSecrets } from "../integration-auth/integration-sync-secret";
import { TKmsServiceFactory } from "../kms/kms-service";
import { KmsDataKey } from "../kms/kms-types";
import { TOrgDALFactory } from "../org/org-dal";
import { TOrgServiceFactory } from "../org/org-service";
import { TProjectDALFactory } from "../project/project-dal";
import { createProjectKey } from "../project/project-fns";
@@ -50,12 +48,12 @@ import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
import { TProjectKeyDALFactory } from "../project-key/project-key-dal";
import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal";
import { TProjectUserMembershipRoleDALFactory } from "../project-membership/project-user-membership-role-dal";
import { TReminderServiceFactory } from "../reminder/reminder-service";
import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal";
import { ResourceMetadataDTO } from "../resource-metadata/resource-metadata-schema";
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
import { fnSecretsV2FromImports } from "../secret-import/secret-import-fns";
import { TSecretReminderRecipientsDALFactory } from "../secret-reminder-recipients/secret-reminder-recipients-dal";
import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";
import { expandSecretReferencesFactory, getAllSecretReferences } from "../secret-v2-bridge/secret-v2-bridge-fns";
import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal";
@@ -93,7 +91,6 @@ type TSecretQueueFactoryDep = {
projectKeyDAL: Pick<TProjectKeyDALFactory, "create">;
projectMembershipDAL: Pick<TProjectMembershipDALFactory, "findAllProjectMembers" | "create">;
smtpService: TSmtpService;
orgDAL: Pick<TOrgDALFactory, "findOrgByProjectId">;
secretVersionDAL: TSecretVersionDALFactory;
secretBlindIndexDAL: TSecretBlindIndexDALFactory;
secretTagDAL: TSecretTagDALFactory;
@@ -113,11 +110,11 @@ type TSecretQueueFactoryDep = {
projectUserMembershipRoleDAL: Pick<TProjectUserMembershipRoleDALFactory, "create">;
resourceMetadataDAL: Pick<TResourceMetadataDALFactory, "insertMany" | "delete">;
folderCommitService: Pick<TFolderCommitServiceFactory, "createCommit">;
secretReminderRecipientsDAL: Pick<
TSecretReminderRecipientsDALFactory,
"delete" | "findUsersBySecretId" | "insertMany" | "transaction"
>;
secretSyncQueue: Pick<TSecretSyncQueueFactory, "queueSecretSyncsSyncSecretsByPath">;
reminderService: Pick<
TReminderServiceFactory,
"createReminderInternal" | "deleteReminderBySecretId" | "removeReminderRecipients"
>;
};
export type TGetSecrets = {
@@ -155,7 +152,6 @@ export const secretQueueFactory = ({
userDAL,
webhookDAL,
projectEnvDAL,
orgDAL,
smtpService,
projectDAL,
projectBotDAL,
@@ -178,9 +174,9 @@ export const secretQueueFactory = ({
projectUserMembershipRoleDAL,
projectKeyDAL,
resourceMetadataDAL,
secretReminderRecipientsDAL,
secretSyncQueue,
folderCommitService
folderCommitService,
reminderService
}: TSecretQueueFactoryDep) => {
const integrationMeter = opentelemetry.metrics.getMeter("Integrations");
const errorHistogram = integrationMeter.createHistogram("integration_secret_sync_errors", {
@@ -190,19 +186,8 @@ export const secretQueueFactory = ({
const removeSecretReminder = async ({ deleteRecipients = true, ...dto }: TRemoveSecretReminderDTO, tx?: Knex) => {
if (deleteRecipients) {
await secretReminderRecipientsDAL.delete({ secretId: dto.secretId }, tx);
await reminderService.deleteReminderBySecretId(dto.secretId, tx);
}
const appCfg = getConfig();
await queueService.stopRepeatableJob(
QueueName.SecretReminder,
QueueJobs.SecretReminder,
{
// on prod it this will be in days, in development this will be second
every: appCfg.NODE_ENV === "development" ? secondsToMillis(dto.repeatDays) : daysToMillisecond(dto.repeatDays)
},
`reminder-${dto.secretId}`
);
};
const $generateActor = async (actorId?: string, isManual?: boolean): Promise<Actor> => {
@@ -238,15 +223,8 @@ export const secretQueueFactory = ({
.replace(":", "-");
};
const addSecretReminder = async ({
oldSecret,
newSecret,
projectId,
deleteRecipients = true
}: TCreateSecretReminderDTO) => {
const addSecretReminder = async ({ oldSecret, newSecret, secretReminderRecipients }: TCreateSecretReminderDTO) => {
try {
const appCfg = getConfig();
if (oldSecret.id !== newSecret.id) {
throw new BadRequestError({
name: "SecretReminderIdMismatch",
@@ -261,38 +239,12 @@ export const secretQueueFactory = ({
});
}
// If the secret already has a reminder, we should remove the existing one first.
if (oldSecret.secretReminderRepeatDays) {
await removeSecretReminder({
repeatDays: oldSecret.secretReminderRepeatDays,
secretId: oldSecret.id,
deleteRecipients
});
}
await queueService.queue(
QueueName.SecretReminder,
QueueJobs.SecretReminder,
{
note: newSecret.secretReminderNote,
projectId,
repeatDays: newSecret.secretReminderRepeatDays,
secretId: newSecret.id
},
{
jobId: `reminder-${newSecret.id}`,
repeat: {
// on prod it this will be in days, in development this will be second
every:
appCfg.NODE_ENV === "development"
? secondsToMillis(newSecret.secretReminderRepeatDays)
: daysToMillisecond(newSecret.secretReminderRepeatDays),
immediately: true
},
removeOnComplete: true,
removeOnFail: true
}
);
await reminderService.createReminderInternal({
secretId: newSecret.id,
message: newSecret.secretReminderNote,
repeatDays: newSecret.secretReminderRepeatDays,
recipients: secretReminderRecipients
});
} catch (err) {
logger.error(err, "Failed to create secret reminder.");
throw new BadRequestError({
@@ -302,58 +254,31 @@ export const secretQueueFactory = ({
}
};
const handleSecretReminder = async ({ newSecret, oldSecret, projectId }: THandleReminderDTO) => {
const handleSecretReminder = async ({ newSecret, oldSecret }: THandleReminderDTO) => {
const { secretReminderRepeatDays, secretReminderNote, secretReminderRecipients } = newSecret;
const recipientsUpdated =
secretReminderRecipients?.some(
(newId) => !oldSecret.secretReminderRecipients?.find((oldId) => newId === oldId)
) || secretReminderRecipients?.length !== oldSecret.secretReminderRecipients?.length;
await secretReminderRecipientsDAL.transaction(async (tx) => {
if (newSecret.type !== SecretType.Personal && secretReminderRepeatDays !== undefined) {
if (
(secretReminderRepeatDays && oldSecret.secretReminderRepeatDays !== secretReminderRepeatDays) ||
(secretReminderNote && oldSecret.secretReminderNote !== secretReminderNote)
) {
await addSecretReminder({
oldSecret,
newSecret,
projectId,
deleteRecipients: false
});
} else if (
secretReminderRepeatDays === null &&
secretReminderNote === null &&
oldSecret.secretReminderRepeatDays
) {
await removeSecretReminder({
secretId: oldSecret.id,
repeatDays: oldSecret.secretReminderRepeatDays
});
}
if (newSecret.type !== SecretType.Personal && secretReminderRepeatDays !== undefined) {
if (
(secretReminderRepeatDays && oldSecret.secretReminderRepeatDays !== secretReminderRepeatDays) ||
(secretReminderNote && oldSecret.secretReminderNote !== secretReminderNote)
) {
await addSecretReminder({
oldSecret,
newSecret,
secretReminderRecipients: secretReminderRecipients ?? [],
deleteRecipients: false
});
} else if (
secretReminderRepeatDays === null &&
secretReminderNote === null &&
oldSecret.secretReminderRepeatDays
) {
await removeSecretReminder({
secretId: oldSecret.id,
repeatDays: oldSecret.secretReminderRepeatDays
});
}
if (recipientsUpdated) {
// if no recipients, delete all existing recipients
if (!secretReminderRecipients?.length) {
const existingRecipients = await secretReminderRecipientsDAL.findUsersBySecretId(newSecret.id, tx);
if (existingRecipients) {
await secretReminderRecipientsDAL.delete({ secretId: newSecret.id }, tx);
}
} else {
await secretReminderRecipientsDAL.delete({ secretId: newSecret.id }, tx);
await secretReminderRecipientsDAL.insertMany(
secretReminderRecipients.map((r) => ({
secretId: newSecret.id,
userId: r,
projectId
})),
tx
);
}
}
});
}
};
const createManySecretsRawFn = createManySecretsRawFnFactory({
projectDAL,
@@ -1111,62 +1036,9 @@ export const secretQueueFactory = ({
}
});
// TODO: remove this queue (needed for queue initialization and perform the migration)
queueService.start(QueueName.SecretReminder, async ({ data }) => {
logger.info(`secretReminderQueue.process: [secretDocument=${data.secretId}]`);
const { projectId } = data;
const organization = await orgDAL.findOrgByProjectId(projectId);
const project = await projectDAL.findById(projectId);
const secret = await secretV2BridgeDAL.findById(data.secretId);
const [folder] = await folderDAL.findSecretPathByFolderIds(project.id, [secret.folderId]);
const recipients = await secretReminderRecipientsDAL.findUsersBySecretId(data.secretId);
if (!organization) {
logger.info(`secretReminderQueue.process: [secretDocument=${data.secretId}] no organization found`);
return;
}
if (!project) {
logger.info(`secretReminderQueue.process: [secretDocument=${data.secretId}] no project found`);
return;
}
const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId);
if (!projectMembers || !projectMembers.length) {
logger.info(`secretReminderQueue.process: [secretDocument=${data.secretId}] no project members found`);
return;
}
const selectedRecipients = recipients?.length
? recipients.map((r) => r.email as string)
: projectMembers.map((m) => m.user.email as string);
await smtpService.sendMail({
template: SmtpTemplates.SecretReminder,
subjectLine: "Infisical secret reminder",
recipients: selectedRecipients,
substitutions: {
reminderNote: data.note, // May not be present.
projectName: project.name,
organizationName: organization.name
}
});
await queueService.queue(QueueName.SecretWebhook, QueueJobs.SecWebhook, {
type: WebhookEvents.SecretReminderExpired,
payload: {
projectName: project.name,
projectId: project.id,
secretPath: folder?.path,
environment: folder?.environmentSlug || "",
reminderNote: data.note,
secretName: secret?.key,
secretId: data.secretId
}
});
logger.info(`(deprecated) secretReminderQueue.process: [secretDocument=${data.secretId}]`);
});
const startSecretV2Migration = async (projectId: string) => {
backend/src/services/secret/secret-service.ts
+45 -11
View File
@@ -48,6 +48,7 @@ import { ChangeType } from "../folder-commit/folder-commit-service";
import { TProjectDALFactory } from "../project/project-dal";
import { TProjectBotServiceFactory } from "../project-bot/project-bot-service";
import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
import { TReminderServiceFactory } from "../reminder/reminder-service";
import { TSecretBlindIndexDALFactory } from "../secret-blind-index/secret-blind-index-dal";
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
@@ -131,6 +132,7 @@ type TSecretServiceFactoryDep = {
"insertMany" | "insertApprovalSecretTags"
>;
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
reminderService: Pick<TReminderServiceFactory, "createReminder">;
};
export type TSecretServiceFactory = ReturnType<typeof secretServiceFactory>;
@@ -153,7 +155,8 @@ export const secretServiceFactory = ({
secretApprovalRequestSecretDAL,
secretV2BridgeService,
secretApprovalRequestService,
licenseService
licenseService,
reminderService
}: TSecretServiceFactoryDep) => {
const getSecretReference = async (projectId: string) => {
// if bot key missing means e2e still exist
@@ -390,8 +393,7 @@ export const secretServiceFactory = ({
id: secrets[0].id,
...inputSecret
},
oldSecret: secrets[0],
projectId
oldSecret: secrets[0]
});
const tags = inputSecret.tags ? await secretTagDAL.findManyTagsById(projectId, inputSecret.tags) : [];
@@ -1660,8 +1662,6 @@ export const secretServiceFactory = ({
secretComment,
secretValue,
tagIds,
reminderNote: secretReminderNote,
reminderRepeatDays: secretReminderRepeatDays,
secretMetadata
}
]
@@ -1824,9 +1824,6 @@ export const secretServiceFactory = ({
secretComment,
secretValue,
tagIds,
reminderNote: secretReminderNote,
reminderRepeatDays: secretReminderRepeatDays,
secretReminderRecipients,
secretMetadata
}
]
@@ -1835,9 +1832,6 @@ export const secretServiceFactory = ({
return { type: SecretProtectionType.Approval as const, approval };
}
const secret = await secretV2BridgeService.updateSecret({
secretReminderRepeatDays,
secretReminderNote,
secretReminderRecipients,
skipMultilineEncoding,
tagIds,
secretComment,
@@ -1855,6 +1849,22 @@ export const secretServiceFactory = ({
secretValue,
secretMetadata
});
if (secretReminderRepeatDays) {
await reminderService.createReminder({
actor,
actorId,
actorOrgId,
actorAuthMethod,
projectId,
reminder: {
secretId: secret.id,
message: secretReminderNote,
repeatDays: secretReminderRepeatDays,
recipients: secretReminderRecipients
}
});
}
return { type: SecretProtectionType.Direct as const, secret };
}
@@ -2246,6 +2256,30 @@ export const secretServiceFactory = ({
secrets: inputSecrets,
mode
});
await Promise.all(
inputSecrets
.filter((el) => el.secretReminderRepeatDays)
.map(async (secret) => {
await reminderService.createReminder({
actor,
actorId,
actorOrgId,
actorAuthMethod,
projectId,
reminder: {
secretId: secrets.find(
(el) =>
(el.secretKey === secret.secretKey || el.secretKey === secret.newSecretName) &&
el.secretPath === (secret.secretPath || secretPath)
)?.id,
message: secret.secretReminderNote,
repeatDays: secret.secretReminderRepeatDays
}
});
})
);
return { type: SecretProtectionType.Direct as const, secrets };
}
backend/src/services/secret/secret-types.ts
+2 -2
View File
@@ -310,6 +310,7 @@ export type TUpdateManySecretRawDTO = Omit<TProjectPermission, "projectId"> & {
secretMetadata?: ResourceMetadataDTO;
secretReminderRepeatDays?: number | null;
secretReminderNote?: string | null;
secretPath?: string;
}[];
};
@@ -403,13 +404,12 @@ export type TFnSecretBlindIndexCheckV2 = {
export type THandleReminderDTO = {
newSecret: TPartialInputSecret;
oldSecret: TPartialSecret;
projectId: string;
};
export type TCreateSecretReminderDTO = {
oldSecret: TPartialSecret;
newSecret: TPartialSecret;
projectId: string;
secretReminderRecipients: string[];
deleteRecipients?: boolean;
};