feat(secret-reminders): rework secret reminders logic

This commit is contained in:
Carlos Monastyrski
2025-07-04 09:47:36 -03:00
parent f6002d81b3
commit 0efc314f33
34 changed files with 1709 additions and 535 deletions
@@ -0,0 +1,135 @@
import { Knex } from "knex";
import { TDbClient } from "@app/db";
import {
TableName,
TOrganizations,
TProjectEnvironments,
TProjects,
TSecretFolders,
TSecretsV2,
TUsers
} from "@app/db/schemas";
import { RemindersSchema } from "@app/db/schemas/reminders";
import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex";
export type TReminderDALFactory = ReturnType<typeof reminderDALFactory>;
export const reminderDALFactory = (db: TDbClient) => {
const reminderOrm = ormify(db, TableName.Reminders);
const getTodayDateRange = () => {
const now = new Date();
const startOfDay = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate(), 0, 0, 0, 0));
const endOfDay = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate(), 23, 59, 59, 999));
return { startOfDay, endOfDay };
};
const findSecretDailyReminders = async (tx?: Knex) => {
const { startOfDay, endOfDay } = getTodayDateRange();
const rawReminders = await (tx || db)(TableName.Reminders)
.whereBetween("nextReminderDate", [startOfDay, endOfDay])
.leftJoin(
TableName.RemindersRecipients,
`${TableName.Reminders}.id`,
`${TableName.RemindersRecipients}.reminderId`
)
.leftJoin<TUsers>(TableName.Users, `${TableName.RemindersRecipients}.userId`, `${TableName.Users}.id`)
.leftJoin<TSecretsV2>(TableName.SecretV2, `${TableName.Reminders}.secretId`, `${TableName.SecretV2}.id`)
.leftJoin<TSecretFolders>(
TableName.SecretFolder,
`${TableName.SecretV2}.folderId`,
`${TableName.SecretFolder}.id`
)
.leftJoin<TProjectEnvironments>(
TableName.Environment,
`${TableName.SecretFolder}.envId`,
`${TableName.Environment}.id`
)
.leftJoin<TProjects>(TableName.Project, `${TableName.Environment}.projectId`, `${TableName.Project}.id`)
.leftJoin<TOrganizations>(TableName.Organization, `${TableName.Project}.orgId`, `${TableName.Organization}.id`)
.select(selectAllTableCols(TableName.Reminders))
.select(db.ref("email").withSchema(TableName.Users))
.select(db.ref("name").withSchema(TableName.Project).as("projectName"))
.select(db.ref("id").withSchema(TableName.Project).as("projectId"))
.select(db.ref("name").withSchema(TableName.Organization).as("organizationName"));
const reminders = sqlNestRelationships({
data: rawReminders,
key: "id",
parentMapper: (el) => ({
_id: el.id,
...RemindersSchema.parse(el),
projectName: el.projectName,
projectId: el.projectId,
organizationName: el.organizationName
}),
childrenMapper: [
{
key: "email",
label: "recipients" as const,
mapper: ({ email }) => ({
email
})
}
]
});
return reminders;
};
const findUpcomingReminders = async (daysAhead: number = 7, tx?: Knex) => {
const { startOfDay } = getTodayDateRange();
const futureDate = new Date(startOfDay);
futureDate.setDate(futureDate.getDate() + daysAhead);
const reminders = await (tx || db)(TableName.Reminders)
.where("nextReminderDate", ">=", startOfDay)
.where("nextReminderDate", "<=", futureDate)
.orderBy("nextReminderDate", "asc")
.leftJoin(
TableName.RemindersRecipients,
`${TableName.Reminders}.id`,
`${TableName.RemindersRecipients}.reminderId`
)
.select(selectAllTableCols(TableName.Reminders))
.select(db.ref("userId").withSchema(TableName.RemindersRecipients));
return reminders;
};
const findSecretReminder = async (secretId: string, tx?: Knex) => {
const rawReminders = await (tx || db)(TableName.Reminders)
.where(`${TableName.Reminders}.secretId`, secretId)
.leftJoin(
TableName.RemindersRecipients,
`${TableName.Reminders}.id`,
`${TableName.RemindersRecipients}.reminderId`
)
.select(selectAllTableCols(TableName.Reminders))
.select(db.ref("userId").withSchema(TableName.RemindersRecipients));
const reminders = sqlNestRelationships({
data: rawReminders,
key: "id",
parentMapper: (el) => ({
_id: el.id,
...RemindersSchema.parse(el)
}),
childrenMapper: [
{
key: "userId",
label: "recipients" as const,
mapper: ({ userId }) => userId
}
]
});
return reminders[0] || null;
};
return {
...reminderOrm,
findSecretDailyReminders,
findUpcomingReminders,
findSecretReminder
};
};
@@ -0,0 +1,190 @@
/* eslint-disable no-await-in-loop */
import RE2 from "re2";
import { logger } from "@app/lib/logger";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import { TSecretReminderRecipientsDALFactory } from "../secret-reminder-recipients/secret-reminder-recipients-dal";
import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";
import { TReminderServiceFactory } from "./reminder-service";
type TDailyReminderQueueServiceFactoryDep = {
reminderService: TReminderServiceFactory;
queueService: TQueueServiceFactory;
secretDAL: Pick<TSecretV2BridgeDALFactory, "findSecretsWithReminderRecipients">;
secretReminderRecipientsDAL: Pick<TSecretReminderRecipientsDALFactory, "delete">;
};
export type TDailyReminderQueueServiceFactory = ReturnType<typeof dailyReminderQueueServiceFactory>;
const uuidRegex = new RE2(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i);
export const dailyReminderQueueServiceFactory = ({
reminderService,
queueService,
secretDAL,
secretReminderRecipientsDAL
}: TDailyReminderQueueServiceFactoryDep) => {
queueService.start(QueueName.DailyReminders, async () => {
logger.info(`${QueueName.DailyReminders}: queue task started`);
await reminderService.sendDailyReminders();
logger.info(`${QueueName.DailyReminders}: queue task completed`);
});
queueService.start(QueueName.SecretReminderMigration, async () => {
const REMINDER_PRUNE_BATCH_SIZE = 5_000;
const MAX_RETRY_ON_FAILURE = 3;
let numberOfRetryOnFailure = 0;
let deletedReminderCount = 0;
logger.info(`${QueueName.SecretReminderMigration}: queue task started`);
try {
const repeatableJobs = await queueService.getRepeatableJobs(QueueName.SecretReminder);
const delayedJobs = await queueService.getDelayedJobs(QueueName.SecretReminder);
logger.info(`${QueueName.SecretReminderMigration}: found ${repeatableJobs.length} secret reminder jobs`);
const reminderJobs = repeatableJobs
.map((job) => ({ secretId: job.id?.replace("reminder-", "") as string, jobKey: job.key }))
.filter(Boolean);
const reminderDelayedJobs = delayedJobs.reduce((map, job) => {
const match = uuidRegex.exec(job.repeatJobKey || "");
if (match) {
map.set(match[0], {
timestamp: job.timestamp,
delay: job.delay
});
}
return map;
}, new Map<string, { timestamp: number; delay: number }>());
if (reminderJobs.length === 0) {
logger.info(`${QueueName.SecretReminderMigration}: no reminder jobs found`);
return;
}
for (let offset = 0; offset < reminderJobs.length; offset += REMINDER_PRUNE_BATCH_SIZE) {
try {
const batch = reminderJobs.slice(offset, offset + REMINDER_PRUNE_BATCH_SIZE);
const batchIds = batch.map((job) => job.secretId);
// Find existing secrets with pagination
// eslint-disable-next-line no-await-in-loop
const secrets = await secretDAL.findSecretsWithReminderRecipients(batchIds, REMINDER_PRUNE_BATCH_SIZE);
const secretsWithReminder = secrets.filter((secret) => secret.reminderRepeatDays);
const foundSecretIds = new Set(secretsWithReminder.map((secret) => secret.id));
// Find IDs that don't exist in either table
const secretIdsNotFound = batchIds.filter((secretId) => !foundSecretIds.has(secretId));
// Delete reminders for non-existent secrets
for (const secretId of secretIdsNotFound) {
const jobKey = reminderJobs.find((r) => r.secretId === secretId)?.jobKey;
if (jobKey) {
// eslint-disable-next-line no-await-in-loop
await queueService.stopRepeatableJobByKey(QueueName.SecretReminder, jobKey);
deletedReminderCount += 1;
}
}
for (const secretId of foundSecretIds) {
const jobKey = reminderJobs.find((r) => r.secretId === secretId)?.jobKey;
if (jobKey) {
await queueService.stopRepeatableJobByKey(QueueName.SecretReminder, jobKey);
deletedReminderCount += 1;
}
}
await reminderService.batchCreateReminders(
secretsWithReminder.map((secret) => {
const delayedJob = reminderDelayedJobs.get(secret.id);
const nextDate = delayedJob ? new Date(delayedJob.timestamp + delayedJob.delay) : undefined;
return {
secretId: secret.id,
message: secret.reminderNote,
repeatDays: secret.reminderRepeatDays,
nextReminderDate: nextDate,
recipients: secret.recipients || []
};
})
);
await secretReminderRecipientsDAL.delete({ $in: { secretId: secretsWithReminder.map((s) => s.id) } });
numberOfRetryOnFailure = 0;
} catch (error) {
numberOfRetryOnFailure += 1;
logger.error(error, `Failed to process batch at offset ${offset}`);
if (numberOfRetryOnFailure >= MAX_RETRY_ON_FAILURE) {
break;
}
// Retry the current batch
offset -= REMINDER_PRUNE_BATCH_SIZE;
// eslint-disable-next-line no-promise-executor-return, @typescript-eslint/no-loop-func, no-await-in-loop
await new Promise((resolve) => setTimeout(resolve, 500 * numberOfRetryOnFailure));
}
// Small delay between batches
// eslint-disable-next-line no-promise-executor-return, @typescript-eslint/no-loop-func, no-await-in-loop
await new Promise((resolve) => setTimeout(resolve, 10));
}
} catch (error) {
logger.error(error, "Failed to complete secret reminder pruning");
} finally {
logger.info(
`${QueueName.SecretReminderMigration}: secret reminders completed. Deleted ${deletedReminderCount} reminders`
);
}
});
// we do a repeat cron job in utc timezone at 12 Midnight each day
const startDailyRemindersJob = async () => {
// clear previous job
await queueService.stopRepeatableJob(
QueueName.DailyReminders,
QueueJobs.DailyReminders,
{ pattern: "0 0 * * *", utc: true },
QueueName.DailyReminders // just a job id
);
await queueService.queue(QueueName.DailyReminders, QueueJobs.DailyReminders, undefined, {
delay: 5000,
jobId: QueueName.DailyReminders,
repeat: { pattern: "0 0 * * *", utc: true }
});
};
// TODO: remove once all the old reminders in queues are migrated
const startSecretReminderMigrationJob = async () => {
// clear previous job
await queueService.stopRepeatableJob(
QueueName.SecretReminderMigration,
QueueJobs.SecretReminderMigration,
{ pattern: "0 */1 * * *", utc: true },
QueueName.SecretReminderMigration // just a job id
);
await queueService.queue(QueueName.SecretReminderMigration, QueueJobs.SecretReminderMigration, undefined, {
delay: 5000,
jobId: QueueName.SecretReminderMigration,
repeat: { pattern: "0 */1 * * *", utc: true }
});
};
queueService.listen(QueueName.DailyReminders, "failed", (_, err) => {
logger.error(err, `${QueueName.DailyReminders}: resource cleanup failed`);
});
queueService.listen(QueueName.SecretReminderMigration, "failed", (_, err) => {
logger.error(err, `${QueueName.SecretReminderMigration}: resource cleanup failed`);
});
return {
startDailyRemindersJob,
startSecretReminderMigrationJob
};
};
@@ -0,0 +1,332 @@
/* eslint-disable no-await-in-loop */
import { ForbiddenError } from "@casl/ability";
import { Knex } from "knex";
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service-types";
import { ProjectPermissionSecretActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
import { BadRequestError } from "@app/lib/errors";
import { logger } from "@app/lib/logger";
import { ActorAuthMethod, ActorType } from "../auth/auth-type";
import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal";
import { TReminderRecipientDALFactory } from "../reminder-recipients/reminder-dal";
import { SmtpTemplates, TSmtpService } from "../smtp/smtp-service";
import { TReminderDALFactory } from "./reminder-dal";
import { TBatchCreateReminderDTO, TCreateReminderDTO } from "./reminder-types";
type TReminderServiceFactoryDep = {
reminderDAL: TReminderDALFactory;
reminderRecipientDAL: TReminderRecipientDALFactory;
smtpService: TSmtpService;
projectMembershipDAL: Pick<TProjectMembershipDALFactory, "findAllProjectMembers">;
permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
};
export type TReminderServiceFactory = ReturnType<typeof reminderServiceFactory>;
export const reminderServiceFactory = ({
reminderDAL,
reminderRecipientDAL,
smtpService,
projectMembershipDAL,
permissionService
}: TReminderServiceFactoryDep) => {
const addDays = (days: number, fromDate: Date = new Date()): Date => {
const result = new Date(fromDate);
result.setDate(result.getDate() + days);
return result;
};
const $manageReminderRecipients = async (reminderId: string, newRecipients?: string[] | null): Promise<void> => {
if (!newRecipients || newRecipients.length === 0) {
// If no recipients provided, remove all existing recipients
await reminderRecipientDAL.deleteById(reminderId);
return;
}
// Remove duplicates from input
const uniqueRecipients = [...new Set(newRecipients)];
// Get existing recipients
const existingRecipients = await reminderRecipientDAL.find({ reminderId });
const existingUserIds = new Set(existingRecipients.map((r) => r.userId));
const newUserIds = new Set(uniqueRecipients);
// Find recipients to add and remove
const recipientsToAdd = uniqueRecipients.filter((userId) => !existingUserIds.has(userId));
const recipientsToRemove = existingRecipients.filter((r) => !newUserIds.has(r.userId));
// Perform database operations
if (recipientsToRemove.length > 0) {
await reminderRecipientDAL.delete({ $in: { id: recipientsToRemove.map((r) => r.id) } });
}
if (recipientsToAdd.length > 0) {
await reminderRecipientDAL.insertMany(
recipientsToAdd.map((userId) => ({
reminderId,
userId
}))
);
}
};
const createReminderInternal = async ({
secretId,
message,
repeatDays,
nextReminderDate: nextReminderDateInput,
recipients
}: {
secretId?: string;
message?: string | null;
repeatDays?: number | null;
nextReminderDate?: string | null;
recipients?: string[] | null;
}) => {
if (!secretId) {
throw new BadRequestError({ message: "secretId is required" });
}
let nextReminderDate;
if (nextReminderDateInput) {
nextReminderDate = new Date(nextReminderDateInput);
}
if (repeatDays && repeatDays > 0) {
nextReminderDate = addDays(repeatDays);
}
if (!nextReminderDate) {
throw new BadRequestError({ message: "repeatDays must be a positive number" });
}
const existingReminder = await reminderDAL.findOne({ secretId });
let reminderId: string;
if (existingReminder) {
// Update existing reminder
await reminderDAL.updateById(existingReminder.id, {
message,
repeatDays,
nextReminderDate
});
reminderId = existingReminder.id;
} else {
// Create new reminder
const newReminder = await reminderDAL.create({
secretId,
message,
repeatDays,
nextReminderDate
});
reminderId = newReminder.id;
}
// Manage recipients (add/update/delete as needed)
await $manageReminderRecipients(reminderId, recipients);
return { id: reminderId, created: !existingReminder };
};
const createReminder = async ({
actor,
actorId,
actorOrgId,
actorAuthMethod,
projectId,
reminder
}: TCreateReminderDTO) => {
const { permission } = await permissionService.getProjectPermission({
actor,
actorId,
projectId,
actorAuthMethod,
actorOrgId
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionSecretActions.Edit, ProjectPermissionSub.Secrets);
const response = await createReminderInternal(reminder);
return response;
};
const getReminder = async ({
secretId,
projectId,
actor,
actorId,
actorOrgId,
actorAuthMethod
}: {
secretId: string;
projectId: string;
actor: ActorType;
actorId: string;
actorOrgId: string;
actorAuthMethod: ActorAuthMethod;
}) => {
const { permission } = await permissionService.getProjectPermission({
actor,
actorId,
projectId,
actorAuthMethod,
actorOrgId
});
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretActions.DescribeSecret,
ProjectPermissionSub.Secrets
);
const reminder = await reminderDAL.findSecretReminder(secretId);
return reminder;
};
const sendDailyReminders = async () => {
const remindersToSend = await reminderDAL.findSecretDailyReminders();
for (const reminder of remindersToSend) {
try {
const recipients: string[] = reminder.recipients
.map((r) => r.email)
.filter((email): email is string => Boolean(email));
if (recipients.length === 0) {
const members = await projectMembershipDAL.findAllProjectMembers(reminder.projectId);
recipients.push(...members.map((m) => m.user.email).filter((email): email is string => Boolean(email)));
}
await smtpService.sendMail({
template: SmtpTemplates.SecretReminder,
subjectLine: "Infisical secret reminder",
recipients,
substitutions: {
reminderNote: reminder.message || "",
projectName: reminder.projectName || "",
organizationName: reminder.organizationName || ""
}
});
if (reminder.repeatDays) {
await reminderDAL.updateById(reminder.id, { nextReminderDate: addDays(reminder.repeatDays) });
} else {
await reminderDAL.deleteById(reminder.id);
}
} catch (error) {
logger.error(
error,
`Failed to send reminder to recipients ${reminder.recipients.map((r) => r.email).join(", ")}`
);
}
}
};
const deleteReminder = async ({
actor,
actorId,
actorOrgId,
actorAuthMethod,
secretId,
projectId
}: {
actor: ActorType;
actorId: string;
actorOrgId: string;
actorAuthMethod: ActorAuthMethod;
secretId: string;
projectId: string;
}) => {
const { permission } = await permissionService.getProjectPermission({
actor,
actorId,
projectId,
actorAuthMethod,
actorOrgId
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionSecretActions.Edit, ProjectPermissionSub.Secrets);
await reminderDAL.delete({ secretId });
};
const removeReminderRecipients = async (secretId: string, tx?: Knex) => {
const reminder = await reminderDAL.findOne({ secretId }, tx);
if (!reminder) {
return;
}
await reminderRecipientDAL.delete({ reminderId: reminder.id }, tx);
};
const deleteReminderBySecretId = async (secretId: string, tx?: Knex) => {
await reminderDAL.delete({ secretId }, tx);
};
const batchCreateReminders = async (remindersData: TBatchCreateReminderDTO) => {
if (!remindersData || remindersData.length === 0) {
return { created: 0, reminderIds: [] };
}
const processedReminders = remindersData.map(
({ secretId, message, repeatDays, nextReminderDate: nextReminderDateInput, recipients }) => {
let nextReminderDate;
if (nextReminderDateInput) {
nextReminderDate = new Date(nextReminderDateInput);
}
if (repeatDays && repeatDays > 0 && !nextReminderDate) {
nextReminderDate = addDays(repeatDays);
}
if (!nextReminderDate) {
throw new BadRequestError({
message: `repeatDays must be a positive number for secretId: ${secretId}`
});
}
return {
secretId,
message,
repeatDays,
nextReminderDate,
recipients: recipients ? [...new Set(recipients)] : []
};
}
);
const newReminders = await reminderDAL.insertMany(
processedReminders.map(({ secretId, message, repeatDays, nextReminderDate }) => ({
secretId,
message,
repeatDays,
nextReminderDate
}))
);
const allRecipientInserts: Array<{ reminderId: string; userId: string }> = [];
newReminders.forEach((reminder, index) => {
const { recipients } = processedReminders[index];
if (recipients && recipients.length > 0) {
recipients.forEach((userId) => {
allRecipientInserts.push({
reminderId: reminder.id,
userId
});
});
}
});
if (allRecipientInserts.length > 0) {
await reminderRecipientDAL.insertMany(allRecipientInserts);
}
return {
created: newReminders.length,
reminderIds: newReminders.map((r) => r.id)
};
};
return {
createReminder,
getReminder,
sendDailyReminders,
deleteReminder,
removeReminderRecipients,
deleteReminderBySecretId,
batchCreateReminders,
createReminderInternal
};
};
@@ -0,0 +1,44 @@
import { ActorAuthMethod, ActorType } from "../auth/auth-type";
export type TReminder = {
id: string;
secretId?: string;
message?: string;
repeatDays?: number;
nextReminderDate?: Date;
createdAt: Date;
};
export type TCreateReminderDTO = {
actor: ActorType;
actorId: string;
actorOrgId: string;
actorAuthMethod: ActorAuthMethod;
projectId: string;
reminder: {
secretId?: string;
message?: string | null;
repeatDays?: number | null;
nextReminderDate?: string | null;
recipients?: string[] | null;
};
};
export type TCreateSecretReminderDTO = {
secretName: string;
projectId: string;
environment: string;
secretPath: string;
message?: string | null;
repeatDays?: number | null;
nextReminderDate?: string | null;
recipients?: string[] | null;
};
export type TBatchCreateReminderDTO = {
secretId: string;
message?: string | null;
repeatDays?: number | null;
nextReminderDate?: string | Date | null;
recipients?: string[] | null;
}[];