Refactor to secret queue

backend-pg/src/@types/fastify.d.ts

@@ -36,7 +36,6 @@ import { TProjectRoleServiceFactory } from "@app/services/project-role/project-r
 import { TSecretServiceFactory } from "@app/services/secret/secret-service";
 import { TSecretFolderServiceFactory } from "@app/services/secret-folder/secret-folder-service";
 import { TSecretImportServiceFactory } from "@app/services/secret-import/secret-import-service";
-import { TSecretReminderServiceFactory } from "@app/services/secret-reminder/secret-reminder-service";
 import { TSecretTagServiceFactory } from "@app/services/secret-tag/secret-tag-service";
 import { TServiceTokenServiceFactory } from "@app/services/service-token/service-token-service";
 import { TSuperAdminServiceFactory } from "@app/services/super-admin/super-admin-service";
@@ -100,7 +99,6 @@ declare module "fastify" {
       secretApprovalPolicy: TSecretApprovalPolicyServiceFactory;
       secretApprovalRequest: TSecretApprovalRequestServiceFactory;
       secretRotation: TSecretRotationServiceFactory;
-      secretReminder: TSecretReminderServiceFactory;
       snapshot: TSecretSnapshotServiceFactory;
       saml: TSamlConfigServiceFactory;
       auditLog: TAuditLogServiceFactory;

backend-pg/src/server/routes/index.ts

@@ -85,8 +85,6 @@ import { secretFolderServiceFactory } from "@app/services/secret-folder/secret-f
 import { secretFolderVersionDALFactory } from "@app/services/secret-folder/secret-folder-version-dal";
 import { secretImportDALFactory } from "@app/services/secret-import/secret-import-dal";
 import { secretImportServiceFactory } from "@app/services/secret-import/secret-import-service";
-import { secretReminderServiceFactory } from "@app/services/secret-reminder";
-import { secretReminderQueueFactory } from "@app/services/secret-reminder/secret-reminder-queue";
 import { secretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
 import { secretTagServiceFactory } from "@app/services/secret-tag/secret-tag-service";
 import { serviceTokenDALFactory } from "@app/services/service-token/service-token-dal";
@@ -355,24 +353,11 @@ export const registerRoutes = async (
     webhookDAL
   });
 
-  const secretReminderQueue = secretReminderQueueFactory({
-    queue: queueService,
-    orgDal: orgDAL,
-    projectDal: projectDAL,
-    projectMembershipDal: projectMembershipDAL,
-    smtpService
-  });
-
-  const secretReminderService = secretReminderServiceFactory({
-    secretReminderQueue
-  });
-
   const secretService = secretServiceFactory({
     folderDAL,
     secretVersionDAL,
     secretVersionTagDAL,
     secretBlindIndexDAL,
-    secretReminderService,
     permissionService,
     secretDAL,
     secretTagDAL,
@@ -483,7 +468,6 @@ export const registerRoutes = async (
     secretApprovalPolicy: sapService,
     secretApprovalRequest: sarService,
     secretRotation: secretRotationService,
-    secretReminder: secretReminderService,
     snapshot: snapshotService,
     saml: samlService,
     auditLog: auditLogService,

backend-pg/src/services/secret-reminder/index.ts

@@ -1 +0,0 @@
-export { secretReminderServiceFactory } from "./secret-reminder-service";

backend-pg/src/services/secret-reminder/secret-reminder-queue/index.ts

@@ -1,105 +0,0 @@
-import { getConfig } from "@app/lib/config/env";
-import { daysToMillisecond, secondsToMillis } from "@app/lib/dates";
-import { logger } from "@app/lib/logger";
-import { QueueJobs, QueueName, TQueueJobTypes, TQueueServiceFactory } from "@app/queue";
-import { TOrgDalFactory } from "@app/services/org/org-dal";
-import { TProjectDalFactory } from "@app/services/project/project-dal";
-import { TProjectMembershipDalFactory } from "@app/services/project-membership/project-membership-dal";
-import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service";
-
-export type TSecretReminderQueueFactory = ReturnType<typeof secretReminderQueueFactory>;
-
-type TSecretReminderQueueFactoryDep = {
-  queue: TQueueServiceFactory;
-  projectMembershipDal: Pick<TProjectMembershipDalFactory, "findAllProjectMembers">;
-  orgDal: Pick<TOrgDalFactory, "findOrgById" | "findOrgByProjectId">;
-  projectDal: Pick<TProjectDalFactory, "findById">;
-  smtpService: TSmtpService;
-};
-
-export const secretReminderQueueFactory = ({
-  queue,
-  projectMembershipDal,
-  smtpService,
-  orgDal,
-  projectDal
-}: TSecretReminderQueueFactoryDep) => {
-  const addToQueue = async (data: TQueueJobTypes["secret-reminder"]["payload"]) => {
-    const appCfg = getConfig();
-    queue.queue(QueueName.SecretReminder, QueueJobs.SecretReminder, data, {
-      jobId: `reminder-${data.secretId}`,
-      repeat: {
-        // on prod it this will be in days, in development this will be second
-        every:
-          appCfg.NODE_ENV === "development"
-            ? secondsToMillis(data.repeatDays)
-            : daysToMillisecond(data.repeatDays),
-        immediately: true
-      }
-    });
-  };
-
-  const removeFromQueue = async (secretId: string, repeatDays: number) => {
-    const appCfg = getConfig();
-    await queue.stopRepeatableJob(
-      QueueName.SecretReminder,
-      QueueJobs.SecretReminder,
-      {
-        // on prod it this will be in days, in development this will be second
-        every:
-          appCfg.NODE_ENV === "development"
-            ? secondsToMillis(repeatDays)
-            : daysToMillisecond(repeatDays)
-      },
-      `reminder-${secretId}`
-    );
-  };
-
-  queue.start(QueueName.SecretReminder, async ({ data }) => {
-    logger.info(`secretReminderQueue.process: [secretDocument=${data.secretId}]`);
-
-    const { projectId } = data;
-
-    const organization = await orgDal.findOrgByProjectId(projectId);
-    const project = await projectDal.findById(projectId);
-
-    if (!organization) {
-      logger.info(
-        `secretReminderQueue.process: [secretDocument=${data.secretId}] no organization found`
-      );
-      return;
-    }
-
-    if (!project) {
-      logger.info(
-        `secretReminderQueue.process: [secretDocument=${data.secretId}] no project found`
-      );
-      return;
-    }
-
-    const projectMembers = await projectMembershipDal.findAllProjectMembers(projectId);
-
-    if (!projectMembers || !projectMembers.length) {
-      logger.info(
-        `secretReminderQueue.process: [secretDocument=${data.secretId}] no project members found`
-      );
-      return;
-    }
-
-    await smtpService.sendMail({
-      template: SmtpTemplates.SecretReminder,
-      subjectLine: "Infisical secret reminder",
-      recipients: [...projectMembers.map((m) => m.user.email)],
-      substitutions: {
-        reminderNote: data.note, // May not be present.
-        projectName: project.name,
-        organizationName: organization.name
-      }
-    });
-  });
-
-  return {
-    addToQueue,
-    removeFromQueue
-  };
-};

backend-pg/src/services/secret-reminder/secret-reminder-service.ts

@@ -1,102 +0,0 @@
-import { BadRequestError } from "@app/lib/errors";
-import { logger } from "@app/lib/logger";
-
-import { TSecretReminderQueueFactory } from "./secret-reminder-queue";
-import {
-  TCreateSecretReminderDTO,
-  TDeleteSecretReminderDTO,
-  THandleReminderDTO
-} from "./secret-reminder-types";
-
-type TSecretReminderServiceFactoryDep = {
-  secretReminderQueue: TSecretReminderQueueFactory;
-};
-
-export type TSecretReminderServiceFactory = ReturnType<typeof secretReminderServiceFactory>;
-
-export const secretReminderServiceFactory = ({
-  secretReminderQueue
-}: TSecretReminderServiceFactoryDep) => {
-  const createReminder = async ({ oldSecret, newSecret, projectId }: TCreateSecretReminderDTO) => {
-    try {
-      if (oldSecret.id !== newSecret.id) {
-        throw new BadRequestError({
-          name: "SecretReminderIdMismatch",
-          message: "Existing secret didn't match the updated secret ID."
-        });
-      }
-
-      if (!newSecret.secretReminderRepeatDays) {
-        throw new BadRequestError({
-          name: "SecretReminderRepeatDaysMissing",
-          message: "Secret reminder repeat days is missing."
-        });
-      }
-
-      // If the secret already has a reminder, we should remove the existing one first.
-      if (oldSecret.secretReminderRepeatDays) {
-        await secretReminderQueue.removeFromQueue(oldSecret.id, oldSecret.secretReminderRepeatDays);
-      }
-
-      await secretReminderQueue.addToQueue({
-        note: newSecret.secretReminderNote,
-        projectId,
-        repeatDays: newSecret.secretReminderRepeatDays,
-        secretId: newSecret.id
-      });
-    } catch (err) {
-      logger.error(err, "Failed to create secret reminder.");
-      throw new BadRequestError({
-        name: "SecretReminderCreateFailed",
-        message: "Failed to create secret reminder."
-      });
-    }
-  };
-
-  const deleteReminder = async ({ secretId, repeatDays }: TDeleteSecretReminderDTO) => {
-    try {
-      logger.info(`[${secretId}] Removing secret reminder from queue.`);
-      await secretReminderQueue.removeFromQueue(secretId, repeatDays);
-    } catch (err) {
-      logger.error(err, "Failed to remove secret reminder from queue.");
-      throw new BadRequestError({
-        name: "SecretReminderDeleteFailed",
-        message: "Failed to delete secret reminder."
-      });
-    }
-  };
-
-  // A handler function to cut down on code duplication in the future.
-  const handleReminder = async ({ newSecret, oldSecret, projectId }: THandleReminderDTO) => {
-    const { secretReminderRepeatDays, secretReminderNote } = newSecret;
-
-    if (newSecret.type !== "personal" && secretReminderRepeatDays !== undefined) {
-      if (
-        (secretReminderRepeatDays &&
-          oldSecret.secretReminderRepeatDays !== secretReminderRepeatDays) ||
-        (secretReminderNote && oldSecret.secretReminderNote !== secretReminderNote)
-      ) {
-        await createReminder({
-          oldSecret,
-          newSecret,
-          projectId
-        });
-      } else if (
-        secretReminderRepeatDays === null &&
-        secretReminderNote === null &&
-        oldSecret.secretReminderRepeatDays
-      ) {
-        await deleteReminder({
-          secretId: oldSecret.id,
-          repeatDays: oldSecret.secretReminderRepeatDays
-        });
-      }
-    }
-  };
-
-  return {
-    createReminder,
-    deleteReminder,
-    handleReminder
-  };
-};

backend-pg/src/services/secret-reminder/secret-reminder-types.ts

@@ -1,25 +0,0 @@
-import { TSecrets } from "@app/db/schemas";
-
-type TPartialSecret = Pick<TSecrets, "id" | "secretReminderRepeatDays" | "secretReminderNote">;
-
-type TPartialInputSecret = Pick<
-  TSecrets,
-  "type" | "secretReminderNote" | "secretReminderRepeatDays" | "id"
->;
-
-export type TCreateSecretReminderDTO = {
-  oldSecret: TPartialSecret;
-  newSecret: TPartialSecret;
-  projectId: string;
-};
-
-export type TDeleteSecretReminderDTO = {
-  secretId: string;
-  repeatDays: number;
-};
-
-export type THandleReminderDTO = {
-  newSecret: TPartialInputSecret;
-  oldSecret: TPartialSecret;
-  projectId: string;
-};

backend-pg/src/services/secret/secret-queue.ts

@@ -1,5 +1,8 @@
 /* eslint-disable no-await-in-loop */
+import { getConfig } from "@app/lib/config/env";
 import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
+import { daysToMillisecond, secondsToMillis } from "@app/lib/dates";
+import { BadRequestError } from "@app/lib/errors";
 import { isSamePath } from "@app/lib/fn";
 import { logger } from "@app/lib/logger";
 import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
@@ -16,6 +19,11 @@ import { TWebhookDALFactory } from "../webhook/webhook-dal";
 import { fnTriggerWebhook } from "../webhook/webhook-fns";
 import { TSecretDALFactory } from "./secret-dal";
 import { interpolateSecrets } from "./secret-fns";
+import {
+  TCreateSecretReminderDTO,
+  THandleReminderDTO,
+  TRemoveSecretReminderDTO
+} from "./secret-types";
 
 export type TSecretQueueFactory = ReturnType<typeof secretQueueFactory>;
 
@@ -78,6 +86,108 @@ export const secretQueueFactory = ({
     await syncIntegrations(dto);
   };
 
+  const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => {
+    const appCfg = getConfig();
+    await queueService.stopRepeatableJob(
+      QueueName.SecretReminder,
+      QueueJobs.SecretReminder,
+      {
+        // on prod it this will be in days, in development this will be second
+        every:
+          appCfg.NODE_ENV === "development"
+            ? secondsToMillis(dto.repeatDays)
+            : daysToMillisecond(dto.repeatDays)
+      },
+      `reminder-${dto.secretId}`
+    );
+  };
+
+  const addSecretReminder = async ({
+    oldSecret,
+    newSecret,
+    projectId
+  }: TCreateSecretReminderDTO) => {
+    try {
+      const appCfg = getConfig();
+
+      if (oldSecret.id !== newSecret.id) {
+        throw new BadRequestError({
+          name: "SecretReminderIdMismatch",
+          message: "Existing secret didn't match the updated secret ID."
+        });
+      }
+
+      if (!newSecret.secretReminderRepeatDays) {
+        throw new BadRequestError({
+          name: "SecretReminderRepeatDaysMissing",
+          message: "Secret reminder repeat days is missing."
+        });
+      }
+
+      // If the secret already has a reminder, we should remove the existing one first.
+      if (oldSecret.secretReminderRepeatDays) {
+        await removeSecretReminder({
+          repeatDays: oldSecret.secretReminderRepeatDays,
+          secretId: oldSecret.id
+        });
+      }
+
+      await queueService.queue(
+        QueueName.SecretReminder,
+        QueueJobs.SecretReminder,
+        {
+          note: newSecret.secretReminderNote,
+          projectId,
+          repeatDays: newSecret.secretReminderRepeatDays,
+          secretId: newSecret.id
+        },
+        {
+          jobId: `reminder-${newSecret.id}`,
+          repeat: {
+            // on prod it this will be in days, in development this will be second
+            every:
+              appCfg.NODE_ENV === "development"
+                ? secondsToMillis(newSecret.secretReminderRepeatDays)
+                : daysToMillisecond(newSecret.secretReminderRepeatDays)
+          }
+        }
+      );
+    } catch (err) {
+      logger.error(err, "Failed to create secret reminder.");
+      throw new BadRequestError({
+        name: "SecretReminderCreateFailed",
+        message: "Failed to create secret reminder."
+      });
+    }
+  };
+
+  const handleSecretReminder = async ({ newSecret, oldSecret, projectId }: THandleReminderDTO) => {
+    const { secretReminderRepeatDays, secretReminderNote } = newSecret;
+
+    if (newSecret.type !== "personal" && secretReminderRepeatDays !== undefined) {
+      if (
+        (secretReminderRepeatDays &&
+          oldSecret.secretReminderRepeatDays !== secretReminderRepeatDays) ||
+        (secretReminderNote && oldSecret.secretReminderNote !== secretReminderNote)
+      ) {
+        await addSecretReminder({
+          oldSecret,
+          newSecret,
+          projectId
+        });
+      } else if (
+        secretReminderRepeatDays === null &&
+        secretReminderNote === null &&
+        oldSecret.secretReminderRepeatDays
+      ) {
+        await removeSecretReminder({
+          secretId: oldSecret.id,
+          repeatDays: oldSecret.secretReminderRepeatDays
+        });
+      }
+    }
+  };
+
   const getIntegrationSecrets = async (dto: TGetSecrets & { folderId: string }, key: string) => {
     const secrets = await secretDAL.findByFolderId(dto.folderId);
     if (!secrets.length) return {};
@@ -229,5 +339,11 @@ export const secretQueueFactory = ({
     await fnTriggerWebhook({ ...job.data, projectEnvDAL, webhookDAL });
   });
 
-  return { syncSecrets, syncIntegrations };
+  return {
+    syncSecrets,
+    syncIntegrations,
+    addSecretReminder,
+    removeSecretReminder,
+    handleSecretReminder
+  };
 };

backend-pg/src/services/secret/secret-service.ts

@@ -24,7 +24,6 @@ import { TProjectBotServiceFactory } from "../project-bot/project-bot-service";
 import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
 import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
 import { fnSecretsFromImports } from "../secret-import/secret-import-fns";
-import { TSecretReminderServiceFactory } from "../secret-reminder/secret-reminder-service";
 import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal";
 import { TSecretBlindIndexDALFactory } from "./secret-blind-index-dal";
 import { TSecretDALFactory } from "./secret-dal";
@@ -65,9 +64,11 @@ type TSecretServiceFactoryDep = {
   >;
   secretBlindIndexDAL: TSecretBlindIndexDALFactory;
   permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
-  secretReminderService: TSecretReminderServiceFactory;
   snapshotService: Pick<TSecretSnapshotServiceFactory, "performSnapshot">;
-  secretQueueService: Pick<TSecretQueueFactory, "syncSecrets">;
+  secretQueueService: Pick<
+    TSecretQueueFactory,
+    "syncSecrets" | "handleSecretReminder" | "removeSecretReminder"
+  >;
   projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
   secretImportDAL: Pick<TSecretImportDALFactory, "find">;
   secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
@@ -81,7 +82,6 @@ export const secretServiceFactory = ({
   folderDAL,
   secretBlindIndexDAL,
   permissionService,
-  secretReminderService,
   snapshotService,
   secretQueueService,
   projectBotService,
@@ -212,8 +212,8 @@ export const secretServiceFactory = ({
     for (const s of deletedSecrets) {
       if (s.secretReminderRepeatDays) {
         // eslint-disable-next-line no-await-in-loop
-        await secretReminderService
+        await secretQueueService
-          .deleteReminder({
+          .removeSecretReminder({
             secretId: s.id,
             repeatDays: s.secretReminderRepeatDays
           })
@@ -425,7 +425,7 @@ export const secretServiceFactory = ({
       newSecretNameBlindIndex = kN2NewBlindIndex[inputSecret.newSecretName];
     }
 
-    await secretReminderService.handleReminder({
+    await secretQueueService.handleSecretReminder({
       newSecret: {
         id: secrets[0].id,
         ...inputSecret

backend-pg/src/services/secret/secret-types.ts

@@ -9,6 +9,13 @@ import {
 } from "@app/db/schemas";
 import { TProjectPermission } from "@app/lib/types";
 
+type TPartialSecret = Pick<TSecrets, "id" | "secretReminderRepeatDays" | "secretReminderNote">;
+
+type TPartialInputSecret = Pick<
+  TSecrets,
+  "type" | "secretReminderNote" | "secretReminderRepeatDays" | "id"
+>;
+
 export type TCreateSecretDTO = {
   secretName: string;
   path: string;
@@ -220,3 +227,20 @@ export type TFnSecretBlindIndexCheckV2 = {
   userId?: string;
   inputSecrets: Array<{ secretBlindIndex: string; type?: SecretType }>;
 };
+
+export type THandleReminderDTO = {
+  newSecret: TPartialInputSecret;
+  oldSecret: TPartialSecret;
+  projectId: string;
+};
+
+export type TCreateSecretReminderDTO = {
+  oldSecret: TPartialSecret;
+  newSecret: TPartialSecret;
+  projectId: string;
+};
+
+export type TRemoveSecretReminderDTO = {
+  secretId: string;
+  repeatDays: number;
+};

backend-pg/src/services/smtp/smtp-service.ts

@@ -51,7 +51,12 @@ export const getTlsOption = (host?: SmtpHost | string, secure?: boolean) => {
 };
 
 export const smtpServiceFactory = (cfg: TSmtpConfig) => {
-  const smtp = createTransport({ ...cfg, ...getTlsOption(cfg.host, cfg.secure) });
+  const smtp = createTransport({
+    ...cfg,
+    ...getTlsOption(cfg.host, cfg.secure),
+    secure: false,
+    port: 587
+  });
   const isSmtpOn = Boolean(cfg.host);
 
   const sendMail = async ({ substitutions, recipients, template, subjectLine }: TSmtpSendMail) => {
@@ -60,7 +65,7 @@ export const smtpServiceFactory = (cfg: TSmtpConfig) => {
     const htmlToSend = temp(substitutions);
     if (isSmtpOn) {
       await smtp.sendMail({
-        from: cfg.from,
+        from: "[email protected]",
         to: recipients.join(", "),
         subject: subjectLine,
         html: htmlToSend