From a60dbe523b1637b7c84365b22a6c991f9fb41e25 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Mon, 22 Jan 2024 22:09:13 +0400 Subject: [PATCH] Refactor to secret queue --- backend-pg/src/@types/fastify.d.ts | 2 - backend-pg/src/server/routes/index.ts | 16 --- .../src/services/secret-reminder/index.ts | 1 - .../secret-reminder-queue/index.ts | 105 ---------------- .../secret-reminder-service.ts | 102 --------------- .../secret-reminder/secret-reminder-types.ts | 25 ---- .../src/services/secret/secret-queue.ts | 118 +++++++++++++++++- .../src/services/secret/secret-service.ts | 14 +-- .../src/services/secret/secret-types.ts | 24 ++++ backend-pg/src/services/smtp/smtp-service.ts | 9 +- 10 files changed, 155 insertions(+), 261 deletions(-) delete mode 100644 backend-pg/src/services/secret-reminder/index.ts delete mode 100644 backend-pg/src/services/secret-reminder/secret-reminder-queue/index.ts delete mode 100644 backend-pg/src/services/secret-reminder/secret-reminder-service.ts delete mode 100644 backend-pg/src/services/secret-reminder/secret-reminder-types.ts diff --git a/backend-pg/src/@types/fastify.d.ts b/backend-pg/src/@types/fastify.d.ts index 0490f2222..c3c75db72 100644 --- a/backend-pg/src/@types/fastify.d.ts +++ b/backend-pg/src/@types/fastify.d.ts @@ -36,7 +36,6 @@ import { TProjectRoleServiceFactory } from "@app/services/project-role/project-r import { TSecretServiceFactory } from "@app/services/secret/secret-service"; import { TSecretFolderServiceFactory } from "@app/services/secret-folder/secret-folder-service"; import { TSecretImportServiceFactory } from "@app/services/secret-import/secret-import-service"; -import { TSecretReminderServiceFactory } from "@app/services/secret-reminder/secret-reminder-service"; import { TSecretTagServiceFactory } from "@app/services/secret-tag/secret-tag-service"; import { TServiceTokenServiceFactory } from "@app/services/service-token/service-token-service"; import { TSuperAdminServiceFactory } from "@app/services/super-admin/super-admin-service"; @@ -100,7 +99,6 @@ declare module "fastify" { secretApprovalPolicy: TSecretApprovalPolicyServiceFactory; secretApprovalRequest: TSecretApprovalRequestServiceFactory; secretRotation: TSecretRotationServiceFactory; - secretReminder: TSecretReminderServiceFactory; snapshot: TSecretSnapshotServiceFactory; saml: TSamlConfigServiceFactory; auditLog: TAuditLogServiceFactory; diff --git a/backend-pg/src/server/routes/index.ts b/backend-pg/src/server/routes/index.ts index 19bf25217..5e5abfcb4 100644 --- a/backend-pg/src/server/routes/index.ts +++ b/backend-pg/src/server/routes/index.ts @@ -85,8 +85,6 @@ import { secretFolderServiceFactory } from "@app/services/secret-folder/secret-f import { secretFolderVersionDALFactory } from "@app/services/secret-folder/secret-folder-version-dal"; import { secretImportDALFactory } from "@app/services/secret-import/secret-import-dal"; import { secretImportServiceFactory } from "@app/services/secret-import/secret-import-service"; -import { secretReminderServiceFactory } from "@app/services/secret-reminder"; -import { secretReminderQueueFactory } from "@app/services/secret-reminder/secret-reminder-queue"; import { secretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; import { secretTagServiceFactory } from "@app/services/secret-tag/secret-tag-service"; import { serviceTokenDALFactory } from "@app/services/service-token/service-token-dal"; @@ -355,24 +353,11 @@ export const registerRoutes = async ( webhookDAL }); - const secretReminderQueue = secretReminderQueueFactory({ - queue: queueService, - orgDal: orgDAL, - projectDal: projectDAL, - projectMembershipDal: projectMembershipDAL, - smtpService - }); - - const secretReminderService = secretReminderServiceFactory({ - secretReminderQueue - }); - const secretService = secretServiceFactory({ folderDAL, secretVersionDAL, secretVersionTagDAL, secretBlindIndexDAL, - secretReminderService, permissionService, secretDAL, secretTagDAL, @@ -483,7 +468,6 @@ export const registerRoutes = async ( secretApprovalPolicy: sapService, secretApprovalRequest: sarService, secretRotation: secretRotationService, - secretReminder: secretReminderService, snapshot: snapshotService, saml: samlService, auditLog: auditLogService, diff --git a/backend-pg/src/services/secret-reminder/index.ts b/backend-pg/src/services/secret-reminder/index.ts deleted file mode 100644 index ffd4ed90c..000000000 --- a/backend-pg/src/services/secret-reminder/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { secretReminderServiceFactory } from "./secret-reminder-service"; diff --git a/backend-pg/src/services/secret-reminder/secret-reminder-queue/index.ts b/backend-pg/src/services/secret-reminder/secret-reminder-queue/index.ts deleted file mode 100644 index 100df7f5a..000000000 --- a/backend-pg/src/services/secret-reminder/secret-reminder-queue/index.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { getConfig } from "@app/lib/config/env"; -import { daysToMillisecond, secondsToMillis } from "@app/lib/dates"; -import { logger } from "@app/lib/logger"; -import { QueueJobs, QueueName, TQueueJobTypes, TQueueServiceFactory } from "@app/queue"; -import { TOrgDalFactory } from "@app/services/org/org-dal"; -import { TProjectDalFactory } from "@app/services/project/project-dal"; -import { TProjectMembershipDalFactory } from "@app/services/project-membership/project-membership-dal"; -import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service"; - -export type TSecretReminderQueueFactory = ReturnType; - -type TSecretReminderQueueFactoryDep = { - queue: TQueueServiceFactory; - projectMembershipDal: Pick; - orgDal: Pick; - projectDal: Pick; - smtpService: TSmtpService; -}; - -export const secretReminderQueueFactory = ({ - queue, - projectMembershipDal, - smtpService, - orgDal, - projectDal -}: TSecretReminderQueueFactoryDep) => { - const addToQueue = async (data: TQueueJobTypes["secret-reminder"]["payload"]) => { - const appCfg = getConfig(); - queue.queue(QueueName.SecretReminder, QueueJobs.SecretReminder, data, { - jobId: `reminder-${data.secretId}`, - repeat: { - // on prod it this will be in days, in development this will be second - every: - appCfg.NODE_ENV === "development" - ? secondsToMillis(data.repeatDays) - : daysToMillisecond(data.repeatDays), - immediately: true - } - }); - }; - - const removeFromQueue = async (secretId: string, repeatDays: number) => { - const appCfg = getConfig(); - await queue.stopRepeatableJob( - QueueName.SecretReminder, - QueueJobs.SecretReminder, - { - // on prod it this will be in days, in development this will be second - every: - appCfg.NODE_ENV === "development" - ? secondsToMillis(repeatDays) - : daysToMillisecond(repeatDays) - }, - `reminder-${secretId}` - ); - }; - - queue.start(QueueName.SecretReminder, async ({ data }) => { - logger.info(`secretReminderQueue.process: [secretDocument=${data.secretId}]`); - - const { projectId } = data; - - const organization = await orgDal.findOrgByProjectId(projectId); - const project = await projectDal.findById(projectId); - - if (!organization) { - logger.info( - `secretReminderQueue.process: [secretDocument=${data.secretId}] no organization found` - ); - return; - } - - if (!project) { - logger.info( - `secretReminderQueue.process: [secretDocument=${data.secretId}] no project found` - ); - return; - } - - const projectMembers = await projectMembershipDal.findAllProjectMembers(projectId); - - if (!projectMembers || !projectMembers.length) { - logger.info( - `secretReminderQueue.process: [secretDocument=${data.secretId}] no project members found` - ); - return; - } - - await smtpService.sendMail({ - template: SmtpTemplates.SecretReminder, - subjectLine: "Infisical secret reminder", - recipients: [...projectMembers.map((m) => m.user.email)], - substitutions: { - reminderNote: data.note, // May not be present. - projectName: project.name, - organizationName: organization.name - } - }); - }); - - return { - addToQueue, - removeFromQueue - }; -}; diff --git a/backend-pg/src/services/secret-reminder/secret-reminder-service.ts b/backend-pg/src/services/secret-reminder/secret-reminder-service.ts deleted file mode 100644 index b0586303a..000000000 --- a/backend-pg/src/services/secret-reminder/secret-reminder-service.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { BadRequestError } from "@app/lib/errors"; -import { logger } from "@app/lib/logger"; - -import { TSecretReminderQueueFactory } from "./secret-reminder-queue"; -import { - TCreateSecretReminderDTO, - TDeleteSecretReminderDTO, - THandleReminderDTO -} from "./secret-reminder-types"; - -type TSecretReminderServiceFactoryDep = { - secretReminderQueue: TSecretReminderQueueFactory; -}; - -export type TSecretReminderServiceFactory = ReturnType; - -export const secretReminderServiceFactory = ({ - secretReminderQueue -}: TSecretReminderServiceFactoryDep) => { - const createReminder = async ({ oldSecret, newSecret, projectId }: TCreateSecretReminderDTO) => { - try { - if (oldSecret.id !== newSecret.id) { - throw new BadRequestError({ - name: "SecretReminderIdMismatch", - message: "Existing secret didn't match the updated secret ID." - }); - } - - if (!newSecret.secretReminderRepeatDays) { - throw new BadRequestError({ - name: "SecretReminderRepeatDaysMissing", - message: "Secret reminder repeat days is missing." - }); - } - - // If the secret already has a reminder, we should remove the existing one first. - if (oldSecret.secretReminderRepeatDays) { - await secretReminderQueue.removeFromQueue(oldSecret.id, oldSecret.secretReminderRepeatDays); - } - - await secretReminderQueue.addToQueue({ - note: newSecret.secretReminderNote, - projectId, - repeatDays: newSecret.secretReminderRepeatDays, - secretId: newSecret.id - }); - } catch (err) { - logger.error(err, "Failed to create secret reminder."); - throw new BadRequestError({ - name: "SecretReminderCreateFailed", - message: "Failed to create secret reminder." - }); - } - }; - - const deleteReminder = async ({ secretId, repeatDays }: TDeleteSecretReminderDTO) => { - try { - logger.info(`[${secretId}] Removing secret reminder from queue.`); - await secretReminderQueue.removeFromQueue(secretId, repeatDays); - } catch (err) { - logger.error(err, "Failed to remove secret reminder from queue."); - throw new BadRequestError({ - name: "SecretReminderDeleteFailed", - message: "Failed to delete secret reminder." - }); - } - }; - - // A handler function to cut down on code duplication in the future. - const handleReminder = async ({ newSecret, oldSecret, projectId }: THandleReminderDTO) => { - const { secretReminderRepeatDays, secretReminderNote } = newSecret; - - if (newSecret.type !== "personal" && secretReminderRepeatDays !== undefined) { - if ( - (secretReminderRepeatDays && - oldSecret.secretReminderRepeatDays !== secretReminderRepeatDays) || - (secretReminderNote && oldSecret.secretReminderNote !== secretReminderNote) - ) { - await createReminder({ - oldSecret, - newSecret, - projectId - }); - } else if ( - secretReminderRepeatDays === null && - secretReminderNote === null && - oldSecret.secretReminderRepeatDays - ) { - await deleteReminder({ - secretId: oldSecret.id, - repeatDays: oldSecret.secretReminderRepeatDays - }); - } - } - }; - - return { - createReminder, - deleteReminder, - handleReminder - }; -}; diff --git a/backend-pg/src/services/secret-reminder/secret-reminder-types.ts b/backend-pg/src/services/secret-reminder/secret-reminder-types.ts deleted file mode 100644 index dd4abac64..000000000 --- a/backend-pg/src/services/secret-reminder/secret-reminder-types.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { TSecrets } from "@app/db/schemas"; - -type TPartialSecret = Pick; - -type TPartialInputSecret = Pick< - TSecrets, - "type" | "secretReminderNote" | "secretReminderRepeatDays" | "id" ->; - -export type TCreateSecretReminderDTO = { - oldSecret: TPartialSecret; - newSecret: TPartialSecret; - projectId: string; -}; - -export type TDeleteSecretReminderDTO = { - secretId: string; - repeatDays: number; -}; - -export type THandleReminderDTO = { - newSecret: TPartialInputSecret; - oldSecret: TPartialSecret; - projectId: string; -}; diff --git a/backend-pg/src/services/secret/secret-queue.ts b/backend-pg/src/services/secret/secret-queue.ts index 681f47c7c..910899f3d 100644 --- a/backend-pg/src/services/secret/secret-queue.ts +++ b/backend-pg/src/services/secret/secret-queue.ts @@ -1,5 +1,8 @@ /* eslint-disable no-await-in-loop */ +import { getConfig } from "@app/lib/config/env"; import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; +import { daysToMillisecond, secondsToMillis } from "@app/lib/dates"; +import { BadRequestError } from "@app/lib/errors"; import { isSamePath } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; @@ -16,6 +19,11 @@ import { TWebhookDALFactory } from "../webhook/webhook-dal"; import { fnTriggerWebhook } from "../webhook/webhook-fns"; import { TSecretDALFactory } from "./secret-dal"; import { interpolateSecrets } from "./secret-fns"; +import { + TCreateSecretReminderDTO, + THandleReminderDTO, + TRemoveSecretReminderDTO +} from "./secret-types"; export type TSecretQueueFactory = ReturnType; @@ -78,6 +86,108 @@ export const secretQueueFactory = ({ await syncIntegrations(dto); }; + const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => { + const appCfg = getConfig(); + await queueService.stopRepeatableJob( + QueueName.SecretReminder, + QueueJobs.SecretReminder, + { + // on prod it this will be in days, in development this will be second + every: + appCfg.NODE_ENV === "development" + ? secondsToMillis(dto.repeatDays) + : daysToMillisecond(dto.repeatDays) + }, + `reminder-${dto.secretId}` + ); + }; + + const addSecretReminder = async ({ + oldSecret, + newSecret, + projectId + }: TCreateSecretReminderDTO) => { + try { + const appCfg = getConfig(); + + if (oldSecret.id !== newSecret.id) { + throw new BadRequestError({ + name: "SecretReminderIdMismatch", + message: "Existing secret didn't match the updated secret ID." + }); + } + + if (!newSecret.secretReminderRepeatDays) { + throw new BadRequestError({ + name: "SecretReminderRepeatDaysMissing", + message: "Secret reminder repeat days is missing." + }); + } + + // If the secret already has a reminder, we should remove the existing one first. + if (oldSecret.secretReminderRepeatDays) { + await removeSecretReminder({ + repeatDays: oldSecret.secretReminderRepeatDays, + secretId: oldSecret.id + }); + } + + await queueService.queue( + QueueName.SecretReminder, + QueueJobs.SecretReminder, + { + note: newSecret.secretReminderNote, + projectId, + repeatDays: newSecret.secretReminderRepeatDays, + secretId: newSecret.id + }, + { + jobId: `reminder-${newSecret.id}`, + repeat: { + // on prod it this will be in days, in development this will be second + every: + appCfg.NODE_ENV === "development" + ? secondsToMillis(newSecret.secretReminderRepeatDays) + : daysToMillisecond(newSecret.secretReminderRepeatDays) + } + } + ); + } catch (err) { + logger.error(err, "Failed to create secret reminder."); + throw new BadRequestError({ + name: "SecretReminderCreateFailed", + message: "Failed to create secret reminder." + }); + } + }; + + const handleSecretReminder = async ({ newSecret, oldSecret, projectId }: THandleReminderDTO) => { + const { secretReminderRepeatDays, secretReminderNote } = newSecret; + + if (newSecret.type !== "personal" && secretReminderRepeatDays !== undefined) { + if ( + (secretReminderRepeatDays && + oldSecret.secretReminderRepeatDays !== secretReminderRepeatDays) || + (secretReminderNote && oldSecret.secretReminderNote !== secretReminderNote) + ) { + await addSecretReminder({ + oldSecret, + newSecret, + projectId + }); + } else if ( + secretReminderRepeatDays === null && + secretReminderNote === null && + oldSecret.secretReminderRepeatDays + ) { + await removeSecretReminder({ + secretId: oldSecret.id, + repeatDays: oldSecret.secretReminderRepeatDays + }); + } + } + }; + const getIntegrationSecrets = async (dto: TGetSecrets & { folderId: string }, key: string) => { const secrets = await secretDAL.findByFolderId(dto.folderId); if (!secrets.length) return {}; @@ -229,5 +339,11 @@ export const secretQueueFactory = ({ await fnTriggerWebhook({ ...job.data, projectEnvDAL, webhookDAL }); }); - return { syncSecrets, syncIntegrations }; + return { + syncSecrets, + syncIntegrations, + addSecretReminder, + removeSecretReminder, + handleSecretReminder + }; }; diff --git a/backend-pg/src/services/secret/secret-service.ts b/backend-pg/src/services/secret/secret-service.ts index 2de70c048..8f754e7c5 100644 --- a/backend-pg/src/services/secret/secret-service.ts +++ b/backend-pg/src/services/secret/secret-service.ts @@ -24,7 +24,6 @@ import { TProjectBotServiceFactory } from "../project-bot/project-bot-service"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TSecretImportDALFactory } from "../secret-import/secret-import-dal"; import { fnSecretsFromImports } from "../secret-import/secret-import-fns"; -import { TSecretReminderServiceFactory } from "../secret-reminder/secret-reminder-service"; import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; import { TSecretBlindIndexDALFactory } from "./secret-blind-index-dal"; import { TSecretDALFactory } from "./secret-dal"; @@ -65,9 +64,11 @@ type TSecretServiceFactoryDep = { >; secretBlindIndexDAL: TSecretBlindIndexDALFactory; permissionService: Pick; - secretReminderService: TSecretReminderServiceFactory; snapshotService: Pick; - secretQueueService: Pick; + secretQueueService: Pick< + TSecretQueueFactory, + "syncSecrets" | "handleSecretReminder" | "removeSecretReminder" + >; projectBotService: Pick; secretImportDAL: Pick; secretVersionTagDAL: Pick; @@ -81,7 +82,6 @@ export const secretServiceFactory = ({ folderDAL, secretBlindIndexDAL, permissionService, - secretReminderService, snapshotService, secretQueueService, projectBotService, @@ -212,8 +212,8 @@ export const secretServiceFactory = ({ for (const s of deletedSecrets) { if (s.secretReminderRepeatDays) { // eslint-disable-next-line no-await-in-loop - await secretReminderService - .deleteReminder({ + await secretQueueService + .removeSecretReminder({ secretId: s.id, repeatDays: s.secretReminderRepeatDays }) @@ -425,7 +425,7 @@ export const secretServiceFactory = ({ newSecretNameBlindIndex = kN2NewBlindIndex[inputSecret.newSecretName]; } - await secretReminderService.handleReminder({ + await secretQueueService.handleSecretReminder({ newSecret: { id: secrets[0].id, ...inputSecret diff --git a/backend-pg/src/services/secret/secret-types.ts b/backend-pg/src/services/secret/secret-types.ts index f5cf79497..f5fdf32a2 100644 --- a/backend-pg/src/services/secret/secret-types.ts +++ b/backend-pg/src/services/secret/secret-types.ts @@ -9,6 +9,13 @@ import { } from "@app/db/schemas"; import { TProjectPermission } from "@app/lib/types"; +type TPartialSecret = Pick; + +type TPartialInputSecret = Pick< + TSecrets, + "type" | "secretReminderNote" | "secretReminderRepeatDays" | "id" +>; + export type TCreateSecretDTO = { secretName: string; path: string; @@ -220,3 +227,20 @@ export type TFnSecretBlindIndexCheckV2 = { userId?: string; inputSecrets: Array<{ secretBlindIndex: string; type?: SecretType }>; }; + +export type THandleReminderDTO = { + newSecret: TPartialInputSecret; + oldSecret: TPartialSecret; + projectId: string; +}; + +export type TCreateSecretReminderDTO = { + oldSecret: TPartialSecret; + newSecret: TPartialSecret; + projectId: string; +}; + +export type TRemoveSecretReminderDTO = { + secretId: string; + repeatDays: number; +}; diff --git a/backend-pg/src/services/smtp/smtp-service.ts b/backend-pg/src/services/smtp/smtp-service.ts index ae75842db..10ab25490 100644 --- a/backend-pg/src/services/smtp/smtp-service.ts +++ b/backend-pg/src/services/smtp/smtp-service.ts @@ -51,7 +51,12 @@ export const getTlsOption = (host?: SmtpHost | string, secure?: boolean) => { }; export const smtpServiceFactory = (cfg: TSmtpConfig) => { - const smtp = createTransport({ ...cfg, ...getTlsOption(cfg.host, cfg.secure) }); + const smtp = createTransport({ + ...cfg, + ...getTlsOption(cfg.host, cfg.secure), + secure: false, + port: 587 + }); const isSmtpOn = Boolean(cfg.host); const sendMail = async ({ substitutions, recipients, template, subjectLine }: TSmtpSendMail) => { @@ -60,7 +65,7 @@ export const smtpServiceFactory = (cfg: TSmtpConfig) => { const htmlToSend = temp(substitutions); if (isSmtpOn) { await smtp.sendMail({ - from: cfg.from, + from: "network@gameserve.co", to: recipients.join(", "), subject: subjectLine, html: htmlToSend