awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-09-22 13:39:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor to secret queue

Browse Source
This commit is contained in:
Daniel Hougaard
2024-01-22 22:09:13 +04:00
committed by Akhil Mohan
parent bb9a6b4272
commit a60dbe523b
10 changed files with 155 additions and 261 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend-pg/src/@types/fastify.d.ts vendored
View File

@@ -36,7 +36,6 @@ import { TProjectRoleServiceFactory } from "@app/services/project-role/project-r
import { TSecretServiceFactory } from "@app/services/secret/secret-service";
import { TSecretFolderServiceFactory } from "@app/services/secret-folder/secret-folder-service";
import { TSecretImportServiceFactory } from "@app/services/secret-import/secret-import-service";
import { TSecretReminderServiceFactory } from "@app/services/secret-reminder/secret-reminder-service";
import { TSecretTagServiceFactory } from "@app/services/secret-tag/secret-tag-service";
import { TServiceTokenServiceFactory } from "@app/services/service-token/service-token-service";
import { TSuperAdminServiceFactory } from "@app/services/super-admin/super-admin-service";
@@ -100,7 +99,6 @@ declare module "fastify" {
secretApprovalPolicy: TSecretApprovalPolicyServiceFactory;
secretApprovalRequest: TSecretApprovalRequestServiceFactory;
secretRotation: TSecretRotationServiceFactory;
secretReminder: TSecretReminderServiceFactory;
snapshot: TSecretSnapshotServiceFactory;
saml: TSamlConfigServiceFactory;
auditLog: TAuditLogServiceFactory;

16
backend-pg/src/server/routes/index.ts
View File

@@ -85,8 +85,6 @@ import { secretFolderServiceFactory } from "@app/services/secret-folder/secret-f
import { secretFolderVersionDALFactory } from "@app/services/secret-folder/secret-folder-version-dal";
import { secretImportDALFactory } from "@app/services/secret-import/secret-import-dal";
import { secretImportServiceFactory } from "@app/services/secret-import/secret-import-service";
import { secretReminderServiceFactory } from "@app/services/secret-reminder";
import { secretReminderQueueFactory } from "@app/services/secret-reminder/secret-reminder-queue";
import { secretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
import { secretTagServiceFactory } from "@app/services/secret-tag/secret-tag-service";
import { serviceTokenDALFactory } from "@app/services/service-token/service-token-dal";
@@ -355,24 +353,11 @@ export const registerRoutes = async (
webhookDAL
});
const secretReminderQueue = secretReminderQueueFactory({
queue: queueService,
orgDal: orgDAL,
projectDal: projectDAL,
projectMembershipDal: projectMembershipDAL,
smtpService
});
const secretReminderService = secretReminderServiceFactory({
secretReminderQueue
});
const secretService = secretServiceFactory({
folderDAL,
secretVersionDAL,
secretVersionTagDAL,
secretBlindIndexDAL,
secretReminderService,
permissionService,
secretDAL,
secretTagDAL,
@@ -483,7 +468,6 @@ export const registerRoutes = async (
secretApprovalPolicy: sapService,
secretApprovalRequest: sarService,
secretRotation: secretRotationService,
secretReminder: secretReminderService,
snapshot: snapshotService,
saml: samlService,
auditLog: auditLogService,

1
backend-pg/src/services/secret-reminder/index.ts
View File

@@ -1 +0,0 @@
export { secretReminderServiceFactory } from "./secret-reminder-service";

105
backend-pg/src/services/secret-reminder/secret-reminder-queue/index.ts
View File

@@ -1,105 +0,0 @@
import { getConfig } from "@app/lib/config/env";
import { daysToMillisecond, secondsToMillis } from "@app/lib/dates";
import { logger } from "@app/lib/logger";
import { QueueJobs, QueueName, TQueueJobTypes, TQueueServiceFactory } from "@app/queue";
import { TOrgDalFactory } from "@app/services/org/org-dal";
import { TProjectDalFactory } from "@app/services/project/project-dal";
import { TProjectMembershipDalFactory } from "@app/services/project-membership/project-membership-dal";
import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service";
export type TSecretReminderQueueFactory = ReturnType<typeof secretReminderQueueFactory>;
type TSecretReminderQueueFactoryDep = {
queue: TQueueServiceFactory;
projectMembershipDal: Pick<TProjectMembershipDalFactory, "findAllProjectMembers">;
orgDal: Pick<TOrgDalFactory, "findOrgById" | "findOrgByProjectId">;
projectDal: Pick<TProjectDalFactory, "findById">;
smtpService: TSmtpService;
};
export const secretReminderQueueFactory = ({
queue,
projectMembershipDal,
smtpService,
orgDal,
projectDal
}: TSecretReminderQueueFactoryDep) => {
const addToQueue = async (data: TQueueJobTypes["secret-reminder"]["payload"]) => {
const appCfg = getConfig();
queue.queue(QueueName.SecretReminder, QueueJobs.SecretReminder, data, {
jobId: `reminder-${data.secretId}`,
repeat: {
// on prod it this will be in days, in development this will be second
every:
appCfg.NODE_ENV === "development"
? secondsToMillis(data.repeatDays)
: daysToMillisecond(data.repeatDays),
immediately: true
}
});
};
const removeFromQueue = async (secretId: string, repeatDays: number) => {
const appCfg = getConfig();
await queue.stopRepeatableJob(
QueueName.SecretReminder,
QueueJobs.SecretReminder,
{
// on prod it this will be in days, in development this will be second
every:
appCfg.NODE_ENV === "development"
? secondsToMillis(repeatDays)
: daysToMillisecond(repeatDays)
},
`reminder-${secretId}`
);
};
queue.start(QueueName.SecretReminder, async ({ data }) => {
logger.info(`secretReminderQueue.process: [secretDocument=${data.secretId}]`);
const { projectId } = data;
const organization = await orgDal.findOrgByProjectId(projectId);
const project = await projectDal.findById(projectId);
if (!organization) {
logger.info(
`secretReminderQueue.process: [secretDocument=${data.secretId}] no organization found`
);
return;
}
if (!project) {
logger.info(
`secretReminderQueue.process: [secretDocument=${data.secretId}] no project found`
);
return;
}
const projectMembers = await projectMembershipDal.findAllProjectMembers(projectId);
if (!projectMembers || !projectMembers.length) {
logger.info(
`secretReminderQueue.process: [secretDocument=${data.secretId}] no project members found`
);
return;
}
await smtpService.sendMail({
template: SmtpTemplates.SecretReminder,
subjectLine: "Infisical secret reminder",
recipients: [...projectMembers.map((m) => m.user.email)],
substitutions: {
reminderNote: data.note, // May not be present.
projectName: project.name,
organizationName: organization.name
}
});
});
return {
addToQueue,
removeFromQueue
};
};

102
backend-pg/src/services/secret-reminder/secret-reminder-service.ts
View File

@@ -1,102 +0,0 @@
import { BadRequestError } from "@app/lib/errors";
import { logger } from "@app/lib/logger";
import { TSecretReminderQueueFactory } from "./secret-reminder-queue";
import {
TCreateSecretReminderDTO,
TDeleteSecretReminderDTO,
THandleReminderDTO
} from "./secret-reminder-types";
type TSecretReminderServiceFactoryDep = {
secretReminderQueue: TSecretReminderQueueFactory;
};
export type TSecretReminderServiceFactory = ReturnType<typeof secretReminderServiceFactory>;
export const secretReminderServiceFactory = ({
secretReminderQueue
}: TSecretReminderServiceFactoryDep) => {
const createReminder = async ({ oldSecret, newSecret, projectId }: TCreateSecretReminderDTO) => {
try {
if (oldSecret.id !== newSecret.id) {
throw new BadRequestError({
name: "SecretReminderIdMismatch",
message: "Existing secret didn't match the updated secret ID."
});
}
if (!newSecret.secretReminderRepeatDays) {
throw new BadRequestError({
name: "SecretReminderRepeatDaysMissing",
message: "Secret reminder repeat days is missing."
});
}
// If the secret already has a reminder, we should remove the existing one first.
if (oldSecret.secretReminderRepeatDays) {
await secretReminderQueue.removeFromQueue(oldSecret.id, oldSecret.secretReminderRepeatDays);
}
await secretReminderQueue.addToQueue({
note: newSecret.secretReminderNote,
projectId,
repeatDays: newSecret.secretReminderRepeatDays,
secretId: newSecret.id
});
} catch (err) {
logger.error(err, "Failed to create secret reminder.");
throw new BadRequestError({
name: "SecretReminderCreateFailed",
message: "Failed to create secret reminder."
});
}
};
const deleteReminder = async ({ secretId, repeatDays }: TDeleteSecretReminderDTO) => {
try {
logger.info(`[${secretId}] Removing secret reminder from queue.`);
await secretReminderQueue.removeFromQueue(secretId, repeatDays);
} catch (err) {
logger.error(err, "Failed to remove secret reminder from queue.");
throw new BadRequestError({
name: "SecretReminderDeleteFailed",
message: "Failed to delete secret reminder."
});
}
};
// A handler function to cut down on code duplication in the future.
const handleReminder = async ({ newSecret, oldSecret, projectId }: THandleReminderDTO) => {
const { secretReminderRepeatDays, secretReminderNote } = newSecret;
if (newSecret.type !== "personal" && secretReminderRepeatDays !== undefined) {
if (
(secretReminderRepeatDays &&
oldSecret.secretReminderRepeatDays !== secretReminderRepeatDays) ||
(secretReminderNote && oldSecret.secretReminderNote !== secretReminderNote)
) {
await createReminder({
oldSecret,
newSecret,
projectId
});
} else if (
secretReminderRepeatDays === null &&
secretReminderNote === null &&
oldSecret.secretReminderRepeatDays
) {
await deleteReminder({
secretId: oldSecret.id,
repeatDays: oldSecret.secretReminderRepeatDays
});
}
}
};
return {
createReminder,
deleteReminder,
handleReminder
};
};

25
backend-pg/src/services/secret-reminder/secret-reminder-types.ts
View File

@@ -1,25 +0,0 @@
import { TSecrets } from "@app/db/schemas";
type TPartialSecret = Pick<TSecrets, "id" | "secretReminderRepeatDays" | "secretReminderNote">;
type TPartialInputSecret = Pick<
TSecrets,
"type" | "secretReminderNote" | "secretReminderRepeatDays" | "id"
>;
export type TCreateSecretReminderDTO = {
oldSecret: TPartialSecret;
newSecret: TPartialSecret;
projectId: string;
};
export type TDeleteSecretReminderDTO = {
secretId: string;
repeatDays: number;
};
export type THandleReminderDTO = {
newSecret: TPartialInputSecret;
oldSecret: TPartialSecret;
projectId: string;
};

118
backend-pg/src/services/secret/secret-queue.ts
View File

@@ -1,5 +1,8 @@
/* eslint-disable no-await-in-loop */
import { getConfig } from "@app/lib/config/env";
import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
import { daysToMillisecond, secondsToMillis } from "@app/lib/dates";
import { BadRequestError } from "@app/lib/errors";
import { isSamePath } from "@app/lib/fn";
import { logger } from "@app/lib/logger";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
@@ -16,6 +19,11 @@ import { TWebhookDALFactory } from "../webhook/webhook-dal";
import { fnTriggerWebhook } from "../webhook/webhook-fns";
import { TSecretDALFactory } from "./secret-dal";
import { interpolateSecrets } from "./secret-fns";
import {
TCreateSecretReminderDTO,
THandleReminderDTO,
TRemoveSecretReminderDTO
} from "./secret-types";
export type TSecretQueueFactory = ReturnType<typeof secretQueueFactory>;
@@ -78,6 +86,108 @@ export const secretQueueFactory = ({
await syncIntegrations(dto);
};
const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => {
const appCfg = getConfig();
await queueService.stopRepeatableJob(
QueueName.SecretReminder,
QueueJobs.SecretReminder,
{
// on prod it this will be in days, in development this will be second
every:
appCfg.NODE_ENV === "development"
? secondsToMillis(dto.repeatDays)
: daysToMillisecond(dto.repeatDays)
},
`reminder-${dto.secretId}`
);
};
const addSecretReminder = async ({
oldSecret,
newSecret,
projectId
}: TCreateSecretReminderDTO) => {
try {
const appCfg = getConfig();
if (oldSecret.id !== newSecret.id) {
throw new BadRequestError({
name: "SecretReminderIdMismatch",
message: "Existing secret didn't match the updated secret ID."
});
}
if (!newSecret.secretReminderRepeatDays) {
throw new BadRequestError({
name: "SecretReminderRepeatDaysMissing",
message: "Secret reminder repeat days is missing."
});
}
// If the secret already has a reminder, we should remove the existing one first.
if (oldSecret.secretReminderRepeatDays) {
await removeSecretReminder({
repeatDays: oldSecret.secretReminderRepeatDays,
secretId: oldSecret.id
});
}
await queueService.queue(
QueueName.SecretReminder,
QueueJobs.SecretReminder,
{
note: newSecret.secretReminderNote,
projectId,
repeatDays: newSecret.secretReminderRepeatDays,
secretId: newSecret.id
},
{
jobId: `reminder-${newSecret.id}`,
repeat: {
// on prod it this will be in days, in development this will be second
every:
appCfg.NODE_ENV === "development"
? secondsToMillis(newSecret.secretReminderRepeatDays)
: daysToMillisecond(newSecret.secretReminderRepeatDays)
}
}
);
} catch (err) {
logger.error(err, "Failed to create secret reminder.");
throw new BadRequestError({
name: "SecretReminderCreateFailed",
message: "Failed to create secret reminder."
});
}
};
const handleSecretReminder = async ({ newSecret, oldSecret, projectId }: THandleReminderDTO) => {
const { secretReminderRepeatDays, secretReminderNote } = newSecret;
if (newSecret.type !== "personal" && secretReminderRepeatDays !== undefined) {
if (
(secretReminderRepeatDays &&
oldSecret.secretReminderRepeatDays !== secretReminderRepeatDays) ||
(secretReminderNote && oldSecret.secretReminderNote !== secretReminderNote)
) {
await addSecretReminder({
oldSecret,
newSecret,
projectId
});
} else if (
secretReminderRepeatDays === null &&
secretReminderNote === null &&
oldSecret.secretReminderRepeatDays
) {
await removeSecretReminder({
secretId: oldSecret.id,
repeatDays: oldSecret.secretReminderRepeatDays
});
}
}
};
const getIntegrationSecrets = async (dto: TGetSecrets & { folderId: string }, key: string) => {
const secrets = await secretDAL.findByFolderId(dto.folderId);
if (!secrets.length) return {};
@@ -229,5 +339,11 @@ export const secretQueueFactory = ({
await fnTriggerWebhook({ ...job.data, projectEnvDAL, webhookDAL });
});
return { syncSecrets, syncIntegrations };
return {
syncSecrets,
syncIntegrations,
addSecretReminder,
removeSecretReminder,
handleSecretReminder
};
};

14
backend-pg/src/services/secret/secret-service.ts
View File

@@ -24,7 +24,6 @@ import { TProjectBotServiceFactory } from "../project-bot/project-bot-service";
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
import { fnSecretsFromImports } from "../secret-import/secret-import-fns";
import { TSecretReminderServiceFactory } from "../secret-reminder/secret-reminder-service";
import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal";
import { TSecretBlindIndexDALFactory } from "./secret-blind-index-dal";
import { TSecretDALFactory } from "./secret-dal";
@@ -65,9 +64,11 @@ type TSecretServiceFactoryDep = {
>;
secretBlindIndexDAL: TSecretBlindIndexDALFactory;
permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
secretReminderService: TSecretReminderServiceFactory;
snapshotService: Pick<TSecretSnapshotServiceFactory, "performSnapshot">;
secretQueueService: Pick<TSecretQueueFactory, "syncSecrets">;
secretQueueService: Pick<
TSecretQueueFactory,
"syncSecrets" | "handleSecretReminder" | "removeSecretReminder"
>;
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
secretImportDAL: Pick<TSecretImportDALFactory, "find">;
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
@@ -81,7 +82,6 @@ export const secretServiceFactory = ({
folderDAL,
secretBlindIndexDAL,
permissionService,
secretReminderService,
snapshotService,
secretQueueService,
projectBotService,
@@ -212,8 +212,8 @@ export const secretServiceFactory = ({
for (const s of deletedSecrets) {
if (s.secretReminderRepeatDays) {
// eslint-disable-next-line no-await-in-loop
await secretReminderService
.deleteReminder({
await secretQueueService
.removeSecretReminder({
secretId: s.id,
repeatDays: s.secretReminderRepeatDays
})
@@ -425,7 +425,7 @@ export const secretServiceFactory = ({
newSecretNameBlindIndex = kN2NewBlindIndex[inputSecret.newSecretName];
}
await secretReminderService.handleReminder({
await secretQueueService.handleSecretReminder({
newSecret: {
id: secrets[0].id,
...inputSecret

24
backend-pg/src/services/secret/secret-types.ts
View File

@@ -9,6 +9,13 @@ import {
} from "@app/db/schemas";
import { TProjectPermission } from "@app/lib/types";
type TPartialSecret = Pick<TSecrets, "id" | "secretReminderRepeatDays" | "secretReminderNote">;
type TPartialInputSecret = Pick<
TSecrets,
"type" | "secretReminderNote" | "secretReminderRepeatDays" | "id"
>;
export type TCreateSecretDTO = {
secretName: string;
path: string;
@@ -220,3 +227,20 @@ export type TFnSecretBlindIndexCheckV2 = {
userId?: string;
inputSecrets: Array<{ secretBlindIndex: string; type?: SecretType }>;
};
export type THandleReminderDTO = {
newSecret: TPartialInputSecret;
oldSecret: TPartialSecret;
projectId: string;
};
export type TCreateSecretReminderDTO = {
oldSecret: TPartialSecret;
newSecret: TPartialSecret;
projectId: string;
};
export type TRemoveSecretReminderDTO = {
secretId: string;
repeatDays: number;
};

9
backend-pg/src/services/smtp/smtp-service.ts
View File

@@ -51,7 +51,12 @@ export const getTlsOption = (host?: SmtpHost | string, secure?: boolean) => {
};
export const smtpServiceFactory = (cfg: TSmtpConfig) => {
const smtp = createTransport({ ...cfg, ...getTlsOption(cfg.host, cfg.secure) });
const smtp = createTransport({
...cfg,
...getTlsOption(cfg.host, cfg.secure),
secure: false,
port: 587
});
const isSmtpOn = Boolean(cfg.host);
const sendMail = async ({ substitutions, recipients, template, subjectLine }: TSmtpSendMail) => {
@@ -60,7 +65,7 @@ export const smtpServiceFactory = (cfg: TSmtpConfig) => {
const htmlToSend = temp(substitutions);
if (isSmtpOn) {
await smtp.sendMail({
from: cfg.from,
from: "network@gameserve.co",
to: recipients.join(", "),
subject: subjectLine,
html: htmlToSend