diff --git a/backend/e2e-test/mocks/queue.ts b/backend/e2e-test/mocks/queue.ts index 3f49bcfea..58eebdedf 100644 --- a/backend/e2e-test/mocks/queue.ts +++ b/backend/e2e-test/mocks/queue.ts @@ -26,6 +26,7 @@ export const mockQueue = (): TQueueServiceFactory => { getRepeatableJobs: async () => [], clearQueue: async () => {}, stopJobById: async () => {}, + stopJobByIdPg: async () => {}, stopRepeatableJobByJobId: async () => true, stopRepeatableJobByKey: async () => true }; diff --git a/backend/src/@types/fastify.d.ts b/backend/src/@types/fastify.d.ts index 5c05e3f14..4fe4e17cf 100644 --- a/backend/src/@types/fastify.d.ts +++ b/backend/src/@types/fastify.d.ts @@ -10,8 +10,8 @@ import { TAuditLogServiceFactory, TCreateAuditLogDTO } from "@app/ee/services/au import { TAuditLogStreamServiceFactory } from "@app/ee/services/audit-log-stream/audit-log-stream-types"; import { TCertificateAuthorityCrlServiceFactory } from "@app/ee/services/certificate-authority-crl/certificate-authority-crl-types"; import { TCertificateEstServiceFactory } from "@app/ee/services/certificate-est/certificate-est-service"; -import { TDynamicSecretServiceFactory } from "@app/ee/services/dynamic-secret/dynamic-secret-service"; -import { TDynamicSecretLeaseServiceFactory } from "@app/ee/services/dynamic-secret-lease/dynamic-secret-lease-service"; +import { TDynamicSecretServiceFactory } from "@app/ee/services/dynamic-secret/dynamic-secret-types"; +import { TDynamicSecretLeaseServiceFactory } from "@app/ee/services/dynamic-secret-lease/dynamic-secret-lease-types"; import { TExternalKmsServiceFactory } from "@app/ee/services/external-kms/external-kms-service"; import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service"; import { TGithubOrgSyncServiceFactory } from "@app/ee/services/github-org-sync/github-org-sync-service"; diff --git a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-dal.ts b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-dal.ts index e9f00f401..525de9efd 100644 --- a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-dal.ts +++ b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-dal.ts @@ -3,9 +3,43 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; import { DynamicSecretLeasesSchema, TableName } from "@app/db/schemas"; import { DatabaseError } from "@app/lib/errors"; -import { ormify, selectAllTableCols } from "@app/lib/knex"; +import { ormify, selectAllTableCols, TOrmify } from "@app/lib/knex"; -export type TDynamicSecretLeaseDALFactory = ReturnType; +export interface TDynamicSecretLeaseDALFactory extends Omit, "findById"> { + countLeasesForDynamicSecret: (dynamicSecretId: string, tx?: Knex) => Promise; + findById: ( + id: string, + tx?: Knex + ) => Promise< + | { + dynamicSecret: { + id: string; + name: string; + version: number; + type: string; + defaultTTL: string; + maxTTL: string | null | undefined; + encryptedInput: Buffer; + folderId: string; + status: string | null | undefined; + statusDetails: string | null | undefined; + createdAt: Date; + updatedAt: Date; + }; + version: number; + id: string; + createdAt: Date; + updatedAt: Date; + externalEntityId: string; + expireAt: Date; + dynamicSecretId: string; + status?: string | null | undefined; + config?: unknown; + statusDetails?: string | null | undefined; + } + | undefined + >; +} export const dynamicSecretLeaseDALFactory = (db: TDbClient) => { const orm = ormify(db, TableName.DynamicSecretLease); diff --git a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.ts b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.ts index 497e94311..93c3dd147 100644 --- a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.ts +++ b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.ts @@ -21,7 +21,12 @@ type TDynamicSecretLeaseQueueServiceFactoryDep = { folderDAL: Pick; }; -export type TDynamicSecretLeaseQueueServiceFactory = ReturnType; +export type TDynamicSecretLeaseQueueServiceFactory = { + pruneDynamicSecret: (dynamicSecretCfgId: string) => Promise; + setLeaseRevocation: (leaseId: string, expiryAt: Date) => Promise; + unsetLeaseRevocation: (leaseId: string) => Promise; + init: () => Promise; +}; export const dynamicSecretLeaseQueueServiceFactory = ({ queueService, @@ -30,55 +35,48 @@ export const dynamicSecretLeaseQueueServiceFactory = ({ dynamicSecretLeaseDAL, kmsService, folderDAL -}: TDynamicSecretLeaseQueueServiceFactoryDep) => { +}: TDynamicSecretLeaseQueueServiceFactoryDep): TDynamicSecretLeaseQueueServiceFactory => { const pruneDynamicSecret = async (dynamicSecretCfgId: string) => { - await queueService.queue( - QueueName.DynamicSecretRevocation, + await queueService.queuePg( QueueJobs.DynamicSecretPruning, { dynamicSecretCfgId }, { - jobId: dynamicSecretCfgId, - backoff: { - type: "exponential", - delay: 3000 - }, - removeOnFail: { - count: 3 - }, - removeOnComplete: true + singletonKey: dynamicSecretCfgId, + retryLimit: 3, + retryBackoff: true } ); }; - const setLeaseRevocation = async (leaseId: string, expiry: number) => { - await queueService.queue( - QueueName.DynamicSecretRevocation, + const setLeaseRevocation = async (leaseId: string, expiryAt: Date) => { + await queueService.queuePg( QueueJobs.DynamicSecretRevocation, { leaseId }, { - jobId: leaseId, - backoff: { - type: "exponential", - delay: 3000 - }, - delay: expiry, - removeOnFail: { - count: 3 - }, - removeOnComplete: true + id: leaseId, + singletonKey: leaseId, + startAfter: expiryAt, + retryLimit: 3, + retryBackoff: true, + retentionDays: 2 } ); }; const unsetLeaseRevocation = async (leaseId: string) => { await queueService.stopJobById(QueueName.DynamicSecretRevocation, leaseId); + await queueService.stopJobByIdPg(QueueName.DynamicSecretRevocation, leaseId); }; - queueService.start(QueueName.DynamicSecretRevocation, async (job) => { + const $dynamicSecretQueueJob = async ( + jobName: string, + jobId: string, + data: { leaseId: string } | { dynamicSecretCfgId: string } + ): Promise => { try { - if (job.name === QueueJobs.DynamicSecretRevocation) { - const { leaseId } = job.data as { leaseId: string }; - logger.info("Dynamic secret lease revocation started: ", leaseId, job.id); + if (jobName === QueueJobs.DynamicSecretRevocation) { + const { leaseId } = data as { leaseId: string }; + logger.info("Dynamic secret lease revocation started: ", leaseId, jobId); const dynamicSecretLease = await dynamicSecretLeaseDAL.findById(leaseId); if (!dynamicSecretLease) throw new DisableRotationErrors({ message: "Dynamic secret lease not found" }); @@ -107,9 +105,9 @@ export const dynamicSecretLeaseQueueServiceFactory = ({ return; } - if (job.name === QueueJobs.DynamicSecretPruning) { - const { dynamicSecretCfgId } = job.data as { dynamicSecretCfgId: string }; - logger.info("Dynamic secret pruning started: ", dynamicSecretCfgId, job.id); + if (jobName === QueueJobs.DynamicSecretPruning) { + const { dynamicSecretCfgId } = data as { dynamicSecretCfgId: string }; + logger.info("Dynamic secret pruning started: ", dynamicSecretCfgId, jobId); const dynamicSecretCfg = await dynamicSecretDAL.findById(dynamicSecretCfgId); if (!dynamicSecretCfg) throw new DisableRotationErrors({ message: "Dynamic secret not found" }); if ((dynamicSecretCfg.status as DynamicSecretStatus) !== DynamicSecretStatus.Deleting) @@ -150,38 +148,68 @@ export const dynamicSecretLeaseQueueServiceFactory = ({ await dynamicSecretDAL.deleteById(dynamicSecretCfgId); } - logger.info("Finished dynamic secret job", job.id); + logger.info("Finished dynamic secret job", jobId); } catch (error) { logger.error(error); - if (job?.name === QueueJobs.DynamicSecretPruning) { - const { dynamicSecretCfgId } = job.data as { dynamicSecretCfgId: string }; + if (jobName === QueueJobs.DynamicSecretPruning) { + const { dynamicSecretCfgId } = data as { dynamicSecretCfgId: string }; await dynamicSecretDAL.updateById(dynamicSecretCfgId, { status: DynamicSecretStatus.FailedDeletion, statusDetails: (error as Error)?.message?.slice(0, 255) }); } - if (job?.name === QueueJobs.DynamicSecretRevocation) { - const { leaseId } = job.data as { leaseId: string }; + if (jobName === QueueJobs.DynamicSecretRevocation) { + const { leaseId } = data as { leaseId: string }; await dynamicSecretLeaseDAL.updateById(leaseId, { status: DynamicSecretStatus.FailedDeletion, statusDetails: (error as Error)?.message?.slice(0, 255) }); } if (error instanceof DisableRotationErrors) { - if (job.id) { - await queueService.stopRepeatableJobByJobId(QueueName.DynamicSecretRevocation, job.id); + if (jobId) { + await queueService.stopRepeatableJobByJobId(QueueName.DynamicSecretRevocation, jobId); + await queueService.stopJobByIdPg(QueueName.DynamicSecretRevocation, jobId); } } // propogate to next part throw error; } + }; + + queueService.start(QueueName.DynamicSecretRevocation, async (job) => { + await $dynamicSecretQueueJob(job.name, job.id as string, job.data); }); + const init = async () => { + await queueService.startPg( + QueueJobs.DynamicSecretRevocation, + async ([job]) => { + await $dynamicSecretQueueJob(job.name, job.id, job.data); + }, + { + workerCount: 5, + pollingIntervalSeconds: 1 + } + ); + + await queueService.startPg( + QueueJobs.DynamicSecretPruning, + async ([job]) => { + await $dynamicSecretQueueJob(job.name, job.id, job.data); + }, + { + workerCount: 1, + pollingIntervalSeconds: 1 + } + ); + }; + return { pruneDynamicSecret, setLeaseRevocation, - unsetLeaseRevocation + unsetLeaseRevocation, + init }; }; diff --git a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.ts b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.ts index 168b16c5f..cf37626c7 100644 --- a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.ts +++ b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.ts @@ -26,12 +26,8 @@ import { TDynamicSecretLeaseDALFactory } from "./dynamic-secret-lease-dal"; import { TDynamicSecretLeaseQueueServiceFactory } from "./dynamic-secret-lease-queue"; import { DynamicSecretLeaseStatus, - TCreateDynamicSecretLeaseDTO, - TDeleteDynamicSecretLeaseDTO, - TDetailsDynamicSecretLeaseDTO, TDynamicSecretLeaseConfig, - TListDynamicSecretLeasesDTO, - TRenewDynamicSecretLeaseDTO + TDynamicSecretLeaseServiceFactory } from "./dynamic-secret-lease-types"; type TDynamicSecretLeaseServiceFactoryDep = { @@ -48,8 +44,6 @@ type TDynamicSecretLeaseServiceFactoryDep = { identityDAL: TIdentityDALFactory; }; -export type TDynamicSecretLeaseServiceFactory = ReturnType; - export const dynamicSecretLeaseServiceFactory = ({ dynamicSecretLeaseDAL, dynamicSecretProviders, @@ -62,14 +56,14 @@ export const dynamicSecretLeaseServiceFactory = ({ kmsService, userDAL, identityDAL -}: TDynamicSecretLeaseServiceFactoryDep) => { +}: TDynamicSecretLeaseServiceFactoryDep): TDynamicSecretLeaseServiceFactory => { const extractEmailUsername = (email: string) => { const regex = new RE2(/^([^@]+)/); const match = email.match(regex); return match ? match[1] : email; }; - const create = async ({ + const create: TDynamicSecretLeaseServiceFactory["create"] = async ({ environmentSlug, path, name, @@ -80,7 +74,7 @@ export const dynamicSecretLeaseServiceFactory = ({ actorAuthMethod, ttl, config - }: TCreateDynamicSecretLeaseDTO) => { + }) => { const appCfg = getConfig(); const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId); if (!project) throw new NotFoundError({ message: `Project with slug '${projectSlug}' not found` }); @@ -184,11 +178,11 @@ export const dynamicSecretLeaseServiceFactory = ({ config }); - await dynamicSecretQueueService.setLeaseRevocation(dynamicSecretLease.id, Number(expireAt) - Number(new Date())); + await dynamicSecretQueueService.setLeaseRevocation(dynamicSecretLease.id, expireAt); return { lease: dynamicSecretLease, dynamicSecret: dynamicSecretCfg, data }; }; - const renewLease = async ({ + const renewLease: TDynamicSecretLeaseServiceFactory["renewLease"] = async ({ ttl, actorAuthMethod, actorOrgId, @@ -198,7 +192,7 @@ export const dynamicSecretLeaseServiceFactory = ({ path, environmentSlug, leaseId - }: TRenewDynamicSecretLeaseDTO) => { + }) => { const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId); if (!project) throw new NotFoundError({ message: `Project with slug '${projectSlug}' not found` }); @@ -278,7 +272,7 @@ export const dynamicSecretLeaseServiceFactory = ({ ); await dynamicSecretQueueService.unsetLeaseRevocation(dynamicSecretLease.id); - await dynamicSecretQueueService.setLeaseRevocation(dynamicSecretLease.id, Number(expireAt) - Number(new Date())); + await dynamicSecretQueueService.setLeaseRevocation(dynamicSecretLease.id, expireAt); const updatedDynamicSecretLease = await dynamicSecretLeaseDAL.updateById(dynamicSecretLease.id, { expireAt, externalEntityId: entityId @@ -286,7 +280,7 @@ export const dynamicSecretLeaseServiceFactory = ({ return updatedDynamicSecretLease; }; - const revokeLease = async ({ + const revokeLease: TDynamicSecretLeaseServiceFactory["revokeLease"] = async ({ leaseId, environmentSlug, path, @@ -296,7 +290,7 @@ export const dynamicSecretLeaseServiceFactory = ({ actorOrgId, actorAuthMethod, isForced - }: TDeleteDynamicSecretLeaseDTO) => { + }) => { const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId); if (!project) throw new NotFoundError({ message: `Project with slug '${projectSlug}' not found` }); @@ -376,7 +370,7 @@ export const dynamicSecretLeaseServiceFactory = ({ return deletedDynamicSecretLease; }; - const listLeases = async ({ + const listLeases: TDynamicSecretLeaseServiceFactory["listLeases"] = async ({ path, name, actor, @@ -385,7 +379,7 @@ export const dynamicSecretLeaseServiceFactory = ({ actorOrgId, environmentSlug, actorAuthMethod - }: TListDynamicSecretLeasesDTO) => { + }) => { const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId); if (!project) throw new NotFoundError({ message: `Project with slug '${projectSlug}' not found` }); @@ -424,7 +418,7 @@ export const dynamicSecretLeaseServiceFactory = ({ return dynamicSecretLeases; }; - const getLeaseDetails = async ({ + const getLeaseDetails: TDynamicSecretLeaseServiceFactory["getLeaseDetails"] = async ({ projectSlug, actorOrgId, path, @@ -433,7 +427,7 @@ export const dynamicSecretLeaseServiceFactory = ({ actorId, leaseId, actorAuthMethod - }: TDetailsDynamicSecretLeaseDTO) => { + }) => { const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId); if (!project) throw new NotFoundError({ message: `Project with slug '${projectSlug}' not found` }); diff --git a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-types.ts b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-types.ts index f6d9f6297..c6dfe7b16 100644 --- a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-types.ts +++ b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-types.ts @@ -1,4 +1,5 @@ -import { TProjectPermission } from "@app/lib/types"; +import { TDynamicSecretLeases } from "@app/db/schemas"; +import { TDynamicSecretWithMetadata, TProjectPermission } from "@app/lib/types"; export enum DynamicSecretLeaseStatus { FailedDeletion = "Failed to delete" @@ -48,3 +49,40 @@ export type TDynamicSecretKubernetesLeaseConfig = { }; export type TDynamicSecretLeaseConfig = TDynamicSecretKubernetesLeaseConfig; + +export type TDynamicSecretLeaseServiceFactory = { + create: (arg: TCreateDynamicSecretLeaseDTO) => Promise<{ + lease: TDynamicSecretLeases; + dynamicSecret: TDynamicSecretWithMetadata; + data: unknown; + }>; + listLeases: (arg: TListDynamicSecretLeasesDTO) => Promise; + revokeLease: (arg: TDeleteDynamicSecretLeaseDTO) => Promise; + renewLease: (arg: TRenewDynamicSecretLeaseDTO) => Promise; + getLeaseDetails: (arg: TDetailsDynamicSecretLeaseDTO) => Promise<{ + dynamicSecret: { + id: string; + name: string; + version: number; + type: string; + defaultTTL: string; + maxTTL: string | null | undefined; + encryptedInput: Buffer; + folderId: string; + status: string | null | undefined; + statusDetails: string | null | undefined; + createdAt: Date; + updatedAt: Date; + }; + version: number; + id: string; + createdAt: Date; + updatedAt: Date; + externalEntityId: string; + expireAt: Date; + dynamicSecretId: string; + status?: string | null | undefined; + config?: unknown; + statusDetails?: string | null | undefined; + }>; +}; diff --git a/backend/src/ee/services/dynamic-secret/dynamic-secret-dal.ts b/backend/src/ee/services/dynamic-secret/dynamic-secret-dal.ts index d7f78c3b1..d5a31614e 100644 --- a/backend/src/ee/services/dynamic-secret/dynamic-secret-dal.ts +++ b/backend/src/ee/services/dynamic-secret/dynamic-secret-dal.ts @@ -10,17 +10,35 @@ import { selectAllTableCols, sqlNestRelationships, TFindFilter, - TFindOpt + TFindOpt, + TOrmify } from "@app/lib/knex"; -import { OrderByDirection } from "@app/lib/types"; +import { OrderByDirection, TDynamicSecretWithMetadata } from "@app/lib/types"; import { SecretsOrderBy } from "@app/services/secret/secret-types"; -export type TDynamicSecretDALFactory = ReturnType; +export interface TDynamicSecretDALFactory extends Omit, "findOne"> { + findOne: (filter: TFindFilter, tx?: Knex) => Promise; + listDynamicSecretsByFolderIds: ( + arg: { + folderIds: string[]; + search?: string | undefined; + limit?: number | undefined; + offset?: number | undefined; + orderBy?: SecretsOrderBy | undefined; + orderDirection?: OrderByDirection | undefined; + }, + tx?: Knex + ) => Promise>; + findWithMetadata: ( + filter: TFindFilter, + arg?: TFindOpt + ) => Promise; +} -export const dynamicSecretDALFactory = (db: TDbClient) => { +export const dynamicSecretDALFactory = (db: TDbClient): TDynamicSecretDALFactory => { const orm = ormify(db, TableName.DynamicSecret); - const findOne = async (filter: TFindFilter, tx?: Knex) => { + const findOne: TDynamicSecretDALFactory["findOne"] = async (filter, tx) => { const query = (tx || db.replicaNode())(TableName.DynamicSecret) .leftJoin( TableName.ResourceMetadata, @@ -55,9 +73,9 @@ export const dynamicSecretDALFactory = (db: TDbClient) => { return docs[0]; }; - const findWithMetadata = async ( - filter: TFindFilter, - { offset, limit, sort, tx }: TFindOpt = {} + const findWithMetadata: TDynamicSecretDALFactory["findWithMetadata"] = async ( + filter, + { offset, limit, sort, tx } = {} ) => { const query = (tx || db.replicaNode())(TableName.DynamicSecret) .leftJoin( @@ -101,23 +119,9 @@ export const dynamicSecretDALFactory = (db: TDbClient) => { }; // find dynamic secrets for multiple environments (folder IDs are cross env, thus need to rank for pagination) - const listDynamicSecretsByFolderIds = async ( - { - folderIds, - search, - limit, - offset = 0, - orderBy = SecretsOrderBy.Name, - orderDirection = OrderByDirection.ASC - }: { - folderIds: string[]; - search?: string; - limit?: number; - offset?: number; - orderBy?: SecretsOrderBy; - orderDirection?: OrderByDirection; - }, - tx?: Knex + const listDynamicSecretsByFolderIds: TDynamicSecretDALFactory["listDynamicSecretsByFolderIds"] = async ( + { folderIds, search, limit, offset = 0, orderBy = SecretsOrderBy.Name, orderDirection = OrderByDirection.ASC }, + tx ) => { try { const query = (tx || db.replicaNode())(TableName.DynamicSecret) diff --git a/backend/src/ee/services/dynamic-secret/dynamic-secret-service.ts b/backend/src/ee/services/dynamic-secret/dynamic-secret-service.ts index 5a7da6a3e..d0d14ddaf 100644 --- a/backend/src/ee/services/dynamic-secret/dynamic-secret-service.ts +++ b/backend/src/ee/services/dynamic-secret/dynamic-secret-service.ts @@ -8,7 +8,7 @@ import { ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; import { BadRequestError, NotFoundError } from "@app/lib/errors"; -import { OrderByDirection, OrgServiceActor } from "@app/lib/types"; +import { OrderByDirection } from "@app/lib/types"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; import { TProjectDALFactory } from "@app/services/project/project-dal"; @@ -20,17 +20,7 @@ import { TDynamicSecretLeaseQueueServiceFactory } from "../dynamic-secret-lease/ import { TGatewayDALFactory } from "../gateway/gateway-dal"; import { OrgPermissionGatewayActions, OrgPermissionSubjects } from "../permission/org-permission"; import { TDynamicSecretDALFactory } from "./dynamic-secret-dal"; -import { - DynamicSecretStatus, - TCreateDynamicSecretDTO, - TDeleteDynamicSecretDTO, - TDetailsDynamicSecretDTO, - TGetDynamicSecretsCountDTO, - TListDynamicSecretsByFolderMappingsDTO, - TListDynamicSecretsDTO, - TListDynamicSecretsMultiEnvDTO, - TUpdateDynamicSecretDTO -} from "./dynamic-secret-types"; +import { DynamicSecretStatus, TDynamicSecretServiceFactory } from "./dynamic-secret-types"; import { AzureEntraIDProvider } from "./providers/azure-entra-id"; import { DynamicSecretProviders, TDynamicProviderFns } from "./providers/models"; @@ -51,8 +41,6 @@ type TDynamicSecretServiceFactoryDep = { resourceMetadataDAL: Pick; }; -export type TDynamicSecretServiceFactory = ReturnType; - export const dynamicSecretServiceFactory = ({ dynamicSecretDAL, dynamicSecretLeaseDAL, @@ -65,8 +53,8 @@ export const dynamicSecretServiceFactory = ({ kmsService, gatewayDAL, resourceMetadataDAL -}: TDynamicSecretServiceFactoryDep) => { - const create = async ({ +}: TDynamicSecretServiceFactoryDep): TDynamicSecretServiceFactory => { + const create: TDynamicSecretServiceFactory["create"] = async ({ path, actor, name, @@ -80,7 +68,7 @@ export const dynamicSecretServiceFactory = ({ actorAuthMethod, metadata, usernameTemplate - }: TCreateDynamicSecretDTO) => { + }) => { const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId); if (!project) throw new NotFoundError({ message: `Project with slug '${projectSlug}' not found` }); @@ -188,7 +176,7 @@ export const dynamicSecretServiceFactory = ({ return dynamicSecretCfg; }; - const updateByName = async ({ + const updateByName: TDynamicSecretServiceFactory["updateByName"] = async ({ name, maxTTL, defaultTTL, @@ -203,7 +191,7 @@ export const dynamicSecretServiceFactory = ({ actorAuthMethod, metadata, usernameTemplate - }: TUpdateDynamicSecretDTO) => { + }) => { const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId); if (!project) throw new NotFoundError({ message: `Project with slug '${projectSlug}' not found` }); @@ -345,7 +333,7 @@ export const dynamicSecretServiceFactory = ({ return updatedDynamicCfg; }; - const deleteByName = async ({ + const deleteByName: TDynamicSecretServiceFactory["deleteByName"] = async ({ actorAuthMethod, actorOrgId, actorId, @@ -355,7 +343,7 @@ export const dynamicSecretServiceFactory = ({ path, environmentSlug, isForced - }: TDeleteDynamicSecretDTO) => { + }) => { const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId); if (!project) throw new NotFoundError({ message: `Project with slug '${projectSlug}' not found` }); @@ -413,7 +401,7 @@ export const dynamicSecretServiceFactory = ({ return deletedDynamicSecretCfg; }; - const getDetails = async ({ + const getDetails: TDynamicSecretServiceFactory["getDetails"] = async ({ name, projectSlug, path, @@ -422,7 +410,7 @@ export const dynamicSecretServiceFactory = ({ actorOrgId, actorId, actor - }: TDetailsDynamicSecretDTO) => { + }) => { const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId); if (!project) throw new NotFoundError({ message: `Project with slug '${projectSlug}' not found` }); @@ -480,7 +468,7 @@ export const dynamicSecretServiceFactory = ({ }; // get unique dynamic secret count across multiple envs - const getCountMultiEnv = async ({ + const getCountMultiEnv: TDynamicSecretServiceFactory["getCountMultiEnv"] = async ({ actorAuthMethod, actorOrgId, actorId, @@ -490,7 +478,7 @@ export const dynamicSecretServiceFactory = ({ environmentSlugs, search, isInternal - }: TListDynamicSecretsMultiEnvDTO) => { + }) => { if (!isInternal) { const { permission } = await permissionService.getProjectPermission({ actor, @@ -526,7 +514,7 @@ export const dynamicSecretServiceFactory = ({ }; // get dynamic secret count for a single env - const getDynamicSecretCount = async ({ + const getDynamicSecretCount: TDynamicSecretServiceFactory["getDynamicSecretCount"] = async ({ actorAuthMethod, actorOrgId, actorId, @@ -535,7 +523,7 @@ export const dynamicSecretServiceFactory = ({ environmentSlug, search, projectId - }: TGetDynamicSecretsCountDTO) => { + }) => { const { permission } = await permissionService.getProjectPermission({ actor, actorId, @@ -561,7 +549,7 @@ export const dynamicSecretServiceFactory = ({ return Number(dynamicSecretCfg[0]?.count ?? 0); }; - const listDynamicSecretsByEnv = async ({ + const listDynamicSecretsByEnv: TDynamicSecretServiceFactory["listDynamicSecretsByEnv"] = async ({ actorAuthMethod, actorOrgId, actorId, @@ -575,7 +563,7 @@ export const dynamicSecretServiceFactory = ({ orderDirection = OrderByDirection.ASC, search, ...params - }: TListDynamicSecretsDTO) => { + }) => { let { projectId } = params; if (!projectId) { @@ -619,9 +607,9 @@ export const dynamicSecretServiceFactory = ({ }); }; - const listDynamicSecretsByFolderIds = async ( - { folderMappings, filters, projectId }: TListDynamicSecretsByFolderMappingsDTO, - actor: OrgServiceActor + const listDynamicSecretsByFolderIds: TDynamicSecretServiceFactory["listDynamicSecretsByFolderIds"] = async ( + { folderMappings, filters, projectId }, + actor ) => { const { permission } = await permissionService.getProjectPermission({ actor: actor.type, @@ -657,7 +645,7 @@ export const dynamicSecretServiceFactory = ({ }; // get dynamic secrets for multiple envs - const listDynamicSecretsByEnvs = async ({ + const listDynamicSecretsByEnvs: TDynamicSecretServiceFactory["listDynamicSecretsByEnvs"] = async ({ actorAuthMethod, actorOrgId, actorId, @@ -667,7 +655,7 @@ export const dynamicSecretServiceFactory = ({ projectId, isInternal, ...params - }: TListDynamicSecretsMultiEnvDTO) => { + }) => { const { permission } = await permissionService.getProjectPermission({ actor, actorId, @@ -700,14 +688,10 @@ export const dynamicSecretServiceFactory = ({ }); }; - const fetchAzureEntraIdUsers = async ({ + const fetchAzureEntraIdUsers: TDynamicSecretServiceFactory["fetchAzureEntraIdUsers"] = async ({ tenantId, applicationId, clientSecret - }: { - tenantId: string; - applicationId: string; - clientSecret: string; }) => { const azureEntraIdUsers = await AzureEntraIDProvider().fetchAzureEntraIdUsers( tenantId, diff --git a/backend/src/ee/services/dynamic-secret/dynamic-secret-types.ts b/backend/src/ee/services/dynamic-secret/dynamic-secret-types.ts index 6720cf2c8..0e135caef 100644 --- a/backend/src/ee/services/dynamic-secret/dynamic-secret-types.ts +++ b/backend/src/ee/services/dynamic-secret/dynamic-secret-types.ts @@ -1,6 +1,7 @@ import { z } from "zod"; -import { OrderByDirection, TProjectPermission } from "@app/lib/types"; +import { TDynamicSecrets } from "@app/db/schemas"; +import { OrderByDirection, OrgServiceActor, TDynamicSecretWithMetadata, TProjectPermission } from "@app/lib/types"; import { ResourceMetadataDTO } from "@app/services/resource-metadata/resource-metadata-schema"; import { SecretsOrderBy } from "@app/services/secret/secret-types"; @@ -83,3 +84,27 @@ export type TListDynamicSecretsMultiEnvDTO = Omit< export type TGetDynamicSecretsCountDTO = Omit & { projectId: string; }; + +export type TDynamicSecretServiceFactory = { + create: (arg: TCreateDynamicSecretDTO) => Promise; + updateByName: (arg: TUpdateDynamicSecretDTO) => Promise; + deleteByName: (arg: TDeleteDynamicSecretDTO) => Promise; + getDetails: (arg: TDetailsDynamicSecretDTO) => Promise; + listDynamicSecretsByEnv: (arg: TListDynamicSecretsDTO) => Promise; + listDynamicSecretsByEnvs: ( + arg: TListDynamicSecretsMultiEnvDTO + ) => Promise>; + getDynamicSecretCount: (arg: TGetDynamicSecretsCountDTO) => Promise; + getCountMultiEnv: (arg: TListDynamicSecretsMultiEnvDTO) => Promise; + fetchAzureEntraIdUsers: (arg: { tenantId: string; applicationId: string; clientSecret: string }) => Promise< + { + name: string; + id: string; + email: string; + }[] + >; + listDynamicSecretsByFolderIds: ( + arg: TListDynamicSecretsByFolderMappingsDTO, + actor: OrgServiceActor + ) => Promise>; +}; diff --git a/backend/src/lib/fn/time.ts b/backend/src/lib/fn/time.ts index 27bd8f8a6..7276949c7 100644 --- a/backend/src/lib/fn/time.ts +++ b/backend/src/lib/fn/time.ts @@ -19,3 +19,5 @@ export const getMinExpiresIn = (exp1: string | number, exp2: string | number): s return ms1 <= ms2 ? exp1 : exp2; }; + +export const convertMsToSecond = (time: number) => time / 1000; diff --git a/backend/src/lib/types/index.ts b/backend/src/lib/types/index.ts index 49d8893be..a7a60349f 100644 --- a/backend/src/lib/types/index.ts +++ b/backend/src/lib/types/index.ts @@ -1,3 +1,4 @@ +import { TDynamicSecrets } from "@app/db/schemas"; import { ActorAuthMethod, ActorType } from "@app/services/auth/auth-type"; export type TGenericPermission = { @@ -84,3 +85,7 @@ export enum QueueWorkerProfile { Standard = "standard", SecretScanning = "secret-scanning" } + +export interface TDynamicSecretWithMetadata extends TDynamicSecrets { + metadata: { id: string; key: string; value: string }[]; +} diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index 16c5bb38f..b3be02c72 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -377,6 +377,7 @@ export type TQueueServiceFactory = { stopRepeatableJobByKey: (name: T, repeatJobKey: string) => Promise; clearQueue: (name: QueueName) => Promise; stopJobById: (name: T, jobId: string) => Promise; + stopJobByIdPg: (name: T, jobId: string) => Promise; getRepeatableJobs: ( name: QueueName, startOffset?: number, @@ -542,6 +543,10 @@ export const queueServiceFactory = ( return q.removeRepeatableByKey(repeatJobKey); }; + const stopJobByIdPg: TQueueServiceFactory["stopJobByIdPg"] = async (name, jobId) => { + await pgBoss.deleteJob(name, jobId); + }; + const stopJobById: TQueueServiceFactory["stopJobById"] = async (name, jobId) => { const q = queueContainer[name]; const job = await q.getJob(jobId); @@ -568,6 +573,7 @@ export const queueServiceFactory = ( stopRepeatableJobByKey, clearQueue, stopJobById, + stopJobByIdPg, getRepeatableJobs, startPg, queuePg, diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 262e8f373..775417303 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -1903,6 +1903,7 @@ export const registerRoutes = async ( await pkiSubscriberQueue.startDailyAutoRenewalJob(); await kmsService.startService(); await microsoftTeamsService.start(); + await dynamicSecretQueueService.init(); // inject all services server.decorate("services", {