Merge pull request #3833 from akhilmhdh/feat/pg-queue

feat: migrated dynamic secret to pg queue and corrected service layer

backend/e2e-test/mocks/queue.ts

@@ -26,6 +26,7 @@ export const mockQueue = (): TQueueServiceFactory => {
     getRepeatableJobs: async () => [],
     clearQueue: async () => {},
     stopJobById: async () => {},
+    stopJobByIdPg: async () => {},
     stopRepeatableJobByJobId: async () => true,
     stopRepeatableJobByKey: async () => true
   };

backend/src/@types/fastify.d.ts

@@ -10,8 +10,8 @@ import { TAuditLogServiceFactory, TCreateAuditLogDTO } from "@app/ee/services/au
 import { TAuditLogStreamServiceFactory } from "@app/ee/services/audit-log-stream/audit-log-stream-types";
 import { TCertificateAuthorityCrlServiceFactory } from "@app/ee/services/certificate-authority-crl/certificate-authority-crl-types";
 import { TCertificateEstServiceFactory } from "@app/ee/services/certificate-est/certificate-est-service";
-import { TDynamicSecretServiceFactory } from "@app/ee/services/dynamic-secret/dynamic-secret-service";
+import { TDynamicSecretServiceFactory } from "@app/ee/services/dynamic-secret/dynamic-secret-types";
-import { TDynamicSecretLeaseServiceFactory } from "@app/ee/services/dynamic-secret-lease/dynamic-secret-lease-service";
+import { TDynamicSecretLeaseServiceFactory } from "@app/ee/services/dynamic-secret-lease/dynamic-secret-lease-types";
 import { TExternalKmsServiceFactory } from "@app/ee/services/external-kms/external-kms-service";
 import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
 import { TGithubOrgSyncServiceFactory } from "@app/ee/services/github-org-sync/github-org-sync-service";

backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-dal.ts

@@ -3,9 +3,43 @@ import { Knex } from "knex";
 import { TDbClient } from "@app/db";
 import { DynamicSecretLeasesSchema, TableName } from "@app/db/schemas";
 import { DatabaseError } from "@app/lib/errors";
-import { ormify, selectAllTableCols } from "@app/lib/knex";
+import { ormify, selectAllTableCols, TOrmify } from "@app/lib/knex";
 
-export type TDynamicSecretLeaseDALFactory = ReturnType<typeof dynamicSecretLeaseDALFactory>;
+export interface TDynamicSecretLeaseDALFactory extends Omit<TOrmify<TableName.DynamicSecretLease>, "findById"> {
+  countLeasesForDynamicSecret: (dynamicSecretId: string, tx?: Knex) => Promise<number>;
+  findById: (
+    id: string,
+    tx?: Knex
+  ) => Promise<
+    | {
+        dynamicSecret: {
+          id: string;
+          name: string;
+          version: number;
+          type: string;
+          defaultTTL: string;
+          maxTTL: string | null | undefined;
+          encryptedInput: Buffer;
+          folderId: string;
+          status: string | null | undefined;
+          statusDetails: string | null | undefined;
+          createdAt: Date;
+          updatedAt: Date;
+        };
+        version: number;
+        id: string;
+        createdAt: Date;
+        updatedAt: Date;
+        externalEntityId: string;
+        expireAt: Date;
+        dynamicSecretId: string;
+        status?: string | null | undefined;
+        config?: unknown;
+        statusDetails?: string | null | undefined;
+      }
+    | undefined
+  >;
+}
 
 export const dynamicSecretLeaseDALFactory = (db: TDbClient) => {
   const orm = ormify(db, TableName.DynamicSecretLease);

backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.ts

@@ -21,7 +21,12 @@ type TDynamicSecretLeaseQueueServiceFactoryDep = {
   folderDAL: Pick<TSecretFolderDALFactory, "findById">;
 };
 
-export type TDynamicSecretLeaseQueueServiceFactory = ReturnType<typeof dynamicSecretLeaseQueueServiceFactory>;
+export type TDynamicSecretLeaseQueueServiceFactory = {
+  pruneDynamicSecret: (dynamicSecretCfgId: string) => Promise<void>;
+  setLeaseRevocation: (leaseId: string, expiryAt: Date) => Promise<void>;
+  unsetLeaseRevocation: (leaseId: string) => Promise<void>;
+  init: () => Promise<void>;
+};
 
 export const dynamicSecretLeaseQueueServiceFactory = ({
   queueService,
@@ -30,55 +35,48 @@ export const dynamicSecretLeaseQueueServiceFactory = ({
   dynamicSecretLeaseDAL,
   kmsService,
   folderDAL
-}: TDynamicSecretLeaseQueueServiceFactoryDep) => {
+}: TDynamicSecretLeaseQueueServiceFactoryDep): TDynamicSecretLeaseQueueServiceFactory => {
   const pruneDynamicSecret = async (dynamicSecretCfgId: string) => {
-    await queueService.queue(
+    await queueService.queuePg<QueueName.DynamicSecretRevocation>(
-      QueueName.DynamicSecretRevocation,
       QueueJobs.DynamicSecretPruning,
       { dynamicSecretCfgId },
       {
-        jobId: dynamicSecretCfgId,
+        singletonKey: dynamicSecretCfgId,
-        backoff: {
+        retryLimit: 3,
-          type: "exponential",
+        retryBackoff: true
-          delay: 3000
-        },
-        removeOnFail: {
-          count: 3
-        },
-        removeOnComplete: true
       }
     );
   };
 
-  const setLeaseRevocation = async (leaseId: string, expiry: number) => {
+  const setLeaseRevocation = async (leaseId: string, expiryAt: Date) => {
-    await queueService.queue(
+    await queueService.queuePg<QueueName.DynamicSecretRevocation>(
-      QueueName.DynamicSecretRevocation,
       QueueJobs.DynamicSecretRevocation,
       { leaseId },
       {
-        jobId: leaseId,
+        id: leaseId,
-        backoff: {
+        singletonKey: leaseId,
-          type: "exponential",
+        startAfter: expiryAt,
-          delay: 3000
+        retryLimit: 3,
-        },
+        retryBackoff: true,
-        delay: expiry,
+        retentionDays: 2
-        removeOnFail: {
-          count: 3
-        },
-        removeOnComplete: true
       }
     );
   };
 
   const unsetLeaseRevocation = async (leaseId: string) => {
     await queueService.stopJobById(QueueName.DynamicSecretRevocation, leaseId);
+    await queueService.stopJobByIdPg(QueueName.DynamicSecretRevocation, leaseId);
   };
 
-  queueService.start(QueueName.DynamicSecretRevocation, async (job) => {
+  const $dynamicSecretQueueJob = async (
+    jobName: string,
+    jobId: string,
+    data: { leaseId: string } | { dynamicSecretCfgId: string }
+  ): Promise<void> => {
     try {
-      if (job.name === QueueJobs.DynamicSecretRevocation) {
+      if (jobName === QueueJobs.DynamicSecretRevocation) {
-        const { leaseId } = job.data as { leaseId: string };
+        const { leaseId } = data as { leaseId: string };
-        logger.info("Dynamic secret lease revocation started: ", leaseId, job.id);
+        logger.info("Dynamic secret lease revocation started: ", leaseId, jobId);
 
         const dynamicSecretLease = await dynamicSecretLeaseDAL.findById(leaseId);
         if (!dynamicSecretLease) throw new DisableRotationErrors({ message: "Dynamic secret lease not found" });
@@ -107,9 +105,9 @@ export const dynamicSecretLeaseQueueServiceFactory = ({
         return;
       }
 
-      if (job.name === QueueJobs.DynamicSecretPruning) {
+      if (jobName === QueueJobs.DynamicSecretPruning) {
-        const { dynamicSecretCfgId } = job.data as { dynamicSecretCfgId: string };
+        const { dynamicSecretCfgId } = data as { dynamicSecretCfgId: string };
-        logger.info("Dynamic secret pruning started: ", dynamicSecretCfgId, job.id);
+        logger.info("Dynamic secret pruning started: ", dynamicSecretCfgId, jobId);
         const dynamicSecretCfg = await dynamicSecretDAL.findById(dynamicSecretCfgId);
         if (!dynamicSecretCfg) throw new DisableRotationErrors({ message: "Dynamic secret not found" });
         if ((dynamicSecretCfg.status as DynamicSecretStatus) !== DynamicSecretStatus.Deleting)
@@ -150,38 +148,68 @@ export const dynamicSecretLeaseQueueServiceFactory = ({
 
         await dynamicSecretDAL.deleteById(dynamicSecretCfgId);
       }
-      logger.info("Finished dynamic secret job", job.id);
+      logger.info("Finished dynamic secret job", jobId);
     } catch (error) {
       logger.error(error);
 
-      if (job?.name === QueueJobs.DynamicSecretPruning) {
+      if (jobName === QueueJobs.DynamicSecretPruning) {
-        const { dynamicSecretCfgId } = job.data as { dynamicSecretCfgId: string };
+        const { dynamicSecretCfgId } = data as { dynamicSecretCfgId: string };
         await dynamicSecretDAL.updateById(dynamicSecretCfgId, {
           status: DynamicSecretStatus.FailedDeletion,
           statusDetails: (error as Error)?.message?.slice(0, 255)
         });
       }
 
-      if (job?.name === QueueJobs.DynamicSecretRevocation) {
+      if (jobName === QueueJobs.DynamicSecretRevocation) {
-        const { leaseId } = job.data as { leaseId: string };
+        const { leaseId } = data as { leaseId: string };
         await dynamicSecretLeaseDAL.updateById(leaseId, {
           status: DynamicSecretStatus.FailedDeletion,
           statusDetails: (error as Error)?.message?.slice(0, 255)
         });
       }
       if (error instanceof DisableRotationErrors) {
-        if (job.id) {
+        if (jobId) {
-          await queueService.stopRepeatableJobByJobId(QueueName.DynamicSecretRevocation, job.id);
+          await queueService.stopRepeatableJobByJobId(QueueName.DynamicSecretRevocation, jobId);
+          await queueService.stopJobByIdPg(QueueName.DynamicSecretRevocation, jobId);
         }
       }
       // propogate to next part
       throw error;
     }
+  };
+
+  queueService.start(QueueName.DynamicSecretRevocation, async (job) => {
+    await $dynamicSecretQueueJob(job.name, job.id as string, job.data);
   });
 
+  const init = async () => {
+    await queueService.startPg<QueueName.DynamicSecretRevocation>(
+      QueueJobs.DynamicSecretRevocation,
+      async ([job]) => {
+        await $dynamicSecretQueueJob(job.name, job.id, job.data);
+      },
+      {
+        workerCount: 5,
+        pollingIntervalSeconds: 1
+      }
+    );
+
+    await queueService.startPg<QueueName.DynamicSecretRevocation>(
+      QueueJobs.DynamicSecretPruning,
+      async ([job]) => {
+        await $dynamicSecretQueueJob(job.name, job.id, job.data);
+      },
+      {
+        workerCount: 1,
+        pollingIntervalSeconds: 1
+      }
+    );
+  };
+
   return {
     pruneDynamicSecret,
     setLeaseRevocation,
-    unsetLeaseRevocation
+    unsetLeaseRevocation,
+    init
   };
 };

backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.ts

@@ -26,12 +26,8 @@ import { TDynamicSecretLeaseDALFactory } from "./dynamic-secret-lease-dal";
 import { TDynamicSecretLeaseQueueServiceFactory } from "./dynamic-secret-lease-queue";
 import {
   DynamicSecretLeaseStatus,
-  TCreateDynamicSecretLeaseDTO,
-  TDeleteDynamicSecretLeaseDTO,
-  TDetailsDynamicSecretLeaseDTO,
   TDynamicSecretLeaseConfig,
-  TListDynamicSecretLeasesDTO,
+  TDynamicSecretLeaseServiceFactory
-  TRenewDynamicSecretLeaseDTO
 } from "./dynamic-secret-lease-types";
 
 type TDynamicSecretLeaseServiceFactoryDep = {
@@ -48,8 +44,6 @@ type TDynamicSecretLeaseServiceFactoryDep = {
   identityDAL: TIdentityDALFactory;
 };
 
-export type TDynamicSecretLeaseServiceFactory = ReturnType<typeof dynamicSecretLeaseServiceFactory>;
-
 export const dynamicSecretLeaseServiceFactory = ({
   dynamicSecretLeaseDAL,
   dynamicSecretProviders,
@@ -62,14 +56,14 @@ export const dynamicSecretLeaseServiceFactory = ({
   kmsService,
   userDAL,
   identityDAL
-}: TDynamicSecretLeaseServiceFactoryDep) => {
+}: TDynamicSecretLeaseServiceFactoryDep): TDynamicSecretLeaseServiceFactory => {
   const extractEmailUsername = (email: string) => {
     const regex = new RE2(/^([^@]+)/);
     const match = email.match(regex);
     return match ? match[1] : email;
   };
 
-  const create = async ({
+  const create: TDynamicSecretLeaseServiceFactory["create"] = async ({
     environmentSlug,
     path,
     name,
@@ -80,7 +74,7 @@ export const dynamicSecretLeaseServiceFactory = ({
     actorAuthMethod,
     ttl,
     config
-  }: TCreateDynamicSecretLeaseDTO) => {
+  }) => {
     const appCfg = getConfig();
     const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
     if (!project) throw new NotFoundError({ message: `Project with slug '${projectSlug}' not found` });
@@ -184,11 +178,11 @@ export const dynamicSecretLeaseServiceFactory = ({
       config
     });
 
-    await dynamicSecretQueueService.setLeaseRevocation(dynamicSecretLease.id, Number(expireAt) - Number(new Date()));
+    await dynamicSecretQueueService.setLeaseRevocation(dynamicSecretLease.id, expireAt);
     return { lease: dynamicSecretLease, dynamicSecret: dynamicSecretCfg, data };
   };
 
-  const renewLease = async ({
+  const renewLease: TDynamicSecretLeaseServiceFactory["renewLease"] = async ({
     ttl,
     actorAuthMethod,
     actorOrgId,
@@ -198,7 +192,7 @@ export const dynamicSecretLeaseServiceFactory = ({
     path,
     environmentSlug,
     leaseId
-  }: TRenewDynamicSecretLeaseDTO) => {
+  }) => {
     const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
     if (!project) throw new NotFoundError({ message: `Project with slug '${projectSlug}' not found` });
 
@@ -278,7 +272,7 @@ export const dynamicSecretLeaseServiceFactory = ({
     );
 
     await dynamicSecretQueueService.unsetLeaseRevocation(dynamicSecretLease.id);
-    await dynamicSecretQueueService.setLeaseRevocation(dynamicSecretLease.id, Number(expireAt) - Number(new Date()));
+    await dynamicSecretQueueService.setLeaseRevocation(dynamicSecretLease.id, expireAt);
     const updatedDynamicSecretLease = await dynamicSecretLeaseDAL.updateById(dynamicSecretLease.id, {
       expireAt,
       externalEntityId: entityId
@@ -286,7 +280,7 @@ export const dynamicSecretLeaseServiceFactory = ({
     return updatedDynamicSecretLease;
   };
 
-  const revokeLease = async ({
+  const revokeLease: TDynamicSecretLeaseServiceFactory["revokeLease"] = async ({
     leaseId,
     environmentSlug,
     path,
@@ -296,7 +290,7 @@ export const dynamicSecretLeaseServiceFactory = ({
     actorOrgId,
     actorAuthMethod,
     isForced
-  }: TDeleteDynamicSecretLeaseDTO) => {
+  }) => {
     const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
     if (!project) throw new NotFoundError({ message: `Project with slug '${projectSlug}' not found` });
 
@@ -376,7 +370,7 @@ export const dynamicSecretLeaseServiceFactory = ({
     return deletedDynamicSecretLease;
   };
 
-  const listLeases = async ({
+  const listLeases: TDynamicSecretLeaseServiceFactory["listLeases"] = async ({
     path,
     name,
     actor,
@@ -385,7 +379,7 @@ export const dynamicSecretLeaseServiceFactory = ({
     actorOrgId,
     environmentSlug,
     actorAuthMethod
-  }: TListDynamicSecretLeasesDTO) => {
+  }) => {
     const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
     if (!project) throw new NotFoundError({ message: `Project with slug '${projectSlug}' not found` });
 
@@ -424,7 +418,7 @@ export const dynamicSecretLeaseServiceFactory = ({
     return dynamicSecretLeases;
   };
 
-  const getLeaseDetails = async ({
+  const getLeaseDetails: TDynamicSecretLeaseServiceFactory["getLeaseDetails"] = async ({
     projectSlug,
     actorOrgId,
     path,
@@ -433,7 +427,7 @@ export const dynamicSecretLeaseServiceFactory = ({
     actorId,
     leaseId,
     actorAuthMethod
-  }: TDetailsDynamicSecretLeaseDTO) => {
+  }) => {
     const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
     if (!project) throw new NotFoundError({ message: `Project with slug '${projectSlug}' not found` });
 

backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-types.ts

@@ -1,4 +1,5 @@
-import { TProjectPermission } from "@app/lib/types";
+import { TDynamicSecretLeases } from "@app/db/schemas";
+import { TDynamicSecretWithMetadata, TProjectPermission } from "@app/lib/types";
 
 export enum DynamicSecretLeaseStatus {
   FailedDeletion = "Failed to delete"
@@ -48,3 +49,40 @@ export type TDynamicSecretKubernetesLeaseConfig = {
 };
 
 export type TDynamicSecretLeaseConfig = TDynamicSecretKubernetesLeaseConfig;
+
+export type TDynamicSecretLeaseServiceFactory = {
+  create: (arg: TCreateDynamicSecretLeaseDTO) => Promise<{
+    lease: TDynamicSecretLeases;
+    dynamicSecret: TDynamicSecretWithMetadata;
+    data: unknown;
+  }>;
+  listLeases: (arg: TListDynamicSecretLeasesDTO) => Promise<TDynamicSecretLeases[]>;
+  revokeLease: (arg: TDeleteDynamicSecretLeaseDTO) => Promise<TDynamicSecretLeases>;
+  renewLease: (arg: TRenewDynamicSecretLeaseDTO) => Promise<TDynamicSecretLeases>;
+  getLeaseDetails: (arg: TDetailsDynamicSecretLeaseDTO) => Promise<{
+    dynamicSecret: {
+      id: string;
+      name: string;
+      version: number;
+      type: string;
+      defaultTTL: string;
+      maxTTL: string | null | undefined;
+      encryptedInput: Buffer;
+      folderId: string;
+      status: string | null | undefined;
+      statusDetails: string | null | undefined;
+      createdAt: Date;
+      updatedAt: Date;
+    };
+    version: number;
+    id: string;
+    createdAt: Date;
+    updatedAt: Date;
+    externalEntityId: string;
+    expireAt: Date;
+    dynamicSecretId: string;
+    status?: string | null | undefined;
+    config?: unknown;
+    statusDetails?: string | null | undefined;
+  }>;
+};

backend/src/ee/services/dynamic-secret/dynamic-secret-dal.ts

@@ -10,17 +10,35 @@ import {
   selectAllTableCols,
   sqlNestRelationships,
   TFindFilter,
-  TFindOpt
+  TFindOpt,
+  TOrmify
 } from "@app/lib/knex";
-import { OrderByDirection } from "@app/lib/types";
+import { OrderByDirection, TDynamicSecretWithMetadata } from "@app/lib/types";
 import { SecretsOrderBy } from "@app/services/secret/secret-types";
 
-export type TDynamicSecretDALFactory = ReturnType<typeof dynamicSecretDALFactory>;
+export interface TDynamicSecretDALFactory extends Omit<TOrmify<TableName.DynamicSecret>, "findOne"> {
+  findOne: (filter: TFindFilter<TDynamicSecrets>, tx?: Knex) => Promise<TDynamicSecretWithMetadata>;
+  listDynamicSecretsByFolderIds: (
+    arg: {
+      folderIds: string[];
+      search?: string | undefined;
+      limit?: number | undefined;
+      offset?: number | undefined;
+      orderBy?: SecretsOrderBy | undefined;
+      orderDirection?: OrderByDirection | undefined;
+    },
+    tx?: Knex
+  ) => Promise<Array<TDynamicSecretWithMetadata & { environment: string }>>;
+  findWithMetadata: (
+    filter: TFindFilter<TDynamicSecrets>,
+    arg?: TFindOpt<TDynamicSecrets>
+  ) => Promise<TDynamicSecretWithMetadata[]>;
+}
 
-export const dynamicSecretDALFactory = (db: TDbClient) => {
+export const dynamicSecretDALFactory = (db: TDbClient): TDynamicSecretDALFactory => {
   const orm = ormify(db, TableName.DynamicSecret);
 
-  const findOne = async (filter: TFindFilter<TDynamicSecrets>, tx?: Knex) => {
+  const findOne: TDynamicSecretDALFactory["findOne"] = async (filter, tx) => {
     const query = (tx || db.replicaNode())(TableName.DynamicSecret)
       .leftJoin(
         TableName.ResourceMetadata,
@@ -55,9 +73,9 @@ export const dynamicSecretDALFactory = (db: TDbClient) => {
     return docs[0];
   };
 
-  const findWithMetadata = async (
+  const findWithMetadata: TDynamicSecretDALFactory["findWithMetadata"] = async (
-    filter: TFindFilter<TDynamicSecrets>,
+    filter,
-    { offset, limit, sort, tx }: TFindOpt<TDynamicSecrets> = {}
+    { offset, limit, sort, tx } = {}
   ) => {
     const query = (tx || db.replicaNode())(TableName.DynamicSecret)
       .leftJoin(
@@ -101,23 +119,9 @@ export const dynamicSecretDALFactory = (db: TDbClient) => {
   };
 
   // find dynamic secrets for multiple environments (folder IDs are cross env, thus need to rank for pagination)
-  const listDynamicSecretsByFolderIds = async (
+  const listDynamicSecretsByFolderIds: TDynamicSecretDALFactory["listDynamicSecretsByFolderIds"] = async (
-    {
+    { folderIds, search, limit, offset = 0, orderBy = SecretsOrderBy.Name, orderDirection = OrderByDirection.ASC },
-      folderIds,
+    tx
-      search,
-      limit,
-      offset = 0,
-      orderBy = SecretsOrderBy.Name,
-      orderDirection = OrderByDirection.ASC
-    }: {
-      folderIds: string[];
-      search?: string;
-      limit?: number;
-      offset?: number;
-      orderBy?: SecretsOrderBy;
-      orderDirection?: OrderByDirection;
-    },
-    tx?: Knex
   ) => {
     try {
       const query = (tx || db.replicaNode())(TableName.DynamicSecret)

backend/src/ee/services/dynamic-secret/dynamic-secret-service.ts

@@ -8,7 +8,7 @@ import {
   ProjectPermissionSub
 } from "@app/ee/services/permission/project-permission";
 import { BadRequestError, NotFoundError } from "@app/lib/errors";
-import { OrderByDirection, OrgServiceActor } from "@app/lib/types";
+import { OrderByDirection } from "@app/lib/types";
 import { TKmsServiceFactory } from "@app/services/kms/kms-service";
 import { KmsDataKey } from "@app/services/kms/kms-types";
 import { TProjectDALFactory } from "@app/services/project/project-dal";
@@ -20,17 +20,7 @@ import { TDynamicSecretLeaseQueueServiceFactory } from "../dynamic-secret-lease/
 import { TGatewayDALFactory } from "../gateway/gateway-dal";
 import { OrgPermissionGatewayActions, OrgPermissionSubjects } from "../permission/org-permission";
 import { TDynamicSecretDALFactory } from "./dynamic-secret-dal";
-import {
+import { DynamicSecretStatus, TDynamicSecretServiceFactory } from "./dynamic-secret-types";
-  DynamicSecretStatus,
-  TCreateDynamicSecretDTO,
-  TDeleteDynamicSecretDTO,
-  TDetailsDynamicSecretDTO,
-  TGetDynamicSecretsCountDTO,
-  TListDynamicSecretsByFolderMappingsDTO,
-  TListDynamicSecretsDTO,
-  TListDynamicSecretsMultiEnvDTO,
-  TUpdateDynamicSecretDTO
-} from "./dynamic-secret-types";
 import { AzureEntraIDProvider } from "./providers/azure-entra-id";
 import { DynamicSecretProviders, TDynamicProviderFns } from "./providers/models";
 
@@ -51,8 +41,6 @@ type TDynamicSecretServiceFactoryDep = {
   resourceMetadataDAL: Pick<TResourceMetadataDALFactory, "insertMany" | "delete">;
 };
 
-export type TDynamicSecretServiceFactory = ReturnType<typeof dynamicSecretServiceFactory>;
-
 export const dynamicSecretServiceFactory = ({
   dynamicSecretDAL,
   dynamicSecretLeaseDAL,
@@ -65,8 +53,8 @@ export const dynamicSecretServiceFactory = ({
   kmsService,
   gatewayDAL,
   resourceMetadataDAL
-}: TDynamicSecretServiceFactoryDep) => {
+}: TDynamicSecretServiceFactoryDep): TDynamicSecretServiceFactory => {
-  const create = async ({
+  const create: TDynamicSecretServiceFactory["create"] = async ({
     path,
     actor,
     name,
@@ -80,7 +68,7 @@ export const dynamicSecretServiceFactory = ({
     actorAuthMethod,
     metadata,
     usernameTemplate
-  }: TCreateDynamicSecretDTO) => {
+  }) => {
     const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
     if (!project) throw new NotFoundError({ message: `Project with slug '${projectSlug}' not found` });
 
@@ -188,7 +176,7 @@ export const dynamicSecretServiceFactory = ({
     return dynamicSecretCfg;
   };
 
-  const updateByName = async ({
+  const updateByName: TDynamicSecretServiceFactory["updateByName"] = async ({
     name,
     maxTTL,
     defaultTTL,
@@ -203,7 +191,7 @@ export const dynamicSecretServiceFactory = ({
     actorAuthMethod,
     metadata,
     usernameTemplate
-  }: TUpdateDynamicSecretDTO) => {
+  }) => {
     const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
     if (!project) throw new NotFoundError({ message: `Project with slug '${projectSlug}' not found` });
 
@@ -345,7 +333,7 @@ export const dynamicSecretServiceFactory = ({
     return updatedDynamicCfg;
   };
 
-  const deleteByName = async ({
+  const deleteByName: TDynamicSecretServiceFactory["deleteByName"] = async ({
     actorAuthMethod,
     actorOrgId,
     actorId,
@@ -355,7 +343,7 @@ export const dynamicSecretServiceFactory = ({
     path,
     environmentSlug,
     isForced
-  }: TDeleteDynamicSecretDTO) => {
+  }) => {
     const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
     if (!project) throw new NotFoundError({ message: `Project with slug '${projectSlug}' not found` });
 
@@ -413,7 +401,7 @@ export const dynamicSecretServiceFactory = ({
     return deletedDynamicSecretCfg;
   };
 
-  const getDetails = async ({
+  const getDetails: TDynamicSecretServiceFactory["getDetails"] = async ({
     name,
     projectSlug,
     path,
@@ -422,7 +410,7 @@ export const dynamicSecretServiceFactory = ({
     actorOrgId,
     actorId,
     actor
-  }: TDetailsDynamicSecretDTO) => {
+  }) => {
     const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
     if (!project) throw new NotFoundError({ message: `Project with slug '${projectSlug}' not found` });
 
@@ -480,7 +468,7 @@ export const dynamicSecretServiceFactory = ({
   };
 
   // get unique dynamic secret count across multiple envs
-  const getCountMultiEnv = async ({
+  const getCountMultiEnv: TDynamicSecretServiceFactory["getCountMultiEnv"] = async ({
     actorAuthMethod,
     actorOrgId,
     actorId,
@@ -490,7 +478,7 @@ export const dynamicSecretServiceFactory = ({
     environmentSlugs,
     search,
     isInternal
-  }: TListDynamicSecretsMultiEnvDTO) => {
+  }) => {
     if (!isInternal) {
       const { permission } = await permissionService.getProjectPermission({
         actor,
@@ -526,7 +514,7 @@ export const dynamicSecretServiceFactory = ({
   };
 
   // get dynamic secret count for a single env
-  const getDynamicSecretCount = async ({
+  const getDynamicSecretCount: TDynamicSecretServiceFactory["getDynamicSecretCount"] = async ({
     actorAuthMethod,
     actorOrgId,
     actorId,
@@ -535,7 +523,7 @@ export const dynamicSecretServiceFactory = ({
     environmentSlug,
     search,
     projectId
-  }: TGetDynamicSecretsCountDTO) => {
+  }) => {
     const { permission } = await permissionService.getProjectPermission({
       actor,
       actorId,
@@ -561,7 +549,7 @@ export const dynamicSecretServiceFactory = ({
     return Number(dynamicSecretCfg[0]?.count ?? 0);
   };
 
-  const listDynamicSecretsByEnv = async ({
+  const listDynamicSecretsByEnv: TDynamicSecretServiceFactory["listDynamicSecretsByEnv"] = async ({
     actorAuthMethod,
     actorOrgId,
     actorId,
@@ -575,7 +563,7 @@ export const dynamicSecretServiceFactory = ({
     orderDirection = OrderByDirection.ASC,
     search,
     ...params
-  }: TListDynamicSecretsDTO) => {
+  }) => {
     let { projectId } = params;
 
     if (!projectId) {
@@ -619,9 +607,9 @@ export const dynamicSecretServiceFactory = ({
     });
   };
 
-  const listDynamicSecretsByFolderIds = async (
+  const listDynamicSecretsByFolderIds: TDynamicSecretServiceFactory["listDynamicSecretsByFolderIds"] = async (
-    { folderMappings, filters, projectId }: TListDynamicSecretsByFolderMappingsDTO,
+    { folderMappings, filters, projectId },
-    actor: OrgServiceActor
+    actor
   ) => {
     const { permission } = await permissionService.getProjectPermission({
       actor: actor.type,
@@ -657,7 +645,7 @@ export const dynamicSecretServiceFactory = ({
   };
 
   // get dynamic secrets for multiple envs
-  const listDynamicSecretsByEnvs = async ({
+  const listDynamicSecretsByEnvs: TDynamicSecretServiceFactory["listDynamicSecretsByEnvs"] = async ({
     actorAuthMethod,
     actorOrgId,
     actorId,
@@ -667,7 +655,7 @@ export const dynamicSecretServiceFactory = ({
     projectId,
     isInternal,
     ...params
-  }: TListDynamicSecretsMultiEnvDTO) => {
+  }) => {
     const { permission } = await permissionService.getProjectPermission({
       actor,
       actorId,
@@ -700,14 +688,10 @@ export const dynamicSecretServiceFactory = ({
     });
   };
 
-  const fetchAzureEntraIdUsers = async ({
+  const fetchAzureEntraIdUsers: TDynamicSecretServiceFactory["fetchAzureEntraIdUsers"] = async ({
     tenantId,
     applicationId,
     clientSecret
-  }: {
-    tenantId: string;
-    applicationId: string;
-    clientSecret: string;
   }) => {
     const azureEntraIdUsers = await AzureEntraIDProvider().fetchAzureEntraIdUsers(
       tenantId,

backend/src/ee/services/dynamic-secret/dynamic-secret-types.ts

@@ -1,6 +1,7 @@
 import { z } from "zod";
 
-import { OrderByDirection, TProjectPermission } from "@app/lib/types";
+import { TDynamicSecrets } from "@app/db/schemas";
+import { OrderByDirection, OrgServiceActor, TDynamicSecretWithMetadata, TProjectPermission } from "@app/lib/types";
 import { ResourceMetadataDTO } from "@app/services/resource-metadata/resource-metadata-schema";
 import { SecretsOrderBy } from "@app/services/secret/secret-types";
 
@@ -83,3 +84,27 @@ export type TListDynamicSecretsMultiEnvDTO = Omit<
 export type TGetDynamicSecretsCountDTO = Omit<TListDynamicSecretsDTO, "projectSlug" | "projectId"> & {
   projectId: string;
 };
+
+export type TDynamicSecretServiceFactory = {
+  create: (arg: TCreateDynamicSecretDTO) => Promise<TDynamicSecrets>;
+  updateByName: (arg: TUpdateDynamicSecretDTO) => Promise<TDynamicSecrets>;
+  deleteByName: (arg: TDeleteDynamicSecretDTO) => Promise<TDynamicSecrets>;
+  getDetails: (arg: TDetailsDynamicSecretDTO) => Promise<TDynamicSecretWithMetadata>;
+  listDynamicSecretsByEnv: (arg: TListDynamicSecretsDTO) => Promise<TDynamicSecretWithMetadata[]>;
+  listDynamicSecretsByEnvs: (
+    arg: TListDynamicSecretsMultiEnvDTO
+  ) => Promise<Array<TDynamicSecretWithMetadata & { environment: string }>>;
+  getDynamicSecretCount: (arg: TGetDynamicSecretsCountDTO) => Promise<number>;
+  getCountMultiEnv: (arg: TListDynamicSecretsMultiEnvDTO) => Promise<number>;
+  fetchAzureEntraIdUsers: (arg: { tenantId: string; applicationId: string; clientSecret: string }) => Promise<
+    {
+      name: string;
+      id: string;
+      email: string;
+    }[]
+  >;
+  listDynamicSecretsByFolderIds: (
+    arg: TListDynamicSecretsByFolderMappingsDTO,
+    actor: OrgServiceActor
+  ) => Promise<Array<TDynamicSecretWithMetadata & { environment: string; path: string }>>;
+};

backend/src/lib/fn/time.ts

@@ -19,3 +19,5 @@ export const getMinExpiresIn = (exp1: string | number, exp2: string | number): s
 
   return ms1 <= ms2 ? exp1 : exp2;
 };
+
+export const convertMsToSecond = (time: number) => time / 1000;

backend/src/lib/types/index.ts

@@ -1,3 +1,4 @@
+import { TDynamicSecrets } from "@app/db/schemas";
 import { ActorAuthMethod, ActorType } from "@app/services/auth/auth-type";
 
 export type TGenericPermission = {
@@ -84,3 +85,7 @@ export enum QueueWorkerProfile {
   Standard = "standard",
   SecretScanning = "secret-scanning"
 }
+
+export interface TDynamicSecretWithMetadata extends TDynamicSecrets {
+  metadata: { id: string; key: string; value: string }[];
+}

backend/src/queue/queue-service.ts

@@ -377,6 +377,7 @@ export type TQueueServiceFactory = {
   stopRepeatableJobByKey: <T extends QueueName>(name: T, repeatJobKey: string) => Promise<boolean>;
   clearQueue: (name: QueueName) => Promise<void>;
   stopJobById: <T extends QueueName>(name: T, jobId: string) => Promise<void | undefined>;
+  stopJobByIdPg: <T extends QueueName>(name: T, jobId: string) => Promise<void | undefined>;
   getRepeatableJobs: (
     name: QueueName,
     startOffset?: number,
@@ -542,6 +543,10 @@ export const queueServiceFactory = (
     return q.removeRepeatableByKey(repeatJobKey);
   };
 
+  const stopJobByIdPg: TQueueServiceFactory["stopJobByIdPg"] = async (name, jobId) => {
+    await pgBoss.deleteJob(name, jobId);
+  };
+
   const stopJobById: TQueueServiceFactory["stopJobById"] = async (name, jobId) => {
     const q = queueContainer[name];
     const job = await q.getJob(jobId);
@@ -568,6 +573,7 @@ export const queueServiceFactory = (
     stopRepeatableJobByKey,
     clearQueue,
     stopJobById,
+    stopJobByIdPg,
     getRepeatableJobs,
     startPg,
     queuePg,

backend/src/server/routes/index.ts

@@ -1903,6 +1903,7 @@ export const registerRoutes = async (
   await pkiSubscriberQueue.startDailyAutoRenewalJob();
   await kmsService.startService();
   await microsoftTeamsService.start();
+  await dynamicSecretQueueService.init();
 
   // inject all services
   server.decorate<FastifyZodProvider["services"]>("services", {