feat: migrated dynamic secret to pg queue and corrected service layer types to non infer version

backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-dal.ts

@@ -3,9 +3,43 @@ import { Knex } from "knex";
 import { TDbClient } from "@app/db";
 import { DynamicSecretLeasesSchema, TableName } from "@app/db/schemas";
 import { DatabaseError } from "@app/lib/errors";
-import { ormify, selectAllTableCols } from "@app/lib/knex";
+import { ormify, selectAllTableCols, TOrmify } from "@app/lib/knex";
 
-export type TDynamicSecretLeaseDALFactory = ReturnType<typeof dynamicSecretLeaseDALFactory>;
+export interface TDynamicSecretLeaseDALFactory extends Omit<TOrmify<TableName.DynamicSecretLease>, "findById"> {
+  countLeasesForDynamicSecret: (dynamicSecretId: string, tx?: Knex) => Promise<number>;
+  findById: (
+    id: string,
+    tx?: Knex
+  ) => Promise<
+    | {
+        dynamicSecret: {
+          id: string;
+          name: string;
+          version: number;
+          type: string;
+          defaultTTL: string;
+          maxTTL: string | null | undefined;
+          encryptedInput: Buffer;
+          folderId: string;
+          status: string | null | undefined;
+          statusDetails: string | null | undefined;
+          createdAt: Date;
+          updatedAt: Date;
+        };
+        version: number;
+        id: string;
+        createdAt: Date;
+        updatedAt: Date;
+        externalEntityId: string;
+        expireAt: Date;
+        dynamicSecretId: string;
+        status?: string | null | undefined;
+        config?: unknown;
+        statusDetails?: string | null | undefined;
+      }
+    | undefined
+  >;
+}
 
 export const dynamicSecretLeaseDALFactory = (db: TDbClient) => {
   const orm = ormify(db, TableName.DynamicSecretLease);

backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.ts

@@ -21,7 +21,12 @@ type TDynamicSecretLeaseQueueServiceFactoryDep = {
   folderDAL: Pick<TSecretFolderDALFactory, "findById">;
 };
 
-export type TDynamicSecretLeaseQueueServiceFactory = ReturnType<typeof dynamicSecretLeaseQueueServiceFactory>;
+export type TDynamicSecretLeaseQueueServiceFactory = {
+  pruneDynamicSecret: (dynamicSecretCfgId: string) => Promise<void>;
+  setLeaseRevocation: (leaseId: string, expiryAt: Date) => Promise<void>;
+  unsetLeaseRevocation: (leaseId: string) => Promise<void>;
+  init: () => Promise<void>;
+};
 
 export const dynamicSecretLeaseQueueServiceFactory = ({
   queueService,
@@ -30,55 +35,47 @@ export const dynamicSecretLeaseQueueServiceFactory = ({
   dynamicSecretLeaseDAL,
   kmsService,
   folderDAL
-}: TDynamicSecretLeaseQueueServiceFactoryDep) => {
+}: TDynamicSecretLeaseQueueServiceFactoryDep): TDynamicSecretLeaseQueueServiceFactory => {
   const pruneDynamicSecret = async (dynamicSecretCfgId: string) => {
-    await queueService.queue(
-      QueueName.DynamicSecretRevocation,
+    await queueService.queuePg<QueueName.DynamicSecretRevocation>(
       QueueJobs.DynamicSecretPruning,
       { dynamicSecretCfgId },
       {
-        jobId: dynamicSecretCfgId,
-        backoff: {
-          type: "exponential",
-          delay: 3000
-        },
-        removeOnFail: {
-          count: 3
-        },
-        removeOnComplete: true
+        singletonKey: dynamicSecretCfgId,
+        retryLimit: 3,
+        retryBackoff: true
       }
     );
   };
 
-  const setLeaseRevocation = async (leaseId: string, expiry: number) => {
-    await queueService.queue(
-      QueueName.DynamicSecretRevocation,
+  const setLeaseRevocation = async (leaseId: string, expiryAt: Date) => {
+    await queueService.queuePg<QueueName.DynamicSecretRevocation>(
       QueueJobs.DynamicSecretRevocation,
       { leaseId },
       {
-        jobId: leaseId,
-        backoff: {
-          type: "exponential",
-          delay: 3000
-        },
-        delay: expiry,
-        removeOnFail: {
-          count: 3
-        },
-        removeOnComplete: true
+        id: leaseId,
+        singletonKey: leaseId,
+        startAfter: expiryAt,
+        retryLimit: 3,
+        retryBackoff: true,
+        retentionDays: 2
       }
     );
   };
 
   const unsetLeaseRevocation = async (leaseId: string) => {
-    await queueService.stopJobById(QueueName.DynamicSecretRevocation, leaseId);
+    await queueService.stopJobByIdPg(QueueName.DynamicSecretRevocation, leaseId);
   };
 
-  queueService.start(QueueName.DynamicSecretRevocation, async (job) => {
+  const $dynamicSecretQueueJob = async (
+    jobName: string,
+    jobId: string,
+    data: { leaseId: string } | { dynamicSecretCfgId: string }
+  ): Promise<void> => {
     try {
-      if (job.name === QueueJobs.DynamicSecretRevocation) {
-        const { leaseId } = job.data as { leaseId: string };
-        logger.info("Dynamic secret lease revocation started: ", leaseId, job.id);
+      if (jobName === QueueJobs.DynamicSecretRevocation) {
+        const { leaseId } = data as { leaseId: string };
+        logger.info("Dynamic secret lease revocation started: ", leaseId, jobId);
 
         const dynamicSecretLease = await dynamicSecretLeaseDAL.findById(leaseId);
         if (!dynamicSecretLease) throw new DisableRotationErrors({ message: "Dynamic secret lease not found" });
@@ -107,9 +104,9 @@ export const dynamicSecretLeaseQueueServiceFactory = ({
         return;
       }
 
-      if (job.name === QueueJobs.DynamicSecretPruning) {
-        const { dynamicSecretCfgId } = job.data as { dynamicSecretCfgId: string };
-        logger.info("Dynamic secret pruning started: ", dynamicSecretCfgId, job.id);
+      if (jobName === QueueJobs.DynamicSecretPruning) {
+        const { dynamicSecretCfgId } = data as { dynamicSecretCfgId: string };
+        logger.info("Dynamic secret pruning started: ", dynamicSecretCfgId, jobId);
         const dynamicSecretCfg = await dynamicSecretDAL.findById(dynamicSecretCfgId);
         if (!dynamicSecretCfg) throw new DisableRotationErrors({ message: "Dynamic secret not found" });
         if ((dynamicSecretCfg.status as DynamicSecretStatus) !== DynamicSecretStatus.Deleting)
@@ -150,38 +147,67 @@ export const dynamicSecretLeaseQueueServiceFactory = ({
 
         await dynamicSecretDAL.deleteById(dynamicSecretCfgId);
       }
-      logger.info("Finished dynamic secret job", job.id);
+      logger.info("Finished dynamic secret job", jobId);
     } catch (error) {
       logger.error(error);
 
-      if (job?.name === QueueJobs.DynamicSecretPruning) {
-        const { dynamicSecretCfgId } = job.data as { dynamicSecretCfgId: string };
+      if (jobName === QueueJobs.DynamicSecretPruning) {
+        const { dynamicSecretCfgId } = data as { dynamicSecretCfgId: string };
         await dynamicSecretDAL.updateById(dynamicSecretCfgId, {
           status: DynamicSecretStatus.FailedDeletion,
           statusDetails: (error as Error)?.message?.slice(0, 255)
         });
       }
 
-      if (job?.name === QueueJobs.DynamicSecretRevocation) {
-        const { leaseId } = job.data as { leaseId: string };
+      if (jobName === QueueJobs.DynamicSecretRevocation) {
+        const { leaseId } = data as { leaseId: string };
         await dynamicSecretLeaseDAL.updateById(leaseId, {
           status: DynamicSecretStatus.FailedDeletion,
           statusDetails: (error as Error)?.message?.slice(0, 255)
         });
       }
       if (error instanceof DisableRotationErrors) {
-        if (job.id) {
-          await queueService.stopRepeatableJobByJobId(QueueName.DynamicSecretRevocation, job.id);
+        if (jobId) {
+          await queueService.stopRepeatableJobByJobId(QueueName.DynamicSecretRevocation, jobId);
         }
       }
       // propogate to next part
       throw error;
     }
+  };
+
+  queueService.start(QueueName.DynamicSecretRevocation, async (job) => {
+    await $dynamicSecretQueueJob(job.name, job.id as string, job.data);
   });
 
+  const init = async () => {
+    await queueService.startPg<QueueName.DynamicSecretRevocation>(
+      QueueJobs.DynamicSecretRevocation,
+      async ([job]) => {
+        await $dynamicSecretQueueJob(job.name, job.id, job.data);
+      },
+      {
+        workerCount: 2,
+        pollingIntervalSeconds: 30
+      }
+    );
+
+    await queueService.startPg<QueueName.DynamicSecretRevocation>(
+      QueueJobs.DynamicSecretPruning,
+      async ([job]) => {
+        await $dynamicSecretQueueJob(job.name, job.id, job.data);
+      },
+      {
+        workerCount: 2,
+        pollingIntervalSeconds: 5
+      }
+    );
+  };
+
   return {
     pruneDynamicSecret,
     setLeaseRevocation,
-    unsetLeaseRevocation
+    unsetLeaseRevocation,
+    init
   };
 };

backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.ts

@@ -26,12 +26,8 @@ import { TDynamicSecretLeaseDALFactory } from "./dynamic-secret-lease-dal";
 import { TDynamicSecretLeaseQueueServiceFactory } from "./dynamic-secret-lease-queue";
 import {
   DynamicSecretLeaseStatus,
-  TCreateDynamicSecretLeaseDTO,
-  TDeleteDynamicSecretLeaseDTO,
-  TDetailsDynamicSecretLeaseDTO,
   TDynamicSecretLeaseConfig,
-  TListDynamicSecretLeasesDTO,
-  TRenewDynamicSecretLeaseDTO
+  TDynamicSecretLeaseServiceFactory
 } from "./dynamic-secret-lease-types";
 
 type TDynamicSecretLeaseServiceFactoryDep = {
@@ -48,8 +44,6 @@ type TDynamicSecretLeaseServiceFactoryDep = {
   identityDAL: TIdentityDALFactory;
 };
 
-export type TDynamicSecretLeaseServiceFactory = ReturnType<typeof dynamicSecretLeaseServiceFactory>;
-
 export const dynamicSecretLeaseServiceFactory = ({
   dynamicSecretLeaseDAL,
   dynamicSecretProviders,
@@ -62,14 +56,14 @@ export const dynamicSecretLeaseServiceFactory = ({
   kmsService,
   userDAL,
   identityDAL
-}: TDynamicSecretLeaseServiceFactoryDep) => {
+}: TDynamicSecretLeaseServiceFactoryDep): TDynamicSecretLeaseServiceFactory => {
   const extractEmailUsername = (email: string) => {
     const regex = new RE2(/^([^@]+)/);
     const match = email.match(regex);
     return match ? match[1] : email;
   };
 
-  const create = async ({
+  const create: TDynamicSecretLeaseServiceFactory["create"] = async ({
     environmentSlug,
     path,
     name,
@@ -80,7 +74,7 @@ export const dynamicSecretLeaseServiceFactory = ({
     actorAuthMethod,
     ttl,
     config
-  }: TCreateDynamicSecretLeaseDTO) => {
+  }) => {
     const appCfg = getConfig();
     const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
     if (!project) throw new NotFoundError({ message: `Project with slug '${projectSlug}' not found` });
@@ -184,11 +178,11 @@ export const dynamicSecretLeaseServiceFactory = ({
       config
     });
 
-    await dynamicSecretQueueService.setLeaseRevocation(dynamicSecretLease.id, Number(expireAt) - Number(new Date()));
+    await dynamicSecretQueueService.setLeaseRevocation(dynamicSecretLease.id, expireAt);
     return { lease: dynamicSecretLease, dynamicSecret: dynamicSecretCfg, data };
   };
 
-  const renewLease = async ({
+  const renewLease: TDynamicSecretLeaseServiceFactory["renewLease"] = async ({
     ttl,
     actorAuthMethod,
     actorOrgId,
@@ -198,7 +192,7 @@ export const dynamicSecretLeaseServiceFactory = ({
     path,
     environmentSlug,
     leaseId
-  }: TRenewDynamicSecretLeaseDTO) => {
+  }) => {
     const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
     if (!project) throw new NotFoundError({ message: `Project with slug '${projectSlug}' not found` });
 
@@ -278,7 +272,7 @@ export const dynamicSecretLeaseServiceFactory = ({
     );
 
     await dynamicSecretQueueService.unsetLeaseRevocation(dynamicSecretLease.id);
-    await dynamicSecretQueueService.setLeaseRevocation(dynamicSecretLease.id, Number(expireAt) - Number(new Date()));
+    await dynamicSecretQueueService.setLeaseRevocation(dynamicSecretLease.id, expireAt);
     const updatedDynamicSecretLease = await dynamicSecretLeaseDAL.updateById(dynamicSecretLease.id, {
       expireAt,
       externalEntityId: entityId
@@ -286,7 +280,7 @@ export const dynamicSecretLeaseServiceFactory = ({
     return updatedDynamicSecretLease;
   };
 
-  const revokeLease = async ({
+  const revokeLease: TDynamicSecretLeaseServiceFactory["revokeLease"] = async ({
     leaseId,
     environmentSlug,
     path,
@@ -296,7 +290,7 @@ export const dynamicSecretLeaseServiceFactory = ({
     actorOrgId,
     actorAuthMethod,
     isForced
-  }: TDeleteDynamicSecretLeaseDTO) => {
+  }) => {
     const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
     if (!project) throw new NotFoundError({ message: `Project with slug '${projectSlug}' not found` });
 
@@ -376,7 +370,7 @@ export const dynamicSecretLeaseServiceFactory = ({
     return deletedDynamicSecretLease;
   };
 
-  const listLeases = async ({
+  const listLeases: TDynamicSecretLeaseServiceFactory["listLeases"] = async ({
     path,
     name,
     actor,
@@ -385,7 +379,7 @@ export const dynamicSecretLeaseServiceFactory = ({
     actorOrgId,
     environmentSlug,
     actorAuthMethod
-  }: TListDynamicSecretLeasesDTO) => {
+  }) => {
     const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
     if (!project) throw new NotFoundError({ message: `Project with slug '${projectSlug}' not found` });
 
@@ -424,7 +418,7 @@ export const dynamicSecretLeaseServiceFactory = ({
     return dynamicSecretLeases;
   };
 
-  const getLeaseDetails = async ({
+  const getLeaseDetails: TDynamicSecretLeaseServiceFactory["getLeaseDetails"] = async ({
     projectSlug,
     actorOrgId,
     path,
@@ -433,7 +427,7 @@ export const dynamicSecretLeaseServiceFactory = ({
     actorId,
     leaseId,
     actorAuthMethod
-  }: TDetailsDynamicSecretLeaseDTO) => {
+  }) => {
     const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
     if (!project) throw new NotFoundError({ message: `Project with slug '${projectSlug}' not found` });
 

backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-types.ts

@@ -1,4 +1,5 @@
-import { TProjectPermission } from "@app/lib/types";
+import { TDynamicSecretLeases } from "@app/db/schemas";
+import { TDynamicSecretWithMetadata, TProjectPermission } from "@app/lib/types";
 
 export enum DynamicSecretLeaseStatus {
   FailedDeletion = "Failed to delete"
@@ -48,3 +49,40 @@ export type TDynamicSecretKubernetesLeaseConfig = {
 };
 
 export type TDynamicSecretLeaseConfig = TDynamicSecretKubernetesLeaseConfig;
+
+export type TDynamicSecretLeaseServiceFactory = {
+  create: (arg: TCreateDynamicSecretLeaseDTO) => Promise<{
+    lease: TDynamicSecretLeases;
+    dynamicSecret: TDynamicSecretWithMetadata;
+    data: unknown;
+  }>;
+  listLeases: (arg: TListDynamicSecretLeasesDTO) => Promise<TDynamicSecretLeases[]>;
+  revokeLease: (arg: TDeleteDynamicSecretLeaseDTO) => Promise<TDynamicSecretLeases>;
+  renewLease: (arg: TRenewDynamicSecretLeaseDTO) => Promise<TDynamicSecretLeases>;
+  getLeaseDetails: (arg: TDetailsDynamicSecretLeaseDTO) => Promise<{
+    dynamicSecret: {
+      id: string;
+      name: string;
+      version: number;
+      type: string;
+      defaultTTL: string;
+      maxTTL: string | null | undefined;
+      encryptedInput: Buffer;
+      folderId: string;
+      status: string | null | undefined;
+      statusDetails: string | null | undefined;
+      createdAt: Date;
+      updatedAt: Date;
+    };
+    version: number;
+    id: string;
+    createdAt: Date;
+    updatedAt: Date;
+    externalEntityId: string;
+    expireAt: Date;
+    dynamicSecretId: string;
+    status?: string | null | undefined;
+    config?: unknown;
+    statusDetails?: string | null | undefined;
+  }>;
+};