Merge branch 'main' into ENG-3041

backend/src/ee/routes/v1/project-router.ts

@@ -111,15 +111,38 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => {
       params: z.object({
         workspaceId: z.string().trim().describe(AUDIT_LOGS.EXPORT.projectId)
       }),
-      querystring: z.object({
-        eventType: z.nativeEnum(EventType).optional().describe(AUDIT_LOGS.EXPORT.eventType),
-        userAgentType: z.nativeEnum(UserAgentType).optional().describe(AUDIT_LOGS.EXPORT.userAgentType),
-        startDate: z.string().datetime().optional().describe(AUDIT_LOGS.EXPORT.startDate),
-        endDate: z.string().datetime().optional().describe(AUDIT_LOGS.EXPORT.endDate),
-        offset: z.coerce.number().default(0).describe(AUDIT_LOGS.EXPORT.offset),
-        limit: z.coerce.number().default(20).describe(AUDIT_LOGS.EXPORT.limit),
-        actor: z.string().optional().describe(AUDIT_LOGS.EXPORT.actor)
-      }),
+      querystring: z
+        .object({
+          eventType: z.nativeEnum(EventType).optional().describe(AUDIT_LOGS.EXPORT.eventType),
+          userAgentType: z.nativeEnum(UserAgentType).optional().describe(AUDIT_LOGS.EXPORT.userAgentType),
+          startDate: z.string().datetime().optional().describe(AUDIT_LOGS.EXPORT.startDate),
+          endDate: z.string().datetime().optional().describe(AUDIT_LOGS.EXPORT.endDate),
+          offset: z.coerce.number().default(0).describe(AUDIT_LOGS.EXPORT.offset),
+          limit: z.coerce.number().max(1000).default(20).describe(AUDIT_LOGS.EXPORT.limit),
+          actor: z.string().optional().describe(AUDIT_LOGS.EXPORT.actor)
+        })
+        .superRefine((el, ctx) => {
+          if (el.endDate && el.startDate) {
+            const startDate = new Date(el.startDate);
+            const endDate = new Date(el.endDate);
+            const maxAllowedDate = new Date(startDate);
+            maxAllowedDate.setMonth(maxAllowedDate.getMonth() + 3);
+            if (endDate < startDate) {
+              ctx.addIssue({
+                code: z.ZodIssueCode.custom,
+                path: ["endDate"],
+                message: "End date cannot be before start date"
+              });
+            }
+            if (endDate > maxAllowedDate) {
+              ctx.addIssue({
+                code: z.ZodIssueCode.custom,
+                path: ["endDate"],
+                message: "Dates must be within 3 months"
+              });
+            }
+          }
+        }),
       response: {
         200: z.object({
           auditLogs: AuditLogsSchema.omit({
@@ -161,7 +184,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => {
         filter: {
           ...req.query,
           projectId: req.params.workspaceId,
-          endDate: req.query.endDate,
+          endDate: req.query.endDate || new Date().toISOString(),
           startDate: req.query.startDate || getLastMidnightDateISO(),
           auditLogActorId: req.query.actor,
           eventType: req.query.eventType ? [req.query.eventType] : undefined

backend/src/ee/services/audit-log/audit-log-dal.ts

@@ -30,10 +30,10 @@ type TFindQuery = {
   actor?: string;
   projectId?: string;
   environment?: string;
-  orgId?: string;
+  orgId: string;
   eventType?: string;
-  startDate?: string;
-  endDate?: string;
+  startDate: string;
+  endDate: string;
   userAgentType?: string;
   limit?: number;
   offset?: number;
@@ -61,18 +61,15 @@ export const auditLogDALFactory = (db: TDbClient) => {
     },
     tx
   ) => {
-    if (!orgId && !projectId) {
-      throw new Error("Either orgId or projectId must be provided");
-    }
-
     try {
       // Find statements
       const sqlQuery = (tx || db.replicaNode())(TableName.AuditLog)
+        .where(`${TableName.AuditLog}.orgId`, orgId)
+        .whereRaw(`"${TableName.AuditLog}"."createdAt" >= ?::timestamptz`, [startDate])
+        .andWhereRaw(`"${TableName.AuditLog}"."createdAt" < ?::timestamptz`, [endDate])
         // eslint-disable-next-line func-names
         .where(function () {
-          if (orgId) {
-            void this.where(`${TableName.AuditLog}.orgId`, orgId);
-          } else if (projectId) {
+          if (projectId) {
             void this.where(`${TableName.AuditLog}.projectId`, projectId);
           }
         });
@@ -135,14 +132,6 @@ export const auditLogDALFactory = (db: TDbClient) => {
         void sqlQuery.whereIn("eventType", eventType);
       }
 
-      // Filter by date range
-      if (startDate) {
-        void sqlQuery.whereRaw(`"${TableName.AuditLog}"."createdAt" >= ?::timestamptz`, [startDate]);
-      }
-      if (endDate) {
-        void sqlQuery.whereRaw(`"${TableName.AuditLog}"."createdAt" <= ?::timestamptz`, [endDate]);
-      }
-
       // we timeout long running queries to prevent DB resource issues (2 minutes)
       const docs = await sqlQuery.timeout(1000 * 120);
 
@@ -174,6 +163,8 @@ export const auditLogDALFactory = (db: TDbClient) => {
       try {
         const findExpiredLogSubQuery = (tx || db)(TableName.AuditLog)
           .where("expiresAt", "<", today)
+          .where("createdAt", "<", today) // to use audit log partition
+          .orderBy(`${TableName.AuditLog}.createdAt`, "desc")
           .select("id")
           .limit(AUDIT_LOG_PRUNE_BATCH_SIZE);
 

backend/src/ee/services/audit-log/audit-log-service.ts

@@ -67,7 +67,8 @@ export const auditLogServiceFactory = ({
       secretPath: filter.secretPath,
       secretKey: filter.secretKey,
       environment: filter.environment,
-      ...(filter.projectId ? { projectId: filter.projectId } : { orgId: actorOrgId })
+      orgId: actorOrgId,
+      ...(filter.projectId ? { projectId: filter.projectId } : {})
     });
 
     return auditLogs.map(({ eventType: logEventType, actor: eActor, actorMetadata, eventMetadata, ...el }) => ({

backend/src/ee/services/audit-log/audit-log-types.ts

@@ -56,8 +56,8 @@ export type TListProjectAuditLogDTO = {
     eventType?: EventType[];
     offset?: number;
     limit: number;
-    endDate?: string;
-    startDate?: string;
+    endDate: string;
+    startDate: string;
     projectId?: string;
     environment?: string;
     auditLogActorId?: string;

backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts

@@ -318,7 +318,7 @@ export const secretScanningV2QueueServiceFactory = async ({
     },
     {
       batchSize: 1,
-      workerCount: 20,
+      workerCount: 2,
       pollingIntervalSeconds: 1
     }
   );
@@ -539,7 +539,7 @@ export const secretScanningV2QueueServiceFactory = async ({
     },
     {
       batchSize: 1,
-      workerCount: 20,
+      workerCount: 2,
       pollingIntervalSeconds: 1
     }
   );
@@ -613,7 +613,7 @@ export const secretScanningV2QueueServiceFactory = async ({
     },
     {
       batchSize: 1,
-      workerCount: 5,
+      workerCount: 2,
       pollingIntervalSeconds: 1
     }
   );