Merge remote-tracking branch 'origin/main' into ENG-3639

backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts

@@ -50,20 +50,19 @@ interface TSecretV2DalArg {
 }
 
 export const SECRET_DAL_TTL = () => applyJitter(10 * 60, 2 * 60);
-export const SECRET_DAL_VERSION_TTL = 15 * 60;
+export const SECRET_DAL_VERSION_TTL = "15m";
 export const MAX_SECRET_CACHE_BYTES = 25 * 1024 * 1024;
 export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
   const secretOrm = ormify(db, TableName.SecretV2);
 
-  const invalidateSecretCacheByProjectId = async (projectId: string) => {
+  const invalidateSecretCacheByProjectId = async (projectId: string, tx?: Knex) => {
     const secretDalVersionKey = SecretServiceCacheKeys.getSecretDalVersion(projectId);
-    await keyStore.incrementBy(secretDalVersionKey, 1);
-    await keyStore.setExpiry(secretDalVersionKey, SECRET_DAL_VERSION_TTL);
+    await keyStore.pgIncrementBy(secretDalVersionKey, { incr: 1, tx, expiry: SECRET_DAL_VERSION_TTL });
   };
 
   const findOne = async (filter: Partial<TSecretsV2>, tx?: Knex) => {
     try {
-      const docs = await (tx || db)(TableName.SecretV2)
+      const docs = await (tx || db.replicaNode())(TableName.SecretV2)
         // eslint-disable-next-line @typescript-eslint/no-misused-promises
         .where(buildFindFilter(filter, TableName.SecretV2))
         .leftJoin(
@@ -144,7 +143,7 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
   const find = async (filter: TFindFilter<TSecretsV2>, opts: TFindOpt<TSecretsV2> = {}) => {
     const { offset, limit, sort, tx } = opts;
     try {
-      const query = (tx || db)(TableName.SecretV2)
+      const query = (tx || db.replicaNode())(TableName.SecretV2)
         // eslint-disable-next-line @typescript-eslint/no-misused-promises
         .where(buildFindFilter(filter))
         .leftJoin(
@@ -888,13 +887,13 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
   const findSecretsWithReminderRecipients = async (ids: string[], limit: number, tx?: Knex) => {
     try {
       // Create a subquery to get limited secret IDs
-      const limitedSecretIds = (tx || db)(TableName.SecretV2)
+      const limitedSecretIds = (tx || db.replicaNode())(TableName.SecretV2)
         .whereIn(`${TableName.SecretV2}.id`, ids)
         .limit(limit)
         .select("id");
 
       // Join with all recipients for the limited secrets
-      const docs = await (tx || db)(TableName.SecretV2)
+      const docs = await (tx || db.replicaNode())(TableName.SecretV2)
         .whereIn(`${TableName.SecretV2}.id`, limitedSecretIds)
         .leftJoin(TableName.Reminder, `${TableName.SecretV2}.id`, `${TableName.Reminder}.secretId`)
         .leftJoin(TableName.ReminderRecipient, `${TableName.Reminder}.id`, `${TableName.ReminderRecipient}.reminderId`)
@@ -926,13 +925,13 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
   const findSecretsWithReminderRecipientsOld = async (ids: string[], limit: number, tx?: Knex) => {
     try {
       // Create a subquery to get limited secret IDs
-      const limitedSecretIds = (tx || db)(TableName.SecretV2)
+      const limitedSecretIds = (tx || db.replicaNode())(TableName.SecretV2)
         .whereIn(`${TableName.SecretV2}.id`, ids)
         .limit(limit)
         .select("id");
 
       // Join with all recipients for the limited secrets
-      const docs = await (tx || db)(TableName.SecretV2)
+      const docs = await (tx || db.replicaNode())(TableName.SecretV2)
         .whereIn(`${TableName.SecretV2}.id`, limitedSecretIds)
         .leftJoin(TableName.Reminder, `${TableName.SecretV2}.id`, `${TableName.Reminder}.secretId`)
         .leftJoin(

backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts

@@ -118,7 +118,7 @@ type TSecretV2BridgeServiceFactoryDep = {
   >;
   snapshotService: Pick<TSecretSnapshotServiceFactory, "performSnapshot">;
   resourceMetadataDAL: Pick<TResourceMetadataDALFactory, "insertMany" | "delete">;
-  keyStore: Pick<TKeyStoreFactory, "getItem" | "setExpiry" | "setItemWithExpiry" | "deleteItem">;
+  keyStore: Pick<TKeyStoreFactory, "getItem" | "setExpiry" | "setItemWithExpiry" | "deleteItem" | "pgGetIntItem">;
   reminderService: Pick<TReminderServiceFactory, "createReminder" | "getReminder">;
 };
 
@@ -373,6 +373,7 @@ export const secretV2BridgeServiceFactory = ({
         tx
       });
 
+      await secretDAL.invalidateSecretCacheByProjectId(projectId, tx);
       return createdSecret;
     });
 
@@ -390,7 +391,6 @@ export const secretV2BridgeServiceFactory = ({
       });
     }
 
-    await secretDAL.invalidateSecretCacheByProjectId(projectId);
     if (inputSecret.type === SecretType.Shared) {
       await snapshotService.performSnapshot(folderId);
       await secretQueueService.syncSecrets({
@@ -579,8 +579,16 @@ export const secretV2BridgeServiceFactory = ({
           }
         : {};
 
-    const updatedSecret = await secretDAL.transaction(async (tx) =>
-      fnSecretBulkUpdate({
+    if (secretValue) {
+      const { nestedReferences, localReferences } = getAllSecretReferences(secretValue);
+      const allSecretReferences = nestedReferences.concat(
+        localReferences.map((el) => ({ secretKey: el, secretPath, environment }))
+      );
+      await $validateSecretReferences(projectId, permission, allSecretReferences);
+    }
+
+    const updatedSecret = await secretDAL.transaction(async (tx) => {
+      const modifiedSecretsInDB = await fnSecretBulkUpdate({
         folderId,
         orgId: actorOrgId,
         resourceMetadataDAL,
@@ -611,8 +619,11 @@ export const secretV2BridgeServiceFactory = ({
           actorId
         },
         tx
-      })
-    );
+      });
+
+      await secretDAL.invalidateSecretCacheByProjectId(projectId, tx);
+      return modifiedSecretsInDB;
+    });
     if (inputSecret.secretReminderRepeatDays) {
       await reminderService.createReminder({
         actor,
@@ -628,7 +639,6 @@ export const secretV2BridgeServiceFactory = ({
       });
     }
 
-    await secretDAL.invalidateSecretCacheByProjectId(projectId);
     if (inputSecret.type === SecretType.Shared) {
       await snapshotService.performSnapshot(folderId);
       await secretQueueService.syncSecrets({
@@ -728,8 +738,8 @@ export const secretV2BridgeServiceFactory = ({
     );
 
     try {
-      const deletedSecret = await secretDAL.transaction(async (tx) =>
-        fnSecretBulkDelete({
+      const deletedSecret = await secretDAL.transaction(async (tx) => {
+        const modifiedSecretsInDB = await fnSecretBulkDelete({
           projectId,
           folderId,
           actorId,
@@ -745,10 +755,11 @@ export const secretV2BridgeServiceFactory = ({
             }
           ],
           tx
-        })
-      );
+        });
+        await secretDAL.invalidateSecretCacheByProjectId(projectId, tx);
+        return modifiedSecretsInDB;
+      });
 
-      await secretDAL.invalidateSecretCacheByProjectId(projectId);
       if (inputSecret.type === SecretType.Shared) {
         await snapshotService.performSnapshot(folderId);
         await secretQueueService.syncSecrets({
@@ -1040,7 +1051,7 @@ export const secretV2BridgeServiceFactory = ({
     });
     throwIfMissingSecretReadValueOrDescribePermission(permission, ProjectPermissionSecretActions.DescribeSecret);
 
-    const cachedSecretDalVersion = await keyStore.getItem(SecretServiceCacheKeys.getSecretDalVersion(projectId));
+    const cachedSecretDalVersion = await keyStore.pgGetIntItem(SecretServiceCacheKeys.getSecretDalVersion(projectId));
     const secretDalVersion = Number(cachedSecretDalVersion || 0);
     const cacheKey = SecretServiceCacheKeys.getSecretsOfServiceLayer(projectId, secretDalVersion, {
       ...dto,
@@ -1705,7 +1716,7 @@ export const secretV2BridgeServiceFactory = ({
       await kmsService.createCipherPairWithDataKey({ type: KmsDataKey.SecretManager, projectId });
 
     const executeBulkInsert = async (tx: Knex) => {
-      return fnSecretBulkInsert({
+      const modifiedSecretsInDB = await fnSecretBulkInsert({
         inputSecrets: inputSecrets.map((el) => {
           const references = secretReferencesGroupByInputSecretKey[el.secretKey]?.nestedReferences;
 
@@ -1741,13 +1752,14 @@ export const secretV2BridgeServiceFactory = ({
         },
         tx
       });
+      await secretDAL.invalidateSecretCacheByProjectId(projectId, tx);
+      return modifiedSecretsInDB;
     };
 
     const newSecrets = providedTx
       ? await executeBulkInsert(providedTx)
       : await secretDAL.transaction(executeBulkInsert);
 
-    await secretDAL.invalidateSecretCacheByProjectId(projectId);
     await snapshotService.performSnapshot(folderId);
     await secretQueueService.syncSecrets({
       actor,
@@ -2112,6 +2124,7 @@ export const secretV2BridgeServiceFactory = ({
         }
       }
 
+      await secretDAL.invalidateSecretCacheByProjectId(projectId, tx);
       return updatedSecrets;
     };
 
@@ -2119,7 +2132,6 @@ export const secretV2BridgeServiceFactory = ({
       ? await executeBulkUpdate(providedTx)
       : await secretDAL.transaction(executeBulkUpdate);
 
-    await secretDAL.invalidateSecretCacheByProjectId(projectId);
     await Promise.allSettled(folders.map((el) => (el?.id ? snapshotService.performSnapshot(el.id) : undefined)));
     await Promise.allSettled(
       folders.map((el) =>
@@ -2246,7 +2258,7 @@ export const secretV2BridgeServiceFactory = ({
     });
 
     const executeBulkDelete = async (tx: Knex) => {
-      return fnSecretBulkDelete({
+      const modifiedSecretsInDB = await fnSecretBulkDelete({
         secretDAL,
         secretQueueService,
         folderCommitService,
@@ -2262,6 +2274,8 @@ export const secretV2BridgeServiceFactory = ({
         commitChanges,
         tx
       });
+      await secretDAL.invalidateSecretCacheByProjectId(projectId, tx);
+      return modifiedSecretsInDB;
     };
 
     try {
@@ -2269,7 +2283,6 @@ export const secretV2BridgeServiceFactory = ({
         ? await executeBulkDelete(providedTx)
         : await secretDAL.transaction(executeBulkDelete);
 
-      await secretDAL.invalidateSecretCacheByProjectId(projectId);
       await snapshotService.performSnapshot(folderId);
       await secretQueueService.syncSecrets({
         actor,

backend/src/services/secret-v2-bridge/secret-version-dal.ts

@@ -72,7 +72,7 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => {
         .where(`${TableName.SecretVersionV2}.folderId`, folderId)
         .join(TableName.SecretV2, `${TableName.SecretV2}.id`, `${TableName.SecretVersionV2}.secretId`)
         .join<TSecretVersionsV2, TSecretVersionsV2 & { secretId: string; max: number }>(
-          (tx || db)(TableName.SecretVersionV2)
+          (tx || db.replicaNode())(TableName.SecretVersionV2)
             .where(`${TableName.SecretVersionV2}.folderId`, folderId)
             .groupBy("secretId")
             .max("version")
@@ -121,7 +121,7 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => {
         .where("folderId", folderId)
         .whereIn(`${TableName.SecretVersionV2}.secretId`, secretIds)
         .join(
-          (tx || db)(TableName.SecretVersionV2)
+          (tx || db.replicaNode())(TableName.SecretVersionV2)
             .groupBy("secretId")
             .max("version")
             .select("secretId")
@@ -189,7 +189,7 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => {
   }) => {
     try {
       const { offset, limit, sort = [["createdAt", "desc"]] } = findOpt;
-      const query = (tx || db)(TableName.SecretVersionV2)
+      const query = (tx || db.replicaNode())(TableName.SecretVersionV2)
         .leftJoin(TableName.Users, `${TableName.Users}.id`, `${TableName.SecretVersionV2}.userActorId`)
         .leftJoin(
           TableName.ProjectMembership,