feat: switched to pull all and selectively replicated strategy and simplified logic

backend/src/services/secret-import/secret-import-service.ts

@@ -13,7 +13,6 @@ import { TProjectDALFactory } from "../project/project-dal";
 import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
 import { TSecretDALFactory } from "../secret/secret-dal";
 import { TSecretQueueFactory } from "../secret/secret-queue";
-import { SecretOperations } from "../secret/secret-types";
 import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
 import { TSecretImportDALFactory } from "./secret-import-dal";
 import { fnSecretsFromImports } from "./secret-import-fns";
@@ -139,24 +138,21 @@ export const secretImportServiceFactory = ({
     });
 
     if (secImport.isReplication && sourceFolder) {
-      const importedSecrets = await secretDAL.find({ folderId: sourceFolder?.id });
       await secretQueueService.replicateSecrets({
         secretPath: secImport.importPath,
         projectId,
         environmentSlug: importEnv.slug,
         pickOnlyImportIds: [secImport.id],
-        folderId: sourceFolder.id,
-        secrets: importedSecrets.map(({ id, version }) => ({ operation: SecretOperations.Create, version, id })),
         actorId,
-        actor,
-        environmentId: importEnv.id
+        actor
       });
     } else {
       await secretQueueService.syncSecrets({
-        secretPath: secImport.importPath,
+        secretPath,
         projectId,
-        environmentSlug: importEnv.slug,
-        excludeReplication: true
+        environmentSlug: environment,
+        actorId,
+        actor
       });
     }
 
@@ -307,7 +303,8 @@ export const secretImportServiceFactory = ({
       secretPath,
       projectId,
       environmentSlug: environment,
-      excludeReplication: true
+      actor,
+      actorId
     });
 
     return secImport;
@@ -372,18 +369,14 @@ export const secretImportServiceFactory = ({
       secretImportDoc.importPath
     );
 
-    const importedSecrets = await secretDAL.find({ folderId: sourceFolder?.id });
     if (membership && sourceFolder) {
       await secretQueueService.replicateSecrets({
         secretPath: secretImportDoc.importPath,
         projectId,
         environmentSlug: secretImportDoc.importEnv.slug,
         pickOnlyImportIds: [secretImportDoc.id],
-        folderId: sourceFolder.id,
-        secrets: importedSecrets.map(({ id, version }) => ({ operation: SecretOperations.Create, version, id })),
         actorId,
-        actor,
-        environmentId: secretImportDoc.importEnv.id
+        actor
       });
     }
 

backend/src/services/secret/secret-dal.ts

@@ -75,7 +75,7 @@ export const secretDALFactory = (db: TDbClient) => {
   };
 
   const deleteMany = async (
-    data: Array<{ blindIndex: string; type: SecretType; isReplicated?: boolean }>,
+    data: Array<{ blindIndex: string; type: SecretType }>,
     folderId: string,
     userId: string,
     tx?: Knex

backend/src/services/secret/secret-fns.ts

@@ -536,7 +536,7 @@ export const fnSecretBulkInsert = async ({
     }))
   );
   const secretVersions = await secretVersionDAL.insertMany(
-    inputSecrets.map(({ tags, references, isReplicated, ...el }) => ({
+    inputSecrets.map(({ tags, references, ...el }) => ({
       ...el,
       folderId,
       secretId: newSecretGroupByBlindIndex[el.secretBlindIndex as string][0].id

backend/src/services/secret/secret-queue.ts

@@ -64,7 +64,7 @@ export type TGetSecrets = {
 };
 
 const MAX_SYNC_SECRET_DEPTH = 5;
-const uniqueSecretQueueKey = (environment: string, secretPath: string) =>
+export const uniqueSecretQueueKey = (environment: string, secretPath: string) =>
   `secret-queue-dedupe-${environment}-${secretPath}`;
 
 type TIntegrationSecret = Record<string, { value: string; comment?: string; skipMultilineEncoding?: boolean }>;
@@ -325,6 +325,7 @@ export const secretQueueFactory = ({
   const syncSecrets = async <T extends boolean = false>({
     // seperate de-dupe queue for integration sync and replication sync
     _deDupeQueue: deDupeQueue = {},
+    _depth: depth = 0,
     _deDupeReplicationQueue: deDupeReplicationQueue = {},
     ...dto
   }: TSyncSecretsDTO<T>) => {
@@ -332,7 +333,11 @@ export const secretQueueFactory = ({
       `syncSecrets: syncing project secrets where [projectId=${dto.projectId}]  [environment=${dto.environmentSlug}] [path=${dto.secretPath}]`
     );
     const deDuplicationKey = uniqueSecretQueueKey(dto.environmentSlug, dto.secretPath);
-    if (!dto.excludeReplication ? deDupeReplicationQueue?.[deDuplicationKey] : deDupeQueue?.[deDuplicationKey]) {
+    if (
+      !dto.excludeReplication
+        ? deDupeReplicationQueue?.[deDuplicationKey]
+        : deDupeQueue?.[deDuplicationKey] || depth > MAX_SYNC_SECRET_DEPTH
+    ) {
       return;
     }
     // eslint-disable-next-line
@@ -342,7 +347,12 @@ export const secretQueueFactory = ({
     await queueService.queue(
       QueueName.SecretSync,
       QueueJobs.SecretSync,
-      { ...dto, _deDupeQueue: deDupeQueue, _deDupeReplicationQueue: deDupeReplicationQueue } as TSyncSecretsDTO,
+      {
+        ...dto,
+        _deDupeQueue: deDupeQueue,
+        _deDupeReplicationQueue: deDupeReplicationQueue,
+        _depth: depth
+      } as TSyncSecretsDTO,
       {
         removeOnFail: true,
         removeOnComplete: true,
@@ -360,22 +370,21 @@ export const secretQueueFactory = ({
     const {
       _deDupeQueue: deDupeQueue,
       _deDupeReplicationQueue: deDupeReplicationQueue,
+      _depth: depth,
       secretPath,
-      environmentId,
       projectId,
       environmentSlug: environment,
-      secrets,
-      folderId,
       excludeReplication,
       actorId,
       actor
     } = job.data;
+
     await queueService.queue(
       QueueName.SecretWebhook,
       QueueJobs.SecWebhook,
       { environment, projectId, secretPath },
       {
-        jobId: `secret-webhook-${environmentId}-${projectId}-${secretPath}`,
+        jobId: `secret-webhook-${environment}-${projectId}-${secretPath}`,
         removeOnFail: { count: 5 },
         removeOnComplete: true,
         delay: 1000,
@@ -390,11 +399,9 @@ export const secretQueueFactory = ({
     if (!excludeReplication) {
       await replicateSecrets({
         _deDupeReplicationQueue: deDupeReplicationQueue,
-        environmentId,
+        _depth: depth,
         projectId,
         secretPath,
-        folderId,
-        secrets,
         actorId,
         actor,
         excludeReplication,
@@ -405,6 +412,7 @@ export const secretQueueFactory = ({
 
   queueService.start(QueueName.IntegrationSync, async (job) => {
     const { environment, projectId, secretPath, depth = 1, deDupeQueue = {} } = job.data;
+    if (depth > MAX_SYNC_SECRET_DEPTH) return;
 
     const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
     if (!folder) {
@@ -450,6 +458,7 @@ export const secretQueueFactory = ({
                 secretPath: foldersGroupedById[folderId][0]?.path as string,
                 environmentSlug: foldersGroupedById[folderId][0]?.environmentSlug as string,
                 _deDupeQueue: deDupeQueue,
+                _depth: depth + 1,
                 excludeReplication: true
               })
             )
@@ -487,6 +496,7 @@ export const secretQueueFactory = ({
                 secretPath: referencedFoldersGroupedById[folderId][0]?.path as string,
                 environmentSlug: referencedFoldersGroupedById[folderId][0]?.environmentSlug as string,
                 _deDupeQueue: deDupeQueue,
+                _depth: depth + 1,
                 excludeReplication: true
               })
             )

backend/src/services/secret/secret-service.ts

@@ -44,7 +44,6 @@ import {
 } from "./secret-fns";
 import { TSecretQueueFactory } from "./secret-queue";
 import {
-  SecretOperations,
   TAttachSecretTagsDTO,
   TBackFillSecretReferencesDTO,
   TCreateBulkSecretDTO,
@@ -238,19 +237,10 @@ export const secretServiceFactory = ({
     await snapshotService.performSnapshot(folderId);
     await secretQueueService.syncSecrets({
       secretPath: path,
-      folderId: folder.id,
       actorId,
       actor,
       projectId,
-      environmentSlug: folder.environment.slug,
-      environmentId: folder.envId,
-      secrets: [
-        {
-          operation: SecretOperations.Create,
-          id: secret[0].id,
-          version: 1
-        }
-      ]
+      environmentSlug: folder.environment.slug
     });
     return { ...secret[0], environment, workspace: projectId, tags, secretPath: path };
   };
@@ -362,7 +352,6 @@ export const secretServiceFactory = ({
                 "secretReminderRepeatDays",
                 "tags"
               ]),
-              isReplicated: false,
               secretBlindIndex: newSecretNameBlindIndex || keyName2BlindIndex[secretName],
               references: references({
                 ciphertext: inputSecret.secretValueCiphertext,
@@ -385,17 +374,8 @@ export const secretServiceFactory = ({
       actor,
       actorId,
       secretPath: path,
-      folderId: folder.id,
       projectId,
-      environmentSlug: folder.environment.slug,
-      environmentId: folder.envId,
-      secrets: [
-        {
-          operation: SecretOperations.Update,
-          id: updatedSecret[0].id,
-          version: updatedSecret[0].version
-        }
-      ]
+      environmentSlug: folder.environment.slug
     });
     return { ...updatedSecret[0], workspace: projectId, environment, secretPath: path };
   };
@@ -469,17 +449,8 @@ export const secretServiceFactory = ({
       actor,
       actorId,
       secretPath: path,
-      folderId: folder.id,
       projectId,
-      environmentSlug: folder.environment.slug,
-      environmentId: folder.envId,
-      secrets: [
-        {
-          operation: SecretOperations.Delete,
-          id: deletedSecret[0].id,
-          version: deletedSecret[0].version
-        }
-      ]
+      environmentSlug: folder.environment.slug
     });
     // TODO(akhilmhdh-pg): licence check, posthog service and snapshot
     return { ...deletedSecret[0], _id: deletedSecret[0].id, workspace: projectId, environment, secretPath: path };
@@ -770,11 +741,8 @@ export const secretServiceFactory = ({
       actor,
       actorId,
       secretPath: path,
-      folderId: folder.id,
       projectId,
-      environmentSlug: folder.environment.slug,
-      environmentId: folder.envId,
-      secrets: newSecrets.map(({ id, version }) => ({ id, version, operation: SecretOperations.Create }))
+      environmentSlug: folder.environment.slug
     });
 
     return newSecrets;
@@ -851,7 +819,6 @@ export const secretServiceFactory = ({
             ...el,
             folderId,
             type: SecretType.Shared,
-            isReplicated: false,
             secretBlindIndex:
               newSecretName && newKeyName2BlindIndex[newSecretName]
                 ? newKeyName2BlindIndex[newSecretName]
@@ -880,11 +847,8 @@ export const secretServiceFactory = ({
       actor,
       actorId,
       secretPath: path,
-      folderId: folder.id,
       projectId,
-      environmentSlug: folder.environment.slug,
-      environmentId: folder.envId,
-      secrets: secrets.map(({ id, version }) => ({ id, version, operation: SecretOperations.Update }))
+      environmentSlug: folder.environment.slug
     });
 
     return secrets;
@@ -953,11 +917,8 @@ export const secretServiceFactory = ({
       actor,
       actorId,
       secretPath: path,
-      folderId: folder.id,
       projectId,
-      environmentSlug: folder.environment.slug,
-      environmentId: folder.envId,
-      secrets: secretsDeleted.map(({ id, version }) => ({ id, version, operation: SecretOperations.Delete }))
+      environmentSlug: folder.environment.slug
     });
 
     return secretsDeleted;

backend/src/services/secret/secret-types.ts

@@ -380,6 +380,7 @@ export enum SecretOperations {
 export type TSyncSecretsDTO<T extends boolean = false> = {
   _deDupeQueue?: Record<string, boolean>;
   _deDupeReplicationQueue?: Record<string, boolean>;
+  _depth?: number;
   secretPath: string;
   projectId: string;
   environmentSlug: string;
@@ -388,15 +389,8 @@ export type TSyncSecretsDTO<T extends boolean = false> = {
 } & (T extends true
   ? object
   : {
-      environmentId: string;
-      folderId: string;
       actor: ActorType;
       actorId: string;
       // used for import creation to trigger replication
       pickOnlyImportIds?: string[];
-      secrets: {
-        operation: SecretOperations;
-        id: string;
-        version: number;
-      }[];
     });