diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index 42e13b445..fbd406a4c 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -421,94 +421,88 @@ export const secretQueueFactory = ({ const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); if (!folder) { - logger.error(new Error("Secret path not found")); - return; + throw new Error("Secret path not found"); } - // start syncing all linked imports also - if (depth < MAX_SYNC_SECRET_DEPTH) { - // find all imports made with the given environment and secret path - const linkSourceDto = { - projectId, - importEnv: folder.environment.id, - importPath: secretPath, - isReplication: false - }; - const imports = await secretImportDAL.find(linkSourceDto); + // find all imports made with the given environment and secret path + const linkSourceDto = { + projectId, + importEnv: folder.environment.id, + importPath: secretPath, + isReplication: false + }; + const imports = await secretImportDAL.find(linkSourceDto); - if (imports.length) { - // keep calling sync secret for all the imports made - const importedFolderIds = unique(imports, (i) => i.folderId).map(({ folderId }) => folderId); - const importedFolders = await folderDAL.findSecretPathByFolderIds(projectId, importedFolderIds); - const foldersGroupedById = groupBy(importedFolders.filter(Boolean), (i) => i?.id as string); - logger.info( - `getIntegrationSecrets: Syncing secret due to link change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}] [secretPath=${job.data.secretPath}] [depth=${depth}]` - ); - await Promise.all( - imports - .filter(({ folderId }) => Boolean(foldersGroupedById[folderId][0]?.path as string)) - // filter out already synced ones - .filter( - ({ folderId }) => - !deDupeQueue[ - uniqueSecretQueueKey( - foldersGroupedById[folderId][0]?.environmentSlug as string, - foldersGroupedById[folderId][0]?.path as string - ) - ] - ) - .map(({ folderId }) => - syncSecrets({ - projectId, - secretPath: foldersGroupedById[folderId][0]?.path as string, - environmentSlug: foldersGroupedById[folderId][0]?.environmentSlug as string, - _deDupeQueue: deDupeQueue, - _depth: depth + 1, - excludeReplication: true - }) - ) - ); - } - - const secretReferences = await secretDAL.findReferencedSecretReferences( - projectId, - folder.environment.slug, - secretPath + if (imports.length) { + // keep calling sync secret for all the imports made + const importedFolderIds = unique(imports, (i) => i.folderId).map(({ folderId }) => folderId); + const importedFolders = await folderDAL.findSecretPathByFolderIds(projectId, importedFolderIds); + const foldersGroupedById = groupBy(importedFolders.filter(Boolean), (i) => i?.id as string); + logger.info( + `getIntegrationSecrets: Syncing secret due to link change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}] [secretPath=${job.data.secretPath}] [depth=${depth}]` + ); + await Promise.all( + imports + .filter(({ folderId }) => Boolean(foldersGroupedById[folderId][0]?.path as string)) + // filter out already synced ones + .filter( + ({ folderId }) => + !deDupeQueue[ + uniqueSecretQueueKey( + foldersGroupedById[folderId][0]?.environmentSlug as string, + foldersGroupedById[folderId][0]?.path as string + ) + ] + ) + .map(({ folderId }) => + syncSecrets({ + projectId, + secretPath: foldersGroupedById[folderId][0]?.path as string, + environmentSlug: foldersGroupedById[folderId][0]?.environmentSlug as string, + _deDupeQueue: deDupeQueue, + _depth: depth + 1, + excludeReplication: true + }) + ) + ); + } + + const secretReferences = await secretDAL.findReferencedSecretReferences( + projectId, + folder.environment.slug, + secretPath + ); + if (secretReferences.length) { + const referencedFolderIds = unique(secretReferences, (i) => i.folderId).map(({ folderId }) => folderId); + const referencedFolders = await folderDAL.findSecretPathByFolderIds(projectId, referencedFolderIds); + const referencedFoldersGroupedById = groupBy(referencedFolders.filter(Boolean), (i) => i?.id as string); + logger.info( + `getIntegrationSecrets: Syncing secret due to reference change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}] [secretPath=${job.data.secretPath}] [depth=${depth}]` + ); + await Promise.all( + secretReferences + .filter(({ folderId }) => Boolean(referencedFoldersGroupedById[folderId][0]?.path)) + // filter out already synced ones + .filter( + ({ folderId }) => + !deDupeQueue[ + uniqueSecretQueueKey( + referencedFoldersGroupedById[folderId][0]?.environmentSlug as string, + referencedFoldersGroupedById[folderId][0]?.path as string + ) + ] + ) + .map(({ folderId }) => + syncSecrets({ + projectId, + secretPath: referencedFoldersGroupedById[folderId][0]?.path as string, + environmentSlug: referencedFoldersGroupedById[folderId][0]?.environmentSlug as string, + _deDupeQueue: deDupeQueue, + _depth: depth + 1, + excludeReplication: true + }) + ) ); - if (secretReferences.length) { - const referencedFolderIds = unique(secretReferences, (i) => i.folderId).map(({ folderId }) => folderId); - const referencedFolders = await folderDAL.findSecretPathByFolderIds(projectId, referencedFolderIds); - const referencedFoldersGroupedById = groupBy(referencedFolders.filter(Boolean), (i) => i?.id as string); - logger.info( - `getIntegrationSecrets: Syncing secret due to reference change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}] [secretPath=${job.data.secretPath}] [depth=${depth}]` - ); - await Promise.all( - secretReferences - .filter(({ folderId }) => Boolean(referencedFoldersGroupedById[folderId][0]?.path)) - // filter out already synced ones - .filter( - ({ folderId }) => - !deDupeQueue[ - uniqueSecretQueueKey( - referencedFoldersGroupedById[folderId][0]?.environmentSlug as string, - referencedFoldersGroupedById[folderId][0]?.path as string - ) - ] - ) - .map(({ folderId }) => - syncSecrets({ - projectId, - secretPath: referencedFoldersGroupedById[folderId][0]?.path as string, - environmentSlug: referencedFoldersGroupedById[folderId][0]?.environmentSlug as string, - _deDupeQueue: deDupeQueue, - _depth: depth + 1, - excludeReplication: true - }) - ) - ); - } - } else { - logger.info(`getIntegrationSecrets: Secret depth exceeded for [projectId=${projectId}] [folderId=${folder.id}]`); } const integrations = await integrationDAL.findByProjectIdV2(projectId, environment); // note: returns array of integrations + integration auths in this environment @@ -571,10 +565,12 @@ export const secretQueueFactory = ({ syncMessage: "", isSynced: true }); - } catch (err: unknown) { + } catch (err) { logger.info("Secret integration sync error: %o", err); + const message = - err instanceof AxiosError ? JSON.stringify((err as AxiosError)?.response?.data) : (err as Error)?.message; + (err instanceof AxiosError ? JSON.stringify(err?.response?.data) : (err as Error)?.message) || + "Unknown error occurred."; await integrationDAL.updateById(integration.id, { lastSyncJobId: job.id,