From 179573a269f27ac2511431894ef802f2861b347c Mon Sep 17 00:00:00 2001 From: Akhil Mohan Date: Tue, 2 Apr 2024 13:20:48 +0530 Subject: [PATCH 1/6] fix(server): added sync secret for imports and added check for avoiding cyclic import --- backend/src/queue/queue-service.ts | 4 +- .../secret-folder/secret-folder-dal.ts | 8 +-- .../secret-import/secret-import-dal.ts | 2 +- .../secret-import/secret-import-service.ts | 25 ++++++++- backend/src/services/secret/secret-queue.ts | 54 +++++++++++++++---- 5 files changed, 77 insertions(+), 16 deletions(-) diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index 7cb443ae1..e1149120d 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -61,11 +61,11 @@ export type TQueueJobTypes = { }; [QueueName.SecretWebhook]: { name: QueueJobs.SecWebhook; - payload: { projectId: string; environment: string; secretPath: string }; + payload: { projectId: string; environment: string; secretPath: string; depth?: number }; }; [QueueName.IntegrationSync]: { name: QueueJobs.IntegrationSync; - payload: { projectId: string; environment: string; secretPath: string }; + payload: { projectId: string; environment: string; secretPath: string; depth?: number }; }; [QueueName.SecretFullRepoScan]: { name: QueueJobs.SecretScan; diff --git a/backend/src/services/secret-folder/secret-folder-dal.ts b/backend/src/services/secret-folder/secret-folder-dal.ts index 023d039ca..8425b97de 100644 --- a/backend/src/services/secret-folder/secret-folder-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-dal.ts @@ -170,7 +170,8 @@ const sqlFindSecretPathByFolderId = (db: Knex, projectId: string, folderIds: str // if the given folder id is root folder id then intial path is set as / instead of /root // if not root folder the path here will be / path: db.raw(`CONCAT('/', (CASE WHEN "parentId" is NULL THEN '' ELSE ${TableName.SecretFolder}.name END))`), - child: db.raw("NULL::uuid") + child: db.raw("NULL::uuid"), + environmentSlug: `${TableName.Environment}.slug` }) .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) .where({ projectId }) @@ -190,14 +191,15 @@ const sqlFindSecretPathByFolderId = (db: Knex, projectId: string, folderIds: str ELSE CONCAT('/', secret_folders.name) END, parent.path )` ), - child: db.raw("COALESCE(parent.child, parent.id)") + child: db.raw("COALESCE(parent.child, parent.id)"), + environmentSlug: "parent.environmentSlug" }) .from(TableName.SecretFolder) .join("parent", "parent.parentId", `${TableName.SecretFolder}.id`) ); }) .select("*") - .from("parent"); + .from("parent"); export type TSecretFolderDALFactory = ReturnType; // never change this. If u do write a migration for it diff --git a/backend/src/services/secret-import/secret-import-dal.ts b/backend/src/services/secret-import/secret-import-dal.ts index f9c6f1be7..a2ad4d82c 100644 --- a/backend/src/services/secret-import/secret-import-dal.ts +++ b/backend/src/services/secret-import/secret-import-dal.ts @@ -49,7 +49,7 @@ export const secretImportDALFactory = (db: TDbClient) => { } }; - const find = async (filter: Partial, tx?: Knex) => { + const find = async (filter: Partial, tx?: Knex) => { try { const docs = await (tx || db)(TableName.SecretImport) .where(filter) diff --git a/backend/src/services/secret-import/secret-import-service.ts b/backend/src/services/secret-import/secret-import-service.ts index 40f9797e4..ecd84e84b 100644 --- a/backend/src/services/secret-import/secret-import-service.ts +++ b/backend/src/services/secret-import/secret-import-service.ts @@ -77,10 +77,19 @@ export const secretImportServiceFactory = ({ const folder = await folderDAL.findBySecretPath(projectId, environment, path); if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create import" }); - // TODO(akhilmhdh-pg): updated permission check add here const [importEnv] = await projectEnvDAL.findBySlugs(projectId, [data.environment]); if (!importEnv) throw new BadRequestError({ error: "Imported env not found", name: "Create import" }); + const sourceFolder = await folderDAL.findBySecretPath(projectId, data.environment, data.path); + if (sourceFolder) { + const existingImport = await secretImportDAL.findOne({ + folderId: sourceFolder.id, + importEnv: folder.environment.id, + importPath: path + }); + if (existingImport) throw new BadRequestError({ message: "Cyclic import not allowed" }); + } + const secImport = await secretImportDAL.transaction(async (tx) => { const lastPos = await secretImportDAL.findLastImportPosition(folder.id, tx); return secretImportDAL.create( @@ -131,6 +140,20 @@ export const secretImportServiceFactory = ({ : await projectEnvDAL.findById(secImpDoc.importEnv); if (!importedEnv) throw new BadRequestError({ error: "Imported env not found", name: "Create import" }); + const sourceFolder = await folderDAL.findBySecretPath( + projectId, + importedEnv.slug, + data.path || secImpDoc.importPath + ); + if (sourceFolder) { + const existingImport = await secretImportDAL.findOne({ + folderId: sourceFolder.id, + importEnv: folder.environment.id, + importPath: path + }); + if (existingImport) throw new BadRequestError({ message: "Cyclic import not allowed" }); + } + const updatedSecImport = await secretImportDAL.transaction(async (tx) => { const secImp = await secretImportDAL.findOne({ folderId: folder.id, id }); if (!secImp) throw ERR_SEC_IMP_NOT_FOUND; diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index d815875df..dfeacdb0f 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -3,7 +3,7 @@ import { getConfig } from "@app/lib/config/env"; import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; import { daysToMillisecond, secondsToMillis } from "@app/lib/dates"; import { BadRequestError } from "@app/lib/errors"; -import { isSamePath } from "@app/lib/fn"; +import { groupBy, isSamePath, unique } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; import { TProjectBotDALFactory } from "@app/services/project-bot/project-bot-dal"; @@ -32,7 +32,6 @@ import { interpolateSecrets } from "./secret-fns"; import { TCreateSecretReminderDTO, THandleReminderDTO, TRemoveSecretReminderDTO } from "./secret-types"; export type TSecretQueueFactory = ReturnType; - type TSecretQueueFactoryDep = { queueService: TQueueServiceFactory; integrationDAL: Pick; @@ -60,6 +59,8 @@ export type TGetSecrets = { environment: string; }; +const MAX_SYNC_SECRET_DEPTH = 5; + export const secretQueueFactory = ({ queueService, integrationDAL, @@ -117,7 +118,10 @@ export const secretQueueFactory = ({ }); }; - const syncSecrets = async (dto: TGetSecrets) => { + const syncSecrets = async (dto: TGetSecrets & { depth?: number }) => { + logger.info( + `Syncing secrets Project: ${dto.projectId} - Environment: ${dto.environment} - Path: ${dto.secretPath}` + ); await queueService.queue(QueueName.SecretWebhook, QueueJobs.SecWebhook, dto, { jobId: `secret-webhook-${dto.environment}-${dto.projectId}-${dto.secretPath}`, removeOnFail: { count: 5 }, @@ -310,20 +314,51 @@ export const secretQueueFactory = ({ }; queueService.start(QueueName.IntegrationSync, async (job) => { - const { environment, projectId, secretPath } = job.data; + const { environment, projectId, secretPath, depth = 1 } = job.data; const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); if (!folder) { - logger.error("Secret path not found"); + logger.error(new Error("Secret path not found")); return; } + // start syncing all linked imports also + if (depth < MAX_SYNC_SECRET_DEPTH) { + // find all imports made with the given environment and secret path + const linkSourceDto = { + projectId, + importEnv: folder.environment.id, + importPath: secretPath + }; + const imports = await secretImportDAL.find(linkSourceDto); + if (imports.length) { + // keep calling sync secret for all the imports made + const importedFolderIds = unique(imports, (i) => i.folderId).map(({ folderId }) => folderId); + const importedFolders = await folderDAL.findSecretPathByFolderIds(projectId, importedFolderIds); + const foldersGroupedById = groupBy(importedFolders, (i) => i.id); + await Promise.all( + imports + .filter(({ folderId }) => Boolean(foldersGroupedById[folderId][0].path)) + .map(({ folderId }) => { + const syncDto = { + depth: depth + 1, + projectId, + secretPath: foldersGroupedById[folderId][0].path, + environment: foldersGroupedById[folderId][0].environmentSlug + }; + logger.info({ sourceLink: linkSourceDto, destination: syncDto }, `Syncing secret due to link change`); + return syncSecrets(syncDto); + }) + ); + } + } + const integrations = await integrationDAL.findByProjectIdV2(projectId, environment); const toBeSyncedIntegrations = integrations.filter( ({ secretPath: integrationSecPath, isActive }) => isActive && isSamePath(secretPath, integrationSecPath) ); if (!integrations.length) return; - logger.info("Secret integration sync started", job.data, job.id); + logger.info({ source: job.data }, "Secret integration sync started - %s", job.id); for (const integration of toBeSyncedIntegrations) { const integrationAuth = { ...integration.integrationAuth, @@ -362,7 +397,7 @@ export const secretQueueFactory = ({ }); } - logger.info("Secret integration sync ended", job.id); + logger.info("Secret integration sync ended: %s", job.id); }); queueService.start(QueueName.SecretReminder, async ({ data }) => { @@ -403,7 +438,7 @@ export const secretQueueFactory = ({ }); queueService.listen(QueueName.IntegrationSync, "failed", (job, err) => { - logger.error("Failed to sync integration", job?.data, err); + logger.error(err, "Failed to sync integration %s", job?.id); }); queueService.start(QueueName.SecretWebhook, async (job) => { @@ -411,7 +446,8 @@ export const secretQueueFactory = ({ }); return { - syncSecrets, + // depth is internal only field thus no need to make it available outside + syncSecrets: (dto: TGetSecrets) => syncSecrets(dto), syncIntegrations, addSecretReminder, removeSecretReminder, From abd62867eb78bbdb660d6af08d0aa1c9b2a3636c Mon Sep 17 00:00:00 2001 From: Akhil Mohan Date: Tue, 2 Apr 2024 13:55:26 +0530 Subject: [PATCH 2/6] fix(server): resolved failing test in import --- backend/e2e-test/routes/v1/secret-import.spec.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/e2e-test/routes/v1/secret-import.spec.ts b/backend/e2e-test/routes/v1/secret-import.spec.ts index ba37b5f42..c184e44e5 100644 --- a/backend/e2e-test/routes/v1/secret-import.spec.ts +++ b/backend/e2e-test/routes/v1/secret-import.spec.ts @@ -46,7 +46,7 @@ const deleteSecretImport = async (id: string) => { describe("Secret Import Router", async () => { test.each([ - { importEnv: "dev", importPath: "/" }, // one in root + { importEnv: "prod", importPath: "/" }, // one in root { importEnv: "staging", importPath: "/" } // then create a deep one creating intermediate ones ])("Create secret import $importEnv with path $importPath", async ({ importPath, importEnv }) => { // check for default environments @@ -66,7 +66,7 @@ describe("Secret Import Router", async () => { }); test("Get secret imports", async () => { - const createdImport1 = await createSecretImport("/", "dev"); + const createdImport1 = await createSecretImport("/", "prod"); const createdImport2 = await createSecretImport("/", "staging"); const res = await testServer.inject({ method: "GET", @@ -103,10 +103,10 @@ describe("Secret Import Router", async () => { }); test("Update secret import position", async () => { - const devImportDetails = { path: "/", envSlug: "dev" }; + const prodImportDetails = { path: "/", envSlug: "prod" }; const stagingImportDetails = { path: "/", envSlug: "staging" }; - const createdImport1 = await createSecretImport(devImportDetails.path, devImportDetails.envSlug); + const createdImport1 = await createSecretImport(prodImportDetails.path, prodImportDetails.envSlug); const createdImport2 = await createSecretImport(stagingImportDetails.path, stagingImportDetails.envSlug); const updateImportRes = await testServer.inject({ @@ -136,7 +136,7 @@ describe("Secret Import Router", async () => { position: 2, importEnv: expect.objectContaining({ name: expect.any(String), - slug: expect.stringMatching(devImportDetails.envSlug), + slug: expect.stringMatching(prodImportDetails.envSlug), id: expect.any(String) }) }) @@ -166,7 +166,7 @@ describe("Secret Import Router", async () => { }); test("Delete secret import position", async () => { - const createdImport1 = await createSecretImport("/", "dev"); + const createdImport1 = await createSecretImport("/", "prod"); const createdImport2 = await createSecretImport("/", "staging"); const deletedImport = await deleteSecretImport(createdImport1.id); // check for default environments From 85489a81ff85c4024b414aa0becbbdd18c0cb736 Mon Sep 17 00:00:00 2001 From: Tuan Dang Date: Wed, 10 Apr 2024 21:18:26 -0700 Subject: [PATCH 3/6] Add resync on integration import creation/deletion and update forward/backward recursive logic for syncing dependent imports --- backend/src/server/routes/index.ts | 17 +-- .../secret-folder/secret-folder-dal.ts | 2 + .../secret-import/secret-import-service.ts | 18 ++- backend/src/services/secret/secret-queue.ts | 107 ++++++++++-------- 4 files changed, 85 insertions(+), 59 deletions(-) diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index d94d00ccb..be0494f0e 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -445,14 +445,6 @@ export const registerRoutes = async ( projectEnvDAL, snapshotService }); - const secretImportService = secretImportServiceFactory({ - projectEnvDAL, - folderDAL, - permissionService, - secretImportDAL, - projectDAL, - secretDAL - }); const integrationAuthService = integrationAuthServiceFactory({ integrationAuthDAL, integrationDAL, @@ -480,6 +472,15 @@ export const registerRoutes = async ( secretTagDAL, secretVersionTagDAL }); + const secretImportService = secretImportServiceFactory({ + projectEnvDAL, + folderDAL, + permissionService, + secretImportDAL, + projectDAL, + secretDAL, + secretQueueService + }); const secretBlindIndexService = secretBlindIndexServiceFactory({ permissionService, secretDAL, diff --git a/backend/src/services/secret-folder/secret-folder-dal.ts b/backend/src/services/secret-folder/secret-folder-dal.ts index 8425b97de..b3147d1fa 100644 --- a/backend/src/services/secret-folder/secret-folder-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-dal.ts @@ -259,10 +259,12 @@ export const secretFolderDALFactory = (db: TDbClient) => { const findSecretPathByFolderIds = async (projectId: string, folderIds: string[], tx?: Knex) => { try { const folders = await sqlFindSecretPathByFolderId(tx || db, projectId, folderIds); + const rootFolders = groupBy( folders.filter(({ parentId }) => parentId === null), (i) => i.child || i.id // root condition then child and parent will null ); + return folderIds.map((folderId) => rootFolders[folderId]?.[0]); } catch (error) { throw new DatabaseError({ error, name: "Find by secret path" }); diff --git a/backend/src/services/secret-import/secret-import-service.ts b/backend/src/services/secret-import/secret-import-service.ts index ecd84e84b..2d59284d4 100644 --- a/backend/src/services/secret-import/secret-import-service.ts +++ b/backend/src/services/secret-import/secret-import-service.ts @@ -7,6 +7,7 @@ import { BadRequestError } from "@app/lib/errors"; import { TProjectDALFactory } from "../project/project-dal"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TSecretDALFactory } from "../secret/secret-dal"; +import { TSecretQueueFactory } from "../secret/secret-queue"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TSecretImportDALFactory } from "./secret-import-dal"; import { fnSecretsFromImports } from "./secret-import-fns"; @@ -25,6 +26,7 @@ type TSecretImportServiceFactoryDep = { projectDAL: Pick; projectEnvDAL: TProjectEnvDALFactory; permissionService: Pick; + secretQueueService: Pick; }; const ERR_SEC_IMP_NOT_FOUND = new BadRequestError({ message: "Secret import not found" }); @@ -37,7 +39,8 @@ export const secretImportServiceFactory = ({ permissionService, folderDAL, projectDAL, - secretDAL + secretDAL, + secretQueueService }: TSecretImportServiceFactoryDep) => { const createImport = async ({ environment, @@ -103,6 +106,12 @@ export const secretImportServiceFactory = ({ ); }); + await secretQueueService.syncSecrets({ + secretPath: secImport.importPath, + projectId, + environment: importEnv.slug + }); + return { ...secImport, importEnv }; }; @@ -208,6 +217,13 @@ export const secretImportServiceFactory = ({ if (!importEnv) throw new BadRequestError({ error: "Imported env not found", name: "Create import" }); return { ...doc, importEnv }; }); + + await secretQueueService.syncSecrets({ + secretPath: path, + projectId, + environment + }); + return secImport; }; diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index dfeacdb0f..1d2561723 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -23,7 +23,6 @@ import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TSecretImportDALFactory } from "../secret-import/secret-import-dal"; -import { fnSecretsFromImports } from "../secret-import/secret-import-fns"; import { SmtpTemplates, TSmtpService } from "../smtp/smtp-service"; import { TWebhookDALFactory } from "../webhook/webhook-dal"; import { fnTriggerWebhook } from "../webhook/webhook-fns"; @@ -231,62 +230,35 @@ export const secretQueueFactory = ({ } }; - const getIntegrationSecrets = async (dto: TGetSecrets & { folderId: string }, key: string) => { + type Content = Record; + + /** + * Return the secrets in a given [folderId] including the secrets from nested imported folders + */ + const getIntegrationSecrets = async (dto: { + projectId: string; + environment: string; + folderId: string; + key: string; + depth: number; + }) => { + let content: Content = {}; + if (dto.depth > MAX_SYNC_SECRET_DEPTH) return content; + const secrets = await secretDAL.findByFolderId(dto.folderId); - - // get imported secrets - const secretImport = await secretImportDAL.find({ folderId: dto.folderId }); - const importedSecrets = await fnSecretsFromImports({ - allowedImports: secretImport, - secretDAL, - folderDAL - }); - - if (!secrets.length && !importedSecrets.length) return {}; - - const content: Record = {}; - - importedSecrets.forEach(({ secrets: secs }) => { - secs.forEach((secret) => { - const secretKey = decryptSymmetric128BitHexKeyUTF8({ - ciphertext: secret.secretKeyCiphertext, - iv: secret.secretKeyIV, - tag: secret.secretKeyTag, - key - }); - const secretValue = decryptSymmetric128BitHexKeyUTF8({ - ciphertext: secret.secretValueCiphertext, - iv: secret.secretValueIV, - tag: secret.secretValueTag, - key - }); - content[secretKey] = { value: secretValue }; - content[secretKey].skipMultilineEncoding = Boolean(secret.skipMultilineEncoding); - - if (secret.secretCommentCiphertext && secret.secretCommentIV && secret.secretCommentTag) { - const commentValue = decryptSymmetric128BitHexKeyUTF8({ - ciphertext: secret.secretCommentCiphertext, - iv: secret.secretCommentIV, - tag: secret.secretCommentTag, - key - }); - content[secretKey].comment = commentValue; - } - }); - }); secrets.forEach((secret) => { const secretKey = decryptSymmetric128BitHexKeyUTF8({ ciphertext: secret.secretKeyCiphertext, iv: secret.secretKeyIV, tag: secret.secretKeyTag, - key + key: dto.key }); const secretValue = decryptSymmetric128BitHexKeyUTF8({ ciphertext: secret.secretValueCiphertext, iv: secret.secretValueIV, tag: secret.secretValueTag, - key + key: dto.key }); content[secretKey] = { value: secretValue }; @@ -296,25 +268,52 @@ export const secretQueueFactory = ({ ciphertext: secret.secretCommentCiphertext, iv: secret.secretCommentIV, tag: secret.secretCommentTag, - key + key: dto.key }); content[secretKey].comment = commentValue; } content[secretKey].skipMultilineEncoding = Boolean(secret.skipMultilineEncoding); }); + const expandSecrets = interpolateSecrets({ projectId: dto.projectId, - secretEncKey: key, + secretEncKey: dto.key, folderDAL, secretDAL }); + await expandSecrets(content); + + const secretImport = await secretImportDAL.find({ folderId: dto.folderId }); + if (!secretImport) return content; + + const importedFolders = await folderDAL.findByManySecretPath( + secretImport.map(({ importEnv, importPath }) => ({ + envId: importEnv.id, + secretPath: importPath + })) + ); + + for await (const folder of importedFolders) { + if (folder) { + const importedSecrets = await getIntegrationSecrets({ + environment: dto.environment, + projectId: dto.projectId, + folderId: folder.id, + key: dto.key, + depth: dto.depth + 1 + }); + content = { ...content, ...importedSecrets }; + } + } + return content; }; queueService.start(QueueName.IntegrationSync, async (job) => { const { environment, projectId, secretPath, depth = 1 } = job.data; + const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); if (!folder) { logger.error(new Error("Secret path not found")); @@ -330,11 +329,12 @@ export const secretQueueFactory = ({ importPath: secretPath }; const imports = await secretImportDAL.find(linkSourceDto); + if (imports.length) { // keep calling sync secret for all the imports made const importedFolderIds = unique(imports, (i) => i.folderId).map(({ folderId }) => folderId); const importedFolders = await folderDAL.findSecretPathByFolderIds(projectId, importedFolderIds); - const foldersGroupedById = groupBy(importedFolders, (i) => i.id); + const foldersGroupedById = groupBy(importedFolders, (i) => i.child || i.id); await Promise.all( imports .filter(({ folderId }) => Boolean(foldersGroupedById[folderId][0].path)) @@ -352,8 +352,9 @@ export const secretQueueFactory = ({ } } - const integrations = await integrationDAL.findByProjectIdV2(projectId, environment); + const integrations = await integrationDAL.findByProjectIdV2(projectId, environment); // note: returns array of integrations + integration auths in this environment const toBeSyncedIntegrations = integrations.filter( + // note: sync only the integrations sourced from secretPath ({ secretPath: integrationSecPath, isActive }) => isActive && isSamePath(secretPath, integrationSecPath) ); @@ -369,7 +370,13 @@ export const secretQueueFactory = ({ const botKey = await projectBotService.getBotKey(projectId); const { accessToken, accessId } = await integrationAuthService.getIntegrationAccessToken(integrationAuth, botKey); - const secrets = await getIntegrationSecrets({ environment, projectId, secretPath, folderId: folder.id }, botKey); + const secrets = await getIntegrationSecrets({ + environment, + projectId, + folderId: folder.id, + key: botKey, + depth: 1 + }); const suffixedSecrets: typeof secrets = {}; const metadata = integration.metadata as Record; if (metadata) { From 2ac785493a321f74ec7fc70a27bfcf6edf419811 Mon Sep 17 00:00:00 2001 From: Tuan Dang Date: Wed, 10 Apr 2024 21:52:33 -0700 Subject: [PATCH 4/6] Add comments to explain new getIntegrationSecrets --- backend/src/services/secret/secret-queue.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index 1d2561723..ddef9269d 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -233,7 +233,8 @@ export const secretQueueFactory = ({ type Content = Record; /** - * Return the secrets in a given [folderId] including the secrets from nested imported folders + * Return the secrets in a given [folderId] including secrets from + * nested imported folders recursively. */ const getIntegrationSecrets = async (dto: { projectId: string; @@ -245,6 +246,7 @@ export const secretQueueFactory = ({ let content: Content = {}; if (dto.depth > MAX_SYNC_SECRET_DEPTH) return content; + // process secrets in current folder const secrets = await secretDAL.findByFolderId(dto.folderId); secrets.forEach((secret) => { const secretKey = decryptSymmetric128BitHexKeyUTF8({ @@ -285,7 +287,10 @@ export const secretQueueFactory = ({ await expandSecrets(content); + // check if current folder has any imports from other folders const secretImport = await secretImportDAL.find({ folderId: dto.folderId }); + + // if no imports then return secrets in the current folder if (!secretImport) return content; const importedFolders = await folderDAL.findByManySecretPath( @@ -297,6 +302,8 @@ export const secretQueueFactory = ({ for await (const folder of importedFolders) { if (folder) { + // get secrets contained in each imported folder by recursively calling + // this function against the imported folder const importedSecrets = await getIntegrationSecrets({ environment: dto.environment, projectId: dto.projectId, @@ -304,6 +311,8 @@ export const secretQueueFactory = ({ key: dto.key, depth: dto.depth + 1 }); + + // add the imported secrets to the current folder secrets content = { ...content, ...importedSecrets }; } } From a01a995585131c97c32aaf0a0fe78546040adfbf Mon Sep 17 00:00:00 2001 From: Tuan Dang Date: Wed, 10 Apr 2024 21:52:33 -0700 Subject: [PATCH 5/6] Add comments to explain new getIntegrationSecrets --- backend/src/services/secret/secret-queue.ts | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index ddef9269d..c96088e7b 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -119,7 +119,7 @@ export const secretQueueFactory = ({ const syncSecrets = async (dto: TGetSecrets & { depth?: number }) => { logger.info( - `Syncing secrets Project: ${dto.projectId} - Environment: ${dto.environment} - Path: ${dto.secretPath}` + `syncSecrets: syncing project secrets where [projectId=${dto.projectId}] [environment=${dto.environment}] [path=${dto.secretPath}]` ); await queueService.queue(QueueName.SecretWebhook, QueueJobs.SecWebhook, dto, { jobId: `secret-webhook-${dto.environment}-${dto.projectId}-${dto.secretPath}`, @@ -244,7 +244,12 @@ export const secretQueueFactory = ({ depth: number; }) => { let content: Content = {}; - if (dto.depth > MAX_SYNC_SECRET_DEPTH) return content; + if (dto.depth > MAX_SYNC_SECRET_DEPTH) { + logger.info( + `getIntegrationSecrets: secret depth exceeded for [projectId=${dto.projectId}] [folderId=${dto.folderId}] [depth=${dto.depth}]` + ); + return content; + } // process secrets in current folder const secrets = await secretDAL.findByFolderId(dto.folderId); @@ -354,11 +359,17 @@ export const secretQueueFactory = ({ secretPath: foldersGroupedById[folderId][0].path, environment: foldersGroupedById[folderId][0].environmentSlug }; - logger.info({ sourceLink: linkSourceDto, destination: syncDto }, `Syncing secret due to link change`); + logger.info( + `getIntegrationSecrets: Syncing secret due to link change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}] [secretPath=${job.data.secretPath}] [depth=${job.data.depth}]` + ); return syncSecrets(syncDto); }) ); } + } else { + logger.info( + `getIntegrationSecrets: Secret depth exceeded for [projectId=${projectId}] [folderId=${folder.id}] [depth=${depth}]` + ); } const integrations = await integrationDAL.findByProjectIdV2(projectId, environment); // note: returns array of integrations + integration auths in this environment @@ -368,7 +379,9 @@ export const secretQueueFactory = ({ ); if (!integrations.length) return; - logger.info({ source: job.data }, "Secret integration sync started - %s", job.id); + logger.info( + `getIntegrationSecrets: secret integration sync started [jobId=${job.id}] [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}] [secretPath=${job.data.secretPath}] [depth=${job.data.depth}]` + ); for (const integration of toBeSyncedIntegrations) { const integrationAuth = { ...integration.integrationAuth, From 0db682c5f05ae9a0d8a2c69a2a684219044a95eb Mon Sep 17 00:00:00 2001 From: Maidul Islam Date: Thu, 11 Apr 2024 10:10:42 -0400 Subject: [PATCH 6/6] remove depth from exceed message --- backend/src/services/secret/secret-queue.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index c96088e7b..1fc6b1109 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -360,16 +360,14 @@ export const secretQueueFactory = ({ environment: foldersGroupedById[folderId][0].environmentSlug }; logger.info( - `getIntegrationSecrets: Syncing secret due to link change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}] [secretPath=${job.data.secretPath}] [depth=${job.data.depth}]` + `getIntegrationSecrets: Syncing secret due to link change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}] [secretPath=${job.data.secretPath}] [depth=${depth}]` ); return syncSecrets(syncDto); }) ); } } else { - logger.info( - `getIntegrationSecrets: Secret depth exceeded for [projectId=${projectId}] [folderId=${folder.id}] [depth=${depth}]` - ); + logger.info(`getIntegrationSecrets: Secret depth exceeded for [projectId=${projectId}] [folderId=${folder.id}]`); } const integrations = await integrationDAL.findByProjectIdV2(projectId, environment); // note: returns array of integrations + integration auths in this environment