From 179573a269f27ac2511431894ef802f2861b347c Mon Sep 17 00:00:00 2001 From: Akhil Mohan Date: Tue, 2 Apr 2024 13:20:48 +0530 Subject: [PATCH] fix(server): added sync secret for imports and added check for avoiding cyclic import --- backend/src/queue/queue-service.ts | 4 +- .../secret-folder/secret-folder-dal.ts | 8 +-- .../secret-import/secret-import-dal.ts | 2 +- .../secret-import/secret-import-service.ts | 25 ++++++++- backend/src/services/secret/secret-queue.ts | 54 +++++++++++++++---- 5 files changed, 77 insertions(+), 16 deletions(-) diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index 7cb443ae1..e1149120d 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -61,11 +61,11 @@ export type TQueueJobTypes = { }; [QueueName.SecretWebhook]: { name: QueueJobs.SecWebhook; - payload: { projectId: string; environment: string; secretPath: string }; + payload: { projectId: string; environment: string; secretPath: string; depth?: number }; }; [QueueName.IntegrationSync]: { name: QueueJobs.IntegrationSync; - payload: { projectId: string; environment: string; secretPath: string }; + payload: { projectId: string; environment: string; secretPath: string; depth?: number }; }; [QueueName.SecretFullRepoScan]: { name: QueueJobs.SecretScan; diff --git a/backend/src/services/secret-folder/secret-folder-dal.ts b/backend/src/services/secret-folder/secret-folder-dal.ts index 023d039ca..8425b97de 100644 --- a/backend/src/services/secret-folder/secret-folder-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-dal.ts @@ -170,7 +170,8 @@ const sqlFindSecretPathByFolderId = (db: Knex, projectId: string, folderIds: str // if the given folder id is root folder id then intial path is set as / instead of /root // if not root folder the path here will be / path: db.raw(`CONCAT('/', (CASE WHEN "parentId" is NULL THEN '' ELSE ${TableName.SecretFolder}.name END))`), - child: db.raw("NULL::uuid") + child: db.raw("NULL::uuid"), + environmentSlug: `${TableName.Environment}.slug` }) .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) .where({ projectId }) @@ -190,14 +191,15 @@ const sqlFindSecretPathByFolderId = (db: Knex, projectId: string, folderIds: str ELSE CONCAT('/', secret_folders.name) END, parent.path )` ), - child: db.raw("COALESCE(parent.child, parent.id)") + child: db.raw("COALESCE(parent.child, parent.id)"), + environmentSlug: "parent.environmentSlug" }) .from(TableName.SecretFolder) .join("parent", "parent.parentId", `${TableName.SecretFolder}.id`) ); }) .select("*") - .from("parent"); + .from("parent"); export type TSecretFolderDALFactory = ReturnType; // never change this. If u do write a migration for it diff --git a/backend/src/services/secret-import/secret-import-dal.ts b/backend/src/services/secret-import/secret-import-dal.ts index f9c6f1be7..a2ad4d82c 100644 --- a/backend/src/services/secret-import/secret-import-dal.ts +++ b/backend/src/services/secret-import/secret-import-dal.ts @@ -49,7 +49,7 @@ export const secretImportDALFactory = (db: TDbClient) => { } }; - const find = async (filter: Partial, tx?: Knex) => { + const find = async (filter: Partial, tx?: Knex) => { try { const docs = await (tx || db)(TableName.SecretImport) .where(filter) diff --git a/backend/src/services/secret-import/secret-import-service.ts b/backend/src/services/secret-import/secret-import-service.ts index 40f9797e4..ecd84e84b 100644 --- a/backend/src/services/secret-import/secret-import-service.ts +++ b/backend/src/services/secret-import/secret-import-service.ts @@ -77,10 +77,19 @@ export const secretImportServiceFactory = ({ const folder = await folderDAL.findBySecretPath(projectId, environment, path); if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create import" }); - // TODO(akhilmhdh-pg): updated permission check add here const [importEnv] = await projectEnvDAL.findBySlugs(projectId, [data.environment]); if (!importEnv) throw new BadRequestError({ error: "Imported env not found", name: "Create import" }); + const sourceFolder = await folderDAL.findBySecretPath(projectId, data.environment, data.path); + if (sourceFolder) { + const existingImport = await secretImportDAL.findOne({ + folderId: sourceFolder.id, + importEnv: folder.environment.id, + importPath: path + }); + if (existingImport) throw new BadRequestError({ message: "Cyclic import not allowed" }); + } + const secImport = await secretImportDAL.transaction(async (tx) => { const lastPos = await secretImportDAL.findLastImportPosition(folder.id, tx); return secretImportDAL.create( @@ -131,6 +140,20 @@ export const secretImportServiceFactory = ({ : await projectEnvDAL.findById(secImpDoc.importEnv); if (!importedEnv) throw new BadRequestError({ error: "Imported env not found", name: "Create import" }); + const sourceFolder = await folderDAL.findBySecretPath( + projectId, + importedEnv.slug, + data.path || secImpDoc.importPath + ); + if (sourceFolder) { + const existingImport = await secretImportDAL.findOne({ + folderId: sourceFolder.id, + importEnv: folder.environment.id, + importPath: path + }); + if (existingImport) throw new BadRequestError({ message: "Cyclic import not allowed" }); + } + const updatedSecImport = await secretImportDAL.transaction(async (tx) => { const secImp = await secretImportDAL.findOne({ folderId: folder.id, id }); if (!secImp) throw ERR_SEC_IMP_NOT_FOUND; diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index d815875df..dfeacdb0f 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -3,7 +3,7 @@ import { getConfig } from "@app/lib/config/env"; import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; import { daysToMillisecond, secondsToMillis } from "@app/lib/dates"; import { BadRequestError } from "@app/lib/errors"; -import { isSamePath } from "@app/lib/fn"; +import { groupBy, isSamePath, unique } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; import { TProjectBotDALFactory } from "@app/services/project-bot/project-bot-dal"; @@ -32,7 +32,6 @@ import { interpolateSecrets } from "./secret-fns"; import { TCreateSecretReminderDTO, THandleReminderDTO, TRemoveSecretReminderDTO } from "./secret-types"; export type TSecretQueueFactory = ReturnType; - type TSecretQueueFactoryDep = { queueService: TQueueServiceFactory; integrationDAL: Pick; @@ -60,6 +59,8 @@ export type TGetSecrets = { environment: string; }; +const MAX_SYNC_SECRET_DEPTH = 5; + export const secretQueueFactory = ({ queueService, integrationDAL, @@ -117,7 +118,10 @@ export const secretQueueFactory = ({ }); }; - const syncSecrets = async (dto: TGetSecrets) => { + const syncSecrets = async (dto: TGetSecrets & { depth?: number }) => { + logger.info( + `Syncing secrets Project: ${dto.projectId} - Environment: ${dto.environment} - Path: ${dto.secretPath}` + ); await queueService.queue(QueueName.SecretWebhook, QueueJobs.SecWebhook, dto, { jobId: `secret-webhook-${dto.environment}-${dto.projectId}-${dto.secretPath}`, removeOnFail: { count: 5 }, @@ -310,20 +314,51 @@ export const secretQueueFactory = ({ }; queueService.start(QueueName.IntegrationSync, async (job) => { - const { environment, projectId, secretPath } = job.data; + const { environment, projectId, secretPath, depth = 1 } = job.data; const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); if (!folder) { - logger.error("Secret path not found"); + logger.error(new Error("Secret path not found")); return; } + // start syncing all linked imports also + if (depth < MAX_SYNC_SECRET_DEPTH) { + // find all imports made with the given environment and secret path + const linkSourceDto = { + projectId, + importEnv: folder.environment.id, + importPath: secretPath + }; + const imports = await secretImportDAL.find(linkSourceDto); + if (imports.length) { + // keep calling sync secret for all the imports made + const importedFolderIds = unique(imports, (i) => i.folderId).map(({ folderId }) => folderId); + const importedFolders = await folderDAL.findSecretPathByFolderIds(projectId, importedFolderIds); + const foldersGroupedById = groupBy(importedFolders, (i) => i.id); + await Promise.all( + imports + .filter(({ folderId }) => Boolean(foldersGroupedById[folderId][0].path)) + .map(({ folderId }) => { + const syncDto = { + depth: depth + 1, + projectId, + secretPath: foldersGroupedById[folderId][0].path, + environment: foldersGroupedById[folderId][0].environmentSlug + }; + logger.info({ sourceLink: linkSourceDto, destination: syncDto }, `Syncing secret due to link change`); + return syncSecrets(syncDto); + }) + ); + } + } + const integrations = await integrationDAL.findByProjectIdV2(projectId, environment); const toBeSyncedIntegrations = integrations.filter( ({ secretPath: integrationSecPath, isActive }) => isActive && isSamePath(secretPath, integrationSecPath) ); if (!integrations.length) return; - logger.info("Secret integration sync started", job.data, job.id); + logger.info({ source: job.data }, "Secret integration sync started - %s", job.id); for (const integration of toBeSyncedIntegrations) { const integrationAuth = { ...integration.integrationAuth, @@ -362,7 +397,7 @@ export const secretQueueFactory = ({ }); } - logger.info("Secret integration sync ended", job.id); + logger.info("Secret integration sync ended: %s", job.id); }); queueService.start(QueueName.SecretReminder, async ({ data }) => { @@ -403,7 +438,7 @@ export const secretQueueFactory = ({ }); queueService.listen(QueueName.IntegrationSync, "failed", (job, err) => { - logger.error("Failed to sync integration", job?.data, err); + logger.error(err, "Failed to sync integration %s", job?.id); }); queueService.start(QueueName.SecretWebhook, async (job) => { @@ -411,7 +446,8 @@ export const secretQueueFactory = ({ }); return { - syncSecrets, + // depth is internal only field thus no need to make it available outside + syncSecrets: (dto: TGetSecrets) => syncSecrets(dto), syncIntegrations, addSecretReminder, removeSecretReminder,