From 4e16b0ac8f14962c682d2352b66b54ab5c6e77b6 Mon Sep 17 00:00:00 2001 From: Carlos Monastyrski Date: Wed, 3 Sep 2025 13:26:22 -0300 Subject: [PATCH 1/4] Fix folder path and environment on secret imports --- .../secret-replication-service.ts | 2 +- .../integration-delete-secret.ts | 4 +- .../integration/integration-service.ts | 2 +- .../secret-import/secret-import-fns.ts | 79 ++++++++++++++++++- .../services/secret-sync/secret-sync-queue.ts | 2 +- .../secret-v2-bridge-service.ts | 2 +- backend/src/services/secret/secret-queue.ts | 2 +- 7 files changed, 85 insertions(+), 8 deletions(-) diff --git a/backend/src/ee/services/secret-replication/secret-replication-service.ts b/backend/src/ee/services/secret-replication/secret-replication-service.ts index 4a5558f46..f08c1560e 100644 --- a/backend/src/ee/services/secret-replication/secret-replication-service.ts +++ b/backend/src/ee/services/secret-replication/secret-replication-service.ts @@ -59,7 +59,7 @@ type TSecretReplicationServiceFactoryDep = { TSecretVersionV2DALFactory, "find" | "insertMany" | "update" | "findLatestVersionMany" >; - secretImportDAL: Pick; + secretImportDAL: Pick; folderDAL: Pick< TSecretFolderDALFactory, "findSecretPathByFolderIds" | "findBySecretPath" | "create" | "findOne" | "findByManySecretPath" diff --git a/backend/src/services/integration-auth/integration-delete-secret.ts b/backend/src/services/integration-auth/integration-delete-secret.ts index 2efec7a47..ccab7f978 100644 --- a/backend/src/services/integration-auth/integration-delete-secret.ts +++ b/backend/src/services/integration-auth/integration-delete-secret.ts @@ -39,7 +39,7 @@ const getIntegrationSecretsV2 = async ( }, secretV2BridgeDAL: Pick, folderDAL: Pick, - secretImportDAL: Pick + secretImportDAL: Pick ) => { const content: Record = {}; if (dto.depth > MAX_SYNC_SECRET_DEPTH) { @@ -300,7 +300,7 @@ export const deleteIntegrationSecrets = async ({ projectBotService: Pick; secretV2BridgeDAL: Pick; folderDAL: Pick; - secretImportDAL: Pick; + secretImportDAL: Pick; secretDAL: Pick; kmsService: Pick; }) => { diff --git a/backend/src/services/integration/integration-service.ts b/backend/src/services/integration/integration-service.ts index 2ef8615eb..498b99866 100644 --- a/backend/src/services/integration/integration-service.ts +++ b/backend/src/services/integration/integration-service.ts @@ -40,7 +40,7 @@ type TIntegrationServiceFactoryDep = { projectBotService: TProjectBotServiceFactory; secretQueueService: Pick; secretV2BridgeDAL: Pick; - secretImportDAL: Pick; + secretImportDAL: Pick; kmsService: Pick; secretDAL: Pick; }; diff --git a/backend/src/services/secret-import/secret-import-fns.ts b/backend/src/services/secret-import/secret-import-fns.ts index 6aa73465d..e8bf123b5 100644 --- a/backend/src/services/secret-import/secret-import-fns.ts +++ b/backend/src/services/secret-import/secret-import-fns.ts @@ -1,3 +1,5 @@ +import RE2 from "re2"; + import { SecretType, TSecretImports, TSecrets, TSecretsV2 } from "@app/db/schemas"; import { groupBy, unique } from "@app/lib/fn"; @@ -54,6 +56,77 @@ type TSecretImportSecretsV2 = { const LEVEL_BREAK = 10; const getImportUniqKey = (envSlug: string, path: string) => `${envSlug}=${path}`; + +/** + * Processes reserved imports by resolving them to their replication source. + */ +const processReservedImports = async < + T extends { + isReserved?: boolean | null; + importPath: string; + importEnv: { id: string; slug: string; name: string }; + folderId: string; + } +>( + imports: T[], + secretImportDAL: Pick +): Promise => { + const reservedImportIds: string[] = []; + + imports.forEach((secretImport) => { + if (secretImport.isReserved) { + const reservedMatch = new RE2("/__reserve_replication_([a-f0-9-]{36})").exec(secretImport.importPath); + if (reservedMatch) { + const referencedImportId = reservedMatch[1]; + reservedImportIds.push(referencedImportId); + } + } + }); + + if (reservedImportIds.length === 0) { + return imports; + } + + try { + const importDetailsMap = new Map< + string, + { importPath: string; importEnv: { id: string; slug: string; name: string } } + >(); + + /* eslint-disable no-await-in-loop */ + for (const importId of reservedImportIds) { + const referencedImport = await secretImportDAL.findById(importId); + if (referencedImport) { + importDetailsMap.set(importId, { + importPath: referencedImport.importPath, + importEnv: referencedImport.importEnv + }); + } + } + /* eslint-enable no-await-in-loop */ + + return imports.map((secretImport) => { + if (secretImport.isReserved) { + const reservedMatch = new RE2("/__reserve_replication_([a-f0-9-]{36})").exec(secretImport.importPath); + if (reservedMatch) { + const referencedImportId = reservedMatch[1]; + const referencedDetails = importDetailsMap.get(referencedImportId); + + if (referencedDetails) { + return { + ...secretImport, + importPath: referencedDetails.importPath, + importEnv: referencedDetails.importEnv + }; + } + } + } + return secretImport; + }); + } catch (error) { + return imports; + } +}; export const fnSecretsFromImports = async ({ allowedImports: possibleCyclicImports, folderDAL, @@ -167,7 +240,7 @@ export const fnSecretsV2FromImports = async ({ folderDAL: Pick; viewSecretValue: boolean; secretDAL: Pick; - secretImportDAL: Pick; + secretImportDAL: Pick; decryptor: (value?: Buffer | null) => string; expandSecretReferences?: (inputSecret: { value?: string; @@ -188,6 +261,10 @@ export const fnSecretsV2FromImports = async ({ })[]; }[] = [{ secretImports: rootSecretImports, depth: 0, parentImportedSecrets: [] }]; + const processedSecretImports = await processReservedImports(rootSecretImports, secretImportDAL); + + stack[0] = { secretImports: processedSecretImports, depth: 0, parentImportedSecrets: [] }; + const processedImports: TSecretImportSecretsV2[] = []; while (stack.length) { diff --git a/backend/src/services/secret-sync/secret-sync-queue.ts b/backend/src/services/secret-sync/secret-sync-queue.ts index 7bef7d8c7..50c0b9dca 100644 --- a/backend/src/services/secret-sync/secret-sync-queue.ts +++ b/backend/src/services/secret-sync/secret-sync-queue.ts @@ -80,7 +80,7 @@ type TSecretSyncQueueFactoryDep = { | "deleteMany" | "invalidateSecretCacheByProjectId" >; - secretImportDAL: Pick; + secretImportDAL: Pick; secretSyncDAL: Pick; auditLogService: Pick; projectMembershipDAL: Pick; diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts index b23cd0c56..4b05582c1 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts @@ -108,7 +108,7 @@ type TSecretV2BridgeServiceFactoryDep = { | "findBySecretPathMultiEnv" | "findSecretPathByFolderIds" >; - secretImportDAL: Pick; + secretImportDAL: Pick; secretQueueService: Pick; secretApprovalPolicyService: Pick; secretApprovalRequestDAL: Pick; diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index dc5713901..5160c4bf3 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -86,7 +86,7 @@ type TSecretQueueFactoryDep = { integrationAuthService: Pick; folderDAL: TSecretFolderDALFactory; secretDAL: TSecretDALFactory; - secretImportDAL: Pick; + secretImportDAL: Pick; webhookDAL: Pick; projectEnvDAL: Pick; projectDAL: TProjectDALFactory; From e263c95a14374689e2d37add8b3ee3166f53e7df Mon Sep 17 00:00:00 2001 From: Carlos Monastyrski Date: Wed, 3 Sep 2025 13:31:49 -0300 Subject: [PATCH 2/4] Extracted duplicated regex pattern to a constant --- backend/src/services/secret-import/secret-import-fns.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/src/services/secret-import/secret-import-fns.ts b/backend/src/services/secret-import/secret-import-fns.ts index e8bf123b5..e1e83be18 100644 --- a/backend/src/services/secret-import/secret-import-fns.ts +++ b/backend/src/services/secret-import/secret-import-fns.ts @@ -56,6 +56,7 @@ type TSecretImportSecretsV2 = { const LEVEL_BREAK = 10; const getImportUniqKey = (envSlug: string, path: string) => `${envSlug}=${path}`; +const RESERVED_IMPORT_REGEX = new RE2("/__reserve_replication_([a-f0-9-]{36})"); /** * Processes reserved imports by resolving them to their replication source. @@ -75,7 +76,7 @@ const processReservedImports = async < imports.forEach((secretImport) => { if (secretImport.isReserved) { - const reservedMatch = new RE2("/__reserve_replication_([a-f0-9-]{36})").exec(secretImport.importPath); + const reservedMatch = RESERVED_IMPORT_REGEX.exec(secretImport.importPath); if (reservedMatch) { const referencedImportId = reservedMatch[1]; reservedImportIds.push(referencedImportId); @@ -107,7 +108,7 @@ const processReservedImports = async < return imports.map((secretImport) => { if (secretImport.isReserved) { - const reservedMatch = new RE2("/__reserve_replication_([a-f0-9-]{36})").exec(secretImport.importPath); + const reservedMatch = RESERVED_IMPORT_REGEX.exec(secretImport.importPath); if (reservedMatch) { const referencedImportId = reservedMatch[1]; const referencedDetails = importDetailsMap.get(referencedImportId); From 48283d28260bff21f3bcb3458ac23169ebef6cc6 Mon Sep 17 00:00:00 2001 From: Carlos Monastyrski Date: Wed, 3 Sep 2025 14:23:30 -0300 Subject: [PATCH 3/4] Fix test to adapt to the new format of the response --- backend/e2e-test/routes/v3/secret-reference.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/e2e-test/routes/v3/secret-reference.spec.ts b/backend/e2e-test/routes/v3/secret-reference.spec.ts index 560565342..a5d8ba61d 100644 --- a/backend/e2e-test/routes/v3/secret-reference.spec.ts +++ b/backend/e2e-test/routes/v3/secret-reference.spec.ts @@ -314,8 +314,8 @@ describe("Secret expansion", () => { expect(listSecrets.imports).toEqual( expect.arrayContaining([ expect.objectContaining({ - secretPath: `/__reserve_replication_${secretImportFromProdToDev.id}`, - environment: seedData1.environment.slug, + secretPath: "/deep/nested", + environment: "prod", secrets: expect.arrayContaining([ expect.objectContaining({ secretKey: "NESTED_KEY_1", From 490fdd56018a3993e2b28b3222bc81a72bb2b435 Mon Sep 17 00:00:00 2001 From: Carlos Monastyrski Date: Fri, 5 Sep 2025 15:50:34 -0300 Subject: [PATCH 4/4] Improve db access to make one single query on imports new logic --- .../secret-replication-service.ts | 2 +- .../integration-delete-secret.ts | 4 ++-- .../integration/integration-service.ts | 2 +- .../secret-import/secret-import-dal.ts | 22 +++++++++++++++++++ .../secret-import/secret-import-fns.ts | 22 ++++++++----------- .../services/secret-sync/secret-sync-queue.ts | 2 +- .../secret-v2-bridge-service.ts | 2 +- backend/src/services/secret/secret-queue.ts | 2 +- 8 files changed, 38 insertions(+), 20 deletions(-) diff --git a/backend/src/ee/services/secret-replication/secret-replication-service.ts b/backend/src/ee/services/secret-replication/secret-replication-service.ts index f08c1560e..db41d00f7 100644 --- a/backend/src/ee/services/secret-replication/secret-replication-service.ts +++ b/backend/src/ee/services/secret-replication/secret-replication-service.ts @@ -59,7 +59,7 @@ type TSecretReplicationServiceFactoryDep = { TSecretVersionV2DALFactory, "find" | "insertMany" | "update" | "findLatestVersionMany" >; - secretImportDAL: Pick; + secretImportDAL: Pick; folderDAL: Pick< TSecretFolderDALFactory, "findSecretPathByFolderIds" | "findBySecretPath" | "create" | "findOne" | "findByManySecretPath" diff --git a/backend/src/services/integration-auth/integration-delete-secret.ts b/backend/src/services/integration-auth/integration-delete-secret.ts index ccab7f978..eaad08df8 100644 --- a/backend/src/services/integration-auth/integration-delete-secret.ts +++ b/backend/src/services/integration-auth/integration-delete-secret.ts @@ -39,7 +39,7 @@ const getIntegrationSecretsV2 = async ( }, secretV2BridgeDAL: Pick, folderDAL: Pick, - secretImportDAL: Pick + secretImportDAL: Pick ) => { const content: Record = {}; if (dto.depth > MAX_SYNC_SECRET_DEPTH) { @@ -300,7 +300,7 @@ export const deleteIntegrationSecrets = async ({ projectBotService: Pick; secretV2BridgeDAL: Pick; folderDAL: Pick; - secretImportDAL: Pick; + secretImportDAL: Pick; secretDAL: Pick; kmsService: Pick; }) => { diff --git a/backend/src/services/integration/integration-service.ts b/backend/src/services/integration/integration-service.ts index 498b99866..463ca12e3 100644 --- a/backend/src/services/integration/integration-service.ts +++ b/backend/src/services/integration/integration-service.ts @@ -40,7 +40,7 @@ type TIntegrationServiceFactoryDep = { projectBotService: TProjectBotServiceFactory; secretQueueService: Pick; secretV2BridgeDAL: Pick; - secretImportDAL: Pick; + secretImportDAL: Pick; kmsService: Pick; secretDAL: Pick; }; diff --git a/backend/src/services/secret-import/secret-import-dal.ts b/backend/src/services/secret-import/secret-import-dal.ts index dbe2f6a84..db611dc6c 100644 --- a/backend/src/services/secret-import/secret-import-dal.ts +++ b/backend/src/services/secret-import/secret-import-dal.ts @@ -127,6 +127,27 @@ export const secretImportDALFactory = (db: TDbClient) => { } }; + const findByIds = async (ids: string[], tx?: Knex) => { + try { + const docs = await (tx || db.replicaNode())(TableName.SecretImport) + .whereIn(`${TableName.SecretImport}.id`, ids) + .join(TableName.Environment, `${TableName.SecretImport}.importEnv`, `${TableName.Environment}.id`) + .select( + db.ref("*").withSchema(TableName.SecretImport) as unknown as keyof TSecretImports, + db.ref("slug").withSchema(TableName.Environment), + db.ref("name").withSchema(TableName.Environment), + db.ref("id").withSchema(TableName.Environment).as("envId") + ); + + return docs.map(({ envId, slug, name, ...el }) => ({ + ...el, + importEnv: { id: envId, slug, name } + })); + } catch (error) { + throw new DatabaseError({ error, name: "Find secret imports by ids" }); + } + }; + const getProjectImportCount = async ( { search, ...filter }: Partial, tx?: Knex @@ -325,6 +346,7 @@ export const secretImportDALFactory = (db: TDbClient) => { ...secretImportOrm, find, findById, + findByIds, findByFolderIds, findLastImportPosition, updateAllPosition, diff --git a/backend/src/services/secret-import/secret-import-fns.ts b/backend/src/services/secret-import/secret-import-fns.ts index e1e83be18..c739c5ad2 100644 --- a/backend/src/services/secret-import/secret-import-fns.ts +++ b/backend/src/services/secret-import/secret-import-fns.ts @@ -70,7 +70,7 @@ const processReservedImports = async < } >( imports: T[], - secretImportDAL: Pick + secretImportDAL: Pick ): Promise => { const reservedImportIds: string[] = []; @@ -94,17 +94,13 @@ const processReservedImports = async < { importPath: string; importEnv: { id: string; slug: string; name: string } } >(); - /* eslint-disable no-await-in-loop */ - for (const importId of reservedImportIds) { - const referencedImport = await secretImportDAL.findById(importId); - if (referencedImport) { - importDetailsMap.set(importId, { - importPath: referencedImport.importPath, - importEnv: referencedImport.importEnv - }); - } - } - /* eslint-enable no-await-in-loop */ + const referencedImports = await secretImportDAL.findByIds(reservedImportIds); + referencedImports.forEach((referencedImport) => { + importDetailsMap.set(referencedImport.id, { + importPath: referencedImport.importPath, + importEnv: referencedImport.importEnv + }); + }); return imports.map((secretImport) => { if (secretImport.isReserved) { @@ -241,7 +237,7 @@ export const fnSecretsV2FromImports = async ({ folderDAL: Pick; viewSecretValue: boolean; secretDAL: Pick; - secretImportDAL: Pick; + secretImportDAL: Pick; decryptor: (value?: Buffer | null) => string; expandSecretReferences?: (inputSecret: { value?: string; diff --git a/backend/src/services/secret-sync/secret-sync-queue.ts b/backend/src/services/secret-sync/secret-sync-queue.ts index 50c0b9dca..1a911710f 100644 --- a/backend/src/services/secret-sync/secret-sync-queue.ts +++ b/backend/src/services/secret-sync/secret-sync-queue.ts @@ -80,7 +80,7 @@ type TSecretSyncQueueFactoryDep = { | "deleteMany" | "invalidateSecretCacheByProjectId" >; - secretImportDAL: Pick; + secretImportDAL: Pick; secretSyncDAL: Pick; auditLogService: Pick; projectMembershipDAL: Pick; diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts index 4b05582c1..48ab07816 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts @@ -108,7 +108,7 @@ type TSecretV2BridgeServiceFactoryDep = { | "findBySecretPathMultiEnv" | "findSecretPathByFolderIds" >; - secretImportDAL: Pick; + secretImportDAL: Pick; secretQueueService: Pick; secretApprovalPolicyService: Pick; secretApprovalRequestDAL: Pick; diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index 5160c4bf3..ed794a664 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -86,7 +86,7 @@ type TSecretQueueFactoryDep = { integrationAuthService: Pick; folderDAL: TSecretFolderDALFactory; secretDAL: TSecretDALFactory; - secretImportDAL: Pick; + secretImportDAL: Pick; webhookDAL: Pick; projectEnvDAL: Pick; projectDAL: TProjectDALFactory;