diff --git a/backend/e2e-test/routes/v3/secret-reference.spec.ts b/backend/e2e-test/routes/v3/secret-reference.spec.ts index 560565342..a5d8ba61d 100644 --- a/backend/e2e-test/routes/v3/secret-reference.spec.ts +++ b/backend/e2e-test/routes/v3/secret-reference.spec.ts @@ -314,8 +314,8 @@ describe("Secret expansion", () => { expect(listSecrets.imports).toEqual( expect.arrayContaining([ expect.objectContaining({ - secretPath: `/__reserve_replication_${secretImportFromProdToDev.id}`, - environment: seedData1.environment.slug, + secretPath: "/deep/nested", + environment: "prod", secrets: expect.arrayContaining([ expect.objectContaining({ secretKey: "NESTED_KEY_1", diff --git a/backend/src/ee/services/secret-replication/secret-replication-service.ts b/backend/src/ee/services/secret-replication/secret-replication-service.ts index 4a5558f46..db41d00f7 100644 --- a/backend/src/ee/services/secret-replication/secret-replication-service.ts +++ b/backend/src/ee/services/secret-replication/secret-replication-service.ts @@ -59,7 +59,7 @@ type TSecretReplicationServiceFactoryDep = { TSecretVersionV2DALFactory, "find" | "insertMany" | "update" | "findLatestVersionMany" >; - secretImportDAL: Pick; + secretImportDAL: Pick; folderDAL: Pick< TSecretFolderDALFactory, "findSecretPathByFolderIds" | "findBySecretPath" | "create" | "findOne" | "findByManySecretPath" diff --git a/backend/src/services/integration-auth/integration-delete-secret.ts b/backend/src/services/integration-auth/integration-delete-secret.ts index 2efec7a47..eaad08df8 100644 --- a/backend/src/services/integration-auth/integration-delete-secret.ts +++ b/backend/src/services/integration-auth/integration-delete-secret.ts @@ -39,7 +39,7 @@ const getIntegrationSecretsV2 = async ( }, secretV2BridgeDAL: Pick, folderDAL: Pick, - secretImportDAL: Pick + secretImportDAL: Pick ) => { const content: Record = {}; if (dto.depth > MAX_SYNC_SECRET_DEPTH) { @@ -300,7 +300,7 @@ export const deleteIntegrationSecrets = async ({ projectBotService: Pick; secretV2BridgeDAL: Pick; folderDAL: Pick; - secretImportDAL: Pick; + secretImportDAL: Pick; secretDAL: Pick; kmsService: Pick; }) => { diff --git a/backend/src/services/integration/integration-service.ts b/backend/src/services/integration/integration-service.ts index 2ef8615eb..463ca12e3 100644 --- a/backend/src/services/integration/integration-service.ts +++ b/backend/src/services/integration/integration-service.ts @@ -40,7 +40,7 @@ type TIntegrationServiceFactoryDep = { projectBotService: TProjectBotServiceFactory; secretQueueService: Pick; secretV2BridgeDAL: Pick; - secretImportDAL: Pick; + secretImportDAL: Pick; kmsService: Pick; secretDAL: Pick; }; diff --git a/backend/src/services/secret-import/secret-import-dal.ts b/backend/src/services/secret-import/secret-import-dal.ts index dbe2f6a84..db611dc6c 100644 --- a/backend/src/services/secret-import/secret-import-dal.ts +++ b/backend/src/services/secret-import/secret-import-dal.ts @@ -127,6 +127,27 @@ export const secretImportDALFactory = (db: TDbClient) => { } }; + const findByIds = async (ids: string[], tx?: Knex) => { + try { + const docs = await (tx || db.replicaNode())(TableName.SecretImport) + .whereIn(`${TableName.SecretImport}.id`, ids) + .join(TableName.Environment, `${TableName.SecretImport}.importEnv`, `${TableName.Environment}.id`) + .select( + db.ref("*").withSchema(TableName.SecretImport) as unknown as keyof TSecretImports, + db.ref("slug").withSchema(TableName.Environment), + db.ref("name").withSchema(TableName.Environment), + db.ref("id").withSchema(TableName.Environment).as("envId") + ); + + return docs.map(({ envId, slug, name, ...el }) => ({ + ...el, + importEnv: { id: envId, slug, name } + })); + } catch (error) { + throw new DatabaseError({ error, name: "Find secret imports by ids" }); + } + }; + const getProjectImportCount = async ( { search, ...filter }: Partial, tx?: Knex @@ -325,6 +346,7 @@ export const secretImportDALFactory = (db: TDbClient) => { ...secretImportOrm, find, findById, + findByIds, findByFolderIds, findLastImportPosition, updateAllPosition, diff --git a/backend/src/services/secret-import/secret-import-fns.ts b/backend/src/services/secret-import/secret-import-fns.ts index 6aa73465d..c739c5ad2 100644 --- a/backend/src/services/secret-import/secret-import-fns.ts +++ b/backend/src/services/secret-import/secret-import-fns.ts @@ -1,3 +1,5 @@ +import RE2 from "re2"; + import { SecretType, TSecretImports, TSecrets, TSecretsV2 } from "@app/db/schemas"; import { groupBy, unique } from "@app/lib/fn"; @@ -54,6 +56,74 @@ type TSecretImportSecretsV2 = { const LEVEL_BREAK = 10; const getImportUniqKey = (envSlug: string, path: string) => `${envSlug}=${path}`; +const RESERVED_IMPORT_REGEX = new RE2("/__reserve_replication_([a-f0-9-]{36})"); + +/** + * Processes reserved imports by resolving them to their replication source. + */ +const processReservedImports = async < + T extends { + isReserved?: boolean | null; + importPath: string; + importEnv: { id: string; slug: string; name: string }; + folderId: string; + } +>( + imports: T[], + secretImportDAL: Pick +): Promise => { + const reservedImportIds: string[] = []; + + imports.forEach((secretImport) => { + if (secretImport.isReserved) { + const reservedMatch = RESERVED_IMPORT_REGEX.exec(secretImport.importPath); + if (reservedMatch) { + const referencedImportId = reservedMatch[1]; + reservedImportIds.push(referencedImportId); + } + } + }); + + if (reservedImportIds.length === 0) { + return imports; + } + + try { + const importDetailsMap = new Map< + string, + { importPath: string; importEnv: { id: string; slug: string; name: string } } + >(); + + const referencedImports = await secretImportDAL.findByIds(reservedImportIds); + referencedImports.forEach((referencedImport) => { + importDetailsMap.set(referencedImport.id, { + importPath: referencedImport.importPath, + importEnv: referencedImport.importEnv + }); + }); + + return imports.map((secretImport) => { + if (secretImport.isReserved) { + const reservedMatch = RESERVED_IMPORT_REGEX.exec(secretImport.importPath); + if (reservedMatch) { + const referencedImportId = reservedMatch[1]; + const referencedDetails = importDetailsMap.get(referencedImportId); + + if (referencedDetails) { + return { + ...secretImport, + importPath: referencedDetails.importPath, + importEnv: referencedDetails.importEnv + }; + } + } + } + return secretImport; + }); + } catch (error) { + return imports; + } +}; export const fnSecretsFromImports = async ({ allowedImports: possibleCyclicImports, folderDAL, @@ -167,7 +237,7 @@ export const fnSecretsV2FromImports = async ({ folderDAL: Pick; viewSecretValue: boolean; secretDAL: Pick; - secretImportDAL: Pick; + secretImportDAL: Pick; decryptor: (value?: Buffer | null) => string; expandSecretReferences?: (inputSecret: { value?: string; @@ -188,6 +258,10 @@ export const fnSecretsV2FromImports = async ({ })[]; }[] = [{ secretImports: rootSecretImports, depth: 0, parentImportedSecrets: [] }]; + const processedSecretImports = await processReservedImports(rootSecretImports, secretImportDAL); + + stack[0] = { secretImports: processedSecretImports, depth: 0, parentImportedSecrets: [] }; + const processedImports: TSecretImportSecretsV2[] = []; while (stack.length) { diff --git a/backend/src/services/secret-sync/secret-sync-queue.ts b/backend/src/services/secret-sync/secret-sync-queue.ts index 797e1dc1f..915ff89bf 100644 --- a/backend/src/services/secret-sync/secret-sync-queue.ts +++ b/backend/src/services/secret-sync/secret-sync-queue.ts @@ -80,7 +80,7 @@ type TSecretSyncQueueFactoryDep = { | "deleteMany" | "invalidateSecretCacheByProjectId" >; - secretImportDAL: Pick; + secretImportDAL: Pick; secretSyncDAL: Pick; auditLogService: Pick; projectMembershipDAL: Pick; diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts index b23cd0c56..48ab07816 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts @@ -108,7 +108,7 @@ type TSecretV2BridgeServiceFactoryDep = { | "findBySecretPathMultiEnv" | "findSecretPathByFolderIds" >; - secretImportDAL: Pick; + secretImportDAL: Pick; secretQueueService: Pick; secretApprovalPolicyService: Pick; secretApprovalRequestDAL: Pick; diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index dc5713901..ed794a664 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -86,7 +86,7 @@ type TSecretQueueFactoryDep = { integrationAuthService: Pick; folderDAL: TSecretFolderDALFactory; secretDAL: TSecretDALFactory; - secretImportDAL: Pick; + secretImportDAL: Pick; webhookDAL: Pick; projectEnvDAL: Pick; projectDAL: TProjectDALFactory;