diff --git a/backend-pg/src/server/routes/index.ts b/backend-pg/src/server/routes/index.ts index 195c0fe07..7544a67c4 100644 --- a/backend-pg/src/server/routes/index.ts +++ b/backend-pg/src/server/routes/index.ts @@ -338,14 +338,14 @@ export const registerRoutes = async ( }); const secretQueueService = secretQueueFactory({ queueService, - webhookService, secretDal, folderDal, - secretImportService, integrationAuthService, projectBotService, integrationDal, - secretImportDal + secretImportDal, + projectEnvDal, + webhookDal }); const secretService = secretServiceFactory({ folderDal, @@ -355,7 +355,9 @@ export const registerRoutes = async ( secretDal, secretTagDal, snapshotService, - secretQueueService + secretQueueService, + secretImportDal, + projectBotService }); const sarService = secretApprovalRequestServiceFactory({ permissionService, diff --git a/backend-pg/src/server/routes/sanitizedSchemas.ts b/backend-pg/src/server/routes/sanitizedSchemas.ts index a81faebf2..6abe65785 100644 --- a/backend-pg/src/server/routes/sanitizedSchemas.ts +++ b/backend-pg/src/server/routes/sanitizedSchemas.ts @@ -27,3 +27,13 @@ export const sapPubSchema = SecretApprovalPoliciesSchema.merge( projectId: z.string() }) ); + +export const secretRawSchema = z.object({ + id: z.string(), + _id: z.string(), + version: z.number(), + type: z.string(), + secretKey: z.string(), + secretValue: z.string(), + secretComment: z.string().optional() +}); diff --git a/backend-pg/src/server/routes/v3/secret-router.ts b/backend-pg/src/server/routes/v3/secret-router.ts index 0cbc683e3..881895090 100644 --- a/backend-pg/src/server/routes/v3/secret-router.ts +++ b/backend-pg/src/server/routes/v3/secret-router.ts @@ -11,7 +11,308 @@ import { CommitType } from "@app/ee/services/secret-approval-request/secret-appr import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { ActorType, AuthMode } from "@app/services/auth/auth-type"; +import { secretRawSchema } from "../sanitizedSchemas"; + export const registerSecretRouter = async (server: FastifyZodProvider) => { + server.route({ + url: "/raw", + method: "GET", + schema: { + querystring: z.object({ + workspaceId: z.string().trim(), + environment: z.string().trim(), + secretPath: z.string().trim().default("/"), + include_imports: z + .enum(["true", "false"]) + .default("false") + .transform((value) => value === "true") + }), + response: { + 200: z.object({ + secrets: secretRawSchema.array(), + imports: z + .object({ + secretPath: z.string(), + environment: z.object({ + id: z.string(), + name: z.string(), + slug: z.string() + }), + folderId: z.string().optional(), + secrets: secretRawSchema.array() + }) + .array() + .optional() + }) + } + }, + onRequest: verifyAuth([ + AuthMode.JWT, + AuthMode.API_KEY, + AuthMode.SERVICE_TOKEN, + AuthMode.IDENTITY_ACCESS_TOKEN + ]), + handler: async (req) => { + const { secrets, imports } = await server.services.secret.getSecretsRaw({ + actorId: req.permission.id, + actor: req.permission.type, + environment: req.query.environment, + projectId: req.query.workspaceId, + path: req.query.secretPath, + includeImports: req.query.include_imports + }); + + await server.services.auditLog.createAuditLog({ + projectId: req.query.workspaceId, + ...req.auditLogInfo, + event: { + type: EventType.GET_SECRETS, + metadata: { + environment: req.query.environment, + secretPath: req.query.secretPath, + numberOfSecrets: secrets.length + } + } + }); + return { secrets, imports }; + } + }); + + server.route({ + url: "/raw/:secretName", + method: "GET", + schema: { + params: z.object({ + secretName: z.string().trim() + }), + querystring: z.object({ + workspaceId: z.string().trim(), + environment: z.string().trim(), + secretPath: z.string().trim().default("/"), + type: z.nativeEnum(SecretType).default(SecretType.Shared), + include_imports: z + .enum(["true", "false"]) + .default("false") + .transform((value) => value === "true") + }), + response: { + 200: z.object({ + secret: secretRawSchema + }) + } + }, + onRequest: verifyAuth([ + AuthMode.JWT, + AuthMode.API_KEY, + AuthMode.SERVICE_TOKEN, + AuthMode.IDENTITY_ACCESS_TOKEN + ]), + handler: async (req) => { + const secret = await server.services.secret.getASecretRaw({ + actorId: req.permission.id, + actor: req.permission.type, + environment: req.query.environment, + projectId: req.query.workspaceId, + path: req.query.secretPath, + secretName: req.params.secretName, + type: req.query.type, + includeImports: req.query.include_imports + }); + + await server.services.auditLog.createAuditLog({ + projectId: req.query.workspaceId, + ...req.auditLogInfo, + event: { + type: EventType.GET_SECRET, + metadata: { + environment: req.query.environment, + secretPath: req.query.secretPath, + secretId: secret.id, + secretKey: req.params.secretName, + secretVersion: secret.version + } + } + }); + return { secret }; + } + }); + + server.route({ + url: "/raw/:secretName", + method: "POST", + schema: { + params: z.object({ + secretName: z.string().trim() + }), + body: z.object({ + workspaceId: z.string().trim(), + environment: z.string().trim(), + secretPath: z.string().trim().default("/"), + secretValue: z + .string() + .transform((val) => (val.at(-1) === "\n" ? `${val.trim()}\n` : val.trim())), + secretComment: z.string().trim().optional().default(""), + skipMultilineEncoding: z.boolean().optional(), + type: z.nativeEnum(SecretType).default(SecretType.Shared) + }), + response: { + 200: z.object({ + secret: secretRawSchema + }) + } + }, + onRequest: verifyAuth([ + AuthMode.JWT, + AuthMode.API_KEY, + AuthMode.SERVICE_TOKEN, + AuthMode.IDENTITY_ACCESS_TOKEN + ]), + handler: async (req) => { + const secret = await server.services.secret.createSecretRaw({ + actorId: req.permission.id, + actor: req.permission.type, + environment: req.body.environment, + projectId: req.body.workspaceId, + secretPath: req.body.secretPath, + secretName: req.params.secretName, + type: req.body.type, + secretValue: req.body.secretValue, + skipMultilineEncoding: req.body.skipMultilineEncoding, + secretComment: req.body.secretComment + }); + + await server.services.auditLog.createAuditLog({ + projectId: req.body.workspaceId, + ...req.auditLogInfo, + event: { + type: EventType.CREATE_SECRET, + metadata: { + environment: req.body.environment, + secretPath: req.body.secretPath, + secretId: secret.id, + secretKey: req.params.secretName, + secretVersion: secret.version + } + } + }); + return { secret }; + } + }); + + server.route({ + url: "/raw/:secretName", + method: "PATCH", + schema: { + params: z.object({ + secretName: z.string().trim() + }), + body: z.object({ + workspaceId: z.string().trim(), + environment: z.string().trim(), + secretValue: z + .string() + .transform((val) => (val.at(-1) === "\n" ? `${val.trim()}\n` : val.trim())), + secretPath: z.string().trim().default("/"), + skipMultilineEncoding: z.boolean().optional(), + type: z.nativeEnum(SecretType).default(SecretType.Shared) + }), + response: { + 200: z.object({ + secret: secretRawSchema + }) + } + }, + onRequest: verifyAuth([ + AuthMode.JWT, + AuthMode.API_KEY, + AuthMode.SERVICE_TOKEN, + AuthMode.IDENTITY_ACCESS_TOKEN + ]), + handler: async (req) => { + const secret = await server.services.secret.updateSecretRaw({ + actorId: req.permission.id, + actor: req.permission.type, + environment: req.body.environment, + projectId: req.body.workspaceId, + secretPath: req.body.secretPath, + secretName: req.params.secretName, + type: req.body.type, + secretValue: req.body.secretValue, + skipMultilineEncoding: req.body.skipMultilineEncoding + }); + + await server.services.auditLog.createAuditLog({ + projectId: req.body.workspaceId, + ...req.auditLogInfo, + event: { + type: EventType.UPDATE_SECRET, + metadata: { + environment: req.body.environment, + secretPath: req.body.secretPath, + secretId: secret.id, + secretKey: req.params.secretName, + secretVersion: secret.version + } + } + }); + return { secret }; + } + }); + + server.route({ + url: "/raw/:secretName", + method: "DELETE", + schema: { + params: z.object({ + secretName: z.string().trim() + }), + body: z.object({ + workspaceId: z.string().trim(), + environment: z.string().trim(), + secretPath: z.string().trim().default("/"), + type: z.nativeEnum(SecretType).default(SecretType.Shared) + }), + response: { + 200: z.object({ + secret: secretRawSchema + }) + } + }, + onRequest: verifyAuth([ + AuthMode.JWT, + AuthMode.API_KEY, + AuthMode.SERVICE_TOKEN, + AuthMode.IDENTITY_ACCESS_TOKEN + ]), + handler: async (req) => { + const secret = await server.services.secret.deleteSecretRaw({ + actorId: req.permission.id, + actor: req.permission.type, + environment: req.body.environment, + projectId: req.body.workspaceId, + secretPath: req.body.secretPath, + secretName: req.params.secretName, + type: req.body.type + }); + + await server.services.auditLog.createAuditLog({ + projectId: req.body.workspaceId, + ...req.auditLogInfo, + event: { + type: EventType.DELETE_SECRET, + metadata: { + environment: req.body.environment, + secretPath: req.body.secretPath, + secretId: secret.id, + secretKey: req.params.secretName, + secretVersion: secret.version + } + } + }); + return { secret }; + } + }); + server.route({ url: "/", method: "GET", @@ -38,7 +339,20 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { }).array() }) ) + .array(), + imports: z + .object({ + secretPath: z.string(), + environment: z.object({ + id: z.string(), + name: z.string(), + slug: z.string() + }), + folderId: z.string().optional(), + secrets: SecretsSchema.omit({ secretBlindIndex: true }).array() + }) .array() + .optional() }) } }, @@ -49,12 +363,13 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { AuthMode.IDENTITY_ACCESS_TOKEN ]), handler: async (req) => { - const secrets = await server.services.secret.getSecrets({ + const { secrets, imports } = await server.services.secret.getSecrets({ actorId: req.permission.id, actor: req.permission.type, environment: req.query.environment, projectId: req.query.workspaceId, - path: req.query.secretPath + path: req.query.secretPath, + includeImports: req.query.include_imports }); await server.services.auditLog.createAuditLog({ @@ -70,7 +385,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } }); - return { secrets }; + return { secrets, imports }; } }); @@ -111,7 +426,8 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { projectId: req.query.workspaceId, path: req.query.secretPath, secretName: req.params.secretName, - type: req.query.type + type: req.query.type, + includeImports: req.query.include_imports }); await server.services.auditLog.createAuditLog({ diff --git a/backend-pg/src/services/secret-import/secret-import-fns.ts b/backend-pg/src/services/secret-import/secret-import-fns.ts new file mode 100644 index 000000000..2f99adb58 --- /dev/null +++ b/backend-pg/src/services/secret-import/secret-import-fns.ts @@ -0,0 +1,42 @@ +import { SecretType, TSecretImports } from "@app/db/schemas"; +import { groupBy } from "@app/lib/fn"; + +import { TSecretDalFactory } from "../secret/secret-dal"; +import { TSecretFolderDalFactory } from "../secret-folder/secret-folder-dal"; + +export const fnSecretsFromImports = async ({ + allowedImports, + folderDal, + secretDal +}: { + allowedImports: (Omit & { + importEnv: { id: string; slug: string; name: string }; + })[]; + folderDal: Pick; + secretDal: Pick; +}) => { + const importedFolders = await folderDal.findByManySecretPath( + allowedImports.map(({ importEnv, importPath }) => ({ + envId: importEnv.id, + secretPath: importPath + })) + ); + const folderIds = importedFolders.map((el) => el?.id).filter(Boolean) as string[]; + if (!folderIds.length) { + return []; + } + const importedSecrets = await secretDal.find({ + $in: { folderId: folderIds }, + type: SecretType.Shared + }); + + const importedSecsGroupByFolderId = groupBy(importedSecrets, (i) => i.folderId); + return allowedImports.map(({ importPath, importEnv }, i) => ({ + secretPath: importPath, + environment: importEnv, + folderId: importedFolders?.[i]?.id, + secrets: importedFolders?.[i]?.id + ? importedSecsGroupByFolderId[importedFolders?.[i]?.id as string] + : [] + })); +}; diff --git a/backend-pg/src/services/secret-import/secret-import-service.ts b/backend-pg/src/services/secret-import/secret-import-service.ts index 07a2c4f04..75de3f54b 100644 --- a/backend-pg/src/services/secret-import/secret-import-service.ts +++ b/backend-pg/src/services/secret-import/secret-import-service.ts @@ -1,18 +1,17 @@ import { ForbiddenError, subject } from "@casl/ability"; -import { SecretType, TSecretImports } from "@app/db/schemas"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; import { BadRequestError } from "@app/lib/errors"; -import { groupBy } from "@app/lib/fn"; import { TProjectEnvDalFactory } from "../project-env/project-env-dal"; import { TSecretDalFactory } from "../secret/secret-dal"; import { TSecretFolderDalFactory } from "../secret-folder/secret-folder-dal"; import { TSecretImportDalFactory } from "./secret-import-dal"; +import { fnSecretsFromImports } from "./secret-import-fns"; import { TCreateSecretImportDTO, TDeleteSecretImportDTO, @@ -176,37 +175,6 @@ export const secretImportServiceFactory = ({ return secImports; }; - const fnSecretsFromImports = async ( - allowedImports: (Omit & { - importEnv: { id: string; slug: string; name: string }; - })[] - ) => { - const importedFolders = await folderDal.findByManySecretPath( - allowedImports.map(({ importEnv, importPath }) => ({ - envId: importEnv.id, - secretPath: importPath - })) - ); - const folderIds = importedFolders.map((el) => el?.id).filter(Boolean) as string[]; - if (!folderIds.length) { - return []; - } - const importedSecrets = await secretDal.find({ - $in: { folderId: folderIds }, - type: SecretType.Shared - }); - - const importedSecsGroupByFolderId = groupBy(importedSecrets, (i) => i.folderId); - return allowedImports.map(({ importPath, importEnv }, i) => ({ - secretPath: importPath, - environment: importEnv, - folderId: importedFolders?.[i]?.id, - secrets: importedFolders?.[i]?.id - ? importedSecsGroupByFolderId[importedFolders?.[i]?.id as string] - : [] - })); - }; - const getSecretsFromImports = async ({ path, environment, @@ -234,7 +202,7 @@ export const secretImportServiceFactory = ({ }) ) ); - return fnSecretsFromImports(allowedImports); + return fnSecretsFromImports({ allowedImports, folderDal, secretDal }); }; return { diff --git a/backend-pg/src/services/secret/secret-fns.ts b/backend-pg/src/services/secret/secret-fns.ts index 03859cdc0..58bb9680e 100644 --- a/backend-pg/src/services/secret/secret-fns.ts +++ b/backend-pg/src/services/secret/secret-fns.ts @@ -1,11 +1,30 @@ /* eslint-disable no-await-in-loop */ import path from "path"; -import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; +import { SecretKeyEncoding, TSecretBlindIndexes, TSecrets } from "@app/db/schemas"; +import { getConfig } from "@app/lib/config/env"; +import { buildSecretBlindIndexFromName, decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; import { TSecretFolderDalFactory } from "../secret-folder/secret-folder-dal"; import { TSecretDalFactory } from "./secret-dal"; +export const generateSecretBlindIndexBySalt = async ( + secretName: string, + secretBlindIndexDoc: TSecretBlindIndexes +) => { + const appCfg = getConfig(); + const secretBlindIndex = await buildSecretBlindIndexFromName({ + secretName, + keyEncoding: secretBlindIndexDoc.keyEncoding as SecretKeyEncoding, + rootEncryptionKey: appCfg.ROOT_ENCRYPTION_KEY, + encryptionKey: appCfg.ENCRYPTION_KEY, + tag: secretBlindIndexDoc.saltTag, + ciphertext: secretBlindIndexDoc.encryptedSaltCipherText, + iv: secretBlindIndexDoc.saltIV + }); + return secretBlindIndex; +}; + type TInterpolateSecretArg = { projectId: string; secretEncKey: string; @@ -177,3 +196,41 @@ export const interpolateSecrets = ({ }; return expandSecrets; }; + +export const decryptSecretRaw = (secret: TSecrets, key: string) => { + const secretKey = decryptSymmetric128BitHexKeyUTF8({ + ciphertext: secret.secretKeyCiphertext, + iv: secret.secretKeyIV, + tag: secret.secretKeyTag, + key + }); + + const secretValue = decryptSymmetric128BitHexKeyUTF8({ + ciphertext: secret.secretValueCiphertext, + iv: secret.secretValueIV, + tag: secret.secretValueTag, + key + }); + + let secretComment = ""; + + if (secret.secretCommentCiphertext && secret.secretCommentIV && secret.secretCommentTag) { + secretComment = decryptSymmetric128BitHexKeyUTF8({ + ciphertext: secret.secretCommentCiphertext, + iv: secret.secretCommentIV, + tag: secret.secretCommentTag, + key + }); + } + + return { + secretKey, + secretValue, + secretComment, + version: secret.version, + type: secret.type, + _id: secret.id, + id: secret.id, + user: secret.userId + }; +}; diff --git a/backend-pg/src/services/secret/secret-queue.ts b/backend-pg/src/services/secret/secret-queue.ts index 2c651c92b..ed6096427 100644 --- a/backend-pg/src/services/secret/secret-queue.ts +++ b/backend-pg/src/services/secret/secret-queue.ts @@ -8,10 +8,12 @@ import { TIntegrationDalFactory } from "../integration/integration-dal"; import { TIntegrationAuthServiceFactory } from "../integration-auth/integration-auth-service"; import { syncIntegrationSecrets } from "../integration-auth/integration-sync-secret"; import { TProjectBotServiceFactory } from "../project-bot/project-bot-service"; +import { TProjectEnvDalFactory } from "../project-env/project-env-dal"; import { TSecretFolderDalFactory } from "../secret-folder/secret-folder-dal"; import { TSecretImportDalFactory } from "../secret-import/secret-import-dal"; -import { TSecretImportServiceFactory } from "../secret-import/secret-import-service"; -import { TWebhookServiceFactory } from "../webhook/webhook-service"; +import { fnSecretsFromImports } from "../secret-import/secret-import-fns"; +import { TWebhookDalFactory } from "../webhook/webhook-dal"; +import { fnTriggerWebhook } from "../webhook/webhook-fns"; import { TSecretDalFactory } from "./secret-dal"; import { interpolateSecrets } from "./secret-fns"; @@ -19,14 +21,14 @@ export type TSecretQueueFactory = ReturnType; type TSecretQueueFactoryDep = { queueService: TQueueServiceFactory; - webhookService: Pick; integrationDal: Pick; projectBotService: Pick; integrationAuthService: Pick; - folderDal: Pick; - secretDal: Pick; + folderDal: Pick; + secretDal: Pick; secretImportDal: Pick; - secretImportService: Pick; + webhookDal: Pick; + projectEnvDal: Pick; }; export type TGetSecrets = { @@ -37,14 +39,14 @@ export type TGetSecrets = { export const secretQueueFactory = ({ queueService, - webhookService, integrationDal, projectBotService, integrationAuthService, secretDal, secretImportDal, - secretImportService, - folderDal + folderDal, + webhookDal, + projectEnvDal }: TSecretQueueFactoryDep) => { const syncSecrets = async (dto: TGetSecrets) => { queueService.queue(QueueName.SecretWebhook, QueueJobs.SecWebhook, dto, { @@ -79,7 +81,11 @@ export const secretQueueFactory = ({ // get imported secrets const secretImport = await secretImportDal.find({ folderId: dto.folderId }); - const importedSecrets = await secretImportService.fnSecretsFromImports(secretImport); + const importedSecrets = await fnSecretsFromImports({ + allowedImports: secretImport, + secretDal, + folderDal + }); const content: Record< string, { value: string; comment?: string; skipMultilineEncoding?: boolean } @@ -217,7 +223,7 @@ export const secretQueueFactory = ({ queueService.start(QueueName.SecretWebhook, async (job) => { logger.info("Secret webhook job started", job.data, job.id); - await webhookService.fnTriggerWebhook(job.data); + await fnTriggerWebhook({ ...job.data, projectEnvDal, webhookDal }); logger.info("Secret webhook job ended", job.id); }); diff --git a/backend-pg/src/services/secret/secret-service.ts b/backend-pg/src/services/secret/secret-service.ts index 087248f13..9287fe8c8 100644 --- a/backend-pg/src/services/secret/secret-service.ts +++ b/backend-pg/src/services/secret/secret-service.ts @@ -1,12 +1,6 @@ import { ForbiddenError, subject } from "@casl/ability"; -import { - SecretEncryptionAlgo, - SecretKeyEncoding, - SecretType, - TableName, - TSecretBlindIndexes -} from "@app/db/schemas"; +import { SecretEncryptionAlgo, SecretKeyEncoding, SecretType, TableName } from "@app/db/schemas"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, @@ -14,31 +8,40 @@ import { } from "@app/ee/services/permission/project-permission"; import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service"; import { getConfig } from "@app/lib/config/env"; -import { buildSecretBlindIndexFromName } from "@app/lib/crypto"; +import { buildSecretBlindIndexFromName, encryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; import { BadRequestError } from "@app/lib/errors"; import { groupBy, pick } from "@app/lib/fn"; import { ActorType } from "../auth/auth-type"; +import { TProjectBotServiceFactory } from "../project-bot/project-bot-service"; import { TSecretFolderDalFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretImportDalFactory } from "../secret-import/secret-import-dal"; +import { fnSecretsFromImports } from "../secret-import/secret-import-fns"; import { TSecretTagDalFactory } from "../secret-tag/secret-tag-dal"; import { TSecretBlindIndexDalFactory } from "./secret-blind-index-dal"; import { TSecretDalFactory } from "./secret-dal"; +import { decryptSecretRaw, generateSecretBlindIndexBySalt } from "./secret-fns"; import { TSecretQueueFactory } from "./secret-queue"; import { TCreateBulkSecretDTO, TCreateSecretDTO, + TCreateSecretRawDTO, TDeleteBulkSecretDTO, TDeleteSecretDTO, + TDeleteSecretRawDTO, TFnSecretBlindIndexCheck, TFnSecretBlindIndexCheckV2, TFnSecretBulkDelete, TFnSecretBulkInsert, TFnSecretBulkUpdate, TGetASecretDTO, + TGetASecretRawDTO, TGetSecretsDTO, + TGetSecretsRawDTO, TListSecretVersionDTO, TUpdateBulkSecretDTO, - TUpdateSecretDTO + TUpdateSecretDTO, + TUpdateSecretRawDTO } from "./secret-types"; import { TSecretVersionDalFactory } from "./secret-version-dal"; @@ -46,32 +49,19 @@ type TSecretServiceFactoryDep = { secretDal: TSecretDalFactory; secretTagDal: TSecretTagDalFactory; secretVersionDal: TSecretVersionDalFactory; - folderDal: Pick; + folderDal: Pick< + TSecretFolderDalFactory, + "findBySecretPath" | "updateById" | "findById" | "findByManySecretPath" + >; secretBlindIndexDal: TSecretBlindIndexDalFactory; permissionService: Pick; snapshotService: Pick; secretQueueService: Pick; + projectBotService: Pick; + secretImportDal: Pick; }; export type TSecretServiceFactory = ReturnType; - -export const generateSecretBlindIndexBySalt = async ( - secretName: string, - secretBlindIndexDoc: TSecretBlindIndexes -) => { - const appCfg = getConfig(); - const secretBlindIndex = await buildSecretBlindIndexFromName({ - secretName, - keyEncoding: secretBlindIndexDoc.keyEncoding as SecretKeyEncoding, - rootEncryptionKey: appCfg.ROOT_ENCRYPTION_KEY, - encryptionKey: appCfg.ENCRYPTION_KEY, - tag: secretBlindIndexDoc.saltTag, - ciphertext: secretBlindIndexDoc.encryptedSaltCipherText, - iv: secretBlindIndexDoc.saltIV - }); - return secretBlindIndex; -}; - export const secretServiceFactory = ({ secretDal, secretTagDal, @@ -80,7 +70,9 @@ export const secretServiceFactory = ({ secretBlindIndexDal, permissionService, snapshotService, - secretQueueService + secretQueueService, + projectBotService, + secretImportDal }: TSecretServiceFactoryDep) => { // utility function to get secret blind index data const interalGenSecBlindIndexByName = async (projectId: string, secretName: string) => { @@ -485,7 +477,14 @@ export const secretServiceFactory = ({ return deletedSecret[0]; }; - const getSecrets = async ({ actorId, path, environment, projectId, actor }: TGetSecretsDTO) => { + const getSecrets = async ({ + actorId, + path, + environment, + projectId, + actor, + includeImports + }: TGetSecretsDTO) => { const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId); ForbiddenError.from(permission).throwUnlessCan( ProjectPermissionActions.Read, @@ -497,7 +496,28 @@ export const secretServiceFactory = ({ const folderId = folder.id; const secrets = await secretDal.findByFolderId(folderId, actorId); - return secrets; + if (includeImports) { + const secretImports = await secretImportDal.find({ folderId }); + const allowedImports = secretImports.filter(({ importEnv, importPath }) => + // if its service token allow full access over imported one + actor === ActorType.SERVICE + ? true + : permission.can( + ProjectPermissionActions.Read, + subject(ProjectPermissionSub.Secrets, { + environment: importEnv.slug, + secretPath: importPath + }) + ) + ); + const importedSecrets = await fnSecretsFromImports({ + allowedImports, + secretDal, + folderDal + }); + return { secrets, imports: importedSecrets }; + } + return { secrets }; }; const getASecret = async ({ @@ -507,7 +527,8 @@ export const secretServiceFactory = ({ environment, path, type, - secretName + secretName, + includeImports }: TGetASecretDTO) => { const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId); ForbiddenError.from(permission).throwUnlessCan( @@ -526,6 +547,36 @@ export const secretServiceFactory = ({ userId: type === SecretType.Personal ? actorId : null, secretBlindIndex }); + // now if secret is not found + // then search for imported secrets + // here we consider the import order also thus starting from bottom + if (!secret && includeImports) { + const secretImports = await secretImportDal.find({ folderId }); + const allowedImports = secretImports.filter(({ importEnv, importPath }) => + // if its service token allow full access over imported one + actor === ActorType.SERVICE + ? true + : permission.can( + ProjectPermissionActions.Read, + subject(ProjectPermissionSub.Secrets, { + environment: importEnv.slug, + secretPath: importPath + }) + ) + ); + const importedSecrets = await fnSecretsFromImports({ + allowedImports, + secretDal, + folderDal + }); + for (let i = importedSecrets.length; i >= 0; i -= 1) { + for (let j = 0; j < importedSecrets[i].secrets.length; j += 1) { + if (secretBlindIndex === importedSecrets[i].secrets[j].secretBlindIndex) { + return importedSecrets[i].secrets[j]; + } + } + } + } if (!secret) throw new BadRequestError({ message: "Secret not found" }); return secret; @@ -738,6 +789,170 @@ export const secretServiceFactory = ({ return secretVersions; }; + const getSecretsRaw = async ({ + projectId, + path, + actor, + actorId, + environment, + includeImports + }: TGetSecretsRawDTO) => { + const botKey = await projectBotService.getBotKey(projectId); + if (!botKey) throw new BadRequestError({ message: "Project bot not found" }); + + const { secrets, imports } = await getSecrets({ + actorId, + projectId, + environment, + actor, + path, + includeImports + }); + + return { + secrets: secrets.map((el) => decryptSecretRaw(el, botKey)), + imports: (imports || [])?.map(({ secrets: importedSecrets, ...el }) => ({ + ...el, + secrets: importedSecrets.map((sec) => decryptSecretRaw(sec, botKey)) + })) + }; + }; + + const getASecretRaw = async ({ + type, + path, + actor, + environment, + projectId, + actorId, + secretName, + includeImports + }: TGetASecretRawDTO) => { + const botKey = await projectBotService.getBotKey(projectId); + if (!botKey) throw new BadRequestError({ message: "Project bot not found" }); + + const secret = await getASecret({ + actorId, + projectId, + environment, + actor, + path, + secretName, + type, + includeImports + }); + return decryptSecretRaw(secret, botKey); + }; + + const createSecretRaw = async ({ + secretName, + actorId, + projectId, + environment, + actor, + type, + secretPath, + secretValue, + secretComment, + skipMultilineEncoding + }: TCreateSecretRawDTO) => { + const botKey = await projectBotService.getBotKey(projectId); + if (!botKey) throw new BadRequestError({ message: "Project bot not found" }); + + const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secretName, botKey); + const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secretValue || "", botKey); + const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secretComment || "", botKey); + + const secret = await createSecret({ + secretName, + projectId, + environment, + type, + path: secretPath, + actor, + actorId, + secretKeyCiphertext: secretKeyEncrypted.ciphertext, + secretKeyIV: secretKeyEncrypted.iv, + secretKeyTag: secretKeyEncrypted.tag, + secretValueCiphertext: secretValueEncrypted.ciphertext, + secretValueIV: secretValueEncrypted.iv, + secretValueTag: secretValueEncrypted.tag, + secretCommentCiphertext: secretCommentEncrypted.ciphertext, + secretCommentIV: secretCommentEncrypted.iv, + secretCommentTag: secretCommentEncrypted.tag, + skipMultilineEncoding + }); + + await snapshotService.performSnapshot(secret.folderId); + await secretQueueService.syncSecrets({ secretPath, projectId, environment }); + + return decryptSecretRaw(secret, botKey); + }; + + const updateSecretRaw = async ({ + secretName, + actorId, + projectId, + environment, + actor, + type, + secretPath, + secretValue, + skipMultilineEncoding + }: TUpdateSecretRawDTO) => { + const botKey = await projectBotService.getBotKey(projectId); + if (!botKey) throw new BadRequestError({ message: "Project bot not found" }); + + const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secretValue || "", botKey); + + const secret = await updateSecret({ + secretName, + projectId, + environment, + type, + path: secretPath, + actor, + actorId, + secretValueCiphertext: secretValueEncrypted.ciphertext, + secretValueIV: secretValueEncrypted.iv, + secretValueTag: secretValueEncrypted.tag, + skipMultilineEncoding + }); + + await snapshotService.performSnapshot(secret.folderId); + await secretQueueService.syncSecrets({ secretPath, projectId, environment }); + + return decryptSecretRaw(secret, botKey); + }; + + const deleteSecretRaw = async ({ + secretName, + actorId, + projectId, + environment, + actor, + type, + secretPath + }: TDeleteSecretRawDTO) => { + const botKey = await projectBotService.getBotKey(projectId); + if (!botKey) throw new BadRequestError({ message: "Project bot not found" }); + + const secret = await deleteSecret({ + secretName, + projectId, + environment, + type, + path: secretPath, + actor, + actorId + }); + + await snapshotService.performSnapshot(secret.folderId); + await secretQueueService.syncSecrets({ secretPath, projectId, environment }); + + return decryptSecretRaw(secret, botKey); + }; + return { createSecret, deleteSecret, @@ -747,6 +962,11 @@ export const secretServiceFactory = ({ deleteManySecret, getASecret, getSecrets, + getSecretsRaw, + getASecretRaw, + createSecretRaw, + updateSecretRaw, + deleteSecretRaw, listSecretVersionsBySecretId, // external services function fnSecretBulkDelete, diff --git a/backend-pg/src/services/secret/secret-types.ts b/backend-pg/src/services/secret/secret-types.ts index a6442bb13..fc0ae8a01 100644 --- a/backend-pg/src/services/secret/secret-types.ts +++ b/backend-pg/src/services/secret/secret-types.ts @@ -61,6 +61,7 @@ export type TDeleteSecretDTO = { export type TGetSecretsDTO = { path: string; environment: string; + includeImports?: boolean; } & TProjectPermission; export type TGetASecretDTO = { @@ -68,6 +69,7 @@ export type TGetASecretDTO = { path: string; environment: string; type: "shared" | "personal"; + includeImports?: boolean; } & TProjectPermission; export type TCreateBulkSecretDTO = { @@ -125,6 +127,46 @@ export type TListSecretVersionDTO = { limit?: number; } & Omit; +export type TGetSecretsRawDTO = { + path: string; + environment: string; + includeImports?: boolean; +} & TProjectPermission; + +export type TGetASecretRawDTO = { + secretName: string; + path: string; + environment: string; + type: "shared" | "personal"; + includeImports?: boolean; +} & TProjectPermission; + +export type TCreateSecretRawDTO = TProjectPermission & { + secretPath: string; + environment: string; + secretName: string; + secretValue: string; + type: SecretType; + secretComment?: string; + skipMultilineEncoding?: boolean; +}; + +export type TUpdateSecretRawDTO = TProjectPermission & { + secretPath: string; + environment: string; + secretName: string; + secretValue?: string; + type: SecretType; + skipMultilineEncoding?: boolean; +}; + +export type TDeleteSecretRawDTO = TProjectPermission & { + secretPath: string; + environment: string; + secretName: string; + type: SecretType; +}; + export type TFnSecretBulkInsert = { folderId: string; tx?: Knex; diff --git a/backend-pg/src/services/webhook/webhook-fns.ts b/backend-pg/src/services/webhook/webhook-fns.ts index 96b8e6e0a..0169b6a42 100644 --- a/backend-pg/src/services/webhook/webhook-fns.ts +++ b/backend-pg/src/services/webhook/webhook-fns.ts @@ -1,9 +1,15 @@ import crypto from "node:crypto"; +import picomatch from "picomatch"; + import { SecretKeyEncoding, TWebhooks } from "@app/db/schemas"; import { getConfig } from "@app/lib/config/env"; import { request } from "@app/lib/config/request"; import { decryptSymmetric, decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; +import { BadRequestError } from "@app/lib/errors"; + +import { TProjectEnvDalFactory } from "../project-env/project-env-dal"; +import { TWebhookDalFactory } from "./webhook-dal"; const WEBHOOK_TRIGGER_TIMEOUT = 15 * 1000; export const triggerWebhookRequest = async ( @@ -64,3 +70,67 @@ export const getWebhookPayload = ( secretPath } }); + +export type TFnTriggerWebhookDTO = { + projectId: string; + secretPath: string; + environment: string; + webhookDal: Pick; + projectEnvDal: Pick; +}; +// this is reusable function +// used in secret queue to trigger webhook and update status when secrets changes +export const fnTriggerWebhook = async ({ + environment, + secretPath, + projectId, + webhookDal, + projectEnvDal +}: TFnTriggerWebhookDTO) => { + const webhooks = await webhookDal.findAllWebhooks(projectId, environment); + const toBeTriggeredHooks = webhooks.filter( + ({ secretPath: hookSecretPath, isDisabled }) => + !isDisabled && picomatch.isMatch(secretPath, hookSecretPath, { strictSlashes: false }) + ); + if (!toBeTriggeredHooks.length) return; + const webhooksTriggered = await Promise.allSettled( + toBeTriggeredHooks.map((hook) => + triggerWebhookRequest( + hook, + getWebhookPayload("secrets.modified", projectId, environment, secretPath) + ) + ) + ); + // filter hooks by status + const successWebhooks = webhooksTriggered + .filter(({ status }) => status === "fulfilled") + .map((_, i) => toBeTriggeredHooks[i].id); + const failedWebhooks = webhooksTriggered + .filter(({ status }) => status === "rejected") + .map((data, i) => ({ + id: toBeTriggeredHooks[i].id, + error: data.status === "rejected" && data.reason.message + })); + + await webhookDal.transaction(async (tx) => { + const env = await projectEnvDal.findOne({ projectId, slug: environment }, tx); + if (!env) throw new BadRequestError({ message: "Env not found" }); + if (successWebhooks.length) { + await webhookDal.update( + { envId: env.id, $in: { id: successWebhooks } }, + { lastStatus: "success", lastRunErrorMessage: null }, + tx + ); + } + if (failedWebhooks.length) { + await webhookDal.bulkUpdate( + failedWebhooks.map(({ id, error }) => ({ + id, + lastRunErrorMessage: error, + lastStatus: "failed" + })), + tx + ); + } + }); +}; diff --git a/backend-pg/src/services/webhook/webhook-service.ts b/backend-pg/src/services/webhook/webhook-service.ts index 2d7dea841..65c70340a 100644 --- a/backend-pg/src/services/webhook/webhook-service.ts +++ b/backend-pg/src/services/webhook/webhook-service.ts @@ -1,5 +1,4 @@ import { ForbiddenError } from "@casl/ability"; -import picomatch from "picomatch"; import { SecretEncryptionAlgo, SecretKeyEncoding, TWebhooksInsert } from "@app/db/schemas"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; @@ -17,7 +16,6 @@ import { getWebhookPayload, triggerWebhookRequest } from "./webhook-fns"; import { TCreateWebhookDTO, TDeleteWebhookDTO, - TFnTriggerWebhookDTO, TListWebhookDTO, TTestWebhookDTO, TUpdateWebhookDTO @@ -170,63 +168,11 @@ export const webhookServiceFactory = ({ return webhookDal.findAllWebhooks(projectId, environment, secretPath); }; - // this is reusable function - // used in secret queue to trigger webhook and update status when secrets changes - const fnTriggerWebhook = async ({ environment, secretPath, projectId }: TFnTriggerWebhookDTO) => { - const webhooks = await webhookDal.findAllWebhooks(projectId, environment); - const toBeTriggeredHooks = webhooks.filter( - ({ secretPath: hookSecretPath, isDisabled }) => - !isDisabled && picomatch.isMatch(secretPath, hookSecretPath, { strictSlashes: false }) - ); - if (!toBeTriggeredHooks.length) return; - const webhooksTriggered = await Promise.allSettled( - toBeTriggeredHooks.map((hook) => - triggerWebhookRequest( - hook, - getWebhookPayload("secrets.modified", projectId, environment, secretPath) - ) - ) - ); - // filter hooks by status - const successWebhooks = webhooksTriggered - .filter(({ status }) => status === "fulfilled") - .map((_, i) => toBeTriggeredHooks[i].id); - const failedWebhooks = webhooksTriggered - .filter(({ status }) => status === "rejected") - .map((data, i) => ({ - id: toBeTriggeredHooks[i].id, - error: data.status === "rejected" && data.reason.message - })); - - await webhookDal.transaction(async (tx) => { - const env = await projectEnvDal.findOne({ projectId, slug: environment }, tx); - if (!env) throw new BadRequestError({ message: "Env not found" }); - if (successWebhooks.length) { - await webhookDal.update( - { envId: env.id, $in: { id: successWebhooks } }, - { lastStatus: "success", lastRunErrorMessage: null }, - tx - ); - } - if (failedWebhooks.length) { - await webhookDal.bulkUpdate( - failedWebhooks.map(({ id, error }) => ({ - id, - lastRunErrorMessage: error, - lastStatus: "failed" - })), - tx - ); - } - }); - }; - return { createWebhook, deleteWebhook, listWebhooks, updateWebhook, - testWebhook, - fnTriggerWebhook + testWebhook }; }; diff --git a/backend-pg/src/services/webhook/webhook-types.ts b/backend-pg/src/services/webhook/webhook-types.ts index ff8feded7..7a6e92c80 100644 --- a/backend-pg/src/services/webhook/webhook-types.ts +++ b/backend-pg/src/services/webhook/webhook-types.ts @@ -24,9 +24,3 @@ export type TListWebhookDTO = { environment?: string; secretPath?: string; } & TProjectPermission; - -export type TFnTriggerWebhookDTO = { - projectId: string; - secretPath: string; - environment: string; -};