diff --git a/backend/src/@types/knex.d.ts b/backend/src/@types/knex.d.ts index 3a6aa549d..9e8a2a752 100644 --- a/backend/src/@types/knex.d.ts +++ b/backend/src/@types/knex.d.ts @@ -231,6 +231,7 @@ import { TWebhooksInsert, TWebhooksUpdate } from "@app/db/schemas"; +import { TSecretReferences, TSecretReferencesInsert, TSecretReferencesUpdate } from "@app/db/schemas/secret-references"; declare module "knex/types/tables" { interface Tables { @@ -304,6 +305,11 @@ declare module "knex/types/tables" { >; [TableName.ProjectKeys]: Knex.CompositeTableType; [TableName.Secret]: Knex.CompositeTableType; + [TableName.SecretReference]: Knex.CompositeTableType< + TSecretReferences, + TSecretReferencesInsert, + TSecretReferencesUpdate + >; [TableName.SecretBlindIndex]: Knex.CompositeTableType< TSecretBlindIndexes, TSecretBlindIndexesInsert, diff --git a/backend/src/db/migrations/20240514141809_inline-secret-reference-sync.ts b/backend/src/db/migrations/20240514141809_inline-secret-reference-sync.ts new file mode 100644 index 000000000..fa6fb4fea --- /dev/null +++ b/backend/src/db/migrations/20240514141809_inline-secret-reference-sync.ts @@ -0,0 +1,24 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; +import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils"; + +export async function up(knex: Knex): Promise { + if (!(await knex.schema.hasTable(TableName.SecretReference))) { + await knex.schema.createTable(TableName.SecretReference, (t) => { + t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); + t.string("environment").notNullable(); + t.string("secretPath").notNullable(); + t.uuid("secretId").notNullable(); + t.foreign("secretId").references("id").inTable(TableName.Secret).onDelete("CASCADE"); + t.timestamps(true, true, true); + }); + + await createOnUpdateTrigger(knex, TableName.SecretReference); + } +} + +export async function down(knex: Knex): Promise { + await knex.schema.dropTableIfExists(TableName.SecretReference); + await dropOnUpdateTrigger(knex, TableName.SecretReference); +} diff --git a/backend/src/db/schemas/models.ts b/backend/src/db/schemas/models.ts index 4951de99e..aaaa0ff9a 100644 --- a/backend/src/db/schemas/models.ts +++ b/backend/src/db/schemas/models.ts @@ -28,6 +28,7 @@ export enum TableName { ProjectUserMembershipRole = "project_user_membership_roles", ProjectKeys = "project_keys", Secret = "secrets", + SecretReference = "secret_references", SecretBlindIndex = "secret_blind_indexes", SecretVersion = "secret_versions", SecretFolder = "secret_folders", diff --git a/backend/src/db/schemas/secret-references.ts b/backend/src/db/schemas/secret-references.ts new file mode 100644 index 000000000..b3e6a8629 --- /dev/null +++ b/backend/src/db/schemas/secret-references.ts @@ -0,0 +1,21 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const SecretReferencesSchema = z.object({ + id: z.string().uuid(), + environment: z.string(), + secretPath: z.string(), + secretId: z.string().uuid(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TSecretReferences = z.infer; +export type TSecretReferencesInsert = Omit, TImmutableDBKeys>; +export type TSecretReferencesUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts index 2e66ab2ce..690d308d2 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts @@ -7,12 +7,15 @@ import { SecretType, TSecretApprovalRequestsSecretsInsert } from "@app/db/schemas"; +import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; import { BadRequestError, UnauthorizedError } from "@app/lib/errors"; import { groupBy, pick, unique } from "@app/lib/fn"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { ActorType } from "@app/services/auth/auth-type"; import { TProjectDALFactory } from "@app/services/project/project-dal"; +import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; import { TSecretDALFactory } from "@app/services/secret/secret-dal"; +import { getAllNestedSecretReferences } from "@app/services/secret/secret-fns"; import { TSecretQueueFactory } from "@app/services/secret/secret-queue"; import { TSecretServiceFactory } from "@app/services/secret/secret-service"; import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal"; @@ -53,6 +56,7 @@ type TSecretApprovalRequestServiceFactoryDep = { secretVersionDAL: Pick; secretVersionTagDAL: Pick; projectDAL: Pick; + projectBotService: Pick; secretService: Pick< TSecretServiceFactory, | "fnSecretBulkInsert" @@ -80,7 +84,8 @@ export const secretApprovalRequestServiceFactory = ({ snapshotService, secretService, secretVersionDAL, - secretQueueService + secretQueueService, + projectBotService }: TSecretApprovalRequestServiceFactoryDep) => { const requestCount = async ({ projectId, actor, actorId, actorOrgId, actorAuthMethod }: TApprovalRequestCountDTO) => { if (actor === ActorType.SERVICE) throw new BadRequestError({ message: "Cannot use service token" }); @@ -352,7 +357,7 @@ export const secretApprovalRequestServiceFactory = ({ } const secretDeletionCommits = secretApprovalSecrets.filter(({ op }) => op === CommitType.Delete); - + const botKey = await projectBotService.getBotKey(projectId).catch(() => null); const mergeStatus = await secretApprovalRequestDAL.transaction(async (tx) => { const newSecrets = secretCreationCommits.length ? await secretService.fnSecretBulkInsert({ @@ -379,7 +384,17 @@ export const secretApprovalRequestServiceFactory = ({ ]), tags: el?.tags.map(({ id }) => id), version: 1, - type: SecretType.Shared + type: SecretType.Shared, + references: botKey + ? getAllNestedSecretReferences( + decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.secretValueCiphertext, + iv: el.secretValueIV, + tag: el.secretValueTag, + key: botKey + }) + ) + : undefined })), secretDAL, secretVersionDAL, @@ -414,7 +429,17 @@ export const secretApprovalRequestServiceFactory = ({ "secretReminderNote", "secretReminderRepeatDays", "secretBlindIndex" - ]) + ]), + references: botKey + ? getAllNestedSecretReferences( + decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.secretValueCiphertext, + iv: el.secretValueIV, + tag: el.secretValueTag, + key: botKey + }) + ) + : undefined } })), secretDAL, diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index e1149120d..bc8ac88ff 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -65,7 +65,13 @@ export type TQueueJobTypes = { }; [QueueName.IntegrationSync]: { name: QueueJobs.IntegrationSync; - payload: { projectId: string; environment: string; secretPath: string; depth?: number }; + payload: { + projectId: string; + environment: string; + secretPath: string; + depth?: number; + deDupeQueue?: Record; + }; }; [QueueName.SecretFullRepoScan]: { name: QueueJobs.SecretScan; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 3b0758d68..3a050bcad 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -607,6 +607,7 @@ export const registerRoutes = async ( }); const sarService = secretApprovalRequestServiceFactory({ permissionService, + projectBotService, folderDAL, secretDAL, secretTagDAL, diff --git a/backend/src/server/routes/v3/secret-router.ts b/backend/src/server/routes/v3/secret-router.ts index e2d8ddb12..6fa574a69 100644 --- a/backend/src/server/routes/v3/secret-router.ts +++ b/backend/src/server/routes/v3/secret-router.ts @@ -1926,4 +1926,41 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { return { secrets }; } }); + + server.route({ + method: "POST", + url: "/backfill-secret-references", + config: { + rateLimit: secretsLimit + }, + schema: { + description: "Backfill secret references", + security: [ + { + bearerAuth: [] + } + ], + body: z.object({ + projectId: z.string().trim().min(1) + }), + response: { + 200: z.object({ + message: z.string() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const { projectId } = req.body; + const message = await server.services.secret.backfillSecretReferences({ + actorId: req.permission.id, + actor: req.permission.type, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId, + projectId + }); + + return message; + } + }); }; diff --git a/backend/src/services/secret/secret-dal.ts b/backend/src/services/secret/secret-dal.ts index 8a5970b83..1a2e414dd 100644 --- a/backend/src/services/secret/secret-dal.ts +++ b/backend/src/services/secret/secret-dal.ts @@ -243,6 +243,74 @@ export const secretDALFactory = (db: TDbClient) => { } }; + const upsertSecretReferences = async ( + data: { + secretId: string; + references: Array<{ environment: string; secretPath: string }>; + }[] = [], + tx?: Knex + ) => { + try { + if (!data.length) return; + + await (tx || db)(TableName.SecretReference) + .whereIn( + "secretId", + data.map(({ secretId }) => secretId) + ) + .delete(); + const newSecretReferences = data + .filter(({ references }) => references.length) + .flatMap(({ secretId, references }) => + references.map(({ environment, secretPath }) => ({ + secretPath, + secretId, + environment + })) + ); + if (!newSecretReferences.length) return; + const secretReferences = await (tx || db)(TableName.SecretReference).insert(newSecretReferences); + return secretReferences; + } catch (error) { + throw new DatabaseError({ error, name: "UpsertSecretReference" }); + } + }; + + const findReferencedSecretReferences = async (projectId: string, envSlug: string, secretPath: string, tx?: Knex) => { + try { + const docs = await (tx || db)(TableName.SecretReference) + .where({ + secretPath, + environment: envSlug + }) + .join(TableName.Secret, `${TableName.Secret}.id`, `${TableName.SecretReference}.secretId`) + .join(TableName.SecretFolder, `${TableName.Secret}.folderId`, `${TableName.SecretFolder}.id`) + .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) + .where("projectId", projectId) + .select(selectAllTableCols(TableName.SecretReference)) + .select("folderId"); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindReferencedSecretReferences" }); + } + }; + + // special query to backfill secret value + const findAllProjectSecretValues = async (projectId: string, tx?: Knex) => { + try { + const docs = await (tx || db)(TableName.Secret) + .join(TableName.SecretFolder, `${TableName.Secret}.folderId`, `${TableName.SecretFolder}.id`) + .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) + .where("projectId", projectId) + // not empty + .whereNotNull("secretValueCiphertext") + .select("secretValueTag", "secretValueCiphertext", "secretValueIV", `${TableName.Secret}.id` as "id"); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindAllProjectSecretValues" }); + } + }; + return { ...secretOrm, update, @@ -252,6 +320,9 @@ export const secretDALFactory = (db: TDbClient) => { getSecretTags, findByFolderId, findByFolderIds, - findByBlindIndexes + findByBlindIndexes, + upsertSecretReferences, + findReferencedSecretReferences, + findAllProjectSecretValues }; }; diff --git a/backend/src/services/secret/secret-fns.ts b/backend/src/services/secret/secret-fns.ts index fb2b90ba9..32860e38c 100644 --- a/backend/src/services/secret/secret-fns.ts +++ b/backend/src/services/secret/secret-fns.ts @@ -194,6 +194,7 @@ type TInterpolateSecretArg = { folderDAL: Pick; }; +const INTERPOLATION_SYNTAX_REG = /\${([^}]+)}/g; export const interpolateSecrets = ({ projectId, secretEncKey, secretDAL, folderDAL }: TInterpolateSecretArg) => { const fetchSecretsCrossEnv = () => { const fetchCache: Record> = {}; @@ -235,7 +236,6 @@ export const interpolateSecrets = ({ projectId, secretEncKey, secretDAL, folderD }; }; - const INTERPOLATION_SYNTAX_REG = /\${([^}]+)}/g; const recursivelyExpandSecret = async ( expandedSec: Record, interpolatedSec: Record, @@ -396,6 +396,16 @@ export const decryptSecretRaw = ( }; }; +export const getAllNestedSecretReferences = (value: string) => { + const references = Array.from(value.matchAll(INTERPOLATION_SYNTAX_REG), (m) => m[1]); + return references + .filter((el) => el.includes(".")) + .map((el) => { + const [environment, ...secretPath] = el.split("."); + return { environment, secretPath: path.join("/", ...secretPath.slice(0, -1)) }; + }); +}; + /** * Checks and handles secrets using a blind index method. * The function generates mappings between secret names and their blind indexes, validates user IDs for personal secrets, and retrieves secrets from the database based on their blind indexes. @@ -467,7 +477,7 @@ export const fnSecretBulkInsert = async ({ tx }: TFnSecretBulkInsert) => { const newSecrets = await secretDAL.insertMany( - inputSecrets.map(({ tags, ...el }) => ({ ...el, folderId })), + inputSecrets.map(({ tags, references, ...el }) => ({ ...el, folderId })), tx ); const newSecretGroupByBlindIndex = groupBy(newSecrets, (item) => item.secretBlindIndex as string); @@ -478,13 +488,19 @@ export const fnSecretBulkInsert = async ({ })) ); const secretVersions = await secretVersionDAL.insertMany( - inputSecrets.map(({ tags, ...el }) => ({ + inputSecrets.map(({ tags, references, ...el }) => ({ ...el, folderId, secretId: newSecretGroupByBlindIndex[el.secretBlindIndex as string][0].id })), tx ); + await secretDAL.upsertSecretReferences( + inputSecrets.map(({ references = [], secretBlindIndex }) => ({ + secretId: newSecretGroupByBlindIndex[secretBlindIndex as string][0].id, + references + })) + ); if (newSecretTags.length) { const secTags = await secretTagDAL.saveTagsToSecret(newSecretTags, tx); const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId); @@ -509,7 +525,7 @@ export const fnSecretBulkUpdate = async ({ secretVersionTagDAL }: TFnSecretBulkUpdate) => { const newSecrets = await secretDAL.bulkUpdate( - inputSecrets.map(({ filter, data: { tags, ...data } }) => ({ + inputSecrets.map(({ filter, data: { tags, references, ...data } }) => ({ filter: { ...filter, folderId }, data })), @@ -522,6 +538,14 @@ export const fnSecretBulkUpdate = async ({ })), tx ); + await secretDAL.upsertSecretReferences( + inputSecrets + .filter(({ data: { references } }) => Boolean(references)) + .map(({ data: { references = [] } }, i) => ({ + secretId: newSecrets[i].id, + references + })) + ); const secsUpdatedTag = inputSecrets.flatMap(({ data: { tags } }, i) => tags !== undefined ? { tags, secretId: newSecrets[i].id } : [] ); @@ -591,50 +615,39 @@ export const createManySecretsRawFnFactory = ({ folderId, isNew: true, blindIndexCfg, + userId, secretDAL }); - const inputSecrets = await Promise.all( - secrets.map(async (secret) => { - const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey); - const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey); - const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey); + const inputSecrets = secrets.map((secret) => { + const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey); + const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey); + const secretReferences = getAllNestedSecretReferences(secret.secretValue || ""); + const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey); - if (secret.type === SecretType.Personal) { - if (!userId) throw new BadRequestError({ message: "Missing user id for personal secret" }); - const sharedExist = await secretDAL.findOne({ - secretBlindIndex: keyName2BlindIndex[secret.secretName], - folderId, - type: SecretType.Shared - }); + return { + type: secret.type, + userId: secret.type === SecretType.Personal ? userId : null, + secretName: secret.secretName, + secretKeyCiphertext: secretKeyEncrypted.ciphertext, + secretKeyIV: secretKeyEncrypted.iv, + secretKeyTag: secretKeyEncrypted.tag, + secretValueCiphertext: secretValueEncrypted.ciphertext, + secretValueIV: secretValueEncrypted.iv, + secretValueTag: secretValueEncrypted.tag, + secretCommentCiphertext: secretCommentEncrypted.ciphertext, + secretCommentIV: secretCommentEncrypted.iv, + secretCommentTag: secretCommentEncrypted.tag, + skipMultilineEncoding: secret.skipMultilineEncoding, + tags: secret.tags, + references: secretReferences + }; + }); - if (!sharedExist) - throw new BadRequestError({ - message: "Failed to create personal secret override for no corresponding shared secret" - }); - } - - const tags = secret.tags ? await secretTagDAL.findManyTagsById(projectId, secret.tags) : []; - if ((secret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" }); - - return { - type: secret.type, - userId: secret.type === SecretType.Personal ? userId : null, - secretName: secret.secretName, - secretKeyCiphertext: secretKeyEncrypted.ciphertext, - secretKeyIV: secretKeyEncrypted.iv, - secretKeyTag: secretKeyEncrypted.tag, - secretValueCiphertext: secretValueEncrypted.ciphertext, - secretValueIV: secretValueEncrypted.iv, - secretValueTag: secretValueEncrypted.tag, - secretCommentCiphertext: secretCommentEncrypted.ciphertext, - secretCommentIV: secretCommentEncrypted.iv, - secretCommentTag: secretCommentEncrypted.tag, - skipMultilineEncoding: secret.skipMultilineEncoding, - tags: secret.tags - }; - }) - ); + // get all tags + const tagIds = inputSecrets.flatMap(({ tags = [] }) => tags); + const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : []; + if (tags.length !== tagIds.length) throw new BadRequestError({ message: "Tag not found" }); const newSecrets = await secretDAL.transaction(async (tx) => fnSecretBulkInsert({ @@ -703,56 +716,35 @@ export const updateManySecretsRawFnFactory = ({ userId }); - const inputSecrets = await Promise.all( - secrets.map(async (secret) => { - if (secret.newSecretName === "") { - throw new BadRequestError({ message: "New secret name cannot be empty" }); - } + const inputSecrets = secrets.map((secret) => { + if (secret.newSecretName === "") { + throw new BadRequestError({ message: "New secret name cannot be empty" }); + } - const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey); - const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey); - const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey); + const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey); + const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey); + const secretReferences = getAllNestedSecretReferences(secret.secretValue || ""); + const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey); - if (secret.type === SecretType.Personal) { - if (!userId) throw new BadRequestError({ message: "Missing user id for personal secret" }); - - const sharedExist = await secretDAL.findOne({ - secretBlindIndex: keyName2BlindIndex[secret.secretName], - folderId, - type: SecretType.Shared - }); - - if (!sharedExist) - throw new BadRequestError({ - message: "Failed to update personal secret override for no corresponding shared secret" - }); - - if (secret.newSecretName) - throw new BadRequestError({ message: "Personal secret cannot change the key name" }); - } - - const tags = secret.tags ? await secretTagDAL.findManyTagsById(projectId, secret.tags) : []; - if ((secret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" }); - - return { - type: secret.type, - userId: secret.type === SecretType.Personal ? userId : null, - secretName: secret.secretName, - newSecretName: secret.newSecretName, - secretKeyCiphertext: secretKeyEncrypted.ciphertext, - secretKeyIV: secretKeyEncrypted.iv, - secretKeyTag: secretKeyEncrypted.tag, - secretValueCiphertext: secretValueEncrypted.ciphertext, - secretValueIV: secretValueEncrypted.iv, - secretValueTag: secretValueEncrypted.tag, - secretCommentCiphertext: secretCommentEncrypted.ciphertext, - secretCommentIV: secretCommentEncrypted.iv, - secretCommentTag: secretCommentEncrypted.tag, - skipMultilineEncoding: secret.skipMultilineEncoding, - tags: secret.tags - }; - }) - ); + return { + type: secret.type, + userId: secret.type === SecretType.Personal ? userId : null, + secretName: secret.secretName, + newSecretName: secret.newSecretName, + secretKeyCiphertext: secretKeyEncrypted.ciphertext, + secretKeyIV: secretKeyEncrypted.iv, + secretKeyTag: secretKeyEncrypted.tag, + secretValueCiphertext: secretValueEncrypted.ciphertext, + secretValueIV: secretValueEncrypted.iv, + secretValueTag: secretValueEncrypted.tag, + secretCommentCiphertext: secretCommentEncrypted.ciphertext, + secretCommentIV: secretCommentEncrypted.iv, + secretCommentTag: secretCommentEncrypted.tag, + skipMultilineEncoding: secret.skipMultilineEncoding, + tags: secret.tags, + references: secretReferences + }; + }); const tagIds = inputSecrets.flatMap(({ tags = [] }) => tags); const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : []; diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index 32b68fe29..e1c8d61af 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -59,6 +59,7 @@ export type TGetSecrets = { }; const MAX_SYNC_SECRET_DEPTH = 5; +const uniqueIntegrationKey = (environment: string, secretPath: string) => `integration-${environment}-${secretPath}`; export const secretQueueFactory = ({ queueService, @@ -102,28 +103,35 @@ export const secretQueueFactory = ({ folderDAL }); - const syncIntegrations = async (dto: TGetSecrets) => { + const syncIntegrations = async (dto: TGetSecrets & { deDupeQueue?: Record }) => { await queueService.queue(QueueName.IntegrationSync, QueueJobs.IntegrationSync, dto, { - attempts: 5, + attempts: 3, delay: 1000, backoff: { type: "exponential", delay: 3000 }, removeOnComplete: true, - removeOnFail: { - count: 5 // keep the most recent jobs - } + removeOnFail: true }); }; - const syncSecrets = async (dto: TGetSecrets & { depth?: number }) => { + const syncSecrets = async ({ + deDupeQueue = {}, + ...dto + }: TGetSecrets & { depth?: number; deDupeQueue?: Record }) => { + const deDuplicationKey = uniqueIntegrationKey(dto.environment, dto.secretPath); + if (deDupeQueue?.[deDuplicationKey]) { + return; + } + // eslint-disable-next-line + deDupeQueue[deDuplicationKey] = true; logger.info( `syncSecrets: syncing project secrets where [projectId=${dto.projectId}] [environment=${dto.environment}] [path=${dto.secretPath}]` ); await queueService.queue(QueueName.SecretWebhook, QueueJobs.SecWebhook, dto, { jobId: `secret-webhook-${dto.environment}-${dto.projectId}-${dto.secretPath}`, - removeOnFail: { count: 5 }, + removeOnFail: true, removeOnComplete: true, delay: 1000, attempts: 5, @@ -132,7 +140,7 @@ export const secretQueueFactory = ({ delay: 3000 } }); - await syncIntegrations(dto); + await syncIntegrations({ ...dto, deDupeQueue }); }; const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => { @@ -326,7 +334,7 @@ export const secretQueueFactory = ({ }; queueService.start(QueueName.IntegrationSync, async (job) => { - const { environment, projectId, secretPath, depth = 1 } = job.data; + const { environment, projectId, secretPath, depth = 1, deDupeQueue = {} } = job.data; const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); if (!folder) { @@ -349,21 +357,68 @@ export const secretQueueFactory = ({ const importedFolderIds = unique(imports, (i) => i.folderId).map(({ folderId }) => folderId); const importedFolders = await folderDAL.findSecretPathByFolderIds(projectId, importedFolderIds); const foldersGroupedById = groupBy(importedFolders, (i) => i.child || i.id); + logger.info( + `getIntegrationSecrets: Syncing secret due to link change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}] [secretPath=${job.data.secretPath}] [depth=${depth}]` + ); await Promise.all( imports .filter(({ folderId }) => Boolean(foldersGroupedById[folderId][0].path)) - .map(({ folderId }) => { - const syncDto = { + // filter out already synced ones + .filter( + ({ folderId }) => + !deDupeQueue[ + uniqueIntegrationKey( + foldersGroupedById[folderId][0].environmentSlug, + foldersGroupedById[folderId][0].path + ) + ] + ) + .map(({ folderId }) => + syncSecrets({ depth: depth + 1, projectId, secretPath: foldersGroupedById[folderId][0].path, - environment: foldersGroupedById[folderId][0].environmentSlug - }; - logger.info( - `getIntegrationSecrets: Syncing secret due to link change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}] [secretPath=${job.data.secretPath}] [depth=${depth}]` - ); - return syncSecrets(syncDto); - }) + environment: foldersGroupedById[folderId][0].environmentSlug, + deDupeQueue + }) + ) + ); + } + + const secretReferences = await secretDAL.findReferencedSecretReferences( + projectId, + folder.environment.slug, + secretPath + ); + if (secretReferences.length) { + const referencedFolderIds = unique(secretReferences, (i) => i.folderId).map(({ folderId }) => folderId); + const referencedFolders = await folderDAL.findSecretPathByFolderIds(projectId, referencedFolderIds); + const referencedFoldersGroupedById = groupBy(referencedFolders, (i) => i.child || i.id); + logger.info( + `getIntegrationSecrets: Syncing secret due to reference change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}] [secretPath=${job.data.secretPath}] [depth=${depth}]` + ); + await Promise.all( + secretReferences + .filter(({ folderId }) => Boolean(referencedFoldersGroupedById[folderId][0].path)) + // filter out already synced ones + .filter( + ({ folderId }) => + !deDupeQueue[ + uniqueIntegrationKey( + referencedFoldersGroupedById[folderId][0].environmentSlug, + referencedFoldersGroupedById[folderId][0].path + ) + ] + ) + .map(({ folderId }) => + syncSecrets({ + depth: depth + 1, + projectId, + secretPath: referencedFoldersGroupedById[folderId][0].path, + environment: referencedFoldersGroupedById[folderId][0].environmentSlug, + deDupeQueue + }) + ) ); } } else { diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index feb8c01d5..37baa5ad4 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -2,12 +2,22 @@ /* eslint-disable no-await-in-loop */ import { ForbiddenError, subject } from "@casl/ability"; -import { SecretEncryptionAlgo, SecretKeyEncoding, SecretsSchema, SecretType } from "@app/db/schemas"; +import { + ProjectMembershipRole, + SecretEncryptionAlgo, + SecretKeyEncoding, + SecretsSchema, + SecretType +} from "@app/db/schemas"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service"; import { getConfig } from "@app/lib/config/env"; -import { buildSecretBlindIndexFromName, encryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; +import { + buildSecretBlindIndexFromName, + decryptSymmetric128BitHexKeyUTF8, + encryptSymmetric128BitHexKeyUTF8 +} from "@app/lib/crypto"; import { BadRequestError } from "@app/lib/errors"; import { groupBy, pick } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; @@ -27,12 +37,14 @@ import { fnSecretBlindIndexCheck, fnSecretBulkInsert, fnSecretBulkUpdate, + getAllNestedSecretReferences, interpolateSecrets, recursivelyGetSecretPaths } from "./secret-fns"; import { TSecretQueueFactory } from "./secret-queue"; import { TAttachSecretTagsDTO, + TBackFillSecretReferencesDTO, TCreateBulkSecretDTO, TCreateManySecretRawDTO, TCreateSecretDTO, @@ -91,6 +103,22 @@ export const secretServiceFactory = ({ secretImportDAL, secretVersionTagDAL }: TSecretServiceFactoryDep) => { + const getSecretReference = async (projectId: string) => { + // if bot key missing means e2e still exist + const botKey = await projectBotService.getBotKey(projectId).catch(() => null); + return (el: { ciphertext?: string; iv: string; tag: string }) => + botKey + ? getAllNestedSecretReferences( + decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.ciphertext || "", + iv: el.iv, + tag: el.tag, + key: botKey + }) + ) + : undefined; + }; + // utility function to get secret blind index data const interalGenSecBlindIndexByName = async (projectId: string, secretName: string) => { const appCfg = getConfig(); @@ -225,6 +253,7 @@ export const secretServiceFactory = ({ if ((inputSecret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" }); const { secretName, type, ...el } = inputSecret; + const references = await getSecretReference(projectId); const secret = await secretDAL.transaction((tx) => fnSecretBulkInsert({ folderId, @@ -237,7 +266,12 @@ export const secretServiceFactory = ({ userId: inputSecret.type === SecretType.Personal ? actorId : null, algorithm: SecretEncryptionAlgo.AES_256_GCM, keyEncoding: SecretKeyEncoding.UTF8, - tags: inputSecret.tags + tags: inputSecret.tags, + references: references({ + ciphertext: inputSecret.secretValueCiphertext, + iv: inputSecret.secretValueIV, + tag: inputSecret.secretValueTag + }) } ], secretDAL, @@ -335,6 +369,7 @@ export const secretServiceFactory = ({ const { secretName, ...el } = inputSecret; + const references = await getSecretReference(projectId); const updatedSecret = await secretDAL.transaction(async (tx) => fnSecretBulkUpdate({ folderId, @@ -360,7 +395,12 @@ export const secretServiceFactory = ({ "secretReminderRepeatDays", "tags" ]), - secretBlindIndex: newSecretNameBlindIndex || keyName2BlindIndex[secretName] + secretBlindIndex: newSecretNameBlindIndex || keyName2BlindIndex[secretName], + references: references({ + ciphertext: inputSecret.secretValueCiphertext, + iv: inputSecret.secretValueIV, + tag: inputSecret.secretValueTag + }) } } ], @@ -700,6 +740,7 @@ export const secretServiceFactory = ({ const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : []; if (tags.length !== tagIds.length) throw new BadRequestError({ message: "Tag not found" }); + const references = await getSecretReference(projectId); const newSecrets = await secretDAL.transaction(async (tx) => fnSecretBulkInsert({ inputSecrets: inputSecrets.map(({ secretName, ...el }) => ({ @@ -708,7 +749,12 @@ export const secretServiceFactory = ({ secretBlindIndex: keyName2BlindIndex[secretName], type: SecretType.Shared, algorithm: SecretEncryptionAlgo.AES_256_GCM, - keyEncoding: SecretKeyEncoding.UTF8 + keyEncoding: SecretKeyEncoding.UTF8, + references: references({ + ciphertext: el.secretValueCiphertext, + iv: el.secretValueIV, + tag: el.secretValueTag + }) })), folderId, secretDAL, @@ -783,6 +829,8 @@ export const secretServiceFactory = ({ const tagIds = inputSecrets.flatMap(({ tags = [] }) => tags); const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : []; if (tagIds.length !== tags.length) throw new BadRequestError({ message: "Tag not found" }); + + const references = await getSecretReference(projectId); const secrets = await secretDAL.transaction(async (tx) => fnSecretBulkUpdate({ folderId, @@ -799,7 +847,15 @@ export const secretServiceFactory = ({ ? newKeyName2BlindIndex[newSecretName] : keyName2BlindIndex[secretName], algorithm: SecretEncryptionAlgo.AES_256_GCM, - keyEncoding: SecretKeyEncoding.UTF8 + keyEncoding: SecretKeyEncoding.UTF8, + references: + el.secretValueIV && el.secretValueTag + ? references({ + ciphertext: el.secretValueCiphertext, + iv: el.secretValueIV, + tag: el.secretValueTag + }) + : undefined } })), secretDAL, @@ -1488,6 +1544,49 @@ export const secretServiceFactory = ({ }; }; + // this is a backfilling API for secret references + // what it does is it will go through all the secret values and parse all references + // populate the secret reference to do sync integrations + const backfillSecretReferences = async ({ + projectId, + actor, + actorId, + actorOrgId, + actorAuthMethod + }: TBackFillSecretReferencesDTO) => { + const { hasRole } = await permissionService.getProjectPermission( + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + ); + + if (!hasRole(ProjectMembershipRole.Admin)) throw new BadRequestError({ message: "Only admins are allowed" }); + + const botKey = await projectBotService.getBotKey(projectId); + if (!botKey) throw new BadRequestError({ message: "Kindly upgrade your project", name: "bot_not_found_error" }); + + await secretDAL.transaction(async (tx) => { + const secrets = await secretDAL.findAllProjectSecretValues(projectId, tx); + await secretDAL.upsertSecretReferences( + secrets.map(({ id, secretValueCiphertext, secretValueIV, secretValueTag }) => ({ + secretId: id, + references: getAllNestedSecretReferences( + decryptSymmetric128BitHexKeyUTF8({ + ciphertext: secretValueCiphertext, + iv: secretValueIV, + tag: secretValueTag, + key: botKey + }) + ) + })) + ); + }); + + return { message: "Successfully backfilled secret references." }; + }; + return { attachTags, detachTags, @@ -1508,6 +1607,7 @@ export const secretServiceFactory = ({ updateManySecretsRaw, deleteManySecretsRaw, getSecretVersions, + backfillSecretReferences, // external services function fnSecretBulkDelete, fnSecretBulkUpdate, diff --git a/backend/src/services/secret/secret-types.ts b/backend/src/services/secret/secret-types.ts index 9f2addc89..7e713a80f 100644 --- a/backend/src/services/secret/secret-types.ts +++ b/backend/src/services/secret/secret-types.ts @@ -223,11 +223,13 @@ export type TGetSecretVersionsDTO = Omit & { secretId: string; }; +export type TSecretReference = { environment: string; secretPath: string }; + export type TFnSecretBulkInsert = { folderId: string; tx?: Knex; - inputSecrets: Array & { tags?: string[] }>; - secretDAL: Pick; + inputSecrets: Array & { tags?: string[]; references?: TSecretReference[] }>; + secretDAL: Pick; secretVersionDAL: Pick; secretTagDAL: Pick; secretVersionTagDAL: Pick; @@ -236,8 +238,11 @@ export type TFnSecretBulkInsert = { export type TFnSecretBulkUpdate = { folderId: string; projectId: string; - inputSecrets: { filter: Partial; data: TSecretsUpdate & { tags?: string[] } }[]; - secretDAL: Pick; + inputSecrets: { + filter: Partial; + data: TSecretsUpdate & { tags?: string[]; references?: TSecretReference[] }; + }[]; + secretDAL: Pick; secretVersionDAL: Pick; secretTagDAL: Pick; secretVersionTagDAL: Pick; @@ -294,6 +299,8 @@ export type TRemoveSecretReminderDTO = { repeatDays: number; }; +export type TBackFillSecretReferencesDTO = TProjectPermission; + // --- export type TCreateManySecretsRawFnFactory = {