diff --git a/backend/e2e-test/mocks/keystore.ts b/backend/e2e-test/mocks/keystore.ts index c85244129..965ea4e31 100644 --- a/backend/e2e-test/mocks/keystore.ts +++ b/backend/e2e-test/mocks/keystore.ts @@ -25,6 +25,9 @@ export const mockKeyStore = (): TKeyStoreFactory => { }, incrementBy: async () => { return 1; + }, + acquireLock: () => { + throw new Error("Not implemented"); } }; }; diff --git a/backend/src/db/migrations/20240430024805_secret-replication.ts b/backend/src/db/migrations/20240430024805_secret-replication.ts index 2cb6b4c83..d3ec4d81f 100644 --- a/backend/src/db/migrations/20240430024805_secret-replication.ts +++ b/backend/src/db/migrations/20240430024805_secret-replication.ts @@ -20,6 +20,12 @@ export async function up(knex: Knex): Promise { t.boolean("isReplicated"); }); } + + if (await knex.schema.hasTable(TableName.SecretApprovalRequestSecret)) { + await knex.schema.alterTable(TableName.SecretApprovalRequestSecret, (t) => { + t.boolean("isReplicated"); + }); + } } export async function down(knex: Knex): Promise { @@ -36,7 +42,13 @@ export async function down(knex: Knex): Promise { } if (await knex.schema.hasTable(TableName.SecretVersion)) { - await knex.schema.alterTable(TableName.Secret, (t) => { + await knex.schema.alterTable(TableName.SecretVersion, (t) => { + t.dropColumns("isReplicated"); + }); + } + + if (await knex.schema.hasTable(TableName.SecretApprovalRequestSecret)) { + await knex.schema.alterTable(TableName.SecretApprovalRequestSecret, (t) => { t.dropColumns("isReplicated"); }); } diff --git a/backend/src/db/schemas/secret-approval-requests-secrets.ts b/backend/src/db/schemas/secret-approval-requests-secrets.ts index b795b47b4..12b97e006 100644 --- a/backend/src/db/schemas/secret-approval-requests-secrets.ts +++ b/backend/src/db/schemas/secret-approval-requests-secrets.ts @@ -31,7 +31,8 @@ export const SecretApprovalRequestsSecretsSchema = z.object({ requestId: z.string().uuid(), op: z.string(), secretId: z.string().uuid().nullable().optional(), - secretVersion: z.string().uuid().nullable().optional() + secretVersion: z.string().uuid().nullable().optional(), + isReplicated: z.boolean().nullable().optional() }); export type TSecretApprovalRequestsSecrets = z.infer; diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts index 690d308d2..c9cbf3b15 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts @@ -16,8 +16,15 @@ import { TProjectDALFactory } from "@app/services/project/project-dal"; import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; import { TSecretDALFactory } from "@app/services/secret/secret-dal"; import { getAllNestedSecretReferences } from "@app/services/secret/secret-fns"; +import { + fnSecretBlindIndexCheck, + fnSecretBlindIndexCheckV2, + fnSecretBulkDelete, + fnSecretBulkInsert, + fnSecretBulkUpdate +} from "@app/services/secret/secret-fns"; import { TSecretQueueFactory } from "@app/services/secret/secret-queue"; -import { TSecretServiceFactory } from "@app/services/secret/secret-service"; +import { SecretOperations } from "@app/services/secret/secret-types"; import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal"; import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal"; import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal"; @@ -32,7 +39,6 @@ import { TSecretApprovalRequestReviewerDALFactory } from "./secret-approval-requ import { TSecretApprovalRequestSecretDALFactory } from "./secret-approval-request-secret-dal"; import { ApprovalStatus, - CommitType, RequestState, TApprovalRequestCountDTO, TGenerateSecretApprovalRequestDTO, @@ -48,7 +54,7 @@ type TSecretApprovalRequestServiceFactoryDep = { secretApprovalRequestDAL: TSecretApprovalRequestDALFactory; secretApprovalRequestSecretDAL: TSecretApprovalRequestSecretDALFactory; secretApprovalRequestReviewerDAL: TSecretApprovalRequestReviewerDALFactory; - folderDAL: Pick; + folderDAL: Pick; secretDAL: TSecretDALFactory; secretTagDAL: Pick; secretBlindIndexDAL: Pick; @@ -56,16 +62,7 @@ type TSecretApprovalRequestServiceFactoryDep = { secretVersionDAL: Pick; secretVersionTagDAL: Pick; projectDAL: Pick; - projectBotService: Pick; - secretService: Pick< - TSecretServiceFactory, - | "fnSecretBulkInsert" - | "fnSecretBulkUpdate" - | "fnSecretBlindIndexCheck" - | "fnSecretBulkDelete" - | "fnSecretBlindIndexCheckV2" - >; - secretQueueService: Pick; + secretQueueService: Pick; }; export type TSecretApprovalRequestServiceFactory = ReturnType; @@ -82,7 +79,6 @@ export const secretApprovalRequestServiceFactory = ({ projectDAL, permissionService, snapshotService, - secretService, secretVersionDAL, secretQueueService, projectBotService @@ -302,11 +298,12 @@ export const secretApprovalRequestServiceFactory = ({ const secretApprovalSecrets = await secretApprovalRequestSecretDAL.findByRequestId(secretApprovalRequest.id); if (!secretApprovalSecrets) throw new BadRequestError({ message: "No secrets found" }); - const conflicts: Array<{ secretId: string; op: CommitType }> = []; - let secretCreationCommits = secretApprovalSecrets.filter(({ op }) => op === CommitType.Create); + const conflicts: Array<{ secretId: string; op: SecretOperations }> = []; + let secretCreationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Create); if (secretCreationCommits.length) { - const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await secretService.fnSecretBlindIndexCheckV2({ + const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await fnSecretBlindIndexCheckV2({ folderId, + secretDAL, inputSecrets: secretCreationCommits.map(({ secretBlindIndex }) => { if (!secretBlindIndex) { throw new BadRequestError({ @@ -319,17 +316,19 @@ export const secretApprovalRequestServiceFactory = ({ secretCreationCommits .filter(({ secretBlindIndex }) => conflictGroupByBlindIndex[secretBlindIndex || ""]) .forEach((el) => { - conflicts.push({ op: CommitType.Create, secretId: el.id }); + conflicts.push({ op: SecretOperations.Create, secretId: el.id }); }); secretCreationCommits = secretCreationCommits.filter( ({ secretBlindIndex }) => !conflictGroupByBlindIndex[secretBlindIndex || ""] ); } - let secretUpdationCommits = secretApprovalSecrets.filter(({ op }) => op === CommitType.Update); + let secretUpdationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Update); if (secretUpdationCommits.length) { - const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await secretService.fnSecretBlindIndexCheckV2({ + const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await fnSecretBlindIndexCheckV2({ folderId, + secretDAL, + userId: "", inputSecrets: secretUpdationCommits .filter(({ secretBlindIndex, secret }) => secret && secret.secretBlindIndex !== secretBlindIndex) .map(({ secretBlindIndex }) => { @@ -347,7 +346,7 @@ export const secretApprovalRequestServiceFactory = ({ (secretBlindIndex && conflictGroupByBlindIndex[secretBlindIndex]) || !secretId ) .forEach((el) => { - conflicts.push({ op: CommitType.Update, secretId: el.id }); + conflicts.push({ op: SecretOperations.Update, secretId: el.id }); }); secretUpdationCommits = secretUpdationCommits.filter( @@ -356,11 +355,11 @@ export const secretApprovalRequestServiceFactory = ({ ); } - const secretDeletionCommits = secretApprovalSecrets.filter(({ op }) => op === CommitType.Delete); - const botKey = await projectBotService.getBotKey(projectId).catch(() => null); + const secretDeletionCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Delete); + const mergeStatus = await secretApprovalRequestDAL.transaction(async (tx) => { const newSecrets = secretCreationCommits.length - ? await secretService.fnSecretBulkInsert({ + ? await fnSecretBulkInsert({ tx, folderId, inputSecrets: secretCreationCommits.map((el) => ({ @@ -380,7 +379,8 @@ export const secretApprovalRequestServiceFactory = ({ "secretReminderRepeatDays", "algorithm", "keyEncoding", - "secretBlindIndex" + "secretBlindIndex", + "isReplicated" ]), tags: el?.tags.map(({ id }) => id), version: 1, @@ -403,7 +403,7 @@ export const secretApprovalRequestServiceFactory = ({ }) : []; const updatedSecrets = secretUpdationCommits.length - ? await secretService.fnSecretBulkUpdate({ + ? await fnSecretBulkUpdate({ folderId, projectId, tx, @@ -425,6 +425,7 @@ export const secretApprovalRequestServiceFactory = ({ "secretKeyTag", "secretKeyIV", "metadata", + "isReplicated", "skipMultilineEncoding", "secretReminderNote", "secretReminderRepeatDays", @@ -449,11 +450,13 @@ export const secretApprovalRequestServiceFactory = ({ }) : []; const deletedSecret = secretDeletionCommits.length - ? await secretService.fnSecretBulkDelete({ + ? await fnSecretBulkDelete({ projectId, folderId, tx, actorId: "", + secretDAL, + secretQueueService, inputSecrets: secretDeletionCommits.map(({ secretBlindIndex }) => { if (!secretBlindIndex) { throw new BadRequestError({ @@ -480,12 +483,33 @@ export const secretApprovalRequestServiceFactory = ({ }; }); await snapshotService.performSnapshot(folderId); - const folder = await folderDAL.findById(folderId); + const [folder] = await folderDAL.findSecretPathByFolderIds(projectId, [folderId]); // TODO(akhilmhdh-pg): change query to do secret path from folder await secretQueueService.syncSecrets({ projectId, - secretPath: "/", - environment: folder?.environment.envSlug as string + secretPath: folder.path, + environmentSlug: folder.environmentSlug, + folderId: folder.id, + membershipId: membership.id, + environmentId: folder.envId, + secrets: mergeStatus.secrets.created + .map(({ id, version }) => ({ + operation: SecretOperations.Create, + version, + id + })) + .concat( + mergeStatus.secrets.updated.map(({ id, version }) => ({ + operation: SecretOperations.Update, + version, + id + })), + mergeStatus.secrets.deleted.map(({ id, version }) => ({ + operation: SecretOperations.Delete, + version, + id + })) + ) }); return mergeStatus; }; @@ -533,9 +557,9 @@ export const secretApprovalRequestServiceFactory = ({ const commits: Omit[] = []; const commitTagIds: Record = {}; // for created secret approval change - const createdSecrets = data[CommitType.Create]; + const createdSecrets = data[SecretOperations.Create]; if (createdSecrets && createdSecrets?.length) { - const { keyName2BlindIndex } = await secretService.fnSecretBlindIndexCheck({ + const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({ inputSecrets: createdSecrets, folderId, isNew: true, @@ -546,7 +570,7 @@ export const secretApprovalRequestServiceFactory = ({ commits.push( ...createdSecrets.map(({ secretName, ...el }) => ({ ...el, - op: CommitType.Create as const, + op: SecretOperations.Create as const, version: 1, secretBlindIndex: keyName2BlindIndex[secretName], algorithm: SecretEncryptionAlgo.AES_256_GCM, @@ -558,12 +582,12 @@ export const secretApprovalRequestServiceFactory = ({ }); } // not secret approval for update operations - const updatedSecrets = data[CommitType.Update]; + const updatedSecrets = data[SecretOperations.Update]; if (updatedSecrets && updatedSecrets?.length) { // get all blind index // Find all those secrets // if not throw not found - const { keyName2BlindIndex, secrets: secretsToBeUpdated } = await secretService.fnSecretBlindIndexCheck({ + const { keyName2BlindIndex, secrets: secretsToBeUpdated } = await fnSecretBlindIndexCheck({ inputSecrets: updatedSecrets, folderId, isNew: false, @@ -574,7 +598,7 @@ export const secretApprovalRequestServiceFactory = ({ // now find any secret that needs to update its name // same process as above const nameUpdatedSecrets = updatedSecrets.filter(({ newSecretName }) => Boolean(newSecretName)); - const { keyName2BlindIndex: newKeyName2BlindIndex } = await secretService.fnSecretBlindIndexCheck({ + const { keyName2BlindIndex: newKeyName2BlindIndex } = await fnSecretBlindIndexCheck({ inputSecrets: nameUpdatedSecrets, folderId, isNew: true, @@ -599,7 +623,7 @@ export const secretApprovalRequestServiceFactory = ({ return { ...latestSecretVersions[secretId], ...el, - op: CommitType.Update as const, + op: SecretOperations.Update as const, secret: secretId, secretVersion: latestSecretVersions[secretId].id, secretBlindIndex, @@ -609,12 +633,12 @@ export const secretApprovalRequestServiceFactory = ({ ); } // deleted secrets - const deletedSecrets = data[CommitType.Delete]; + const deletedSecrets = data[SecretOperations.Delete]; if (deletedSecrets && deletedSecrets.length) { // get all blind index // Find all those secrets // if not throw not found - const { keyName2BlindIndex, secrets } = await secretService.fnSecretBlindIndexCheck({ + const { keyName2BlindIndex, secrets } = await fnSecretBlindIndexCheck({ inputSecrets: deletedSecrets, folderId, isNew: false, @@ -635,7 +659,7 @@ export const secretApprovalRequestServiceFactory = ({ if (!latestSecretVersions[secretId].secretBlindIndex) throw new BadRequestError({ message: "Failed to find secret blind index" }); return { - op: CommitType.Delete as const, + op: SecretOperations.Delete as const, ...latestSecretVersions[secretId], secretBlindIndex: latestSecretVersions[secretId].secretBlindIndex as string, secret: secretId, diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-types.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-types.ts index 008b977e6..1fbb75418 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-types.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-types.ts @@ -1,11 +1,6 @@ import { TImmutableDBKeys, TSecretApprovalPolicies, TSecretApprovalRequestsSecrets } from "@app/db/schemas"; import { TProjectPermission } from "@app/lib/types"; - -export enum CommitType { - Create = "create", - Update = "update", - Delete = "delete" -} +import { SecretOperations } from "@app/services/secret/secret-types"; export enum RequestState { Open = "open", @@ -18,14 +13,14 @@ export enum ApprovalStatus { REJECTED = "rejected" } -type TApprovalCreateSecret = Omit< +export type TApprovalCreateSecret = Omit< TSecretApprovalRequestsSecrets, TImmutableDBKeys | "version" | "algorithm" | "keyEncoding" | "requestId" | "op" | "secretVersion" | "secretBlindIndex" > & { secretName: string; tagIds?: string[]; }; -type TApprovalUpdateSecret = Partial & { +export type TApprovalUpdateSecret = Partial & { secretName: string; newSecretName?: string; tagIds?: string[]; @@ -36,9 +31,9 @@ export type TGenerateSecretApprovalRequestDTO = { secretPath: string; policy: TSecretApprovalPolicies; data: { - [CommitType.Create]?: TApprovalCreateSecret[]; - [CommitType.Update]?: TApprovalUpdateSecret[]; - [CommitType.Delete]?: { secretName: string }[]; + [SecretOperations.Create]?: TApprovalCreateSecret[]; + [SecretOperations.Update]?: TApprovalUpdateSecret[]; + [SecretOperations.Delete]?: { secretName: string }[]; }; } & TProjectPermission; diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index 32af011ec..f0ce62318 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -7,7 +7,7 @@ import { TScanFullRepoEventPayload, TScanPushEventPayload } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types"; -import { TSyncSecretReplicationDTO } from "@app/services/secret-replication/secret-replication-types"; +import { TSyncSecretsDTO } from "@app/services/secret/secret-types"; export enum QueueName { SecretRotation = "secret-rotation", @@ -23,7 +23,8 @@ export enum QueueName { SecretPushEventScan = "secret-push-event-scan", UpgradeProjectToGhost = "upgrade-project-to-ghost", DynamicSecretRevocation = "dynamic-secret-revocation", - SecretReplication = "secret-replication" + SecretReplication = "secret-replication", + SecretSync = "secret-sync" // parent queue to push integration sync, webhook, and secret replication } export enum QueueJobs { @@ -40,7 +41,8 @@ export enum QueueJobs { UpgradeProjectToGhost = "upgrade-project-to-ghost-job", DynamicSecretRevocation = "dynamic-secret-revocation", DynamicSecretPruning = "dynamic-secret-pruning", - SecretReplication = "secret-replication" + SecretReplication = "secret-replication", + SecretSync = "secret-sync" // parent queue to push integration sync, webhook, and secret replication } export type TQueueJobTypes = { @@ -121,7 +123,11 @@ export type TQueueJobTypes = { }; [QueueName.SecretReplication]: { name: QueueJobs.SecretReplication; - payload: TSyncSecretReplicationDTO; + payload: Omit; + }; + [QueueName.SecretSync]: { + name: QueueJobs.SecretSync; + payload: TSyncSecretsDTO; }; }; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index c3fdb87cf..605678011 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -243,8 +243,8 @@ export const registerRoutes = async ( const sapApproverDAL = secretApprovalPolicyApproverDALFactory(db); const secretApprovalPolicyDAL = secretApprovalPolicyDALFactory(db); const secretApprovalRequestDAL = secretApprovalRequestDALFactory(db); - const sarReviewerDAL = secretApprovalRequestReviewerDALFactory(db); - const sarSecretDAL = secretApprovalRequestSecretDALFactory(db); + const secretApprovalRequestReviewerDAL = secretApprovalRequestReviewerDALFactory(db); + const secretApprovalRequestSecretDAL = secretApprovalRequestSecretDALFactory(db); const secretRotationDAL = secretRotationDALFactory(db); const snapshotDAL = snapshotDALFactory(db); @@ -492,7 +492,7 @@ export const registerRoutes = async ( projectBotDAL, projectMembershipDAL, secretApprovalRequestDAL, - secretApprovalSecretDAL: sarSecretDAL, + secretApprovalSecretDAL: secretApprovalRequestSecretDAL, projectUserMembershipRoleDAL }); @@ -603,17 +603,6 @@ export const registerRoutes = async ( secretDAL, secretBlindIndexDAL }); - const secretReplicationService = secretReplicationServiceFactory({ - secretTagDAL, - secretVersionTagDAL, - secretDAL, - secretVersionDAL, - secretImportDAL, - keyStore, - queueService, - secretReplicationDAL, - folderDAL - }); const secretService = secretServiceFactory({ folderDAL, secretVersionDAL, @@ -625,7 +614,6 @@ export const registerRoutes = async ( secretTagDAL, snapshotService, secretQueueService, - secretReplicationService, secretImportDAL, projectEnvDAL, projectBotService @@ -636,19 +624,18 @@ export const registerRoutes = async ( secretSharingDAL }); - const sarService = secretApprovalRequestServiceFactory({ + const secretApprovalRequestService = secretApprovalRequestServiceFactory({ permissionService, projectBotService, folderDAL, secretDAL, secretTagDAL, - secretApprovalRequestSecretDAL: sarSecretDAL, - secretApprovalRequestReviewerDAL: sarReviewerDAL, + secretApprovalRequestSecretDAL, + secretApprovalRequestReviewerDAL, projectDAL, secretVersionDAL, secretBlindIndexDAL, secretApprovalRequestDAL, - secretService, snapshotService, secretVersionTagDAL, secretQueueService @@ -677,6 +664,21 @@ export const registerRoutes = async ( accessApprovalPolicyApproverDAL }); + const secretReplicationService = secretReplicationServiceFactory({ + secretTagDAL, + secretVersionTagDAL, + secretDAL, + secretVersionDAL, + secretImportDAL, + keyStore, + queueService, + secretReplicationDAL, + folderDAL, + secretApprovalPolicyService, + secretBlindIndexDAL, + secretApprovalRequestDAL, + secretApprovalRequestSecretDAL + }); const secretRotationQueue = secretRotationQueueFactory({ telemetryService, secretRotationDAL, @@ -858,10 +860,10 @@ export const registerRoutes = async ( identityGcpAuth: identityGcpAuthService, identityAwsAuth: identityAwsAuthService, identityAzureAuth: identityAzureAuthService, - secretApprovalPolicy: sapService, accessApprovalPolicy: accessApprovalPolicyService, accessApprovalRequest: accessApprovalRequestService, - secretApprovalRequest: sarService, + secretApprovalPolicy: secretApprovalPolicyService, + secretApprovalRequest: secretApprovalRequestService, secretRotation: secretRotationService, dynamicSecret: dynamicSecretService, dynamicSecretLease: dynamicSecretLeaseService, diff --git a/backend/src/server/routes/v3/secret-router.ts b/backend/src/server/routes/v3/secret-router.ts index 6fa574a69..05db617b9 100644 --- a/backend/src/server/routes/v3/secret-router.ts +++ b/backend/src/server/routes/v3/secret-router.ts @@ -9,7 +9,6 @@ import { ServiceTokenScopes } from "@app/db/schemas"; import { EventType } from "@app/ee/services/audit-log/audit-log-types"; -import { CommitType } from "@app/ee/services/secret-approval-request/secret-approval-request-types"; import { RAW_SECRETS, SECRETS } from "@app/lib/api-docs"; import { BadRequestError } from "@app/lib/errors"; import { removeTrailingSlash } from "@app/lib/fn"; @@ -19,6 +18,7 @@ import { getUserAgentType } from "@app/server/plugins/audit-log"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { ActorType, AuthMode } from "@app/services/auth/auth-type"; import { ProjectFilterType } from "@app/services/project/project-types"; +import { SecretOperations } from "@app/services/secret/secret-types"; import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types"; import { secretRawSchema } from "../sanitizedSchemas"; @@ -902,7 +902,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { projectId, policy, data: { - [CommitType.Create]: [ + [SecretOperations.Create]: [ { secretName: req.params.secretName, secretValueCiphertext, @@ -1084,7 +1084,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { projectId, policy, data: { - [CommitType.Update]: [ + [SecretOperations.Update]: [ { secretName: req.params.secretName, newSecretName, @@ -1234,7 +1234,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { projectId, policy, data: { - [CommitType.Delete]: [ + [SecretOperations.Delete]: [ { secretName: req.params.secretName } @@ -1364,7 +1364,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { projectId, policy, data: { - [CommitType.Create]: inputSecrets + [SecretOperations.Create]: inputSecrets } }); @@ -1491,7 +1491,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { projectId, policy, data: { - [CommitType.Update]: inputSecrets.filter(({ type }) => type === "shared") + [SecretOperations.Update]: inputSecrets.filter(({ type }) => type === "shared") } }); @@ -1606,7 +1606,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { projectId, policy, data: { - [CommitType.Delete]: inputSecrets.filter(({ type }) => type === "shared") + [SecretOperations.Delete]: inputSecrets.filter(({ type }) => type === "shared") } }); await server.services.auditLog.createAuditLog({ diff --git a/backend/src/services/secret-import/secret-import-service.ts b/backend/src/services/secret-import/secret-import-service.ts index 2aba9d89c..5503470e4 100644 --- a/backend/src/services/secret-import/secret-import-service.ts +++ b/backend/src/services/secret-import/secret-import-service.ts @@ -111,7 +111,8 @@ export const secretImportServiceFactory = ({ await secretQueueService.syncSecrets({ secretPath: secImport.importPath, projectId, - environment: importEnv.slug + environmentSlug: importEnv.slug, + excludeReplication: true }); return { ...secImport, importEnv }; @@ -223,7 +224,8 @@ export const secretImportServiceFactory = ({ await secretQueueService.syncSecrets({ secretPath: path, projectId, - environment + environmentSlug: environment, + excludeReplication: true }); return secImport; diff --git a/backend/src/services/secret-replication/secret-replication-service.ts b/backend/src/services/secret-replication/secret-replication-service.ts index 2a98193ea..3b8c12f58 100644 --- a/backend/src/services/secret-replication/secret-replication-service.ts +++ b/backend/src/services/secret-replication/secret-replication-service.ts @@ -1,28 +1,40 @@ +import { TSecretApprovalPolicyServiceFactory } from "@app/ee/services/secret-approval-policy/secret-approval-policy-service"; +import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal"; +import { TSecretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal"; import { TKeyStoreFactory } from "@app/keystore/keystore"; import { groupBy } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; +import { alphaNumericNanoId } from "@app/lib/nanoid"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; import { TSecretDALFactory } from "../secret/secret-dal"; import { fnSecretBulkInsert, fnSecretBulkUpdate } from "../secret/secret-fns"; +import { SecretOperations, TSyncSecretsDTO } from "../secret/secret-types"; import { TSecretVersionDALFactory } from "../secret/secret-version-dal"; import { TSecretVersionTagDALFactory } from "../secret/secret-version-tag-dal"; +import { TSecretBlindIndexDALFactory } from "../secret-blind-index/secret-blind-index-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TSecretImportDALFactory } from "../secret-import/secret-import-dal"; import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; import { TSecretReplicationDALFactory } from "./secret-replication-dal"; -import { SecretReplicationOperations, TSyncSecretReplicationDTO } from "./secret-replication-types"; type TSecretReplicationServiceFactoryDep = { secretReplicationDAL: TSecretReplicationDALFactory; secretDAL: Pick; + secretVersionDAL: Pick; secretImportDAL: Pick; - folderDAL: Pick; - secretVersionDAL: Pick; - secretTagDAL: Pick; + folderDAL: Pick; secretVersionTagDAL: Pick; queueService: Pick; + secretApprovalPolicyService: Pick; keyStore: Pick; + secretBlindIndexDAL: Pick; + secretTagDAL: Pick; + secretApprovalRequestDAL: Pick; + secretApprovalRequestSecretDAL: Pick< + TSecretApprovalRequestSecretDALFactory, + "insertMany" | "insertApprovalSecretTags" + >; }; export type TSecretReplicationServiceFactory = ReturnType; @@ -44,18 +56,21 @@ export const secretReplicationServiceFactory = ({ keyStore, secretVersionTagDAL, secretTagDAL, - folderDAL + folderDAL, + secretApprovalPolicyService, + secretApprovalRequestSecretDAL, + secretApprovalRequestDAL }: TSecretReplicationServiceFactoryDep) => { queueService.start(QueueName.SecretReplication, async (job) => { logger.info(job.data, "Replication started"); - const { secrets, folderId, secretPath, environmentId, projectId } = job.data; + const { secrets, folderId, secretPath, environmentId, projectId, membershipId } = job.data; const secretImports = await secretImportDAL.find({ importPath: secretPath, importEnv: environmentId, isReplication: true }); console.log(">>>> Secret Imports replics ", secretImports.length, secretPath, environmentId); - if (!secretImports.length) return; + if (!secretImports.length || !secrets.length) return; // unfiltered secrets to be replicated console.log(secrets.length); @@ -75,7 +90,8 @@ export const secretReplicationServiceFactory = ({ try { /* eslint-disable no-await-in-loop */ for (const secretImport of secretImports) { - const importFolderId = secretImport.folderId; + const [importedFolder] = await folderDAL.findSecretPathByFolderIds(projectId, [secretImport.folderId]); + const importFolderId = importedFolder.id; const localSecrets = await secretDAL.find({ $in: { secretBlindIndex: replicatedSecrets.map(({ secretBlindIndex }) => secretBlindIndex) }, @@ -86,14 +102,14 @@ export const secretReplicationServiceFactory = ({ const locallyCreatedSecrets = secrets.filter(({ operation, id }) => { return ( - (operation === SecretReplicationOperations.Create || operation === SecretReplicationOperations.Update) && + (operation === SecretOperations.Create || operation === SecretOperations.Update) && !localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0] ); }); const locallyUpdatedSecrets = secrets.filter( ({ operation, id }) => - (operation === SecretReplicationOperations.Create || operation === SecretReplicationOperations.Update) && + (operation === SecretOperations.Create || operation === SecretOperations.Update) && localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0] ); @@ -102,35 +118,48 @@ export const secretReplicationServiceFactory = ({ console.log("update", locallyUpdatedSecrets); console.log("local board", localSecrets); - const locallyDeletedSecrets = secrets - .filter( - ({ operation, id }) => - operation === SecretReplicationOperations.Delete && - Boolean(replicatedSecretsGroupBySecretId[id]?.[0]?.secretBlindIndex) && - localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0] - ) - .map( - ({ id }) => - localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string][0] - ); + const locallyDeletedSecrets = secrets.filter( + ({ operation, id }) => + operation === SecretOperations.Delete && + Boolean(replicatedSecretsGroupBySecretId[id]?.[0]?.secretBlindIndex) && + localSecretsGroupedByBlindIndex[replicatedSecretsGroupBySecretId[id][0].secretBlindIndex as string]?.[0] + ); - let nestedImportSecrets: TSyncSecretReplicationDTO["secrets"] = []; - await secretReplicationDAL.transaction(async (tx) => { - if (locallyCreatedSecrets.length) { - const newSecrets = await fnSecretBulkInsert({ - folderId: importFolderId, - secretVersionDAL, - secretDAL, - tx, - secretTagDAL, - secretVersionTagDAL, - inputSecrets: locallyCreatedSecrets.map(({ id }) => { + const policy = await secretApprovalPolicyService.getSecretApprovalPolicy( + projectId, + importedFolder.environmentSlug, + importedFolder.path + ); + // this means it should be a approval request rather than direct replication + if (policy) { + const localSecretsLatestVersions = localSecrets.map(({ id }) => id); + const latestSecretVersions = await secretVersionDAL.findLatestVersionMany( + importFolderId, + localSecretsLatestVersions + ); + await secretApprovalRequestDAL.transaction(async (tx) => { + const approvalRequestDoc = await secretApprovalRequestDAL.create( + { + folderId: importFolderId, + slug: alphaNumericNanoId(), + policyId: policy.id, + status: "open", + hasMerged: false, + committerId: membershipId + }, + tx + ); + const commits = locallyCreatedSecrets + .concat(locallyUpdatedSecrets) + .concat(locallyDeletedSecrets) + .map(({ id, operation }) => { const doc = replicatedSecretsGroupBySecretId[id][0]; + const localSecret = localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string]?.[0]; return { + op: operation, keyEncoding: doc.keyEncoding, algorithm: doc.algorithm, - folderId, - type: doc.type, + requestId: approvalRequestDoc.id, metadata: doc.metadata, secretKeyIV: doc.secretKeyIV, secretKeyTag: doc.secretKeyTag, @@ -143,31 +172,31 @@ export const secretReplicationServiceFactory = ({ secretCommentTag: doc.secretCommentTag, secretCommentCiphertext: doc.secretCommentCiphertext, isReplicated: true, - skipMultilineEncoding: doc.skipMultilineEncoding + skipMultilineEncoding: doc.skipMultilineEncoding, + // except create operation other two needs the secret id and version id + ...(operation !== SecretOperations.Create + ? { secretId: localSecret.id, secretVersion: latestSecretVersions[localSecret.id].id } + : {}) }; - }) - }); - nestedImportSecrets = nestedImportSecrets.concat( - ...newSecrets.map(({ id, version }) => ({ operation: SecretReplicationOperations.Create, version, id })) - ); - } - if (locallyUpdatedSecrets.length) { - const newSecrets = await fnSecretBulkUpdate({ - projectId, - folderId: importFolderId, - secretVersionDAL, - secretDAL, - tx, - secretTagDAL, - secretVersionTagDAL, - inputSecrets: locallyUpdatedSecrets.map(({ id }) => { - const doc = replicatedSecretsGroupBySecretId[id][0]; - return { - filter: { - folderId: importFolderId, - id: localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string][0].id - }, - data: { + }); + const approvalCommits = await secretApprovalRequestSecretDAL.insertMany(commits, tx); + + return { ...approvalRequestDoc, commits: approvalCommits }; + }); + } else { + let nestedImportSecrets: TSyncSecretsDTO["secrets"] = []; + await secretReplicationDAL.transaction(async (tx) => { + if (locallyCreatedSecrets.length) { + const newSecrets = await fnSecretBulkInsert({ + folderId: importFolderId, + secretVersionDAL, + secretDAL, + tx, + secretTagDAL, + secretVersionTagDAL, + inputSecrets: locallyCreatedSecrets.map(({ id }) => { + const doc = replicatedSecretsGroupBySecretId[id][0]; + return { keyEncoding: doc.keyEncoding, algorithm: doc.algorithm, type: doc.type, @@ -184,39 +213,80 @@ export const secretReplicationServiceFactory = ({ secretCommentCiphertext: doc.secretCommentCiphertext, isReplicated: true, skipMultilineEncoding: doc.skipMultilineEncoding - } - }; - }) - }); - nestedImportSecrets = nestedImportSecrets.concat( - ...newSecrets.map(({ id, version }) => ({ operation: SecretReplicationOperations.Update, version, id })) - ); - } - if (locallyDeletedSecrets.length) { - const newSecrets = await secretDAL.delete( - { - $in: { - id: locallyDeletedSecrets.map(({ id }) => id) + }; + }) + }); + nestedImportSecrets = nestedImportSecrets.concat( + newSecrets.map(({ id, version }) => ({ operation: SecretOperations.Create, version, id })) + ); + } + if (locallyUpdatedSecrets.length) { + const newSecrets = await fnSecretBulkUpdate({ + projectId, + folderId: importFolderId, + secretVersionDAL, + secretDAL, + tx, + secretTagDAL, + secretVersionTagDAL, + inputSecrets: locallyUpdatedSecrets.map(({ id }) => { + const doc = replicatedSecretsGroupBySecretId[id][0]; + return { + filter: { + folderId: importFolderId, + id: localSecretsGroupedByBlindIndex[doc.secretBlindIndex as string][0].id + }, + data: { + keyEncoding: doc.keyEncoding, + algorithm: doc.algorithm, + type: doc.type, + metadata: doc.metadata, + secretKeyIV: doc.secretKeyIV, + secretKeyTag: doc.secretKeyTag, + secretKeyCiphertext: doc.secretKeyCiphertext, + secretValueIV: doc.secretValueIV, + secretValueTag: doc.secretValueTag, + secretValueCiphertext: doc.secretValueCiphertext, + secretBlindIndex: doc.secretBlindIndex, + secretCommentIV: doc.secretCommentIV, + secretCommentTag: doc.secretCommentTag, + secretCommentCiphertext: doc.secretCommentCiphertext, + isReplicated: true, + skipMultilineEncoding: doc.skipMultilineEncoding + } + }; + }) + }); + nestedImportSecrets = nestedImportSecrets.concat( + newSecrets.map(({ id, version }) => ({ operation: SecretOperations.Update, version, id })) + ); + } + if (locallyDeletedSecrets.length) { + const newSecrets = await secretDAL.delete( + { + $in: { + id: locallyDeletedSecrets.map(({ id }) => id) + }, + isReplicated: true, + folderId: importFolderId }, - isReplicated: true, - folderId: importFolderId - }, - tx - ); - nestedImportSecrets = nestedImportSecrets.concat( - ...newSecrets.map(({ id, version }) => ({ operation: SecretReplicationOperations.Delete, version, id })) - ); - } - }); - const [folder] = await folderDAL.findSecretPathByFolderIds(projectId, [secretImport.folderId]); - console.log("Environment ID -> slug", folder.envId, folder.environmentSlug); - await queueService.queue(QueueName.SecretReplication, QueueJobs.SecretReplication, { - folderId: folder.id, - projectId, - secrets: nestedImportSecrets, - secretPath: folder.path, - environmentId: folder.envId - }); + tx + ); + nestedImportSecrets = nestedImportSecrets.concat( + newSecrets.map(({ id, version }) => ({ operation: SecretOperations.Delete, version, id })) + ); + } + }); + console.log("Environment ID -> slug", importedFolder.envId, importedFolder.environmentSlug); + await queueService.queue(QueueName.SecretReplication, QueueJobs.SecretReplication, { + folderId: importedFolder.id, + projectId, + secrets: nestedImportSecrets, + secretPath: importedFolder.path, + environmentId: importedFolder.envId, + membershipId + }); + } } await secretVersionDAL.update({ $in: { id: replicatedSecrets.map(({ id }) => id) } }, { isReplicated: true }); /* eslint-enable no-await-in-loop */ @@ -228,20 +298,4 @@ export const secretReplicationServiceFactory = ({ queueService.listen(QueueName.SecretReplication, "failed", async (job, err) => { logger.error(err, "Failed to replicate secret", job?.data); }); - - const replicate = async (data: TSyncSecretReplicationDTO) => { - await queueService.queue(QueueName.SecretReplication, QueueJobs.SecretReplication, data, { - attempts: 3, - backoff: { - type: "exponential", - delay: 1000 - }, - removeOnComplete: true, - removeOnFail: true - }); - }; - - return { - replicate - }; }; diff --git a/backend/src/services/secret-replication/secret-replication-types.ts b/backend/src/services/secret-replication/secret-replication-types.ts index 509bf5b02..1b32f1f4a 100644 --- a/backend/src/services/secret-replication/secret-replication-types.ts +++ b/backend/src/services/secret-replication/secret-replication-types.ts @@ -1,17 +1,3 @@ -export enum SecretReplicationOperations { - Create = "create", - Update = "update", - Delete = "delete" -} - export type TSyncSecretReplicationDTO = { - secretPath: string; - projectId: string; - environmentId: string; - folderId: string; - secrets: { - operation: SecretReplicationOperations; - id: string; - version: number; - }[]; + id: string; }; diff --git a/backend/src/services/secret/secret-fns.ts b/backend/src/services/secret/secret-fns.ts index e59804bec..f21144cfc 100644 --- a/backend/src/services/secret/secret-fns.ts +++ b/backend/src/services/secret/secret-fns.ts @@ -32,6 +32,8 @@ import { TCreateManySecretsRawFn, TCreateManySecretsRawFnFactory, TFnSecretBlindIndexCheck, + TFnSecretBlindIndexCheckV2, + TFnSecretBulkDelete, TFnSecretBulkInsert, TFnSecretBulkUpdate, TUpdateManySecretsRawFn, @@ -396,6 +398,30 @@ export const decryptSecretRaw = ( }; }; +// this is used when secret blind index already exist +// mainly for secret approval +export const fnSecretBlindIndexCheckV2 = async ({ + inputSecrets, + folderId, + userId, + secretDAL +}: TFnSecretBlindIndexCheckV2) => { + if (inputSecrets.some(({ type }) => type === SecretType.Personal) && !userId) { + throw new BadRequestError({ message: "Missing user id for personal secret" }); + } + const secrets = await secretDAL.findByBlindIndexes( + folderId, + inputSecrets.map(({ secretBlindIndex, type }) => ({ + blindIndex: secretBlindIndex, + type: type || SecretType.Shared + })), + userId + ); + const secsGroupedByBlindIndex = groupBy(secrets, (i) => i.secretBlindIndex as string); + + return { secsGroupedByBlindIndex, secrets }; +}; + /** * Grabs and processes nested secret references from a string * @@ -598,6 +624,35 @@ export const fnSecretBulkUpdate = async ({ return newSecrets.map((secret) => ({ ...secret, _id: secret.id })); }; +export const fnSecretBulkDelete = async ({ + folderId, + inputSecrets, + tx, + actorId, + secretDAL, + secretQueueService +}: TFnSecretBulkDelete) => { + const deletedSecrets = await secretDAL.deleteMany( + inputSecrets.map(({ type, secretBlindIndex }) => ({ + blindIndex: secretBlindIndex, + type + })), + folderId, + actorId, + tx + ); + + await Promise.allSettled( + deletedSecrets + .filter(({ secretReminderRepeatDays }) => Boolean(secretReminderRepeatDays)) + .map(({ id, secretReminderRepeatDays }) => + secretQueueService.removeSecretReminder({ secretId: id, repeatDays: secretReminderRepeatDays as number }) + ) + ); + + return deletedSecrets; +}; + export const createManySecretsRawFnFactory = ({ projectDAL, projectBotDAL, diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index f3e3f1731..20dff881c 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -28,7 +28,12 @@ import { TWebhookDALFactory } from "../webhook/webhook-dal"; import { fnTriggerWebhook } from "../webhook/webhook-fns"; import { TSecretDALFactory } from "./secret-dal"; import { interpolateSecrets } from "./secret-fns"; -import { TCreateSecretReminderDTO, THandleReminderDTO, TRemoveSecretReminderDTO } from "./secret-types"; +import { + TCreateSecretReminderDTO, + THandleReminderDTO, + TRemoveSecretReminderDTO, + TSyncSecretsDTO +} from "./secret-types"; export type TSecretQueueFactory = ReturnType; type TSecretQueueFactoryDep = { @@ -116,21 +121,17 @@ export const secretQueueFactory = ({ }); }; - const syncSecrets = async ({ - deDupeQueue = {}, - ...dto - }: TGetSecrets & { depth?: number; deDupeQueue?: Record }) => { - const deDuplicationKey = uniqueIntegrationKey(dto.environment, dto.secretPath); + const syncSecrets = async ({_deDupeQueue:deDupeQueue = {},_depth = 0, ...dto}: TSyncSecretsDTO) => { + logger.info( + `syncSecrets: syncing project secrets where [projectId=${dto.projectId}] [environment=${dto.environmentSlug}] [path=${dto.secretPath}]` + ); + const deDuplicationKey = uniqueIntegrationKey(dto.environmentSlug, dto.secretPath); if (deDupeQueue?.[deDuplicationKey]) { return; } // eslint-disable-next-line deDupeQueue[deDuplicationKey] = true; - logger.info( - `syncSecrets: syncing project secrets where [projectId=${dto.projectId}] [environment=${dto.environment}] [path=${dto.secretPath}]` - ); - await queueService.queue(QueueName.SecretWebhook, QueueJobs.SecWebhook, dto, { - jobId: `secret-webhook-${dto.environment}-${dto.projectId}-${dto.secretPath}`, + await queueService.queue(QueueName.SecretSync, QueueJobs.SecretSync, dto as TSyncSecretsDTO, { removeOnFail: true, removeOnComplete: true, delay: 1000, @@ -140,7 +141,6 @@ export const secretQueueFactory = ({ delay: 3000 } }); - await syncIntegrations({ ...dto, deDupeQueue }); }; const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => { @@ -333,6 +333,52 @@ export const secretQueueFactory = ({ return content; }; + queueService.start(QueueName.SecretSync, async (job) => { + const { + secretPath, + environmentId, + projectId, + environmentSlug: environment, + secrets, + folderId, + excludeReplication, + membershipId + } = job.data; + await queueService.queue( + QueueName.SecretWebhook, + QueueJobs.SecWebhook, + { environment, projectId, secretPath }, + { + jobId: `secret-webhook-${environmentId}-${projectId}-${secretPath}`, + removeOnFail: { count: 5 }, + removeOnComplete: true, + delay: 1000, + attempts: 5, + backoff: { + type: "exponential", + delay: 3000 + } + } + ); + await syncIntegrations({ secretPath, projectId, environment }); + if (!excludeReplication) { + await queueService.queue( + QueueName.SecretReplication, + QueueJobs.SecretReplication, + { environmentId, projectId, secretPath, folderId, secrets, membershipId }, + { + attempts: 3, + backoff: { + type: "exponential", + delay: 1000 + }, + removeOnComplete: true, + removeOnFail: true + } + ); + } + }); + queueService.start(QueueName.IntegrationSync, async (job) => { const { environment, projectId, secretPath, depth = 1, deDupeQueue = {} } = job.data; @@ -375,11 +421,12 @@ export const secretQueueFactory = ({ ) .map(({ folderId }) => syncSecrets({ - depth: depth + 1, + _depth: depth + 1, projectId, secretPath: foldersGroupedById[folderId][0].path, - environment: foldersGroupedById[folderId][0].environmentSlug, - deDupeQueue + environmentSlug: foldersGroupedById[folderId][0].environmentSlug, + _deDupeQueue: deDupeQueue, + excludeReplication: true }) ) ); @@ -412,11 +459,12 @@ export const secretQueueFactory = ({ ) .map(({ folderId }) => syncSecrets({ - depth: depth + 1, + _depth: depth + 1, projectId, secretPath: referencedFoldersGroupedById[folderId][0].path, - environment: referencedFoldersGroupedById[folderId][0].environmentSlug, - deDupeQueue + environmentSlug: referencedFoldersGroupedById[folderId][0].environmentSlug, + _deDupeQueue: deDupeQueue, + excludeReplication: true }) ) ); @@ -546,7 +594,7 @@ export const secretQueueFactory = ({ return { // depth is internal only field thus no need to make it available outside - syncSecrets: (dto: TGetSecrets) => syncSecrets(dto), + syncSecrets, syncIntegrations, addSecretReminder, removeSecretReminder, diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index da8c48b5b..be7333abe 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -30,13 +30,12 @@ import { TSecretBlindIndexDALFactory } from "../secret-blind-index/secret-blind- import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TSecretImportDALFactory } from "../secret-import/secret-import-dal"; import { fnSecretsFromImports } from "../secret-import/secret-import-fns"; -import { TSecretReplicationServiceFactory } from "../secret-replication/secret-replication-service"; -import { SecretReplicationOperations } from "../secret-replication/secret-replication-types"; import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; import { TSecretDALFactory } from "./secret-dal"; import { decryptSecretRaw, fnSecretBlindIndexCheck, + fnSecretBulkDelete, fnSecretBulkInsert, fnSecretBulkUpdate, getAllNestedSecretReferences, @@ -45,6 +44,7 @@ import { } from "./secret-fns"; import { TSecretQueueFactory } from "./secret-queue"; import { + SecretOperations, TAttachSecretTagsDTO, TBackFillSecretReferencesDTO, TCreateBulkSecretDTO, @@ -55,8 +55,6 @@ import { TDeleteManySecretRawDTO, TDeleteSecretDTO, TDeleteSecretRawDTO, - TFnSecretBlindIndexCheckV2, - TFnSecretBulkDelete, TGetASecretDTO, TGetASecretRawDTO, TGetSecretsDTO, @@ -87,7 +85,6 @@ type TSecretServiceFactoryDep = { projectBotService: Pick; secretImportDAL: Pick; secretVersionTagDAL: Pick; - secretReplicationService: Pick; }; export type TSecretServiceFactory = ReturnType; @@ -104,8 +101,7 @@ export const secretServiceFactory = ({ projectDAL, projectBotService, secretImportDAL, - secretVersionTagDAL, - secretReplicationService + secretVersionTagDAL }: TSecretServiceFactoryDep) => { const getSecretReference = async (projectId: string) => { // if bot key missing means e2e still exist @@ -143,53 +139,6 @@ export const secretServiceFactory = ({ return secretBlindIndex; }; - const fnSecretBulkDelete = async ({ folderId, inputSecrets, tx, actorId }: TFnSecretBulkDelete) => { - const deletedSecrets = await secretDAL.deleteMany( - inputSecrets.map(({ type, secretBlindIndex }) => ({ - blindIndex: secretBlindIndex, - type - })), - folderId, - actorId, - tx - ); - - for (const s of deletedSecrets) { - if (s.secretReminderRepeatDays) { - // eslint-disable-next-line no-await-in-loop - await secretQueueService - .removeSecretReminder({ - secretId: s.id, - repeatDays: s.secretReminderRepeatDays - }) - .catch((err) => { - logger.error(err, `Failed to delete secret reminder for secret with ID ${s?.id}`); - }); - } - } - - return deletedSecrets; - }; - - // this is used when secret blind index already exist - // mainly for secret approval - const fnSecretBlindIndexCheckV2 = async ({ inputSecrets, folderId, userId }: TFnSecretBlindIndexCheckV2) => { - if (inputSecrets.some(({ type }) => type === SecretType.Personal) && !userId) { - throw new BadRequestError({ message: "Missing user id for personal secret" }); - } - const secrets = await secretDAL.findByBlindIndexes( - folderId, - inputSecrets.map(({ secretBlindIndex, type }) => ({ - blindIndex: secretBlindIndex, - type: type || SecretType.Shared - })), - userId - ); - const secsGroupedByBlindIndex = groupBy(secrets, (i) => i.secretBlindIndex as string); - - return { secsGroupedByBlindIndex, secrets }; - }; - const createSecret = async ({ path, actor, @@ -200,7 +149,7 @@ export const secretServiceFactory = ({ projectId, ...inputSecret }: TCreateSecretDTO) => { - const { permission } = await permissionService.getProjectPermission( + const { permission, membership } = await permissionService.getProjectPermission( actor, actorId, projectId, @@ -287,16 +236,17 @@ export const secretServiceFactory = ({ ); await snapshotService.performSnapshot(folderId); - await secretQueueService.syncSecrets({ secretPath: path, projectId, environment }); - // TODO(akhilmhdh-pg): licence check, posthog service and snapshot - await secretReplicationService.replicate({ - folderId, - projectId, - environmentId: folder.envId, + await secretQueueService.syncSecrets({ secretPath: path, + // if secret service reached means there was no secret policy + // TODO(akhilmhdh): The policy based replication will fail if machine identity is used. + membershipId: membership?.id as string, + projectId, + environmentSlug: folder.environment.slug, + environmentId: folder.envId, secrets: [ { - operation: SecretReplicationOperations.Create, + operation: SecretOperations.Create, id: secret[0].id, version: 1 } @@ -315,7 +265,7 @@ export const secretServiceFactory = ({ projectId, ...inputSecret }: TUpdateSecretDTO) => { - const { permission } = await permissionService.getProjectPermission( + const { permission, membership } = await permissionService.getProjectPermission( actor, actorId, projectId, @@ -430,15 +380,18 @@ export const secretServiceFactory = ({ ); await snapshotService.performSnapshot(folderId); - await secretQueueService.syncSecrets({ secretPath: path, projectId, environment }); - await secretReplicationService.replicate({ - folderId, - projectId, - environmentId: folder.envId, + await secretQueueService.syncSecrets({ + // if secret service reached means there was no secret policy + // TODO(akhilmhdh): The policy based replication will fail if machine identity is used. + membershipId: membership?.id as string, secretPath: path, + folderId: folder.id, + projectId, + environmentSlug: folder.environment.slug, + environmentId: folder.envId, secrets: [ { - operation: SecretReplicationOperations.Update, + operation: SecretOperations.Update, id: updatedSecret[0].id, version: updatedSecret[0].version } @@ -457,7 +410,7 @@ export const secretServiceFactory = ({ projectId, ...inputSecret }: TDeleteSecretDTO) => { - const { permission } = await permissionService.getProjectPermission( + const { permission, membership } = await permissionService.getProjectPermission( actor, actorId, projectId, @@ -499,6 +452,8 @@ export const secretServiceFactory = ({ projectId, folderId, actorId, + secretDAL, + secretQueueService, inputSecrets: [ { type: inputSecret.type as SecretType, @@ -510,15 +465,18 @@ export const secretServiceFactory = ({ ); await snapshotService.performSnapshot(folderId); - await secretQueueService.syncSecrets({ secretPath: path, projectId, environment }); - await secretReplicationService.replicate({ - folderId, - projectId, - environmentId: folder.envId, + await secretQueueService.syncSecrets({ + // if secret service reached means there was no secret policy + // TODO(akhilmhdh): The policy based replication will fail if machine identity is used. + membershipId: membership?.id as string, secretPath: path, + folderId: folder.id, + projectId, + environmentSlug: folder.environment.slug, + environmentId: folder.envId, secrets: [ { - operation: SecretReplicationOperations.Delete, + operation: SecretOperations.Delete, id: deletedSecret[0].id, version: deletedSecret[0].version } @@ -744,7 +702,7 @@ export const secretServiceFactory = ({ projectId, secrets: inputSecrets }: TCreateBulkSecretDTO) => { - const { permission } = await permissionService.getProjectPermission( + const { permission, membership } = await permissionService.getProjectPermission( actor, actorId, projectId, @@ -808,13 +766,16 @@ export const secretServiceFactory = ({ ); await snapshotService.performSnapshot(folderId); - await secretQueueService.syncSecrets({ secretPath: path, projectId, environment }); - await secretReplicationService.replicate({ - folderId, - projectId, - environmentId: folder.envId, + await secretQueueService.syncSecrets({ + // if secret service reached means there was no secret policy + // TODO(akhilmhdh): The policy based replication will fail if machine identity is used. + membershipId: membership?.id as string, secretPath: path, - secrets: newSecrets.map(({ id, version }) => ({ id, version, operation: SecretReplicationOperations.Create })) + folderId: folder.id, + projectId, + environmentSlug: folder.environment.slug, + environmentId: folder.envId, + secrets: newSecrets.map(({ id, version }) => ({ id, version, operation: SecretOperations.Create })) }); return newSecrets; @@ -830,7 +791,7 @@ export const secretServiceFactory = ({ projectId, secrets: inputSecrets }: TUpdateBulkSecretDTO) => { - const { permission } = await permissionService.getProjectPermission( + const { permission, membership } = await permissionService.getProjectPermission( actor, actorId, projectId, @@ -915,13 +876,16 @@ export const secretServiceFactory = ({ ); await snapshotService.performSnapshot(folderId); - await secretQueueService.syncSecrets({ secretPath: path, projectId, environment }); - await secretReplicationService.replicate({ - folderId, - projectId, - environmentId: folder.envId, + await secretQueueService.syncSecrets({ + // if secret service reached means there was no secret policy + // TODO(akhilmhdh): The policy based replication will fail if machine identity is used. + membershipId: membership?.id as string, secretPath: path, - secrets: secrets.map(({ id, version }) => ({ id, version, operation: SecretReplicationOperations.Update })) + folderId: folder.id, + projectId, + environmentSlug: folder.environment.slug, + environmentId: folder.envId, + secrets: secrets.map(({ id, version }) => ({ id, version, operation: SecretOperations.Update })) }); return secrets; @@ -937,7 +901,7 @@ export const secretServiceFactory = ({ actorAuthMethod, actorOrgId }: TDeleteBulkSecretDTO) => { - const { permission } = await permissionService.getProjectPermission( + const { permission, membership } = await permissionService.getProjectPermission( actor, actorId, projectId, @@ -972,6 +936,8 @@ export const secretServiceFactory = ({ const secretsDeleted = await secretDAL.transaction(async (tx) => fnSecretBulkDelete({ + secretDAL, + secretQueueService, inputSecrets: inputSecrets.map(({ type, secretName }) => ({ secretBlindIndex: keyName2BlindIndex[secretName], type @@ -984,13 +950,16 @@ export const secretServiceFactory = ({ ); await snapshotService.performSnapshot(folderId); - await secretQueueService.syncSecrets({ secretPath: path, projectId, environment }); - await secretReplicationService.replicate({ - folderId, - projectId, - environmentId: folder.envId, + await secretQueueService.syncSecrets({ + // if secret service reached means there was no secret policy + // TODO(akhilmhdh): The policy based replication will fail if machine identity is used. + membershipId: membership?.id as string, secretPath: path, - secrets: secretsDeleted.map(({ id, version }) => ({ id, version, operation: SecretReplicationOperations.Delete })) + folderId: folder.id, + projectId, + environmentSlug: folder.environment.slug, + environmentId: folder.envId, + secrets: secretsDeleted.map(({ id, version }) => ({ id, version, operation: SecretOperations.Delete })) }); return secretsDeleted; @@ -1171,9 +1140,6 @@ export const secretServiceFactory = ({ skipMultilineEncoding }); - await snapshotService.performSnapshot(secret.folderId); - await secretQueueService.syncSecrets({ secretPath, projectId, environment }); - return decryptSecretRaw(secret, botKey); }; @@ -1212,8 +1178,6 @@ export const secretServiceFactory = ({ }); await snapshotService.performSnapshot(secret.folderId); - await secretQueueService.syncSecrets({ secretPath, projectId, environment }); - return decryptSecretRaw(secret, botKey); }; @@ -1243,9 +1207,6 @@ export const secretServiceFactory = ({ actorAuthMethod }); - await snapshotService.performSnapshot(secret.folderId); - await secretQueueService.syncSecrets({ secretPath, projectId, environment }); - return decryptSecretRaw(secret, botKey); }; @@ -1294,12 +1255,7 @@ export const secretServiceFactory = ({ }) }); - await snapshotService.performSnapshot(secrets[0].folderId); - await secretQueueService.syncSecrets({ secretPath, projectId, environment }); - - return secrets.map((secret) => - decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey) - ); + return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey)); }; const updateManySecretsRaw = async ({ @@ -1348,12 +1304,7 @@ export const secretServiceFactory = ({ }) }); - await snapshotService.performSnapshot(secrets[0].folderId); - await secretQueueService.syncSecrets({ secretPath, projectId, environment }); - - return secrets.map((secret) => - decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey) - ); + return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey)); }; const deleteManySecretsRaw = async ({ @@ -1384,12 +1335,7 @@ export const secretServiceFactory = ({ secrets: inputSecrets.map(({ secretKey }) => ({ secretName: secretKey, type: SecretType.Shared })) }); - await snapshotService.performSnapshot(secrets[0].folderId); - await secretQueueService.syncSecrets({ secretPath, projectId, environment }); - - return secrets.map((secret) => - decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey) - ); + return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey)); }; const getSecretVersions = async ({ @@ -1510,7 +1456,12 @@ export const secretServiceFactory = ({ ); await snapshotService.performSnapshot(folder.id); - await secretQueueService.syncSecrets({ secretPath, projectId: project.id, environment }); + await secretQueueService.syncSecrets({ + secretPath, + projectId: project.id, + environmentSlug: environment, + excludeReplication: true + }); return { ...updatedSecret[0], @@ -1612,7 +1563,12 @@ export const secretServiceFactory = ({ ); await snapshotService.performSnapshot(folder.id); - await secretQueueService.syncSecrets({ secretPath, projectId: project.id, environment }); + await secretQueueService.syncSecrets({ + secretPath, + projectId: project.id, + environmentSlug: environment, + excludeReplication: true + }); return { ...updatedSecret[0], @@ -1685,13 +1641,6 @@ export const secretServiceFactory = ({ createManySecretsRaw, updateManySecretsRaw, deleteManySecretsRaw, - getSecretVersions, - backfillSecretReferences, - // external services function - fnSecretBulkDelete, - fnSecretBulkUpdate, - fnSecretBlindIndexCheck, - fnSecretBulkInsert, - fnSecretBlindIndexCheckV2 + getSecretVersions }; }; diff --git a/backend/src/services/secret/secret-types.ts b/backend/src/services/secret/secret-types.ts index 7e713a80f..ac1eb6cb8 100644 --- a/backend/src/services/secret/secret-types.ts +++ b/backend/src/services/secret/secret-types.ts @@ -264,6 +264,10 @@ export type TFnSecretBulkDelete = { inputSecrets: Array<{ type: SecretType; secretBlindIndex: string }>; actorId: string; tx?: Knex; + secretDAL: Pick; + secretQueueService: { + removeSecretReminder: (data: TRemoveSecretReminderDTO) => Promise; + }; }; export type TFnSecretBlindIndexCheck = { @@ -277,6 +281,7 @@ export type TFnSecretBlindIndexCheck = { // when blind index is already present export type TFnSecretBlindIndexCheckV2 = { + secretDAL: Pick; folderId: string; userId?: string; inputSecrets: Array<{ secretBlindIndex: string; type?: SecretType }>; @@ -363,3 +368,30 @@ export type TUpdateManySecretsRawFn = { }[]; userId?: string; }; + +export enum SecretOperations { + Create = "create", + Update = "update", + Delete = "delete" +} + +export type TSyncSecretsDTO = { + _depth?: number; + _deDupeQueue?: Record; + secretPath: string; + projectId: string; + environmentSlug: string; + // cases for just doing sync integration and webhook + excludeReplication?: T; +} & (T extends true + ? object + : { + environmentId: string; + folderId: string; + membershipId: string; + secrets: { + operation: SecretOperations; + id: string; + version: number; + }[]; + }); diff --git a/backend/src/services/secret/secret-version-dal.ts b/backend/src/services/secret/secret-version-dal.ts index 758352ed2..203406e30 100644 --- a/backend/src/services/secret/secret-version-dal.ts +++ b/backend/src/services/secret/secret-version-dal.ts @@ -89,6 +89,7 @@ export const secretVersionDALFactory = (db: TDbClient) => { const findLatestVersionMany = async (folderId: string, secretIds: string[], tx?: Knex) => { try { + if (!secretIds.length) return {}; const docs: Array = await (tx || db)(TableName.SecretVersion) .where("folderId", folderId) .whereIn(`${TableName.SecretVersion}.secretId`, secretIds)