diff --git a/backend/e2e-test/mocks/keystore.ts b/backend/e2e-test/mocks/keystore.ts index c85244129..965ea4e31 100644 --- a/backend/e2e-test/mocks/keystore.ts +++ b/backend/e2e-test/mocks/keystore.ts @@ -25,6 +25,9 @@ export const mockKeyStore = (): TKeyStoreFactory => { }, incrementBy: async () => { return 1; + }, + acquireLock: () => { + throw new Error("Not implemented"); } }; }; diff --git a/backend/src/@types/fastify.d.ts b/backend/src/@types/fastify.d.ts index 2bc1d9f26..3f1ca94e9 100644 --- a/backend/src/@types/fastify.d.ts +++ b/backend/src/@types/fastify.d.ts @@ -52,6 +52,7 @@ import { TSecretServiceFactory } from "@app/services/secret/secret-service"; import { TSecretBlindIndexServiceFactory } from "@app/services/secret-blind-index/secret-blind-index-service"; import { TSecretFolderServiceFactory } from "@app/services/secret-folder/secret-folder-service"; import { TSecretImportServiceFactory } from "@app/services/secret-import/secret-import-service"; +import { TSecretReplicationServiceFactory } from "@app/services/secret-replication/secret-replication-service"; import { TSecretSharingServiceFactory } from "@app/services/secret-sharing/secret-sharing-service"; import { TSecretTagServiceFactory } from "@app/services/secret-tag/secret-tag-service"; import { TServiceTokenServiceFactory } from "@app/services/service-token/service-token-service"; @@ -108,6 +109,7 @@ declare module "fastify" { projectKey: TProjectKeyServiceFactory; projectRole: TProjectRoleServiceFactory; secret: TSecretServiceFactory; + secretReplication: TSecretReplicationServiceFactory; secretTag: TSecretTagServiceFactory; secretImport: TSecretImportServiceFactory; projectBot: TProjectBotServiceFactory; diff --git a/backend/src/db/migrations/20240529111503_secret-replication.ts b/backend/src/db/migrations/20240529111503_secret-replication.ts new file mode 100644 index 000000000..ddb965df4 --- /dev/null +++ b/backend/src/db/migrations/20240529111503_secret-replication.ts @@ -0,0 +1,85 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + const doesSecretImportIsReplicationExist = await knex.schema.hasColumn(TableName.SecretImport, "isReplication"); + const doesSecretImportIsReplicationSuccessExist = await knex.schema.hasColumn( + TableName.SecretImport, + "isReplicationSuccess" + ); + const doesSecretImportReplicationStatusExist = await knex.schema.hasColumn( + TableName.SecretImport, + "replicationStatus" + ); + const doesSecretImportLastReplicatedExist = await knex.schema.hasColumn(TableName.SecretImport, "lastReplicated"); + const doesSecretImportIsReservedExist = await knex.schema.hasColumn(TableName.SecretImport, "isReserved"); + + if (await knex.schema.hasTable(TableName.SecretImport)) { + await knex.schema.alterTable(TableName.SecretImport, (t) => { + if (!doesSecretImportIsReplicationExist) t.boolean("isReplication").defaultTo(false); + if (!doesSecretImportIsReplicationSuccessExist) t.boolean("isReplicationSuccess").nullable(); + if (!doesSecretImportReplicationStatusExist) t.text("replicationStatus").nullable(); + if (!doesSecretImportLastReplicatedExist) t.datetime("lastReplicated").nullable(); + if (!doesSecretImportIsReservedExist) t.boolean("isReserved").defaultTo(false); + }); + } + + const doesSecretFolderReservedExist = await knex.schema.hasColumn(TableName.SecretFolder, "isReserved"); + if (await knex.schema.hasTable(TableName.SecretFolder)) { + await knex.schema.alterTable(TableName.SecretFolder, (t) => { + if (!doesSecretFolderReservedExist) t.boolean("isReserved").defaultTo(false); + }); + } + + const doesSecretApprovalRequestIsReplicatedExist = await knex.schema.hasColumn( + TableName.SecretApprovalRequest, + "isReplicated" + ); + if (await knex.schema.hasTable(TableName.SecretApprovalRequest)) { + await knex.schema.alterTable(TableName.SecretApprovalRequest, (t) => { + if (!doesSecretApprovalRequestIsReplicatedExist) t.boolean("isReplicated"); + }); + } +} + +export async function down(knex: Knex): Promise { + const doesSecretImportIsReplicationExist = await knex.schema.hasColumn(TableName.SecretImport, "isReplication"); + const doesSecretImportIsReplicationSuccessExist = await knex.schema.hasColumn( + TableName.SecretImport, + "isReplicationSuccess" + ); + const doesSecretImportReplicationStatusExist = await knex.schema.hasColumn( + TableName.SecretImport, + "replicationStatus" + ); + const doesSecretImportLastReplicatedExist = await knex.schema.hasColumn(TableName.SecretImport, "lastReplicated"); + const doesSecretImportIsReservedExist = await knex.schema.hasColumn(TableName.SecretImport, "isReserved"); + + if (await knex.schema.hasTable(TableName.SecretImport)) { + await knex.schema.alterTable(TableName.SecretImport, (t) => { + if (doesSecretImportIsReplicationExist) t.dropColumn("isReplication"); + if (doesSecretImportIsReplicationSuccessExist) t.dropColumn("isReplicationSuccess"); + if (doesSecretImportReplicationStatusExist) t.dropColumn("replicationStatus"); + if (doesSecretImportLastReplicatedExist) t.dropColumn("lastReplicated"); + if (doesSecretImportIsReservedExist) t.dropColumn("isReserved"); + }); + } + + const doesSecretFolderReservedExist = await knex.schema.hasColumn(TableName.SecretFolder, "isReserved"); + if (await knex.schema.hasTable(TableName.SecretFolder)) { + await knex.schema.alterTable(TableName.SecretFolder, (t) => { + if (doesSecretFolderReservedExist) t.dropColumn("isReserved"); + }); + } + + const doesSecretApprovalRequestIsReplicatedExist = await knex.schema.hasColumn( + TableName.SecretApprovalRequest, + "isReplicated" + ); + if (await knex.schema.hasTable(TableName.SecretApprovalRequest)) { + await knex.schema.alterTable(TableName.SecretApprovalRequest, (t) => { + if (doesSecretApprovalRequestIsReplicatedExist) t.dropColumn("isReplicated"); + }); + } +} diff --git a/backend/src/db/schemas/secret-approval-requests.ts b/backend/src/db/schemas/secret-approval-requests.ts index 6ee97fbb6..77ad370b7 100644 --- a/backend/src/db/schemas/secret-approval-requests.ts +++ b/backend/src/db/schemas/secret-approval-requests.ts @@ -18,7 +18,8 @@ export const SecretApprovalRequestsSchema = z.object({ statusChangeBy: z.string().uuid().nullable().optional(), committerId: z.string().uuid(), createdAt: z.date(), - updatedAt: z.date() + updatedAt: z.date(), + isReplicated: z.boolean().nullable().optional() }); export type TSecretApprovalRequests = z.infer; diff --git a/backend/src/db/schemas/secret-folders.ts b/backend/src/db/schemas/secret-folders.ts index 0f9684d0e..ad43ed1ad 100644 --- a/backend/src/db/schemas/secret-folders.ts +++ b/backend/src/db/schemas/secret-folders.ts @@ -14,7 +14,8 @@ export const SecretFoldersSchema = z.object({ createdAt: z.date(), updatedAt: z.date(), envId: z.string().uuid(), - parentId: z.string().uuid().nullable().optional() + parentId: z.string().uuid().nullable().optional(), + isReserved: z.boolean().default(false).nullable().optional() }); export type TSecretFolders = z.infer; diff --git a/backend/src/db/schemas/secret-imports.ts b/backend/src/db/schemas/secret-imports.ts index 9d42d8da5..4bb1e669d 100644 --- a/backend/src/db/schemas/secret-imports.ts +++ b/backend/src/db/schemas/secret-imports.ts @@ -15,7 +15,12 @@ export const SecretImportsSchema = z.object({ position: z.number(), createdAt: z.date(), updatedAt: z.date(), - folderId: z.string().uuid() + folderId: z.string().uuid(), + isReplication: z.boolean().default(false).nullable().optional(), + isReplicationSuccess: z.boolean().nullable().optional(), + replicationStatus: z.string().nullable().optional(), + lastReplicated: z.date().nullable().optional(), + isReserved: z.boolean().default(false).nullable().optional() }); export type TSecretImports = z.infer; diff --git a/backend/src/ee/routes/v1/secret-approval-request-router.ts b/backend/src/ee/routes/v1/secret-approval-request-router.ts index 2a9cc405d..b7204f72e 100644 --- a/backend/src/ee/routes/v1/secret-approval-request-router.ts +++ b/backend/src/ee/routes/v1/secret-approval-request-router.ts @@ -32,22 +32,20 @@ export const registerSecretApprovalRequestRouter = async (server: FastifyZodProv }), response: { 200: z.object({ - approvals: SecretApprovalRequestsSchema.merge( - z.object({ - // secretPath: z.string(), - policy: z.object({ - id: z.string(), - name: z.string(), - approvals: z.number(), - approvers: z.string().array(), - secretPath: z.string().optional().nullable() - }), - commits: z.object({ op: z.string(), secretId: z.string().nullable().optional() }).array(), - environment: z.string(), - reviewers: z.object({ member: z.string(), status: z.string() }).array(), - approvers: z.string().array() - }) - ).array() + approvals: SecretApprovalRequestsSchema.extend({ + // secretPath: z.string(), + policy: z.object({ + id: z.string(), + name: z.string(), + approvals: z.number(), + approvers: z.string().array(), + secretPath: z.string().optional().nullable() + }), + commits: z.object({ op: z.string(), secretId: z.string().nullable().optional() }).array(), + environment: z.string(), + reviewers: z.object({ member: z.string(), status: z.string() }).array(), + approvers: z.string().array() + }).array() }) } }, diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts index 690d308d2..5d0977134 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts @@ -15,9 +15,16 @@ import { ActorType } from "@app/services/auth/auth-type"; import { TProjectDALFactory } from "@app/services/project/project-dal"; import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; import { TSecretDALFactory } from "@app/services/secret/secret-dal"; -import { getAllNestedSecretReferences } from "@app/services/secret/secret-fns"; +import { + fnSecretBlindIndexCheck, + fnSecretBlindIndexCheckV2, + fnSecretBulkDelete, + fnSecretBulkInsert, + fnSecretBulkUpdate, + getAllNestedSecretReferences +} from "@app/services/secret/secret-fns"; import { TSecretQueueFactory } from "@app/services/secret/secret-queue"; -import { TSecretServiceFactory } from "@app/services/secret/secret-service"; +import { SecretOperations } from "@app/services/secret/secret-types"; import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal"; import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal"; import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal"; @@ -32,7 +39,6 @@ import { TSecretApprovalRequestReviewerDALFactory } from "./secret-approval-requ import { TSecretApprovalRequestSecretDALFactory } from "./secret-approval-request-secret-dal"; import { ApprovalStatus, - CommitType, RequestState, TApprovalRequestCountDTO, TGenerateSecretApprovalRequestDTO, @@ -45,10 +51,11 @@ import { type TSecretApprovalRequestServiceFactoryDep = { permissionService: Pick; + projectBotService: Pick; secretApprovalRequestDAL: TSecretApprovalRequestDALFactory; secretApprovalRequestSecretDAL: TSecretApprovalRequestSecretDALFactory; secretApprovalRequestReviewerDAL: TSecretApprovalRequestReviewerDALFactory; - folderDAL: Pick; + folderDAL: Pick; secretDAL: TSecretDALFactory; secretTagDAL: Pick; secretBlindIndexDAL: Pick; @@ -56,16 +63,7 @@ type TSecretApprovalRequestServiceFactoryDep = { secretVersionDAL: Pick; secretVersionTagDAL: Pick; projectDAL: Pick; - projectBotService: Pick; - secretService: Pick< - TSecretServiceFactory, - | "fnSecretBulkInsert" - | "fnSecretBulkUpdate" - | "fnSecretBlindIndexCheck" - | "fnSecretBulkDelete" - | "fnSecretBlindIndexCheckV2" - >; - secretQueueService: Pick; + secretQueueService: Pick; }; export type TSecretApprovalRequestServiceFactory = ReturnType; @@ -82,7 +80,6 @@ export const secretApprovalRequestServiceFactory = ({ projectDAL, permissionService, snapshotService, - secretService, secretVersionDAL, secretQueueService, projectBotService @@ -302,11 +299,12 @@ export const secretApprovalRequestServiceFactory = ({ const secretApprovalSecrets = await secretApprovalRequestSecretDAL.findByRequestId(secretApprovalRequest.id); if (!secretApprovalSecrets) throw new BadRequestError({ message: "No secrets found" }); - const conflicts: Array<{ secretId: string; op: CommitType }> = []; - let secretCreationCommits = secretApprovalSecrets.filter(({ op }) => op === CommitType.Create); + const conflicts: Array<{ secretId: string; op: SecretOperations }> = []; + let secretCreationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Create); if (secretCreationCommits.length) { - const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await secretService.fnSecretBlindIndexCheckV2({ + const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await fnSecretBlindIndexCheckV2({ folderId, + secretDAL, inputSecrets: secretCreationCommits.map(({ secretBlindIndex }) => { if (!secretBlindIndex) { throw new BadRequestError({ @@ -319,17 +317,19 @@ export const secretApprovalRequestServiceFactory = ({ secretCreationCommits .filter(({ secretBlindIndex }) => conflictGroupByBlindIndex[secretBlindIndex || ""]) .forEach((el) => { - conflicts.push({ op: CommitType.Create, secretId: el.id }); + conflicts.push({ op: SecretOperations.Create, secretId: el.id }); }); secretCreationCommits = secretCreationCommits.filter( ({ secretBlindIndex }) => !conflictGroupByBlindIndex[secretBlindIndex || ""] ); } - let secretUpdationCommits = secretApprovalSecrets.filter(({ op }) => op === CommitType.Update); + let secretUpdationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Update); if (secretUpdationCommits.length) { - const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await secretService.fnSecretBlindIndexCheckV2({ + const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await fnSecretBlindIndexCheckV2({ folderId, + secretDAL, + userId: "", inputSecrets: secretUpdationCommits .filter(({ secretBlindIndex, secret }) => secret && secret.secretBlindIndex !== secretBlindIndex) .map(({ secretBlindIndex }) => { @@ -347,7 +347,7 @@ export const secretApprovalRequestServiceFactory = ({ (secretBlindIndex && conflictGroupByBlindIndex[secretBlindIndex]) || !secretId ) .forEach((el) => { - conflicts.push({ op: CommitType.Update, secretId: el.id }); + conflicts.push({ op: SecretOperations.Update, secretId: el.id }); }); secretUpdationCommits = secretUpdationCommits.filter( @@ -356,11 +356,11 @@ export const secretApprovalRequestServiceFactory = ({ ); } - const secretDeletionCommits = secretApprovalSecrets.filter(({ op }) => op === CommitType.Delete); + const secretDeletionCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Delete); const botKey = await projectBotService.getBotKey(projectId).catch(() => null); const mergeStatus = await secretApprovalRequestDAL.transaction(async (tx) => { const newSecrets = secretCreationCommits.length - ? await secretService.fnSecretBulkInsert({ + ? await fnSecretBulkInsert({ tx, folderId, inputSecrets: secretCreationCommits.map((el) => ({ @@ -403,7 +403,7 @@ export const secretApprovalRequestServiceFactory = ({ }) : []; const updatedSecrets = secretUpdationCommits.length - ? await secretService.fnSecretBulkUpdate({ + ? await fnSecretBulkUpdate({ folderId, projectId, tx, @@ -449,11 +449,13 @@ export const secretApprovalRequestServiceFactory = ({ }) : []; const deletedSecret = secretDeletionCommits.length - ? await secretService.fnSecretBulkDelete({ + ? await fnSecretBulkDelete({ projectId, folderId, tx, actorId: "", + secretDAL, + secretQueueService, inputSecrets: secretDeletionCommits.map(({ secretBlindIndex }) => { if (!secretBlindIndex) { throw new BadRequestError({ @@ -480,12 +482,14 @@ export const secretApprovalRequestServiceFactory = ({ }; }); await snapshotService.performSnapshot(folderId); - const folder = await folderDAL.findById(folderId); - // TODO(akhilmhdh-pg): change query to do secret path from folder + const [folder] = await folderDAL.findSecretPathByFolderIds(projectId, [folderId]); + if (!folder) throw new BadRequestError({ message: "Folder not found" }); await secretQueueService.syncSecrets({ projectId, - secretPath: "/", - environment: folder?.environment.envSlug as string + secretPath: folder.path, + environmentSlug: folder.environmentSlug, + actorId, + actor }); return mergeStatus; }; @@ -533,9 +537,9 @@ export const secretApprovalRequestServiceFactory = ({ const commits: Omit[] = []; const commitTagIds: Record = {}; // for created secret approval change - const createdSecrets = data[CommitType.Create]; + const createdSecrets = data[SecretOperations.Create]; if (createdSecrets && createdSecrets?.length) { - const { keyName2BlindIndex } = await secretService.fnSecretBlindIndexCheck({ + const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({ inputSecrets: createdSecrets, folderId, isNew: true, @@ -546,7 +550,7 @@ export const secretApprovalRequestServiceFactory = ({ commits.push( ...createdSecrets.map(({ secretName, ...el }) => ({ ...el, - op: CommitType.Create as const, + op: SecretOperations.Create as const, version: 1, secretBlindIndex: keyName2BlindIndex[secretName], algorithm: SecretEncryptionAlgo.AES_256_GCM, @@ -558,12 +562,12 @@ export const secretApprovalRequestServiceFactory = ({ }); } // not secret approval for update operations - const updatedSecrets = data[CommitType.Update]; + const updatedSecrets = data[SecretOperations.Update]; if (updatedSecrets && updatedSecrets?.length) { // get all blind index // Find all those secrets // if not throw not found - const { keyName2BlindIndex, secrets: secretsToBeUpdated } = await secretService.fnSecretBlindIndexCheck({ + const { keyName2BlindIndex, secrets: secretsToBeUpdated } = await fnSecretBlindIndexCheck({ inputSecrets: updatedSecrets, folderId, isNew: false, @@ -574,8 +578,8 @@ export const secretApprovalRequestServiceFactory = ({ // now find any secret that needs to update its name // same process as above const nameUpdatedSecrets = updatedSecrets.filter(({ newSecretName }) => Boolean(newSecretName)); - const { keyName2BlindIndex: newKeyName2BlindIndex } = await secretService.fnSecretBlindIndexCheck({ - inputSecrets: nameUpdatedSecrets, + const { keyName2BlindIndex: newKeyName2BlindIndex } = await fnSecretBlindIndexCheck({ + inputSecrets: nameUpdatedSecrets.map(({ newSecretName }) => ({ secretName: newSecretName as string })), folderId, isNew: true, blindIndexCfg, @@ -592,14 +596,14 @@ export const secretApprovalRequestServiceFactory = ({ const secretId = secsGroupedByBlindIndex[keyName2BlindIndex[secretName]][0].id; const secretBlindIndex = newSecretName && newKeyName2BlindIndex[newSecretName] - ? newKeyName2BlindIndex?.[secretName] + ? newKeyName2BlindIndex?.[newSecretName] : keyName2BlindIndex[secretName]; // add tags if (tagIds?.length) commitTagIds[keyName2BlindIndex[secretName]] = tagIds; return { ...latestSecretVersions[secretId], ...el, - op: CommitType.Update as const, + op: SecretOperations.Update as const, secret: secretId, secretVersion: latestSecretVersions[secretId].id, secretBlindIndex, @@ -609,12 +613,12 @@ export const secretApprovalRequestServiceFactory = ({ ); } // deleted secrets - const deletedSecrets = data[CommitType.Delete]; + const deletedSecrets = data[SecretOperations.Delete]; if (deletedSecrets && deletedSecrets.length) { // get all blind index // Find all those secrets // if not throw not found - const { keyName2BlindIndex, secrets } = await secretService.fnSecretBlindIndexCheck({ + const { keyName2BlindIndex, secrets } = await fnSecretBlindIndexCheck({ inputSecrets: deletedSecrets, folderId, isNew: false, @@ -635,7 +639,7 @@ export const secretApprovalRequestServiceFactory = ({ if (!latestSecretVersions[secretId].secretBlindIndex) throw new BadRequestError({ message: "Failed to find secret blind index" }); return { - op: CommitType.Delete as const, + op: SecretOperations.Delete as const, ...latestSecretVersions[secretId], secretBlindIndex: latestSecretVersions[secretId].secretBlindIndex as string, secret: secretId, diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-types.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-types.ts index 008b977e6..1fbb75418 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-types.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-types.ts @@ -1,11 +1,6 @@ import { TImmutableDBKeys, TSecretApprovalPolicies, TSecretApprovalRequestsSecrets } from "@app/db/schemas"; import { TProjectPermission } from "@app/lib/types"; - -export enum CommitType { - Create = "create", - Update = "update", - Delete = "delete" -} +import { SecretOperations } from "@app/services/secret/secret-types"; export enum RequestState { Open = "open", @@ -18,14 +13,14 @@ export enum ApprovalStatus { REJECTED = "rejected" } -type TApprovalCreateSecret = Omit< +export type TApprovalCreateSecret = Omit< TSecretApprovalRequestsSecrets, TImmutableDBKeys | "version" | "algorithm" | "keyEncoding" | "requestId" | "op" | "secretVersion" | "secretBlindIndex" > & { secretName: string; tagIds?: string[]; }; -type TApprovalUpdateSecret = Partial & { +export type TApprovalUpdateSecret = Partial & { secretName: string; newSecretName?: string; tagIds?: string[]; @@ -36,9 +31,9 @@ export type TGenerateSecretApprovalRequestDTO = { secretPath: string; policy: TSecretApprovalPolicies; data: { - [CommitType.Create]?: TApprovalCreateSecret[]; - [CommitType.Update]?: TApprovalUpdateSecret[]; - [CommitType.Delete]?: { secretName: string }[]; + [SecretOperations.Create]?: TApprovalCreateSecret[]; + [SecretOperations.Update]?: TApprovalUpdateSecret[]; + [SecretOperations.Delete]?: { secretName: string }[]; }; } & TProjectPermission; diff --git a/backend/src/ee/services/secret-replication/secret-replication-constants.ts b/backend/src/ee/services/secret-replication/secret-replication-constants.ts new file mode 100644 index 000000000..88c9ee166 --- /dev/null +++ b/backend/src/ee/services/secret-replication/secret-replication-constants.ts @@ -0,0 +1 @@ +export const MAX_REPLICATION_DEPTH = 5; diff --git a/backend/src/ee/services/secret-replication/secret-replication-dal.ts b/backend/src/ee/services/secret-replication/secret-replication-dal.ts new file mode 100644 index 000000000..3c4c021fd --- /dev/null +++ b/backend/src/ee/services/secret-replication/secret-replication-dal.ts @@ -0,0 +1,10 @@ +import { TDbClient } from "@app/db"; +import { TableName } from "@app/db/schemas"; +import { ormify } from "@app/lib/knex"; + +export type TSecretReplicationDALFactory = ReturnType; + +export const secretReplicationDALFactory = (db: TDbClient) => { + const orm = ormify(db, TableName.SecretVersion); + return orm; +}; diff --git a/backend/src/ee/services/secret-replication/secret-replication-service.ts b/backend/src/ee/services/secret-replication/secret-replication-service.ts new file mode 100644 index 000000000..fd2f7cc1a --- /dev/null +++ b/backend/src/ee/services/secret-replication/secret-replication-service.ts @@ -0,0 +1,485 @@ +import { SecretType, TSecrets } from "@app/db/schemas"; +import { TSecretApprovalPolicyServiceFactory } from "@app/ee/services/secret-approval-policy/secret-approval-policy-service"; +import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal"; +import { TSecretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal"; +import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore"; +import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; +import { BadRequestError } from "@app/lib/errors"; +import { groupBy, unique } from "@app/lib/fn"; +import { logger } from "@app/lib/logger"; +import { alphaNumericNanoId } from "@app/lib/nanoid"; +import { QueueName, TQueueServiceFactory } from "@app/queue"; +import { ActorType } from "@app/services/auth/auth-type"; +import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; +import { TProjectMembershipDALFactory } from "@app/services/project-membership/project-membership-dal"; +import { TSecretDALFactory } from "@app/services/secret/secret-dal"; +import { fnSecretBulkInsert, fnSecretBulkUpdate } from "@app/services/secret/secret-fns"; +import { TSecretQueueFactory, uniqueSecretQueueKey } from "@app/services/secret/secret-queue"; +import { SecretOperations } from "@app/services/secret/secret-types"; +import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal"; +import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal"; +import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal"; +import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; +import { ReservedFolders } from "@app/services/secret-folder/secret-folder-types"; +import { TSecretImportDALFactory } from "@app/services/secret-import/secret-import-dal"; +import { fnSecretsFromImports } from "@app/services/secret-import/secret-import-fns"; +import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; + +import { MAX_REPLICATION_DEPTH } from "./secret-replication-constants"; + +type TSecretReplicationServiceFactoryDep = { + secretDAL: Pick< + TSecretDALFactory, + "find" | "findByBlindIndexes" | "insertMany" | "bulkUpdate" | "delete" | "upsertSecretReferences" | "transaction" + >; + secretVersionDAL: Pick; + secretImportDAL: Pick; + folderDAL: Pick< + TSecretFolderDALFactory, + "findSecretPathByFolderIds" | "findBySecretPath" | "create" | "findOne" | "findByManySecretPath" + >; + secretVersionTagDAL: Pick; + secretQueueService: Pick; + queueService: Pick; + secretApprovalPolicyService: Pick; + keyStore: Pick; + secretBlindIndexDAL: Pick; + secretTagDAL: Pick; + secretApprovalRequestDAL: Pick; + projectMembershipDAL: Pick; + secretApprovalRequestSecretDAL: Pick< + TSecretApprovalRequestSecretDALFactory, + "insertMany" | "insertApprovalSecretTags" + >; + projectBotService: Pick; +}; + +export type TSecretReplicationServiceFactory = ReturnType; +const SECRET_IMPORT_SUCCESS_LOCK = 10; + +const keystoreReplicationSuccessKey = (jobId: string, secretImportId: string) => `${jobId}-${secretImportId}`; +const getReplicationKeyLockPrefix = (projectId: string, environmentSlug: string, secretPath: string) => + `REPLICATION_SECRET_${projectId}-${environmentSlug}-${secretPath}`; +export const getReplicationFolderName = (importId: string) => `${ReservedFolders.SecretReplication}${importId}`; + +const getDecryptedKeyValue = (key: string, secret: TSecrets) => { + const secretKey = decryptSymmetric128BitHexKeyUTF8({ + ciphertext: secret.secretKeyCiphertext, + iv: secret.secretKeyIV, + tag: secret.secretKeyTag, + key + }); + + const secretValue = decryptSymmetric128BitHexKeyUTF8({ + ciphertext: secret.secretValueCiphertext, + iv: secret.secretValueIV, + tag: secret.secretValueTag, + key + }); + return { key: secretKey, value: secretValue }; +}; + +export const secretReplicationServiceFactory = ({ + secretDAL, + queueService, + secretVersionDAL, + secretImportDAL, + keyStore, + secretVersionTagDAL, + secretTagDAL, + folderDAL, + secretApprovalPolicyService, + secretApprovalRequestSecretDAL, + secretApprovalRequestDAL, + secretQueueService, + projectMembershipDAL, + projectBotService +}: TSecretReplicationServiceFactoryDep) => { + const getReplicatedSecrets = ( + botKey: string, + localSecrets: TSecrets[], + importedSecrets: { secrets: TSecrets[] }[] + ) => { + const deDupe = new Set(); + const secrets = localSecrets + .filter(({ secretBlindIndex }) => Boolean(secretBlindIndex)) + .map((el) => { + const decryptedSecret = getDecryptedKeyValue(botKey, el); + deDupe.add(decryptedSecret.key); + return { ...el, secretKey: decryptedSecret.key, secretValue: decryptedSecret.value }; + }); + + for (let i = importedSecrets.length - 1; i >= 0; i = -1) { + importedSecrets[i].secrets.forEach((el) => { + const decryptedSecret = getDecryptedKeyValue(botKey, el); + if (deDupe.has(decryptedSecret.key) || !el.secretBlindIndex) { + return; + } + deDupe.add(decryptedSecret.key); + secrets.push({ ...el, secretKey: decryptedSecret.key, secretValue: decryptedSecret.value }); + }); + } + return secrets; + }; + + // IMPORTANT NOTE BEFORE READING THE FUNCTION + // SOURCE - Where secrets are copied from + // DESTINATION - Where the replicated imports that points to SOURCE from Destination + queueService.start(QueueName.SecretReplication, async (job) => { + logger.info(job.data, "Replication started"); + const { + secretPath, + environmentSlug, + projectId, + actorId, + actor, + pickOnlyImportIds, + _deDupeReplicationQueue: deDupeReplicationQueue, + _deDupeQueue: deDupeQueue, + _depth: depth = 0 + } = job.data; + if (depth > MAX_REPLICATION_DEPTH) return; + + const folder = await folderDAL.findBySecretPath(projectId, environmentSlug, secretPath); + if (!folder) return; + + // the the replicated imports made to the source. These are the destinations + const destinationSecretImports = await secretImportDAL.find({ + importPath: secretPath, + importEnv: folder.envId + }); + + // CASE: normal mode <- link import <- replicated import + const nonReplicatedDestinationImports = destinationSecretImports.filter(({ isReplication }) => !isReplication); + if (nonReplicatedDestinationImports.length) { + // keep calling sync secret for all the imports made + const importedFolderIds = unique(nonReplicatedDestinationImports, (i) => i.folderId).map( + ({ folderId }) => folderId + ); + const importedFolders = await folderDAL.findSecretPathByFolderIds(projectId, importedFolderIds); + const foldersGroupedById = groupBy(importedFolders.filter(Boolean), (i) => i?.id as string); + await Promise.all( + nonReplicatedDestinationImports + .filter(({ folderId }) => Boolean(foldersGroupedById[folderId][0]?.path as string)) + // filter out already synced ones + .filter( + ({ folderId }) => + !deDupeQueue?.[ + uniqueSecretQueueKey( + foldersGroupedById[folderId][0]?.environmentSlug as string, + foldersGroupedById[folderId][0]?.path as string + ) + ] + ) + .map(({ folderId }) => + secretQueueService.replicateSecrets({ + projectId, + secretPath: foldersGroupedById[folderId][0]?.path as string, + environmentSlug: foldersGroupedById[folderId][0]?.environmentSlug as string, + actorId, + actor, + _depth: depth + 1, + _deDupeReplicationQueue: deDupeReplicationQueue, + _deDupeQueue: deDupeQueue + }) + ) + ); + } + + let destinationReplicatedSecretImports = destinationSecretImports.filter(({ isReplication }) => + Boolean(isReplication) + ); + destinationReplicatedSecretImports = pickOnlyImportIds + ? destinationReplicatedSecretImports.filter(({ id }) => pickOnlyImportIds?.includes(id)) + : destinationReplicatedSecretImports; + if (!destinationReplicatedSecretImports.length) return; + + const botKey = await projectBotService.getBotKey(projectId); + + // these are the secrets to be added in replicated folders + const sourceLocalSecrets = await secretDAL.find({ folderId: folder.id, type: SecretType.Shared }); + const sourceSecretImports = await secretImportDAL.find({ folderId: folder.id }); + const sourceImportedSecrets = await fnSecretsFromImports({ + allowedImports: sourceSecretImports, + secretDAL, + folderDAL, + secretImportDAL + }); + // secrets that gets replicated across imports + const sourceSecrets = getReplicatedSecrets(botKey, sourceLocalSecrets, sourceImportedSecrets); + const sourceSecretsGroupByBlindIndex = groupBy(sourceSecrets, (i) => i.secretBlindIndex as string); + + const lock = await keyStore.acquireLock( + [getReplicationKeyLockPrefix(projectId, environmentSlug, secretPath)], + 5000 + ); + + try { + /* eslint-disable no-await-in-loop */ + for (const destinationSecretImport of destinationReplicatedSecretImports) { + try { + const hasJobCompleted = await keyStore.getItem( + keystoreReplicationSuccessKey(job.id as string, destinationSecretImport.id), + KeyStorePrefixes.SecretReplication + ); + if (hasJobCompleted) { + logger.info( + { jobId: job.id, importId: destinationSecretImport.id }, + "Skipping this job as this has been successfully replicated." + ); + // eslint-disable-next-line + continue; + } + + const [destinationFolder] = await folderDAL.findSecretPathByFolderIds(projectId, [ + destinationSecretImport.folderId + ]); + if (!destinationFolder) throw new BadRequestError({ message: "Imported folder not found" }); + + let destinationReplicationFolder = await folderDAL.findOne({ + parentId: destinationFolder.id, + name: getReplicationFolderName(destinationSecretImport.id), + isReserved: true + }); + if (!destinationReplicationFolder) { + destinationReplicationFolder = await folderDAL.create({ + parentId: destinationFolder.id, + name: getReplicationFolderName(destinationSecretImport.id), + envId: destinationFolder.envId, + isReserved: true + }); + } + const destinationReplicationFolderId = destinationReplicationFolder.id; + + const destinationLocalSecretsFromDB = await secretDAL.find({ + folderId: destinationReplicationFolderId + }); + const destinationLocalSecrets = destinationLocalSecretsFromDB.map((el) => { + const decryptedSecret = getDecryptedKeyValue(botKey, el); + return { ...el, secretKey: decryptedSecret.key, secretValue: decryptedSecret.value }; + }); + + const destinationLocalSecretsGroupedByBlindIndex = groupBy( + destinationLocalSecrets.filter(({ secretBlindIndex }) => Boolean(secretBlindIndex)), + (i) => i.secretBlindIndex as string + ); + + const locallyCreatedSecrets = sourceSecrets + .filter( + ({ secretBlindIndex }) => !destinationLocalSecretsGroupedByBlindIndex[secretBlindIndex as string]?.[0] + ) + .map((el) => ({ ...el, operation: SecretOperations.Create })); // rewrite update ops to create + + const locallyUpdatedSecrets = sourceSecrets + .filter( + ({ secretBlindIndex, secretKey, secretValue }) => + destinationLocalSecretsGroupedByBlindIndex[secretBlindIndex as string]?.[0] && + // if key or value changed + (destinationLocalSecretsGroupedByBlindIndex[secretBlindIndex as string]?.[0]?.secretKey !== secretKey || + destinationLocalSecretsGroupedByBlindIndex[secretBlindIndex as string]?.[0]?.secretValue !== + secretValue) + ) + .map((el) => ({ ...el, operation: SecretOperations.Update })); // rewrite update ops to create + + const locallyDeletedSecrets = destinationLocalSecrets + .filter(({ secretBlindIndex }) => !sourceSecretsGroupByBlindIndex[secretBlindIndex as string]?.[0]) + .map((el) => ({ ...el, operation: SecretOperations.Delete })); + + const isEmtpy = + locallyCreatedSecrets.length + locallyUpdatedSecrets.length + locallyDeletedSecrets.length === 0; + // eslint-disable-next-line + if (isEmtpy) continue; + + const policy = await secretApprovalPolicyService.getSecretApprovalPolicy( + projectId, + destinationFolder.environmentSlug, + destinationFolder.path + ); + // this means it should be a approval request rather than direct replication + if (policy && actor === ActorType.USER) { + const membership = await projectMembershipDAL.findOne({ projectId, userId: actorId }); + if (!membership) { + logger.error("Project membership not found in %s for user %s", projectId, actorId); + return; + } + + const localSecretsLatestVersions = destinationLocalSecrets.map(({ id }) => id); + const latestSecretVersions = await secretVersionDAL.findLatestVersionMany( + destinationReplicationFolderId, + localSecretsLatestVersions + ); + await secretApprovalRequestDAL.transaction(async (tx) => { + const approvalRequestDoc = await secretApprovalRequestDAL.create( + { + folderId: destinationReplicationFolderId, + slug: alphaNumericNanoId(), + policyId: policy.id, + status: "open", + hasMerged: false, + committerId: membership.id, + isReplicated: true + }, + tx + ); + const commits = locallyCreatedSecrets + .concat(locallyUpdatedSecrets) + .concat(locallyDeletedSecrets) + .map((doc) => { + const { operation } = doc; + const localSecret = destinationLocalSecretsGroupedByBlindIndex[doc.secretBlindIndex as string]?.[0]; + + return { + op: operation, + keyEncoding: doc.keyEncoding, + algorithm: doc.algorithm, + requestId: approvalRequestDoc.id, + metadata: doc.metadata, + secretKeyIV: doc.secretKeyIV, + secretKeyTag: doc.secretKeyTag, + secretKeyCiphertext: doc.secretKeyCiphertext, + secretValueIV: doc.secretValueIV, + secretValueTag: doc.secretValueTag, + secretValueCiphertext: doc.secretValueCiphertext, + secretBlindIndex: doc.secretBlindIndex, + secretCommentIV: doc.secretCommentIV, + secretCommentTag: doc.secretCommentTag, + secretCommentCiphertext: doc.secretCommentCiphertext, + skipMultilineEncoding: doc.skipMultilineEncoding, + // except create operation other two needs the secret id and version id + ...(operation !== SecretOperations.Create + ? { secretId: localSecret.id, secretVersion: latestSecretVersions[localSecret.id].id } + : {}) + }; + }); + const approvalCommits = await secretApprovalRequestSecretDAL.insertMany(commits, tx); + + return { ...approvalRequestDoc, commits: approvalCommits }; + }); + } else { + await secretDAL.transaction(async (tx) => { + if (locallyCreatedSecrets.length) { + await fnSecretBulkInsert({ + folderId: destinationReplicationFolderId, + secretVersionDAL, + secretDAL, + tx, + secretTagDAL, + secretVersionTagDAL, + inputSecrets: locallyCreatedSecrets.map((doc) => { + return { + keyEncoding: doc.keyEncoding, + algorithm: doc.algorithm, + type: doc.type, + metadata: doc.metadata, + secretKeyIV: doc.secretKeyIV, + secretKeyTag: doc.secretKeyTag, + secretKeyCiphertext: doc.secretKeyCiphertext, + secretValueIV: doc.secretValueIV, + secretValueTag: doc.secretValueTag, + secretValueCiphertext: doc.secretValueCiphertext, + secretBlindIndex: doc.secretBlindIndex, + secretCommentIV: doc.secretCommentIV, + secretCommentTag: doc.secretCommentTag, + secretCommentCiphertext: doc.secretCommentCiphertext, + skipMultilineEncoding: doc.skipMultilineEncoding + }; + }) + }); + } + if (locallyUpdatedSecrets.length) { + await fnSecretBulkUpdate({ + projectId, + folderId: destinationReplicationFolderId, + secretVersionDAL, + secretDAL, + tx, + secretTagDAL, + secretVersionTagDAL, + inputSecrets: locallyUpdatedSecrets.map((doc) => { + return { + filter: { + folderId: destinationReplicationFolderId, + id: destinationLocalSecretsGroupedByBlindIndex[doc.secretBlindIndex as string][0].id + }, + data: { + keyEncoding: doc.keyEncoding, + algorithm: doc.algorithm, + type: doc.type, + metadata: doc.metadata, + secretKeyIV: doc.secretKeyIV, + secretKeyTag: doc.secretKeyTag, + secretKeyCiphertext: doc.secretKeyCiphertext, + secretValueIV: doc.secretValueIV, + secretValueTag: doc.secretValueTag, + secretValueCiphertext: doc.secretValueCiphertext, + secretBlindIndex: doc.secretBlindIndex, + secretCommentIV: doc.secretCommentIV, + secretCommentTag: doc.secretCommentTag, + secretCommentCiphertext: doc.secretCommentCiphertext, + skipMultilineEncoding: doc.skipMultilineEncoding + } + }; + }) + }); + } + if (locallyDeletedSecrets.length) { + await secretDAL.delete( + { + $in: { + id: locallyDeletedSecrets.map(({ id }) => id) + }, + folderId: destinationReplicationFolderId + }, + tx + ); + } + }); + + await secretQueueService.syncSecrets({ + projectId, + secretPath: destinationFolder.path, + environmentSlug: destinationFolder.environmentSlug, + actorId, + actor, + _depth: depth + 1, + _deDupeReplicationQueue: deDupeReplicationQueue, + _deDupeQueue: deDupeQueue + }); + } + + // this is used to avoid multiple times generating secret approval by failed one + await keyStore.setItemWithExpiry( + keystoreReplicationSuccessKey(job.id as string, destinationSecretImport.id), + SECRET_IMPORT_SUCCESS_LOCK, + 1, + KeyStorePrefixes.SecretReplication + ); + + await secretImportDAL.updateById(destinationSecretImport.id, { + lastReplicated: new Date(), + replicationStatus: null, + isReplicationSuccess: true + }); + } catch (err) { + logger.error( + err, + `Failed to replicate secret with import id=[${destinationSecretImport.id}] env=[${destinationSecretImport.importEnv.slug}] path=[${destinationSecretImport.importPath}]` + ); + await secretImportDAL.updateById(destinationSecretImport.id, { + lastReplicated: new Date(), + replicationStatus: (err as Error)?.message.slice(0, 500), + isReplicationSuccess: false + }); + } + } + /* eslint-enable no-await-in-loop */ + } finally { + await lock.release(); + logger.info(job.data, "Replication finished"); + } + }); + + queueService.listen(QueueName.SecretReplication, "failed", (job, err) => { + logger.error(err, "Failed to replicate secret", job?.data); + }); +}; diff --git a/backend/src/ee/services/secret-replication/secret-replication-types.ts b/backend/src/ee/services/secret-replication/secret-replication-types.ts new file mode 100644 index 000000000..1b32f1f4a --- /dev/null +++ b/backend/src/ee/services/secret-replication/secret-replication-types.ts @@ -0,0 +1,3 @@ +export type TSyncSecretReplicationDTO = { + id: string; +}; diff --git a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts index 0e71ad126..bd8750577 100644 --- a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts +++ b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts @@ -220,7 +220,7 @@ export const secretSnapshotServiceFactory = ({ const deletedTopLevelSecsGroupById = groupBy(deletedTopLevelSecs, (item) => item.id); // this will remove all secrets and folders on child // due to sql foreign key and link list connection removing the folders removes everything below too - const deletedFolders = await folderDAL.delete({ parentId: snapshot.folderId }, tx); + const deletedFolders = await folderDAL.delete({ parentId: snapshot.folderId, isReserved: false }, tx); const deletedTopLevelFolders = groupBy( deletedFolders.filter(({ parentId }) => parentId === snapshot.folderId), (item) => item.id diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.ts index 5e2c3aab3..76cd6d522 100644 --- a/backend/src/keystore/keystore.ts +++ b/backend/src/keystore/keystore.ts @@ -1,20 +1,42 @@ import { Redis } from "ioredis"; +import { Redlock, Settings } from "@app/lib/red-lock"; + export type TKeyStoreFactory = ReturnType; +// all the key prefixes used must be set here to avoid conflict +export enum KeyStorePrefixes { + SecretReplication = "secret-replication-import-lock" +} + export const keyStoreFactory = (redisUrl: string) => { const redis = new Redis(redisUrl); + const redisLock = new Redlock([redis], { retryCount: 2, retryDelay: 200 }); - const setItem = async (key: string, value: string | number | Buffer) => redis.set(key, value); + const setItem = async (key: string, value: string | number | Buffer, prefix?: string) => + redis.set(prefix ? `${prefix}:${key}` : key, value); - const getItem = async (key: string) => redis.get(key); + const getItem = async (key: string, prefix?: string) => redis.get(prefix ? `${prefix}:${key}` : key); - const setItemWithExpiry = async (key: string, exp: number | string, value: string | number | Buffer) => - redis.setex(key, exp, value); + const setItemWithExpiry = async ( + key: string, + exp: number | string, + value: string | number | Buffer, + prefix?: string + ) => redis.setex(prefix ? `${prefix}:${key}` : key, exp, value); const deleteItem = async (key: string) => redis.del(key); const incrementBy = async (key: string, value: number) => redis.incrby(key, value); - return { setItem, getItem, setItemWithExpiry, deleteItem, incrementBy }; + return { + setItem, + getItem, + setItemWithExpiry, + deleteItem, + incrementBy, + acquireLock(resources: string[], duration: number, settings?: Partial) { + return redisLock.acquire(resources, duration, settings); + } + }; }; diff --git a/backend/src/lib/red-lock/index.ts b/backend/src/lib/red-lock/index.ts new file mode 100644 index 000000000..e1cc4f587 --- /dev/null +++ b/backend/src/lib/red-lock/index.ts @@ -0,0 +1,682 @@ +/* eslint-disable */ +// Source code credits: https://github.com/mike-marcacci/node-redlock +// Taken to avoid external dependency +import { randomBytes, createHash } from "crypto"; +import { EventEmitter } from "events"; + +// AbortController became available as a global in node version 16. Once version +// 14 reaches its end-of-life, this can be removed. + +import { Redis as IORedisClient, Cluster as IORedisCluster } from "ioredis"; + +type Client = IORedisClient | IORedisCluster; + +// Define script constants. +const ACQUIRE_SCRIPT = ` + -- Return 0 if an entry already exists. + for i, key in ipairs(KEYS) do + if redis.call("exists", key) == 1 then + return 0 + end + end + + -- Create an entry for each provided key. + for i, key in ipairs(KEYS) do + redis.call("set", key, ARGV[1], "PX", ARGV[2]) + end + + -- Return the number of entries added. + return #KEYS +`; + +const EXTEND_SCRIPT = ` + -- Return 0 if an entry exists with a *different* lock value. + for i, key in ipairs(KEYS) do + if redis.call("get", key) ~= ARGV[1] then + return 0 + end + end + + -- Update the entry for each provided key. + for i, key in ipairs(KEYS) do + redis.call("set", key, ARGV[1], "PX", ARGV[2]) + end + + -- Return the number of entries updated. + return #KEYS +`; + +const RELEASE_SCRIPT = ` + local count = 0 + for i, key in ipairs(KEYS) do + -- Only remove entries for *this* lock value. + if redis.call("get", key) == ARGV[1] then + redis.pcall("del", key) + count = count + 1 + end + end + + -- Return the number of entries removed. + return count +`; + +export type ClientExecutionResult = + | { + client: Client; + vote: "for"; + value: number; + } + | { + client: Client; + vote: "against"; + error: Error; + }; + +/* + * This object contains a summary of results. + */ +export type ExecutionStats = { + readonly membershipSize: number; + readonly quorumSize: number; + readonly votesFor: Set; + readonly votesAgainst: Map; +}; + +/* + * This object contains a summary of results. Because the result of an attempt + * can sometimes be determined before all requests are finished, each attempt + * contains a Promise that will resolve ExecutionStats once all requests are + * finished. A rejection of these promises should be considered undefined + * behavior and should cause a crash. + */ +export type ExecutionResult = { + attempts: ReadonlyArray>; + start: number; +}; + +/** + * + */ +export interface Settings { + readonly driftFactor: number; + readonly retryCount: number; + readonly retryDelay: number; + readonly retryJitter: number; + readonly automaticExtensionThreshold: number; +} + +// Define default settings. +const defaultSettings: Readonly = { + driftFactor: 0.01, + retryCount: 10, + retryDelay: 200, + retryJitter: 100, + automaticExtensionThreshold: 500 +}; + +// Modifyng this object is forbidden. +Object.freeze(defaultSettings); + +/* + * This error indicates a failure due to the existence of another lock for one + * or more of the requested resources. + */ +export class ResourceLockedError extends Error { + constructor(public readonly message: string) { + super(); + this.name = "ResourceLockedError"; + } +} + +/* + * This error indicates a failure of an operation to pass with a quorum. + */ +export class ExecutionError extends Error { + constructor( + public readonly message: string, + public readonly attempts: ReadonlyArray> + ) { + super(); + this.name = "ExecutionError"; + } +} + +/* + * An object of this type is returned when a resource is successfully locked. It + * contains convenience methods `release` and `extend` which perform the + * associated Redlock method on itself. + */ +export class Lock { + constructor( + public readonly redlock: Redlock, + public readonly resources: string[], + public readonly value: string, + public readonly attempts: ReadonlyArray>, + public expiration: number + ) {} + + async release(): Promise { + return this.redlock.release(this); + } + + async extend(duration: number): Promise { + return this.redlock.extend(this, duration); + } +} + +export type RedlockAbortSignal = AbortSignal & { error?: Error }; + +/** + * A redlock object is instantiated with an array of at least one redis client + * and an optional `options` object. Properties of the Redlock object should NOT + * be changed after it is first used, as doing so could have unintended + * consequences for live locks. + */ +export class Redlock extends EventEmitter { + public readonly clients: Set; + public readonly settings: Settings; + public readonly scripts: { + readonly acquireScript: { value: string; hash: string }; + readonly extendScript: { value: string; hash: string }; + readonly releaseScript: { value: string; hash: string }; + }; + + public constructor( + clients: Iterable, + settings: Partial = {}, + scripts: { + readonly acquireScript?: string | ((script: string) => string); + readonly extendScript?: string | ((script: string) => string); + readonly releaseScript?: string | ((script: string) => string); + } = {} + ) { + super(); + + // Prevent crashes on error events. + this.on("error", () => { + // Because redlock is designed for high availability, it does not care if + // a minority of redis instances/clusters fail at an operation. + // + // However, it can be helpful to monitor and log such cases. Redlock emits + // an "error" event whenever it encounters an error, even if the error is + // ignored in its normal operation. + // + // This function serves to prevent node's default behavior of crashing + // when an "error" event is emitted in the absence of listeners. + }); + + // Create a new array of client, to ensure no accidental mutation. + this.clients = new Set(clients); + if (this.clients.size === 0) { + throw new Error("Redlock must be instantiated with at least one redis client."); + } + + // Customize the settings for this instance. + this.settings = { + driftFactor: typeof settings.driftFactor === "number" ? settings.driftFactor : defaultSettings.driftFactor, + retryCount: typeof settings.retryCount === "number" ? settings.retryCount : defaultSettings.retryCount, + retryDelay: typeof settings.retryDelay === "number" ? settings.retryDelay : defaultSettings.retryDelay, + retryJitter: typeof settings.retryJitter === "number" ? settings.retryJitter : defaultSettings.retryJitter, + automaticExtensionThreshold: + typeof settings.automaticExtensionThreshold === "number" + ? settings.automaticExtensionThreshold + : defaultSettings.automaticExtensionThreshold + }; + + // Use custom scripts and script modifiers. + const acquireScript = + typeof scripts.acquireScript === "function" ? scripts.acquireScript(ACQUIRE_SCRIPT) : ACQUIRE_SCRIPT; + const extendScript = + typeof scripts.extendScript === "function" ? scripts.extendScript(EXTEND_SCRIPT) : EXTEND_SCRIPT; + const releaseScript = + typeof scripts.releaseScript === "function" ? scripts.releaseScript(RELEASE_SCRIPT) : RELEASE_SCRIPT; + + this.scripts = { + acquireScript: { + value: acquireScript, + hash: this._hash(acquireScript) + }, + extendScript: { + value: extendScript, + hash: this._hash(extendScript) + }, + releaseScript: { + value: releaseScript, + hash: this._hash(releaseScript) + } + }; + } + + /** + * Generate a sha1 hash compatible with redis evalsha. + */ + private _hash(value: string): string { + return createHash("sha1").update(value).digest("hex"); + } + + /** + * Generate a cryptographically random string. + */ + private _random(): string { + return randomBytes(16).toString("hex"); + } + + /** + * This method runs `.quit()` on all client connections. + */ + public async quit(): Promise { + const results = []; + for (const client of this.clients) { + results.push(client.quit()); + } + + await Promise.all(results); + } + + /** + * This method acquires a locks on the resources for the duration specified by + * the `duration`. + */ + public async acquire(resources: string[], duration: number, settings?: Partial): Promise { + if (Math.floor(duration) !== duration) { + throw new Error("Duration must be an integer value in milliseconds."); + } + + const value = this._random(); + + try { + const { attempts, start } = await this._execute( + this.scripts.acquireScript, + resources, + [value, duration], + settings + ); + + // Add 2 milliseconds to the drift to account for Redis expires precision, + // which is 1 ms, plus the configured allowable drift factor. + const drift = Math.round((settings?.driftFactor ?? this.settings.driftFactor) * duration) + 2; + + return new Lock(this, resources, value, attempts, start + duration - drift); + } catch (error) { + // If there was an error acquiring the lock, release any partial lock + // state that may exist on a minority of clients. + await this._execute(this.scripts.releaseScript, resources, [value], { + retryCount: 0 + }).catch(() => { + // Any error here will be ignored. + }); + + throw error; + } + } + + /** + * This method unlocks the provided lock from all servers still persisting it. + * It will fail with an error if it is unable to release the lock on a quorum + * of nodes, but will make no attempt to restore the lock in the case of a + * failure to release. It is safe to re-attempt a release or to ignore the + * error, as the lock will automatically expire after its timeout. + */ + public async release(lock: Lock, settings?: Partial): Promise { + // Immediately invalidate the lock. + lock.expiration = 0; + + // Attempt to release the lock. + return this._execute(this.scripts.releaseScript, lock.resources, [lock.value], settings); + } + + /** + * This method extends a valid lock by the provided `duration`. + */ + public async extend(existing: Lock, duration: number, settings?: Partial): Promise { + if (Math.floor(duration) !== duration) { + throw new Error("Duration must be an integer value in milliseconds."); + } + + // The lock has already expired. + if (existing.expiration < Date.now()) { + throw new ExecutionError("Cannot extend an already-expired lock.", []); + } + + const { attempts, start } = await this._execute( + this.scripts.extendScript, + existing.resources, + [existing.value, duration], + settings + ); + + // Invalidate the existing lock. + existing.expiration = 0; + + // Add 2 milliseconds to the drift to account for Redis expires precision, + // which is 1 ms, plus the configured allowable drift factor. + const drift = Math.round((settings?.driftFactor ?? this.settings.driftFactor) * duration) + 2; + + const replacement = new Lock(this, existing.resources, existing.value, attempts, start + duration - drift); + + return replacement; + } + + /** + * Execute a script on all clients. The resulting promise is resolved or + * rejected as soon as this quorum is reached; the resolution or rejection + * will contains a `stats` property that is resolved once all votes are in. + */ + private async _execute( + script: { value: string; hash: string }, + keys: string[], + args: (string | number)[], + _settings?: Partial + ): Promise { + const settings = _settings + ? { + ...this.settings, + ..._settings + } + : this.settings; + + // For the purpose of easy config serialization, we treat a retryCount of + // -1 a equivalent to Infinity. + const maxAttempts = settings.retryCount === -1 ? Infinity : settings.retryCount + 1; + + const attempts: Promise[] = []; + + while (true) { + const { vote, stats, start } = await this._attemptOperation(script, keys, args); + + attempts.push(stats); + + // The operation achieved a quorum in favor. + if (vote === "for") { + return { attempts, start }; + } + + // Wait before reattempting. + if (attempts.length < maxAttempts) { + await new Promise((resolve) => { + setTimeout( + resolve, + Math.max(0, settings.retryDelay + Math.floor((Math.random() * 2 - 1) * settings.retryJitter)), + undefined + ); + }); + } else { + throw new ExecutionError("The operation was unable to achieve a quorum during its retry window.", attempts); + } + } + } + + private async _attemptOperation( + script: { value: string; hash: string }, + keys: string[], + args: (string | number)[] + ): Promise< + | { vote: "for"; stats: Promise; start: number } + | { vote: "against"; stats: Promise; start: number } + > { + const start = Date.now(); + + return await new Promise((resolve) => { + const clientResults = []; + for (const client of this.clients) { + clientResults.push(this._attemptOperationOnClient(client, script, keys, args)); + } + + const stats: ExecutionStats = { + membershipSize: clientResults.length, + quorumSize: Math.floor(clientResults.length / 2) + 1, + votesFor: new Set(), + votesAgainst: new Map() + }; + + let done: () => void; + const statsPromise = new Promise((resolve) => { + done = () => resolve(stats); + }); + + // This is the expected flow for all successful and unsuccessful requests. + const onResultResolve = (clientResult: ClientExecutionResult): void => { + switch (clientResult.vote) { + case "for": + stats.votesFor.add(clientResult.client); + break; + case "against": + stats.votesAgainst.set(clientResult.client, clientResult.error); + break; + } + + // A quorum has determined a success. + if (stats.votesFor.size === stats.quorumSize) { + resolve({ + vote: "for", + stats: statsPromise, + start + }); + } + + // A quorum has determined a failure. + if (stats.votesAgainst.size === stats.quorumSize) { + resolve({ + vote: "against", + stats: statsPromise, + start + }); + } + + // All votes are in. + if (stats.votesFor.size + stats.votesAgainst.size === stats.membershipSize) { + done(); + } + }; + + // This is unexpected and should crash to prevent undefined behavior. + const onResultReject = (error: Error): void => { + throw error; + }; + + for (const result of clientResults) { + result.then(onResultResolve, onResultReject); + } + }); + } + + private async _attemptOperationOnClient( + client: Client, + script: { value: string; hash: string }, + keys: string[], + args: (string | number)[] + ): Promise { + try { + let result: number; + try { + // Attempt to evaluate the script by its hash. + // @ts-expect-error + const shaResult = (await client.evalsha(script.hash, keys.length, [...keys, ...args])) as unknown; + + if (typeof shaResult !== "number") { + throw new Error(`Unexpected result of type ${typeof shaResult} returned from redis.`); + } + + result = shaResult; + } catch (error) { + // If the redis server does not already have the script cached, + // reattempt the request with the script's raw text. + if (!(error instanceof Error) || !error.message.startsWith("NOSCRIPT")) { + throw error; + } + // @ts-expect-error + const rawResult = (await client.eval(script.value, keys.length, [...keys, ...args])) as unknown; + + if (typeof rawResult !== "number") { + throw new Error(`Unexpected result of type ${typeof rawResult} returned from redis.`); + } + + result = rawResult; + } + + // One or more of the resources was already locked. + if (result !== keys.length) { + throw new ResourceLockedError( + `The operation was applied to: ${result} of the ${keys.length} requested resources.` + ); + } + + return { + vote: "for", + client, + value: result + }; + } catch (error) { + if (!(error instanceof Error)) { + throw new Error(`Unexpected type ${typeof error} thrown with value: ${error}`); + } + + // Emit the error on the redlock instance for observability. + this.emit("error", error); + + return { + vote: "against", + client, + error + }; + } + } + + /** + * Wrap and execute a routine in the context of an auto-extending lock, + * returning a promise of the routine's value. In the case that auto-extension + * fails, an AbortSignal will be updated to indicate that abortion of the + * routine is in order, and to pass along the encountered error. + * + * @example + * ```ts + * await redlock.using([senderId, recipientId], 5000, { retryCount: 5 }, async (signal) => { + * const senderBalance = await getBalance(senderId); + * const recipientBalance = await getBalance(recipientId); + * + * if (senderBalance < amountToSend) { + * throw new Error("Insufficient balance."); + * } + * + * // The abort signal will be true if: + * // 1. the above took long enough that the lock needed to be extended + * // 2. redlock was unable to extend the lock + * // + * // In such a case, exclusivity can no longer be guaranteed for further + * // operations, and should be handled as an exceptional case. + * if (signal.aborted) { + * throw signal.error; + * } + * + * await setBalances([ + * {id: senderId, balance: senderBalance - amountToSend}, + * {id: recipientId, balance: recipientBalance + amountToSend}, + * ]); + * }); + * ``` + */ + + public async using( + resources: string[], + duration: number, + settings: Partial, + routine?: (signal: RedlockAbortSignal) => Promise + ): Promise; + + public async using( + resources: string[], + duration: number, + routine: (signal: RedlockAbortSignal) => Promise + ): Promise; + + public async using( + resources: string[], + duration: number, + settingsOrRoutine: undefined | Partial | ((signal: RedlockAbortSignal) => Promise), + optionalRoutine?: (signal: RedlockAbortSignal) => Promise + ): Promise { + if (Math.floor(duration) !== duration) { + throw new Error("Duration must be an integer value in milliseconds."); + } + + const settings = + settingsOrRoutine && typeof settingsOrRoutine !== "function" + ? { + ...this.settings, + ...settingsOrRoutine + } + : this.settings; + + const routine = optionalRoutine ?? settingsOrRoutine; + if (typeof routine !== "function") { + throw new Error("INVARIANT: routine is not a function."); + } + + if (settings.automaticExtensionThreshold > duration - 100) { + throw new Error( + "A lock `duration` must be at least 100ms greater than the `automaticExtensionThreshold` setting." + ); + } + + // The AbortController/AbortSignal pattern allows the routine to be notified + // of a failure to extend the lock, and subsequent expiration. In the event + // of an abort, the error object will be made available at `signal.error`. + const controller = new AbortController(); + + const signal = controller.signal as RedlockAbortSignal; + + function queue(): void { + timeout = setTimeout( + () => (extension = extend()), + lock.expiration - Date.now() - settings.automaticExtensionThreshold + ); + } + + async function extend(): Promise { + timeout = undefined; + + try { + lock = await lock.extend(duration); + queue(); + } catch (error) { + if (!(error instanceof Error)) { + throw new Error(`Unexpected thrown ${typeof error}: ${error}.`); + } + + if (lock.expiration > Date.now()) { + return (extension = extend()); + } + + signal.error = error instanceof Error ? error : new Error(`${error}`); + controller.abort(); + } + } + + let timeout: undefined | NodeJS.Timeout; + let extension: undefined | Promise; + let lock = await this.acquire(resources, duration, settings); + queue(); + + try { + return await routine(signal); + } finally { + // Clean up the timer. + if (timeout) { + clearTimeout(timeout); + timeout = undefined; + } + + // Wait for an in-flight extension to finish. + if (extension) { + await extension.catch(() => { + // An error here doesn't matter at all, because the routine has + // already completed, and a release will be attempted regardless. The + // only reason for waiting here is to prevent possible contention + // between the extension and release. + }); + } + + await lock.release(); + } + } +} diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index 9d85b6015..7046058b7 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -7,6 +7,7 @@ import { TScanFullRepoEventPayload, TScanPushEventPayload } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types"; +import { TSyncSecretsDTO } from "@app/services/secret/secret-types"; export enum QueueName { SecretRotation = "secret-rotation", @@ -21,7 +22,9 @@ export enum QueueName { SecretFullRepoScan = "secret-full-repo-scan", SecretPushEventScan = "secret-push-event-scan", UpgradeProjectToGhost = "upgrade-project-to-ghost", - DynamicSecretRevocation = "dynamic-secret-revocation" + DynamicSecretRevocation = "dynamic-secret-revocation", + SecretReplication = "secret-replication", + SecretSync = "secret-sync" // parent queue to push integration sync, webhook, and secret replication } export enum QueueJobs { @@ -37,7 +40,9 @@ export enum QueueJobs { SecretScan = "secret-scan", UpgradeProjectToGhost = "upgrade-project-to-ghost-job", DynamicSecretRevocation = "dynamic-secret-revocation", - DynamicSecretPruning = "dynamic-secret-pruning" + DynamicSecretPruning = "dynamic-secret-pruning", + SecretReplication = "secret-replication", + SecretSync = "secret-sync" // parent queue to push integration sync, webhook, and secret replication } export type TQueueJobTypes = { @@ -116,6 +121,14 @@ export type TQueueJobTypes = { dynamicSecretCfgId: string; }; }; + [QueueName.SecretReplication]: { + name: QueueJobs.SecretReplication; + payload: TSyncSecretsDTO; + }; + [QueueName.SecretSync]: { + name: QueueJobs.SecretSync; + payload: TSyncSecretsDTO; + }; }; export type TQueueServiceFactory = ReturnType; @@ -132,7 +145,7 @@ export const queueServiceFactory = (redisUrl: string) => { const start = ( name: T, - jobFn: (job: Job) => Promise, + jobFn: (job: Job, token?: string) => Promise, queueSettings: Omit = {} ) => { if (queueContainer[name]) { @@ -166,7 +179,7 @@ export const queueServiceFactory = (redisUrl: string) => { name: T, job: TQueueJobTypes[T]["name"], data: TQueueJobTypes[T]["payload"], - opts: JobsOptions & { jobId?: string } + opts?: JobsOptions & { jobId?: string } ) => { const q = queueContainer[name]; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 1593515ec..ed2d31254 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -44,6 +44,7 @@ import { secretApprovalRequestDALFactory } from "@app/ee/services/secret-approva import { secretApprovalRequestReviewerDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-reviewer-dal"; import { secretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal"; import { secretApprovalRequestServiceFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-service"; +import { secretReplicationServiceFactory } from "@app/ee/services/secret-replication/secret-replication-service"; import { secretRotationDALFactory } from "@app/ee/services/secret-rotation/secret-rotation-dal"; import { secretRotationQueueFactory } from "@app/ee/services/secret-rotation/secret-rotation-queue"; import { secretRotationServiceFactory } from "@app/ee/services/secret-rotation/secret-rotation-service"; @@ -240,8 +241,8 @@ export const registerRoutes = async ( const sapApproverDAL = secretApprovalPolicyApproverDALFactory(db); const secretApprovalPolicyDAL = secretApprovalPolicyDALFactory(db); const secretApprovalRequestDAL = secretApprovalRequestDALFactory(db); - const sarReviewerDAL = secretApprovalRequestReviewerDALFactory(db); - const sarSecretDAL = secretApprovalRequestSecretDALFactory(db); + const secretApprovalRequestReviewerDAL = secretApprovalRequestReviewerDALFactory(db); + const secretApprovalRequestSecretDAL = secretApprovalRequestSecretDALFactory(db); const secretRotationDAL = secretRotationDALFactory(db); const snapshotDAL = snapshotDALFactory(db); @@ -288,7 +289,7 @@ export const registerRoutes = async ( permissionService, auditLogStreamDAL }); - const sapService = secretApprovalPolicyServiceFactory({ + const secretApprovalPolicyService = secretApprovalPolicyServiceFactory({ projectMembershipDAL, projectEnvDAL, secretApprovalPolicyApproverDAL: sapApproverDAL, @@ -489,7 +490,7 @@ export const registerRoutes = async ( projectBotDAL, projectMembershipDAL, secretApprovalRequestDAL, - secretApprovalSecretDAL: sarSecretDAL, + secretApprovalSecretDAL: secretApprovalRequestSecretDAL, projectUserMembershipRoleDAL }); @@ -587,6 +588,7 @@ export const registerRoutes = async ( secretVersionTagDAL }); const secretImportService = secretImportServiceFactory({ + licenseService, projectEnvDAL, folderDAL, permissionService, @@ -621,19 +623,18 @@ export const registerRoutes = async ( secretSharingDAL }); - const sarService = secretApprovalRequestServiceFactory({ + const secretApprovalRequestService = secretApprovalRequestServiceFactory({ permissionService, projectBotService, folderDAL, secretDAL, secretTagDAL, - secretApprovalRequestSecretDAL: sarSecretDAL, - secretApprovalRequestReviewerDAL: sarReviewerDAL, + secretApprovalRequestSecretDAL, + secretApprovalRequestReviewerDAL, projectDAL, secretVersionDAL, secretBlindIndexDAL, secretApprovalRequestDAL, - secretService, snapshotService, secretVersionTagDAL, secretQueueService @@ -662,6 +663,23 @@ export const registerRoutes = async ( accessApprovalPolicyApproverDAL }); + const secretReplicationService = secretReplicationServiceFactory({ + secretTagDAL, + secretVersionTagDAL, + secretDAL, + secretVersionDAL, + secretImportDAL, + keyStore, + queueService, + folderDAL, + secretApprovalPolicyService, + secretBlindIndexDAL, + secretApprovalRequestDAL, + secretApprovalRequestSecretDAL, + secretQueueService, + projectMembershipDAL, + projectBotService + }); const secretRotationQueue = secretRotationQueueFactory({ telemetryService, secretRotationDAL, @@ -826,6 +844,7 @@ export const registerRoutes = async ( projectEnv: projectEnvService, projectRole: projectRoleService, secret: secretService, + secretReplication: secretReplicationService, secretTag: secretTagService, folder: folderService, secretImport: secretImportService, @@ -842,10 +861,10 @@ export const registerRoutes = async ( identityGcpAuth: identityGcpAuthService, identityAwsAuth: identityAwsAuthService, identityAzureAuth: identityAzureAuthService, - secretApprovalPolicy: sapService, accessApprovalPolicy: accessApprovalPolicyService, accessApprovalRequest: accessApprovalRequestService, - secretApprovalRequest: sarService, + secretApprovalPolicy: secretApprovalPolicyService, + secretApprovalRequest: secretApprovalRequestService, secretRotation: secretRotationService, dynamicSecret: dynamicSecretService, dynamicSecretLease: dynamicSecretLeaseService, diff --git a/backend/src/server/routes/v1/secret-import-router.ts b/backend/src/server/routes/v1/secret-import-router.ts index d036fdbdd..50311273c 100644 --- a/backend/src/server/routes/v1/secret-import-router.ts +++ b/backend/src/server/routes/v1/secret-import-router.ts @@ -29,7 +29,8 @@ export const registerSecretImportRouter = async (server: FastifyZodProvider) => import: z.object({ environment: z.string().trim().describe(SECRET_IMPORTS.CREATE.import.environment), path: z.string().trim().transform(removeTrailingSlash).describe(SECRET_IMPORTS.CREATE.import.path) - }) + }), + isReplication: z.boolean().default(false) }), response: { 200: z.object({ @@ -210,6 +211,49 @@ export const registerSecretImportRouter = async (server: FastifyZodProvider) => } }); + server.route({ + method: "POST", + url: "/:secretImportId/replication-resync", + config: { + rateLimit: secretsLimit + }, + schema: { + description: "Resync secret replication of secret imports", + security: [ + { + bearerAuth: [] + } + ], + params: z.object({ + secretImportId: z.string().trim().describe(SECRET_IMPORTS.UPDATE.secretImportId) + }), + body: z.object({ + workspaceId: z.string().trim().describe(SECRET_IMPORTS.UPDATE.workspaceId), + environment: z.string().trim().describe(SECRET_IMPORTS.UPDATE.environment), + path: z.string().trim().default("/").transform(removeTrailingSlash).describe(SECRET_IMPORTS.UPDATE.path) + }), + response: { + 200: z.object({ + message: z.string() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const { message } = await server.services.secretImport.resyncSecretImportReplication({ + actorId: req.permission.id, + actor: req.permission.type, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId, + id: req.params.secretImportId, + ...req.body, + projectId: req.body.workspaceId + }); + + return { message }; + } + }); + server.route({ method: "GET", url: "/", @@ -232,11 +276,9 @@ export const registerSecretImportRouter = async (server: FastifyZodProvider) => 200: z.object({ message: z.string(), secretImports: SecretImportsSchema.omit({ importEnv: true }) - .merge( - z.object({ - importEnv: z.object({ name: z.string(), slug: z.string(), id: z.string() }) - }) - ) + .extend({ + importEnv: z.object({ name: z.string(), slug: z.string(), id: z.string() }) + }) .array() }) } diff --git a/backend/src/server/routes/v3/secret-router.ts b/backend/src/server/routes/v3/secret-router.ts index 6fa574a69..05db617b9 100644 --- a/backend/src/server/routes/v3/secret-router.ts +++ b/backend/src/server/routes/v3/secret-router.ts @@ -9,7 +9,6 @@ import { ServiceTokenScopes } from "@app/db/schemas"; import { EventType } from "@app/ee/services/audit-log/audit-log-types"; -import { CommitType } from "@app/ee/services/secret-approval-request/secret-approval-request-types"; import { RAW_SECRETS, SECRETS } from "@app/lib/api-docs"; import { BadRequestError } from "@app/lib/errors"; import { removeTrailingSlash } from "@app/lib/fn"; @@ -19,6 +18,7 @@ import { getUserAgentType } from "@app/server/plugins/audit-log"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { ActorType, AuthMode } from "@app/services/auth/auth-type"; import { ProjectFilterType } from "@app/services/project/project-types"; +import { SecretOperations } from "@app/services/secret/secret-types"; import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types"; import { secretRawSchema } from "../sanitizedSchemas"; @@ -902,7 +902,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { projectId, policy, data: { - [CommitType.Create]: [ + [SecretOperations.Create]: [ { secretName: req.params.secretName, secretValueCiphertext, @@ -1084,7 +1084,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { projectId, policy, data: { - [CommitType.Update]: [ + [SecretOperations.Update]: [ { secretName: req.params.secretName, newSecretName, @@ -1234,7 +1234,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { projectId, policy, data: { - [CommitType.Delete]: [ + [SecretOperations.Delete]: [ { secretName: req.params.secretName } @@ -1364,7 +1364,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { projectId, policy, data: { - [CommitType.Create]: inputSecrets + [SecretOperations.Create]: inputSecrets } }); @@ -1491,7 +1491,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { projectId, policy, data: { - [CommitType.Update]: inputSecrets.filter(({ type }) => type === "shared") + [SecretOperations.Update]: inputSecrets.filter(({ type }) => type === "shared") } }); @@ -1606,7 +1606,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { projectId, policy, data: { - [CommitType.Delete]: inputSecrets.filter(({ type }) => type === "shared") + [SecretOperations.Delete]: inputSecrets.filter(({ type }) => type === "shared") } }); await server.services.auditLog.createAuditLog({ diff --git a/backend/src/services/secret-folder/secret-folder-dal.ts b/backend/src/services/secret-folder/secret-folder-dal.ts index b3147d1fa..0e896d0c6 100644 --- a/backend/src/services/secret-folder/secret-folder-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-dal.ts @@ -169,6 +169,7 @@ const sqlFindSecretPathByFolderId = (db: Knex, projectId: string, folderIds: str // this is for root condition // if the given folder id is root folder id then intial path is set as / instead of /root // if not root folder the path here will be / + depth: 1, path: db.raw(`CONCAT('/', (CASE WHEN "parentId" is NULL THEN '' ELSE ${TableName.SecretFolder}.name END))`), child: db.raw("NULL::uuid"), environmentSlug: `${TableName.Environment}.slug` @@ -185,6 +186,7 @@ const sqlFindSecretPathByFolderId = (db: Knex, projectId: string, folderIds: str .select({ // then we join join this folder name behind previous as we are going from child to parent // the root folder check is used to avoid last / and also root name in folders + depth: db.raw("parent.depth + 1"), path: db.raw( `CONCAT( CASE WHEN ${TableName.SecretFolder}."parentId" is NULL THEN '' @@ -199,7 +201,7 @@ const sqlFindSecretPathByFolderId = (db: Knex, projectId: string, folderIds: str ); }) .select("*") - .from("parent"); + .from("parent"); export type TSecretFolderDALFactory = ReturnType; // never change this. If u do write a migration for it @@ -260,12 +262,23 @@ export const secretFolderDALFactory = (db: TDbClient) => { try { const folders = await sqlFindSecretPathByFolderId(tx || db, projectId, folderIds); + // travelling all the way from leaf node to root contains real path const rootFolders = groupBy( folders.filter(({ parentId }) => parentId === null), (i) => i.child || i.id // root condition then child and parent will null ); + const actualFolders = groupBy( + folders.filter(({ depth }) => depth === 1), + (i) => i.id // root condition then child and parent will null + ); - return folderIds.map((folderId) => rootFolders[folderId]?.[0]); + return folderIds.map((folderId) => { + if (!rootFolders[folderId]?.[0]) return; + + const actualId = rootFolders[folderId][0].child || rootFolders[folderId][0].id; + const folder = actualFolders[actualId][0]; + return { ...folder, path: rootFolders[folderId]?.[0].path }; + }); } catch (error) { throw new DatabaseError({ error, name: "Find by secret path" }); } diff --git a/backend/src/services/secret-folder/secret-folder-service.ts b/backend/src/services/secret-folder/secret-folder-service.ts index da429d88a..97258c006 100644 --- a/backend/src/services/secret-folder/secret-folder-service.ts +++ b/backend/src/services/secret-folder/secret-folder-service.ts @@ -253,7 +253,7 @@ export const secretFolderServiceFactory = ({ const env = await projectEnvDAL.findOne({ projectId, slug: environment }); if (!env) throw new BadRequestError({ message: "Environment not found", name: "Update folder" }); const folder = await folderDAL - .findOne({ envId: env.id, id, parentId: parentFolder.id }) + .findOne({ envId: env.id, id, parentId: parentFolder.id, isReserved: false }) // now folder api accepts id based change // this is for cli backward compatiability and when cli removes this, we will remove this logic .catch(() => folderDAL.findOne({ envId: env.id, name: id, parentId: parentFolder.id })); @@ -276,7 +276,11 @@ export const secretFolderServiceFactory = ({ } const newFolder = await folderDAL.transaction(async (tx) => { - const [doc] = await folderDAL.update({ envId: env.id, id: folder.id, parentId: parentFolder.id }, { name }, tx); + const [doc] = await folderDAL.update( + { envId: env.id, id: folder.id, parentId: parentFolder.id, isReserved: false }, + { name }, + tx + ); await folderVersionDAL.create( { name: doc.name, @@ -324,7 +328,12 @@ export const secretFolderServiceFactory = ({ if (!parentFolder) throw new BadRequestError({ message: "Secret path not found" }); const [doc] = await folderDAL.delete( - { envId: env.id, [uuidValidate(idOrName) ? "id" : "name"]: idOrName, parentId: parentFolder.id }, + { + envId: env.id, + [uuidValidate(idOrName) ? "id" : "name"]: idOrName, + parentId: parentFolder.id, + isReserved: false + }, tx ); if (!doc) throw new BadRequestError({ message: "Folder not found", name: "Delete folder" }); @@ -354,7 +363,7 @@ export const secretFolderServiceFactory = ({ const parentFolder = await folderDAL.findBySecretPath(projectId, environment, secretPath); if (!parentFolder) return []; - const folders = await folderDAL.find({ envId: env.id, parentId: parentFolder.id }); + const folders = await folderDAL.find({ envId: env.id, parentId: parentFolder.id, isReserved: false }); return folders; }; diff --git a/backend/src/services/secret-folder/secret-folder-types.ts b/backend/src/services/secret-folder/secret-folder-types.ts index 1405f8bd7..c01d5f7b8 100644 --- a/backend/src/services/secret-folder/secret-folder-types.ts +++ b/backend/src/services/secret-folder/secret-folder-types.ts @@ -1,5 +1,9 @@ import { TProjectPermission } from "@app/lib/types"; +export enum ReservedFolders { + SecretReplication = "__reserve_replication_" +} + export type TCreateFolderDTO = { environment: string; path: string; diff --git a/backend/src/services/secret-folder/secret-folder-version-dal.ts b/backend/src/services/secret-folder/secret-folder-version-dal.ts index f133308cf..73b536b48 100644 --- a/backend/src/services/secret-folder/secret-folder-version-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-version-dal.ts @@ -15,7 +15,7 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => { try { const docs = await (tx || db)(TableName.SecretFolderVersion) .join(TableName.SecretFolder, `${TableName.SecretFolderVersion}.folderId`, `${TableName.SecretFolder}.id`) - .where({ parentId: folderId }) + .where({ parentId: folderId, isReserved: false }) .join( (tx || db)(TableName.SecretFolderVersion) .groupBy("envId", "folderId") diff --git a/backend/src/services/secret-import/secret-import-dal.ts b/backend/src/services/secret-import/secret-import-dal.ts index aa45d410d..0e73a8c23 100644 --- a/backend/src/services/secret-import/secret-import-dal.ts +++ b/backend/src/services/secret-import/secret-import-dal.ts @@ -20,14 +20,14 @@ export const secretImportDALFactory = (db: TDbClient) => { return lastPos?.position || 0; }; - const updateAllPosition = async (folderId: string, pos: number, targetPos: number, tx?: Knex) => { + const updateAllPosition = async (folderId: string, pos: number, targetPos: number, positionInc = 1, tx?: Knex) => { try { if (targetPos === -1) { // this means delete await (tx || db)(TableName.SecretImport) .where({ folderId }) .andWhere("position", ">", pos) - .decrement("position", 1); + .decrement("position", positionInc); return; } @@ -36,13 +36,13 @@ export const secretImportDALFactory = (db: TDbClient) => { .where({ folderId }) .where("position", "<=", targetPos) .andWhere("position", ">", pos) - .decrement("position", 1); + .decrement("position", positionInc); } else { await (tx || db)(TableName.SecretImport) .where({ folderId }) .where("position", ">=", targetPos) .andWhere("position", "<", pos) - .increment("position", 1); + .increment("position", positionInc); } } catch (error) { throw new DatabaseError({ error, name: "Update position" }); @@ -74,6 +74,7 @@ export const secretImportDALFactory = (db: TDbClient) => { try { const docs = await (tx || db)(TableName.SecretImport) .whereIn("folderId", folderIds) + .where("isReplication", false) .join(TableName.Environment, `${TableName.SecretImport}.importEnv`, `${TableName.Environment}.id`) .select( db.ref("*").withSchema(TableName.SecretImport) as unknown as keyof TSecretImports, diff --git a/backend/src/services/secret-import/secret-import-fns.ts b/backend/src/services/secret-import/secret-import-fns.ts index fffc22a99..06ffbc903 100644 --- a/backend/src/services/secret-import/secret-import-fns.ts +++ b/backend/src/services/secret-import/secret-import-fns.ts @@ -79,7 +79,7 @@ export const fnSecretsFromImports = async ({ let secretsFromDeeperImports: TSecretImportSecrets[] = []; if (deeperImports.length) { secretsFromDeeperImports = await fnSecretsFromImports({ - allowedImports: deeperImports, + allowedImports: deeperImports.filter(({ isReplication }) => !isReplication), secretImportDAL, folderDAL, secretDAL, diff --git a/backend/src/services/secret-import/secret-import-service.ts b/backend/src/services/secret-import/secret-import-service.ts index 43676ba04..237c7cfe4 100644 --- a/backend/src/services/secret-import/secret-import-service.ts +++ b/backend/src/services/secret-import/secret-import-service.ts @@ -1,7 +1,12 @@ +import path from "node:path"; + import { ForbiddenError, subject } from "@casl/ability"; +import { TableName } from "@app/db/schemas"; +import { TLicenseServiceFactory } from "@app/ee/services/license/license-service"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; +import { getReplicationFolderName } from "@app/ee/services/secret-replication/secret-replication-service"; import { BadRequestError } from "@app/lib/errors"; import { TProjectDALFactory } from "../project/project-dal"; @@ -16,6 +21,7 @@ import { TDeleteSecretImportDTO, TGetSecretImportsDTO, TGetSecretsFromImportDTO, + TResyncSecretImportReplicationDTO, TUpdateSecretImportDTO } from "./secret-import-types"; @@ -26,7 +32,8 @@ type TSecretImportServiceFactoryDep = { projectDAL: Pick; projectEnvDAL: TProjectEnvDALFactory; permissionService: Pick; - secretQueueService: Pick; + secretQueueService: Pick; + licenseService: Pick; }; const ERR_SEC_IMP_NOT_FOUND = new BadRequestError({ message: "Secret import not found" }); @@ -40,7 +47,8 @@ export const secretImportServiceFactory = ({ folderDAL, projectDAL, secretDAL, - secretQueueService + secretQueueService, + licenseService }: TSecretImportServiceFactoryDep) => { const createImport = async ({ environment, @@ -50,7 +58,8 @@ export const secretImportServiceFactory = ({ actorOrgId, actorAuthMethod, projectId, - path + isReplication, + path: secretPath }: TCreateSecretImportDTO) => { const { permission } = await permissionService.getProjectPermission( actor, @@ -63,7 +72,7 @@ export const secretImportServiceFactory = ({ // check if user has permission to import into destination path ForbiddenError.from(permission).throwUnlessCan( ProjectPermissionActions.Create, - subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) + subject(ProjectPermissionSub.Secrets, { environment, secretPath }) ); // check if user has permission to import from target path @@ -74,10 +83,18 @@ export const secretImportServiceFactory = ({ secretPath: data.path }) ); + if (isReplication) { + const plan = await licenseService.getPlan(actorOrgId); + if (!plan.secretApproval) { + throw new BadRequestError({ + message: "Failed to create secret replication due to plan restriction. Upgrade plan to create replication." + }); + } + } await projectDAL.checkProjectUpgradeStatus(projectId); - const folder = await folderDAL.findBySecretPath(projectId, environment, path); + const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create import" }); const [importEnv] = await projectEnvDAL.findBySlugs(projectId, [data.environment]); @@ -88,35 +105,62 @@ export const secretImportServiceFactory = ({ const existingImport = await secretImportDAL.findOne({ folderId: sourceFolder.id, importEnv: folder.environment.id, - importPath: path + importPath: secretPath }); if (existingImport) throw new BadRequestError({ message: "Cyclic import not allowed" }); } const secImport = await secretImportDAL.transaction(async (tx) => { const lastPos = await secretImportDAL.findLastImportPosition(folder.id, tx); - return secretImportDAL.create( + const doc = await secretImportDAL.create( { folderId: folder.id, position: lastPos + 1, importEnv: importEnv.id, - importPath: data.path + importPath: data.path, + isReplication }, tx ); + if (doc.isReplication) { + await secretImportDAL.create( + { + folderId: folder.id, + position: lastPos + 2, + isReserved: true, + importEnv: folder.environment.id, + importPath: path.join(secretPath, getReplicationFolderName(doc.id)) + }, + tx + ); + } + return doc; }); - await secretQueueService.syncSecrets({ - secretPath: secImport.importPath, - projectId, - environment: importEnv.slug - }); + if (secImport.isReplication && sourceFolder) { + await secretQueueService.replicateSecrets({ + secretPath: secImport.importPath, + projectId, + environmentSlug: importEnv.slug, + pickOnlyImportIds: [secImport.id], + actorId, + actor + }); + } else { + await secretQueueService.syncSecrets({ + secretPath, + projectId, + environmentSlug: environment, + actorId, + actor + }); + } return { ...secImport, importEnv }; }; const updateImport = async ({ - path, + path: secretPath, environment, projectId, actor, @@ -135,10 +179,10 @@ export const secretImportServiceFactory = ({ ); ForbiddenError.from(permission).throwUnlessCan( ProjectPermissionActions.Edit, - subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) + subject(ProjectPermissionSub.Secrets, { environment, secretPath }) ); - const folder = await folderDAL.findBySecretPath(projectId, environment, path); + const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Update import" }); const secImpDoc = await secretImportDAL.findOne({ folderId: folder.id, id }); @@ -158,7 +202,7 @@ export const secretImportServiceFactory = ({ const existingImport = await secretImportDAL.findOne({ folderId: sourceFolder.id, importEnv: folder.environment.id, - importPath: path + importPath: secretPath }); if (existingImport) throw new BadRequestError({ message: "Cyclic import not allowed" }); } @@ -167,12 +211,31 @@ export const secretImportServiceFactory = ({ const secImp = await secretImportDAL.findOne({ folderId: folder.id, id }); if (!secImp) throw ERR_SEC_IMP_NOT_FOUND; if (data.position) { - await secretImportDAL.updateAllPosition(folder.id, secImp.position, data.position, tx); + if (secImp.isReplication) { + await secretImportDAL.updateAllPosition(folder.id, secImp.position, data.position, 2, tx); + } else { + await secretImportDAL.updateAllPosition(folder.id, secImp.position, data.position, 1, tx); + } + } + if (secImp.isReplication) { + const replicationFolderPath = path.join(secretPath, getReplicationFolderName(secImp.id)); + await secretImportDAL.update( + { + folderId: folder.id, + importEnv: folder.environment.id, + importPath: replicationFolderPath, + isReserved: true + }, + { position: data?.position ? data.position + 1 : undefined }, + tx + ); } const [doc] = await secretImportDAL.update( { id, folderId: folder.id }, { - position: data?.position, + // when moving replicated import, the position is meant for reserved import + // replicated one should always be behind the reserved import + position: data.position, importEnv: data?.environment ? importedEnv.id : undefined, importPath: data?.path }, @@ -184,7 +247,7 @@ export const secretImportServiceFactory = ({ }; const deleteImport = async ({ - path, + path: secretPath, environment, projectId, actor, @@ -202,16 +265,34 @@ export const secretImportServiceFactory = ({ ); ForbiddenError.from(permission).throwUnlessCan( ProjectPermissionActions.Delete, - subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) + subject(ProjectPermissionSub.Secrets, { environment, secretPath }) ); - const folder = await folderDAL.findBySecretPath(projectId, environment, path); + const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Delete import" }); const secImport = await secretImportDAL.transaction(async (tx) => { const [doc] = await secretImportDAL.delete({ folderId: folder.id, id }, tx); if (!doc) throw new BadRequestError({ name: "Sec imp del", message: "Secret import doc not found" }); - await secretImportDAL.updateAllPosition(folder.id, doc.position, -1, tx); + if (doc.isReplication) { + const replicationFolderPath = path.join(secretPath, getReplicationFolderName(doc.id)); + const replicatedFolder = await folderDAL.findBySecretPath(projectId, environment, replicationFolderPath, tx); + if (replicatedFolder) { + await secretImportDAL.delete( + { + folderId: folder.id, + importEnv: folder.environment.id, + importPath: replicationFolderPath, + isReserved: true + }, + tx + ); + await folderDAL.deleteById(replicatedFolder.id, tx); + } + await secretImportDAL.updateAllPosition(folder.id, doc.position, -1, 2, tx); + } else { + await secretImportDAL.updateAllPosition(folder.id, doc.position, -1, 1, tx); + } const importEnv = await projectEnvDAL.findById(doc.importEnv); if (!importEnv) throw new BadRequestError({ error: "Imported env not found", name: "Create import" }); @@ -219,16 +300,91 @@ export const secretImportServiceFactory = ({ }); await secretQueueService.syncSecrets({ - secretPath: path, + secretPath, projectId, - environment + environmentSlug: environment, + actor, + actorId }); return secImport; }; + const resyncSecretImportReplication = async ({ + environment, + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + path: secretPath, + id: secretImportDocId + }: TResyncSecretImportReplicationDTO) => { + const { permission, membership } = await permissionService.getProjectPermission( + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + ); + + // check if user has permission to import into destination path + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionActions.Create, + subject(ProjectPermissionSub.Secrets, { environment, secretPath }) + ); + + const plan = await licenseService.getPlan(actorOrgId); + if (!plan.secretApproval) { + throw new BadRequestError({ + message: "Failed to create secret replication due to plan restriction. Upgrade plan to create replication." + }); + } + + const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); + if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Update import" }); + + const [secretImportDoc] = await secretImportDAL.find({ + folderId: folder.id, + [`${TableName.SecretImport}.id` as "id"]: secretImportDocId + }); + if (!secretImportDoc) throw new BadRequestError({ message: "Failed to find secret import" }); + + if (!secretImportDoc.isReplication) throw new BadRequestError({ message: "Import is not in replication mode" }); + + // check if user has permission to import from target path + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionActions.Create, + subject(ProjectPermissionSub.Secrets, { + environment: secretImportDoc.importEnv.slug, + secretPath: secretImportDoc.importPath + }) + ); + + await projectDAL.checkProjectUpgradeStatus(projectId); + + const sourceFolder = await folderDAL.findBySecretPath( + projectId, + secretImportDoc.importEnv.slug, + secretImportDoc.importPath + ); + + if (membership && sourceFolder) { + await secretQueueService.replicateSecrets({ + secretPath: secretImportDoc.importPath, + projectId, + environmentSlug: secretImportDoc.importEnv.slug, + pickOnlyImportIds: [secretImportDoc.id], + actorId, + actor + }); + } + + return { message: "replication started" }; + }; + const getImports = async ({ - path, + path: secretPath, environment, projectId, actor, @@ -245,10 +401,10 @@ export const secretImportServiceFactory = ({ ); ForbiddenError.from(permission).throwUnlessCan( ProjectPermissionActions.Read, - subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) + subject(ProjectPermissionSub.Secrets, { environment, secretPath }) ); - const folder = await folderDAL.findBySecretPath(projectId, environment, path); + const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Get imports" }); const secImports = await secretImportDAL.find({ folderId: folder.id }); @@ -256,7 +412,7 @@ export const secretImportServiceFactory = ({ }; const getSecretsFromImports = async ({ - path, + path: secretPath, environment, projectId, actor, @@ -273,13 +429,13 @@ export const secretImportServiceFactory = ({ ); ForbiddenError.from(permission).throwUnlessCan( ProjectPermissionActions.Read, - subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) + subject(ProjectPermissionSub.Secrets, { environment, secretPath }) ); - const folder = await folderDAL.findBySecretPath(projectId, environment, path); + const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); if (!folder) return []; // this will already order by position // so anything based on this order will also be in right position - const secretImports = await secretImportDAL.find({ folderId: folder.id }); + const secretImports = await secretImportDAL.find({ folderId: folder.id, isReplication: false }); const allowedImports = secretImports.filter(({ importEnv, importPath }) => permission.can( @@ -299,6 +455,7 @@ export const secretImportServiceFactory = ({ deleteImport, getImports, getSecretsFromImports, + resyncSecretImportReplication, fnSecretsFromImports }; }; diff --git a/backend/src/services/secret-import/secret-import-types.ts b/backend/src/services/secret-import/secret-import-types.ts index d123f28da..01847738b 100644 --- a/backend/src/services/secret-import/secret-import-types.ts +++ b/backend/src/services/secret-import/secret-import-types.ts @@ -7,6 +7,7 @@ export type TCreateSecretImportDTO = { environment: string; path: string; }; + isReplication?: boolean; } & TProjectPermission; export type TUpdateSecretImportDTO = { @@ -16,6 +17,12 @@ export type TUpdateSecretImportDTO = { data: Partial<{ environment: string; path: string; position: number }>; } & TProjectPermission; +export type TResyncSecretImportReplicationDTO = { + environment: string; + path: string; + id: string; +} & TProjectPermission; + export type TDeleteSecretImportDTO = { environment: string; path: string; diff --git a/backend/src/services/secret/secret-fns.ts b/backend/src/services/secret/secret-fns.ts index 51ad7a6aa..3cd6c4e6e 100644 --- a/backend/src/services/secret/secret-fns.ts +++ b/backend/src/services/secret/secret-fns.ts @@ -32,6 +32,8 @@ import { TCreateManySecretsRawFn, TCreateManySecretsRawFnFactory, TFnSecretBlindIndexCheck, + TFnSecretBlindIndexCheckV2, + TFnSecretBulkDelete, TFnSecretBulkInsert, TFnSecretBulkUpdate, TUpdateManySecretsRawFn, @@ -149,7 +151,8 @@ export const recursivelyGetSecretPaths = ({ // Fetch all folders in env once with a single query const folders = await folderDAL.find({ - envId: env.id + envId: env.id, + isReserved: false }); // Build the folder hierarchy map @@ -396,6 +399,30 @@ export const decryptSecretRaw = ( }; }; +// this is used when secret blind index already exist +// mainly for secret approval +export const fnSecretBlindIndexCheckV2 = async ({ + inputSecrets, + folderId, + userId, + secretDAL +}: TFnSecretBlindIndexCheckV2) => { + if (inputSecrets.some(({ type }) => type === SecretType.Personal) && !userId) { + throw new BadRequestError({ message: "Missing user id for personal secret" }); + } + const secrets = await secretDAL.findByBlindIndexes( + folderId, + inputSecrets.map(({ secretBlindIndex, type }) => ({ + blindIndex: secretBlindIndex, + type: type || SecretType.Shared + })), + userId + ); + const secsGroupedByBlindIndex = groupBy(secrets, (i) => i.secretBlindIndex as string); + + return { secsGroupedByBlindIndex, secrets }; +}; + /** * Grabs and processes nested secret references from a string * @@ -598,6 +625,35 @@ export const fnSecretBulkUpdate = async ({ return newSecrets.map((secret) => ({ ...secret, _id: secret.id })); }; +export const fnSecretBulkDelete = async ({ + folderId, + inputSecrets, + tx, + actorId, + secretDAL, + secretQueueService +}: TFnSecretBulkDelete) => { + const deletedSecrets = await secretDAL.deleteMany( + inputSecrets.map(({ type, secretBlindIndex }) => ({ + blindIndex: secretBlindIndex, + type + })), + folderId, + actorId, + tx + ); + + await Promise.allSettled( + deletedSecrets + .filter(({ secretReminderRepeatDays }) => Boolean(secretReminderRepeatDays)) + .map(({ id, secretReminderRepeatDays }) => + secretQueueService.removeSecretReminder({ secretId: id, repeatDays: secretReminderRepeatDays as number }) + ) + ); + + return deletedSecrets; +}; + export const createManySecretsRawFnFactory = ({ projectDAL, projectBotDAL, diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index f3e3f1731..d40a18e5e 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -28,7 +28,12 @@ import { TWebhookDALFactory } from "../webhook/webhook-dal"; import { fnTriggerWebhook } from "../webhook/webhook-fns"; import { TSecretDALFactory } from "./secret-dal"; import { interpolateSecrets } from "./secret-fns"; -import { TCreateSecretReminderDTO, THandleReminderDTO, TRemoveSecretReminderDTO } from "./secret-types"; +import { + TCreateSecretReminderDTO, + THandleReminderDTO, + TRemoveSecretReminderDTO, + TSyncSecretsDTO +} from "./secret-types"; export type TSecretQueueFactory = ReturnType; type TSecretQueueFactoryDep = { @@ -59,8 +64,10 @@ export type TGetSecrets = { }; const MAX_SYNC_SECRET_DEPTH = 5; -const uniqueIntegrationKey = (environment: string, secretPath: string) => `integration-${environment}-${secretPath}`; +export const uniqueSecretQueueKey = (environment: string, secretPath: string) => + `secret-queue-dedupe-${environment}-${secretPath}`; +type TIntegrationSecret = Record; export const secretQueueFactory = ({ queueService, integrationDAL, @@ -81,68 +88,6 @@ export const secretQueueFactory = ({ secretTagDAL, secretVersionTagDAL }: TSecretQueueFactoryDep) => { - const createManySecretsRawFn = createManySecretsRawFnFactory({ - projectDAL, - projectBotDAL, - secretDAL, - secretVersionDAL, - secretBlindIndexDAL, - secretTagDAL, - secretVersionTagDAL, - folderDAL - }); - - const updateManySecretsRawFn = updateManySecretsRawFnFactory({ - projectDAL, - projectBotDAL, - secretDAL, - secretVersionDAL, - secretBlindIndexDAL, - secretTagDAL, - secretVersionTagDAL, - folderDAL - }); - - const syncIntegrations = async (dto: TGetSecrets & { deDupeQueue?: Record }) => { - await queueService.queue(QueueName.IntegrationSync, QueueJobs.IntegrationSync, dto, { - attempts: 3, - delay: 1000, - backoff: { - type: "exponential", - delay: 3000 - }, - removeOnComplete: true, - removeOnFail: true - }); - }; - - const syncSecrets = async ({ - deDupeQueue = {}, - ...dto - }: TGetSecrets & { depth?: number; deDupeQueue?: Record }) => { - const deDuplicationKey = uniqueIntegrationKey(dto.environment, dto.secretPath); - if (deDupeQueue?.[deDuplicationKey]) { - return; - } - // eslint-disable-next-line - deDupeQueue[deDuplicationKey] = true; - logger.info( - `syncSecrets: syncing project secrets where [projectId=${dto.projectId}] [environment=${dto.environment}] [path=${dto.secretPath}]` - ); - await queueService.queue(QueueName.SecretWebhook, QueueJobs.SecWebhook, dto, { - jobId: `secret-webhook-${dto.environment}-${dto.projectId}-${dto.secretPath}`, - removeOnFail: true, - removeOnComplete: true, - delay: 1000, - attempts: 5, - backoff: { - type: "exponential", - delay: 3000 - } - }); - await syncIntegrations({ ...dto, deDupeQueue }); - }; - const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => { const appCfg = getConfig(); await queueService.stopRepeatableJob( @@ -237,8 +182,27 @@ export const secretQueueFactory = ({ } } }; + const createManySecretsRawFn = createManySecretsRawFnFactory({ + projectDAL, + projectBotDAL, + secretDAL, + secretVersionDAL, + secretBlindIndexDAL, + secretTagDAL, + secretVersionTagDAL, + folderDAL + }); - type Content = Record; + const updateManySecretsRawFn = updateManySecretsRawFnFactory({ + projectDAL, + projectBotDAL, + secretDAL, + secretVersionDAL, + secretBlindIndexDAL, + secretTagDAL, + secretVersionTagDAL, + folderDAL + }); /** * Return the secrets in a given [folderId] including secrets from @@ -251,7 +215,7 @@ export const secretQueueFactory = ({ key: string; depth: number; }) => { - let content: Content = {}; + let content: TIntegrationSecret = {}; if (dto.depth > MAX_SYNC_SECRET_DEPTH) { logger.info( `getIntegrationSecrets: secret depth exceeded for [projectId=${dto.projectId}] [folderId=${dto.folderId}] [depth=${dto.depth}]` @@ -301,7 +265,7 @@ export const secretQueueFactory = ({ await expandSecrets(content); // check if current folder has any imports from other folders - const secretImport = await secretImportDAL.find({ folderId: dto.folderId }); + const secretImport = await secretImportDAL.find({ folderId: dto.folderId, isReplication: false }); // if no imports then return secrets in the current folder if (!secretImport) return content; @@ -333,8 +297,122 @@ export const secretQueueFactory = ({ return content; }; + const syncIntegrations = async (dto: TGetSecrets & { deDupeQueue?: Record }) => { + await queueService.queue(QueueName.IntegrationSync, QueueJobs.IntegrationSync, dto, { + attempts: 3, + delay: 1000, + backoff: { + type: "exponential", + delay: 3000 + }, + removeOnComplete: true, + removeOnFail: true + }); + }; + + const replicateSecrets = async (dto: Omit) => { + await queueService.queue(QueueName.SecretReplication, QueueJobs.SecretReplication, dto, { + attempts: 3, + backoff: { + type: "exponential", + delay: 2000 + }, + removeOnComplete: true, + removeOnFail: true + }); + }; + + const syncSecrets = async ({ + // seperate de-dupe queue for integration sync and replication sync + _deDupeQueue: deDupeQueue = {}, + _depth: depth = 0, + _deDupeReplicationQueue: deDupeReplicationQueue = {}, + ...dto + }: TSyncSecretsDTO) => { + logger.info( + `syncSecrets: syncing project secrets where [projectId=${dto.projectId}] [environment=${dto.environmentSlug}] [path=${dto.secretPath}]` + ); + const deDuplicationKey = uniqueSecretQueueKey(dto.environmentSlug, dto.secretPath); + if ( + !dto.excludeReplication + ? deDupeReplicationQueue?.[deDuplicationKey] + : deDupeQueue?.[deDuplicationKey] || depth > MAX_SYNC_SECRET_DEPTH + ) { + return; + } + // eslint-disable-next-line + deDupeQueue[deDuplicationKey] = true; + // eslint-disable-next-line + deDupeReplicationQueue[deDuplicationKey] = true; + await queueService.queue( + QueueName.SecretSync, + QueueJobs.SecretSync, + { + ...dto, + _deDupeQueue: deDupeQueue, + _deDupeReplicationQueue: deDupeReplicationQueue, + _depth: depth + } as TSyncSecretsDTO, + { + removeOnFail: true, + removeOnComplete: true, + delay: 1000, + attempts: 5, + backoff: { + type: "exponential", + delay: 3000 + } + } + ); + }; + + queueService.start(QueueName.SecretSync, async (job) => { + const { + _deDupeQueue: deDupeQueue, + _deDupeReplicationQueue: deDupeReplicationQueue, + _depth: depth, + secretPath, + projectId, + environmentSlug: environment, + excludeReplication, + actorId, + actor + } = job.data; + + await queueService.queue( + QueueName.SecretWebhook, + QueueJobs.SecWebhook, + { environment, projectId, secretPath }, + { + jobId: `secret-webhook-${environment}-${projectId}-${secretPath}`, + removeOnFail: { count: 5 }, + removeOnComplete: true, + delay: 1000, + attempts: 5, + backoff: { + type: "exponential", + delay: 3000 + } + } + ); + await syncIntegrations({ secretPath, projectId, environment, deDupeQueue }); + if (!excludeReplication) { + await replicateSecrets({ + _deDupeReplicationQueue: deDupeReplicationQueue, + _depth: depth, + projectId, + secretPath, + actorId, + actor, + excludeReplication, + environmentSlug: environment + }); + } + }); + queueService.start(QueueName.IntegrationSync, async (job) => { const { environment, projectId, secretPath, depth = 1, deDupeQueue = {} } = job.data; + if (depth > MAX_SYNC_SECRET_DEPTH) return; const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); if (!folder) { @@ -348,7 +426,8 @@ export const secretQueueFactory = ({ const linkSourceDto = { projectId, importEnv: folder.environment.id, - importPath: secretPath + importPath: secretPath, + isReplication: false }; const imports = await secretImportDAL.find(linkSourceDto); @@ -356,30 +435,31 @@ export const secretQueueFactory = ({ // keep calling sync secret for all the imports made const importedFolderIds = unique(imports, (i) => i.folderId).map(({ folderId }) => folderId); const importedFolders = await folderDAL.findSecretPathByFolderIds(projectId, importedFolderIds); - const foldersGroupedById = groupBy(importedFolders, (i) => i.child || i.id); + const foldersGroupedById = groupBy(importedFolders.filter(Boolean), (i) => i?.id as string); logger.info( `getIntegrationSecrets: Syncing secret due to link change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}] [secretPath=${job.data.secretPath}] [depth=${depth}]` ); await Promise.all( imports - .filter(({ folderId }) => Boolean(foldersGroupedById[folderId][0].path)) + .filter(({ folderId }) => Boolean(foldersGroupedById[folderId][0]?.path as string)) // filter out already synced ones .filter( ({ folderId }) => !deDupeQueue[ - uniqueIntegrationKey( - foldersGroupedById[folderId][0].environmentSlug, - foldersGroupedById[folderId][0].path + uniqueSecretQueueKey( + foldersGroupedById[folderId][0]?.environmentSlug as string, + foldersGroupedById[folderId][0]?.path as string ) ] ) .map(({ folderId }) => syncSecrets({ - depth: depth + 1, projectId, - secretPath: foldersGroupedById[folderId][0].path, - environment: foldersGroupedById[folderId][0].environmentSlug, - deDupeQueue + secretPath: foldersGroupedById[folderId][0]?.path as string, + environmentSlug: foldersGroupedById[folderId][0]?.environmentSlug as string, + _deDupeQueue: deDupeQueue, + _depth: depth + 1, + excludeReplication: true }) ) ); @@ -393,30 +473,31 @@ export const secretQueueFactory = ({ if (secretReferences.length) { const referencedFolderIds = unique(secretReferences, (i) => i.folderId).map(({ folderId }) => folderId); const referencedFolders = await folderDAL.findSecretPathByFolderIds(projectId, referencedFolderIds); - const referencedFoldersGroupedById = groupBy(referencedFolders, (i) => i.child || i.id); + const referencedFoldersGroupedById = groupBy(referencedFolders.filter(Boolean), (i) => i?.id as string); logger.info( `getIntegrationSecrets: Syncing secret due to reference change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}] [secretPath=${job.data.secretPath}] [depth=${depth}]` ); await Promise.all( secretReferences - .filter(({ folderId }) => Boolean(referencedFoldersGroupedById[folderId][0].path)) + .filter(({ folderId }) => Boolean(referencedFoldersGroupedById[folderId][0]?.path)) // filter out already synced ones .filter( ({ folderId }) => !deDupeQueue[ - uniqueIntegrationKey( - referencedFoldersGroupedById[folderId][0].environmentSlug, - referencedFoldersGroupedById[folderId][0].path + uniqueSecretQueueKey( + referencedFoldersGroupedById[folderId][0]?.environmentSlug as string, + referencedFoldersGroupedById[folderId][0]?.path as string ) ] ) .map(({ folderId }) => syncSecrets({ - depth: depth + 1, projectId, - secretPath: referencedFoldersGroupedById[folderId][0].path, - environment: referencedFoldersGroupedById[folderId][0].environmentSlug, - deDupeQueue + secretPath: referencedFoldersGroupedById[folderId][0]?.path as string, + environmentSlug: referencedFoldersGroupedById[folderId][0]?.environmentSlug as string, + _deDupeQueue: deDupeQueue, + _depth: depth + 1, + excludeReplication: true }) ) ); @@ -546,10 +627,11 @@ export const secretQueueFactory = ({ return { // depth is internal only field thus no need to make it available outside - syncSecrets: (dto: TGetSecrets) => syncSecrets(dto), + syncSecrets, syncIntegrations, addSecretReminder, removeSecretReminder, - handleSecretReminder + handleSecretReminder, + replicateSecrets }; }; diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 39e47a28e..5688f7f15 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -35,6 +35,7 @@ import { TSecretDALFactory } from "./secret-dal"; import { decryptSecretRaw, fnSecretBlindIndexCheck, + fnSecretBulkDelete, fnSecretBulkInsert, fnSecretBulkUpdate, getAllNestedSecretReferences, @@ -53,8 +54,6 @@ import { TDeleteManySecretRawDTO, TDeleteSecretDTO, TDeleteSecretRawDTO, - TFnSecretBlindIndexCheckV2, - TFnSecretBulkDelete, TGetASecretDTO, TGetASecretRawDTO, TGetSecretsDTO, @@ -139,53 +138,6 @@ export const secretServiceFactory = ({ return secretBlindIndex; }; - const fnSecretBulkDelete = async ({ folderId, inputSecrets, tx, actorId }: TFnSecretBulkDelete) => { - const deletedSecrets = await secretDAL.deleteMany( - inputSecrets.map(({ type, secretBlindIndex }) => ({ - blindIndex: secretBlindIndex, - type - })), - folderId, - actorId, - tx - ); - - for (const s of deletedSecrets) { - if (s.secretReminderRepeatDays) { - // eslint-disable-next-line no-await-in-loop - await secretQueueService - .removeSecretReminder({ - secretId: s.id, - repeatDays: s.secretReminderRepeatDays - }) - .catch((err) => { - logger.error(err, `Failed to delete secret reminder for secret with ID ${s?.id}`); - }); - } - } - - return deletedSecrets; - }; - - // this is used when secret blind index already exist - // mainly for secret approval - const fnSecretBlindIndexCheckV2 = async ({ inputSecrets, folderId, userId }: TFnSecretBlindIndexCheckV2) => { - if (inputSecrets.some(({ type }) => type === SecretType.Personal) && !userId) { - throw new BadRequestError({ message: "Missing user id for personal secret" }); - } - const secrets = await secretDAL.findByBlindIndexes( - folderId, - inputSecrets.map(({ secretBlindIndex, type }) => ({ - blindIndex: secretBlindIndex, - type: type || SecretType.Shared - })), - userId - ); - const secsGroupedByBlindIndex = groupBy(secrets, (i) => i.secretBlindIndex as string); - - return { secsGroupedByBlindIndex, secrets }; - }; - const createSecret = async ({ path, actor, @@ -283,8 +235,13 @@ export const secretServiceFactory = ({ ); await snapshotService.performSnapshot(folderId); - await secretQueueService.syncSecrets({ secretPath: path, projectId, environment }); - // TODO(akhilmhdh-pg): licence check, posthog service and snapshot + await secretQueueService.syncSecrets({ + secretPath: path, + actorId, + actor, + projectId, + environmentSlug: folder.environment.slug + }); return { ...secret[0], environment, workspace: projectId, tags, secretPath: path }; }; @@ -413,8 +370,13 @@ export const secretServiceFactory = ({ ); await snapshotService.performSnapshot(folderId); - await secretQueueService.syncSecrets({ secretPath: path, projectId, environment }); - // TODO(akhilmhdh-pg): licence check, posthog service and snapshot + await secretQueueService.syncSecrets({ + actor, + actorId, + secretPath: path, + projectId, + environmentSlug: folder.environment.slug + }); return { ...updatedSecret[0], workspace: projectId, environment, secretPath: path }; }; @@ -470,6 +432,8 @@ export const secretServiceFactory = ({ projectId, folderId, actorId, + secretDAL, + secretQueueService, inputSecrets: [ { type: inputSecret.type as SecretType, @@ -481,8 +445,13 @@ export const secretServiceFactory = ({ ); await snapshotService.performSnapshot(folderId); - await secretQueueService.syncSecrets({ secretPath: path, projectId, environment }); - + await secretQueueService.syncSecrets({ + actor, + actorId, + secretPath: path, + projectId, + environmentSlug: folder.environment.slug + }); // TODO(akhilmhdh-pg): licence check, posthog service and snapshot return { ...deletedSecret[0], _id: deletedSecret[0].id, workspace: projectId, environment, secretPath: path }; }; @@ -551,7 +520,8 @@ export const secretServiceFactory = ({ if (includeImports) { const secretImports = await secretImportDAL.findByFolderIds(paths.map((p) => p.folderId)); - const allowedImports = secretImports.filter(({ importEnv, importPath }) => + const allowedImports = secretImports.filter(({ importEnv, importPath, isReplication }) => + !isReplication && // if its service token allow full access over imported one actor === ActorType.SERVICE ? true @@ -656,7 +626,7 @@ export const secretServiceFactory = ({ // then search for imported secrets // here we consider the import order also thus starting from bottom if (!secret && includeImports) { - const secretImports = await secretImportDAL.find({ folderId }); + const secretImports = await secretImportDAL.find({ folderId, isReplication: false }); const allowedImports = secretImports.filter(({ importEnv, importPath }) => // if its service token allow full access over imported one actor === ActorType.SERVICE @@ -767,7 +737,13 @@ export const secretServiceFactory = ({ ); await snapshotService.performSnapshot(folderId); - await secretQueueService.syncSecrets({ secretPath: path, projectId, environment }); + await secretQueueService.syncSecrets({ + actor, + actorId, + secretPath: path, + projectId, + environmentSlug: folder.environment.slug + }); return newSecrets; }; @@ -867,7 +843,13 @@ export const secretServiceFactory = ({ ); await snapshotService.performSnapshot(folderId); - await secretQueueService.syncSecrets({ secretPath: path, projectId, environment }); + await secretQueueService.syncSecrets({ + actor, + actorId, + secretPath: path, + projectId, + environmentSlug: folder.environment.slug + }); return secrets; }; @@ -917,6 +899,8 @@ export const secretServiceFactory = ({ const secretsDeleted = await secretDAL.transaction(async (tx) => fnSecretBulkDelete({ + secretDAL, + secretQueueService, inputSecrets: inputSecrets.map(({ type, secretName }) => ({ secretBlindIndex: keyName2BlindIndex[secretName], type @@ -929,7 +913,13 @@ export const secretServiceFactory = ({ ); await snapshotService.performSnapshot(folderId); - await secretQueueService.syncSecrets({ secretPath: path, projectId, environment }); + await secretQueueService.syncSecrets({ + actor, + actorId, + secretPath: path, + projectId, + environmentSlug: folder.environment.slug + }); return secretsDeleted; }; @@ -1109,9 +1099,6 @@ export const secretServiceFactory = ({ skipMultilineEncoding }); - await snapshotService.performSnapshot(secret.folderId); - await secretQueueService.syncSecrets({ secretPath, projectId, environment }); - return decryptSecretRaw(secret, botKey); }; @@ -1150,8 +1137,6 @@ export const secretServiceFactory = ({ }); await snapshotService.performSnapshot(secret.folderId); - await secretQueueService.syncSecrets({ secretPath, projectId, environment }); - return decryptSecretRaw(secret, botKey); }; @@ -1181,9 +1166,6 @@ export const secretServiceFactory = ({ actorAuthMethod }); - await snapshotService.performSnapshot(secret.folderId); - await secretQueueService.syncSecrets({ secretPath, projectId, environment }); - return decryptSecretRaw(secret, botKey); }; @@ -1232,9 +1214,6 @@ export const secretServiceFactory = ({ }) }); - await snapshotService.performSnapshot(secrets[0].folderId); - await secretQueueService.syncSecrets({ secretPath, projectId, environment }); - return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey) ); @@ -1286,9 +1265,6 @@ export const secretServiceFactory = ({ }) }); - await snapshotService.performSnapshot(secrets[0].folderId); - await secretQueueService.syncSecrets({ secretPath, projectId, environment }); - return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey) ); @@ -1322,9 +1298,6 @@ export const secretServiceFactory = ({ secrets: inputSecrets.map(({ secretKey }) => ({ secretName: secretKey, type: SecretType.Shared })) }); - await snapshotService.performSnapshot(secrets[0].folderId); - await secretQueueService.syncSecrets({ secretPath, projectId, environment }); - return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey) ); @@ -1448,7 +1421,12 @@ export const secretServiceFactory = ({ ); await snapshotService.performSnapshot(folder.id); - await secretQueueService.syncSecrets({ secretPath, projectId: project.id, environment }); + await secretQueueService.syncSecrets({ + secretPath, + projectId: project.id, + environmentSlug: environment, + excludeReplication: true + }); return { ...updatedSecret[0], @@ -1550,7 +1528,12 @@ export const secretServiceFactory = ({ ); await snapshotService.performSnapshot(folder.id); - await secretQueueService.syncSecrets({ secretPath, projectId: project.id, environment }); + await secretQueueService.syncSecrets({ + secretPath, + projectId: project.id, + environmentSlug: environment, + excludeReplication: true + }); return { ...updatedSecret[0], @@ -1624,12 +1607,6 @@ export const secretServiceFactory = ({ updateManySecretsRaw, deleteManySecretsRaw, getSecretVersions, - backfillSecretReferences, - // external services function - fnSecretBulkDelete, - fnSecretBulkUpdate, - fnSecretBlindIndexCheck, - fnSecretBulkInsert, - fnSecretBlindIndexCheckV2 + backfillSecretReferences }; }; diff --git a/backend/src/services/secret/secret-types.ts b/backend/src/services/secret/secret-types.ts index 7e713a80f..18a0077fe 100644 --- a/backend/src/services/secret/secret-types.ts +++ b/backend/src/services/secret/secret-types.ts @@ -11,6 +11,8 @@ import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/se import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; +import { ActorType } from "../auth/auth-type"; + type TPartialSecret = Pick; type TPartialInputSecret = Pick; @@ -264,6 +266,10 @@ export type TFnSecretBulkDelete = { inputSecrets: Array<{ type: SecretType; secretBlindIndex: string }>; actorId: string; tx?: Knex; + secretDAL: Pick; + secretQueueService: { + removeSecretReminder: (data: TRemoveSecretReminderDTO) => Promise; + }; }; export type TFnSecretBlindIndexCheck = { @@ -277,6 +283,7 @@ export type TFnSecretBlindIndexCheck = { // when blind index is already present export type TFnSecretBlindIndexCheckV2 = { + secretDAL: Pick; folderId: string; userId?: string; inputSecrets: Array<{ secretBlindIndex: string; type?: SecretType }>; @@ -363,3 +370,27 @@ export type TUpdateManySecretsRawFn = { }[]; userId?: string; }; + +export enum SecretOperations { + Create = "create", + Update = "update", + Delete = "delete" +} + +export type TSyncSecretsDTO = { + _deDupeQueue?: Record; + _deDupeReplicationQueue?: Record; + _depth?: number; + secretPath: string; + projectId: string; + environmentSlug: string; + // cases for just doing sync integration and webhook + excludeReplication?: T; +} & (T extends true + ? object + : { + actor: ActorType; + actorId: string; + // used for import creation to trigger replication + pickOnlyImportIds?: string[]; + }); diff --git a/backend/src/services/secret/secret-version-dal.ts b/backend/src/services/secret/secret-version-dal.ts index 758352ed2..203406e30 100644 --- a/backend/src/services/secret/secret-version-dal.ts +++ b/backend/src/services/secret/secret-version-dal.ts @@ -89,6 +89,7 @@ export const secretVersionDALFactory = (db: TDbClient) => { const findLatestVersionMany = async (folderId: string, secretIds: string[], tx?: Knex) => { try { + if (!secretIds.length) return {}; const docs: Array = await (tx || db)(TableName.SecretVersion) .where("folderId", folderId) .whereIn(`${TableName.SecretVersion}.secretId`, secretIds) diff --git a/frontend/src/hooks/api/secretApprovalRequest/queries.tsx b/frontend/src/hooks/api/secretApprovalRequest/queries.tsx index 23017c15d..aaf84941a 100644 --- a/frontend/src/hooks/api/secretApprovalRequest/queries.tsx +++ b/frontend/src/hooks/api/secretApprovalRequest/queries.tsx @@ -220,6 +220,7 @@ export const useGetSecretApprovalRequestCount = ({ }) => useQuery({ queryKey: secretApprovalRequestKeys.count({ workspaceId }), + refetchInterval: 5000, queryFn: () => fetchSecretApprovalRequestCount({ workspaceId }), enabled: Boolean(workspaceId) && (options?.enabled ?? true) }); diff --git a/frontend/src/hooks/api/secretApprovalRequest/types.ts b/frontend/src/hooks/api/secretApprovalRequest/types.ts index 32fe31c6b..8c2ba6963 100644 --- a/frontend/src/hooks/api/secretApprovalRequest/types.ts +++ b/frontend/src/hooks/api/secretApprovalRequest/types.ts @@ -44,6 +44,7 @@ export type TSecretApprovalSecChange = { export type TSecretApprovalRequest = { id: string; + isReplicated?: boolean; slug: string; createdAt: string; committerId: string; diff --git a/frontend/src/hooks/api/secretFolders/types.ts b/frontend/src/hooks/api/secretFolders/types.ts index 8fde9c63d..412f2686d 100644 --- a/frontend/src/hooks/api/secretFolders/types.ts +++ b/frontend/src/hooks/api/secretFolders/types.ts @@ -1,3 +1,7 @@ +export enum ReservedFolders { + SecretReplication = "__reserve_replication_" +} + export type TSecretFolder = { id: string; name: string; diff --git a/frontend/src/hooks/api/secretImports/index.ts b/frontend/src/hooks/api/secretImports/index.ts index f30506b6b..fed0f13d4 100644 --- a/frontend/src/hooks/api/secretImports/index.ts +++ b/frontend/src/hooks/api/secretImports/index.ts @@ -1,4 +1,9 @@ -export { useCreateSecretImport, useDeleteSecretImport, useUpdateSecretImport } from "./mutation"; +export { + useCreateSecretImport, + useDeleteSecretImport, + useResyncSecretReplication, + useUpdateSecretImport +} from "./mutation"; export { useGetImportedFoldersByEnv, useGetImportedSecretsAllEnvs, diff --git a/frontend/src/hooks/api/secretImports/mutation.tsx b/frontend/src/hooks/api/secretImports/mutation.tsx index 928322a3c..04f1f01e6 100644 --- a/frontend/src/hooks/api/secretImports/mutation.tsx +++ b/frontend/src/hooks/api/secretImports/mutation.tsx @@ -3,18 +3,24 @@ import { useMutation, useQueryClient } from "@tanstack/react-query"; import { apiRequest } from "@app/config/request"; import { secretImportKeys } from "./queries"; -import { TCreateSecretImportDTO, TDeleteSecretImportDTO, TUpdateSecretImportDTO } from "./types"; +import { + TCreateSecretImportDTO, + TDeleteSecretImportDTO, + TResyncSecretReplicationDTO, + TUpdateSecretImportDTO +} from "./types"; export const useCreateSecretImport = () => { const queryClient = useQueryClient(); return useMutation<{}, {}, TCreateSecretImportDTO>({ - mutationFn: async ({ import: secretImport, environment, projectId, path }) => { + mutationFn: async ({ import: secretImport, environment, isReplication, projectId, path }) => { const { data } = await apiRequest.post("/api/v1/secret-imports", { import: secretImport, environment, workspaceId: projectId, - path + path, + isReplication }); return data; }, @@ -53,6 +59,19 @@ export const useUpdateSecretImport = () => { }); }; +export const useResyncSecretReplication = () => { + return useMutation<{}, {}, TResyncSecretReplicationDTO>({ + mutationFn: async ({ environment, projectId, path, id }) => { + const { data } = await apiRequest.post(`/api/v1/secret-imports/${id}/replication-resync`, { + environment, + path, + workspaceId: projectId + }); + return data; + } + }); +}; + export const useDeleteSecretImport = () => { const queryClient = useQueryClient(); diff --git a/frontend/src/hooks/api/secretImports/types.ts b/frontend/src/hooks/api/secretImports/types.ts index 950fc20c4..1a6c06dd3 100644 --- a/frontend/src/hooks/api/secretImports/types.ts +++ b/frontend/src/hooks/api/secretImports/types.ts @@ -10,6 +10,11 @@ export type TSecretImport = { position: string; createdAt: string; updatedAt: string; + isReserved?: boolean; + isReplication?: boolean; + isReplicationSuccess?: boolean; + replicationStatus?: string; + lastReplicated?: string; }; export type TGetImportedFoldersByEnvDTO = { @@ -60,6 +65,7 @@ export type TCreateSecretImportDTO = { environment: string; path: string; }; + isReplication?: boolean; }; export type TUpdateSecretImportDTO = { @@ -74,6 +80,13 @@ export type TUpdateSecretImportDTO = { }>; }; +export type TResyncSecretReplicationDTO = { + id: string; + projectId: string; + environment: string; + path?: string; +}; + export type TDeleteSecretImportDTO = { id: string; projectId: string; diff --git a/frontend/src/lib/fn/string.ts b/frontend/src/lib/fn/string.ts new file mode 100644 index 000000000..9d3d01cc5 --- /dev/null +++ b/frontend/src/lib/fn/string.ts @@ -0,0 +1,9 @@ +import { ReservedFolders } from "@app/hooks/api/secretFolders/types"; + +export const formatReservedPaths = (secretPath: string) => { + const i = secretPath.indexOf(ReservedFolders.SecretReplication); + if (i !== -1) { + return `${secretPath.slice(0, i)} - (replication)`; + } + return secretPath; +}; diff --git a/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/SecretApprovalRequest.tsx b/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/SecretApprovalRequest.tsx index 0d0c6213a..ca4d3b897 100644 --- a/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/SecretApprovalRequest.tsx +++ b/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/SecretApprovalRequest.tsx @@ -212,7 +212,8 @@ export const SecretApprovalRequest = () => { createdAt, policy, reviewers, - status + status, + isReplicated: isReplication } = secretApproval; const isApprover = policy?.approvers?.indexOf(myMembershipId || "") !== -1; const isReviewed = @@ -240,8 +241,9 @@ export const SecretApprovalRequest = () => { Opened {formatDistance(new Date(createdAt), new Date())} ago by{" "} {membersGroupById?.[committerId]?.user?.firstName}{" "} {membersGroupById?.[committerId]?.user?.lastName} ( - {membersGroupById?.[committerId]?.user?.email}){" "} - {isApprover && !isReviewed && status === "open" && "- Review required"} + {membersGroupById?.[committerId]?.user?.email}) + {isReplication && " via replication"} + {isApprover && !isReviewed && status === "open" && " - Review required"} ); diff --git a/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/components/SecretApprovalRequestChanges.tsx b/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/components/SecretApprovalRequestChanges.tsx index 80dbe9f73..85d970d4b 100644 --- a/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/components/SecretApprovalRequestChanges.tsx +++ b/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/components/SecretApprovalRequestChanges.tsx @@ -20,6 +20,7 @@ import { useUpdateSecretApprovalReviewStatus } from "@app/hooks/api"; import { ApprovalStatus, CommitType, TWorkspaceUser } from "@app/hooks/api/types"; +import { formatReservedPaths } from "@app/lib/fn/string"; import { SecretApprovalRequestAction } from "./SecretApprovalRequestAction"; import { SecretApprovalRequestChangeItem } from "./SecretApprovalRequestChangeItem"; @@ -185,6 +186,9 @@ export const SecretApprovalRequestChanges = ({
{generateCommitText(secretApprovalRequestDetails.commits)} + {secretApprovalRequestDetails.isReplicated && ( + (replication) + )}
{committer?.user?.firstName} @@ -197,7 +201,11 @@ export const SecretApprovalRequestChanges = ({
-
{secretApprovalRequestDetails.secretPath}
+ +
+ {formatReservedPaths(secretApprovalRequestDetails.secretPath)} +
+
diff --git a/frontend/src/views/SecretMainPage/components/ActionBar/ActionBar.tsx b/frontend/src/views/SecretMainPage/components/ActionBar/ActionBar.tsx index 9b235867b..bed5d6a84 100644 --- a/frontend/src/views/SecretMainPage/components/ActionBar/ActionBar.tsx +++ b/frontend/src/views/SecretMainPage/components/ActionBar/ActionBar.tsx @@ -482,6 +482,7 @@ export const ActionBar = ({ environment={environment} workspaceId={workspaceId} secretPath={secretPath} + onUpgradePlan={() => handlePopUpOpen("upgradePlan")} isOpen={popUp.addSecretImport.isOpen} onClose={() => handlePopUpClose("addSecretImport")} onTogglePopUp={(isOpen) => handlePopUpToggle("addSecretImport", isOpen)} diff --git a/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx b/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx index a6dfe7c0e..7f9f2a4fe 100644 --- a/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx +++ b/frontend/src/views/SecretMainPage/components/ActionBar/CreateSecretImportForm.tsx @@ -4,9 +4,16 @@ import { AxiosError } from "axios"; import { z } from "zod"; import { createNotification } from "@app/components/notifications"; -import { Button, FormControl, Modal, ModalContent, Select, SelectItem } from "@app/components/v2"; +import { + Button, + FormControl, + Modal, + ModalContent, + Select, + SelectItem +} from "@app/components/v2"; import { SecretPathInput } from "@app/components/v2/SecretPathInput"; -import { useWorkspace } from "@app/context"; +import { useSubscription, useWorkspace } from "@app/context"; import { useCreateSecretImport } from "@app/hooks/api"; const typeSchema = z.object({ @@ -16,7 +23,8 @@ const typeSchema = z.object({ .trim() .transform((val) => typeof val === "string" && val.at(-1) === "/" && val.length > 1 ? val.slice(0, -1) : val - ) + ), + isReplication: z.boolean().default(false) }); type TFormSchema = z.infer; @@ -29,6 +37,7 @@ type Props = { isOpen?: boolean; onClose: () => void; onTogglePopUp: (isOpen: boolean) => void; + onUpgradePlan: () => void; }; export const CreateSecretImportForm = ({ @@ -37,7 +46,8 @@ export const CreateSecretImportForm = ({ secretPath = "/", isOpen, onClose, - onTogglePopUp + onTogglePopUp, + onUpgradePlan }: Props) => { const { handleSubmit, @@ -49,18 +59,26 @@ export const CreateSecretImportForm = ({ const { currentWorkspace } = useWorkspace(); const environments = currentWorkspace?.environments || []; const selectedEnvironment = watch("environment"); + const { subscription } = useSubscription(); const { mutateAsync: createSecretImport } = useCreateSecretImport(); const handleFormSubmit = async ({ environment: importedEnv, - secretPath: importedSecPath + secretPath: importedSecPath, + isReplication }: TFormSchema) => { try { + if (isReplication && !subscription?.secretApproval) { + onUpgradePlan(); + return; + } + await createSecretImport({ environment, projectId: workspaceId, path: secretPath, + isReplication, import: { environment: importedEnv, path: importedSecPath @@ -70,7 +88,8 @@ export const CreateSecretImportForm = ({ reset(); createNotification({ type: "success", - text: "Successfully linked" + text: `Successfully linked. ${isReplication ? "Please refresh the dashboard to view changes" : "" + }` }); } catch (err) { console.error(err); @@ -127,7 +146,31 @@ export const CreateSecretImportForm = ({ )} /> - + ( + + + + )} + />