diff --git a/backend/scripts/generate-schema-types.ts b/backend/scripts/generate-schema-types.ts index fc398c2ac..40fbc35b1 100644 --- a/backend/scripts/generate-schema-types.ts +++ b/backend/scripts/generate-schema-types.ts @@ -84,6 +84,10 @@ const getZodDefaultValue = (type: unknown, value: string | number | boolean | Ob } }; +const bigIntegerColumns = { + "folder_commits": ["commitId"] +} + const main = async () => { const tables = ( await db("information_schema.tables") @@ -108,6 +112,9 @@ const main = async () => { const columnName = columnNames[colNum]; const colInfo = columns[columnName]; let ztype = getZodPrimitiveType(colInfo.type); + if (bigIntegerColumns[tableName]?.includes(columnName)) { + ztype = "z.coerce.bigint()"; + } if (["zodBuffer"].includes(ztype)) { zodImportSet.add(ztype); } diff --git a/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts b/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts index 75f1ca89f..30d1ec49b 100644 --- a/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts +++ b/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts @@ -101,12 +101,29 @@ export async function up(knex: Knex): Promise { }); } - await createOnUpdateTrigger(knex, TableName.FolderCommit); - await createOnUpdateTrigger(knex, TableName.FolderCommitChanges); - await createOnUpdateTrigger(knex, TableName.FolderCheckpoint); - await createOnUpdateTrigger(knex, TableName.FolderCheckpointResources); - await createOnUpdateTrigger(knex, TableName.FolderTreeCheckpoint); - await createOnUpdateTrigger(knex, TableName.FolderTreeCheckpointResources); + if (!hasFolderCommitTable) { + await createOnUpdateTrigger(knex, TableName.FolderCommit); + } + + if (!hasFolderCommitChangesTable) { + await createOnUpdateTrigger(knex, TableName.FolderCommitChanges); + } + + if (!hasFolderCheckpointTable) { + await createOnUpdateTrigger(knex, TableName.FolderCheckpoint); + } + + if (!hasFolderCheckpointResourcesTable) { + await createOnUpdateTrigger(knex, TableName.FolderCheckpointResources); + } + + if (!hasFolderTreeCheckpointTable) { + await createOnUpdateTrigger(knex, TableName.FolderTreeCheckpoint); + } + + if (!hasFolderTreeCheckpointResourcesTable) { + await createOnUpdateTrigger(knex, TableName.FolderTreeCheckpointResources); + } } export async function down(knex: Knex): Promise { diff --git a/backend/src/db/migrations/20250507185010_pit-projects-commits-initialization.ts b/backend/src/db/migrations/20250507185010_pit-projects-commits-initialization.ts index 5c254afcd..c5b2f8068 100644 --- a/backend/src/db/migrations/20250507185010_pit-projects-commits-initialization.ts +++ b/backend/src/db/migrations/20250507185010_pit-projects-commits-initialization.ts @@ -11,12 +11,10 @@ export async function up(knex: Knex): Promise { const keyStore = inMemoryKeyStore(); const { folderCommitService } = await getMigrationPITServices({ db: knex, keyStore }); const projects = await knex(TableName.Project).where({ version: 3, type: ProjectType.SecretManager }).select("id"); - await knex.transaction(async (tx) => { - for (const project of projects) { - // eslint-disable-next-line no-await-in-loop - await folderCommitService.initializeProject(project.id, tx); - } - }); + for (const project of projects) { + // eslint-disable-next-line no-await-in-loop + await folderCommitService.initializeProject(project.id); + } } } diff --git a/backend/src/db/schemas/folder-commits.ts b/backend/src/db/schemas/folder-commits.ts index fc9b83489..ade480eda 100644 --- a/backend/src/db/schemas/folder-commits.ts +++ b/backend/src/db/schemas/folder-commits.ts @@ -9,7 +9,7 @@ import { TImmutableDBKeys } from "./models"; export const FolderCommitsSchema = z.object({ id: z.string().uuid(), - commitId: z.coerce.number(), + commitId: z.coerce.bigint(), actorMetadata: z.unknown(), actorType: z.string(), message: z.string().nullable().optional(), diff --git a/backend/src/ee/routes/v1/pit-router.ts b/backend/src/ee/routes/v1/pit-router.ts index 57ed20081..ab88b5bd3 100644 --- a/backend/src/ee/routes/v1/pit-router.ts +++ b/backend/src/ee/routes/v1/pit-router.ts @@ -1,14 +1,18 @@ /* eslint-disable @typescript-eslint/no-base-to-string */ +import { ForbiddenError } from "@casl/ability"; import { z } from "zod"; +import { ActionProjectType } from "@app/db/schemas"; import { EventType } from "@app/ee/services/audit-log/audit-log-types"; +import { ProjectPermissionCommitsActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; import { NotFoundError } from "@app/lib/errors"; import { removeTrailingSlash } from "@app/lib/fn"; import { readLimit } from "@app/server/config/rateLimiter"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { booleanSchema } from "@app/server/routes/sanitizedSchemas"; import { AuthMode } from "@app/services/auth/auth-type"; -import { ChangeType } from "@app/services/folder-commit/folder-commit-service"; +import { ChangeType, ResourceChange } from "@app/services/folder-commit/folder-commit-service"; +import { commitChangesResponseSchema } from "@app/services/folder-commit/folder-commit-types"; const commitHistoryItemSchema = z.object({ id: z.string(), @@ -21,19 +25,6 @@ const commitHistoryItemSchema = z.object({ envId: z.string() }); -const versionSchema = z.object({ - secretKey: z.string().optional(), - secretComment: z.string().optional().nullable(), - skipMultilineEncoding: z.boolean().optional().nullable(), - secretReminderRepeatDays: z.number().optional().nullable(), - secretReminderNote: z.string().optional().nullable(), - metadata: z.unknown().optional().nullable(), - tags: z.array(z.string()).optional().nullable(), - secretReminderRecipients: z.array(z.any()).optional().nullable(), - secretValue: z.string().optional().nullable(), - name: z.string().optional().nullable() -}); - const folderStateSchema = z.array( z.object({ type: z.string(), @@ -50,17 +41,15 @@ export const registerPITRouter = async (server: FastifyZodProvider) => { // Get commits count for a folder server.route({ method: "GET", - url: "/commits/count/:workspaceId", + url: "/commits/count", config: { rateLimit: readLimit }, schema: { - params: z.object({ - workspaceId: z.string().trim() - }), querystring: z.object({ environment: z.string().trim(), - path: z.string().trim().default("/").transform(removeTrailingSlash) + path: z.string().trim().default("/").transform(removeTrailingSlash), + workspaceId: z.string().trim() }), response: { 200: z.object({ @@ -76,14 +65,14 @@ export const registerPITRouter = async (server: FastifyZodProvider) => { actorId: req.permission?.id, actorOrgId: req.permission?.orgId, actorAuthMethod: req.permission?.authMethod, - projectId: req.params.workspaceId, + projectId: req.query.workspaceId, environment: req.query.environment, path: req.query.path }); await server.services.auditLog.createAuditLog({ ...req.auditLogInfo, - projectId: req.params.workspaceId, + projectId: req.query.workspaceId, event: { type: EventType.GET_PROJECT_PIT_COMMIT_COUNT, metadata: { @@ -101,17 +90,15 @@ export const registerPITRouter = async (server: FastifyZodProvider) => { // Get all commits for a folder server.route({ method: "GET", - url: "/commits/:workspaceId", + url: "/commits", config: { rateLimit: readLimit }, schema: { - params: z.object({ - workspaceId: z.string().trim() - }), querystring: z.object({ environment: z.string().trim(), - path: z.string().trim().default("/").transform(removeTrailingSlash) + path: z.string().trim().default("/").transform(removeTrailingSlash), + workspaceId: z.string().trim() }), response: { 200: commitHistoryItemSchema.array() @@ -124,14 +111,14 @@ export const registerPITRouter = async (server: FastifyZodProvider) => { actorId: req.permission?.id, actorOrgId: req.permission?.orgId, actorAuthMethod: req.permission?.authMethod, - projectId: req.params.workspaceId, + projectId: req.query.workspaceId, environment: req.query.environment, path: req.query.path }); await server.services.auditLog.createAuditLog({ ...req.auditLogInfo, - projectId: req.params.workspaceId, + projectId: req.query.workspaceId, event: { type: EventType.GET_PROJECT_PIT_COMMITS, metadata: { @@ -149,72 +136,75 @@ export const registerPITRouter = async (server: FastifyZodProvider) => { } }); + const getChangeVersions = async ( + change: ResourceChange, + previousVersion: string, + actorId: string, + actor: string, + actorOrgId: string, + actorAuthMethod: string, + folderId: string + ) => { + if (change.secretVersion) { + const currentVersion = change.secretVersion || "1"; + const secretId = change.secretId ? change.secretId : change.id; + // eslint-disable-next-line no-await-in-loop + const versions = await server.services.secret.getSecretVersionsV2ByIds({ + actorId, + actor, + actorOrgId, + actorAuthMethod, + secretId, + // if it's update add also the previous secretversionid + secretVersions: + change.isUpdate || change.changeType === ChangeType.UPDATE + ? [currentVersion, previousVersion] + : [currentVersion], + folderId + }); + return versions?.map((v) => ({ + secretKey: v.secretKey, + secretComment: v.secretComment, + skipMultilineEncoding: v.skipMultilineEncoding, + secretReminderRepeatDays: v.secretReminderRepeatDays, + secretReminderNote: v.secretReminderNote, + metadata: v.metadata, + tags: v.tags?.map((t) => t.name), + secretReminderRecipients: v.secretReminderRecipients?.map((r) => r.toString()), + secretValue: v.secretValue + })); + } + }; + + const getFolderVersions = async (change: ResourceChange, fromVersion: string, folderId: string) => { + const currentVersion = change.folderVersion || "1"; + // eslint-disable-next-line no-await-in-loop + const versions = await server.services.folder.getFolderVersionsByIds({ + folderId, + folderVersions: + change.isUpdate || change.changeType === ChangeType.UPDATE ? [currentVersion, fromVersion] : [currentVersion] + }); + return versions.map((v) => ({ + name: v.name + })); + }; + // Get commit changes for a specific commit server.route({ method: "GET", - url: "/commits/:workspaceId/:commitId/changes", + url: "/commits/:commitId/changes", config: { rateLimit: readLimit }, schema: { params: z.object({ - workspaceId: z.string().trim(), commitId: z.string().trim() }), + querystring: z.object({ + workspaceId: z.string().trim() + }), response: { - 200: z.object({ - changes: z.object({ - id: z.string(), - commitId: z.string(), - actorMetadata: z - .union([ - z.object({ - id: z.string().optional(), - name: z.string().optional() - }), - z.unknown() - ]) - .optional(), - actorType: z.string(), - message: z.string().optional().nullable(), - folderId: z.string(), - envId: z.string(), - createdAt: z.string().or(z.date()), - updatedAt: z.string().or(z.date()), - changes: z.array( - z.object({ - id: z.string(), - folderCommitId: z.string(), - changeType: z.string(), - isUpdate: z.boolean().optional(), - secretVersionId: z.string().optional().nullable(), - folderVersionId: z.string().optional().nullable(), - // Fix these two fields to accept either string or Date objects - createdAt: z.union([z.string(), z.date()]), - updatedAt: z.union([z.string(), z.date()]), - folderName: z.string().optional().nullable(), - folderChangeId: z.string().optional().nullable(), - folderVersion: z.union([z.string(), z.number()]).optional().nullable(), - secretKey: z.string().optional().nullable(), - secretVersion: z.union([z.string(), z.number()]).optional().nullable(), - secretId: z.string().optional().nullable(), - actorMetadata: z - .union([ - z.object({ - id: z.string().optional(), - name: z.string().optional() - }), - z.unknown() - ]) - .optional(), - actorType: z.string().optional(), - message: z.string().optional().nullable(), - folderId: z.string().optional().nullable(), - versions: z.array(versionSchema).optional() - }) - ) - }) - }) + 200: commitChangesResponseSchema } }, onRequest: verifyAuth([AuthMode.JWT]), @@ -224,53 +214,36 @@ export const registerPITRouter = async (server: FastifyZodProvider) => { actorId: req.permission?.id, actorOrgId: req.permission?.orgId, actorAuthMethod: req.permission?.authMethod, - projectId: req.params.workspaceId, + projectId: req.query.workspaceId, commitId: req.params.commitId }); for (const change of changes.changes) { if (change.secretVersionId) { - const currentVersion = change.secretVersion || "1"; - const previousVersion = (Number.parseInt(currentVersion, 10) - 1).toString(); - if (change.secretId) { - // eslint-disable-next-line no-await-in-loop - const versions = await server.services.secret.getSecretVersionsV2ByIds({ - actorId: req.permission?.id, - actor: req.permission?.type, - actorOrgId: req.permission?.orgId, - actorAuthMethod: req.permission?.authMethod, - secretId: change.secretId, - secretVersions: change.isUpdate ? [currentVersion, previousVersion] : [currentVersion], - folderId: change.folderId - }); - change.versions = versions?.map((v) => ({ - secretKey: v.secretKey, - secretComment: v.secretComment, - skipMultilineEncoding: v.skipMultilineEncoding, - secretReminderRepeatDays: v.secretReminderRepeatDays, - secretReminderNote: v.secretReminderNote, - metadata: v.secretMetadata, - tags: v.tags?.map((t) => t.name), - secretReminderRecipients: v.secretReminderRecipients?.map((r) => r.toString()), - secretValue: v.secretValue - })); - } - } else if (change.folderVersionId && change.folderChangeId) { - const currentVersion = change.folderVersion || "1"; - const previousVersion = (Number.parseInt(currentVersion, 10) - 1).toString(); + change.objectType = "secret"; // eslint-disable-next-line no-await-in-loop - const versions = await server.services.folder.getFolderVersionsByIds({ - folderId: change.folderChangeId, - folderVersions: change.isUpdate ? [currentVersion, previousVersion] : [currentVersion] - }); - change.versions = versions.map((v) => ({ - name: v.name - })); + change.versions = await getChangeVersions( + change, + (Number.parseInt(change.secretVersion, 10) - 1).toString(), + req.permission?.id, + req.permission?.type, + req.permission?.orgId, + req.permission?.authMethod, + change.folderId + ); + } else if (change.folderVersionId && change.folderChangeId) { + change.objectType = "folder"; + // eslint-disable-next-line no-await-in-loop + change.versions = await getFolderVersions( + change, + (Number.parseInt(change.folderVersion, 10) - 1).toString(), + change.folderChangeId + ); } } await server.services.auditLog.createAuditLog({ ...req.auditLogInfo, - projectId: req.params.workspaceId, + projectId: req.query.workspaceId, event: { type: EventType.GET_PROJECT_PIT_COMMIT_CHANGES, metadata: { @@ -292,20 +265,20 @@ export const registerPITRouter = async (server: FastifyZodProvider) => { // Retrieve rollback changes for a commit server.route({ method: "GET", - url: "/commits/:workspaceId/:commitId/compare", + url: "/commits/:commitId/compare", config: { rateLimit: readLimit }, schema: { params: z.object({ - workspaceId: z.string().trim(), commitId: z.string().trim() }), querystring: z.object({ folderId: z.string().trim(), envId: z.string().trim(), deepRollback: booleanSchema.default(false), - secretPath: z.string().trim().default("/").transform(removeTrailingSlash) + secretPath: z.string().trim().default("/").transform(removeTrailingSlash), + workspaceId: z.string().trim() }), response: { 200: z.array( @@ -326,7 +299,7 @@ export const registerPITRouter = async (server: FastifyZodProvider) => { actorId: req.permission?.id, actorOrgId: req.permission?.orgId, actorAuthMethod: req.permission?.authMethod, - projectId: req.params.workspaceId + projectId: req.query.workspaceId }); if (!latestCommit) { throw new NotFoundError({ message: "Latest commit not found" }); @@ -337,9 +310,7 @@ export const registerPITRouter = async (server: FastifyZodProvider) => { diffs = await server.services.folderCommit.deepCompareFolder({ targetCommitId: req.params.commitId, envId: req.query.envId, - actorId: req.permission?.id, - actorType: req.permission?.type, - projectId: req.params.workspaceId + projectId: req.query.workspaceId }); } else { const folder = await server.services.folder.getFolderById({ @@ -365,51 +336,27 @@ export const registerPITRouter = async (server: FastifyZodProvider) => { for (const diff of diffs) { for (const change of diff.changes) { if (change.secretKey) { - const currentVersion = change.secretVersion || "1"; - const previousVersion = change.fromVersion || "1"; // eslint-disable-next-line no-await-in-loop - const versions = await server.services.secret.getSecretVersionsV2ByIds({ - actorId: req.permission?.id, - actor: req.permission?.type, - actorOrgId: req.permission?.orgId, - actorAuthMethod: req.permission?.authMethod, - secretId: change.id, - // if it's update add also the previous secretversionid - secretVersions: - change.changeType === ChangeType.UPDATE ? [currentVersion, previousVersion] : [currentVersion], - folderId: req.query.folderId - }); - change.versions = versions?.map((v) => ({ - secretKey: v.secretKey, - secretComment: v.secretComment, - skipMultilineEncoding: v.skipMultilineEncoding, - secretReminderRepeatDays: v.secretReminderRepeatDays, - secretReminderNote: v.secretReminderNote, - metadata: v.metadata, - tags: v.tags?.map((t) => t.name), - secretReminderRecipients: v.secretReminderRecipients?.map((r) => r.toString()), - secretValue: v.secretValue - })); + change.versions = await getChangeVersions( + change, + change.fromVersion || "1", + req.permission?.id, + req.permission?.type, + req.permission?.orgId, + req.permission?.authMethod, + diff.folderId + ); } if (change.folderVersion) { - const currentVersion = change.folderVersion || "1"; - const previousVersion = change.fromVersion || "1"; // eslint-disable-next-line no-await-in-loop - const versions = await server.services.folder.getFolderVersionsByIds({ - folderId: change.id, - folderVersions: - change.changeType === ChangeType.UPDATE ? [currentVersion, previousVersion] : [currentVersion] - }); - change.versions = versions.map((v) => ({ - name: v.name - })); + change.versions = await getFolderVersions(change, change.fromVersion || "1", change.id); } } } await server.services.auditLog.createAuditLog({ ...req.auditLogInfo, - projectId: req.params.workspaceId, + projectId: req.query.workspaceId, event: { type: EventType.PIT_COMPARE_FOLDER_STATES, metadata: { @@ -428,20 +375,20 @@ export const registerPITRouter = async (server: FastifyZodProvider) => { // Rollback to a previous commit server.route({ method: "POST", - url: "/commits/:workspaceId/:commitId/rollback", + url: "/commits/:commitId/rollback", config: { rateLimit: readLimit }, schema: { params: z.object({ - workspaceId: z.string().trim(), commitId: z.string().trim() }), body: z.object({ folderId: z.string().trim(), deepRollback: z.boolean().default(false), message: z.string().trim().optional(), - envId: z.string().trim() + envId: z.string().trim(), + workspaceId: z.string().trim() }), response: { 200: z.object({ @@ -454,13 +401,26 @@ export const registerPITRouter = async (server: FastifyZodProvider) => { }, onRequest: verifyAuth([AuthMode.JWT]), handler: async (req) => { + const { permission } = await server.services.permission.getProjectPermission({ + actor: req.permission?.type, + actorId: req.permission?.id, + projectId: req.body.workspaceId, + actorAuthMethod: req.permission?.authMethod, + actorOrgId: req.permission?.orgId, + actionProjectType: ActionProjectType.SecretManager + }); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionCommitsActions.PerformRollback, + ProjectPermissionSub.Commits + ); const latestCommit = await server.services.folderCommit.getLatestCommit({ folderId: req.body.folderId, actor: req.permission?.type, actorId: req.permission?.id, actorOrgId: req.permission?.orgId, actorAuthMethod: req.permission?.authMethod, - projectId: req.params.workspaceId + projectId: req.body.workspaceId }); if (!latestCommit) { throw new NotFoundError({ message: "Latest commit not found" }); @@ -472,7 +432,7 @@ export const registerPITRouter = async (server: FastifyZodProvider) => { req.body.envId, req.permission.id, req.permission.type, - req.params.workspaceId + req.body.workspaceId ); return { success: true }; } @@ -489,13 +449,13 @@ export const registerPITRouter = async (server: FastifyZodProvider) => { message: req.body.message || "Rollback to previous commit" }, folderId: req.body.folderId, - projectId: req.params.workspaceId, + projectId: req.body.workspaceId, reconstructNewFolders: req.body.deepRollback }); await server.services.auditLog.createAuditLog({ ...req.auditLogInfo, - projectId: req.params.workspaceId, + projectId: req.body.workspaceId, event: { type: EventType.PIT_ROLLBACK_COMMIT, metadata: { @@ -520,15 +480,17 @@ export const registerPITRouter = async (server: FastifyZodProvider) => { // Revert commit server.route({ method: "POST", - url: "/commits/:workspaceId/:commitId/revert", + url: "/commits/:commitId/revert", config: { rateLimit: readLimit }, schema: { params: z.object({ - workspaceId: z.string().trim(), commitId: z.string().trim() }), + querystring: z.object({ + workspaceId: z.string().trim() + }), response: { 200: z.object({ success: z.boolean(), @@ -547,12 +509,12 @@ export const registerPITRouter = async (server: FastifyZodProvider) => { actorId: req.permission?.id, actorAuthMethod: req.permission?.authMethod, actorOrgId: req.permission?.orgId, - projectId: req.params.workspaceId + projectId: req.query.workspaceId }); await server.services.auditLog.createAuditLog({ ...req.auditLogInfo, - projectId: req.params.workspaceId, + projectId: req.query.workspaceId, event: { type: EventType.PIT_REVERT_COMMIT, metadata: { @@ -570,17 +532,17 @@ export const registerPITRouter = async (server: FastifyZodProvider) => { // Folder state at commit server.route({ method: "GET", - url: "/commits/:workspaceId/:commitId", + url: "/commits/:commitId", config: { rateLimit: readLimit }, schema: { params: z.object({ - workspaceId: z.string().trim(), commitId: z.string().trim() }), querystring: z.object({ - folderId: z.string().trim() + folderId: z.string().trim(), + workspaceId: z.string().trim() }), response: { 200: folderStateSchema @@ -588,11 +550,24 @@ export const registerPITRouter = async (server: FastifyZodProvider) => { }, onRequest: verifyAuth([AuthMode.JWT]), handler: async (req) => { + const { permission } = await server.services.permission.getProjectPermission({ + actor: req.permission?.type, + actorId: req.permission?.id, + projectId: req.query.workspaceId, + actorAuthMethod: req.permission?.authMethod, + actorOrgId: req.permission?.orgId, + actionProjectType: ActionProjectType.SecretManager + }); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionCommitsActions.Read, + ProjectPermissionSub.Commits + ); const response = await server.services.folderCommit.reconstructFolderState(req.params.commitId); await server.services.auditLog.createAuditLog({ ...req.auditLogInfo, - projectId: req.params.workspaceId, + projectId: req.query.workspaceId, event: { type: EventType.PIT_GET_FOLDER_STATE, metadata: { diff --git a/backend/src/ee/services/license/license-fns.ts b/backend/src/ee/services/license/license-fns.ts index cff6ee181..b7ae6f7ee 100644 --- a/backend/src/ee/services/license/license-fns.ts +++ b/backend/src/ee/services/license/license-fns.ts @@ -17,44 +17,44 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({ environmentsUsed: 0, identityLimit: null, identitiesUsed: 0, - dynamicSecret: true, + dynamicSecret: false, secretVersioning: true, - pitRecovery: true, - ipAllowlisting: true, - rbac: true, - githubOrgSync: true, - customRateLimits: true, - customAlerts: true, - secretAccessInsights: true, - auditLogs: true, + pitRecovery: false, + ipAllowlisting: false, + rbac: false, + githubOrgSync: false, + customRateLimits: false, + customAlerts: false, + secretAccessInsights: false, + auditLogs: false, auditLogsRetentionDays: 0, - auditLogStreams: true, + auditLogStreams: false, auditLogStreamLimit: 3, - samlSSO: true, - hsm: true, - oidcSSO: true, - scim: true, - ldap: true, - groups: true, + samlSSO: false, + hsm: false, + oidcSSO: false, + scim: false, + ldap: false, + groups: false, status: null, trial_end: null, has_used_trial: true, - secretApproval: true, - secretRotation: true, - caCrl: true, - instanceUserManagement: true, - externalKms: true, + secretApproval: false, + secretRotation: false, + caCrl: false, + instanceUserManagement: false, + externalKms: false, rateLimits: { readLimit: 60, writeLimit: 200, secretsLimit: 40 }, - pkiEst: true, - enforceMfa: true, - projectTemplates: true, - kmip: true, - gateway: true, - sshHostGroups: true + pkiEst: false, + enforceMfa: false, + projectTemplates: false, + kmip: false, + gateway: false, + sshHostGroups: false }); export const setupLicenseRequestWithStore = (baseURL: string, refreshUrl: string, licenseKey: string) => { diff --git a/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts b/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts index 90377f828..d792ac6e6 100644 --- a/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts +++ b/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts @@ -14,7 +14,7 @@ import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; import { ActorType } from "@app/services/auth/auth-type"; -import { TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; +import { CommitType, TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; @@ -350,7 +350,7 @@ export const secretRotationQueueFactory = ({ message: "Changed by Secret rotation", folderId: secretVersions[0].folderId, changes: secretVersions.map((sv) => ({ - type: "add", + type: CommitType.ADD, isUpdate: true, secretVersionId: sv.id })) diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.ts index 6da6c4fa4..ed4317f0b 100644 --- a/backend/src/keystore/keystore.ts +++ b/backend/src/keystore/keystore.ts @@ -27,6 +27,7 @@ export const KeyStorePrefixes = { KmsOrgDataKeyCreation: "kms-org-data-key-creation-lock", WaitUntilReadyKmsOrgKeyCreation: "wait-until-ready-kms-org-key-creation-", WaitUntilReadyKmsOrgDataKeyCreation: "wait-until-ready-kms-org-data-key-creation-", + FolderTreeCheckpoint: (envId: string) => `folder-tree-checkpoint-${envId}`, WaitUntilReadyProjectEnvironmentOperation: (projectId: string) => `wait-until-ready-project-environments-operation-${projectId}`, diff --git a/backend/src/lib/config/env.ts b/backend/src/lib/config/env.ts index e6d9dd624..352b86770 100644 --- a/backend/src/lib/config/env.ts +++ b/backend/src/lib/config/env.ts @@ -230,8 +230,8 @@ const envSchema = z DATADOG_HOSTNAME: zpStr(z.string().optional()), // PIT - PIT_CHECKPOINT_WINDOW: zpStr(z.string().optional().default("10")), - PIT_TREE_CHECKPOINT_WINDOW: zpStr(z.string().optional().default("100")), + PIT_CHECKPOINT_WINDOW: zpStr(z.string().optional().default("30")), + PIT_TREE_CHECKPOINT_WINDOW: zpStr(z.string().optional().default("30")), /* CORS ----------------------------------------------------------------------------- */ diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 331bd00e9..bfff8e6de 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -577,6 +577,7 @@ export const registerRoutes = async ( const folderCommitQueueService = folderCommitQueueServiceFactory({ queueService, folderTreeCheckpointDAL, + keyStore, folderTreeCheckpointResourcesDAL, folderCommitDAL, folderDAL diff --git a/backend/src/services/external-migration/external-migration-fns.ts b/backend/src/services/external-migration/external-migration-fns.ts index 645d58463..018d7bd43 100644 --- a/backend/src/services/external-migration/external-migration-fns.ts +++ b/backend/src/services/external-migration/external-migration-fns.ts @@ -10,7 +10,7 @@ import { chunkArray } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; -import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; +import { CommitType, TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "../kms/kms-service"; import { KmsDataKey } from "../kms/kms-types"; import { TProjectDALFactory } from "../project/project-dal"; @@ -627,7 +627,7 @@ export const importDataIntoInfisicalFn = async ({ folderId: parentEnv.rootFolderId, changes: [ { - type: "add", + type: CommitType.ADD, folderVersionId: newFolderVersion.id } ] diff --git a/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts b/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts index 94cf1b0d4..7dc150ab5 100644 --- a/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts +++ b/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts @@ -21,7 +21,8 @@ export const folderCheckpointDALFactory = (db: TDbClient) => { const findByCommitId = async (folderCommitId: string, tx?: Knex): Promise => { try { const doc = await (tx || db.replicaNode())(TableName.FolderCheckpoint) - .where({ folderCommitId }) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ folderCommitId }, TableName.FolderCheckpoint)) .select(selectAllTableCols(TableName.FolderCheckpoint)) .first(); return doc; @@ -87,20 +88,11 @@ export const folderCheckpointDALFactory = (db: TDbClient) => { }; const findNearestCheckpoint = async ( - folderCommitId: string, + folderCommitId: bigint, + folderId: string, tx?: Knex - ): Promise<(CheckpointWithCommitInfo & { commitId: number }) | undefined> => { + ): Promise<(CheckpointWithCommitInfo & { commitId: bigint }) | undefined> => { try { - // First, get the commit info to find the folder ID and commit sequence number - const commit = await (tx || db.replicaNode())(TableName.FolderCommit) - .where({ id: folderCommitId }) - .select("commitId", "folderId") - .first(); - - if (!commit) { - return undefined; - } - // Get the checkpoint with the highest commitId that's still less than or equal to our commit const nearestCheckpoint = await (tx || db.replicaNode())(TableName.FolderCheckpoint) .join( @@ -108,8 +100,8 @@ export const folderCheckpointDALFactory = (db: TDbClient) => { `${TableName.FolderCheckpoint}.folderCommitId`, `${TableName.FolderCommit}.id` ) - .where(`${TableName.FolderCommit}.folderId`, "=", commit.folderId) - .where(`${TableName.FolderCommit}.commitId`, "<=", commit.commitId) + .where(`${TableName.FolderCommit}.folderId`, "=", folderId) + .where(`${TableName.FolderCommit}.commitId`, "<=", folderCommitId) .select(selectAllTableCols(TableName.FolderCheckpoint)) .select( db.ref("actorMetadata").withSchema(TableName.FolderCommit), @@ -121,7 +113,6 @@ export const folderCheckpointDALFactory = (db: TDbClient) => { ) .orderBy(`${TableName.FolderCommit}.commitId`, "desc") .first(); - return nearestCheckpoint; } catch (error) { throw new DatabaseError({ error, name: "FindNearestCheckpoint" }); diff --git a/backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts b/backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts index d027558fa..de3d9463b 100644 --- a/backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts +++ b/backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts @@ -5,11 +5,12 @@ import { TableName, TFolderCommitChanges, TFolderCommits, + TProjectEnvironments, TSecretFolderVersions, TSecretVersionsV2 } from "@app/db/schemas"; import { DatabaseError } from "@app/lib/errors"; -import { ormify, selectAllTableCols } from "@app/lib/knex"; +import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex"; export type TFolderCommitChangesDALFactory = ReturnType; @@ -41,10 +42,15 @@ type CommitChangeWithCommitInfo = TFolderCommitChanges & { export const folderCommitChangesDALFactory = (db: TDbClient) => { const folderCommitChangesOrm = ormify(db, TableName.FolderCommitChanges); - const findByCommitId = async (folderCommitId: string, tx?: Knex): Promise => { + const findByCommitId = async ( + folderCommitId: string, + projectId: string, + tx?: Knex + ): Promise => { try { const docs = await (tx || db.replicaNode())(TableName.FolderCommitChanges) - .where({ folderCommitId }) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ folderCommitId }, TableName.FolderCommitChanges)) .leftJoin( TableName.FolderCommit, `${TableName.FolderCommitChanges}.folderCommitId`, @@ -60,6 +66,16 @@ export const folderCommitChangesDALFactory = (db: TDbClient) => { `${TableName.FolderCommitChanges}.folderVersionId`, `${TableName.SecretFolderVersion}.id` ) + .leftJoin( + TableName.Environment, + `${TableName.FolderCommit}.envId`, + `${TableName.Environment}.id` + ) + .where((qb) => { + if (projectId) { + void qb.where(`${TableName.Environment}.projectId`, "=", projectId); + } + }) .select(selectAllTableCols(TableName.FolderCommitChanges)) .select( db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderName"), @@ -90,7 +106,8 @@ export const folderCommitChangesDALFactory = (db: TDbClient) => { TFolderCommitChanges & Pick >(TableName.FolderCommitChanges) - .where({ secretVersionId }) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ secretVersionId }, TableName.FolderCommitChanges)) .select(selectAllTableCols(TableName.FolderCommitChanges)) .join(TableName.FolderCommit, `${TableName.FolderCommitChanges}.folderCommitId`, `${TableName.FolderCommit}.id`) .select( @@ -112,7 +129,8 @@ export const folderCommitChangesDALFactory = (db: TDbClient) => { TFolderCommitChanges & Pick >(TableName.FolderCommitChanges) - .where({ folderVersionId }) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ folderVersionId }, TableName.FolderCommitChanges)) .select(selectAllTableCols(TableName.FolderCommitChanges)) .join(TableName.FolderCommit, `${TableName.FolderCommitChanges}.folderCommitId`, `${TableName.FolderCommit}.id`) .select( diff --git a/backend/src/services/folder-commit/folder-commit-dal.ts b/backend/src/services/folder-commit/folder-commit-dal.ts index 616aba72b..3e7e080ae 100644 --- a/backend/src/services/folder-commit/folder-commit-dal.ts +++ b/backend/src/services/folder-commit/folder-commit-dal.ts @@ -5,6 +5,7 @@ import { TableName, TFolderCommitChanges, TFolderCommits, + TProjectEnvironments, TSecretFolderVersions, TSecretVersionsV2 } from "@app/db/schemas"; @@ -54,10 +55,20 @@ export const folderCommitDALFactory = (db: TDbClient) => { } }; - const findLatestCommit = async (folderId: string, tx?: Knex): Promise => { + const findLatestCommit = async ( + folderId: string, + projectId?: string, + tx?: Knex + ): Promise => { try { const doc = await (tx || db.replicaNode())(TableName.FolderCommit) .where({ folderId }) + .leftJoin(TableName.Environment, `${TableName.FolderCommit}.envId`, `${TableName.Environment}.id`) + .where((qb) => { + if (projectId) { + void qb.where(`${TableName.Environment}.projectId`, "=", projectId); + } + }) .select(selectAllTableCols(TableName.FolderCommit)) .orderBy("commitId", "desc") .first(); @@ -178,8 +189,8 @@ export const folderCommitDALFactory = (db: TDbClient) => { const findCommitsToRecreate = async ( folderId: string, - targetCommitNumber: number, - checkpointCommitNumber: number, + targetCommitNumber: bigint, + checkpointCommitNumber: bigint, tx?: Knex ): Promise< (TFolderCommits & { @@ -213,7 +224,7 @@ export const folderCommitDALFactory = (db: TDbClient) => { // Get all changes for these commits in a single query const allChanges = await (tx || db.replicaNode())(TableName.FolderCommitChanges) - .whereIn("folderCommitId", commitIds) + .whereIn(`${TableName.FolderCommitChanges}.folderCommitId`, commitIds) .leftJoin( TableName.SecretVersionV2, `${TableName.FolderCommitChanges}.secretVersionId`, @@ -291,7 +302,6 @@ export const folderCommitDALFactory = (db: TDbClient) => { endCommitId, tx }: { - folderId?: string; envId?: string; startCommitId?: string; endCommitId?: string; @@ -323,7 +333,6 @@ export const folderCommitDALFactory = (db: TDbClient) => { startCommitId, tx }: { - folderId?: string; envId?: string; startCommitId?: string; tx?: Knex; @@ -346,7 +355,11 @@ export const folderCommitDALFactory = (db: TDbClient) => { } }; - const findPreviousCommitTo = async (folderId: string, commitId: string, tx?: Knex): Promise => { + const findPreviousCommitTo = async ( + folderId: string, + commitId: string, + tx?: Knex + ): Promise => { try { const doc = await (tx || db.replicaNode())(TableName.FolderCommit) .where({ folderId }) @@ -360,6 +373,30 @@ export const folderCommitDALFactory = (db: TDbClient) => { } }; + const findById = async (id: string, tx?: Knex, projectId?: string): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ id }, TableName.FolderCommit)) + .leftJoin( + TableName.Environment, + `${TableName.FolderCommit}.envId`, + `${TableName.Environment}.id` + ) + .where((qb) => { + if (projectId) { + void qb.where(`${TableName.Environment}.projectId`, "=", projectId); + } + }) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("commitId", "desc") + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindById" }); + } + }; + return { ...restOfOrm, findByFolderId, @@ -373,6 +410,7 @@ export const folderCommitDALFactory = (db: TDbClient) => { getEnvNumberOfCommitsSince, findLatestCommitByFolderIds, findAllFolderCommitsAfter, - findPreviousCommitTo + findPreviousCommitTo, + findById }; }; diff --git a/backend/src/services/folder-commit/folder-commit-queue.ts b/backend/src/services/folder-commit/folder-commit-queue.ts index d68298337..fcb348784 100644 --- a/backend/src/services/folder-commit/folder-commit-queue.ts +++ b/backend/src/services/folder-commit/folder-commit-queue.ts @@ -1,6 +1,7 @@ import { Knex } from "knex"; import { TSecretFolders } from "@app/db/schemas"; +import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore"; import { getConfig } from "@app/lib/config/env"; import { logger } from "@app/lib/logger"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; @@ -10,8 +11,16 @@ import { TFolderTreeCheckpointResourcesDALFactory } from "../folder-tree-checkpo import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TFolderCommitDALFactory } from "./folder-commit-dal"; +// Define types for job data +type TCreateFolderTreeCheckpointDTO = { + envId: string; + failedToAcquireLockCount?: number; + folderCommitId?: string; +}; + type TFolderCommitQueueServiceFactoryDep = { queueService: TQueueServiceFactory; + keyStore: Pick; folderTreeCheckpointDAL: Pick< TFolderTreeCheckpointDALFactory, "create" | "findLatestByEnvId" | "findNearestCheckpoint" @@ -31,6 +40,7 @@ export type TFolderCommitQueueServiceFactory = ReturnType { const appCfg = getConfig(); - const scheduleTreeCheckpoint = async (envId: string) => { - await queueService.queue( - QueueName.FolderTreeCheckpoint, - QueueJobs.CreateFolderTreeCheckpoint, - { envId }, - { - jobId: `${envId}`, - backoff: { - type: "exponential", - delay: 3000 - }, - removeOnFail: { - count: 3 - }, - removeOnComplete: true - } - ); + // Helper function to calculate delay for requeuing + const getRequeueDelay = (failureCount?: number) => { + if (!failureCount) return 0; + + const baseDelay = 5000; + const maxDelay = 30000; + + const delay = Math.min(baseDelay * 2 ** failureCount, maxDelay); + const jitter = delay * (0.5 + Math.random() * 0.5); + + return jitter; }; - const schedulePeriodicTreeCheckpoint = async (envId: string, intervalMs: number) => { - await queueService.queue( - QueueName.FolderTreeCheckpoint, - QueueJobs.CreateFolderTreeCheckpoint, - { envId }, - { - jobId: `periodic-${envId}`, - repeat: { - every: intervalMs - }, - backoff: { - type: "exponential", - delay: 3000 - }, - removeOnFail: false, - removeOnComplete: false - } - ); - }; + const scheduleTreeCheckpoint = async (payload: TCreateFolderTreeCheckpointDTO) => { + const { envId, failedToAcquireLockCount = 0 } = payload; - const cancelScheduledTreeCheckpoint = async (envId: string) => { - await queueService.stopJobById(QueueName.FolderTreeCheckpoint, envId); - await queueService.stopRepeatableJobByJobId(QueueName.FolderTreeCheckpoint, `periodic-${envId}`); + // Create a unique jobId for each retry to prevent conflicts + const jobId = + failedToAcquireLockCount > 0 ? `${envId}-retry-${failedToAcquireLockCount}-${Date.now()}` : `${envId}`; + + await queueService.queue(QueueName.FolderTreeCheckpoint, QueueJobs.CreateFolderTreeCheckpoint, payload, { + jobId, + delay: getRequeueDelay(failedToAcquireLockCount), + backoff: { + type: "exponential", + delay: 3000 + }, + removeOnFail: { + count: 3 + }, + removeOnComplete: true + }); }; // Sort folders by hierarchy (copied from the source code) @@ -121,66 +121,152 @@ export const folderCommitQueueServiceFactory = ({ return result; }; - const createFolderTreeCheckpoint = async (envId: string, folderCommitId?: string, tx?: Knex) => { - logger.info("Folder tree checkpoint creation started:", envId); + const createFolderTreeCheckpoint = async (jobData: TCreateFolderTreeCheckpointDTO, tx?: Knex) => { + const { envId, folderCommitId, failedToAcquireLockCount = 0 } = jobData; - const latestTreeCheckpoint = await folderTreeCheckpointDAL.findLatestByEnvId(envId, tx); + logger.info(`Folder tree checkpoint creation started [envId=${envId}] [attempt=${failedToAcquireLockCount + 1}]`); - let latestCommit; - if (folderCommitId) { - latestCommit = await folderCommitDAL.findById(folderCommitId, tx); - } else { - latestCommit = await folderCommitDAL.findLatestEnvCommit(envId, tx); - } - if (!latestCommit) { - logger.info(`Latest commit ID not found for envId ${envId}`); - return; - } - const latestCommitId = latestCommit.id; - - if (latestTreeCheckpoint) { - const commitsSinceLastCheckpoint = await folderCommitDAL.getEnvNumberOfCommitsSince( - envId, - latestTreeCheckpoint.folderCommitId, - tx - ); - if (commitsSinceLastCheckpoint < Number(appCfg.PIT_TREE_CHECKPOINT_WINDOW)) { - logger.info( - `Commits since last checkpoint ${commitsSinceLastCheckpoint} is less than ${appCfg.PIT_TREE_CHECKPOINT_WINDOW}` - ); - return; + // First, try to clear any stale locks before attempting to acquire + if (failedToAcquireLockCount > 1) { + try { + await keyStore.deleteItem(KeyStorePrefixes.FolderTreeCheckpoint(envId)); + logger.info(`Cleared potential stale lock for envId ${envId} before attempt ${failedToAcquireLockCount + 1}`); + } catch (error) { + // This is fine if it fails, we'll still try to acquire the lock + logger.info(`No stale lock found for envId ${envId}`); } } - const folders = await folderDAL.findByEnvId(envId, tx); - const sortedFolders = sortFoldersByHierarchy(folders); - const filteredFoldersIds = sortedFolders.filter((folder) => !folder.isReserved).map((folder) => folder.id); + let lock: Awaited> | undefined; - const folderCommits = await folderCommitDAL.findMultipleLatestCommits(filteredFoldersIds, tx); - const folderTreeCheckpoint = await folderTreeCheckpointDAL.create( - { - folderCommitId: latestCommitId - }, - tx - ); + try { + // Attempt to acquire the lock with a shorter timeout for first attempts + const timeout = failedToAcquireLockCount > 3 ? 60 * 1000 : 15 * 1000; - await folderTreeCheckpointResourcesDAL.insertMany( - folderCommits.map((folderCommit) => ({ - folderTreeCheckpointId: folderTreeCheckpoint.id, - folderId: folderCommit.folderId, - folderCommitId: folderCommit.id - })), - tx - ); + logger.info(`Attempting to acquire lock for envId=${envId} with timeout ${timeout}ms`); - logger.info("Folder tree checkpoint created successfully:", folderTreeCheckpoint.id); + lock = await keyStore.acquireLock([KeyStorePrefixes.FolderTreeCheckpoint(envId)], timeout); + + logger.info(`Successfully acquired lock for envId=${envId}`); + } catch (e) { + logger.info( + `Failed to acquire lock for folder tree checkpoint [envId=${envId}] [attempt=${failedToAcquireLockCount + 1}]` + ); + + // Requeue with incremented failure count if under max attempts + if (failedToAcquireLockCount < 10) { + // Force a delay between retries + const nextRetryCount = failedToAcquireLockCount + 1; + + logger.info(`Scheduling retry #${nextRetryCount} for folder tree checkpoint [envId=${envId}]`); + + // Create a new job with incremented counter + await scheduleTreeCheckpoint({ + envId, + folderCommitId, + failedToAcquireLockCount: nextRetryCount + }); + } else { + // Max retries reached + logger.error(`Maximum lock acquisition attempts (10) reached for envId ${envId}. Giving up.`); + // Try to force-clear the lock for next time + try { + await keyStore.deleteItem(KeyStorePrefixes.FolderTreeCheckpoint(envId)); + } catch (clearError) { + logger.error(clearError, `Failed to clear lock after maximum retries for envId=${envId}`); + } + } + return; + } + + if (!lock) { + logger.error(`Lock is undefined after acquisition for envId=${envId}. This should never happen.`); + return; + } + + try { + logger.info(`Processing tree checkpoint data for envId=${envId}`); + + const latestTreeCheckpoint = await folderTreeCheckpointDAL.findLatestByEnvId(envId, tx); + + let latestCommit; + if (folderCommitId) { + latestCommit = await folderCommitDAL.findById(folderCommitId, tx); + } else { + latestCommit = await folderCommitDAL.findLatestEnvCommit(envId, tx); + } + if (!latestCommit) { + logger.info(`Latest commit ID not found for envId ${envId}`); + return; + } + const latestCommitId = latestCommit.id; + + if (latestTreeCheckpoint) { + const commitsSinceLastCheckpoint = await folderCommitDAL.getEnvNumberOfCommitsSince( + envId, + latestTreeCheckpoint.folderCommitId, + tx + ); + if (commitsSinceLastCheckpoint < Number(appCfg.PIT_TREE_CHECKPOINT_WINDOW)) { + logger.info( + `Commits since last checkpoint ${commitsSinceLastCheckpoint} is less than ${appCfg.PIT_TREE_CHECKPOINT_WINDOW}` + ); + return; + } + } + + const folders = await folderDAL.findByEnvId(envId, tx); + const sortedFolders = sortFoldersByHierarchy(folders); + const filteredFoldersIds = sortedFolders.filter((folder) => !folder.isReserved).map((folder) => folder.id); + + const folderCommits = await folderCommitDAL.findMultipleLatestCommits(filteredFoldersIds, tx); + const folderTreeCheckpoint = await folderTreeCheckpointDAL.create( + { + folderCommitId: latestCommitId + }, + tx + ); + + await folderTreeCheckpointResourcesDAL.insertMany( + folderCommits.map((folderCommit) => ({ + folderTreeCheckpointId: folderTreeCheckpoint.id, + folderId: folderCommit.folderId, + folderCommitId: folderCommit.id + })), + tx + ); + + logger.info(`Folder tree checkpoint created successfully: ${folderTreeCheckpoint.id}`); + } catch (error) { + logger.error(error, `Error processing folder tree checkpoint [envId=${envId}]`); + throw error; + } finally { + // Always release the lock + try { + if (lock) { + await lock.release(); + logger.info(`Released lock for folder tree checkpoint [envId=${envId}]`); + } else { + logger.error(`No lock to release for envId=${envId}. This should never happen.`); + } + } catch (releaseError) { + logger.error(releaseError, `Error releasing lock for folder tree checkpoint [envId=${envId}]`); + // Try to force delete the lock if release fails + try { + await keyStore.deleteItem(KeyStorePrefixes.FolderTreeCheckpoint(envId)); + logger.info(`Force deleted lock after release failure for envId=${envId}`); + } catch (deleteError) { + logger.error(deleteError, `Failed to force delete lock after release failure for envId=${envId}`); + } + } + } }; queueService.start(QueueName.FolderTreeCheckpoint, async (job) => { try { if (job.name === QueueJobs.CreateFolderTreeCheckpoint) { - const { envId } = job.data as { envId: string }; - await createFolderTreeCheckpoint(envId); + const jobData = job.data as TCreateFolderTreeCheckpointDTO; + await createFolderTreeCheckpoint(jobData); } } catch (error) { logger.error(error, "Error creating folder tree checkpoint:"); @@ -189,9 +275,8 @@ export const folderCommitQueueServiceFactory = ({ }); return { - scheduleTreeCheckpoint, - schedulePeriodicTreeCheckpoint, - cancelScheduledTreeCheckpoint, - createFolderTreeCheckpoint + scheduleTreeCheckpoint: (envId: string) => scheduleTreeCheckpoint({ envId }), + createFolderTreeCheckpoint: (envId: string, folderCommitId?: string, tx?: Knex) => + createFolderTreeCheckpoint({ envId, folderCommitId }, tx) }; }; diff --git a/backend/src/services/folder-commit/folder-commit-service.test.ts b/backend/src/services/folder-commit/folder-commit-service.test.ts index bc327da81..26c37f0ff 100644 --- a/backend/src/services/folder-commit/folder-commit-service.test.ts +++ b/backend/src/services/folder-commit/folder-commit-service.test.ts @@ -5,7 +5,12 @@ import { TSecretFolderVersions, TSecretVersionsV2 } from "@app/db/schemas"; import { BadRequestError, NotFoundError } from "@app/lib/errors"; import { ActorType } from "../auth/auth-type"; -import { ChangeType, folderCommitServiceFactory, TFolderCommitServiceFactory } from "./folder-commit-service"; +import { + ChangeType, + CommitType, + folderCommitServiceFactory, + TFolderCommitServiceFactory +} from "./folder-commit-service"; // Mock config vi.mock("@app/lib/config/env", () => ({ @@ -204,7 +209,7 @@ describe("folderCommitServiceFactory", () => { folderId: folderData.id, changes: [ { - type: "add", + type: CommitType.ADD, secretVersionId: "secret-version-1" } ] @@ -264,7 +269,7 @@ describe("folderCommitServiceFactory", () => { folderId: folderData.id, changes: [ { - type: "add", + type: CommitType.ADD, folderVersionId: "folder-version-1" } ] @@ -331,7 +336,7 @@ describe("folderCommitServiceFactory", () => { const data = { folderCommitId: commitData.id, - changeType: "add", + changeType: CommitType.ADD, secretVersionId: "secret-version-1" }; @@ -348,7 +353,7 @@ describe("folderCommitServiceFactory", () => { // Arrange const data = { folderCommitId: "commit-id", - changeType: "add" + changeType: CommitType.ADD }; // Act & Assert @@ -361,7 +366,7 @@ describe("folderCommitServiceFactory", () => { const data = { folderCommitId: "non-existent-commit", - changeType: "add", + changeType: CommitType.ADD, secretVersionId: "secret-version-1" }; diff --git a/backend/src/services/folder-commit/folder-commit-service.ts b/backend/src/services/folder-commit/folder-commit-service.ts index 87e03f31b..21a50ed9f 100644 --- a/backend/src/services/folder-commit/folder-commit-service.ts +++ b/backend/src/services/folder-commit/folder-commit-service.ts @@ -32,6 +32,11 @@ export enum ChangeType { CREATE = "create" } +export enum CommitType { + ADD = "add", + DELETE = "delete" +} + enum ResourceType { SECRET = "secret", FOLDER = "folder" @@ -68,11 +73,12 @@ export type ResourceChange = { versionId: string; oldVersionId?: string; changeType: ChangeType; - commitId: number; + commitId: bigint; createdAt?: Date; parentId?: string; secretKey?: string; secretVersion?: string; + secretId?: string; folderName?: string; folderVersion?: string; fromVersion?: string; @@ -209,7 +215,7 @@ export const folderCommitServiceFactory = ({ const latestCheckpoint = await folderCheckpointDAL.findLatestByFolderId(folderId, tx); if (!latestCommitId) { - const latestCommit = await folderCommitDAL.findLatestCommit(folderId, tx); + const latestCommit = await folderCommitDAL.findLatestCommit(folderId, undefined, tx); if (!latestCommit) { throw new BadRequestError({ message: "Latest commit ID not found" }); } @@ -264,7 +270,11 @@ export const folderCommitServiceFactory = ({ throw new NotFoundError({ message: `Commit with ID ${folderCommitId} not found` }); } - const nearestCheckpoint = await folderCheckpointDAL.findNearestCheckpoint(folderCommitId, tx); + const nearestCheckpoint = await folderCheckpointDAL.findNearestCheckpoint( + targetCommit.commitId, + targetCommit.folderId, + tx + ); if (!nearestCheckpoint) { throw new NotFoundError({ message: `Nearest checkpoint not found for commit ${folderCommitId}` }); } @@ -288,7 +298,7 @@ export const folderCommitServiceFactory = ({ checkpointResources.forEach((resource) => { if (resource.secretVersionId && resource.referencedSecretId) { folderState[`secret-${resource.referencedSecretId}`] = { - type: "secret", + type: ResourceType.SECRET, id: resource.referencedSecretId, versionId: resource.secretVersionId, secretKey: resource.secretKey, @@ -296,7 +306,7 @@ export const folderCommitServiceFactory = ({ }; } else if (resource.folderVersionId && resource.referencedFolderId) { folderState[`folder-${resource.referencedFolderId}`] = { - type: "folder", + type: ResourceType.FOLDER, id: resource.referencedFolderId, versionId: resource.folderVersionId, folderName: resource.folderName, @@ -323,7 +333,7 @@ export const folderCommitServiceFactory = ({ if (change.changeType.toLowerCase() === "add") { folderState[key] = { - type: "secret", + type: ResourceType.SECRET, id: change.referencedSecretId, versionId: change.secretVersionId, secretKey: change.secretKey, @@ -337,7 +347,7 @@ export const folderCommitServiceFactory = ({ if (change.changeType.toLowerCase() === "add") { folderState[key] = { - type: "folder", + type: ResourceType.FOLDER, id: change.referencedFolderId, versionId: change.folderVersionId, folderName: change.folderName, @@ -902,12 +912,14 @@ export const folderCommitServiceFactory = ({ const secretVersions = await secretVersionV2BridgeDAL.findByIdsWithLatestVersion( folderId, secretChanges.map((diff) => diff.id), - secretChanges.map((diff) => diff.versionId) + secretChanges.map((diff) => diff.versionId), + tx ); const folderVersions = await folderVersionDAL.findByIdsWithLatestVersion( folderChanges.map((diff) => diff.id), - folderChanges.map((diff) => diff.versionId) + folderChanges.map((diff) => diff.versionId), + tx ); // Process changes in parallel @@ -1072,8 +1084,8 @@ export const folderCommitServiceFactory = ({ actorOrgId, projectId }); - const changes = await folderCommitChangesDAL.findByCommitId(commitId); - const commit = await folderCommitDAL.findById(commitId); + const changes = await folderCommitChangesDAL.findByCommitId(commitId, projectId); + const commit = await folderCommitDAL.findById(commitId, undefined, projectId); return { ...commit, changes }; }; @@ -1244,8 +1256,6 @@ export const folderCommitServiceFactory = ({ }: { targetCommitId: string; envId: string; - actorId: string; - actorType: ActorType; projectId: string; tx?: Knex; }) => { @@ -1254,7 +1264,7 @@ export const folderCommitServiceFactory = ({ throw new NotFoundError({ message: `No commit found for commit ID ${targetCommitId}` }); } - const checkpoint = await folderTreeCheckpointDAL.findNearestCheckpoint(targetCommitId, envId, tx); + const checkpoint = await folderTreeCheckpointDAL.findNearestCheckpoint(targetCommit.commitId, envId, tx); if (!checkpoint) { throw new NotFoundError({ message: `No checkpoint found for commit ID ${targetCommitId}` }); } @@ -1267,7 +1277,7 @@ export const folderCommitServiceFactory = ({ }); // Group commits by folderId and keep only the latest - const folderGroups = new Map(); + const folderGroups = new Map(); if (folderCheckpointCommits && folderCheckpointCommits.length > 0) { for (const commit of folderCheckpointCommits) { @@ -1358,7 +1368,7 @@ export const folderCommitServiceFactory = ({ throw new NotFoundError({ message: `No commit found for commit ID ${targetCommitId}` }); } - const checkpoint = await folderTreeCheckpointDAL.findNearestCheckpoint(targetCommitId, envId, tx); + const checkpoint = await folderTreeCheckpointDAL.findNearestCheckpoint(targetCommit.commitId, envId, tx); if (!checkpoint) { throw new NotFoundError({ message: `No checkpoint found for commit ID ${targetCommitId}` }); } @@ -1371,7 +1381,7 @@ export const folderCommitServiceFactory = ({ }); // Group commits by folderId and keep only the latest - const folderGroups = new Map(); + const folderGroups = new Map(); if (folderCheckpointCommits && folderCheckpointCommits.length > 0) { for (const commit of folderCheckpointCommits) { @@ -1421,9 +1431,9 @@ export const folderCommitServiceFactory = ({ const foldersToDelete = new Set(); // Process all DELETE operations to build a complete set of folders to be deleted - for (const [folderId, changes] of folderDiffs.entries()) { + for (const changes of folderDiffs.values()) { for (const change of changes) { - if (change.changeType === ChangeType.DELETE && change.type === "folder") { + if (change.changeType === ChangeType.DELETE && change.type === ResourceType.FOLDER) { foldersToDelete.add(change.id); } } @@ -1483,7 +1493,7 @@ export const folderCommitServiceFactory = ({ actorOrgId, projectId }); - return folderCommitDAL.findLatestCommit(folderId); + return folderCommitDAL.findLatestCommit(folderId, projectId); }; /** @@ -1506,6 +1516,19 @@ export const folderCommitServiceFactory = ({ projectId: string; message?: string; }) => { + const { permission } = await permissionService.getProjectPermission({ + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId, + actionProjectType: ActionProjectType.SecretManager + }); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionCommitsActions.PerformRollback, + ProjectPermissionSub.Commits + ); // Check permissions first await checkProjectPermission({ actor, @@ -1516,7 +1539,7 @@ export const folderCommitServiceFactory = ({ }); // Get the commit to revert - const commitToRevert = await folderCommitDAL.findById(commitId); + const commitToRevert = await folderCommitDAL.findById(commitId, undefined, projectId); if (!commitToRevert) { throw new NotFoundError({ message: `Commit with ID ${commitId} not found` }); } diff --git a/backend/src/services/folder-commit/folder-commit-types.ts b/backend/src/services/folder-commit/folder-commit-types.ts new file mode 100644 index 000000000..b848417f9 --- /dev/null +++ b/backend/src/services/folder-commit/folder-commit-types.ts @@ -0,0 +1,84 @@ +import { z } from "zod"; + +const secretVersionSchema = z.object({ + secretKey: z.string(), + secretComment: z.string().optional().nullable(), + skipMultilineEncoding: z.boolean().optional().nullable(), + secretReminderRepeatDays: z.number().optional().nullable(), + secretReminderNote: z.string().optional().nullable(), + metadata: z.unknown().optional().nullable(), + tags: z.array(z.string()).optional().nullable(), + secretReminderRecipients: z.array(z.any()).optional().nullable(), + secretValue: z.string().optional().nullable() +}); + +const folderVersionSchema = z.object({ + name: z.string() +}); + +const baseChangeSchema = z.object({ + id: z.string(), + folderCommitId: z.string(), + changeType: z.string(), + isUpdate: z.boolean().optional(), + createdAt: z.union([z.string(), z.date()]), + updatedAt: z.union([z.string(), z.date()]), + actorMetadata: z + .union([ + z.object({ + id: z.string().optional(), + name: z.string().optional() + }), + z.unknown() + ]) + .optional(), + actorType: z.string().optional(), + message: z.string().optional().nullable(), + folderId: z.string().optional() +}); + +const secretChangeSchema = baseChangeSchema.extend({ + objectType: z.literal("secret"), + secretVersionId: z.string(), + folderVersionId: z.null(), + folderName: z.null(), + folderChangeId: z.null(), + secretKey: z.string(), + secretVersion: z.union([z.string(), z.number()]), + secretId: z.string(), + versions: z.array(secretVersionSchema).optional() +}); + +const folderChangeSchema = baseChangeSchema.extend({ + objectType: z.literal("folder"), + secretVersionId: z.null(), + folderVersionId: z.string(), + folderName: z.string(), + folderChangeId: z.string(), + folderVersion: z.union([z.string(), z.number()]), + secretKey: z.null(), + secretId: z.null(), + versions: z.array(folderVersionSchema).optional() +}); + +const commitChangeSchema = z.discriminatedUnion("objectType", [secretChangeSchema, folderChangeSchema]); + +const commitSchema = z.object({ + id: z.string(), + commitId: z.string(), + actorMetadata: z.object({ + id: z.string(), + name: z.string() + }), + actorType: z.string(), + message: z.string().nullable(), + folderId: z.string(), + envId: z.string(), + createdAt: z.union([z.string(), z.date()]), + updatedAt: z.union([z.string(), z.date()]), + changes: z.array(commitChangeSchema) +}); + +export const commitChangesResponseSchema = z.object({ + changes: commitSchema +}); diff --git a/backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts b/backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts index 5d3fac9c2..58651a48a 100644 --- a/backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts +++ b/backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts @@ -8,7 +8,7 @@ import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex"; export type TFolderTreeCheckpointResourcesDALFactory = ReturnType; type TFolderTreeCheckpointResourcesWithCommitId = TFolderTreeCheckpointResources & { - commitId: number; + commitId: bigint; }; export const folderTreeCheckpointResourcesDALFactory = (db: TDbClient) => { diff --git a/backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts b/backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts index a421b86e9..5eced9da3 100644 --- a/backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts +++ b/backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts @@ -3,12 +3,12 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; import { TableName, TFolderCommits, TFolderTreeCheckpoints } from "@app/db/schemas"; import { DatabaseError } from "@app/lib/errors"; -import { ormify, selectAllTableCols } from "@app/lib/knex"; +import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex"; export type TFolderTreeCheckpointDALFactory = ReturnType; type TreeCheckpointWithCommitInfo = TFolderTreeCheckpoints & { - commitId: number; + commitId: bigint; }; export const folderTreeCheckpointDALFactory = (db: TDbClient) => { @@ -17,7 +17,8 @@ export const folderTreeCheckpointDALFactory = (db: TDbClient) => { const findByCommitId = async (folderCommitId: string, tx?: Knex): Promise => { try { const doc = await (tx || db.replicaNode())(TableName.FolderTreeCheckpoint) - .where({ folderCommitId }) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ folderCommitId }, TableName.FolderTreeCheckpoint)) .select(selectAllTableCols(TableName.FolderTreeCheckpoint)) .first(); return doc; @@ -27,27 +28,20 @@ export const folderTreeCheckpointDALFactory = (db: TDbClient) => { }; const findNearestCheckpoint = async ( - folderCommitId: string, + folderCommitId: bigint, envId: string, tx?: Knex ): Promise => { try { - const targetCommit = await (tx || db.replicaNode())(TableName.FolderCommit) - .where({ id: folderCommitId }) - .select("id", "commitId", "folderId", "envId") - .first(); - - if (!targetCommit) { - return undefined; - } - const nearestCheckpoint = await (tx || db.replicaNode())(TableName.FolderTreeCheckpoint) - .leftJoin( + .join( TableName.FolderCommit, `${TableName.FolderTreeCheckpoint}.folderCommitId`, `${TableName.FolderCommit}.id` ) - .where(`${TableName.FolderCommit}.envId`, "=", targetCommit.envId) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(`${TableName.FolderCommit}.envId`, "=", envId) + .where(`${TableName.FolderCommit}.commitId`, "<=", folderCommitId) .select(selectAllTableCols(TableName.FolderTreeCheckpoint)) .select(db.ref("commitId").withSchema(TableName.FolderCommit)) .orderBy(`${TableName.FolderCommit}.commitId`, "desc") @@ -67,12 +61,12 @@ export const folderTreeCheckpointDALFactory = (db: TDbClient) => { `${TableName.FolderTreeCheckpoint}.folderCommitId`, `${TableName.FolderCommit}.id` ) - .where(`${TableName.FolderCommit}.envId`, envId) + .where(`${TableName.FolderCommit}.envId`, "=", envId) .orderBy(`${TableName.FolderTreeCheckpoint}.createdAt`, "desc") .first(); return doc; } catch (error) { - throw new DatabaseError({ error, name: "FindLatestByFolderId" }); + throw new DatabaseError({ error, name: "FindLatestByEnvId" }); } }; diff --git a/backend/src/services/secret-folder/secret-folder-service.ts b/backend/src/services/secret-folder/secret-folder-service.ts index 87a37b707..c8352cc67 100644 --- a/backend/src/services/secret-folder/secret-folder-service.ts +++ b/backend/src/services/secret-folder/secret-folder-service.ts @@ -10,7 +10,7 @@ import { BadRequestError, NotFoundError } from "@app/lib/errors"; import { OrderByDirection, OrgServiceActor } from "@app/lib/types"; import { buildFolderPath } from "@app/services/secret-folder/secret-folder-fns"; -import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; +import { CommitType, TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TProjectDALFactory } from "../project/project-dal"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TSecretFolderDALFactory } from "./secret-folder-dal"; @@ -134,7 +134,7 @@ export const secretFolderServiceFactory = ({ message: "Folder created", folderId: parentFolderId, changes: folderVersions.map((fv) => ({ - type: "add", + type: CommitType.ADD, folderVersionId: fv.id })) }, @@ -168,7 +168,7 @@ export const secretFolderServiceFactory = ({ folderId: parentFolderId, changes: [ { - type: "add", + type: CommitType.ADD, folderVersionId: folderVersion.id } ] @@ -285,7 +285,7 @@ export const secretFolderServiceFactory = ({ folderId: parentFolder.id, changes: [ { - type: "add", + type: CommitType.ADD, isUpdate: true, folderVersionId: folderVersion.id } @@ -401,7 +401,7 @@ export const secretFolderServiceFactory = ({ folderId: parentFolder.id, changes: [ { - type: "add", + type: CommitType.ADD, isUpdate: true, folderVersionId: folderVersion.id } @@ -463,7 +463,7 @@ export const secretFolderServiceFactory = ({ if (!doc) throw new NotFoundError({ message: `Failed to delete folder with ID '${idOrName}', not found` }); - const folderVersions = await folderVersionDAL.findLatestFolderVersions([doc.id]); + const folderVersions = await folderVersionDAL.findLatestFolderVersions([doc.id], tx); await folderCommitService.createCommit( { @@ -477,7 +477,7 @@ export const secretFolderServiceFactory = ({ folderId: parentFolder.id, changes: [ { - type: "delete", + type: CommitType.DELETE, folderVersionId: folderVersions[doc.id].id } ] diff --git a/backend/src/services/secret-folder/secret-folder-version-dal.ts b/backend/src/services/secret-folder/secret-folder-version-dal.ts index 48cd03231..dc8d88bd5 100644 --- a/backend/src/services/secret-folder/secret-folder-version-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-version-dal.ts @@ -97,67 +97,78 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => { logger.info(`${QueueName.DailyResourceCleanUp}: pruning secret folder versions completed`); }; + // Get latest versions by folderIds + const getLatestFolderVersions = async (folderIds: string[], tx?: Knex): Promise> => { + if (!folderIds.length) return []; + + const knexInstance = tx || db.replicaNode(); + return knexInstance(TableName.SecretFolderVersion) + .whereIn(`${TableName.SecretFolderVersion}.folderId`, folderIds) + .join( + knexInstance(TableName.SecretFolderVersion) + .groupBy("folderId") + .max("version") + .select("folderId") + .as("latestVersion"), + (bd) => { + bd.on(`${TableName.SecretFolderVersion}.folderId`, "latestVersion.folderId").andOn( + `${TableName.SecretFolderVersion}.version`, + "latestVersion.max" + ); + } + ); + }; + + // Get specific versions and update with max version + const getSpecificFolderVersionsWithLatest = async ( + versionIds: string[], + tx?: Knex + ): Promise> => { + if (!versionIds.length) return []; + + const knexInstance = tx || db.replicaNode(); + + // Get specific versions + const specificVersions = await knexInstance(TableName.SecretFolderVersion).whereIn("id", versionIds); + + // Get folderIds from these versions + const specificFolderIds = [...new Set(specificVersions.map((v) => v.folderId).filter(Boolean))]; + + if (!specificFolderIds.length) return specificVersions; + + // Get max versions for these folderIds + const maxVersionsQuery = await knexInstance(TableName.SecretFolderVersion) + .whereIn("folderId", specificFolderIds) + .groupBy("folderId") + .select("folderId") + .max("version", { as: "maxVersion" }); + + // Create lookup map for max versions + const maxVersionMap = maxVersionsQuery.reduce>((acc, item) => { + acc[item.folderId] = item.maxVersion; + return acc; + }, {}); + + // Replace version with max version + return specificVersions.map((version) => ({ + ...version, + version: maxVersionMap[version.folderId] || version.version + })); + }; + const findByIdsWithLatestVersion = async (folderIds: string[], versionIds?: string[], tx?: Knex) => { try { if (!folderIds.length && (!versionIds || !versionIds.length)) return {}; - const knexInstance = tx || db.replicaNode(); - let allDocs: Array = []; + const [latestVersions, specificVersionsWithLatest] = await Promise.all([ + folderIds.length ? getLatestFolderVersions(folderIds, tx) : [], + versionIds?.length ? getSpecificFolderVersionsWithLatest(versionIds, tx) : [] + ]); - // Process latest versions for folderIds - if (folderIds.length) { - const latestVersions: Array = await knexInstance( - TableName.SecretFolderVersion - ) - .whereIn(`${TableName.SecretFolderVersion}.folderId`, folderIds) - .join( - knexInstance(TableName.SecretFolderVersion) - .groupBy("folderId") - .max("version") - .select("folderId") - .as("latestVersion"), - (bd) => { - bd.on(`${TableName.SecretFolderVersion}.folderId`, "latestVersion.folderId").andOn( - `${TableName.SecretFolderVersion}.version`, - "latestVersion.max" - ); - } - ); + const allDocs = [...latestVersions, ...specificVersionsWithLatest]; - allDocs = [...allDocs, ...latestVersions]; - } - - // Process specific versions by versionIds - if (versionIds && versionIds.length) { - // Get the specific versions - const specificVersions = await knexInstance(TableName.SecretFolderVersion).whereIn("id", versionIds); - - // Get the folderIds from these versions - const specificFolderIds = [...new Set(specificVersions.map((v) => v.folderId).filter(Boolean))]; - - // Get max versions for these folderIds - const maxVersionsQuery = (await knexInstance(TableName.SecretFolderVersion) - .whereIn("folderId", specificFolderIds) - .groupBy("folderId") - .select("folderId") - .max("version as max")) as Array<{ folderId: string; max: number }>; - - // Create a lookup map for max versions - const maxVersionMap = maxVersionsQuery.reduce>((acc, item) => { - acc[item.folderId] = item.max; - return acc; - }, {}); - - // Add max version to each specific version - const specificVersionsWithMax = specificVersions.map((version) => ({ - ...version, - max: maxVersionMap[version.folderId] - })); - - allDocs = [...allDocs, ...specificVersionsWithMax]; - } - - return allDocs.reduce>( + // Convert array to record with folderId as key + return allDocs.reduce>( (prev, curr) => ({ ...prev, [curr.folderId || ""]: curr }), {} ); diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts index 8b976aebf..0af512501 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts @@ -8,6 +8,7 @@ import { groupBy } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; import { ActorType } from "../auth/auth-type"; +import { CommitType } from "../folder-commit/folder-commit-service"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { ResourceMetadataDTO } from "../resource-metadata/resource-metadata-schema"; import { INFISICAL_SECRET_VALUE_HIDDEN_MASK } from "../secret/secret-fns"; @@ -135,7 +136,7 @@ export const fnSecretBulkInsert = async ({ const commitChanges = secretVersions .filter(({ type }) => type === SecretType.Shared) .map((sv) => ({ - type: "add", + type: CommitType.ADD, secretVersionId: sv.id })); @@ -289,7 +290,7 @@ export const fnSecretBulkUpdate = async ({ const commitChanges = secretVersions .filter(({ type }) => type === SecretType.Shared) .map((sv) => ({ - type: "add", + type: CommitType.ADD, isUpdate: true, secretVersionId: sv.id })); @@ -421,7 +422,7 @@ export const fnSecretBulkDelete = async ({ const commitChanges = deletedSecrets .filter(({ type }) => type === SecretType.Shared) .map(({ id }) => ({ - type: "delete", + type: CommitType.DELETE, secretVersionId: secretVersions[id].id })); if (commitChanges.length > 0) { diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts index 0e8ce29fc..d21156631 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts @@ -16,7 +16,7 @@ import { } from "@app/ee/services/permission/permission-fns"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { - ProjectPermissionActions, + ProjectPermissionCommitsActions, ProjectPermissionSecretActions, ProjectPermissionSet, ProjectPermissionSub @@ -2173,7 +2173,7 @@ export const secretV2BridgeServiceFactory = ({ actorOrgId, actionProjectType: ActionProjectType.SecretManager }); - ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback); + ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionCommitsActions.Read, ProjectPermissionSub.Commits); const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ type: KmsDataKey.SecretManager, projectId: folder.projectId @@ -2889,7 +2889,7 @@ export const secretV2BridgeServiceFactory = ({ actorOrgId, actionProjectType: ActionProjectType.SecretManager }); - ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback); + ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionCommitsActions.Read, ProjectPermissionSub.Commits); const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ type: KmsDataKey.SecretManager, projectId: folder.projectId diff --git a/backend/src/services/secret-v2-bridge/secret-version-dal.ts b/backend/src/services/secret-v2-bridge/secret-version-dal.ts index e62f3e05c..e9c9b1cd9 100644 --- a/backend/src/services/secret-v2-bridge/secret-version-dal.ts +++ b/backend/src/services/secret-v2-bridge/secret-version-dal.ts @@ -272,6 +272,79 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { } }; + // Function to fetch latest versions by secretIds + const getLatestVersionsBySecretIds = async ( + folderId: string, + secretIds: string[], + tx?: Knex + ): Promise> => { + if (!secretIds.length) return []; + + const knexInstance = tx || db.replicaNode(); + return knexInstance(TableName.SecretVersionV2) + .where("folderId", folderId) + .whereIn(`${TableName.SecretVersionV2}.secretId`, secretIds) + .join( + knexInstance(TableName.SecretVersionV2) + .groupBy("secretId") + .max("version") + .select("secretId") + .as("latestVersion"), + (bd) => { + bd.on(`${TableName.SecretVersionV2}.secretId`, "latestVersion.secretId").andOn( + `${TableName.SecretVersionV2}.version`, + "latestVersion.max" + ); + } + ); + }; + + // Function to fetch specific versions by versionIds + const getSpecificVersionsWithLatestInfo = async ( + folderId: string, + versionIds: string[], + tx?: Knex + ): Promise> => { + if (!versionIds.length) return []; + + const knexInstance = tx || db.replicaNode(); + + // Get the specific versions + const specificVersions = await knexInstance(TableName.SecretVersionV2) + .where("folderId", folderId) + .whereIn("id", versionIds); + + // Get the secretIds from these versions + const specificSecretIds = [...new Set(specificVersions.map((v) => v.secretId).filter(Boolean))]; + + if (!specificSecretIds.length) return specificVersions; + + // Get max versions for these secretIds + const maxVersionsQuery = await knexInstance(TableName.SecretVersionV2) + .whereIn("secretId", specificSecretIds) + .groupBy("secretId") + .select("secretId") + .max("version as maxVersion"); + + // Create a lookup map for max versions + const maxVersionMap = maxVersionsQuery.reduce( + (acc, item) => { + acc[item.secretId] = item.maxVersion; + return acc; + }, + {} as Record + ); + + // Update the version field with maxVersion when needed + return specificVersions.map((version) => { + // Replace version with maxVersion + return { + ...version, + version: maxVersionMap[version.secretId] || version.version + }; + }); + }; + const findByIdsWithLatestVersion = async ( folderId: string, secretIds: string[], @@ -281,67 +354,15 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { try { if (!secretIds.length && (!versionIds || !versionIds.length)) return {}; - const knexInstance = tx || db.replicaNode(); - let allDocs: Array = []; + const [latestVersions, specificVersionsWithLatest] = await Promise.all([ + secretIds.length ? getLatestVersionsBySecretIds(folderId, secretIds, tx) : [], + versionIds?.length ? getSpecificVersionsWithLatestInfo(folderId, versionIds, tx) : [] + ]); - // Process latest versions for secretIds - if (secretIds.length) { - const latestVersions: Array = await knexInstance(TableName.SecretVersionV2) - .where("folderId", folderId) - .whereIn(`${TableName.SecretVersionV2}.secretId`, secretIds) - .join( - knexInstance(TableName.SecretVersionV2) - .groupBy("secretId") - .max("version") - .select("secretId") - .as("latestVersion"), - (bd) => { - bd.on(`${TableName.SecretVersionV2}.secretId`, "latestVersion.secretId").andOn( - `${TableName.SecretVersionV2}.version`, - "latestVersion.max" - ); - } - ); + const allDocs = [...latestVersions, ...specificVersionsWithLatest]; - allDocs = [...allDocs, ...latestVersions]; - } - - // Process specific versions by versionIds - if (versionIds && versionIds.length) { - // Get the specific versions - const specificVersions = await knexInstance(TableName.SecretVersionV2) - .where("folderId", folderId) - .whereIn("id", versionIds); - - // Get the secretIds from these versions - const specificSecretIds = [...new Set(specificVersions.map((v) => v.secretId).filter(Boolean))]; - - // Get max versions for these secretIds - const maxVersionsQuery = (await knexInstance(TableName.SecretVersionV2) - .whereIn("secretId", specificSecretIds) - .groupBy("secretId") - .select("secretId") - .max("version as max")) as Array<{ secretId: string; max: number }>; - - // Create a lookup map for max versions - const maxVersionMap = maxVersionsQuery.reduce( - (acc, item) => { - acc[item.secretId] = item.max; - return acc; - }, - {} as Record - ); - - // Add max version to each specific version - const specificVersionsWithMax = specificVersions.map((version) => ({ - ...version, - max: maxVersionMap[version.secretId] - })); - - allDocs = [...allDocs, ...specificVersionsWithMax]; - } - - return allDocs.reduce>( + // Convert array to record with secretId as key + return allDocs.reduce>( (prev, curr) => ({ ...prev, [curr.secretId || ""]: curr }), {} ); diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 874ed8f90..136a2e228 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -2533,21 +2533,15 @@ export const secretServiceFactory = ({ secretVersions: string[]; folderId: string; }) => { - const secretVersionV2 = await secretV2BridgeService - .getSecretVersionsByIds({ - actorId, - actor, - actorOrgId, - actorAuthMethod, - secretId, - folderId, - secretVersionNumbers: secretVersions - }) - .catch((err) => { - if ((err as Error).message === "BadRequest: Failed to find secret") { - return null; - } - }); + const secretVersionV2 = await secretV2BridgeService.getSecretVersionsByIds({ + actorId, + actor, + actorOrgId, + actorAuthMethod, + secretId, + folderId, + secretVersionNumbers: secretVersions + }); return secretVersionV2; }; diff --git a/docs/internals/permissions/project-permissions.mdx b/docs/internals/permissions/project-permissions.mdx index acf95485b..0f5502688 100644 --- a/docs/internals/permissions/project-permissions.mdx +++ b/docs/internals/permissions/project-permissions.mdx @@ -176,6 +176,13 @@ Supports conditions and permission inversion | `read` | View secret versions and snapshots | | `create` | Roll back secrets to snapshots | +#### Subject: `commits` + +| Action | Description | +| -------- | ---------------------------------- | +| `read` | View commits and changes across folders | +| `perform-rollback` | Roll back commits changes and restore folders to previous state| + #### Subject: `secret-approval` | Action | Description | diff --git a/frontend/src/hooks/api/folderCommits/queries.tsx b/frontend/src/hooks/api/folderCommits/queries.tsx index 5914fcf15..a7b4a7a84 100644 --- a/frontend/src/hooks/api/folderCommits/queries.tsx +++ b/frontend/src/hooks/api/folderCommits/queries.tsx @@ -53,11 +53,12 @@ const fetchFolderCommitsCount = async ({ directory?: string; }) => { const res = await apiRequest.get<{ count: number; folderId: string }>( - `/api/v1/pit/commits/count/${workspaceId}`, + "/api/v1/pit/commits/count", { params: { environment, - path: directory + path: directory, + workspaceId } } ); @@ -69,10 +70,11 @@ const fetchFolderCommitHistory = async ( environment: string, directory: string ): Promise => { - const res = await apiRequest.get(`/api/v1/pit/commits/${workspaceId}`, { + const res = await apiRequest.get("/api/v1/pit/commits", { params: { environment, - path: directory + path: directory, + workspaceId } }); return res.data; @@ -80,7 +82,12 @@ const fetchFolderCommitHistory = async ( export const fetchCommitDetails = async (workspaceId: string, commitId: string) => { const { data } = await apiRequest.get( - `/api/v1/pit/commits/${workspaceId}/${commitId}/changes` + `/api/v1/pit/commits/${commitId}/changes`, + { + params: { + workspaceId + } + } ); return data; }; @@ -89,18 +96,19 @@ export const fetchRollbackPreview = async ( folderId: string, commitId: string, envId: string, - projectId: string, + workspaceId: string, deepRollback: boolean, secretPath: string ): Promise => { const { data } = await apiRequest.get( - `/api/v1/pit/commits/${projectId}/${commitId}/compare`, + `/api/v1/pit/commits/${commitId}/compare`, { params: { folderId, envId, deepRollback, - secretPath + secretPath, + workspaceId } } ); @@ -110,26 +118,30 @@ export const fetchRollbackPreview = async ( const fetchRollback = async ( folderId: string, commitId: string, - projectId: string, + workspaceId: string, deepRollback: boolean, message?: string, envId?: string ) => { const { data } = await apiRequest.post<{ success: boolean }>( - `/api/v1/pit/commits/${projectId}/${commitId}/rollback`, + `/api/v1/pit/commits/${commitId}/rollback`, { folderId, deepRollback, message, - envId + envId, + workspaceId } ); return data; }; -const fetchRevert = async (commitId: string, projectId: string) => { +const fetchRevert = async (commitId: string, workspaceId: string) => { const { data } = await apiRequest.post<{ success: boolean; message: string }>( - `/api/v1/pit/commits/${projectId}/${commitId}/revert` + `/api/v1/pit/commits/${commitId}/revert`, + { + workspaceId + } ); return data; }; diff --git a/frontend/src/hooks/api/folderCommits/types.ts b/frontend/src/hooks/api/folderCommits/types.ts index c70c62af1..e248277db 100644 --- a/frontend/src/hooks/api/folderCommits/types.ts +++ b/frontend/src/hooks/api/folderCommits/types.ts @@ -1,4 +1,4 @@ -import { SecretVersions } from "../types"; +import { CommitType, SecretVersions } from "../types"; export type CommitHistoryItem = { id: string; @@ -18,7 +18,7 @@ export type CommitHistoryItem = { export type TFolderCommitChanges = { id: string; folderCommitId: string; - changeType: "add" | "delete"; + changeType: CommitType; isUpdate: boolean; secretVersionId: string | null; folderVersionId: string | null; diff --git a/frontend/src/pages/project/RoleDetailsBySlugPage/components/ProjectRoleModifySection.utils.tsx b/frontend/src/pages/project/RoleDetailsBySlugPage/components/ProjectRoleModifySection.utils.tsx index ec11244a6..e96f140ad 100644 --- a/frontend/src/pages/project/RoleDetailsBySlugPage/components/ProjectRoleModifySection.utils.tsx +++ b/frontend/src/pages/project/RoleDetailsBySlugPage/components/ProjectRoleModifySection.utils.tsx @@ -1291,7 +1291,7 @@ const SecretsManagerPermissionSubjects = (enabled = false) => ({ [ProjectPermissionSub.Tags]: enabled, [ProjectPermissionSub.Webhooks]: enabled, [ProjectPermissionSub.IpAllowList]: enabled, - [ProjectPermissionSub.SecretRollback]: enabled, + [ProjectPermissionSub.SecretRollback]: false, [ProjectPermissionSub.SecretRotation]: enabled, [ProjectPermissionSub.ServiceTokens]: enabled, [ProjectPermissionSub.Commits]: enabled diff --git a/frontend/src/pages/secret-manager/CommitDetailsPage/components/RollbackPreviewTab/RollbackPreviewTab.tsx b/frontend/src/pages/secret-manager/CommitDetailsPage/components/RollbackPreviewTab/RollbackPreviewTab.tsx index 1a8893989..86aa70b97 100644 --- a/frontend/src/pages/secret-manager/CommitDetailsPage/components/RollbackPreviewTab/RollbackPreviewTab.tsx +++ b/frontend/src/pages/secret-manager/CommitDetailsPage/components/RollbackPreviewTab/RollbackPreviewTab.tsx @@ -213,38 +213,35 @@ export const RollbackPreviewTab = (): JSX.Element => { )} - + {deepRollback && nestedFolderChanges.length > 0 && ( <>
- - Affected Child Folders - + Affected Child Folders
- {(nestedFolderChanges.map((folder) => ( -
setSelectedFolderId(folder.folderId)} - > -
-
- - - {folder.folderPath || folder.folderName} - -
- {folder.changes.length > 0 && ( - - {folder.changes.length} - - )} + {nestedFolderChanges.map((folder) => ( +
setSelectedFolderId(folder.folderId)} + > +
+
+ + + {folder.folderPath || folder.folderName} +
+ {folder.changes.length > 0 && ( + + {folder.changes.length} + + )}
- )) - )} +
+ ))} )}
@@ -305,10 +302,10 @@ export const RollbackPreviewTab = (): JSX.Element => { description="This will restore all changes to how they appeared at the point in time of this commit. Any modifications made after this commit will be undone." /> -
- {renderSidebar()} - {renderMainContent()} -
+
+ {renderSidebar()} + {renderMainContent()} +
@@ -353,7 +350,13 @@ export const RollbackPreviewTab = (): JSX.Element => { }} colorSchema="primary" className="px-6 py-2" - isDisabled={message.length === 0 || !rollbackChangesNested || !rollbackChangesNested?.some((folder) => folder.changes.length > 0)} + isDisabled={ + message.length === 0 || + !rollbackChangesNested || + !rollbackChangesNested?.some((folder) => { + return folder.changes.length > 0; + }) + } > Restore