From 32a3e1d200720e2da0ff2270bf6d2f1e372f0458 Mon Sep 17 00:00:00 2001 From: carlosmonastyrski Date: Tue, 6 May 2025 08:11:50 -0300 Subject: [PATCH 1/9] commit --- backend/src/@types/knex.d.ts | 48 +++++++ .../20250505194916_add-pit-revamp-tables.ts | 130 +++++++++++++++++ .../db/schemas/folder-checkpoint-resources.ts | 23 +++ backend/src/db/schemas/folder-checkpoints.ts | 20 +++ .../src/db/schemas/folder-commit-changes.ts | 22 +++ backend/src/db/schemas/folder-commits.ts | 23 +++ .../folder-tree-checkpoint-resources.ts | 26 ++++ .../src/db/schemas/folder-tree-checkpoints.ts | 20 +++ backend/src/db/schemas/index.ts | 6 + backend/src/db/schemas/models.ts | 8 +- .../src/ee/services/license/license-fns.ts | 54 ++++---- .../folder-checkpoint-resources-dal.ts | 79 +++++++++++ .../folder-checkpoint-dal.ts | 86 ++++++++++++ .../folder-commit-changes-dal.ts | 80 +++++++++++ .../folder-commit/folder-commit-dal.ts | 56 ++++++++ .../folder-tree-checkpoint-resources-dal.ts | 131 ++++++++++++++++++ .../folder-tree-checkpoint-dal.ts | 111 +++++++++++++++ 17 files changed, 895 insertions(+), 28 deletions(-) create mode 100644 backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts create mode 100644 backend/src/db/schemas/folder-checkpoint-resources.ts create mode 100644 backend/src/db/schemas/folder-checkpoints.ts create mode 100644 backend/src/db/schemas/folder-commit-changes.ts create mode 100644 backend/src/db/schemas/folder-commits.ts create mode 100644 backend/src/db/schemas/folder-tree-checkpoint-resources.ts create mode 100644 backend/src/db/schemas/folder-tree-checkpoints.ts create mode 100644 backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts create mode 100644 backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts create mode 100644 backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts create mode 100644 backend/src/services/folder-commit/folder-commit-dal.ts create mode 100644 backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts create mode 100644 backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts diff --git a/backend/src/@types/knex.d.ts b/backend/src/@types/knex.d.ts index 13f3bc306..928fddc36 100644 --- a/backend/src/@types/knex.d.ts +++ b/backend/src/@types/knex.d.ts @@ -74,6 +74,24 @@ import { TExternalKms, TExternalKmsInsert, TExternalKmsUpdate, + TFolderCheckpointResources, + TFolderCheckpointResourcesInsert, + TFolderCheckpointResourcesUpdate, + TFolderCheckpoints, + TFolderCheckpointsInsert, + TFolderCheckpointsUpdate, + TFolderCommitChanges, + TFolderCommitChangesInsert, + TFolderCommitChangesUpdate, + TFolderCommits, + TFolderCommitsInsert, + TFolderCommitsUpdate, + TFolderTreeCheckpointResources, + TFolderTreeCheckpointResourcesInsert, + TFolderTreeCheckpointResourcesUpdate, + TFolderTreeCheckpoints, + TFolderTreeCheckpointsInsert, + TFolderTreeCheckpointsUpdate, TGateways, TGatewaysInsert, TGatewaysUpdate, @@ -1048,5 +1066,35 @@ declare module "knex/types/tables" { TGithubOrgSyncConfigsInsert, TGithubOrgSyncConfigsUpdate >; + [TableName.FolderCommit]: KnexOriginal.CompositeTableType< + TFolderCommits, + TFolderCommitsInsert, + TFolderCommitsUpdate + >; + [TableName.FolderCommitChanges]: KnexOriginal.CompositeTableType< + TFolderCommitChanges, + TFolderCommitChangesInsert, + TFolderCommitChangesUpdate + >; + [TableName.FolderCheckpoint]: KnexOriginal.CompositeTableType< + TFolderCheckpoints, + TFolderCheckpointsInsert, + TFolderCheckpointsUpdate + >; + [TableName.FolderCheckpointResources]: KnexOriginal.CompositeTableType< + TFolderCheckpointResources, + TFolderCheckpointResourcesInsert, + TFolderCheckpointResourcesUpdate + >; + [TableName.FolderTreeCheckpoint]: KnexOriginal.CompositeTableType< + TFolderTreeCheckpoints, + TFolderTreeCheckpointsInsert, + TFolderTreeCheckpointsUpdate + >; + [TableName.FolderTreeCheckpointResources]: KnexOriginal.CompositeTableType< + TFolderTreeCheckpointResources, + TFolderTreeCheckpointResourcesInsert, + TFolderTreeCheckpointResourcesUpdate + >; } } diff --git a/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts b/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts new file mode 100644 index 000000000..f86c800a1 --- /dev/null +++ b/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts @@ -0,0 +1,130 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; +import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils"; + +export async function up(knex: Knex): Promise { + const hasFolderCommitTable = await knex.schema.hasTable(TableName.FolderCommit); + if (!hasFolderCommitTable) { + await knex.schema.createTable(TableName.FolderCommit, (t) => { + t.bigIncrements("id").primary(); + t.string("actorName").notNullable(); + t.string("actorType").notNullable(); + t.string("message"); + t.timestamp("date").notNullable().defaultTo(knex.fn.now()); + t.uuid("folderId").notNullable(); + t.foreign("folderId").references("id").inTable(TableName.SecretFolder).onDelete("CASCADE"); + t.timestamps(true, true, true); + + t.index("folderId"); + }); + } + + const hasFolderCommitChangesTable = await knex.schema.hasTable(TableName.FolderCommitChanges); + if (!hasFolderCommitChangesTable) { + await knex.schema.createTable(TableName.FolderCommitChanges, (t) => { + t.uuid("id").primary().defaultTo(knex.fn.uuid()); + t.bigInteger("folderCommitId").notNullable(); + t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE"); + t.string("changeType").notNullable(); + t.uuid("secretVersionId"); + t.foreign("secretVersionId").references("id").inTable(TableName.SecretVersionV2).onDelete("CASCADE"); + t.uuid("folderVersionId"); + t.foreign("folderVersionId").references("id").inTable(TableName.SecretFolderVersion).onDelete("CASCADE"); + t.timestamps(true, true, true); + + t.index("folderCommitId"); + t.index("secretVersionId"); + t.index("folderVersionId"); + }); + } + + const hasFolderCheckpointTable = await knex.schema.hasTable(TableName.FolderCheckpoint); + if (!hasFolderCheckpointTable) { + await knex.schema.createTable(TableName.FolderCheckpoint, (t) => { + t.uuid("id").primary().defaultTo(knex.fn.uuid()); + t.bigInteger("folderCommitId").notNullable(); + t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE"); + t.timestamp("date").notNullable().defaultTo(knex.fn.now()); + t.timestamps(true, true, true); + + t.index("folderCommitId"); + }); + } + + const hasFolderCheckpointResourcesTable = await knex.schema.hasTable(TableName.FolderCheckpointResources); + if (!hasFolderCheckpointResourcesTable) { + await knex.schema.createTable(TableName.FolderCheckpointResources, (t) => { + t.uuid("id").primary().defaultTo(knex.fn.uuid()); + t.uuid("folderCheckpointId").notNullable(); + t.foreign("folderCheckpointId").references("id").inTable(TableName.FolderCheckpoint).onDelete("CASCADE"); + t.uuid("secretVersionId"); + t.foreign("secretVersionId").references("id").inTable(TableName.SecretVersionV2).onDelete("CASCADE"); + t.uuid("folderVersionId"); + t.foreign("folderVersionId").references("id").inTable(TableName.SecretFolderVersion).onDelete("CASCADE"); + t.timestamps(true, true, true); + + t.index("folderCheckpointId"); + t.index("secretVersionId"); + t.index("folderVersionId"); + }); + } + + const hasFolderTreeCheckpointTable = await knex.schema.hasTable(TableName.FolderTreeCheckpoint); + if (!hasFolderTreeCheckpointTable) { + await knex.schema.createTable(TableName.FolderTreeCheckpoint, (t) => { + t.uuid("id").primary().defaultTo(knex.fn.uuid()); + t.bigInteger("folderCommitId").notNullable(); + t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE"); + t.timestamp("date").notNullable().defaultTo(knex.fn.now()); + t.timestamps(true, true, true); + + t.index("folderCommitId"); + }); + } + + const hasFolderTreeCheckpointResourcesTable = await knex.schema.hasTable(TableName.FolderTreeCheckpointResources); + if (!hasFolderTreeCheckpointResourcesTable) { + await knex.schema.createTable(TableName.FolderTreeCheckpointResources, (t) => { + t.uuid("id").primary().defaultTo(knex.fn.uuid()); + t.uuid("folderTreeCheckpointId").notNullable(); + t.foreign("folderTreeCheckpointId").references("id").inTable(TableName.FolderTreeCheckpoint).onDelete("CASCADE"); + t.uuid("folderId").notNullable(); + t.foreign("folderId").references("id").inTable(TableName.SecretFolder).onDelete("CASCADE"); + t.bigInteger("folderCommitId").notNullable(); + t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE"); + t.timestamps(true, true, true); + + t.index("folderTreeCheckpointId"); + t.index("folderId"); + t.index("folderCommitId"); + }); + } + + await createOnUpdateTrigger(knex, TableName.FolderCommit); + await createOnUpdateTrigger(knex, TableName.FolderCommitChanges); + await createOnUpdateTrigger(knex, TableName.FolderCheckpoint); + await createOnUpdateTrigger(knex, TableName.FolderCheckpointResources); + await createOnUpdateTrigger(knex, TableName.FolderTreeCheckpoint); + await createOnUpdateTrigger(knex, TableName.FolderTreeCheckpointResources); +} + +export async function down(knex: Knex): Promise { + await dropOnUpdateTrigger(knex, TableName.FolderTreeCheckpointResources); + await knex.schema.dropTableIfExists(TableName.FolderTreeCheckpointResources); + + await dropOnUpdateTrigger(knex, TableName.FolderTreeCheckpoint); + await knex.schema.dropTableIfExists(TableName.FolderTreeCheckpoint); + + await dropOnUpdateTrigger(knex, TableName.FolderCheckpointResources); + await knex.schema.dropTableIfExists(TableName.FolderCheckpointResources); + + await dropOnUpdateTrigger(knex, TableName.FolderCheckpoint); + await knex.schema.dropTableIfExists(TableName.FolderCheckpoint); + + await dropOnUpdateTrigger(knex, TableName.FolderCommitChanges); + await knex.schema.dropTableIfExists(TableName.FolderCommitChanges); + + await dropOnUpdateTrigger(knex, TableName.FolderCommit); + await knex.schema.dropTableIfExists(TableName.FolderCommit); +} diff --git a/backend/src/db/schemas/folder-checkpoint-resources.ts b/backend/src/db/schemas/folder-checkpoint-resources.ts new file mode 100644 index 000000000..5fa8215cf --- /dev/null +++ b/backend/src/db/schemas/folder-checkpoint-resources.ts @@ -0,0 +1,23 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const FolderCheckpointResourcesSchema = z.object({ + id: z.string().uuid(), + folderCheckpointId: z.string().uuid(), + secretVersionId: z.string().uuid().nullable().optional(), + folderVersionId: z.string().uuid().nullable().optional(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TFolderCheckpointResources = z.infer; +export type TFolderCheckpointResourcesInsert = Omit, TImmutableDBKeys>; +export type TFolderCheckpointResourcesUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/folder-checkpoints.ts b/backend/src/db/schemas/folder-checkpoints.ts new file mode 100644 index 000000000..fd7ccae84 --- /dev/null +++ b/backend/src/db/schemas/folder-checkpoints.ts @@ -0,0 +1,20 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const FolderCheckpointsSchema = z.object({ + id: z.string().uuid(), + folderCommitId: z.coerce.number(), + date: z.date(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TFolderCheckpoints = z.infer; +export type TFolderCheckpointsInsert = Omit, TImmutableDBKeys>; +export type TFolderCheckpointsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/folder-commit-changes.ts b/backend/src/db/schemas/folder-commit-changes.ts new file mode 100644 index 000000000..9ee228595 --- /dev/null +++ b/backend/src/db/schemas/folder-commit-changes.ts @@ -0,0 +1,22 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const FolderCommitChangesSchema = z.object({ + id: z.string().uuid(), + folderCommitId: z.coerce.number(), + changeType: z.string(), + secretVersionId: z.string().uuid().nullable().optional(), + folderVersionId: z.string().uuid().nullable().optional(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TFolderCommitChanges = z.infer; +export type TFolderCommitChangesInsert = Omit, TImmutableDBKeys>; +export type TFolderCommitChangesUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/folder-commits.ts b/backend/src/db/schemas/folder-commits.ts new file mode 100644 index 000000000..4d5f2d2af --- /dev/null +++ b/backend/src/db/schemas/folder-commits.ts @@ -0,0 +1,23 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const FolderCommitsSchema = z.object({ + id: z.coerce.number(), + actorName: z.string(), + actorType: z.string(), + message: z.string().nullable().optional(), + date: z.date(), + folderId: z.string().uuid(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TFolderCommits = z.infer; +export type TFolderCommitsInsert = Omit, TImmutableDBKeys>; +export type TFolderCommitsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/folder-tree-checkpoint-resources.ts b/backend/src/db/schemas/folder-tree-checkpoint-resources.ts new file mode 100644 index 000000000..1c65e833d --- /dev/null +++ b/backend/src/db/schemas/folder-tree-checkpoint-resources.ts @@ -0,0 +1,26 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const FolderTreeCheckpointResourcesSchema = z.object({ + id: z.string().uuid(), + folderTreeCheckpointId: z.string().uuid(), + folderId: z.string().uuid(), + folderCommitId: z.coerce.number(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TFolderTreeCheckpointResources = z.infer; +export type TFolderTreeCheckpointResourcesInsert = Omit< + z.input, + TImmutableDBKeys +>; +export type TFolderTreeCheckpointResourcesUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/folder-tree-checkpoints.ts b/backend/src/db/schemas/folder-tree-checkpoints.ts new file mode 100644 index 000000000..8f637b49b --- /dev/null +++ b/backend/src/db/schemas/folder-tree-checkpoints.ts @@ -0,0 +1,20 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const FolderTreeCheckpointsSchema = z.object({ + id: z.string().uuid(), + folderCommitId: z.coerce.number(), + date: z.date(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TFolderTreeCheckpoints = z.infer; +export type TFolderTreeCheckpointsInsert = Omit, TImmutableDBKeys>; +export type TFolderTreeCheckpointsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/index.ts b/backend/src/db/schemas/index.ts index b71d51908..5755697cc 100644 --- a/backend/src/db/schemas/index.ts +++ b/backend/src/db/schemas/index.ts @@ -22,6 +22,12 @@ export * from "./dynamic-secret-leases"; export * from "./dynamic-secrets"; export * from "./external-group-org-role-mappings"; export * from "./external-kms"; +export * from "./folder-checkpoint-resources"; +export * from "./folder-checkpoints"; +export * from "./folder-commit-changes"; +export * from "./folder-commits"; +export * from "./folder-tree-checkpoint-resources"; +export * from "./folder-tree-checkpoints"; export * from "./gateways"; export * from "./git-app-install-sessions"; export * from "./git-app-org"; diff --git a/backend/src/db/schemas/models.ts b/backend/src/db/schemas/models.ts index 7fd77da6c..0b80be80f 100644 --- a/backend/src/db/schemas/models.ts +++ b/backend/src/db/schemas/models.ts @@ -152,7 +152,13 @@ export enum TableName { MicrosoftTeamsIntegrations = "microsoft_teams_integrations", ProjectMicrosoftTeamsConfigs = "project_microsoft_teams_configs", SecretReminderRecipients = "secret_reminder_recipients", - GithubOrgSyncConfig = "github_org_sync_configs" + GithubOrgSyncConfig = "github_org_sync_configs", + FolderCommit = "folder_commits", + FolderCommitChanges = "folder_commit_changes", + FolderCheckpoint = "folder_checkpoints", + FolderCheckpointResources = "folder_checkpoint_resources", + FolderTreeCheckpoint = "folder_tree_checkpoints", + FolderTreeCheckpointResources = "folder_tree_checkpoint_resources" } export type TImmutableDBKeys = "id" | "createdAt" | "updatedAt"; diff --git a/backend/src/ee/services/license/license-fns.ts b/backend/src/ee/services/license/license-fns.ts index b7ae6f7ee..cff6ee181 100644 --- a/backend/src/ee/services/license/license-fns.ts +++ b/backend/src/ee/services/license/license-fns.ts @@ -17,44 +17,44 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({ environmentsUsed: 0, identityLimit: null, identitiesUsed: 0, - dynamicSecret: false, + dynamicSecret: true, secretVersioning: true, - pitRecovery: false, - ipAllowlisting: false, - rbac: false, - githubOrgSync: false, - customRateLimits: false, - customAlerts: false, - secretAccessInsights: false, - auditLogs: false, + pitRecovery: true, + ipAllowlisting: true, + rbac: true, + githubOrgSync: true, + customRateLimits: true, + customAlerts: true, + secretAccessInsights: true, + auditLogs: true, auditLogsRetentionDays: 0, - auditLogStreams: false, + auditLogStreams: true, auditLogStreamLimit: 3, - samlSSO: false, - hsm: false, - oidcSSO: false, - scim: false, - ldap: false, - groups: false, + samlSSO: true, + hsm: true, + oidcSSO: true, + scim: true, + ldap: true, + groups: true, status: null, trial_end: null, has_used_trial: true, - secretApproval: false, - secretRotation: false, - caCrl: false, - instanceUserManagement: false, - externalKms: false, + secretApproval: true, + secretRotation: true, + caCrl: true, + instanceUserManagement: true, + externalKms: true, rateLimits: { readLimit: 60, writeLimit: 200, secretsLimit: 40 }, - pkiEst: false, - enforceMfa: false, - projectTemplates: false, - kmip: false, - gateway: false, - sshHostGroups: false + pkiEst: true, + enforceMfa: true, + projectTemplates: true, + kmip: true, + gateway: true, + sshHostGroups: true }); export const setupLicenseRequestWithStore = (baseURL: string, refreshUrl: string, licenseKey: string) => { diff --git a/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts b/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts new file mode 100644 index 000000000..d0b2200ec --- /dev/null +++ b/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts @@ -0,0 +1,79 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { TableName, TFolderCheckpointResources, TFolderCheckpoints } from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TFolderCheckpointResourcesDALFactory = ReturnType; + +type ResourceWithCheckpointInfo = TFolderCheckpointResources & { + folderCommitId: string; + date: Date; +}; + +export const folderCheckpointResourcesDALFactory = (db: TDbClient) => { + const folderCheckpointResourcesOrm = ormify(db, TableName.FolderCheckpointResources); + + const findByCheckpointId = async (folderCheckpointId: string, tx?: Knex): Promise => { + try { + const docs = await (tx || db.replicaNode())(TableName.FolderCheckpointResources) + .where({ folderCheckpointId }) + .select(selectAllTableCols(TableName.FolderCheckpointResources)); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindByCheckpointId" }); + } + }; + + const findBySecretVersionId = async (secretVersionId: string, tx?: Knex): Promise => { + try { + const docs = await (tx || db.replicaNode())< + TFolderCheckpointResources & Pick + >(TableName.FolderCheckpointResources) + .where({ secretVersionId }) + .select(selectAllTableCols(TableName.FolderCheckpointResources)) + .join( + TableName.FolderCheckpoint, + `${TableName.FolderCheckpointResources}.folderCheckpointId`, + `${TableName.FolderCheckpoint}.id` + ) + .select( + db.ref("folderCommitId").withSchema(TableName.FolderCheckpoint), + db.ref("date").withSchema(TableName.FolderCheckpoint) + ); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindBySecretVersionId" }); + } + }; + + const findByFolderVersionId = async (folderVersionId: string, tx?: Knex): Promise => { + try { + const docs = await (tx || db.replicaNode())< + TFolderCheckpointResources & Pick + >(TableName.FolderCheckpointResources) + .where({ folderVersionId }) + .select(selectAllTableCols(TableName.FolderCheckpointResources)) + .join( + TableName.FolderCheckpoint, + `${TableName.FolderCheckpointResources}.folderCheckpointId`, + `${TableName.FolderCheckpoint}.id` + ) + .select( + db.ref("folderCommitId").withSchema(TableName.FolderCheckpoint), + db.ref("date").withSchema(TableName.FolderCheckpoint) + ); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindByFolderVersionId" }); + } + }; + + return { + ...folderCheckpointResourcesOrm, + findByCheckpointId, + findBySecretVersionId, + findByFolderVersionId + }; +}; diff --git a/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts b/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts new file mode 100644 index 000000000..c047c0504 --- /dev/null +++ b/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts @@ -0,0 +1,86 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { TableName, TFolderCheckpoints } from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TFolderCheckpointDALFactory = ReturnType; + +type CheckpointWithCommitInfo = TFolderCheckpoints & { + actorName: string; + actorType: string; + message: string | null; + commitDate: Date; + folderId: string; +}; + +export const folderCheckpointDALFactory = (db: TDbClient) => { + const folderCheckpointOrm = ormify(db, TableName.FolderCheckpoint); + + const findByCommitId = async (folderCommitId: string, tx?: Knex): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCheckpoint) + .where({ folderCommitId }) + .select(selectAllTableCols(TableName.FolderCheckpoint)) + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindByCommitId" }); + } + }; + + const findByFolderId = async (folderId: string, limit?: number, tx?: Knex): Promise => { + try { + let query = (tx || db.replicaNode())(TableName.FolderCheckpoint) + .join(TableName.FolderCommit, `${TableName.FolderCheckpoint}.folderCommitId`, `${TableName.FolderCommit}.id`) + .where({ folderId }) + .select(selectAllTableCols(TableName.FolderCheckpoint)) + .select( + db.ref("actorName").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("date").withSchema(TableName.FolderCommit).as("commitDate"), + db.ref("folderId").withSchema(TableName.FolderCommit) + ) + .orderBy(`${TableName.FolderCheckpoint}.date`, "desc"); + + if (limit !== undefined) { + query = query.limit(limit); + } + + const docs = await query; + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindByFolderId" }); + } + }; + + const findLatestByFolderId = async (folderId: string, tx?: Knex): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCheckpoint) + .join(TableName.FolderCommit, `${TableName.FolderCheckpoint}.folderCommitId`, `${TableName.FolderCommit}.id`) + .where({ folderId }) + .select(selectAllTableCols(TableName.FolderCheckpoint)) + .select( + db.ref("actorName").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("date").withSchema(TableName.FolderCommit).as("commitDate"), + db.ref("folderId").withSchema(TableName.FolderCommit) + ) + .orderBy(`${TableName.FolderCheckpoint}.date`, "desc") + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestByFolderId" }); + } + }; + + return { + ...folderCheckpointOrm, + findByCommitId, + findByFolderId, + findLatestByFolderId + }; +}; diff --git a/backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts b/backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts new file mode 100644 index 000000000..46905bf95 --- /dev/null +++ b/backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts @@ -0,0 +1,80 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { TableName, TFolderCommitChanges, TFolderCommits } from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TFolderCommitChangesDALFactory = ReturnType; + +type CommitChangeWithCommitInfo = TFolderCommitChanges & { + actorName: string; + actorType: string; + message: string | null; + date: Date; + folderId: string; +}; + +export const folderCommitChangesDALFactory = (db: TDbClient) => { + const folderCommitChangesOrm = ormify(db, TableName.FolderCommitChanges); + + const findByCommitId = async (folderCommitId: string, tx?: Knex): Promise => { + try { + const docs = await (tx || db.replicaNode())(TableName.FolderCommitChanges) + .where({ folderCommitId }) + .select(selectAllTableCols(TableName.FolderCommitChanges)); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindByCommitId" }); + } + }; + + const findBySecretVersionId = async (secretVersionId: string, tx?: Knex): Promise => { + try { + const docs = await (tx || db.replicaNode())< + TFolderCommitChanges & Pick + >(TableName.FolderCommitChanges) + .where({ secretVersionId }) + .select(selectAllTableCols(TableName.FolderCommitChanges)) + .join(TableName.FolderCommit, `${TableName.FolderCommitChanges}.folderCommitId`, `${TableName.FolderCommit}.id`) + .select( + db.ref("actorName").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("date").withSchema(TableName.FolderCommit), + db.ref("folderId").withSchema(TableName.FolderCommit) + ); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindBySecretVersionId" }); + } + }; + + const findByFolderVersionId = async (folderVersionId: string, tx?: Knex): Promise => { + try { + const docs = await (tx || db.replicaNode())< + TFolderCommitChanges & Pick + >(TableName.FolderCommitChanges) + .where({ folderVersionId }) + .select(selectAllTableCols(TableName.FolderCommitChanges)) + .join(TableName.FolderCommit, `${TableName.FolderCommitChanges}.folderCommitId`, `${TableName.FolderCommit}.id`) + .select( + db.ref("actorName").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("date").withSchema(TableName.FolderCommit), + db.ref("folderId").withSchema(TableName.FolderCommit) + ); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindByFolderVersionId" }); + } + }; + + return { + ...folderCommitChangesOrm, + findByCommitId, + findBySecretVersionId, + findByFolderVersionId + }; +}; diff --git a/backend/src/services/folder-commit/folder-commit-dal.ts b/backend/src/services/folder-commit/folder-commit-dal.ts new file mode 100644 index 000000000..2ad973bcd --- /dev/null +++ b/backend/src/services/folder-commit/folder-commit-dal.ts @@ -0,0 +1,56 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { TableName, TFolderCommits } from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TFolderCommitDALFactory = ReturnType; + +export const folderCommitDALFactory = (db: TDbClient) => { + const folderCommitOrm = ormify(db, TableName.FolderCommit); + + const findByFolderId = async (folderId: string, tx?: Knex): Promise => { + try { + const docs = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ folderId }) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("date", "desc"); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindByFolderId" }); + } + }; + + const findById = async (id: string, tx?: Knex): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ id }) + .select(selectAllTableCols(TableName.FolderCommit)) + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindById" }); + } + }; + + const findLatestCommit = async (folderId: string, tx?: Knex): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ folderId }) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("date", "desc") + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestCommit" }); + } + }; + + return { + ...folderCommitOrm, + findByFolderId, + findById, + findLatestCommit + }; +}; diff --git a/backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts b/backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts new file mode 100644 index 000000000..7b4e0bffb --- /dev/null +++ b/backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts @@ -0,0 +1,131 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { + TableName, + TEnvironments, + TFolderTreeCheckpointResources, + TFolderTreeCheckpoints, + TSecretFolders +} from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TFolderTreeCheckpointResourcesDALFactory = ReturnType; + +type ResourceWithCheckpointInfo = TFolderTreeCheckpointResources & { + folderCommitId: string; + date: Date; +}; + +type ResourceWithFolderInfo = TFolderTreeCheckpointResources & { + name: string; + parentId: string | null; + slug: string; + envName: string; +}; + +export const folderTreeCheckpointResourcesDALFactory = (db: TDbClient) => { + const folderTreeCheckpointResourcesOrm = ormify( + db, + TableName.FolderTreeCheckpointResources + ); + + const findByTreeCheckpointId = async ( + folderTreeCheckpointId: string, + tx?: Knex + ): Promise => { + try { + const docs = await (tx || db.replicaNode())( + TableName.FolderTreeCheckpointResources + ) + .where({ folderTreeCheckpointId }) + .select(selectAllTableCols(TableName.FolderTreeCheckpointResources)); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindByTreeCheckpointId" }); + } + }; + + const findByFolderId = async (folderId: string, tx?: Knex): Promise => { + try { + const docs = await (tx || db.replicaNode())< + TFolderTreeCheckpointResources & Pick + >(TableName.FolderTreeCheckpointResources) + .where({ folderId }) + .select(selectAllTableCols(TableName.FolderTreeCheckpointResources)) + .join( + TableName.FolderTreeCheckpoint, + `${TableName.FolderTreeCheckpointResources}.folderTreeCheckpointId`, + `${TableName.FolderTreeCheckpoint}.id` + ) + .select( + db.ref("folderCommitId").withSchema(TableName.FolderTreeCheckpoint), + db.ref("date").withSchema(TableName.FolderTreeCheckpoint) + ); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindByFolderId" }); + } + }; + + const findByFolderCommitId = async ( + folderCommitId: string, + tx?: Knex + ): Promise<(TFolderTreeCheckpointResources & { date: Date })[]> => { + try { + const docs = await (tx || db.replicaNode())< + TFolderTreeCheckpointResources & Pick + >(TableName.FolderTreeCheckpointResources) + .where({ folderCommitId }) + .select(selectAllTableCols(TableName.FolderTreeCheckpointResources)) + .join( + TableName.FolderTreeCheckpoint, + `${TableName.FolderTreeCheckpointResources}.folderTreeCheckpointId`, + `${TableName.FolderTreeCheckpoint}.id` + ) + .select(db.ref("date").withSchema(TableName.FolderTreeCheckpoint)); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindByFolderCommitId" }); + } + }; + + const findFoldersInTreeCheckpoint = async ( + folderTreeCheckpointId: string, + tx?: Knex + ): Promise => { + try { + const docs = await (tx || db.replicaNode())< + TFolderTreeCheckpointResources & + Pick & + Pick & { envName: string } + >(TableName.FolderTreeCheckpointResources) + .where({ folderTreeCheckpointId }) + .select(selectAllTableCols(TableName.FolderTreeCheckpointResources)) + .join( + TableName.SecretFolder, + `${TableName.FolderTreeCheckpointResources}.folderId`, + `${TableName.SecretFolder}.id` + ) + .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) + .select( + db.ref("name").withSchema(TableName.SecretFolder), + db.ref("parentId").withSchema(TableName.SecretFolder), + db.ref("slug").withSchema(TableName.Environment), + db.ref("name").withSchema(TableName.Environment).as("envName") + ); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindFoldersInTreeCheckpoint" }); + } + }; + + return { + ...folderTreeCheckpointResourcesOrm, + findByTreeCheckpointId, + findByFolderId, + findByFolderCommitId, + findFoldersInTreeCheckpoint + }; +}; diff --git a/backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts b/backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts new file mode 100644 index 000000000..3377ca643 --- /dev/null +++ b/backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts @@ -0,0 +1,111 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { TableName, TFolderCommits, TFolderTreeCheckpoints } from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TFolderTreeCheckpointDALFactory = ReturnType; + +type TreeCheckpointWithCommitInfo = TFolderTreeCheckpoints & { + actorName: string; + actorType: string; + message: string | null; + commitDate: Date; + folderId: string; +}; + +export const folderTreeCheckpointDALFactory = (db: TDbClient) => { + const folderTreeCheckpointOrm = ormify(db, TableName.FolderTreeCheckpoint); + + const findByCommitId = async (folderCommitId: string, tx?: Knex): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderTreeCheckpoint) + .where({ folderCommitId }) + .select(selectAllTableCols(TableName.FolderTreeCheckpoint)) + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindByCommitId" }); + } + }; + + const findByProjectId = async ( + projectId: string, + limit?: number, + tx?: Knex + ): Promise => { + try { + const query = (tx || db.replicaNode())< + TFolderTreeCheckpoints & + Pick & { commitDate: Date } + >(TableName.FolderTreeCheckpoint) + .join( + TableName.FolderCommit, + `${TableName.FolderTreeCheckpoint}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) + .join(TableName.SecretFolder, `${TableName.FolderCommit}.folderId`, `${TableName.SecretFolder}.id`) + .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) + .where({ projectId }) + .select(selectAllTableCols(TableName.FolderTreeCheckpoint)) + .select( + db.ref("actorName").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("date").withSchema(TableName.FolderCommit).as("commitDate"), + db.ref("folderId").withSchema(TableName.FolderCommit) + ) + .orderBy(`${TableName.FolderTreeCheckpoint}.date`, "desc"); + + if (limit) { + void query.limit(limit); + } + + const docs = await query; + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindByProjectId" }); + } + }; + + const findLatestByProjectId = async ( + projectId: string, + tx?: Knex + ): Promise => { + try { + const doc = await (tx || db.replicaNode())< + TFolderTreeCheckpoints & + Pick & { commitDate: Date } + >(TableName.FolderTreeCheckpoint) + .join( + TableName.FolderCommit, + `${TableName.FolderTreeCheckpoint}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) + .join(TableName.SecretFolder, `${TableName.FolderCommit}.folderId`, `${TableName.SecretFolder}.id`) + .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) + .where({ projectId }) + .select(selectAllTableCols(TableName.FolderTreeCheckpoint)) + .select( + db.ref("actorName").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("date").withSchema(TableName.FolderCommit).as("commitDate"), + db.ref("folderId").withSchema(TableName.FolderCommit) + ) + .orderBy(`${TableName.FolderTreeCheckpoint}.date`, "desc") + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestByProjectId" }); + } + }; + + return { + ...folderTreeCheckpointOrm, + findByCommitId, + findByProjectId, + findLatestByProjectId + }; +}; From f493a617b107f9454ec87938c5be98d8febae90b Mon Sep 17 00:00:00 2001 From: carlosmonastyrski Date: Tue, 6 May 2025 18:57:25 -0300 Subject: [PATCH 2/9] Add new commit logic on every folder/secret operation --- backend/src/@types/fastify.d.ts | 2 + .../20250505194916_add-pit-revamp-tables.ts | 64 ++++--- backend/src/db/schemas/folder-checkpoints.ts | 3 +- .../src/db/schemas/folder-commit-changes.ts | 2 +- backend/src/db/schemas/folder-commits.ts | 6 +- .../folder-tree-checkpoint-resources.ts | 2 +- .../src/db/schemas/folder-tree-checkpoints.ts | 3 +- backend/src/db/schemas/models.ts | 2 +- .../src/ee/services/license/license-fns.ts | 54 +++--- .../secret-approval-request-service.ts | 26 ++- .../secret-replication-service.ts | 5 + .../secret-rotation-v2-service.ts | 23 ++- .../secret-rotation-queue.ts | 20 ++- backend/src/server/routes/index.ts | 39 +++- .../external-migration-fns.ts | 37 ++++ .../external-migration-queue.ts | 8 + .../folder-checkpoint-resources-dal.ts | 11 +- .../folder-checkpoint-dal.ts | 18 +- .../folder-commit-changes-dal.ts | 21 +-- .../folder-commit/folder-commit-dal.ts | 5 +- .../folder-commit/folder-commit-service.ts | 168 ++++++++++++++++++ .../folder-tree-checkpoint-resources-dal.ts | 25 +-- .../folder-tree-checkpoint-dal.ts | 22 +-- .../secret-folder/secret-folder-service.ts | 110 +++++++++++- .../secret-folder-version-dal.ts | 2 +- .../services/secret-sync/secret-sync-queue.ts | 11 +- .../secret-v2-bridge/secret-v2-bridge-fns.ts | 84 ++++++++- .../secret-v2-bridge-service.ts | 16 ++ .../secret-v2-bridge-types.ts | 6 + backend/src/services/secret/secret-fns.ts | 6 +- backend/src/services/secret/secret-queue.ts | 11 +- backend/src/services/secret/secret-types.ts | 3 + 32 files changed, 675 insertions(+), 140 deletions(-) create mode 100644 backend/src/services/folder-commit/folder-commit-service.ts diff --git a/backend/src/@types/fastify.d.ts b/backend/src/@types/fastify.d.ts index 6ec542c6b..f83555c50 100644 --- a/backend/src/@types/fastify.d.ts +++ b/backend/src/@types/fastify.d.ts @@ -57,6 +57,7 @@ import { TCertificateTemplateServiceFactory } from "@app/services/certificate-te import { TCmekServiceFactory } from "@app/services/cmek/cmek-service"; import { TExternalGroupOrgRoleMappingServiceFactory } from "@app/services/external-group-org-role-mapping/external-group-org-role-mapping-service"; import { TExternalMigrationServiceFactory } from "@app/services/external-migration/external-migration-service"; +import { TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { TGroupProjectServiceFactory } from "@app/services/group-project/group-project-service"; import { THsmServiceFactory } from "@app/services/hsm/hsm-service"; import { TIdentityServiceFactory } from "@app/services/identity/identity-service"; @@ -252,6 +253,7 @@ declare module "fastify" { microsoftTeams: TMicrosoftTeamsServiceFactory; assumePrivileges: TAssumePrivilegeServiceFactory; githubOrgSync: TGithubOrgSyncServiceFactory; + folderCommit: TFolderCommitServiceFactory; }; // this is exclusive use for middlewares in which we need to inject data // everywhere else access using service layer diff --git a/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts b/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts index f86c800a1..ea4c8e9bb 100644 --- a/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts +++ b/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts @@ -7,11 +7,11 @@ export async function up(knex: Knex): Promise { const hasFolderCommitTable = await knex.schema.hasTable(TableName.FolderCommit); if (!hasFolderCommitTable) { await knex.schema.createTable(TableName.FolderCommit, (t) => { - t.bigIncrements("id").primary(); - t.string("actorName").notNullable(); + t.uuid("id").primary().defaultTo(knex.fn.uuid()); + t.bigIncrements("commitId"); + t.jsonb("actorMetadata").notNullable(); t.string("actorType").notNullable(); t.string("message"); - t.timestamp("date").notNullable().defaultTo(knex.fn.now()); t.uuid("folderId").notNullable(); t.foreign("folderId").references("id").inTable(TableName.SecretFolder).onDelete("CASCADE"); t.timestamps(true, true, true); @@ -24,7 +24,7 @@ export async function up(knex: Knex): Promise { if (!hasFolderCommitChangesTable) { await knex.schema.createTable(TableName.FolderCommitChanges, (t) => { t.uuid("id").primary().defaultTo(knex.fn.uuid()); - t.bigInteger("folderCommitId").notNullable(); + t.uuid("folderCommitId").notNullable(); t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE"); t.string("changeType").notNullable(); t.uuid("secretVersionId"); @@ -43,9 +43,8 @@ export async function up(knex: Knex): Promise { if (!hasFolderCheckpointTable) { await knex.schema.createTable(TableName.FolderCheckpoint, (t) => { t.uuid("id").primary().defaultTo(knex.fn.uuid()); - t.bigInteger("folderCommitId").notNullable(); + t.uuid("folderCommitId").notNullable(); t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE"); - t.timestamp("date").notNullable().defaultTo(knex.fn.now()); t.timestamps(true, true, true); t.index("folderCommitId"); @@ -74,9 +73,8 @@ export async function up(knex: Knex): Promise { if (!hasFolderTreeCheckpointTable) { await knex.schema.createTable(TableName.FolderTreeCheckpoint, (t) => { t.uuid("id").primary().defaultTo(knex.fn.uuid()); - t.bigInteger("folderCommitId").notNullable(); + t.uuid("folderCommitId").notNullable(); t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE"); - t.timestamp("date").notNullable().defaultTo(knex.fn.now()); t.timestamps(true, true, true); t.index("folderCommitId"); @@ -91,7 +89,7 @@ export async function up(knex: Knex): Promise { t.foreign("folderTreeCheckpointId").references("id").inTable(TableName.FolderTreeCheckpoint).onDelete("CASCADE"); t.uuid("folderId").notNullable(); t.foreign("folderId").references("id").inTable(TableName.SecretFolder).onDelete("CASCADE"); - t.bigInteger("folderCommitId").notNullable(); + t.uuid("folderCommitId").notNullable(); t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE"); t.timestamps(true, true, true); @@ -110,21 +108,45 @@ export async function up(knex: Knex): Promise { } export async function down(knex: Knex): Promise { - await dropOnUpdateTrigger(knex, TableName.FolderTreeCheckpointResources); - await knex.schema.dropTableIfExists(TableName.FolderTreeCheckpointResources); + const hasFolderCheckpointResourcesTable = await knex.schema.hasTable(TableName.FolderCheckpointResources); + const hasFolderTreeCheckpointResourcesTable = await knex.schema.hasTable(TableName.FolderTreeCheckpointResources); + const hasFolderCommitTable = await knex.schema.hasTable(TableName.FolderCommit); + const hasFolderCommitChangesTable = await knex.schema.hasTable(TableName.FolderCommitChanges); + const hasFolderTreeCheckpointTable = await knex.schema.hasTable(TableName.FolderTreeCheckpoint); + const hasFolderCheckpointTable = await knex.schema.hasTable(TableName.FolderCheckpoint); - await dropOnUpdateTrigger(knex, TableName.FolderTreeCheckpoint); - await knex.schema.dropTableIfExists(TableName.FolderTreeCheckpoint); + if (hasFolderCheckpointResourcesTable) { + await dropOnUpdateTrigger(knex, TableName.FolderCheckpointResources); + await knex.schema.dropTableIfExists(TableName.FolderCheckpointResources); + } - await dropOnUpdateTrigger(knex, TableName.FolderCheckpointResources); - await knex.schema.dropTableIfExists(TableName.FolderCheckpointResources); + if (hasFolderTreeCheckpointResourcesTable) { + await dropOnUpdateTrigger(knex, TableName.FolderTreeCheckpointResources); + await knex.schema.dropTableIfExists(TableName.FolderTreeCheckpointResources); + } - await dropOnUpdateTrigger(knex, TableName.FolderCheckpoint); - await knex.schema.dropTableIfExists(TableName.FolderCheckpoint); + if (hasFolderTreeCheckpointTable) { + await dropOnUpdateTrigger(knex, TableName.FolderTreeCheckpoint); + await knex.schema.dropTableIfExists(TableName.FolderTreeCheckpoint); + } - await dropOnUpdateTrigger(knex, TableName.FolderCommitChanges); - await knex.schema.dropTableIfExists(TableName.FolderCommitChanges); + if (hasFolderCheckpointResourcesTable) { + await dropOnUpdateTrigger(knex, TableName.FolderCheckpointResources); + await knex.schema.dropTableIfExists(TableName.FolderCheckpointResources); + } - await dropOnUpdateTrigger(knex, TableName.FolderCommit); - await knex.schema.dropTableIfExists(TableName.FolderCommit); + if (hasFolderCheckpointTable) { + await dropOnUpdateTrigger(knex, TableName.FolderCheckpoint); + await knex.schema.dropTableIfExists(TableName.FolderCheckpoint); + } + + if (hasFolderCommitChangesTable) { + await dropOnUpdateTrigger(knex, TableName.FolderCommitChanges); + await knex.schema.dropTableIfExists(TableName.FolderCommitChanges); + } + + if (hasFolderCommitTable) { + await dropOnUpdateTrigger(knex, TableName.FolderCommit); + await knex.schema.dropTableIfExists(TableName.FolderCommit); + } } diff --git a/backend/src/db/schemas/folder-checkpoints.ts b/backend/src/db/schemas/folder-checkpoints.ts index fd7ccae84..ba0ce6f71 100644 --- a/backend/src/db/schemas/folder-checkpoints.ts +++ b/backend/src/db/schemas/folder-checkpoints.ts @@ -9,8 +9,7 @@ import { TImmutableDBKeys } from "./models"; export const FolderCheckpointsSchema = z.object({ id: z.string().uuid(), - folderCommitId: z.coerce.number(), - date: z.date(), + folderCommitId: z.string().uuid(), createdAt: z.date(), updatedAt: z.date() }); diff --git a/backend/src/db/schemas/folder-commit-changes.ts b/backend/src/db/schemas/folder-commit-changes.ts index 9ee228595..063874f43 100644 --- a/backend/src/db/schemas/folder-commit-changes.ts +++ b/backend/src/db/schemas/folder-commit-changes.ts @@ -9,7 +9,7 @@ import { TImmutableDBKeys } from "./models"; export const FolderCommitChangesSchema = z.object({ id: z.string().uuid(), - folderCommitId: z.coerce.number(), + folderCommitId: z.string().uuid(), changeType: z.string(), secretVersionId: z.string().uuid().nullable().optional(), folderVersionId: z.string().uuid().nullable().optional(), diff --git a/backend/src/db/schemas/folder-commits.ts b/backend/src/db/schemas/folder-commits.ts index 4d5f2d2af..ec166e7c2 100644 --- a/backend/src/db/schemas/folder-commits.ts +++ b/backend/src/db/schemas/folder-commits.ts @@ -8,11 +8,11 @@ import { z } from "zod"; import { TImmutableDBKeys } from "./models"; export const FolderCommitsSchema = z.object({ - id: z.coerce.number(), - actorName: z.string(), + id: z.string().uuid(), + commitId: z.coerce.number(), + actorMetadata: z.unknown(), actorType: z.string(), message: z.string().nullable().optional(), - date: z.date(), folderId: z.string().uuid(), createdAt: z.date(), updatedAt: z.date() diff --git a/backend/src/db/schemas/folder-tree-checkpoint-resources.ts b/backend/src/db/schemas/folder-tree-checkpoint-resources.ts index 1c65e833d..06d5770cc 100644 --- a/backend/src/db/schemas/folder-tree-checkpoint-resources.ts +++ b/backend/src/db/schemas/folder-tree-checkpoint-resources.ts @@ -11,7 +11,7 @@ export const FolderTreeCheckpointResourcesSchema = z.object({ id: z.string().uuid(), folderTreeCheckpointId: z.string().uuid(), folderId: z.string().uuid(), - folderCommitId: z.coerce.number(), + folderCommitId: z.string().uuid(), createdAt: z.date(), updatedAt: z.date() }); diff --git a/backend/src/db/schemas/folder-tree-checkpoints.ts b/backend/src/db/schemas/folder-tree-checkpoints.ts index 8f637b49b..ea500af6b 100644 --- a/backend/src/db/schemas/folder-tree-checkpoints.ts +++ b/backend/src/db/schemas/folder-tree-checkpoints.ts @@ -9,8 +9,7 @@ import { TImmutableDBKeys } from "./models"; export const FolderTreeCheckpointsSchema = z.object({ id: z.string().uuid(), - folderCommitId: z.coerce.number(), - date: z.date(), + folderCommitId: z.string().uuid(), createdAt: z.date(), updatedAt: z.date() }); diff --git a/backend/src/db/schemas/models.ts b/backend/src/db/schemas/models.ts index 0b80be80f..47c5cad54 100644 --- a/backend/src/db/schemas/models.ts +++ b/backend/src/db/schemas/models.ts @@ -161,7 +161,7 @@ export enum TableName { FolderTreeCheckpointResources = "folder_tree_checkpoint_resources" } -export type TImmutableDBKeys = "id" | "createdAt" | "updatedAt"; +export type TImmutableDBKeys = "id" | "createdAt" | "updatedAt" | "commitId"; export const UserDeviceSchema = z .object({ diff --git a/backend/src/ee/services/license/license-fns.ts b/backend/src/ee/services/license/license-fns.ts index cff6ee181..b7ae6f7ee 100644 --- a/backend/src/ee/services/license/license-fns.ts +++ b/backend/src/ee/services/license/license-fns.ts @@ -17,44 +17,44 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({ environmentsUsed: 0, identityLimit: null, identitiesUsed: 0, - dynamicSecret: true, + dynamicSecret: false, secretVersioning: true, - pitRecovery: true, - ipAllowlisting: true, - rbac: true, - githubOrgSync: true, - customRateLimits: true, - customAlerts: true, - secretAccessInsights: true, - auditLogs: true, + pitRecovery: false, + ipAllowlisting: false, + rbac: false, + githubOrgSync: false, + customRateLimits: false, + customAlerts: false, + secretAccessInsights: false, + auditLogs: false, auditLogsRetentionDays: 0, - auditLogStreams: true, + auditLogStreams: false, auditLogStreamLimit: 3, - samlSSO: true, - hsm: true, - oidcSSO: true, - scim: true, - ldap: true, - groups: true, + samlSSO: false, + hsm: false, + oidcSSO: false, + scim: false, + ldap: false, + groups: false, status: null, trial_end: null, has_used_trial: true, - secretApproval: true, - secretRotation: true, - caCrl: true, - instanceUserManagement: true, - externalKms: true, + secretApproval: false, + secretRotation: false, + caCrl: false, + instanceUserManagement: false, + externalKms: false, rateLimits: { readLimit: 60, writeLimit: 200, secretsLimit: 40 }, - pkiEst: true, - enforceMfa: true, - projectTemplates: true, - kmip: true, - gateway: true, - sshHostGroups: true + pkiEst: false, + enforceMfa: false, + projectTemplates: false, + kmip: false, + gateway: false, + sshHostGroups: false }); export const setupLicenseRequestWithStore = (baseURL: string, refreshUrl: string, licenseKey: string) => { diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts index 262e8e5cf..054374c7a 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts @@ -20,6 +20,7 @@ import { EnforcementLevel } from "@app/lib/types"; import { triggerWorkflowIntegrationNotification } from "@app/lib/workflow-integrations/trigger-notification"; import { TriggerFeature } from "@app/lib/workflow-integrations/types"; import { ActorType } from "@app/services/auth/auth-type"; +import { TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; import { TMicrosoftTeamsServiceFactory } from "@app/services/microsoft-teams/microsoft-teams-service"; @@ -130,6 +131,7 @@ type TSecretApprovalRequestServiceFactoryDep = { licenseService: Pick; projectMicrosoftTeamsConfigDAL: Pick; microsoftTeamsService: Pick; + folderCommitService: Pick; }; export type TSecretApprovalRequestServiceFactory = ReturnType; @@ -161,7 +163,8 @@ export const secretApprovalRequestServiceFactory = ({ projectSlackConfigDAL, resourceMetadataDAL, projectMicrosoftTeamsConfigDAL, - microsoftTeamsService + microsoftTeamsService, + folderCommitService }: TSecretApprovalRequestServiceFactoryDep) => { const requestCount = async ({ projectId, actor, actorId, actorOrgId, actorAuthMethod }: TApprovalRequestCountDTO) => { if (actor === ActorType.SERVICE) throw new BadRequestError({ message: "Cannot use service token" }); @@ -596,6 +599,10 @@ export const secretApprovalRequestServiceFactory = ({ ? await fnSecretV2BridgeBulkInsert({ tx, folderId, + actor: { + actorId, + type: actor + }, orgId: actorOrgId, inputSecrets: secretCreationCommits.map((el) => ({ tagIds: el?.tags.map(({ id }) => id), @@ -618,13 +625,18 @@ export const secretApprovalRequestServiceFactory = ({ secretDAL: secretV2BridgeDAL, secretVersionDAL: secretVersionV2BridgeDAL, secretTagDAL, - secretVersionTagDAL: secretVersionTagV2BridgeDAL + secretVersionTagDAL: secretVersionTagV2BridgeDAL, + folderCommitService }) : []; const updatedSecrets = secretUpdationCommits.length ? await fnSecretV2BridgeBulkUpdate({ folderId, orgId: actorOrgId, + actor: { + actorId, + type: actor + }, tx, inputSecrets: secretUpdationCommits.map((el) => { const encryptedValue = @@ -658,7 +670,8 @@ export const secretApprovalRequestServiceFactory = ({ secretVersionDAL: secretVersionV2BridgeDAL, secretTagDAL, secretVersionTagDAL: secretVersionTagV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }) : []; const deletedSecret = secretDeletionCommits.length @@ -666,10 +679,13 @@ export const secretApprovalRequestServiceFactory = ({ projectId, folderId, tx, - actorId: "", + actorId, + actorType: actor, secretDAL: secretV2BridgeDAL, secretQueueService, - inputSecrets: secretDeletionCommits.map(({ key }) => ({ secretKey: key, type: SecretType.Shared })) + inputSecrets: secretDeletionCommits.map(({ key }) => ({ secretKey: key, type: SecretType.Shared })), + folderCommitService, + secretVersionDAL: secretVersionV2BridgeDAL }) : []; const updatedSecretApproval = await secretApprovalRequestDAL.updateById( diff --git a/backend/src/ee/services/secret-replication/secret-replication-service.ts b/backend/src/ee/services/secret-replication/secret-replication-service.ts index 90fdf561e..c18126871 100644 --- a/backend/src/ee/services/secret-replication/secret-replication-service.ts +++ b/backend/src/ee/services/secret-replication/secret-replication-service.ts @@ -10,6 +10,7 @@ import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { QueueName, TQueueServiceFactory } from "@app/queue"; import { ActorType } from "@app/services/auth/auth-type"; +import { TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; @@ -87,6 +88,7 @@ type TSecretReplicationServiceFactoryDep = { projectBotService: Pick; kmsService: Pick; + folderCommitService: Pick; }; export type TSecretReplicationServiceFactory = ReturnType; @@ -132,6 +134,7 @@ export const secretReplicationServiceFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, kmsService, + folderCommitService, resourceMetadataDAL }: TSecretReplicationServiceFactoryDep) => { const $getReplicatedSecrets = ( @@ -446,6 +449,7 @@ export const secretReplicationServiceFactory = ({ tx, secretTagDAL, resourceMetadataDAL, + folderCommitService, secretVersionTagDAL: secretVersionV2TagBridgeDAL, inputSecrets: locallyCreatedSecrets.map((doc) => { return { @@ -466,6 +470,7 @@ export const secretReplicationServiceFactory = ({ orgId, folderId: destinationReplicationFolderId, secretVersionDAL: secretVersionV2BridgeDAL, + folderCommitService, secretDAL: secretV2BridgeDAL, tx, resourceMetadataDAL, diff --git a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts index 69743f133..979cb5b76 100644 --- a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts +++ b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts @@ -61,6 +61,7 @@ import { TAppConnectionDALFactory } from "@app/services/app-connection/app-conne import { decryptAppConnection } from "@app/services/app-connection/app-connection-fns"; import { TAppConnectionServiceFactory } from "@app/services/app-connection/app-connection-service"; import { ActorType } from "@app/services/auth/auth-type"; +import { TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; @@ -96,7 +97,7 @@ export type TSecretRotationV2ServiceFactoryDep = { TSecretV2BridgeDALFactory, "bulkUpdate" | "insertMany" | "deleteMany" | "upsertSecretReferences" | "find" | "invalidateSecretCacheByProjectId" >; - secretVersionV2BridgeDAL: Pick; + secretVersionV2BridgeDAL: Pick; secretVersionTagV2BridgeDAL: Pick; resourceMetadataDAL: Pick; secretTagDAL: Pick; @@ -104,6 +105,7 @@ export type TSecretRotationV2ServiceFactoryDep = { snapshotService: Pick; queueService: Pick; appConnectionDAL: Pick; + folderCommitService: Pick; }; export type TSecretRotationV2ServiceFactory = ReturnType; @@ -141,6 +143,7 @@ export const secretRotationV2ServiceFactory = ({ snapshotService, keyStore, queueService, + folderCommitService, appConnectionDAL }: TSecretRotationV2ServiceFactoryDep) => { const $queueSendSecretRotationStatusNotification = async (secretRotation: TSecretRotationV2Raw) => { @@ -533,7 +536,12 @@ export const secretRotationV2ServiceFactory = ({ secretVersionDAL: secretVersionV2BridgeDAL, secretVersionTagDAL: secretVersionTagV2BridgeDAL, secretTagDAL, - resourceMetadataDAL + folderCommitService, + resourceMetadataDAL, + actor: { + type: actor.type, + actorId: actor.id + } }); await secretRotationV2DAL.insertSecretMappings( @@ -668,7 +676,12 @@ export const secretRotationV2ServiceFactory = ({ secretVersionDAL: secretVersionV2BridgeDAL, secretVersionTagDAL: secretVersionTagV2BridgeDAL, secretTagDAL, - resourceMetadataDAL + folderCommitService, + resourceMetadataDAL, + actor: { + type: actor.type, + actorId: actor.id + } }); secretsMappingUpdated = true; @@ -786,6 +799,9 @@ export const secretRotationV2ServiceFactory = ({ projectId, folderId, actorId: actor.id, // not actually used since rotated secrets are shared + actorType: actor.type, + folderCommitService, + secretVersionDAL: secretVersionV2BridgeDAL, tx }); } @@ -926,6 +942,7 @@ export const secretRotationV2ServiceFactory = ({ secretDAL: secretV2BridgeDAL, secretVersionDAL: secretVersionV2BridgeDAL, secretVersionTagDAL: secretVersionTagV2BridgeDAL, + folderCommitService, secretTagDAL, resourceMetadataDAL }); diff --git a/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts b/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts index 2c6124348..ec92a736f 100644 --- a/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts +++ b/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts @@ -14,6 +14,7 @@ import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; import { ActorType } from "@app/services/auth/auth-type"; +import { TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; @@ -53,6 +54,7 @@ type TSecretRotationQueueFactoryDep = { secretVersionV2BridgeDAL: Pick; telemetryService: Pick; kmsService: Pick; + folderCommitService: Pick; }; // These error should stop the repeatable job and ask user to reconfigure rotation @@ -77,6 +79,7 @@ export const secretRotationQueueFactory = ({ telemetryService, secretV2BridgeDAL, secretVersionV2BridgeDAL, + folderCommitService, kmsService }: TSecretRotationQueueFactoryDep) => { const addToQueue = async (rotationId: string, interval: number) => { @@ -330,7 +333,7 @@ export const secretRotationQueueFactory = ({ })), tx ); - await secretVersionV2BridgeDAL.insertMany( + const secretVersions = await secretVersionV2BridgeDAL.insertMany( updatedSecrets.map(({ id, updatedAt, createdAt, ...el }) => ({ ...el, actorType: ActorType.PLATFORM, @@ -338,6 +341,21 @@ export const secretRotationQueueFactory = ({ })), tx ); + + await folderCommitService.createCommit( + { + actor: { + type: ActorType.PLATFORM + }, + message: "Secret rotation", + folderId: secretVersions[0].folderId, + changes: secretVersions.map((sv) => ({ + type: "add", + secretVersionId: sv.id + })) + }, + tx + ); }); await secretV2BridgeDAL.invalidateSecretCacheByProjectId(secretRotation.projectId); diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index d15058bcb..5ba8954fe 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -140,6 +140,11 @@ import { externalGroupOrgRoleMappingDALFactory } from "@app/services/external-gr import { externalGroupOrgRoleMappingServiceFactory } from "@app/services/external-group-org-role-mapping/external-group-org-role-mapping-service"; import { externalMigrationQueueFactory } from "@app/services/external-migration/external-migration-queue"; import { externalMigrationServiceFactory } from "@app/services/external-migration/external-migration-service"; +import { folderCheckpointDALFactory } from "@app/services/folder-checkpoint/folder-checkpoint-dal"; +import { folderCommitDALFactory } from "@app/services/folder-commit/folder-commit-dal"; +import { folderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; +import { folderCommitChangesDALFactory } from "@app/services/folder-commit-changes/folder-commit-changes-dal"; +import { folderTreeCheckpointDALFactory } from "@app/services/folder-tree-checkpoint/folder-tree-checkpoint-dal"; import { groupProjectDALFactory } from "@app/services/group-project/group-project-dal"; import { groupProjectMembershipRoleDALFactory } from "@app/services/group-project/group-project-membership-role-dal"; import { groupProjectServiceFactory } from "@app/services/group-project/group-project-service"; @@ -551,6 +556,19 @@ export const registerRoutes = async ( projectRoleDAL, permissionService }); + + const folderCommitChangesDAL = folderCommitChangesDALFactory(db); + const folderCheckpointDAL = folderCheckpointDALFactory(db); + const folderTreeCheckpointDAL = folderTreeCheckpointDALFactory(db); + const folderCommitDAL = folderCommitDALFactory(db); + const folderCommitService = folderCommitServiceFactory({ + folderCommitDAL, + folderCommitChangesDAL, + folderCheckpointDAL, + folderTreeCheckpointDAL, + userDAL, + identityDAL + }); const scimService = scimServiceFactory({ licenseService, scimDAL, @@ -973,6 +991,7 @@ export const registerRoutes = async ( projectMembershipDAL, projectBotDAL, secretDAL, + folderCommitService, secretBlindIndexDAL, secretVersionDAL, secretTagDAL, @@ -1019,6 +1038,7 @@ export const registerRoutes = async ( secretReminderRecipientsDAL, orgService, resourceMetadataDAL, + folderCommitService, secretSyncQueue }); @@ -1120,7 +1140,8 @@ export const registerRoutes = async ( folderVersionDAL, projectEnvDAL, snapshotService, - projectDAL + projectDAL, + folderCommitService }); const secretImportService = secretImportServiceFactory({ @@ -1145,6 +1166,7 @@ export const registerRoutes = async ( const secretV2BridgeService = secretV2BridgeServiceFactory({ folderDAL, secretVersionDAL: secretVersionV2BridgeDAL, + folderCommitService, secretQueueService, secretDAL: secretV2BridgeDAL, permissionService, @@ -1188,7 +1210,8 @@ export const registerRoutes = async ( projectSlackConfigDAL, resourceMetadataDAL, projectMicrosoftTeamsConfigDAL, - microsoftTeamsService + microsoftTeamsService, + folderCommitService }); const secretService = secretServiceFactory({ @@ -1273,7 +1296,8 @@ export const registerRoutes = async ( secretV2BridgeDAL, secretVersionV2TagBridgeDAL: secretVersionTagV2BridgeDAL, secretVersionV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }); const secretRotationQueue = secretRotationQueueFactory({ @@ -1285,6 +1309,7 @@ export const registerRoutes = async ( projectBotService, secretVersionV2BridgeDAL, secretV2BridgeDAL, + folderCommitService, kmsService }); @@ -1557,7 +1582,9 @@ export const registerRoutes = async ( secretDAL: secretV2BridgeDAL, queueService, secretV2BridgeService, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService, + folderVersionDAL }); const migrationService = externalMigrationServiceFactory({ @@ -1619,6 +1646,7 @@ export const registerRoutes = async ( auditLogService, secretV2BridgeDAL, secretTagDAL, + folderCommitService, secretVersionTagV2BridgeDAL, secretVersionV2BridgeDAL, keyStore, @@ -1746,7 +1774,8 @@ export const registerRoutes = async ( secretRotationV2: secretRotationV2Service, microsoftTeams: microsoftTeamsService, assumePrivileges: assumePrivilegeService, - githubOrgSync: githubOrgSyncConfigService + githubOrgSync: githubOrgSyncConfigService, + folderCommit: folderCommitService }); const cronJobs: CronJob[] = []; diff --git a/backend/src/services/external-migration/external-migration-fns.ts b/backend/src/services/external-migration/external-migration-fns.ts index 856b39012..f3c2326de 100644 --- a/backend/src/services/external-migration/external-migration-fns.ts +++ b/backend/src/services/external-migration/external-migration-fns.ts @@ -10,6 +10,7 @@ import { chunkArray } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "../kms/kms-service"; import { KmsDataKey } from "../kms/kms-types"; import { TProjectDALFactory } from "../project/project-dal"; @@ -18,6 +19,7 @@ import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TProjectEnvServiceFactory } from "../project-env/project-env-service"; import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal"; import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; import { fnSecretBulkInsert, getAllSecretReferences } from "../secret-v2-bridge/secret-v2-bridge-fns"; @@ -42,6 +44,8 @@ export type TImportDataIntoInfisicalDTO = { projectService: Pick; projectEnvService: Pick; secretV2BridgeService: Pick; + folderCommitService: Pick; + folderVersionDAL: Pick; input: TImportInfisicalDataCreate; }; @@ -507,6 +511,8 @@ export const importDataIntoInfisicalFn = async ({ secretVersionTagDAL, folderDAL, resourceMetadataDAL, + folderVersionDAL, + folderCommitService, input: { data, actor, actorId, actorOrgId, actorAuthMethod } }: TImportDataIntoInfisicalDTO) => { // Import data to infisical @@ -599,6 +605,36 @@ export const importDataIntoInfisicalFn = async ({ tx ); + const newFolderVersion = await folderVersionDAL.create( + { + name: newFolder.name, + envId: newFolder.envId, + version: newFolder.version, + folderId: newFolder.id + }, + tx + ); + + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "External migration", + folderId: parentEnv.rootFolderId, + changes: [ + { + type: "add", + folderVersionId: newFolderVersion.id + } + ] + }, + tx + ); + originalToNewFolderId.set(folder.id, { folderId: newFolder.id, projectId: parentEnv.projectId @@ -772,6 +808,7 @@ export const importDataIntoInfisicalFn = async ({ secretVersionDAL, secretTagDAL, secretVersionTagDAL, + folderCommitService, actor: { type: actor, actorId diff --git a/backend/src/services/external-migration/external-migration-queue.ts b/backend/src/services/external-migration/external-migration-queue.ts index 8aa46b94c..66f2c73e7 100644 --- a/backend/src/services/external-migration/external-migration-queue.ts +++ b/backend/src/services/external-migration/external-migration-queue.ts @@ -3,6 +3,7 @@ import { infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption"; import { logger } from "@app/lib/logger"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "../kms/kms-service"; import { TProjectDALFactory } from "../project/project-dal"; import { TProjectServiceFactory } from "../project/project-service"; @@ -10,6 +11,7 @@ import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TProjectEnvServiceFactory } from "../project-env/project-env-service"; import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal"; import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; import { TSecretV2BridgeServiceFactory } from "../secret-v2-bridge/secret-v2-bridge-service"; @@ -36,6 +38,8 @@ export type TExternalMigrationQueueFactoryDep = { projectService: Pick; projectEnvService: Pick; secretV2BridgeService: Pick; + folderCommitService: Pick; + folderVersionDAL: Pick; resourceMetadataDAL: Pick; }; @@ -56,6 +60,8 @@ export const externalMigrationQueueFactory = ({ secretTagDAL, secretVersionTagDAL, folderDAL, + folderCommitService, + folderVersionDAL, resourceMetadataDAL }: TExternalMigrationQueueFactoryDep) => { const startImport = async (dto: { @@ -114,6 +120,8 @@ export const externalMigrationQueueFactory = ({ projectService, projectEnvService, secretV2BridgeService, + folderCommitService, + folderVersionDAL, resourceMetadataDAL }); diff --git a/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts b/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts index d0b2200ec..2457abe14 100644 --- a/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts +++ b/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts @@ -9,11 +9,10 @@ export type TFolderCheckpointResourcesDALFactory = ReturnType { - const folderCheckpointResourcesOrm = ormify(db, TableName.FolderCheckpointResources); + const folderCheckpointResourcesOrm = ormify(db, TableName.FolderCheckpointResources); const findByCheckpointId = async (folderCheckpointId: string, tx?: Knex): Promise => { try { @@ -29,7 +28,7 @@ export const folderCheckpointResourcesDALFactory = (db: TDbClient) => { const findBySecretVersionId = async (secretVersionId: string, tx?: Knex): Promise => { try { const docs = await (tx || db.replicaNode())< - TFolderCheckpointResources & Pick + TFolderCheckpointResources & Pick >(TableName.FolderCheckpointResources) .where({ secretVersionId }) .select(selectAllTableCols(TableName.FolderCheckpointResources)) @@ -40,7 +39,7 @@ export const folderCheckpointResourcesDALFactory = (db: TDbClient) => { ) .select( db.ref("folderCommitId").withSchema(TableName.FolderCheckpoint), - db.ref("date").withSchema(TableName.FolderCheckpoint) + db.ref("createdAt").withSchema(TableName.FolderCheckpoint) ); return docs; } catch (error) { @@ -51,7 +50,7 @@ export const folderCheckpointResourcesDALFactory = (db: TDbClient) => { const findByFolderVersionId = async (folderVersionId: string, tx?: Knex): Promise => { try { const docs = await (tx || db.replicaNode())< - TFolderCheckpointResources & Pick + TFolderCheckpointResources & Pick >(TableName.FolderCheckpointResources) .where({ folderVersionId }) .select(selectAllTableCols(TableName.FolderCheckpointResources)) @@ -62,7 +61,7 @@ export const folderCheckpointResourcesDALFactory = (db: TDbClient) => { ) .select( db.ref("folderCommitId").withSchema(TableName.FolderCheckpoint), - db.ref("date").withSchema(TableName.FolderCheckpoint) + db.ref("createdAt").withSchema(TableName.FolderCheckpoint) ); return docs; } catch (error) { diff --git a/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts b/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts index c047c0504..5471f9b61 100644 --- a/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts +++ b/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts @@ -8,15 +8,15 @@ import { ormify, selectAllTableCols } from "@app/lib/knex"; export type TFolderCheckpointDALFactory = ReturnType; type CheckpointWithCommitInfo = TFolderCheckpoints & { - actorName: string; + actorMetadata: unknown; actorType: string; - message: string | null; + message?: string | null; commitDate: Date; folderId: string; }; export const folderCheckpointDALFactory = (db: TDbClient) => { - const folderCheckpointOrm = ormify(db, TableName.FolderCheckpoint); + const folderCheckpointOrm = ormify(db, TableName.FolderCheckpoint); const findByCommitId = async (folderCommitId: string, tx?: Knex): Promise => { try { @@ -37,13 +37,13 @@ export const folderCheckpointDALFactory = (db: TDbClient) => { .where({ folderId }) .select(selectAllTableCols(TableName.FolderCheckpoint)) .select( - db.ref("actorName").withSchema(TableName.FolderCommit), + db.ref("actorMetadata").withSchema(TableName.FolderCommit), db.ref("actorType").withSchema(TableName.FolderCommit), db.ref("message").withSchema(TableName.FolderCommit), - db.ref("date").withSchema(TableName.FolderCommit).as("commitDate"), + db.ref("createdAt").withSchema(TableName.FolderCommit).as("commitDate"), db.ref("folderId").withSchema(TableName.FolderCommit) ) - .orderBy(`${TableName.FolderCheckpoint}.date`, "desc"); + .orderBy(`${TableName.FolderCheckpoint}.createdAt`, "desc"); if (limit !== undefined) { query = query.limit(limit); @@ -63,13 +63,13 @@ export const folderCheckpointDALFactory = (db: TDbClient) => { .where({ folderId }) .select(selectAllTableCols(TableName.FolderCheckpoint)) .select( - db.ref("actorName").withSchema(TableName.FolderCommit), + db.ref("actorMetadata").withSchema(TableName.FolderCommit), db.ref("actorType").withSchema(TableName.FolderCommit), db.ref("message").withSchema(TableName.FolderCommit), - db.ref("date").withSchema(TableName.FolderCommit).as("commitDate"), + db.ref("createdAt").withSchema(TableName.FolderCommit).as("commitDate"), db.ref("folderId").withSchema(TableName.FolderCommit) ) - .orderBy(`${TableName.FolderCheckpoint}.date`, "desc") + .orderBy(`${TableName.FolderCheckpoint}.createdAt`, "desc") .first(); return doc; } catch (error) { diff --git a/backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts b/backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts index 46905bf95..0b06f1362 100644 --- a/backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts +++ b/backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts @@ -8,15 +8,14 @@ import { ormify, selectAllTableCols } from "@app/lib/knex"; export type TFolderCommitChangesDALFactory = ReturnType; type CommitChangeWithCommitInfo = TFolderCommitChanges & { - actorName: string; + actorMetadata: unknown; actorType: string; - message: string | null; - date: Date; + message?: string | null; folderId: string; }; export const folderCommitChangesDALFactory = (db: TDbClient) => { - const folderCommitChangesOrm = ormify(db, TableName.FolderCommitChanges); + const folderCommitChangesOrm = ormify(db, TableName.FolderCommitChanges); const findByCommitId = async (folderCommitId: string, tx?: Knex): Promise => { try { @@ -32,16 +31,17 @@ export const folderCommitChangesDALFactory = (db: TDbClient) => { const findBySecretVersionId = async (secretVersionId: string, tx?: Knex): Promise => { try { const docs = await (tx || db.replicaNode())< - TFolderCommitChanges & Pick + TFolderCommitChanges & + Pick >(TableName.FolderCommitChanges) .where({ secretVersionId }) .select(selectAllTableCols(TableName.FolderCommitChanges)) .join(TableName.FolderCommit, `${TableName.FolderCommitChanges}.folderCommitId`, `${TableName.FolderCommit}.id`) .select( - db.ref("actorName").withSchema(TableName.FolderCommit), + db.ref("actorMetadata").withSchema(TableName.FolderCommit), db.ref("actorType").withSchema(TableName.FolderCommit), db.ref("message").withSchema(TableName.FolderCommit), - db.ref("date").withSchema(TableName.FolderCommit), + db.ref("createdAt").withSchema(TableName.FolderCommit), db.ref("folderId").withSchema(TableName.FolderCommit) ); return docs; @@ -53,16 +53,17 @@ export const folderCommitChangesDALFactory = (db: TDbClient) => { const findByFolderVersionId = async (folderVersionId: string, tx?: Knex): Promise => { try { const docs = await (tx || db.replicaNode())< - TFolderCommitChanges & Pick + TFolderCommitChanges & + Pick >(TableName.FolderCommitChanges) .where({ folderVersionId }) .select(selectAllTableCols(TableName.FolderCommitChanges)) .join(TableName.FolderCommit, `${TableName.FolderCommitChanges}.folderCommitId`, `${TableName.FolderCommit}.id`) .select( - db.ref("actorName").withSchema(TableName.FolderCommit), + db.ref("actorMetadata").withSchema(TableName.FolderCommit), db.ref("actorType").withSchema(TableName.FolderCommit), db.ref("message").withSchema(TableName.FolderCommit), - db.ref("date").withSchema(TableName.FolderCommit), + db.ref("createdAt").withSchema(TableName.FolderCommit), db.ref("folderId").withSchema(TableName.FolderCommit) ); return docs; diff --git a/backend/src/services/folder-commit/folder-commit-dal.ts b/backend/src/services/folder-commit/folder-commit-dal.ts index 2ad973bcd..c6888f8cd 100644 --- a/backend/src/services/folder-commit/folder-commit-dal.ts +++ b/backend/src/services/folder-commit/folder-commit-dal.ts @@ -8,7 +8,8 @@ import { ormify, selectAllTableCols } from "@app/lib/knex"; export type TFolderCommitDALFactory = ReturnType; export const folderCommitDALFactory = (db: TDbClient) => { - const folderCommitOrm = ormify(db, TableName.FolderCommit); + const folderCommitOrm = ormify(db, TableName.FolderCommit); + const { delete: deleteOp, deleteById, ...restOfOrm } = folderCommitOrm; const findByFolderId = async (folderId: string, tx?: Knex): Promise => { try { @@ -48,7 +49,7 @@ export const folderCommitDALFactory = (db: TDbClient) => { }; return { - ...folderCommitOrm, + ...restOfOrm, findByFolderId, findById, findLatestCommit diff --git a/backend/src/services/folder-commit/folder-commit-service.ts b/backend/src/services/folder-commit/folder-commit-service.ts new file mode 100644 index 000000000..c82de598c --- /dev/null +++ b/backend/src/services/folder-commit/folder-commit-service.ts @@ -0,0 +1,168 @@ +import { Knex } from "knex"; + +import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors"; + +import { ActorType } from "../auth/auth-type"; +import { TFolderCheckpointDALFactory } from "../folder-checkpoint/folder-checkpoint-dal"; +import { TFolderCommitChangesDALFactory } from "../folder-commit-changes/folder-commit-changes-dal"; +import { TFolderTreeCheckpointDALFactory } from "../folder-tree-checkpoint/folder-tree-checkpoint-dal"; +import { TIdentityDALFactory } from "../identity/identity-dal"; +import { TUserDALFactory } from "../user/user-dal"; +import { TFolderCommitDALFactory } from "./folder-commit-dal"; + +type TFolderCommitServiceFactoryDep = { + folderCommitDAL: Pick< + TFolderCommitDALFactory, + "create" | "findById" | "findByFolderId" | "findLatestCommit" | "transaction" + >; + folderCommitChangesDAL: Pick; + folderCheckpointDAL: Pick; + folderTreeCheckpointDAL: Pick< + TFolderTreeCheckpointDALFactory, + "create" | "findByProjectId" | "findLatestByProjectId" + >; + userDAL: Pick; + identityDAL: Pick; +}; + +export type TCreateCommitDTO = { + actor: { + type: string; + metadata?: { + name?: string; + id?: string; + }; + }; + message?: string; + folderId: string; + changes: { + type: string; + secretVersionId?: string; + folderVersionId?: string; + }[]; +}; + +export type TCommitChangeDTO = { + folderCommitId: string; + changeType: string; + secretVersionId?: string; + folderVersionId?: string; +}; + +export const folderCommitServiceFactory = ({ + folderCommitDAL, + folderCommitChangesDAL, + folderCheckpointDAL, + folderTreeCheckpointDAL, + userDAL, + identityDAL +}: TFolderCommitServiceFactoryDep) => { + const createCommit = async (data: TCreateCommitDTO, tx?: Knex) => { + const metadata = data.actor.metadata || {}; + try { + if (data.actor.type === ActorType.USER && data.actor.metadata?.id) { + const user = await userDAL.findById(data.actor.metadata?.id, tx); + metadata.name = user?.username; + } + if (data.actor.type === ActorType.IDENTITY && data.actor.metadata?.id) { + const identity = await identityDAL.findById(data.actor.metadata?.id, tx); + metadata.name = identity?.name; + } + const newCommit = await folderCommitDAL.create( + { + actorMetadata: metadata, + actorType: data.actor.type, + message: data.message, + folderId: data.folderId + }, + tx + ); + for (const change of data.changes) { + // eslint-disable-next-line no-await-in-loop + await folderCommitChangesDAL.create( + { + folderCommitId: newCommit.id, + changeType: change.type, + secretVersionId: change.secretVersionId, + folderVersionId: change.folderVersionId + }, + tx + ); + } + + return newCommit; + } catch (error) { + throw new DatabaseError({ error, name: "CreateCommit" }); + } + }; + + // Add a change to a commit and trigger checkpoints as needed + const addCommitChange = async (data: TCommitChangeDTO, tx?: Knex) => { + try { + if (!data.secretVersionId && !data.folderVersionId) { + throw new BadRequestError({ message: "Either secretVersionId or folderVersionId must be provided" }); + } + + const commit = await folderCommitDAL.findById(data.folderCommitId, tx); + if (!commit) { + throw new NotFoundError({ message: `Commit with ID ${data.folderCommitId} not found` }); + } + + return await folderCommitChangesDAL.create(data, tx); + } catch (error) { + if (error instanceof NotFoundError || error instanceof BadRequestError) { + throw error; + } + throw new DatabaseError({ error, name: "AddCommitChange" }); + } + }; + + // Retrieve a commit by ID + const getCommitById = async (id: string, tx?: Knex) => { + return folderCommitDAL.findById(id, tx); + }; + + // Get all commits for a folder + const getCommitsByFolderId = async (folderId: string, tx?: Knex) => { + return folderCommitDAL.findByFolderId(folderId, tx); + }; + + // Get changes for a commit + const getCommitChanges = async (commitId: string, tx?: Knex) => { + return folderCommitChangesDAL.findByCommitId(commitId, tx); + }; + + // Get checkpoints for a folder + const getCheckpointsByFolderId = async (folderId: string, limit?: number, tx?: Knex) => { + return folderCheckpointDAL.findByFolderId(folderId, limit, tx); + }; + + // Get the latest checkpoint for a folder + const getLatestCheckpoint = async (folderId: string, tx?: Knex) => { + return folderCheckpointDAL.findLatestByFolderId(folderId, tx); + }; + + // Get tree checkpoints for a project + const getTreeCheckpointsByProjectId = async (projectId: string, limit?: number, tx?: Knex) => { + return folderTreeCheckpointDAL.findByProjectId(projectId, limit, tx); + }; + + // Get the latest tree checkpoint for a project + const getLatestTreeCheckpoint = async (projectId: string, tx?: Knex) => { + return folderTreeCheckpointDAL.findLatestByProjectId(projectId, tx); + }; + + return { + createCommit, + addCommitChange, + getCommitById, + getCommitsByFolderId, + getCommitChanges, + getCheckpointsByFolderId, + getLatestCheckpoint, + getTreeCheckpointsByProjectId, + getLatestTreeCheckpoint + }; +}; + +export type TFolderCommitServiceFactory = ReturnType; diff --git a/backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts b/backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts index 7b4e0bffb..8813784fd 100644 --- a/backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts +++ b/backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts @@ -3,9 +3,9 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; import { TableName, - TEnvironments, TFolderTreeCheckpointResources, TFolderTreeCheckpoints, + TProjectEnvironments, TSecretFolders } from "@app/db/schemas"; import { DatabaseError } from "@app/lib/errors"; @@ -15,21 +15,17 @@ export type TFolderTreeCheckpointResourcesDALFactory = ReturnType { - const folderTreeCheckpointResourcesOrm = ormify( - db, - TableName.FolderTreeCheckpointResources - ); + const folderTreeCheckpointResourcesOrm = ormify(db, TableName.FolderTreeCheckpointResources); const findByTreeCheckpointId = async ( folderTreeCheckpointId: string, @@ -50,7 +46,7 @@ export const folderTreeCheckpointResourcesDALFactory = (db: TDbClient) => { const findByFolderId = async (folderId: string, tx?: Knex): Promise => { try { const docs = await (tx || db.replicaNode())< - TFolderTreeCheckpointResources & Pick + TFolderTreeCheckpointResources & Pick >(TableName.FolderTreeCheckpointResources) .where({ folderId }) .select(selectAllTableCols(TableName.FolderTreeCheckpointResources)) @@ -61,7 +57,7 @@ export const folderTreeCheckpointResourcesDALFactory = (db: TDbClient) => { ) .select( db.ref("folderCommitId").withSchema(TableName.FolderTreeCheckpoint), - db.ref("date").withSchema(TableName.FolderTreeCheckpoint) + db.ref("createdAt").withSchema(TableName.FolderTreeCheckpoint) ); return docs; } catch (error) { @@ -69,13 +65,10 @@ export const folderTreeCheckpointResourcesDALFactory = (db: TDbClient) => { } }; - const findByFolderCommitId = async ( - folderCommitId: string, - tx?: Knex - ): Promise<(TFolderTreeCheckpointResources & { date: Date })[]> => { + const findByFolderCommitId = async (folderCommitId: string, tx?: Knex): Promise => { try { const docs = await (tx || db.replicaNode())< - TFolderTreeCheckpointResources & Pick + TFolderTreeCheckpointResources & Pick >(TableName.FolderTreeCheckpointResources) .where({ folderCommitId }) .select(selectAllTableCols(TableName.FolderTreeCheckpointResources)) @@ -84,7 +77,7 @@ export const folderTreeCheckpointResourcesDALFactory = (db: TDbClient) => { `${TableName.FolderTreeCheckpointResources}.folderTreeCheckpointId`, `${TableName.FolderTreeCheckpoint}.id` ) - .select(db.ref("date").withSchema(TableName.FolderTreeCheckpoint)); + .select(db.ref("createdAt").withSchema(TableName.FolderTreeCheckpoint)); return docs; } catch (error) { throw new DatabaseError({ error, name: "FindByFolderCommitId" }); @@ -99,7 +92,7 @@ export const folderTreeCheckpointResourcesDALFactory = (db: TDbClient) => { const docs = await (tx || db.replicaNode())< TFolderTreeCheckpointResources & Pick & - Pick & { envName: string } + Pick & { envName: string } >(TableName.FolderTreeCheckpointResources) .where({ folderTreeCheckpointId }) .select(selectAllTableCols(TableName.FolderTreeCheckpointResources)) diff --git a/backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts b/backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts index 3377ca643..4e9381f9c 100644 --- a/backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts +++ b/backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts @@ -8,15 +8,15 @@ import { ormify, selectAllTableCols } from "@app/lib/knex"; export type TFolderTreeCheckpointDALFactory = ReturnType; type TreeCheckpointWithCommitInfo = TFolderTreeCheckpoints & { - actorName: string; + actorMetadata: unknown; actorType: string; - message: string | null; + message?: string | null; commitDate: Date; folderId: string; }; export const folderTreeCheckpointDALFactory = (db: TDbClient) => { - const folderTreeCheckpointOrm = ormify(db, TableName.FolderTreeCheckpoint); + const folderTreeCheckpointOrm = ormify(db, TableName.FolderTreeCheckpoint); const findByCommitId = async (folderCommitId: string, tx?: Knex): Promise => { try { @@ -38,7 +38,7 @@ export const folderTreeCheckpointDALFactory = (db: TDbClient) => { try { const query = (tx || db.replicaNode())< TFolderTreeCheckpoints & - Pick & { commitDate: Date } + Pick & { commitDate: Date } >(TableName.FolderTreeCheckpoint) .join( TableName.FolderCommit, @@ -50,13 +50,13 @@ export const folderTreeCheckpointDALFactory = (db: TDbClient) => { .where({ projectId }) .select(selectAllTableCols(TableName.FolderTreeCheckpoint)) .select( - db.ref("actorName").withSchema(TableName.FolderCommit), + db.ref("actorMetadata").withSchema(TableName.FolderCommit), db.ref("actorType").withSchema(TableName.FolderCommit), db.ref("message").withSchema(TableName.FolderCommit), - db.ref("date").withSchema(TableName.FolderCommit).as("commitDate"), + db.ref("createdAt").withSchema(TableName.FolderCommit).as("commitDate"), db.ref("folderId").withSchema(TableName.FolderCommit) ) - .orderBy(`${TableName.FolderTreeCheckpoint}.date`, "desc"); + .orderBy(`${TableName.FolderTreeCheckpoint}.createdAt`, "desc"); if (limit) { void query.limit(limit); @@ -76,7 +76,7 @@ export const folderTreeCheckpointDALFactory = (db: TDbClient) => { try { const doc = await (tx || db.replicaNode())< TFolderTreeCheckpoints & - Pick & { commitDate: Date } + Pick & { commitDate: Date } >(TableName.FolderTreeCheckpoint) .join( TableName.FolderCommit, @@ -88,13 +88,13 @@ export const folderTreeCheckpointDALFactory = (db: TDbClient) => { .where({ projectId }) .select(selectAllTableCols(TableName.FolderTreeCheckpoint)) .select( - db.ref("actorName").withSchema(TableName.FolderCommit), + db.ref("actorMetadata").withSchema(TableName.FolderCommit), db.ref("actorType").withSchema(TableName.FolderCommit), db.ref("message").withSchema(TableName.FolderCommit), - db.ref("date").withSchema(TableName.FolderCommit).as("commitDate"), + db.ref("createdAt").withSchema(TableName.FolderCommit).as("commitDate"), db.ref("folderId").withSchema(TableName.FolderCommit) ) - .orderBy(`${TableName.FolderTreeCheckpoint}.date`, "desc") + .orderBy(`${TableName.FolderTreeCheckpoint}.createdAt`, "desc") .first(); return doc; } catch (error) { diff --git a/backend/src/services/secret-folder/secret-folder-service.ts b/backend/src/services/secret-folder/secret-folder-service.ts index 842eb2bb7..fbb40bb5d 100644 --- a/backend/src/services/secret-folder/secret-folder-service.ts +++ b/backend/src/services/secret-folder/secret-folder-service.ts @@ -10,6 +10,7 @@ import { BadRequestError, NotFoundError } from "@app/lib/errors"; import { OrderByDirection, OrgServiceActor } from "@app/lib/types"; import { buildFolderPath } from "@app/services/secret-folder/secret-folder-fns"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TProjectDALFactory } from "../project/project-dal"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TSecretFolderDALFactory } from "./secret-folder-dal"; @@ -29,7 +30,8 @@ type TSecretFolderServiceFactoryDep = { snapshotService: Pick; folderDAL: TSecretFolderDALFactory; projectEnvDAL: Pick; - folderVersionDAL: TSecretFolderVersionDALFactory; + folderVersionDAL: Pick; + folderCommitService: Pick; projectDAL: Pick; }; @@ -41,6 +43,7 @@ export const secretFolderServiceFactory = ({ permissionService, projectEnvDAL, folderVersionDAL, + folderCommitService, projectDAL }: TSecretFolderServiceFactoryDep) => { const createFolder = async ({ @@ -111,7 +114,7 @@ export const secretFolderServiceFactory = ({ }); parentFolderId = newFolders.at(-1)?.id as string; const docs = await folderDAL.insertMany(newFolders, tx); - await folderVersionDAL.insertMany( + const folderVersions = await folderVersionDAL.insertMany( docs.map((doc) => ({ name: doc.name, envId: doc.envId, @@ -120,6 +123,23 @@ export const secretFolderServiceFactory = ({ })), tx ); + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "Folder created", + folderId: parentFolderId, + changes: folderVersions.map((fv) => ({ + type: "add", + folderVersionId: fv.id + })) + }, + tx + ); } } @@ -127,7 +147,7 @@ export const secretFolderServiceFactory = ({ { name, envId: env.id, version: 1, parentId: parentFolderId, description }, tx ); - await folderVersionDAL.create( + const folderVersion = await folderVersionDAL.create( { name: doc.name, envId: doc.envId, @@ -136,6 +156,25 @@ export const secretFolderServiceFactory = ({ }, tx ); + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "Folder created", + folderId: parentFolderId, + changes: [ + { + type: "add", + folderVersionId: folderVersion.id + } + ] + }, + tx + ); return doc; }); @@ -225,7 +264,7 @@ export const secretFolderServiceFactory = ({ { name, description }, tx ); - await folderVersionDAL.create( + const folderVersion = await folderVersionDAL.create( { name: doc.name, envId: doc.envId, @@ -234,6 +273,25 @@ export const secretFolderServiceFactory = ({ }, tx ); + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "Folder updated", + folderId: parentFolder.id, + changes: [ + { + type: "add", + folderVersionId: folderVersion.id + } + ] + }, + tx + ); if (!doc) { throw new NotFoundError({ message: `Failed to update folder with id '${id}', not found`, @@ -321,7 +379,7 @@ export const secretFolderServiceFactory = ({ { name, description }, tx ); - await folderVersionDAL.create( + const folderVersion = await folderVersionDAL.create( { name: doc.name, envId: doc.envId, @@ -330,6 +388,25 @@ export const secretFolderServiceFactory = ({ }, tx ); + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "Folder updated", + folderId: parentFolder.id, + changes: [ + { + type: "add", + folderVersionId: folderVersion.id + } + ] + }, + tx + ); if (!doc) throw new NotFoundError({ message: `Failed to update folder with ID '${id}'`, name: "UpdateFolder" }); return doc; }); @@ -381,7 +458,30 @@ export const secretFolderServiceFactory = ({ }, tx ); + if (!doc) throw new NotFoundError({ message: `Failed to delete folder with ID '${idOrName}', not found` }); + + const folderVersions = await folderVersionDAL.findLatestFolderVersions([doc.id]); + + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "Folder deleted", + folderId: parentFolder.id, + changes: [ + { + type: "delete", + folderVersionId: folderVersions[doc.id].id + } + ] + }, + tx + ); return doc; }); diff --git a/backend/src/services/secret-folder/secret-folder-version-dal.ts b/backend/src/services/secret-folder/secret-folder-version-dal.ts index 9425c5864..437b1e1eb 100644 --- a/backend/src/services/secret-folder/secret-folder-version-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-version-dal.ts @@ -43,7 +43,7 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => { const docs: Array = await (tx || db.replicaNode())( TableName.SecretFolderVersion ) - .whereIn("folderId", folderIds) + .whereIn(`${TableName.SecretFolderVersion}.folderId`, folderIds) .join( (tx || db)(TableName.SecretFolderVersion) .groupBy("folderId") diff --git a/backend/src/services/secret-sync/secret-sync-queue.ts b/backend/src/services/secret-sync/secret-sync-queue.ts index 62b4ba3cc..6b306cb51 100644 --- a/backend/src/services/secret-sync/secret-sync-queue.ts +++ b/backend/src/services/secret-sync/secret-sync-queue.ts @@ -58,6 +58,7 @@ import { TSecretVersionV2TagDALFactory } from "@app/services/secret-v2-bridge/se import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service"; import { TAppConnectionDALFactory } from "../app-connection/app-connection-dal"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; export type TSecretSyncQueueFactory = ReturnType; @@ -93,6 +94,7 @@ type TSecretSyncQueueFactoryDep = { secretVersionV2BridgeDAL: Pick; secretVersionTagV2BridgeDAL: Pick; resourceMetadataDAL: Pick; + folderCommitService: Pick; }; type SecretSyncActionJob = Job< @@ -133,7 +135,8 @@ export const secretSyncQueueFactory = ({ secretVersionTagDAL, secretVersionV2BridgeDAL, secretVersionTagV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }: TSecretSyncQueueFactoryDep) => { const appCfg = getConfig(); @@ -164,7 +167,8 @@ export const secretSyncQueueFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, secretVersionTagV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }); const $updateManySecretsRawFn = updateManySecretsRawFnFactory({ @@ -180,7 +184,8 @@ export const secretSyncQueueFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, secretVersionTagV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }); const $getInfisicalSecrets = async ( diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts index 6fdcadeff..eb2c77878 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts @@ -73,6 +73,7 @@ export const fnSecretBulkInsert = async ({ resourceMetadataDAL, secretTagDAL, secretVersionTagDAL, + folderCommitService, actor, tx }: TFnSecretBulkInsert) => { @@ -131,6 +132,30 @@ export const fnSecretBulkInsert = async ({ tx ); + const commitChanges = secretVersions + .filter(({ type }) => type === SecretType.Shared) + .map((sv) => ({ + type: "add", + secretVersionId: sv.id + })); + + if (commitChanges.length > 0) { + await folderCommitService.createCommit( + { + actor: { + type: actorType || ActorType.PLATFORM, + metadata: { + id: actor?.actorId + } + }, + message: "Secret Creation", + folderId, + changes: commitChanges + }, + tx + ); + } + await secretDAL.upsertSecretReferences( inputSecrets.map(({ references = [], key }) => ({ secretId: newSecretGroupedByKeyName[key][0].id, @@ -185,6 +210,7 @@ export const fnSecretBulkUpdate = async ({ orgId, secretDAL, secretVersionDAL, + folderCommitService, secretTagDAL, secretVersionTagDAL, resourceMetadataDAL, @@ -259,6 +285,30 @@ export const fnSecretBulkUpdate = async ({ ), tx ); + + const commitChanges = secretVersions + .filter(({ type }) => type === SecretType.Shared) + .map((sv) => ({ + type: "add", + secretVersionId: sv.id + })); + if (commitChanges.length > 0) { + await folderCommitService.createCommit( + { + actor: { + type: actorType || ActorType.PLATFORM, + metadata: { + id: actor?.actorId + } + }, + message: "Secret Update", + folderId, + changes: commitChanges + }, + tx + ); + } + await secretDAL.upsertSecretReferences( inputSecrets .filter(({ data: { references } }) => Boolean(references)) @@ -337,8 +387,11 @@ export const fnSecretBulkDelete = async ({ inputSecrets, tx, actorId, + actorType, secretDAL, - secretQueueService + secretQueueService, + folderCommitService, + secretVersionDAL }: TFnSecretBulkDelete) => { const deletedSecrets = await secretDAL.deleteMany( inputSecrets.map(({ type, secretKey }) => ({ @@ -358,6 +411,35 @@ export const fnSecretBulkDelete = async ({ ) ); + const secretVersions = await secretVersionDAL.findLatestVersionMany( + folderId, + deletedSecrets.map(({ id }) => id), + tx + ); + + const commitChanges = deletedSecrets + .filter(({ type }) => type === SecretType.Shared) + .map(({ id }) => ({ + type: "delete", + secretVersionId: secretVersions[id].id + })); + if (commitChanges.length > 0) { + await folderCommitService.createCommit( + { + actor: { + type: actorType || ActorType.PLATFORM, + metadata: { + id: actorId + } + }, + message: "Secret Delete", + folderId, + changes: commitChanges + }, + tx + ); + } + return deletedSecrets; }; diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts index 1ef4a2d41..33d3cf0f1 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts @@ -34,6 +34,7 @@ import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { ActorType } from "../auth/auth-type"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "../kms/kms-service"; import { KmsDataKey } from "../kms/kms-types"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; @@ -90,6 +91,7 @@ type TSecretV2BridgeServiceFactoryDep = { secretVersionTagDAL: Pick; secretTagDAL: TSecretTagDALFactory; permissionService: Pick; + folderCommitService: Pick; projectEnvDAL: Pick; folderDAL: Pick< TSecretFolderDALFactory, @@ -124,6 +126,7 @@ export const secretV2BridgeServiceFactory = ({ projectEnvDAL, secretTagDAL, secretVersionDAL, + folderCommitService, folderDAL, permissionService, snapshotService, @@ -327,6 +330,7 @@ export const secretV2BridgeServiceFactory = ({ resourceMetadataDAL, secretDAL, secretVersionDAL, + folderCommitService, secretTagDAL, secretVersionTagDAL, actor: { @@ -510,6 +514,7 @@ export const secretV2BridgeServiceFactory = ({ folderId, orgId: actorOrgId, resourceMetadataDAL, + folderCommitService, inputSecrets: [ { filter: { id: secretId }, @@ -650,6 +655,9 @@ export const secretV2BridgeServiceFactory = ({ projectId, folderId, actorId, + actorType: actor, + folderCommitService, + secretVersionDAL, secretDAL, secretQueueService, inputSecrets: [ @@ -1590,6 +1598,7 @@ export const secretV2BridgeServiceFactory = ({ orgId: actorOrgId, secretDAL, resourceMetadataDAL, + folderCommitService, secretVersionDAL, secretTagDAL, secretVersionTagDAL, @@ -1859,6 +1868,7 @@ export const secretV2BridgeServiceFactory = ({ const bulkUpdatedSecrets = await fnSecretBulkUpdate({ folderId, orgId: actorOrgId, + folderCommitService, tx, inputSecrets: secretsToUpdate.map((el) => { const originalSecret = secretsToUpdateInDBGroupedByKey[el.secretKey][0]; @@ -1928,6 +1938,7 @@ export const secretV2BridgeServiceFactory = ({ secretVersionDAL, secretTagDAL, secretVersionTagDAL, + folderCommitService, actor: { type: actor, actorId @@ -2061,6 +2072,8 @@ export const secretV2BridgeServiceFactory = ({ fnSecretBulkDelete({ secretDAL, secretQueueService, + folderCommitService, + secretVersionDAL, inputSecrets: inputSecrets.map(({ type, secretKey }) => ({ secretKey, type: type || SecretType.Shared @@ -2068,6 +2081,7 @@ export const secretV2BridgeServiceFactory = ({ projectId, folderId, actorId, + actorType: actor, tx }) ); @@ -2469,6 +2483,7 @@ export const secretV2BridgeServiceFactory = ({ tx, secretTagDAL, resourceMetadataDAL, + folderCommitService, secretVersionTagDAL, actor: { type: actor, @@ -2495,6 +2510,7 @@ export const secretV2BridgeServiceFactory = ({ folderId: destinationFolder.id, orgId: actorOrgId, resourceMetadataDAL, + folderCommitService, secretVersionDAL, secretDAL, tx, diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts index f4a27d4c5..b9ae74208 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts @@ -8,6 +8,7 @@ import { SecretsOrderBy } from "@app/services/secret/secret-types"; import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal"; import { ResourceMetadataDTO } from "../resource-metadata/resource-metadata-schema"; import { TSecretV2BridgeDALFactory } from "./secret-v2-bridge-dal"; @@ -178,6 +179,7 @@ export type TFnSecretBulkInsert = { secretVersionDAL: Pick; secretTagDAL: Pick; secretVersionTagDAL: Pick; + folderCommitService: Pick; actor?: { type: string; actorId: string; @@ -206,6 +208,7 @@ export type TFnSecretBulkUpdate = { secretVersionDAL: Pick; secretTagDAL: Pick; secretVersionTagDAL: Pick; + folderCommitService: Pick; actor?: { type: string; actorId: string; @@ -218,11 +221,14 @@ export type TFnSecretBulkDelete = { projectId: string; inputSecrets: Array<{ type: SecretType; secretKey: string }>; actorId: string; + actorType?: string; tx?: Knex; secretDAL: Pick; secretQueueService: { removeSecretReminder: (data: TRemoveSecretReminderDTO, tx?: Knex) => Promise; }; + folderCommitService: Pick; + secretVersionDAL: Pick; }; export type THandleReminderDTO = { diff --git a/backend/src/services/secret/secret-fns.ts b/backend/src/services/secret/secret-fns.ts index e5f3acdea..96f89ab5f 100644 --- a/backend/src/services/secret/secret-fns.ts +++ b/backend/src/services/secret/secret-fns.ts @@ -778,6 +778,7 @@ export const createManySecretsRawFnFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, secretVersionTagV2BridgeDAL, + folderCommitService, kmsService, resourceMetadataDAL }: TCreateManySecretsRawFnFactory) => { @@ -850,6 +851,7 @@ export const createManySecretsRawFnFactory = ({ secretVersionDAL: secretVersionV2BridgeDAL, secretTagDAL, secretVersionTagDAL: secretVersionTagV2BridgeDAL, + folderCommitService, tx }) ); @@ -942,6 +944,7 @@ export const updateManySecretsRawFnFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, resourceMetadataDAL, + folderCommitService, kmsService }: TUpdateManySecretsRawFnFactory) => { const getBotKeyFn = getBotKeyFnFactory(projectBotDAL, projectDAL); @@ -1032,7 +1035,8 @@ export const updateManySecretsRawFnFactory = ({ secretDAL: secretV2BridgeDAL, secretVersionDAL: secretVersionV2BridgeDAL, secretTagDAL, - secretVersionTagDAL: secretVersionTagV2BridgeDAL + secretVersionTagDAL: secretVersionTagV2BridgeDAL, + folderCommitService }) ); diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index 714df0d3f..87b2eb467 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -35,6 +35,7 @@ import { TSecretSyncQueueFactory } from "@app/services/secret-sync/secret-sync-q import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; import { ActorType } from "../auth/auth-type"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TIntegrationDALFactory } from "../integration/integration-dal"; import { TIntegrationAuthDALFactory } from "../integration-auth/integration-auth-dal"; import { TIntegrationAuthServiceFactory } from "../integration-auth/integration-auth-service"; @@ -112,6 +113,7 @@ type TSecretQueueFactoryDep = { orgService: Pick; projectUserMembershipRoleDAL: Pick; resourceMetadataDAL: Pick; + folderCommitService: Pick; secretReminderRecipientsDAL: Pick< TSecretReminderRecipientsDALFactory, "delete" | "findUsersBySecretId" | "insertMany" | "transaction" @@ -178,7 +180,8 @@ export const secretQueueFactory = ({ projectKeyDAL, resourceMetadataDAL, secretReminderRecipientsDAL, - secretSyncQueue + secretSyncQueue, + folderCommitService }: TSecretQueueFactoryDep) => { const integrationMeter = opentelemetry.metrics.getMeter("Integrations"); const errorHistogram = integrationMeter.createHistogram("integration_secret_sync_errors", { @@ -366,7 +369,8 @@ export const secretQueueFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, secretVersionTagV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }); const updateManySecretsRawFn = updateManySecretsRawFnFactory({ @@ -382,7 +386,8 @@ export const secretQueueFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, secretVersionTagV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }); /** diff --git a/backend/src/services/secret/secret-types.ts b/backend/src/services/secret/secret-types.ts index 30e3dfafa..91fc2eb6a 100644 --- a/backend/src/services/secret/secret-types.ts +++ b/backend/src/services/secret/secret-types.ts @@ -14,6 +14,7 @@ import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-fold import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; import { ActorType } from "../auth/auth-type"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "../kms/kms-service"; import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal"; import { ResourceMetadataDTO } from "../resource-metadata/resource-metadata-schema"; @@ -441,6 +442,7 @@ export type TCreateManySecretsRawFnFactory = { secretVersionV2BridgeDAL: Pick; secretVersionTagV2BridgeDAL: Pick; resourceMetadataDAL: Pick; + folderCommitService: Pick; }; export type TCreateManySecretsRawFn = { @@ -478,6 +480,7 @@ export type TUpdateManySecretsRawFnFactory = { secretVersionV2BridgeDAL: Pick; secretVersionTagV2BridgeDAL: Pick; resourceMetadataDAL: Pick; + folderCommitService: Pick; }; export type TUpdateManySecretsRawFn = { From e58dbe853ef8c21f3055b7119a9d0293c2c0dffa Mon Sep 17 00:00:00 2001 From: carlosmonastyrski Date: Wed, 7 May 2025 08:38:19 -0300 Subject: [PATCH 3/9] Minor improvements on commits code quality --- .../20250505194916_add-pit-revamp-tables.ts | 15 ++++-------- .../secret-rotation-v2-service.ts | 3 +++ .../secret-rotation-queue.ts | 2 +- .../external-migration-fns.ts | 2 +- .../folder-checkpoint-dal.ts | 8 ++++--- .../folder-commit/folder-commit-dal.ts | 10 ++++---- .../folder-commit/folder-commit-service.ts | 23 ++++++++----------- .../secret-v2-bridge-types.ts | 4 ++-- 8 files changed, 32 insertions(+), 35 deletions(-) diff --git a/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts b/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts index ea4c8e9bb..9855d301e 100644 --- a/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts +++ b/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts @@ -115,26 +115,21 @@ export async function down(knex: Knex): Promise { const hasFolderTreeCheckpointTable = await knex.schema.hasTable(TableName.FolderTreeCheckpoint); const hasFolderCheckpointTable = await knex.schema.hasTable(TableName.FolderCheckpoint); - if (hasFolderCheckpointResourcesTable) { - await dropOnUpdateTrigger(knex, TableName.FolderCheckpointResources); - await knex.schema.dropTableIfExists(TableName.FolderCheckpointResources); - } - if (hasFolderTreeCheckpointResourcesTable) { await dropOnUpdateTrigger(knex, TableName.FolderTreeCheckpointResources); await knex.schema.dropTableIfExists(TableName.FolderTreeCheckpointResources); } - if (hasFolderTreeCheckpointTable) { - await dropOnUpdateTrigger(knex, TableName.FolderTreeCheckpoint); - await knex.schema.dropTableIfExists(TableName.FolderTreeCheckpoint); - } - if (hasFolderCheckpointResourcesTable) { await dropOnUpdateTrigger(knex, TableName.FolderCheckpointResources); await knex.schema.dropTableIfExists(TableName.FolderCheckpointResources); } + if (hasFolderTreeCheckpointTable) { + await dropOnUpdateTrigger(knex, TableName.FolderTreeCheckpoint); + await knex.schema.dropTableIfExists(TableName.FolderTreeCheckpoint); + } + if (hasFolderCheckpointTable) { await dropOnUpdateTrigger(knex, TableName.FolderCheckpoint); await knex.schema.dropTableIfExists(TableName.FolderCheckpoint); diff --git a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts index 979cb5b76..0a7a5c12c 100644 --- a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts +++ b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts @@ -943,6 +943,9 @@ export const secretRotationV2ServiceFactory = ({ secretVersionDAL: secretVersionV2BridgeDAL, secretVersionTagDAL: secretVersionTagV2BridgeDAL, folderCommitService, + actor: { + type: ActorType.PLATFORM + }, secretTagDAL, resourceMetadataDAL }); diff --git a/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts b/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts index ec92a736f..95094f767 100644 --- a/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts +++ b/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts @@ -347,7 +347,7 @@ export const secretRotationQueueFactory = ({ actor: { type: ActorType.PLATFORM }, - message: "Secret rotation", + message: "Changed by Secret rotation", folderId: secretVersions[0].folderId, changes: secretVersions.map((sv) => ({ type: "add", diff --git a/backend/src/services/external-migration/external-migration-fns.ts b/backend/src/services/external-migration/external-migration-fns.ts index f3c2326de..645d58463 100644 --- a/backend/src/services/external-migration/external-migration-fns.ts +++ b/backend/src/services/external-migration/external-migration-fns.ts @@ -623,7 +623,7 @@ export const importDataIntoInfisicalFn = async ({ id: actorId } }, - message: "External migration", + message: "Changed by external migration", folderId: parentEnv.rootFolderId, changes: [ { diff --git a/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts b/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts index 5471f9b61..c598a8ea0 100644 --- a/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts +++ b/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts @@ -3,7 +3,7 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; import { TableName, TFolderCheckpoints } from "@app/db/schemas"; import { DatabaseError } from "@app/lib/errors"; -import { ormify, selectAllTableCols } from "@app/lib/knex"; +import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex"; export type TFolderCheckpointDALFactory = ReturnType; @@ -34,7 +34,8 @@ export const folderCheckpointDALFactory = (db: TDbClient) => { try { let query = (tx || db.replicaNode())(TableName.FolderCheckpoint) .join(TableName.FolderCommit, `${TableName.FolderCheckpoint}.folderCommitId`, `${TableName.FolderCommit}.id`) - .where({ folderId }) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ folderId }, TableName.FolderCommit)) .select(selectAllTableCols(TableName.FolderCheckpoint)) .select( db.ref("actorMetadata").withSchema(TableName.FolderCommit), @@ -60,7 +61,8 @@ export const folderCheckpointDALFactory = (db: TDbClient) => { try { const doc = await (tx || db.replicaNode())(TableName.FolderCheckpoint) .join(TableName.FolderCommit, `${TableName.FolderCheckpoint}.folderCommitId`, `${TableName.FolderCommit}.id`) - .where({ folderId }) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ folderId }, TableName.FolderCommit)) .select(selectAllTableCols(TableName.FolderCheckpoint)) .select( db.ref("actorMetadata").withSchema(TableName.FolderCommit), diff --git a/backend/src/services/folder-commit/folder-commit-dal.ts b/backend/src/services/folder-commit/folder-commit-dal.ts index c6888f8cd..1b454be02 100644 --- a/backend/src/services/folder-commit/folder-commit-dal.ts +++ b/backend/src/services/folder-commit/folder-commit-dal.ts @@ -13,10 +13,10 @@ export const folderCommitDALFactory = (db: TDbClient) => { const findByFolderId = async (folderId: string, tx?: Knex): Promise => { try { - const docs = await (tx || db.replicaNode())(TableName.FolderCommit) + const docs = await (tx || db.replicaNode())(TableName.FolderCommit) .where({ folderId }) .select(selectAllTableCols(TableName.FolderCommit)) - .orderBy("date", "desc"); + .orderBy("createdAt", "desc"); return docs; } catch (error) { throw new DatabaseError({ error, name: "FindByFolderId" }); @@ -25,7 +25,7 @@ export const folderCommitDALFactory = (db: TDbClient) => { const findById = async (id: string, tx?: Knex): Promise => { try { - const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) .where({ id }) .select(selectAllTableCols(TableName.FolderCommit)) .first(); @@ -37,10 +37,10 @@ export const folderCommitDALFactory = (db: TDbClient) => { const findLatestCommit = async (folderId: string, tx?: Knex): Promise => { try { - const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) .where({ folderId }) .select(selectAllTableCols(TableName.FolderCommit)) - .orderBy("date", "desc") + .orderBy("createdAt", "desc") .first(); return doc; } catch (error) { diff --git a/backend/src/services/folder-commit/folder-commit-service.ts b/backend/src/services/folder-commit/folder-commit-service.ts index c82de598c..6a1dd2c89 100644 --- a/backend/src/services/folder-commit/folder-commit-service.ts +++ b/backend/src/services/folder-commit/folder-commit-service.ts @@ -77,18 +77,15 @@ export const folderCommitServiceFactory = ({ }, tx ); - for (const change of data.changes) { - // eslint-disable-next-line no-await-in-loop - await folderCommitChangesDAL.create( - { - folderCommitId: newCommit.id, - changeType: change.type, - secretVersionId: change.secretVersionId, - folderVersionId: change.folderVersionId - }, - tx - ); - } + await folderCommitChangesDAL.insertMany( + data.changes.map((change) => ({ + folderCommitId: newCommit.id, + changeType: change.type, + secretVersionId: change.secretVersionId, + folderVersionId: change.folderVersionId + })), + tx + ); return newCommit; } catch (error) { @@ -96,7 +93,7 @@ export const folderCommitServiceFactory = ({ } }; - // Add a change to a commit and trigger checkpoints as needed + // Add a change to an existing commit const addCommitChange = async (data: TCommitChangeDTO, tx?: Knex) => { try { if (!data.secretVersionId && !data.folderVersionId) { diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts index b9ae74208..f4171b1a7 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts @@ -182,7 +182,7 @@ export type TFnSecretBulkInsert = { folderCommitService: Pick; actor?: { type: string; - actorId: string; + actorId?: string; }; }; @@ -211,7 +211,7 @@ export type TFnSecretBulkUpdate = { folderCommitService: Pick; actor?: { type: string; - actorId: string; + actorId?: string; }; tx?: Knex; }; From e8d424bbb0961a44b94f45313ffb0e710e1ebead Mon Sep 17 00:00:00 2001 From: carlosmonastyrski Date: Thu, 8 May 2025 09:41:01 -0300 Subject: [PATCH 4/9] PIT: Add initialization and checkpoint logic --- ...010_pit-projects-commits-initialization.ts | 26 +++ backend/src/db/migrations/utils/services.ts | 41 +++++ backend/src/lib/config/env.ts | 3 + backend/src/server/routes/index.ts | 9 +- .../folder-commit/folder-commit-dal.ts | 38 +++-- .../folder-commit/folder-commit-service.ts | 153 +++++++++++++++++- .../secret-folder/secret-folder-dal.ts | 14 +- 7 files changed, 260 insertions(+), 24 deletions(-) create mode 100644 backend/src/db/migrations/20250507185010_pit-projects-commits-initialization.ts diff --git a/backend/src/db/migrations/20250507185010_pit-projects-commits-initialization.ts b/backend/src/db/migrations/20250507185010_pit-projects-commits-initialization.ts new file mode 100644 index 000000000..7ad865b10 --- /dev/null +++ b/backend/src/db/migrations/20250507185010_pit-projects-commits-initialization.ts @@ -0,0 +1,26 @@ +import { Knex } from "knex"; + +import { ProjectType, TableName } from "../schemas"; +import { getMigrationPITServices } from "./utils/services"; + +export async function up(knex: Knex): Promise { + const hasFolderCommitTable = await knex.schema.hasTable(TableName.FolderCommit); + if (hasFolderCommitTable) { + const { folderCommitService } = await getMigrationPITServices({ db: knex }); + const projects = await knex(TableName.Project).where({ version: 3, type: ProjectType.SecretManager }).select("id"); + await knex.transaction(async (tx) => { + for (const project of projects) { + // eslint-disable-next-line no-await-in-loop + await folderCommitService.initializeProject(project.id, tx); + } + }); + } +} + +export async function down(knex: Knex): Promise { + const hasFolderCommitTable = await knex.schema.hasTable(TableName.FolderCommit); + if (hasFolderCommitTable) { + // delete all existing entries + await knex(TableName.FolderCommit).del(); + } +} diff --git a/backend/src/db/migrations/utils/services.ts b/backend/src/db/migrations/utils/services.ts index 731f703e2..610112618 100644 --- a/backend/src/db/migrations/utils/services.ts +++ b/backend/src/db/migrations/utils/services.ts @@ -3,12 +3,23 @@ import { Knex } from "knex"; import { initializeHsmModule } from "@app/ee/services/hsm/hsm-fns"; import { hsmServiceFactory } from "@app/ee/services/hsm/hsm-service"; import { TKeyStoreFactory } from "@app/keystore/keystore"; +import { folderCheckpointDALFactory } from "@app/services/folder-checkpoint/folder-checkpoint-dal"; +import { folderCheckpointResourcesDALFactory } from "@app/services/folder-checkpoint-resources/folder-checkpoint-resources-dal"; +import { folderCommitDALFactory } from "@app/services/folder-commit/folder-commit-dal"; +import { folderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; +import { folderCommitChangesDALFactory } from "@app/services/folder-commit-changes/folder-commit-changes-dal"; +import { folderTreeCheckpointDALFactory } from "@app/services/folder-tree-checkpoint/folder-tree-checkpoint-dal"; +import { identityDALFactory } from "@app/services/identity/identity-dal"; import { internalKmsDALFactory } from "@app/services/kms/internal-kms-dal"; import { kmskeyDALFactory } from "@app/services/kms/kms-key-dal"; import { kmsRootConfigDALFactory } from "@app/services/kms/kms-root-config-dal"; import { kmsServiceFactory } from "@app/services/kms/kms-service"; import { orgDALFactory } from "@app/services/org/org-dal"; import { projectDALFactory } from "@app/services/project/project-dal"; +import { secretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; +import { secretFolderVersionDALFactory } from "@app/services/secret-folder/secret-folder-version-dal"; +import { secretVersionV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-version-dal"; +import { userDALFactory } from "@app/services/user/user-dal"; import { TMigrationEnvConfig } from "./env-config"; @@ -50,3 +61,33 @@ export const getMigrationEncryptionServices = async ({ envConfig, db, keyStore } return { kmsService }; }; + +export const getMigrationPITServices = async ({ db }: { db: Knex }) => { + const projectDAL = projectDALFactory(db); + const folderCommitDAL = folderCommitDALFactory(db); + const folderCommitChangesDAL = folderCommitChangesDALFactory(db); + const folderCheckpointDAL = folderCheckpointDALFactory(db); + const folderTreeCheckpointDAL = folderTreeCheckpointDALFactory(db); + const userDAL = userDALFactory(db); + const identityDAL = identityDALFactory(db); + const folderDAL = secretFolderDALFactory(db); + const folderVersionDAL = secretFolderVersionDALFactory(db); + const secretVersionV2BridgeDAL = secretVersionV2BridgeDALFactory(db); + const folderCheckpointResourcesDAL = folderCheckpointResourcesDALFactory(db); + + const folderCommitService = folderCommitServiceFactory({ + folderCommitDAL, + folderCommitChangesDAL, + folderCheckpointDAL, + folderTreeCheckpointDAL, + userDAL, + identityDAL, + folderDAL, + folderVersionDAL, + secretVersionV2BridgeDAL, + projectDAL, + folderCheckpointResourcesDAL + }); + + return { folderCommitService }; +}; diff --git a/backend/src/lib/config/env.ts b/backend/src/lib/config/env.ts index 907884433..38e252b2a 100644 --- a/backend/src/lib/config/env.ts +++ b/backend/src/lib/config/env.ts @@ -228,6 +228,9 @@ const envSchema = z DATADOG_SERVICE: zpStr(z.string().optional().default("infisical-core")), DATADOG_HOSTNAME: zpStr(z.string().optional()), + // PIT + CHECKPOINT_WINDOW: zpStr(z.string().optional().default("10")), + /* CORS ----------------------------------------------------------------------------- */ CORS_ALLOWED_ORIGINS: zpStr( diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 5ba8954fe..4367f69c5 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -141,6 +141,7 @@ import { externalGroupOrgRoleMappingServiceFactory } from "@app/services/externa import { externalMigrationQueueFactory } from "@app/services/external-migration/external-migration-queue"; import { externalMigrationServiceFactory } from "@app/services/external-migration/external-migration-service"; import { folderCheckpointDALFactory } from "@app/services/folder-checkpoint/folder-checkpoint-dal"; +import { folderCheckpointResourcesDALFactory } from "@app/services/folder-checkpoint-resources/folder-checkpoint-resources-dal"; import { folderCommitDALFactory } from "@app/services/folder-commit/folder-commit-dal"; import { folderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { folderCommitChangesDALFactory } from "@app/services/folder-commit-changes/folder-commit-changes-dal"; @@ -559,6 +560,7 @@ export const registerRoutes = async ( const folderCommitChangesDAL = folderCommitChangesDALFactory(db); const folderCheckpointDAL = folderCheckpointDALFactory(db); + const folderCheckpointResourcesDAL = folderCheckpointResourcesDALFactory(db); const folderTreeCheckpointDAL = folderTreeCheckpointDALFactory(db); const folderCommitDAL = folderCommitDALFactory(db); const folderCommitService = folderCommitServiceFactory({ @@ -567,7 +569,12 @@ export const registerRoutes = async ( folderCheckpointDAL, folderTreeCheckpointDAL, userDAL, - identityDAL + identityDAL, + folderDAL, + folderVersionDAL, + secretVersionV2BridgeDAL, + projectDAL, + folderCheckpointResourcesDAL }); const scimService = scimServiceFactory({ licenseService, diff --git a/backend/src/services/folder-commit/folder-commit-dal.ts b/backend/src/services/folder-commit/folder-commit-dal.ts index 1b454be02..68394ddb3 100644 --- a/backend/src/services/folder-commit/folder-commit-dal.ts +++ b/backend/src/services/folder-commit/folder-commit-dal.ts @@ -23,24 +23,12 @@ export const folderCommitDALFactory = (db: TDbClient) => { } }; - const findById = async (id: string, tx?: Knex): Promise => { - try { - const doc = await (tx || db.replicaNode())(TableName.FolderCommit) - .where({ id }) - .select(selectAllTableCols(TableName.FolderCommit)) - .first(); - return doc; - } catch (error) { - throw new DatabaseError({ error, name: "FindById" }); - } - }; - const findLatestCommit = async (folderId: string, tx?: Knex): Promise => { try { const doc = await (tx || db.replicaNode())(TableName.FolderCommit) .where({ folderId }) .select(selectAllTableCols(TableName.FolderCommit)) - .orderBy("createdAt", "desc") + .orderBy("commitId", "desc") .first(); return doc; } catch (error) { @@ -48,10 +36,30 @@ export const folderCommitDALFactory = (db: TDbClient) => { } }; + const getNumberOfCommitsSince = async (folderId: string, folderCommitId: string, tx?: Knex): Promise => { + try { + const referencedCommit = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ id: folderCommitId }) + .select("commitId") + .first(); + + if (referencedCommit?.commitId) { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ folderId }) + .where("commitId", ">", referencedCommit.commitId) + .count(); + return Number(doc?.[0].count); + } + return 0; + } catch (error) { + throw new DatabaseError({ error, name: "getNumberOfCommitsSince" }); + } + }; + return { ...restOfOrm, findByFolderId, - findById, - findLatestCommit + findLatestCommit, + getNumberOfCommitsSince }; }; diff --git a/backend/src/services/folder-commit/folder-commit-service.ts b/backend/src/services/folder-commit/folder-commit-service.ts index 6a1dd2c89..df834b88e 100644 --- a/backend/src/services/folder-commit/folder-commit-service.ts +++ b/backend/src/services/folder-commit/folder-commit-service.ts @@ -1,28 +1,37 @@ import { Knex } from "knex"; +import { TSecretFolders } from "@app/db/schemas"; +import { getConfig } from "@app/lib/config/env"; import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors"; import { ActorType } from "../auth/auth-type"; import { TFolderCheckpointDALFactory } from "../folder-checkpoint/folder-checkpoint-dal"; +import { TFolderCheckpointResourcesDALFactory } from "../folder-checkpoint-resources/folder-checkpoint-resources-dal"; import { TFolderCommitChangesDALFactory } from "../folder-commit-changes/folder-commit-changes-dal"; import { TFolderTreeCheckpointDALFactory } from "../folder-tree-checkpoint/folder-tree-checkpoint-dal"; import { TIdentityDALFactory } from "../identity/identity-dal"; +import { TProjectDALFactory } from "../project/project-dal"; +import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal"; +import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal"; import { TUserDALFactory } from "../user/user-dal"; import { TFolderCommitDALFactory } from "./folder-commit-dal"; type TFolderCommitServiceFactoryDep = { folderCommitDAL: Pick< TFolderCommitDALFactory, - "create" | "findById" | "findByFolderId" | "findLatestCommit" | "transaction" + "create" | "findById" | "findByFolderId" | "findLatestCommit" | "transaction" | "getNumberOfCommitsSince" >; folderCommitChangesDAL: Pick; folderCheckpointDAL: Pick; - folderTreeCheckpointDAL: Pick< - TFolderTreeCheckpointDALFactory, - "create" | "findByProjectId" | "findLatestByProjectId" - >; + folderCheckpointResourcesDAL: Pick; + folderTreeCheckpointDAL: Pick; userDAL: Pick; identityDAL: Pick; + folderDAL: Pick; + folderVersionDAL: Pick; + secretVersionV2BridgeDAL: Pick; + projectDAL: Pick; }; export type TCreateCommitDTO = { @@ -54,9 +63,72 @@ export const folderCommitServiceFactory = ({ folderCommitChangesDAL, folderCheckpointDAL, folderTreeCheckpointDAL, + folderCheckpointResourcesDAL, userDAL, - identityDAL + identityDAL, + folderDAL, + folderVersionDAL, + secretVersionV2BridgeDAL, + projectDAL }: TFolderCommitServiceFactoryDep) => { + const appCfg = getConfig(); + + const getFolderResources = async (folderId: string, tx?: Knex) => { + const resources = []; + const subFolders = await folderDAL.findByParentId(folderId, tx); + if (subFolders.length > 0) { + const subFolderIds = subFolders.map((folder) => folder.id); + const folderVersions = await folderVersionDAL.findLatestFolderVersions(subFolderIds, tx); + resources.push(...Object.values(folderVersions).map((folderVersion) => ({ folderVersionId: folderVersion.id }))); + } + const secretVersions = await secretVersionV2BridgeDAL.findLatestVersionByFolderId(folderId, tx); + if (secretVersions.length > 0) { + resources.push(...secretVersions.map((secretVersion) => ({ secretVersionId: secretVersion.id }))); + } + return resources; + }; + + const createFolderCheckpoint = async ({ + folderId, + folderCommitId, + force = false, + tx + }: { + folderId: string; + folderCommitId?: string; + force?: boolean; + tx?: Knex; + }) => { + let latestCommitId = folderCommitId; + if (!latestCommitId) { + latestCommitId = (await folderCheckpointDAL.findLatestByFolderId(folderId, tx))?.folderCommitId; + } + if (!latestCommitId) { + throw new BadRequestError({ message: "Latest commit ID not found" }); + return; + } + if (!force) { + const commitsSinceLastCheckpoint = await folderCommitDAL.getNumberOfCommitsSince(folderId, latestCommitId, tx); + if (commitsSinceLastCheckpoint < Number(appCfg.CHECKPOINT_WINDOW)) { + return; + } + } + const checkpointResources = await getFolderResources(folderId, tx); + + if (checkpointResources.length > 0) { + const newCheckpoint = await folderCheckpointDAL.create( + { + folderCommitId: latestCommitId + }, + tx + ); + await folderCheckpointResourcesDAL.insertMany( + checkpointResources.map((resource) => ({ folderCheckpointId: newCheckpoint.id, ...resource })), + tx + ); + } + }; + const createCommit = async (data: TCreateCommitDTO, tx?: Knex) => { const metadata = data.actor.metadata || {}; try { @@ -87,6 +159,7 @@ export const folderCommitServiceFactory = ({ tx ); + await createFolderCheckpoint({ folderId: data.folderId, folderCommitId: newCommit.id, tx }); return newCommit; } catch (error) { throw new DatabaseError({ error, name: "CreateCommit" }); @@ -149,6 +222,69 @@ export const folderCommitServiceFactory = ({ return folderTreeCheckpointDAL.findLatestByProjectId(projectId, tx); }; + const initializeFolder = async (folderId: string, tx?: Knex) => { + const folderResources = await getFolderResources(folderId, tx); + const changes = folderResources.map((resource) => ({ type: "add", ...resource })); + if (changes.length > 0) { + const newCommit = await createCommit( + { + actor: { + type: ActorType.PLATFORM + }, + message: "Initialized folder", + folderId, + changes + }, + tx + ); + await createFolderCheckpoint({ folderId, folderCommitId: newCommit.id, force: true, tx }); + } + }; + + function sortFoldersByHierarchy(folders: TSecretFolders[]) { + // Create a map for quick lookup of children by parent ID + const childrenMap: Map = new Map(); + folders.forEach((folder) => { + const { parentId } = folder; + if (!childrenMap.has(parentId || null)) { + childrenMap.set(parentId || null, []); + } + childrenMap.get(parentId || null)?.push(folder); + }); + + // Start with root folders (null parentId) + const result = []; + const rootFolders = childrenMap.get(null) || []; + + // Process each level of the hierarchy + let currentLevel = rootFolders; + result.push(...currentLevel); + + while (currentLevel.length > 0) { + const nextLevel = []; + + for (const folder of currentLevel) { + const children = childrenMap.get(folder.id) || []; + nextLevel.push(...children); + } + + result.push(...nextLevel); + currentLevel = nextLevel; + } + + return result; + } + + const initializeProject = async (projectId: string, tx?: Knex) => { + const project = await projectDAL.findById(projectId, tx); + if (!project) { + throw new NotFoundError({ message: `Project with ID ${projectId} not found` }); + } + const folders = await folderDAL.findByProjectId(projectId, tx); + const sortedFolders = sortFoldersByHierarchy(folders); + await Promise.all(sortedFolders.map((folder) => initializeFolder(folder.id, tx))); + }; + return { createCommit, addCommitChange, @@ -158,7 +294,10 @@ export const folderCommitServiceFactory = ({ getCheckpointsByFolderId, getLatestCheckpoint, getTreeCheckpointsByProjectId, - getLatestTreeCheckpoint + getLatestTreeCheckpoint, + initializeFolder, + initializeProject, + createFolderCheckpoint }; }; diff --git a/backend/src/services/secret-folder/secret-folder-dal.ts b/backend/src/services/secret-folder/secret-folder-dal.ts index e136c5a50..d6d5c4aad 100644 --- a/backend/src/services/secret-folder/secret-folder-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-dal.ts @@ -488,6 +488,17 @@ export const secretFolderDALFactory = (db: TDbClient) => { } }; + const findByParentId = async (parentId: string, tx?: Knex) => { + try { + const folders = await (tx || db.replicaNode())(TableName.SecretFolder) + .where({ parentId }) + .select(selectAllTableCols(TableName.SecretFolder)); + return folders; + } catch (error) { + throw new DatabaseError({ error, name: "findByParentId" }); + } + }; + return { ...secretFolderOrm, update, @@ -499,6 +510,7 @@ export const secretFolderDALFactory = (db: TDbClient) => { findClosestFolder, findByProjectId, findByMultiEnv, - findByEnvsDeep + findByEnvsDeep, + findByParentId }; }; From 78d5bc823d364fd38e7549b60a49d4b6315590b0 Mon Sep 17 00:00:00 2001 From: carlosmonastyrski Date: Fri, 9 May 2025 09:20:17 -0300 Subject: [PATCH 5/9] PIT: Add folder reconstruction functions --- backend/src/ee/routes/v1/index.ts | 2 + backend/src/ee/routes/v1/pit-router.ts | 27 ++ .../folder-checkpoint-resources-dal.ts | 36 ++- .../folder-checkpoint-dal.ts | 62 ++++- .../folder-commit/folder-commit-dal.ts | 82 ++++++- .../folder-commit/folder-commit-service.ts | 232 +++++++++++++++--- 6 files changed, 397 insertions(+), 44 deletions(-) create mode 100644 backend/src/ee/routes/v1/pit-router.ts diff --git a/backend/src/ee/routes/v1/index.ts b/backend/src/ee/routes/v1/index.ts index 0b8c78586..9ba7f734e 100644 --- a/backend/src/ee/routes/v1/index.ts +++ b/backend/src/ee/routes/v1/index.ts @@ -18,6 +18,7 @@ import { registerLdapRouter } from "./ldap-router"; import { registerLicenseRouter } from "./license-router"; import { registerOidcRouter } from "./oidc-router"; import { registerOrgRoleRouter } from "./org-role-router"; +import { registerPITRouter } from "./pit-router"; import { registerProjectRoleRouter } from "./project-role-router"; import { registerProjectRouter } from "./project-router"; import { registerRateLimitRouter } from "./rate-limit-router"; @@ -53,6 +54,7 @@ export const registerV1EERoutes = async (server: FastifyZodProvider) => { { prefix: "/workspace" } ); await server.register(registerSnapshotRouter, { prefix: "/secret-snapshot" }); + await server.register(registerPITRouter, { prefix: "/pit" }); await server.register(registerSecretApprovalPolicyRouter, { prefix: "/secret-approvals" }); await server.register(registerSecretApprovalRequestRouter, { prefix: "/secret-approval-requests" diff --git a/backend/src/ee/routes/v1/pit-router.ts b/backend/src/ee/routes/v1/pit-router.ts new file mode 100644 index 000000000..0045a7efe --- /dev/null +++ b/backend/src/ee/routes/v1/pit-router.ts @@ -0,0 +1,27 @@ +import { z } from "zod"; + +import { readLimit } from "@app/server/config/rateLimiter"; + +export const registerPITRouter = async (server: FastifyZodProvider) => { + server.route({ + method: "GET", + url: "/diff", + config: { + rateLimit: readLimit + }, + schema: { + querystring: z.object({ + fromCommit: z.string().trim(), + toCommit: z.string().trim() + }), + response: { + 200: z.any() + } + }, + handler: async (req) => { + const backup = await server.services.folderCommit.compareFolderStates(req.query.fromCommit, req.query.toCommit); + + return backup; + } + }); +}; diff --git a/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts b/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts index 2457abe14..7a36ec641 100644 --- a/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts +++ b/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts @@ -1,24 +1,52 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; -import { TableName, TFolderCheckpointResources, TFolderCheckpoints } from "@app/db/schemas"; +import { + TableName, + TFolderCheckpointResources, + TFolderCheckpoints, + TSecretFolderVersions, + TSecretVersionsV2 +} from "@app/db/schemas"; import { DatabaseError } from "@app/lib/errors"; import { ormify, selectAllTableCols } from "@app/lib/knex"; export type TFolderCheckpointResourcesDALFactory = ReturnType; -type ResourceWithCheckpointInfo = TFolderCheckpointResources & { +export type ResourceWithCheckpointInfo = TFolderCheckpointResources & { folderCommitId: string; }; export const folderCheckpointResourcesDALFactory = (db: TDbClient) => { const folderCheckpointResourcesOrm = ormify(db, TableName.FolderCheckpointResources); - const findByCheckpointId = async (folderCheckpointId: string, tx?: Knex): Promise => { + const findByCheckpointId = async ( + folderCheckpointId: string, + tx?: Knex + ): Promise< + (TFolderCheckpointResources & { + referencedSecretId?: string; + referencedFolderId?: string; + })[] + > => { try { const docs = await (tx || db.replicaNode())(TableName.FolderCheckpointResources) .where({ folderCheckpointId }) - .select(selectAllTableCols(TableName.FolderCheckpointResources)); + .leftJoin( + TableName.SecretVersionV2, + `${TableName.FolderCheckpointResources}.secretVersionId`, + `${TableName.SecretVersionV2}.id` + ) + .leftJoin( + TableName.SecretFolderVersion, + `${TableName.FolderCheckpointResources}.folderVersionId`, + `${TableName.SecretFolderVersion}.id` + ) + .select(selectAllTableCols(TableName.FolderCheckpointResources)) + .select( + db.ref("secretId").withSchema(TableName.SecretVersionV2).as("referencedSecretId"), + db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("referencedFolderId") + ); return docs; } catch (error) { throw new DatabaseError({ error, name: "FindByCheckpointId" }); diff --git a/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts b/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts index c598a8ea0..5ebb39f5b 100644 --- a/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts +++ b/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts @@ -1,7 +1,7 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; -import { TableName, TFolderCheckpoints } from "@app/db/schemas"; +import { TableName, TFolderCheckpoints, TFolderCommits } from "@app/db/schemas"; import { DatabaseError } from "@app/lib/errors"; import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex"; @@ -33,7 +33,11 @@ export const folderCheckpointDALFactory = (db: TDbClient) => { const findByFolderId = async (folderId: string, limit?: number, tx?: Knex): Promise => { try { let query = (tx || db.replicaNode())(TableName.FolderCheckpoint) - .join(TableName.FolderCommit, `${TableName.FolderCheckpoint}.folderCommitId`, `${TableName.FolderCommit}.id`) + .join( + TableName.FolderCommit, + `${TableName.FolderCheckpoint}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) // eslint-disable-next-line @typescript-eslint/no-misused-promises .where(buildFindFilter({ folderId }, TableName.FolderCommit)) .select(selectAllTableCols(TableName.FolderCheckpoint)) @@ -50,8 +54,7 @@ export const folderCheckpointDALFactory = (db: TDbClient) => { query = query.limit(limit); } - const docs = await query; - return docs; + return await query; } catch (error) { throw new DatabaseError({ error, name: "FindByFolderId" }); } @@ -60,7 +63,11 @@ export const folderCheckpointDALFactory = (db: TDbClient) => { const findLatestByFolderId = async (folderId: string, tx?: Knex): Promise => { try { const doc = await (tx || db.replicaNode())(TableName.FolderCheckpoint) - .join(TableName.FolderCommit, `${TableName.FolderCheckpoint}.folderCommitId`, `${TableName.FolderCommit}.id`) + .join( + TableName.FolderCommit, + `${TableName.FolderCheckpoint}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) // eslint-disable-next-line @typescript-eslint/no-misused-promises .where(buildFindFilter({ folderId }, TableName.FolderCommit)) .select(selectAllTableCols(TableName.FolderCheckpoint)) @@ -79,10 +86,53 @@ export const folderCheckpointDALFactory = (db: TDbClient) => { } }; + const findNearestCheckpoint = async ( + folderCommitId: string, + tx?: Knex + ): Promise<(CheckpointWithCommitInfo & { commitId: number }) | undefined> => { + try { + // First, get the commit info to find the folder ID and commit sequence number + const commit = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ id: folderCommitId }) + .select("commitId", "folderId") + .first(); + + if (!commit) { + return undefined; + } + + // Get the checkpoint with the highest commitId that's still less than or equal to our commit + const nearestCheckpoint = await (tx || db.replicaNode())(TableName.FolderCheckpoint) + .join( + TableName.FolderCommit, + `${TableName.FolderCheckpoint}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) + .where(`${TableName.FolderCommit}.folderId`, commit.folderId) + .where(`${TableName.FolderCommit}.commitId`, "<=", commit.commitId.toString()) + .select(selectAllTableCols(TableName.FolderCheckpoint)) + .select( + db.ref("actorMetadata").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("commitId").withSchema(TableName.FolderCommit), + db.ref("createdAt").withSchema(TableName.FolderCommit).as("commitDate"), + db.ref("folderId").withSchema(TableName.FolderCommit) + ) + .orderBy(`${TableName.FolderCommit}.commitId`, "desc") + .first(); + + return nearestCheckpoint; + } catch (error) { + throw new DatabaseError({ error, name: "FindNearestCheckpoint" }); + } + }; + return { ...folderCheckpointOrm, findByCommitId, findByFolderId, - findLatestByFolderId + findLatestByFolderId, + findNearestCheckpoint }; }; diff --git a/backend/src/services/folder-commit/folder-commit-dal.ts b/backend/src/services/folder-commit/folder-commit-dal.ts index 68394ddb3..f52da95b5 100644 --- a/backend/src/services/folder-commit/folder-commit-dal.ts +++ b/backend/src/services/folder-commit/folder-commit-dal.ts @@ -1,9 +1,15 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; -import { TableName, TFolderCommits } from "@app/db/schemas"; +import { + TableName, + TFolderCommitChanges, + TFolderCommits, + TSecretFolderVersions, + TSecretVersionsV2 +} from "@app/db/schemas"; import { DatabaseError } from "@app/lib/errors"; -import { ormify, selectAllTableCols } from "@app/lib/knex"; +import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex"; export type TFolderCommitDALFactory = ReturnType; @@ -56,10 +62,80 @@ export const folderCommitDALFactory = (db: TDbClient) => { } }; + const findCommitsToRecreate = async ( + folderId: string, + targetCommitNumber: number, + checkpointCommitNumber: number, + tx?: Knex + ): Promise< + (TFolderCommits & { + changes: (TFolderCommitChanges & { referencedSecretId?: string; referencedFolderId?: string })[]; + })[] + > => { + try { + // First get all the commits in the range + const commits = await (tx || db.replicaNode())(TableName.FolderCommit) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ folderId }, TableName.FolderCommit)) + .andWhere(`${TableName.FolderCommit}.commitId`, ">", checkpointCommitNumber) + .andWhere(`${TableName.FolderCommit}.commitId`, "<=", targetCommitNumber) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy(`${TableName.FolderCommit}.commitId`, "asc"); + + // If no commits found, return empty array + if (!commits.length) { + return []; + } + + // Get all the commit IDs + const commitIds = commits.map((commit) => commit.id); + + // Get all changes for these commits in a single query + const allChanges = await (tx || db.replicaNode())(TableName.FolderCommitChanges) + .whereIn("folderCommitId", commitIds) + .leftJoin( + TableName.SecretVersionV2, + `${TableName.FolderCommitChanges}.secretVersionId`, + `${TableName.SecretVersionV2}.id` + ) + .leftJoin( + TableName.SecretFolderVersion, + `${TableName.FolderCommitChanges}.folderVersionId`, + `${TableName.SecretFolderVersion}.id` + ) + .select(selectAllTableCols(TableName.FolderCommitChanges)) + .select( + db.ref("secretId").withSchema(TableName.SecretVersionV2).as("referencedSecretId"), + db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("referencedFolderId") + ); + + // Organize changes by commit ID + const changesByCommitId = allChanges.reduce( + (acc, change) => { + if (!acc[change.folderCommitId]) { + acc[change.folderCommitId] = []; + } + acc[change.folderCommitId].push(change); + return acc; + }, + {} as Record + ); + + // Attach changes to each commit + return commits.map((commit) => ({ + ...commit, + changes: changesByCommitId[commit.id] || [] + })); + } catch (error) { + throw new DatabaseError({ error, name: "FindCommitsToRecreate" }); + } + }; + return { ...restOfOrm, findByFolderId, findLatestCommit, - getNumberOfCommitsSince + getNumberOfCommitsSince, + findCommitsToRecreate }; }; diff --git a/backend/src/services/folder-commit/folder-commit-service.ts b/backend/src/services/folder-commit/folder-commit-service.ts index df834b88e..c39aaa7e7 100644 --- a/backend/src/services/folder-commit/folder-commit-service.ts +++ b/backend/src/services/folder-commit/folder-commit-service.ts @@ -1,3 +1,4 @@ +/* eslint-disable no-await-in-loop */ import { Knex } from "knex"; import { TSecretFolders } from "@app/db/schemas"; @@ -20,17 +21,32 @@ import { TFolderCommitDALFactory } from "./folder-commit-dal"; type TFolderCommitServiceFactoryDep = { folderCommitDAL: Pick< TFolderCommitDALFactory, - "create" | "findById" | "findByFolderId" | "findLatestCommit" | "transaction" | "getNumberOfCommitsSince" + | "create" + | "findById" + | "findByFolderId" + | "findLatestCommit" + | "transaction" + | "getNumberOfCommitsSince" + | "findCommitsToRecreate" >; folderCommitChangesDAL: Pick; - folderCheckpointDAL: Pick; - folderCheckpointResourcesDAL: Pick; + folderCheckpointDAL: Pick< + TFolderCheckpointDALFactory, + "create" | "findByFolderId" | "findLatestByFolderId" | "findNearestCheckpoint" + >; + folderCheckpointResourcesDAL: Pick; folderTreeCheckpointDAL: Pick; userDAL: Pick; identityDAL: Pick; folderDAL: Pick; - folderVersionDAL: Pick; - secretVersionV2BridgeDAL: Pick; + folderVersionDAL: Pick< + TSecretFolderVersionDALFactory, + "findLatestFolderVersions" | "findById" | "deleteById" | "create" | "updateById" + >; + secretVersionV2BridgeDAL: Pick< + TSecretVersionV2DALFactory, + "findLatestVersionByFolderId" | "findById" | "deleteById" | "create" | "updateById" + >; projectDAL: Pick; }; @@ -100,15 +116,20 @@ export const folderCommitServiceFactory = ({ tx?: Knex; }) => { let latestCommitId = folderCommitId; + const latestCheckpoint = await folderCheckpointDAL.findLatestByFolderId(folderId, tx); if (!latestCommitId) { - latestCommitId = (await folderCheckpointDAL.findLatestByFolderId(folderId, tx))?.folderCommitId; + latestCommitId = (await folderCommitDAL.findLatestCommit(folderId, tx))?.id; } if (!latestCommitId) { throw new BadRequestError({ message: "Latest commit ID not found" }); return; } - if (!force) { - const commitsSinceLastCheckpoint = await folderCommitDAL.getNumberOfCommitsSince(folderId, latestCommitId, tx); + if (!force && latestCheckpoint) { + const commitsSinceLastCheckpoint = await folderCommitDAL.getNumberOfCommitsSince( + folderId, + latestCheckpoint.folderCommitId, + tx + ); if (commitsSinceLastCheckpoint < Number(appCfg.CHECKPOINT_WINDOW)) { return; } @@ -129,6 +150,175 @@ export const folderCommitServiceFactory = ({ } }; + const reconstructFolderState = async ( + folderCommitId: string, + tx?: Knex + ): Promise<{ type: string; id: string; versionId: string }[]> => { + const targetCommit = await folderCommitDAL.findById(folderCommitId, tx); + if (!targetCommit) { + throw new NotFoundError({ message: `Commit with ID ${folderCommitId} not found` }); + } + + const nearestCheckpoint = await folderCheckpointDAL.findNearestCheckpoint(folderCommitId, tx); + if (!nearestCheckpoint) { + throw new NotFoundError({ message: `Nearest checkpoint not found for commit ${folderCommitId}` }); + } + + const checkpointResources = await folderCheckpointResourcesDAL.findByCheckpointId(nearestCheckpoint.id, tx); + + const folderState: Record = {}; + + // Add all checkpoint resources to initial state + checkpointResources.forEach((resource) => { + if (resource.secretVersionId && resource.referencedSecretId) { + folderState[`secret-${resource.referencedSecretId}`] = { + type: "secret", + id: resource.referencedSecretId, + versionId: resource.secretVersionId + }; + } else if (resource.folderVersionId && resource.referencedFolderId) { + folderState[`folder-${resource.referencedFolderId}`] = { + type: "folder", + id: resource.referencedFolderId, + versionId: resource.folderVersionId + }; + } + }); + + const commitsToRecreate = await folderCommitDAL.findCommitsToRecreate( + targetCommit.folderId, + targetCommit.commitId, + nearestCheckpoint.commitId, + tx + ); + + // Process commits to recreate final state + for (const commit of commitsToRecreate) { + // eslint-disable-next-line no-continue + if (!commit.changes) continue; + + for (const change of commit.changes) { + if (change.secretVersionId && change.referencedSecretId) { + const key = `secret-${change.referencedSecretId}`; + + if (change.changeType.toLowerCase() === "add") { + folderState[key] = { + type: "secret", + id: change.referencedSecretId, + versionId: change.secretVersionId + }; + } else if (change.changeType.toLowerCase() === "delete") { + delete folderState[key]; + } + } else if (change.folderVersionId && change.referencedFolderId) { + const key = `folder-${change.referencedFolderId}`; + + if (change.changeType.toLowerCase() === "add") { + folderState[key] = { + type: "folder", + id: change.referencedFolderId, + versionId: change.folderVersionId + }; + } else if (change.changeType.toLowerCase() === "delete") { + delete folderState[key]; + } + } + } + } + return Object.values(folderState); + }; + + const compareFolderStates = async (currentCommitId: string, targetCommitId: string, tx?: Knex) => { + // Reconstruct state for both commits + const currentState = await reconstructFolderState(currentCommitId, tx); + const targetState = await reconstructFolderState(targetCommitId, tx); + + // Create lookup maps for easier comparison + const currentMap: Record = {}; + const targetMap: Record = {}; + + // Build lookup map for current state + currentState.forEach((resource) => { + const key = `${resource.type}-${resource.id}`; + currentMap[key] = resource; + }); + + // Build lookup map for target state + targetState.forEach((resource) => { + const key = `${resource.type}-${resource.id}`; + targetMap[key] = resource; + }); + + // Track differences + const differences: { + type: string; + id: string; + versionId: string; + changeType: "create" | "update" | "delete"; + }[] = []; + + // Find deletes and updates (resources in current but not in target, or with different versions) + Object.keys(currentMap).forEach((key) => { + const currentResource = currentMap[key]; + const targetResource = targetMap[key]; + + if (!targetResource) { + // Resource exists in current but not in target - it's a delete + differences.push({ + type: currentResource.type, + id: currentResource.id, + versionId: currentResource.versionId, + changeType: "delete" + }); + } else if (currentResource.versionId !== targetResource.versionId) { + // Resource exists in both but with different versions - it's an update + differences.push({ + type: targetResource.type, + id: targetResource.id, + versionId: targetResource.versionId, + changeType: "update" + }); + } + // If versions are the same, it's unchanged - exclude from result + }); + + // Find creates (resources in target but not in current) + Object.keys(targetMap).forEach((key) => { + if (!currentMap[key]) { + const targetResource = targetMap[key]; + differences.push({ + type: targetResource.type, + id: targetResource.id, + versionId: targetResource.versionId, + changeType: "create" + }); + } + }); + + return differences; + }; + + // Add a change to an existing commit + const addCommitChange = async (data: TCommitChangeDTO, tx?: Knex) => { + try { + if (!data.secretVersionId && !data.folderVersionId) { + throw new BadRequestError({ message: "Either secretVersionId or folderVersionId must be provided" }); + } + + const commit = await folderCommitDAL.findById(data.folderCommitId, tx); + if (!commit) { + throw new NotFoundError({ message: `Commit with ID ${data.folderCommitId} not found` }); + } + + return await folderCommitChangesDAL.create(data, tx); + } catch (error) { + if (error instanceof NotFoundError || error instanceof BadRequestError) { + throw error; + } + throw new DatabaseError({ error, name: "AddCommitChange" }); + } + }; + const createCommit = async (data: TCreateCommitDTO, tx?: Knex) => { const metadata = data.actor.metadata || {}; try { @@ -159,34 +349,13 @@ export const folderCommitServiceFactory = ({ tx ); - await createFolderCheckpoint({ folderId: data.folderId, folderCommitId: newCommit.id, tx }); + await createFolderCheckpoint({ folderId: data.folderId, tx }); return newCommit; } catch (error) { throw new DatabaseError({ error, name: "CreateCommit" }); } }; - // Add a change to an existing commit - const addCommitChange = async (data: TCommitChangeDTO, tx?: Knex) => { - try { - if (!data.secretVersionId && !data.folderVersionId) { - throw new BadRequestError({ message: "Either secretVersionId or folderVersionId must be provided" }); - } - - const commit = await folderCommitDAL.findById(data.folderCommitId, tx); - if (!commit) { - throw new NotFoundError({ message: `Commit with ID ${data.folderCommitId} not found` }); - } - - return await folderCommitChangesDAL.create(data, tx); - } catch (error) { - if (error instanceof NotFoundError || error instanceof BadRequestError) { - throw error; - } - throw new DatabaseError({ error, name: "AddCommitChange" }); - } - }; - // Retrieve a commit by ID const getCommitById = async (id: string, tx?: Knex) => { return folderCommitDAL.findById(id, tx); @@ -297,7 +466,8 @@ export const folderCommitServiceFactory = ({ getLatestTreeCheckpoint, initializeFolder, initializeProject, - createFolderCheckpoint + createFolderCheckpoint, + compareFolderStates }; }; From 9e1d38a27ba64e9eba1c6f5a42adfd0309eb3c0e Mon Sep 17 00:00:00 2001 From: carlosmonastyrski Date: Fri, 9 May 2025 16:03:50 -0300 Subject: [PATCH 6/9] Add PIT rollback --- ...010_pit-projects-commits-initialization.ts | 5 +- backend/src/db/migrations/utils/services.ts | 7 +- backend/src/ee/routes/v1/pit-router.ts | 34 +++ backend/src/server/routes/index.ts | 3 +- .../folder-commit/folder-commit-service.ts | 287 +++++++++++++++++- .../secret-folder-version-dal.ts | 77 ++++- .../secret-v2-bridge/secret-version-dal.ts | 83 ++++- 7 files changed, 484 insertions(+), 12 deletions(-) diff --git a/backend/src/db/migrations/20250507185010_pit-projects-commits-initialization.ts b/backend/src/db/migrations/20250507185010_pit-projects-commits-initialization.ts index 7ad865b10..5c254afcd 100644 --- a/backend/src/db/migrations/20250507185010_pit-projects-commits-initialization.ts +++ b/backend/src/db/migrations/20250507185010_pit-projects-commits-initialization.ts @@ -1,12 +1,15 @@ import { Knex } from "knex"; +import { inMemoryKeyStore } from "@app/keystore/memory"; + import { ProjectType, TableName } from "../schemas"; import { getMigrationPITServices } from "./utils/services"; export async function up(knex: Knex): Promise { const hasFolderCommitTable = await knex.schema.hasTable(TableName.FolderCommit); if (hasFolderCommitTable) { - const { folderCommitService } = await getMigrationPITServices({ db: knex }); + const keyStore = inMemoryKeyStore(); + const { folderCommitService } = await getMigrationPITServices({ db: knex, keyStore }); const projects = await knex(TableName.Project).where({ version: 3, type: ProjectType.SecretManager }).select("id"); await knex.transaction(async (tx) => { for (const project of projects) { diff --git a/backend/src/db/migrations/utils/services.ts b/backend/src/db/migrations/utils/services.ts index 610112618..1ee81536c 100644 --- a/backend/src/db/migrations/utils/services.ts +++ b/backend/src/db/migrations/utils/services.ts @@ -18,6 +18,7 @@ import { orgDALFactory } from "@app/services/org/org-dal"; import { projectDALFactory } from "@app/services/project/project-dal"; import { secretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; import { secretFolderVersionDALFactory } from "@app/services/secret-folder/secret-folder-version-dal"; +import { secretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal"; import { secretVersionV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-version-dal"; import { userDALFactory } from "@app/services/user/user-dal"; @@ -62,7 +63,7 @@ export const getMigrationEncryptionServices = async ({ envConfig, db, keyStore } return { kmsService }; }; -export const getMigrationPITServices = async ({ db }: { db: Knex }) => { +export const getMigrationPITServices = async ({ db, keyStore }: { db: Knex; keyStore: TKeyStoreFactory }) => { const projectDAL = projectDALFactory(db); const folderCommitDAL = folderCommitDALFactory(db); const folderCommitChangesDAL = folderCommitChangesDALFactory(db); @@ -74,6 +75,7 @@ export const getMigrationPITServices = async ({ db }: { db: Knex }) => { const folderVersionDAL = secretFolderVersionDALFactory(db); const secretVersionV2BridgeDAL = secretVersionV2BridgeDALFactory(db); const folderCheckpointResourcesDAL = folderCheckpointResourcesDALFactory(db); + const secretV2BridgeDAL = secretV2BridgeDALFactory({ db, keyStore }); const folderCommitService = folderCommitServiceFactory({ folderCommitDAL, @@ -86,7 +88,8 @@ export const getMigrationPITServices = async ({ db }: { db: Knex }) => { folderVersionDAL, secretVersionV2BridgeDAL, projectDAL, - folderCheckpointResourcesDAL + folderCheckpointResourcesDAL, + secretV2BridgeDAL }); return { folderCommitService }; diff --git a/backend/src/ee/routes/v1/pit-router.ts b/backend/src/ee/routes/v1/pit-router.ts index 0045a7efe..9abef5382 100644 --- a/backend/src/ee/routes/v1/pit-router.ts +++ b/backend/src/ee/routes/v1/pit-router.ts @@ -24,4 +24,38 @@ export const registerPITRouter = async (server: FastifyZodProvider) => { return backup; } }); + + server.route({ + method: "POST", + url: "/rollback", + config: { + rateLimit: readLimit + }, + schema: { + body: z.object({ + fromCommit: z.string().trim(), + toCommit: z.string().trim(), + folderId: z.string().trim(), + projectId: z.string().trim() + }), + response: { + 200: z.any() + } + }, + handler: async (req) => { + const diff = await server.services.folderCommit.compareFolderStates(req.body.fromCommit, req.body.toCommit); + const response = await server.services.folderCommit.applyFolderStateDifferences( + diff, + { + actorType: req.permission?.type || "PLATFORM", + actorId: req.permission?.id, + message: "Rollback to previous commit" + }, + req.body.folderId, + req.body.projectId + ); + + return response; + } + }); }; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 4367f69c5..38c5f5dd0 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -574,7 +574,8 @@ export const registerRoutes = async ( folderVersionDAL, secretVersionV2BridgeDAL, projectDAL, - folderCheckpointResourcesDAL + folderCheckpointResourcesDAL, + secretV2BridgeDAL }); const scimService = scimServiceFactory({ licenseService, diff --git a/backend/src/services/folder-commit/folder-commit-service.ts b/backend/src/services/folder-commit/folder-commit-service.ts index c39aaa7e7..e40847d06 100644 --- a/backend/src/services/folder-commit/folder-commit-service.ts +++ b/backend/src/services/folder-commit/folder-commit-service.ts @@ -14,6 +14,7 @@ import { TIdentityDALFactory } from "../identity/identity-dal"; import { TProjectDALFactory } from "../project/project-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal"; +import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal"; import { TUserDALFactory } from "../user/user-dal"; import { TFolderCommitDALFactory } from "./folder-commit-dal"; @@ -38,14 +39,33 @@ type TFolderCommitServiceFactoryDep = { folderTreeCheckpointDAL: Pick; userDAL: Pick; identityDAL: Pick; - folderDAL: Pick; + folderDAL: Pick< + TSecretFolderDALFactory, + "findByParentId" | "findByProjectId" | "deleteById" | "create" | "updateById" | "update" + >; folderVersionDAL: Pick< TSecretFolderVersionDALFactory, - "findLatestFolderVersions" | "findById" | "deleteById" | "create" | "updateById" + | "findLatestFolderVersions" + | "findById" + | "deleteById" + | "create" + | "updateById" + | "find" + | "findByIdsWithLatestVersion" >; secretVersionV2BridgeDAL: Pick< TSecretVersionV2DALFactory, - "findLatestVersionByFolderId" | "findById" | "deleteById" | "create" | "updateById" + | "findLatestVersionByFolderId" + | "findById" + | "deleteById" + | "create" + | "updateById" + | "find" + | "findByIdsWithLatestVersion" + >; + secretV2BridgeDAL: Pick< + TSecretV2BridgeDALFactory, + "deleteById" | "create" | "updateById" | "update" | "insertMany" | "invalidateSecretCacheByProjectId" >; projectDAL: Pick; }; @@ -85,7 +105,8 @@ export const folderCommitServiceFactory = ({ folderDAL, folderVersionDAL, secretVersionV2BridgeDAL, - projectDAL + projectDAL, + secretV2BridgeDAL }: TFolderCommitServiceFactoryDep) => { const appCfg = getConfig(); @@ -356,6 +377,261 @@ export const folderCommitServiceFactory = ({ } }; + const applyFolderStateDifferences = async ( + differences: Array<{ + type: string; + id: string; + versionId: string; + oldVersionId?: string; + changeType: "create" | "update" | "delete"; + }>, + actorInfo: { + actorType: string; + actorId?: string; + message?: string; + }, + folderId: string, + projectId: string + ) => { + let result = {}; + await folderCommitDAL.transaction(async (tx) => { + // Group differences by type for more efficient processing + const secretChanges = differences.filter((diff) => diff.type === "secret"); + const folderChanges = differences.filter((diff) => diff.type === "folder"); + + const secretVersions = await secretVersionV2BridgeDAL.findByIdsWithLatestVersion( + folderId, + secretChanges.map((diff) => diff.id), + secretChanges.map((diff) => diff.versionId) + ); + const folderVersions = await folderVersionDAL.findByIdsWithLatestVersion( + folderChanges.map((diff) => diff.id), + folderChanges.map((diff) => diff.versionId) + ); + + // Track all changes for commit recording + const commitChanges = []; + + // Process secret changes + for (const change of secretChanges) { + const secretVersion = secretVersions[change.id]; + switch (change.changeType) { + case "create": + if (secretVersion) { + const newSecret = [ + { + id: change.id, + skipMultilineEncoding: secretVersion.skipMultilineEncoding, + version: secretVersion.version + 1, + type: secretVersion.type, + key: secretVersion.key, + reminderNote: secretVersion.reminderNote, + reminderRepeatDays: secretVersion.reminderRepeatDays, + encryptedValue: secretVersion.encryptedValue, + encryptedComment: secretVersion.encryptedComment, + userId: secretVersion.userId, + metadata: secretVersion.metadata, + folderId + } + ]; + await secretV2BridgeDAL.insertMany(newSecret, tx); + + const newVersion = await secretVersionV2BridgeDAL.create( + { + folderId, + secretId: secretVersion.secretId, + version: secretVersion.version + 1, + encryptedValue: secretVersion.encryptedValue, + key: secretVersion.key, + encryptedComment: secretVersion.encryptedComment, + skipMultilineEncoding: secretVersion.skipMultilineEncoding, + reminderNote: secretVersion.reminderNote, + reminderRepeatDays: secretVersion.reminderRepeatDays, + userId: secretVersion.userId, + metadata: secretVersion.metadata, + actorType: actorInfo.actorType, + envId: secretVersion.envId, + ...(actorInfo.actorType === ActorType.IDENTITY && { identityActorId: actorInfo.actorId }), + ...(actorInfo.actorType === ActorType.USER && { userActorId: actorInfo.actorId }) + }, + tx + ); + + commitChanges.push({ + type: "add", + secretVersionId: newVersion.id + }); + } + break; + + case "update": + // Update secret to specific version + if (secretVersion) { + await secretV2BridgeDAL.updateById( + change.id, + { + skipMultilineEncoding: secretVersion?.skipMultilineEncoding, + version: secretVersion?.version, + type: secretVersion?.type, + key: secretVersion?.key, + reminderNote: secretVersion?.reminderNote, + reminderRepeatDays: secretVersion?.reminderRepeatDays, + encryptedValue: secretVersion?.encryptedValue, + encryptedComment: secretVersion?.encryptedComment, + userId: secretVersion?.userId, + metadata: secretVersion?.metadata + }, + tx + ); + + const newVersion = await secretVersionV2BridgeDAL.create( + { + version: secretVersion.version + 1, + encryptedValue: secretVersion.encryptedValue, + key: secretVersion.key, + encryptedComment: secretVersion.encryptedComment, + skipMultilineEncoding: secretVersion.skipMultilineEncoding, + reminderNote: secretVersion.reminderNote, + reminderRepeatDays: secretVersion.reminderRepeatDays, + userId: secretVersion.userId, + metadata: secretVersion.metadata, + actorType: actorInfo.actorType, + envId: secretVersion.envId, + folderId, + secretId: secretVersion.secretId, + ...(actorInfo.actorType === ActorType.IDENTITY && { identityActorId: actorInfo.actorId }), + ...(actorInfo.actorType === ActorType.USER && { userActorId: actorInfo.actorId }) + }, + tx + ); + + commitChanges.push({ + type: "add", + secretVersionId: newVersion.id + }); + } + break; + + case "delete": + await secretV2BridgeDAL.deleteById(change.id, tx); + + commitChanges.push({ + type: "delete", + secretVersionId: change.versionId + }); + break; + + default: + throw new BadRequestError({ message: `Unknown change type: ${change.changeType as string}` }); + } + } + + // process folder changes + for (const change of folderChanges) { + const folderVersion = folderVersions[change.id]; + switch (change.changeType) { + case "create": + // Add new folder + if (folderVersion) { + const newFolder = { + id: change.id, + parentId: folderId, + envId: folderVersion.envId, + version: (folderVersion.version || 1) + 1, + name: folderVersion.name + }; + await folderDAL.create(newFolder, tx); + + const newFolderVersion = await folderVersionDAL.create( + { + folderId: change.id, + version: (folderVersion.version || 1) + 1, + name: folderVersion.name, + envId: folderVersion.envId + }, + tx + ); + + commitChanges.push({ + type: "add", + folderVersionId: newFolderVersion.id + }); + } + break; + + case "update": + // Update folder to specific version + if (change.versionId) { + await folderVersionDAL.findById(change.versionId, tx).then(async (versionDetails) => { + if (versionDetails) { + await folderDAL.updateById( + change.id, + { + parentId: folderId, + envId: versionDetails.envId, + version: (versionDetails.version || 1) + 1, + name: versionDetails.name + }, + tx + ); + + const newFolderVersion = await folderVersionDAL.create( + { + folderId: change.id, + version: (versionDetails.version || 1) + 1, + name: versionDetails.name, + envId: versionDetails.envId + }, + tx + ); + + commitChanges.push({ + type: "add", + folderVersionId: newFolderVersion.id + }); + } + }); + } + break; + + case "delete": + await folderDAL.deleteById(change.id, tx); + + commitChanges.push({ + type: "delete", + folderVersionId: change.versionId + }); + break; + + default: + throw new BadRequestError({ message: `Unknown change type: ${change.changeType as string}` }); + } + } + + await createCommit( + { + actor: { + type: actorInfo.actorType, + metadata: { id: actorInfo.actorId } + }, + message: actorInfo.message || "Rolled back folder state", + folderId, + changes: commitChanges + }, + tx + ); + + result = { + secretChangesCount: secretChanges.length, + folderChangesCount: folderChanges.length, + totalChanges: differences.length + }; + await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId); + }); + + return result; + }; + // Retrieve a commit by ID const getCommitById = async (id: string, tx?: Knex) => { return folderCommitDAL.findById(id, tx); @@ -467,7 +743,8 @@ export const folderCommitServiceFactory = ({ initializeFolder, initializeProject, createFolderCheckpoint, - compareFolderStates + compareFolderStates, + applyFolderStateDifferences }; }; diff --git a/backend/src/services/secret-folder/secret-folder-version-dal.ts b/backend/src/services/secret-folder/secret-folder-version-dal.ts index 437b1e1eb..56ac4f1f7 100644 --- a/backend/src/services/secret-folder/secret-folder-version-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-version-dal.ts @@ -96,5 +96,80 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => { logger.info(`${QueueName.DailyResourceCleanUp}: pruning secret folder versions completed`); }; - return { ...secretFolderVerOrm, findLatestFolderVersions, findLatestVersionByFolderId, pruneExcessVersions }; + const findByIdsWithLatestVersion = async (folderIds: string[], versionIds?: string[], tx?: Knex) => { + try { + if (!folderIds.length && (!versionIds || !versionIds.length)) return {}; + + const knexInstance = tx || db.replicaNode(); + let allDocs: Array = []; + + // Process latest versions for folderIds + if (folderIds.length) { + const latestVersions: Array = await knexInstance( + TableName.SecretFolderVersion + ) + .whereIn(`${TableName.SecretFolderVersion}.folderId`, folderIds) + .join( + knexInstance(TableName.SecretFolderVersion) + .groupBy("folderId") + .max("version") + .select("folderId") + .as("latestVersion"), + (bd) => { + bd.on(`${TableName.SecretFolderVersion}.folderId`, "latestVersion.folderId").andOn( + `${TableName.SecretFolderVersion}.version`, + "latestVersion.max" + ); + } + ); + + allDocs = [...allDocs, ...latestVersions]; + } + + // Process specific versions by versionIds + if (versionIds && versionIds.length) { + // Get the specific versions + const specificVersions = await knexInstance(TableName.SecretFolderVersion).whereIn("id", versionIds); + + // Get the folderIds from these versions + const specificFolderIds = [...new Set(specificVersions.map((v) => v.folderId).filter(Boolean))]; + + // Get max versions for these folderIds + const maxVersionsQuery = (await knexInstance(TableName.SecretFolderVersion) + .whereIn("folderId", specificFolderIds) + .groupBy("folderId") + .select("folderId") + .max("version as max")) as Array<{ folderId: string; max: number }>; + + // Create a lookup map for max versions + const maxVersionMap = maxVersionsQuery.reduce>((acc, item) => { + acc[item.folderId] = item.max; + return acc; + }, {}); + + // Add max version to each specific version + const specificVersionsWithMax = specificVersions.map((version) => ({ + ...version, + max: maxVersionMap[version.folderId] + })); + + allDocs = [...allDocs, ...specificVersionsWithMax]; + } + + return allDocs.reduce>( + (prev, curr) => ({ ...prev, [curr.folderId || ""]: curr }), + {} + ); + } catch (error) { + throw new DatabaseError({ error, name: "FindByIdsWithLatestVersion" }); + } + }; + + return { + ...secretFolderVerOrm, + findLatestFolderVersions, + findLatestVersionByFolderId, + pruneExcessVersions, + findByIdsWithLatestVersion + }; }; diff --git a/backend/src/services/secret-v2-bridge/secret-version-dal.ts b/backend/src/services/secret-v2-bridge/secret-version-dal.ts index 2f9574ac9..291fec22f 100644 --- a/backend/src/services/secret-v2-bridge/secret-version-dal.ts +++ b/backend/src/services/secret-v2-bridge/secret-version-dal.ts @@ -138,7 +138,7 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { {} ); } catch (error) { - throw new DatabaseError({ error, name: "FindLatestVersinMany" }); + throw new DatabaseError({ error, name: "FindLatestVersionMany" }); } }; @@ -261,6 +261,84 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { } }; + const findByIdsWithLatestVersion = async ( + folderId: string, + secretIds: string[], + versionIds?: string[], + tx?: Knex + ) => { + try { + if (!secretIds.length && (!versionIds || !versionIds.length)) return {}; + + const knexInstance = tx || db.replicaNode(); + let allDocs: Array = []; + + // Process latest versions for secretIds + if (secretIds.length) { + const latestVersions: Array = await knexInstance(TableName.SecretVersionV2) + .where("folderId", folderId) + .whereIn(`${TableName.SecretVersionV2}.secretId`, secretIds) + .join( + knexInstance(TableName.SecretVersionV2) + .groupBy("secretId") + .max("version") + .select("secretId") + .as("latestVersion"), + (bd) => { + bd.on(`${TableName.SecretVersionV2}.secretId`, "latestVersion.secretId").andOn( + `${TableName.SecretVersionV2}.version`, + "latestVersion.max" + ); + } + ); + + allDocs = [...allDocs, ...latestVersions]; + } + + // Process specific versions by versionIds + if (versionIds && versionIds.length) { + // Get the specific versions + const specificVersions = await knexInstance(TableName.SecretVersionV2) + .where("folderId", folderId) + .whereIn("id", versionIds); + + // Get the secretIds from these versions + const specificSecretIds = [...new Set(specificVersions.map((v) => v.secretId).filter(Boolean))]; + + // Get max versions for these secretIds + const maxVersionsQuery = (await knexInstance(TableName.SecretVersionV2) + .whereIn("secretId", specificSecretIds) + .groupBy("secretId") + .select("secretId") + .max("version as max")) as Array<{ secretId: string; max: number }>; + + // Create a lookup map for max versions + const maxVersionMap = maxVersionsQuery.reduce( + (acc, item) => { + acc[item.secretId] = item.max; + return acc; + }, + {} as Record + ); + + // Add max version to each specific version + const specificVersionsWithMax = specificVersions.map((version) => ({ + ...version, + max: maxVersionMap[version.secretId] + })); + + allDocs = [...allDocs, ...specificVersionsWithMax]; + } + + return allDocs.reduce>( + (prev, curr) => ({ ...prev, [curr.secretId || ""]: curr }), + {} + ); + } catch (error) { + throw new DatabaseError({ error, name: "FindByIdsWithLatestVersion" }); + } + }; + return { ...secretVersionV2Orm, pruneExcessVersions, @@ -268,6 +346,7 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { bulkUpdate, findLatestVersionByFolderId, findVersionsBySecretIdWithActors, - findBySecretId + findBySecretId, + findByIdsWithLatestVersion }; }; From 8bfd3913da82c95923d6e52543fa35b3e7805a83 Mon Sep 17 00:00:00 2001 From: carlosmonastyrski Date: Wed, 14 May 2025 10:26:41 -0300 Subject: [PATCH 7/9] PIT: add backend logic for deep PIT and rollback --- .../20250505194916_add-pit-revamp-tables.ts | 5 +- backend/src/db/migrations/utils/services.ts | 5 +- backend/src/db/schemas/folder-commits.ts | 1 + backend/src/ee/routes/v1/pit-router.ts | 109 +- backend/src/lib/config/env.ts | 3 +- backend/src/queue/queue-service.ts | 12 +- backend/src/server/routes/index.ts | 14 +- .../folder-commit/folder-commit-dal.ts | 157 ++- .../folder-commit/folder-commit-queue.ts | 180 ++++ .../folder-commit-service.test.ts | 571 +++++++++++ .../folder-commit/folder-commit-service.ts | 952 ++++++++++++------ .../folder-tree-checkpoint-resources-dal.ts | 110 +- .../folder-tree-checkpoint-dal.ts | 101 +- .../secret-folder/secret-folder-dal.ts | 60 +- 14 files changed, 1818 insertions(+), 462 deletions(-) create mode 100644 backend/src/services/folder-commit/folder-commit-queue.ts create mode 100644 backend/src/services/folder-commit/folder-commit-service.test.ts diff --git a/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts b/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts index 9855d301e..a2a8ac8c2 100644 --- a/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts +++ b/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts @@ -13,10 +13,12 @@ export async function up(knex: Knex): Promise { t.string("actorType").notNullable(); t.string("message"); t.uuid("folderId").notNullable(); - t.foreign("folderId").references("id").inTable(TableName.SecretFolder).onDelete("CASCADE"); + t.uuid("envId").notNullable(); + t.foreign("envId").references("id").inTable(TableName.Environment).onDelete("CASCADE"); t.timestamps(true, true, true); t.index("folderId"); + t.index("envId"); }); } @@ -88,7 +90,6 @@ export async function up(knex: Knex): Promise { t.uuid("folderTreeCheckpointId").notNullable(); t.foreign("folderTreeCheckpointId").references("id").inTable(TableName.FolderTreeCheckpoint).onDelete("CASCADE"); t.uuid("folderId").notNullable(); - t.foreign("folderId").references("id").inTable(TableName.SecretFolder).onDelete("CASCADE"); t.uuid("folderCommitId").notNullable(); t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE"); t.timestamps(true, true, true); diff --git a/backend/src/db/migrations/utils/services.ts b/backend/src/db/migrations/utils/services.ts index 1ee81536c..4e0188c98 100644 --- a/backend/src/db/migrations/utils/services.ts +++ b/backend/src/db/migrations/utils/services.ts @@ -9,6 +9,7 @@ import { folderCommitDALFactory } from "@app/services/folder-commit/folder-commi import { folderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { folderCommitChangesDALFactory } from "@app/services/folder-commit-changes/folder-commit-changes-dal"; import { folderTreeCheckpointDALFactory } from "@app/services/folder-tree-checkpoint/folder-tree-checkpoint-dal"; +import { folderTreeCheckpointResourcesDALFactory } from "@app/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal"; import { identityDALFactory } from "@app/services/identity/identity-dal"; import { internalKmsDALFactory } from "@app/services/kms/internal-kms-dal"; import { kmskeyDALFactory } from "@app/services/kms/kms-key-dal"; @@ -76,6 +77,7 @@ export const getMigrationPITServices = async ({ db, keyStore }: { db: Knex; keyS const secretVersionV2BridgeDAL = secretVersionV2BridgeDALFactory(db); const folderCheckpointResourcesDAL = folderCheckpointResourcesDALFactory(db); const secretV2BridgeDAL = secretV2BridgeDALFactory({ db, keyStore }); + const folderTreeCheckpointResourcesDAL = folderTreeCheckpointResourcesDALFactory(db); const folderCommitService = folderCommitServiceFactory({ folderCommitDAL, @@ -89,7 +91,8 @@ export const getMigrationPITServices = async ({ db, keyStore }: { db: Knex; keyS secretVersionV2BridgeDAL, projectDAL, folderCheckpointResourcesDAL, - secretV2BridgeDAL + secretV2BridgeDAL, + folderTreeCheckpointResourcesDAL }); return { folderCommitService }; diff --git a/backend/src/db/schemas/folder-commits.ts b/backend/src/db/schemas/folder-commits.ts index ec166e7c2..fc9b83489 100644 --- a/backend/src/db/schemas/folder-commits.ts +++ b/backend/src/db/schemas/folder-commits.ts @@ -14,6 +14,7 @@ export const FolderCommitsSchema = z.object({ actorType: z.string(), message: z.string().nullable().optional(), folderId: z.string().uuid(), + envId: z.string().uuid(), createdAt: z.date(), updatedAt: z.date() }); diff --git a/backend/src/ee/routes/v1/pit-router.ts b/backend/src/ee/routes/v1/pit-router.ts index 9abef5382..a3ec8dcc9 100644 --- a/backend/src/ee/routes/v1/pit-router.ts +++ b/backend/src/ee/routes/v1/pit-router.ts @@ -3,9 +3,52 @@ import { z } from "zod"; import { readLimit } from "@app/server/config/rateLimiter"; export const registerPITRouter = async (server: FastifyZodProvider) => { + // Get all commits for a folder server.route({ method: "GET", - url: "/diff", + url: "/commits/:folderId", + config: { + rateLimit: readLimit + }, + schema: { + params: z.object({ + folderId: z.string().trim() + }), + response: { + 200: z.any() + } + }, + handler: async (req) => { + const commits = await server.services.folderCommit.getCommitsByFolderId(req.params.folderId); + return commits; + } + }); + + // Get commit changes for a specific commit + server.route({ + method: "GET", + url: "/commits/:commitId/changes", + config: { + rateLimit: readLimit + }, + schema: { + params: z.object({ + commitId: z.string().trim() + }), + response: { + 200: z.any() + } + }, + handler: async (req) => { + const changes = await server.services.folderCommit.getCommitChanges(req.params.commitId); + return changes; + } + }); + + // Compare folder states between commits + server.route({ + method: "GET", + url: "/compare", config: { rateLimit: readLimit }, @@ -19,12 +62,16 @@ export const registerPITRouter = async (server: FastifyZodProvider) => { } }, handler: async (req) => { - const backup = await server.services.folderCommit.compareFolderStates(req.query.fromCommit, req.query.toCommit); + const diff = await server.services.folderCommit.compareFolderStates({ + currentCommitId: req.query.fromCommit, + targetCommitId: req.query.toCommit + }); - return backup; + return diff; } }); + // Rollback to a previous commit server.route({ method: "POST", url: "/rollback", @@ -36,6 +83,46 @@ export const registerPITRouter = async (server: FastifyZodProvider) => { fromCommit: z.string().trim(), toCommit: z.string().trim(), folderId: z.string().trim(), + projectId: z.string().trim(), + reconstructNewFolders: z.boolean().default(false) + }), + response: { + 200: z.any() + } + }, + handler: async (req) => { + const diff = await server.services.folderCommit.compareFolderStates({ + currentCommitId: req.body.fromCommit, + targetCommitId: req.body.toCommit + }); + + const response = await server.services.folderCommit.applyFolderStateDifferences({ + differences: diff, + actorInfo: { + actorType: req.permission?.type || "PLATFORM", + actorId: req.permission?.id, + message: "Rollback to previous commit" + }, + folderId: req.body.folderId, + projectId: req.body.projectId, + reconstructNewFolders: req.body.reconstructNewFolders + }); + + return response; + } + }); + + // Deep rollback to a specific commit + server.route({ + method: "POST", + url: "/deep-rollback", + config: { + rateLimit: readLimit + }, + schema: { + body: z.object({ + commitId: z.string().trim(), + envId: z.string().trim(), projectId: z.string().trim() }), response: { @@ -43,19 +130,15 @@ export const registerPITRouter = async (server: FastifyZodProvider) => { } }, handler: async (req) => { - const diff = await server.services.folderCommit.compareFolderStates(req.body.fromCommit, req.body.toCommit); - const response = await server.services.folderCommit.applyFolderStateDifferences( - diff, - { - actorType: req.permission?.type || "PLATFORM", - actorId: req.permission?.id, - message: "Rollback to previous commit" - }, - req.body.folderId, + await server.services.folderCommit.deepRollbackFolder( + req.body.commitId, + req.body.envId, + req.permission?.id || "PLATFORM", + req.permission?.type || "PLATFORM", req.body.projectId ); - return response; + return { success: true }; } }); }; diff --git a/backend/src/lib/config/env.ts b/backend/src/lib/config/env.ts index 38e252b2a..74da0b399 100644 --- a/backend/src/lib/config/env.ts +++ b/backend/src/lib/config/env.ts @@ -229,7 +229,8 @@ const envSchema = z DATADOG_HOSTNAME: zpStr(z.string().optional()), // PIT - CHECKPOINT_WINDOW: zpStr(z.string().optional().default("10")), + PIT_CHECKPOINT_WINDOW: zpStr(z.string().optional().default("10")), + PIT_TREE_CHECKPOINT_WINDOW: zpStr(z.string().optional().default("100")), /* CORS ----------------------------------------------------------------------------- */ diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index ae1a3e821..f02519e3a 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -49,7 +49,8 @@ export enum QueueName { AccessTokenStatusUpdate = "access-token-status-update", ImportSecretsFromExternalSource = "import-secrets-from-external-source", AppConnectionSecretSync = "app-connection-secret-sync", - SecretRotationV2 = "secret-rotation-v2" + SecretRotationV2 = "secret-rotation-v2", + FolderTreeCheckpoint = "folder-tree-checkpoint" } export enum QueueJobs { @@ -81,7 +82,8 @@ export enum QueueJobs { SecretSyncSendActionFailedNotifications = "secret-sync-send-action-failed-notifications", SecretRotationV2QueueRotations = "secret-rotation-v2-queue-rotations", SecretRotationV2RotateSecrets = "secret-rotation-v2-rotate-secrets", - SecretRotationV2SendNotification = "secret-rotation-v2-send-notification" + SecretRotationV2SendNotification = "secret-rotation-v2-send-notification", + CreateFolderTreeCheckpoint = "create-folder-tree-checkpoint" } export type TQueueJobTypes = { @@ -191,6 +193,12 @@ export type TQueueJobTypes = { name: QueueJobs.ProjectV3Migration; payload: { projectId: string }; }; + [QueueName.FolderTreeCheckpoint]: { + name: QueueJobs.CreateFolderTreeCheckpoint; + payload: { + envId: string; + }; + }; [QueueName.ImportSecretsFromExternalSource]: { name: QueueJobs.ImportSecretsFromExternalSource; payload: { diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 38c5f5dd0..2960519bd 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -143,9 +143,11 @@ import { externalMigrationServiceFactory } from "@app/services/external-migratio import { folderCheckpointDALFactory } from "@app/services/folder-checkpoint/folder-checkpoint-dal"; import { folderCheckpointResourcesDALFactory } from "@app/services/folder-checkpoint-resources/folder-checkpoint-resources-dal"; import { folderCommitDALFactory } from "@app/services/folder-commit/folder-commit-dal"; +import { folderCommitQueueServiceFactory } from "@app/services/folder-commit/folder-commit-queue"; import { folderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { folderCommitChangesDALFactory } from "@app/services/folder-commit-changes/folder-commit-changes-dal"; import { folderTreeCheckpointDALFactory } from "@app/services/folder-tree-checkpoint/folder-tree-checkpoint-dal"; +import { folderTreeCheckpointResourcesDALFactory } from "@app/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal"; import { groupProjectDALFactory } from "@app/services/group-project/group-project-dal"; import { groupProjectMembershipRoleDALFactory } from "@app/services/group-project/group-project-membership-role-dal"; import { groupProjectServiceFactory } from "@app/services/group-project/group-project-service"; @@ -563,6 +565,14 @@ export const registerRoutes = async ( const folderCheckpointResourcesDAL = folderCheckpointResourcesDALFactory(db); const folderTreeCheckpointDAL = folderTreeCheckpointDALFactory(db); const folderCommitDAL = folderCommitDALFactory(db); + const folderTreeCheckpointResourcesDAL = folderTreeCheckpointResourcesDALFactory(db); + const folderCommitQueueService = folderCommitQueueServiceFactory({ + queueService, + folderTreeCheckpointDAL, + folderTreeCheckpointResourcesDAL, + folderCommitDAL, + folderDAL + }); const folderCommitService = folderCommitServiceFactory({ folderCommitDAL, folderCommitChangesDAL, @@ -575,7 +585,9 @@ export const registerRoutes = async ( secretVersionV2BridgeDAL, projectDAL, folderCheckpointResourcesDAL, - secretV2BridgeDAL + secretV2BridgeDAL, + folderTreeCheckpointResourcesDAL, + folderCommitQueueService }); const scimService = scimServiceFactory({ licenseService, diff --git a/backend/src/services/folder-commit/folder-commit-dal.ts b/backend/src/services/folder-commit/folder-commit-dal.ts index f52da95b5..7f015ddc0 100644 --- a/backend/src/services/folder-commit/folder-commit-dal.ts +++ b/backend/src/services/folder-commit/folder-commit-dal.ts @@ -42,6 +42,75 @@ export const folderCommitDALFactory = (db: TDbClient) => { } }; + const findLatestCommitByFolderIds = async (folderIds: string[], tx?: Knex): Promise => { + try { + // First get max commitId for each folderId + const maxCommitIdSubquery = (tx || db.replicaNode())(TableName.FolderCommit) + .select("folderId") + .max("commitId as maxCommitId") + .whereIn("folderId", folderIds) + .groupBy("folderId"); + + // Join with main table to get complete records for each max commitId + const docs = await (tx || db.replicaNode())(TableName.FolderCommit) + .select(selectAllTableCols(TableName.FolderCommit)) + // eslint-disable-next-line func-names + .join(maxCommitIdSubquery.as("latest"), function () { + this.on(`${TableName.FolderCommit}.folderId`, "=", "latest.folderId").andOn( + `${TableName.FolderCommit}.commitId`, + "=", + "latest.maxCommitId" + ); + }); + + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestCommitByFolderIds" }); + } + }; + + const findLatestEnvCommit = async (envId: string, tx?: Knex): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + .where(`${TableName.FolderCommit}.envId`, "=", envId) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("commitId", "desc") + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestCommit" }); + } + }; + + const findMultipleLatestCommits = async (folderIds: string[], tx?: Knex): Promise => { + try { + const knexInstance = tx || db.replicaNode(); + + // Get the latest commitId for each folderId + const subquery = knexInstance(TableName.FolderCommit) + .whereIn("folderId", folderIds) + .groupBy("folderId") + .select("folderId") + .max("commitId as maxCommitId"); + + // Then fetch the complete rows matching those latest commits + const docs = await knexInstance(TableName.FolderCommit) + // eslint-disable-next-line func-names + .innerJoin(subquery.as("latest"), function () { + this.on(`${TableName.FolderCommit}.folderId`, "=", "latest.folderId").andOn( + `${TableName.FolderCommit}.commitId`, + "=", + "latest.maxCommitId" + ); + }) + .select(selectAllTableCols(TableName.FolderCommit)); + + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindMultipleLatestCommits" }); + } + }; + const getNumberOfCommitsSince = async (folderId: string, folderCommitId: string, tx?: Knex): Promise => { try { const referencedCommit = await (tx || db.replicaNode())(TableName.FolderCommit) @@ -62,6 +131,26 @@ export const folderCommitDALFactory = (db: TDbClient) => { } }; + const getEnvNumberOfCommitsSince = async (envId: string, folderCommitId: string, tx?: Knex): Promise => { + try { + const referencedCommit = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ id: folderCommitId }) + .select("commitId") + .first(); + + if (referencedCommit?.commitId) { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + .where(`${TableName.FolderCommit}.envId`, "=", envId) + .where("commitId", ">", referencedCommit.commitId) + .count(); + return Number(doc?.[0].count); + } + return 0; + } catch (error) { + throw new DatabaseError({ error, name: "getNumberOfCommitsSince" }); + } + }; + const findCommitsToRecreate = async ( folderId: string, targetCommitNumber: number, @@ -131,11 +220,77 @@ export const folderCommitDALFactory = (db: TDbClient) => { } }; + const findLatestCommitBetween = async ({ + folderId, + startCommitId, + endCommitId, + tx + }: { + folderId: string; + startCommitId?: string; + endCommitId: string; + tx?: Knex; + }): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + .where("commitId", "<=", endCommitId) + .where({ folderId }) + .where((qb) => { + if (startCommitId) { + void qb.where("commitId", ">=", startCommitId); + } + }) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("commitId", "desc") + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestCommitBetween" }); + } + }; + + const findAllCommitsBetween = async ({ + envId, + startCommitId, + endCommitId, + tx + }: { + folderId?: string; + envId?: string; + startCommitId?: string; + endCommitId: string; + tx?: Knex; + }): Promise => { + try { + const docs = await (tx || db.replicaNode())(TableName.FolderCommit) + .where("commitId", "<=", endCommitId) + .where((qb) => { + if (envId) { + void qb.where(`${TableName.FolderCommit}.envId`, "=", envId); + } + if (startCommitId) { + void qb.where("commitId", ">=", startCommitId); + } + }) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("commitId", "desc"); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestCommitBetween" }); + } + }; + return { ...restOfOrm, findByFolderId, findLatestCommit, getNumberOfCommitsSince, - findCommitsToRecreate + findCommitsToRecreate, + findMultipleLatestCommits, + findAllCommitsBetween, + findLatestCommitBetween, + findLatestEnvCommit, + getEnvNumberOfCommitsSince, + findLatestCommitByFolderIds }; }; diff --git a/backend/src/services/folder-commit/folder-commit-queue.ts b/backend/src/services/folder-commit/folder-commit-queue.ts new file mode 100644 index 000000000..06aeaa0c9 --- /dev/null +++ b/backend/src/services/folder-commit/folder-commit-queue.ts @@ -0,0 +1,180 @@ +import { TSecretFolders } from "@app/db/schemas"; +import { getConfig } from "@app/lib/config/env"; +import { logger } from "@app/lib/logger"; +import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; + +import { TFolderTreeCheckpointDALFactory } from "../folder-tree-checkpoint/folder-tree-checkpoint-dal"; +import { TFolderTreeCheckpointResourcesDALFactory } from "../folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal"; +import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TFolderCommitDALFactory } from "./folder-commit-dal"; + +type TFolderCommitQueueServiceFactoryDep = { + queueService: TQueueServiceFactory; + folderTreeCheckpointDAL: Pick< + TFolderTreeCheckpointDALFactory, + "create" | "findLatestByEnvId" | "findNearestCheckpoint" + >; + folderTreeCheckpointResourcesDAL: Pick< + TFolderTreeCheckpointResourcesDALFactory, + "insertMany" | "findByTreeCheckpointId" + >; + folderCommitDAL: Pick< + TFolderCommitDALFactory, + "findLatestEnvCommit" | "getEnvNumberOfCommitsSince" | "findMultipleLatestCommits" + >; + folderDAL: Pick; +}; + +export type TFolderCommitQueueServiceFactory = ReturnType; + +export const folderCommitQueueServiceFactory = ({ + queueService, + folderTreeCheckpointDAL, + folderTreeCheckpointResourcesDAL, + folderCommitDAL, + folderDAL +}: TFolderCommitQueueServiceFactoryDep) => { + const appCfg = getConfig(); + + const scheduleTreeCheckpoint = async (envId: string) => { + await queueService.queue( + QueueName.FolderTreeCheckpoint, + QueueJobs.CreateFolderTreeCheckpoint, + { envId }, + { + jobId: envId, + backoff: { + type: "exponential", + delay: 3000 + }, + removeOnFail: { + count: 3 + }, + removeOnComplete: true + } + ); + }; + + const schedulePeriodicTreeCheckpoint = async (envId: string, intervalMs: number) => { + await queueService.queue( + QueueName.FolderTreeCheckpoint, + QueueJobs.CreateFolderTreeCheckpoint, + { envId }, + { + jobId: `periodic-${envId}`, + repeat: { + every: intervalMs + }, + backoff: { + type: "exponential", + delay: 3000 + }, + removeOnFail: false, + removeOnComplete: false + } + ); + }; + + const cancelScheduledTreeCheckpoint = async (envId: string) => { + await queueService.stopJobById(QueueName.FolderTreeCheckpoint, envId); + await queueService.stopRepeatableJobByJobId(QueueName.FolderTreeCheckpoint, `periodic-${envId}`); + }; + + // Sort folders by hierarchy (copied from the source code) + const sortFoldersByHierarchy = (folders: TSecretFolders[]) => { + const childrenMap = new Map(); + const allFolderIds = new Set(); + + folders.forEach((folder) => { + if (folder.id) allFolderIds.add(folder.id); + }); + + folders.forEach((folder) => { + if (folder.parentId) { + const children = childrenMap.get(folder.parentId) || []; + children.push(folder); + childrenMap.set(folder.parentId, children); + } + }); + + const rootFolders = folders.filter((folder) => !folder.parentId || !allFolderIds.has(folder.parentId)); + + const result = []; + let currentLevel = rootFolders; + + while (currentLevel.length > 0) { + result.push(...currentLevel); + + const nextLevel = []; + for (const folder of currentLevel) { + if (folder.id) { + const children = childrenMap.get(folder.id) || []; + nextLevel.push(...children); + } + } + + currentLevel = nextLevel; + } + + return result; + }; + + queueService.start(QueueName.FolderTreeCheckpoint, async (job) => { + try { + if (job.name === QueueJobs.CreateFolderTreeCheckpoint) { + const { envId } = job.data as { envId: string }; + logger.info("Folder tree checkpoint creation started:", envId, job.id); + + const latestTreeCheckpoint = await folderTreeCheckpointDAL.findLatestByEnvId(envId); + + const latestCommit = await folderCommitDAL.findLatestEnvCommit(envId); + if (!latestCommit) { + logger.info(`Latest commit ID not found for envId ${envId}`); + return; + } + const latestCommitId = latestCommit.id; + + if (latestTreeCheckpoint) { + const commitsSinceLastCheckpoint = await folderCommitDAL.getEnvNumberOfCommitsSince( + envId, + latestTreeCheckpoint.folderCommitId + ); + if (commitsSinceLastCheckpoint < Number(appCfg.PIT_TREE_CHECKPOINT_WINDOW)) { + logger.info( + `Commits since last checkpoint ${commitsSinceLastCheckpoint} is less than ${appCfg.PIT_TREE_CHECKPOINT_WINDOW}` + ); + return; + } + } + + const folders = await folderDAL.findByEnvId(envId); + const sortedFolders = sortFoldersByHierarchy(folders); + const filteredFoldersIds = sortedFolders.filter((folder) => !folder.isReserved).map((folder) => folder.id); + + const folderCommits = await folderCommitDAL.findMultipleLatestCommits(filteredFoldersIds); + const folderTreeCheckpoint = await folderTreeCheckpointDAL.create({ + folderCommitId: latestCommitId + }); + + await folderTreeCheckpointResourcesDAL.insertMany( + folderCommits.map((folderCommit) => ({ + folderTreeCheckpointId: folderTreeCheckpoint.id, + folderId: folderCommit.folderId, + folderCommitId: folderCommit.id + })) + ); + + logger.info("Folder tree checkpoint created successfully:", folderTreeCheckpoint.id); + } + } catch (error) { + logger.error(error, "Error creating folder tree checkpoint:"); + throw error; + } + }); + + return { + scheduleTreeCheckpoint, + schedulePeriodicTreeCheckpoint, + cancelScheduledTreeCheckpoint + }; +}; diff --git a/backend/src/services/folder-commit/folder-commit-service.test.ts b/backend/src/services/folder-commit/folder-commit-service.test.ts new file mode 100644 index 000000000..f00e12283 --- /dev/null +++ b/backend/src/services/folder-commit/folder-commit-service.test.ts @@ -0,0 +1,571 @@ +import { Knex } from "knex"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { TSecretFolderVersions, TSecretVersionsV2 } from "@app/db/schemas"; +import { BadRequestError, NotFoundError } from "@app/lib/errors"; + +import { ActorType } from "../auth/auth-type"; +import { ChangeType, folderCommitServiceFactory, TFolderCommitServiceFactory } from "./folder-commit-service"; + +// Mock config +vi.mock("@app/lib/config/env", () => ({ + getConfig: () => ({ + PIT_CHECKPOINT_WINDOW: 5, + PIT_TREE_CHECKPOINT_WINDOW: 10 + }) +})); + +// Mock logger +vi.mock("@app/lib/logger", () => ({ + logger: { + info: vi.fn(), + error: vi.fn() + } +})); + +describe("folderCommitServiceFactory", () => { + // Properly type the mock functions + type TransactionCallback = (trx: Knex) => Promise; + + // Mock dependencies + const mockFolderCommitDAL = { + create: vi.fn().mockResolvedValue({}), + findById: vi.fn().mockResolvedValue({}), + findByFolderId: vi.fn().mockResolvedValue([]), + findLatestCommit: vi.fn().mockResolvedValue({}), + transaction: vi.fn().mockImplementation((callback: TransactionCallback) => callback({} as Knex)), + getNumberOfCommitsSince: vi.fn().mockResolvedValue(0), + getEnvNumberOfCommitsSince: vi.fn().mockResolvedValue(0), + findCommitsToRecreate: vi.fn().mockResolvedValue([]), + findMultipleLatestCommits: vi.fn().mockResolvedValue([]), + findLatestCommitBetween: vi.fn().mockResolvedValue({}), + findAllCommitsBetween: vi.fn().mockResolvedValue([]), + findLatestEnvCommit: vi.fn().mockResolvedValue({}), + findLatestCommitByFolderIds: vi.fn().mockResolvedValue({}) + }; + + const mockFolderCommitChangesDAL = { + create: vi.fn().mockResolvedValue({}), + findByCommitId: vi.fn().mockResolvedValue([]), + insertMany: vi.fn().mockResolvedValue([]) + }; + + const mockFolderCheckpointDAL = { + create: vi.fn().mockResolvedValue({}), + findByFolderId: vi.fn().mockResolvedValue([]), + findLatestByFolderId: vi.fn().mockResolvedValue(null), + findNearestCheckpoint: vi.fn().mockResolvedValue({}) + }; + + const mockFolderCheckpointResourcesDAL = { + insertMany: vi.fn().mockResolvedValue([]), + findByCheckpointId: vi.fn().mockResolvedValue([]) + }; + + const mockFolderTreeCheckpointDAL = { + create: vi.fn().mockResolvedValue({}), + findByProjectId: vi.fn().mockResolvedValue([]), + findLatestByProjectId: vi.fn().mockResolvedValue({}), + findNearestCheckpoint: vi.fn().mockResolvedValue({}), + findLatestByEnvId: vi.fn().mockResolvedValue({}) + }; + + const mockFolderTreeCheckpointResourcesDAL = { + insertMany: vi.fn().mockResolvedValue([]), + findByTreeCheckpointId: vi.fn().mockResolvedValue([]) + }; + + const mockUserDAL = { + findById: vi.fn().mockResolvedValue({}) + }; + + const mockIdentityDAL = { + findById: vi.fn().mockResolvedValue({}) + }; + + const mockFolderDAL = { + findByParentId: vi.fn().mockResolvedValue([]), + findByProjectId: vi.fn().mockResolvedValue([]), + deleteById: vi.fn().mockResolvedValue({}), + create: vi.fn().mockResolvedValue({}), + updateById: vi.fn().mockResolvedValue({}), + update: vi.fn().mockResolvedValue({}), + find: vi.fn().mockResolvedValue([]), + findById: vi.fn().mockResolvedValue({}), + findByEnvId: vi.fn().mockResolvedValue([]), + findFoldersByRootAndIds: vi.fn().mockResolvedValue([]) + }; + + const mockFolderVersionDAL = { + findLatestFolderVersions: vi.fn().mockResolvedValue({}), + findById: vi.fn().mockResolvedValue({}), + deleteById: vi.fn().mockResolvedValue({}), + create: vi.fn().mockResolvedValue({}), + updateById: vi.fn().mockResolvedValue({}), + find: vi.fn().mockResolvedValue([]), + findByIdsWithLatestVersion: vi.fn().mockResolvedValue({}) + }; + + const mockSecretVersionV2BridgeDAL = { + findLatestVersionByFolderId: vi.fn().mockResolvedValue([]), + findById: vi.fn().mockResolvedValue({}), + deleteById: vi.fn().mockResolvedValue({}), + create: vi.fn().mockResolvedValue({}), + updateById: vi.fn().mockResolvedValue({}), + find: vi.fn().mockResolvedValue([]), + findByIdsWithLatestVersion: vi.fn().mockResolvedValue({}) + }; + + const mockSecretV2BridgeDAL = { + deleteById: vi.fn().mockResolvedValue({}), + create: vi.fn().mockResolvedValue({}), + updateById: vi.fn().mockResolvedValue({}), + update: vi.fn().mockResolvedValue({}), + insertMany: vi.fn().mockResolvedValue([]), + invalidateSecretCacheByProjectId: vi.fn().mockResolvedValue({}) + }; + + const mockProjectDAL = { + findById: vi.fn().mockResolvedValue({}) + }; + + const mockFolderCommitQueueService = { + scheduleTreeCheckpoint: vi.fn().mockResolvedValue({}) + }; + + let folderCommitService: TFolderCommitServiceFactory; + + beforeEach(() => { + vi.clearAllMocks(); + + folderCommitService = folderCommitServiceFactory({ + folderCommitDAL: mockFolderCommitDAL, + folderCommitChangesDAL: mockFolderCommitChangesDAL, + folderCheckpointDAL: mockFolderCheckpointDAL, + folderCheckpointResourcesDAL: mockFolderCheckpointResourcesDAL, + folderTreeCheckpointDAL: mockFolderTreeCheckpointDAL, + folderTreeCheckpointResourcesDAL: mockFolderTreeCheckpointResourcesDAL, + userDAL: mockUserDAL, + identityDAL: mockIdentityDAL, + folderDAL: mockFolderDAL, + folderVersionDAL: mockFolderVersionDAL, + secretVersionV2BridgeDAL: mockSecretVersionV2BridgeDAL, + projectDAL: mockProjectDAL, + secretV2BridgeDAL: mockSecretV2BridgeDAL, + folderCommitQueueService: mockFolderCommitQueueService + }); + }); + + afterEach(() => { + vi.resetAllMocks(); + }); + + describe("createCommit", () => { + it("should successfully create a commit with user actor", async () => { + // Arrange + const userData = { id: "user-id", username: "testuser" }; + const folderData = { id: "folder-id", envId: "env-id" }; + const commitData = { id: "commit-id", folderId: "folder-id" }; + + mockUserDAL.findById.mockResolvedValue(userData); + mockFolderDAL.findById.mockResolvedValue(folderData); + mockFolderCommitDAL.create.mockResolvedValue(commitData); + mockFolderCheckpointDAL.findLatestByFolderId.mockResolvedValue(null); + mockFolderCommitDAL.findLatestCommit.mockResolvedValue({ id: "latest-commit-id" }); + mockFolderDAL.findByParentId.mockResolvedValue([]); + mockSecretVersionV2BridgeDAL.findLatestVersionByFolderId.mockResolvedValue([]); + + const data = { + actor: { + type: ActorType.USER, + metadata: { id: userData.id } + }, + message: "Test commit", + folderId: folderData.id, + changes: [ + { + type: "add", + secretVersionId: "secret-version-1" + } + ] + }; + + // Act + const result = await folderCommitService.createCommit(data); + + // Assert + expect(mockUserDAL.findById).toHaveBeenCalledWith(userData.id, undefined); + expect(mockFolderDAL.findById).toHaveBeenCalledWith(folderData.id, undefined); + expect(mockFolderCommitDAL.create).toHaveBeenCalledWith( + expect.objectContaining({ + actorType: ActorType.USER, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + actorMetadata: expect.objectContaining({ name: userData.username }), + message: data.message, + folderId: data.folderId, + envId: folderData.envId + }), + undefined + ); + expect(mockFolderCommitChangesDAL.insertMany).toHaveBeenCalledWith( + expect.arrayContaining([ + expect.objectContaining({ + folderCommitId: commitData.id, + changeType: data.changes[0].type, + secretVersionId: data.changes[0].secretVersionId + }) + ]), + undefined + ); + expect(mockFolderCommitQueueService.scheduleTreeCheckpoint).toHaveBeenCalledWith(folderData.envId); + expect(result).toEqual(commitData); + }); + + it("should successfully create a commit with identity actor", async () => { + // Arrange + const identityData = { id: "identity-id", name: "testidentity" }; + const folderData = { id: "folder-id", envId: "env-id" }; + const commitData = { id: "commit-id", folderId: "folder-id" }; + + mockIdentityDAL.findById.mockResolvedValue(identityData); + mockFolderDAL.findById.mockResolvedValue(folderData); + mockFolderCommitDAL.create.mockResolvedValue(commitData); + mockFolderCheckpointDAL.findLatestByFolderId.mockResolvedValue(null); + mockFolderCommitDAL.findLatestCommit.mockResolvedValue({ id: "latest-commit-id" }); + mockFolderDAL.findByParentId.mockResolvedValue([]); + mockSecretVersionV2BridgeDAL.findLatestVersionByFolderId.mockResolvedValue([]); + + const data = { + actor: { + type: ActorType.IDENTITY, + metadata: { id: identityData.id } + }, + message: "Test commit", + folderId: folderData.id, + changes: [ + { + type: "add", + folderVersionId: "folder-version-1" + } + ] + }; + + // Act + const result = await folderCommitService.createCommit(data); + + // Assert + expect(mockIdentityDAL.findById).toHaveBeenCalledWith(identityData.id, undefined); + expect(mockFolderDAL.findById).toHaveBeenCalledWith(folderData.id, undefined); + expect(mockFolderCommitDAL.create).toHaveBeenCalledWith( + expect.objectContaining({ + actorType: ActorType.IDENTITY, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + actorMetadata: expect.objectContaining({ name: identityData.name }), + message: data.message, + folderId: data.folderId, + envId: folderData.envId + }), + undefined + ); + expect(mockFolderCommitChangesDAL.insertMany).toHaveBeenCalledWith( + expect.arrayContaining([ + expect.objectContaining({ + folderCommitId: commitData.id, + changeType: data.changes[0].type, + folderVersionId: data.changes[0].folderVersionId + }) + ]), + undefined + ); + expect(mockFolderCommitQueueService.scheduleTreeCheckpoint).toHaveBeenCalledWith(folderData.envId); + expect(result).toEqual(commitData); + }); + + it("should throw NotFoundError when folder does not exist", async () => { + // Arrange + mockFolderDAL.findById.mockResolvedValue(null); + + const data = { + actor: { + type: ActorType.PLATFORM + }, + message: "Test commit", + folderId: "non-existent-folder", + changes: [] + }; + + // Act & Assert + await expect(folderCommitService.createCommit(data)).rejects.toThrow(NotFoundError); + expect(mockFolderDAL.findById).toHaveBeenCalledWith("non-existent-folder", undefined); + }); + }); + + describe("addCommitChange", () => { + it("should successfully add a change to an existing commit", async () => { + // Arrange + const commitData = { id: "commit-id", folderId: "folder-id" }; + const changeData = { id: "change-id", folderCommitId: "commit-id" }; + + mockFolderCommitDAL.findById.mockResolvedValue(commitData); + mockFolderCommitChangesDAL.create.mockResolvedValue(changeData); + + const data = { + folderCommitId: commitData.id, + changeType: "add", + secretVersionId: "secret-version-1" + }; + + // Act + const result = await folderCommitService.addCommitChange(data); + + // Assert + expect(mockFolderCommitDAL.findById).toHaveBeenCalledWith(commitData.id, undefined); + expect(mockFolderCommitChangesDAL.create).toHaveBeenCalledWith(data, undefined); + expect(result).toEqual(changeData); + }); + + it("should throw BadRequestError when neither secretVersionId nor folderVersionId is provided", async () => { + // Arrange + const data = { + folderCommitId: "commit-id", + changeType: "add" + }; + + // Act & Assert + await expect(folderCommitService.addCommitChange(data)).rejects.toThrow(BadRequestError); + }); + + it("should throw NotFoundError when commit does not exist", async () => { + // Arrange + mockFolderCommitDAL.findById.mockResolvedValue(null); + + const data = { + folderCommitId: "non-existent-commit", + changeType: "add", + secretVersionId: "secret-version-1" + }; + + // Act & Assert + await expect(folderCommitService.addCommitChange(data)).rejects.toThrow(NotFoundError); + expect(mockFolderCommitDAL.findById).toHaveBeenCalledWith("non-existent-commit", undefined); + }); + }); + + // Note: reconstructFolderState is an internal function not exposed in the public API + // We'll test it indirectly through compareFolderStates + + describe("compareFolderStates", () => { + it("should mark all resources as creates when currentCommitId is not provided", async () => { + // Arrange + const targetCommitId = "target-commit-id"; + const targetCommit = { id: targetCommitId, commitId: 1, folderId: "folder-id" }; + + mockFolderCommitDAL.findById.mockResolvedValue(targetCommit); + // Mock how compareFolderStates would process the results internally + mockFolderCheckpointDAL.findNearestCheckpoint.mockResolvedValue({ id: "checkpoint-id", commitId: "hash-0" }); + mockFolderCheckpointResourcesDAL.findByCheckpointId.mockResolvedValue([ + { secretVersionId: "secret-version-1", referencedSecretId: "secret-1" }, + { folderVersionId: "folder-version-1", referencedFolderId: "folder-1" } + ]); + mockFolderCommitDAL.findCommitsToRecreate.mockResolvedValue([]); + + // Act + const result = await folderCommitService.compareFolderStates({ + targetCommitId + }); + + // Assert + expect(mockFolderCommitDAL.findById).toHaveBeenCalledWith(targetCommitId, undefined); + + // Verify we get resources marked as create + expect(result).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + changeType: "create", + commitId: targetCommit.commitId + }) + ]) + ); + }); + }); + + describe("createFolderCheckpoint", () => { + it("should successfully create a checkpoint when force is true", async () => { + // Arrange + const folderCommitId = "commit-id"; + const folderId = "folder-id"; + const checkpointData = { id: "checkpoint-id", folderCommitId }; + + mockFolderDAL.findByParentId.mockResolvedValue([{ id: "subfolder-id" }]); + mockFolderVersionDAL.findLatestFolderVersions.mockResolvedValue({ "subfolder-id": { id: "folder-version-1" } }); + mockSecretVersionV2BridgeDAL.findLatestVersionByFolderId.mockResolvedValue([{ id: "secret-version-1" }]); + mockFolderCheckpointDAL.create.mockResolvedValue(checkpointData); + + // Act + const result = await folderCommitService.createFolderCheckpoint({ + folderId, + folderCommitId, + force: true + }); + + // Assert + expect(mockFolderCheckpointDAL.create).toHaveBeenCalledWith({ folderCommitId }, undefined); + expect(mockFolderCheckpointResourcesDAL.insertMany).toHaveBeenCalled(); + expect(result).toBe(folderCommitId); + }); + }); + + describe("deepRollbackFolder", () => { + it("should throw NotFoundError when commit doesn't exist", async () => { + // Arrange + const targetCommitId = "non-existent-commit"; + const envId = "env-id"; + const actorId = "user-id"; + const actorType = ActorType.USER; + const projectId = "project-id"; + + mockFolderCommitDAL.findById.mockResolvedValue(null); + + // Act & Assert + await expect( + folderCommitService.deepRollbackFolder(targetCommitId, envId, actorId, actorType, projectId) + ).rejects.toThrow(NotFoundError); + }); + }); + + describe("createFolderTreeCheckpoint", () => { + it("should create a tree checkpoint when checkpoint window is exceeded", async () => { + // Arrange + const envId = "env-id"; + const folderCommitId = "commit-id"; + const latestCommit = { id: folderCommitId }; + const latestTreeCheckpoint = { id: "tree-checkpoint-id", folderCommitId: "old-commit-id" }; + const folders = [ + { id: "folder-1", isReserved: false }, + { id: "folder-2", isReserved: false }, + { id: "folder-3", isReserved: true } // Reserved folders should be filtered out + ]; + const folderCommits = [ + { folderId: "folder-1", id: "commit-1" }, + { folderId: "folder-2", id: "commit-2" } + ]; + const treeCheckpoint = { id: "new-tree-checkpoint-id" }; + + mockFolderCommitDAL.findLatestEnvCommit.mockResolvedValue(latestCommit); + mockFolderTreeCheckpointDAL.findLatestByEnvId.mockResolvedValue(latestTreeCheckpoint); + mockFolderCommitDAL.getEnvNumberOfCommitsSince.mockResolvedValue(15); // More than PIT_TREE_CHECKPOINT_WINDOW (10) + mockFolderDAL.findByEnvId.mockResolvedValue(folders); + mockFolderCommitDAL.findMultipleLatestCommits.mockResolvedValue(folderCommits); + mockFolderTreeCheckpointDAL.create.mockResolvedValue(treeCheckpoint); + + // Act + await folderCommitService.createFolderTreeCheckpoint(envId); + + // Assert + expect(mockFolderCommitDAL.findLatestEnvCommit).toHaveBeenCalledWith(envId, undefined); + expect(mockFolderTreeCheckpointDAL.create).toHaveBeenCalledWith({ folderCommitId }, undefined); + }); + }); + + describe("applyFolderStateDifferences", () => { + it("should process changes correctly", async () => { + // Arrange + const folderId = "folder-id"; + const projectId = "project-id"; + const actorId = "user-id"; + const actorType = ActorType.USER; + + const differences = [ + { type: "secret", id: "secret-1", versionId: "v1", changeType: ChangeType.CREATE, commitId: 1 }, + { type: "folder", id: "folder-1", versionId: "v2", changeType: ChangeType.UPDATE, commitId: 1 } + ]; + + const secretVersions = { + "secret-1": { + id: "secret-version-1", + createdAt: new Date(), + updatedAt: new Date(), + type: "shared", + folderId: "folder-1", + secretId: "secret-1", + version: 1, + key: "SECRET_KEY", + encryptedValue: Buffer.from("encrypted"), + encryptedComment: Buffer.from("comment"), + skipMultilineEncoding: false, + userId: "user-1", + envId: "env-1", + metadata: {} + } as TSecretVersionsV2 + }; + + const folderVersions = { + "folder-1": { + folderId: "folder-1", + version: 1, + name: "Test Folder", + envId: "env-1" + } as TSecretFolderVersions + }; + + // Mock folder lookup for the folder being processed + mockFolderDAL.findById.mockImplementation((id) => { + if (id === folderId) { + return Promise.resolve({ id: folderId, envId: "env-1" }); + } + return Promise.resolve(null); + }); + + // Mock latest commit lookup + mockFolderCommitDAL.findLatestCommit.mockImplementation((id) => { + if (id === folderId) { + return Promise.resolve({ id: "latest-commit-id", folderId }); + } + return Promise.resolve(null); + }); + + // Make sure findByParentId returns an array, not undefined + mockFolderDAL.findByParentId.mockResolvedValue([]); + + // Make sure other required functions return appropriate values + mockFolderCheckpointDAL.findLatestByFolderId.mockResolvedValue(null); + mockSecretVersionV2BridgeDAL.findLatestVersionByFolderId.mockResolvedValue([]); + + // These mocks need to return objects with an id field + mockSecretVersionV2BridgeDAL.findByIdsWithLatestVersion.mockResolvedValue(secretVersions); + mockFolderVersionDAL.findByIdsWithLatestVersion.mockResolvedValue(folderVersions); + mockSecretV2BridgeDAL.insertMany.mockResolvedValue([{ id: "new-secret-1" }]); + mockSecretVersionV2BridgeDAL.create.mockResolvedValue({ id: "new-secret-version-1" }); + mockFolderDAL.updateById.mockResolvedValue({ id: "updated-folder-1" }); + mockFolderVersionDAL.create.mockResolvedValue({ id: "new-folder-version-1" }); + mockFolderCommitDAL.create.mockResolvedValue({ id: "new-commit-id" }); + + // Mock transaction + mockFolderCommitDAL.transaction.mockImplementation((callback: TransactionCallback) => callback({} as Knex)); + + // Act + const result = await folderCommitService.applyFolderStateDifferences({ + differences, + actorInfo: { + actorType, + actorId, + message: "Applying changes" + }, + folderId, + projectId, + reconstructNewFolders: false + }); + + // Assert + expect(mockFolderCommitDAL.create).toHaveBeenCalled(); + expect(mockSecretV2BridgeDAL.invalidateSecretCacheByProjectId).toHaveBeenCalledWith(projectId); + + // Check that we got the right counts + expect(result).toEqual({ + secretChangesCount: 1, + folderChangesCount: 1, + totalChanges: 2 + }); + }); + }); +}); diff --git a/backend/src/services/folder-commit/folder-commit-service.ts b/backend/src/services/folder-commit/folder-commit-service.ts index e40847d06..1fc4b456d 100644 --- a/backend/src/services/folder-commit/folder-commit-service.ts +++ b/backend/src/services/folder-commit/folder-commit-service.ts @@ -1,23 +1,87 @@ /* eslint-disable no-await-in-loop */ import { Knex } from "knex"; -import { TSecretFolders } from "@app/db/schemas"; +import { TSecretFolders, TSecretFolderVersions, TSecretVersionsV2 } from "@app/db/schemas"; import { getConfig } from "@app/lib/config/env"; import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors"; +import { logger } from "@app/lib/logger"; import { ActorType } from "../auth/auth-type"; import { TFolderCheckpointDALFactory } from "../folder-checkpoint/folder-checkpoint-dal"; import { TFolderCheckpointResourcesDALFactory } from "../folder-checkpoint-resources/folder-checkpoint-resources-dal"; import { TFolderCommitChangesDALFactory } from "../folder-commit-changes/folder-commit-changes-dal"; import { TFolderTreeCheckpointDALFactory } from "../folder-tree-checkpoint/folder-tree-checkpoint-dal"; +import { TFolderTreeCheckpointResourcesDALFactory } from "../folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal"; import { TIdentityDALFactory } from "../identity/identity-dal"; import { TProjectDALFactory } from "../project/project-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal"; -import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; +import * as secretV2BridgeDal from "../secret-v2-bridge/secret-v2-bridge-dal"; import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal"; import { TUserDALFactory } from "../user/user-dal"; import { TFolderCommitDALFactory } from "./folder-commit-dal"; +import { TFolderCommitQueueServiceFactory } from "./folder-commit-queue"; + +// Define enums for better type safety +export enum ChangeType { + ADD = "add", + DELETE = "delete", + UPDATE = "update", + CREATE = "create" +} + +enum ResourceType { + SECRET = "secret", + FOLDER = "folder" +} + +// Improved types for DTO objects +type TCreateCommitDTO = { + actor: { + type: string; + metadata?: { + name?: string; + id?: string; + }; + }; + message?: string; + folderId: string; + changes: { + type: string; + secretVersionId?: string; + folderVersionId?: string; + }[]; +}; + +type TCommitChangeDTO = { + folderCommitId: string; + changeType: string; + secretVersionId?: string; + folderVersionId?: string; +}; + +export type ResourceChange = { + type: string; + id: string; + versionId: string; + oldVersionId?: string; + changeType: ChangeType; + commitId: number; + createdAt?: Date; + parentId?: string; +}; + +type ActorInfo = { + actorType: string; + actorId?: string; + message?: string; +}; + +type StateChangeResult = { + secretChangesCount: number; + folderChangesCount: number; + totalChanges: number; +}; type TFolderCommitServiceFactoryDep = { folderCommitDAL: Pick< @@ -28,7 +92,13 @@ type TFolderCommitServiceFactoryDep = { | "findLatestCommit" | "transaction" | "getNumberOfCommitsSince" + | "getEnvNumberOfCommitsSince" | "findCommitsToRecreate" + | "findMultipleLatestCommits" + | "findLatestCommitBetween" + | "findAllCommitsBetween" + | "findLatestEnvCommit" + | "findLatestCommitByFolderIds" >; folderCommitChangesDAL: Pick; folderCheckpointDAL: Pick< @@ -36,12 +106,28 @@ type TFolderCommitServiceFactoryDep = { "create" | "findByFolderId" | "findLatestByFolderId" | "findNearestCheckpoint" >; folderCheckpointResourcesDAL: Pick; - folderTreeCheckpointDAL: Pick; + folderTreeCheckpointDAL: Pick< + TFolderTreeCheckpointDALFactory, + "create" | "findNearestCheckpoint" | "findLatestByEnvId" + >; + folderTreeCheckpointResourcesDAL: Pick< + TFolderTreeCheckpointResourcesDALFactory, + "insertMany" | "findByTreeCheckpointId" + >; userDAL: Pick; identityDAL: Pick; folderDAL: Pick< TSecretFolderDALFactory, - "findByParentId" | "findByProjectId" | "deleteById" | "create" | "updateById" | "update" + | "findByParentId" + | "findByProjectId" + | "deleteById" + | "create" + | "updateById" + | "update" + | "find" + | "findById" + | "findByEnvId" + | "findFoldersByRootAndIds" >; folderVersionDAL: Pick< TSecretFolderVersionDALFactory, @@ -64,34 +150,11 @@ type TFolderCommitServiceFactoryDep = { | "findByIdsWithLatestVersion" >; secretV2BridgeDAL: Pick< - TSecretV2BridgeDALFactory, + secretV2BridgeDal.TSecretV2BridgeDALFactory, "deleteById" | "create" | "updateById" | "update" | "insertMany" | "invalidateSecretCacheByProjectId" >; projectDAL: Pick; -}; - -export type TCreateCommitDTO = { - actor: { - type: string; - metadata?: { - name?: string; - id?: string; - }; - }; - message?: string; - folderId: string; - changes: { - type: string; - secretVersionId?: string; - folderVersionId?: string; - }[]; -}; - -export type TCommitChangeDTO = { - folderCommitId: string; - changeType: string; - secretVersionId?: string; - folderVersionId?: string; + folderCommitQueueService?: Pick; }; export const folderCommitServiceFactory = ({ @@ -106,25 +169,36 @@ export const folderCommitServiceFactory = ({ folderVersionDAL, secretVersionV2BridgeDAL, projectDAL, - secretV2BridgeDAL + secretV2BridgeDAL, + folderTreeCheckpointResourcesDAL, + folderCommitQueueService }: TFolderCommitServiceFactoryDep) => { const appCfg = getConfig(); + /** + * Fetches all resources within a folder + */ const getFolderResources = async (folderId: string, tx?: Knex) => { const resources = []; const subFolders = await folderDAL.findByParentId(folderId, tx); + if (subFolders.length > 0) { const subFolderIds = subFolders.map((folder) => folder.id); const folderVersions = await folderVersionDAL.findLatestFolderVersions(subFolderIds, tx); resources.push(...Object.values(folderVersions).map((folderVersion) => ({ folderVersionId: folderVersion.id }))); } + const secretVersions = await secretVersionV2BridgeDAL.findLatestVersionByFolderId(folderId, tx); if (secretVersions.length > 0) { resources.push(...secretVersions.map((secretVersion) => ({ secretVersionId: secretVersion.id }))); } + return resources; }; + /** + * Creates a checkpoint for a folder if necessary + */ const createFolderCheckpoint = async ({ folderId, folderCommitId, @@ -138,23 +212,26 @@ export const folderCommitServiceFactory = ({ }) => { let latestCommitId = folderCommitId; const latestCheckpoint = await folderCheckpointDAL.findLatestByFolderId(folderId, tx); + if (!latestCommitId) { - latestCommitId = (await folderCommitDAL.findLatestCommit(folderId, tx))?.id; - } - if (!latestCommitId) { - throw new BadRequestError({ message: "Latest commit ID not found" }); - return; + const latestCommit = await folderCommitDAL.findLatestCommit(folderId, tx); + if (!latestCommit) { + throw new BadRequestError({ message: "Latest commit ID not found" }); + } + latestCommitId = latestCommit.id; } + if (!force && latestCheckpoint) { const commitsSinceLastCheckpoint = await folderCommitDAL.getNumberOfCommitsSince( folderId, latestCheckpoint.folderCommitId, tx ); - if (commitsSinceLastCheckpoint < Number(appCfg.CHECKPOINT_WINDOW)) { + if (commitsSinceLastCheckpoint < Number(appCfg.PIT_CHECKPOINT_WINDOW)) { return; } } + const checkpointResources = await getFolderResources(folderId, tx); if (checkpointResources.length > 0) { @@ -169,8 +246,12 @@ export const folderCommitServiceFactory = ({ tx ); } + return latestCommitId; }; + /** + * Reconstructs the state of a folder at a specific commit + */ const reconstructFolderState = async ( folderCommitId: string, tx?: Knex @@ -249,8 +330,37 @@ export const folderCommitServiceFactory = ({ return Object.values(folderState); }; - const compareFolderStates = async (currentCommitId: string, targetCommitId: string, tx?: Knex) => { - // Reconstruct state for both commits + /** + * Compares folder states between two commits and returns the differences + */ + const compareFolderStates = async ({ + currentCommitId, + targetCommitId, + tx + }: { + currentCommitId?: string; + targetCommitId: string; + tx?: Knex; + }) => { + const targetCommit = await folderCommitDAL.findById(targetCommitId, tx); + if (!targetCommit) { + throw new NotFoundError({ message: `Commit with ID ${targetCommitId} not found` }); + } + + // If currentCommitId is not provided, mark all resources in target as creates + if (!currentCommitId) { + const targetState = await reconstructFolderState(targetCommitId, tx); + + return targetState.map((resource) => ({ + type: resource.type, + id: resource.id, + versionId: resource.versionId, + changeType: "create", + commitId: targetCommit.commitId + })) as ResourceChange[]; + } + + // Original logic for when currentCommitId is provided const currentState = await reconstructFolderState(currentCommitId, tx); const targetState = await reconstructFolderState(targetCommitId, tx); @@ -258,52 +368,45 @@ export const folderCommitServiceFactory = ({ const currentMap: Record = {}; const targetMap: Record = {}; - // Build lookup map for current state + // Build lookup maps currentState.forEach((resource) => { const key = `${resource.type}-${resource.id}`; currentMap[key] = resource; }); - // Build lookup map for target state targetState.forEach((resource) => { const key = `${resource.type}-${resource.id}`; targetMap[key] = resource; }); // Track differences - const differences: { - type: string; - id: string; - versionId: string; - changeType: "create" | "update" | "delete"; - }[] = []; + const differences: ResourceChange[] = []; - // Find deletes and updates (resources in current but not in target, or with different versions) + // Find deletes and updates Object.keys(currentMap).forEach((key) => { const currentResource = currentMap[key]; const targetResource = targetMap[key]; if (!targetResource) { - // Resource exists in current but not in target - it's a delete differences.push({ type: currentResource.type, id: currentResource.id, versionId: currentResource.versionId, - changeType: "delete" + changeType: ChangeType.DELETE, + commitId: targetCommit.commitId }); } else if (currentResource.versionId !== targetResource.versionId) { - // Resource exists in both but with different versions - it's an update differences.push({ type: targetResource.type, id: targetResource.id, versionId: targetResource.versionId, - changeType: "update" + changeType: ChangeType.UPDATE, + commitId: targetCommit.commitId }); } - // If versions are the same, it's unchanged - exclude from result }); - // Find creates (resources in target but not in current) + // Find creates Object.keys(targetMap).forEach((key) => { if (!currentMap[key]) { const targetResource = targetMap[key]; @@ -311,7 +414,9 @@ export const folderCommitServiceFactory = ({ type: targetResource.type, id: targetResource.id, versionId: targetResource.versionId, - changeType: "create" + changeType: ChangeType.CREATE, + commitId: targetCommit.commitId, + createdAt: targetCommit.createdAt }); } }); @@ -319,7 +424,9 @@ export const folderCommitServiceFactory = ({ return differences; }; - // Add a change to an existing commit + /** + * Adds a change to an existing commit + */ const addCommitChange = async (data: TCommitChangeDTO, tx?: Knex) => { try { if (!data.secretVersionId && !data.folderVersionId) { @@ -340,26 +447,39 @@ export const folderCommitServiceFactory = ({ } }; + /** + * Creates a new commit with the provided changes + */ const createCommit = async (data: TCreateCommitDTO, tx?: Knex) => { - const metadata = data.actor.metadata || {}; try { + const metadata = { ...data.actor.metadata } || {}; + if (data.actor.type === ActorType.USER && data.actor.metadata?.id) { const user = await userDAL.findById(data.actor.metadata?.id, tx); metadata.name = user?.username; } + if (data.actor.type === ActorType.IDENTITY && data.actor.metadata?.id) { const identity = await identityDAL.findById(data.actor.metadata?.id, tx); metadata.name = identity?.name; } + + const folder = await folderDAL.findById(data.folderId, tx); + if (!folder) { + throw new NotFoundError({ message: `Folder with ID ${data.folderId} not found` }); + } + const newCommit = await folderCommitDAL.create( { actorMetadata: metadata, actorType: data.actor.type, message: data.message, - folderId: data.folderId + folderId: data.folderId, + envId: folder.envId }, tx ); + await folderCommitChangesDAL.insertMany( data.changes.map((change) => ({ folderCommitId: newCommit.id, @@ -371,167 +491,184 @@ export const folderCommitServiceFactory = ({ ); await createFolderCheckpoint({ folderId: data.folderId, tx }); + if (folderCommitQueueService) { + await folderCommitQueueService.scheduleTreeCheckpoint(folder.envId); + } return newCommit; } catch (error) { + if (error instanceof NotFoundError || error instanceof BadRequestError) { + throw error; + } throw new DatabaseError({ error, name: "CreateCommit" }); } }; - const applyFolderStateDifferences = async ( - differences: Array<{ - type: string; - id: string; - versionId: string; - oldVersionId?: string; - changeType: "create" | "update" | "delete"; - }>, - actorInfo: { - actorType: string; - actorId?: string; - message?: string; - }, + /** + * Process secret changes when applying folder state differences + */ + const processSecretChanges = async ( + changes: ResourceChange[], + secretVersions: Record, + actorInfo: ActorInfo, folderId: string, - projectId: string + tx?: Knex ) => { - let result = {}; - await folderCommitDAL.transaction(async (tx) => { - // Group differences by type for more efficient processing - const secretChanges = differences.filter((diff) => diff.type === "secret"); - const folderChanges = differences.filter((diff) => diff.type === "folder"); + const commitChanges = []; - const secretVersions = await secretVersionV2BridgeDAL.findByIdsWithLatestVersion( - folderId, - secretChanges.map((diff) => diff.id), - secretChanges.map((diff) => diff.versionId) - ); - const folderVersions = await folderVersionDAL.findByIdsWithLatestVersion( - folderChanges.map((diff) => diff.id), - folderChanges.map((diff) => diff.versionId) - ); + for (const change of changes) { + const secretVersion = secretVersions[change.id]; + // eslint-disable-next-line no-continue + if (!secretVersion) continue; - // Track all changes for commit recording - const commitChanges = []; + switch (change.changeType) { + case "create": + { + const newSecret = [ + { + id: change.id, + skipMultilineEncoding: secretVersion.skipMultilineEncoding, + version: secretVersion.version + 1, + type: secretVersion.type, + key: secretVersion.key, + reminderNote: secretVersion.reminderNote, + reminderRepeatDays: secretVersion.reminderRepeatDays, + encryptedValue: secretVersion.encryptedValue, + encryptedComment: secretVersion.encryptedComment, + userId: secretVersion.userId, + metadata: secretVersion.metadata, + folderId + } + ]; + await secretV2BridgeDAL.insertMany(newSecret, tx); - // Process secret changes - for (const change of secretChanges) { - const secretVersion = secretVersions[change.id]; - switch (change.changeType) { - case "create": - if (secretVersion) { - const newSecret = [ - { - id: change.id, - skipMultilineEncoding: secretVersion.skipMultilineEncoding, - version: secretVersion.version + 1, - type: secretVersion.type, - key: secretVersion.key, - reminderNote: secretVersion.reminderNote, - reminderRepeatDays: secretVersion.reminderRepeatDays, - encryptedValue: secretVersion.encryptedValue, - encryptedComment: secretVersion.encryptedComment, - userId: secretVersion.userId, - metadata: secretVersion.metadata, - folderId - } - ]; - await secretV2BridgeDAL.insertMany(newSecret, tx); - - const newVersion = await secretVersionV2BridgeDAL.create( - { - folderId, - secretId: secretVersion.secretId, - version: secretVersion.version + 1, - encryptedValue: secretVersion.encryptedValue, - key: secretVersion.key, - encryptedComment: secretVersion.encryptedComment, - skipMultilineEncoding: secretVersion.skipMultilineEncoding, - reminderNote: secretVersion.reminderNote, - reminderRepeatDays: secretVersion.reminderRepeatDays, - userId: secretVersion.userId, - metadata: secretVersion.metadata, - actorType: actorInfo.actorType, - envId: secretVersion.envId, - ...(actorInfo.actorType === ActorType.IDENTITY && { identityActorId: actorInfo.actorId }), - ...(actorInfo.actorType === ActorType.USER && { userActorId: actorInfo.actorId }) - }, - tx - ); - - commitChanges.push({ - type: "add", - secretVersionId: newVersion.id - }); - } - break; - - case "update": - // Update secret to specific version - if (secretVersion) { - await secretV2BridgeDAL.updateById( - change.id, - { - skipMultilineEncoding: secretVersion?.skipMultilineEncoding, - version: secretVersion?.version, - type: secretVersion?.type, - key: secretVersion?.key, - reminderNote: secretVersion?.reminderNote, - reminderRepeatDays: secretVersion?.reminderRepeatDays, - encryptedValue: secretVersion?.encryptedValue, - encryptedComment: secretVersion?.encryptedComment, - userId: secretVersion?.userId, - metadata: secretVersion?.metadata - }, - tx - ); - - const newVersion = await secretVersionV2BridgeDAL.create( - { - version: secretVersion.version + 1, - encryptedValue: secretVersion.encryptedValue, - key: secretVersion.key, - encryptedComment: secretVersion.encryptedComment, - skipMultilineEncoding: secretVersion.skipMultilineEncoding, - reminderNote: secretVersion.reminderNote, - reminderRepeatDays: secretVersion.reminderRepeatDays, - userId: secretVersion.userId, - metadata: secretVersion.metadata, - actorType: actorInfo.actorType, - envId: secretVersion.envId, - folderId, - secretId: secretVersion.secretId, - ...(actorInfo.actorType === ActorType.IDENTITY && { identityActorId: actorInfo.actorId }), - ...(actorInfo.actorType === ActorType.USER && { userActorId: actorInfo.actorId }) - }, - tx - ); - - commitChanges.push({ - type: "add", - secretVersionId: newVersion.id - }); - } - break; - - case "delete": - await secretV2BridgeDAL.deleteById(change.id, tx); + const newVersion = await secretVersionV2BridgeDAL.create( + { + folderId, + secretId: secretVersion.secretId, + version: secretVersion.version + 1, + encryptedValue: secretVersion.encryptedValue, + key: secretVersion.key, + encryptedComment: secretVersion.encryptedComment, + skipMultilineEncoding: secretVersion.skipMultilineEncoding, + reminderNote: secretVersion.reminderNote, + reminderRepeatDays: secretVersion.reminderRepeatDays, + userId: secretVersion.userId, + metadata: secretVersion.metadata, + actorType: actorInfo.actorType, + envId: secretVersion.envId, + ...(actorInfo.actorType === ActorType.IDENTITY && { identityActorId: actorInfo.actorId }), + ...(actorInfo.actorType === ActorType.USER && { userActorId: actorInfo.actorId }) + }, + tx + ); commitChanges.push({ - type: "delete", - secretVersionId: change.versionId + type: ChangeType.ADD, + secretVersionId: newVersion.id }); - break; + } + break; - default: - throw new BadRequestError({ message: `Unknown change type: ${change.changeType as string}` }); - } + case "update": + { + await secretV2BridgeDAL.updateById( + change.id, + { + skipMultilineEncoding: secretVersion?.skipMultilineEncoding, + version: secretVersion?.version, + type: secretVersion?.type, + key: secretVersion?.key, + reminderNote: secretVersion?.reminderNote, + reminderRepeatDays: secretVersion?.reminderRepeatDays, + encryptedValue: secretVersion?.encryptedValue, + encryptedComment: secretVersion?.encryptedComment, + userId: secretVersion?.userId, + metadata: secretVersion?.metadata + }, + tx + ); + + const newVersion = await secretVersionV2BridgeDAL.create( + { + version: secretVersion.version + 1, + encryptedValue: secretVersion.encryptedValue, + key: secretVersion.key, + encryptedComment: secretVersion.encryptedComment, + skipMultilineEncoding: secretVersion.skipMultilineEncoding, + reminderNote: secretVersion.reminderNote, + reminderRepeatDays: secretVersion.reminderRepeatDays, + userId: secretVersion.userId, + metadata: secretVersion.metadata, + actorType: actorInfo.actorType, + envId: secretVersion.envId, + folderId, + secretId: secretVersion.secretId, + ...(actorInfo.actorType === ActorType.IDENTITY && { identityActorId: actorInfo.actorId }), + ...(actorInfo.actorType === ActorType.USER && { userActorId: actorInfo.actorId }) + }, + tx + ); + + commitChanges.push({ + type: ChangeType.ADD, + secretVersionId: newVersion.id + }); + } + break; + + case "delete": + await secretV2BridgeDAL.deleteById(change.id, tx); + + commitChanges.push({ + type: ChangeType.DELETE, + secretVersionId: change.versionId + }); + break; + + default: + throw new BadRequestError({ message: `Unknown change type: ${change.changeType}` }); } + } - // process folder changes - for (const change of folderChanges) { + return commitChanges; + }; + + /** + * Core function to apply folder state differences + */ + const applyFolderStateDifferencesFn = async ({ + differences, + actorInfo, + folderId, + projectId, + reconstructNewFolders, + reconstructUpToCommit, + step = 0, + tx + }: { + differences: ResourceChange[]; + actorInfo: ActorInfo; + folderId: string; + projectId: string; + reconstructNewFolders: boolean; + reconstructUpToCommit?: string; + step: number; + tx?: Knex; + }): Promise => { + /** + * Process folder changes when applying folder state differences + */ + const processFolderChanges = async ( + changes: ResourceChange[], + folderVersions: Record + ) => { + const commitChanges = []; + + for (const change of changes) { const folderVersion = folderVersions[change.id]; + switch (change.changeType) { case "create": - // Add new folder if (folderVersion) { const newFolder = { id: change.id, @@ -552,45 +689,69 @@ export const folderCommitServiceFactory = ({ tx ); + if (reconstructNewFolders && reconstructUpToCommit && step < 20) { + const subFolderLatestCommit = await folderCommitDAL.findLatestCommitBetween({ + folderId: change.id, + endCommitId: reconstructUpToCommit, + tx + }); + if (subFolderLatestCommit) { + const subFolderDiff = await compareFolderStates({ + targetCommitId: subFolderLatestCommit.id, + tx + }); + if (subFolderDiff?.length > 0) { + await applyFolderStateDifferencesFn({ + differences: subFolderDiff, + actorInfo, + folderId: change.id, + projectId, + reconstructNewFolders, + reconstructUpToCommit, + step: step + 1, + tx + }); + } + } + } + commitChanges.push({ - type: "add", + type: ChangeType.ADD, folderVersionId: newFolderVersion.id }); } break; case "update": - // Update folder to specific version if (change.versionId) { - await folderVersionDAL.findById(change.versionId, tx).then(async (versionDetails) => { - if (versionDetails) { - await folderDAL.updateById( - change.id, - { - parentId: folderId, - envId: versionDetails.envId, - version: (versionDetails.version || 1) + 1, - name: versionDetails.name - }, - tx - ); + const versionDetails = await folderVersionDAL.findById(change.versionId, tx); + if (versionDetails) { + await folderDAL.updateById( + change.id, + { + parentId: folderId, + envId: versionDetails.envId, + version: (versionDetails.version || 1) + 1, + name: versionDetails.name + }, + tx + ); - const newFolderVersion = await folderVersionDAL.create( - { - folderId: change.id, - version: (versionDetails.version || 1) + 1, - name: versionDetails.name, - envId: versionDetails.envId - }, - tx - ); + const newFolderVersion = await folderVersionDAL.create( + { + folderId: change.id, + version: (versionDetails.version || 1) + 1, + name: versionDetails.name, + envId: versionDetails.envId + }, + tx + ); - commitChanges.push({ - type: "add", - folderVersionId: newFolderVersion.id - }); - } - }); + commitChanges.push({ + type: ChangeType.ADD, + folderVersionId: newFolderVersion.id + }); + } } break; @@ -598,78 +759,130 @@ export const folderCommitServiceFactory = ({ await folderDAL.deleteById(change.id, tx); commitChanges.push({ - type: "delete", + type: ChangeType.DELETE, folderVersionId: change.versionId }); break; default: - throw new BadRequestError({ message: `Unknown change type: ${change.changeType as string}` }); + throw new BadRequestError({ message: `Unknown change type: ${change.changeType}` }); } } - await createCommit( - { - actor: { - type: actorInfo.actorType, - metadata: { id: actorInfo.actorId } - }, - message: actorInfo.message || "Rolled back folder state", - folderId, - changes: commitChanges + return commitChanges; + }; + // Group differences by type for more efficient processing + const secretChanges = differences.filter((diff) => diff.type === ResourceType.SECRET); + const folderChanges = differences.filter((diff) => diff.type === ResourceType.FOLDER); + + // Batch fetch necessary data + const secretVersions = await secretVersionV2BridgeDAL.findByIdsWithLatestVersion( + folderId, + secretChanges.map((diff) => diff.id), + secretChanges.map((diff) => diff.versionId) + ); + + const folderVersions = await folderVersionDAL.findByIdsWithLatestVersion( + folderChanges.map((diff) => diff.id), + folderChanges.map((diff) => diff.versionId) + ); + + // Process changes in parallel + const [secretCommitChanges, folderCommitChanges] = await Promise.all([ + processSecretChanges(secretChanges, secretVersions, actorInfo, folderId, tx), + processFolderChanges(folderChanges, folderVersions) + ]); + + // Combine all changes + const allCommitChanges = [...secretCommitChanges, ...folderCommitChanges]; + + // Create a commit with all the changes + await createCommit( + { + actor: { + type: actorInfo.actorType, + metadata: { id: actorInfo.actorId } }, - tx - ); + message: actorInfo.message || "Rolled back folder state", + folderId, + changes: allCommitChanges + }, + tx + ); - result = { - secretChangesCount: secretChanges.length, - folderChangesCount: folderChanges.length, - totalChanges: differences.length - }; - await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId); - }); + // Invalidate cache to reflect the changes + await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId); - return result; + return { + secretChangesCount: secretChanges.length, + folderChangesCount: folderChanges.length, + totalChanges: differences.length + }; }; - // Retrieve a commit by ID + /** + * Apply folder state differences with transaction handling + */ + const applyFolderStateDifferences = async (params: { + differences: ResourceChange[]; + actorInfo: ActorInfo; + folderId: string; + projectId: string; + reconstructNewFolders: boolean; + reconstructUpToCommit?: string; + tx?: Knex; + }): Promise => { + // If a transaction was provided, use it directly + if (params.tx) { + return applyFolderStateDifferencesFn({ ...params, step: 0 }); + } + + // Otherwise, start a new transaction + return folderCommitDAL.transaction((newTx) => applyFolderStateDifferencesFn({ ...params, tx: newTx, step: 0 })); + }; + + /** + * Retrieve a commit by ID + */ const getCommitById = async (id: string, tx?: Knex) => { return folderCommitDAL.findById(id, tx); }; - // Get all commits for a folder + /** + * Get all commits for a folder + */ const getCommitsByFolderId = async (folderId: string, tx?: Knex) => { return folderCommitDAL.findByFolderId(folderId, tx); }; - // Get changes for a commit + /** + * Get changes for a commit + */ const getCommitChanges = async (commitId: string, tx?: Knex) => { return folderCommitChangesDAL.findByCommitId(commitId, tx); }; - // Get checkpoints for a folder + /** + * Get checkpoints for a folder + */ const getCheckpointsByFolderId = async (folderId: string, limit?: number, tx?: Knex) => { return folderCheckpointDAL.findByFolderId(folderId, limit, tx); }; - // Get the latest checkpoint for a folder + /** + * Get the latest checkpoint for a folder + */ const getLatestCheckpoint = async (folderId: string, tx?: Knex) => { return folderCheckpointDAL.findLatestByFolderId(folderId, tx); }; - // Get tree checkpoints for a project - const getTreeCheckpointsByProjectId = async (projectId: string, limit?: number, tx?: Knex) => { - return folderTreeCheckpointDAL.findByProjectId(projectId, limit, tx); - }; - - // Get the latest tree checkpoint for a project - const getLatestTreeCheckpoint = async (projectId: string, tx?: Knex) => { - return folderTreeCheckpointDAL.findLatestByProjectId(projectId, tx); - }; - + /** + * Initialize a folder with its current state + */ const initializeFolder = async (folderId: string, tx?: Knex) => { const folderResources = await getFolderResources(folderId, tx); - const changes = folderResources.map((resource) => ({ type: "add", ...resource })); + const changes = folderResources.map((resource) => ({ type: ChangeType.ADD, ...resource })); + if (changes.length > 0) { const newCommit = await createCommit( { @@ -686,50 +899,225 @@ export const folderCommitServiceFactory = ({ } }; - function sortFoldersByHierarchy(folders: TSecretFolders[]) { + /** + * Sort folders by hierarchy (parents before children) + */ + const sortFoldersByHierarchy = (folders: TSecretFolders[]) => { // Create a map for quick lookup of children by parent ID - const childrenMap: Map = new Map(); + const childrenMap = new Map(); + + // Set of all folder IDs + const allFolderIds = new Set(); + + // Build the set of all folder IDs folders.forEach((folder) => { - const { parentId } = folder; - if (!childrenMap.has(parentId || null)) { - childrenMap.set(parentId || null, []); + if (folder.id) { + allFolderIds.add(folder.id); } - childrenMap.get(parentId || null)?.push(folder); }); - // Start with root folders (null parentId) - const result = []; - const rootFolders = childrenMap.get(null) || []; + // Group folders by their parentId + folders.forEach((folder) => { + if (folder.parentId) { + const children = childrenMap.get(folder.parentId) || []; + children.push(folder); + childrenMap.set(folder.parentId, children); + } + }); + + // Find root folders - those with no parentId or with a parentId that doesn't exist + const rootFolders = folders.filter((folder) => !folder.parentId || !allFolderIds.has(folder.parentId)); // Process each level of the hierarchy + const result = []; let currentLevel = rootFolders; - result.push(...currentLevel); while (currentLevel.length > 0) { - const nextLevel = []; + result.push(...currentLevel); + const nextLevel = []; for (const folder of currentLevel) { - const children = childrenMap.get(folder.id) || []; - nextLevel.push(...children); + if (folder.id) { + const children = childrenMap.get(folder.id) || []; + nextLevel.push(...children); + } } - result.push(...nextLevel); currentLevel = nextLevel; } return result; - } + }; + /** + * Create a checkpoint for a folder tree + */ + const createFolderTreeCheckpoint = async (envId: string, folderCommitId?: string, tx?: Knex) => { + let latestCommitId = folderCommitId; + const latestTreeCheckpoint = await folderTreeCheckpointDAL.findLatestByEnvId(envId, tx); + + if (!latestCommitId) { + const latestCommit = await folderCommitDAL.findLatestEnvCommit(envId, tx); + if (!latestCommit) { + logger.info(`createFolderTreeCheckpoint - Latest commit ID not found for envId ${envId}`); + return; + } + latestCommitId = latestCommit.id; + } + + if (latestTreeCheckpoint) { + const commitsSinceLastCheckpoint = await folderCommitDAL.getEnvNumberOfCommitsSince( + envId, + latestTreeCheckpoint.folderCommitId, + tx + ); + if (commitsSinceLastCheckpoint < Number(appCfg.PIT_TREE_CHECKPOINT_WINDOW)) { + logger.info( + `createFolderTreeCheckpoint - Commits since last checkpoint ${commitsSinceLastCheckpoint} is less than ${appCfg.PIT_TREE_CHECKPOINT_WINDOW}` + ); + return; + } + } + + const folders = await folderDAL.findByEnvId(envId, tx); + const sortedFolders = sortFoldersByHierarchy(folders); + const filteredFoldersIds = sortedFolders.filter((folder) => !folder.isReserved).map((folder) => folder.id); + const folderCommits = await folderCommitDAL.findMultipleLatestCommits(filteredFoldersIds, tx); + const folderTreeCheckpoint = await folderTreeCheckpointDAL.create( + { + folderCommitId: latestCommitId + }, + tx + ); + await folderTreeCheckpointResourcesDAL.insertMany( + folderCommits.map((folderCommit) => ({ + folderTreeCheckpointId: folderTreeCheckpoint.id, + folderId: folderCommit.folderId, + folderCommitId: folderCommit.id + })), + tx + ); + }; + + /** + * Initialize a project with its current state + */ const initializeProject = async (projectId: string, tx?: Knex) => { const project = await projectDAL.findById(projectId, tx); if (!project) { throw new NotFoundError({ message: `Project with ID ${projectId} not found` }); } + const folders = await folderDAL.findByProjectId(projectId, tx); const sortedFolders = sortFoldersByHierarchy(folders); + await Promise.all(sortedFolders.map((folder) => initializeFolder(folder.id, tx))); + + const envIds = [...new Set(folders.map((folder) => folder.envId))]; + await Promise.all( + envIds.map(async (envId) => { + await createFolderTreeCheckpoint(envId, undefined, tx); + }) + ); }; + /** + * Roll back a folder tree to a specific commit + */ + const deepRollbackFolder = async ( + targetCommitId: string, + envId: string, + actorId: string, + actorType: ActorType, + projectId: string, + tx?: Knex + ) => { + const targetCommit = await folderCommitDAL.findById(targetCommitId, tx); + if (!targetCommit) { + throw new NotFoundError({ message: `No commit found for commit ID ${targetCommitId}` }); + } + + const checkpoint = await folderTreeCheckpointDAL.findNearestCheckpoint(targetCommitId, envId, tx); + if (!checkpoint) { + throw new NotFoundError({ message: `No checkpoint found for commit ID ${targetCommitId}` }); + } + + const folderCheckpointCommits = await folderTreeCheckpointResourcesDAL.findByTreeCheckpointId(checkpoint.id, tx); + const folderCommits = await folderCommitDAL.findAllCommitsBetween({ + envId, + endCommitId: targetCommit.commitId.toString(), + startCommitId: checkpoint.commitId.toString(), + tx + }); + + // Group commits by folderId and keep only the latest + const folderGroups = new Map(); + + if (folderCheckpointCommits && folderCheckpointCommits.length > 0) { + for (const commit of folderCheckpointCommits) { + folderGroups.set(commit.folderId, { + createdAt: commit.createdAt, + id: commit.folderCommitId + }); + } + } + + if (folderCommits && folderCommits.length > 0) { + for (const commit of folderCommits) { + const { folderId, createdAt, id } = commit; + const existingCommit = folderGroups.get(folderId); + + if (!existingCommit || createdAt.getTime() > existingCommit.createdAt.getTime()) { + folderGroups.set(folderId, { createdAt, id }); + } + } + } + + const folderDiffs = new Map(); + + // Process each folder to determine differences + await Promise.all( + Array.from(folderGroups.entries()).map(async ([folderId, commit]) => { + const latestFolderCommit = await folderCommitDAL.findLatestCommit(folderId, tx); + if (latestFolderCommit && latestFolderCommit.id !== commit.id) { + const diff = await compareFolderStates({ + currentCommitId: latestFolderCommit.id, + targetCommitId: commit.id, + tx + }); + if (diff?.length > 0) { + folderDiffs.set(folderId, diff); + } + } + }) + ); + + // Apply changes in hierarchical order + const folderIds = Array.from(folderDiffs.keys()); + const folders = await folderDAL.findFoldersByRootAndIds({ rootId: targetCommit.folderId, folderIds }, tx); + const sortedFolders = sortFoldersByHierarchy(folders); + + for (const folder of sortedFolders) { + const diff = folderDiffs.get(folder.id); + if (diff) { + await applyFolderStateDifferences({ + differences: diff, + actorInfo: { + actorType, + actorId, + message: "Deep rollback" + }, + folderId: folder.id, + projectId, + reconstructNewFolders: true, + reconstructUpToCommit: targetCommit.commitId.toString(), + tx + }); + } + } + }; + + // Return the public interface return { createCommit, addCommitChange, @@ -738,13 +1126,13 @@ export const folderCommitServiceFactory = ({ getCommitChanges, getCheckpointsByFolderId, getLatestCheckpoint, - getTreeCheckpointsByProjectId, - getLatestTreeCheckpoint, initializeFolder, initializeProject, createFolderCheckpoint, compareFolderStates, - applyFolderStateDifferences + applyFolderStateDifferences, + createFolderTreeCheckpoint, + deepRollbackFolder }; }; diff --git a/backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts b/backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts index 8813784fd..5d3fac9c2 100644 --- a/backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts +++ b/backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts @@ -1,27 +1,14 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; -import { - TableName, - TFolderTreeCheckpointResources, - TFolderTreeCheckpoints, - TProjectEnvironments, - TSecretFolders -} from "@app/db/schemas"; +import { TableName, TFolderTreeCheckpointResources } from "@app/db/schemas"; import { DatabaseError } from "@app/lib/errors"; -import { ormify, selectAllTableCols } from "@app/lib/knex"; +import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex"; export type TFolderTreeCheckpointResourcesDALFactory = ReturnType; -type ResourceWithCheckpointInfo = TFolderTreeCheckpointResources & { - folderCommitId: string; -}; - -type ResourceWithFolderInfo = TFolderTreeCheckpointResources & { - name: string; - parentId?: string | null; - slug: string; - envName: string; +type TFolderTreeCheckpointResourcesWithCommitId = TFolderTreeCheckpointResources & { + commitId: number; }; export const folderTreeCheckpointResourcesDALFactory = (db: TDbClient) => { @@ -30,95 +17,28 @@ export const folderTreeCheckpointResourcesDALFactory = (db: TDbClient) => { const findByTreeCheckpointId = async ( folderTreeCheckpointId: string, tx?: Knex - ): Promise => { + ): Promise => { try { const docs = await (tx || db.replicaNode())( TableName.FolderTreeCheckpointResources ) - .where({ folderTreeCheckpointId }) - .select(selectAllTableCols(TableName.FolderTreeCheckpointResources)); + .join( + TableName.FolderCommit, + `${TableName.FolderTreeCheckpointResources}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ folderTreeCheckpointId }, TableName.FolderTreeCheckpointResources)) + .select(selectAllTableCols(TableName.FolderTreeCheckpointResources)) + .select(db.ref("commitId").withSchema(TableName.FolderCommit).as("commitId")); return docs; } catch (error) { throw new DatabaseError({ error, name: "FindByTreeCheckpointId" }); } }; - const findByFolderId = async (folderId: string, tx?: Knex): Promise => { - try { - const docs = await (tx || db.replicaNode())< - TFolderTreeCheckpointResources & Pick - >(TableName.FolderTreeCheckpointResources) - .where({ folderId }) - .select(selectAllTableCols(TableName.FolderTreeCheckpointResources)) - .join( - TableName.FolderTreeCheckpoint, - `${TableName.FolderTreeCheckpointResources}.folderTreeCheckpointId`, - `${TableName.FolderTreeCheckpoint}.id` - ) - .select( - db.ref("folderCommitId").withSchema(TableName.FolderTreeCheckpoint), - db.ref("createdAt").withSchema(TableName.FolderTreeCheckpoint) - ); - return docs; - } catch (error) { - throw new DatabaseError({ error, name: "FindByFolderId" }); - } - }; - - const findByFolderCommitId = async (folderCommitId: string, tx?: Knex): Promise => { - try { - const docs = await (tx || db.replicaNode())< - TFolderTreeCheckpointResources & Pick - >(TableName.FolderTreeCheckpointResources) - .where({ folderCommitId }) - .select(selectAllTableCols(TableName.FolderTreeCheckpointResources)) - .join( - TableName.FolderTreeCheckpoint, - `${TableName.FolderTreeCheckpointResources}.folderTreeCheckpointId`, - `${TableName.FolderTreeCheckpoint}.id` - ) - .select(db.ref("createdAt").withSchema(TableName.FolderTreeCheckpoint)); - return docs; - } catch (error) { - throw new DatabaseError({ error, name: "FindByFolderCommitId" }); - } - }; - - const findFoldersInTreeCheckpoint = async ( - folderTreeCheckpointId: string, - tx?: Knex - ): Promise => { - try { - const docs = await (tx || db.replicaNode())< - TFolderTreeCheckpointResources & - Pick & - Pick & { envName: string } - >(TableName.FolderTreeCheckpointResources) - .where({ folderTreeCheckpointId }) - .select(selectAllTableCols(TableName.FolderTreeCheckpointResources)) - .join( - TableName.SecretFolder, - `${TableName.FolderTreeCheckpointResources}.folderId`, - `${TableName.SecretFolder}.id` - ) - .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) - .select( - db.ref("name").withSchema(TableName.SecretFolder), - db.ref("parentId").withSchema(TableName.SecretFolder), - db.ref("slug").withSchema(TableName.Environment), - db.ref("name").withSchema(TableName.Environment).as("envName") - ); - return docs; - } catch (error) { - throw new DatabaseError({ error, name: "FindFoldersInTreeCheckpoint" }); - } - }; - return { ...folderTreeCheckpointResourcesOrm, - findByTreeCheckpointId, - findByFolderId, - findByFolderCommitId, - findFoldersInTreeCheckpoint + findByTreeCheckpointId }; }; diff --git a/backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts b/backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts index 4e9381f9c..7589c4b49 100644 --- a/backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts +++ b/backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts @@ -8,11 +8,7 @@ import { ormify, selectAllTableCols } from "@app/lib/knex"; export type TFolderTreeCheckpointDALFactory = ReturnType; type TreeCheckpointWithCommitInfo = TFolderTreeCheckpoints & { - actorMetadata: unknown; - actorType: string; - message?: string | null; - commitDate: Date; - folderId: string; + commitId: number; }; export const folderTreeCheckpointDALFactory = (db: TDbClient) => { @@ -30,82 +26,61 @@ export const folderTreeCheckpointDALFactory = (db: TDbClient) => { } }; - const findByProjectId = async ( - projectId: string, - limit?: number, - tx?: Knex - ): Promise => { - try { - const query = (tx || db.replicaNode())< - TFolderTreeCheckpoints & - Pick & { commitDate: Date } - >(TableName.FolderTreeCheckpoint) - .join( - TableName.FolderCommit, - `${TableName.FolderTreeCheckpoint}.folderCommitId`, - `${TableName.FolderCommit}.id` - ) - .join(TableName.SecretFolder, `${TableName.FolderCommit}.folderId`, `${TableName.SecretFolder}.id`) - .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) - .where({ projectId }) - .select(selectAllTableCols(TableName.FolderTreeCheckpoint)) - .select( - db.ref("actorMetadata").withSchema(TableName.FolderCommit), - db.ref("actorType").withSchema(TableName.FolderCommit), - db.ref("message").withSchema(TableName.FolderCommit), - db.ref("createdAt").withSchema(TableName.FolderCommit).as("commitDate"), - db.ref("folderId").withSchema(TableName.FolderCommit) - ) - .orderBy(`${TableName.FolderTreeCheckpoint}.createdAt`, "desc"); - - if (limit) { - void query.limit(limit); - } - - const docs = await query; - return docs; - } catch (error) { - throw new DatabaseError({ error, name: "FindByProjectId" }); - } - }; - - const findLatestByProjectId = async ( - projectId: string, + const findNearestCheckpoint = async ( + folderCommitId: string, + envId: string, tx?: Knex ): Promise => { try { - const doc = await (tx || db.replicaNode())< - TFolderTreeCheckpoints & - Pick & { commitDate: Date } - >(TableName.FolderTreeCheckpoint) - .join( + const targetCommit = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ id: folderCommitId }) + .select("id", "commitId", "folderId") + .first(); + + if (!targetCommit) { + return undefined; + } + + const nearestCheckpoint = await (tx || db.replicaNode())(TableName.FolderTreeCheckpoint) + .join( TableName.FolderCommit, `${TableName.FolderTreeCheckpoint}.folderCommitId`, `${TableName.FolderCommit}.id` ) - .join(TableName.SecretFolder, `${TableName.FolderCommit}.folderId`, `${TableName.SecretFolder}.id`) - .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) - .where({ projectId }) + .where(`${TableName.FolderCommit}.commitId`, "<=", targetCommit.commitId.toString()) + .where(`${TableName.FolderCommit}.envId`, envId) .select(selectAllTableCols(TableName.FolderTreeCheckpoint)) - .select( - db.ref("actorMetadata").withSchema(TableName.FolderCommit), - db.ref("actorType").withSchema(TableName.FolderCommit), - db.ref("message").withSchema(TableName.FolderCommit), - db.ref("createdAt").withSchema(TableName.FolderCommit).as("commitDate"), - db.ref("folderId").withSchema(TableName.FolderCommit) + .select(db.ref("commitId").withSchema(TableName.FolderCommit)) + .orderBy(`${TableName.FolderCommit}.commitId`, "desc") + .first(); + + return nearestCheckpoint; + } catch (error) { + throw new DatabaseError({ error, name: "FindNearestCheckpoint" }); + } + }; + + const findLatestByEnvId = async (envId: string, tx?: Knex): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderTreeCheckpoint) + .join( + TableName.FolderCommit, + `${TableName.FolderTreeCheckpoint}.folderCommitId`, + `${TableName.FolderCommit}.id` ) + .where(`${TableName.FolderCommit}.envId`, envId) .orderBy(`${TableName.FolderTreeCheckpoint}.createdAt`, "desc") .first(); return doc; } catch (error) { - throw new DatabaseError({ error, name: "FindLatestByProjectId" }); + throw new DatabaseError({ error, name: "FindLatestByFolderId" }); } }; return { ...folderTreeCheckpointOrm, findByCommitId, - findByProjectId, - findLatestByProjectId + findNearestCheckpoint, + findLatestByEnvId }; }; diff --git a/backend/src/services/secret-folder/secret-folder-dal.ts b/backend/src/services/secret-folder/secret-folder-dal.ts index d6d5c4aad..5e5809ba5 100644 --- a/backend/src/services/secret-folder/secret-folder-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-dal.ts @@ -488,6 +488,51 @@ export const secretFolderDALFactory = (db: TDbClient) => { } }; + const findFoldersByRootAndIds = async ({ rootId, folderIds }: { rootId: string; folderIds: string[] }, tx?: Knex) => { + try { + // First, get all descendant folders of rootId + const descendants = await (tx || db.replicaNode()) + .withRecursive("descendants", (qb) => + qb + .select( + selectAllTableCols(TableName.SecretFolder), + db.raw("0 as depth"), + db.raw(`'/' as path`), + db.ref(`${TableName.Environment}.slug`).as("environment") + ) + .from(TableName.SecretFolder) + .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) + .where(`${TableName.SecretFolder}.id`, rootId) + .union((un) => { + void un + .select( + selectAllTableCols(TableName.SecretFolder), + db.raw("descendants.depth + 1 as depth"), + db.raw( + `CONCAT( + CASE WHEN descendants.path = '/' THEN '' ELSE descendants.path END, + CASE WHEN ${TableName.SecretFolder}."parentId" is NULL THEN '' ELSE CONCAT('/', secret_folders.name) END + )` + ), + db.ref("descendants.environment") + ) + .from(TableName.SecretFolder) + .where(`${TableName.SecretFolder}.isReserved`, false) + .join("descendants", `${TableName.SecretFolder}.parentId`, "descendants.id"); + }) + ) + .select<(TSecretFolders & { path: string; depth: number; environment: string })[]>("*") + .from("descendants") + .whereIn(`id`, folderIds) + .orderBy("depth") + .orderBy(`name`); + + return descendants; + } catch (error) { + throw new DatabaseError({ error, name: "FindFoldersByRootAndIds" }); + } + }; + const findByParentId = async (parentId: string, tx?: Knex) => { try { const folders = await (tx || db.replicaNode())(TableName.SecretFolder) @@ -499,6 +544,17 @@ export const secretFolderDALFactory = (db: TDbClient) => { } }; + const findByEnvId = async (envId: string, tx?: Knex) => { + try { + const folders = await (tx || db.replicaNode())(TableName.SecretFolder) + .where({ envId }) + .select(selectAllTableCols(TableName.SecretFolder)); + return folders; + } catch (error) { + throw new DatabaseError({ error, name: "findByEnvId" }); + } + }; + return { ...secretFolderOrm, update, @@ -511,6 +567,8 @@ export const secretFolderDALFactory = (db: TDbClient) => { findByProjectId, findByMultiEnv, findByEnvsDeep, - findByParentId + findByParentId, + findByEnvId, + findFoldersByRootAndIds }; }; From 6319f538023a6aad9f7d9d38822996331d577702 Mon Sep 17 00:00:00 2001 From: carlosmonastyrski Date: Tue, 20 May 2025 08:22:14 -0300 Subject: [PATCH 8/9] PIT: UI views --- .../20250505194916_add-pit-revamp-tables.ts | 1 + .../src/db/schemas/folder-commit-changes.ts | 1 + backend/src/ee/routes/v1/pit-router.ts | 564 ++++++++++++++-- .../ee/services/audit-log/audit-log-types.ts | 84 ++- .../services/permission/project-permission.ts | 26 +- .../secret-rotation-queue.ts | 1 + backend/src/server/routes/index.ts | 3 +- .../folder-checkpoint-resources-dal.ts | 16 +- .../folder-commit-changes-dal.ts | 64 +- .../folder-commit/folder-commit-dal.ts | 77 ++- .../folder-commit-service.test.ts | 45 +- .../folder-commit/folder-commit-service.ts | 602 +++++++++++++++--- .../secret-folder/secret-folder-service.ts | 23 +- .../secret-v2-bridge/secret-v2-bridge-fns.ts | 7 +- .../secret-v2-bridge-service.ts | 82 ++- .../secret-v2-bridge/secret-version-dal.ts | 22 +- backend/src/services/secret/secret-service.ts | 33 +- .../SecretDashboardPathBreadcrumb.tsx | 6 +- frontend/src/const/routes.ts | 12 + .../context/ProjectPermissionContext/types.ts | 11 +- .../src/hooks/api/folderCommits/index.tsx | 2 + .../src/hooks/api/folderCommits/queries.tsx | 247 +++++++ frontend/src/hooks/api/folderCommits/types.ts | 63 ++ .../src/hooks/api/secretFolders/queries.tsx | 30 + frontend/src/hooks/api/secrets/mutations.tsx | 51 ++ .../ProjectRoleModifySection.utils.tsx | 19 +- .../CommitDetailsPage/CommitDetailsPage.tsx | 81 +++ .../CommitDetailsTab/CommitDetailsTab.tsx | 327 ++++++++++ .../components/CommitDetailsTab/index.ts | 1 + .../components/CommitDetailsTab/types.ts | 14 + .../RollbackPreviewTab/RollbackPreviewTab.tsx | 376 +++++++++++ .../components/RollbackPreviewTab/index.ts | 1 + .../components/RollbackPreviewTab/route.tsx | 91 +++ .../SecretVersionDiffView.tsx | 585 +++++++++++++++++ .../components/SecretVersionDiffView/index.ts | 1 + .../CommitDetailsPage/index.tsx | 1 + .../CommitDetailsPage/route.tsx | 76 +++ .../CommitsPage/CommitsPage.tsx | 65 ++ .../CommitHistoryTab/CommitHistoryTab.tsx | 281 ++++++++ .../components/CommitHistoryTab/index.tsx | 1 + .../secret-manager/CommitsPage/index.tsx | 1 + .../secret-manager/CommitsPage/route.tsx | 71 +++ .../SecretDashboardPage.tsx | 54 +- .../components/ActionBar/ActionBar.tsx | 4 +- .../SecretListView/SecretListView.tsx | 13 + frontend/src/routeTree.gen.ts | 184 +++++- frontend/src/routes.ts | 10 + 47 files changed, 4134 insertions(+), 196 deletions(-) create mode 100644 frontend/src/hooks/api/folderCommits/index.tsx create mode 100644 frontend/src/hooks/api/folderCommits/queries.tsx create mode 100644 frontend/src/hooks/api/folderCommits/types.ts create mode 100644 frontend/src/pages/secret-manager/CommitDetailsPage/CommitDetailsPage.tsx create mode 100644 frontend/src/pages/secret-manager/CommitDetailsPage/components/CommitDetailsTab/CommitDetailsTab.tsx create mode 100644 frontend/src/pages/secret-manager/CommitDetailsPage/components/CommitDetailsTab/index.ts create mode 100644 frontend/src/pages/secret-manager/CommitDetailsPage/components/CommitDetailsTab/types.ts create mode 100644 frontend/src/pages/secret-manager/CommitDetailsPage/components/RollbackPreviewTab/RollbackPreviewTab.tsx create mode 100644 frontend/src/pages/secret-manager/CommitDetailsPage/components/RollbackPreviewTab/index.ts create mode 100644 frontend/src/pages/secret-manager/CommitDetailsPage/components/RollbackPreviewTab/route.tsx create mode 100644 frontend/src/pages/secret-manager/CommitDetailsPage/components/SecretVersionDiffView/SecretVersionDiffView.tsx create mode 100644 frontend/src/pages/secret-manager/CommitDetailsPage/components/SecretVersionDiffView/index.ts create mode 100644 frontend/src/pages/secret-manager/CommitDetailsPage/index.tsx create mode 100644 frontend/src/pages/secret-manager/CommitDetailsPage/route.tsx create mode 100644 frontend/src/pages/secret-manager/CommitsPage/CommitsPage.tsx create mode 100644 frontend/src/pages/secret-manager/CommitsPage/components/CommitHistoryTab/CommitHistoryTab.tsx create mode 100644 frontend/src/pages/secret-manager/CommitsPage/components/CommitHistoryTab/index.tsx create mode 100644 frontend/src/pages/secret-manager/CommitsPage/index.tsx create mode 100644 frontend/src/pages/secret-manager/CommitsPage/route.tsx diff --git a/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts b/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts index a2a8ac8c2..75f1ca89f 100644 --- a/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts +++ b/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts @@ -29,6 +29,7 @@ export async function up(knex: Knex): Promise { t.uuid("folderCommitId").notNullable(); t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE"); t.string("changeType").notNullable(); + t.boolean("isUpdate").notNullable().defaultTo(false); t.uuid("secretVersionId"); t.foreign("secretVersionId").references("id").inTable(TableName.SecretVersionV2).onDelete("CASCADE"); t.uuid("folderVersionId"); diff --git a/backend/src/db/schemas/folder-commit-changes.ts b/backend/src/db/schemas/folder-commit-changes.ts index 063874f43..2bee0c5b3 100644 --- a/backend/src/db/schemas/folder-commit-changes.ts +++ b/backend/src/db/schemas/folder-commit-changes.ts @@ -11,6 +11,7 @@ export const FolderCommitChangesSchema = z.object({ id: z.string().uuid(), folderCommitId: z.string().uuid(), changeType: z.string(), + isUpdate: z.boolean().default(false), secretVersionId: z.string().uuid().nullable().optional(), folderVersionId: z.string().uuid().nullable().optional(), createdAt: z.date(), diff --git a/backend/src/ee/routes/v1/pit-router.ts b/backend/src/ee/routes/v1/pit-router.ts index a3ec8dcc9..274df1b11 100644 --- a/backend/src/ee/routes/v1/pit-router.ts +++ b/backend/src/ee/routes/v1/pit-router.ts @@ -1,144 +1,612 @@ +/* eslint-disable @typescript-eslint/no-base-to-string */ import { z } from "zod"; +import { EventType } from "@app/ee/services/audit-log/audit-log-types"; +import { NotFoundError } from "@app/lib/errors"; +import { removeTrailingSlash } from "@app/lib/fn"; import { readLimit } from "@app/server/config/rateLimiter"; +import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; +import { booleanSchema } from "@app/server/routes/sanitizedSchemas"; +import { AuthMode } from "@app/services/auth/auth-type"; +import { ChangeType } from "@app/services/folder-commit/folder-commit-service"; + +const commitHistoryItemSchema = z.object({ + id: z.string(), + folderId: z.string(), + actorType: z.string(), + actorMetadata: z.unknown().optional(), + message: z.string().optional().nullable(), + commitId: z.string(), + createdAt: z.string().or(z.date()), + envId: z.string() +}); + +const versionSchema = z.object({ + secretKey: z.string().optional(), + secretComment: z.string().optional().nullable(), + skipMultilineEncoding: z.boolean().optional().nullable(), + secretReminderRepeatDays: z.number().optional().nullable(), + secretReminderNote: z.string().optional().nullable(), + metadata: z.unknown().optional().nullable(), + tags: z.array(z.string()).optional().nullable(), + secretReminderRecipients: z.array(z.any()).optional().nullable(), + secretValue: z.string().optional().nullable(), + name: z.string().optional().nullable() +}); + +const folderStateSchema = z.array( + z.object({ + type: z.string(), + id: z.string(), + versionId: z.string(), + secretKey: z.string().optional(), + secretVersion: z.number().optional(), + folderName: z.string().optional(), + folderVersion: z.number().optional() + }) +); export const registerPITRouter = async (server: FastifyZodProvider) => { - // Get all commits for a folder + // Get commits count for a folder server.route({ method: "GET", - url: "/commits/:folderId", + url: "/commits/count/:workspaceId", config: { rateLimit: readLimit }, schema: { params: z.object({ - folderId: z.string().trim() + workspaceId: z.string().trim() + }), + querystring: z.object({ + environment: z.string().trim(), + path: z.string().trim().default("/").transform(removeTrailingSlash) }), response: { - 200: z.any() + 200: z.object({ + count: z.number(), + folderId: z.string() + }) } }, + onRequest: verifyAuth([AuthMode.JWT]), handler: async (req) => { - const commits = await server.services.folderCommit.getCommitsByFolderId(req.params.folderId); - return commits; + const res = await server.services.folderCommit.getCommitsCount({ + actor: req.permission?.type, + actorId: req.permission?.id, + actorOrgId: req.permission?.orgId, + actorAuthMethod: req.permission?.authMethod, + projectId: req.params.workspaceId, + environment: req.query.environment, + path: req.query.path + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.params.workspaceId, + event: { + type: EventType.GET_PROJECT_PIT_COMMIT_COUNT, + metadata: { + environment: req.query.environment, + path: req.query.path, + commitCount: res.count.toString() + } + } + }); + + return res; + } + }); + + // Get all commits for a folder + server.route({ + method: "GET", + url: "/commits/:workspaceId", + config: { + rateLimit: readLimit + }, + schema: { + params: z.object({ + workspaceId: z.string().trim() + }), + querystring: z.object({ + environment: z.string().trim(), + path: z.string().trim().default("/").transform(removeTrailingSlash) + }), + response: { + 200: commitHistoryItemSchema.array() + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const commits = await server.services.folderCommit.getCommitsForFolder({ + actor: req.permission?.type, + actorId: req.permission?.id, + actorOrgId: req.permission?.orgId, + actorAuthMethod: req.permission?.authMethod, + projectId: req.params.workspaceId, + environment: req.query.environment, + path: req.query.path + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.params.workspaceId, + event: { + type: EventType.GET_PROJECT_PIT_COMMITS, + metadata: { + environment: req.query.environment, + path: req.query.path, + commitCount: commits.length.toString() + } + } + }); + + return commits.map((commit) => ({ + ...commit, + commitId: commit.commitId.toString() + })); } }); // Get commit changes for a specific commit server.route({ method: "GET", - url: "/commits/:commitId/changes", + url: "/commits/:workspaceId/:commitId/changes", config: { rateLimit: readLimit }, schema: { params: z.object({ + workspaceId: z.string().trim(), commitId: z.string().trim() }), response: { - 200: z.any() + 200: z.object({ + changes: z.object({ + id: z.string(), + commitId: z.string(), + actorMetadata: z + .union([ + z.object({ + id: z.string().optional(), + name: z.string().optional() + }), + z.unknown() + ]) + .optional(), + actorType: z.string(), + message: z.string().optional().nullable(), + folderId: z.string(), + envId: z.string(), + createdAt: z.string().or(z.date()), + updatedAt: z.string().or(z.date()), + changes: z.array( + z.object({ + id: z.string(), + folderCommitId: z.string(), + changeType: z.string(), + isUpdate: z.boolean().optional(), + secretVersionId: z.string().optional().nullable(), + folderVersionId: z.string().optional().nullable(), + // Fix these two fields to accept either string or Date objects + createdAt: z.union([z.string(), z.date()]), + updatedAt: z.union([z.string(), z.date()]), + folderName: z.string().optional().nullable(), + folderChangeId: z.string().optional().nullable(), + folderVersion: z.union([z.string(), z.number()]).optional().nullable(), + secretKey: z.string().optional().nullable(), + secretVersion: z.union([z.string(), z.number()]).optional().nullable(), + secretId: z.string().optional().nullable(), + actorMetadata: z + .union([ + z.object({ + id: z.string().optional(), + name: z.string().optional() + }), + z.unknown() + ]) + .optional(), + actorType: z.string().optional(), + message: z.string().optional().nullable(), + folderId: z.string().optional().nullable(), + versions: z.array(versionSchema).optional() + }) + ) + }) + }) } }, + onRequest: verifyAuth([AuthMode.JWT]), handler: async (req) => { - const changes = await server.services.folderCommit.getCommitChanges(req.params.commitId); - return changes; + const changes = await server.services.folderCommit.getCommitChanges({ + actor: req.permission?.type, + actorId: req.permission?.id, + actorOrgId: req.permission?.orgId, + actorAuthMethod: req.permission?.authMethod, + projectId: req.params.workspaceId, + commitId: req.params.commitId + }); + for (const change of changes.changes) { + if (change.secretVersionId) { + const currentVersion = change.secretVersion || "1"; + const previousVersion = (Number.parseInt(currentVersion, 10) - 1).toString(); + if (change.secretId) { + // eslint-disable-next-line no-await-in-loop + const versions = await server.services.secret.getSecretVersionsV2ByIds({ + actorId: req.permission?.id, + actor: req.permission?.type, + actorOrgId: req.permission?.orgId, + actorAuthMethod: req.permission?.authMethod, + secretId: change.secretId, + secretVersions: change.isUpdate ? [currentVersion, previousVersion] : [currentVersion], + folderId: change.folderId + }); + change.versions = versions?.map((v) => ({ + secretKey: v.secretKey, + secretComment: v.secretComment, + skipMultilineEncoding: v.skipMultilineEncoding, + secretReminderRepeatDays: v.secretReminderRepeatDays, + secretReminderNote: v.secretReminderNote, + metadata: v.secretMetadata, + tags: v.tags?.map((t) => t.name), + secretReminderRecipients: v.secretReminderRecipients?.map((r) => r.toString()), + secretValue: v.secretValue + })); + } + } else if (change.folderVersionId && change.folderChangeId) { + const currentVersion = change.folderVersion || "1"; + const previousVersion = (Number.parseInt(currentVersion, 10) - 1).toString(); + // eslint-disable-next-line no-await-in-loop + const versions = await server.services.folder.getFolderVersionsByIds({ + folderId: change.folderChangeId, + folderVersions: change.isUpdate ? [currentVersion, previousVersion] : [currentVersion] + }); + change.versions = versions.map((v) => ({ + name: v.name + })); + } + } + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.params.workspaceId, + event: { + type: EventType.GET_PROJECT_PIT_COMMIT_CHANGES, + metadata: { + commitId: req.params.commitId, + changesCount: (changes.changes?.length || 0).toString() + } + } + }); + + return { + changes: { + ...changes, + commitId: changes.commitId.toString() + } + }; } }); - // Compare folder states between commits + // Retrieve rollback changes for a commit server.route({ method: "GET", - url: "/compare", + url: "/commits/:workspaceId/:commitId/compare", config: { rateLimit: readLimit }, schema: { + params: z.object({ + workspaceId: z.string().trim(), + commitId: z.string().trim() + }), querystring: z.object({ - fromCommit: z.string().trim(), - toCommit: z.string().trim() + folderId: z.string().trim(), + envId: z.string().trim(), + deepRollback: booleanSchema.default(false), + secretPath: z.string().trim().default("/").transform(removeTrailingSlash) }), response: { - 200: z.any() + 200: z.array( + z.object({ + folderId: z.string(), + folderName: z.string(), + folderPath: z.string().optional(), + changes: z.any() + }) + ) } }, + onRequest: verifyAuth([AuthMode.JWT]), handler: async (req) => { - const diff = await server.services.folderCommit.compareFolderStates({ - currentCommitId: req.query.fromCommit, - targetCommitId: req.query.toCommit + const latestCommit = await server.services.folderCommit.getLatestCommit({ + folderId: req.query.folderId, + actor: req.permission?.type, + actorId: req.permission?.id, + actorOrgId: req.permission?.orgId, + actorAuthMethod: req.permission?.authMethod, + projectId: req.params.workspaceId + }); + if (!latestCommit) { + throw new NotFoundError({ message: "Latest commit not found" }); + } + + let diffs; + if (req.query.deepRollback) { + diffs = await server.services.folderCommit.deepCompareFolder({ + targetCommitId: req.params.commitId, + envId: req.query.envId, + actorId: req.permission?.id, + actorType: req.permission?.type, + projectId: req.params.workspaceId + }); + } else { + const folder = await server.services.folder.getFolderById({ + actor: req.permission?.type, + actorId: req.permission?.id, + actorOrgId: req.permission?.orgId, + actorAuthMethod: req.permission?.authMethod, + id: req.query.folderId + }); + diffs = [ + { + folderId: folder.id, + folderName: folder.name, + folderPath: req.query.secretPath, + changes: await server.services.folderCommit.compareFolderStates({ + targetCommitId: req.params.commitId, + currentCommitId: latestCommit.id + }) + } + ]; + } + + for (const diff of diffs) { + for (const change of diff.changes) { + if (change.secretKey) { + const currentVersion = change.secretVersion || "1"; + const previousVersion = change.fromVersion || "1"; + // eslint-disable-next-line no-await-in-loop + const versions = await server.services.secret.getSecretVersionsV2ByIds({ + actorId: req.permission?.id, + actor: req.permission?.type, + actorOrgId: req.permission?.orgId, + actorAuthMethod: req.permission?.authMethod, + secretId: change.id, + // if it's update add also the previous secretversionid + secretVersions: + change.changeType === ChangeType.UPDATE ? [currentVersion, previousVersion] : [currentVersion], + folderId: req.query.folderId + }); + change.versions = versions?.map((v) => ({ + secretKey: v.secretKey, + secretComment: v.secretComment, + skipMultilineEncoding: v.skipMultilineEncoding, + secretReminderRepeatDays: v.secretReminderRepeatDays, + secretReminderNote: v.secretReminderNote, + metadata: v.metadata, + tags: v.tags?.map((t) => t.name), + secretReminderRecipients: v.secretReminderRecipients?.map((r) => r.toString()), + secretValue: v.secretValue + })); + } + if (change.folderVersion) { + const currentVersion = change.folderVersion || "1"; + const previousVersion = change.fromVersion || "1"; + // eslint-disable-next-line no-await-in-loop + const versions = await server.services.folder.getFolderVersionsByIds({ + folderId: change.id, + folderVersions: + change.changeType === ChangeType.UPDATE ? [currentVersion, previousVersion] : [currentVersion] + }); + change.versions = versions.map((v) => ({ + name: v.name + })); + } + } + } + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.params.workspaceId, + event: { + type: EventType.PIT_COMPARE_FOLDER_STATES, + metadata: { + targetCommitId: req.params.commitId, + folderId: req.query.folderId, + deepRollback: req.query.deepRollback, + diffsCount: diffs.length.toString() + } + } }); - return diff; + return diffs; } }); // Rollback to a previous commit server.route({ method: "POST", - url: "/rollback", + url: "/commits/:workspaceId/:commitId/rollback", config: { rateLimit: readLimit }, schema: { + params: z.object({ + workspaceId: z.string().trim(), + commitId: z.string().trim() + }), body: z.object({ - fromCommit: z.string().trim(), - toCommit: z.string().trim(), folderId: z.string().trim(), - projectId: z.string().trim(), - reconstructNewFolders: z.boolean().default(false) + deepRollback: z.boolean().default(false), + message: z.string().trim().optional() }), response: { - 200: z.any() + 200: z.object({ + success: z.boolean(), + secretChangesCount: z.number().optional(), + folderChangesCount: z.number().optional(), + totalChanges: z.number().optional() + }) } }, + onRequest: verifyAuth([AuthMode.JWT]), handler: async (req) => { + const latestCommit = await server.services.folderCommit.getLatestCommit({ + folderId: req.body.folderId, + actor: req.permission?.type, + actorId: req.permission?.id, + actorOrgId: req.permission?.orgId, + actorAuthMethod: req.permission?.authMethod, + projectId: req.params.workspaceId + }); + if (!latestCommit) { + throw new NotFoundError({ message: "Latest commit not found" }); + } + + if (req.body.deepRollback) { + await server.services.folderCommit.deepRollbackFolder( + latestCommit.id, + req.body.folderId, + req.permission.id, + req.permission.type, + req.params.workspaceId + ); + return { success: true }; + } const diff = await server.services.folderCommit.compareFolderStates({ - currentCommitId: req.body.fromCommit, - targetCommitId: req.body.toCommit + currentCommitId: latestCommit.id, + targetCommitId: req.params.commitId }); const response = await server.services.folderCommit.applyFolderStateDifferences({ differences: diff, actorInfo: { - actorType: req.permission?.type || "PLATFORM", - actorId: req.permission?.id, - message: "Rollback to previous commit" + actorType: req.permission.type, + actorId: req.permission.id, + message: req.body.message || "Rollback to previous commit" }, folderId: req.body.folderId, - projectId: req.body.projectId, - reconstructNewFolders: req.body.reconstructNewFolders + projectId: req.params.workspaceId, + reconstructNewFolders: req.body.deepRollback + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.params.workspaceId, + event: { + type: EventType.PIT_ROLLBACK_COMMIT, + metadata: { + targetCommitId: req.params.commitId, + folderId: req.body.folderId, + deepRollback: req.body.deepRollback, + message: req.body.message || "Rollback to previous commit", + totalChanges: response.totalChanges.toString() + } + } + }); + + return { + success: true, + secretChangesCount: response.secretChangesCount, + folderChangesCount: response.folderChangesCount, + totalChanges: response.totalChanges + }; + } + }); + + // Revert commit + server.route({ + method: "POST", + url: "/commits/:workspaceId/:commitId/revert", + config: { + rateLimit: readLimit + }, + schema: { + params: z.object({ + workspaceId: z.string().trim(), + commitId: z.string().trim() + }), + response: { + 200: z.object({ + success: z.boolean(), + message: z.string(), + originalCommitId: z.string(), + revertCommitId: z.string().optional(), + changesReverted: z.number().optional() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const response = await server.services.folderCommit.revertCommitChanges({ + commitId: req.params.commitId, + actor: req.permission?.type, + actorId: req.permission?.id, + actorAuthMethod: req.permission?.authMethod, + actorOrgId: req.permission?.orgId, + projectId: req.params.workspaceId + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.params.workspaceId, + event: { + type: EventType.PIT_REVERT_COMMIT, + metadata: { + commitId: req.params.commitId, + revertCommitId: response.revertCommitId, + changesReverted: response.changesReverted?.toString() + } + } }); return response; } }); - // Deep rollback to a specific commit + // Folder state at commit server.route({ - method: "POST", - url: "/deep-rollback", + method: "GET", + url: "/commits/:workspaceId/:commitId", config: { rateLimit: readLimit }, schema: { - body: z.object({ - commitId: z.string().trim(), - envId: z.string().trim(), - projectId: z.string().trim() + params: z.object({ + workspaceId: z.string().trim(), + commitId: z.string().trim() + }), + querystring: z.object({ + folderId: z.string().trim() }), response: { - 200: z.any() + 200: folderStateSchema } }, + onRequest: verifyAuth([AuthMode.JWT]), handler: async (req) => { - await server.services.folderCommit.deepRollbackFolder( - req.body.commitId, - req.body.envId, - req.permission?.id || "PLATFORM", - req.permission?.type || "PLATFORM", - req.body.projectId - ); + const response = await server.services.folderCommit.reconstructFolderState(req.params.commitId); - return { success: true }; + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.params.workspaceId, + event: { + type: EventType.PIT_GET_FOLDER_STATE, + metadata: { + commitId: req.params.commitId, + folderId: req.query.folderId, + resourceCount: response.length.toString() + } + } + }); + + return response.map((item) => ({ + ...item, + secretVersion: item.secretVersion ? Number(item.secretVersion) : undefined, + folderVersion: item.folderVersion ? Number(item.folderVersion) : undefined + })); } }); }; diff --git a/backend/src/ee/services/audit-log/audit-log-types.ts b/backend/src/ee/services/audit-log/audit-log-types.ts index 1f4badfb5..6bd4d2607 100644 --- a/backend/src/ee/services/audit-log/audit-log-types.ts +++ b/backend/src/ee/services/audit-log/audit-log-types.ts @@ -343,7 +343,15 @@ export enum EventType { MICROSOFT_TEAMS_WORKFLOW_INTEGRATION_LIST = "microsoft-teams-workflow-integration-list", PROJECT_ASSUME_PRIVILEGE_SESSION_START = "project-assume-privileges-session-start", - PROJECT_ASSUME_PRIVILEGE_SESSION_END = "project-assume-privileges-session-end" + PROJECT_ASSUME_PRIVILEGE_SESSION_END = "project-assume-privileges-session-end", + + GET_PROJECT_PIT_COMMITS = "get-project-pit-commits", + GET_PROJECT_PIT_COMMIT_CHANGES = "get-project-pit-commit-changes", + GET_PROJECT_PIT_COMMIT_COUNT = "get-project-pit-commit-count", + PIT_ROLLBACK_COMMIT = "pit-rollback-commit", + PIT_REVERT_COMMIT = "pit-revert-commit", + PIT_GET_FOLDER_STATE = "pit-get-folder-state", + PIT_COMPARE_FOLDER_STATES = "pit-compare-folder-states" } export const filterableSecretEvents: EventType[] = [ @@ -2694,6 +2702,71 @@ interface MicrosoftTeamsWorkflowIntegrationUpdateEvent { }; } +interface GetProjectPitCommitsEvent { + type: EventType.GET_PROJECT_PIT_COMMITS; + metadata: { + commitCount: string; + environment: string; + path: string; + }; +} + +interface GetProjectPitCommitChangesEvent { + type: EventType.GET_PROJECT_PIT_COMMIT_CHANGES; + metadata: { + changesCount: string; + commitId: string; + }; +} + +interface GetProjectPitCommitCountEvent { + type: EventType.GET_PROJECT_PIT_COMMIT_COUNT; + metadata: { + environment: string; + path: string; + commitCount: string; + }; +} + +interface PitRollbackCommitEvent { + type: EventType.PIT_ROLLBACK_COMMIT; + metadata: { + targetCommitId: string; + folderId: string; + deepRollback: boolean; + message: string; + totalChanges: string; + }; +} + +interface PitRevertCommitEvent { + type: EventType.PIT_REVERT_COMMIT; + metadata: { + commitId: string; + revertCommitId?: string; + changesReverted?: string; + }; +} + +interface PitGetFolderStateEvent { + type: EventType.PIT_GET_FOLDER_STATE; + metadata: { + commitId: string; + folderId: string; + resourceCount: string; + }; +} + +interface PitCompareFolderStatesEvent { + type: EventType.PIT_COMPARE_FOLDER_STATES; + metadata: { + targetCommitId: string; + folderId: string; + deepRollback: boolean; + diffsCount: string; + }; +} + export type Event = | GetSecretsEvent | GetSecretEvent @@ -2941,4 +3014,11 @@ export type Event = | MicrosoftTeamsWorkflowIntegrationGetTeamsEvent | MicrosoftTeamsWorkflowIntegrationGetEvent | MicrosoftTeamsWorkflowIntegrationListEvent - | MicrosoftTeamsWorkflowIntegrationUpdateEvent; + | MicrosoftTeamsWorkflowIntegrationUpdateEvent + | GetProjectPitCommitsEvent + | GetProjectPitCommitChangesEvent + | PitRollbackCommitEvent + | GetProjectPitCommitCountEvent + | PitRevertCommitEvent + | PitCompareFolderStatesEvent + | PitGetFolderStateEvent; diff --git a/backend/src/ee/services/permission/project-permission.ts b/backend/src/ee/services/permission/project-permission.ts index 319a0259a..aab0c8667 100644 --- a/backend/src/ee/services/permission/project-permission.ts +++ b/backend/src/ee/services/permission/project-permission.ts @@ -17,6 +17,11 @@ export enum ProjectPermissionActions { Delete = "delete" } +export enum ProjectPermissionCommitsActions { + Read = "read", + PerformRollback = "perform-rollback" +} + export enum ProjectPermissionSecretActions { DescribeAndReadValue = "read", DescribeSecret = "describeSecret", @@ -126,6 +131,7 @@ export enum ProjectPermissionSub { SecretRollback = "secret-rollback", SecretApproval = "secret-approval", SecretRotation = "secret-rotation", + Commits = "commits", Identity = "identity", CertificateAuthorities = "certificate-authorities", Certificates = "certificates", @@ -251,7 +257,8 @@ export type ProjectPermissionSet = | [ProjectPermissionActions.Edit, ProjectPermissionSub.Project] | [ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback] | [ProjectPermissionActions.Create, ProjectPermissionSub.SecretRollback] - | [ProjectPermissionActions.Edit, ProjectPermissionSub.Kms]; + | [ProjectPermissionActions.Edit, ProjectPermissionSub.Kms] + | [ProjectPermissionCommitsActions, ProjectPermissionSub.Commits]; const SECRET_PATH_MISSING_SLASH_ERR_MSG = "Invalid Secret Path; it must start with a '/'"; const SECRET_PATH_PERMISSION_OPERATOR_SCHEMA = z.union([ @@ -557,6 +564,12 @@ const GeneralPermissionSchema = [ action: CASL_ACTION_SCHEMA_NATIVE_ENUM(ProjectPermissionKmipActions).describe( "Describe what action an entity can take." ) + }), + z.object({ + subject: z.literal(ProjectPermissionSub.Commits).describe("The entity this permission pertains to."), + action: CASL_ACTION_SCHEMA_NATIVE_ENUM(ProjectPermissionCommitsActions).describe( + "Describe what action an entity can take." + ) }) ]; @@ -829,6 +842,11 @@ const buildAdminPermissionRules = () => { ProjectPermissionSub.SecretRotation ); + can( + [ProjectPermissionCommitsActions.Read, ProjectPermissionCommitsActions.PerformRollback], + ProjectPermissionSub.Commits + ); + return rules; }; @@ -1011,6 +1029,11 @@ const buildMemberPermissionRules = () => { ProjectPermissionSub.SecretSyncs ); + can( + [ProjectPermissionCommitsActions.Read, ProjectPermissionCommitsActions.PerformRollback], + ProjectPermissionSub.Commits + ); + return rules; }; @@ -1046,6 +1069,7 @@ const buildViewerPermissionRules = () => { can(ProjectPermissionActions.Read, ProjectPermissionSub.SshCertificates); can(ProjectPermissionActions.Read, ProjectPermissionSub.SshCertificateTemplates); can(ProjectPermissionSecretSyncActions.Read, ProjectPermissionSub.SecretSyncs); + can(ProjectPermissionCommitsActions.Read, ProjectPermissionSub.Commits); return rules; }; diff --git a/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts b/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts index 95094f767..90377f828 100644 --- a/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts +++ b/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts @@ -351,6 +351,7 @@ export const secretRotationQueueFactory = ({ folderId: secretVersions[0].folderId, changes: secretVersions.map((sv) => ({ type: "add", + isUpdate: true, secretVersionId: sv.id })) }, diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 2960519bd..aeecaee7b 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -587,7 +587,8 @@ export const registerRoutes = async ( folderCheckpointResourcesDAL, secretV2BridgeDAL, folderTreeCheckpointResourcesDAL, - folderCommitQueueService + folderCommitQueueService, + permissionService }); const scimService = scimServiceFactory({ licenseService, diff --git a/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts b/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts index 7a36ec641..3e2d09d0b 100644 --- a/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts +++ b/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts @@ -27,6 +27,10 @@ export const folderCheckpointResourcesDALFactory = (db: TDbClient) => { (TFolderCheckpointResources & { referencedSecretId?: string; referencedFolderId?: string; + folderName?: string; + folderVersion?: string; + secretKey?: string; + secretVersion?: string; })[] > => { try { @@ -45,9 +49,17 @@ export const folderCheckpointResourcesDALFactory = (db: TDbClient) => { .select(selectAllTableCols(TableName.FolderCheckpointResources)) .select( db.ref("secretId").withSchema(TableName.SecretVersionV2).as("referencedSecretId"), - db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("referencedFolderId") + db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("referencedFolderId"), + db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderName"), + db.ref("version").withSchema(TableName.SecretFolderVersion).as("folderVersion"), + db.ref("key").withSchema(TableName.SecretVersionV2).as("secretKey"), + db.ref("version").withSchema(TableName.SecretVersionV2).as("secretVersion") ); - return docs; + return docs.map((doc) => ({ + ...doc, + folderVersion: doc.folderVersion?.toString(), + secretVersion: doc.secretVersion?.toString() + })); } catch (error) { throw new DatabaseError({ error, name: "FindByCheckpointId" }); } diff --git a/backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts b/backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts index 0b06f1362..d027558fa 100644 --- a/backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts +++ b/backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts @@ -1,7 +1,13 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; -import { TableName, TFolderCommitChanges, TFolderCommits } from "@app/db/schemas"; +import { + TableName, + TFolderCommitChanges, + TFolderCommits, + TSecretFolderVersions, + TSecretVersionsV2 +} from "@app/db/schemas"; import { DatabaseError } from "@app/lib/errors"; import { ormify, selectAllTableCols } from "@app/lib/knex"; @@ -12,17 +18,67 @@ type CommitChangeWithCommitInfo = TFolderCommitChanges & { actorType: string; message?: string | null; folderId: string; + folderName?: string; + folderVersion?: string; + secretKey?: string; + secretVersion?: string; + secretId?: string; + folderChangeId?: string; + versions?: { + secretKey?: string; + secretComment?: string; + skipMultilineEncoding?: boolean | null; + secretReminderRepeatDays?: number | null; + secretReminderNote?: string | null; + metadata?: unknown; + tags?: string[] | null; + secretReminderRecipients?: string[] | null; + secretValue?: string; + name?: string; + }[]; }; export const folderCommitChangesDALFactory = (db: TDbClient) => { const folderCommitChangesOrm = ormify(db, TableName.FolderCommitChanges); - const findByCommitId = async (folderCommitId: string, tx?: Knex): Promise => { + const findByCommitId = async (folderCommitId: string, tx?: Knex): Promise => { try { const docs = await (tx || db.replicaNode())(TableName.FolderCommitChanges) .where({ folderCommitId }) - .select(selectAllTableCols(TableName.FolderCommitChanges)); - return docs; + .leftJoin( + TableName.FolderCommit, + `${TableName.FolderCommitChanges}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) + .leftJoin( + TableName.SecretVersionV2, + `${TableName.FolderCommitChanges}.secretVersionId`, + `${TableName.SecretVersionV2}.id` + ) + .leftJoin( + TableName.SecretFolderVersion, + `${TableName.FolderCommitChanges}.folderVersionId`, + `${TableName.SecretFolderVersion}.id` + ) + .select(selectAllTableCols(TableName.FolderCommitChanges)) + .select( + db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderName"), + db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("folderChangeId"), + db.ref("version").withSchema(TableName.SecretFolderVersion).as("folderVersion"), + db.ref("key").withSchema(TableName.SecretVersionV2).as("secretKey"), + db.ref("version").withSchema(TableName.SecretVersionV2).as("secretVersion"), + db.ref("secretId").withSchema(TableName.SecretVersionV2), + db.ref("actorMetadata").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("createdAt").withSchema(TableName.FolderCommit), + db.ref("folderId").withSchema(TableName.FolderCommit) + ); + return docs.map((doc) => ({ + ...doc, + folderVersion: doc.folderVersion?.toString(), + secretVersion: doc.secretVersion?.toString() + })); } catch (error) { throw new DatabaseError({ error, name: "FindByCommitId" }); } diff --git a/backend/src/services/folder-commit/folder-commit-dal.ts b/backend/src/services/folder-commit/folder-commit-dal.ts index 7f015ddc0..5992a4672 100644 --- a/backend/src/services/folder-commit/folder-commit-dal.ts +++ b/backend/src/services/folder-commit/folder-commit-dal.ts @@ -19,11 +19,36 @@ export const folderCommitDALFactory = (db: TDbClient) => { const findByFolderId = async (folderId: string, tx?: Knex): Promise => { try { - const docs = await (tx || db.replicaNode())(TableName.FolderCommit) + const trx = tx || db.replicaNode(); + + // First, get all folder commits + const folderCommits = await trx(TableName.FolderCommit) .where({ folderId }) - .select(selectAllTableCols(TableName.FolderCommit)) + .select("*") .orderBy("createdAt", "desc"); - return docs; + + if (folderCommits.length === 0) return []; + + // Get all commit IDs + const commitIds = folderCommits.map((commit) => commit.id); + + // Then get all related changes + const changes = await trx(TableName.FolderCommitChanges).whereIn("folderCommitId", commitIds).select("*"); + + const changesMap = changes.reduce( + (acc, change) => { + const { folderCommitId } = change; + if (!acc[folderCommitId]) acc[folderCommitId] = []; + acc[folderCommitId].push(change); + return acc; + }, + {} as Record + ); + + return folderCommits.map((commit) => ({ + ...commit, + changes: changesMap[commit.id] || [] + })); } catch (error) { throw new DatabaseError({ error, name: "FindByFolderId" }); } @@ -158,7 +183,14 @@ export const folderCommitDALFactory = (db: TDbClient) => { tx?: Knex ): Promise< (TFolderCommits & { - changes: (TFolderCommitChanges & { referencedSecretId?: string; referencedFolderId?: string })[]; + changes: (TFolderCommitChanges & { + referencedSecretId?: string; + referencedFolderId?: string; + folderName?: string; + folderVersion?: string; + secretKey?: string; + secretVersion?: string; + })[]; })[] > => { try { @@ -195,7 +227,11 @@ export const folderCommitDALFactory = (db: TDbClient) => { .select(selectAllTableCols(TableName.FolderCommitChanges)) .select( db.ref("secretId").withSchema(TableName.SecretVersionV2).as("referencedSecretId"), - db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("referencedFolderId") + db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("referencedFolderId"), + db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderName"), + db.ref("version").withSchema(TableName.SecretFolderVersion).as("folderVersion"), + db.ref("key").withSchema(TableName.SecretVersionV2).as("secretKey"), + db.ref("version").withSchema(TableName.SecretVersionV2).as("secretVersion") ); // Organize changes by commit ID @@ -280,6 +316,34 @@ export const folderCommitDALFactory = (db: TDbClient) => { } }; + const findAllFolderCommitsAfter = async ({ + envId, + startCommitId, + tx + }: { + folderId?: string; + envId?: string; + startCommitId?: string; + tx?: Knex; + }): Promise => { + try { + const docs = await (tx || db.replicaNode())(TableName.FolderCommit) + .where((qb) => { + if (envId) { + void qb.where(`${TableName.FolderCommit}.envId`, "=", envId); + } + if (startCommitId) { + void qb.where("commitId", ">=", startCommitId); + } + }) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("commitId", "desc"); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestCommitBetween" }); + } + }; + return { ...restOfOrm, findByFolderId, @@ -291,6 +355,7 @@ export const folderCommitDALFactory = (db: TDbClient) => { findLatestCommitBetween, findLatestEnvCommit, getEnvNumberOfCommitsSince, - findLatestCommitByFolderIds + findLatestCommitByFolderIds, + findAllFolderCommitsAfter }; }; diff --git a/backend/src/services/folder-commit/folder-commit-service.test.ts b/backend/src/services/folder-commit/folder-commit-service.test.ts index f00e12283..c25ead567 100644 --- a/backend/src/services/folder-commit/folder-commit-service.test.ts +++ b/backend/src/services/folder-commit/folder-commit-service.test.ts @@ -113,7 +113,8 @@ describe("folderCommitServiceFactory", () => { create: vi.fn().mockResolvedValue({}), updateById: vi.fn().mockResolvedValue({}), find: vi.fn().mockResolvedValue([]), - findByIdsWithLatestVersion: vi.fn().mockResolvedValue({}) + findByIdsWithLatestVersion: vi.fn().mockResolvedValue({}), + findLatestVersionMany: vi.fn().mockResolvedValue({}) }; const mockSecretV2BridgeDAL = { @@ -133,26 +134,44 @@ describe("folderCommitServiceFactory", () => { scheduleTreeCheckpoint: vi.fn().mockResolvedValue({}) }; + const mockPermissionService = { + getProjectPermission: vi.fn().mockResolvedValue({}) + }; + let folderCommitService: TFolderCommitServiceFactory; beforeEach(() => { vi.clearAllMocks(); folderCommitService = folderCommitServiceFactory({ + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing folderCommitDAL: mockFolderCommitDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing folderCommitChangesDAL: mockFolderCommitChangesDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing folderCheckpointDAL: mockFolderCheckpointDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing folderCheckpointResourcesDAL: mockFolderCheckpointResourcesDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing folderTreeCheckpointDAL: mockFolderTreeCheckpointDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing folderTreeCheckpointResourcesDAL: mockFolderTreeCheckpointResourcesDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing userDAL: mockUserDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing identityDAL: mockIdentityDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing folderDAL: mockFolderDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing folderVersionDAL: mockFolderVersionDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing secretVersionV2BridgeDAL: mockSecretVersionV2BridgeDAL, projectDAL: mockProjectDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing secretV2BridgeDAL: mockSecretV2BridgeDAL, - folderCommitQueueService: mockFolderCommitQueueService + folderCommitQueueService: mockFolderCommitQueueService, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + permissionService: mockPermissionService }); }); @@ -532,13 +551,31 @@ describe("folderCommitServiceFactory", () => { mockSecretVersionV2BridgeDAL.findLatestVersionByFolderId.mockResolvedValue([]); // These mocks need to return objects with an id field - mockSecretVersionV2BridgeDAL.findByIdsWithLatestVersion.mockResolvedValue(secretVersions); - mockFolderVersionDAL.findByIdsWithLatestVersion.mockResolvedValue(folderVersions); + mockSecretVersionV2BridgeDAL.findByIdsWithLatestVersion.mockResolvedValue(Object.values(secretVersions)); + mockFolderVersionDAL.findByIdsWithLatestVersion.mockResolvedValue(Object.values(folderVersions)); mockSecretV2BridgeDAL.insertMany.mockResolvedValue([{ id: "new-secret-1" }]); mockSecretVersionV2BridgeDAL.create.mockResolvedValue({ id: "new-secret-version-1" }); mockFolderDAL.updateById.mockResolvedValue({ id: "updated-folder-1" }); mockFolderVersionDAL.create.mockResolvedValue({ id: "new-folder-version-1" }); mockFolderCommitDAL.create.mockResolvedValue({ id: "new-commit-id" }); + mockSecretVersionV2BridgeDAL.findLatestVersionMany.mockResolvedValue([ + { + id: "secret-version-1", + createdAt: new Date(), + updatedAt: new Date(), + type: "shared", + folderId: "folder-1", + secretId: "secret-1", + version: 1, + key: "SECRET_KEY", + encryptedValue: Buffer.from("encrypted"), + encryptedComment: Buffer.from("comment"), + skipMultilineEncoding: false, + userId: "user-1", + envId: "env-1", + metadata: {} + } + ]); // Mock transaction mockFolderCommitDAL.transaction.mockImplementation((callback: TransactionCallback) => callback({} as Knex)); diff --git a/backend/src/services/folder-commit/folder-commit-service.ts b/backend/src/services/folder-commit/folder-commit-service.ts index 1fc4b456d..9d598f2ba 100644 --- a/backend/src/services/folder-commit/folder-commit-service.ts +++ b/backend/src/services/folder-commit/folder-commit-service.ts @@ -1,12 +1,15 @@ /* eslint-disable no-await-in-loop */ +import { ForbiddenError } from "@casl/ability"; import { Knex } from "knex"; -import { TSecretFolders, TSecretFolderVersions, TSecretVersionsV2 } from "@app/db/schemas"; +import { ActionProjectType, TSecretFolders, TSecretFolderVersions, TSecretVersionsV2 } from "@app/db/schemas"; +import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; +import { ProjectPermissionCommitsActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; import { getConfig } from "@app/lib/config/env"; import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors"; import { logger } from "@app/lib/logger"; -import { ActorType } from "../auth/auth-type"; +import { ActorAuthMethod, ActorType } from "../auth/auth-type"; import { TFolderCheckpointDALFactory } from "../folder-checkpoint/folder-checkpoint-dal"; import { TFolderCheckpointResourcesDALFactory } from "../folder-checkpoint-resources/folder-checkpoint-resources-dal"; import { TFolderCommitChangesDALFactory } from "../folder-commit-changes/folder-commit-changes-dal"; @@ -22,7 +25,6 @@ import { TUserDALFactory } from "../user/user-dal"; import { TFolderCommitDALFactory } from "./folder-commit-dal"; import { TFolderCommitQueueServiceFactory } from "./folder-commit-queue"; -// Define enums for better type safety export enum ChangeType { ADD = "add", DELETE = "delete", @@ -35,7 +37,6 @@ enum ResourceType { FOLDER = "folder" } -// Improved types for DTO objects type TCreateCommitDTO = { actor: { type: string; @@ -50,6 +51,7 @@ type TCreateCommitDTO = { type: string; secretVersionId?: string; folderVersionId?: string; + isUpdate?: boolean; }[]; }; @@ -69,6 +71,23 @@ export type ResourceChange = { commitId: number; createdAt?: Date; parentId?: string; + secretKey?: string; + secretVersion?: string; + folderName?: string; + folderVersion?: string; + fromVersion?: string; + versions?: { + secretKey?: string; + secretComment?: string; + skipMultilineEncoding?: boolean | null; + secretReminderRepeatDays?: number | null; + secretReminderNote?: string | null; + metadata?: unknown; + tags?: string[] | null; + secretReminderRecipients?: string[] | null; + secretValue?: string; + name?: string; + }[]; }; type ActorInfo = { @@ -84,77 +103,21 @@ type StateChangeResult = { }; type TFolderCommitServiceFactoryDep = { - folderCommitDAL: Pick< - TFolderCommitDALFactory, - | "create" - | "findById" - | "findByFolderId" - | "findLatestCommit" - | "transaction" - | "getNumberOfCommitsSince" - | "getEnvNumberOfCommitsSince" - | "findCommitsToRecreate" - | "findMultipleLatestCommits" - | "findLatestCommitBetween" - | "findAllCommitsBetween" - | "findLatestEnvCommit" - | "findLatestCommitByFolderIds" - >; - folderCommitChangesDAL: Pick; - folderCheckpointDAL: Pick< - TFolderCheckpointDALFactory, - "create" | "findByFolderId" | "findLatestByFolderId" | "findNearestCheckpoint" - >; - folderCheckpointResourcesDAL: Pick; - folderTreeCheckpointDAL: Pick< - TFolderTreeCheckpointDALFactory, - "create" | "findNearestCheckpoint" | "findLatestByEnvId" - >; - folderTreeCheckpointResourcesDAL: Pick< - TFolderTreeCheckpointResourcesDALFactory, - "insertMany" | "findByTreeCheckpointId" - >; - userDAL: Pick; - identityDAL: Pick; - folderDAL: Pick< - TSecretFolderDALFactory, - | "findByParentId" - | "findByProjectId" - | "deleteById" - | "create" - | "updateById" - | "update" - | "find" - | "findById" - | "findByEnvId" - | "findFoldersByRootAndIds" - >; - folderVersionDAL: Pick< - TSecretFolderVersionDALFactory, - | "findLatestFolderVersions" - | "findById" - | "deleteById" - | "create" - | "updateById" - | "find" - | "findByIdsWithLatestVersion" - >; - secretVersionV2BridgeDAL: Pick< - TSecretVersionV2DALFactory, - | "findLatestVersionByFolderId" - | "findById" - | "deleteById" - | "create" - | "updateById" - | "find" - | "findByIdsWithLatestVersion" - >; - secretV2BridgeDAL: Pick< - secretV2BridgeDal.TSecretV2BridgeDALFactory, - "deleteById" | "create" | "updateById" | "update" | "insertMany" | "invalidateSecretCacheByProjectId" - >; + folderCommitDAL: TFolderCommitDALFactory; + folderCommitChangesDAL: TFolderCommitChangesDALFactory; + folderCheckpointDAL: TFolderCheckpointDALFactory; + folderCheckpointResourcesDAL: TFolderCheckpointResourcesDALFactory; + folderTreeCheckpointDAL: TFolderTreeCheckpointDALFactory; + folderTreeCheckpointResourcesDAL: TFolderTreeCheckpointResourcesDALFactory; + userDAL: TUserDALFactory; + identityDAL: TIdentityDALFactory; + folderDAL: TSecretFolderDALFactory; + folderVersionDAL: TSecretFolderVersionDALFactory; + secretVersionV2BridgeDAL: TSecretVersionV2DALFactory; + secretV2BridgeDAL: secretV2BridgeDal.TSecretV2BridgeDALFactory; projectDAL: Pick; folderCommitQueueService?: Pick; + permissionService?: TPermissionServiceFactory; }; export const folderCommitServiceFactory = ({ @@ -171,10 +134,39 @@ export const folderCommitServiceFactory = ({ projectDAL, secretV2BridgeDAL, folderTreeCheckpointResourcesDAL, - folderCommitQueueService + folderCommitQueueService, + permissionService }: TFolderCommitServiceFactoryDep) => { const appCfg = getConfig(); + const checkProjectPermission = async ({ + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + }: { + actor: ActorType; + actorId: string; + projectId: string; + actorAuthMethod: ActorAuthMethod; + actorOrgId: string; + }) => { + if (!permissionService) { + throw new Error("Permission service not initialized"); + } + const { permission } = await permissionService.getProjectPermission({ + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId, + actionProjectType: ActionProjectType.SecretManager + }); + + ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionCommitsActions.Read, ProjectPermissionSub.Commits); + }; + /** * Fetches all resources within a folder */ @@ -255,7 +247,17 @@ export const folderCommitServiceFactory = ({ const reconstructFolderState = async ( folderCommitId: string, tx?: Knex - ): Promise<{ type: string; id: string; versionId: string }[]> => { + ): Promise< + { + type: string; + id: string; + versionId: string; + secretKey?: string; + secretVersion?: string; + folderName?: string; + folderVersion?: string; + }[] + > => { const targetCommit = await folderCommitDAL.findById(folderCommitId, tx); if (!targetCommit) { throw new NotFoundError({ message: `Commit with ID ${folderCommitId} not found` }); @@ -268,7 +270,18 @@ export const folderCommitServiceFactory = ({ const checkpointResources = await folderCheckpointResourcesDAL.findByCheckpointId(nearestCheckpoint.id, tx); - const folderState: Record = {}; + const folderState: Record< + string, + { + type: string; + id: string; + versionId: string; + secretKey?: string; + secretVersion?: string; + folderName?: string; + folderVersion?: string; + } + > = {}; // Add all checkpoint resources to initial state checkpointResources.forEach((resource) => { @@ -276,13 +289,17 @@ export const folderCommitServiceFactory = ({ folderState[`secret-${resource.referencedSecretId}`] = { type: "secret", id: resource.referencedSecretId, - versionId: resource.secretVersionId + versionId: resource.secretVersionId, + secretKey: resource.secretKey, + secretVersion: resource.secretVersion }; } else if (resource.folderVersionId && resource.referencedFolderId) { folderState[`folder-${resource.referencedFolderId}`] = { type: "folder", id: resource.referencedFolderId, - versionId: resource.folderVersionId + versionId: resource.folderVersionId, + folderName: resource.folderName, + folderVersion: resource.folderVersion }; } }); @@ -307,7 +324,9 @@ export const folderCommitServiceFactory = ({ folderState[key] = { type: "secret", id: change.referencedSecretId, - versionId: change.secretVersionId + versionId: change.secretVersionId, + secretKey: change.secretKey, + secretVersion: change.secretVersion }; } else if (change.changeType.toLowerCase() === "delete") { delete folderState[key]; @@ -319,7 +338,9 @@ export const folderCommitServiceFactory = ({ folderState[key] = { type: "folder", id: change.referencedFolderId, - versionId: change.folderVersionId + versionId: change.folderVersionId, + folderName: change.folderName, + folderVersion: change.folderVersion }; } else if (change.changeType.toLowerCase() === "delete") { delete folderState[key]; @@ -356,7 +377,11 @@ export const folderCommitServiceFactory = ({ id: resource.id, versionId: resource.versionId, changeType: "create", - commitId: targetCommit.commitId + commitId: targetCommit.commitId, + secretKey: resource.secretKey, + secretVersion: resource.secretVersion, + folderName: resource.folderName, + folderVersion: resource.folderVersion })) as ResourceChange[]; } @@ -365,8 +390,32 @@ export const folderCommitServiceFactory = ({ const targetState = await reconstructFolderState(targetCommitId, tx); // Create lookup maps for easier comparison - const currentMap: Record = {}; - const targetMap: Record = {}; + const currentMap: Record< + string, + { + type: string; + id: string; + versionId: string; + secretKey?: string; + secretVersion?: string; + folderName?: string; + folderVersion?: string; + fromVersion?: string; + } + > = {}; + const targetMap: Record< + string, + { + type: string; + id: string; + versionId: string; + secretKey?: string; + secretVersion?: string; + folderName?: string; + folderVersion?: string; + fromVersion?: string; + } + > = {}; // Build lookup maps currentState.forEach((resource) => { @@ -393,7 +442,12 @@ export const folderCommitServiceFactory = ({ id: currentResource.id, versionId: currentResource.versionId, changeType: ChangeType.DELETE, - commitId: targetCommit.commitId + commitId: targetCommit.commitId, + secretKey: currentResource.secretKey, + secretVersion: currentResource.secretVersion, + folderName: currentResource.folderName, + folderVersion: currentResource.folderVersion, + fromVersion: currentResource.versionId }); } else if (currentResource.versionId !== targetResource.versionId) { differences.push({ @@ -401,12 +455,16 @@ export const folderCommitServiceFactory = ({ id: targetResource.id, versionId: targetResource.versionId, changeType: ChangeType.UPDATE, - commitId: targetCommit.commitId + commitId: targetCommit.commitId, + secretKey: targetResource.secretKey, + secretVersion: targetResource.secretVersion, + folderName: targetResource.folderName, + folderVersion: targetResource.folderVersion, + fromVersion: currentResource.folderVersion || currentResource.secretVersion }); } }); - // Find creates Object.keys(targetMap).forEach((key) => { if (!currentMap[key]) { const targetResource = targetMap[key]; @@ -416,7 +474,11 @@ export const folderCommitServiceFactory = ({ versionId: targetResource.versionId, changeType: ChangeType.CREATE, commitId: targetCommit.commitId, - createdAt: targetCommit.createdAt + createdAt: targetCommit.createdAt, + secretKey: targetResource.secretKey, + secretVersion: targetResource.secretVersion, + folderName: targetResource.folderName, + folderVersion: targetResource.folderVersion }); } }); @@ -485,7 +547,8 @@ export const folderCommitServiceFactory = ({ folderCommitId: newCommit.id, changeType: change.type, secretVersionId: change.secretVersionId, - folderVersionId: change.folderVersionId + folderVersionId: change.folderVersionId, + isUpdate: change.isUpdate || false })), tx ); @@ -515,11 +578,21 @@ export const folderCommitServiceFactory = ({ ) => { const commitChanges = []; + // Collect all secretIds for batch lookup + const secretIds = changes.map((change) => secretVersions[change.id]?.secretId).filter(Boolean); + + // Fetch all latest versions in one call + const latestVersionsMap = await secretVersionV2BridgeDAL.findLatestVersionMany(folderId, secretIds, tx); + for (const change of changes) { const secretVersion = secretVersions[change.id]; // eslint-disable-next-line no-continue if (!secretVersion) continue; + // Get the latest version from our batch result + const latestVersion = latestVersionsMap[secretVersion.secretId]; + const nextVersion = latestVersion ? latestVersion.version + 1 : 1; + switch (change.changeType) { case "create": { @@ -527,7 +600,7 @@ export const folderCommitServiceFactory = ({ { id: change.id, skipMultilineEncoding: secretVersion.skipMultilineEncoding, - version: secretVersion.version + 1, + version: nextVersion, type: secretVersion.type, key: secretVersion.key, reminderNote: secretVersion.reminderNote, @@ -545,7 +618,7 @@ export const folderCommitServiceFactory = ({ { folderId, secretId: secretVersion.secretId, - version: secretVersion.version + 1, + version: nextVersion, encryptedValue: secretVersion.encryptedValue, key: secretVersion.key, encryptedComment: secretVersion.encryptedComment, @@ -575,7 +648,7 @@ export const folderCommitServiceFactory = ({ change.id, { skipMultilineEncoding: secretVersion?.skipMultilineEncoding, - version: secretVersion?.version, + version: nextVersion, type: secretVersion?.type, key: secretVersion?.key, reminderNote: secretVersion?.reminderNote, @@ -590,7 +663,7 @@ export const folderCommitServiceFactory = ({ const newVersion = await secretVersionV2BridgeDAL.create( { - version: secretVersion.version + 1, + version: nextVersion, encryptedValue: secretVersion.encryptedValue, key: secretVersion.key, encryptedComment: secretVersion.encryptedComment, @@ -611,14 +684,15 @@ export const folderCommitServiceFactory = ({ commitChanges.push({ type: ChangeType.ADD, + isUpdate: true, secretVersionId: newVersion.id }); } break; + // Delete case remains unchanged case "delete": await secretV2BridgeDAL.deleteById(change.id, tx); - commitChanges.push({ type: ChangeType.DELETE, secretVersionId: change.versionId @@ -724,8 +798,13 @@ export const folderCommitServiceFactory = ({ case "update": if (change.versionId) { - const versionDetails = await folderVersionDAL.findById(change.versionId, tx); - if (versionDetails) { + const latestVersionDetails = await folderVersionDAL.findByIdsWithLatestVersion( + [change.versionId], + undefined, + tx + ); + if (latestVersionDetails && Object.keys(latestVersionDetails).length > 0) { + const versionDetails = latestVersionDetails[0]; await folderDAL.updateById( change.id, { @@ -749,6 +828,7 @@ export const folderCommitServiceFactory = ({ commitChanges.push({ type: ChangeType.ADD, + isUpdate: true, folderVersionId: newFolderVersion.id }); } @@ -855,11 +935,103 @@ export const folderCommitServiceFactory = ({ return folderCommitDAL.findByFolderId(folderId, tx); }; + const getCommitsForFolder = async ({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId, + environment, + path + }: { + actor: ActorType; + actorId: string; + actorAuthMethod: ActorAuthMethod; + actorOrgId: string; + projectId: string; + environment: string; + path: string; + }) => { + await checkProjectPermission({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId + }); + const folder = await folderDAL.findBySecretPath(projectId, environment, path); + if (!folder) { + throw new NotFoundError({ + message: `Folder not found for project ID ${projectId}, environment ${environment}, path ${path}` + }); + } + const folderCommits = await folderCommitDAL.findByFolderId(folder.id); + return folderCommits; + }; + + const getCommitsCount = async ({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId, + environment, + path + }: { + actor: ActorType; + actorId: string; + actorAuthMethod: ActorAuthMethod; + actorOrgId: string; + projectId: string; + environment: string; + path: string; + }) => { + await checkProjectPermission({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId + }); + + const folder = await folderDAL.findBySecretPath(projectId, environment, path); + if (!folder) { + throw new NotFoundError({ + message: `Folder not found for project ID ${projectId}, environment ${environment}, path ${path}` + }); + } + const folderCommits = await folderCommitDAL.findByFolderId(folder.id); + return { count: folderCommits.length, folderId: folder.id }; + }; + /** * Get changes for a commit */ - const getCommitChanges = async (commitId: string, tx?: Knex) => { - return folderCommitChangesDAL.findByCommitId(commitId, tx); + const getCommitChanges = async ({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId, + commitId + }: { + actor: ActorType; + actorId: string; + actorAuthMethod: ActorAuthMethod; + actorOrgId: string; + projectId: string; + commitId: string; + }) => { + await checkProjectPermission({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId + }); + const changes = await folderCommitChangesDAL.findByCommitId(commitId); + const commit = await folderCommitDAL.findById(commitId); + return { ...commit, changes }; }; /** @@ -1021,6 +1193,100 @@ export const folderCommitServiceFactory = ({ ); }; + const deepCompareFolder = async ({ + targetCommitId, + envId, + projectId, + tx + }: { + targetCommitId: string; + envId: string; + actorId: string; + actorType: ActorType; + projectId: string; + tx?: Knex; + }) => { + const targetCommit = await folderCommitDAL.findById(targetCommitId, tx); + if (!targetCommit) { + throw new NotFoundError({ message: `No commit found for commit ID ${targetCommitId}` }); + } + + const checkpoint = await folderTreeCheckpointDAL.findNearestCheckpoint(targetCommitId, envId, tx); + if (!checkpoint) { + throw new NotFoundError({ message: `No checkpoint found for commit ID ${targetCommitId}` }); + } + + const folderCheckpointCommits = await folderTreeCheckpointResourcesDAL.findByTreeCheckpointId(checkpoint.id, tx); + const folderCommits = await folderCommitDAL.findAllCommitsBetween({ + envId, + endCommitId: targetCommit.commitId.toString(), + startCommitId: checkpoint.commitId.toString(), + tx + }); + + // Group commits by folderId and keep only the latest + const folderGroups = new Map(); + + if (folderCheckpointCommits && folderCheckpointCommits.length > 0) { + for (const commit of folderCheckpointCommits) { + folderGroups.set(commit.folderId, { + createdAt: commit.createdAt, + id: commit.folderCommitId + }); + } + } + + if (folderCommits && folderCommits.length > 0) { + for (const commit of folderCommits) { + const { folderId, createdAt, id } = commit; + const existingCommit = folderGroups.get(folderId); + + if (!existingCommit || createdAt.getTime() > existingCommit.createdAt.getTime()) { + folderGroups.set(folderId, { createdAt, id }); + } + } + } + + const folderDiffs = new Map(); + + // Process each folder to determine differences + await Promise.all( + Array.from(folderGroups.entries()).map(async ([folderId, commit]) => { + const latestFolderCommit = await folderCommitDAL.findLatestCommit(folderId, tx); + if (latestFolderCommit && latestFolderCommit.id !== commit.id) { + const diff = await compareFolderStates({ + currentCommitId: latestFolderCommit.id, + targetCommitId: commit.id, + tx + }); + if (diff?.length > 0) { + folderDiffs.set(folderId, diff); + } + } + }) + ); + + // Apply changes in hierarchical order + const folderIds = Array.from(folderDiffs.keys()); + const folders = await folderDAL.findFoldersByRootAndIds({ rootId: targetCommit.folderId, folderIds }, tx); + const sortedFolders = sortFoldersByHierarchy(folders); + + const response = []; + for (const folder of sortedFolders) { + const diff = folderDiffs.get(folder.id); + if (diff) { + const folderPath = await folderDAL.findSecretPathByFolderIds(projectId, [folder.id]); + response.push({ + folderId: folder.id, + folderName: folder.name, + changes: diff, + folderPath: folderPath?.[0]?.path + }); + } + } + return response; + }; + /** * Roll back a folder tree to a specific commit */ @@ -1117,7 +1383,139 @@ export const folderCommitServiceFactory = ({ } }; - // Return the public interface + const getLatestCommit = async ({ + folderId, + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId + }: { + folderId: string; + actor: ActorType; + actorId: string; + actorAuthMethod: ActorAuthMethod; + actorOrgId: string; + projectId: string; + }) => { + await checkProjectPermission({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId + }); + return folderCommitDAL.findLatestCommit(folderId); + }; + + /** + * Revert changes made in a specific commit + */ + const revertCommitChanges = async ({ + commitId, + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId, + message = "Revert commit changes" + }: { + commitId: string; + actor: ActorType; + actorId: string; + actorAuthMethod: ActorAuthMethod; + actorOrgId: string; + projectId: string; + message?: string; + }) => { + // Check permissions first + await checkProjectPermission({ + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + }); + + // Get the commit to revert + const commitToRevert = await folderCommitDAL.findById(commitId); + if (!commitToRevert) { + throw new NotFoundError({ message: `Commit with ID ${commitId} not found` }); + } + + // Get all commits for this folder + const allCommits = await folderCommitDAL.findByFolderId(commitToRevert.folderId); + if (!allCommits || allCommits.length === 0) { + throw new NotFoundError({ message: `No commits found for folder ${commitToRevert.folderId}` }); + } + + // Sort commits by commitId (which appears to be numeric) + const sortedCommits = allCommits.sort((a, b) => a.commitId - b.commitId); + + // Find the index of the commit to revert + const commitIndex = sortedCommits.findIndex((c) => c.id === commitId); + if (commitIndex === -1) { + throw new NotFoundError({ message: `Commit ${commitId} not found in the commit history` }); + } + + // If it's the first commit, we can't revert it (nothing before it) + if (commitIndex === 0) { + throw new BadRequestError({ message: "Cannot revert the first commit" }); + } + + // Get the commit just before the one we want to revert + const previousCommit = sortedCommits[commitIndex - 1]; + + // Calculate the changes needed to go from current commit back to the previous one + const inverseChanges = await compareFolderStates({ + currentCommitId: commitToRevert.id, + targetCommitId: previousCommit.id + }); + + const latestCommit = await folderCommitDAL.findLatestCommit(commitToRevert.folderId); + if (!latestCommit) { + throw new NotFoundError({ message: `Latest commit not found for folder ${commitToRevert.folderId}` }); + } + const currentState = await reconstructFolderState(latestCommit.id); + + const filteredChanges = inverseChanges.filter( + (change) => + ((change.changeType === ChangeType.DELETE || change.changeType === ChangeType.UPDATE) && + (currentState.some((c) => c.id === change.id) || currentState.some((c) => c.id === change.id))) || + (change.changeType === ChangeType.CREATE && + (currentState.every((c) => c.id !== change.id) || currentState.every((c) => c.id !== change.id))) + ); + + if (!filteredChanges || filteredChanges.length === 0) { + return { + success: true, + message: "No changes to revert", + originalCommitId: commitId + }; + } + + // Apply the changes to revert the commit + const revertResult = await applyFolderStateDifferences({ + differences: filteredChanges, + actorInfo: { + actorType: actor, + actorId, + message: message || `Reverted changes from commit ${commitId}` + }, + folderId: commitToRevert.folderId, + projectId, + reconstructNewFolders: false + }); + + return { + success: true, + message: "Changes reverted successfully", + originalCommitId: commitId, + revertCommitId: latestCommit?.id, + changesReverted: revertResult.totalChanges + }; + }; + return { createCommit, addCommitChange, @@ -1132,7 +1530,13 @@ export const folderCommitServiceFactory = ({ compareFolderStates, applyFolderStateDifferences, createFolderTreeCheckpoint, - deepRollbackFolder + deepRollbackFolder, + getCommitsCount, + getLatestCommit, + deepCompareFolder, + reconstructFolderState, + getCommitsForFolder, + revertCommitChanges }; }; diff --git a/backend/src/services/secret-folder/secret-folder-service.ts b/backend/src/services/secret-folder/secret-folder-service.ts index fbb40bb5d..87a37b707 100644 --- a/backend/src/services/secret-folder/secret-folder-service.ts +++ b/backend/src/services/secret-folder/secret-folder-service.ts @@ -30,7 +30,7 @@ type TSecretFolderServiceFactoryDep = { snapshotService: Pick; folderDAL: TSecretFolderDALFactory; projectEnvDAL: Pick; - folderVersionDAL: Pick; + folderVersionDAL: Pick; folderCommitService: Pick; projectDAL: Pick; }; @@ -286,6 +286,7 @@ export const secretFolderServiceFactory = ({ changes: [ { type: "add", + isUpdate: true, folderVersionId: folderVersion.id } ] @@ -401,6 +402,7 @@ export const secretFolderServiceFactory = ({ changes: [ { type: "add", + isUpdate: true, folderVersionId: folderVersion.id } ] @@ -765,6 +767,22 @@ export const secretFolderServiceFactory = ({ return environmentFolders; }; + const getFolderVersionsByIds = async ({ + folderId, + folderVersions + }: { + folderId: string; + folderVersions: string[]; + }) => { + const versions = await folderVersionDAL.find({ + folderId, + $in: { + version: folderVersions.map((v) => Number.parseInt(v, 10)) + } + }); + return versions; + }; + return { createFolder, updateFolder, @@ -775,6 +793,7 @@ export const secretFolderServiceFactory = ({ getProjectFolderCount, getFoldersMultiEnv, getFoldersDeepByEnvs, - getProjectEnvironmentsFolders + getProjectEnvironmentsFolders, + getFolderVersionsByIds }; }; diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts index eb2c77878..8b976aebf 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts @@ -148,7 +148,7 @@ export const fnSecretBulkInsert = async ({ id: actor?.actorId } }, - message: "Secret Creation", + message: "Secret Created", folderId, changes: commitChanges }, @@ -290,6 +290,7 @@ export const fnSecretBulkUpdate = async ({ .filter(({ type }) => type === SecretType.Shared) .map((sv) => ({ type: "add", + isUpdate: true, secretVersionId: sv.id })); if (commitChanges.length > 0) { @@ -301,7 +302,7 @@ export const fnSecretBulkUpdate = async ({ id: actor?.actorId } }, - message: "Secret Update", + message: "Secret Updated", folderId, changes: commitChanges }, @@ -432,7 +433,7 @@ export const fnSecretBulkDelete = async ({ id: actorId } }, - message: "Secret Delete", + message: "Secret Deleted", folderId, changes: commitChanges }, diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts index 33d3cf0f1..0e8ce29fc 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts @@ -2178,10 +2178,14 @@ export const secretV2BridgeServiceFactory = ({ type: KmsDataKey.SecretManager, projectId: folder.projectId }); - const secretVersions = await secretVersionDAL.findVersionsBySecretIdWithActors(secretId, folder.projectId, { - offset, - limit, - sort: [["createdAt", "desc"]] + const secretVersions = await secretVersionDAL.findVersionsBySecretIdWithActors({ + secretId, + projectId: folder.projectId, + findOpt: { + offset, + limit, + sort: [["createdAt", "desc"]] + } }); return secretVersions.map((el) => { const secretValueHidden = !hasSecretReadValueOrDescribePermission( @@ -2856,6 +2860,73 @@ export const secretV2BridgeServiceFactory = ({ }; }; + const getSecretVersionsByIds = async ({ + actorId, + actor, + actorOrgId, + actorAuthMethod, + secretId, + secretVersionNumbers, + folderId + }: TGetSecretVersionsDTO & { + secretVersionNumbers: string[]; + folderId: string; + }) => { + const folder = await folderDAL.findById(folderId); + if (!folder) throw new NotFoundError({ message: `Folder with ID '${folderId}' not found` }); + + const [folderWithPath] = await folderDAL.findSecretPathByFolderIds(folder.projectId, [folder.id]); + + if (!folderWithPath) { + throw new NotFoundError({ message: `Folder with ID '${folder.id}' not found` }); + } + + const { permission } = await permissionService.getProjectPermission({ + actor, + actorId, + projectId: folder.projectId, + actorAuthMethod, + actorOrgId, + actionProjectType: ActionProjectType.SecretManager + }); + ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback); + const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId: folder.projectId + }); + const secretVersions = await secretVersionDAL.findVersionsBySecretIdWithActors({ + secretId, + projectId: folder.projectId, + secretVersions: secretVersionNumbers + }); + return secretVersions.map((el) => { + const secretValueHidden = !hasSecretReadValueOrDescribePermission( + permission, + ProjectPermissionSecretActions.ReadValue, + { + environment: folder.environment.envSlug, + secretPath: folderWithPath.path, + secretName: el.key, + ...(el.tags?.length && { + secretTags: el.tags.map((tag) => tag.slug) + }) + } + ); + + return reshapeBridgeSecret( + folder.projectId, + folder.environment.envSlug, + folderWithPath.path, + { + ...el, + value: el.encryptedValue ? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString() : "", + comment: el.encryptedComment ? secretManagerDecryptor({ cipherTextBlob: el.encryptedComment }).toString() : "" + }, + secretValueHidden + ); + }); + }; + return { createSecret, deleteSecret, @@ -2874,6 +2945,7 @@ export const secretV2BridgeServiceFactory = ({ getSecretReferenceTree, getSecretsByFolderMappings, getSecretById, - getAccessibleSecrets + getAccessibleSecrets, + getSecretVersionsByIds }; }; diff --git a/backend/src/services/secret-v2-bridge/secret-version-dal.ts b/backend/src/services/secret-v2-bridge/secret-version-dal.ts index 291fec22f..62a8203ea 100644 --- a/backend/src/services/secret-v2-bridge/secret-version-dal.ts +++ b/backend/src/services/secret-v2-bridge/secret-version-dal.ts @@ -173,13 +173,21 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { logger.info(`${QueueName.DailyResourceCleanUp}: pruning secret version v2 completed`); }; - const findVersionsBySecretIdWithActors = async ( - secretId: string, - projectId: string, - { offset, limit, sort = [["createdAt", "desc"]] }: TFindOpt = {}, - tx?: Knex - ) => { + const findVersionsBySecretIdWithActors = async ({ + secretId, + projectId, + secretVersions, + findOpt = {}, + tx + }: { + secretId: string; + projectId: string; + secretVersions?: string[]; + findOpt?: TFindOpt; + tx?: Knex; + }) => { try { + const { offset, limit, sort = [["createdAt", "desc"]] } = findOpt; const query = (tx || db)(TableName.SecretVersionV2) .leftJoin(TableName.Users, `${TableName.Users}.id`, `${TableName.SecretVersionV2}.userActorId`) .leftJoin( @@ -202,10 +210,12 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { .where((qb) => { void qb.where(`${TableName.SecretVersionV2}.secretId`, secretId); void qb.where(`${TableName.ProjectMembership}.projectId`, projectId); + if (secretVersions?.length) void qb.whereIn(`${TableName.SecretVersionV2}.version`, secretVersions); }) .orWhere((qb) => { void qb.where(`${TableName.SecretVersionV2}.secretId`, secretId); void qb.whereNull(`${TableName.ProjectMembership}.projectId`); + if (secretVersions?.length) void qb.whereIn(`${TableName.SecretVersionV2}.version`, secretVersions); }) .select( selectAllTableCols(TableName.SecretVersionV2), diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 46ed7ef83..874ed8f90 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -2521,6 +2521,36 @@ export const secretServiceFactory = ({ }); }; + const getSecretVersionsV2ByIds = async ({ + actorId, + actor, + actorOrgId, + actorAuthMethod, + secretId, + secretVersions, + folderId + }: TGetSecretVersionsDTO & { + secretVersions: string[]; + folderId: string; + }) => { + const secretVersionV2 = await secretV2BridgeService + .getSecretVersionsByIds({ + actorId, + actor, + actorOrgId, + actorAuthMethod, + secretId, + folderId, + secretVersionNumbers: secretVersions + }) + .catch((err) => { + if ((err as Error).message === "BadRequest: Failed to find secret") { + return null; + } + }); + return secretVersionV2; + }; + const attachTags = async ({ secretName, tagSlugs, @@ -3309,6 +3339,7 @@ export const secretServiceFactory = ({ getSecretsRawByFolderMappings, getSecretAccessList, getSecretByIdRaw, - getAccessibleSecrets + getAccessibleSecrets, + getSecretVersionsV2ByIds }; }; diff --git a/frontend/src/components/navigation/SecretDashboardPathBreadcrumb.tsx b/frontend/src/components/navigation/SecretDashboardPathBreadcrumb.tsx index 584a3f6c7..31a626866 100644 --- a/frontend/src/components/navigation/SecretDashboardPathBreadcrumb.tsx +++ b/frontend/src/components/navigation/SecretDashboardPathBreadcrumb.tsx @@ -14,13 +14,15 @@ type Props = { selectedPathSegmentIndex: number; environmentSlug: string; projectId: string; + disableCopy?: boolean; }; export const SecretDashboardPathBreadcrumb = ({ secretPathSegments, selectedPathSegmentIndex, environmentSlug, - projectId + projectId, + disableCopy }: Props) => { const [, isCopying, setIsCopying] = useTimedReset({ initialState: false @@ -32,7 +34,7 @@ export const SecretDashboardPathBreadcrumb = ({ return (
- {isLastItem ? ( + {isLastItem && !disableCopy ? (
; diff --git a/frontend/src/hooks/api/folderCommits/index.tsx b/frontend/src/hooks/api/folderCommits/index.tsx new file mode 100644 index 000000000..ffdec812a --- /dev/null +++ b/frontend/src/hooks/api/folderCommits/index.tsx @@ -0,0 +1,2 @@ +export { useGetFolderCommitHistory, useGetFolderCommitsCount } from "./queries"; +export * from "./types"; diff --git a/frontend/src/hooks/api/folderCommits/queries.tsx b/frontend/src/hooks/api/folderCommits/queries.tsx new file mode 100644 index 000000000..2fabb7936 --- /dev/null +++ b/frontend/src/hooks/api/folderCommits/queries.tsx @@ -0,0 +1,247 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; + +import { apiRequest } from "@app/config/request"; + +import { CommitHistoryItem, CommitWithChanges, RollbackPreview } from "./types"; + +export const commitKeys = { + count: ({ + workspaceId, + environment, + directory + }: { + workspaceId: string; + environment: string; + directory?: string; + }) => [{ workspaceId, environment, directory }, "folder-commits-count"] as const, + + history: ({ + workspaceId, + environment, + directory + }: { + workspaceId: string; + environment: string; + directory?: string; + }) => [{ workspaceId, environment, directory }, "folder-commits"] as const, + + details: ({ workspaceId, commitId }: { workspaceId: string; commitId: string }) => + [{ workspaceId, commitId }, "commit-details"] as const, + + rollbackPreview: ({ + folderId, + commitId, + envId, + projectId, + deepRollback + }: { + folderId: string; + commitId: string; + envId: string; + projectId: string; + deepRollback: boolean; + }) => [{ folderId, commitId, envId, projectId, deepRollback }, "rollback-preview"] as const +}; + +const fetchFolderCommitsCount = async ({ + workspaceId, + environment, + directory +}: { + workspaceId: string; + environment: string; + directory?: string; +}) => { + const res = await apiRequest.get<{ count: number; folderId: string }>( + `/api/v1/pit/commits/count/${workspaceId}`, + { + params: { + environment, + path: directory + } + } + ); + return res.data; +}; + +const fetchFolderCommitHistory = async ( + workspaceId: string, + environment: string, + directory: string +): Promise => { + const res = await apiRequest.get(`/api/v1/pit/commits/${workspaceId}`, { + params: { + environment, + path: directory + } + }); + return res.data; +}; + +export const fetchCommitDetails = async (workspaceId: string, commitId: string) => { + const { data } = await apiRequest.get( + `/api/v1/pit/commits/${workspaceId}/${commitId}/changes` + ); + return data; +}; + +export const fetchRollbackPreview = async ( + folderId: string, + commitId: string, + envId: string, + projectId: string, + deepRollback: boolean, + secretPath: string +): Promise => { + const { data } = await apiRequest.get( + `/api/v1/pit/commits/${projectId}/${commitId}/compare`, + { + params: { + folderId, + envId, + deepRollback, + secretPath + } + } + ); + return data; +}; + +const fetchRollback = async ( + folderId: string, + commitId: string, + projectId: string, + deepRollback: boolean, + message?: string +) => { + const { data } = await apiRequest.post<{ success: boolean }>( + `/api/v1/pit/commits/${projectId}/${commitId}/rollback`, + { + folderId, + deepRollback, + message + } + ); + return data; +}; + +const fetchRevert = async (commitId: string, projectId: string) => { + const { data } = await apiRequest.post<{ success: boolean; message: string }>( + `/api/v1/pit/commits/${projectId}/${commitId}/revert` + ); + return data; +}; + +export const useCommitRevert = ({ + commitId, + projectId, + environment, + directory +}: { + commitId: string; + projectId: string; + environment: string; + directory: string; +}) => { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: () => fetchRevert(commitId, projectId), + onSuccess: () => { + queryClient.invalidateQueries({ + queryKey: [ + commitKeys.details({ workspaceId: projectId, commitId }), + commitKeys.history({ workspaceId: projectId, environment, directory }), + commitKeys.count({ workspaceId: projectId, environment, directory }) + ] + }); + } + }); +}; + +export const useCommitRollback = ({ + workspaceId, + commitId, + folderId, + deepRollback, + environment, + directory +}: { + workspaceId: string; + commitId: string; + folderId: string; + deepRollback: boolean; + environment: string; + directory: string; +}) => { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: (message: string) => + fetchRollback(folderId, commitId, workspaceId, deepRollback, message), + onSuccess: () => { + queryClient.invalidateQueries({ + queryKey: [ + commitKeys.details({ workspaceId, commitId }), + commitKeys.history({ workspaceId, environment, directory }), + commitKeys.count({ workspaceId, environment, directory }) + ] + }); + } + }); +}; + +export const useGetFolderCommitsCount = ({ + workspaceId, + environment, + directory, + isPaused +}: { + workspaceId: string; + environment: string; + directory: string; + isPaused?: boolean; +}) => + useQuery({ + enabled: Boolean(workspaceId && environment) && !isPaused, + queryKey: commitKeys.count({ workspaceId, environment, directory }), + queryFn: () => fetchFolderCommitsCount({ workspaceId, environment, directory }) + }); + +export const useGetFolderCommitHistory = ({ + workspaceId, + environment, + directory +}: { + workspaceId: string; + environment: string; + directory: string; +}) => { + return useQuery({ + queryKey: commitKeys.history({ workspaceId, environment, directory }), + queryFn: () => fetchFolderCommitHistory(workspaceId, environment, directory), + enabled: Boolean(workspaceId && environment) + }); +}; + +export const useGetCommitDetails = (workspaceId: string, commitId: string) => { + return useQuery({ + queryKey: commitKeys.details({ workspaceId, commitId }), + queryFn: () => fetchCommitDetails(workspaceId, commitId), + enabled: Boolean(workspaceId) && Boolean(commitId) + }); +}; + +export const useGetRollbackPreview = ( + folderId: string, + commitId: string, + envId: string, + projectId: string, + deepRollback: boolean, + secretPath: string +) => { + return useQuery({ + queryKey: commitKeys.rollbackPreview({ folderId, commitId, envId, projectId, deepRollback }), + queryFn: () => + fetchRollbackPreview(folderId, commitId, envId, projectId, deepRollback, secretPath), + enabled: Boolean(folderId) && Boolean(commitId) + }); +}; diff --git a/frontend/src/hooks/api/folderCommits/types.ts b/frontend/src/hooks/api/folderCommits/types.ts new file mode 100644 index 000000000..c70c62af1 --- /dev/null +++ b/frontend/src/hooks/api/folderCommits/types.ts @@ -0,0 +1,63 @@ +import { SecretVersions } from "../types"; + +export type CommitHistoryItem = { + id: string; + commitId: string; + actorMetadata: { + id: string; + name?: string; + }; + actorType: string; + message: string; + folderId: string; + envId: string; + createdAt: string; + updatedAt: string; +}; + +export type TFolderCommitChanges = { + id: string; + folderCommitId: string; + changeType: "add" | "delete"; + isUpdate: boolean; + secretVersionId: string | null; + folderVersionId: string | null; + createdAt: string; + updatedAt: string; + versions: SecretVersions[]; + secretKey?: string; + folderName?: string; + secretVersion?: string; + folderVersion?: string; +}; + +export type FolderReconstructedItem = { + type: string; + id: string; + versionId: string; + folderName?: string; + folderVersion?: number; + secretKey?: string; + secretVersion?: number; +}; + +export type CommitWithChanges = { + changes: CommitHistoryItem & { + changes: TFolderCommitChanges[]; + }; +}; + +export type RollbackChange = { + type: "folder" | "secret"; + id: string; + versionId: string; + changeType: "create" | "update" | "delete"; + commitId: string; +}; + +export type RollbackPreview = { + folderId: string; + folderName: string; + folderPath: string; + changes: RollbackChange[]; +}; diff --git a/frontend/src/hooks/api/secretFolders/queries.tsx b/frontend/src/hooks/api/secretFolders/queries.tsx index b49fbd6cc..9e12f329d 100644 --- a/frontend/src/hooks/api/secretFolders/queries.tsx +++ b/frontend/src/hooks/api/secretFolders/queries.tsx @@ -10,6 +10,7 @@ import { import { apiRequest } from "@app/config/request"; import { dashboardKeys } from "@app/hooks/api/dashboard/queries"; +import { commitKeys } from "../folderCommits/queries"; import { secretSnapshotKeys } from "../secretSnapshots/queries"; import { TCreateFolderDTO, @@ -166,6 +167,9 @@ export const useCreateFolder = () => { queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ workspaceId: projectId, environment, directory: path }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId: projectId, environment, directory: path }) + }); } }); }; @@ -200,6 +204,12 @@ export const useUpdateFolder = () => { queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ workspaceId: projectId, environment, directory: path }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId: projectId, environment, directory: path }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId: projectId, environment, directory: path }) + }); } }); }; @@ -234,6 +244,12 @@ export const useDeleteFolder = () => { queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ workspaceId: projectId, environment, directory: path }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId: projectId, environment, directory: path }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId: projectId, environment, directory: path }) + }); } }); }; @@ -279,6 +295,20 @@ export const useUpdateFolderBatch = () => { directory: folder.path }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ + workspaceId: projectId, + environment: folder.environment, + directory: folder.path + }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ + workspaceId: projectId, + environment: folder.environment, + directory: folder.path + }) + }); }); } }); diff --git a/frontend/src/hooks/api/secrets/mutations.tsx b/frontend/src/hooks/api/secrets/mutations.tsx index 3862d1f8d..82bf623e6 100644 --- a/frontend/src/hooks/api/secrets/mutations.tsx +++ b/frontend/src/hooks/api/secrets/mutations.tsx @@ -3,6 +3,7 @@ import { MutationOptions, useMutation, useQueryClient } from "@tanstack/react-qu import { apiRequest } from "@app/config/request"; import { dashboardKeys } from "@app/hooks/api/dashboard/queries"; +import { commitKeys } from "../folderCommits/queries"; import { secretApprovalRequestKeys } from "../secretApprovalRequest/queries"; import { secretSnapshotKeys } from "../secretSnapshots/queries"; import { secretKeys } from "./queries"; @@ -59,6 +60,12 @@ export const useCreateSecretV3 = ({ queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ environment, workspaceId, directory: secretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId, environment, directory: secretPath }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId, environment, directory: secretPath }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId }) }); }, ...options @@ -118,6 +125,12 @@ export const useUpdateSecretV3 = ({ queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ environment, workspaceId, directory: secretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId, environment, directory: secretPath }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId, environment, directory: secretPath }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId }) }); }, ...options @@ -164,6 +177,12 @@ export const useDeleteSecretV3 = ({ queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ environment, workspaceId, directory: secretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId, environment, directory: secretPath }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId, environment, directory: secretPath }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId }) }); }, ...options @@ -200,6 +219,12 @@ export const useCreateSecretBatch = ({ queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ environment, workspaceId, directory: secretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId, environment, directory: secretPath }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId, environment, directory: secretPath }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId }) }); }, ...options @@ -236,6 +261,12 @@ export const useUpdateSecretBatch = ({ queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ environment, workspaceId, directory: secretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId, environment, directory: secretPath }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId, environment, directory: secretPath }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId }) }); }, ...options @@ -274,6 +305,12 @@ export const useDeleteSecretBatch = ({ queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ environment, workspaceId, directory: secretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId, environment, directory: secretPath }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId, environment, directory: secretPath }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId }) }); }, ...options @@ -347,6 +384,20 @@ export const useMoveSecrets = ({ directory: sourceSecretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ + workspaceId: projectId, + environment: sourceEnvironment, + directory: sourceSecretPath + }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ + workspaceId: projectId, + environment: sourceEnvironment, + directory: sourceSecretPath + }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId: projectId }) }); diff --git a/frontend/src/pages/project/RoleDetailsBySlugPage/components/ProjectRoleModifySection.utils.tsx b/frontend/src/pages/project/RoleDetailsBySlugPage/components/ProjectRoleModifySection.utils.tsx index 3a1bdb1d2..bbc304401 100644 --- a/frontend/src/pages/project/RoleDetailsBySlugPage/components/ProjectRoleModifySection.utils.tsx +++ b/frontend/src/pages/project/RoleDetailsBySlugPage/components/ProjectRoleModifySection.utils.tsx @@ -11,6 +11,7 @@ import { } from "@app/context"; import { PermissionConditionOperators, + ProjectPermissionCommitsActions, ProjectPermissionDynamicSecretActions, ProjectPermissionGroupActions, ProjectPermissionIdentityActions, @@ -70,6 +71,11 @@ const SecretSyncPolicyActionSchema = z.object({ [ProjectPermissionSecretSyncActions.RemoveSecrets]: z.boolean().optional() }); +const CommitPolicyActionSchema = z.object({ + [ProjectPermissionCommitsActions.Read]: z.boolean().optional(), + [ProjectPermissionCommitsActions.PerformRollback]: z.boolean().optional() +}); + const SecretRotationPolicyActionSchema = z.object({ [ProjectPermissionSecretRotationActions.Read]: z.boolean().optional(), [ProjectPermissionSecretRotationActions.ReadGeneratedCredentials]: z.boolean().optional(), @@ -248,7 +254,8 @@ export const projectRoleFormSchema = z.object({ [ProjectPermissionSub.Kms]: GeneralPolicyActionSchema.array().default([]), [ProjectPermissionSub.Cmek]: CmekPolicyActionSchema.array().default([]), [ProjectPermissionSub.SecretSyncs]: SecretSyncPolicyActionSchema.array().default([]), - [ProjectPermissionSub.Kmip]: KmipPolicyActionSchema.array().default([]) + [ProjectPermissionSub.Kmip]: KmipPolicyActionSchema.array().default([]), + [ProjectPermissionSub.Commits]: CommitPolicyActionSchema.array().default([]) }) .partial() .optional() @@ -382,7 +389,8 @@ export const rolePermission2Form = (permissions: TProjectPermission[] = []) => { ProjectPermissionSub.SshCertificateTemplates, ProjectPermissionSub.SshCertificateAuthorities, ProjectPermissionSub.SshCertificates, - ProjectPermissionSub.SshHostGroups + ProjectPermissionSub.SshHostGroups, + ProjectPermissionSub.Commits ].includes(subject) ) { // from above statement we are sure it won't be undefined @@ -975,6 +983,13 @@ export const PROJECT_PERMISSION_OBJECT: TProjectPermissionObject = { { label: "Remove", value: "delete" } ] }, + [ProjectPermissionSub.Commits]: { + title: "Commits", + actions: [ + { label: "View", value: ProjectPermissionCommitsActions.Read }, + { label: "Perform Rollback", value: ProjectPermissionCommitsActions.PerformRollback } + ] + }, [ProjectPermissionSub.Tags]: { title: "Tags", actions: [ diff --git a/frontend/src/pages/secret-manager/CommitDetailsPage/CommitDetailsPage.tsx b/frontend/src/pages/secret-manager/CommitDetailsPage/CommitDetailsPage.tsx new file mode 100644 index 000000000..9a5a4f313 --- /dev/null +++ b/frontend/src/pages/secret-manager/CommitDetailsPage/CommitDetailsPage.tsx @@ -0,0 +1,81 @@ +import { Helmet } from "react-helmet"; +import { useTranslation } from "react-i18next"; +import { useNavigate, useParams, useSearch } from "@tanstack/react-router"; + +import { ROUTE_PATHS } from "@app/const/routes"; +import { useWorkspace } from "@app/context"; +import { ProjectType } from "@app/hooks/api/workspace/types"; + +import { CommitDetailsTab } from "./components/CommitDetailsTab"; + +export const CommitDetailsPage = () => { + const { t } = useTranslation(); + const envSlug = useParams({ + from: ROUTE_PATHS.SecretManager.CommitDetailsPage.id, + select: (el) => el.environment + }); + const selectedCommitId = useParams({ + from: ROUTE_PATHS.SecretManager.CommitDetailsPage.id, + select: (el) => el.commitId + }); + const folderId = useParams({ + from: ROUTE_PATHS.SecretManager.CommitDetailsPage.id, + select: (el) => el.folderId + }); + const { currentWorkspace } = useWorkspace(); + + const navigate = useNavigate(); + const routerQueryParams: { secretPath?: string } = useSearch({ + from: ROUTE_PATHS.SecretManager.CommitDetailsPage.id + }); + + const secretPath = (routerQueryParams.secretPath as string) || "/"; + + const handleGoBackToHistory = () => { + navigate({ + to: `/${ProjectType.SecretManager}/$projectId/commits/$environment/$folderId` as const, + params: { + projectId: currentWorkspace.id, + folderId, + environment: envSlug + }, + search: (query) => ({ + ...query, + secretPath + }) + }); + }; + + const handleGoToRollbackPreview = () => { + navigate({ + to: `/${ProjectType.SecretManager}/$projectId/commits/$environment/$folderId/$commitId/restore` as const, + params: { + projectId: currentWorkspace.id, + folderId, + environment: envSlug, + commitId: selectedCommitId + }, + search: (query) => ({ + ...query, + secretPath + }) + }); + }; + + return ( +
+ + {t("common.head-title", { title: "Commit History" })} + +
+ +
+
+ ); +}; diff --git a/frontend/src/pages/secret-manager/CommitDetailsPage/components/CommitDetailsTab/CommitDetailsTab.tsx b/frontend/src/pages/secret-manager/CommitDetailsPage/components/CommitDetailsTab/CommitDetailsTab.tsx new file mode 100644 index 000000000..5e275ed15 --- /dev/null +++ b/frontend/src/pages/secret-manager/CommitDetailsPage/components/CommitDetailsTab/CommitDetailsTab.tsx @@ -0,0 +1,327 @@ +import { useEffect, useState } from "react"; +import { faAngleDown } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { DropdownMenuItem } from "@radix-ui/react-dropdown-menu"; +import { useSearch } from "@tanstack/react-router"; + +import { createNotification } from "@app/components/notifications"; +import { + DeleteActionModal, + DropdownMenu, + DropdownMenuContent, + DropdownMenuTrigger, + IconButton, + Spinner +} from "@app/components/v2"; +import { ROUTE_PATHS } from "@app/const/routes"; +import { usePopUp } from "@app/hooks"; +import { CommitWithChanges } from "@app/hooks/api/folderCommits"; +import { useCommitRevert, useGetCommitDetails } from "@app/hooks/api/folderCommits/queries"; + +import { SecretVersionDiffView } from "../SecretVersionDiffView"; +import { MergedItem } from "./types"; + +const formatDisplayDate = (dateString: string): string => { + try { + const date = new Date(dateString); + const options: Intl.DateTimeFormatOptions = { + year: "numeric", + month: "short", + day: "numeric", + hour: "numeric", + minute: "numeric", + hour12: true + }; + return new Intl.DateTimeFormat("en-US", options).format(date); + } catch { + return dateString; + } +}; + +export const CommitDetailsTab = ({ + selectedCommitId, + workspaceId, + envSlug, + goBackToHistory, + goToRollbackPreview +}: { + selectedCommitId: string; + workspaceId: string; + envSlug: string; + goBackToHistory: () => void; + goToRollbackPreview: () => void; +}): JSX.Element => { + // State for tracking collapsed items (empty by default means all are expanded) + const [collapsedItems, setCollapsedItems] = useState>({}); + + const { handlePopUpToggle, popUp, handlePopUpOpen, handlePopUpClose } = usePopUp([ + "revertChanges" + ] as const); + + const { data: commitDetails, isLoading } = useGetCommitDetails(workspaceId, selectedCommitId); + + const routerQueryParams: { secretPath?: string } = useSearch({ + from: ROUTE_PATHS.SecretManager.CommitDetailsPage.id + }); + const secretPath = (routerQueryParams.secretPath as string) || "/"; + + const { mutateAsync: revert } = useCommitRevert({ + commitId: selectedCommitId, + projectId: workspaceId, + environment: envSlug, + directory: secretPath + }); + + useEffect(() => { + setCollapsedItems({}); + }, [selectedCommitId]); + + const toggleItemCollapsed = (itemId: string): void => { + setCollapsedItems((prev) => ({ ...prev, [itemId]: !prev[itemId] })); + }; + + const handleRevertChanges = async (): Promise => { + const response = await revert(); + if (!response.success) { + createNotification({ + type: "error", + text: response.message + }); + return; + } + createNotification({ + type: "success", + text: response.message + }); + + handlePopUpClose("revertChanges"); + + goBackToHistory(); + }; + + // If no commit is selected or data is loading, show appropriate message + if (!selectedCommitId) { + return ( +
+

Select a commit to view details

+
+ ); + } + + if (isLoading) { + return ( +
+ +
+ ); + } + + if (!commitDetails) { + return ( +
+

No details found for this commit

+
+ ); + } + + // Parse the commit details if it's a string + let parsedCommitDetails: CommitWithChanges; + try { + parsedCommitDetails = + typeof commitDetails === "string" ? JSON.parse(commitDetails) : commitDetails; + } catch (error) { + console.error("Failed to parse commit details:", error); + return ( +
+

Error parsing commit details

+
+ ); + } + + // Get all changes from the commit + const commitChanges = parsedCommitDetails.changes?.changes || []; + + // Separate changes by type + const addedChanges = commitChanges.filter((c) => c.changeType === "add" && !c.isUpdate); + const updatedChanges = commitChanges.filter((c) => c.changeType === "add" && c.isUpdate); + const deletedChanges = commitChanges.filter((c) => c.changeType === "delete"); + + // Create merged item list from changes only + const changedItems: MergedItem[] = []; + + // Add items from added changes + addedChanges.forEach((change) => { + changedItems.push({ + id: change.id, + type: change.secretVersionId || change.secretKey ? "secret" : "folder", + versionId: change.secretVersionId || change.id, + folderName: change.folderName, + folderVersion: change.folderVersion, + secretKey: change.secretKey, + secretVersion: change.secretVersion, + isAdded: true, + versions: change.versions, + changeId: change.id + }); + }); + + // Add items from updated changes + updatedChanges.forEach((change) => { + changedItems.push({ + id: change.id, + type: change.secretVersionId || change.secretKey ? "secret" : "folder", + versionId: change.secretVersionId || change.id, + folderName: change.folderName, + folderVersion: change.folderVersion, + secretKey: change.secretKey, + secretVersion: change.secretVersion, + isUpdated: true, + versions: change.versions, + changeId: change.id + }); + }); + + // Add deleted items + deletedChanges.forEach((change) => { + changedItems.push({ + id: change.id, + type: change.secretVersionId || change.secretKey ? "secret" : "folder", + secretKey: change.secretKey, + folderName: change.folderName, + secretVersion: change.secretVersion, + folderVersion: change.folderVersion, + isDeleted: true, + versions: change.versions, + changeId: change.id + }); + }); + + // Sort items: deleted first, then folders, then secrets, all alphabetically + const sortedChangedItems = [...changedItems].sort((a, b) => { + // First sort deleted items to the top + if (a.isDeleted !== b.isDeleted) { + return a.isDeleted ? -1 : 1; + } + + // Then sort by type (folders before secrets) + if (a.type !== b.type) { + return a.type === "folder" ? -1 : 1; + } + + // Finally sort alphabetically by name + const aName = a.type === "folder" ? a.folderName || "" : a.secretKey || ""; + const bName = b.type === "folder" ? b.folderName || "" : b.secretKey || ""; + return aName.localeCompare(bName); + }); + + // Render an item from the merged list + const renderMergedItem = (item: MergedItem): JSX.Element => { + return ( +
+ toggleItemCollapsed(id)} + /> +
+ ); + }; + + // Format actor display + const actorDisplay = + parsedCommitDetails.changes?.actorMetadata?.name || + parsedCommitDetails.changes?.actorType || + "Unknown"; + + return ( +
+
+
+
+
+
+

+ {parsedCommitDetails.changes?.message || "No message"} +

+
+
+
+

+ Commited by + {actorDisplay} + on + + {formatDisplayDate( + parsedCommitDetails.changes?.createdAt || new Date().toISOString() + )} + +

+
+
+
+ + + +

Restore Options

+ + + + + goToRollbackPreview()} + > +
+ Restore entire folder +
+ + + handlePopUpOpen("revertChanges")} + > +
+ Revert commit changes +
+ + + +
+
+ +
+
+
+ {sortedChangedItems.length > 0 ? ( + sortedChangedItems.map((item) => renderMergedItem(item)) + ) : ( +
+

No changed items found

+
+ )} +
+
+
+
+ + handlePopUpToggle("revertChanges", isOpen)} + onDeleteApproved={handleRevertChanges} + buttonText="Revert Changes" + /> +
+ ); +}; diff --git a/frontend/src/pages/secret-manager/CommitDetailsPage/components/CommitDetailsTab/index.ts b/frontend/src/pages/secret-manager/CommitDetailsPage/components/CommitDetailsTab/index.ts new file mode 100644 index 000000000..490d67f1f --- /dev/null +++ b/frontend/src/pages/secret-manager/CommitDetailsPage/components/CommitDetailsTab/index.ts @@ -0,0 +1 @@ +export { CommitDetailsTab } from "./CommitDetailsTab"; diff --git a/frontend/src/pages/secret-manager/CommitDetailsPage/components/CommitDetailsTab/types.ts b/frontend/src/pages/secret-manager/CommitDetailsPage/components/CommitDetailsTab/types.ts new file mode 100644 index 000000000..f3c914d8f --- /dev/null +++ b/frontend/src/pages/secret-manager/CommitDetailsPage/components/CommitDetailsTab/types.ts @@ -0,0 +1,14 @@ +export interface MergedItem { + id: string; + type: "secret" | "folder"; + versionId?: string; + folderName?: string; + folderVersion?: string; + secretKey?: string; + secretVersion?: string; + isAdded?: boolean; + isUpdated?: boolean; + isDeleted?: boolean; + versions?: any[]; + changeId: string; +} diff --git a/frontend/src/pages/secret-manager/CommitDetailsPage/components/RollbackPreviewTab/RollbackPreviewTab.tsx b/frontend/src/pages/secret-manager/CommitDetailsPage/components/RollbackPreviewTab/RollbackPreviewTab.tsx new file mode 100644 index 000000000..0778b4f25 --- /dev/null +++ b/frontend/src/pages/secret-manager/CommitDetailsPage/components/RollbackPreviewTab/RollbackPreviewTab.tsx @@ -0,0 +1,376 @@ +/* eslint-disable jsx-a11y/no-static-element-interactions */ +/* eslint-disable jsx-a11y/click-events-have-key-events */ +import { useEffect, useState } from "react"; +import { faFolder, faInfoCircle } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { useNavigate, useParams, useSearch } from "@tanstack/react-router"; + +import { createNotification } from "@app/components/notifications"; +import { + Button, + DeleteActionModal, + Input, + PageHeader, + Spinner, + Switch, + Tooltip +} from "@app/components/v2"; +import { ROUTE_PATHS } from "@app/const/routes"; +import { useWorkspace } from "@app/context"; +import { usePopUp } from "@app/hooks"; +import { useCommitRollback, useGetRollbackPreview } from "@app/hooks/api/folderCommits/queries"; +import { ProjectType } from "@app/hooks/api/workspace/types"; + +import { SecretVersionDiffView } from "../SecretVersionDiffView"; + +interface Version { + // Common fields + id?: string; + version: number; + createdAt?: string; + updatedAt?: string; + + // Secret-specific fields + secretKey?: string; + secretValue?: string; + secretComment?: string; + skipMultilineEncoding?: boolean; + secretReminderRepeatDays?: number | null; + secretReminderNote?: string | null; + secretReminderRecipients?: string[]; + tags?: string[]; + metadata?: Record; + + // Folder-specific fields + name?: string; + envId?: string; + folderId?: string; + + [key: string]: any; +} + +interface RollbackChange { + type: "secret" | "folder"; + id: string; + versionId: string; + fromVersion?: number; + changeType: "create" | "update" | "delete"; + commitId: string; + secretKey?: string; + secretVersion?: number; + folderName?: string; + folderVersion?: number; + versions?: Version[]; + createdAt?: string; +} + +interface FolderChanges { + folderId: string; + folderName: string; + folderPath: string; + changes: RollbackChange[]; +} + +export const RollbackPreviewTab = (): JSX.Element => { + const [deepRollback, setDeepRollback] = useState(false); + const [message, setMessage] = useState(""); + const [selectedFolderId, setSelectedFolderId] = useState(null); + const { currentWorkspace } = useWorkspace(); + const envSlug = useParams({ + from: ROUTE_PATHS.SecretManager.RollbackPreviewPage.id, + select: (el) => el.environment + }); + const selectedCommitId = useParams({ + from: ROUTE_PATHS.SecretManager.RollbackPreviewPage.id, + select: (el) => el.commitId + }); + const folderId = useParams({ + from: ROUTE_PATHS.SecretManager.RollbackPreviewPage.id, + select: (el) => el.folderId + }); + + const navigate = useNavigate(); + const routerQueryParams = useSearch({ + from: ROUTE_PATHS.SecretManager.RollbackPreviewPage.id + }); + + const secretPath = (routerQueryParams.secretPath as string) || "/"; + + const goBackToHistory = () => { + navigate({ + to: `/${ProjectType.SecretManager}/$projectId/commits/$environment/$folderId` as const, + params: { + projectId: currentWorkspace.id, + folderId, + environment: envSlug + }, + search: (query) => ({ + ...query, + secretPath + }) + }); + }; + + const { handlePopUpToggle, popUp, handlePopUpOpen, handlePopUpClose } = usePopUp([ + "rollbackConfirm" + ] as const); + + const { mutateAsync: rollback } = useCommitRollback({ + workspaceId: currentWorkspace.id, + commitId: selectedCommitId, + folderId, + deepRollback, + environment: envSlug, + directory: secretPath + }); + + const { data: rollbackChangesNested, isLoading } = useGetRollbackPreview( + folderId, + selectedCommitId, + currentWorkspace.environments.find((e) => e.slug === envSlug)?.id || "", + currentWorkspace.id, + deepRollback, + secretPath + ); + + const handleRollback = async (): Promise => { + try { + await rollback(message); + + createNotification({ + type: "success", + text: "Rollback completed successfully" + }); + + handlePopUpClose("rollbackConfirm"); + goBackToHistory(); + } catch (error) { + createNotification({ + type: "error", + text: error instanceof Error ? error.message : "Failed to rollback changes" + }); + } + }; + + const folderChanges: FolderChanges[] = rollbackChangesNested || []; + + const currentFolderChanges: FolderChanges = folderChanges.find( + (folder) => folder.folderId === folderId + ) || { + folderId: folderId || "", + folderName: "Current Folder", + folderPath: secretPath, + changes: [] + }; + const nestedFolderChanges: FolderChanges[] = folderChanges.filter( + (folder) => folder.folderId !== folderId + ); + + useEffect(() => { + // Select the current folder by default + if (folderChanges.length > 0) { + setSelectedFolderId(currentFolderChanges.folderId); + } + }, [folderChanges, currentFolderChanges.folderId]); + + if (!selectedCommitId) { + return ( +
+

Select a commit to view rollback preview

+
+ ); + } + + if (isLoading) { + return ( +
+ +
+ ); + } + + const renderSidebar = (): JSX.Element => { + return ( +
+
setSelectedFolderId(currentFolderChanges.folderId)} + > +
+
+ + + {currentFolderChanges.folderPath || currentFolderChanges.folderName} + +
+
+
+ + {deepRollback && nestedFolderChanges.length > 0 && ( + <> +
+ Affected Child Folders +
+ {nestedFolderChanges.map((folder) => ( +
setSelectedFolderId(folder.folderId)} + > +
+
+ + + {folder.folderPath || folder.folderName} + +
+
+
+ ))} + + )} +
+ ); + }; + + const getSelectedFolderChanges = (): RollbackChange[] => { + if (!selectedFolderId) return []; + + const folder = folderChanges.find((f) => f.folderId === selectedFolderId); + return folder?.changes || []; + }; + + const renderMainContent = (): JSX.Element => { + const selectedFolderChanges = getSelectedFolderChanges(); + const selectedFolder = folderChanges.find((f) => f.folderId === selectedFolderId); + + if (!selectedFolder || selectedFolderChanges.length === 0) { + return ( +
+

No changes in selected folder

+
+ ); + } + + return ( +
+
+ {selectedFolderChanges.map((change) => ( +
+ +
+ ))} +
+
+ ); + }; + + return ( +
+
+
+
+ + + {folderChanges.length > 0 ? ( +
+ {renderSidebar()} + {renderMainContent()} +
+ ) : ( +
+

No changes will be applied with this restore

+
+ )} + +
+
+
+ + + + + Recursively Restore Child Folders + +
+
+
+ +
+
+
+ setMessage(e.target.value)} + className="w-full border-mineshaft-500 bg-mineshaft-700 text-sm" + /> +
+
+ +
+
+
+
+ + handlePopUpToggle("rollbackConfirm", isOpen)} + onDeleteApproved={handleRollback} + buttonText="Restore" + /> +
{" "} +
{" "} +
+ ); +}; diff --git a/frontend/src/pages/secret-manager/CommitDetailsPage/components/RollbackPreviewTab/index.ts b/frontend/src/pages/secret-manager/CommitDetailsPage/components/RollbackPreviewTab/index.ts new file mode 100644 index 000000000..9c840b949 --- /dev/null +++ b/frontend/src/pages/secret-manager/CommitDetailsPage/components/RollbackPreviewTab/index.ts @@ -0,0 +1 @@ +export { RollbackPreviewTab } from "./RollbackPreviewTab"; diff --git a/frontend/src/pages/secret-manager/CommitDetailsPage/components/RollbackPreviewTab/route.tsx b/frontend/src/pages/secret-manager/CommitDetailsPage/components/RollbackPreviewTab/route.tsx new file mode 100644 index 000000000..34b0e59bd --- /dev/null +++ b/frontend/src/pages/secret-manager/CommitDetailsPage/components/RollbackPreviewTab/route.tsx @@ -0,0 +1,91 @@ +import { createFileRoute, linkOptions, stripSearchParams } from "@tanstack/react-router"; +import { zodValidator } from "@tanstack/zod-adapter"; +import { z } from "zod"; + +import { SecretDashboardPathBreadcrumb } from "@app/components/navigation/SecretDashboardPathBreadcrumb"; +import { BreadcrumbTypes } from "@app/components/v2"; + +import { RollbackPreviewTab } from "./RollbackPreviewTab"; + +const RollbackPreviewTabQueryParamsSchema = z.object({ + secretPath: z.string().catch("/") +}); + +export const Route = createFileRoute( + "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/$commitId/restore" +)({ + component: RollbackPreviewTab, + validateSearch: zodValidator(RollbackPreviewTabQueryParamsSchema), + search: { + middlewares: [stripSearchParams({ secretPath: "/" })] + }, + beforeLoad: ({ context, params, search }) => { + const secretPathSegments = search.secretPath.split("/").filter(Boolean); + + return { + breadcrumbs: [ + ...context.breadcrumbs, + { + type: BreadcrumbTypes.Dropdown, + label: + context.project.environments.find((el) => el.slug === params.environment)?.name || "", + dropdownTitle: "Environments", + links: context.project.environments.map((el) => ({ + label: el.name, + link: linkOptions({ + to: "/secret-manager/$projectId/secrets/$envSlug", + params: { + projectId: params.projectId, + envSlug: el.slug + } + }) + })) + }, + ...secretPathSegments.map((_, index) => ({ + type: BreadcrumbTypes.Component, + component: () => ( + + ) + })), + { + label: "Commits", + link: linkOptions({ + to: "/secret-manager/$projectId/commits/$environment/$folderId", + params: { + projectId: params.projectId, + environment: params.environment, + folderId: params.folderId + }, + search: { + secretPath: search.secretPath + } + }) + }, + { + label: params.commitId, + link: linkOptions({ + to: "/secret-manager/$projectId/commits/$environment/$folderId/$commitId", + params: { + projectId: params.projectId, + environment: params.environment, + folderId: params.folderId, + commitId: params.commitId + }, + search: { + secretPath: search.secretPath + } + }) + }, + { + label: "Restore" + } + ] + }; + } +}); diff --git a/frontend/src/pages/secret-manager/CommitDetailsPage/components/SecretVersionDiffView/SecretVersionDiffView.tsx b/frontend/src/pages/secret-manager/CommitDetailsPage/components/SecretVersionDiffView/SecretVersionDiffView.tsx new file mode 100644 index 000000000..1548cca90 --- /dev/null +++ b/frontend/src/pages/secret-manager/CommitDetailsPage/components/SecretVersionDiffView/SecretVersionDiffView.tsx @@ -0,0 +1,585 @@ +/* eslint-disable react/prop-types */ +import { useRef, useState } from "react"; +import { faChevronDown, faChevronUp } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; + +export interface Version { + id?: string; + version: number; + // Secret-specific fields + secretKey?: string; + secretValue?: string; + secretComment?: string; + skipMultilineEncoding?: boolean; + // Folder-specific fields + name?: string; + // Allow other properties + [key: string]: any; +} + +export interface DiffViewItem { + type: "secret" | "folder"; + isAdded?: boolean; + isDeleted?: boolean; + isUpdated?: boolean; + versions?: Version[]; + isRollback?: boolean; + id: string; + secretKey?: string; + folderName?: string; +} + +interface SecretVersionDiffViewProps { + item: DiffViewItem; + isCollapsed?: boolean; + onToggleCollapse?: (id: string) => void; + showHeader?: boolean; + customHeader?: JSX.Element; +} + +const highlightChangedFields = ( + json: any, + changedFields: Set, + isOldVersion: boolean +): JSX.Element => { + const lines = JSON.stringify(json, null, 2).split("\n"); + return ( +
+ {lines.map((line, idx) => { + const fieldMatch = line.match(/"([^"]+)":/); + if (fieldMatch && changedFields.has(fieldMatch[1])) { + // Check for different value types and highlight them + + // 1. String values: "field": "value" + const stringMatch = line.match(/: "([^"]*)"(,?)$/); + if (stringMatch) { + const beforeValue = line.substring(0, line.indexOf(': "')); + const value = `"${stringMatch[1]}"`; + const afterValue = stringMatch[2] || ""; + + return ( +
+
{isOldVersion ? "-" : "+"}
+
+ {beforeValue}:{" "} + + {value} + + {afterValue} +
+
+ ); + } + + // 2. Number values: "field": 123 + const numberMatch = line.match(/: (-?\d+(?:\.\d+)?)(,?)$/); + if (numberMatch) { + const beforeValue = line.substring(0, line.indexOf(": ") + 2); + const value = numberMatch[1]; + const afterValue = numberMatch[2] || ""; + + return ( +
+
{isOldVersion ? "-" : "+"}
+
+ {beforeValue.substring(0, beforeValue.length - 2)}:{" "} + + {value} + + {afterValue} +
+
+ ); + } + + // 3. Null values: "field": null + const nullMatch = line.match(/: (null)(,?)$/); + if (nullMatch) { + const beforeValue = line.substring(0, line.indexOf(": ") + 2); + const value = nullMatch[1]; + const afterValue = nullMatch[2] || ""; + + return ( +
+
{isOldVersion ? "-" : "+"}
+
+ {beforeValue.substring(0, beforeValue.length - 2)}:{" "} + + {value} + + {afterValue} +
+
+ ); + } + + // 4. Boolean values: "field": true|false + const booleanMatch = line.match(/: (true|false)(,?)$/); + if (booleanMatch) { + const beforeValue = line.substring(0, line.indexOf(": ") + 2); + const value = booleanMatch[1]; + const afterValue = booleanMatch[2] || ""; + + return ( +
+
{isOldVersion ? "-" : "+"}
+
+ {beforeValue.substring(0, beforeValue.length - 2)}:{" "} + + {value} + + {afterValue} +
+
+ ); + } + + // 5. Array values: Handle the first line of an array - "field": [ + const arrayStartMatch = line.match(/: \[(,?)$/); + if (arrayStartMatch) { + // This is the start of an array, highlight the whole line + return ( +
+
{isOldVersion ? "-" : "+"}
+
{line}
+
+ ); + } + + // 6. Empty Array values: "field": [] + const emptyArrayMatch = line.match(/: \[\](,?)$/); + if (emptyArrayMatch) { + const beforeValue = line.substring(0, line.indexOf(": ") + 2); + const value = "[]"; + const afterValue = emptyArrayMatch[1] || ""; + + return ( +
+
{isOldVersion ? "-" : "+"}
+
+ {beforeValue.substring(0, beforeValue.length - 2)}:{" "} + + {value} + + {afterValue} +
+
+ ); + } + + // Check if this is part of an array or object that belongs to a changed field + // This handles array items, closing brackets, and other complex structure contents + const belongsToChangedField = () => { + // If we're inside an array or object of a changed field, highlight it + let openBrackets = 0; + let openBraces = 0; + let currentFieldName = null; + + for (let i = idx - 1; i >= 0; i -= 1) { + const prevLine = lines[i]; + + // Count brackets and braces to track nesting + openBrackets += (prevLine.match(/\[/g) || []).length; + openBrackets -= (prevLine.match(/\]/g) || []).length; + openBraces += (prevLine.match(/{/g) || []).length; + openBraces -= (prevLine.match(/}/g) || []).length; + + // If we find a field and we're still inside its value, check if it's a changed field + const fieldNameMatch = prevLine.match(/"([^"]+)":/); + if (fieldNameMatch && (openBrackets > 0 || openBraces > 0)) { + [, currentFieldName] = fieldNameMatch; + return changedFields.has(currentFieldName); + } + + // If we've reached the root level, stop looking + if (openBrackets <= 0 && openBraces <= 0) { + return false; + } + } + + return false; + }; + + // Regular line with changed field or part of a changed complex structure + if (belongsToChangedField()) { + return ( +
+
{isOldVersion ? "-" : "+"}
+
{line}
+
+ ); + } + + // Simple fallback for any other cases of changed fields + return ( +
+
{isOldVersion ? "-" : "+"}
+
{line}
+
+ ); + } + + // Unchanged lines + return ( +
+
+
{line}
+
+ ); + })} +
+ ); +}; + +// Helper: Format added JSON with + in a separate column +const formatAddedJson = (json: any): JSX.Element => { + const lines = JSON.stringify(json, null, 2).split("\n"); + return ( +
+ {lines.map((line, idx) => ( +
+
+
+
{line}
+
+ ))} +
+ ); +}; + +// Helper: Format deleted JSON with - in a separate column +const formatDeletedJson = (json: any): JSX.Element => { + const lines = JSON.stringify(json, null, 2).split("\n"); + return ( +
+ {lines.map((line, idx) => ( +
+
-
+
{line}
+
+ ))} +
+ ); +}; + +// Helper: Get differences between secret versions +export const getVersionDifferences = (versions: Version[]) => { + if (!versions || versions.length === 0) return []; + + // Sort versions by version number (descending) + const sortedVersions = [...versions].sort((a, b) => b.version - a.version); + const newVersion = sortedVersions[0]; + + // Fields to process + const fieldsToProcess = [ + "secretKey", + "secretValue", + "secretComment", + "skipMultilineEncoding", + "secretReminderRepeatDays", + "secretReminderNote", + "metadata", + "tags", + "secretReminderRecipients", + "name" + ]; + + // If only one version exists + if (sortedVersions.length === 1) { + return fieldsToProcess.reduce( + (differences, field) => { + if (newVersion[field] !== undefined && newVersion[field] !== null) { + let newVal = newVersion[field]; + + if (field === "tags" && Array.isArray(newVersion[field])) { + if (newVersion[field].length > 0 && typeof newVersion[field][0] === "object") { + newVal = newVersion[field].map((tag) => tag.name).join(", "); + } else if (Array.isArray(newVersion[field])) { + newVal = newVersion[field].join(", "); + } + } + + differences.push({ + field, + oldValue: null, + newValue: newVal + }); + } + return differences; + }, + [] as { field: string; oldValue: any; newValue: any }[] + ); + } + + // Otherwise, compare the two versions + const oldVersion = sortedVersions[1]; + + return fieldsToProcess.reduce( + (differences, field) => { + if (JSON.stringify(oldVersion[field]) !== JSON.stringify(newVersion[field])) { + let oldVal = oldVersion[field]; + let newVal = newVersion[field]; + + if (field === "tags" && Array.isArray(oldVersion[field])) { + if (oldVersion[field].length > 0 && typeof oldVersion[field][0] === "object") { + oldVal = oldVersion[field].map((tag) => tag.name).join(", "); + } else if (Array.isArray(oldVersion[field])) { + oldVal = oldVersion[field].join(", "); + } + } + + if (field === "tags" && Array.isArray(newVersion[field])) { + if (newVersion[field].length > 0 && typeof newVersion[field][0] === "object") { + newVal = newVersion[field].map((tag) => tag.name).join(", "); + } else if (Array.isArray(newVersion[field])) { + newVal = newVersion[field].join(", "); + } + } + + differences.push({ + field, + oldValue: oldVal, + newValue: newVal + }); + } + return differences; + }, + [] as { field: string; oldValue: any; newValue: any }[] + ); +}; + +// Helper: Get differences between folder versions +export const getFolderDifferences = (versions: Version[]) => { + if (!versions || versions.length === 0) return []; + + const sortedVersions = [...versions].sort((a, b) => b.version - a.version); + const newVersion = sortedVersions[0]; + + if (sortedVersions.length === 1) { + return [ + { + field: "folderName", + oldValue: null, + newValue: newVersion.name + } + ]; + } + + const oldVersion = sortedVersions[1]; + return [ + { + field: "folderName", + oldValue: oldVersion.name, + newValue: newVersion.name + } + ]; +}; + +export const SecretVersionDiffView: React.FC = ({ + item, + isCollapsed = false, + onToggleCollapse, + showHeader = true, + customHeader +}) => { + const oldContainerRef = useRef(null); + const newContainerRef = useRef(null); + const [isScrollingSynced, setIsScrollingSynced] = useState(false); + const [internalCollapsed, setInternalCollapsed] = useState(isCollapsed); + + const collapsed = onToggleCollapse ? isCollapsed : internalCollapsed; + + const handleToggle = () => { + if (onToggleCollapse && item.id) { + onToggleCollapse(item.id); + } else { + setInternalCollapsed(!internalCollapsed); + } + }; + + if (!item.versions) { + return
No details available
; + } + + const differences = getVersionDifferences(item.versions); + + if (differences.length === 0) { + return
No details available
; + } + + const changedFields = new Set(); + differences.forEach((diff) => { + if (JSON.stringify(diff.oldValue) !== JSON.stringify(diff.newValue)) { + changedFields.add(diff.field); + } + }); + + const handleScroll = (container: "old" | "new") => { + if (isScrollingSynced) return; + setIsScrollingSynced(true); + + if (container === "old" && oldContainerRef.current && newContainerRef.current) { + newContainerRef.current.scrollTop = oldContainerRef.current.scrollTop; + } else if (container === "new" && oldContainerRef.current && newContainerRef.current) { + oldContainerRef.current.scrollTop = newContainerRef.current.scrollTop; + } + + setTimeout(() => setIsScrollingSynced(false), 50); + }; + + const sortedVersions = [...item.versions].sort((a, b) => b.version - a.version); + + let oldVersion = null; + let newVersion = null; + let oldVersionContent = null; + let newVersionContent = null; + + if (item.isUpdated) { + if (item.isRollback) { + [oldVersion, newVersion] = sortedVersions; + } else { + [newVersion, oldVersion] = sortedVersions; + } + oldVersionContent = highlightChangedFields(oldVersion, changedFields, true); + newVersionContent = highlightChangedFields(newVersion, changedFields, false); + } else if (item.isAdded) { + [newVersion] = sortedVersions; + oldVersionContent =
; + newVersionContent = formatAddedJson(newVersion); + } else if (item.isDeleted) { + [oldVersion] = sortedVersions; + oldVersionContent = formatDeletedJson(oldVersion); + newVersionContent =
; + } + + const renderHeader = () => { + if (customHeader) { + return customHeader; + } + + const isSecret = item.type === "secret"; + const key = isSecret ? item.secretKey || "" : item.folderName || ""; + let textStyle = "text-white"; + let changeBadge = null; + + if (item.isDeleted) { + textStyle = "line-through text-red-300"; + changeBadge = ( + + {isSecret ? "Secret" : "Folder"} Deleted + + ); + } else if (item.isAdded) { + changeBadge = ( + + {isSecret ? "Secret" : "Folder"} Added + + ); + } else if (item.isUpdated) { + changeBadge = ( + + {isSecret ? "Secret" : "Folder"} Updated + + ); + } + + return ( +
{ + if (e.key === "Enter" || e.key === " ") { + handleToggle(); + e.preventDefault(); + } + }} + role="button" + tabIndex={0} + aria-expanded={!collapsed} + > +
+ {key} + {changeBadge} +
+ +
+ ); + }; + + return ( +
+ {showHeader && renderHeader()} + + {!collapsed && ( +
+
+
handleScroll("old")} + > + {oldVersionContent} +
+ +
handleScroll("new")} + > + {newVersionContent} +
+
+
+ )} +
+ ); +}; diff --git a/frontend/src/pages/secret-manager/CommitDetailsPage/components/SecretVersionDiffView/index.ts b/frontend/src/pages/secret-manager/CommitDetailsPage/components/SecretVersionDiffView/index.ts new file mode 100644 index 000000000..7c1cfdd79 --- /dev/null +++ b/frontend/src/pages/secret-manager/CommitDetailsPage/components/SecretVersionDiffView/index.ts @@ -0,0 +1 @@ +export { SecretVersionDiffView } from "./SecretVersionDiffView"; diff --git a/frontend/src/pages/secret-manager/CommitDetailsPage/index.tsx b/frontend/src/pages/secret-manager/CommitDetailsPage/index.tsx new file mode 100644 index 000000000..0623db621 --- /dev/null +++ b/frontend/src/pages/secret-manager/CommitDetailsPage/index.tsx @@ -0,0 +1 @@ +export { CommitDetailsPage } from "./CommitDetailsPage"; diff --git a/frontend/src/pages/secret-manager/CommitDetailsPage/route.tsx b/frontend/src/pages/secret-manager/CommitDetailsPage/route.tsx new file mode 100644 index 000000000..8cf046d2e --- /dev/null +++ b/frontend/src/pages/secret-manager/CommitDetailsPage/route.tsx @@ -0,0 +1,76 @@ +import { createFileRoute, linkOptions, stripSearchParams } from "@tanstack/react-router"; +import { zodValidator } from "@tanstack/zod-adapter"; +import { z } from "zod"; + +import { SecretDashboardPathBreadcrumb } from "@app/components/navigation/SecretDashboardPathBreadcrumb"; +import { BreadcrumbTypes } from "@app/components/v2"; + +import { CommitDetailsPage } from "./CommitDetailsPage"; + +const CommitDetailsPageQueryParamsSchema = z.object({ + secretPath: z.string().catch("/") +}); + +export const Route = createFileRoute( + "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/$commitId/" +)({ + component: CommitDetailsPage, + validateSearch: zodValidator(CommitDetailsPageQueryParamsSchema), + search: { + middlewares: [stripSearchParams({ secretPath: "/" })] + }, + beforeLoad: ({ context, params, search }) => { + const secretPathSegments = search.secretPath.split("/").filter(Boolean); + + return { + breadcrumbs: [ + ...context.breadcrumbs, + { + type: BreadcrumbTypes.Dropdown, + label: + context.project.environments.find((el) => el.slug === params.environment)?.name || "", + dropdownTitle: "Environments", + links: context.project.environments.map((el) => ({ + label: el.name, + link: linkOptions({ + to: "/secret-manager/$projectId/secrets/$envSlug", + params: { + projectId: params.projectId, + envSlug: el.slug + } + }) + })) + }, + ...secretPathSegments.map((_, index) => ({ + type: BreadcrumbTypes.Component, + component: () => ( + + ) + })), + { + label: "Commits", + link: linkOptions({ + to: "/secret-manager/$projectId/commits/$environment/$folderId", + params: { + projectId: params.projectId, + environment: params.environment, + folderId: params.folderId + }, + search: { + secretPath: search.secretPath + } + }) + }, + { + label: params.commitId + } + ] + }; + } +}); diff --git a/frontend/src/pages/secret-manager/CommitsPage/CommitsPage.tsx b/frontend/src/pages/secret-manager/CommitsPage/CommitsPage.tsx new file mode 100644 index 000000000..a7f5b6dd3 --- /dev/null +++ b/frontend/src/pages/secret-manager/CommitsPage/CommitsPage.tsx @@ -0,0 +1,65 @@ +import { Helmet } from "react-helmet"; +import { useTranslation } from "react-i18next"; +import { useNavigate, useParams, useSearch } from "@tanstack/react-router"; + +import { PageHeader } from "@app/components/v2"; +import { ROUTE_PATHS } from "@app/const/routes"; +import { useWorkspace } from "@app/context"; +import { ProjectType } from "@app/hooks/api/workspace/types"; + +import { CommitHistoryTab } from "./components/CommitHistoryTab"; + +export const CommitsPage = () => { + const { t } = useTranslation(); + const envSlug = useParams({ + from: ROUTE_PATHS.SecretManager.CommitsPage.id, + select: (el) => el.environment + }); + const { currentWorkspace } = useWorkspace(); + const navigate = useNavigate(); + const folderId = useParams({ + from: ROUTE_PATHS.SecretManager.CommitsPage.id, + select: (el) => el.folderId + }); + const routerQueryParams: { secretPath?: string } = useSearch({ + from: ROUTE_PATHS.SecretManager.CommitsPage.id + }); + + const secretPath = routerQueryParams?.secretPath || "/"; + + const handleSelectCommit = (commitId: string) => { + navigate({ + to: `/${ProjectType.SecretManager}/$projectId/commits/$environment/$folderId/$commitId` as const, + params: { + projectId: currentWorkspace.id, + folderId, + environment: envSlug, + commitId + }, + search: (query) => ({ + ...query, + secretPath + }) + }); + }; + + return ( +
+ + {t("common.head-title", { title: "Commit History" })} + +
+ + +
+
+ ); +}; diff --git a/frontend/src/pages/secret-manager/CommitsPage/components/CommitHistoryTab/CommitHistoryTab.tsx b/frontend/src/pages/secret-manager/CommitsPage/components/CommitHistoryTab/CommitHistoryTab.tsx new file mode 100644 index 000000000..19ff6df1b --- /dev/null +++ b/frontend/src/pages/secret-manager/CommitsPage/components/CommitHistoryTab/CommitHistoryTab.tsx @@ -0,0 +1,281 @@ +import { useCallback, useMemo, useState } from "react"; +import { + faArrowDownWideShort, + faArrowUpWideShort, + faCopy, + faSearch +} from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { format, formatDistanceToNow } from "date-fns"; + +import { Button, Input, Spinner } from "@app/components/v2"; +import { CopyButton } from "@app/components/v2/CopyButton"; +import { useGetFolderCommitHistory } from "@app/hooks/api/folderCommits"; + +interface CommitActorMetadata { + email?: string; + name?: string; +} + +interface Commit { + id: string; + message: string; + createdAt: string; + actorType: string; + actorMetadata?: CommitActorMetadata; +} + +const formatTimeAgo = (timestamp: string): string => { + return formatDistanceToNow(new Date(timestamp), { addSuffix: true }); +}; + +/** + * Commit Item component for displaying a single commit + */ +const CommitItem = ({ + commit, + onSelectCommit +}: { + commit: Commit; + onSelectCommit: (commitId: string, tab: string) => void; +}) => { + return ( +
+
+
+
+
+ +
+

+ + {commit.actorMetadata?.email || commit.actorMetadata?.name || commit.actorType} +

committed

+ + +

+
+
+
+ + +
+
+
+
+
+ ); +}; + +/** + * Date Group component for displaying commits grouped by date + */ +const DateGroup = ({ + date, + commits, + onSelectCommit +}: { + date: string; + commits: Commit[]; + onSelectCommit: (commitId: string, tab: string) => void; +}) => { + return ( +
+
+
+
+
+
+

Commits on {date}

+
+ +
+
+
+ {commits.map((commit) => ( +
+
+ +
+
+ ))} +
+
+
+ ); +}; + +export const CommitHistoryTab = ({ + onSelectCommit, + projectId, + environment, + secretPath +}: { + onSelectCommit: (commitId: string, tab: string) => void; + projectId: string; + environment: string; + secretPath: string; +}) => { + const [searchTerm, setSearchTerm] = useState(""); + const [sortDirection, setSortDirection] = useState<"asc" | "desc">("desc"); + const [visibleCommits, setVisibleCommits] = useState(10); + + const { data: commits, isLoading } = useGetFolderCommitHistory({ + workspaceId: projectId, + environment, + directory: secretPath + }); + + const filteredCommits = useMemo(() => { + if (!commits?.length) return []; + + return commits.filter( + (commit) => + commit.id?.includes(searchTerm) || + commit.message?.toLowerCase().includes(searchTerm.toLowerCase()) || + commit.actorMetadata?.name?.toLowerCase().includes(searchTerm.toLowerCase()) + ); + }, [commits, searchTerm]); + + const sortedCommits = useMemo(() => { + if (!filteredCommits?.length) return []; + + return [...filteredCommits].sort((a, b) => { + const dateA = new Date(a.createdAt).getTime(); + const dateB = new Date(b.createdAt).getTime(); + return sortDirection === "desc" ? dateB - dateA : dateA - dateB; + }); + }, [filteredCommits, sortDirection]); + + const displayedCommits = useMemo(() => { + return sortedCommits.slice(0, visibleCommits); + }, [sortedCommits, visibleCommits]); + const groupedCommits = useMemo(() => { + return displayedCommits.reduce( + (acc, commit) => { + const date = format(new Date(commit.createdAt), "MMM d, yyyy"); + if (!acc[date]) { + acc[date] = []; + } + acc[date].push(commit); + return acc; + }, + {} as Record + ); + }, [displayedCommits]); + + const handleSort = useCallback(() => { + setSortDirection((prev) => (prev === "desc" ? "asc" : "desc")); + }, []); + + const loadMoreCommits = useCallback(() => { + setVisibleCommits((prev) => prev + 10); + }, []); + + if (isLoading) { + return ( +
+ +
+ ); + } + + const hasMoreCommits = sortedCommits.length > visibleCommits; + + return ( +
+
+
+
+ setSearchTerm(e.target.value)} + aria-label="Search commits" + /> +
+
+
+ +
+
+ +
+ {Object.keys(groupedCommits).length > 0 ? ( + <> + {Object.entries(groupedCommits).map(([date, dateCommits]) => ( + + ))} + + ) : ( +
+
+ )} + + {hasMoreCommits && ( +
+ +
+ )} +
+
+ ); +}; diff --git a/frontend/src/pages/secret-manager/CommitsPage/components/CommitHistoryTab/index.tsx b/frontend/src/pages/secret-manager/CommitsPage/components/CommitHistoryTab/index.tsx new file mode 100644 index 000000000..de271b16a --- /dev/null +++ b/frontend/src/pages/secret-manager/CommitsPage/components/CommitHistoryTab/index.tsx @@ -0,0 +1 @@ +export { CommitHistoryTab } from "./CommitHistoryTab"; diff --git a/frontend/src/pages/secret-manager/CommitsPage/index.tsx b/frontend/src/pages/secret-manager/CommitsPage/index.tsx new file mode 100644 index 000000000..8203e3900 --- /dev/null +++ b/frontend/src/pages/secret-manager/CommitsPage/index.tsx @@ -0,0 +1 @@ +export { CommitsPage } from "./CommitsPage"; diff --git a/frontend/src/pages/secret-manager/CommitsPage/route.tsx b/frontend/src/pages/secret-manager/CommitsPage/route.tsx new file mode 100644 index 000000000..6ea9159d2 --- /dev/null +++ b/frontend/src/pages/secret-manager/CommitsPage/route.tsx @@ -0,0 +1,71 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { createFileRoute, linkOptions, stripSearchParams } from "@tanstack/react-router"; +import { zodValidator } from "@tanstack/zod-adapter"; +import { z } from "zod"; + +import { SecretDashboardPathBreadcrumb } from "@app/components/navigation/SecretDashboardPathBreadcrumb"; +import { BreadcrumbTypes } from "@app/components/v2"; + +import { CommitsPage } from "./CommitsPage"; + +const CommitsPageQueryParamsSchema = z.object({ + secretPath: z.string().catch("/") +}); + +export const Route = createFileRoute( + "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/" +)({ + component: CommitsPage, + validateSearch: zodValidator(CommitsPageQueryParamsSchema), + search: { + middlewares: [stripSearchParams({ secretPath: "/" })] + }, + beforeLoad: ({ context, params, search }) => { + const secretPathSegments = search.secretPath.split("/").filter(Boolean); + + return { + breadcrumbs: [ + ...context.breadcrumbs, + { + type: BreadcrumbTypes.Dropdown, + label: + context.project.environments.find((el) => el.slug === params.environment)?.name || "", + dropdownTitle: "Environments", + links: context.project.environments.map((el) => ({ + label: el.name, + link: linkOptions({ + to: "/secret-manager/$projectId/secrets/$envSlug", + params: { + projectId: params.projectId, + envSlug: el.slug + } + }) + })) + }, + ...secretPathSegments.map((_, index) => ({ + type: BreadcrumbTypes.Component, + component: () => ( + + ) + })), + { + label: "Commits", + link: linkOptions({ + to: "/secret-manager/$projectId/commits/$environment/$folderId", + params: { + projectId: params.projectId, + environment: params.environment, + folderId: params.folderId + } + }) + } + ] + }; + } +}); diff --git a/frontend/src/pages/secret-manager/SecretDashboardPage/SecretDashboardPage.tsx b/frontend/src/pages/secret-manager/SecretDashboardPage/SecretDashboardPage.tsx index d28f28392..c0445df92 100644 --- a/frontend/src/pages/secret-manager/SecretDashboardPage/SecretDashboardPage.tsx +++ b/frontend/src/pages/secret-manager/SecretDashboardPage/SecretDashboardPage.tsx @@ -39,6 +39,7 @@ import { } from "@app/hooks/api"; import { useGetProjectSecretsDetails } from "@app/hooks/api/dashboard"; import { DashboardSecretsOrderBy } from "@app/hooks/api/dashboard/types"; +import { useGetFolderCommitsCount } from "@app/hooks/api/folderCommits"; import { OrderByDirection } from "@app/hooks/api/generic/types"; import { ProjectType } from "@app/hooks/api/workspace/types"; import { hasSecretReadValueOrDescribePermission } from "@app/lib/fn/permission"; @@ -265,6 +266,17 @@ const Page = () => { limit: 10 }); + const { + data: { count: folderCommitsCount, folderId } = { count: 0, folderId: "" }, + isPending: isFolderCommitsCountLoading, + isFetching: isFolderCommitsCountFetching + } = useGetFolderCommitsCount({ + directory: secretPath, + workspaceId, + environment, + isPaused: !canDoReadRollback + }); + const { data: snapshotCount, isPending: isSnapshotCountLoading, @@ -276,6 +288,39 @@ const Page = () => { isPaused: !canDoReadRollback }); + const isPITEnabled = true; + + const changesCount = useMemo(() => { + return isPITEnabled ? folderCommitsCount : snapshotCount; + }, [folderCommitsCount, snapshotCount]); + + const isChangesCountPending = useMemo(() => { + return isPITEnabled ? isFolderCommitsCountLoading || isSnapshotCountLoading : false; + }, [isFolderCommitsCountLoading, isSnapshotCountLoading]); + + const isChangesCountFetching = useMemo(() => { + return isPITEnabled ? isFolderCommitsCountFetching || isSnapshotCountFetching : false; + }, [isFolderCommitsCountFetching, isSnapshotCountFetching]); + + const handleOnClickRollbackMode = () => { + if (isPITEnabled) { + navigate({ + to: `/${ProjectType.SecretManager}/$projectId/commits/$environment/$folderId` as const, + params: { + projectId: workspaceId, + folderId, + environment + }, + search: (query) => ({ + ...query, + secretPath + }) + }); + } else { + handlePopUpToggle("snapshots", true); + } + }; + const noAccessSecretCount = Math.max( (page * perPage > totalCount ? totalCount % perPage : perPage) - (imports?.length || 0) - @@ -436,13 +481,14 @@ const Page = () => { onVisibilityToggle={handleToggleVisibility} onSearchChange={handleSearchChange} onToggleTagFilter={handleTagToggle} - snapshotCount={snapshotCount || 0} - isSnapshotCountLoading={isSnapshotCountLoading && isSnapshotCountFetching} + snapshotCount={changesCount || 0} + isSnapshotCountLoading={isChangesCountPending && isChangesCountFetching} onToggleRowType={handleToggleRowType} - onClickRollbackMode={() => handlePopUpToggle("snapshots", true)} + onClickRollbackMode={handleOnClickRollbackMode} protectedBranchPolicyName={boardPolicy?.name} importedBy={importedBy} usedBySecretSyncs={usedBySecretSyncs} + isPITEnabled={isPITEnabled} />
@@ -606,7 +652,7 @@ const Page = () => { secretPath={secretPath} secrets={secrets} folders={folders} - snapshotCount={snapshotCount} + snapshotCount={changesCount} onGoBack={handleResetSnapshot} onClickListSnapshot={() => handlePopUpToggle("snapshots", true)} /> diff --git a/frontend/src/pages/secret-manager/SecretDashboardPage/components/ActionBar/ActionBar.tsx b/frontend/src/pages/secret-manager/SecretDashboardPage/components/ActionBar/ActionBar.tsx index 13b6d45b7..1d917f816 100644 --- a/frontend/src/pages/secret-manager/SecretDashboardPage/components/ActionBar/ActionBar.tsx +++ b/frontend/src/pages/secret-manager/SecretDashboardPage/components/ActionBar/ActionBar.tsx @@ -123,6 +123,7 @@ type Props = { isImported: boolean; }[]; }[]; + isPITEnabled: boolean; }; export const ActionBar = ({ @@ -142,6 +143,7 @@ export const ActionBar = ({ onToggleRowType, protectedBranchPolicyName, importedBy, + isPITEnabled = false, usedBySecretSyncs }: Props) => { const { handlePopUpOpen, handlePopUpToggle, handlePopUpClose, popUp } = usePopUp([ @@ -796,7 +798,7 @@ export const ActionBar = ({ className="h-10" isDisabled={!isAllowed} > - {`${snapshotCount} ${snapshotCount === 1 ? "Snapshot" : "Snapshots"}`} + {`${snapshotCount} ${isPITEnabled ? "Commit" : "Snapshot"}${snapshotCount === 1 ? "" : "s"}`} )} diff --git a/frontend/src/pages/secret-manager/SecretDashboardPage/components/SecretListView/SecretListView.tsx b/frontend/src/pages/secret-manager/SecretDashboardPage/components/SecretListView/SecretListView.tsx index 70d783d2b..89c1cae25 100644 --- a/frontend/src/pages/secret-manager/SecretDashboardPage/components/SecretListView/SecretListView.tsx +++ b/frontend/src/pages/secret-manager/SecretDashboardPage/components/SecretListView/SecretListView.tsx @@ -9,6 +9,7 @@ import { usePopUp } from "@app/hooks"; import { useCreateSecretV3, useDeleteSecretV3, useUpdateSecretV3 } from "@app/hooks/api"; import { dashboardKeys } from "@app/hooks/api/dashboard/queries"; import { UsedBySecretSyncs } from "@app/hooks/api/dashboard/types"; +import { commitKeys } from "@app/hooks/api/folderCommits/queries"; import { secretApprovalRequestKeys } from "@app/hooks/api/secretApprovalRequest/queries"; import { secretKeys } from "@app/hooks/api/secrets/queries"; import { SecretType, SecretV3RawSanitized } from "@app/hooks/api/secrets/types"; @@ -264,6 +265,12 @@ export const SecretListView = ({ queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ workspaceId, environment, directory: secretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId, environment, directory: secretPath }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId, environment, directory: secretPath }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId }) }); @@ -315,6 +322,12 @@ export const SecretListView = ({ queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ workspaceId, environment, directory: secretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId, environment, directory: secretPath }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId, environment, directory: secretPath }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId }) }); diff --git a/frontend/src/routeTree.gen.ts b/frontend/src/routeTree.gen.ts index 15d67ab46..1104b6f27 100644 --- a/frontend/src/routeTree.gen.ts +++ b/frontend/src/routeTree.gen.ts @@ -191,6 +191,9 @@ import { Route as secretManagerIntegrationsGcpSecretManagerOauthCallbackPageRout import { Route as secretManagerIntegrationsBitbucketOauthCallbackPageRouteImport } from './pages/secret-manager/integrations/BitbucketOauthCallbackPage/route' import { Route as secretManagerIntegrationsAzureKeyVaultOauthCallbackPageRouteImport } from './pages/secret-manager/integrations/AzureKeyVaultOauthCallbackPage/route' import { Route as secretManagerIntegrationsAzureAppConfigurationOauthCallbackPageRouteImport } from './pages/secret-manager/integrations/AzureAppConfigurationOauthCallbackPage/route' +import { Route as secretManagerCommitsPageRouteImport } from './pages/secret-manager/CommitsPage/route' +import { Route as secretManagerCommitDetailsPageComponentsRollbackPreviewTabRouteImport } from './pages/secret-manager/CommitDetailsPage/components/RollbackPreviewTab/route' +import { Route as secretManagerCommitDetailsPageRouteImport } from './pages/secret-manager/CommitDetailsPage/route' // Create Virtual Routes @@ -246,6 +249,14 @@ const AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLa createFileRoute( '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/integrations', )() +const AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdImport = + createFileRoute( + '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId', + )() +const AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdCommitIdImport = + createFileRoute( + '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/$commitId', + )() // Create/Update Routes @@ -1082,6 +1093,15 @@ const secretManagerIntegrationsListPageRouteRoute = AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutIntegrationsRoute, } as any) +const AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdRoute = + AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdImport.update( + { + id: '/commits/$environment/$folderId', + path: '/commits/$environment/$folderId', + getParentRoute: () => secretManagerLayoutRoute, + } as any, + ) + const secretManagerIntegrationsWindmillConfigurePageRouteRoute = secretManagerIntegrationsWindmillConfigurePageRouteImport.update({ id: '/windmill/create', @@ -1608,6 +1628,16 @@ const secretManagerIntegrationsAwsParameterStoreAuthorizePageRouteRoute = AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutIntegrationsRoute, } as any) +const AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdCommitIdRoute = + AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdCommitIdImport.update( + { + id: '/$commitId', + path: '/$commitId', + getParentRoute: () => + AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdRoute, + } as any, + ) + const secretManagerIntegrationsVercelOauthCallbackPageRouteRoute = secretManagerIntegrationsVercelOauthCallbackPageRouteImport.update({ id: '/vercel/oauth2/callback', @@ -1690,6 +1720,30 @@ const secretManagerIntegrationsAzureAppConfigurationOauthCallbackPageRouteRoute } as any, ) +const secretManagerCommitsPageRouteRoute = + secretManagerCommitsPageRouteImport.update({ + id: '/', + path: '/', + getParentRoute: () => + AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdRoute, + } as any) + +const secretManagerCommitDetailsPageComponentsRollbackPreviewTabRouteRoute = + secretManagerCommitDetailsPageComponentsRollbackPreviewTabRouteImport.update({ + id: '/restore', + path: '/restore', + getParentRoute: () => + AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdCommitIdRoute, + } as any) + +const secretManagerCommitDetailsPageRouteRoute = + secretManagerCommitDetailsPageRouteImport.update({ + id: '/', + path: '/', + getParentRoute: () => + AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdCommitIdRoute, + } as any) + // Populate the FileRoutesByPath interface declare module '@tanstack/react-router' { @@ -2975,6 +3029,20 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof secretManagerIntegrationsWindmillConfigurePageRouteImport parentRoute: typeof AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutIntegrationsImport } + '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId': { + id: '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId' + path: '/commits/$environment/$folderId' + fullPath: '/secret-manager/$projectId/commits/$environment/$folderId' + preLoaderRoute: typeof AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdImport + parentRoute: typeof secretManagerLayoutImport + } + '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/': { + id: '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/' + path: '/' + fullPath: '/secret-manager/$projectId/commits/$environment/$folderId/' + preLoaderRoute: typeof secretManagerCommitsPageRouteImport + parentRoute: typeof AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdImport + } '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/integrations/azure-app-configuration/oauth2/callback': { id: '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/integrations/azure-app-configuration/oauth2/callback' path: '/azure-app-configuration/oauth2/callback' @@ -3045,6 +3113,27 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof secretManagerIntegrationsVercelOauthCallbackPageRouteImport parentRoute: typeof AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutIntegrationsImport } + '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/$commitId': { + id: '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/$commitId' + path: '/$commitId' + fullPath: '/secret-manager/$projectId/commits/$environment/$folderId/$commitId' + preLoaderRoute: typeof AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdCommitIdImport + parentRoute: typeof AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdImport + } + '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/$commitId/': { + id: '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/$commitId/' + path: '/' + fullPath: '/secret-manager/$projectId/commits/$environment/$folderId/$commitId/' + preLoaderRoute: typeof secretManagerCommitDetailsPageRouteImport + parentRoute: typeof AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdCommitIdImport + } + '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/$commitId/restore': { + id: '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/$commitId/restore' + path: '/restore' + fullPath: '/secret-manager/$projectId/commits/$environment/$folderId/$commitId/restore' + preLoaderRoute: typeof secretManagerCommitDetailsPageComponentsRollbackPreviewTabRouteImport + parentRoute: typeof AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdCommitIdImport + } } } @@ -3553,6 +3642,41 @@ const AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLa AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutIntegrationsRouteChildren, ) +interface AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdCommitIdRouteChildren { + secretManagerCommitDetailsPageRouteRoute: typeof secretManagerCommitDetailsPageRouteRoute + secretManagerCommitDetailsPageComponentsRollbackPreviewTabRouteRoute: typeof secretManagerCommitDetailsPageComponentsRollbackPreviewTabRouteRoute +} + +const AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdCommitIdRouteChildren: AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdCommitIdRouteChildren = + { + secretManagerCommitDetailsPageRouteRoute: + secretManagerCommitDetailsPageRouteRoute, + secretManagerCommitDetailsPageComponentsRollbackPreviewTabRouteRoute: + secretManagerCommitDetailsPageComponentsRollbackPreviewTabRouteRoute, + } + +const AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdCommitIdRouteWithChildren = + AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdCommitIdRoute._addFileChildren( + AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdCommitIdRouteChildren, + ) + +interface AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdRouteChildren { + secretManagerCommitsPageRouteRoute: typeof secretManagerCommitsPageRouteRoute + AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdCommitIdRoute: typeof AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdCommitIdRouteWithChildren +} + +const AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdRouteChildren: AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdRouteChildren = + { + secretManagerCommitsPageRouteRoute: secretManagerCommitsPageRouteRoute, + AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdCommitIdRoute: + AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdCommitIdRouteWithChildren, + } + +const AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdRouteWithChildren = + AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdRoute._addFileChildren( + AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdRouteChildren, + ) + interface secretManagerLayoutRouteChildren { secretManagerIPAllowlistPageRouteRoute: typeof secretManagerIPAllowlistPageRouteRoute secretManagerSecretApprovalsPageRouteRoute: typeof secretManagerSecretApprovalsPageRouteRoute @@ -3565,6 +3689,7 @@ interface secretManagerLayoutRouteChildren { projectIdentityDetailsByIDPageRouteSecretManagerRoute: typeof projectIdentityDetailsByIDPageRouteSecretManagerRoute projectMemberDetailsByIDPageRouteSecretManagerRoute: typeof projectMemberDetailsByIDPageRouteSecretManagerRoute projectRoleDetailsBySlugPageRouteSecretManagerRoute: typeof projectRoleDetailsBySlugPageRouteSecretManagerRoute + AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdRoute: typeof AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdRouteWithChildren } const secretManagerLayoutRouteChildren: secretManagerLayoutRouteChildren = { @@ -3588,6 +3713,8 @@ const secretManagerLayoutRouteChildren: secretManagerLayoutRouteChildren = { projectMemberDetailsByIDPageRouteSecretManagerRoute, projectRoleDetailsBySlugPageRouteSecretManagerRoute: projectRoleDetailsBySlugPageRouteSecretManagerRoute, + AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdRoute: + AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdRouteWithChildren, } const secretManagerLayoutRouteWithChildren = @@ -4018,6 +4145,8 @@ export interface FileRoutesByFullPath { '/secret-manager/$projectId/integrations/vercel/create': typeof secretManagerIntegrationsVercelConfigurePageRouteRoute '/secret-manager/$projectId/integrations/windmill/authorize': typeof secretManagerIntegrationsWindmillAuthorizePageRouteRoute '/secret-manager/$projectId/integrations/windmill/create': typeof secretManagerIntegrationsWindmillConfigurePageRouteRoute + '/secret-manager/$projectId/commits/$environment/$folderId': typeof AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdRouteWithChildren + '/secret-manager/$projectId/commits/$environment/$folderId/': typeof secretManagerCommitsPageRouteRoute '/secret-manager/$projectId/integrations/azure-app-configuration/oauth2/callback': typeof secretManagerIntegrationsAzureAppConfigurationOauthCallbackPageRouteRoute '/secret-manager/$projectId/integrations/azure-key-vault/oauth2/callback': typeof secretManagerIntegrationsAzureKeyVaultOauthCallbackPageRouteRoute '/secret-manager/$projectId/integrations/bitbucket/oauth2/callback': typeof secretManagerIntegrationsBitbucketOauthCallbackPageRouteRoute @@ -4028,6 +4157,9 @@ export interface FileRoutesByFullPath { '/secret-manager/$projectId/integrations/netlify/oauth2/callback': typeof secretManagerIntegrationsNetlifyOauthCallbackPageRouteRoute '/secret-manager/$projectId/integrations/secret-syncs/$destination/$syncId': typeof secretManagerSecretSyncDetailsByIDPageRouteRoute '/secret-manager/$projectId/integrations/vercel/oauth2/callback': typeof secretManagerIntegrationsVercelOauthCallbackPageRouteRoute + '/secret-manager/$projectId/commits/$environment/$folderId/$commitId': typeof AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdCommitIdRouteWithChildren + '/secret-manager/$projectId/commits/$environment/$folderId/$commitId/': typeof secretManagerCommitDetailsPageRouteRoute + '/secret-manager/$projectId/commits/$environment/$folderId/$commitId/restore': typeof secretManagerCommitDetailsPageComponentsRollbackPreviewTabRouteRoute } export interface FileRoutesByTo { @@ -4196,6 +4328,7 @@ export interface FileRoutesByTo { '/secret-manager/$projectId/integrations/vercel/create': typeof secretManagerIntegrationsVercelConfigurePageRouteRoute '/secret-manager/$projectId/integrations/windmill/authorize': typeof secretManagerIntegrationsWindmillAuthorizePageRouteRoute '/secret-manager/$projectId/integrations/windmill/create': typeof secretManagerIntegrationsWindmillConfigurePageRouteRoute + '/secret-manager/$projectId/commits/$environment/$folderId': typeof secretManagerCommitsPageRouteRoute '/secret-manager/$projectId/integrations/azure-app-configuration/oauth2/callback': typeof secretManagerIntegrationsAzureAppConfigurationOauthCallbackPageRouteRoute '/secret-manager/$projectId/integrations/azure-key-vault/oauth2/callback': typeof secretManagerIntegrationsAzureKeyVaultOauthCallbackPageRouteRoute '/secret-manager/$projectId/integrations/bitbucket/oauth2/callback': typeof secretManagerIntegrationsBitbucketOauthCallbackPageRouteRoute @@ -4206,6 +4339,8 @@ export interface FileRoutesByTo { '/secret-manager/$projectId/integrations/netlify/oauth2/callback': typeof secretManagerIntegrationsNetlifyOauthCallbackPageRouteRoute '/secret-manager/$projectId/integrations/secret-syncs/$destination/$syncId': typeof secretManagerSecretSyncDetailsByIDPageRouteRoute '/secret-manager/$projectId/integrations/vercel/oauth2/callback': typeof secretManagerIntegrationsVercelOauthCallbackPageRouteRoute + '/secret-manager/$projectId/commits/$environment/$folderId/$commitId': typeof secretManagerCommitDetailsPageRouteRoute + '/secret-manager/$projectId/commits/$environment/$folderId/$commitId/restore': typeof secretManagerCommitDetailsPageComponentsRollbackPreviewTabRouteRoute } export interface FileRoutesById { @@ -4393,6 +4528,8 @@ export interface FileRoutesById { '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/integrations/vercel/create': typeof secretManagerIntegrationsVercelConfigurePageRouteRoute '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/integrations/windmill/authorize': typeof secretManagerIntegrationsWindmillAuthorizePageRouteRoute '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/integrations/windmill/create': typeof secretManagerIntegrationsWindmillConfigurePageRouteRoute + '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId': typeof AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdRouteWithChildren + '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/': typeof secretManagerCommitsPageRouteRoute '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/integrations/azure-app-configuration/oauth2/callback': typeof secretManagerIntegrationsAzureAppConfigurationOauthCallbackPageRouteRoute '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/integrations/azure-key-vault/oauth2/callback': typeof secretManagerIntegrationsAzureKeyVaultOauthCallbackPageRouteRoute '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/integrations/bitbucket/oauth2/callback': typeof secretManagerIntegrationsBitbucketOauthCallbackPageRouteRoute @@ -4403,6 +4540,9 @@ export interface FileRoutesById { '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/integrations/netlify/oauth2/callback': typeof secretManagerIntegrationsNetlifyOauthCallbackPageRouteRoute '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/integrations/secret-syncs/$destination/$syncId': typeof secretManagerSecretSyncDetailsByIDPageRouteRoute '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/integrations/vercel/oauth2/callback': typeof secretManagerIntegrationsVercelOauthCallbackPageRouteRoute + '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/$commitId': typeof AuthenticateInjectOrgDetailsOrgLayoutSecretManagerProjectIdSecretManagerLayoutCommitsEnvironmentFolderIdCommitIdRouteWithChildren + '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/$commitId/': typeof secretManagerCommitDetailsPageRouteRoute + '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/$commitId/restore': typeof secretManagerCommitDetailsPageComponentsRollbackPreviewTabRouteRoute } export interface FileRouteTypes { @@ -4582,6 +4722,8 @@ export interface FileRouteTypes { | '/secret-manager/$projectId/integrations/vercel/create' | '/secret-manager/$projectId/integrations/windmill/authorize' | '/secret-manager/$projectId/integrations/windmill/create' + | '/secret-manager/$projectId/commits/$environment/$folderId' + | '/secret-manager/$projectId/commits/$environment/$folderId/' | '/secret-manager/$projectId/integrations/azure-app-configuration/oauth2/callback' | '/secret-manager/$projectId/integrations/azure-key-vault/oauth2/callback' | '/secret-manager/$projectId/integrations/bitbucket/oauth2/callback' @@ -4592,6 +4734,9 @@ export interface FileRouteTypes { | '/secret-manager/$projectId/integrations/netlify/oauth2/callback' | '/secret-manager/$projectId/integrations/secret-syncs/$destination/$syncId' | '/secret-manager/$projectId/integrations/vercel/oauth2/callback' + | '/secret-manager/$projectId/commits/$environment/$folderId/$commitId' + | '/secret-manager/$projectId/commits/$environment/$folderId/$commitId/' + | '/secret-manager/$projectId/commits/$environment/$folderId/$commitId/restore' fileRoutesByTo: FileRoutesByTo to: | '/' @@ -4759,6 +4904,7 @@ export interface FileRouteTypes { | '/secret-manager/$projectId/integrations/vercel/create' | '/secret-manager/$projectId/integrations/windmill/authorize' | '/secret-manager/$projectId/integrations/windmill/create' + | '/secret-manager/$projectId/commits/$environment/$folderId' | '/secret-manager/$projectId/integrations/azure-app-configuration/oauth2/callback' | '/secret-manager/$projectId/integrations/azure-key-vault/oauth2/callback' | '/secret-manager/$projectId/integrations/bitbucket/oauth2/callback' @@ -4769,6 +4915,8 @@ export interface FileRouteTypes { | '/secret-manager/$projectId/integrations/netlify/oauth2/callback' | '/secret-manager/$projectId/integrations/secret-syncs/$destination/$syncId' | '/secret-manager/$projectId/integrations/vercel/oauth2/callback' + | '/secret-manager/$projectId/commits/$environment/$folderId/$commitId' + | '/secret-manager/$projectId/commits/$environment/$folderId/$commitId/restore' id: | '__root__' | '/' @@ -4954,6 +5102,8 @@ export interface FileRouteTypes { | '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/integrations/vercel/create' | '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/integrations/windmill/authorize' | '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/integrations/windmill/create' + | '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId' + | '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/' | '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/integrations/azure-app-configuration/oauth2/callback' | '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/integrations/azure-key-vault/oauth2/callback' | '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/integrations/bitbucket/oauth2/callback' @@ -4964,6 +5114,9 @@ export interface FileRouteTypes { | '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/integrations/netlify/oauth2/callback' | '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/integrations/secret-syncs/$destination/$syncId' | '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/integrations/vercel/oauth2/callback' + | '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/$commitId' + | '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/$commitId/' + | '/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/$commitId/restore' fileRoutesById: FileRoutesById } @@ -5403,7 +5556,8 @@ export const routeTree = rootRoute "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/secrets/$envSlug", "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/identities/$identityId", "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/members/$membershipId", - "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/roles/$roleSlug" + "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/roles/$roleSlug", + "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId" ] }, "/_authenticate/_inject-org-details/_org-layout/ssh/$projectId/_ssh-layout": { @@ -5970,6 +6124,18 @@ export const routeTree = rootRoute "filePath": "secret-manager/integrations/WindmillConfigurePage/route.tsx", "parent": "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/integrations" }, + "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId": { + "filePath": "", + "parent": "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout", + "children": [ + "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/", + "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/$commitId" + ] + }, + "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/": { + "filePath": "secret-manager/CommitsPage/route.tsx", + "parent": "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId" + }, "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/integrations/azure-app-configuration/oauth2/callback": { "filePath": "secret-manager/integrations/AzureAppConfigurationOauthCallbackPage/route.tsx", "parent": "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/integrations" @@ -6009,6 +6175,22 @@ export const routeTree = rootRoute "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/integrations/vercel/oauth2/callback": { "filePath": "secret-manager/integrations/VercelOauthCallbackPage/route.tsx", "parent": "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/integrations" + }, + "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/$commitId": { + "filePath": "", + "parent": "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId", + "children": [ + "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/$commitId/", + "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/$commitId/restore" + ] + }, + "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/$commitId/": { + "filePath": "secret-manager/CommitDetailsPage/route.tsx", + "parent": "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/$commitId" + }, + "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/$commitId/restore": { + "filePath": "secret-manager/CommitDetailsPage/components/RollbackPreviewTab/route.tsx", + "parent": "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/$commitId" } } } diff --git a/frontend/src/routes.ts b/frontend/src/routes.ts index 3d9889ab3..9954fb650 100644 --- a/frontend/src/routes.ts +++ b/frontend/src/routes.ts @@ -47,6 +47,16 @@ const secretManagerRoutes = route("/secret-manager/$projectId", [ route("/approval", "secret-manager/SecretApprovalsPage/route.tsx"), route("/secret-rotation", "secret-manager/SecretRotationPage/route.tsx"), route("/settings", "secret-manager/SettingsPage/route.tsx"), + route("/commits/$environment/$folderId", [ + index("secret-manager/CommitsPage/route.tsx"), + route("/$commitId", [ + index("secret-manager/CommitDetailsPage/route.tsx"), + route( + "/restore", + "secret-manager/CommitDetailsPage/components/RollbackPreviewTab/route.tsx" + ) + ]) + ]), route("/integrations", [ index("secret-manager/IntegrationsListPage/route.tsx"), route("/$integrationId", "secret-manager/IntegrationsDetailsByIDPage/route.tsx"), From 9ea9f9092850b9502014734f4dd9bdf3fbc78e9f Mon Sep 17 00:00:00 2001 From: carlosmonastyrski Date: Tue, 20 May 2025 09:34:43 -0300 Subject: [PATCH 9/9] PIT: add envID to rollback endpoint --- backend/src/ee/routes/v1/pit-router.ts | 7 ++++--- frontend/src/hooks/api/folderCommits/queries.tsx | 12 ++++++++---- .../RollbackPreviewTab/RollbackPreviewTab.tsx | 3 ++- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/backend/src/ee/routes/v1/pit-router.ts b/backend/src/ee/routes/v1/pit-router.ts index 274df1b11..57ed20081 100644 --- a/backend/src/ee/routes/v1/pit-router.ts +++ b/backend/src/ee/routes/v1/pit-router.ts @@ -440,7 +440,8 @@ export const registerPITRouter = async (server: FastifyZodProvider) => { body: z.object({ folderId: z.string().trim(), deepRollback: z.boolean().default(false), - message: z.string().trim().optional() + message: z.string().trim().optional(), + envId: z.string().trim() }), response: { 200: z.object({ @@ -467,8 +468,8 @@ export const registerPITRouter = async (server: FastifyZodProvider) => { if (req.body.deepRollback) { await server.services.folderCommit.deepRollbackFolder( - latestCommit.id, - req.body.folderId, + req.params.commitId, + req.body.envId, req.permission.id, req.permission.type, req.params.workspaceId diff --git a/frontend/src/hooks/api/folderCommits/queries.tsx b/frontend/src/hooks/api/folderCommits/queries.tsx index 2fabb7936..5914fcf15 100644 --- a/frontend/src/hooks/api/folderCommits/queries.tsx +++ b/frontend/src/hooks/api/folderCommits/queries.tsx @@ -112,14 +112,16 @@ const fetchRollback = async ( commitId: string, projectId: string, deepRollback: boolean, - message?: string + message?: string, + envId?: string ) => { const { data } = await apiRequest.post<{ success: boolean }>( `/api/v1/pit/commits/${projectId}/${commitId}/rollback`, { folderId, deepRollback, - message + message, + envId } ); return data; @@ -164,7 +166,8 @@ export const useCommitRollback = ({ folderId, deepRollback, environment, - directory + directory, + envId }: { workspaceId: string; commitId: string; @@ -172,11 +175,12 @@ export const useCommitRollback = ({ deepRollback: boolean; environment: string; directory: string; + envId: string; }) => { const queryClient = useQueryClient(); return useMutation({ mutationFn: (message: string) => - fetchRollback(folderId, commitId, workspaceId, deepRollback, message), + fetchRollback(folderId, commitId, workspaceId, deepRollback, message, envId), onSuccess: () => { queryClient.invalidateQueries({ queryKey: [ diff --git a/frontend/src/pages/secret-manager/CommitDetailsPage/components/RollbackPreviewTab/RollbackPreviewTab.tsx b/frontend/src/pages/secret-manager/CommitDetailsPage/components/RollbackPreviewTab/RollbackPreviewTab.tsx index 0778b4f25..ecb2056e6 100644 --- a/frontend/src/pages/secret-manager/CommitDetailsPage/components/RollbackPreviewTab/RollbackPreviewTab.tsx +++ b/frontend/src/pages/secret-manager/CommitDetailsPage/components/RollbackPreviewTab/RollbackPreviewTab.tsx @@ -121,7 +121,8 @@ export const RollbackPreviewTab = (): JSX.Element => { folderId, deepRollback, environment: envSlug, - directory: secretPath + directory: secretPath, + envId: currentWorkspace.environments.find((e) => e.slug === envSlug)?.id || "" }); const { data: rollbackChangesNested, isLoading } = useGetRollbackPreview(