Merge pull request #3546 from Infisical/feat/point-in-time-revamp

feat(PIT): Point In Time Revamp

backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts

@@ -0,0 +1,166 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  const hasFolderCommitTable = await knex.schema.hasTable(TableName.FolderCommit);
+  if (!hasFolderCommitTable) {
+    await knex.schema.createTable(TableName.FolderCommit, (t) => {
+      t.uuid("id").primary().defaultTo(knex.fn.uuid());
+      t.bigIncrements("commitId");
+      t.jsonb("actorMetadata").notNullable();
+      t.string("actorType").notNullable();
+      t.string("message");
+      t.uuid("folderId").notNullable();
+      t.uuid("envId").notNullable();
+      t.foreign("envId").references("id").inTable(TableName.Environment).onDelete("CASCADE");
+      t.timestamps(true, true, true);
+
+      t.index("folderId");
+      t.index("envId");
+    });
+  }
+
+  const hasFolderCommitChangesTable = await knex.schema.hasTable(TableName.FolderCommitChanges);
+  if (!hasFolderCommitChangesTable) {
+    await knex.schema.createTable(TableName.FolderCommitChanges, (t) => {
+      t.uuid("id").primary().defaultTo(knex.fn.uuid());
+      t.uuid("folderCommitId").notNullable();
+      t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE");
+      t.string("changeType").notNullable();
+      t.boolean("isUpdate").notNullable().defaultTo(false);
+      t.uuid("secretVersionId");
+      t.foreign("secretVersionId").references("id").inTable(TableName.SecretVersionV2).onDelete("CASCADE");
+      t.uuid("folderVersionId");
+      t.foreign("folderVersionId").references("id").inTable(TableName.SecretFolderVersion).onDelete("CASCADE");
+      t.timestamps(true, true, true);
+
+      t.index("folderCommitId");
+      t.index("secretVersionId");
+      t.index("folderVersionId");
+    });
+  }
+
+  const hasFolderCheckpointTable = await knex.schema.hasTable(TableName.FolderCheckpoint);
+  if (!hasFolderCheckpointTable) {
+    await knex.schema.createTable(TableName.FolderCheckpoint, (t) => {
+      t.uuid("id").primary().defaultTo(knex.fn.uuid());
+      t.uuid("folderCommitId").notNullable();
+      t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE");
+      t.timestamps(true, true, true);
+
+      t.index("folderCommitId");
+    });
+  }
+
+  const hasFolderCheckpointResourcesTable = await knex.schema.hasTable(TableName.FolderCheckpointResources);
+  if (!hasFolderCheckpointResourcesTable) {
+    await knex.schema.createTable(TableName.FolderCheckpointResources, (t) => {
+      t.uuid("id").primary().defaultTo(knex.fn.uuid());
+      t.uuid("folderCheckpointId").notNullable();
+      t.foreign("folderCheckpointId").references("id").inTable(TableName.FolderCheckpoint).onDelete("CASCADE");
+      t.uuid("secretVersionId");
+      t.foreign("secretVersionId").references("id").inTable(TableName.SecretVersionV2).onDelete("CASCADE");
+      t.uuid("folderVersionId");
+      t.foreign("folderVersionId").references("id").inTable(TableName.SecretFolderVersion).onDelete("CASCADE");
+      t.timestamps(true, true, true);
+
+      t.index("folderCheckpointId");
+      t.index("secretVersionId");
+      t.index("folderVersionId");
+    });
+  }
+
+  const hasFolderTreeCheckpointTable = await knex.schema.hasTable(TableName.FolderTreeCheckpoint);
+  if (!hasFolderTreeCheckpointTable) {
+    await knex.schema.createTable(TableName.FolderTreeCheckpoint, (t) => {
+      t.uuid("id").primary().defaultTo(knex.fn.uuid());
+      t.uuid("folderCommitId").notNullable();
+      t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE");
+      t.timestamps(true, true, true);
+
+      t.index("folderCommitId");
+    });
+  }
+
+  const hasFolderTreeCheckpointResourcesTable = await knex.schema.hasTable(TableName.FolderTreeCheckpointResources);
+  if (!hasFolderTreeCheckpointResourcesTable) {
+    await knex.schema.createTable(TableName.FolderTreeCheckpointResources, (t) => {
+      t.uuid("id").primary().defaultTo(knex.fn.uuid());
+      t.uuid("folderTreeCheckpointId").notNullable();
+      t.foreign("folderTreeCheckpointId").references("id").inTable(TableName.FolderTreeCheckpoint).onDelete("CASCADE");
+      t.uuid("folderId").notNullable();
+      t.uuid("folderCommitId").notNullable();
+      t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE");
+      t.timestamps(true, true, true);
+
+      t.index("folderTreeCheckpointId");
+      t.index("folderId");
+      t.index("folderCommitId");
+    });
+  }
+
+  if (!hasFolderCommitTable) {
+    await createOnUpdateTrigger(knex, TableName.FolderCommit);
+  }
+
+  if (!hasFolderCommitChangesTable) {
+    await createOnUpdateTrigger(knex, TableName.FolderCommitChanges);
+  }
+
+  if (!hasFolderCheckpointTable) {
+    await createOnUpdateTrigger(knex, TableName.FolderCheckpoint);
+  }
+
+  if (!hasFolderCheckpointResourcesTable) {
+    await createOnUpdateTrigger(knex, TableName.FolderCheckpointResources);
+  }
+
+  if (!hasFolderTreeCheckpointTable) {
+    await createOnUpdateTrigger(knex, TableName.FolderTreeCheckpoint);
+  }
+
+  if (!hasFolderTreeCheckpointResourcesTable) {
+    await createOnUpdateTrigger(knex, TableName.FolderTreeCheckpointResources);
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const hasFolderCheckpointResourcesTable = await knex.schema.hasTable(TableName.FolderCheckpointResources);
+  const hasFolderTreeCheckpointResourcesTable = await knex.schema.hasTable(TableName.FolderTreeCheckpointResources);
+  const hasFolderCommitTable = await knex.schema.hasTable(TableName.FolderCommit);
+  const hasFolderCommitChangesTable = await knex.schema.hasTable(TableName.FolderCommitChanges);
+  const hasFolderTreeCheckpointTable = await knex.schema.hasTable(TableName.FolderTreeCheckpoint);
+  const hasFolderCheckpointTable = await knex.schema.hasTable(TableName.FolderCheckpoint);
+
+  if (hasFolderTreeCheckpointResourcesTable) {
+    await dropOnUpdateTrigger(knex, TableName.FolderTreeCheckpointResources);
+    await knex.schema.dropTableIfExists(TableName.FolderTreeCheckpointResources);
+  }
+
+  if (hasFolderCheckpointResourcesTable) {
+    await dropOnUpdateTrigger(knex, TableName.FolderCheckpointResources);
+    await knex.schema.dropTableIfExists(TableName.FolderCheckpointResources);
+  }
+
+  if (hasFolderTreeCheckpointTable) {
+    await dropOnUpdateTrigger(knex, TableName.FolderTreeCheckpoint);
+    await knex.schema.dropTableIfExists(TableName.FolderTreeCheckpoint);
+  }
+
+  if (hasFolderCheckpointTable) {
+    await dropOnUpdateTrigger(knex, TableName.FolderCheckpoint);
+    await knex.schema.dropTableIfExists(TableName.FolderCheckpoint);
+  }
+
+  if (hasFolderCommitChangesTable) {
+    await dropOnUpdateTrigger(knex, TableName.FolderCommitChanges);
+    await knex.schema.dropTableIfExists(TableName.FolderCommitChanges);
+  }
+
+  if (hasFolderCommitTable) {
+    await dropOnUpdateTrigger(knex, TableName.FolderCommit);
+    await knex.schema.dropTableIfExists(TableName.FolderCommit);
+  }
+}

backend/src/db/migrations/20250507185010_pit-projects-commits-initialization.ts

@@ -0,0 +1,29 @@
+import { Knex } from "knex";
+
+import { inMemoryKeyStore } from "@app/keystore/memory";
+
+import { ProjectType, TableName } from "../schemas";
+import { getMigrationEnvConfig } from "./utils/env-config";
+import { getMigrationPITServices } from "./utils/services";
+
+export async function up(knex: Knex): Promise<void> {
+  const hasFolderCommitTable = await knex.schema.hasTable(TableName.FolderCommit);
+  if (hasFolderCommitTable) {
+    const keyStore = inMemoryKeyStore();
+    const envConfig = getMigrationEnvConfig();
+    const { folderCommitService } = await getMigrationPITServices({ db: knex, keyStore, envConfig });
+    const projects = await knex(TableName.Project).where({ version: 3, type: ProjectType.SecretManager }).select("id");
+    for (const project of projects) {
+      // eslint-disable-next-line no-await-in-loop
+      await folderCommitService.initializeProject(project.id, knex);
+    }
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const hasFolderCommitTable = await knex.schema.hasTable(TableName.FolderCommit);
+  if (hasFolderCommitTable) {
+    // delete all existing entries
+    await knex(TableName.FolderCommit).del();
+  }
+}

backend/src/db/migrations/20250528110936_add-folder-description-to-versioning.ts

@@ -0,0 +1,19 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasColumn(TableName.SecretFolderVersion, "description"))) {
+    await knex.schema.alterTable(TableName.SecretFolderVersion, (t) => {
+      t.string("description").nullable();
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  if (await knex.schema.hasColumn(TableName.SecretFolderVersion, "description")) {
+    await knex.schema.alterTable(TableName.SecretFolderVersion, (t) => {
+      t.dropColumn("description");
+    });
+  }
+}

backend/src/db/migrations/utils/services.ts

@@ -3,12 +3,27 @@ import { Knex } from "knex";
 import { initializeHsmModule } from "@app/ee/services/hsm/hsm-fns";
 import { hsmServiceFactory } from "@app/ee/services/hsm/hsm-service";
 import { TKeyStoreFactory } from "@app/keystore/keystore";
+import { folderCheckpointDALFactory } from "@app/services/folder-checkpoint/folder-checkpoint-dal";
+import { folderCheckpointResourcesDALFactory } from "@app/services/folder-checkpoint-resources/folder-checkpoint-resources-dal";
+import { folderCommitDALFactory } from "@app/services/folder-commit/folder-commit-dal";
+import { folderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service";
+import { folderCommitChangesDALFactory } from "@app/services/folder-commit-changes/folder-commit-changes-dal";
+import { folderTreeCheckpointDALFactory } from "@app/services/folder-tree-checkpoint/folder-tree-checkpoint-dal";
+import { folderTreeCheckpointResourcesDALFactory } from "@app/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal";
+import { identityDALFactory } from "@app/services/identity/identity-dal";
 import { internalKmsDALFactory } from "@app/services/kms/internal-kms-dal";
 import { kmskeyDALFactory } from "@app/services/kms/kms-key-dal";
 import { kmsRootConfigDALFactory } from "@app/services/kms/kms-root-config-dal";
 import { kmsServiceFactory } from "@app/services/kms/kms-service";
 import { orgDALFactory } from "@app/services/org/org-dal";
 import { projectDALFactory } from "@app/services/project/project-dal";
+import { resourceMetadataDALFactory } from "@app/services/resource-metadata/resource-metadata-dal";
+import { secretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
+import { secretFolderVersionDALFactory } from "@app/services/secret-folder/secret-folder-version-dal";
+import { secretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
+import { secretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal";
+import { secretVersionV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-version-dal";
+import { userDALFactory } from "@app/services/user/user-dal";
 
 import { TMigrationEnvConfig } from "./env-config";
 
@@ -50,3 +65,77 @@ export const getMigrationEncryptionServices = async ({ envConfig, db, keyStore }
 
   return { kmsService };
 };
+
+export const getMigrationPITServices = async ({
+  db,
+  keyStore,
+  envConfig
+}: {
+  db: Knex;
+  keyStore: TKeyStoreFactory;
+  envConfig: TMigrationEnvConfig;
+}) => {
+  const projectDAL = projectDALFactory(db);
+  const folderCommitDAL = folderCommitDALFactory(db);
+  const folderCommitChangesDAL = folderCommitChangesDALFactory(db);
+  const folderCheckpointDAL = folderCheckpointDALFactory(db);
+  const folderTreeCheckpointDAL = folderTreeCheckpointDALFactory(db);
+  const userDAL = userDALFactory(db);
+  const identityDAL = identityDALFactory(db);
+  const folderDAL = secretFolderDALFactory(db);
+  const folderVersionDAL = secretFolderVersionDALFactory(db);
+  const secretVersionV2BridgeDAL = secretVersionV2BridgeDALFactory(db);
+  const folderCheckpointResourcesDAL = folderCheckpointResourcesDALFactory(db);
+  const secretV2BridgeDAL = secretV2BridgeDALFactory({ db, keyStore });
+  const folderTreeCheckpointResourcesDAL = folderTreeCheckpointResourcesDALFactory(db);
+  const secretTagDAL = secretTagDALFactory(db);
+
+  const orgDAL = orgDALFactory(db);
+  const kmsRootConfigDAL = kmsRootConfigDALFactory(db);
+  const kmsDAL = kmskeyDALFactory(db);
+  const internalKmsDAL = internalKmsDALFactory(db);
+  const resourceMetadataDAL = resourceMetadataDALFactory(db);
+
+  const hsmModule = initializeHsmModule(envConfig);
+  hsmModule.initialize();
+
+  const hsmService = hsmServiceFactory({
+    hsmModule: hsmModule.getModule(),
+    envConfig
+  });
+
+  const kmsService = kmsServiceFactory({
+    kmsRootConfigDAL,
+    keyStore,
+    kmsDAL,
+    internalKmsDAL,
+    orgDAL,
+    projectDAL,
+    hsmService,
+    envConfig
+  });
+
+  await hsmService.startService();
+  await kmsService.startService();
+
+  const folderCommitService = folderCommitServiceFactory({
+    folderCommitDAL,
+    folderCommitChangesDAL,
+    folderCheckpointDAL,
+    folderTreeCheckpointDAL,
+    userDAL,
+    identityDAL,
+    folderDAL,
+    folderVersionDAL,
+    secretVersionV2BridgeDAL,
+    projectDAL,
+    folderCheckpointResourcesDAL,
+    secretV2BridgeDAL,
+    folderTreeCheckpointResourcesDAL,
+    kmsService,
+    secretTagDAL,
+    resourceMetadataDAL
+  });
+
+  return { folderCommitService };
+};

backend/src/db/schemas/folder-checkpoint-resources.ts

@@ -0,0 +1,23 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const FolderCheckpointResourcesSchema = z.object({
+  id: z.string().uuid(),
+  folderCheckpointId: z.string().uuid(),
+  secretVersionId: z.string().uuid().nullable().optional(),
+  folderVersionId: z.string().uuid().nullable().optional(),
+  createdAt: z.date(),
+  updatedAt: z.date()
+});
+
+export type TFolderCheckpointResources = z.infer<typeof FolderCheckpointResourcesSchema>;
+export type TFolderCheckpointResourcesInsert = Omit<z.input<typeof FolderCheckpointResourcesSchema>, TImmutableDBKeys>;
+export type TFolderCheckpointResourcesUpdate = Partial<
+  Omit<z.input<typeof FolderCheckpointResourcesSchema>, TImmutableDBKeys>
+>;

backend/src/db/schemas/folder-checkpoints.ts

@@ -0,0 +1,19 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const FolderCheckpointsSchema = z.object({
+  id: z.string().uuid(),
+  folderCommitId: z.string().uuid(),
+  createdAt: z.date(),
+  updatedAt: z.date()
+});
+
+export type TFolderCheckpoints = z.infer<typeof FolderCheckpointsSchema>;
+export type TFolderCheckpointsInsert = Omit<z.input<typeof FolderCheckpointsSchema>, TImmutableDBKeys>;
+export type TFolderCheckpointsUpdate = Partial<Omit<z.input<typeof FolderCheckpointsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/folder-commit-changes.ts

@@ -0,0 +1,23 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const FolderCommitChangesSchema = z.object({
+  id: z.string().uuid(),
+  folderCommitId: z.string().uuid(),
+  changeType: z.string(),
+  isUpdate: z.boolean().default(false),
+  secretVersionId: z.string().uuid().nullable().optional(),
+  folderVersionId: z.string().uuid().nullable().optional(),
+  createdAt: z.date(),
+  updatedAt: z.date()
+});
+
+export type TFolderCommitChanges = z.infer<typeof FolderCommitChangesSchema>;
+export type TFolderCommitChangesInsert = Omit<z.input<typeof FolderCommitChangesSchema>, TImmutableDBKeys>;
+export type TFolderCommitChangesUpdate = Partial<Omit<z.input<typeof FolderCommitChangesSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/folder-commits.ts

@@ -0,0 +1,24 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const FolderCommitsSchema = z.object({
+  id: z.string().uuid(),
+  commitId: z.coerce.bigint(),
+  actorMetadata: z.unknown(),
+  actorType: z.string(),
+  message: z.string().nullable().optional(),
+  folderId: z.string().uuid(),
+  envId: z.string().uuid(),
+  createdAt: z.date(),
+  updatedAt: z.date()
+});
+
+export type TFolderCommits = z.infer<typeof FolderCommitsSchema>;
+export type TFolderCommitsInsert = Omit<z.input<typeof FolderCommitsSchema>, TImmutableDBKeys>;
+export type TFolderCommitsUpdate = Partial<Omit<z.input<typeof FolderCommitsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/folder-tree-checkpoint-resources.ts

@@ -0,0 +1,26 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const FolderTreeCheckpointResourcesSchema = z.object({
+  id: z.string().uuid(),
+  folderTreeCheckpointId: z.string().uuid(),
+  folderId: z.string().uuid(),
+  folderCommitId: z.string().uuid(),
+  createdAt: z.date(),
+  updatedAt: z.date()
+});
+
+export type TFolderTreeCheckpointResources = z.infer<typeof FolderTreeCheckpointResourcesSchema>;
+export type TFolderTreeCheckpointResourcesInsert = Omit<
+  z.input<typeof FolderTreeCheckpointResourcesSchema>,
+  TImmutableDBKeys
+>;
+export type TFolderTreeCheckpointResourcesUpdate = Partial<
+  Omit<z.input<typeof FolderTreeCheckpointResourcesSchema>, TImmutableDBKeys>
+>;

backend/src/db/schemas/folder-tree-checkpoints.ts

@@ -0,0 +1,19 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const FolderTreeCheckpointsSchema = z.object({
+  id: z.string().uuid(),
+  folderCommitId: z.string().uuid(),
+  createdAt: z.date(),
+  updatedAt: z.date()
+});
+
+export type TFolderTreeCheckpoints = z.infer<typeof FolderTreeCheckpointsSchema>;
+export type TFolderTreeCheckpointsInsert = Omit<z.input<typeof FolderTreeCheckpointsSchema>, TImmutableDBKeys>;
+export type TFolderTreeCheckpointsUpdate = Partial<Omit<z.input<typeof FolderTreeCheckpointsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/index.ts

@@ -23,6 +23,12 @@ export * from "./dynamic-secrets";
 export * from "./external-certificate-authorities";
 export * from "./external-group-org-role-mappings";
 export * from "./external-kms";
+export * from "./folder-checkpoint-resources";
+export * from "./folder-checkpoints";
+export * from "./folder-commit-changes";
+export * from "./folder-commits";
+export * from "./folder-tree-checkpoint-resources";
+export * from "./folder-tree-checkpoints";
 export * from "./gateways";
 export * from "./git-app-install-sessions";
 export * from "./git-app-org";

backend/src/db/schemas/models.ts

@@ -157,10 +157,16 @@ export enum TableName {
   MicrosoftTeamsIntegrations = "microsoft_teams_integrations",
   ProjectMicrosoftTeamsConfigs = "project_microsoft_teams_configs",
   SecretReminderRecipients = "secret_reminder_recipients",
-  GithubOrgSyncConfig = "github_org_sync_configs"
+  GithubOrgSyncConfig = "github_org_sync_configs",
+  FolderCommit = "folder_commits",
+  FolderCommitChanges = "folder_commit_changes",
+  FolderCheckpoint = "folder_checkpoints",
+  FolderCheckpointResources = "folder_checkpoint_resources",
+  FolderTreeCheckpoint = "folder_tree_checkpoints",
+  FolderTreeCheckpointResources = "folder_tree_checkpoint_resources"
 }
 
-export type TImmutableDBKeys = "id" | "createdAt" | "updatedAt";
+export type TImmutableDBKeys = "id" | "createdAt" | "updatedAt" | "commitId";
 
 export const UserDeviceSchema = z
   .object({

backend/src/db/schemas/secret-folder-versions.ts

@@ -14,7 +14,8 @@ export const SecretFolderVersionsSchema = z.object({
   createdAt: z.date(),
   updatedAt: z.date(),
   envId: z.string().uuid(),
-  folderId: z.string().uuid()
+  folderId: z.string().uuid(),
+  description: z.string().nullable().optional()
 });
 
 export type TSecretFolderVersions = z.infer<typeof SecretFolderVersionsSchema>;