diff --git a/backend/scripts/generate-schema-types.ts b/backend/scripts/generate-schema-types.ts index fc398c2ac..c0e18a763 100644 --- a/backend/scripts/generate-schema-types.ts +++ b/backend/scripts/generate-schema-types.ts @@ -84,6 +84,11 @@ const getZodDefaultValue = (type: unknown, value: string | number | boolean | Ob } }; +const bigIntegerColumns: Record = { + "folder_commits": ["commitId"] +}; + + const main = async () => { const tables = ( await db("information_schema.tables") @@ -108,6 +113,9 @@ const main = async () => { const columnName = columnNames[colNum]; const colInfo = columns[columnName]; let ztype = getZodPrimitiveType(colInfo.type); + if (bigIntegerColumns[tableName]?.includes(columnName)) { + ztype = "z.coerce.bigint()"; + } if (["zodBuffer"].includes(ztype)) { zodImportSet.add(ztype); } diff --git a/backend/src/@types/fastify.d.ts b/backend/src/@types/fastify.d.ts index 344d1e02e..9f825b6b6 100644 --- a/backend/src/@types/fastify.d.ts +++ b/backend/src/@types/fastify.d.ts @@ -26,6 +26,7 @@ import { TLdapConfigServiceFactory } from "@app/ee/services/ldap-config/ldap-con import { TLicenseServiceFactory } from "@app/ee/services/license/license-service"; import { TOidcConfigServiceFactory } from "@app/ee/services/oidc/oidc-config-service"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; +import { TPitServiceFactory } from "@app/ee/services/pit/pit-service"; import { TProjectTemplateServiceFactory } from "@app/ee/services/project-template/project-template-service"; import { TProjectUserAdditionalPrivilegeServiceFactory } from "@app/ee/services/project-user-additional-privilege/project-user-additional-privilege-service"; import { TRateLimitServiceFactory } from "@app/ee/services/rate-limit/rate-limit-service"; @@ -59,6 +60,7 @@ import { TCertificateTemplateServiceFactory } from "@app/services/certificate-te import { TCmekServiceFactory } from "@app/services/cmek/cmek-service"; import { TExternalGroupOrgRoleMappingServiceFactory } from "@app/services/external-group-org-role-mapping/external-group-org-role-mapping-service"; import { TExternalMigrationServiceFactory } from "@app/services/external-migration/external-migration-service"; +import { TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { TGroupProjectServiceFactory } from "@app/services/group-project/group-project-service"; import { THsmServiceFactory } from "@app/services/hsm/hsm-service"; import { TIdentityServiceFactory } from "@app/services/identity/identity-service"; @@ -276,6 +278,8 @@ declare module "fastify" { microsoftTeams: TMicrosoftTeamsServiceFactory; assumePrivileges: TAssumePrivilegeServiceFactory; githubOrgSync: TGithubOrgSyncServiceFactory; + folderCommit: TFolderCommitServiceFactory; + pit: TPitServiceFactory; secretScanningV2: TSecretScanningV2ServiceFactory; internalCertificateAuthority: TInternalCertificateAuthorityServiceFactory; pkiTemplate: TPkiTemplatesServiceFactory; diff --git a/backend/src/@types/knex.d.ts b/backend/src/@types/knex.d.ts index 44cd6bc79..8fd073a49 100644 --- a/backend/src/@types/knex.d.ts +++ b/backend/src/@types/knex.d.ts @@ -80,6 +80,24 @@ import { TExternalKms, TExternalKmsInsert, TExternalKmsUpdate, + TFolderCheckpointResources, + TFolderCheckpointResourcesInsert, + TFolderCheckpointResourcesUpdate, + TFolderCheckpoints, + TFolderCheckpointsInsert, + TFolderCheckpointsUpdate, + TFolderCommitChanges, + TFolderCommitChangesInsert, + TFolderCommitChangesUpdate, + TFolderCommits, + TFolderCommitsInsert, + TFolderCommitsUpdate, + TFolderTreeCheckpointResources, + TFolderTreeCheckpointResourcesInsert, + TFolderTreeCheckpointResourcesUpdate, + TFolderTreeCheckpoints, + TFolderTreeCheckpointsInsert, + TFolderTreeCheckpointsUpdate, TGateways, TGatewaysInsert, TGatewaysUpdate, @@ -1122,6 +1140,36 @@ declare module "knex/types/tables" { TGithubOrgSyncConfigsInsert, TGithubOrgSyncConfigsUpdate >; + [TableName.FolderCommit]: KnexOriginal.CompositeTableType< + TFolderCommits, + TFolderCommitsInsert, + TFolderCommitsUpdate + >; + [TableName.FolderCommitChanges]: KnexOriginal.CompositeTableType< + TFolderCommitChanges, + TFolderCommitChangesInsert, + TFolderCommitChangesUpdate + >; + [TableName.FolderCheckpoint]: KnexOriginal.CompositeTableType< + TFolderCheckpoints, + TFolderCheckpointsInsert, + TFolderCheckpointsUpdate + >; + [TableName.FolderCheckpointResources]: KnexOriginal.CompositeTableType< + TFolderCheckpointResources, + TFolderCheckpointResourcesInsert, + TFolderCheckpointResourcesUpdate + >; + [TableName.FolderTreeCheckpoint]: KnexOriginal.CompositeTableType< + TFolderTreeCheckpoints, + TFolderTreeCheckpointsInsert, + TFolderTreeCheckpointsUpdate + >; + [TableName.FolderTreeCheckpointResources]: KnexOriginal.CompositeTableType< + TFolderTreeCheckpointResources, + TFolderTreeCheckpointResourcesInsert, + TFolderTreeCheckpointResourcesUpdate + >; [TableName.SecretScanningDataSource]: KnexOriginal.CompositeTableType< TSecretScanningDataSources, TSecretScanningDataSourcesInsert, diff --git a/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts b/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts new file mode 100644 index 000000000..30d1ec49b --- /dev/null +++ b/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts @@ -0,0 +1,166 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; +import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils"; + +export async function up(knex: Knex): Promise { + const hasFolderCommitTable = await knex.schema.hasTable(TableName.FolderCommit); + if (!hasFolderCommitTable) { + await knex.schema.createTable(TableName.FolderCommit, (t) => { + t.uuid("id").primary().defaultTo(knex.fn.uuid()); + t.bigIncrements("commitId"); + t.jsonb("actorMetadata").notNullable(); + t.string("actorType").notNullable(); + t.string("message"); + t.uuid("folderId").notNullable(); + t.uuid("envId").notNullable(); + t.foreign("envId").references("id").inTable(TableName.Environment).onDelete("CASCADE"); + t.timestamps(true, true, true); + + t.index("folderId"); + t.index("envId"); + }); + } + + const hasFolderCommitChangesTable = await knex.schema.hasTable(TableName.FolderCommitChanges); + if (!hasFolderCommitChangesTable) { + await knex.schema.createTable(TableName.FolderCommitChanges, (t) => { + t.uuid("id").primary().defaultTo(knex.fn.uuid()); + t.uuid("folderCommitId").notNullable(); + t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE"); + t.string("changeType").notNullable(); + t.boolean("isUpdate").notNullable().defaultTo(false); + t.uuid("secretVersionId"); + t.foreign("secretVersionId").references("id").inTable(TableName.SecretVersionV2).onDelete("CASCADE"); + t.uuid("folderVersionId"); + t.foreign("folderVersionId").references("id").inTable(TableName.SecretFolderVersion).onDelete("CASCADE"); + t.timestamps(true, true, true); + + t.index("folderCommitId"); + t.index("secretVersionId"); + t.index("folderVersionId"); + }); + } + + const hasFolderCheckpointTable = await knex.schema.hasTable(TableName.FolderCheckpoint); + if (!hasFolderCheckpointTable) { + await knex.schema.createTable(TableName.FolderCheckpoint, (t) => { + t.uuid("id").primary().defaultTo(knex.fn.uuid()); + t.uuid("folderCommitId").notNullable(); + t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE"); + t.timestamps(true, true, true); + + t.index("folderCommitId"); + }); + } + + const hasFolderCheckpointResourcesTable = await knex.schema.hasTable(TableName.FolderCheckpointResources); + if (!hasFolderCheckpointResourcesTable) { + await knex.schema.createTable(TableName.FolderCheckpointResources, (t) => { + t.uuid("id").primary().defaultTo(knex.fn.uuid()); + t.uuid("folderCheckpointId").notNullable(); + t.foreign("folderCheckpointId").references("id").inTable(TableName.FolderCheckpoint).onDelete("CASCADE"); + t.uuid("secretVersionId"); + t.foreign("secretVersionId").references("id").inTable(TableName.SecretVersionV2).onDelete("CASCADE"); + t.uuid("folderVersionId"); + t.foreign("folderVersionId").references("id").inTable(TableName.SecretFolderVersion).onDelete("CASCADE"); + t.timestamps(true, true, true); + + t.index("folderCheckpointId"); + t.index("secretVersionId"); + t.index("folderVersionId"); + }); + } + + const hasFolderTreeCheckpointTable = await knex.schema.hasTable(TableName.FolderTreeCheckpoint); + if (!hasFolderTreeCheckpointTable) { + await knex.schema.createTable(TableName.FolderTreeCheckpoint, (t) => { + t.uuid("id").primary().defaultTo(knex.fn.uuid()); + t.uuid("folderCommitId").notNullable(); + t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE"); + t.timestamps(true, true, true); + + t.index("folderCommitId"); + }); + } + + const hasFolderTreeCheckpointResourcesTable = await knex.schema.hasTable(TableName.FolderTreeCheckpointResources); + if (!hasFolderTreeCheckpointResourcesTable) { + await knex.schema.createTable(TableName.FolderTreeCheckpointResources, (t) => { + t.uuid("id").primary().defaultTo(knex.fn.uuid()); + t.uuid("folderTreeCheckpointId").notNullable(); + t.foreign("folderTreeCheckpointId").references("id").inTable(TableName.FolderTreeCheckpoint).onDelete("CASCADE"); + t.uuid("folderId").notNullable(); + t.uuid("folderCommitId").notNullable(); + t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE"); + t.timestamps(true, true, true); + + t.index("folderTreeCheckpointId"); + t.index("folderId"); + t.index("folderCommitId"); + }); + } + + if (!hasFolderCommitTable) { + await createOnUpdateTrigger(knex, TableName.FolderCommit); + } + + if (!hasFolderCommitChangesTable) { + await createOnUpdateTrigger(knex, TableName.FolderCommitChanges); + } + + if (!hasFolderCheckpointTable) { + await createOnUpdateTrigger(knex, TableName.FolderCheckpoint); + } + + if (!hasFolderCheckpointResourcesTable) { + await createOnUpdateTrigger(knex, TableName.FolderCheckpointResources); + } + + if (!hasFolderTreeCheckpointTable) { + await createOnUpdateTrigger(knex, TableName.FolderTreeCheckpoint); + } + + if (!hasFolderTreeCheckpointResourcesTable) { + await createOnUpdateTrigger(knex, TableName.FolderTreeCheckpointResources); + } +} + +export async function down(knex: Knex): Promise { + const hasFolderCheckpointResourcesTable = await knex.schema.hasTable(TableName.FolderCheckpointResources); + const hasFolderTreeCheckpointResourcesTable = await knex.schema.hasTable(TableName.FolderTreeCheckpointResources); + const hasFolderCommitTable = await knex.schema.hasTable(TableName.FolderCommit); + const hasFolderCommitChangesTable = await knex.schema.hasTable(TableName.FolderCommitChanges); + const hasFolderTreeCheckpointTable = await knex.schema.hasTable(TableName.FolderTreeCheckpoint); + const hasFolderCheckpointTable = await knex.schema.hasTable(TableName.FolderCheckpoint); + + if (hasFolderTreeCheckpointResourcesTable) { + await dropOnUpdateTrigger(knex, TableName.FolderTreeCheckpointResources); + await knex.schema.dropTableIfExists(TableName.FolderTreeCheckpointResources); + } + + if (hasFolderCheckpointResourcesTable) { + await dropOnUpdateTrigger(knex, TableName.FolderCheckpointResources); + await knex.schema.dropTableIfExists(TableName.FolderCheckpointResources); + } + + if (hasFolderTreeCheckpointTable) { + await dropOnUpdateTrigger(knex, TableName.FolderTreeCheckpoint); + await knex.schema.dropTableIfExists(TableName.FolderTreeCheckpoint); + } + + if (hasFolderCheckpointTable) { + await dropOnUpdateTrigger(knex, TableName.FolderCheckpoint); + await knex.schema.dropTableIfExists(TableName.FolderCheckpoint); + } + + if (hasFolderCommitChangesTable) { + await dropOnUpdateTrigger(knex, TableName.FolderCommitChanges); + await knex.schema.dropTableIfExists(TableName.FolderCommitChanges); + } + + if (hasFolderCommitTable) { + await dropOnUpdateTrigger(knex, TableName.FolderCommit); + await knex.schema.dropTableIfExists(TableName.FolderCommit); + } +} diff --git a/backend/src/db/migrations/20250528110936_add-folder-description-to-versioning.ts b/backend/src/db/migrations/20250528110936_add-folder-description-to-versioning.ts new file mode 100644 index 000000000..8c1da7ebc --- /dev/null +++ b/backend/src/db/migrations/20250528110936_add-folder-description-to-versioning.ts @@ -0,0 +1,19 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + if (!(await knex.schema.hasColumn(TableName.SecretFolderVersion, "description"))) { + await knex.schema.alterTable(TableName.SecretFolderVersion, (t) => { + t.string("description").nullable(); + }); + } +} + +export async function down(knex: Knex): Promise { + if (await knex.schema.hasColumn(TableName.SecretFolderVersion, "description")) { + await knex.schema.alterTable(TableName.SecretFolderVersion, (t) => { + t.dropColumn("description"); + }); + } +} diff --git a/backend/src/db/migrations/20250602155451_fix-secret-versions.ts b/backend/src/db/migrations/20250602155451_fix-secret-versions.ts new file mode 100644 index 000000000..f525e85f3 --- /dev/null +++ b/backend/src/db/migrations/20250602155451_fix-secret-versions.ts @@ -0,0 +1,139 @@ +/* eslint-disable no-await-in-loop */ +import { Knex } from "knex"; + +import { chunkArray } from "@app/lib/fn"; +import { selectAllTableCols } from "@app/lib/knex"; +import { logger } from "@app/lib/logger"; + +import { SecretType, TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + logger.info("Starting secret version fix migration"); + + // Get all shared secret IDs first to optimize versions query + const secretIds = await knex(TableName.SecretV2) + .where("type", SecretType.Shared) + .select("id") + .then((rows) => rows.map((row) => row.id)); + + logger.info(`Found ${secretIds.length} shared secrets to process`); + + if (secretIds.length === 0) { + logger.info("No shared secrets found"); + return; + } + + const secretIdChunks = chunkArray(secretIds, 5000); + + for (let chunkIndex = 0; chunkIndex < secretIdChunks.length; chunkIndex += 1) { + const currentSecretIds = secretIdChunks[chunkIndex]; + logger.info(`Processing chunk ${chunkIndex + 1} of ${secretIdChunks.length}`); + + // Get secrets and versions for current chunk + const [sharedSecrets, allVersions] = await Promise.all([ + knex(TableName.SecretV2).whereIn("id", currentSecretIds).select(selectAllTableCols(TableName.SecretV2)), + knex(TableName.SecretVersionV2).whereIn("secretId", currentSecretIds).select("secretId", "version") + ]); + + const versionsBySecretId = new Map(); + + allVersions.forEach((v) => { + const versions = versionsBySecretId.get(v.secretId); + if (versions) { + versions.push(v.version); + } else { + versionsBySecretId.set(v.secretId, [v.version]); + } + }); + + const versionsToAdd = []; + const secretsToUpdate = []; + + // Process each shared secret + for (const secret of sharedSecrets) { + const existingVersions = versionsBySecretId.get(secret.id) || []; + + if (existingVersions.length === 0) { + // No versions exist - add current version + versionsToAdd.push({ + secretId: secret.id, + version: secret.version, + key: secret.key, + encryptedValue: secret.encryptedValue, + encryptedComment: secret.encryptedComment, + reminderNote: secret.reminderNote, + reminderRepeatDays: secret.reminderRepeatDays, + skipMultilineEncoding: secret.skipMultilineEncoding, + metadata: secret.metadata, + folderId: secret.folderId, + actorType: "platform" + }); + } else { + const latestVersion = Math.max(...existingVersions); + + if (latestVersion !== secret.version) { + // Latest version doesn't match - create new version and update secret + const nextVersion = latestVersion + 1; + + versionsToAdd.push({ + secretId: secret.id, + version: nextVersion, + key: secret.key, + encryptedValue: secret.encryptedValue, + encryptedComment: secret.encryptedComment, + reminderNote: secret.reminderNote, + reminderRepeatDays: secret.reminderRepeatDays, + skipMultilineEncoding: secret.skipMultilineEncoding, + metadata: secret.metadata, + folderId: secret.folderId, + actorType: "platform" + }); + + secretsToUpdate.push({ + id: secret.id, + newVersion: nextVersion + }); + } + } + } + + logger.info( + `Chunk ${chunkIndex + 1}: Adding ${versionsToAdd.length} versions, updating ${secretsToUpdate.length} secrets` + ); + + // Batch insert new versions + if (versionsToAdd.length > 0) { + const insertBatches = chunkArray(versionsToAdd, 9000); + for (let i = 0; i < insertBatches.length; i += 1) { + await knex.batchInsert(TableName.SecretVersionV2, insertBatches[i]); + } + } + + if (secretsToUpdate.length > 0) { + const updateBatches = chunkArray(secretsToUpdate, 1000); + + for (const updateBatch of updateBatches) { + const ids = updateBatch.map((u) => u.id); + const versionCases = updateBatch.map((u) => `WHEN '${u.id}' THEN ${u.newVersion}`).join(" "); + + await knex.raw( + ` + UPDATE ${TableName.SecretV2} + SET version = CASE id ${versionCases} END, + "updatedAt" = NOW() + WHERE id IN (${ids.map(() => "?").join(",")}) + `, + ids + ); + } + } + } + + logger.info("Secret version fix migration completed"); +} + +export async function down(): Promise { + logger.info("Rollback not implemented for secret version fix migration"); + // Note: Rolling back this migration would be complex and potentially destructive + // as it would require tracking which version entries were added +} diff --git a/backend/src/db/migrations/20250602155452_pit-projects-commits-initialization.ts b/backend/src/db/migrations/20250602155452_pit-projects-commits-initialization.ts new file mode 100644 index 000000000..dd4034c57 --- /dev/null +++ b/backend/src/db/migrations/20250602155452_pit-projects-commits-initialization.ts @@ -0,0 +1,345 @@ +import { Knex } from "knex"; + +import { chunkArray } from "@app/lib/fn"; +import { selectAllTableCols } from "@app/lib/knex"; +import { logger } from "@app/lib/logger"; +import { ActorType } from "@app/services/auth/auth-type"; +import { ChangeType } from "@app/services/folder-commit/folder-commit-service"; + +import { + ProjectType, + SecretType, + TableName, + TFolderCheckpoints, + TFolderCommits, + TFolderTreeCheckpoints, + TSecretFolders +} from "../schemas"; + +const sortFoldersByHierarchy = (folders: TSecretFolders[]) => { + // Create a map for quick lookup of children by parent ID + const childrenMap = new Map(); + + // Set of all folder IDs + const allFolderIds = new Set(); + + // Build the set of all folder IDs + folders.forEach((folder) => { + if (folder.id) { + allFolderIds.add(folder.id); + } + }); + + // Group folders by their parentId + folders.forEach((folder) => { + if (folder.parentId) { + const children = childrenMap.get(folder.parentId) || []; + children.push(folder); + childrenMap.set(folder.parentId, children); + } + }); + + // Find root folders - those with no parentId or with a parentId that doesn't exist + const rootFolders = folders.filter((folder) => !folder.parentId || !allFolderIds.has(folder.parentId)); + + // Process each level of the hierarchy + const result = []; + let currentLevel = rootFolders; + + while (currentLevel.length > 0) { + result.push(...currentLevel); + + const nextLevel = []; + for (const folder of currentLevel) { + if (folder.id) { + const children = childrenMap.get(folder.id) || []; + nextLevel.push(...children); + } + } + + currentLevel = nextLevel; + } + + return result.reverse(); +}; + +const getSecretsByFolderIds = async (knex: Knex, folderIds: string[]): Promise> => { + const secrets = await knex(TableName.SecretV2) + .whereIn(`${TableName.SecretV2}.folderId`, folderIds) + .where(`${TableName.SecretV2}.type`, SecretType.Shared) + .join(TableName.SecretVersionV2, (queryBuilder) => { + void queryBuilder + .on(`${TableName.SecretVersionV2}.secretId`, `${TableName.SecretV2}.id`) + .andOn(`${TableName.SecretVersionV2}.version`, `${TableName.SecretV2}.version`); + }) + .select(selectAllTableCols(TableName.SecretV2)) + .select(knex.ref("id").withSchema(TableName.SecretVersionV2).as("secretVersionId")); + + const secretsMap: Record = {}; + + secrets.forEach((secret) => { + if (!secretsMap[secret.folderId]) { + secretsMap[secret.folderId] = []; + } + secretsMap[secret.folderId].push(secret.secretVersionId); + }); + + return secretsMap; +}; + +const getFoldersByParentIds = async (knex: Knex, parentIds: string[]): Promise> => { + const folders = await knex(TableName.SecretFolder) + .whereIn(`${TableName.SecretFolder}.parentId`, parentIds) + .where(`${TableName.SecretFolder}.isReserved`, false) + .join(TableName.SecretFolderVersion, (queryBuilder) => { + void queryBuilder + .on(`${TableName.SecretFolderVersion}.folderId`, `${TableName.SecretFolder}.id`) + .andOn(`${TableName.SecretFolderVersion}.version`, `${TableName.SecretFolder}.version`); + }) + .select(selectAllTableCols(TableName.SecretFolder)) + .select(knex.ref("id").withSchema(TableName.SecretFolderVersion).as("folderVersionId")); + + const foldersMap: Record = {}; + + folders.forEach((folder) => { + if (!folder.parentId) { + return; + } + if (!foldersMap[folder.parentId]) { + foldersMap[folder.parentId] = []; + } + foldersMap[folder.parentId].push(folder.folderVersionId); + }); + + return foldersMap; +}; + +export async function up(knex: Knex): Promise { + logger.info("Initializing folder commits"); + const hasFolderCommitTable = await knex.schema.hasTable(TableName.FolderCommit); + if (hasFolderCommitTable) { + // Get Projects to Initialize + const projects = await knex(TableName.Project) + .where(`${TableName.Project}.version`, 3) + .where(`${TableName.Project}.type`, ProjectType.SecretManager) + .select(selectAllTableCols(TableName.Project)); + logger.info(`Found ${projects.length} projects to initialize`); + + // Process Projects in batches of 100 + const batches = chunkArray(projects, 100); + let i = 0; + for (const batch of batches) { + i += 1; + logger.info(`Processing project batch ${i} of ${batches.length}`); + let foldersCommitsList = []; + + const rootFoldersMap: Record = {}; + const envRootFoldersMap: Record = {}; + + // Get All Folders for the Project + // eslint-disable-next-line no-await-in-loop + const folders = await knex(TableName.SecretFolder) + .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) + .whereIn( + `${TableName.Environment}.projectId`, + batch.map((project) => project.id) + ) + .where(`${TableName.SecretFolder}.isReserved`, false) + .select(selectAllTableCols(TableName.SecretFolder)); + logger.info(`Found ${folders.length} folders to initialize in project batch ${i} of ${batches.length}`); + + // Sort Folders by Hierarchy (parents before nested folders) + const sortedFolders = sortFoldersByHierarchy(folders); + + // eslint-disable-next-line no-await-in-loop + const folderSecretsMap = await getSecretsByFolderIds( + knex, + sortedFolders.map((folder) => folder.id) + ); + // eslint-disable-next-line no-await-in-loop + const folderFoldersMap = await getFoldersByParentIds( + knex, + sortedFolders.map((folder) => folder.id) + ); + + // Get folder commit changes + for (const folder of sortedFolders) { + const subFolderVersionIds = folderFoldersMap[folder.id]; + const secretVersionIds = folderSecretsMap[folder.id]; + const changes = []; + if (subFolderVersionIds) { + changes.push( + ...subFolderVersionIds.map((folderVersionId) => ({ + folderId: folder.id, + changeType: ChangeType.ADD, + secretVersionId: undefined, + folderVersionId, + isUpdate: false + })) + ); + } + if (secretVersionIds) { + changes.push( + ...secretVersionIds.map((secretVersionId) => ({ + folderId: folder.id, + changeType: ChangeType.ADD, + secretVersionId, + folderVersionId: undefined, + isUpdate: false + })) + ); + } + if (changes.length > 0) { + const folderCommit = { + commit: { + actorMetadata: {}, + actorType: ActorType.PLATFORM, + message: "Initialized folder", + folderId: folder.id, + envId: folder.envId + }, + changes + }; + foldersCommitsList.push(folderCommit); + if (!folder.parentId) { + rootFoldersMap[folder.id] = folder.envId; + envRootFoldersMap[folder.envId] = folder.id; + } + } + } + logger.info(`Retrieved folder changes for project batch ${i} of ${batches.length}`); + + const filteredBrokenProjectFolders: string[] = []; + + foldersCommitsList = foldersCommitsList.filter((folderCommit) => { + if (!envRootFoldersMap[folderCommit.commit.envId]) { + filteredBrokenProjectFolders.push(folderCommit.commit.folderId); + return false; + } + return true; + }); + + logger.info( + `Filtered ${filteredBrokenProjectFolders.length} broken project folders: ${JSON.stringify(filteredBrokenProjectFolders)}` + ); + + // Insert New Commits in batches of 9000 + const newCommits = foldersCommitsList.map((folderCommit) => folderCommit.commit); + const commitBatches = chunkArray(newCommits, 9000); + + let j = 0; + for (const commitBatch of commitBatches) { + j += 1; + logger.info(`Inserting folder commits - batch ${j} of ${commitBatches.length}`); + // Create folder commit + // eslint-disable-next-line no-await-in-loop + const newCommitsInserted = (await knex + .batchInsert(TableName.FolderCommit, commitBatch) + .returning("*")) as TFolderCommits[]; + + logger.info(`Finished inserting folder commits - batch ${j} of ${commitBatches.length}`); + + const newCommitsMap: Record = {}; + const newCommitsMapInverted: Record = {}; + const newCheckpointsMap: Record = {}; + newCommitsInserted.forEach((commit) => { + newCommitsMap[commit.folderId] = commit.id; + newCommitsMapInverted[commit.id] = commit.folderId; + }); + + // Create folder checkpoints + // eslint-disable-next-line no-await-in-loop + const newCheckpoints = (await knex + .batchInsert( + TableName.FolderCheckpoint, + Object.values(newCommitsMap).map((commitId) => ({ + folderCommitId: commitId + })) + ) + .returning("*")) as TFolderCheckpoints[]; + + logger.info(`Finished inserting folder checkpoints - batch ${j} of ${commitBatches.length}`); + + newCheckpoints.forEach((checkpoint) => { + newCheckpointsMap[newCommitsMapInverted[checkpoint.folderCommitId]] = checkpoint.id; + }); + + // Create folder commit changes + // eslint-disable-next-line no-await-in-loop + await knex.batchInsert( + TableName.FolderCommitChanges, + foldersCommitsList + .map((folderCommit) => folderCommit.changes) + .flat() + .map((change) => ({ + folderCommitId: newCommitsMap[change.folderId], + changeType: change.changeType, + secretVersionId: change.secretVersionId, + folderVersionId: change.folderVersionId, + isUpdate: false + })) + ); + + logger.info(`Finished inserting folder commit changes - batch ${j} of ${commitBatches.length}`); + + // Create folder checkpoint resources + // eslint-disable-next-line no-await-in-loop + await knex.batchInsert( + TableName.FolderCheckpointResources, + foldersCommitsList + .map((folderCommit) => folderCommit.changes) + .flat() + .map((change) => ({ + folderCheckpointId: newCheckpointsMap[change.folderId], + folderVersionId: change.folderVersionId, + secretVersionId: change.secretVersionId + })) + ); + + logger.info(`Finished inserting folder checkpoint resources - batch ${j} of ${commitBatches.length}`); + + // Create Folder Tree Checkpoint + // eslint-disable-next-line no-await-in-loop + const newTreeCheckpoints = (await knex + .batchInsert( + TableName.FolderTreeCheckpoint, + Object.keys(rootFoldersMap).map((folderId) => ({ + folderCommitId: newCommitsMap[folderId] + })) + ) + .returning("*")) as TFolderTreeCheckpoints[]; + + logger.info(`Finished inserting folder tree checkpoints - batch ${j} of ${commitBatches.length}`); + + const newTreeCheckpointsMap: Record = {}; + newTreeCheckpoints.forEach((checkpoint) => { + newTreeCheckpointsMap[rootFoldersMap[newCommitsMapInverted[checkpoint.folderCommitId]]] = checkpoint.id; + }); + + // Create Folder Tree Checkpoint Resources + // eslint-disable-next-line no-await-in-loop + await knex + .batchInsert( + TableName.FolderTreeCheckpointResources, + newCommitsInserted.map((folderCommit) => ({ + folderTreeCheckpointId: newTreeCheckpointsMap[folderCommit.envId], + folderId: folderCommit.folderId, + folderCommitId: folderCommit.id + })) + ) + .returning("*"); + + logger.info(`Finished inserting folder tree checkpoint resources - batch ${j} of ${commitBatches.length}`); + } + } + } + logger.info("Folder commits initialized"); +} + +export async function down(knex: Knex): Promise { + const hasFolderCommitTable = await knex.schema.hasTable(TableName.FolderCommit); + if (hasFolderCommitTable) { + // delete all existing entries + await knex(TableName.FolderCommit).del(); + } +} diff --git a/backend/src/db/migrations/20250606134139_add-project-snapshots-legacy-option.ts b/backend/src/db/migrations/20250606134139_add-project-snapshots-legacy-option.ts new file mode 100644 index 000000000..f5f73d7fe --- /dev/null +++ b/backend/src/db/migrations/20250606134139_add-project-snapshots-legacy-option.ts @@ -0,0 +1,21 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + const hasShowSnapshotsLegacyColumn = await knex.schema.hasColumn(TableName.Project, "showSnapshotsLegacy"); + if (!hasShowSnapshotsLegacyColumn) { + await knex.schema.table(TableName.Project, (table) => { + table.boolean("showSnapshotsLegacy").notNullable().defaultTo(false); + }); + } +} + +export async function down(knex: Knex): Promise { + const hasShowSnapshotsLegacyColumn = await knex.schema.hasColumn(TableName.Project, "showSnapshotsLegacy"); + if (hasShowSnapshotsLegacyColumn) { + await knex.schema.table(TableName.Project, (table) => { + table.dropColumn("showSnapshotsLegacy"); + }); + } +} diff --git a/backend/src/db/migrations/20250610143920_add-dynamic-secret-lease-config.ts b/backend/src/db/migrations/20250610143920_add-dynamic-secret-lease-config.ts new file mode 100644 index 000000000..30d6f1854 --- /dev/null +++ b/backend/src/db/migrations/20250610143920_add-dynamic-secret-lease-config.ts @@ -0,0 +1,21 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + const hasConfigColumn = await knex.schema.hasColumn(TableName.DynamicSecretLease, "config"); + if (!hasConfigColumn) { + await knex.schema.alterTable(TableName.DynamicSecretLease, (table) => { + table.jsonb("config"); + }); + } +} + +export async function down(knex: Knex): Promise { + const hasConfigColumn = await knex.schema.hasColumn(TableName.DynamicSecretLease, "config"); + if (hasConfigColumn) { + await knex.schema.alterTable(TableName.DynamicSecretLease, (table) => { + table.dropColumn("config"); + }); + } +} diff --git a/backend/src/db/migrations/20250610173646_identity-kubernetes-auth-optional-host-url.ts b/backend/src/db/migrations/20250610173646_identity-kubernetes-auth-optional-host-url.ts new file mode 100644 index 000000000..6d2dd6664 --- /dev/null +++ b/backend/src/db/migrations/20250610173646_identity-kubernetes-auth-optional-host-url.ts @@ -0,0 +1,45 @@ +import { Knex } from "knex"; + +import { selectAllTableCols } from "@app/lib/knex"; + +import { TableName } from "../schemas"; + +const BATCH_SIZE = 1000; + +export async function up(knex: Knex): Promise { + const hasKubernetesHostColumn = await knex.schema.hasColumn(TableName.IdentityKubernetesAuth, "kubernetesHost"); + + if (hasKubernetesHostColumn) { + await knex.schema.alterTable(TableName.IdentityKubernetesAuth, (table) => { + table.string("kubernetesHost").nullable().alter(); + }); + } +} + +export async function down(knex: Knex): Promise { + const hasKubernetesHostColumn = await knex.schema.hasColumn(TableName.IdentityKubernetesAuth, "kubernetesHost"); + + // find all rows where kubernetesHost is null + const rows = await knex(TableName.IdentityKubernetesAuth) + .whereNull("kubernetesHost") + .select(selectAllTableCols(TableName.IdentityKubernetesAuth)); + + if (rows.length > 0) { + for (let i = 0; i < rows.length; i += BATCH_SIZE) { + const batch = rows.slice(i, i + BATCH_SIZE); + // eslint-disable-next-line no-await-in-loop + await knex(TableName.IdentityKubernetesAuth) + .whereIn( + "id", + batch.map((row) => row.id) + ) + .update({ kubernetesHost: "" }); + } + } + + if (hasKubernetesHostColumn) { + await knex.schema.alterTable(TableName.IdentityKubernetesAuth, (table) => { + table.string("kubernetesHost").notNullable().alter(); + }); + } +} diff --git a/backend/src/db/migrations/utils/services.ts b/backend/src/db/migrations/utils/services.ts index 731f703e2..0e071e6fe 100644 --- a/backend/src/db/migrations/utils/services.ts +++ b/backend/src/db/migrations/utils/services.ts @@ -3,12 +3,27 @@ import { Knex } from "knex"; import { initializeHsmModule } from "@app/ee/services/hsm/hsm-fns"; import { hsmServiceFactory } from "@app/ee/services/hsm/hsm-service"; import { TKeyStoreFactory } from "@app/keystore/keystore"; +import { folderCheckpointDALFactory } from "@app/services/folder-checkpoint/folder-checkpoint-dal"; +import { folderCheckpointResourcesDALFactory } from "@app/services/folder-checkpoint-resources/folder-checkpoint-resources-dal"; +import { folderCommitDALFactory } from "@app/services/folder-commit/folder-commit-dal"; +import { folderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; +import { folderCommitChangesDALFactory } from "@app/services/folder-commit-changes/folder-commit-changes-dal"; +import { folderTreeCheckpointDALFactory } from "@app/services/folder-tree-checkpoint/folder-tree-checkpoint-dal"; +import { folderTreeCheckpointResourcesDALFactory } from "@app/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal"; +import { identityDALFactory } from "@app/services/identity/identity-dal"; import { internalKmsDALFactory } from "@app/services/kms/internal-kms-dal"; import { kmskeyDALFactory } from "@app/services/kms/kms-key-dal"; import { kmsRootConfigDALFactory } from "@app/services/kms/kms-root-config-dal"; import { kmsServiceFactory } from "@app/services/kms/kms-service"; import { orgDALFactory } from "@app/services/org/org-dal"; import { projectDALFactory } from "@app/services/project/project-dal"; +import { resourceMetadataDALFactory } from "@app/services/resource-metadata/resource-metadata-dal"; +import { secretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; +import { secretFolderVersionDALFactory } from "@app/services/secret-folder/secret-folder-version-dal"; +import { secretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; +import { secretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal"; +import { secretVersionV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-version-dal"; +import { userDALFactory } from "@app/services/user/user-dal"; import { TMigrationEnvConfig } from "./env-config"; @@ -50,3 +65,77 @@ export const getMigrationEncryptionServices = async ({ envConfig, db, keyStore } return { kmsService }; }; + +export const getMigrationPITServices = async ({ + db, + keyStore, + envConfig +}: { + db: Knex; + keyStore: TKeyStoreFactory; + envConfig: TMigrationEnvConfig; +}) => { + const projectDAL = projectDALFactory(db); + const folderCommitDAL = folderCommitDALFactory(db); + const folderCommitChangesDAL = folderCommitChangesDALFactory(db); + const folderCheckpointDAL = folderCheckpointDALFactory(db); + const folderTreeCheckpointDAL = folderTreeCheckpointDALFactory(db); + const userDAL = userDALFactory(db); + const identityDAL = identityDALFactory(db); + const folderDAL = secretFolderDALFactory(db); + const folderVersionDAL = secretFolderVersionDALFactory(db); + const secretVersionV2BridgeDAL = secretVersionV2BridgeDALFactory(db); + const folderCheckpointResourcesDAL = folderCheckpointResourcesDALFactory(db); + const secretV2BridgeDAL = secretV2BridgeDALFactory({ db, keyStore }); + const folderTreeCheckpointResourcesDAL = folderTreeCheckpointResourcesDALFactory(db); + const secretTagDAL = secretTagDALFactory(db); + + const orgDAL = orgDALFactory(db); + const kmsRootConfigDAL = kmsRootConfigDALFactory(db); + const kmsDAL = kmskeyDALFactory(db); + const internalKmsDAL = internalKmsDALFactory(db); + const resourceMetadataDAL = resourceMetadataDALFactory(db); + + const hsmModule = initializeHsmModule(envConfig); + hsmModule.initialize(); + + const hsmService = hsmServiceFactory({ + hsmModule: hsmModule.getModule(), + envConfig + }); + + const kmsService = kmsServiceFactory({ + kmsRootConfigDAL, + keyStore, + kmsDAL, + internalKmsDAL, + orgDAL, + projectDAL, + hsmService, + envConfig + }); + + await hsmService.startService(); + await kmsService.startService(); + + const folderCommitService = folderCommitServiceFactory({ + folderCommitDAL, + folderCommitChangesDAL, + folderCheckpointDAL, + folderTreeCheckpointDAL, + userDAL, + identityDAL, + folderDAL, + folderVersionDAL, + secretVersionV2BridgeDAL, + projectDAL, + folderCheckpointResourcesDAL, + secretV2BridgeDAL, + folderTreeCheckpointResourcesDAL, + kmsService, + secretTagDAL, + resourceMetadataDAL + }); + + return { folderCommitService }; +}; diff --git a/backend/src/db/schemas/dynamic-secret-leases.ts b/backend/src/db/schemas/dynamic-secret-leases.ts index 8c16bcb55..ef16b1a30 100644 --- a/backend/src/db/schemas/dynamic-secret-leases.ts +++ b/backend/src/db/schemas/dynamic-secret-leases.ts @@ -16,7 +16,8 @@ export const DynamicSecretLeasesSchema = z.object({ statusDetails: z.string().nullable().optional(), dynamicSecretId: z.string().uuid(), createdAt: z.date(), - updatedAt: z.date() + updatedAt: z.date(), + config: z.unknown().nullable().optional() }); export type TDynamicSecretLeases = z.infer; diff --git a/backend/src/db/schemas/folder-checkpoint-resources.ts b/backend/src/db/schemas/folder-checkpoint-resources.ts new file mode 100644 index 000000000..5fa8215cf --- /dev/null +++ b/backend/src/db/schemas/folder-checkpoint-resources.ts @@ -0,0 +1,23 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const FolderCheckpointResourcesSchema = z.object({ + id: z.string().uuid(), + folderCheckpointId: z.string().uuid(), + secretVersionId: z.string().uuid().nullable().optional(), + folderVersionId: z.string().uuid().nullable().optional(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TFolderCheckpointResources = z.infer; +export type TFolderCheckpointResourcesInsert = Omit, TImmutableDBKeys>; +export type TFolderCheckpointResourcesUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/folder-checkpoints.ts b/backend/src/db/schemas/folder-checkpoints.ts new file mode 100644 index 000000000..ba0ce6f71 --- /dev/null +++ b/backend/src/db/schemas/folder-checkpoints.ts @@ -0,0 +1,19 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const FolderCheckpointsSchema = z.object({ + id: z.string().uuid(), + folderCommitId: z.string().uuid(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TFolderCheckpoints = z.infer; +export type TFolderCheckpointsInsert = Omit, TImmutableDBKeys>; +export type TFolderCheckpointsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/folder-commit-changes.ts b/backend/src/db/schemas/folder-commit-changes.ts new file mode 100644 index 000000000..2bee0c5b3 --- /dev/null +++ b/backend/src/db/schemas/folder-commit-changes.ts @@ -0,0 +1,23 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const FolderCommitChangesSchema = z.object({ + id: z.string().uuid(), + folderCommitId: z.string().uuid(), + changeType: z.string(), + isUpdate: z.boolean().default(false), + secretVersionId: z.string().uuid().nullable().optional(), + folderVersionId: z.string().uuid().nullable().optional(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TFolderCommitChanges = z.infer; +export type TFolderCommitChangesInsert = Omit, TImmutableDBKeys>; +export type TFolderCommitChangesUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/folder-commits.ts b/backend/src/db/schemas/folder-commits.ts new file mode 100644 index 000000000..ade480eda --- /dev/null +++ b/backend/src/db/schemas/folder-commits.ts @@ -0,0 +1,24 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const FolderCommitsSchema = z.object({ + id: z.string().uuid(), + commitId: z.coerce.bigint(), + actorMetadata: z.unknown(), + actorType: z.string(), + message: z.string().nullable().optional(), + folderId: z.string().uuid(), + envId: z.string().uuid(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TFolderCommits = z.infer; +export type TFolderCommitsInsert = Omit, TImmutableDBKeys>; +export type TFolderCommitsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/folder-tree-checkpoint-resources.ts b/backend/src/db/schemas/folder-tree-checkpoint-resources.ts new file mode 100644 index 000000000..06d5770cc --- /dev/null +++ b/backend/src/db/schemas/folder-tree-checkpoint-resources.ts @@ -0,0 +1,26 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const FolderTreeCheckpointResourcesSchema = z.object({ + id: z.string().uuid(), + folderTreeCheckpointId: z.string().uuid(), + folderId: z.string().uuid(), + folderCommitId: z.string().uuid(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TFolderTreeCheckpointResources = z.infer; +export type TFolderTreeCheckpointResourcesInsert = Omit< + z.input, + TImmutableDBKeys +>; +export type TFolderTreeCheckpointResourcesUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/folder-tree-checkpoints.ts b/backend/src/db/schemas/folder-tree-checkpoints.ts new file mode 100644 index 000000000..ea500af6b --- /dev/null +++ b/backend/src/db/schemas/folder-tree-checkpoints.ts @@ -0,0 +1,19 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const FolderTreeCheckpointsSchema = z.object({ + id: z.string().uuid(), + folderCommitId: z.string().uuid(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TFolderTreeCheckpoints = z.infer; +export type TFolderTreeCheckpointsInsert = Omit, TImmutableDBKeys>; +export type TFolderTreeCheckpointsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/identity-kubernetes-auths.ts b/backend/src/db/schemas/identity-kubernetes-auths.ts index 8a351014a..deb78bf8a 100644 --- a/backend/src/db/schemas/identity-kubernetes-auths.ts +++ b/backend/src/db/schemas/identity-kubernetes-auths.ts @@ -18,7 +18,7 @@ export const IdentityKubernetesAuthsSchema = z.object({ createdAt: z.date(), updatedAt: z.date(), identityId: z.string().uuid(), - kubernetesHost: z.string(), + kubernetesHost: z.string().nullable().optional(), encryptedCaCert: z.string().nullable().optional(), caCertIV: z.string().nullable().optional(), caCertTag: z.string().nullable().optional(), diff --git a/backend/src/db/schemas/index.ts b/backend/src/db/schemas/index.ts index 6743a23cc..7a27caf9e 100644 --- a/backend/src/db/schemas/index.ts +++ b/backend/src/db/schemas/index.ts @@ -24,6 +24,12 @@ export * from "./dynamic-secrets"; export * from "./external-certificate-authorities"; export * from "./external-group-org-role-mappings"; export * from "./external-kms"; +export * from "./folder-checkpoint-resources"; +export * from "./folder-checkpoints"; +export * from "./folder-commit-changes"; +export * from "./folder-commits"; +export * from "./folder-tree-checkpoint-resources"; +export * from "./folder-tree-checkpoints"; export * from "./gateways"; export * from "./git-app-install-sessions"; export * from "./git-app-org"; diff --git a/backend/src/db/schemas/models.ts b/backend/src/db/schemas/models.ts index 6722ce235..df0a858b9 100644 --- a/backend/src/db/schemas/models.ts +++ b/backend/src/db/schemas/models.ts @@ -160,6 +160,12 @@ export enum TableName { ProjectMicrosoftTeamsConfigs = "project_microsoft_teams_configs", SecretReminderRecipients = "secret_reminder_recipients", GithubOrgSyncConfig = "github_org_sync_configs", + FolderCommit = "folder_commits", + FolderCommitChanges = "folder_commit_changes", + FolderCheckpoint = "folder_checkpoints", + FolderCheckpointResources = "folder_checkpoint_resources", + FolderTreeCheckpoint = "folder_tree_checkpoints", + FolderTreeCheckpointResources = "folder_tree_checkpoint_resources", SecretScanningDataSource = "secret_scanning_data_sources", SecretScanningResource = "secret_scanning_resources", SecretScanningScan = "secret_scanning_scans", @@ -167,7 +173,7 @@ export enum TableName { SecretScanningConfig = "secret_scanning_configs" } -export type TImmutableDBKeys = "id" | "createdAt" | "updatedAt"; +export type TImmutableDBKeys = "id" | "createdAt" | "updatedAt" | "commitId"; export const UserDeviceSchema = z .object({ diff --git a/backend/src/db/schemas/projects.ts b/backend/src/db/schemas/projects.ts index c1e96e8ce..b4c98d8a2 100644 --- a/backend/src/db/schemas/projects.ts +++ b/backend/src/db/schemas/projects.ts @@ -28,7 +28,8 @@ export const ProjectsSchema = z.object({ type: z.string(), enforceCapitalization: z.boolean().default(false), hasDeleteProtection: z.boolean().default(false).nullable().optional(), - secretSharing: z.boolean().default(true) + secretSharing: z.boolean().default(true), + showSnapshotsLegacy: z.boolean().default(false) }); export type TProjects = z.infer; diff --git a/backend/src/db/schemas/secret-folder-versions.ts b/backend/src/db/schemas/secret-folder-versions.ts index 8bef6e83f..3d444c566 100644 --- a/backend/src/db/schemas/secret-folder-versions.ts +++ b/backend/src/db/schemas/secret-folder-versions.ts @@ -14,7 +14,8 @@ export const SecretFolderVersionsSchema = z.object({ createdAt: z.date(), updatedAt: z.date(), envId: z.string().uuid(), - folderId: z.string().uuid() + folderId: z.string().uuid(), + description: z.string().nullable().optional() }); export type TSecretFolderVersions = z.infer; diff --git a/backend/src/ee/routes/v1/dynamic-secret-lease-router.ts b/backend/src/ee/routes/v1/dynamic-secret-lease-router.ts index 7c42c7f99..26c27d0d3 100644 --- a/backend/src/ee/routes/v1/dynamic-secret-lease-router.ts +++ b/backend/src/ee/routes/v1/dynamic-secret-lease-router.ts @@ -36,7 +36,8 @@ export const registerDynamicSecretLeaseRouter = async (server: FastifyZodProvide ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be less than a day" }); }), path: z.string().trim().default("/").transform(removeTrailingSlash).describe(DYNAMIC_SECRET_LEASES.CREATE.path), - environmentSlug: z.string().min(1).describe(DYNAMIC_SECRET_LEASES.CREATE.path) + environmentSlug: z.string().min(1).describe(DYNAMIC_SECRET_LEASES.CREATE.environmentSlug), + config: z.any().optional() }), response: { 200: z.object({ diff --git a/backend/src/ee/routes/v1/dynamic-secret-lease-routers/kubernetes-lease-router.ts b/backend/src/ee/routes/v1/dynamic-secret-lease-routers/kubernetes-lease-router.ts new file mode 100644 index 000000000..f2751c635 --- /dev/null +++ b/backend/src/ee/routes/v1/dynamic-secret-lease-routers/kubernetes-lease-router.ts @@ -0,0 +1,67 @@ +import { z } from "zod"; + +import { DynamicSecretLeasesSchema } from "@app/db/schemas"; +import { ApiDocsTags, DYNAMIC_SECRET_LEASES } from "@app/lib/api-docs"; +import { daysToMillisecond } from "@app/lib/dates"; +import { removeTrailingSlash } from "@app/lib/fn"; +import { ms } from "@app/lib/ms"; +import { writeLimit } from "@app/server/config/rateLimiter"; +import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; +import { SanitizedDynamicSecretSchema } from "@app/server/routes/sanitizedSchemas"; +import { AuthMode } from "@app/services/auth/auth-type"; + +export const registerKubernetesDynamicSecretLeaseRouter = async (server: FastifyZodProvider) => { + server.route({ + method: "POST", + url: "/", + config: { + rateLimit: writeLimit + }, + schema: { + hide: false, + tags: [ApiDocsTags.DynamicSecrets], + body: z.object({ + dynamicSecretName: z.string().min(1).describe(DYNAMIC_SECRET_LEASES.CREATE.dynamicSecretName).toLowerCase(), + projectSlug: z.string().min(1).describe(DYNAMIC_SECRET_LEASES.CREATE.projectSlug), + ttl: z + .string() + .optional() + .describe(DYNAMIC_SECRET_LEASES.CREATE.ttl) + .superRefine((val, ctx) => { + if (!val) return; + const valMs = ms(val); + if (valMs < 60 * 1000) + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be greater than 1min" }); + if (valMs > daysToMillisecond(1)) + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be less than a day" }); + }), + path: z.string().trim().default("/").transform(removeTrailingSlash).describe(DYNAMIC_SECRET_LEASES.CREATE.path), + environmentSlug: z.string().min(1).describe(DYNAMIC_SECRET_LEASES.CREATE.environmentSlug), + config: z + .object({ + namespace: z.string().min(1).optional().describe(DYNAMIC_SECRET_LEASES.KUBERNETES.CREATE.config.namespace) + }) + .optional() + }), + response: { + 200: z.object({ + lease: DynamicSecretLeasesSchema, + dynamicSecret: SanitizedDynamicSecretSchema, + data: z.unknown() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { data, lease, dynamicSecret } = await server.services.dynamicSecretLease.create({ + actor: req.permission.type, + actorId: req.permission.id, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId, + name: req.body.dynamicSecretName, + ...req.body + }); + return { lease, data, dynamicSecret }; + } + }); +}; diff --git a/backend/src/ee/routes/v1/index.ts b/backend/src/ee/routes/v1/index.ts index 0b8c78586..8f3b69dfa 100644 --- a/backend/src/ee/routes/v1/index.ts +++ b/backend/src/ee/routes/v1/index.ts @@ -6,6 +6,7 @@ import { registerAssumePrivilegeRouter } from "./assume-privilege-router"; import { registerAuditLogStreamRouter } from "./audit-log-stream-router"; import { registerCaCrlRouter } from "./certificate-authority-crl-router"; import { registerDynamicSecretLeaseRouter } from "./dynamic-secret-lease-router"; +import { registerKubernetesDynamicSecretLeaseRouter } from "./dynamic-secret-lease-routers/kubernetes-lease-router"; import { registerDynamicSecretRouter } from "./dynamic-secret-router"; import { registerExternalKmsRouter } from "./external-kms-router"; import { registerGatewayRouter } from "./gateway-router"; @@ -18,6 +19,7 @@ import { registerLdapRouter } from "./ldap-router"; import { registerLicenseRouter } from "./license-router"; import { registerOidcRouter } from "./oidc-router"; import { registerOrgRoleRouter } from "./org-role-router"; +import { registerPITRouter } from "./pit-router"; import { registerProjectRoleRouter } from "./project-role-router"; import { registerProjectRouter } from "./project-router"; import { registerRateLimitRouter } from "./rate-limit-router"; @@ -53,6 +55,7 @@ export const registerV1EERoutes = async (server: FastifyZodProvider) => { { prefix: "/workspace" } ); await server.register(registerSnapshotRouter, { prefix: "/secret-snapshot" }); + await server.register(registerPITRouter, { prefix: "/pit" }); await server.register(registerSecretApprovalPolicyRouter, { prefix: "/secret-approvals" }); await server.register(registerSecretApprovalRequestRouter, { prefix: "/secret-approval-requests" @@ -69,6 +72,7 @@ export const registerV1EERoutes = async (server: FastifyZodProvider) => { async (dynamicSecretRouter) => { await dynamicSecretRouter.register(registerDynamicSecretRouter); await dynamicSecretRouter.register(registerDynamicSecretLeaseRouter, { prefix: "/leases" }); + await dynamicSecretRouter.register(registerKubernetesDynamicSecretLeaseRouter, { prefix: "/leases/kubernetes" }); }, { prefix: "/dynamic-secrets" } ); diff --git a/backend/src/ee/routes/v1/pit-router.ts b/backend/src/ee/routes/v1/pit-router.ts new file mode 100644 index 000000000..f993e31d7 --- /dev/null +++ b/backend/src/ee/routes/v1/pit-router.ts @@ -0,0 +1,416 @@ +/* eslint-disable @typescript-eslint/no-base-to-string */ +import { z } from "zod"; + +import { EventType } from "@app/ee/services/audit-log/audit-log-types"; +import { removeTrailingSlash } from "@app/lib/fn"; +import { readLimit } from "@app/server/config/rateLimiter"; +import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; +import { booleanSchema } from "@app/server/routes/sanitizedSchemas"; +import { AuthMode } from "@app/services/auth/auth-type"; +import { commitChangesResponseSchema, resourceChangeSchema } from "@app/services/folder-commit/folder-commit-schemas"; + +const commitHistoryItemSchema = z.object({ + id: z.string(), + folderId: z.string(), + actorType: z.string(), + actorMetadata: z.unknown().optional(), + message: z.string().optional().nullable(), + commitId: z.string(), + createdAt: z.string().or(z.date()), + envId: z.string() +}); + +const folderStateSchema = z.array( + z.object({ + type: z.string(), + id: z.string(), + versionId: z.string(), + secretKey: z.string().optional(), + secretVersion: z.number().optional(), + folderName: z.string().optional(), + folderVersion: z.number().optional() + }) +); + +export const registerPITRouter = async (server: FastifyZodProvider) => { + // Get commits count for a folder + server.route({ + method: "GET", + url: "/commits/count", + config: { + rateLimit: readLimit + }, + schema: { + querystring: z.object({ + environment: z.string().trim(), + path: z.string().trim().default("/").transform(removeTrailingSlash), + projectId: z.string().trim() + }), + response: { + 200: z.object({ + count: z.number(), + folderId: z.string() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const result = await server.services.pit.getCommitsCount({ + actor: req.permission?.type, + actorId: req.permission?.id, + actorOrgId: req.permission?.orgId, + actorAuthMethod: req.permission?.authMethod, + projectId: req.query.projectId, + environment: req.query.environment, + path: req.query.path + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.query.projectId, + event: { + type: EventType.GET_PROJECT_PIT_COMMIT_COUNT, + metadata: { + environment: req.query.environment, + path: req.query.path, + commitCount: result.count.toString() + } + } + }); + + return result; + } + }); + + // Get all commits for a folder + server.route({ + method: "GET", + url: "/commits", + config: { + rateLimit: readLimit + }, + schema: { + querystring: z.object({ + environment: z.string().trim(), + path: z.string().trim().default("/").transform(removeTrailingSlash), + projectId: z.string().trim(), + offset: z.coerce.number().min(0).default(0), + limit: z.coerce.number().min(1).max(100).default(20), + search: z.string().trim().optional(), + sort: z.enum(["asc", "desc"]).default("desc") + }), + response: { + 200: z.object({ + commits: commitHistoryItemSchema.array(), + total: z.number(), + hasMore: z.boolean() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const result = await server.services.pit.getCommitsForFolder({ + actor: req.permission?.type, + actorId: req.permission?.id, + actorOrgId: req.permission?.orgId, + actorAuthMethod: req.permission?.authMethod, + projectId: req.query.projectId, + environment: req.query.environment, + path: req.query.path, + offset: req.query.offset, + limit: req.query.limit, + search: req.query.search, + sort: req.query.sort + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.query.projectId, + event: { + type: EventType.GET_PROJECT_PIT_COMMITS, + metadata: { + environment: req.query.environment, + path: req.query.path, + commitCount: result.commits.length.toString(), + offset: req.query.offset.toString(), + limit: req.query.limit.toString(), + search: req.query.search, + sort: req.query.sort + } + } + }); + + return result; + } + }); + + // Get commit changes for a specific commit + server.route({ + method: "GET", + url: "/commits/:commitId/changes", + config: { + rateLimit: readLimit + }, + schema: { + params: z.object({ + commitId: z.string().trim() + }), + querystring: z.object({ + projectId: z.string().trim() + }), + response: { + 200: commitChangesResponseSchema + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const result = await server.services.pit.getCommitChanges({ + actor: req.permission?.type, + actorId: req.permission?.id, + actorOrgId: req.permission?.orgId, + actorAuthMethod: req.permission?.authMethod, + projectId: req.query.projectId, + commitId: req.params.commitId + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.query.projectId, + event: { + type: EventType.GET_PROJECT_PIT_COMMIT_CHANGES, + metadata: { + commitId: req.params.commitId, + changesCount: (result.changes.changes?.length || 0).toString() + } + } + }); + + return result; + } + }); + + // Retrieve rollback changes for a commit + server.route({ + method: "GET", + url: "/commits/:commitId/compare", + config: { + rateLimit: readLimit + }, + schema: { + params: z.object({ + commitId: z.string().trim() + }), + querystring: z.object({ + folderId: z.string().trim(), + environment: z.string().trim(), + deepRollback: booleanSchema.default(false), + secretPath: z.string().trim().default("/").transform(removeTrailingSlash), + projectId: z.string().trim() + }), + response: { + 200: z.array( + z.object({ + folderId: z.string(), + folderName: z.string(), + folderPath: z.string().optional(), + changes: z.array(resourceChangeSchema) + }) + ) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const result = await server.services.pit.compareCommitChanges({ + actor: req.permission?.type, + actorId: req.permission?.id, + actorOrgId: req.permission?.orgId, + actorAuthMethod: req.permission?.authMethod, + projectId: req.query.projectId, + commitId: req.params.commitId, + folderId: req.query.folderId, + environment: req.query.environment, + deepRollback: req.query.deepRollback, + secretPath: req.query.secretPath + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.query.projectId, + event: { + type: EventType.PIT_COMPARE_FOLDER_STATES, + metadata: { + targetCommitId: req.params.commitId, + folderId: req.query.folderId, + deepRollback: req.query.deepRollback, + diffsCount: result.length.toString(), + environment: req.query.environment, + folderPath: req.query.secretPath + } + } + }); + + return result; + } + }); + + // Rollback to a previous commit + server.route({ + method: "POST", + url: "/commits/:commitId/rollback", + config: { + rateLimit: readLimit + }, + schema: { + params: z.object({ + commitId: z.string().trim() + }), + body: z.object({ + folderId: z.string().trim(), + deepRollback: z.boolean().default(false), + message: z.string().max(256).trim().optional(), + environment: z.string().trim(), + projectId: z.string().trim() + }), + response: { + 200: z.object({ + success: z.boolean(), + secretChangesCount: z.number().optional(), + folderChangesCount: z.number().optional(), + totalChanges: z.number().optional() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const result = await server.services.pit.rollbackToCommit({ + actor: req.permission?.type, + actorId: req.permission?.id, + actorOrgId: req.permission?.orgId, + actorAuthMethod: req.permission?.authMethod, + projectId: req.body.projectId, + commitId: req.params.commitId, + folderId: req.body.folderId, + deepRollback: req.body.deepRollback, + message: req.body.message, + environment: req.body.environment + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.body.projectId, + event: { + type: EventType.PIT_ROLLBACK_COMMIT, + metadata: { + targetCommitId: req.params.commitId, + environment: req.body.environment, + folderId: req.body.folderId, + deepRollback: req.body.deepRollback, + message: req.body.message || "Rollback to previous commit", + totalChanges: result.totalChanges?.toString() || "0" + } + } + }); + + return result; + } + }); + + // Revert commit + server.route({ + method: "POST", + url: "/commits/:commitId/revert", + config: { + rateLimit: readLimit + }, + schema: { + params: z.object({ + commitId: z.string().trim() + }), + body: z.object({ + projectId: z.string().trim() + }), + response: { + 200: z.object({ + success: z.boolean(), + message: z.string(), + originalCommitId: z.string(), + revertCommitId: z.string().optional(), + changesReverted: z.number().optional() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const result = await server.services.pit.revertCommit({ + actor: req.permission?.type, + actorId: req.permission?.id, + actorOrgId: req.permission?.orgId, + actorAuthMethod: req.permission?.authMethod, + projectId: req.body.projectId, + commitId: req.params.commitId + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.body.projectId, + event: { + type: EventType.PIT_REVERT_COMMIT, + metadata: { + commitId: req.params.commitId, + revertCommitId: result.revertCommitId, + changesReverted: result.changesReverted?.toString() + } + } + }); + + return result; + } + }); + + // Folder state at commit + server.route({ + method: "GET", + url: "/commits/:commitId", + config: { + rateLimit: readLimit + }, + schema: { + params: z.object({ + commitId: z.string().trim() + }), + querystring: z.object({ + folderId: z.string().trim(), + projectId: z.string().trim() + }), + response: { + 200: folderStateSchema + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const result = await server.services.pit.getFolderStateAtCommit({ + actor: req.permission?.type, + actorId: req.permission?.id, + actorOrgId: req.permission?.orgId, + actorAuthMethod: req.permission?.authMethod, + projectId: req.query.projectId, + commitId: req.params.commitId + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.query.projectId, + event: { + type: EventType.PIT_GET_FOLDER_STATE, + metadata: { + commitId: req.params.commitId, + folderId: req.query.folderId, + resourceCount: result.length.toString() + } + } + }); + + return result; + } + }); +}; diff --git a/backend/src/ee/routes/v1/snapshot-router.ts b/backend/src/ee/routes/v1/snapshot-router.ts index 3ee80adce..c14d97192 100644 --- a/backend/src/ee/routes/v1/snapshot-router.ts +++ b/backend/src/ee/routes/v1/snapshot-router.ts @@ -65,9 +65,10 @@ export const registerSnapshotRouter = async (server: FastifyZodProvider) => { rateLimit: writeLimit }, schema: { - hide: false, + hide: true, + deprecated: true, tags: [ApiDocsTags.Projects], - description: "Roll back project secrets to those captured in a secret snapshot version.", + description: "(Deprecated) Roll back project secrets to those captured in a secret snapshot version.", security: [ { bearerAuth: [] @@ -84,6 +85,10 @@ export const registerSnapshotRouter = async (server: FastifyZodProvider) => { }, onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.IDENTITY_ACCESS_TOKEN]), handler: async (req) => { + throw new Error( + "This endpoint is deprecated. Please use the new PIT recovery system. More information is available at: https://infisical.com/docs/documentation/platform/pit-recovery." + ); + const secretSnapshot = await server.services.snapshot.rollbackSnapshot({ actor: req.permission.type, actorId: req.permission.id, diff --git a/backend/src/ee/services/audit-log/audit-log-types.ts b/backend/src/ee/services/audit-log/audit-log-types.ts index cfc01741f..1b07982ca 100644 --- a/backend/src/ee/services/audit-log/audit-log-types.ts +++ b/backend/src/ee/services/audit-log/audit-log-types.ts @@ -44,6 +44,7 @@ import { TSecretSyncRaw, TUpdateSecretSyncDTO } from "@app/services/secret-sync/secret-sync-types"; +import { TWebhookPayloads } from "@app/services/webhook/webhook-types"; import { WorkflowIntegration } from "@app/services/workflow-integration/workflow-integration-types"; import { KmipPermission } from "../kmip/kmip-enum"; @@ -206,6 +207,7 @@ export enum EventType { CREATE_WEBHOOK = "create-webhook", UPDATE_WEBHOOK_STATUS = "update-webhook-status", DELETE_WEBHOOK = "delete-webhook", + WEBHOOK_TRIGGERED = "webhook-triggered", GET_SECRET_IMPORTS = "get-secret-imports", GET_SECRET_IMPORT = "get-secret-import", CREATE_SECRET_IMPORT = "create-secret-import", @@ -393,6 +395,13 @@ export enum EventType { PROJECT_ASSUME_PRIVILEGE_SESSION_START = "project-assume-privileges-session-start", PROJECT_ASSUME_PRIVILEGE_SESSION_END = "project-assume-privileges-session-end", + GET_PROJECT_PIT_COMMITS = "get-project-pit-commits", + GET_PROJECT_PIT_COMMIT_CHANGES = "get-project-pit-commit-changes", + GET_PROJECT_PIT_COMMIT_COUNT = "get-project-pit-commit-count", + PIT_ROLLBACK_COMMIT = "pit-rollback-commit", + PIT_REVERT_COMMIT = "pit-revert-commit", + PIT_GET_FOLDER_STATE = "pit-get-folder-state", + PIT_COMPARE_FOLDER_STATES = "pit-compare-folder-states", SECRET_SCANNING_DATA_SOURCE_LIST = "secret-scanning-data-source-list", SECRET_SCANNING_DATA_SOURCE_CREATE = "secret-scanning-data-source-create", SECRET_SCANNING_DATA_SOURCE_UPDATE = "secret-scanning-data-source-update", @@ -1440,6 +1449,14 @@ interface DeleteWebhookEvent { }; } +export interface WebhookTriggeredEvent { + type: EventType.WEBHOOK_TRIGGERED; + metadata: { + webhookId: string; + status: string; + } & TWebhookPayloads; +} + interface GetSecretImportsEvent { type: EventType.GET_SECRET_IMPORTS; metadata: { @@ -2979,6 +2996,78 @@ interface MicrosoftTeamsWorkflowIntegrationUpdateEvent { }; } +interface GetProjectPitCommitsEvent { + type: EventType.GET_PROJECT_PIT_COMMITS; + metadata: { + commitCount: string; + environment: string; + path: string; + offset: string; + limit: string; + search?: string; + sort: string; + }; +} + +interface GetProjectPitCommitChangesEvent { + type: EventType.GET_PROJECT_PIT_COMMIT_CHANGES; + metadata: { + changesCount: string; + commitId: string; + }; +} + +interface GetProjectPitCommitCountEvent { + type: EventType.GET_PROJECT_PIT_COMMIT_COUNT; + metadata: { + environment: string; + path: string; + commitCount: string; + }; +} + +interface PitRollbackCommitEvent { + type: EventType.PIT_ROLLBACK_COMMIT; + metadata: { + targetCommitId: string; + folderId: string; + deepRollback: boolean; + message: string; + totalChanges: string; + environment: string; + }; +} + +interface PitRevertCommitEvent { + type: EventType.PIT_REVERT_COMMIT; + metadata: { + commitId: string; + revertCommitId?: string; + changesReverted?: string; + }; +} + +interface PitGetFolderStateEvent { + type: EventType.PIT_GET_FOLDER_STATE; + metadata: { + commitId: string; + folderId: string; + resourceCount: string; + }; +} + +interface PitCompareFolderStatesEvent { + type: EventType.PIT_COMPARE_FOLDER_STATES; + metadata: { + targetCommitId: string; + folderId: string; + deepRollback: boolean; + diffsCount: string; + environment: string; + folderPath: string; + }; +} + interface SecretScanningDataSourceListEvent { type: EventType.SECRET_SCANNING_DATA_SOURCE_LIST; metadata: { @@ -3221,6 +3310,7 @@ export type Event = | CreateWebhookEvent | UpdateWebhookStatusEvent | DeleteWebhookEvent + | WebhookTriggeredEvent | GetSecretImportsEvent | GetSecretImportEvent | CreateSecretImportEvent @@ -3397,6 +3487,13 @@ export type Event = | MicrosoftTeamsWorkflowIntegrationGetEvent | MicrosoftTeamsWorkflowIntegrationListEvent | MicrosoftTeamsWorkflowIntegrationUpdateEvent + | GetProjectPitCommitsEvent + | GetProjectPitCommitChangesEvent + | PitRollbackCommitEvent + | GetProjectPitCommitCountEvent + | PitRevertCommitEvent + | PitCompareFolderStatesEvent + | PitGetFolderStateEvent | SecretScanningDataSourceListEvent | SecretScanningDataSourceGetEvent | SecretScanningDataSourceCreateEvent diff --git a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.ts b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.ts index c38a8f146..497e94311 100644 --- a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.ts +++ b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.ts @@ -10,6 +10,7 @@ import { TDynamicSecretDALFactory } from "../dynamic-secret/dynamic-secret-dal"; import { DynamicSecretStatus } from "../dynamic-secret/dynamic-secret-types"; import { DynamicSecretProviders, TDynamicProviderFns } from "../dynamic-secret/providers/models"; import { TDynamicSecretLeaseDALFactory } from "./dynamic-secret-lease-dal"; +import { TDynamicSecretLeaseConfig } from "./dynamic-secret-lease-types"; type TDynamicSecretLeaseQueueServiceFactoryDep = { queueService: TQueueServiceFactory; @@ -134,10 +135,15 @@ export const dynamicSecretLeaseQueueServiceFactory = ({ await Promise.all(dynamicSecretLeases.map(({ id }) => unsetLeaseRevocation(id))); await Promise.all( - dynamicSecretLeases.map(({ externalEntityId }) => - selectedProvider.revoke(decryptedStoredInput, externalEntityId, { - projectId: folder.projectId - }) + dynamicSecretLeases.map(({ externalEntityId, config }) => + selectedProvider.revoke( + decryptedStoredInput, + externalEntityId, + { + projectId: folder.projectId + }, + config as TDynamicSecretLeaseConfig + ) ) ); } diff --git a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.ts b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.ts index 561b2170c..4b72ff7e9 100644 --- a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.ts +++ b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.ts @@ -29,6 +29,7 @@ import { TCreateDynamicSecretLeaseDTO, TDeleteDynamicSecretLeaseDTO, TDetailsDynamicSecretLeaseDTO, + TDynamicSecretLeaseConfig, TListDynamicSecretLeasesDTO, TRenewDynamicSecretLeaseDTO } from "./dynamic-secret-lease-types"; @@ -77,7 +78,8 @@ export const dynamicSecretLeaseServiceFactory = ({ actorId, actorOrgId, actorAuthMethod, - ttl + ttl, + config }: TCreateDynamicSecretLeaseDTO) => { const appCfg = getConfig(); const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId); @@ -163,7 +165,8 @@ export const dynamicSecretLeaseServiceFactory = ({ expireAt: expireAt.getTime(), usernameTemplate: dynamicSecretCfg.usernameTemplate, identity, - metadata: { projectId } + metadata: { projectId }, + config }); } catch (error: unknown) { if (error && typeof error === "object" && error !== null && "sqlMessage" in error) { @@ -177,8 +180,10 @@ export const dynamicSecretLeaseServiceFactory = ({ expireAt, version: 1, dynamicSecretId: dynamicSecretCfg.id, - externalEntityId: entityId + externalEntityId: entityId, + config }); + await dynamicSecretQueueService.setLeaseRevocation(dynamicSecretLease.id, Number(expireAt) - Number(new Date())); return { lease: dynamicSecretLease, dynamicSecret: dynamicSecretCfg, data }; }; @@ -342,7 +347,12 @@ export const dynamicSecretLeaseServiceFactory = ({ ) as object; const revokeResponse = await selectedProvider - .revoke(decryptedStoredInput, dynamicSecretLease.externalEntityId, { projectId }) + .revoke( + decryptedStoredInput, + dynamicSecretLease.externalEntityId, + { projectId }, + dynamicSecretLease.config as TDynamicSecretLeaseConfig + ) .catch(async (err) => { // only propogate this error if forced is false if (!isForced) return { error: err as Error }; diff --git a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-types.ts b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-types.ts index bf182b349..f6d9f6297 100644 --- a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-types.ts +++ b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-types.ts @@ -10,6 +10,7 @@ export type TCreateDynamicSecretLeaseDTO = { environmentSlug: string; ttl?: string; projectSlug: string; + config?: TDynamicSecretLeaseConfig; } & Omit; export type TDetailsDynamicSecretLeaseDTO = { @@ -41,3 +42,9 @@ export type TRenewDynamicSecretLeaseDTO = { ttl?: string; projectSlug: string; } & Omit; + +export type TDynamicSecretKubernetesLeaseConfig = { + namespace?: string; +}; + +export type TDynamicSecretLeaseConfig = TDynamicSecretKubernetesLeaseConfig; diff --git a/backend/src/ee/services/dynamic-secret/providers/kubernetes.ts b/backend/src/ee/services/dynamic-secret/providers/kubernetes.ts index cf8f2b3e0..45cc06e7c 100644 --- a/backend/src/ee/services/dynamic-secret/providers/kubernetes.ts +++ b/backend/src/ee/services/dynamic-secret/providers/kubernetes.ts @@ -1,13 +1,14 @@ -import axios from "axios"; +import axios, { AxiosError } from "axios"; import handlebars from "handlebars"; import https from "https"; -import { InternalServerError } from "@app/lib/errors"; +import { BadRequestError, InternalServerError } from "@app/lib/errors"; import { GatewayHttpProxyActions, GatewayProxyProtocol, withGatewayProxy } from "@app/lib/gateway"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { blockLocalAndPrivateIpAddresses } from "@app/lib/validator"; import { TKubernetesTokenRequest } from "@app/services/identity-kubernetes-auth/identity-kubernetes-auth-types"; +import { TDynamicSecretKubernetesLeaseConfig } from "../../dynamic-secret-lease/dynamic-secret-lease-types"; import { TGatewayServiceFactory } from "../../gateway/gateway-service"; import { DynamicSecretKubernetesSchema, @@ -19,6 +20,9 @@ import { const EXTERNAL_REQUEST_TIMEOUT = 10 * 1000; +// This value is just a placeholder. When using gateway auth method, the url is irrelevant. +const GATEWAY_AUTH_DEFAULT_URL = "https://kubernetes.default.svc.cluster.local"; + type TKubernetesProviderDTO = { gatewayService: Pick; }; @@ -36,7 +40,7 @@ const generateUsername = (usernameTemplate?: string | null) => { export const KubernetesProvider = ({ gatewayService }: TKubernetesProviderDTO): TDynamicProviderFns => { const validateProviderInputs = async (inputs: unknown) => { const providerInputs = await DynamicSecretKubernetesSchema.parseAsync(inputs); - if (!providerInputs.gatewayId) { + if (!providerInputs.gatewayId && providerInputs.url) { await blockLocalAndPrivateIpAddresses(providerInputs.url); } @@ -103,135 +107,173 @@ export const KubernetesProvider = ({ gatewayService }: TKubernetesProviderDTO): const serviceAccountName = generateUsername(); const roleBindingName = `${serviceAccountName}-role-binding`; - // 1. Create a test service account - await axios.post( - `${baseUrl}/api/v1/namespaces/${providerInputs.namespace}/serviceaccounts`, - { - metadata: { - name: serviceAccountName, - namespace: providerInputs.namespace - } - }, - { - headers: { - "Content-Type": "application/json", - ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway - ? { "x-infisical-action": GatewayHttpProxyActions.InjectGatewayK8sServiceAccountToken } - : { Authorization: `Bearer ${providerInputs.clusterToken}` }) - }, - signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), - timeout: EXTERNAL_REQUEST_TIMEOUT, - httpsAgent - } - ); + const namespaces = providerInputs.namespace.split(",").map((namespace) => namespace.trim()); - // 2. Create a test role binding - const roleBindingUrl = - providerInputs.roleType === KubernetesRoleType.ClusterRole - ? `${baseUrl}/apis/rbac.authorization.k8s.io/v1/clusterrolebindings` - : `${baseUrl}/apis/rbac.authorization.k8s.io/v1/namespaces/${providerInputs.namespace}/rolebindings`; - - const roleBindingMetadata = { - name: roleBindingName, - ...(providerInputs.roleType !== KubernetesRoleType.ClusterRole && { namespace: providerInputs.namespace }) - }; - - await axios.post( - roleBindingUrl, - { - metadata: roleBindingMetadata, - roleRef: { - kind: providerInputs.roleType === KubernetesRoleType.ClusterRole ? "ClusterRole" : "Role", - name: providerInputs.role, - apiGroup: "rbac.authorization.k8s.io" - }, - subjects: [ + // Test each namespace sequentially instead of in parallel to simplify cleanup + for await (const namespace of namespaces) { + try { + // 1. Create a test service account + await axios.post( + `${baseUrl}/api/v1/namespaces/${namespace}/serviceaccounts`, { - kind: "ServiceAccount", - name: serviceAccountName, - namespace: providerInputs.namespace + metadata: { + name: serviceAccountName, + namespace + } + }, + { + headers: { + "Content-Type": "application/json", + ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway + ? { "x-infisical-action": GatewayHttpProxyActions.UseGatewayK8sServiceAccount } + : { Authorization: `Bearer ${providerInputs.clusterToken}` }) + }, + ...(providerInputs.authMethod === KubernetesAuthMethod.Api + ? { + httpsAgent + } + : {}), + signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), + timeout: EXTERNAL_REQUEST_TIMEOUT } - ] - }, - { - headers: { - "Content-Type": "application/json", - ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway - ? { "x-infisical-action": GatewayHttpProxyActions.InjectGatewayK8sServiceAccountToken } - : { Authorization: `Bearer ${providerInputs.clusterToken}` }) - }, - signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), - timeout: EXTERNAL_REQUEST_TIMEOUT, - httpsAgent - } - ); + ); - // 3. Request a token for the test service account - await axios.post( - `${baseUrl}/api/v1/namespaces/${providerInputs.namespace}/serviceaccounts/${serviceAccountName}/token`, - { - spec: { - expirationSeconds: 600, // 10 minutes - ...(providerInputs.audiences?.length ? { audiences: providerInputs.audiences } : {}) + // 2. Create a test role binding + const roleBindingUrl = + providerInputs.roleType === KubernetesRoleType.ClusterRole + ? `${baseUrl}/apis/rbac.authorization.k8s.io/v1/clusterrolebindings` + : `${baseUrl}/apis/rbac.authorization.k8s.io/v1/namespaces/${namespace}/rolebindings`; + + const roleBindingMetadata = { + name: roleBindingName, + ...(providerInputs.roleType !== KubernetesRoleType.ClusterRole && { namespace }) + }; + + await axios.post( + roleBindingUrl, + { + metadata: roleBindingMetadata, + roleRef: { + kind: providerInputs.roleType === KubernetesRoleType.ClusterRole ? "ClusterRole" : "Role", + name: providerInputs.role, + apiGroup: "rbac.authorization.k8s.io" + }, + subjects: [ + { + kind: "ServiceAccount", + name: serviceAccountName, + namespace + } + ] + }, + { + headers: { + "Content-Type": "application/json", + ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway + ? { "x-infisical-action": GatewayHttpProxyActions.UseGatewayK8sServiceAccount } + : { Authorization: `Bearer ${providerInputs.clusterToken}` }) + }, + ...(providerInputs.authMethod === KubernetesAuthMethod.Api + ? { + httpsAgent + } + : {}), + signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), + timeout: EXTERNAL_REQUEST_TIMEOUT + } + ); + + // 3. Request a token for the test service account + await axios.post( + `${baseUrl}/api/v1/namespaces/${namespace}/serviceaccounts/${serviceAccountName}/token`, + { + spec: { + expirationSeconds: 600, // 10 minutes + ...(providerInputs.audiences?.length ? { audiences: providerInputs.audiences } : {}) + } + }, + { + headers: { + "Content-Type": "application/json", + ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway + ? { "x-infisical-action": GatewayHttpProxyActions.UseGatewayK8sServiceAccount } + : { Authorization: `Bearer ${providerInputs.clusterToken}` }) + }, + ...(providerInputs.authMethod === KubernetesAuthMethod.Api + ? { + httpsAgent + } + : {}), + signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), + timeout: EXTERNAL_REQUEST_TIMEOUT + } + ); + + // 4. Cleanup: delete role binding and service account + if (providerInputs.roleType === KubernetesRoleType.Role) { + await axios.delete( + `${baseUrl}/apis/rbac.authorization.k8s.io/v1/namespaces/${namespace}/rolebindings/${roleBindingName}`, + { + headers: { + "Content-Type": "application/json", + ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway + ? { "x-infisical-action": GatewayHttpProxyActions.UseGatewayK8sServiceAccount } + : { Authorization: `Bearer ${providerInputs.clusterToken}` }) + }, + ...(providerInputs.authMethod === KubernetesAuthMethod.Api + ? { + httpsAgent + } + : {}), + signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), + timeout: EXTERNAL_REQUEST_TIMEOUT + } + ); + } else { + await axios.delete(`${baseUrl}/apis/rbac.authorization.k8s.io/v1/clusterrolebindings/${roleBindingName}`, { + headers: { + "Content-Type": "application/json", + ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway + ? { "x-infisical-action": GatewayHttpProxyActions.UseGatewayK8sServiceAccount } + : { Authorization: `Bearer ${providerInputs.clusterToken}` }) + }, + ...(providerInputs.authMethod === KubernetesAuthMethod.Api + ? { + httpsAgent + } + : {}), + signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), + timeout: EXTERNAL_REQUEST_TIMEOUT + }); } - }, - { - headers: { - "Content-Type": "application/json", - ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway - ? { "x-infisical-action": GatewayHttpProxyActions.InjectGatewayK8sServiceAccountToken } - : { Authorization: `Bearer ${providerInputs.clusterToken}` }) - }, - signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), - timeout: EXTERNAL_REQUEST_TIMEOUT, - httpsAgent - } - ); - // 4. Cleanup: delete role binding and service account - if (providerInputs.roleType === KubernetesRoleType.Role) { - await axios.delete( - `${baseUrl}/apis/rbac.authorization.k8s.io/v1/namespaces/${providerInputs.namespace}/rolebindings/${roleBindingName}`, - { + await axios.delete(`${baseUrl}/api/v1/namespaces/${namespace}/serviceaccounts/${serviceAccountName}`, { headers: { "Content-Type": "application/json", ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway - ? { "x-infisical-action": GatewayHttpProxyActions.InjectGatewayK8sServiceAccountToken } + ? { "x-infisical-action": GatewayHttpProxyActions.UseGatewayK8sServiceAccount } : { Authorization: `Bearer ${providerInputs.clusterToken}` }) }, + ...(providerInputs.authMethod === KubernetesAuthMethod.Api + ? { + httpsAgent + } + : {}), signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), - timeout: EXTERNAL_REQUEST_TIMEOUT, - httpsAgent + timeout: EXTERNAL_REQUEST_TIMEOUT + }); + } catch (error) { + const cleanupInfo = `You may need to manually clean up the following resources in namespace "${namespace}": Service Account - ${serviceAccountName}, ${providerInputs.roleType === KubernetesRoleType.Role ? "Role" : "Cluster Role"} Binding - ${roleBindingName}.`; + let mainErrorMessage = "Unknown error"; + if (error instanceof AxiosError) { + mainErrorMessage = (error.response?.data as { message: string })?.message; + } else if (error instanceof Error) { + mainErrorMessage = error.message; } - ); - } else { - await axios.delete(`${baseUrl}/apis/rbac.authorization.k8s.io/v1/clusterrolebindings/${roleBindingName}`, { - headers: { - "Content-Type": "application/json", - ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway - ? { "x-infisical-action": GatewayHttpProxyActions.InjectGatewayK8sServiceAccountToken } - : { Authorization: `Bearer ${providerInputs.clusterToken}` }) - }, - signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), - timeout: EXTERNAL_REQUEST_TIMEOUT, - httpsAgent - }); - } - await axios.delete( - `${baseUrl}/api/v1/namespaces/${providerInputs.namespace}/serviceaccounts/${serviceAccountName}`, - { - headers: { - "Content-Type": "application/json", - ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway - ? { "x-infisical-action": GatewayHttpProxyActions.InjectGatewayK8sServiceAccountToken } - : { Authorization: `Bearer ${providerInputs.clusterToken}` }) - }, - signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), - timeout: EXTERNAL_REQUEST_TIMEOUT, - httpsAgent + throw new Error(`${mainErrorMessage}. ${cleanupInfo}`); } - ); + } }; const serviceAccountStaticCallback = async (host: string, port: number, httpsAgent?: https.Agent) => { @@ -247,17 +289,23 @@ export const KubernetesProvider = ({ gatewayService }: TKubernetesProviderDTO): headers: { "Content-Type": "application/json", ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway - ? { "x-infisical-action": GatewayHttpProxyActions.InjectGatewayK8sServiceAccountToken } + ? { "x-infisical-action": GatewayHttpProxyActions.UseGatewayK8sServiceAccount } : { Authorization: `Bearer ${providerInputs.clusterToken}` }) }, + ...(providerInputs.authMethod === KubernetesAuthMethod.Api + ? { + httpsAgent + } + : {}), signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), - timeout: EXTERNAL_REQUEST_TIMEOUT, - httpsAgent + timeout: EXTERNAL_REQUEST_TIMEOUT } ); }; - const url = new URL(providerInputs.url); + const rawUrl = + providerInputs.authMethod === KubernetesAuthMethod.Gateway ? GATEWAY_AUTH_DEFAULT_URL : providerInputs.url || ""; + const url = new URL(rawUrl); const k8sGatewayHost = url.hostname; const k8sPort = url.port ? Number(url.port) : 443; const k8sHost = `${url.protocol}//${url.hostname}`; @@ -315,11 +363,13 @@ export const KubernetesProvider = ({ gatewayService }: TKubernetesProviderDTO): const create = async ({ inputs, expireAt, - usernameTemplate + usernameTemplate, + config }: { inputs: unknown; expireAt: number; usernameTemplate?: string | null; + config?: TDynamicSecretKubernetesLeaseConfig; }) => { const providerInputs = await validateProviderInputs(inputs); @@ -331,26 +381,44 @@ export const KubernetesProvider = ({ gatewayService }: TKubernetesProviderDTO): const baseUrl = port ? `${host}:${port}` : host; const serviceAccountName = generateUsername(usernameTemplate); const roleBindingName = `${serviceAccountName}-role-binding`; + const allowedNamespaces = providerInputs.namespace.split(",").map((namespace) => namespace.trim()); + + if (config?.namespace && !allowedNamespaces?.includes(config?.namespace)) { + throw new BadRequestError({ + message: `Namespace ${config?.namespace} is not allowed. Allowed namespaces: ${allowedNamespaces?.join(", ")}` + }); + } + + const namespace = config?.namespace || allowedNamespaces[0]; + if (!namespace) { + throw new BadRequestError({ + message: "No namespace provided" + }); + } // 1. Create the service account await axios.post( - `${baseUrl}/api/v1/namespaces/${providerInputs.namespace}/serviceaccounts`, + `${baseUrl}/api/v1/namespaces/${namespace}/serviceaccounts`, { metadata: { name: serviceAccountName, - namespace: providerInputs.namespace + namespace } }, { headers: { "Content-Type": "application/json", ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway - ? { "x-infisical-action": GatewayHttpProxyActions.InjectGatewayK8sServiceAccountToken } + ? { "x-infisical-action": GatewayHttpProxyActions.UseGatewayK8sServiceAccount } : { Authorization: `Bearer ${providerInputs.clusterToken}` }) }, + ...(providerInputs.authMethod === KubernetesAuthMethod.Api + ? { + httpsAgent + } + : {}), signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), - timeout: EXTERNAL_REQUEST_TIMEOUT, - httpsAgent + timeout: EXTERNAL_REQUEST_TIMEOUT } ); @@ -358,11 +426,11 @@ export const KubernetesProvider = ({ gatewayService }: TKubernetesProviderDTO): const roleBindingUrl = providerInputs.roleType === KubernetesRoleType.ClusterRole ? `${baseUrl}/apis/rbac.authorization.k8s.io/v1/clusterrolebindings` - : `${baseUrl}/apis/rbac.authorization.k8s.io/v1/namespaces/${providerInputs.namespace}/rolebindings`; + : `${baseUrl}/apis/rbac.authorization.k8s.io/v1/namespaces/${namespace}/rolebindings`; const roleBindingMetadata = { name: roleBindingName, - ...(providerInputs.roleType !== KubernetesRoleType.ClusterRole && { namespace: providerInputs.namespace }) + ...(providerInputs.roleType !== KubernetesRoleType.ClusterRole && { namespace }) }; await axios.post( @@ -378,7 +446,7 @@ export const KubernetesProvider = ({ gatewayService }: TKubernetesProviderDTO): { kind: "ServiceAccount", name: serviceAccountName, - namespace: providerInputs.namespace + namespace } ] }, @@ -386,18 +454,22 @@ export const KubernetesProvider = ({ gatewayService }: TKubernetesProviderDTO): headers: { "Content-Type": "application/json", ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway - ? { "x-infisical-action": GatewayHttpProxyActions.InjectGatewayK8sServiceAccountToken } + ? { "x-infisical-action": GatewayHttpProxyActions.UseGatewayK8sServiceAccount } : { Authorization: `Bearer ${providerInputs.clusterToken}` }) }, + ...(providerInputs.authMethod === KubernetesAuthMethod.Api + ? { + httpsAgent + } + : {}), signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), - timeout: EXTERNAL_REQUEST_TIMEOUT, - httpsAgent + timeout: EXTERNAL_REQUEST_TIMEOUT } ); // 3. Request a token for the service account const res = await axios.post( - `${baseUrl}/api/v1/namespaces/${providerInputs.namespace}/serviceaccounts/${serviceAccountName}/token`, + `${baseUrl}/api/v1/namespaces/${namespace}/serviceaccounts/${serviceAccountName}/token`, { spec: { expirationSeconds: Math.floor((expireAt - Date.now()) / 1000), @@ -408,12 +480,16 @@ export const KubernetesProvider = ({ gatewayService }: TKubernetesProviderDTO): headers: { "Content-Type": "application/json", ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway - ? { "x-infisical-action": GatewayHttpProxyActions.InjectGatewayK8sServiceAccountToken } + ? { "x-infisical-action": GatewayHttpProxyActions.UseGatewayK8sServiceAccount } : { Authorization: `Bearer ${providerInputs.clusterToken}` }) }, + ...(providerInputs.authMethod === KubernetesAuthMethod.Api + ? { + httpsAgent + } + : {}), signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), - timeout: EXTERNAL_REQUEST_TIMEOUT, - httpsAgent + timeout: EXTERNAL_REQUEST_TIMEOUT } ); @@ -425,6 +501,12 @@ export const KubernetesProvider = ({ gatewayService }: TKubernetesProviderDTO): throw new Error("invalid callback"); } + if (config?.namespace && config.namespace !== providerInputs.namespace) { + throw new BadRequestError({ + message: `Namespace ${config?.namespace} is not allowed. Allowed namespace: ${providerInputs.namespace}.` + }); + } + const baseUrl = port ? `${host}:${port}` : host; const res = await axios.post( @@ -439,19 +521,25 @@ export const KubernetesProvider = ({ gatewayService }: TKubernetesProviderDTO): headers: { "Content-Type": "application/json", ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway - ? { "x-infisical-action": GatewayHttpProxyActions.InjectGatewayK8sServiceAccountToken } + ? { "x-infisical-action": GatewayHttpProxyActions.UseGatewayK8sServiceAccount } : { Authorization: `Bearer ${providerInputs.clusterToken}` }) }, + ...(providerInputs.authMethod === KubernetesAuthMethod.Api + ? { + httpsAgent + } + : {}), signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), - timeout: EXTERNAL_REQUEST_TIMEOUT, - httpsAgent + timeout: EXTERNAL_REQUEST_TIMEOUT } ); return { ...res.data, serviceAccountName: providerInputs.serviceAccountName }; }; - const url = new URL(providerInputs.url); + const rawUrl = + providerInputs.authMethod === KubernetesAuthMethod.Gateway ? GATEWAY_AUTH_DEFAULT_URL : providerInputs.url || ""; + const url = new URL(rawUrl); const k8sHost = `${url.protocol}//${url.hostname}`; const k8sGatewayHost = url.hostname; const k8sPort = url.port ? Number(url.port) : 443; @@ -511,7 +599,13 @@ export const KubernetesProvider = ({ gatewayService }: TKubernetesProviderDTO): } }; - const revoke = async (inputs: unknown, entityId: string) => { + const revoke = async ( + inputs: unknown, + entityId: string, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + _metadata: { projectId: string }, + config?: TDynamicSecretKubernetesLeaseConfig + ) => { const providerInputs = await validateProviderInputs(inputs); const serviceAccountDynamicCallback = async (host: string, port: number, httpsAgent?: https.Agent) => { @@ -522,19 +616,25 @@ export const KubernetesProvider = ({ gatewayService }: TKubernetesProviderDTO): const baseUrl = port ? `${host}:${port}` : host; const roleBindingName = `${entityId}-role-binding`; + const namespace = config?.namespace ?? providerInputs.namespace.split(",")[0].trim(); + if (providerInputs.roleType === KubernetesRoleType.Role) { await axios.delete( - `${baseUrl}/apis/rbac.authorization.k8s.io/v1/namespaces/${providerInputs.namespace}/rolebindings/${roleBindingName}`, + `${baseUrl}/apis/rbac.authorization.k8s.io/v1/namespaces/${namespace}/rolebindings/${roleBindingName}`, { headers: { "Content-Type": "application/json", ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway - ? { "x-infisical-action": GatewayHttpProxyActions.InjectGatewayK8sServiceAccountToken } + ? { "x-infisical-action": GatewayHttpProxyActions.UseGatewayK8sServiceAccount } : { Authorization: `Bearer ${providerInputs.clusterToken}` }) }, + ...(providerInputs.authMethod === KubernetesAuthMethod.Api + ? { + httpsAgent + } + : {}), signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), - timeout: EXTERNAL_REQUEST_TIMEOUT, - httpsAgent + timeout: EXTERNAL_REQUEST_TIMEOUT } ); } else { @@ -542,31 +642,44 @@ export const KubernetesProvider = ({ gatewayService }: TKubernetesProviderDTO): headers: { "Content-Type": "application/json", ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway - ? { "x-infisical-action": GatewayHttpProxyActions.InjectGatewayK8sServiceAccountToken } + ? { "x-infisical-action": GatewayHttpProxyActions.UseGatewayK8sServiceAccount } : { Authorization: `Bearer ${providerInputs.clusterToken}` }) }, + ...(providerInputs.authMethod === KubernetesAuthMethod.Api + ? { + httpsAgent + } + : {}), signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), - timeout: EXTERNAL_REQUEST_TIMEOUT, - httpsAgent + timeout: EXTERNAL_REQUEST_TIMEOUT }); } // Delete the service account - await axios.delete(`${baseUrl}/api/v1/namespaces/${providerInputs.namespace}/serviceaccounts/${entityId}`, { + await axios.delete(`${baseUrl}/api/v1/namespaces/${namespace}/serviceaccounts/${entityId}`, { headers: { "Content-Type": "application/json", ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway - ? { "x-infisical-action": GatewayHttpProxyActions.InjectGatewayK8sServiceAccountToken } + ? { "x-infisical-action": GatewayHttpProxyActions.UseGatewayK8sServiceAccount } : { Authorization: `Bearer ${providerInputs.clusterToken}` }) }, + ...(providerInputs.authMethod === KubernetesAuthMethod.Api + ? { + httpsAgent + } + : {}), signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), - timeout: EXTERNAL_REQUEST_TIMEOUT, - httpsAgent + timeout: EXTERNAL_REQUEST_TIMEOUT }); }; if (providerInputs.credentialType === KubernetesCredentialType.Dynamic) { - const url = new URL(providerInputs.url); + const rawUrl = + providerInputs.authMethod === KubernetesAuthMethod.Gateway + ? GATEWAY_AUTH_DEFAULT_URL + : providerInputs.url || ""; + + const url = new URL(rawUrl); const k8sGatewayHost = url.hostname; const k8sPort = url.port ? Number(url.port) : 443; const k8sHost = `${url.protocol}//${url.hostname}`; diff --git a/backend/src/ee/services/dynamic-secret/providers/models.ts b/backend/src/ee/services/dynamic-secret/providers/models.ts index b2496eebd..32cc46d22 100644 --- a/backend/src/ee/services/dynamic-secret/providers/models.ts +++ b/backend/src/ee/services/dynamic-secret/providers/models.ts @@ -1,5 +1,10 @@ +import RE2 from "re2"; import { z } from "zod"; +import { CharacterType, characterValidator } from "@app/lib/validator/validate-string"; + +import { TDynamicSecretLeaseConfig } from "../../dynamic-secret-lease/dynamic-secret-lease-types"; + export type PasswordRequirements = { length: number; required: { @@ -323,24 +328,54 @@ export const LdapSchema = z.union([ export const DynamicSecretKubernetesSchema = z .discriminatedUnion("credentialType", [ z.object({ - url: z.string().url().trim().min(1), + url: z + .string() + .optional() + .refine((val: string | undefined) => !val || new RE2(/^https?:\/\/.+/).test(val), { + message: "Invalid URL. Must start with http:// or https:// (e.g. https://example.com)" + }), clusterToken: z.string().trim().optional(), ca: z.string().optional(), sslEnabled: z.boolean().default(false), credentialType: z.literal(KubernetesCredentialType.Static), serviceAccountName: z.string().trim().min(1), - namespace: z.string().trim().min(1), + namespace: z + .string() + .trim() + .min(1) + .refine((val) => !val.includes(","), "Namespace must be a single value, not a comma-separated list") + .refine( + (val) => characterValidator([CharacterType.AlphaNumeric, CharacterType.Hyphen])(val), + "Invalid namespace format" + ), gatewayId: z.string().optional(), audiences: z.array(z.string().trim().min(1)), authMethod: z.nativeEnum(KubernetesAuthMethod).default(KubernetesAuthMethod.Api) }), z.object({ - url: z.string().url().trim().min(1), + url: z + .string() + .url() + .optional() + .refine((val: string | undefined) => !val || new RE2(/^https?:\/\/.+/).test(val), { + message: "Invalid URL. Must start with http:// or https:// (e.g. https://example.com)" + }), clusterToken: z.string().trim().optional(), ca: z.string().optional(), sslEnabled: z.boolean().default(false), credentialType: z.literal(KubernetesCredentialType.Dynamic), - namespace: z.string().trim().min(1), + namespace: z + .string() + .trim() + .min(1) + .refine((val) => { + const namespaces = val.split(",").map((ns) => ns.trim()); + return ( + namespaces.length > 0 && + namespaces.every((ns) => ns.length > 0) && + namespaces.every((ns) => characterValidator([CharacterType.AlphaNumeric, CharacterType.Hyphen])(ns)) + ); + }, "Must be a valid comma-separated list of namespace values"), gatewayId: z.string().optional(), audiences: z.array(z.string().trim().min(1)), roleType: z.nativeEnum(KubernetesRoleType), @@ -356,12 +391,21 @@ export const DynamicSecretKubernetesSchema = z message: "When auth method is set to Gateway, a gateway must be selected" }); } - if ((data.authMethod === KubernetesAuthMethod.Api || !data.authMethod) && !data.clusterToken) { - ctx.addIssue({ - path: ["clusterToken"], - code: z.ZodIssueCode.custom, - message: "When auth method is set to Manual Token, a cluster token must be provided" - }); + if (data.authMethod === KubernetesAuthMethod.Api || !data.authMethod) { + if (!data.clusterToken) { + ctx.addIssue({ + path: ["clusterToken"], + code: z.ZodIssueCode.custom, + message: "When auth method is set to Token, a cluster token must be provided" + }); + } + if (!data.url) { + ctx.addIssue({ + path: ["url"], + code: z.ZodIssueCode.custom, + message: "When auth method is set to Token, a cluster URL must be provided" + }); + } } }); @@ -475,10 +519,16 @@ export type TDynamicProviderFns = { name: string; }; metadata: { projectId: string }; + config?: TDynamicSecretLeaseConfig; }) => Promise<{ entityId: string; data: unknown }>; validateConnection: (inputs: unknown, metadata: { projectId: string }) => Promise; validateProviderInputs: (inputs: object, metadata: { projectId: string }) => Promise; - revoke: (inputs: unknown, entityId: string, metadata: { projectId: string }) => Promise<{ entityId: string }>; + revoke: ( + inputs: unknown, + entityId: string, + metadata: { projectId: string }, + config?: TDynamicSecretLeaseConfig + ) => Promise<{ entityId: string }>; renew: ( inputs: unknown, entityId: string, diff --git a/backend/src/ee/services/permission/default-roles.ts b/backend/src/ee/services/permission/default-roles.ts index 40c5310ae..cca4efaf2 100644 --- a/backend/src/ee/services/permission/default-roles.ts +++ b/backend/src/ee/services/permission/default-roles.ts @@ -4,6 +4,7 @@ import { ProjectPermissionActions, ProjectPermissionCertificateActions, ProjectPermissionCmekActions, + ProjectPermissionCommitsActions, ProjectPermissionDynamicSecretActions, ProjectPermissionGroupActions, ProjectPermissionIdentityActions, @@ -90,6 +91,11 @@ const buildAdminPermissionRules = () => { ProjectPermissionSub.Certificates ); + can( + [ProjectPermissionCommitsActions.Read, ProjectPermissionCommitsActions.PerformRollback], + ProjectPermissionSub.Commits + ); + can( [ ProjectPermissionSshHostActions.Edit, @@ -292,6 +298,11 @@ const buildMemberPermissionRules = () => { ProjectPermissionSub.SecretImports ); + can( + [ProjectPermissionCommitsActions.Read, ProjectPermissionCommitsActions.PerformRollback], + ProjectPermissionSub.Commits + ); + can([ProjectPermissionActions.Read], ProjectPermissionSub.SecretApproval); can([ProjectPermissionSecretRotationActions.Read], ProjectPermissionSub.SecretRotation); @@ -479,6 +490,7 @@ const buildViewerPermissionRules = () => { can(ProjectPermissionActions.Read, ProjectPermissionSub.SshCertificates); can(ProjectPermissionActions.Read, ProjectPermissionSub.SshCertificateTemplates); can(ProjectPermissionSecretSyncActions.Read, ProjectPermissionSub.SecretSyncs); + can(ProjectPermissionCommitsActions.Read, ProjectPermissionSub.Commits); can( [ diff --git a/backend/src/ee/services/permission/project-permission.ts b/backend/src/ee/services/permission/project-permission.ts index d1ae69574..f61c4b1a4 100644 --- a/backend/src/ee/services/permission/project-permission.ts +++ b/backend/src/ee/services/permission/project-permission.ts @@ -17,6 +17,11 @@ export enum ProjectPermissionActions { Delete = "delete" } +export enum ProjectPermissionCommitsActions { + Read = "read", + PerformRollback = "perform-rollback" +} + export enum ProjectPermissionCertificateActions { Read = "read", Create = "create", @@ -172,6 +177,7 @@ export enum ProjectPermissionSub { SecretRollback = "secret-rollback", SecretApproval = "secret-approval", SecretRotation = "secret-rotation", + Commits = "commits", Identity = "identity", CertificateAuthorities = "certificate-authorities", Certificates = "certificates", @@ -325,6 +331,7 @@ export type ProjectPermissionSet = | [ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback] | [ProjectPermissionActions.Create, ProjectPermissionSub.SecretRollback] | [ProjectPermissionActions.Edit, ProjectPermissionSub.Kms] + | [ProjectPermissionCommitsActions, ProjectPermissionSub.Commits] | [ProjectPermissionSecretScanningDataSourceActions, ProjectPermissionSub.SecretScanningDataSources] | [ProjectPermissionSecretScanningFindingActions, ProjectPermissionSub.SecretScanningFindings] | [ProjectPermissionSecretScanningConfigActions, ProjectPermissionSub.SecretScanningConfigs]; @@ -676,6 +683,12 @@ const GeneralPermissionSchema = [ "Describe what action an entity can take." ) }), + z.object({ + subject: z.literal(ProjectPermissionSub.Commits).describe("The entity this permission pertains to."), + action: CASL_ACTION_SCHEMA_NATIVE_ENUM(ProjectPermissionCommitsActions).describe( + "Describe what action an entity can take." + ) + }), z.object({ subject: z .literal(ProjectPermissionSub.SecretScanningDataSources) diff --git a/backend/src/ee/services/pit/pit-service.ts b/backend/src/ee/services/pit/pit-service.ts new file mode 100644 index 000000000..160729123 --- /dev/null +++ b/backend/src/ee/services/pit/pit-service.ts @@ -0,0 +1,485 @@ +/* eslint-disable no-await-in-loop */ +import { ForbiddenError } from "@casl/ability"; + +import { ActionProjectType } from "@app/db/schemas"; +import { ProjectPermissionCommitsActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; +import { NotFoundError } from "@app/lib/errors"; +import { logger } from "@app/lib/logger"; +import { ActorAuthMethod, ActorType } from "@app/services/auth/auth-type"; +import { ResourceType, TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; +import { + isFolderCommitChange, + isSecretCommitChange +} from "@app/services/folder-commit-changes/folder-commit-changes-dal"; +import { TProjectEnvDALFactory } from "@app/services/project-env/project-env-dal"; +import { TSecretServiceFactory } from "@app/services/secret/secret-service"; +import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; +import { TSecretFolderServiceFactory } from "@app/services/secret-folder/secret-folder-service"; + +import { TPermissionServiceFactory } from "../permission/permission-service"; + +type TPitServiceFactoryDep = { + folderCommitService: TFolderCommitServiceFactory; + secretService: Pick; + folderService: Pick; + permissionService: Pick; + folderDAL: Pick; + projectEnvDAL: Pick; +}; + +export type TPitServiceFactory = ReturnType; + +export const pitServiceFactory = ({ + folderCommitService, + secretService, + folderService, + permissionService, + folderDAL, + projectEnvDAL +}: TPitServiceFactoryDep) => { + const getCommitsCount = async ({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + environment, + path + }: { + actor: ActorType; + actorId: string; + actorOrgId: string; + actorAuthMethod: ActorAuthMethod; + projectId: string; + environment: string; + path: string; + }) => { + const result = await folderCommitService.getCommitsCount({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + environment, + path + }); + + return result; + }; + + const getCommitsForFolder = async ({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + environment, + path, + offset, + limit, + search, + sort + }: { + actor: ActorType; + actorId: string; + actorOrgId: string; + actorAuthMethod: ActorAuthMethod; + projectId: string; + environment: string; + path: string; + offset: number; + limit: number; + search?: string; + sort: "asc" | "desc"; + }) => { + const result = await folderCommitService.getCommitsForFolder({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + environment, + path, + offset, + limit, + search, + sort + }); + + return { + commits: result.commits.map((commit) => ({ + ...commit, + commitId: commit.commitId.toString() + })), + total: result.total, + hasMore: result.hasMore + }; + }; + + const getCommitChanges = async ({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + commitId + }: { + actor: ActorType; + actorId: string; + actorOrgId: string; + actorAuthMethod: ActorAuthMethod; + projectId: string; + commitId: string; + }) => { + const changes = await folderCommitService.getCommitChanges({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + commitId + }); + + const [folderWithPath] = await folderDAL.findSecretPathByFolderIds(projectId, [changes.folderId]); + + for (const change of changes.changes) { + if (isSecretCommitChange(change)) { + change.versions = await secretService.getChangeVersions( + { + secretVersion: change.secretVersion, + secretId: change.secretId, + id: change.id, + isUpdate: change.isUpdate, + changeType: change.changeType + }, + (Number.parseInt(change.secretVersion, 10) - 1).toString(), + actorId, + actor, + actorOrgId, + actorAuthMethod, + changes.envId, + projectId, + folderWithPath?.path || "" + ); + } else if (isFolderCommitChange(change)) { + change.versions = await folderService.getFolderVersions( + change, + (Number.parseInt(change.folderVersion, 10) - 1).toString(), + change.folderChangeId + ); + } + } + + return { + changes: { + ...changes, + commitId: changes.commitId.toString() + } + }; + }; + + const compareCommitChanges = async ({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + commitId, + folderId, + environment, + deepRollback, + secretPath + }: { + actor: ActorType; + actorId: string; + actorOrgId: string; + actorAuthMethod: ActorAuthMethod; + projectId: string; + commitId: string; + folderId: string; + environment: string; + deepRollback: boolean; + secretPath: string; + }) => { + const latestCommit = await folderCommitService.getLatestCommit({ + folderId, + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId + }); + + const targetCommit = await folderCommitService.getCommitById({ + commitId, + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId + }); + + const env = await projectEnvDAL.findOne({ + projectId, + slug: environment + }); + + if (!latestCommit) { + throw new NotFoundError({ message: "Latest commit not found" }); + } + + let diffs; + if (deepRollback) { + diffs = await folderCommitService.deepCompareFolder({ + targetCommitId: targetCommit.id, + envId: env.id, + projectId + }); + } else { + const folderData = await folderService.getFolderById({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + id: folderId + }); + + diffs = [ + { + folderId: folderData.id, + folderName: folderData.name, + folderPath: secretPath, + changes: await folderCommitService.compareFolderStates({ + targetCommitId: commitId, + currentCommitId: latestCommit.id + }) + } + ]; + } + + for (const diff of diffs) { + for (const change of diff.changes) { + // Use discriminated union type checking + if (change.type === ResourceType.SECRET) { + // TypeScript now knows this is a SecretChange + if (change.secretKey && change.secretVersion && change.secretId) { + change.versions = await secretService.getChangeVersions( + { + secretVersion: change.secretVersion, + secretId: change.secretId, + id: change.id, + isUpdate: change.isUpdate, + changeType: change.changeType + }, + change.fromVersion || "1", + actorId, + actor, + actorOrgId, + actorAuthMethod, + env.id, + projectId, + diff.folderPath || "" + ); + } + } else if (change.type === ResourceType.FOLDER) { + // TypeScript now knows this is a FolderChange + if (change.folderVersion) { + change.versions = await folderService.getFolderVersions(change, change.fromVersion || "1", change.id); + } + } + } + } + + return diffs; + }; + + const rollbackToCommit = async ({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + commitId, + folderId, + deepRollback, + message, + environment + }: { + actor: ActorType; + actorId: string; + actorOrgId: string; + actorAuthMethod: ActorAuthMethod; + projectId: string; + commitId: string; + folderId: string; + deepRollback: boolean; + message?: string; + environment: string; + }) => { + const { permission: userPermission } = await permissionService.getProjectPermission({ + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId, + actionProjectType: ActionProjectType.SecretManager + }); + + ForbiddenError.from(userPermission).throwUnlessCan( + ProjectPermissionCommitsActions.PerformRollback, + ProjectPermissionSub.Commits + ); + + const latestCommit = await folderCommitService.getLatestCommit({ + folderId, + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId + }); + + if (!latestCommit) { + throw new NotFoundError({ message: "Latest commit not found" }); + } + + logger.info(`PIT - Attempting to rollback folder ${folderId} from commit ${latestCommit.id} to commit ${commitId}`); + + const targetCommit = await folderCommitService.getCommitById({ + commitId, + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId + }); + + const env = await projectEnvDAL.findOne({ + projectId, + slug: environment + }); + + if (!targetCommit || targetCommit.folderId !== folderId || targetCommit.envId !== env.id) { + throw new NotFoundError({ message: "Target commit not found" }); + } + + if (!latestCommit || latestCommit.envId !== env.id) { + throw new NotFoundError({ message: "Latest commit not found" }); + } + + if (deepRollback) { + await folderCommitService.deepRollbackFolder(commitId, env.id, actorId, actor, projectId, message); + return { success: true }; + } + + const diff = await folderCommitService.compareFolderStates({ + currentCommitId: latestCommit.id, + targetCommitId: commitId + }); + + const response = await folderCommitService.applyFolderStateDifferences({ + differences: diff, + actorInfo: { + actorType: actor, + actorId, + message: message || "Rollback to previous commit" + }, + folderId, + projectId, + reconstructNewFolders: deepRollback + }); + + return { + success: true, + secretChangesCount: response.secretChangesCount, + folderChangesCount: response.folderChangesCount, + totalChanges: response.totalChanges + }; + }; + + const revertCommit = async ({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + commitId + }: { + actor: ActorType; + actorId: string; + actorOrgId: string; + actorAuthMethod: ActorAuthMethod; + projectId: string; + commitId: string; + }) => { + const response = await folderCommitService.revertCommitChanges({ + commitId, + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId + }); + + return response; + }; + + const getFolderStateAtCommit = async ({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + commitId + }: { + actor: ActorType; + actorId: string; + actorOrgId: string; + actorAuthMethod: ActorAuthMethod; + projectId: string; + commitId: string; + }) => { + const commit = await folderCommitService.getCommitById({ + commitId, + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId + }); + + if (!commit) { + throw new NotFoundError({ message: `Commit with ID ${commitId} not found` }); + } + + const response = await folderCommitService.reconstructFolderState(commitId); + + return response.map((item) => { + if (item.type === ResourceType.SECRET) { + return { + ...item, + secretVersion: Number(item.secretVersion) + }; + } + + if (item.type === ResourceType.FOLDER) { + return { + ...item, + folderVersion: Number(item.folderVersion) + }; + } + + return item; + }); + }; + + return { + getCommitsCount, + getCommitsForFolder, + getCommitChanges, + compareCommitChanges, + rollbackToCommit, + revertCommit, + getFolderStateAtCommit + }; +}; diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts index 217181281..9f1ee5307 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts @@ -1,3 +1,4 @@ +/* eslint-disable no-nested-ternary */ import { ForbiddenError, subject } from "@casl/ability"; import { @@ -20,6 +21,7 @@ import { EnforcementLevel } from "@app/lib/types"; import { triggerWorkflowIntegrationNotification } from "@app/lib/workflow-integrations/trigger-notification"; import { TriggerFeature } from "@app/lib/workflow-integrations/types"; import { ActorType } from "@app/services/auth/auth-type"; +import { TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; import { TMicrosoftTeamsServiceFactory } from "@app/services/microsoft-teams/microsoft-teams-service"; @@ -130,6 +132,7 @@ type TSecretApprovalRequestServiceFactoryDep = { licenseService: Pick; projectMicrosoftTeamsConfigDAL: Pick; microsoftTeamsService: Pick; + folderCommitService: Pick; }; export type TSecretApprovalRequestServiceFactory = ReturnType; @@ -161,7 +164,8 @@ export const secretApprovalRequestServiceFactory = ({ projectSlackConfigDAL, resourceMetadataDAL, projectMicrosoftTeamsConfigDAL, - microsoftTeamsService + microsoftTeamsService, + folderCommitService }: TSecretApprovalRequestServiceFactoryDep) => { const requestCount = async ({ projectId, actor, actorId, actorOrgId, actorAuthMethod }: TApprovalRequestCountDTO) => { if (actor === ActorType.SERVICE) throw new BadRequestError({ message: "Cannot use service token" }); @@ -243,7 +247,7 @@ export const secretApprovalRequestServiceFactory = ({ const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId); const { policy } = secretApprovalRequest; - const { hasRole } = await permissionService.getProjectPermission({ + const { hasRole, permission } = await permissionService.getProjectPermission({ actor, actorId, projectId, @@ -259,6 +263,12 @@ export const secretApprovalRequestServiceFactory = ({ throw new ForbiddenRequestError({ message: "User has insufficient privileges" }); } + const hasSecretReadAccess = permission.can( + ProjectPermissionSecretActions.DescribeAndReadValue, + ProjectPermissionSub.Secrets + ); + const hiddenSecretValue = "******"; + let secrets; if (shouldUseSecretV2Bridge) { const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ @@ -275,9 +285,9 @@ export const secretApprovalRequestServiceFactory = ({ version: el.version, secretMetadata: el.secretMetadata as ResourceMetadataDTO, isRotatedSecret: el.secret?.isRotatedSecret ?? false, - secretValue: - // eslint-disable-next-line no-nested-ternary - el.secret && el.secret.isRotatedSecret + secretValue: !hasSecretReadAccess + ? hiddenSecretValue + : el.secret && el.secret.isRotatedSecret ? undefined : el.encryptedValue ? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString() @@ -290,9 +300,11 @@ export const secretApprovalRequestServiceFactory = ({ secretKey: el.secret.key, id: el.secret.id, version: el.secret.version, - secretValue: el.secret.encryptedValue - ? secretManagerDecryptor({ cipherTextBlob: el.secret.encryptedValue }).toString() - : "", + secretValue: !hasSecretReadAccess + ? hiddenSecretValue + : el.secret.encryptedValue + ? secretManagerDecryptor({ cipherTextBlob: el.secret.encryptedValue }).toString() + : "", secretComment: el.secret.encryptedComment ? secretManagerDecryptor({ cipherTextBlob: el.secret.encryptedComment }).toString() : "" @@ -303,9 +315,11 @@ export const secretApprovalRequestServiceFactory = ({ secretKey: el.secretVersion.key, id: el.secretVersion.id, version: el.secretVersion.version, - secretValue: el.secretVersion.encryptedValue - ? secretManagerDecryptor({ cipherTextBlob: el.secretVersion.encryptedValue }).toString() - : "", + secretValue: !hasSecretReadAccess + ? hiddenSecretValue + : el.secretVersion.encryptedValue + ? secretManagerDecryptor({ cipherTextBlob: el.secretVersion.encryptedValue }).toString() + : "", secretComment: el.secretVersion.encryptedComment ? secretManagerDecryptor({ cipherTextBlob: el.secretVersion.encryptedComment }).toString() : "", @@ -597,6 +611,10 @@ export const secretApprovalRequestServiceFactory = ({ ? await fnSecretV2BridgeBulkInsert({ tx, folderId, + actor: { + actorId, + type: actor + }, orgId: actorOrgId, inputSecrets: secretCreationCommits.map((el) => ({ tagIds: el?.tags.map(({ id }) => id), @@ -619,13 +637,18 @@ export const secretApprovalRequestServiceFactory = ({ secretDAL: secretV2BridgeDAL, secretVersionDAL: secretVersionV2BridgeDAL, secretTagDAL, - secretVersionTagDAL: secretVersionTagV2BridgeDAL + secretVersionTagDAL: secretVersionTagV2BridgeDAL, + folderCommitService }) : []; const updatedSecrets = secretUpdationCommits.length ? await fnSecretV2BridgeBulkUpdate({ folderId, orgId: actorOrgId, + actor: { + actorId, + type: actor + }, tx, inputSecrets: secretUpdationCommits.map((el) => { const encryptedValue = @@ -659,7 +682,8 @@ export const secretApprovalRequestServiceFactory = ({ secretVersionDAL: secretVersionV2BridgeDAL, secretTagDAL, secretVersionTagDAL: secretVersionTagV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }) : []; const deletedSecret = secretDeletionCommits.length @@ -667,10 +691,13 @@ export const secretApprovalRequestServiceFactory = ({ projectId, folderId, tx, - actorId: "", + actorId, + actorType: actor, secretDAL: secretV2BridgeDAL, secretQueueService, - inputSecrets: secretDeletionCommits.map(({ key }) => ({ secretKey: key, type: SecretType.Shared })) + inputSecrets: secretDeletionCommits.map(({ key }) => ({ secretKey: key, type: SecretType.Shared })), + folderCommitService, + secretVersionDAL: secretVersionV2BridgeDAL }) : []; const updatedSecretApproval = await secretApprovalRequestDAL.updateById( diff --git a/backend/src/ee/services/secret-replication/secret-replication-service.ts b/backend/src/ee/services/secret-replication/secret-replication-service.ts index 90fdf561e..628f8e310 100644 --- a/backend/src/ee/services/secret-replication/secret-replication-service.ts +++ b/backend/src/ee/services/secret-replication/secret-replication-service.ts @@ -10,6 +10,7 @@ import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { QueueName, TQueueServiceFactory } from "@app/queue"; import { ActorType } from "@app/services/auth/auth-type"; +import { TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; @@ -87,6 +88,7 @@ type TSecretReplicationServiceFactoryDep = { projectBotService: Pick; kmsService: Pick; + folderCommitService: Pick; }; export type TSecretReplicationServiceFactory = ReturnType; @@ -132,6 +134,7 @@ export const secretReplicationServiceFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, kmsService, + folderCommitService, resourceMetadataDAL }: TSecretReplicationServiceFactoryDep) => { const $getReplicatedSecrets = ( @@ -419,7 +422,7 @@ export const secretReplicationServiceFactory = ({ return { op: operation, requestId: approvalRequestDoc.id, - metadata: doc.metadata, + metadata: doc.metadata ? JSON.stringify(doc.metadata) : [], secretMetadata: JSON.stringify(doc.secretMetadata), key: doc.key, encryptedValue: doc.encryptedValue, @@ -446,11 +449,12 @@ export const secretReplicationServiceFactory = ({ tx, secretTagDAL, resourceMetadataDAL, + folderCommitService, secretVersionTagDAL: secretVersionV2TagBridgeDAL, inputSecrets: locallyCreatedSecrets.map((doc) => { return { type: doc.type, - metadata: doc.metadata, + metadata: doc.metadata ? JSON.stringify(doc.metadata) : [], key: doc.key, encryptedValue: doc.encryptedValue, encryptedComment: doc.encryptedComment, @@ -466,6 +470,7 @@ export const secretReplicationServiceFactory = ({ orgId, folderId: destinationReplicationFolderId, secretVersionDAL: secretVersionV2BridgeDAL, + folderCommitService, secretDAL: secretV2BridgeDAL, tx, resourceMetadataDAL, @@ -479,7 +484,7 @@ export const secretReplicationServiceFactory = ({ }, data: { type: doc.type, - metadata: doc.metadata, + metadata: doc.metadata ? JSON.stringify(doc.metadata) : [], key: doc.key, encryptedValue: doc.encryptedValue as Buffer, encryptedComment: doc.encryptedComment, diff --git a/backend/src/ee/services/secret-rotation-v2/azure-client-secret/azure-client-secret-rotation-fns.ts b/backend/src/ee/services/secret-rotation-v2/azure-client-secret/azure-client-secret-rotation-fns.ts index 037df50ac..e2969ac1d 100644 --- a/backend/src/ee/services/secret-rotation-v2/azure-client-secret/azure-client-secret-rotation-fns.ts +++ b/backend/src/ee/services/secret-rotation-v2/azure-client-secret/azure-client-secret-rotation-fns.ts @@ -101,10 +101,56 @@ export const azureClientSecretRotationFactory: TRotationFactory< } }; + /** + * Checks if a credential with the given keyId exists. + */ + const credentialExists = async (keyId: string): Promise => { + const accessToken = await getAzureConnectionAccessToken(connection.id, appConnectionDAL, kmsService); + const endpoint = `${GRAPH_API_BASE}/applications/${objectId}/passwordCredentials`; + + try { + const { data } = await request.get<{ value: Array<{ keyId: string }> }>(endpoint, { + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + } + }); + + return data.value?.some((credential) => credential.keyId === keyId) || false; + } catch (error: unknown) { + if (error instanceof AxiosError) { + let message; + if ( + error.response?.data && + typeof error.response.data === "object" && + "error" in error.response.data && + typeof (error.response.data as AzureErrorResponse).error.message === "string" + ) { + message = (error.response.data as AzureErrorResponse).error.message; + } + throw new BadRequestError({ + message: `Failed to check credential existence for app ${objectId}: ${ + message || error.message || "Unknown error" + }` + }); + } + throw new BadRequestError({ + message: "Unable to validate connection: verify credentials" + }); + } + }; + /** * Revokes a client secret from the Azure app using its keyId. + * First checks if the credential exists before attempting revocation. */ const revokeCredential = async (keyId: string) => { + // Check if credential exists before attempting revocation + const exists = await credentialExists(keyId); + if (!exists) { + return; // Credential doesn't exist, nothing to revoke + } + const accessToken = await getAzureConnectionAccessToken(connection.id, appConnectionDAL, kmsService); const endpoint = `${GRAPH_API_BASE}/applications/${objectId}/removePassword`; diff --git a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts index 9518e603c..986cb0ee4 100644 --- a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts +++ b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts @@ -63,6 +63,7 @@ import { TAppConnectionDALFactory } from "@app/services/app-connection/app-conne import { decryptAppConnection } from "@app/services/app-connection/app-connection-fns"; import { TAppConnectionServiceFactory } from "@app/services/app-connection/app-connection-service"; import { ActorType } from "@app/services/auth/auth-type"; +import { TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; @@ -98,7 +99,7 @@ export type TSecretRotationV2ServiceFactoryDep = { TSecretV2BridgeDALFactory, "bulkUpdate" | "insertMany" | "deleteMany" | "upsertSecretReferences" | "find" | "invalidateSecretCacheByProjectId" >; - secretVersionV2BridgeDAL: Pick; + secretVersionV2BridgeDAL: Pick; secretVersionTagV2BridgeDAL: Pick; resourceMetadataDAL: Pick; secretTagDAL: Pick; @@ -106,6 +107,7 @@ export type TSecretRotationV2ServiceFactoryDep = { snapshotService: Pick; queueService: Pick; appConnectionDAL: Pick; + folderCommitService: Pick; }; export type TSecretRotationV2ServiceFactory = ReturnType; @@ -146,6 +148,7 @@ export const secretRotationV2ServiceFactory = ({ snapshotService, keyStore, queueService, + folderCommitService, appConnectionDAL }: TSecretRotationV2ServiceFactoryDep) => { const $queueSendSecretRotationStatusNotification = async (secretRotation: TSecretRotationV2Raw) => { @@ -539,7 +542,12 @@ export const secretRotationV2ServiceFactory = ({ secretVersionDAL: secretVersionV2BridgeDAL, secretVersionTagDAL: secretVersionTagV2BridgeDAL, secretTagDAL, - resourceMetadataDAL + folderCommitService, + resourceMetadataDAL, + actor: { + type: actor.type, + actorId: actor.id + } }); await secretRotationV2DAL.insertSecretMappings( @@ -675,7 +683,12 @@ export const secretRotationV2ServiceFactory = ({ secretVersionDAL: secretVersionV2BridgeDAL, secretVersionTagDAL: secretVersionTagV2BridgeDAL, secretTagDAL, - resourceMetadataDAL + folderCommitService, + resourceMetadataDAL, + actor: { + type: actor.type, + actorId: actor.id + } }); secretsMappingUpdated = true; @@ -793,6 +806,9 @@ export const secretRotationV2ServiceFactory = ({ projectId, folderId, actorId: actor.id, // not actually used since rotated secrets are shared + actorType: actor.type, + folderCommitService, + secretVersionDAL: secretVersionV2BridgeDAL, tx }); } @@ -936,6 +952,10 @@ export const secretRotationV2ServiceFactory = ({ secretDAL: secretV2BridgeDAL, secretVersionDAL: secretVersionV2BridgeDAL, secretVersionTagDAL: secretVersionTagV2BridgeDAL, + folderCommitService, + actor: { + type: ActorType.PLATFORM + }, secretTagDAL, resourceMetadataDAL }); diff --git a/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts b/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts index 2c6124348..d792ac6e6 100644 --- a/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts +++ b/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts @@ -14,6 +14,7 @@ import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; import { ActorType } from "@app/services/auth/auth-type"; +import { CommitType, TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; @@ -53,6 +54,7 @@ type TSecretRotationQueueFactoryDep = { secretVersionV2BridgeDAL: Pick; telemetryService: Pick; kmsService: Pick; + folderCommitService: Pick; }; // These error should stop the repeatable job and ask user to reconfigure rotation @@ -77,6 +79,7 @@ export const secretRotationQueueFactory = ({ telemetryService, secretV2BridgeDAL, secretVersionV2BridgeDAL, + folderCommitService, kmsService }: TSecretRotationQueueFactoryDep) => { const addToQueue = async (rotationId: string, interval: number) => { @@ -330,7 +333,7 @@ export const secretRotationQueueFactory = ({ })), tx ); - await secretVersionV2BridgeDAL.insertMany( + const secretVersions = await secretVersionV2BridgeDAL.insertMany( updatedSecrets.map(({ id, updatedAt, createdAt, ...el }) => ({ ...el, actorType: ActorType.PLATFORM, @@ -338,6 +341,22 @@ export const secretRotationQueueFactory = ({ })), tx ); + + await folderCommitService.createCommit( + { + actor: { + type: ActorType.PLATFORM + }, + message: "Changed by Secret rotation", + folderId: secretVersions[0].folderId, + changes: secretVersions.map((sv) => ({ + type: CommitType.ADD, + isUpdate: true, + secretVersionId: sv.id + })) + }, + tx + ); }); await secretV2BridgeDAL.invalidateSecretCacheByProjectId(secretRotation.projectId); diff --git a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts index 015a8d420..8cae3dfb5 100644 --- a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts +++ b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts @@ -8,6 +8,7 @@ import { InternalServerError, NotFoundError } from "@app/lib/errors"; import { groupBy } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; import { ActorType } from "@app/services/auth/auth-type"; +import { CommitType, TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; @@ -51,8 +52,8 @@ type TSecretSnapshotServiceFactoryDep = { snapshotSecretV2BridgeDAL: TSnapshotSecretV2DALFactory; snapshotFolderDAL: TSnapshotFolderDALFactory; secretVersionDAL: Pick; - secretVersionV2BridgeDAL: Pick; - folderVersionDAL: Pick; + secretVersionV2BridgeDAL: Pick; + folderVersionDAL: Pick; secretDAL: Pick; secretV2BridgeDAL: Pick; secretTagDAL: Pick; @@ -63,6 +64,7 @@ type TSecretSnapshotServiceFactoryDep = { licenseService: Pick; kmsService: Pick; projectBotService: Pick; + folderCommitService: Pick; }; export type TSecretSnapshotServiceFactory = ReturnType; @@ -84,7 +86,8 @@ export const secretSnapshotServiceFactory = ({ snapshotSecretV2BridgeDAL, secretVersionV2TagBridgeDAL, kmsService, - projectBotService + projectBotService, + folderCommitService }: TSecretSnapshotServiceFactoryDep) => { const projectSecretSnapshotCount = async ({ environment, @@ -403,6 +406,18 @@ export const secretSnapshotServiceFactory = ({ .filter((el) => el.isRotatedSecret) .map((el) => el.secretId); + const deletedSecretsChanges = new Map(); // secretId -> version info + const deletedFoldersChanges = new Map(); // folderId -> version info + const addedSecretsChanges = new Map(); // secretId -> version info + const addedFoldersChanges = new Map(); // folderId -> version info + const commitChanges: { + type: string; + secretVersionId?: string; + folderVersionId?: string; + isUpdate?: boolean; + folderId?: string; + }[] = []; + // this will remove all secrets in current folder except rotated secrets which we ignore const deletedTopLevelSecs = await secretV2BridgeDAL.delete( { @@ -424,7 +439,35 @@ export const secretSnapshotServiceFactory = ({ }, tx ); + + await Promise.all( + deletedTopLevelSecs.map(async (sec) => { + const version = await secretVersionV2BridgeDAL.findOne({ secretId: sec.id, version: sec.version }, tx); + deletedSecretsChanges.set(sec.id, { + id: sec.id, + version: sec.version, + // Store the version ID if available from the snapshot + versionId: version?.id + }); + }) + ); + const deletedTopLevelSecsGroupById = groupBy(deletedTopLevelSecs, (item) => item.id); + + const deletedFoldersData = await folderDAL.delete({ parentId: snapshot.folderId, isReserved: false }, tx); + + await Promise.all( + deletedFoldersData.map(async (folder) => { + const version = await folderVersionDAL.findOne({ folderId: folder.id, version: folder.version }, tx); + deletedFoldersChanges.set(folder.id, { + id: folder.id, + version: folder.version, + // Store the version ID if available + versionId: version?.id + }); + }) + ); + // this will remove all secrets and folders on child // due to sql foreign key and link list connection removing the folders removes everything below too const deletedFolders = await folderDAL.delete({ parentId: snapshot.folderId, isReserved: false }, tx); @@ -489,14 +532,21 @@ export const secretSnapshotServiceFactory = ({ }); await secretTagDAL.saveTagsToSecretV2(secretTagsToBeInsert, tx); const folderVersions = await folderVersionDAL.insertMany( - folders.map(({ version, name, id, envId }) => ({ + folders.map(({ version, name, id, envId, description }) => ({ name, version, folderId: id, - envId + envId, + description })), tx ); + + // Track added folders + folderVersions.forEach((fv) => { + addedFoldersChanges.set(fv.folderId, fv); + }); + const userActorId = actor === ActorType.USER ? actorId : undefined; const identityActorId = actor !== ActorType.USER ? actorId : undefined; const actorType = actor || ActorType.PLATFORM; @@ -511,6 +561,11 @@ export const secretSnapshotServiceFactory = ({ })), tx ); + + secretVersions.forEach((sv) => { + addedSecretsChanges.set(sv.secretId, sv); + }); + await secretVersionV2TagBridgeDAL.insertMany( secretVersions.flatMap(({ secretId, id }) => secretVerTagToBeInsert?.[secretId]?.length @@ -522,6 +577,70 @@ export const secretSnapshotServiceFactory = ({ ), tx ); + + // Compute commit changes + // Handle secrets + deletedSecretsChanges.forEach((deletedInfo, secretId) => { + const addedSecret = addedSecretsChanges.get(secretId); + if (addedSecret) { + // Secret was deleted and re-added - this is an update only if versions are different + if (deletedInfo.versionId !== addedSecret.id) { + commitChanges.push({ + type: CommitType.ADD, // In the commit system, updates are tracked as "add" with isUpdate=true + secretVersionId: addedSecret.id, + isUpdate: true + }); + } + // Remove from addedSecrets since we've handled it + addedSecretsChanges.delete(secretId); + } else if (deletedInfo.versionId) { + // Secret was only deleted + commitChanges.push({ + type: CommitType.DELETE, + secretVersionId: deletedInfo.versionId + }); + } + }); + // Add remaining new secrets (not updates) + addedSecretsChanges.forEach((addedSecret) => { + commitChanges.push({ + type: CommitType.ADD, + secretVersionId: addedSecret.id + }); + }); + + // Handle folders + deletedFoldersChanges.forEach((deletedInfo, folderId) => { + const addedFolder = addedFoldersChanges.get(folderId); + if (addedFolder) { + // Folder was deleted and re-added - this is an update only if versions are different + if (deletedInfo.versionId !== addedFolder.id) { + commitChanges.push({ + type: CommitType.ADD, + folderVersionId: addedFolder.id, + isUpdate: true + }); + } + // Remove from addedFolders since we've handled it + addedFoldersChanges.delete(folderId); + } else if (deletedInfo.versionId) { + // Folder was only deleted + commitChanges.push({ + type: CommitType.DELETE, + folderVersionId: deletedInfo.versionId, + folderId: deletedInfo.id + }); + } + }); + + // Add remaining new folders (not updates) + addedFoldersChanges.forEach((addedFolder) => { + commitChanges.push({ + type: CommitType.ADD, + folderVersionId: addedFolder.id + }); + }); + const newSnapshot = await snapshotDAL.create( { folderId: snapshot.folderId, @@ -550,6 +669,22 @@ export const secretSnapshotServiceFactory = ({ })), tx ); + if (commitChanges.length > 0) { + await folderCommitService.createCommit( + { + actor: { + type: actorType, + metadata: { + id: userActorId || identityActorId + } + }, + message: "Rollback to snapshot", + folderId: snapshot.folderId, + changes: commitChanges + }, + tx + ); + } return { ...newSnapshot, snapshotSecrets, snapshotFolders }; }); @@ -609,11 +744,12 @@ export const secretSnapshotServiceFactory = ({ }); await secretTagDAL.saveTagsToSecret(secretTagsToBeInsert, tx); const folderVersions = await folderVersionDAL.insertMany( - folders.map(({ version, name, id, envId }) => ({ + folders.map(({ version, name, id, envId, description }) => ({ name, version, folderId: id, - envId + envId, + description })), tx ); diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.ts index c6ec5dccb..d3f19168a 100644 --- a/backend/src/keystore/keystore.ts +++ b/backend/src/keystore/keystore.ts @@ -27,6 +27,7 @@ export const KeyStorePrefixes = { KmsOrgDataKeyCreation: "kms-org-data-key-creation-lock", WaitUntilReadyKmsOrgKeyCreation: "wait-until-ready-kms-org-key-creation-", WaitUntilReadyKmsOrgDataKeyCreation: "wait-until-ready-kms-org-data-key-creation-", + FolderTreeCheckpoint: (envId: string) => `folder-tree-checkpoint-${envId}`, WaitUntilReadyProjectEnvironmentOperation: (projectId: string) => `wait-until-ready-project-environments-operation-${projectId}`, diff --git a/backend/src/lib/api-docs/constants.ts b/backend/src/lib/api-docs/constants.ts index a10fcc67e..caac30556 100644 --- a/backend/src/lib/api-docs/constants.ts +++ b/backend/src/lib/api-docs/constants.ts @@ -626,7 +626,8 @@ export const PROJECTS = { autoCapitalization: "Disable or enable auto-capitalization for the project.", slug: "An optional slug for the project. (must be unique within the organization)", hasDeleteProtection: "Enable or disable delete protection for the project.", - secretSharing: "Enable or disable secret sharing for the project." + secretSharing: "Enable or disable secret sharing for the project.", + showSnapshotsLegacy: "Enable or disable legacy snapshots for the project." }, GET_KEY: { workspaceId: "The ID of the project to get the key from." @@ -1112,6 +1113,14 @@ export const DYNAMIC_SECRET_LEASES = { leaseId: "The ID of the dynamic secret lease.", isForced: "A boolean flag to delete the the dynamic secret from Infisical without trying to remove it from external provider. Used when the dynamic secret got modified externally." + }, + KUBERNETES: { + CREATE: { + config: { + namespace: + "The Kubernetes namespace to create the lease in. If not specified, the first namespace defined in the configuration will be used." + } + } } } as const; export const SECRET_TAGS = { diff --git a/backend/src/lib/config/env.ts b/backend/src/lib/config/env.ts index e2fc73d8a..6b9d33c2a 100644 --- a/backend/src/lib/config/env.ts +++ b/backend/src/lib/config/env.ts @@ -261,6 +261,10 @@ const envSchema = z DATADOG_SERVICE: zpStr(z.string().optional().default("infisical-core")), DATADOG_HOSTNAME: zpStr(z.string().optional()), + // PIT + PIT_CHECKPOINT_WINDOW: zpStr(z.string().optional().default("2")), + PIT_TREE_CHECKPOINT_WINDOW: zpStr(z.string().optional().default("30")), + /* CORS ----------------------------------------------------------------------------- */ CORS_ALLOWED_ORIGINS: zpStr( z diff --git a/backend/src/lib/gateway/gateway.ts b/backend/src/lib/gateway/gateway.ts index 179c29fc8..46481a049 100644 --- a/backend/src/lib/gateway/gateway.ts +++ b/backend/src/lib/gateway/gateway.ts @@ -149,8 +149,8 @@ const setupProxyServer = async ({ protocol = GatewayProxyProtocol.Tcp, httpsAgent }: { - targetHost: string; - targetPort: number; + targetHost?: string; + targetPort?: number; relayPort: number; relayHost: string; tlsOptions: TGatewayTlsOptions; @@ -183,27 +183,44 @@ const setupProxyServer = async ({ let command: string; if (protocol === GatewayProxyProtocol.Http) { - const targetUrl = `${targetHost}:${targetPort}`; // note(daniel): targetHost MUST include the scheme (https|http) - command = `FORWARD-HTTP ${targetUrl}`; - logger.debug(`Using HTTP proxy mode: ${command.trim()}`); + if (!targetHost && !targetPort) { + command = `FORWARD-HTTP`; + logger.debug(`Using HTTP proxy mode, no target URL provided [command=${command.trim()}]`); + } else { + if (!targetHost || targetPort === undefined) { + throw new BadRequestError({ + message: `Target host and port are required for HTTP proxy mode with custom target` + }); + } - // extract ca certificate from httpsAgent if present - if (httpsAgent && targetHost.startsWith("https://")) { - const agentOptions = httpsAgent.options; - if (agentOptions && agentOptions.ca) { - const caCert = Array.isArray(agentOptions.ca) ? agentOptions.ca.join("\n") : agentOptions.ca; - const caB64 = Buffer.from(caCert as string).toString("base64"); - command += ` ca=${caB64}`; + const targetUrl = `${targetHost}:${targetPort}`; // note(daniel): targetHost MUST include the scheme (https|http) + command = `FORWARD-HTTP ${targetUrl}`; + logger.debug(`Using HTTP proxy mode, custom target URL provided [command=${command.trim()}]`); - const rejectUnauthorized = agentOptions.rejectUnauthorized !== false; - command += ` verify=${rejectUnauthorized}`; + // extract ca certificate from httpsAgent if present + if (httpsAgent && targetHost.startsWith("https://")) { + const agentOptions = httpsAgent.options; + if (agentOptions && agentOptions.ca) { + const caCert = Array.isArray(agentOptions.ca) ? agentOptions.ca.join("\n") : agentOptions.ca; + const caB64 = Buffer.from(caCert as string).toString("base64"); + command += ` ca=${caB64}`; - logger.debug(`Using HTTP proxy mode [command=${command.trim()}]`); + const rejectUnauthorized = agentOptions.rejectUnauthorized !== false; + command += ` verify=${rejectUnauthorized}`; + + logger.debug(`Using HTTP proxy mode, custom target URL provided [command=${command.trim()}]`); + } } } command += "\n"; } else if (protocol === GatewayProxyProtocol.Tcp) { + if (!targetHost || !targetPort) { + throw new BadRequestError({ + message: `Target host and port are required for TCP proxy mode` + }); + } + // For TCP mode, send FORWARD-TCP with host:port command = `FORWARD-TCP ${targetHost}:${targetPort}\n`; logger.debug(`Using TCP proxy mode: ${command.trim()}`); diff --git a/backend/src/lib/gateway/types.ts b/backend/src/lib/gateway/types.ts index 5d0ac8237..8552fbf54 100644 --- a/backend/src/lib/gateway/types.ts +++ b/backend/src/lib/gateway/types.ts @@ -10,12 +10,13 @@ export enum GatewayProxyProtocol { } export enum GatewayHttpProxyActions { - InjectGatewayK8sServiceAccountToken = "inject-k8s-sa-auth-token" + InjectGatewayK8sServiceAccountToken = "inject-k8s-sa-auth-token", + UseGatewayK8sServiceAccount = "use-k8s-sa" } export interface IGatewayProxyOptions { - targetHost: string; - targetPort: number; + targetHost?: string; + targetPort?: number; relayHost: string; relayPort: number; tlsOptions: TGatewayTlsOptions; diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index e4d654998..25677841d 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -60,6 +60,7 @@ export enum QueueName { ImportSecretsFromExternalSource = "import-secrets-from-external-source", AppConnectionSecretSync = "app-connection-secret-sync", SecretRotationV2 = "secret-rotation-v2", + FolderTreeCheckpoint = "folder-tree-checkpoint", InvalidateCache = "invalidate-cache", SecretScanningV2 = "secret-scanning-v2" } @@ -94,6 +95,7 @@ export enum QueueJobs { SecretRotationV2QueueRotations = "secret-rotation-v2-queue-rotations", SecretRotationV2RotateSecrets = "secret-rotation-v2-rotate-secrets", SecretRotationV2SendNotification = "secret-rotation-v2-send-notification", + CreateFolderTreeCheckpoint = "create-folder-tree-checkpoint", InvalidateCache = "invalidate-cache", SecretScanningV2FullScan = "secret-scanning-v2-full-scan", SecretScanningV2DiffScan = "secret-scanning-v2-diff-scan", @@ -209,6 +211,12 @@ export type TQueueJobTypes = { name: QueueJobs.ProjectV3Migration; payload: { projectId: string }; }; + [QueueName.FolderTreeCheckpoint]: { + name: QueueJobs.CreateFolderTreeCheckpoint; + payload: { + envId: string; + }; + }; [QueueName.ImportSecretsFromExternalSource]: { name: QueueJobs.ImportSecretsFromExternalSource; payload: { diff --git a/backend/src/server/plugins/serve-ui.ts b/backend/src/server/plugins/serve-ui.ts index 22c097726..b71451b6e 100644 --- a/backend/src/server/plugins/serve-ui.ts +++ b/backend/src/server/plugins/serve-ui.ts @@ -57,9 +57,12 @@ export const registerServeUI = async ( reply.callNotFound(); return; } - // reference: https://github.com/fastify/fastify-static?tab=readme-ov-file#managing-cache-control-headers - // to avoid ui bundle skew on new deployment - return reply.sendFile("index.html", { maxAge: 0, immutable: false }); + + // This should help avoid caching any chunks (temp fix) + void reply.header("Cache-Control", "no-cache, no-store, must-revalidate, private, max-age=0"); + void reply.header("Pragma", "no-cache"); + void reply.header("Expires", "0"); + return reply.sendFile("index.html"); } }); } diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index aa38bb0e8..5513fb49b 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -60,6 +60,7 @@ import { oidcConfigDALFactory } from "@app/ee/services/oidc/oidc-config-dal"; import { oidcConfigServiceFactory } from "@app/ee/services/oidc/oidc-config-service"; import { permissionDALFactory } from "@app/ee/services/permission/permission-dal"; import { permissionServiceFactory } from "@app/ee/services/permission/permission-service"; +import { pitServiceFactory } from "@app/ee/services/pit/pit-service"; import { projectTemplateDALFactory } from "@app/ee/services/project-template/project-template-dal"; import { projectTemplateServiceFactory } from "@app/ee/services/project-template/project-template-service"; import { projectUserAdditionalPrivilegeDALFactory } from "@app/ee/services/project-user-additional-privilege/project-user-additional-privilege-dal"; @@ -154,6 +155,14 @@ import { externalGroupOrgRoleMappingDALFactory } from "@app/services/external-gr import { externalGroupOrgRoleMappingServiceFactory } from "@app/services/external-group-org-role-mapping/external-group-org-role-mapping-service"; import { externalMigrationQueueFactory } from "@app/services/external-migration/external-migration-queue"; import { externalMigrationServiceFactory } from "@app/services/external-migration/external-migration-service"; +import { folderCheckpointDALFactory } from "@app/services/folder-checkpoint/folder-checkpoint-dal"; +import { folderCheckpointResourcesDALFactory } from "@app/services/folder-checkpoint-resources/folder-checkpoint-resources-dal"; +import { folderCommitDALFactory } from "@app/services/folder-commit/folder-commit-dal"; +import { folderCommitQueueServiceFactory } from "@app/services/folder-commit/folder-commit-queue"; +import { folderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; +import { folderCommitChangesDALFactory } from "@app/services/folder-commit-changes/folder-commit-changes-dal"; +import { folderTreeCheckpointDALFactory } from "@app/services/folder-tree-checkpoint/folder-tree-checkpoint-dal"; +import { folderTreeCheckpointResourcesDALFactory } from "@app/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal"; import { groupProjectDALFactory } from "@app/services/group-project/group-project-dal"; import { groupProjectMembershipRoleDALFactory } from "@app/services/group-project/group-project-membership-role-dal"; import { groupProjectServiceFactory } from "@app/services/group-project/group-project-service"; @@ -583,6 +592,41 @@ export const registerRoutes = async ( projectRoleDAL, permissionService }); + + const folderCommitChangesDAL = folderCommitChangesDALFactory(db); + const folderCheckpointDAL = folderCheckpointDALFactory(db); + const folderCheckpointResourcesDAL = folderCheckpointResourcesDALFactory(db); + const folderTreeCheckpointDAL = folderTreeCheckpointDALFactory(db); + const folderCommitDAL = folderCommitDALFactory(db); + const folderTreeCheckpointResourcesDAL = folderTreeCheckpointResourcesDALFactory(db); + const folderCommitQueueService = folderCommitQueueServiceFactory({ + queueService, + folderTreeCheckpointDAL, + keyStore, + folderTreeCheckpointResourcesDAL, + folderCommitDAL, + folderDAL + }); + const folderCommitService = folderCommitServiceFactory({ + folderCommitDAL, + folderCommitChangesDAL, + folderCheckpointDAL, + folderTreeCheckpointDAL, + userDAL, + identityDAL, + folderDAL, + folderVersionDAL, + secretVersionV2BridgeDAL, + projectDAL, + folderCheckpointResourcesDAL, + secretV2BridgeDAL, + folderTreeCheckpointResourcesDAL, + folderCommitQueueService, + permissionService, + kmsService, + secretTagDAL, + resourceMetadataDAL + }); const scimService = scimServiceFactory({ licenseService, scimDAL, @@ -987,6 +1031,7 @@ export const registerRoutes = async ( projectMembershipDAL, projectBotDAL, secretDAL, + folderCommitService, secretBlindIndexDAL, secretVersionDAL, secretTagDAL, @@ -1034,6 +1079,7 @@ export const registerRoutes = async ( secretReminderRecipientsDAL, orgService, resourceMetadataDAL, + folderCommitService, secretSyncQueue }); @@ -1110,6 +1156,7 @@ export const registerRoutes = async ( snapshotDAL, snapshotFolderDAL, snapshotSecretDAL, + folderCommitService, secretVersionDAL, folderVersionDAL, secretTagDAL, @@ -1136,7 +1183,8 @@ export const registerRoutes = async ( folderVersionDAL, projectEnvDAL, snapshotService, - projectDAL + projectDAL, + folderCommitService }); const secretImportService = secretImportServiceFactory({ @@ -1161,6 +1209,7 @@ export const registerRoutes = async ( const secretV2BridgeService = secretV2BridgeServiceFactory({ folderDAL, secretVersionDAL: secretVersionV2BridgeDAL, + folderCommitService, secretQueueService, secretDAL: secretV2BridgeDAL, permissionService, @@ -1204,7 +1253,8 @@ export const registerRoutes = async ( projectSlackConfigDAL, resourceMetadataDAL, projectMicrosoftTeamsConfigDAL, - microsoftTeamsService + microsoftTeamsService, + folderCommitService }); const secretService = secretServiceFactory({ @@ -1291,7 +1341,8 @@ export const registerRoutes = async ( secretV2BridgeDAL, secretVersionV2TagBridgeDAL: secretVersionTagV2BridgeDAL, secretVersionV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }); const secretRotationQueue = secretRotationQueueFactory({ @@ -1303,6 +1354,7 @@ export const registerRoutes = async ( projectBotService, secretVersionV2BridgeDAL, secretV2BridgeDAL, + folderCommitService, kmsService }); @@ -1454,6 +1506,15 @@ export const registerRoutes = async ( permissionService }); + const pitService = pitServiceFactory({ + folderCommitService, + secretService, + folderService, + permissionService, + folderDAL, + projectEnvDAL + }); + const identityOidcAuthService = identityOidcAuthServiceFactory({ identityOidcAuthDAL, identityOrgMembershipDAL, @@ -1597,7 +1658,9 @@ export const registerRoutes = async ( secretDAL: secretV2BridgeDAL, queueService, secretV2BridgeService, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService, + folderVersionDAL }); const migrationService = externalMigrationServiceFactory({ @@ -1707,6 +1770,7 @@ export const registerRoutes = async ( auditLogService, secretV2BridgeDAL, secretTagDAL, + folderCommitService, secretVersionTagV2BridgeDAL, secretVersionV2BridgeDAL, keyStore, @@ -1895,6 +1959,7 @@ export const registerRoutes = async ( certificateTemplate: certificateTemplateService, certificateAuthorityCrl: certificateAuthorityCrlService, certificateEst: certificateEstService, + pit: pitService, pkiAlert: pkiAlertService, pkiCollection: pkiCollectionService, pkiSubscriber: pkiSubscriberService, @@ -1929,6 +1994,7 @@ export const registerRoutes = async ( microsoftTeams: microsoftTeamsService, assumePrivileges: assumePrivilegeService, githubOrgSync: githubOrgSyncConfigService, + folderCommit: folderCommitService, secretScanningV2: secretScanningV2Service }); diff --git a/backend/src/server/routes/sanitizedSchemas.ts b/backend/src/server/routes/sanitizedSchemas.ts index a26293ac8..ce51b1079 100644 --- a/backend/src/server/routes/sanitizedSchemas.ts +++ b/backend/src/server/routes/sanitizedSchemas.ts @@ -262,7 +262,8 @@ export const SanitizedProjectSchema = ProjectsSchema.pick({ kmsCertificateKeyId: true, auditLogsRetentionDays: true, hasDeleteProtection: true, - secretSharing: true + secretSharing: true, + showSnapshotsLegacy: true }); export const SanitizedTagSchema = SecretTagsSchema.pick({ diff --git a/backend/src/server/routes/v1/identity-kubernetes-auth-router.ts b/backend/src/server/routes/v1/identity-kubernetes-auth-router.ts index 5eb0c6990..879310790 100644 --- a/backend/src/server/routes/v1/identity-kubernetes-auth-router.ts +++ b/backend/src/server/routes/v1/identity-kubernetes-auth-router.ts @@ -108,17 +108,21 @@ export const registerIdentityKubernetesRouter = async (server: FastifyZodProvide .string() .trim() .min(1) + .nullable() .describe(KUBERNETES_AUTH.ATTACH.kubernetesHost) .refine( - (val) => - characterValidator([ + (val) => { + if (val === null) return true; + + return characterValidator([ CharacterType.Alphabets, CharacterType.Numbers, CharacterType.Colon, CharacterType.Period, CharacterType.ForwardSlash, CharacterType.Hyphen - ])(val), + ])(val); + }, { message: "Kubernetes host must only contain alphabets, numbers, colons, periods, hyphen, and forward slashes." @@ -164,6 +168,13 @@ export const registerIdentityKubernetesRouter = async (server: FastifyZodProvide .describe(KUBERNETES_AUTH.ATTACH.accessTokenNumUsesLimit) }) .superRefine((data, ctx) => { + if (data.tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Api && !data.kubernetesHost) { + ctx.addIssue({ + path: ["kubernetesHost"], + code: z.ZodIssueCode.custom, + message: "When token review mode is set to API, a Kubernetes host must be provided" + }); + } if (data.tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Gateway && !data.gatewayId) { ctx.addIssue({ path: ["gatewayId"], @@ -171,6 +182,7 @@ export const registerIdentityKubernetesRouter = async (server: FastifyZodProvide message: "When token review mode is set to Gateway, a gateway must be selected" }); } + if (data.accessTokenTTL > data.accessTokenMaxTTL) { ctx.addIssue({ path: ["accessTokenTTL"], @@ -203,7 +215,7 @@ export const registerIdentityKubernetesRouter = async (server: FastifyZodProvide type: EventType.ADD_IDENTITY_KUBERNETES_AUTH, metadata: { identityId: identityKubernetesAuth.identityId, - kubernetesHost: identityKubernetesAuth.kubernetesHost, + kubernetesHost: identityKubernetesAuth.kubernetesHost ?? "", allowedNamespaces: identityKubernetesAuth.allowedNamespaces, allowedNames: identityKubernetesAuth.allowedNames, accessTokenTTL: identityKubernetesAuth.accessTokenTTL, @@ -243,6 +255,7 @@ export const registerIdentityKubernetesRouter = async (server: FastifyZodProvide .string() .trim() .min(1) + .nullable() .optional() .describe(KUBERNETES_AUTH.UPDATE.kubernetesHost) .refine( @@ -345,7 +358,7 @@ export const registerIdentityKubernetesRouter = async (server: FastifyZodProvide type: EventType.UPDATE_IDENTITY_KUBENETES_AUTH, metadata: { identityId: identityKubernetesAuth.identityId, - kubernetesHost: identityKubernetesAuth.kubernetesHost, + kubernetesHost: identityKubernetesAuth.kubernetesHost ?? "", allowedNamespaces: identityKubernetesAuth.allowedNamespaces, allowedNames: identityKubernetesAuth.allowedNames, accessTokenTTL: identityKubernetesAuth.accessTokenTTL, diff --git a/backend/src/server/routes/v1/project-router.ts b/backend/src/server/routes/v1/project-router.ts index 2a868864e..cc94adede 100644 --- a/backend/src/server/routes/v1/project-router.ts +++ b/backend/src/server/routes/v1/project-router.ts @@ -376,7 +376,8 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { }) .optional() .describe(PROJECTS.UPDATE.slug), - secretSharing: z.boolean().optional().describe(PROJECTS.UPDATE.secretSharing) + secretSharing: z.boolean().optional().describe(PROJECTS.UPDATE.secretSharing), + showSnapshotsLegacy: z.boolean().optional().describe(PROJECTS.UPDATE.showSnapshotsLegacy) }), response: { 200: z.object({ @@ -397,7 +398,8 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { autoCapitalization: req.body.autoCapitalization, hasDeleteProtection: req.body.hasDeleteProtection, slug: req.body.slug, - secretSharing: req.body.secretSharing + secretSharing: req.body.secretSharing, + showSnapshotsLegacy: req.body.showSnapshotsLegacy }, actorAuthMethod: req.permission.authMethod, actorId: req.permission.id, diff --git a/backend/src/services/external-migration/external-migration-fns.ts b/backend/src/services/external-migration/external-migration-fns.ts index 856b39012..018d7bd43 100644 --- a/backend/src/services/external-migration/external-migration-fns.ts +++ b/backend/src/services/external-migration/external-migration-fns.ts @@ -10,6 +10,7 @@ import { chunkArray } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; +import { CommitType, TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "../kms/kms-service"; import { KmsDataKey } from "../kms/kms-types"; import { TProjectDALFactory } from "../project/project-dal"; @@ -18,6 +19,7 @@ import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TProjectEnvServiceFactory } from "../project-env/project-env-service"; import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal"; import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; import { fnSecretBulkInsert, getAllSecretReferences } from "../secret-v2-bridge/secret-v2-bridge-fns"; @@ -42,6 +44,8 @@ export type TImportDataIntoInfisicalDTO = { projectService: Pick; projectEnvService: Pick; secretV2BridgeService: Pick; + folderCommitService: Pick; + folderVersionDAL: Pick; input: TImportInfisicalDataCreate; }; @@ -507,6 +511,8 @@ export const importDataIntoInfisicalFn = async ({ secretVersionTagDAL, folderDAL, resourceMetadataDAL, + folderVersionDAL, + folderCommitService, input: { data, actor, actorId, actorOrgId, actorAuthMethod } }: TImportDataIntoInfisicalDTO) => { // Import data to infisical @@ -599,6 +605,36 @@ export const importDataIntoInfisicalFn = async ({ tx ); + const newFolderVersion = await folderVersionDAL.create( + { + name: newFolder.name, + envId: newFolder.envId, + version: newFolder.version, + folderId: newFolder.id + }, + tx + ); + + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "Changed by external migration", + folderId: parentEnv.rootFolderId, + changes: [ + { + type: CommitType.ADD, + folderVersionId: newFolderVersion.id + } + ] + }, + tx + ); + originalToNewFolderId.set(folder.id, { folderId: newFolder.id, projectId: parentEnv.projectId @@ -772,6 +808,7 @@ export const importDataIntoInfisicalFn = async ({ secretVersionDAL, secretTagDAL, secretVersionTagDAL, + folderCommitService, actor: { type: actor, actorId diff --git a/backend/src/services/external-migration/external-migration-queue.ts b/backend/src/services/external-migration/external-migration-queue.ts index 8aa46b94c..66f2c73e7 100644 --- a/backend/src/services/external-migration/external-migration-queue.ts +++ b/backend/src/services/external-migration/external-migration-queue.ts @@ -3,6 +3,7 @@ import { infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption"; import { logger } from "@app/lib/logger"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "../kms/kms-service"; import { TProjectDALFactory } from "../project/project-dal"; import { TProjectServiceFactory } from "../project/project-service"; @@ -10,6 +11,7 @@ import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TProjectEnvServiceFactory } from "../project-env/project-env-service"; import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal"; import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; import { TSecretV2BridgeServiceFactory } from "../secret-v2-bridge/secret-v2-bridge-service"; @@ -36,6 +38,8 @@ export type TExternalMigrationQueueFactoryDep = { projectService: Pick; projectEnvService: Pick; secretV2BridgeService: Pick; + folderCommitService: Pick; + folderVersionDAL: Pick; resourceMetadataDAL: Pick; }; @@ -56,6 +60,8 @@ export const externalMigrationQueueFactory = ({ secretTagDAL, secretVersionTagDAL, folderDAL, + folderCommitService, + folderVersionDAL, resourceMetadataDAL }: TExternalMigrationQueueFactoryDep) => { const startImport = async (dto: { @@ -114,6 +120,8 @@ export const externalMigrationQueueFactory = ({ projectService, projectEnvService, secretV2BridgeService, + folderCommitService, + folderVersionDAL, resourceMetadataDAL }); diff --git a/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts b/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts new file mode 100644 index 000000000..3e2d09d0b --- /dev/null +++ b/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts @@ -0,0 +1,118 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { + TableName, + TFolderCheckpointResources, + TFolderCheckpoints, + TSecretFolderVersions, + TSecretVersionsV2 +} from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TFolderCheckpointResourcesDALFactory = ReturnType; + +export type ResourceWithCheckpointInfo = TFolderCheckpointResources & { + folderCommitId: string; +}; + +export const folderCheckpointResourcesDALFactory = (db: TDbClient) => { + const folderCheckpointResourcesOrm = ormify(db, TableName.FolderCheckpointResources); + + const findByCheckpointId = async ( + folderCheckpointId: string, + tx?: Knex + ): Promise< + (TFolderCheckpointResources & { + referencedSecretId?: string; + referencedFolderId?: string; + folderName?: string; + folderVersion?: string; + secretKey?: string; + secretVersion?: string; + })[] + > => { + try { + const docs = await (tx || db.replicaNode())(TableName.FolderCheckpointResources) + .where({ folderCheckpointId }) + .leftJoin( + TableName.SecretVersionV2, + `${TableName.FolderCheckpointResources}.secretVersionId`, + `${TableName.SecretVersionV2}.id` + ) + .leftJoin( + TableName.SecretFolderVersion, + `${TableName.FolderCheckpointResources}.folderVersionId`, + `${TableName.SecretFolderVersion}.id` + ) + .select(selectAllTableCols(TableName.FolderCheckpointResources)) + .select( + db.ref("secretId").withSchema(TableName.SecretVersionV2).as("referencedSecretId"), + db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("referencedFolderId"), + db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderName"), + db.ref("version").withSchema(TableName.SecretFolderVersion).as("folderVersion"), + db.ref("key").withSchema(TableName.SecretVersionV2).as("secretKey"), + db.ref("version").withSchema(TableName.SecretVersionV2).as("secretVersion") + ); + return docs.map((doc) => ({ + ...doc, + folderVersion: doc.folderVersion?.toString(), + secretVersion: doc.secretVersion?.toString() + })); + } catch (error) { + throw new DatabaseError({ error, name: "FindByCheckpointId" }); + } + }; + + const findBySecretVersionId = async (secretVersionId: string, tx?: Knex): Promise => { + try { + const docs = await (tx || db.replicaNode())< + TFolderCheckpointResources & Pick + >(TableName.FolderCheckpointResources) + .where({ secretVersionId }) + .select(selectAllTableCols(TableName.FolderCheckpointResources)) + .join( + TableName.FolderCheckpoint, + `${TableName.FolderCheckpointResources}.folderCheckpointId`, + `${TableName.FolderCheckpoint}.id` + ) + .select( + db.ref("folderCommitId").withSchema(TableName.FolderCheckpoint), + db.ref("createdAt").withSchema(TableName.FolderCheckpoint) + ); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindBySecretVersionId" }); + } + }; + + const findByFolderVersionId = async (folderVersionId: string, tx?: Knex): Promise => { + try { + const docs = await (tx || db.replicaNode())< + TFolderCheckpointResources & Pick + >(TableName.FolderCheckpointResources) + .where({ folderVersionId }) + .select(selectAllTableCols(TableName.FolderCheckpointResources)) + .join( + TableName.FolderCheckpoint, + `${TableName.FolderCheckpointResources}.folderCheckpointId`, + `${TableName.FolderCheckpoint}.id` + ) + .select( + db.ref("folderCommitId").withSchema(TableName.FolderCheckpoint), + db.ref("createdAt").withSchema(TableName.FolderCheckpoint) + ); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindByFolderVersionId" }); + } + }; + + return { + ...folderCheckpointResourcesOrm, + findByCheckpointId, + findBySecretVersionId, + findByFolderVersionId + }; +}; diff --git a/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts b/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts new file mode 100644 index 000000000..51bba9cc0 --- /dev/null +++ b/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts @@ -0,0 +1,129 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { TableName, TFolderCheckpoints, TFolderCommits } from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TFolderCheckpointDALFactory = ReturnType; + +type CheckpointWithCommitInfo = TFolderCheckpoints & { + actorMetadata: unknown; + actorType: string; + message?: string | null; + commitDate: Date; + folderId: string; +}; + +export const folderCheckpointDALFactory = (db: TDbClient) => { + const folderCheckpointOrm = ormify(db, TableName.FolderCheckpoint); + + const findByCommitId = async (folderCommitId: string, tx?: Knex): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCheckpoint) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ folderCommitId }, TableName.FolderCheckpoint)) + .select(selectAllTableCols(TableName.FolderCheckpoint)) + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindByCommitId" }); + } + }; + + const findByFolderId = async (folderId: string, limit?: number, tx?: Knex): Promise => { + try { + let query = (tx || db.replicaNode())(TableName.FolderCheckpoint) + .join( + TableName.FolderCommit, + `${TableName.FolderCheckpoint}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ folderId }, TableName.FolderCommit)) + .select(selectAllTableCols(TableName.FolderCheckpoint)) + .select( + db.ref("actorMetadata").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("createdAt").withSchema(TableName.FolderCommit).as("commitDate"), + db.ref("folderId").withSchema(TableName.FolderCommit) + ) + .orderBy(`${TableName.FolderCheckpoint}.createdAt`, "desc"); + + if (limit !== undefined) { + query = query.limit(limit); + } + + return await query; + } catch (error) { + throw new DatabaseError({ error, name: "FindByFolderId" }); + } + }; + + const findLatestByFolderId = async (folderId: string, tx?: Knex): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCheckpoint) + .join( + TableName.FolderCommit, + `${TableName.FolderCheckpoint}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ folderId }, TableName.FolderCommit)) + .select(selectAllTableCols(TableName.FolderCheckpoint)) + .select( + db.ref("actorMetadata").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("createdAt").withSchema(TableName.FolderCommit).as("commitDate"), + db.ref("folderId").withSchema(TableName.FolderCommit) + ) + .orderBy(`${TableName.FolderCheckpoint}.createdAt`, "desc") + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestByFolderId" }); + } + }; + + const findNearestCheckpoint = async ( + folderCommitId: bigint, + folderId: string, + tx?: Knex + ): Promise<(CheckpointWithCommitInfo & { commitId: bigint }) | undefined> => { + try { + // Get the checkpoint with the highest commitId that's still less than or equal to our commit + const nearestCheckpoint = await (tx || db.replicaNode())(TableName.FolderCheckpoint) + .join( + TableName.FolderCommit, + `${TableName.FolderCheckpoint}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) + .where(`${TableName.FolderCommit}.folderId`, "=", folderId) + .where(`${TableName.FolderCommit}.commitId`, "<=", folderCommitId.toString()) + .select(selectAllTableCols(TableName.FolderCheckpoint)) + .select( + db.ref("actorMetadata").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("commitId").withSchema(TableName.FolderCommit), + db.ref("createdAt").withSchema(TableName.FolderCommit).as("commitDate"), + db.ref("folderId").withSchema(TableName.FolderCommit) + ) + .orderBy(`${TableName.FolderCommit}.commitId`, "desc") + .first(); + return nearestCheckpoint; + } catch (error) { + throw new DatabaseError({ error, name: "FindNearestCheckpoint" }); + } + }; + + return { + ...folderCheckpointOrm, + findByCommitId, + findByFolderId, + findLatestByFolderId, + findNearestCheckpoint + }; +}; diff --git a/backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts b/backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts new file mode 100644 index 000000000..2c30c5bcf --- /dev/null +++ b/backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts @@ -0,0 +1,233 @@ +/* eslint-disable @typescript-eslint/no-misused-promises */ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { + TableName, + TFolderCommitChanges, + TFolderCommits, + TProjectEnvironments, + TSecretFolderVersions, + TSecretVersionsV2 +} from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TFolderCommitChangesDALFactory = ReturnType; + +// Base type with common fields +type BaseCommitChangeInfo = TFolderCommitChanges & { + actorMetadata: unknown; + actorType: string; + message?: string | null; + folderId: string; + createdAt: Date; +}; + +// Secret-specific change +export type SecretCommitChange = BaseCommitChangeInfo & { + resourceType: "secret"; + secretKey: string; + changeType: string; + secretVersionId?: string | null; + secretVersion: string; + secretId: string; + versions?: { + secretKey: string; + secretComment: string; + skipMultilineEncoding?: boolean | null; + secretReminderRepeatDays?: number | null; + secretReminderNote?: string | null; + metadata?: unknown; + tags?: string[] | null; + secretReminderRecipients?: string[] | null; + secretValue: string; + }[]; +}; + +// Folder-specific change +export type FolderCommitChange = BaseCommitChangeInfo & { + resourceType: "folder"; + folderName: string; + folderVersion: string; + folderChangeId: string; + versions?: { + version: string; + name?: string; + }[]; +}; + +// Discriminated union +export type CommitChangeWithCommitInfo = SecretCommitChange | FolderCommitChange; + +// Type guards +export const isSecretCommitChange = (change: CommitChangeWithCommitInfo): change is SecretCommitChange => + change.resourceType === "secret"; + +export const isFolderCommitChange = (change: CommitChangeWithCommitInfo): change is FolderCommitChange => + change.resourceType === "folder"; + +export const folderCommitChangesDALFactory = (db: TDbClient) => { + const folderCommitChangesOrm = ormify(db, TableName.FolderCommitChanges); + + const findByCommitId = async ( + folderCommitId: string, + projectId: string, + tx?: Knex + ): Promise => { + try { + const docs = await (tx || db.replicaNode())(TableName.FolderCommitChanges) + .where(buildFindFilter({ folderCommitId }, TableName.FolderCommitChanges)) + .leftJoin( + TableName.FolderCommit, + `${TableName.FolderCommitChanges}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) + .leftJoin( + TableName.SecretVersionV2, + `${TableName.FolderCommitChanges}.secretVersionId`, + `${TableName.SecretVersionV2}.id` + ) + .leftJoin( + TableName.SecretFolderVersion, + `${TableName.FolderCommitChanges}.folderVersionId`, + `${TableName.SecretFolderVersion}.id` + ) + .leftJoin( + TableName.Environment, + `${TableName.FolderCommit}.envId`, + `${TableName.Environment}.id` + ) + .where((qb) => { + if (projectId) { + void qb.where(`${TableName.Environment}.projectId`, "=", projectId); + } + }) + .select(selectAllTableCols(TableName.FolderCommitChanges)) + .select( + db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderName"), + db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("folderChangeId"), + db.ref("version").withSchema(TableName.SecretFolderVersion).as("folderVersion"), + db.ref("key").withSchema(TableName.SecretVersionV2).as("secretKey"), + db.ref("version").withSchema(TableName.SecretVersionV2).as("secretVersion"), + db.ref("secretId").withSchema(TableName.SecretVersionV2), + db.ref("actorMetadata").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("createdAt").withSchema(TableName.FolderCommit), + db.ref("folderId").withSchema(TableName.FolderCommit) + ); + + return docs.map((doc) => { + // Determine if this is a secret or folder change based on populated fields + if (doc.secretKey && doc.secretVersion !== null && doc.secretId) { + return { + ...doc, + resourceType: "secret", + secretKey: doc.secretKey, + secretVersion: doc.secretVersion.toString(), + secretId: doc.secretId + } as SecretCommitChange; + } + return { + ...doc, + resourceType: "folder", + folderName: doc.folderName, + folderVersion: doc.folderVersion.toString(), + folderChangeId: doc.folderChangeId + } as FolderCommitChange; + }); + } catch (error) { + throw new DatabaseError({ error, name: "FindByCommitId" }); + } + }; + + const findBySecretVersionId = async (secretVersionId: string, tx?: Knex): Promise => { + try { + const docs = await (tx || db.replicaNode())< + TFolderCommitChanges & + Pick + >(TableName.FolderCommitChanges) + .where(buildFindFilter({ secretVersionId }, TableName.FolderCommitChanges)) + .select(selectAllTableCols(TableName.FolderCommitChanges)) + .join(TableName.FolderCommit, `${TableName.FolderCommitChanges}.folderCommitId`, `${TableName.FolderCommit}.id`) + .leftJoin( + TableName.SecretVersionV2, + `${TableName.FolderCommitChanges}.secretVersionId`, + `${TableName.SecretVersionV2}.id` + ) + .select( + db.ref("actorMetadata").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("createdAt").withSchema(TableName.FolderCommit), + db.ref("folderId").withSchema(TableName.FolderCommit), + db.ref("key").withSchema(TableName.SecretVersionV2).as("secretKey"), + db.ref("version").withSchema(TableName.SecretVersionV2).as("secretVersion"), + db.ref("secretId").withSchema(TableName.SecretVersionV2) + ); + + return docs + .filter((doc) => doc.secretKey && doc.secretVersion !== null && doc.secretId) + .map( + (doc): SecretCommitChange => ({ + ...doc, + resourceType: "secret", + secretKey: doc.secretKey, + secretVersion: doc.secretVersion.toString(), + secretId: doc.secretId + }) + ); + } catch (error) { + throw new DatabaseError({ error, name: "FindBySecretVersionId" }); + } + }; + + const findByFolderVersionId = async (folderVersionId: string, tx?: Knex): Promise => { + try { + const docs = await (tx || db.replicaNode())< + TFolderCommitChanges & + Pick + >(TableName.FolderCommitChanges) + .where(buildFindFilter({ folderVersionId }, TableName.FolderCommitChanges)) + .select(selectAllTableCols(TableName.FolderCommitChanges)) + .join(TableName.FolderCommit, `${TableName.FolderCommitChanges}.folderCommitId`, `${TableName.FolderCommit}.id`) + .leftJoin( + TableName.SecretFolderVersion, + `${TableName.FolderCommitChanges}.folderVersionId`, + `${TableName.SecretFolderVersion}.id` + ) + .select( + db.ref("actorMetadata").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("createdAt").withSchema(TableName.FolderCommit), + db.ref("folderId").withSchema(TableName.FolderCommit), + db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderName"), + db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("folderChangeId"), + db.ref("version").withSchema(TableName.SecretFolderVersion).as("folderVersion") + ); + + return docs + .filter((doc) => doc.folderName && doc.folderVersion !== null && doc.folderChangeId) + .map( + (doc): FolderCommitChange => ({ + ...doc, + resourceType: "folder", + folderName: doc.folderName, + folderVersion: doc.folderVersion!.toString(), + folderChangeId: doc.folderChangeId + }) + ); + } catch (error) { + throw new DatabaseError({ error, name: "FindByFolderVersionId" }); + } + }; + + return { + ...folderCommitChangesOrm, + findByCommitId, + findBySecretVersionId, + findByFolderVersionId + }; +}; diff --git a/backend/src/services/folder-commit/folder-commit-dal.ts b/backend/src/services/folder-commit/folder-commit-dal.ts new file mode 100644 index 000000000..e95dbb839 --- /dev/null +++ b/backend/src/services/folder-commit/folder-commit-dal.ts @@ -0,0 +1,513 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { + TableName, + TFolderCommitChanges, + TFolderCommits, + TProjectEnvironments, + TSecretFolderVersions, + TSecretVersionsV2 +} from "@app/db/schemas"; +import { DatabaseError, NotFoundError } from "@app/lib/errors"; +import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TFolderCommitDALFactory = ReturnType; + +export const folderCommitDALFactory = (db: TDbClient) => { + const folderCommitOrm = ormify(db, TableName.FolderCommit); + const { delete: deleteOp, deleteById, ...restOfOrm } = folderCommitOrm; + + const findByFolderId = async (folderId: string, tx?: Knex): Promise => { + try { + const trx = tx || db.replicaNode(); + + // First, get all folder commits + const folderCommits = await trx(TableName.FolderCommit) + .where({ folderId }) + .select("*") + .orderBy("createdAt", "desc"); + + if (folderCommits.length === 0) return []; + + // Get all commit IDs + const commitIds = folderCommits.map((commit) => commit.id); + + // Then get all related changes + const changes = await trx(TableName.FolderCommitChanges).whereIn("folderCommitId", commitIds).select("*"); + + const changesMap = changes.reduce( + (acc, change) => { + const { folderCommitId } = change; + if (!acc[folderCommitId]) acc[folderCommitId] = []; + acc[folderCommitId].push(change); + return acc; + }, + {} as Record + ); + + return folderCommits.map((commit) => ({ + ...commit, + changes: changesMap[commit.id] || [] + })); + } catch (error) { + throw new DatabaseError({ error, name: "FindByFolderId" }); + } + }; + + const findLatestCommit = async ( + folderId: string, + projectId?: string, + tx?: Knex + ): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ folderId }) + .leftJoin(TableName.Environment, `${TableName.FolderCommit}.envId`, `${TableName.Environment}.id`) + .where((qb) => { + if (projectId) { + void qb.where(`${TableName.Environment}.projectId`, "=", projectId); + } + }) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("commitId", "desc") + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestCommit" }); + } + }; + + const findLatestCommitByFolderIds = async (folderIds: string[], tx?: Knex): Promise => { + try { + // First get max commitId for each folderId + const maxCommitIdSubquery = (tx || db.replicaNode())(TableName.FolderCommit) + .select("folderId") + .max("commitId as maxCommitId") + .whereIn("folderId", folderIds) + .groupBy("folderId"); + + // Join with main table to get complete records for each max commitId + const docs = await (tx || db.replicaNode())(TableName.FolderCommit) + .select(selectAllTableCols(TableName.FolderCommit)) + // eslint-disable-next-line func-names + .join(maxCommitIdSubquery.as("latest"), function () { + this.on(`${TableName.FolderCommit}.folderId`, "=", "latest.folderId").andOn( + `${TableName.FolderCommit}.commitId`, + "=", + "latest.maxCommitId" + ); + }); + + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestCommitByFolderIds" }); + } + }; + + const findLatestEnvCommit = async (envId: string, tx?: Knex): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + .where(`${TableName.FolderCommit}.envId`, "=", envId) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("commitId", "desc") + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestCommit" }); + } + }; + + const findMultipleLatestCommits = async (folderIds: string[], tx?: Knex): Promise => { + try { + const knexInstance = tx || db.replicaNode(); + + // Get the latest commitId for each folderId + const subquery = knexInstance(TableName.FolderCommit) + .whereIn("folderId", folderIds) + .groupBy("folderId") + .select("folderId") + .max("commitId as maxCommitId"); + + // Then fetch the complete rows matching those latest commits + const docs = await knexInstance(TableName.FolderCommit) + // eslint-disable-next-line func-names + .innerJoin(subquery.as("latest"), function () { + this.on(`${TableName.FolderCommit}.folderId`, "=", "latest.folderId").andOn( + `${TableName.FolderCommit}.commitId`, + "=", + "latest.maxCommitId" + ); + }) + .select(selectAllTableCols(TableName.FolderCommit)); + + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindMultipleLatestCommits" }); + } + }; + + const getNumberOfCommitsSince = async (folderId: string, folderCommitId: string, tx?: Knex): Promise => { + try { + const referencedCommit = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ id: folderCommitId }) + .select("commitId") + .first(); + + if (referencedCommit?.commitId) { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ folderId }) + .where("commitId", ">", referencedCommit.commitId) + .count(); + return Number(doc?.[0].count); + } + return 0; + } catch (error) { + throw new DatabaseError({ error, name: "getNumberOfCommitsSince" }); + } + }; + + const getEnvNumberOfCommitsSince = async (envId: string, folderCommitId: string, tx?: Knex): Promise => { + try { + const referencedCommit = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ id: folderCommitId }) + .select("commitId") + .first(); + + if (referencedCommit?.commitId) { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + .where(`${TableName.FolderCommit}.envId`, "=", envId) + .where("commitId", ">", referencedCommit.commitId) + .count(); + return Number(doc?.[0].count); + } + return 0; + } catch (error) { + throw new DatabaseError({ error, name: "getNumberOfCommitsSince" }); + } + }; + + const findCommitsToRecreate = async ( + folderId: string, + targetCommitNumber: bigint, + checkpointCommitNumber: bigint, + tx?: Knex + ): Promise< + (TFolderCommits & { + changes: (TFolderCommitChanges & { + referencedSecretId?: string; + referencedFolderId?: string; + folderName?: string; + folderVersion?: string; + secretKey?: string; + secretVersion?: string; + })[]; + })[] + > => { + try { + // First get all the commits in the range + const commits = await (tx || db.replicaNode())(TableName.FolderCommit) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ folderId }, TableName.FolderCommit)) + .andWhere(`${TableName.FolderCommit}.commitId`, ">", checkpointCommitNumber.toString()) + .andWhere(`${TableName.FolderCommit}.commitId`, "<=", targetCommitNumber.toString()) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy(`${TableName.FolderCommit}.commitId`, "asc"); + + // If no commits found, return empty array + if (!commits.length) { + return []; + } + + // Get all the commit IDs + const commitIds = commits.map((commit) => commit.id); + + // Get all changes for these commits in a single query + const allChanges = await (tx || db.replicaNode())(TableName.FolderCommitChanges) + .whereIn(`${TableName.FolderCommitChanges}.folderCommitId`, commitIds) + .leftJoin( + TableName.SecretVersionV2, + `${TableName.FolderCommitChanges}.secretVersionId`, + `${TableName.SecretVersionV2}.id` + ) + .leftJoin( + TableName.SecretFolderVersion, + `${TableName.FolderCommitChanges}.folderVersionId`, + `${TableName.SecretFolderVersion}.id` + ) + .select(selectAllTableCols(TableName.FolderCommitChanges)) + .select( + db.ref("secretId").withSchema(TableName.SecretVersionV2).as("referencedSecretId"), + db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("referencedFolderId"), + db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderName"), + db.ref("version").withSchema(TableName.SecretFolderVersion).as("folderVersion"), + db.ref("key").withSchema(TableName.SecretVersionV2).as("secretKey"), + db.ref("version").withSchema(TableName.SecretVersionV2).as("secretVersion") + ); + + // Organize changes by commit ID + const changesByCommitId = allChanges.reduce( + (acc, change) => { + if (!acc[change.folderCommitId]) { + acc[change.folderCommitId] = []; + } + acc[change.folderCommitId].push(change); + return acc; + }, + {} as Record + ); + + // Attach changes to each commit + return commits.map((commit) => ({ + ...commit, + changes: changesByCommitId[commit.id] || [] + })); + } catch (error) { + throw new DatabaseError({ error, name: "FindCommitsToRecreate" }); + } + }; + + const findLatestCommitBetween = async ({ + folderId, + startCommitId, + endCommitId, + tx + }: { + folderId: string; + startCommitId?: string; + endCommitId: string; + tx?: Knex; + }): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + .where("commitId", "<=", endCommitId) + .where({ folderId }) + .where((qb) => { + if (startCommitId) { + void qb.where("commitId", ">=", startCommitId); + } + }) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("commitId", "desc") + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestCommitBetween" }); + } + }; + + const findAllCommitsBetween = async ({ + envId, + startCommitId, + endCommitId, + tx + }: { + envId?: string; + startCommitId?: string; + endCommitId?: string; + tx?: Knex; + }): Promise => { + try { + const docs = await (tx || db.replicaNode())(TableName.FolderCommit) + .where((qb) => { + if (envId) { + void qb.where(`${TableName.FolderCommit}.envId`, "=", envId); + } + if (startCommitId) { + void qb.where("commitId", ">=", startCommitId); + } + if (endCommitId) { + void qb.where("commitId", "<=", endCommitId); + } + }) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("commitId", "desc"); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestCommitBetween" }); + } + }; + + const findAllFolderCommitsAfter = async ({ + envId, + startCommitId, + tx + }: { + envId?: string; + startCommitId?: string; + tx?: Knex; + }): Promise => { + try { + const docs = await (tx || db.replicaNode())(TableName.FolderCommit) + .where((qb) => { + if (envId) { + void qb.where(`${TableName.FolderCommit}.envId`, "=", envId); + } + if (startCommitId) { + void qb.where("commitId", ">=", startCommitId); + } + }) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("commitId", "desc"); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestCommitBetween" }); + } + }; + + const findPreviousCommitTo = async ( + folderId: string, + commitId: string, + tx?: Knex + ): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ folderId }) + .where("commitId", "<=", commitId) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("commitId", "desc") + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindPreviousCommitTo" }); + } + }; + + const findById = async (id: string, tx?: Knex, projectId?: string): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ id }, TableName.FolderCommit)) + .leftJoin( + TableName.Environment, + `${TableName.FolderCommit}.envId`, + `${TableName.Environment}.id` + ) + .where((qb) => { + if (projectId) { + void qb.where(`${TableName.Environment}.projectId`, "=", projectId); + } + }) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("commitId", "desc") + .first(); + if (!doc) { + throw new NotFoundError({ + message: `Folder commit not found for ID ${id}` + }); + } + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindById" }); + } + }; + + const findByFolderIdPaginated = async ( + folderId: string, + options: { + offset?: number; + limit?: number; + search?: string; + sort?: "asc" | "desc"; + } = {}, + tx?: Knex + ): Promise<{ + commits: TFolderCommits[]; + total: number; + hasMore: boolean; + }> => { + try { + const { offset = 0, limit = 20, search, sort = "desc" } = options; + const trx = tx || db.replicaNode(); + + // Build base query + let baseQuery = trx(TableName.FolderCommit).where({ folderId }); + + // Add search functionality + if (search) { + baseQuery = baseQuery.where((qb) => { + void qb.whereILike("message", `%${search}%`); + }); + } + + // Get total count + const totalResult = await baseQuery.clone().count("*", { as: "count" }).first(); + const total = Number(totalResult?.count || 0); + + // Get paginated commits + const folderCommits = await baseQuery.select("*").orderBy("createdAt", sort).limit(limit).offset(offset); + + if (folderCommits.length === 0) { + return { commits: [], total, hasMore: false }; + } + + // Get all commit IDs for changes + const commitIds = folderCommits.map((commit) => commit.id); + + // Get all related changes + const changes = await trx(TableName.FolderCommitChanges).whereIn("folderCommitId", commitIds).select("*"); + + const changesMap = changes.reduce( + (acc, change) => { + const { folderCommitId } = change; + if (!acc[folderCommitId]) acc[folderCommitId] = []; + acc[folderCommitId].push(change); + return acc; + }, + {} as Record + ); + + const commitsWithChanges = folderCommits.map((commit) => ({ + ...commit, + changes: changesMap[commit.id] || [] + })); + + const hasMore = offset + limit < total; + + return { + commits: commitsWithChanges, + total, + hasMore + }; + } catch (error) { + throw new DatabaseError({ error, name: "FindByFolderIdPaginated" }); + } + }; + + const findCommitBefore = async ( + folderId: string, + commitId: bigint, + tx?: Knex + ): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ folderId }) + .where("commitId", "<", commitId.toString()) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("commitId", "desc") + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindCommitBefore" }); + } + }; + + return { + ...restOfOrm, + findByFolderId, + findLatestCommit, + getNumberOfCommitsSince, + findCommitsToRecreate, + findMultipleLatestCommits, + findAllCommitsBetween, + findLatestCommitBetween, + findLatestEnvCommit, + getEnvNumberOfCommitsSince, + findLatestCommitByFolderIds, + findAllFolderCommitsAfter, + findPreviousCommitTo, + findById, + findByFolderIdPaginated, + findCommitBefore + }; +}; diff --git a/backend/src/services/folder-commit/folder-commit-queue.ts b/backend/src/services/folder-commit/folder-commit-queue.ts new file mode 100644 index 000000000..fcb348784 --- /dev/null +++ b/backend/src/services/folder-commit/folder-commit-queue.ts @@ -0,0 +1,282 @@ +import { Knex } from "knex"; + +import { TSecretFolders } from "@app/db/schemas"; +import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore"; +import { getConfig } from "@app/lib/config/env"; +import { logger } from "@app/lib/logger"; +import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; + +import { TFolderTreeCheckpointDALFactory } from "../folder-tree-checkpoint/folder-tree-checkpoint-dal"; +import { TFolderTreeCheckpointResourcesDALFactory } from "../folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal"; +import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TFolderCommitDALFactory } from "./folder-commit-dal"; + +// Define types for job data +type TCreateFolderTreeCheckpointDTO = { + envId: string; + failedToAcquireLockCount?: number; + folderCommitId?: string; +}; + +type TFolderCommitQueueServiceFactoryDep = { + queueService: TQueueServiceFactory; + keyStore: Pick; + folderTreeCheckpointDAL: Pick< + TFolderTreeCheckpointDALFactory, + "create" | "findLatestByEnvId" | "findNearestCheckpoint" + >; + folderTreeCheckpointResourcesDAL: Pick< + TFolderTreeCheckpointResourcesDALFactory, + "insertMany" | "findByTreeCheckpointId" + >; + folderCommitDAL: Pick< + TFolderCommitDALFactory, + "findLatestEnvCommit" | "getEnvNumberOfCommitsSince" | "findMultipleLatestCommits" | "findById" + >; + folderDAL: Pick; +}; + +export type TFolderCommitQueueServiceFactory = ReturnType; + +export const folderCommitQueueServiceFactory = ({ + queueService, + keyStore, + folderTreeCheckpointDAL, + folderTreeCheckpointResourcesDAL, + folderCommitDAL, + folderDAL +}: TFolderCommitQueueServiceFactoryDep) => { + const appCfg = getConfig(); + + // Helper function to calculate delay for requeuing + const getRequeueDelay = (failureCount?: number) => { + if (!failureCount) return 0; + + const baseDelay = 5000; + const maxDelay = 30000; + + const delay = Math.min(baseDelay * 2 ** failureCount, maxDelay); + const jitter = delay * (0.5 + Math.random() * 0.5); + + return jitter; + }; + + const scheduleTreeCheckpoint = async (payload: TCreateFolderTreeCheckpointDTO) => { + const { envId, failedToAcquireLockCount = 0 } = payload; + + // Create a unique jobId for each retry to prevent conflicts + const jobId = + failedToAcquireLockCount > 0 ? `${envId}-retry-${failedToAcquireLockCount}-${Date.now()}` : `${envId}`; + + await queueService.queue(QueueName.FolderTreeCheckpoint, QueueJobs.CreateFolderTreeCheckpoint, payload, { + jobId, + delay: getRequeueDelay(failedToAcquireLockCount), + backoff: { + type: "exponential", + delay: 3000 + }, + removeOnFail: { + count: 3 + }, + removeOnComplete: true + }); + }; + + // Sort folders by hierarchy (copied from the source code) + const sortFoldersByHierarchy = (folders: TSecretFolders[]) => { + const childrenMap = new Map(); + const allFolderIds = new Set(); + + folders.forEach((folder) => { + if (folder.id) allFolderIds.add(folder.id); + }); + + folders.forEach((folder) => { + if (folder.parentId) { + const children = childrenMap.get(folder.parentId) || []; + children.push(folder); + childrenMap.set(folder.parentId, children); + } + }); + + const rootFolders = folders.filter((folder) => !folder.parentId || !allFolderIds.has(folder.parentId)); + + const result = []; + let currentLevel = rootFolders; + + while (currentLevel.length > 0) { + result.push(...currentLevel); + + const nextLevel = []; + for (const folder of currentLevel) { + if (folder.id) { + const children = childrenMap.get(folder.id) || []; + nextLevel.push(...children); + } + } + + currentLevel = nextLevel; + } + + return result; + }; + + const createFolderTreeCheckpoint = async (jobData: TCreateFolderTreeCheckpointDTO, tx?: Knex) => { + const { envId, folderCommitId, failedToAcquireLockCount = 0 } = jobData; + + logger.info(`Folder tree checkpoint creation started [envId=${envId}] [attempt=${failedToAcquireLockCount + 1}]`); + + // First, try to clear any stale locks before attempting to acquire + if (failedToAcquireLockCount > 1) { + try { + await keyStore.deleteItem(KeyStorePrefixes.FolderTreeCheckpoint(envId)); + logger.info(`Cleared potential stale lock for envId ${envId} before attempt ${failedToAcquireLockCount + 1}`); + } catch (error) { + // This is fine if it fails, we'll still try to acquire the lock + logger.info(`No stale lock found for envId ${envId}`); + } + } + + let lock: Awaited> | undefined; + + try { + // Attempt to acquire the lock with a shorter timeout for first attempts + const timeout = failedToAcquireLockCount > 3 ? 60 * 1000 : 15 * 1000; + + logger.info(`Attempting to acquire lock for envId=${envId} with timeout ${timeout}ms`); + + lock = await keyStore.acquireLock([KeyStorePrefixes.FolderTreeCheckpoint(envId)], timeout); + + logger.info(`Successfully acquired lock for envId=${envId}`); + } catch (e) { + logger.info( + `Failed to acquire lock for folder tree checkpoint [envId=${envId}] [attempt=${failedToAcquireLockCount + 1}]` + ); + + // Requeue with incremented failure count if under max attempts + if (failedToAcquireLockCount < 10) { + // Force a delay between retries + const nextRetryCount = failedToAcquireLockCount + 1; + + logger.info(`Scheduling retry #${nextRetryCount} for folder tree checkpoint [envId=${envId}]`); + + // Create a new job with incremented counter + await scheduleTreeCheckpoint({ + envId, + folderCommitId, + failedToAcquireLockCount: nextRetryCount + }); + } else { + // Max retries reached + logger.error(`Maximum lock acquisition attempts (10) reached for envId ${envId}. Giving up.`); + // Try to force-clear the lock for next time + try { + await keyStore.deleteItem(KeyStorePrefixes.FolderTreeCheckpoint(envId)); + } catch (clearError) { + logger.error(clearError, `Failed to clear lock after maximum retries for envId=${envId}`); + } + } + return; + } + + if (!lock) { + logger.error(`Lock is undefined after acquisition for envId=${envId}. This should never happen.`); + return; + } + + try { + logger.info(`Processing tree checkpoint data for envId=${envId}`); + + const latestTreeCheckpoint = await folderTreeCheckpointDAL.findLatestByEnvId(envId, tx); + + let latestCommit; + if (folderCommitId) { + latestCommit = await folderCommitDAL.findById(folderCommitId, tx); + } else { + latestCommit = await folderCommitDAL.findLatestEnvCommit(envId, tx); + } + if (!latestCommit) { + logger.info(`Latest commit ID not found for envId ${envId}`); + return; + } + const latestCommitId = latestCommit.id; + + if (latestTreeCheckpoint) { + const commitsSinceLastCheckpoint = await folderCommitDAL.getEnvNumberOfCommitsSince( + envId, + latestTreeCheckpoint.folderCommitId, + tx + ); + if (commitsSinceLastCheckpoint < Number(appCfg.PIT_TREE_CHECKPOINT_WINDOW)) { + logger.info( + `Commits since last checkpoint ${commitsSinceLastCheckpoint} is less than ${appCfg.PIT_TREE_CHECKPOINT_WINDOW}` + ); + return; + } + } + + const folders = await folderDAL.findByEnvId(envId, tx); + const sortedFolders = sortFoldersByHierarchy(folders); + const filteredFoldersIds = sortedFolders.filter((folder) => !folder.isReserved).map((folder) => folder.id); + + const folderCommits = await folderCommitDAL.findMultipleLatestCommits(filteredFoldersIds, tx); + const folderTreeCheckpoint = await folderTreeCheckpointDAL.create( + { + folderCommitId: latestCommitId + }, + tx + ); + + await folderTreeCheckpointResourcesDAL.insertMany( + folderCommits.map((folderCommit) => ({ + folderTreeCheckpointId: folderTreeCheckpoint.id, + folderId: folderCommit.folderId, + folderCommitId: folderCommit.id + })), + tx + ); + + logger.info(`Folder tree checkpoint created successfully: ${folderTreeCheckpoint.id}`); + } catch (error) { + logger.error(error, `Error processing folder tree checkpoint [envId=${envId}]`); + throw error; + } finally { + // Always release the lock + try { + if (lock) { + await lock.release(); + logger.info(`Released lock for folder tree checkpoint [envId=${envId}]`); + } else { + logger.error(`No lock to release for envId=${envId}. This should never happen.`); + } + } catch (releaseError) { + logger.error(releaseError, `Error releasing lock for folder tree checkpoint [envId=${envId}]`); + // Try to force delete the lock if release fails + try { + await keyStore.deleteItem(KeyStorePrefixes.FolderTreeCheckpoint(envId)); + logger.info(`Force deleted lock after release failure for envId=${envId}`); + } catch (deleteError) { + logger.error(deleteError, `Failed to force delete lock after release failure for envId=${envId}`); + } + } + } + }; + + queueService.start(QueueName.FolderTreeCheckpoint, async (job) => { + try { + if (job.name === QueueJobs.CreateFolderTreeCheckpoint) { + const jobData = job.data as TCreateFolderTreeCheckpointDTO; + await createFolderTreeCheckpoint(jobData); + } + } catch (error) { + logger.error(error, "Error creating folder tree checkpoint:"); + throw error; + } + }); + + return { + scheduleTreeCheckpoint: (envId: string) => scheduleTreeCheckpoint({ envId }), + createFolderTreeCheckpoint: (envId: string, folderCommitId?: string, tx?: Knex) => + createFolderTreeCheckpoint({ envId, folderCommitId }, tx) + }; +}; diff --git a/backend/src/services/folder-commit/folder-commit-schemas.ts b/backend/src/services/folder-commit/folder-commit-schemas.ts new file mode 100644 index 000000000..9f99bd2cc --- /dev/null +++ b/backend/src/services/folder-commit/folder-commit-schemas.ts @@ -0,0 +1,143 @@ +import { z } from "zod"; + +// Base schema shared by both secret and folder changes +const baseChangeSchema = z.object({ + id: z.string(), + folderCommitId: z.string(), + changeType: z.string(), + isUpdate: z.boolean().optional(), + createdAt: z.union([z.string(), z.date()]), + updatedAt: z.union([z.string(), z.date()]), + actorMetadata: z + .union([ + z.object({ + id: z.string().optional(), + name: z.string().optional() + }), + z.unknown() + ]) + .optional(), + actorType: z.string(), + message: z.string().nullable().optional(), + folderId: z.string() +}); + +// Secret-specific versions schema +const secretVersionSchema = z.object({ + secretKey: z.string(), + secretComment: z.string(), + skipMultilineEncoding: z.boolean().nullable().optional(), + tags: z.array(z.string()).nullable().optional(), + metadata: z.unknown().nullable().optional(), + secretValue: z.string() +}); + +// Folder-specific versions schema +const folderVersionSchema = z.object({ + version: z.string().optional(), + name: z.string().optional(), + description: z.string().optional().nullable() +}); + +// Secret commit change schema +const secretCommitChangeSchema = baseChangeSchema.extend({ + resourceType: z.literal("secret"), + secretVersionId: z.string().optional().nullable(), + secretKey: z.string(), + secretVersion: z.union([z.string(), z.number()]), + secretId: z.string(), + versions: z.array(secretVersionSchema).optional() +}); + +// Folder commit change schema +const folderCommitChangeSchema = baseChangeSchema.extend({ + resourceType: z.literal("folder"), + folderVersionId: z.string().optional().nullable(), + folderName: z.string(), + folderChangeId: z.string(), + folderVersion: z.union([z.string(), z.number()]), + versions: z.array(folderVersionSchema).optional() +}); + +// Discriminated union for commit changes +export const commitChangeSchema = z.discriminatedUnion("resourceType", [ + secretCommitChangeSchema, + folderCommitChangeSchema +]); + +// Commit schema +const commitSchema = z.object({ + id: z.string(), + commitId: z.string(), + actorMetadata: z + .union([ + z.object({ + id: z.string().optional(), + name: z.string().optional() + }), + z.unknown() + ]) + .optional(), + actorType: z.string(), + message: z.string().nullable().optional(), + folderId: z.string(), + envId: z.string(), + createdAt: z.union([z.string(), z.date()]), + updatedAt: z.union([z.string(), z.date()]), + isLatest: z.boolean().default(false), + changes: z.array(commitChangeSchema).optional() +}); + +// Response schema +export const commitChangesResponseSchema = z.object({ + changes: commitSchema +}); + +// Base resource change schema for comparison results +const baseResourceChangeSchema = z.object({ + id: z.string(), + versionId: z.string(), + oldVersionId: z.string().optional(), + changeType: z.enum(["add", "delete", "update", "create"]), + commitId: z.union([z.string(), z.bigint()]), + createdAt: z.union([z.string(), z.date()]).optional(), + parentId: z.string().optional(), + isUpdate: z.boolean().optional(), + fromVersion: z.union([z.string(), z.number()]).optional() +}); + +// Secret resource change schema +const secretResourceChangeSchema = baseResourceChangeSchema.extend({ + type: z.literal("secret"), + secretKey: z.string(), + secretVersion: z.union([z.string(), z.number()]), + secretId: z.string(), + versions: z + .array( + z.object({ + secretKey: z.string().optional(), + secretComment: z.string().optional(), + skipMultilineEncoding: z.boolean().nullable().optional(), + secretReminderRepeatDays: z.number().nullable().optional(), + tags: z.array(z.string()).nullable().optional(), + metadata: z.unknown().nullable().optional(), + secretReminderNote: z.string().nullable().optional(), + secretValue: z.string().optional() + }) + ) + .optional() +}); + +// Folder resource change schema +const folderResourceChangeSchema = baseResourceChangeSchema.extend({ + type: z.literal("folder"), + folderName: z.string(), + folderVersion: z.union([z.string(), z.number()]), + versions: z.array(folderVersionSchema).optional() +}); + +// Discriminated union for resource changes +export const resourceChangeSchema = z.discriminatedUnion("type", [ + secretResourceChangeSchema, + folderResourceChangeSchema +]); diff --git a/backend/src/services/folder-commit/folder-commit-service.test.ts b/backend/src/services/folder-commit/folder-commit-service.test.ts new file mode 100644 index 000000000..1879cf493 --- /dev/null +++ b/backend/src/services/folder-commit/folder-commit-service.test.ts @@ -0,0 +1,671 @@ +/* eslint-disable @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/return-await */ +/* eslint-disable @typescript-eslint/no-unsafe-return */ +import { Knex } from "knex"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { ProjectType, TSecretFolderVersions, TSecretVersionsV2 } from "@app/db/schemas"; +import { BadRequestError, NotFoundError } from "@app/lib/errors"; + +import { ActorType } from "../auth/auth-type"; +import { + ChangeType, + CommitType, + folderCommitServiceFactory, + ResourceChange, + TFolderCommitServiceFactory +} from "./folder-commit-service"; + +// Mock config +vi.mock("@app/lib/config/env", () => ({ + getConfig: () => ({ + PIT_CHECKPOINT_WINDOW: 5, + PIT_TREE_CHECKPOINT_WINDOW: 10 + }) +})); + +// Mock logger +vi.mock("@app/lib/logger", () => ({ + logger: { + info: vi.fn(), + error: vi.fn() + } +})); + +describe("folderCommitServiceFactory", () => { + // Properly type the mock functions + type TransactionCallback = (trx: Knex) => Promise; + + // Mock dependencies + const mockFolderCommitDAL = { + create: vi.fn().mockResolvedValue({}), + findById: vi.fn().mockResolvedValue({}), + findByFolderId: vi.fn().mockResolvedValue([]), + findLatestCommit: vi.fn().mockResolvedValue({}), + transaction: vi.fn().mockImplementation((callback: TransactionCallback) => callback({} as Knex)), + getNumberOfCommitsSince: vi.fn().mockResolvedValue(0), + getEnvNumberOfCommitsSince: vi.fn().mockResolvedValue(0), + findCommitsToRecreate: vi.fn().mockResolvedValue([]), + findMultipleLatestCommits: vi.fn().mockResolvedValue([]), + findLatestCommitBetween: vi.fn().mockResolvedValue({}), + findAllCommitsBetween: vi.fn().mockResolvedValue([]), + findLatestEnvCommit: vi.fn().mockResolvedValue({}), + findLatestCommitByFolderIds: vi.fn().mockResolvedValue({}) + }; + + const mockKmsService = { + createCipherPairWithDataKey: vi.fn().mockResolvedValue({}) + }; + + const mockFolderCommitChangesDAL = { + create: vi.fn().mockResolvedValue({}), + findByCommitId: vi.fn().mockResolvedValue([]), + insertMany: vi.fn().mockResolvedValue([]) + }; + + const mockFolderCheckpointDAL = { + create: vi.fn().mockResolvedValue({}), + findByFolderId: vi.fn().mockResolvedValue([]), + findLatestByFolderId: vi.fn().mockResolvedValue(null), + findNearestCheckpoint: vi.fn().mockResolvedValue({}) + }; + + const mockFolderCheckpointResourcesDAL = { + insertMany: vi.fn().mockResolvedValue([]), + findByCheckpointId: vi.fn().mockResolvedValue([]) + }; + + const mockFolderTreeCheckpointDAL = { + create: vi.fn().mockResolvedValue({}), + findByProjectId: vi.fn().mockResolvedValue([]), + findLatestByProjectId: vi.fn().mockResolvedValue({}), + findNearestCheckpoint: vi.fn().mockResolvedValue({}), + findLatestByEnvId: vi.fn().mockResolvedValue({}) + }; + + const mockFolderTreeCheckpointResourcesDAL = { + insertMany: vi.fn().mockResolvedValue([]), + findByTreeCheckpointId: vi.fn().mockResolvedValue([]) + }; + + const mockUserDAL = { + findById: vi.fn().mockResolvedValue({}) + }; + + const mockIdentityDAL = { + findById: vi.fn().mockResolvedValue({}) + }; + + const mockFolderDAL = { + findByParentId: vi.fn().mockResolvedValue([]), + findByProjectId: vi.fn().mockResolvedValue([]), + deleteById: vi.fn().mockResolvedValue({}), + create: vi.fn().mockResolvedValue({}), + updateById: vi.fn().mockResolvedValue({}), + update: vi.fn().mockResolvedValue({}), + find: vi.fn().mockResolvedValue([]), + findById: vi.fn().mockResolvedValue({}), + findByEnvId: vi.fn().mockResolvedValue([]), + findFoldersByRootAndIds: vi.fn().mockResolvedValue([]) + }; + + const mockFolderVersionDAL = { + findLatestFolderVersions: vi.fn().mockResolvedValue({}), + findById: vi.fn().mockResolvedValue({}), + deleteById: vi.fn().mockResolvedValue({}), + create: vi.fn().mockResolvedValue({}), + updateById: vi.fn().mockResolvedValue({}), + find: vi.fn().mockResolvedValue({}), // Changed from [] to {} to match Object.values() expectation + findByIdsWithLatestVersion: vi.fn().mockResolvedValue({}) + }; + + const mockSecretVersionV2BridgeDAL = { + findLatestVersionByFolderId: vi.fn().mockResolvedValue([]), + findById: vi.fn().mockResolvedValue({}), + deleteById: vi.fn().mockResolvedValue({}), + create: vi.fn().mockResolvedValue({}), + updateById: vi.fn().mockResolvedValue({}), + find: vi.fn().mockResolvedValue([]), + findByIdsWithLatestVersion: vi.fn().mockResolvedValue({}), + findLatestVersionMany: vi.fn().mockResolvedValue({}) + }; + + const mockSecretV2BridgeDAL = { + deleteById: vi.fn().mockResolvedValue({}), + create: vi.fn().mockResolvedValue({}), + updateById: vi.fn().mockResolvedValue({}), + update: vi.fn().mockResolvedValue({}), + insertMany: vi.fn().mockResolvedValue([]), + invalidateSecretCacheByProjectId: vi.fn().mockResolvedValue({}) + }; + + const mockProjectDAL = { + findById: vi.fn().mockResolvedValue({}), + findProjectByEnvId: vi.fn().mockResolvedValue({}) + }; + + const mockFolderCommitQueueService = { + scheduleTreeCheckpoint: vi.fn().mockResolvedValue({}), + createFolderTreeCheckpoint: vi.fn().mockResolvedValue({}) + }; + + const mockPermissionService = { + getProjectPermission: vi.fn().mockResolvedValue({}) + }; + + const mockSecretTagDAL = { + findSecretTagsByVersionId: vi.fn().mockResolvedValue([]), + saveTagsToSecretV2: vi.fn().mockResolvedValue([]), + findSecretTagsBySecretId: vi.fn().mockResolvedValue([]), + deleteTagsToSecretV2: vi.fn().mockResolvedValue([]), + saveTagsToSecretVersionV2: vi.fn().mockResolvedValue([]) + }; + + const mockResourceMetadataDAL = { + find: vi.fn().mockResolvedValue([]), + insertMany: vi.fn().mockResolvedValue([]), + delete: vi.fn().mockResolvedValue([]) + }; + + let folderCommitService: TFolderCommitServiceFactory; + + beforeEach(() => { + vi.clearAllMocks(); + + folderCommitService = folderCommitServiceFactory({ + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + folderCommitDAL: mockFolderCommitDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + folderCommitChangesDAL: mockFolderCommitChangesDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + folderCheckpointDAL: mockFolderCheckpointDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + folderCheckpointResourcesDAL: mockFolderCheckpointResourcesDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + folderTreeCheckpointDAL: mockFolderTreeCheckpointDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + folderTreeCheckpointResourcesDAL: mockFolderTreeCheckpointResourcesDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + userDAL: mockUserDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + identityDAL: mockIdentityDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + folderDAL: mockFolderDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + folderVersionDAL: mockFolderVersionDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + secretVersionV2BridgeDAL: mockSecretVersionV2BridgeDAL, + projectDAL: mockProjectDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + secretV2BridgeDAL: mockSecretV2BridgeDAL, + folderCommitQueueService: mockFolderCommitQueueService, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + permissionService: mockPermissionService, + kmsService: mockKmsService, + secretTagDAL: mockSecretTagDAL, + resourceMetadataDAL: mockResourceMetadataDAL + }); + }); + + afterEach(() => { + vi.resetAllMocks(); + }); + + describe("createCommit", () => { + it("should successfully create a commit with user actor", async () => { + // Arrange + const userData = { id: "user-id", username: "testuser" }; + const folderData = { id: "folder-id", envId: "env-id" }; + const commitData = { id: "commit-id", folderId: "folder-id" }; + + mockUserDAL.findById.mockResolvedValue(userData); + mockFolderDAL.findById.mockResolvedValue(folderData); + mockFolderCommitDAL.create.mockResolvedValue(commitData); + mockFolderCheckpointDAL.findLatestByFolderId.mockResolvedValue(null); + mockFolderCommitDAL.findLatestCommit.mockResolvedValue({ id: "latest-commit-id" }); + mockFolderDAL.findByParentId.mockResolvedValue([]); + mockSecretVersionV2BridgeDAL.findLatestVersionByFolderId.mockResolvedValue([]); + + const data = { + actor: { + type: ActorType.USER, + metadata: { id: userData.id } + }, + message: "Test commit", + folderId: folderData.id, + changes: [ + { + type: CommitType.ADD, + secretVersionId: "secret-version-1" + } + ] + }; + + // Act + const result = await folderCommitService.createCommit(data); + + // Assert + expect(mockUserDAL.findById).toHaveBeenCalledWith(userData.id, undefined); + expect(mockFolderDAL.findById).toHaveBeenCalledWith(folderData.id, undefined); + expect(mockFolderCommitDAL.create).toHaveBeenCalledWith( + expect.objectContaining({ + actorType: ActorType.USER, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + actorMetadata: expect.objectContaining({ name: userData.username }), + message: data.message, + folderId: data.folderId, + envId: folderData.envId + }), + undefined + ); + expect(mockFolderCommitChangesDAL.insertMany).toHaveBeenCalledWith( + expect.arrayContaining([ + expect.objectContaining({ + folderCommitId: commitData.id, + changeType: data.changes[0].type, + secretVersionId: data.changes[0].secretVersionId + }) + ]), + undefined + ); + expect(mockFolderCommitQueueService.scheduleTreeCheckpoint).toHaveBeenCalledWith(folderData.envId); + expect(result).toEqual(commitData); + }); + + it("should successfully create a commit with identity actor", async () => { + // Arrange + const identityData = { id: "identity-id", name: "testidentity" }; + const folderData = { id: "folder-id", envId: "env-id" }; + const commitData = { id: "commit-id", folderId: "folder-id" }; + + mockIdentityDAL.findById.mockResolvedValue(identityData); + mockFolderDAL.findById.mockResolvedValue(folderData); + mockFolderCommitDAL.create.mockResolvedValue(commitData); + mockFolderCheckpointDAL.findLatestByFolderId.mockResolvedValue(null); + mockFolderCommitDAL.findLatestCommit.mockResolvedValue({ id: "latest-commit-id" }); + mockFolderDAL.findByParentId.mockResolvedValue([]); + mockSecretVersionV2BridgeDAL.findLatestVersionByFolderId.mockResolvedValue([]); + + // Mock folderVersionDAL.find to return an object with folder version data + mockFolderVersionDAL.find.mockResolvedValue({ + "folder-version-1": { + id: "folder-version-1", + folderId: "sub-folder-id", + envId: "env-id", + name: "Test Folder", + version: 1 + } + }); + + const data = { + actor: { + type: ActorType.IDENTITY, + metadata: { id: identityData.id } + }, + message: "Test commit", + folderId: folderData.id, + changes: [ + { + type: CommitType.ADD, + folderVersionId: "folder-version-1" + } + ], + omitIgnoreFilter: true + }; + + // Act + const result = await folderCommitService.createCommit(data); + + // Assert + expect(mockIdentityDAL.findById).toHaveBeenCalledWith(identityData.id, undefined); + expect(mockFolderDAL.findById).toHaveBeenCalledWith(folderData.id, undefined); + expect(mockFolderCommitDAL.create).toHaveBeenCalledWith( + expect.objectContaining({ + actorType: ActorType.IDENTITY, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + actorMetadata: expect.objectContaining({ name: identityData.name }), + message: data.message, + folderId: data.folderId, + envId: folderData.envId + }), + undefined + ); + expect(mockFolderCommitChangesDAL.insertMany).toHaveBeenCalledWith( + expect.arrayContaining([ + expect.objectContaining({ + folderCommitId: commitData.id, + changeType: data.changes[0].type, + folderVersionId: data.changes[0].folderVersionId + }) + ]), + undefined + ); + expect(mockFolderCommitQueueService.scheduleTreeCheckpoint).toHaveBeenCalledWith(folderData.envId); + expect(result).toEqual(commitData); + }); + + it("should throw NotFoundError when folder does not exist", async () => { + // Arrange + mockFolderDAL.findById.mockResolvedValue(null); + + const data = { + actor: { + type: ActorType.PLATFORM + }, + message: "Test commit", + folderId: "non-existent-folder", + changes: [] + }; + + // Act & Assert + await expect(folderCommitService.createCommit(data)).rejects.toThrow(NotFoundError); + expect(mockFolderDAL.findById).toHaveBeenCalledWith("non-existent-folder", undefined); + }); + }); + + describe("addCommitChange", () => { + it("should successfully add a change to an existing commit", async () => { + // Arrange + const commitData = { id: "commit-id", folderId: "folder-id" }; + const changeData = { id: "change-id", folderCommitId: "commit-id" }; + + mockFolderCommitDAL.findById.mockResolvedValue(commitData); + mockFolderCommitChangesDAL.create.mockResolvedValue(changeData); + + const data = { + folderCommitId: commitData.id, + changeType: CommitType.ADD, + secretVersionId: "secret-version-1" + }; + + // Act + const result = await folderCommitService.addCommitChange(data); + + // Assert + expect(mockFolderCommitDAL.findById).toHaveBeenCalledWith(commitData.id, undefined); + expect(mockFolderCommitChangesDAL.create).toHaveBeenCalledWith(data, undefined); + expect(result).toEqual(changeData); + }); + + it("should throw BadRequestError when neither secretVersionId nor folderVersionId is provided", async () => { + // Arrange + const data = { + folderCommitId: "commit-id", + changeType: CommitType.ADD + }; + + // Act & Assert + await expect(folderCommitService.addCommitChange(data)).rejects.toThrow(BadRequestError); + }); + + it("should throw NotFoundError when commit does not exist", async () => { + // Arrange + mockFolderCommitDAL.findById.mockResolvedValue(null); + + const data = { + folderCommitId: "non-existent-commit", + changeType: CommitType.ADD, + secretVersionId: "secret-version-1" + }; + + // Act & Assert + await expect(folderCommitService.addCommitChange(data)).rejects.toThrow(NotFoundError); + expect(mockFolderCommitDAL.findById).toHaveBeenCalledWith("non-existent-commit", undefined); + }); + }); + + // Note: reconstructFolderState is an internal function not exposed in the public API + // We'll test it indirectly through compareFolderStates + + describe("compareFolderStates", () => { + it("should mark all resources as creates when currentCommitId is not provided", async () => { + // Arrange + const targetCommitId = "target-commit-id"; + const targetCommit = { id: targetCommitId, commitId: 1, folderId: "folder-id" }; + + mockFolderCommitDAL.findById.mockResolvedValue(targetCommit); + // Mock how compareFolderStates would process the results internally + mockFolderCheckpointDAL.findNearestCheckpoint.mockResolvedValue({ id: "checkpoint-id", commitId: "hash-0" }); + mockFolderCheckpointResourcesDAL.findByCheckpointId.mockResolvedValue([ + { secretVersionId: "secret-version-1", referencedSecretId: "secret-1" }, + { folderVersionId: "folder-version-1", referencedFolderId: "folder-1" } + ]); + mockFolderCommitDAL.findCommitsToRecreate.mockResolvedValue([]); + mockProjectDAL.findProjectByEnvId.mockResolvedValue({ + id: "project-id", + name: "test-project", + type: ProjectType.SecretManager + }); + + // Act + const result = await folderCommitService.compareFolderStates({ + targetCommitId + }); + + // Assert + expect(mockFolderCommitDAL.findById).toHaveBeenCalledWith(targetCommitId, undefined); + + // Verify we get resources marked as create + expect(result).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + changeType: "create", + commitId: targetCommit.commitId + }) + ]) + ); + }); + }); + + describe("createFolderCheckpoint", () => { + it("should successfully create a checkpoint when force is true", async () => { + // Arrange + const folderCommitId = "commit-id"; + const folderId = "folder-id"; + const checkpointData = { id: "checkpoint-id", folderCommitId }; + + mockFolderDAL.findByParentId.mockResolvedValue([{ id: "subfolder-id" }]); + mockFolderVersionDAL.findLatestFolderVersions.mockResolvedValue({ "subfolder-id": { id: "folder-version-1" } }); + mockSecretVersionV2BridgeDAL.findLatestVersionByFolderId.mockResolvedValue([{ id: "secret-version-1" }]); + mockFolderCheckpointDAL.create.mockResolvedValue(checkpointData); + + // Act + const result = await folderCommitService.createFolderCheckpoint({ + folderId, + folderCommitId, + force: true + }); + + // Assert + expect(mockFolderCheckpointDAL.create).toHaveBeenCalledWith({ folderCommitId }, undefined); + expect(mockFolderCheckpointResourcesDAL.insertMany).toHaveBeenCalled(); + expect(result).toBe(folderCommitId); + }); + }); + + describe("deepRollbackFolder", () => { + it("should throw NotFoundError when commit doesn't exist", async () => { + // Arrange + const targetCommitId = "non-existent-commit"; + const envId = "env-id"; + const actorId = "user-id"; + const actorType = ActorType.USER; + const projectId = "project-id"; + + // Mock the transaction to properly handle the error + mockFolderCommitDAL.transaction.mockImplementation(async (callback) => { + return await callback({} as Knex); + }); + + // Mock findById to return null inside the transaction + mockFolderCommitDAL.findById.mockResolvedValue(null); + + // Act & Assert + await expect( + folderCommitService.deepRollbackFolder(targetCommitId, envId, actorId, actorType, projectId) + ).rejects.toThrow(NotFoundError); + }); + }); + + describe("createFolderTreeCheckpoint", () => { + it("should create a tree checkpoint when checkpoint window is exceeded", async () => { + // Arrange + const envId = "env-id"; + const folderCommitId = "commit-id"; + const latestCommit = { id: folderCommitId }; + const latestTreeCheckpoint = { id: "tree-checkpoint-id", folderCommitId: "old-commit-id" }; + const folders = [ + { id: "folder-1", isReserved: false }, + { id: "folder-2", isReserved: false }, + { id: "folder-3", isReserved: true } // Reserved folders should be filtered out + ]; + const folderCommits = [ + { folderId: "folder-1", id: "commit-1" }, + { folderId: "folder-2", id: "commit-2" } + ]; + const treeCheckpoint = { id: "new-tree-checkpoint-id" }; + + mockFolderCommitDAL.findLatestEnvCommit.mockResolvedValue(latestCommit); + mockFolderTreeCheckpointDAL.findLatestByEnvId.mockResolvedValue(latestTreeCheckpoint); + mockFolderCommitDAL.getEnvNumberOfCommitsSince.mockResolvedValue(15); // More than PIT_TREE_CHECKPOINT_WINDOW (10) + mockFolderDAL.findByEnvId.mockResolvedValue(folders); + mockFolderCommitDAL.findMultipleLatestCommits.mockResolvedValue(folderCommits); + mockFolderTreeCheckpointDAL.create.mockResolvedValue(treeCheckpoint); + + // Act + await folderCommitService.createFolderTreeCheckpoint(envId); + + // Assert + expect(mockFolderCommitDAL.findLatestEnvCommit).toHaveBeenCalledWith(envId, undefined); + expect(mockFolderTreeCheckpointDAL.create).toHaveBeenCalledWith({ folderCommitId }, undefined); + }); + }); + + describe("applyFolderStateDifferences", () => { + it("should process changes correctly", async () => { + // Arrange + const folderId = "folder-id"; + const projectId = "project-id"; + const actorId = "user-id"; + const actorType = ActorType.USER; + + const differences = [ + { + id: "secret-1", + versionId: "v1", + changeType: ChangeType.CREATE, + commitId: BigInt(1) + } as ResourceChange, + { + id: "folder-1", + versionId: "v2", + changeType: ChangeType.UPDATE, + commitId: BigInt(1), + folderName: "Test Folder", + folderVersion: "v2" + } as ResourceChange + ]; + + const secretVersions = { + "secret-1": { + id: "secret-version-1", + createdAt: new Date(), + updatedAt: new Date(), + type: "shared", + folderId: "folder-1", + secretId: "secret-1", + version: 1, + key: "SECRET_KEY", + encryptedValue: Buffer.from("encrypted"), + encryptedComment: Buffer.from("comment"), + skipMultilineEncoding: false, + userId: "user-1", + envId: "env-1", + metadata: {} + } as TSecretVersionsV2 + }; + + const folderVersions = { + "folder-1": { + folderId: "folder-1", + version: 1, + name: "Test Folder", + envId: "env-1" + } as TSecretFolderVersions + }; + + // Mock folder lookup for the folder being processed + mockFolderDAL.findById.mockImplementation((id) => { + if (id === folderId) { + return Promise.resolve({ id: folderId, envId: "env-1" }); + } + return Promise.resolve(null); + }); + + // Mock latest commit lookup + mockFolderCommitDAL.findLatestCommit.mockImplementation((id) => { + if (id === folderId) { + return Promise.resolve({ id: "latest-commit-id", folderId }); + } + return Promise.resolve(null); + }); + + // Make sure findByParentId returns an array, not undefined + mockFolderDAL.findByParentId.mockResolvedValue([]); + + // Make sure other required functions return appropriate values + mockFolderCheckpointDAL.findLatestByFolderId.mockResolvedValue(null); + mockSecretVersionV2BridgeDAL.findLatestVersionByFolderId.mockResolvedValue([]); + + // These mocks need to return objects with an id field + mockSecretVersionV2BridgeDAL.findByIdsWithLatestVersion.mockResolvedValue(Object.values(secretVersions)); + mockFolderVersionDAL.findByIdsWithLatestVersion.mockResolvedValue(Object.values(folderVersions)); + mockSecretV2BridgeDAL.insertMany.mockResolvedValue([{ id: "new-secret-1" }]); + mockSecretVersionV2BridgeDAL.create.mockResolvedValue({ id: "new-secret-version-1" }); + mockFolderDAL.updateById.mockResolvedValue({ id: "updated-folder-1" }); + mockFolderVersionDAL.create.mockResolvedValue({ id: "new-folder-version-1" }); + mockFolderCommitDAL.create.mockResolvedValue({ id: "new-commit-id" }); + mockSecretVersionV2BridgeDAL.findLatestVersionMany.mockResolvedValue([ + { + id: "secret-version-1", + createdAt: new Date(), + updatedAt: new Date(), + type: "shared", + folderId: "folder-1", + secretId: "secret-1", + version: 1, + key: "SECRET_KEY", + encryptedValue: Buffer.from("encrypted"), + encryptedComment: Buffer.from("comment"), + skipMultilineEncoding: false, + userId: "user-1", + envId: "env-1", + metadata: {} + } + ]); + + // Mock transaction + mockFolderCommitDAL.transaction.mockImplementation((callback: TransactionCallback) => callback({} as Knex)); + + // Act + const result = await folderCommitService.applyFolderStateDifferences({ + differences, + actorInfo: { + actorType, + actorId, + message: "Applying changes" + }, + folderId, + projectId, + reconstructNewFolders: false + }); + + // Assert + expect(mockFolderCommitDAL.create).toHaveBeenCalled(); + expect(mockSecretV2BridgeDAL.invalidateSecretCacheByProjectId).toHaveBeenCalledWith(projectId); + + // Check that we got the right counts + expect(result.totalChanges).toEqual(2); + }); + }); +}); diff --git a/backend/src/services/folder-commit/folder-commit-service.ts b/backend/src/services/folder-commit/folder-commit-service.ts new file mode 100644 index 000000000..8c0bc8ebf --- /dev/null +++ b/backend/src/services/folder-commit/folder-commit-service.ts @@ -0,0 +1,2173 @@ +/* eslint-disable no-await-in-loop */ +import { ForbiddenError } from "@casl/ability"; +import { Knex } from "knex"; + +import { + ActionProjectType, + TSecretFolders, + TSecretFolderVersions, + TSecretV2TagJunctionInsert, + TSecretVersionsV2 +} from "@app/db/schemas"; +import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; +import { ProjectPermissionCommitsActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; +import { getConfig } from "@app/lib/config/env"; +import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors"; +import { chunkArray } from "@app/lib/fn"; +import { logger } from "@app/lib/logger"; + +import { ActorAuthMethod, ActorType } from "../auth/auth-type"; +import { TFolderCheckpointDALFactory } from "../folder-checkpoint/folder-checkpoint-dal"; +import { TFolderCheckpointResourcesDALFactory } from "../folder-checkpoint-resources/folder-checkpoint-resources-dal"; +import { TFolderCommitChangesDALFactory } from "../folder-commit-changes/folder-commit-changes-dal"; +import { TFolderTreeCheckpointDALFactory } from "../folder-tree-checkpoint/folder-tree-checkpoint-dal"; +import { TFolderTreeCheckpointResourcesDALFactory } from "../folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal"; +import { TIdentityDALFactory } from "../identity/identity-dal"; +import { TKmsServiceFactory } from "../kms/kms-service"; +import { KmsDataKey } from "../kms/kms-types"; +import { TProjectDALFactory } from "../project/project-dal"; +import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal"; +import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal"; +import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; +import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; +import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal"; +import { TUserDALFactory } from "../user/user-dal"; +import { TFolderCommitDALFactory } from "./folder-commit-dal"; +import { TFolderCommitQueueServiceFactory } from "./folder-commit-queue"; + +export enum ChangeType { + ADD = "add", + DELETE = "delete", + UPDATE = "update", + CREATE = "create" +} + +export enum CommitType { + ADD = "add", + DELETE = "delete" +} + +export enum ResourceType { + SECRET = "secret", + FOLDER = "folder" +} + +type TCreateCommitDTO = { + actor: { + type: string; + metadata?: { + name?: string; + id?: string; + }; + }; + message?: string; + folderId: string; + changes: { + type: string; + secretVersionId?: string; + folderVersionId?: string; + isUpdate?: boolean; + folderId?: string; + }[]; + omitIgnoreFilter?: boolean; +}; + +type TCommitChangeDTO = { + folderCommitId: string; + changeType: string; + secretVersionId?: string; + folderVersionId?: string; +}; + +type BaseChange = { + id: string; + versionId: string; + oldVersionId?: string; + changeType: ChangeType; + commitId: bigint; + createdAt?: Date; + parentId?: string; + isUpdate?: boolean; + fromVersion?: string; +}; + +type SecretChange = BaseChange & { + type: ResourceType.SECRET; + secretKey: string; + secretVersion: string; + secretId: string; + versions?: { + secretKey?: string; + secretComment?: string; + skipMultilineEncoding?: boolean | null; + metadata?: unknown; + tags?: string[] | null; + secretValue?: string; + }[]; +}; + +type FolderChange = BaseChange & { + type: ResourceType.FOLDER; + folderName: string; + folderVersion: string; + versions?: { + name: string; + description?: string | null; + }[]; +}; + +type SecretTargetChange = { + type: ResourceType.SECRET; + id: string; + versionId: string; + secretKey: string; + secretVersion: string; + fromVersion?: string; +}; + +type FolderTargetChange = { + type: ResourceType.FOLDER; + id: string; + versionId: string; + folderName: string; + folderVersion: string; + fromVersion?: string; +}; + +export type ResourceChange = SecretChange | FolderChange; + +type ActorInfo = { + actorType: string; + actorId?: string; + message?: string; +}; + +type StateChangeResult = { + secretChangesCount: number; + folderChangesCount: number; + totalChanges: number; +}; + +type TFolderCommitServiceFactoryDep = { + folderCommitDAL: TFolderCommitDALFactory; + folderCommitChangesDAL: TFolderCommitChangesDALFactory; + folderCheckpointDAL: TFolderCheckpointDALFactory; + folderCheckpointResourcesDAL: TFolderCheckpointResourcesDALFactory; + folderTreeCheckpointDAL: TFolderTreeCheckpointDALFactory; + folderTreeCheckpointResourcesDAL: TFolderTreeCheckpointResourcesDALFactory; + userDAL: TUserDALFactory; + identityDAL: TIdentityDALFactory; + folderDAL: TSecretFolderDALFactory; + folderVersionDAL: TSecretFolderVersionDALFactory; + secretVersionV2BridgeDAL: TSecretVersionV2DALFactory; + secretV2BridgeDAL: TSecretV2BridgeDALFactory; + projectDAL: Pick; + folderCommitQueueService?: Pick< + TFolderCommitQueueServiceFactory, + "scheduleTreeCheckpoint" | "createFolderTreeCheckpoint" + >; + permissionService?: TPermissionServiceFactory; + kmsService: Pick; + secretTagDAL: Pick< + TSecretTagDALFactory, + | "findSecretTagsByVersionId" + | "saveTagsToSecretV2" + | "findSecretTagsBySecretId" + | "deleteTagsToSecretV2" + | "saveTagsToSecretVersionV2" + >; + resourceMetadataDAL: Pick; +}; + +export const folderCommitServiceFactory = ({ + folderCommitDAL, + folderCommitChangesDAL, + folderCheckpointDAL, + folderTreeCheckpointDAL, + folderCheckpointResourcesDAL, + userDAL, + identityDAL, + folderDAL, + folderVersionDAL, + secretVersionV2BridgeDAL, + projectDAL, + secretV2BridgeDAL, + folderTreeCheckpointResourcesDAL, + folderCommitQueueService, + permissionService, + kmsService, + secretTagDAL, + resourceMetadataDAL +}: TFolderCommitServiceFactoryDep) => { + const appCfg = getConfig(); + + const checkProjectCommitReadPermission = async ({ + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + }: { + actor: ActorType; + actorId: string; + projectId: string; + actorAuthMethod: ActorAuthMethod; + actorOrgId: string; + }) => { + if (!permissionService) { + throw new Error("Permission service not initialized"); + } + const { permission } = await permissionService.getProjectPermission({ + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId, + actionProjectType: ActionProjectType.SecretManager + }); + + ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionCommitsActions.Read, ProjectPermissionSub.Commits); + }; + + /** + * Fetches all resources within a folder + */ + const getFolderResources = async (folderId: string, tx?: Knex) => { + const resources = []; + const subFolders = await folderDAL.findByParentId(folderId, tx); + + if (subFolders.length > 0) { + const subFolderIds = subFolders.map((folder) => folder.id); + const folderVersions = await folderVersionDAL.findLatestFolderVersions(subFolderIds, tx); + resources.push( + ...Object.values(folderVersions).map((folderVersion) => ({ + folderVersionId: folderVersion.id, + secretVersionId: undefined + })) + ); + } + + const secretVersions = await secretVersionV2BridgeDAL.findLatestVersionByFolderId(folderId, tx); + if (secretVersions.length > 0) { + resources.push( + ...secretVersions.map((secretVersion) => ({ secretVersionId: secretVersion.id, folderVersionId: undefined })) + ); + } + + return resources; + }; + + /** + * Creates a checkpoint for a folder if necessary + */ + const createFolderCheckpoint = async ({ + folderId, + folderCommitId, + force = false, + tx + }: { + folderId: string; + folderCommitId?: string; + force?: boolean; + tx?: Knex; + }) => { + let latestCommitId = folderCommitId; + const latestCheckpoint = await folderCheckpointDAL.findLatestByFolderId(folderId, tx); + + if (!latestCommitId) { + const latestCommit = await folderCommitDAL.findLatestCommit(folderId, undefined, tx); + if (!latestCommit) { + throw new BadRequestError({ message: "Latest commit ID not found" }); + } + latestCommitId = latestCommit.id; + } + + if (!force && latestCheckpoint) { + const commitsSinceLastCheckpoint = await folderCommitDAL.getNumberOfCommitsSince( + folderId, + latestCheckpoint.folderCommitId, + tx + ); + if (commitsSinceLastCheckpoint < Number(appCfg.PIT_CHECKPOINT_WINDOW)) { + return; + } + } + + const checkpointResources = await getFolderResources(folderId, tx); + + const newCheckpoint = await folderCheckpointDAL.create( + { + folderCommitId: latestCommitId + }, + tx + ); + const batchSize = 500; + const chunks = chunkArray(checkpointResources, batchSize); + + await Promise.all( + chunks.map(async (chunk) => { + await folderCheckpointResourcesDAL.insertMany( + chunk.map((resource) => ({ folderCheckpointId: newCheckpoint.id, ...resource })), + tx + ); + }) + ); + + return latestCommitId; + }; + + /** + * Reconstructs the state of a folder at a specific commit + */ + const reconstructFolderState = async ( + folderCommitId: string, + tx?: Knex + ): Promise<(SecretTargetChange | FolderTargetChange)[]> => { + const targetCommit = await folderCommitDAL.findById(folderCommitId, tx); + if (!targetCommit) { + throw new NotFoundError({ message: `Commit with ID ${folderCommitId} not found` }); + } + + const nearestCheckpoint = await folderCheckpointDAL.findNearestCheckpoint( + targetCommit.commitId, + targetCommit.folderId, + tx + ); + if (!nearestCheckpoint) { + throw new NotFoundError({ message: `Nearest checkpoint not found for commit ${folderCommitId}` }); + } + + const checkpointResources = await folderCheckpointResourcesDAL.findByCheckpointId(nearestCheckpoint.id, tx); + + const folderState: Record = {}; + + // Add all checkpoint resources to initial state + checkpointResources.forEach((resource) => { + if (resource.secretVersionId && resource.referencedSecretId) { + folderState[`secret-${resource.referencedSecretId}`] = { + type: ResourceType.SECRET, + id: resource.referencedSecretId, + versionId: resource.secretVersionId, + secretKey: resource.secretKey, + secretVersion: resource.secretVersion + } as SecretTargetChange; + } else if (resource.folderVersionId && resource.referencedFolderId) { + folderState[`folder-${resource.referencedFolderId}`] = { + type: ResourceType.FOLDER, + id: resource.referencedFolderId, + versionId: resource.folderVersionId, + folderName: resource.folderName, + folderVersion: resource.folderVersion + } as FolderTargetChange; + } + }); + + const commitsToRecreate = await folderCommitDAL.findCommitsToRecreate( + targetCommit.folderId, + targetCommit.commitId, + nearestCheckpoint.commitId, + tx + ); + + // Process commits to recreate final state + for (const commit of commitsToRecreate) { + // eslint-disable-next-line no-continue + if (!commit.changes) continue; + + for (const change of commit.changes) { + if (change.secretVersionId && change.referencedSecretId) { + const key = `secret-${change.referencedSecretId}`; + + if (change.changeType.toLowerCase() === "add") { + folderState[key] = { + type: ResourceType.SECRET, + id: change.referencedSecretId, + versionId: change.secretVersionId, + secretKey: change.secretKey, + secretVersion: change.secretVersion + } as SecretTargetChange; + } else if (change.changeType.toLowerCase() === "delete") { + delete folderState[key]; + } + } else if (change.folderVersionId && change.referencedFolderId) { + const key = `folder-${change.referencedFolderId}`; + + if (change.changeType.toLowerCase() === "add") { + folderState[key] = { + type: ResourceType.FOLDER, + id: change.referencedFolderId, + versionId: change.folderVersionId, + folderName: change.folderName, + folderVersion: change.folderVersion + } as FolderTargetChange; + } else if (change.changeType.toLowerCase() === "delete") { + delete folderState[key]; + } + } + } + } + return Object.values(folderState); + }; + + /** + * Compares folder states between two commits and returns the differences + */ + const compareFolderStates = async ({ + currentCommitId, + targetCommitId, + defaultOperation = "create", + tx + }: { + currentCommitId?: string; + targetCommitId: string; + defaultOperation?: "create" | "update" | "delete"; + tx?: Knex; + }): Promise => { + const targetCommit = await folderCommitDAL.findById(targetCommitId, tx); + if (!targetCommit) { + throw new NotFoundError({ message: `Commit with ID ${targetCommitId} not found` }); + } + + const project = await projectDAL.findProjectByEnvId(targetCommit.envId, tx); + + if (!project) { + throw new NotFoundError({ message: `No project found for envId ${targetCommit.envId}` }); + } + + // If currentCommitId is not provided, mark all resources in target as creates + if (!currentCommitId) { + const targetState = await reconstructFolderState(targetCommitId, tx); + + return targetState + .map((resource): ResourceChange | null => { + if (resource.type === ResourceType.SECRET) { + return { + type: ResourceType.SECRET, + id: resource.id, + versionId: resource.versionId, + changeType: defaultOperation as ChangeType, + commitId: targetCommit.commitId, + secretKey: resource.secretKey, + secretVersion: resource.secretVersion, + secretId: resource.id + }; + } + if (resource.type === ResourceType.FOLDER) { + return { + type: ResourceType.FOLDER, + id: resource.id, + versionId: resource.versionId, + changeType: defaultOperation as ChangeType, + commitId: targetCommit.commitId, + folderName: resource.folderName, + folderVersion: resource.folderVersion + }; + } + return null; + }) + .filter((change): change is ResourceChange => !!change); + } + + // Original logic for when currentCommitId is provided + const currentState = await reconstructFolderState(currentCommitId, tx); + const targetState = await reconstructFolderState(targetCommitId, tx); + + // Create lookup maps for easier comparison + const currentMap: Record = {}; + const targetMap: Record< + string, + { + type: string; + id: string; + versionId: string; + secretKey?: string; + secretVersion?: string; + folderName?: string; + folderVersion?: string; + fromVersion?: string; + } + > = {}; + + // Build lookup maps + currentState.forEach((resource) => { + const key = `${resource.type}-${resource.id}`; + currentMap[key] = resource; + }); + + targetState.forEach((resource) => { + const key = `${resource.type}-${resource.id}`; + targetMap[key] = resource; + }); + + // Track differences + const differences: ResourceChange[] = []; + + // Find deletes and updates + Object.keys(currentMap).forEach((key) => { + const currentResource = currentMap[key]; + const targetResource = targetMap[key]; + + if (!targetResource) { + // Resource was deleted + if (currentResource.type === ResourceType.SECRET) { + differences.push({ + type: ResourceType.SECRET, + id: currentResource.id, + versionId: currentResource.versionId, + changeType: ChangeType.DELETE, + commitId: targetCommit.commitId, + secretKey: currentResource.secretKey, + secretVersion: currentResource.secretVersion, + secretId: currentResource.id, + fromVersion: currentResource.versionId + }); + } else if (currentResource.type === ResourceType.FOLDER) { + differences.push({ + type: ResourceType.FOLDER, + id: currentResource.id, + versionId: currentResource.versionId, + changeType: ChangeType.DELETE, + commitId: targetCommit.commitId, + folderName: currentResource.folderName, + folderVersion: currentResource.folderVersion, + fromVersion: currentResource.versionId + }); + } + } else if (currentResource.versionId !== targetResource.versionId) { + // Resource was updated + if (targetResource.type === ResourceType.SECRET) { + const secretCurrentResource = currentResource as SecretTargetChange; + const secretTargetResource = targetResource as SecretTargetChange; + differences.push({ + type: ResourceType.SECRET, + id: secretTargetResource.id, + versionId: secretTargetResource.versionId, + changeType: ChangeType.UPDATE, + commitId: targetCommit.commitId, + secretKey: secretTargetResource.secretKey, + secretVersion: secretTargetResource.secretVersion, + secretId: secretTargetResource.id, + fromVersion: secretCurrentResource.secretVersion + }); + } else if (targetResource.type === ResourceType.FOLDER) { + const folderCurrentResource = currentResource as FolderTargetChange; + const folderTargetResource = targetResource as FolderTargetChange; + + differences.push({ + type: ResourceType.FOLDER, + id: folderTargetResource.id, + versionId: folderTargetResource.versionId, + changeType: ChangeType.UPDATE, + commitId: targetCommit.commitId, + folderName: folderTargetResource.folderName, + folderVersion: folderTargetResource.folderVersion, + fromVersion: folderCurrentResource.folderVersion + }); + } + } + }); + + // Find new resources + Object.keys(targetMap).forEach((key) => { + if (!currentMap[key]) { + const targetResource = targetMap[key]; + if (targetResource.type === ResourceType.SECRET) { + const secretTargetResource = targetResource as SecretTargetChange; + differences.push({ + type: ResourceType.SECRET, + id: secretTargetResource.id, + versionId: secretTargetResource.versionId, + changeType: ChangeType.CREATE, + commitId: targetCommit.commitId, + createdAt: targetCommit.createdAt, + secretKey: secretTargetResource.secretKey, + secretVersion: secretTargetResource.secretVersion, + secretId: secretTargetResource.id + }); + } else if (targetResource.type === ResourceType.FOLDER) { + const folderTargetResource = targetResource as FolderTargetChange; + differences.push({ + type: ResourceType.FOLDER, + id: folderTargetResource.id, + versionId: folderTargetResource.versionId, + changeType: ChangeType.CREATE, + commitId: targetCommit.commitId, + createdAt: targetCommit.createdAt, + folderName: folderTargetResource.folderName, + folderVersion: folderTargetResource.folderVersion + }); + } + } + }); + + const removeNoChangeUpdate: string[] = []; + + const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId: project.id + }); + + await Promise.all( + differences.map(async (change) => { + if (change.changeType === ChangeType.UPDATE) { + if (change.type === ResourceType.FOLDER && change.folderVersion && change.fromVersion) { + const versions = await folderVersionDAL.find({ + folderId: change.id, + $in: { + version: [Number(change.folderVersion), Number(change.fromVersion)] + } + }); + const versionsShaped = versions.map((version) => ({ + name: version.name, + description: version.description + })); + const uniqueVersions = versionsShaped.filter( + (item, index, arr) => + arr.findIndex((other) => + Object.entries(item).every( + ([key, value]) => JSON.stringify(value) === JSON.stringify(other[key as keyof typeof other]) + ) + ) === index + ); + if (uniqueVersions.length === 1) { + removeNoChangeUpdate.push(change.id); + } + } else if (change.type === ResourceType.SECRET && change.secretVersion && change.fromVersion) { + const versions = await secretVersionV2BridgeDAL.findVersionsBySecretIdWithActors({ + secretId: change.id, + projectId: project.id, + secretVersions: [change.secretVersion, change.fromVersion] + }); + const versionsShaped = versions.map((el) => ({ + secretKey: el.key, + secretComment: el.encryptedComment + ? secretManagerDecryptor({ cipherTextBlob: el.encryptedComment }).toString() + : "", + skipMultilineEncoding: el.skipMultilineEncoding, + secretReminderRepeatDays: el.reminderRepeatDays, + tags: el.tags, + metadata: el.metadata, + secretReminderNote: el.reminderNote, + secretValue: el.encryptedValue + ? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString() + : "" + })); + const uniqueVersions = versionsShaped.filter( + (item, index, arr) => + arr.findIndex((other) => + Object.entries(item).every( + ([key, value]) => JSON.stringify(value) === JSON.stringify(other[key as keyof typeof other]) + ) + ) === index + ); + if (uniqueVersions.length === 1) { + removeNoChangeUpdate.push(change.id); + } + } + } + }) + ); + return differences.filter((change) => !removeNoChangeUpdate.includes(change.id)); + }; + + /** + * Adds a change to an existing commit + */ + const addCommitChange = async (data: TCommitChangeDTO, tx?: Knex) => { + try { + if (!data.secretVersionId && !data.folderVersionId) { + throw new BadRequestError({ message: "Either secretVersionId or folderVersionId must be provided" }); + } + + const commit = await folderCommitDAL.findById(data.folderCommitId, tx); + if (!commit) { + throw new NotFoundError({ message: `Commit with ID ${data.folderCommitId} not found` }); + } + + return await folderCommitChangesDAL.create(data, tx); + } catch (error) { + if (error instanceof NotFoundError || error instanceof BadRequestError) { + throw error; + } + throw new DatabaseError({ error, name: "AddCommitChange" }); + } + }; + + const createDeleteCommitForNestedFolders = async ({ + folderId, + actorMetadata, + actorType, + envId, + parentFolderName, + step = 1, + tx + }: { + folderId: string; + actorMetadata: Record; + actorType: string; + envId: string; + parentFolderName: string; + step?: number; + tx?: Knex; + }) => { + if (step > 20) { + logger.info(`createDeleteCommitForNestedFolders - Max step reached for folder ${folderId}`); + return; + } + logger.info(`Creating delete commit for nested folders ${folderId}`); + const folderVersion = await folderVersionDAL.findLatestVersion(folderId, tx); + if (!folderVersion) { + logger.info(`No folder version found for ${folderId}`); + return; + } + const lastFolderCommit = await folderCommitDAL.findLatestCommit(folderId, undefined, tx); + if (!lastFolderCommit) { + logger.info(`No commit found for folder ${folderId}`); + return; + } + const folderState = await reconstructFolderState(lastFolderCommit.id, tx); + const changes = folderState.map((resource) => ({ + type: ChangeType.DELETE, + folderId: resource.id, + folderName: resource.type === ResourceType.FOLDER ? resource.folderName : undefined, + secretVersionId: resource.type === ResourceType.SECRET ? resource.versionId : undefined, + folderVersionId: resource.type === ResourceType.FOLDER ? resource.versionId : undefined, + secretKey: resource.type === ResourceType.SECRET ? resource.secretKey : undefined + })); + logger.info(`Found ${changes.length} changes for ${folderId}`); + + const newCommit = await folderCommitDAL.create( + { + actorMetadata, + actorType, + message: `Parent folder ${parentFolderName} deleted`, + folderId, + envId + }, + tx + ); + + const batchSize = 500; + const chunks = chunkArray(changes, batchSize); + + await Promise.all( + chunks.map(async (chunk) => { + await folderCommitChangesDAL.insertMany( + chunk.map((change) => ({ + folderCommitId: newCommit.id, + changeType: CommitType.DELETE, + secretVersionId: change.secretVersionId, + folderVersionId: change.folderVersionId, + isUpdate: false + })), + tx + ); + }) + ); + + await Promise.all( + changes + .filter((change) => change.type === ChangeType.DELETE && change.folderVersionId) + .map(async (change) => { + await createDeleteCommitForNestedFolders({ + folderId: change.folderId, + actorMetadata, + actorType, + envId, + parentFolderName: folderVersion.name, + step: step + 1, + tx + }); + }) + ); + }; + + const compareSecretVersions = async ( + version1: TSecretVersionsV2 & { tags: { id: string }[] }, + version2: TSecretVersionsV2 & { tags: { id: string }[] }, + projectId: string + ) => { + const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + const objectsEqual = (o1: unknown, o2: unknown): boolean => { + if (typeof o1 !== "object" || o1 === null || typeof o2 !== "object" || o2 === null) { + return o1 === o2; + } + + const obj1 = o1 as Record; + const obj2 = o2 as Record; + return ( + Object.keys(obj1).length === Object.keys(obj2).length && Object.keys(obj1).every((p) => obj1[p] === obj2[p]) + ); + }; + + const arraysEqual = (a1: unknown[], a2: unknown[]) => + a1.length === a2.length && a1.every((obj1) => a2.some((obj2) => objectsEqual(obj1, obj2))); + + const version1Reshaped = { + ...version1, + encryptedValue: version1.encryptedValue + ? secretManagerDecryptor({ cipherTextBlob: version1.encryptedValue }).toString() + : "", + encryptedComment: version1.encryptedComment + ? secretManagerDecryptor({ cipherTextBlob: version1.encryptedComment }).toString() + : "", + metadata: Array.isArray(version1.metadata) ? (version1.metadata as { key: string; value: string }[]) : [], + tags: version1.tags.map((tag) => tag.id) + }; + const version2Reshaped = { + ...version2, + encryptedValue: version2.encryptedValue + ? secretManagerDecryptor({ cipherTextBlob: version2.encryptedValue }).toString() + : "", + encryptedComment: version2.encryptedComment + ? secretManagerDecryptor({ cipherTextBlob: version2.encryptedComment }).toString() + : "", + metadata: Array.isArray(version2.metadata) ? (version2.metadata as { key: string; value: string }[]) : [], + tags: version2.tags.map((tag) => tag.id) + }; + return ( + version1Reshaped.key === version2Reshaped.key && + version1Reshaped.encryptedValue === version2Reshaped.encryptedValue && + version1Reshaped.encryptedComment === version2Reshaped.encryptedComment && + version1Reshaped.skipMultilineEncoding === version2Reshaped.skipMultilineEncoding && + arraysEqual(version1Reshaped.metadata, version2Reshaped.metadata) && + version1Reshaped.tags.length === version2Reshaped.tags.length && + version1Reshaped.tags.every((tag) => version2Reshaped.tags.includes(tag)) + ); + }; + + const filterIgnoredChanges = async ( + changes: { + type: string; + secretVersionId?: string; + folderVersionId?: string; + isUpdate?: boolean; + folderId?: string; + }[], + projectId: string, + tx?: Knex + ) => { + let filteredChanges = [...changes]; + for (const change of changes) { + if (change.type === ChangeType.ADD && change.isUpdate && change.secretVersionId) { + const secretVersions = await secretVersionV2BridgeDAL.findByIdAndPreviousVersion(change.secretVersionId, tx); + const comparison = await compareSecretVersions(secretVersions[0], secretVersions[1], projectId); + if (comparison) { + filteredChanges = filteredChanges.filter( + (filteredChange) => filteredChange.secretVersionId !== change.secretVersionId + ); + } + } + } + return filteredChanges; + }; + + /** + * Creates a new commit with the provided changes + */ + const createCommit = async (data: TCreateCommitDTO, tx?: Knex) => { + try { + const metadata = { ...data.actor.metadata } || {}; + + if (data.actor.type === ActorType.USER && data.actor.metadata?.id) { + const user = await userDAL.findById(data.actor.metadata?.id, tx); + metadata.name = user?.username; + } + + if (data.actor.type === ActorType.IDENTITY && data.actor.metadata?.id) { + const identity = await identityDAL.findById(data.actor.metadata?.id, tx); + metadata.name = identity?.name; + } + + const folder = await folderDAL.findById(data.folderId, tx); + if (!folder) { + throw new NotFoundError({ message: `Folder with ID ${data.folderId} not found` }); + } + + let { changes } = data; + if (!data.omitIgnoreFilter) { + const project = await projectDAL.findProjectByEnvId(folder.envId, tx); + + if (!project) { + return; + } + + changes = await filterIgnoredChanges(data.changes, project.id, tx); + if (changes.length === 0) { + return; + } + } + + const newCommit = await folderCommitDAL.create( + { + actorMetadata: metadata, + actorType: data.actor.type, + message: data.message, + folderId: data.folderId, + envId: folder.envId + }, + tx + ); + + const batchSize = 500; + const chunks = chunkArray(changes, batchSize); + + await Promise.all( + chunks.map(async (chunk) => { + await folderCommitChangesDAL.insertMany( + chunk.map((change) => ({ + folderCommitId: newCommit.id, + changeType: change.type, + secretVersionId: change.secretVersionId, + folderVersionId: change.folderVersionId, + isUpdate: change.isUpdate || false + })), + tx + ); + }) + ); + + await Promise.all( + changes.map(async (change) => { + if (change.type === ChangeType.DELETE && change.folderId) { + await createDeleteCommitForNestedFolders({ + folderId: change.folderId, + actorMetadata: metadata, + actorType: data.actor.type, + envId: folder.envId, + parentFolderName: folder.name, + tx + }); + } + }) + ); + + await createFolderCheckpoint({ folderId: data.folderId, folderCommitId: newCommit.id, tx }); + if (folderCommitQueueService) { + if (!folder.parentId) { + const previousTreeCommit = await folderTreeCheckpointDAL.findLatestByEnvId(folder.envId); + if (!previousTreeCommit) { + await folderCommitQueueService.createFolderTreeCheckpoint(folder.envId, newCommit.id, tx); + } + } + await folderCommitQueueService.scheduleTreeCheckpoint(folder.envId); + } + return newCommit; + } catch (error) { + if (error instanceof NotFoundError || error instanceof BadRequestError) { + throw error; + } + throw new DatabaseError({ error, name: "CreateCommit" }); + } + }; + + /** + * Process secret changes when applying folder state differences + */ + const processSecretChanges = async ( + changes: ResourceChange[], + secretVersions: Record, + actorInfo: ActorInfo, + folderId: string, + tx?: Knex + ) => { + const commitChanges = []; + const folder = await folderDAL.findById(folderId, tx); + if (!folder) { + return []; + } + const project = await projectDAL.findById(folder.projectId, tx); + + // Filter only secret changes using discriminated union + const secretChanges = changes.filter( + (change): change is ResourceChange & SecretChange => change.type === ResourceType.SECRET + ); + + // Collect all secretIds for batch lookup + const secretIds = secretChanges.map((change) => secretVersions[change.id]?.secretId).filter(Boolean); + + // Fetch all latest versions in one call + const latestVersionsMap = await secretVersionV2BridgeDAL.findLatestVersionMany(folderId, secretIds, tx); + + for (const change of secretChanges) { + const secretVersion = secretVersions[change.id]; + // eslint-disable-next-line no-continue + if (!secretVersion) continue; + + // Get the latest version from our batch result + const latestVersion = latestVersionsMap[secretVersion.secretId]; + const nextVersion = latestVersion ? latestVersion.version + 1 : 1; + + switch (change.changeType) { + case "create": + { + const newSecret = [ + { + id: change.id, + skipMultilineEncoding: secretVersion.skipMultilineEncoding, + version: nextVersion, + type: secretVersion.type, + key: secretVersion.key, + reminderNote: secretVersion.reminderNote, + reminderRepeatDays: secretVersion.reminderRepeatDays, + encryptedValue: secretVersion.encryptedValue, + encryptedComment: secretVersion.encryptedComment, + userId: secretVersion.userId, + folderId + } + ]; + await secretV2BridgeDAL.insertMany(newSecret, tx); + + const metadata: { key: string; value: string }[] = + (secretVersion.metadata as { key: string; value: string }[]) || []; + if (metadata.length > 0) { + await resourceMetadataDAL.insertMany( + metadata.map(({ key, value }) => ({ + key, + value, + secretId: change.id, + orgId: project.orgId + })), + tx + ); + } + + const newVersion = await secretVersionV2BridgeDAL.create( + { + folderId, + secretId: secretVersion.secretId, + version: nextVersion, + encryptedValue: secretVersion.encryptedValue, + key: secretVersion.key, + encryptedComment: secretVersion.encryptedComment, + skipMultilineEncoding: secretVersion.skipMultilineEncoding, + reminderNote: secretVersion.reminderNote, + reminderRepeatDays: secretVersion.reminderRepeatDays, + userId: secretVersion.userId, + actorType: actorInfo.actorType, + envId: secretVersion.envId, + metadata: JSON.stringify(metadata), + ...(actorInfo.actorType === ActorType.IDENTITY && { identityActorId: actorInfo.actorId }), + ...(actorInfo.actorType === ActorType.USER && { userActorId: actorInfo.actorId }) + }, + tx + ); + + const secretTagsToBeInsert: TSecretV2TagJunctionInsert[] = []; + const secretTags = await secretTagDAL.findSecretTagsByVersionId(secretVersion.id, tx); + secretTags.forEach((tag) => { + secretTagsToBeInsert.push({ secrets_v2Id: change.id, secret_tagsId: tag.secret_tagsId }); + }); + await secretTagDAL.saveTagsToSecretV2(secretTagsToBeInsert, tx); + await secretTagDAL.saveTagsToSecretVersionV2( + secretTagsToBeInsert.map((tag) => ({ + secret_tagsId: tag.secret_tagsId, + secret_versions_v2Id: newVersion.id + })), + tx + ); + + commitChanges.push({ + type: ChangeType.ADD, + secretVersionId: newVersion.id + }); + } + break; + + case "update": + { + await secretV2BridgeDAL.updateById( + change.id, + { + skipMultilineEncoding: secretVersion?.skipMultilineEncoding, + version: nextVersion, + type: secretVersion?.type, + key: secretVersion?.key, + reminderNote: secretVersion?.reminderNote, + reminderRepeatDays: secretVersion?.reminderRepeatDays, + encryptedValue: secretVersion?.encryptedValue, + encryptedComment: secretVersion?.encryptedComment, + userId: secretVersion?.userId + }, + tx + ); + + const metadata: { key: string; value: string }[] = + (secretVersion.metadata as { key: string; value: string }[]) || []; + await resourceMetadataDAL.delete({ secretId: change.id }, tx); + if (metadata.length > 0) { + await resourceMetadataDAL.insertMany( + metadata.map(({ key, value }) => ({ + key, + value, + secretId: change.id, + orgId: project.orgId + })), + tx + ); + } + + const newVersion = await secretVersionV2BridgeDAL.create( + { + version: nextVersion, + encryptedValue: secretVersion.encryptedValue, + key: secretVersion.key, + encryptedComment: secretVersion.encryptedComment, + skipMultilineEncoding: secretVersion.skipMultilineEncoding, + reminderNote: secretVersion.reminderNote, + reminderRepeatDays: secretVersion.reminderRepeatDays, + userId: secretVersion.userId, + metadata: JSON.stringify(metadata), + actorType: actorInfo.actorType, + envId: secretVersion.envId, + folderId, + secretId: secretVersion.secretId, + ...(actorInfo.actorType === ActorType.IDENTITY && { identityActorId: actorInfo.actorId }), + ...(actorInfo.actorType === ActorType.USER && { userActorId: actorInfo.actorId }) + }, + tx + ); + + let secretTagsToBeInsert: TSecretV2TagJunctionInsert[] = []; + const secretTagsToBeDelete: string[] = []; + const secretTags = await secretTagDAL.findSecretTagsByVersionId(secretVersion.id, tx); + secretTags.forEach((tag) => { + secretTagsToBeInsert.push({ secrets_v2Id: change.id, secret_tagsId: tag.secret_tagsId }); + }); + const currentTags = await secretTagDAL.findSecretTagsBySecretId(change.id, tx); + currentTags.forEach((tag) => { + if (!secretTagsToBeInsert.find((t) => t.secret_tagsId === tag.secret_tagsId)) { + secretTagsToBeDelete.push(tag.secret_tagsId); + secretTagsToBeInsert = secretTagsToBeInsert.filter((t) => t.secret_tagsId !== tag.secret_tagsId); + } + }); + await secretTagDAL.saveTagsToSecretV2(secretTagsToBeInsert, tx); + await secretTagDAL.saveTagsToSecretVersionV2( + secretTagsToBeInsert.map((tag) => ({ + secret_tagsId: tag.secret_tagsId, + secret_versions_v2Id: newVersion.id + })), + tx + ); + await secretTagDAL.deleteTagsToSecretV2( + { $in: { secret_tagsId: secretTagsToBeDelete }, secrets_v2Id: change.id }, + tx + ); + + commitChanges.push({ + type: ChangeType.ADD, + isUpdate: true, + secretVersionId: newVersion.id + }); + } + break; + + // Delete case remains unchanged + case "delete": + await secretV2BridgeDAL.deleteById(change.id, tx); + commitChanges.push({ + type: ChangeType.DELETE, + secretVersionId: change.versionId + }); + break; + + default: + throw new BadRequestError({ message: `Unknown change type: ${change.changeType}` }); + } + } + + return commitChanges; + }; + + /** + * Core function to apply folder state differences + */ + const applyFolderStateDifferencesFn = async ({ + differences, + actorInfo, + folderId, + projectId, + reconstructNewFolders, + reconstructUpToCommit, + step = 0, + tx + }: { + differences: ResourceChange[]; + actorInfo: ActorInfo; + folderId: string; + projectId: string; + reconstructNewFolders: boolean; + reconstructUpToCommit?: string; + step: number; + tx?: Knex; + }): Promise => { + /** + * Process folder changes when applying folder state differences + */ + const processFolderChanges = async ( + changes: ResourceChange[], + folderVersions: Record + ) => { + const commitChanges = []; + + // Filter only folder changes using discriminated union + const folderChanges = changes.filter( + (change): change is ResourceChange & FolderChange => change.type === ResourceType.FOLDER + ); + + for (const change of folderChanges) { + const folderVersion = folderVersions[change.id]; + + switch (change.changeType) { + case "create": + if (folderVersion) { + const newFolder = { + id: change.id, + parentId: folderId, + envId: folderVersion.envId, + version: (folderVersion.version || 1) + 1, + name: folderVersion.name, + description: folderVersion.description + }; + await folderDAL.create(newFolder, tx); + + const newFolderVersion = await folderVersionDAL.create( + { + folderId: change.id, + version: (folderVersion.version || 1) + 1, + name: folderVersion.name, + description: folderVersion.description, + envId: folderVersion.envId + }, + tx + ); + + if (reconstructNewFolders && reconstructUpToCommit && step < 20) { + const subFolderLatestCommit = await folderCommitDAL.findLatestCommitBetween({ + folderId: change.id, + endCommitId: reconstructUpToCommit, + tx + }); + if (subFolderLatestCommit) { + const subFolderDiff = await compareFolderStates({ + targetCommitId: subFolderLatestCommit.id, + tx + }); + if (subFolderDiff?.length > 0) { + await applyFolderStateDifferencesFn({ + differences: subFolderDiff, + actorInfo, + folderId: change.id, + projectId, + reconstructNewFolders, + reconstructUpToCommit, + step: step + 1, + tx + }); + } + } + } + + commitChanges.push({ + type: ChangeType.ADD, + folderVersionId: newFolderVersion.id + }); + } + break; + + case "update": + if (change.versionId) { + const latestVersionDetails = await folderVersionDAL.findByIdsWithLatestVersion( + [change.id], + [change.versionId], + tx + ); + if (latestVersionDetails && Object.keys(latestVersionDetails).length > 0) { + const versionDetails = Object.values(latestVersionDetails)[0]; + await folderDAL.updateById( + change.id, + { + parentId: folderId, + envId: versionDetails.envId, + version: (versionDetails.version || 1) + 1, + name: versionDetails.name, + description: versionDetails.description + }, + tx + ); + + const newFolderVersion = await folderVersionDAL.create( + { + folderId: change.id, + version: (versionDetails.version || 1) + 1, + name: versionDetails.name, + description: versionDetails.description, + envId: versionDetails.envId + }, + tx + ); + + commitChanges.push({ + type: ChangeType.ADD, + isUpdate: true, + folderVersionId: newFolderVersion.id + }); + } + } + break; + + case "delete": + await folderDAL.deleteById(change.id, tx); + + commitChanges.push({ + type: ChangeType.DELETE, + folderVersionId: change.versionId, + folderId: change.id + }); + break; + + default: + throw new BadRequestError({ message: `Unknown change type: ${change.changeType}` }); + } + } + return commitChanges; + }; + + // Group differences by type for more efficient processing using discriminated unions + const secretChanges = differences.filter( + (diff): diff is ResourceChange & SecretChange => diff.type === ResourceType.SECRET + ); + const folderChanges = differences.filter( + (diff): diff is ResourceChange & FolderChange => diff.type === ResourceType.FOLDER + ); + + // Batch fetch necessary data + const secretVersions = await secretVersionV2BridgeDAL.findByIdsWithLatestVersion( + folderId, + secretChanges.map((diff) => diff.id), + secretChanges.map((diff) => diff.versionId), + tx + ); + + const folderVersions = await folderVersionDAL.findByIdsWithLatestVersion( + folderChanges.map((diff) => diff.id), + folderChanges.map((diff) => diff.versionId), + tx + ); + + // Process changes in parallel + const [secretCommitChanges, folderCommitChanges] = await Promise.all([ + processSecretChanges(differences, secretVersions, actorInfo, folderId, tx), + processFolderChanges(differences, folderVersions) + ]); + + // Combine all changes + const allCommitChanges = [...secretCommitChanges, ...folderCommitChanges]; + + // Create a commit with all the changes + await createCommit( + { + actor: { + type: actorInfo.actorType, + metadata: { id: actorInfo.actorId } + }, + message: actorInfo.message || "Rolled back folder state", + folderId, + changes: allCommitChanges, + omitIgnoreFilter: true + }, + tx + ); + + // Invalidate cache to reflect the changes + await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId); + + return { + secretChangesCount: secretChanges.length, + folderChangesCount: folderChanges.length, + totalChanges: differences.length + }; + }; + + /** + * Apply folder state differences with transaction handling + */ + const applyFolderStateDifferences = async (params: { + differences: ResourceChange[]; + actorInfo: ActorInfo; + folderId: string; + projectId: string; + reconstructNewFolders: boolean; + reconstructUpToCommit?: string; + tx?: Knex; + }): Promise => { + // If a transaction was provided, use it directly + if (params.tx) { + return applyFolderStateDifferencesFn({ ...params, step: 0 }); + } + + // Otherwise, start a new transaction + return folderCommitDAL.transaction((newTx) => applyFolderStateDifferencesFn({ ...params, tx: newTx, step: 0 })); + }; + + /** + * Retrieve a commit by ID + */ + const getCommitById = async ({ + commitId, + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId, + tx + }: { + commitId: string; + actor: ActorType; + actorId: string; + actorAuthMethod: ActorAuthMethod; + actorOrgId: string; + projectId: string; + tx?: Knex; + }) => { + await checkProjectCommitReadPermission({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId + }); + return folderCommitDAL.findById(commitId, tx, projectId); + }; + + /** + * Get all commits for a folder + */ + const getCommitsByFolderId = async (folderId: string, tx?: Knex) => { + return folderCommitDAL.findByFolderId(folderId, tx); + }; + + const getCommitsForFolder = async ({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId, + environment, + path, + offset = 0, + limit = 20, + search, + sort = "desc" + }: { + actor: ActorType; + actorId: string; + actorAuthMethod: ActorAuthMethod; + actorOrgId: string; + projectId: string; + environment: string; + path: string; + offset: number; + limit: number; + search?: string; + sort: "asc" | "desc"; + }) => { + await checkProjectCommitReadPermission({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId + }); + const folder = await folderDAL.findBySecretPath(projectId, environment, path); + if (!folder) { + throw new NotFoundError({ + message: `Folder not found for project ID ${projectId}, environment ${environment}, path ${path}` + }); + } + const folderCommits = await folderCommitDAL.findByFolderIdPaginated(folder.id, { + offset, + limit, + search, + sort + }); + return folderCommits; + }; + + const getCommitsCount = async ({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId, + environment, + path + }: { + actor: ActorType; + actorId: string; + actorAuthMethod: ActorAuthMethod; + actorOrgId: string; + projectId: string; + environment: string; + path: string; + }) => { + await checkProjectCommitReadPermission({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId + }); + + const folder = await folderDAL.findBySecretPath(projectId, environment, path); + if (!folder) { + throw new NotFoundError({ + message: `Folder not found for project ID ${projectId}, environment ${environment}, path ${path}` + }); + } + const folderCommits = await folderCommitDAL.findByFolderId(folder.id); + return { count: folderCommits.length, folderId: folder.id }; + }; + + /** + * Get changes for a commit + */ + const getCommitChanges = async ({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId, + commitId + }: { + actor: ActorType; + actorId: string; + actorAuthMethod: ActorAuthMethod; + actorOrgId: string; + projectId: string; + commitId: string; + }) => { + await checkProjectCommitReadPermission({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId + }); + const changes = await folderCommitChangesDAL.findByCommitId(commitId, projectId); + const commit = await folderCommitDAL.findById(commitId, undefined, projectId); + const latestCommit = await folderCommitDAL.findLatestCommit(commit.folderId, projectId); + return { ...commit, changes, isLatest: commit.id === latestCommit?.id }; + }; + + /** + * Get checkpoints for a folder + */ + const getCheckpointsByFolderId = async (folderId: string, limit?: number, tx?: Knex) => { + return folderCheckpointDAL.findByFolderId(folderId, limit, tx); + }; + + /** + * Get the latest checkpoint for a folder + */ + const getLatestCheckpoint = async (folderId: string, tx?: Knex) => { + return folderCheckpointDAL.findLatestByFolderId(folderId, tx); + }; + + /** + * Initialize a folder with its current state + */ + const getFolderInitialChanges = async (folderId: string, envId: string, tx?: Knex) => { + const folderResources = await getFolderResources(folderId, tx); + const changes = folderResources.map((resource) => ({ type: ChangeType.ADD, ...resource })); + + if (changes.length > 0) { + return { + commit: { + actorMetadata: {}, + actorType: ActorType.PLATFORM, + message: "Initialized folder", + folderId, + envId + }, + changes: changes.map((change) => ({ + folderId, + changeType: change.type, + secretVersionId: change.secretVersionId, + folderVersionId: change.folderVersionId, + isUpdate: false + })) + }; + } + return {}; + }; + + /** + * Sort folders by hierarchy (parents before children) + */ + const sortFoldersByHierarchy = (folders: TSecretFolders[]) => { + // Create a map for quick lookup of children by parent ID + const childrenMap = new Map(); + + // Set of all folder IDs + const allFolderIds = new Set(); + + // Build the set of all folder IDs + folders.forEach((folder) => { + if (folder.id) { + allFolderIds.add(folder.id); + } + }); + + // Group folders by their parentId + folders.forEach((folder) => { + if (folder.parentId) { + const children = childrenMap.get(folder.parentId) || []; + children.push(folder); + childrenMap.set(folder.parentId, children); + } + }); + + // Find root folders - those with no parentId or with a parentId that doesn't exist + const rootFolders = folders.filter((folder) => !folder.parentId || !allFolderIds.has(folder.parentId)); + + // Process each level of the hierarchy + const result = []; + let currentLevel = rootFolders; + + while (currentLevel.length > 0) { + result.push(...currentLevel); + + const nextLevel = []; + for (const folder of currentLevel) { + if (folder.id) { + const children = childrenMap.get(folder.id) || []; + nextLevel.push(...children); + } + } + + currentLevel = nextLevel; + } + + return result; + }; + + /** + * Create a checkpoint for a folder tree + */ + const createFolderTreeCheckpoint = async (envId: string, folderCommitId?: string, tx?: Knex) => { + let latestCommitId = folderCommitId; + const latestTreeCheckpoint = await folderTreeCheckpointDAL.findLatestByEnvId(envId, tx); + + if (!latestCommitId) { + const latestCommit = await folderCommitDAL.findLatestEnvCommit(envId, tx); + if (!latestCommit) { + logger.info(`createFolderTreeCheckpoint - Latest commit ID not found for envId ${envId}`); + return; + } + latestCommitId = latestCommit.id; + } + + if (latestTreeCheckpoint) { + const commitsSinceLastCheckpoint = await folderCommitDAL.getEnvNumberOfCommitsSince( + envId, + latestTreeCheckpoint.folderCommitId, + tx + ); + if (commitsSinceLastCheckpoint < Number(appCfg.PIT_TREE_CHECKPOINT_WINDOW)) { + logger.info( + `createFolderTreeCheckpoint - Commits since last checkpoint ${commitsSinceLastCheckpoint} is less than ${appCfg.PIT_TREE_CHECKPOINT_WINDOW}` + ); + return; + } + } + + const folders = await folderDAL.findByEnvId(envId, tx); + const sortedFolders = sortFoldersByHierarchy(folders); + const filteredFoldersIds = sortedFolders.filter((folder) => !folder.isReserved).map((folder) => folder.id); + const folderCommits = await folderCommitDAL.findMultipleLatestCommits(filteredFoldersIds, tx); + const folderTreeCheckpoint = await folderTreeCheckpointDAL.create( + { + folderCommitId: latestCommitId + }, + tx + ); + await folderTreeCheckpointResourcesDAL.insertMany( + folderCommits.map((folderCommit) => ({ + folderTreeCheckpointId: folderTreeCheckpoint.id, + folderId: folderCommit.folderId, + folderCommitId: folderCommit.id + })), + tx + ); + }; + + const addNestedFolderChanges = async ({ + changes, + beforeCommit, + folderId, + folderName, + folderPath, + step = 1, + tx + }: { + changes: { + folderId: string; + folderName: string; + changes: ResourceChange[]; + folderPath?: string; + }[]; + beforeCommit: bigint; + folderId: string; + folderName?: string; + folderPath?: string; + step?: number; + tx?: Knex; + }) => { + if (step > 20) { + return; + } + const latestFolderCommit = await folderCommitDAL.findCommitBefore(folderId, beforeCommit, tx); + if (!latestFolderCommit) { + return; + } + const diff = await compareFolderStates({ + targetCommitId: latestFolderCommit.id, + tx + }); + changes.push({ + folderId, + folderName: folderName || "", + changes: diff, + folderPath: folderPath || "" + }); + await Promise.all( + diff.map(async (change) => { + if (change.type === ResourceType.FOLDER && change.changeType === ChangeType.CREATE) { + await addNestedFolderChanges({ + changes, + beforeCommit, + folderId: change.id, + folderName: change.folderName, + folderPath: `${folderPath}/${change.folderName}`, + step: step + 1, + tx + }); + } + }) + ); + }; + + const deepCompareFolder = async ({ + targetCommitId, + envId, + projectId, + tx + }: { + targetCommitId: string; + envId: string; + projectId: string; + tx?: Knex; + }) => { + const targetCommit = await folderCommitDAL.findById(targetCommitId, tx); + if (!targetCommit) { + throw new NotFoundError({ message: `No commit found for commit ID ${targetCommitId}` }); + } + + const checkpoint = await folderTreeCheckpointDAL.findNearestCheckpoint(targetCommit.commitId, envId, tx); + if (!checkpoint) { + throw new NotFoundError({ message: `No checkpoint found for commit ID ${targetCommitId}` }); + } + + const folderCheckpointCommits = await folderTreeCheckpointResourcesDAL.findByTreeCheckpointId(checkpoint.id, tx); + const folderCommits = await folderCommitDAL.findAllCommitsBetween({ + envId, + startCommitId: checkpoint.commitId.toString(), + tx + }); + + // Group commits by folderId and keep only the latest + const folderGroups = new Map(); + + if (folderCheckpointCommits && folderCheckpointCommits.length > 0) { + for (const commit of folderCheckpointCommits) { + if (commit.commitId > targetCommit.commitId) { + folderGroups.set(commit.folderId, { + commitId: commit.commitId, + id: commit.folderCommitId + }); + } + } + } + + if (folderCommits && folderCommits.length > 0) { + for (const commit of folderCommits) { + const { folderId, commitId, id } = commit; + const existingCommit = folderGroups.get(folderId); + + if ((!existingCommit || commitId > existingCommit.commitId) && commitId > targetCommit.commitId) { + folderGroups.set(folderId, { commitId, id }); + } + } + } + + const folderDiffs = new Map(); + + // Process each folder to determine differences + await Promise.all( + Array.from(folderGroups.entries()).map(async ([folderId, commit]) => { + const previousCommit = await folderCommitDAL.findPreviousCommitTo( + folderId, + targetCommit.commitId.toString(), + tx + ); + let diff = []; + if (previousCommit && previousCommit.id !== commit.id) { + diff = await compareFolderStates({ + currentCommitId: commit.id, + targetCommitId: previousCommit.id, + tx + }); + } else { + diff = await compareFolderStates({ + targetCommitId: commit.id, + defaultOperation: "delete", + tx + }); + } + if (diff?.length > 0) { + folderDiffs.set(folderId, diff); + } + }) + ); + + // Apply changes in hierarchical order + const folderIds = Array.from(folderDiffs.keys()); + const folders = await folderDAL.findFoldersByRootAndIds({ rootId: targetCommit.folderId, folderIds }, tx); + const sortedFolders = sortFoldersByHierarchy(folders); + + const response: { + folderId: string; + folderName: string; + changes: ResourceChange[]; + folderPath?: string; + }[] = []; + for (const folder of sortedFolders) { + const diff = folderDiffs.get(folder.id); + if (diff) { + const folderPath = await folderDAL.findSecretPathByFolderIds(projectId, [folder.id]); + response.push({ + folderId: folder.id, + folderName: folder.name, + changes: diff, + folderPath: folderPath?.[0]?.path + }); + const recreatedFolders = diff + .filter( + (change): change is FolderChange => + change.type === ResourceType.FOLDER && change.changeType === ChangeType.CREATE + ) + .map((change) => ({ + id: change.id, + folderName: change.folderName, + folderPath: folderPath?.[0]?.path + })); + await Promise.all( + recreatedFolders.map(async (change) => { + const nestedFolderPath = folderPath?.[0]?.path; + await addNestedFolderChanges({ + changes: response, + beforeCommit: targetCommit.commitId, + folderId: change.id, + folderName: change.folderName, + folderPath: `${nestedFolderPath !== "/" ? nestedFolderPath : ""}/${change.folderName}`, + tx + }); + }) + ); + } + } + return response; + }; + + /** + * Roll back a folder tree to a specific commit + */ + const deepRollbackFolder = async ( + targetCommitId: string, + envId: string, + actorId: string, + actorType: ActorType, + projectId: string, + message?: string + ) => { + await folderCommitDAL.transaction(async (tx) => { + const targetCommit = await folderCommitDAL.findById(targetCommitId, tx); + if (!targetCommit) { + throw new NotFoundError({ message: `No commit found for commit ID ${targetCommitId}` }); + } + + const checkpoint = await folderTreeCheckpointDAL.findNearestCheckpoint(targetCommit.commitId, envId, tx); + if (!checkpoint) { + throw new NotFoundError({ message: `No checkpoint found for commit ID ${targetCommitId}` }); + } + + const folderCheckpointCommits = await folderTreeCheckpointResourcesDAL.findByTreeCheckpointId(checkpoint.id, tx); + const folderCommits = await folderCommitDAL.findAllCommitsBetween({ + envId, + startCommitId: checkpoint.commitId.toString(), + tx + }); + + // Group commits by folderId and keep only the latest + const folderGroups = new Map(); + + if (folderCheckpointCommits && folderCheckpointCommits.length > 0) { + for (const commit of folderCheckpointCommits) { + if (commit.commitId > targetCommit.commitId) { + folderGroups.set(commit.folderId, { + commitId: commit.commitId, + id: commit.folderCommitId + }); + } + } + } + + if (folderCommits && folderCommits.length > 0) { + for (const commit of folderCommits) { + const { folderId, commitId, id } = commit; + const existingCommit = folderGroups.get(folderId); + + if ((!existingCommit || commitId > existingCommit.commitId) && commitId > targetCommit.commitId) { + folderGroups.set(folderId, { commitId, id }); + } + } + } + + const folderDiffs = new Map(); + + // Process each folder to determine differences + await Promise.all( + Array.from(folderGroups.entries()).map(async ([folderId, { id }]) => { + const previousCommit = await folderCommitDAL.findPreviousCommitTo( + folderId, + targetCommit.commitId.toString(), + tx + ); + if (previousCommit && previousCommit.id !== id) { + const diff = await compareFolderStates({ + currentCommitId: id, + targetCommitId: previousCommit.id, + tx + }); + if (diff?.length > 0) { + folderDiffs.set(folderId, diff); + } + } + }) + ); + + const foldersToDelete = new Set(); + + // Process all DELETE operations to build a complete set of folders to be deleted + for (const changes of folderDiffs.values()) { + for (const change of changes) { + if (change.changeType === ChangeType.DELETE && change.type === ResourceType.FOLDER) { + foldersToDelete.add(change.id); + } + } + } + + // Now, remove any folder that is being deleted from the folderDiffs map + // before applying any changes + for (const folderId of foldersToDelete) { + folderDiffs.delete(folderId); + } + + // Apply changes in hierarchical order + const folderIds = Array.from(folderDiffs.keys()); + const folders = await folderDAL.findFoldersByRootAndIds({ rootId: targetCommit.folderId, folderIds }, tx); + const sortedFolders = sortFoldersByHierarchy(folders); + + for (const folder of sortedFolders) { + const diff = folderDiffs.get(folder.id); + if (diff) { + await applyFolderStateDifferences({ + differences: diff, + actorInfo: { + actorType, + actorId, + message: message || "Deep rollback" + }, + folderId: folder.id, + projectId, + reconstructNewFolders: true, + reconstructUpToCommit: targetCommit.commitId.toString(), + tx + }); + } + } + }); + }; + + const getLatestCommit = async ({ + folderId, + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId + }: { + folderId: string; + actor: ActorType; + actorId: string; + actorAuthMethod: ActorAuthMethod; + actorOrgId: string; + projectId: string; + }) => { + await checkProjectCommitReadPermission({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId + }); + return folderCommitDAL.findLatestCommit(folderId, projectId); + }; + + /** + * Revert changes made in a specific commit + */ + const revertCommitChanges = async ({ + commitId, + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId, + message = "Revert commit changes" + }: { + commitId: string; + actor: ActorType; + actorId: string; + actorAuthMethod: ActorAuthMethod; + actorOrgId: string; + projectId: string; + message?: string; + }) => { + if (!permissionService) { + throw new Error("Permission service not initialized"); + } + const { permission } = await permissionService.getProjectPermission({ + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId, + actionProjectType: ActionProjectType.SecretManager + }); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionCommitsActions.PerformRollback, + ProjectPermissionSub.Commits + ); + // Check permissions first + await checkProjectCommitReadPermission({ + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + }); + + // Get the commit to revert + const commitToRevert = await folderCommitDAL.findById(commitId, undefined, projectId); + if (!commitToRevert) { + throw new NotFoundError({ message: `Commit with ID ${commitId} not found` }); + } + + const previousCommit = await folderCommitDAL.findCommitBefore(commitToRevert.folderId, commitToRevert.commitId); + + if (!previousCommit) { + throw new BadRequestError({ message: "Cannot revert the first commit" }); + } + + // Calculate the changes needed to go from current commit back to the previous one + const inverseChanges = await compareFolderStates({ + currentCommitId: commitToRevert.id, + targetCommitId: previousCommit.id + }); + + const latestCommit = await folderCommitDAL.findLatestCommit(commitToRevert.folderId); + if (!latestCommit) { + throw new NotFoundError({ message: `Latest commit not found for folder ${commitToRevert.folderId}` }); + } + const currentState = await reconstructFolderState(latestCommit.id); + + const filteredChanges = inverseChanges.filter( + (change) => + ((change.changeType === ChangeType.DELETE || change.changeType === ChangeType.UPDATE) && + (currentState.some((c) => c.id === change.id) || currentState.some((c) => c.id === change.id))) || + (change.changeType === ChangeType.CREATE && + (currentState.every((c) => c.id !== change.id) || currentState.every((c) => c.id !== change.id))) + ); + + if (!filteredChanges || filteredChanges.length === 0) { + return { + success: true, + message: "No changes to revert", + originalCommitId: commitId + }; + } + + // Apply the changes to revert the commit + const revertResult = await applyFolderStateDifferences({ + differences: filteredChanges, + actorInfo: { + actorType: actor, + actorId, + message: message || `Reverted changes from commit ${commitId}` + }, + folderId: commitToRevert.folderId, + projectId, + reconstructNewFolders: true, + reconstructUpToCommit: commitToRevert.commitId.toString() + }); + + return { + success: true, + message: "Changes reverted successfully", + originalCommitId: commitId, + revertCommitId: latestCommit?.id, + changesReverted: revertResult.totalChanges + }; + }; + + return { + createCommit, + addCommitChange, + getCommitById, + getCommitsByFolderId, + getCommitChanges, + getCheckpointsByFolderId, + getLatestCheckpoint, + getFolderInitialChanges, + createFolderCheckpoint, + compareFolderStates, + applyFolderStateDifferences, + createFolderTreeCheckpoint, + deepRollbackFolder, + getCommitsCount, + getLatestCommit, + deepCompareFolder, + reconstructFolderState, + getCommitsForFolder, + revertCommitChanges + }; +}; + +export type TFolderCommitServiceFactory = ReturnType; diff --git a/backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts b/backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts new file mode 100644 index 000000000..58651a48a --- /dev/null +++ b/backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts @@ -0,0 +1,44 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { TableName, TFolderTreeCheckpointResources } from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TFolderTreeCheckpointResourcesDALFactory = ReturnType; + +type TFolderTreeCheckpointResourcesWithCommitId = TFolderTreeCheckpointResources & { + commitId: bigint; +}; + +export const folderTreeCheckpointResourcesDALFactory = (db: TDbClient) => { + const folderTreeCheckpointResourcesOrm = ormify(db, TableName.FolderTreeCheckpointResources); + + const findByTreeCheckpointId = async ( + folderTreeCheckpointId: string, + tx?: Knex + ): Promise => { + try { + const docs = await (tx || db.replicaNode())( + TableName.FolderTreeCheckpointResources + ) + .join( + TableName.FolderCommit, + `${TableName.FolderTreeCheckpointResources}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ folderTreeCheckpointId }, TableName.FolderTreeCheckpointResources)) + .select(selectAllTableCols(TableName.FolderTreeCheckpointResources)) + .select(db.ref("commitId").withSchema(TableName.FolderCommit).as("commitId")); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindByTreeCheckpointId" }); + } + }; + + return { + ...folderTreeCheckpointResourcesOrm, + findByTreeCheckpointId + }; +}; diff --git a/backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts b/backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts new file mode 100644 index 000000000..cc0634e74 --- /dev/null +++ b/backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts @@ -0,0 +1,79 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { TableName, TFolderCommits, TFolderTreeCheckpoints } from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TFolderTreeCheckpointDALFactory = ReturnType; + +type TreeCheckpointWithCommitInfo = TFolderTreeCheckpoints & { + commitId: bigint; +}; + +export const folderTreeCheckpointDALFactory = (db: TDbClient) => { + const folderTreeCheckpointOrm = ormify(db, TableName.FolderTreeCheckpoint); + + const findByCommitId = async (folderCommitId: string, tx?: Knex): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderTreeCheckpoint) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ folderCommitId }, TableName.FolderTreeCheckpoint)) + .select(selectAllTableCols(TableName.FolderTreeCheckpoint)) + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindByCommitId" }); + } + }; + + const findNearestCheckpoint = async ( + folderCommitId: bigint, + envId: string, + tx?: Knex + ): Promise => { + try { + const nearestCheckpoint = await (tx || db.replicaNode())(TableName.FolderTreeCheckpoint) + .join( + TableName.FolderCommit, + `${TableName.FolderTreeCheckpoint}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(`${TableName.FolderCommit}.envId`, "=", envId) + .andWhere(`${TableName.FolderCommit}.commitId`, "<=", folderCommitId.toString()) + .select(selectAllTableCols(TableName.FolderTreeCheckpoint)) + .select(db.ref("commitId").withSchema(TableName.FolderCommit)) + .orderBy(`${TableName.FolderCommit}.commitId`, "desc") + .first(); + + return nearestCheckpoint; + } catch (error) { + throw new DatabaseError({ error, name: "FindNearestCheckpoint" }); + } + }; + + const findLatestByEnvId = async (envId: string, tx?: Knex): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderTreeCheckpoint) + .join( + TableName.FolderCommit, + `${TableName.FolderTreeCheckpoint}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) + .where(`${TableName.FolderCommit}.envId`, "=", envId) + .orderBy(`${TableName.FolderTreeCheckpoint}.createdAt`, "desc") + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestByEnvId" }); + } + }; + + return { + ...folderTreeCheckpointOrm, + findByCommitId, + findNearestCheckpoint, + findLatestByEnvId + }; +}; diff --git a/backend/src/services/identity-kubernetes-auth/identity-kubernetes-auth-service.ts b/backend/src/services/identity-kubernetes-auth/identity-kubernetes-auth-service.ts index a1231c353..1f89745a9 100644 --- a/backend/src/services/identity-kubernetes-auth/identity-kubernetes-auth-service.ts +++ b/backend/src/services/identity-kubernetes-auth/identity-kubernetes-auth-service.ts @@ -72,8 +72,8 @@ export const identityKubernetesAuthServiceFactory = ({ const $gatewayProxyWrapper = async ( inputs: { gatewayId: string; - targetHost: string; - targetPort: number; + targetHost?: string; + targetPort?: number; caCert?: string; reviewTokenThroughGateway: boolean; }, @@ -104,11 +104,15 @@ export const identityKubernetesAuthServiceFactory = ({ cert: relayDetails.certificate, key: relayDetails.privateKey.toString() }, - // we always pass this, because its needed for both tcp and http protocol - httpsAgent: new https.Agent({ - ca: inputs.caCert, - rejectUnauthorized: Boolean(inputs.caCert) - }) + // only needed for TCP protocol, because the gateway as reviewer will use the pod's CA cert for auth directly + ...(!inputs.reviewTokenThroughGateway + ? { + httpsAgent: new https.Agent({ + ca: inputs.caCert, + rejectUnauthorized: Boolean(inputs.caCert) + }) + } + : {}) } ); @@ -142,8 +146,15 @@ export const identityKubernetesAuthServiceFactory = ({ caCert = decryptor({ cipherTextBlob: identityKubernetesAuth.encryptedKubernetesCaCertificate }).toString(); } - const tokenReviewCallbackRaw = async (host: string = identityKubernetesAuth.kubernetesHost, port?: number) => { + const tokenReviewCallbackRaw = async (host = identityKubernetesAuth.kubernetesHost, port?: number) => { logger.info({ host, port }, "tokenReviewCallbackRaw: Processing kubernetes token review using raw API"); + + if (!host || !identityKubernetesAuth.kubernetesHost) { + throw new BadRequestError({ + message: "Kubernetes host is required when token review mode is set to API" + }); + } + let tokenReviewerJwt = ""; if (identityKubernetesAuth.encryptedKubernetesTokenReviewerJwt) { tokenReviewerJwt = decryptor({ @@ -211,11 +222,7 @@ export const identityKubernetesAuthServiceFactory = ({ return res.data; }; - const tokenReviewCallbackThroughGateway = async ( - host: string = identityKubernetesAuth.kubernetesHost, - port?: number, - httpsAgent?: https.Agent - ) => { + const tokenReviewCallbackThroughGateway = async (host: string, port?: number) => { logger.info( { host, @@ -224,11 +231,9 @@ export const identityKubernetesAuthServiceFactory = ({ "tokenReviewCallbackThroughGateway: Processing kubernetes token review using gateway" ); - const baseUrl = port ? `${host}:${port}` : host; - const res = await axios .post( - `${baseUrl}/apis/authentication.k8s.io/v1/tokenreviews`, + `${host}:${port}/apis/authentication.k8s.io/v1/tokenreviews`, { apiVersion: "authentication.k8s.io/v1", kind: "TokenReview", @@ -240,11 +245,10 @@ export const identityKubernetesAuthServiceFactory = ({ { headers: { "Content-Type": "application/json", - "x-infisical-action": GatewayHttpProxyActions.InjectGatewayK8sServiceAccountToken + "x-infisical-action": GatewayHttpProxyActions.UseGatewayK8sServiceAccount }, signal: AbortSignal.timeout(10000), - timeout: 10000, - ...(httpsAgent ? { httpsAgent } : {}) + timeout: 10000 } ) .catch((err) => { @@ -273,29 +277,6 @@ export const identityKubernetesAuthServiceFactory = ({ let data: TCreateTokenReviewResponse | undefined; if (identityKubernetesAuth.tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Gateway) { - const { kubernetesHost } = identityKubernetesAuth; - - let urlString = kubernetesHost; - if (!kubernetesHost.startsWith("http://") && !kubernetesHost.startsWith("https://")) { - urlString = `https://${kubernetesHost}`; - } - - const url = new URL(urlString); - let { port: k8sPort } = url; - const { protocol, hostname: k8sHost } = url; - - const cleanedProtocol = new RE2(/[^a-zA-Z0-9]/g).replace(protocol, "").toLowerCase(); - - if (!["https", "http"].includes(cleanedProtocol)) { - throw new BadRequestError({ - message: "Invalid Kubernetes host URL, must start with http:// or https://" - }); - } - - if (!k8sPort) { - k8sPort = cleanedProtocol === "https" ? "443" : "80"; - } - if (!identityKubernetesAuth.gatewayId) { throw new BadRequestError({ message: "Gateway ID is required when token review mode is set to Gateway" @@ -305,14 +286,17 @@ export const identityKubernetesAuthServiceFactory = ({ data = await $gatewayProxyWrapper( { gatewayId: identityKubernetesAuth.gatewayId, - targetHost: `${cleanedProtocol}://${k8sHost}`, // note(daniel): must include the protocol (https|http) - targetPort: k8sPort ? Number(k8sPort) : 443, - caCert, reviewTokenThroughGateway: true }, tokenReviewCallbackThroughGateway ); } else if (identityKubernetesAuth.tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Api) { + if (!identityKubernetesAuth.kubernetesHost) { + throw new BadRequestError({ + message: "Kubernetes host is required when token review mode is set to API" + }); + } + let { kubernetesHost } = identityKubernetesAuth; if (kubernetesHost.startsWith("https://") || kubernetesHost.startsWith("http://")) { kubernetesHost = new RE2("^https?:\\/\\/").replace(kubernetesHost, ""); diff --git a/backend/src/services/identity-kubernetes-auth/identity-kubernetes-auth-types.ts b/backend/src/services/identity-kubernetes-auth/identity-kubernetes-auth-types.ts index 03dd7fd77..269fa19e0 100644 --- a/backend/src/services/identity-kubernetes-auth/identity-kubernetes-auth-types.ts +++ b/backend/src/services/identity-kubernetes-auth/identity-kubernetes-auth-types.ts @@ -12,7 +12,7 @@ export enum IdentityKubernetesAuthTokenReviewMode { export type TAttachKubernetesAuthDTO = { identityId: string; - kubernetesHost: string; + kubernetesHost: string | null; caCert: string; tokenReviewerJwt?: string; tokenReviewMode: IdentityKubernetesAuthTokenReviewMode; @@ -29,7 +29,7 @@ export type TAttachKubernetesAuthDTO = { export type TUpdateKubernetesAuthDTO = { identityId: string; - kubernetesHost?: string; + kubernetesHost?: string | null; caCert?: string; tokenReviewerJwt?: string | null; tokenReviewMode?: IdentityKubernetesAuthTokenReviewMode; diff --git a/backend/src/services/identity-ua/identity-ua-service.ts b/backend/src/services/identity-ua/identity-ua-service.ts index 53404788e..dec172e4e 100644 --- a/backend/src/services/identity-ua/identity-ua-service.ts +++ b/backend/src/services/identity-ua/identity-ua-service.ts @@ -122,9 +122,9 @@ export const identityUaServiceFactory = ({ } : { accessTokenTTL: identityUa.accessTokenPeriod, - // Setting Max TTL to 2 × period ensures that clients can always renew their token - // at least once, and matches client logic that checks if renewing would exceed Max TTL. - accessTokenMaxTTL: 2 * identityUa.accessTokenPeriod + // We set a very large Max TTL for periodic tokens to ensure that clients (even outdated ones) can always renew their token + // without them having to update their SDKs, CLIs, etc. This workaround sets it to 30 years to emulate "forever" + accessTokenMaxTTL: 1000000000 }; const identityAccessToken = await identityUaDAL.transaction(async (tx) => { diff --git a/backend/src/services/org/org-service.ts b/backend/src/services/org/org-service.ts index 63cb8935d..031b906a6 100644 --- a/backend/src/services/org/org-service.ts +++ b/backend/src/services/org/org-service.ts @@ -1211,8 +1211,8 @@ export const orgServiceFactory = ({ subjectLine: "Infisical organization invitation", recipients: [el.email], substitutions: { - inviterFirstName: invitingUser.firstName, - inviterUsername: invitingUser.email, + inviterFirstName: invitingUser?.firstName, + inviterUsername: invitingUser?.email, organizationName: org?.name, email: el.email, organizationId: org?.id.toString(), diff --git a/backend/src/services/project/project-dal.ts b/backend/src/services/project/project-dal.ts index 54bef02d1..7f733503c 100644 --- a/backend/src/services/project/project-dal.ts +++ b/backend/src/services/project/project-dal.ts @@ -12,7 +12,7 @@ import { TProjectsUpdate } from "@app/db/schemas"; import { BadRequestError, DatabaseError, NotFoundError, UnauthorizedError } from "@app/lib/errors"; -import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex"; +import { buildFindFilter, ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex"; import { ActorType } from "../auth/auth-type"; import { Filter, ProjectFilterType, SearchProjectSortBy } from "./project-types"; @@ -475,6 +475,16 @@ export const projectDALFactory = (db: TDbClient) => { return { docs, totalCount: Number(docs?.[0]?.count ?? 0) }; }; + const findProjectByEnvId = async (envId: string, tx?: Knex) => { + const project = await (tx || db.replicaNode())(TableName.Project) + .leftJoin(TableName.Environment, `${TableName.Environment}.projectId`, `${TableName.Project}.id`) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ id: envId }, TableName.Environment)) + .select(selectAllTableCols(TableName.Project)) + .first(); + return project; + }; + const countOfOrgProjects = async (orgId: string | null, tx?: Knex) => { try { const doc = await (tx || db.replicaNode())(TableName.Project) @@ -504,6 +514,7 @@ export const projectDALFactory = (db: TDbClient) => { checkProjectUpgradeStatus, getProjectFromSplitId, searchProjects, + findProjectByEnvId, countOfOrgProjects }; }; diff --git a/backend/src/services/project/project-service.ts b/backend/src/services/project/project-service.ts index d8eee188a..680b0187c 100644 --- a/backend/src/services/project/project-service.ts +++ b/backend/src/services/project/project-service.ts @@ -671,7 +671,8 @@ export const projectServiceFactory = ({ enforceCapitalization: update.autoCapitalization, hasDeleteProtection: update.hasDeleteProtection, slug: update.slug, - secretSharing: update.secretSharing + secretSharing: update.secretSharing, + showSnapshotsLegacy: update.showSnapshotsLegacy }); return updatedProject; diff --git a/backend/src/services/project/project-types.ts b/backend/src/services/project/project-types.ts index be052f1cb..8ef72492a 100644 --- a/backend/src/services/project/project-types.ts +++ b/backend/src/services/project/project-types.ts @@ -94,6 +94,7 @@ export type TUpdateProjectDTO = { hasDeleteProtection?: boolean; slug?: string; secretSharing?: boolean; + showSnapshotsLegacy?: boolean; }; } & Omit; diff --git a/backend/src/services/secret-folder/secret-folder-dal.ts b/backend/src/services/secret-folder/secret-folder-dal.ts index e136c5a50..7dfeaddcf 100644 --- a/backend/src/services/secret-folder/secret-folder-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-dal.ts @@ -488,6 +488,75 @@ export const secretFolderDALFactory = (db: TDbClient) => { } }; + const findFoldersByRootAndIds = async ({ rootId, folderIds }: { rootId: string; folderIds: string[] }, tx?: Knex) => { + try { + // First, get all descendant folders of rootId + const descendants = await (tx || db.replicaNode()) + .withRecursive("descendants", (qb) => + qb + .select( + selectAllTableCols(TableName.SecretFolder), + db.raw("0 as depth"), + db.raw(`'/' as path`), + db.ref(`${TableName.Environment}.slug`).as("environment") + ) + .from(TableName.SecretFolder) + .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) + .where(`${TableName.SecretFolder}.id`, rootId) + .union((un) => { + void un + .select( + selectAllTableCols(TableName.SecretFolder), + db.raw("descendants.depth + 1 as depth"), + db.raw( + `CONCAT( + CASE WHEN descendants.path = '/' THEN '' ELSE descendants.path END, + CASE WHEN ${TableName.SecretFolder}."parentId" is NULL THEN '' ELSE CONCAT('/', secret_folders.name) END + )` + ), + db.ref("descendants.environment") + ) + .from(TableName.SecretFolder) + .where(`${TableName.SecretFolder}.isReserved`, false) + .join("descendants", `${TableName.SecretFolder}.parentId`, "descendants.id"); + }) + ) + .select<(TSecretFolders & { path: string; depth: number; environment: string })[]>("*") + .from("descendants") + .whereIn(`id`, folderIds) + .orderBy("depth") + .orderBy(`name`); + + return descendants; + } catch (error) { + throw new DatabaseError({ error, name: "FindFoldersByRootAndIds" }); + } + }; + + const findByParentId = async (parentId: string, tx?: Knex) => { + try { + const folders = await (tx || db.replicaNode())(TableName.SecretFolder) + .where({ parentId }) + .andWhere({ isReserved: false }) + .select(selectAllTableCols(TableName.SecretFolder)); + return folders; + } catch (error) { + throw new DatabaseError({ error, name: "findByParentId" }); + } + }; + + const findByEnvId = async (envId: string, tx?: Knex) => { + try { + const folders = await (tx || db.replicaNode())(TableName.SecretFolder) + .where({ envId }) + .andWhere({ isReserved: false }) + .select(selectAllTableCols(TableName.SecretFolder)); + return folders; + } catch (error) { + throw new DatabaseError({ error, name: "findByEnvId" }); + } + }; + return { ...secretFolderOrm, update, @@ -499,6 +568,9 @@ export const secretFolderDALFactory = (db: TDbClient) => { findClosestFolder, findByProjectId, findByMultiEnv, - findByEnvsDeep + findByEnvsDeep, + findByParentId, + findByEnvId, + findFoldersByRootAndIds }; }; diff --git a/backend/src/services/secret-folder/secret-folder-service.ts b/backend/src/services/secret-folder/secret-folder-service.ts index 842eb2bb7..5722734f8 100644 --- a/backend/src/services/secret-folder/secret-folder-service.ts +++ b/backend/src/services/secret-folder/secret-folder-service.ts @@ -10,6 +10,7 @@ import { BadRequestError, NotFoundError } from "@app/lib/errors"; import { OrderByDirection, OrgServiceActor } from "@app/lib/types"; import { buildFolderPath } from "@app/services/secret-folder/secret-folder-fns"; +import { ChangeType, CommitType, TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TProjectDALFactory } from "../project/project-dal"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TSecretFolderDALFactory } from "./secret-folder-dal"; @@ -29,7 +30,8 @@ type TSecretFolderServiceFactoryDep = { snapshotService: Pick; folderDAL: TSecretFolderDALFactory; projectEnvDAL: Pick; - folderVersionDAL: TSecretFolderVersionDALFactory; + folderVersionDAL: Pick; + folderCommitService: Pick; projectDAL: Pick; }; @@ -41,6 +43,7 @@ export const secretFolderServiceFactory = ({ permissionService, projectEnvDAL, folderVersionDAL, + folderCommitService, projectDAL }: TSecretFolderServiceFactoryDep) => { const createFolder = async ({ @@ -111,15 +114,33 @@ export const secretFolderServiceFactory = ({ }); parentFolderId = newFolders.at(-1)?.id as string; const docs = await folderDAL.insertMany(newFolders, tx); - await folderVersionDAL.insertMany( + const folderVersions = await folderVersionDAL.insertMany( docs.map((doc) => ({ name: doc.name, envId: doc.envId, version: doc.version, - folderId: doc.id + folderId: doc.id, + description: doc.description })), tx ); + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "Folder created", + folderId: parentFolderId, + changes: folderVersions.map((fv) => ({ + type: CommitType.ADD, + folderVersionId: fv.id + })) + }, + tx + ); } } @@ -127,12 +148,32 @@ export const secretFolderServiceFactory = ({ { name, envId: env.id, version: 1, parentId: parentFolderId, description }, tx ); - await folderVersionDAL.create( + const folderVersion = await folderVersionDAL.create( { name: doc.name, envId: doc.envId, version: doc.version, - folderId: doc.id + folderId: doc.id, + description: doc.description + }, + tx + ); + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "Folder created", + folderId: parentFolderId, + changes: [ + { + type: CommitType.ADD, + folderVersionId: folderVersion.id + } + ] }, tx ); @@ -225,12 +266,33 @@ export const secretFolderServiceFactory = ({ { name, description }, tx ); - await folderVersionDAL.create( + const folderVersion = await folderVersionDAL.create( { name: doc.name, envId: doc.envId, version: doc.version, - folderId: doc.id + folderId: doc.id, + description: doc.description + }, + tx + ); + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "Folder updated", + folderId: parentFolder.id, + changes: [ + { + type: CommitType.ADD, + isUpdate: true, + folderVersionId: folderVersion.id + } + ] }, tx ); @@ -321,12 +383,33 @@ export const secretFolderServiceFactory = ({ { name, description }, tx ); - await folderVersionDAL.create( + const folderVersion = await folderVersionDAL.create( { name: doc.name, envId: doc.envId, version: doc.version, - folderId: doc.id + folderId: doc.id, + description: doc.description + }, + tx + ); + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "Folder updated", + folderId: parentFolder.id, + changes: [ + { + type: CommitType.ADD, + isUpdate: true, + folderVersionId: folderVersion.id + } + ] }, tx ); @@ -381,7 +464,31 @@ export const secretFolderServiceFactory = ({ }, tx ); + if (!doc) throw new NotFoundError({ message: `Failed to delete folder with ID '${idOrName}', not found` }); + + const folderVersions = await folderVersionDAL.findLatestFolderVersions([doc.id], tx); + + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "Folder deleted", + folderId: parentFolder.id, + changes: [ + { + type: CommitType.DELETE, + folderVersionId: folderVersions[doc.id].id, + folderId: doc.id + } + ] + }, + tx + ); return doc; }); @@ -665,6 +772,45 @@ export const secretFolderServiceFactory = ({ return environmentFolders; }; + const getFolderVersionsByIds = async ({ + folderId, + folderVersions + }: { + folderId: string; + folderVersions: string[]; + }) => { + const versions = await folderVersionDAL.find({ + folderId, + $in: { + version: folderVersions.map((v) => Number.parseInt(v, 10)) + } + }); + return versions; + }; + + const getFolderVersions = async ( + change: { + folderVersion?: string; + isUpdate?: boolean; + changeType?: string; + }, + fromVersion: string, + folderId: string + ) => { + const currentVersion = change.folderVersion || "1"; + // eslint-disable-next-line no-await-in-loop + const versions = await getFolderVersionsByIds({ + folderId, + folderVersions: + change.isUpdate || change.changeType === ChangeType.UPDATE ? [currentVersion, fromVersion] : [currentVersion] + }); + return versions.map((v) => ({ + version: v.version?.toString() || "1", + name: v.name, + description: v.description + })); + }; + return { createFolder, updateFolder, @@ -675,6 +821,8 @@ export const secretFolderServiceFactory = ({ getProjectFolderCount, getFoldersMultiEnv, getFoldersDeepByEnvs, - getProjectEnvironmentsFolders + getProjectEnvironmentsFolders, + getFolderVersionsByIds, + getFolderVersions }; }; diff --git a/backend/src/services/secret-folder/secret-folder-version-dal.ts b/backend/src/services/secret-folder/secret-folder-version-dal.ts index 78186333d..46ff49692 100644 --- a/backend/src/services/secret-folder/secret-folder-version-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-version-dal.ts @@ -43,7 +43,7 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => { const docs: Array = await (tx || db.replicaNode())( TableName.SecretFolderVersion ) - .whereIn("folderId", folderIds) + .whereIn(`${TableName.SecretFolderVersion}.folderId`, folderIds) .join( (tx || db)(TableName.SecretFolderVersion) .groupBy("folderId") @@ -85,6 +85,8 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => { .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) .join("folder_cte", "folder_cte.id", `${TableName.SecretFolderVersion}.id`) .whereRaw(`folder_cte.row_num > ${TableName.Project}."pitVersionLimit"`) + // Projects with version >= 3 will require to have all folder versions for PIT + .andWhere(`${TableName.Project}.version`, "<", 3) .delete(); } catch (error) { throw new DatabaseError({ @@ -95,5 +97,107 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => { logger.info(`${QueueName.DailyResourceCleanUp}: pruning secret folder versions completed`); }; - return { ...secretFolderVerOrm, findLatestFolderVersions, findLatestVersionByFolderId, pruneExcessVersions }; + // Get latest versions by folderIds + const getLatestFolderVersions = async (folderIds: string[], tx?: Knex): Promise> => { + if (!folderIds.length) return []; + + const knexInstance = tx || db.replicaNode(); + return knexInstance(TableName.SecretFolderVersion) + .whereIn(`${TableName.SecretFolderVersion}.folderId`, folderIds) + .join( + knexInstance(TableName.SecretFolderVersion) + .groupBy("folderId") + .max("version") + .select("folderId") + .as("latestVersion"), + (bd) => { + bd.on(`${TableName.SecretFolderVersion}.folderId`, "latestVersion.folderId").andOn( + `${TableName.SecretFolderVersion}.version`, + "latestVersion.max" + ); + } + ); + }; + + // Get specific versions and update with max version + const getSpecificFolderVersionsWithLatest = async ( + versionIds: string[], + tx?: Knex + ): Promise> => { + if (!versionIds.length) return []; + + const knexInstance = tx || db.replicaNode(); + + // Get specific versions + const specificVersions = await knexInstance(TableName.SecretFolderVersion).whereIn("id", versionIds); + + // Get folderIds from these versions + const specificFolderIds = [...new Set(specificVersions.map((v) => v.folderId).filter(Boolean))]; + + if (!specificFolderIds.length) return specificVersions; + + // Get max versions for these folderIds + const maxVersionsQuery = await knexInstance(TableName.SecretFolderVersion) + .whereIn("folderId", specificFolderIds) + .groupBy("folderId") + .select("folderId") + .max("version", { as: "maxVersion" }); + + // Create lookup map for max versions + const maxVersionMap = maxVersionsQuery.reduce>((acc, item) => { + if (item.maxVersion) { + acc[item.folderId] = item.maxVersion; + } + return acc; + }, {}); + + // Replace version with max version + return specificVersions.map((version) => ({ + ...version, + version: maxVersionMap[version.folderId] || version.version + })); + }; + + const findByIdsWithLatestVersion = async (folderIds: string[], versionIds?: string[], tx?: Knex) => { + try { + if (!folderIds.length && (!versionIds || !versionIds.length)) return {}; + + // Run both queries in parallel + const [latestVersions, specificVersionsWithLatest] = await Promise.all([ + folderIds.length ? getLatestFolderVersions(folderIds, tx) : [], + versionIds?.length ? getSpecificFolderVersionsWithLatest(versionIds, tx) : [] + ]); + + const allDocs = [...latestVersions, ...specificVersionsWithLatest]; + + // Convert array to record with folderId as key + return allDocs.reduce>( + (prev, curr) => ({ ...prev, [curr.folderId || ""]: curr }), + {} + ); + } catch (error) { + throw new DatabaseError({ error, name: "FindByIdsWithLatestVersion" }); + } + }; + + const findLatestVersion = async (folderId: string, tx?: Knex) => { + try { + const doc = await (tx || db.replicaNode())(TableName.SecretFolderVersion) + .where(`${TableName.SecretFolderVersion}.folderId`, folderId) + .select(selectAllTableCols(TableName.SecretFolderVersion)) + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "findLatestVersion" }); + } + }; + + return { + ...secretFolderVerOrm, + findLatestFolderVersions, + findLatestVersionByFolderId, + pruneExcessVersions, + findByIdsWithLatestVersion, + findLatestVersion + }; }; diff --git a/backend/src/services/secret-sync/secret-sync-queue.ts b/backend/src/services/secret-sync/secret-sync-queue.ts index 6f627c24e..66e83661f 100644 --- a/backend/src/services/secret-sync/secret-sync-queue.ts +++ b/backend/src/services/secret-sync/secret-sync-queue.ts @@ -59,6 +59,7 @@ import { TSecretVersionV2TagDALFactory } from "@app/services/secret-v2-bridge/se import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service"; import { TAppConnectionDALFactory } from "../app-connection/app-connection-dal"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; export type TSecretSyncQueueFactory = ReturnType; @@ -94,6 +95,7 @@ type TSecretSyncQueueFactoryDep = { secretVersionV2BridgeDAL: Pick; secretVersionTagV2BridgeDAL: Pick; resourceMetadataDAL: Pick; + folderCommitService: Pick; licenseService: Pick; }; @@ -136,6 +138,7 @@ export const secretSyncQueueFactory = ({ secretVersionV2BridgeDAL, secretVersionTagV2BridgeDAL, resourceMetadataDAL, + folderCommitService, licenseService }: TSecretSyncQueueFactoryDep) => { const appCfg = getConfig(); @@ -167,7 +170,8 @@ export const secretSyncQueueFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, secretVersionTagV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }); const $updateManySecretsRawFn = updateManySecretsRawFnFactory({ @@ -183,7 +187,8 @@ export const secretSyncQueueFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, secretVersionTagV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }); const $getInfisicalSecrets = async ( @@ -373,7 +378,7 @@ export const secretSyncQueueFactory = ({ if (Object.hasOwn(secretMap, key)) { // Only update secrets if the source value is not empty - if (value) { + if (value && value !== secretMap[key].value) { secretsToUpdate.push(secret); if (importBehavior === SecretSyncImportBehavior.PrioritizeDestination) importedSecretMap[key] = secretData; } diff --git a/backend/src/services/secret-tag/secret-tag-dal.ts b/backend/src/services/secret-tag/secret-tag-dal.ts index 3b9151557..8768990fe 100644 --- a/backend/src/services/secret-tag/secret-tag-dal.ts +++ b/backend/src/services/secret-tag/secret-tag-dal.ts @@ -11,6 +11,7 @@ export const secretTagDALFactory = (db: TDbClient) => { const secretTagOrm = ormify(db, TableName.SecretTag); const secretJnTagOrm = ormify(db, TableName.JnSecretTag); const secretV2JnTagOrm = ormify(db, TableName.SecretV2JnTag); + const secretVersionV2TagOrm = ormify(db, TableName.SecretVersionV2Tag); const findManyTagsById = async (projectId: string, ids: string[], tx?: Knex) => { try { @@ -48,14 +49,39 @@ export const secretTagDALFactory = (db: TDbClient) => { } }; + const findSecretTagsByVersionId = async (versionId: string, tx?: Knex) => { + try { + const tags = await (tx || db.replicaNode())(TableName.SecretVersionV2Tag) + .where(`${TableName.SecretVersionV2Tag}.${TableName.SecretVersionV2}Id`, versionId) + .select(selectAllTableCols(TableName.SecretVersionV2Tag)); + return tags; + } catch (error) { + throw new DatabaseError({ error, name: "Find all by version id" }); + } + }; + + const findSecretTagsBySecretId = async (secretId: string, tx?: Knex) => { + try { + const tags = await (tx || db.replicaNode())(TableName.SecretV2JnTag) + .where(`${TableName.SecretV2JnTag}.${TableName.SecretV2}Id`, secretId) + .select(selectAllTableCols(TableName.SecretV2JnTag)); + return tags; + } catch (error) { + throw new DatabaseError({ error, name: "Find all by secret id" }); + } + }; + return { ...secretTagOrm, saveTagsToSecret: secretJnTagOrm.insertMany, deleteTagsToSecret: secretJnTagOrm.delete, saveTagsToSecretV2: secretV2JnTagOrm.batchInsert, deleteTagsToSecretV2: secretV2JnTagOrm.delete, + saveTagsToSecretVersionV2: secretVersionV2TagOrm.insertMany, findSecretTagsByProjectId, deleteTagsManySecret, - findManyTagsById + findManyTagsById, + findSecretTagsByVersionId, + findSecretTagsBySecretId }; }; diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts index 6fdcadeff..1adfac22c 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts @@ -8,6 +8,7 @@ import { groupBy } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; import { ActorType } from "../auth/auth-type"; +import { CommitType } from "../folder-commit/folder-commit-service"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { ResourceMetadataDTO } from "../resource-metadata/resource-metadata-schema"; import { INFISICAL_SECRET_VALUE_HIDDEN_MASK } from "../secret/secret-fns"; @@ -73,6 +74,7 @@ export const fnSecretBulkInsert = async ({ resourceMetadataDAL, secretTagDAL, secretVersionTagDAL, + folderCommitService, actor, tx }: TFnSecretBulkInsert) => { @@ -126,11 +128,36 @@ export const fnSecretBulkInsert = async ({ userActorId, identityActorId, actorType, + metadata: el.metadata ? JSON.stringify(el.metadata) : [], secretId: newSecretGroupedByKeyName[el.key][0].id })), tx ); + const commitChanges = secretVersions + .filter(({ type }) => type === SecretType.Shared) + .map((sv) => ({ + type: CommitType.ADD, + secretVersionId: sv.id + })); + + if (commitChanges.length > 0) { + await folderCommitService.createCommit( + { + actor: { + type: actorType || ActorType.PLATFORM, + metadata: { + id: actor?.actorId + } + }, + message: "Secret Created", + folderId, + changes: commitChanges + }, + tx + ); + } + await secretDAL.upsertSecretReferences( inputSecrets.map(({ references = [], key }) => ({ secretId: newSecretGroupedByKeyName[key][0].id, @@ -185,6 +212,7 @@ export const fnSecretBulkUpdate = async ({ orgId, secretDAL, secretVersionDAL, + folderCommitService, secretTagDAL, secretVersionTagDAL, resourceMetadataDAL, @@ -246,7 +274,7 @@ export const fnSecretBulkUpdate = async ({ userId, encryptedComment, version, - metadata, + metadata: metadata ? JSON.stringify(metadata) : [], reminderNote, encryptedValue, reminderRepeatDays, @@ -259,6 +287,7 @@ export const fnSecretBulkUpdate = async ({ ), tx ); + await secretDAL.upsertSecretReferences( inputSecrets .filter(({ data: { references } }) => Boolean(references)) @@ -329,6 +358,31 @@ export const fnSecretBulkUpdate = async ({ }, { tx } ); + + const commitChanges = secretVersions + .filter(({ type }) => type === SecretType.Shared) + .map((sv) => ({ + type: CommitType.ADD, + isUpdate: true, + secretVersionId: sv.id + })); + if (commitChanges.length > 0) { + await folderCommitService.createCommit( + { + actor: { + type: actorType || ActorType.PLATFORM, + metadata: { + id: actor?.actorId + } + }, + message: "Secret Updated", + folderId, + changes: commitChanges + }, + tx + ); + } + return secretsWithTags.map((secret) => ({ ...secret, _id: secret.id })); }; @@ -337,8 +391,11 @@ export const fnSecretBulkDelete = async ({ inputSecrets, tx, actorId, + actorType, secretDAL, - secretQueueService + secretQueueService, + folderCommitService, + secretVersionDAL }: TFnSecretBulkDelete) => { const deletedSecrets = await secretDAL.deleteMany( inputSecrets.map(({ type, secretKey }) => ({ @@ -358,6 +415,35 @@ export const fnSecretBulkDelete = async ({ ) ); + const secretVersions = await secretVersionDAL.findLatestVersionMany( + folderId, + deletedSecrets.map(({ id }) => id), + tx + ); + + const commitChanges = deletedSecrets + .filter(({ type }) => type === SecretType.Shared) + .map(({ id }) => ({ + type: CommitType.DELETE, + secretVersionId: secretVersions[id].id + })); + if (commitChanges.length > 0) { + await folderCommitService.createCommit( + { + actor: { + type: actorType || ActorType.PLATFORM, + metadata: { + id: actorId + } + }, + message: "Secret Deleted", + folderId, + changes: commitChanges + }, + tx + ); + } + return deletedSecrets; }; diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts index 1ef4a2d41..c4b5835c4 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts @@ -17,6 +17,7 @@ import { import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, + ProjectPermissionCommitsActions, ProjectPermissionSecretActions, ProjectPermissionSet, ProjectPermissionSub @@ -34,6 +35,7 @@ import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { ActorType } from "../auth/auth-type"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "../kms/kms-service"; import { KmsDataKey } from "../kms/kms-types"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; @@ -90,6 +92,7 @@ type TSecretV2BridgeServiceFactoryDep = { secretVersionTagDAL: Pick; secretTagDAL: TSecretTagDALFactory; permissionService: Pick; + folderCommitService: Pick; projectEnvDAL: Pick; folderDAL: Pick< TSecretFolderDALFactory, @@ -124,6 +127,7 @@ export const secretV2BridgeServiceFactory = ({ projectEnvDAL, secretTagDAL, secretVersionDAL, + folderCommitService, folderDAL, permissionService, snapshotService, @@ -321,12 +325,14 @@ export const secretV2BridgeServiceFactory = ({ userId: inputSecret.type === SecretType.Personal ? actorId : null, tagIds: inputSecret.tagIds, references: nestedReferences, + metadata: secretMetadata ? JSON.stringify(secretMetadata) : [], secretMetadata } ], resourceMetadataDAL, secretDAL, secretVersionDAL, + folderCommitService, secretTagDAL, secretVersionTagDAL, actor: { @@ -510,6 +516,7 @@ export const secretV2BridgeServiceFactory = ({ folderId, orgId: actorOrgId, resourceMetadataDAL, + folderCommitService, inputSecrets: [ { filter: { id: secretId }, @@ -523,6 +530,7 @@ export const secretV2BridgeServiceFactory = ({ skipMultilineEncoding: inputSecret.skipMultilineEncoding, key: inputSecret.newSecretName || secretName, tags: inputSecret.tagIds, + metadata: secretMetadata ? JSON.stringify(secretMetadata) : [], secretMetadata, ...encryptedValue } @@ -650,6 +658,9 @@ export const secretV2BridgeServiceFactory = ({ projectId, folderId, actorId, + actorType: actor, + folderCommitService, + secretVersionDAL, secretDAL, secretQueueService, inputSecrets: [ @@ -1590,6 +1601,7 @@ export const secretV2BridgeServiceFactory = ({ orgId: actorOrgId, secretDAL, resourceMetadataDAL, + folderCommitService, secretVersionDAL, secretTagDAL, secretVersionTagDAL, @@ -1859,6 +1871,7 @@ export const secretV2BridgeServiceFactory = ({ const bulkUpdatedSecrets = await fnSecretBulkUpdate({ folderId, orgId: actorOrgId, + folderCommitService, tx, inputSecrets: secretsToUpdate.map((el) => { const originalSecret = secretsToUpdateInDBGroupedByKey[el.secretKey][0]; @@ -1928,6 +1941,7 @@ export const secretV2BridgeServiceFactory = ({ secretVersionDAL, secretTagDAL, secretVersionTagDAL, + folderCommitService, actor: { type: actor, actorId @@ -2061,6 +2075,8 @@ export const secretV2BridgeServiceFactory = ({ fnSecretBulkDelete({ secretDAL, secretQueueService, + folderCommitService, + secretVersionDAL, inputSecrets: inputSecrets.map(({ type, secretKey }) => ({ secretKey, type: type || SecretType.Shared @@ -2068,6 +2084,7 @@ export const secretV2BridgeServiceFactory = ({ projectId, folderId, actorId, + actorType: actor, tx }) ); @@ -2159,15 +2176,25 @@ export const secretV2BridgeServiceFactory = ({ actorOrgId, actionProjectType: ActionProjectType.SecretManager }); - ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback); + + const canRead = + permission.can(ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback) || + permission.can(ProjectPermissionCommitsActions.Read, ProjectPermissionSub.Commits); + + if (!canRead) throw new ForbiddenRequestError({ message: "You do not have permission to read secret versions" }); + const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ type: KmsDataKey.SecretManager, projectId: folder.projectId }); - const secretVersions = await secretVersionDAL.findVersionsBySecretIdWithActors(secretId, folder.projectId, { - offset, - limit, - sort: [["createdAt", "desc"]] + const secretVersions = await secretVersionDAL.findVersionsBySecretIdWithActors({ + secretId, + projectId: folder.projectId, + findOpt: { + offset, + limit, + sort: [["createdAt", "desc"]] + } }); return secretVersions.map((el) => { const secretValueHidden = !hasSecretReadValueOrDescribePermission( @@ -2469,6 +2496,7 @@ export const secretV2BridgeServiceFactory = ({ tx, secretTagDAL, resourceMetadataDAL, + folderCommitService, secretVersionTagDAL, actor: { type: actor, @@ -2495,6 +2523,7 @@ export const secretV2BridgeServiceFactory = ({ folderId: destinationFolder.id, orgId: actorOrgId, resourceMetadataDAL, + folderCommitService, secretVersionDAL, secretDAL, tx, @@ -2840,6 +2869,76 @@ export const secretV2BridgeServiceFactory = ({ }; }; + const getSecretVersionsByIds = async ({ + actorId, + actor, + actorOrgId, + actorAuthMethod, + secretId, + secretVersionNumbers, + secretPath, + envId, + projectId + }: TGetSecretVersionsDTO & { + secretVersionNumbers: string[]; + secretPath: string; + envId: string; + projectId: string; + }) => { + const environment = await projectEnvDAL.findOne({ id: envId, projectId }); + + const { permission } = await permissionService.getProjectPermission({ + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId, + actionProjectType: ActionProjectType.SecretManager + }); + + const canRead = + permission.can(ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback) || + permission.can(ProjectPermissionCommitsActions.Read, ProjectPermissionSub.Commits); + + if (!canRead) throw new ForbiddenRequestError({ message: "You do not have permission to read secret versions" }); + + const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + const secretVersions = await secretVersionDAL.findVersionsBySecretIdWithActors({ + secretId, + projectId, + secretVersions: secretVersionNumbers + }); + return secretVersions.map((el) => { + const secretValueHidden = !hasSecretReadValueOrDescribePermission( + permission, + ProjectPermissionSecretActions.ReadValue, + { + environment: environment.slug, + secretPath, + secretName: el.key, + ...(el.tags?.length && { + secretTags: el.tags.map((tag) => tag.slug) + }) + } + ); + + return reshapeBridgeSecret( + projectId, + environment.slug, + secretPath, + { + ...el, + value: el.encryptedValue ? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString() : "", + comment: el.encryptedComment ? secretManagerDecryptor({ cipherTextBlob: el.encryptedComment }).toString() : "" + }, + secretValueHidden + ); + }); + }; + return { createSecret, deleteSecret, @@ -2858,6 +2957,7 @@ export const secretV2BridgeServiceFactory = ({ getSecretReferenceTree, getSecretsByFolderMappings, getSecretById, - getAccessibleSecrets + getAccessibleSecrets, + getSecretVersionsByIds }; }; diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts index f4a27d4c5..f4171b1a7 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts @@ -8,6 +8,7 @@ import { SecretsOrderBy } from "@app/services/secret/secret-types"; import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal"; import { ResourceMetadataDTO } from "../resource-metadata/resource-metadata-schema"; import { TSecretV2BridgeDALFactory } from "./secret-v2-bridge-dal"; @@ -178,9 +179,10 @@ export type TFnSecretBulkInsert = { secretVersionDAL: Pick; secretTagDAL: Pick; secretVersionTagDAL: Pick; + folderCommitService: Pick; actor?: { type: string; - actorId: string; + actorId?: string; }; }; @@ -206,9 +208,10 @@ export type TFnSecretBulkUpdate = { secretVersionDAL: Pick; secretTagDAL: Pick; secretVersionTagDAL: Pick; + folderCommitService: Pick; actor?: { type: string; - actorId: string; + actorId?: string; }; tx?: Knex; }; @@ -218,11 +221,14 @@ export type TFnSecretBulkDelete = { projectId: string; inputSecrets: Array<{ type: SecretType; secretKey: string }>; actorId: string; + actorType?: string; tx?: Knex; secretDAL: Pick; secretQueueService: { removeSecretReminder: (data: TRemoveSecretReminderDTO, tx?: Knex) => Promise; }; + folderCommitService: Pick; + secretVersionDAL: Pick; }; export type THandleReminderDTO = { diff --git a/backend/src/services/secret-v2-bridge/secret-version-dal.ts b/backend/src/services/secret-v2-bridge/secret-version-dal.ts index b54b073a6..9537b79e3 100644 --- a/backend/src/services/secret-v2-bridge/secret-version-dal.ts +++ b/backend/src/services/secret-v2-bridge/secret-version-dal.ts @@ -4,7 +4,7 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; import { SecretVersionsV2Schema, TableName, TSecretVersionsV2, TSecretVersionsV2Update } from "@app/db/schemas"; import { BadRequestError, DatabaseError } from "@app/lib/errors"; -import { ormify, selectAllTableCols, sqlNestRelationships, TFindOpt } from "@app/lib/knex"; +import { buildFindFilter, ormify, selectAllTableCols, sqlNestRelationships, TFindOpt } from "@app/lib/knex"; import { logger } from "@app/lib/logger"; import { QueueName } from "@app/queue"; @@ -138,7 +138,7 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { {} ); } catch (error) { - throw new DatabaseError({ error, name: "FindLatestVersinMany" }); + throw new DatabaseError({ error, name: "FindLatestVersionMany" }); } }; @@ -162,6 +162,8 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) .join("version_cte", "version_cte.id", `${TableName.SecretVersionV2}.id`) .whereRaw(`version_cte.row_num > ${TableName.Project}."pitVersionLimit"`) + // Projects with version >= 3 will require to have all secret versions for PIT + .andWhere(`${TableName.Project}.version`, "<", 3) .delete(); } catch (error) { throw new DatabaseError({ @@ -172,13 +174,21 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { logger.info(`${QueueName.DailyResourceCleanUp}: pruning secret version v2 completed`); }; - const findVersionsBySecretIdWithActors = async ( - secretId: string, - projectId: string, - { offset, limit, sort = [["createdAt", "desc"]] }: TFindOpt = {}, - tx?: Knex - ) => { + const findVersionsBySecretIdWithActors = async ({ + secretId, + projectId, + secretVersions, + findOpt = {}, + tx + }: { + secretId: string; + projectId: string; + secretVersions?: string[]; + findOpt?: TFindOpt; + tx?: Knex; + }) => { try { + const { offset, limit, sort = [["createdAt", "desc"]] } = findOpt; const query = (tx || db)(TableName.SecretVersionV2) .leftJoin(TableName.Users, `${TableName.Users}.id`, `${TableName.SecretVersionV2}.userActorId`) .leftJoin( @@ -189,22 +199,24 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { .leftJoin(TableName.Identity, `${TableName.Identity}.id`, `${TableName.SecretVersionV2}.identityActorId`) .leftJoin(TableName.SecretV2, `${TableName.SecretVersionV2}.secretId`, `${TableName.SecretV2}.id`) .leftJoin( - TableName.SecretV2JnTag, - `${TableName.SecretV2}.id`, - `${TableName.SecretV2JnTag}.${TableName.SecretV2}Id` + TableName.SecretVersionV2Tag, + `${TableName.SecretVersionV2}.id`, + `${TableName.SecretVersionV2Tag}.${TableName.SecretVersionV2}Id` ) .leftJoin( TableName.SecretTag, - `${TableName.SecretV2JnTag}.${TableName.SecretTag}Id`, + `${TableName.SecretVersionV2Tag}.${TableName.SecretTag}Id`, `${TableName.SecretTag}.id` ) .where((qb) => { void qb.where(`${TableName.SecretVersionV2}.secretId`, secretId); void qb.where(`${TableName.ProjectMembership}.projectId`, projectId); + if (secretVersions?.length) void qb.whereIn(`${TableName.SecretVersionV2}.version`, secretVersions); }) .orWhere((qb) => { void qb.where(`${TableName.SecretVersionV2}.secretId`, secretId); void qb.whereNull(`${TableName.ProjectMembership}.projectId`); + if (secretVersions?.length) void qb.whereIn(`${TableName.SecretVersionV2}.version`, secretVersions); }) .select( selectAllTableCols(TableName.SecretVersionV2), @@ -260,6 +272,178 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { } }; + // Function to fetch latest versions by secretIds + const getLatestVersionsBySecretIds = async ( + folderId: string, + secretIds: string[], + tx?: Knex + ): Promise> => { + if (!secretIds.length) return []; + + const knexInstance = tx || db.replicaNode(); + return knexInstance(TableName.SecretVersionV2) + .where("folderId", folderId) + .whereIn(`${TableName.SecretVersionV2}.secretId`, secretIds) + .join( + knexInstance(TableName.SecretVersionV2) + .groupBy("secretId") + .max("version") + .select("secretId") + .as("latestVersion"), + (bd) => { + bd.on(`${TableName.SecretVersionV2}.secretId`, "latestVersion.secretId").andOn( + `${TableName.SecretVersionV2}.version`, + "latestVersion.max" + ); + } + ); + }; + + // Function to fetch specific versions by versionIds + const getSpecificVersionsWithLatestInfo = async ( + folderId: string, + versionIds: string[], + tx?: Knex + ): Promise> => { + if (!versionIds.length) return []; + + const knexInstance = tx || db.replicaNode(); + + // Get the specific versions + const specificVersions = await knexInstance(TableName.SecretVersionV2) + .where("folderId", folderId) + .whereIn("id", versionIds); + + // Get the secretIds from these versions + const specificSecretIds = [...new Set(specificVersions.map((v) => v.secretId).filter(Boolean))]; + + if (!specificSecretIds.length) return specificVersions; + + // Get max versions for these secretIds + const maxVersionsQuery = await knexInstance(TableName.SecretVersionV2) + .whereIn("secretId", specificSecretIds) + .groupBy("secretId") + .select("secretId") + .max("version", { as: "maxVersion" }); + + // Create a lookup map for max versions + const maxVersionMap = maxVersionsQuery.reduce( + (acc, item) => { + acc[item.secretId] = item.maxVersion; + return acc; + }, + {} as Record + ); + + // Update the version field with maxVersion when needed + return specificVersions.map((version) => { + // Replace version with maxVersion + return { + ...version, + version: maxVersionMap[version.secretId] || version.version + }; + }); + }; + + const findByIdsWithLatestVersion = async ( + folderId: string, + secretIds: string[], + versionIds?: string[], + tx?: Knex + ) => { + try { + if (!secretIds.length && (!versionIds || !versionIds.length)) return {}; + + const [latestVersions, specificVersionsWithLatest] = await Promise.all([ + secretIds.length ? getLatestVersionsBySecretIds(folderId, secretIds, tx) : [], + versionIds?.length ? getSpecificVersionsWithLatestInfo(folderId, versionIds, tx) : [] + ]); + + const allDocs = [...latestVersions, ...specificVersionsWithLatest]; + + // Convert array to record with secretId as key + return allDocs.reduce>( + (prev, curr) => ({ ...prev, [curr.secretId || ""]: curr }), + {} + ); + } catch (error) { + throw new DatabaseError({ error, name: "FindByIdsWithLatestVersion" }); + } + }; + + const findByIdAndPreviousVersion = async (secretVersionId: string, tx?: Knex) => { + try { + const targetSecretVersion = await (tx || db.replicaNode())(TableName.SecretVersionV2) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ id: secretVersionId }, TableName.SecretVersionV2)) + .leftJoin( + TableName.SecretVersionV2Tag, + `${TableName.SecretVersionV2}.id`, + `${TableName.SecretVersionV2Tag}.${TableName.SecretVersionV2}Id` + ) + .leftJoin( + TableName.SecretTag, + `${TableName.SecretVersionV2Tag}.${TableName.SecretTag}Id`, + `${TableName.SecretTag}.id` + ) + .select(selectAllTableCols(TableName.SecretVersionV2)) + .select(db.ref("id").withSchema(TableName.SecretTag).as("tagId")) + .select(db.ref("color").withSchema(TableName.SecretTag).as("tagColor")) + .select(db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug")) + .first(); + if (targetSecretVersion) { + const previousSecretVersion = await (tx || db.replicaNode())(TableName.SecretVersionV2) + .where( + // eslint-disable-next-line @typescript-eslint/no-misused-promises + buildFindFilter( + { version: targetSecretVersion.version - 1, secretId: targetSecretVersion.secretId }, + TableName.SecretVersionV2 + ) + ) + .leftJoin( + TableName.SecretVersionV2Tag, + `${TableName.SecretVersionV2}.id`, + `${TableName.SecretVersionV2Tag}.${TableName.SecretVersionV2}Id` + ) + .leftJoin( + TableName.SecretTag, + `${TableName.SecretVersionV2Tag}.${TableName.SecretTag}Id`, + `${TableName.SecretTag}.id` + ) + .select(selectAllTableCols(TableName.SecretVersionV2)) + .select(db.ref("id").withSchema(TableName.SecretTag).as("tagId")) + .select(db.ref("color").withSchema(TableName.SecretTag).as("tagColor")) + .select(db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug")) + .first(); + if (!previousSecretVersion) return []; + const docs = [previousSecretVersion, targetSecretVersion]; + + const data = sqlNestRelationships({ + data: docs, + key: "id", + parentMapper: (el) => ({ _id: el.id, ...SecretVersionsV2Schema.parse(el) }), + childrenMapper: [ + { + key: "tagId", + label: "tags" as const, + mapper: ({ tagId: id, tagColor: color, tagSlug: slug }) => ({ + id, + color, + slug, + name: slug + }) + } + ] + }); + + return data; + } + return []; + } catch (error) { + throw new DatabaseError({ error, name: "FindByIdAndPreviousVersion" }); + } + }; + return { ...secretVersionV2Orm, pruneExcessVersions, @@ -267,6 +451,8 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { bulkUpdate, findLatestVersionByFolderId, findVersionsBySecretIdWithActors, - findBySecretId + findBySecretId, + findByIdsWithLatestVersion, + findByIdAndPreviousVersion }; }; diff --git a/backend/src/services/secret/secret-fns.ts b/backend/src/services/secret/secret-fns.ts index e5f3acdea..96f89ab5f 100644 --- a/backend/src/services/secret/secret-fns.ts +++ b/backend/src/services/secret/secret-fns.ts @@ -778,6 +778,7 @@ export const createManySecretsRawFnFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, secretVersionTagV2BridgeDAL, + folderCommitService, kmsService, resourceMetadataDAL }: TCreateManySecretsRawFnFactory) => { @@ -850,6 +851,7 @@ export const createManySecretsRawFnFactory = ({ secretVersionDAL: secretVersionV2BridgeDAL, secretTagDAL, secretVersionTagDAL: secretVersionTagV2BridgeDAL, + folderCommitService, tx }) ); @@ -942,6 +944,7 @@ export const updateManySecretsRawFnFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, resourceMetadataDAL, + folderCommitService, kmsService }: TUpdateManySecretsRawFnFactory) => { const getBotKeyFn = getBotKeyFnFactory(projectBotDAL, projectDAL); @@ -1032,7 +1035,8 @@ export const updateManySecretsRawFnFactory = ({ secretDAL: secretV2BridgeDAL, secretVersionDAL: secretVersionV2BridgeDAL, secretTagDAL, - secretVersionTagDAL: secretVersionTagV2BridgeDAL + secretVersionTagDAL: secretVersionTagV2BridgeDAL, + folderCommitService }) ); diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index 714df0d3f..e9c6f2d88 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -35,6 +35,7 @@ import { TSecretSyncQueueFactory } from "@app/services/secret-sync/secret-sync-q import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; import { ActorType } from "../auth/auth-type"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TIntegrationDALFactory } from "../integration/integration-dal"; import { TIntegrationAuthDALFactory } from "../integration-auth/integration-auth-dal"; import { TIntegrationAuthServiceFactory } from "../integration-auth/integration-auth-service"; @@ -112,6 +113,7 @@ type TSecretQueueFactoryDep = { orgService: Pick; projectUserMembershipRoleDAL: Pick; resourceMetadataDAL: Pick; + folderCommitService: Pick; secretReminderRecipientsDAL: Pick< TSecretReminderRecipientsDALFactory, "delete" | "findUsersBySecretId" | "insertMany" | "transaction" @@ -178,7 +180,8 @@ export const secretQueueFactory = ({ projectKeyDAL, resourceMetadataDAL, secretReminderRecipientsDAL, - secretSyncQueue + secretSyncQueue, + folderCommitService }: TSecretQueueFactoryDep) => { const integrationMeter = opentelemetry.metrics.getMeter("Integrations"); const errorHistogram = integrationMeter.createHistogram("integration_secret_sync_errors", { @@ -366,7 +369,8 @@ export const secretQueueFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, secretVersionTagV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }); const updateManySecretsRawFn = updateManySecretsRawFnFactory({ @@ -382,7 +386,8 @@ export const secretQueueFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, secretVersionTagV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }); /** @@ -1581,6 +1586,7 @@ export const secretQueueFactory = ({ projectDAL, webhookDAL, event: job.data, + auditLogService, secretManagerDecryptor: (value) => secretManagerDecryptor({ cipherTextBlob: value }).toString() }); }); diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 46ed7ef83..4fa7404d3 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -44,7 +44,8 @@ import { TGetSecretsRawByFolderMappingsDTO } from "@app/services/secret-v2-bridge/secret-v2-bridge-types"; -import { ActorType } from "../auth/auth-type"; +import { ActorAuthMethod, ActorType } from "../auth/auth-type"; +import { ChangeType } from "../folder-commit/folder-commit-service"; import { TProjectDALFactory } from "../project/project-dal"; import { TProjectBotServiceFactory } from "../project-bot/project-bot-service"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; @@ -2521,6 +2522,36 @@ export const secretServiceFactory = ({ }); }; + const getSecretVersionsV2ByIds = async ({ + actorId, + actor, + actorOrgId, + actorAuthMethod, + secretId, + secretVersions, + secretPath, + envId, + projectId + }: TGetSecretVersionsDTO & { + secretVersions: string[]; + secretPath: string; + envId: string; + projectId: string; + }) => { + const secretVersionV2 = await secretV2BridgeService.getSecretVersionsByIds({ + actorId, + actor, + actorOrgId, + actorAuthMethod, + secretId, + secretVersionNumbers: secretVersions, + secretPath, + envId, + projectId + }); + return secretVersionV2; + }; + const attachTags = async ({ secretName, tagSlugs, @@ -3279,6 +3310,53 @@ export const secretServiceFactory = ({ return secrets; }; + const getChangeVersions = async ( + change: { + secretVersion: string; + secretId?: string; + id?: string; + isUpdate?: boolean; + changeType?: string; + }, + previousVersion: string, + actorId: string, + actor: ActorType, + actorOrgId: string, + actorAuthMethod: ActorAuthMethod, + envId: string, + projectId: string, + secretPath: string + ) => { + const currentVersion = change.secretVersion; + const secretId = change.secretId ? change.secretId : change.id; + if (!secretId) { + return; + } + const versions = await getSecretVersionsV2ByIds({ + actorId, + actor, + actorOrgId, + actorAuthMethod, + secretId, + // if it's update add also the previous secretversionid + secretVersions: + change.isUpdate || change.changeType === ChangeType.UPDATE + ? [currentVersion, previousVersion] + : [currentVersion], + secretPath, + envId, + projectId + }); + return versions?.map((v) => ({ + secretKey: v.secretKey, + secretComment: v.secretComment, + skipMultilineEncoding: v.skipMultilineEncoding, + tags: v.tags?.map((tag) => tag.slug), + metadata: v.metadata, + secretValue: v.secretValue + })); + }; + return { attachTags, detachTags, @@ -3309,6 +3387,8 @@ export const secretServiceFactory = ({ getSecretsRawByFolderMappings, getSecretAccessList, getSecretByIdRaw, - getAccessibleSecrets + getAccessibleSecrets, + getSecretVersionsV2ByIds, + getChangeVersions }; }; diff --git a/backend/src/services/secret/secret-types.ts b/backend/src/services/secret/secret-types.ts index 30e3dfafa..91fc2eb6a 100644 --- a/backend/src/services/secret/secret-types.ts +++ b/backend/src/services/secret/secret-types.ts @@ -14,6 +14,7 @@ import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-fold import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; import { ActorType } from "../auth/auth-type"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "../kms/kms-service"; import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal"; import { ResourceMetadataDTO } from "../resource-metadata/resource-metadata-schema"; @@ -441,6 +442,7 @@ export type TCreateManySecretsRawFnFactory = { secretVersionV2BridgeDAL: Pick; secretVersionTagV2BridgeDAL: Pick; resourceMetadataDAL: Pick; + folderCommitService: Pick; }; export type TCreateManySecretsRawFn = { @@ -478,6 +480,7 @@ export type TUpdateManySecretsRawFnFactory = { secretVersionV2BridgeDAL: Pick; secretVersionTagV2BridgeDAL: Pick; resourceMetadataDAL: Pick; + folderCommitService: Pick; }; export type TUpdateManySecretsRawFn = { diff --git a/backend/src/services/smtp/emails/OrganizationInvitationTemplate.tsx b/backend/src/services/smtp/emails/OrganizationInvitationTemplate.tsx index b281e75d6..27092a843 100644 --- a/backend/src/services/smtp/emails/OrganizationInvitationTemplate.tsx +++ b/backend/src/services/smtp/emails/OrganizationInvitationTemplate.tsx @@ -5,8 +5,8 @@ import { BaseEmailWrapper, BaseEmailWrapperProps } from "./BaseEmailWrapper"; interface OrganizationInvitationTemplateProps extends Omit { metadata?: string; - inviterFirstName: string; - inviterUsername: string; + inviterFirstName?: string; + inviterUsername?: string; organizationName: string; email: string; organizationId: string; @@ -38,11 +38,19 @@ export const OrganizationInvitationTemplate = ({
- {inviterFirstName} ( - - {inviterUsername} - - ) has invited you to collaborate on {organizationName}. + {inviterFirstName && inviterUsername ? ( + <> + {inviterFirstName} ( + + {inviterUsername} + + ) has invited you to collaborate on {organizationName}. + + ) : ( + <> + You have been invited to collaborate on {organizationName}. + + )}
diff --git a/backend/src/services/user/user-service.ts b/backend/src/services/user/user-service.ts index aae32d91f..39300fec5 100644 --- a/backend/src/services/user/user-service.ts +++ b/backend/src/services/user/user-service.ts @@ -107,7 +107,8 @@ export const userServiceFactory = ({ }); await userDAL.updateById(user.id, { - isEmailVerified: true + isEmailVerified: true, + username: usersByusername.length === 1 && user.email ? user.email.toLowerCase() : undefined }); }; diff --git a/backend/src/services/webhook/webhook-fns.ts b/backend/src/services/webhook/webhook-fns.ts index a16158e14..6729e9a37 100644 --- a/backend/src/services/webhook/webhook-fns.ts +++ b/backend/src/services/webhook/webhook-fns.ts @@ -4,9 +4,12 @@ import { AxiosError } from "axios"; import picomatch from "picomatch"; import { TWebhooks } from "@app/db/schemas"; +import { TAuditLogServiceFactory } from "@app/ee/services/audit-log/audit-log-service"; +import { EventType, WebhookTriggeredEvent } from "@app/ee/services/audit-log/audit-log-types"; import { request } from "@app/lib/config/request"; import { NotFoundError } from "@app/lib/errors"; import { logger } from "@app/lib/logger"; +import { ActorType } from "@app/services/auth/auth-type"; import { TProjectDALFactory } from "../project/project-dal"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; @@ -163,6 +166,7 @@ export type TFnTriggerWebhookDTO = { projectEnvDAL: Pick; projectDAL: Pick; secretManagerDecryptor: (value: Buffer) => string; + auditLogService: Pick; }; // this is reusable function @@ -175,7 +179,8 @@ export const fnTriggerWebhook = async ({ projectEnvDAL, event, secretManagerDecryptor, - projectDAL + projectDAL, + auditLogService }: TFnTriggerWebhookDTO) => { const webhooks = await webhookDAL.findAllWebhooks(projectId, environment); const toBeTriggeredHooks = webhooks.filter( @@ -200,16 +205,43 @@ export const fnTriggerWebhook = async ({ }) ); + const eventPayloads: WebhookTriggeredEvent["metadata"][] = []; // filter hooks by status const successWebhooks = webhooksTriggered .filter(({ status }) => status === "fulfilled") - .map((_, i) => toBeTriggeredHooks[i].id); + .map((_, i) => { + eventPayloads.push({ + webhookId: toBeTriggeredHooks[i].id, + type: event.type, + payload: { + type: toBeTriggeredHooks[i].type!, + ...event.payload, + projectName + }, + status: "success" + } as WebhookTriggeredEvent["metadata"]); + + return toBeTriggeredHooks[i].id; + }); const failedWebhooks = webhooksTriggered .filter(({ status }) => status === "rejected") - .map((data, i) => ({ - id: toBeTriggeredHooks[i].id, - error: data.status === "rejected" ? (data.reason as AxiosError).message : "" - })); + .map((data, i) => { + eventPayloads.push({ + webhookId: toBeTriggeredHooks[i].id, + type: event.type, + payload: { + type: toBeTriggeredHooks[i].type!, + ...event.payload, + projectName + }, + status: "failed" + } as WebhookTriggeredEvent["metadata"]); + + return { + id: toBeTriggeredHooks[i].id, + error: data.status === "rejected" ? (data.reason as AxiosError).message : "" + }; + }); await webhookDAL.transaction(async (tx) => { const env = await projectEnvDAL.findOne({ projectId, slug: environment }, tx); @@ -236,5 +268,21 @@ export const fnTriggerWebhook = async ({ ); } }); + + for (const eventPayload of eventPayloads) { + // eslint-disable-next-line no-await-in-loop + await auditLogService.createAuditLog({ + actor: { + type: ActorType.PLATFORM, + metadata: {} + }, + projectId, + event: { + type: EventType.WEBHOOK_TRIGGERED, + metadata: eventPayload + } + }); + } + logger.info({ environment, secretPath, projectId }, "Secret webhook job ended"); }; diff --git a/cli/go.mod b/cli/go.mod index fc7322f61..e6d55eb49 100644 --- a/cli/go.mod +++ b/cli/go.mod @@ -14,7 +14,7 @@ require ( github.com/fatih/semgroup v1.2.0 github.com/gitleaks/go-gitdiff v0.9.1 github.com/h2non/filetype v1.1.3 - github.com/infisical/go-sdk v0.5.95 + github.com/infisical/go-sdk v0.5.96 github.com/infisical/infisical-kmip v0.3.5 github.com/mattn/go-isatty v0.0.20 github.com/muesli/ansi v0.0.0-20221106050444-61f0cd9a192a @@ -25,6 +25,7 @@ require ( github.com/pion/logging v0.2.3 github.com/pion/turn/v4 v4.0.0 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c + github.com/pkg/errors v0.9.1 github.com/posthog/posthog-go v0.0.0-20221221115252-24dfed35d71a github.com/quic-go/quic-go v0.50.0 github.com/rs/cors v1.11.0 @@ -106,7 +107,6 @@ require ( github.com/pion/randutil v0.1.0 // indirect github.com/pion/stun/v3 v3.0.0 // indirect github.com/pion/transport/v3 v3.0.7 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/shopspring/decimal v1.4.0 // indirect diff --git a/cli/go.sum b/cli/go.sum index aa8dc1f61..2e41c756b 100644 --- a/cli/go.sum +++ b/cli/go.sum @@ -292,12 +292,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/infisical/go-sdk v0.5.92 h1:PoCnVndrd6Dbkipuxl9fFiwlD5vCKsabtQo09mo8lUE= -github.com/infisical/go-sdk v0.5.92/go.mod h1:ExjqFLRz7LSpZpGluqDLvFl6dFBLq5LKyLW7GBaMAIs= -github.com/infisical/go-sdk v0.5.94 h1:wKBj+KpJEe+ZzOJ7koXQZDR0dLL9bt0Kqgf/1q+7tG4= -github.com/infisical/go-sdk v0.5.94/go.mod h1:ExjqFLRz7LSpZpGluqDLvFl6dFBLq5LKyLW7GBaMAIs= -github.com/infisical/go-sdk v0.5.95 h1:so0YwPofbT7j6Ao8Xcxee/o3ia33meuEVDU2vWr9yfs= -github.com/infisical/go-sdk v0.5.95/go.mod h1:ExjqFLRz7LSpZpGluqDLvFl6dFBLq5LKyLW7GBaMAIs= +github.com/infisical/go-sdk v0.5.96 h1:huky6bQ1Y3oRdPb5MO3Ru868qZaPHUxZ7kP7FPNRn48= +github.com/infisical/go-sdk v0.5.96/go.mod h1:ExjqFLRz7LSpZpGluqDLvFl6dFBLq5LKyLW7GBaMAIs= github.com/infisical/infisical-kmip v0.3.5 h1:QM3s0e18B+mYv3a9HQNjNAlbwZJBzXq5BAJM2scIeiE= github.com/infisical/infisical-kmip v0.3.5/go.mod h1:bO1M4YtKyutNg1bREPmlyZspC5duSR7hyQ3lPmLzrIs= github.com/jedib0t/go-pretty v4.3.0+incompatible h1:CGs8AVhEKg/n9YbUenWmNStRW2PHJzaeDodcfvRAbIo= diff --git a/cli/packages/cmd/agent.go b/cli/packages/cmd/agent.go index 445941674..cb10050dd 100644 --- a/cli/packages/cmd/agent.go +++ b/cli/packages/cmd/agent.go @@ -576,7 +576,7 @@ func (tm *AgentManager) FetchUniversalAuthAccessToken() (credential infisicalSdk } tm.cachedUniversalAuthClientSecret = clientSecret - if tm.removeUniversalAuthClientSecretOnRead { + if universalAuthConfig.RemoveClientSecretOnRead { defer os.Remove(universalAuthConfig.ClientSecretPath) } @@ -718,7 +718,7 @@ func (tm *AgentManager) FetchNewAccessToken() error { } // Refreshes the existing access token -func (tm *AgentManager) RefreshAccessToken() error { +func (tm *AgentManager) RefreshAccessToken(accessToken string) error { httpClient, err := util.GetRestyClientWithCustomHeaders() if err != nil { return err @@ -728,7 +728,6 @@ func (tm *AgentManager) RefreshAccessToken() error { SetRetryMaxWaitTime(20 * time.Second). SetRetryWaitTime(5 * time.Second) - accessToken := tm.GetToken() response, err := api.CallMachineIdentityRefreshAccessToken(httpClient, api.UniversalAuthRefreshRequest{AccessToken: accessToken}) if err != nil { return err @@ -752,18 +751,37 @@ func (tm *AgentManager) ManageTokenLifecycle() { accessTokenRefreshedTime = tm.accessTokenFetchedTime } - nextAccessTokenExpiresInTime := accessTokenRefreshedTime.Add(tm.accessTokenTTL - (5 * time.Second)) + // Calculate next expiry time at 2/3 of the TTL + nextAccessTokenExpiresInTime := accessTokenRefreshedTime.Add(tm.accessTokenTTL * 2 / 3) if tm.accessTokenFetchedTime.IsZero() && tm.accessTokenRefreshedTime.IsZero() { - // case: init login to get access token - log.Info().Msg("attempting to authenticate...") - err := tm.FetchNewAccessToken() - if err != nil { - log.Error().Msgf("unable to authenticate because %v. Will retry in 30 seconds", err) + // try to fetch token from sink files first + // if token is found, refresh the token right away and continue from there + isSavedTokenValid := false + token := tm.FetchTokenFromFiles() + if token != "" { + log.Info().Msg("found existing token in file, attempting to refresh...") + err := tm.RefreshAccessToken(token) + isSavedTokenValid = err == nil + if isSavedTokenValid { + log.Info().Msg("token refreshed successfully from saved file") + tm.accessTokenFetchedTime = time.Now() + } else { + log.Error().Msg("unable to refresh token from saved file") + } + } - // wait a bit before trying again - time.Sleep((30 * time.Second)) - continue + if !isSavedTokenValid { + // case: init login to get access token + log.Info().Msg("attempting to authenticate...") + err := tm.FetchNewAccessToken() + if err != nil { + log.Error().Msgf("unable to authenticate because %v. Will retry in 30 seconds", err) + + // wait a bit before trying again + time.Sleep((30 * time.Second)) + continue + } } } else if time.Now().After(accessTokenMaxTTLExpiresInTime) { // case: token has reached max ttl and we should re-authenticate entirely (cannot refresh) @@ -779,7 +797,7 @@ func (tm *AgentManager) ManageTokenLifecycle() { } else { // case: token ttl has expired, but the token is still within max ttl, so we can refresh log.Info().Msgf("attempting to refresh existing token...") - err := tm.RefreshAccessToken() + err := tm.RefreshAccessToken(tm.GetToken()) if err != nil { log.Error().Msgf("unable to refresh token because %v. Will retry in 30 seconds", err) @@ -800,15 +818,18 @@ func (tm *AgentManager) ManageTokenLifecycle() { accessTokenRefreshedTime = tm.accessTokenRefreshedTime } - nextAccessTokenExpiresInTime = accessTokenRefreshedTime.Add(tm.accessTokenTTL - (5 * time.Second)) + // Recalculate next expiry time at 2/3 of the TTL + nextAccessTokenExpiresInTime = accessTokenRefreshedTime.Add(tm.accessTokenTTL * 2 / 3) accessTokenMaxTTLExpiresInTime = tm.accessTokenFetchedTime.Add(tm.accessTokenMaxTTL - (5 * time.Second)) if nextAccessTokenExpiresInTime.After(accessTokenMaxTTLExpiresInTime) { - // case: Refreshed so close that the next refresh would occur beyond max ttl (this is because currently, token renew tries to add +access-token-ttl amount of time) - // example: access token ttl is 11 sec and max ttl is 30 sec. So it will start with 11 seconds, then 22 seconds but the next time you call refresh it would try to extend it to 33 but max ttl only allows 30, so the token will be valid until 30 before we need to reauth - time.Sleep(tm.accessTokenTTL - nextAccessTokenExpiresInTime.Sub(accessTokenMaxTTLExpiresInTime)) + // case: Refreshed so close that the next refresh would occur beyond max ttl + // Sleep until we're at 2/3 of the remaining time to max TTL + remainingTime := accessTokenMaxTTLExpiresInTime.Sub(time.Now()) + time.Sleep(remainingTime * 2 / 3) } else { - time.Sleep(tm.accessTokenTTL - (5 * time.Second)) + // Sleep until we're at 2/3 of the TTL + time.Sleep(tm.accessTokenTTL * 2 / 3) } } } @@ -830,6 +851,24 @@ func (tm *AgentManager) WriteTokenToFiles() { } } +func (tm *AgentManager) FetchTokenFromFiles() string { + for _, sinkFile := range tm.filePaths { + if sinkFile.Type == "file" { + tokenBytes, err := ioutil.ReadFile(sinkFile.Config.Path) + if err != nil { + log.Debug().Msgf("unable to read token from file '%s' because %v", sinkFile.Config.Path, err) + continue + } + + token := string(tokenBytes) + if token != "" { + return token + } + } + } + return "" +} + func (tm *AgentManager) WriteTemplateToFile(bytes *bytes.Buffer, template *Template) { if err := WriteBytesToFile(bytes, template.DestinationPath); err != nil { log.Error().Msgf("template engine: unable to write secrets to path because %s. Will try again on next cycle", err) diff --git a/cli/packages/cmd/dynamic_secrets.go b/cli/packages/cmd/dynamic_secrets.go index 8761b84ef..0443e7714 100644 --- a/cli/packages/cmd/dynamic_secrets.go +++ b/cli/packages/cmd/dynamic_secrets.go @@ -49,6 +49,11 @@ func getDynamicSecretList(cmd *cobra.Command, args []string) { util.HandleError(err, "Unable to parse flag") } + projectSlug, err := cmd.Flags().GetString("project-slug") + if err != nil { + util.HandleError(err, "Unable to parse flag") + } + secretsPath, err := cmd.Flags().GetString("path") if err != nil { util.HandleError(err, "Unable to parse path flag") @@ -60,10 +65,10 @@ func getDynamicSecretList(cmd *cobra.Command, args []string) { util.HandleError(err, "Unable to get resty client with custom headers") } - if projectId == "" { + if projectId == "" && projectSlug == "" { workspaceFile, err := util.GetWorkSpaceFromFile() if err != nil { - util.PrintErrorMessageAndExit("Please either run infisical init to connect to a project or pass in project id with --projectId flag") + util.PrintErrorMessageAndExit("Please either run infisical init to connect to a project, pass in project slug with --project-slug flag, or pass in project id with --projectId flag") } projectId = workspaceFile.WorkspaceId } @@ -100,13 +105,16 @@ func getDynamicSecretList(cmd *cobra.Command, args []string) { }) infisicalClient.Auth().SetAccessToken(infisicalToken) - projectDetails, err := api.CallGetProjectById(httpClient, projectId) - if err != nil { - util.HandleError(err, "To fetch project details") + if projectSlug == "" { + projectDetails, err := api.CallGetProjectById(httpClient, projectId) + if err != nil { + util.HandleError(err, "To fetch project details") + } + projectSlug = projectDetails.Slug } dynamicSecretRootCredentials, err := infisicalClient.DynamicSecrets().List(infisicalSdk.ListDynamicSecretsRootCredentialsOptions{ - ProjectSlug: projectDetails.Slug, + ProjectSlug: projectSlug, SecretPath: secretsPath, EnvironmentSlug: environmentName, }) @@ -156,6 +164,11 @@ func createDynamicSecretLeaseByName(cmd *cobra.Command, args []string) { util.HandleError(err, "Unable to parse flag") } + projectSlug, err := cmd.Flags().GetString("project-slug") + if err != nil { + util.HandleError(err, "Unable to parse flag") + } + ttl, err := cmd.Flags().GetString("ttl") if err != nil { util.HandleError(err, "Unable to parse flag") @@ -177,10 +190,10 @@ func createDynamicSecretLeaseByName(cmd *cobra.Command, args []string) { util.HandleError(err, "Unable to get resty client with custom headers") } - if projectId == "" { + if projectId == "" && projectSlug == "" { workspaceFile, err := util.GetWorkSpaceFromFile() if err != nil { - util.PrintErrorMessageAndExit("Please either run infisical init to connect to a project or pass in project id with --projectId flag") + util.PrintErrorMessageAndExit("Please either run infisical init to connect to a project, pass in project id with --projectId flag, or pass in project slug with --project-slug flag") } projectId = workspaceFile.WorkspaceId } @@ -216,14 +229,17 @@ func createDynamicSecretLeaseByName(cmd *cobra.Command, args []string) { }) infisicalClient.Auth().SetAccessToken(infisicalToken) - projectDetails, err := api.CallGetProjectById(httpClient, projectId) - if err != nil { - util.HandleError(err, "To fetch project details") + if projectSlug == "" { + projectDetails, err := api.CallGetProjectById(httpClient, projectId) + if err != nil { + util.HandleError(err, "To fetch project details") + } + projectSlug = projectDetails.Slug } dynamicSecretRootCredential, err := infisicalClient.DynamicSecrets().GetByName(infisicalSdk.GetDynamicSecretRootCredentialByNameOptions{ DynamicSecretName: dynamicSecretRootCredentialName, - ProjectSlug: projectDetails.Slug, + ProjectSlug: projectSlug, SecretPath: secretsPath, EnvironmentSlug: environmentName, }) @@ -232,13 +248,26 @@ func createDynamicSecretLeaseByName(cmd *cobra.Command, args []string) { util.HandleError(err, "To fetch dynamic secret root credentials details") } + // for Kubernetes dynamic secrets only + kubernetesNamespace, err := cmd.Flags().GetString("kubernetes-namespace") + if err != nil { + util.HandleError(err, "Unable to parse flag") + } + + config := map[string]any{} + if kubernetesNamespace != "" { + config["namespace"] = kubernetesNamespace + } + leaseCredentials, _, leaseDetails, err := infisicalClient.DynamicSecrets().Leases().Create(infisicalSdk.CreateDynamicSecretLeaseOptions{ DynamicSecretName: dynamicSecretRootCredential.Name, - ProjectSlug: projectDetails.Slug, + ProjectSlug: projectSlug, TTL: ttl, SecretPath: secretsPath, EnvironmentSlug: environmentName, + Config: config, }) + if err != nil { util.HandleError(err, "To lease dynamic secret") } @@ -291,6 +320,11 @@ func renewDynamicSecretLeaseByName(cmd *cobra.Command, args []string) { util.HandleError(err, "Unable to parse flag") } + projectSlug, err := cmd.Flags().GetString("project-slug") + if err != nil { + util.HandleError(err, "Unable to parse flag") + } + ttl, err := cmd.Flags().GetString("ttl") if err != nil { util.HandleError(err, "Unable to parse flag") @@ -307,10 +341,10 @@ func renewDynamicSecretLeaseByName(cmd *cobra.Command, args []string) { util.HandleError(err, "Unable to get resty client with custom headers") } - if projectId == "" { + if projectId == "" && projectSlug == "" { workspaceFile, err := util.GetWorkSpaceFromFile() if err != nil { - util.PrintErrorMessageAndExit("Please either run infisical init to connect to a project or pass in project id with --projectId flag") + util.PrintErrorMessageAndExit("Please either run infisical init to connect to a project, pass in project slug with --project-slug flag, or pass in project id with --projectId flag") } projectId = workspaceFile.WorkspaceId } @@ -347,9 +381,12 @@ func renewDynamicSecretLeaseByName(cmd *cobra.Command, args []string) { }) infisicalClient.Auth().SetAccessToken(infisicalToken) - projectDetails, err := api.CallGetProjectById(httpClient, projectId) - if err != nil { - util.HandleError(err, "To fetch project details") + if projectSlug == "" { + projectDetails, err := api.CallGetProjectById(httpClient, projectId) + if err != nil { + util.HandleError(err, "To fetch project details") + } + projectSlug = projectDetails.Slug } if err != nil { @@ -357,7 +394,7 @@ func renewDynamicSecretLeaseByName(cmd *cobra.Command, args []string) { } leaseDetails, err := infisicalClient.DynamicSecrets().Leases().RenewById(infisicalSdk.RenewDynamicSecretLeaseOptions{ - ProjectSlug: projectDetails.Slug, + ProjectSlug: projectSlug, TTL: ttl, SecretPath: secretsPath, EnvironmentSlug: environmentName, @@ -403,6 +440,11 @@ func revokeDynamicSecretLeaseByName(cmd *cobra.Command, args []string) { util.HandleError(err, "Unable to parse flag") } + projectSlug, err := cmd.Flags().GetString("project-slug") + if err != nil { + util.HandleError(err, "Unable to parse flag") + } + secretsPath, err := cmd.Flags().GetString("path") if err != nil { util.HandleError(err, "Unable to parse path flag") @@ -414,10 +456,10 @@ func revokeDynamicSecretLeaseByName(cmd *cobra.Command, args []string) { util.HandleError(err, "Unable to get resty client with custom headers") } - if projectId == "" { + if projectId == "" && projectSlug == "" { workspaceFile, err := util.GetWorkSpaceFromFile() if err != nil { - util.PrintErrorMessageAndExit("Please either run infisical init to connect to a project or pass in project id with --projectId flag") + util.PrintErrorMessageAndExit("Please either run infisical init to connect to a project, pass in project slug with --project-slug flag, or pass in project id with --projectId flag") } projectId = workspaceFile.WorkspaceId } @@ -454,9 +496,12 @@ func revokeDynamicSecretLeaseByName(cmd *cobra.Command, args []string) { }) infisicalClient.Auth().SetAccessToken(infisicalToken) - projectDetails, err := api.CallGetProjectById(httpClient, projectId) - if err != nil { - util.HandleError(err, "To fetch project details") + if projectSlug == "" { + projectDetails, err := api.CallGetProjectById(httpClient, projectId) + if err != nil { + util.HandleError(err, "To fetch project details") + } + projectSlug = projectDetails.Slug } if err != nil { @@ -464,11 +509,12 @@ func revokeDynamicSecretLeaseByName(cmd *cobra.Command, args []string) { } leaseDetails, err := infisicalClient.DynamicSecrets().Leases().DeleteById(infisicalSdk.DeleteDynamicSecretLeaseOptions{ - ProjectSlug: projectDetails.Slug, + ProjectSlug: projectSlug, SecretPath: secretsPath, EnvironmentSlug: environmentName, LeaseId: dynamicSecretLeaseId, }) + if err != nil { util.HandleError(err, "To revoke dynamic secret lease") } @@ -509,6 +555,11 @@ func listDynamicSecretLeaseByName(cmd *cobra.Command, args []string) { util.HandleError(err, "Unable to parse flag") } + projectSlug, err := cmd.Flags().GetString("project-slug") + if err != nil { + util.HandleError(err, "Unable to parse flag") + } + secretsPath, err := cmd.Flags().GetString("path") if err != nil { util.HandleError(err, "Unable to parse path flag") @@ -520,10 +571,10 @@ func listDynamicSecretLeaseByName(cmd *cobra.Command, args []string) { util.HandleError(err, "Unable to get resty client with custom headers") } - if projectId == "" { + if projectId == "" && projectSlug == "" { workspaceFile, err := util.GetWorkSpaceFromFile() if err != nil { - util.PrintErrorMessageAndExit("Please either run infisical init to connect to a project or pass in project id with --projectId flag") + util.PrintErrorMessageAndExit("Please either run infisical init to connect to a project, pass in project slug with --project-slug flag, or pass in project id with --projectId flag") } projectId = workspaceFile.WorkspaceId } @@ -559,14 +610,17 @@ func listDynamicSecretLeaseByName(cmd *cobra.Command, args []string) { }) infisicalClient.Auth().SetAccessToken(infisicalToken) - projectDetails, err := api.CallGetProjectById(httpClient, projectId) - if err != nil { - util.HandleError(err, "To fetch project details") + if projectSlug == "" { + projectDetails, err := api.CallGetProjectById(httpClient, projectId) + if err != nil { + util.HandleError(err, "To fetch project details") + } + projectSlug = projectDetails.Slug } dynamicSecretLeases, err := infisicalClient.DynamicSecrets().Leases().List(infisicalSdk.ListDynamicSecretLeasesOptions{ DynamicSecretName: dynamicSecretRootCredentialName, - ProjectSlug: projectDetails.Slug, + ProjectSlug: projectSlug, SecretPath: secretsPath, EnvironmentSlug: environmentName, }) @@ -583,30 +637,39 @@ func init() { dynamicSecretLeaseCreateCmd.Flags().StringP("path", "p", "/", "The path from where dynamic secret should be leased from") dynamicSecretLeaseCreateCmd.Flags().String("token", "", "Create dynamic secret leases using machine identity access token") dynamicSecretLeaseCreateCmd.Flags().String("projectId", "", "Manually set the projectId to fetch leased from when using machine identity based auth") + dynamicSecretLeaseCreateCmd.Flags().String("project-slug", "", "Manually set the project-slug to create lease in") dynamicSecretLeaseCreateCmd.Flags().String("ttl", "", "The lease lifetime TTL. If not provided the default TTL of dynamic secret will be used.") dynamicSecretLeaseCreateCmd.Flags().Bool("plain", false, "Print leased credentials without formatting, one per line") + + // Kubernetes specific flags + dynamicSecretLeaseCreateCmd.Flags().String("kubernetes-namespace", "", "The namespace to create the lease in. Only used for Kubernetes dynamic secrets.") + dynamicSecretLeaseCmd.AddCommand(dynamicSecretLeaseCreateCmd) dynamicSecretLeaseListCmd.Flags().StringP("path", "p", "/", "The path from where dynamic secret should be leased from") dynamicSecretLeaseListCmd.Flags().String("token", "", "Fetch dynamic secret leases machine identity access token") dynamicSecretLeaseListCmd.Flags().String("projectId", "", "Manually set the projectId to fetch leased from when using machine identity based auth") + dynamicSecretLeaseListCmd.Flags().String("project-slug", "", "Manually set the project-slug to list leases from") dynamicSecretLeaseCmd.AddCommand(dynamicSecretLeaseListCmd) dynamicSecretLeaseRenewCmd.Flags().StringP("path", "p", "/", "The path from where dynamic secret should be leased from") dynamicSecretLeaseRenewCmd.Flags().String("token", "", "Renew dynamic secrets machine identity access token") dynamicSecretLeaseRenewCmd.Flags().String("projectId", "", "Manually set the projectId to fetch leased from when using machine identity based auth") + dynamicSecretLeaseRenewCmd.Flags().String("project-slug", "", "Manually set the project-slug to renew lease in") dynamicSecretLeaseRenewCmd.Flags().String("ttl", "", "The lease lifetime TTL. If not provided the default TTL of dynamic secret will be used.") dynamicSecretLeaseCmd.AddCommand(dynamicSecretLeaseRenewCmd) dynamicSecretLeaseRevokeCmd.Flags().StringP("path", "p", "/", "The path from where dynamic secret should be leased from") dynamicSecretLeaseRevokeCmd.Flags().String("token", "", "Delete dynamic secrets using machine identity access token") dynamicSecretLeaseRevokeCmd.Flags().String("projectId", "", "Manually set the projectId to fetch leased from when using machine identity based auth") + dynamicSecretLeaseRevokeCmd.Flags().String("project-slug", "", "Manually set the project-slug to revoke lease from") dynamicSecretLeaseCmd.AddCommand(dynamicSecretLeaseRevokeCmd) dynamicSecretCmd.AddCommand(dynamicSecretLeaseCmd) dynamicSecretCmd.Flags().String("token", "", "Fetch secrets using service token or machine identity access token") dynamicSecretCmd.Flags().String("projectId", "", "Manually set the projectId to fetch dynamic-secret when using machine identity based auth") + dynamicSecretCmd.Flags().String("project-slug", "", "Manually set the project-slug to fetch dynamic-secret from") dynamicSecretCmd.PersistentFlags().String("env", "dev", "Used to select the environment name on which actions should be taken on") dynamicSecretCmd.Flags().String("path", "/", "get dynamic secret within a folder path") rootCmd.AddCommand(dynamicSecretCmd) diff --git a/cli/packages/gateway/connection.go b/cli/packages/gateway/connection.go index 3f4ffdf03..980137374 100644 --- a/cli/packages/gateway/connection.go +++ b/cli/packages/gateway/connection.go @@ -108,17 +108,17 @@ func handleStream(stream quic.Stream, quicConn quic.Connection) { return case "FORWARD-HTTP": + targetURL := "" argParts := bytes.Split(args, []byte(" ")) - if len(argParts) == 0 { - log.Error().Msg("FORWARD-HTTP requires target URL") - return - } - targetURL := string(argParts[0]) - - if !isValidURL(targetURL) { - log.Error().Msgf("Invalid target URL: %s", targetURL) - return + if len(argParts) == 0 || len(argParts[0]) == 0 { + log.Warn().Msg("FORWARD-HTTP used without a target URL.") + } else { + targetURL = string(argParts[0]) + if !isValidURL(targetURL) { + log.Error().Msgf("Invalid target URL: %s", targetURL) + return + } } // Parse optional parameters @@ -183,11 +183,6 @@ func handleHTTPProxy(stream quic.Stream, reader *bufio.Reader, targetURL string, transport.TLSClientConfig = tlsConfig } - client := &http.Client{ - Transport: transport, - Timeout: 30 * time.Second, - } - // Loop to handle multiple HTTP requests on the same stream for { req, err := http.ReadRequest(reader) @@ -201,18 +196,53 @@ func handleHTTPProxy(stream quic.Stream, reader *bufio.Reader, targetURL string, } log.Info().Msgf("Received HTTP request: %s", req.URL.Path) - actionHeader := req.Header.Get("x-infisical-action") + actionHeader := HttpProxyAction(req.Header.Get(INFISICAL_HTTP_PROXY_ACTION_HEADER)) if actionHeader != "" { - if actionHeader == "inject-k8s-sa-auth-token" { - token, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token") + if actionHeader == HttpProxyActionInjectGatewayK8sServiceAccountToken { + token, err := os.ReadFile(KUBERNETES_SERVICE_ACCOUNT_TOKEN_PATH) if err != nil { stream.Write([]byte(buildHttpInternalServerError("failed to read k8s sa auth token"))) continue // Continue to next request instead of returning } req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", string(token))) log.Info().Msgf("Injected gateway k8s SA auth token in request to %s", targetURL) + } else if actionHeader == HttpProxyActionUseGatewayK8sServiceAccount { // will work without a target URL set + // set the ca cert to the pod's k8s service account ca cert: + caCert, err := os.ReadFile(KUBERNETES_SERVICE_ACCOUNT_CA_CERT_PATH) + if err != nil { + stream.Write([]byte(buildHttpInternalServerError("failed to read k8s sa ca cert"))) + continue + } + + caCertPool := x509.NewCertPool() + if ok := caCertPool.AppendCertsFromPEM(caCert); !ok { + stream.Write([]byte(buildHttpInternalServerError("failed to parse k8s sa ca cert"))) + continue + } + + transport.TLSClientConfig = &tls.Config{ + RootCAs: caCertPool, + } + + // set authorization header to the pod's k8s service account token: + token, err := os.ReadFile(KUBERNETES_SERVICE_ACCOUNT_TOKEN_PATH) + if err != nil { + stream.Write([]byte(buildHttpInternalServerError("failed to read k8s sa auth token"))) + continue + } + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", string(token))) + + // update the target URL to point to the kubernetes API server: + kubernetesServiceHost := os.Getenv(KUBERNETES_SERVICE_HOST_ENV_NAME) + kubernetesServicePort := os.Getenv(KUBERNETES_SERVICE_PORT_HTTPS_ENV_NAME) + + fullBaseUrl := fmt.Sprintf("https://%s:%s", kubernetesServiceHost, kubernetesServicePort) + targetURL = fullBaseUrl + + log.Info().Msgf("Redirected request to Kubernetes API server: %s", targetURL) } - req.Header.Del("x-infisical-action") + + req.Header.Del(INFISICAL_HTTP_PROXY_ACTION_HEADER) } // Build full target URL @@ -242,6 +272,11 @@ func handleHTTPProxy(stream quic.Stream, reader *bufio.Reader, targetURL string, log.Info().Msgf("Proxying %s %s to %s", req.Method, req.URL.Path, targetFullURL) + client := &http.Client{ + Transport: transport, + Timeout: 30 * time.Second, + } + resp, err := client.Do(proxyReq) if err != nil { log.Error().Msgf("Failed to reach target: %v", err) diff --git a/cli/packages/gateway/constants.go b/cli/packages/gateway/constants.go new file mode 100644 index 000000000..aa260ed2e --- /dev/null +++ b/cli/packages/gateway/constants.go @@ -0,0 +1,17 @@ +package gateway + +const ( + KUBERNETES_SERVICE_HOST_ENV_NAME = "KUBERNETES_SERVICE_HOST" + KUBERNETES_SERVICE_PORT_HTTPS_ENV_NAME = "KUBERNETES_SERVICE_PORT_HTTPS" + KUBERNETES_SERVICE_ACCOUNT_CA_CERT_PATH = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + KUBERNETES_SERVICE_ACCOUNT_TOKEN_PATH = "/var/run/secrets/kubernetes.io/serviceaccount/token" + + INFISICAL_HTTP_PROXY_ACTION_HEADER = "x-infisical-action" +) + +type HttpProxyAction string + +const ( + HttpProxyActionInjectGatewayK8sServiceAccountToken HttpProxyAction = "inject-k8s-sa-auth-token" + HttpProxyActionUseGatewayK8sServiceAccount HttpProxyAction = "use-k8s-sa" +) diff --git a/docs/api-reference/endpoints/dynamic-secrets/kubernetes/create-lease.mdx b/docs/api-reference/endpoints/dynamic-secrets/kubernetes/create-lease.mdx new file mode 100644 index 000000000..1a4e26709 --- /dev/null +++ b/docs/api-reference/endpoints/dynamic-secrets/kubernetes/create-lease.mdx @@ -0,0 +1,4 @@ +--- +title: "Create Kubernetes Lease" +openapi: "POST /api/v1/dynamic-secrets/leases/kubernetes" +--- diff --git a/docs/cli/commands/dynamic-secrets.mdx b/docs/cli/commands/dynamic-secrets.mdx index c345c3e2d..76ea96cf5 100644 --- a/docs/cli/commands/dynamic-secrets.mdx +++ b/docs/cli/commands/dynamic-secrets.mdx @@ -49,7 +49,7 @@ export INFISICAL_DISABLE_UPDATE_CHECK=true ### Flags - The project ID to fetch dynamic secrets from. This is required when using a machine identity to authenticate. + The project ID to fetch dynamic secrets from. ```bash # Example @@ -58,6 +58,16 @@ infisical dynamic-secrets --projectId= + + The project slug to fetch dynamic secrets from. + +```bash +# Example +infisical dynamic-secrets --project-slug= +``` + + + The authenticated token to fetch dynamic secrets from. This is required when using a machine identity to authenticate. @@ -119,7 +129,7 @@ infisical dynamic-secrets lease create --path="/" --env=de - The project ID of the dynamic secrets to lease from. This is required when using a machine identity to authenticate. + The project ID of the dynamic secrets to lease from. ```bash # Example @@ -128,6 +138,16 @@ infisical dynamic-secrets lease create --projectId= + + The project slug of the dynamic secrets to lease from. + +```bash +# Example +infisical dynamic-secrets lease create --project-slug= +``` + + + The authenticated token to create dynamic secret leases. This is required when using a machine identity to authenticate. @@ -148,6 +168,22 @@ infisical dynamic-secrets lease create --ttl= +### Provider-specific flags + +The following flags are specific to certain providers or integrations: + + + + The namespace to create the lease in. Only used for Kubernetes dynamic secrets. + + ```bash + # Example + infisical dynamic-secrets lease create --kubernetes-namespace= + ``` + + + + This command is used to list leases for a dynamic secret. @@ -174,7 +210,7 @@ infisical dynamic-secrets lease list --path="/" --env=dev - The project ID of the dynamic secrets to list leases from. This is required when using a machine identity to authenticate. + The project ID of the dynamic secrets to list leases from. ```bash # Example @@ -183,6 +219,16 @@ infisical dynamic-secrets lease list --projectId= + + The project slug of the dynamic secrets to list leases from. + +```bash +# Example +infisical dynamic-secrets lease list --project-slug= +``` + + + The authenticated token to list dynamic secret leases. This is required when using a machine identity to authenticate. @@ -219,7 +265,7 @@ infisical dynamic-secrets lease renew --path="/" --env=dev - The project ID of the dynamic secret's lease from. This is required when using a machine identity to authenticate. + The project ID of the dynamic secret to lease from. ```bash # Example @@ -228,6 +274,16 @@ infisical dynamic-secrets lease renew --projectId= + + The project slug of the dynamic secret to lease from. + +```bash +# Example +infisical dynamic-secrets lease renew --project-slug= +``` + + + The authenticated token to create dynamic secret leases. This is required when using a machine identity to authenticate. @@ -274,7 +330,7 @@ infisical dynamic-secrets lease delete --path="/" --env=dev - The project ID of the dynamic secret's lease from. This is required when using a machine identity to authenticate. + The project ID of the dynamic secret to delete lease from. ```bash # Example @@ -283,6 +339,16 @@ infisical dynamic-secrets lease delete --projectId= + + The project slug of the dynamic secret to delete lease from. + +```bash +# Example +infisical dynamic-secrets lease delete --project-slug= +``` + + + The authenticated token to delete dynamic secret leases. This is required when using a machine identity to authenticate. diff --git a/docs/cli/usage.mdx b/docs/cli/usage.mdx index a3bfd83f0..8d0236ed5 100644 --- a/docs/cli/usage.mdx +++ b/docs/cli/usage.mdx @@ -120,6 +120,12 @@ The CLI is designed for a variety of secret management applications ranging from + + Starting with CLI version v0.4.0, you can now choose to log in via Infisical Cloud (US/EU) or your own self-hosted instance by simply running `infisical login` and following the on-screen instructions — no need to manually set the `INFISICAL_API_URL` environment variable. + + For versions prior to v0.4.0, the CLI defaults to the US Cloud. To connect to the EU Cloud or a self-hosted instance, set the `INFISICAL_API_URL` environment variable to `https://eu.infisical.com` or your custom URL. + + ## Custom Request Headers diff --git a/docs/documentation/platform/audit-log-streams/audit-log-streams.mdx b/docs/documentation/platform/audit-log-streams/audit-log-streams.mdx index 2f38a3930..2ee4dff92 100644 --- a/docs/documentation/platform/audit-log-streams/audit-log-streams.mdx +++ b/docs/documentation/platform/audit-log-streams/audit-log-streams.mdx @@ -5,12 +5,12 @@ description: "Learn how to stream Infisical Audit Logs to external logging provi Audit log streams is a paid feature. - + If you're using Infisical Cloud, then it is available under the **Enterprise Tier**. If you're self-hosting Infisical, then you should contact team@infisical.com to purchase an enterprise license to use it. -Infisical Audit Log Streaming enables you to transmit your organization's Audit Logs to external logging providers for monitoring and analysis. +Infisical Audit Log Streaming enables you to transmit your organization's Audit Logs to external logging providers for monitoring and analysis. The logs are formatted in JSON, requiring your logging provider to support JSON-based log parsing. @@ -118,7 +118,7 @@ Each log entry sent to the external logging provider will follow the same struct ### Audit Logs Structure - The unique identifier for the log entry. + The unique identifier for the log entry. @@ -168,7 +168,7 @@ Each log entry sent to the external logging provider will follow the same struct If the `actor` field is set to `platform`, `scimClient`, or `unknownUser`, the `actorMetadata` field will be an empty object. - + @@ -178,7 +178,7 @@ Each log entry sent to the external logging provider will follow the same struct The type of event that occurred. Below you can see a list of possible event types. More event types will be added in the future as we expand our audit logs further. - `get-secrets`, `delete-secrets`, `get-secret`, `create-secret`, `update-secret`, `delete-secret`, `get-workspace-key`, `authorize-integration`, `update-integration-auth`, `unauthorize-integration`, `create-integration`, `delete-integration`, `add-trusted-ip`, `update-trusted-ip`, `delete-trusted-ip`, `create-service-token`, `delete-service-token`, `create-identity`, `update-identity`, `delete-identity`, `login-identity-universal-auth`, `add-identity-universal-auth`, `update-identity-universal-auth`, `get-identity-universal-auth`, `create-identity-universal-auth-client-secret`, `revoke-identity-universal-auth-client-secret`, `get-identity-universal-auth-client-secret`, `create-environment`, `update-environment`, `delete-environment`, `add-workspace-member`, `remove-workspace-member`, `create-folder`, `update-folder`, `delete-folder`, `create-webhook`, `update-webhook-status`, `delete-webhook`, `get-secret-imports`, `create-secret-import`, `update-secret-import`, `delete-secret-import`, `update-user-workspace-role`, `update-user-workspace-denied-permissions`, `create-certificate-authority`, `get-certificate-authority`, `update-certificate-authority`, `delete-certificate-authority`, `get-certificate-authority-csr`, `get-certificate-authority-cert`, `sign-intermediate`, `import-certificate-authority-cert`, `get-certificate-authority-crl`, `issue-cert`, `get-cert`, `delete-cert`, `revoke-cert`, `get-cert-body`, `create-pki-alert`, `get-pki-alert`, `update-pki-alert`, `delete-pki-alert`, `create-pki-collection`, `get-pki-collection`, `update-pki-collection`, `delete-pki-collection`, `get-pki-collection-items`, `add-pki-collection-item`, `delete-pki-collection-item`, `org-admin-accessed-project`, `create-certificate-template`, `update-certificate-template`, `delete-certificate-template`, `get-certificate-template`, `create-certificate-template-est-config`, `update-certificate-template-est-config`, `get-certificate-template-est-config`, `update-project-slack-config`, `get-project-slack-config`, `integration-synced`, `create-shared-secret`, `delete-shared-secret`, `read-shared-secret`. + `get-secrets`, `delete-secrets`, `get-secret`, `create-secret`, `update-secret`, `delete-secret`, `get-workspace-key`, `authorize-integration`, `update-integration-auth`, `unauthorize-integration`, `create-integration`, `delete-integration`, `add-trusted-ip`, `update-trusted-ip`, `delete-trusted-ip`, `create-service-token`, `delete-service-token`, `create-identity`, `update-identity`, `delete-identity`, `login-identity-universal-auth`, `add-identity-universal-auth`, `update-identity-universal-auth`, `get-identity-universal-auth`, `create-identity-universal-auth-client-secret`, `revoke-identity-universal-auth-client-secret`, `get-identity-universal-auth-client-secret`, `create-environment`, `update-environment`, `delete-environment`, `add-workspace-member`, `remove-workspace-member`, `create-folder`, `update-folder`, `delete-folder`, `create-webhook`, `update-webhook-status`, `delete-webhook`, `webhook-triggered`, `get-secret-imports`, `create-secret-import`, `update-secret-import`, `delete-secret-import`, `update-user-workspace-role`, `update-user-workspace-denied-permissions`, `create-certificate-authority`, `get-certificate-authority`, `update-certificate-authority`, `delete-certificate-authority`, `get-certificate-authority-csr`, `get-certificate-authority-cert`, `sign-intermediate`, `import-certificate-authority-cert`, `get-certificate-authority-crl`, `issue-cert`, `get-cert`, `delete-cert`, `revoke-cert`, `get-cert-body`, `create-pki-alert`, `get-pki-alert`, `update-pki-alert`, `delete-pki-alert`, `create-pki-collection`, `get-pki-collection`, `update-pki-collection`, `delete-pki-collection`, `get-pki-collection-items`, `add-pki-collection-item`, `delete-pki-collection-item`, `org-admin-accessed-project`, `create-certificate-template`, `update-certificate-template`, `delete-certificate-template`, `get-certificate-template`, `create-certificate-template-est-config`, `update-certificate-template-est-config`, `get-certificate-template-est-config`, `update-project-slack-config`, `get-project-slack-config`, `integration-synced`, `create-shared-secret`, `delete-shared-secret`, `read-shared-secret`. @@ -219,4 +219,4 @@ Each log entry sent to the external logging provider will follow the same struct The name of the project where the event occurred. The `projectName` field will only be present if the event occurred at the project level, not the organization level. - \ No newline at end of file + diff --git a/docs/documentation/platform/dynamic-secrets/aws-iam.mdx b/docs/documentation/platform/dynamic-secrets/aws-iam.mdx index aa10e4029..a02d80a5c 100644 --- a/docs/documentation/platform/dynamic-secrets/aws-iam.mdx +++ b/docs/documentation/platform/dynamic-secrets/aws-iam.mdx @@ -32,7 +32,8 @@ Infisical needs an initial AWS IAM user with the required permissions to create "iam:ListUserPolicies", "iam:PutUserPolicy", "iam:AddUserToGroup", - "iam:RemoveUserFromGroup" + "iam:RemoveUserFromGroup", + "iam:TagUser" ], "Resource": ["*"] } diff --git a/docs/documentation/platform/dynamic-secrets/kubernetes.mdx b/docs/documentation/platform/dynamic-secrets/kubernetes.mdx index 713aefae6..87c5b3e89 100644 --- a/docs/documentation/platform/dynamic-secrets/kubernetes.mdx +++ b/docs/documentation/platform/dynamic-secrets/kubernetes.mdx @@ -162,6 +162,12 @@ This feature is ideal for scenarios where you need to: tokens for the target service account. + + When using Gateway authentication, the Gateway will access the Kubernetes API server + using its internal cluster URL (typically https://kubernetes.default.svc) and TLS configuration. + You don't need to specify these values separately in the dynamic secret configuration. + + 1. Deploy the Infisical Gateway in your cluster 2. Set up RBAC permissions for the Gateway's service account: ```yaml rbac.yaml @@ -206,6 +212,7 @@ This feature is ideal for scenarios where you need to: - Automatically clean up service accounts after token expiration - Assign different roles to different users or applications - Maintain strict control over service account permissions + - Support multiple namespaces with a single dynamic secret configuration ### Prerequisites @@ -213,6 +220,16 @@ This feature is ideal for scenarios where you need to: - Cluster access token with permissions to create service accounts and manage RBAC - (Optional) [Gateway](/documentation/platform/gateways/overview) for private cluster access + ### Namespace Support + + When configuring a dynamic secret, you can specify multiple allowed namespaces as a comma-separated list. During lease creation, you can then specify which namespace to use from this allowed list. This provides flexibility while maintaining security by: + + - Allowing a single dynamic secret configuration to support multiple namespaces + - Restricting service account creation to only the specified allowed namespaces + - Enabling fine-grained control over which namespaces can be used for each lease + + For example, if you configure a dynamic secret with allowed namespaces "default,kube-system,monitoring", you can create leases that use any of these namespaces while preventing access to other namespaces in your cluster. + ### Authentication Setup Choose your authentication method: @@ -318,6 +335,12 @@ This feature is ideal for scenarios where you need to: manage service accounts, their tokens, and RBAC resources. + + When using Gateway authentication, the Gateway will access the Kubernetes API server + using its internal cluster URL (typically https://kubernetes.default.svc) and TLS configuration. + You don't need to specify these values separately in the dynamic secret configuration. + + 1. Deploy the Infisical Gateway in your cluster 2. Set up RBAC permissions for the Gateway's service account: ```yaml rbac.yaml @@ -401,13 +424,13 @@ This feature is ideal for scenarios where you need to: Select a gateway for private cluster access. If not specified, the Internet Gateway will be used. - Kubernetes API server URL (e.g., https://kubernetes.default.svc) + Kubernetes API server URL (e.g., https://kubernetes.default.svc). Not required when using Gateway authentication as the Gateway will use its internal cluster URL. - Whether to enable SSL verification for the Kubernetes API server connection. + Whether to enable SSL verification for the Kubernetes API server connection. Not required when using Gateway authentication as the Gateway will use its internal TLS configuration. - Custom CA certificate for the Kubernetes API server. Leave blank to use the system/public CA. + Custom CA certificate for the Kubernetes API server. Leave blank to use the system/public CA. Not required when using Gateway authentication as the Gateway will use its internal TLS configuration. Choose between Token (API) or Gateway authentication. If using Gateway, the Gateway must be deployed in your Kubernetes cluster. @@ -418,18 +441,30 @@ This feature is ideal for scenarios where you need to: Choose between Static (predefined service account) or Dynamic (temporary service accounts with role assignments) - - Name of the service account to generate tokens for (required for Static credentials) - - - Kubernetes namespace where the service account exists or will be created - - - Type of role to assign (ClusterRole or Role) (required for Dynamic credentials) - - - Name of the role to assign to the temporary service account (required for Dynamic credentials) - + + + + + Name of the service account to generate tokens for + + + Kubernetes namespace where the service account exists + + + + + + Kubernetes namespace(s) where the service accounts will be created. You can specify multiple namespaces as a comma-separated list (e.g., "default,kube-system"). During lease creation, you can specify which namespace to use from this allowed list. + + + Type of role to assign (ClusterRole or Role) + + + Name of the role to assign to the temporary service account + + + + Optional list of audiences to include in the generated token diff --git a/docs/documentation/platform/pit-recovery.mdx b/docs/documentation/platform/pit-recovery.mdx index 448faddbb..24d0071a7 100644 --- a/docs/documentation/platform/pit-recovery.mdx +++ b/docs/documentation/platform/pit-recovery.mdx @@ -4,38 +4,130 @@ description: "Learn how to rollback secrets and configurations to any snapshot w --- - Point-in-Time Recovery is a paid feature. - - If you're using Infisical Cloud, then it is available under the **Pro Tier**. If you're self-hosting Infisical, - then you should contact sales@infisical.com to purchase an enterprise license to use it. + Point-in-Time Recovery is a paid feature. If you're using Infisical Cloud, + then it is available under the **Pro Tier**. If you're self-hosting Infisical, + then you should contact sales@infisical.com to purchase an enterprise license + to use it. Infisical's point-in-time recovery functionality allows secrets to be rolled back to any point in time for any given [folder](./folder) or [environment](/documentation/platform/project#project-environments). -Every time a secret is updated, a new snapshot is taken – capturing the state of the folder and environment at that point of time. -## Snapshots + + + ## Understanding Commits -Similar to Git, a commit (also known as snapshot) in Infisical is the state of your project's secrets at a specific point in time scoped to -an environment and [folder](./folder) within it. + Similar to Git, a commit in Infisical represents a snapshot of changes made to your project's resources at a specific point in time. Each commit is scoped to an environment and [folder](./folder) within it. Unlike the legacy snapshot system, the new commits interface provides granular tracking of individual changes, allowing you to see exactly what was modified, added, or removed in each commit. -To view a list of snapshots for the current folder, press the **Commits** button. + ### Accessing Commits -![PIT commits](../../images/platform/pit-recovery/pit-recovery-commits.png) + From your secrets management interface, you can access the commits functionality by clicking the **Commits Button**. This button is located in the top-right area of your secrets view and shows the number of commits for the current folder (e.g., "4 Commits"). -This opens up a sidebar from which you can select to view a particular snapshot: + ![Commits Button](../../images/platform/pit-recovery/pit-recovery-revamp/pit-commits-button.png) -![PIT snapshots](../../images/platform/pit-recovery/pit-recovery-commits-drawer.png) + ### Commits List View -## Rolling back + The commits page displays a comprehensive chronological history of all changes made to your environment and folders: -After pressing on a snapshot from the sidebar, you can view it and roll back the state -of the folder to that point in time by pressing the **Rollback** button. + ![Commits List View](../../images/platform/pit-recovery/pit-recovery-revamp/pit-commits-history.png) -![PIT snapshot](../../images/platform/pit-recovery/pit-recovery-rollback.png) + - **Chronological Sorting**: Commits are grouped by date + - **Commit Information**: Each commit shows: + - Commit message + - Author information + - Relative timestamp + - Unique commit hash identifier + - **Search Functionality**: Use the search bar to quickly find specific commits + - **Sorting Options**: Sort commits by various criteria using the sort controls -Rolling back secrets to a past snapshot creates a creates a snapshot at the top of the stack and updates secret versions. + ### Detailed Commit Inspection - -Rollbacks are localized to not affect other folders within the same environment. This means each [folder](./folder) maintains its own independent history of changes, offering precise and isolated control over rollback actions. -Put differently, every [folder](./folder) possesses a distinct and separate timeline, providing granular control when managing your secrets. - \ No newline at end of file + Clicking on any commit from the list opens a detailed view showing the list of changes made in that commit. + + ![Detailed Commit Inspection](../../images/platform/pit-recovery/pit-recovery-revamp/pit-commit-changes.png) + + #### Change Categories + + The commit changes details can be grouped into the following categories: + + **Folder Changes** + - Shows folder additions, modifications, or deletions + - Displays the folder properties changes in JSON format, including: + - Folder name + - Folder description + + **Secret Changes** + - Lists all secrets that were added, updated, or removed + - Shows the complete secret configuration including: + - Secret key and value + - Comments, tags and metadata + - Encoding settings (e.g., skipMultilineEncoding) + - Values are displayed with appropriate masking for security + + **Visual Indicators** + - Green "+" indicators show additions + - Red "-" indicators show deletions + - Modified content shows both old and new states + + ### Restoration Options + + Each commit provides two distinct restoration methods accessible via the **Restore Options** dropdown: + + ![Restore Options](../../images/platform/pit-recovery/pit-recovery-revamp/pit-commit-changes-options.png) + + #### Revert changes + This option provides surgical precision for undoing specific modifications: + + - **Granular Control**: Reverts only the specific changes introduced in that individual commit + - **Selective Restoration**: Preserves all other changes made after the commit + - **Targeted Undo**: Perfect for reversing a specific problematic change without affecting other work + - **Minimal Impact**: Only affects the resources that were modified in that particular commit + - **Use Case**: Ideal when you want to undo a specific change while keeping all other modifications intact + + #### Roll back to this commit + This option performs a complete restoration to the selected point in time: + + ![Rollback](../../images/platform/pit-recovery/pit-recovery-revamp/pit-commit-restore.png) + + - **Complete State Restoration**: Returns the entire folder to its exact state at the time of this commit + - **Restore All Child Folders**: If enabled, it'll also restore all nested folders to their exact state at the time of this commit. + - **Destructive Operation**: Discards ALL changes made after the selected commit + - **New Commit Creation**: Creates a new commit representing this rollback operation + - **Use Case**: Ideal when you want to completely undo a series of changes and return to a known good state + + **Warning**: This operation will undo all modifications made after the selected commit, which may include multiple secrets and configuration changes. + + + + + The snapshots interface is deprecated and will be removed in a future version. Please use the new Commits interface for more granular point-in-time recovery operations. + + + ## Snapshots + + Similar to Git, a commit (also known as snapshot) in Infisical is the state of your project's secrets at a specific point in time scoped to + an environment and [folder](./folder) within it. + + To view a list of snapshots for the current folder, press the **Commits** button. + + ![PIT commits](../../images/platform/pit-recovery/pit-recovery-commits.png) + + This opens up a sidebar from which you can select to view a particular snapshot: + + ![PIT snapshots](../../images/platform/pit-recovery/pit-recovery-commits-drawer.png) + + ## Rolling back + + After pressing on a snapshot from the sidebar, you can view it and roll back the state + of the folder to that point in time by pressing the **Rollback** button. + + ![PIT snapshot](../../images/platform/pit-recovery/pit-recovery-rollback.png) + + Rolling back secrets to a past snapshot creates a snapshot at the top of the stack and updates secret versions. + + + Rollbacks are localized to not affect other folders within the same environment. This means each [folder](./folder) maintains its own independent history of changes, offering precise and isolated control over rollback actions. + Put differently, every [folder](./folder) possesses a distinct and separate timeline, providing granular control when managing your secrets. + + + + diff --git a/docs/documentation/platform/webhooks.mdx b/docs/documentation/platform/webhooks.mdx index 92d3ff8b8..e6e71b9b0 100644 --- a/docs/documentation/platform/webhooks.mdx +++ b/docs/documentation/platform/webhooks.mdx @@ -27,7 +27,7 @@ If the signature in the header matches the signature that you generated, then yo ```json { - "event": "secret.modified", + "event": "secrets.modified", "project": { "workspaceId": "the workspace id", "environment": "project environment", diff --git a/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commit-changes-options.png b/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commit-changes-options.png new file mode 100644 index 000000000..60d9dff01 Binary files /dev/null and b/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commit-changes-options.png differ diff --git a/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commit-changes.png b/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commit-changes.png new file mode 100644 index 000000000..56357e9ef Binary files /dev/null and b/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commit-changes.png differ diff --git a/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commit-restore.png b/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commit-restore.png new file mode 100644 index 000000000..3ed22c63b Binary files /dev/null and b/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commit-restore.png differ diff --git a/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commits-button.png b/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commits-button.png new file mode 100644 index 000000000..273760ca5 Binary files /dev/null and b/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commits-button.png differ diff --git a/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commits-history.png b/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commits-history.png new file mode 100644 index 000000000..3832209f4 Binary files /dev/null and b/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commits-history.png differ diff --git a/docs/integrations/frameworks/packer.mdx b/docs/integrations/frameworks/packer.mdx new file mode 100644 index 000000000..dd4e3a0ff --- /dev/null +++ b/docs/integrations/frameworks/packer.mdx @@ -0,0 +1,95 @@ +--- +title: "Packer" +description: "Learn how to fetch secrets from Infisical with Packer using a data source" +--- + +This guide demonstrates how to use the Infisical Packer plugin to fetch secret data using a data source. The Packer plugin supports both [Infisical Cloud](https://app.infisical.com) and [self-hosted instances of Infisical](https://infisical.com/docs/self-hosting/overview). + +## Prerequisites + +Before you begin, make sure you have: + +- [Packer](https://developer.hashicorp.com/packer/install) installed +- An Infisical account with access to a project +- Basic understanding of Packer + +## Project Setup + +### Configure Provider + +First, specify the Infisical provider in your Packer configuration: + +```hcl +packer { + required_plugins { + infisical = { + source = "github.com/infisical/infisical" + version = ">=0.0.1" + } + } +} +``` + +### Authentication + +Using a Machine Identity, you can authenticate with [Universal Auth](https://infisical.com/docs/documentation/platform/identities/universal-auth). + +```hcl +data "infisical-secrets" "dev-secrets" { + folder_path = "/" + env_slug = "dev" # The environment to list secrets from (e.g. dev, staging, prod) + project_id = "00000000-0000-0000-0000-000000000000" + host = "https://app.infisical.com" # Optional for cloud, required for self-hosted + + universal_auth { + client_id = "00000000-0000-0000-0000-000000000000" + client_secret = "..." # Optional if using INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET env variable + } +} +``` + +Learn more about [machine identities](/documentation/platform/identities/machine-identities). + +## Using Secrets in Packer + +You're able to fetch secrets from Infisical using the `infisical-secrets` Data Source: + +```hcl +# Fetch all secrets from a folder +data "infisical-secrets" "dev-secrets" { + folder_path = "/" + env_slug = "dev" + project_id = "00000000-0000-0000-0000-000000000000" + + universal_auth { + ... + } +} + +locals { + secrets = data.infisical-secrets.dev-secrets.secrets +} + +source "null" "basic-example" { + communicator = "none" +} + +build { + sources = [ + "source.null.basic-example" + ] + + provisioner "shell-local" { + inline = [ + "echo secret_key: ${local.secrets["SECRET_KEY"].secret_value}", + ] + } +} +``` + +The `local.secrets` object maps secret keys to [secret objects](https://github.com/Infisical/packer-plugin-infisical/blob/main/docs/datasources/secrets.md#secret-object). + +See also: +- [Packer Plugin Repository Example](https://github.com/Infisical/packer-plugin-infisical/blob/main/example/build.pkr.hcl) +- [Packer Plugin Repository Docs](https://github.com/Infisical/packer-plugin-infisical/tree/main/docs) +- [Machine Identity setup guide](/documentation/platform/identities/machine-identities) diff --git a/docs/internals/permissions/project-permissions.mdx b/docs/internals/permissions/project-permissions.mdx index 98f3bfeb2..da9351188 100644 --- a/docs/internals/permissions/project-permissions.mdx +++ b/docs/internals/permissions/project-permissions.mdx @@ -176,6 +176,13 @@ Supports conditions and permission inversion | `read` | View secret versions and snapshots | | `create` | Roll back secrets to snapshots | +#### Subject: `commits` + +| Action | Description | +| -------- | ---------------------------------- | +| `read` | View commits and changes across folders | +| `perform-rollback` | Roll back commits changes and restore folders to previous state| + #### Subject: `secret-approval` | Action | Description | diff --git a/docs/mint.json b/docs/mint.json index 92bc2cc29..e2c0e58bc 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -450,6 +450,8 @@ { "group": "Infrastructure Integrations", "pages": [ + "integrations/platforms/ansible", + "integrations/platforms/apache-airflow", { "group": "Container orchestrators", "pages": [ @@ -468,7 +470,6 @@ "integrations/platforms/ecs-with-agent" ] }, - "integrations/platforms/infisical-agent", { "group": "Docker", "pages": [ @@ -478,10 +479,10 @@ "integrations/platforms/docker-compose" ] }, - "integrations/frameworks/terraform", + "integrations/platforms/infisical-agent", + "integrations/frameworks/packer", "integrations/frameworks/pulumi", - "integrations/platforms/ansible", - "integrations/platforms/apache-airflow" + "integrations/frameworks/terraform" ] }, { @@ -828,8 +829,7 @@ "api-reference/endpoints/workspaces/delete-workspace", "api-reference/endpoints/workspaces/get-workspace", "api-reference/endpoints/workspaces/update-workspace", - "api-reference/endpoints/workspaces/secret-snapshots", - "api-reference/endpoints/workspaces/rollback-snapshot" + "api-reference/endpoints/workspaces/secret-snapshots" ] }, { @@ -929,6 +929,12 @@ { "group": "Dynamic Secrets", "pages": [ + { + "group": "Kubernetes", + "pages": [ + "api-reference/endpoints/dynamic-secrets/kubernetes/create-lease" + ] + }, "api-reference/endpoints/dynamic-secrets/create", "api-reference/endpoints/dynamic-secrets/update", "api-reference/endpoints/dynamic-secrets/delete", diff --git a/docs/self-hosting/deployment-options/kubernetes-helm.mdx b/docs/self-hosting/deployment-options/kubernetes-helm.mdx index e0e5ba75d..f8a2a6864 100644 --- a/docs/self-hosting/deployment-options/kubernetes-helm.mdx +++ b/docs/self-hosting/deployment-options/kubernetes-helm.mdx @@ -29,7 +29,7 @@ description: "Learn how to use Helm chart to install Infisical on your Kubernete infisical: image: repository: infisical/infisical - tag: "v0.46.2-postgres" #<-- update + tag: "<>" #<-- select tag from Dockerhub from the above link pullPolicy: IfNotPresent ``` diff --git a/frontend/src/components/navigation/SecretDashboardPathBreadcrumb.tsx b/frontend/src/components/navigation/SecretDashboardPathBreadcrumb.tsx index 584a3f6c7..31a626866 100644 --- a/frontend/src/components/navigation/SecretDashboardPathBreadcrumb.tsx +++ b/frontend/src/components/navigation/SecretDashboardPathBreadcrumb.tsx @@ -14,13 +14,15 @@ type Props = { selectedPathSegmentIndex: number; environmentSlug: string; projectId: string; + disableCopy?: boolean; }; export const SecretDashboardPathBreadcrumb = ({ secretPathSegments, selectedPathSegmentIndex, environmentSlug, - projectId + projectId, + disableCopy }: Props) => { const [, isCopying, setIsCopying] = useTimedReset({ initialState: false @@ -32,7 +34,7 @@ export const SecretDashboardPathBreadcrumb = ({ return (
- {isLastItem ? ( + {isLastItem && !disableCopy ? (
{ TCreateDynamicSecretLeaseDTO >({ mutationFn: async (dto) => { + if (dto.provider === DynamicSecretProviders.Kubernetes) { + const { data } = await apiRequest.post<{ lease: TDynamicSecretLease; data: unknown }>( + "/api/v1/dynamic-secrets/leases/kubernetes", + dto + ); + return data; + } + const { data } = await apiRequest.post<{ lease: TDynamicSecretLease; data: unknown }>( "/api/v1/dynamic-secrets/leases", dto diff --git a/frontend/src/hooks/api/dynamicSecretLease/types.ts b/frontend/src/hooks/api/dynamicSecretLease/types.ts index 76bedc8b3..51ee64c4b 100644 --- a/frontend/src/hooks/api/dynamicSecretLease/types.ts +++ b/frontend/src/hooks/api/dynamicSecretLease/types.ts @@ -1,3 +1,5 @@ +import { DynamicSecretProviders } from "../dynamicSecret/types"; + export enum DynamicSecretLeaseStatus { FailedDeletion = "Failed to delete" } @@ -13,12 +15,20 @@ export type TDynamicSecretLease = { updatedAt: string; }; +export type TDynamicSecretKubernetesLeaseConfig = { + namespace?: string; +}; + +export type TDynamicSecretLeaseConfig = TDynamicSecretKubernetesLeaseConfig; + export type TCreateDynamicSecretLeaseDTO = { dynamicSecretName: string; projectSlug: string; ttl?: string; path: string; environmentSlug: string; + config?: TDynamicSecretLeaseConfig; + provider: DynamicSecretProviders; }; export type TRenewDynamicSecretLeaseDTO = { diff --git a/frontend/src/hooks/api/folderCommits/index.tsx b/frontend/src/hooks/api/folderCommits/index.tsx new file mode 100644 index 000000000..ffdec812a --- /dev/null +++ b/frontend/src/hooks/api/folderCommits/index.tsx @@ -0,0 +1,2 @@ +export { useGetFolderCommitHistory, useGetFolderCommitsCount } from "./queries"; +export * from "./types"; diff --git a/frontend/src/hooks/api/folderCommits/queries.tsx b/frontend/src/hooks/api/folderCommits/queries.tsx new file mode 100644 index 000000000..8d0883c5b --- /dev/null +++ b/frontend/src/hooks/api/folderCommits/queries.tsx @@ -0,0 +1,294 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; + +import { apiRequest } from "@app/config/request"; + +import { CommitHistoryItem, CommitWithChanges, RollbackPreview } from "./types"; + +export const commitKeys = { + count: ({ + workspaceId, + environment, + directory + }: { + workspaceId: string; + environment: string; + directory?: string; + }) => [{ workspaceId, environment, directory }, "folder-commits-count"] as const, + + history: ({ + workspaceId, + environment, + directory + }: { + workspaceId: string; + environment: string; + directory?: string; + }) => [{ workspaceId, environment, directory }, "folder-commits"] as const, + + details: ({ workspaceId, commitId }: { workspaceId: string; commitId: string }) => + [{ workspaceId, commitId }, "commit-details"] as const, + + rollbackPreview: ({ + folderId, + commitId, + envSlug, + projectId, + deepRollback + }: { + folderId: string; + commitId: string; + envSlug: string; + projectId: string; + deepRollback: boolean; + }) => [{ folderId, commitId, envSlug, projectId, deepRollback }, "rollback-preview"] as const +}; + +const fetchFolderCommitsCount = async ({ + workspaceId, + environment, + directory +}: { + workspaceId: string; + environment: string; + directory?: string; +}) => { + const res = await apiRequest.get<{ count: number; folderId: string }>( + "/api/v1/pit/commits/count", + { + params: { + environment, + path: directory, + projectId: workspaceId + } + } + ); + return res.data; +}; + +const fetchFolderCommitHistory = async ( + workspaceId: string, + environment: string, + directory: string, + offset: number = 0, + limit: number = 20, + search?: string, + sort: "asc" | "desc" = "desc" +): Promise<{ + commits: CommitHistoryItem[]; + total: number; + hasMore: boolean; +}> => { + const res = await apiRequest.get<{ + commits: CommitHistoryItem[]; + total: number; + hasMore: boolean; + }>("/api/v1/pit/commits", { + params: { + environment, + path: directory, + projectId: workspaceId, + offset, + limit, + search, + sort + } + }); + return res.data; +}; + +export const fetchCommitDetails = async (workspaceId: string, commitId: string) => { + const { data } = await apiRequest.get( + `/api/v1/pit/commits/${commitId}/changes`, + { + params: { + projectId: workspaceId + } + } + ); + return data; +}; + +export const fetchRollbackPreview = async ( + folderId: string, + commitId: string, + envSlug: string, + workspaceId: string, + deepRollback: boolean, + secretPath: string +): Promise => { + const { data } = await apiRequest.get( + `/api/v1/pit/commits/${commitId}/compare`, + { + params: { + folderId, + environment: envSlug, + deepRollback, + secretPath, + projectId: workspaceId + } + } + ); + return data; +}; + +const fetchRollback = async ( + folderId: string, + commitId: string, + workspaceId: string, + deepRollback: boolean, + message?: string, + envSlug?: string +) => { + const { data } = await apiRequest.post<{ success: boolean }>( + `/api/v1/pit/commits/${commitId}/rollback`, + { + folderId, + deepRollback, + message, + environment: envSlug, + projectId: workspaceId + } + ); + return data; +}; + +const fetchRevert = async (commitId: string, workspaceId: string) => { + const { data } = await apiRequest.post<{ success: boolean; message: string }>( + `/api/v1/pit/commits/${commitId}/revert`, + { + projectId: workspaceId + } + ); + return data; +}; + +export const useCommitRevert = ({ + commitId, + projectId, + environment, + directory +}: { + commitId: string; + projectId: string; + environment: string; + directory: string; +}) => { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: () => fetchRevert(commitId, projectId), + onSuccess: () => { + queryClient.invalidateQueries({ + queryKey: [ + commitKeys.details({ workspaceId: projectId, commitId }), + commitKeys.history({ workspaceId: projectId, environment, directory }), + commitKeys.count({ workspaceId: projectId, environment, directory }) + ] + }); + } + }); +}; + +export const useCommitRollback = ({ + workspaceId, + commitId, + folderId, + deepRollback, + environment, + directory, + envSlug +}: { + workspaceId: string; + commitId: string; + folderId: string; + deepRollback: boolean; + environment: string; + directory: string; + envSlug: string; +}) => { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: (message: string) => + fetchRollback(folderId, commitId, workspaceId, deepRollback, message, envSlug), + onSuccess: () => { + queryClient.invalidateQueries({ + queryKey: [ + commitKeys.details({ workspaceId, commitId }), + commitKeys.history({ workspaceId, environment, directory }), + commitKeys.count({ workspaceId, environment, directory }) + ] + }); + } + }); +}; + +export const useGetFolderCommitsCount = ({ + workspaceId, + environment, + directory, + isPaused +}: { + workspaceId: string; + environment: string; + directory: string; + isPaused?: boolean; +}) => + useQuery({ + enabled: Boolean(workspaceId && environment) && !isPaused, + queryKey: commitKeys.count({ workspaceId, environment, directory }), + queryFn: () => fetchFolderCommitsCount({ workspaceId, environment, directory }) + }); + +export const useGetFolderCommitHistory = ({ + workspaceId, + environment, + directory, + offset = 0, + limit = 20, + search, + sort = "desc" +}: { + workspaceId: string; + environment: string; + directory: string; + offset?: number; + limit?: number; + search?: string; + sort?: "asc" | "desc"; +}) => { + return useQuery({ + queryKey: [ + commitKeys.history({ workspaceId, environment, directory }), + offset, + limit, + search, + sort + ], + queryFn: () => + fetchFolderCommitHistory(workspaceId, environment, directory, offset, limit, search, sort), + enabled: Boolean(workspaceId && environment) + }); +}; + +export const useGetCommitDetails = (workspaceId: string, commitId: string) => { + return useQuery({ + queryKey: commitKeys.details({ workspaceId, commitId }), + queryFn: () => fetchCommitDetails(workspaceId, commitId), + enabled: Boolean(workspaceId) && Boolean(commitId) + }); +}; + +export const useGetRollbackPreview = ( + folderId: string, + commitId: string, + envSlug: string, + projectId: string, + deepRollback: boolean, + secretPath: string +) => { + return useQuery({ + queryKey: commitKeys.rollbackPreview({ folderId, commitId, envSlug, projectId, deepRollback }), + queryFn: () => + fetchRollbackPreview(folderId, commitId, envSlug, projectId, deepRollback, secretPath), + enabled: Boolean(folderId) && Boolean(commitId) + }); +}; diff --git a/frontend/src/hooks/api/folderCommits/types.ts b/frontend/src/hooks/api/folderCommits/types.ts new file mode 100644 index 000000000..878e3224d --- /dev/null +++ b/frontend/src/hooks/api/folderCommits/types.ts @@ -0,0 +1,64 @@ +import { CommitType, SecretVersions } from "../types"; + +export type CommitHistoryItem = { + id: string; + commitId: string; + actorMetadata: { + id: string; + name?: string; + }; + actorType: string; + message: string; + folderId: string; + envId: string; + createdAt: string; + updatedAt: string; + isLatest: boolean; +}; + +export type TFolderCommitChanges = { + id: string; + folderCommitId: string; + changeType: CommitType; + isUpdate: boolean; + secretVersionId: string | null; + folderVersionId: string | null; + createdAt: string; + updatedAt: string; + versions: SecretVersions[]; + secretKey?: string; + folderName?: string; + secretVersion?: string; + folderVersion?: string; +}; + +export type FolderReconstructedItem = { + type: string; + id: string; + versionId: string; + folderName?: string; + folderVersion?: number; + secretKey?: string; + secretVersion?: number; +}; + +export type CommitWithChanges = { + changes: CommitHistoryItem & { + changes: TFolderCommitChanges[]; + }; +}; + +export type RollbackChange = { + type: "folder" | "secret"; + id: string; + versionId: string; + changeType: "create" | "update" | "delete"; + commitId: string; +}; + +export type RollbackPreview = { + folderId: string; + folderName: string; + folderPath: string; + changes: RollbackChange[]; +}; diff --git a/frontend/src/hooks/api/identities/types.ts b/frontend/src/hooks/api/identities/types.ts index f5e0f1065..6b00eb40c 100644 --- a/frontend/src/hooks/api/identities/types.ts +++ b/frontend/src/hooks/api/identities/types.ts @@ -403,7 +403,7 @@ export type IdentityKubernetesAuth = { export type AddIdentityKubernetesAuthDTO = { organizationId: string; identityId: string; - kubernetesHost: string; + kubernetesHost: string | null; tokenReviewerJwt?: string; tokenReviewMode: IdentityKubernetesAuthTokenReviewMode; allowedNamespaces: string; @@ -422,7 +422,7 @@ export type AddIdentityKubernetesAuthDTO = { export type UpdateIdentityKubernetesAuthDTO = { organizationId: string; identityId: string; - kubernetesHost?: string; + kubernetesHost?: string | null; tokenReviewerJwt?: string | null; tokenReviewMode?: IdentityKubernetesAuthTokenReviewMode; allowedNamespaces?: string; diff --git a/frontend/src/hooks/api/secretApprovalRequest/types.ts b/frontend/src/hooks/api/secretApprovalRequest/types.ts index 3ac2574b5..241d3c1e2 100644 --- a/frontend/src/hooks/api/secretApprovalRequest/types.ts +++ b/frontend/src/hooks/api/secretApprovalRequest/types.ts @@ -11,7 +11,8 @@ export enum ApprovalStatus { export enum CommitType { DELETE = "delete", UPDATE = "update", - CREATE = "create" + CREATE = "create", + ADD = "add" } export type TSecretApprovalSecChangeData = { diff --git a/frontend/src/hooks/api/secretFolders/queries.tsx b/frontend/src/hooks/api/secretFolders/queries.tsx index b49fbd6cc..9e12f329d 100644 --- a/frontend/src/hooks/api/secretFolders/queries.tsx +++ b/frontend/src/hooks/api/secretFolders/queries.tsx @@ -10,6 +10,7 @@ import { import { apiRequest } from "@app/config/request"; import { dashboardKeys } from "@app/hooks/api/dashboard/queries"; +import { commitKeys } from "../folderCommits/queries"; import { secretSnapshotKeys } from "../secretSnapshots/queries"; import { TCreateFolderDTO, @@ -166,6 +167,9 @@ export const useCreateFolder = () => { queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ workspaceId: projectId, environment, directory: path }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId: projectId, environment, directory: path }) + }); } }); }; @@ -200,6 +204,12 @@ export const useUpdateFolder = () => { queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ workspaceId: projectId, environment, directory: path }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId: projectId, environment, directory: path }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId: projectId, environment, directory: path }) + }); } }); }; @@ -234,6 +244,12 @@ export const useDeleteFolder = () => { queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ workspaceId: projectId, environment, directory: path }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId: projectId, environment, directory: path }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId: projectId, environment, directory: path }) + }); } }); }; @@ -279,6 +295,20 @@ export const useUpdateFolderBatch = () => { directory: folder.path }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ + workspaceId: projectId, + environment: folder.environment, + directory: folder.path + }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ + workspaceId: projectId, + environment: folder.environment, + directory: folder.path + }) + }); }); } }); diff --git a/frontend/src/hooks/api/secrets/mutations.tsx b/frontend/src/hooks/api/secrets/mutations.tsx index 3862d1f8d..82bf623e6 100644 --- a/frontend/src/hooks/api/secrets/mutations.tsx +++ b/frontend/src/hooks/api/secrets/mutations.tsx @@ -3,6 +3,7 @@ import { MutationOptions, useMutation, useQueryClient } from "@tanstack/react-qu import { apiRequest } from "@app/config/request"; import { dashboardKeys } from "@app/hooks/api/dashboard/queries"; +import { commitKeys } from "../folderCommits/queries"; import { secretApprovalRequestKeys } from "../secretApprovalRequest/queries"; import { secretSnapshotKeys } from "../secretSnapshots/queries"; import { secretKeys } from "./queries"; @@ -59,6 +60,12 @@ export const useCreateSecretV3 = ({ queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ environment, workspaceId, directory: secretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId, environment, directory: secretPath }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId, environment, directory: secretPath }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId }) }); }, ...options @@ -118,6 +125,12 @@ export const useUpdateSecretV3 = ({ queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ environment, workspaceId, directory: secretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId, environment, directory: secretPath }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId, environment, directory: secretPath }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId }) }); }, ...options @@ -164,6 +177,12 @@ export const useDeleteSecretV3 = ({ queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ environment, workspaceId, directory: secretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId, environment, directory: secretPath }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId, environment, directory: secretPath }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId }) }); }, ...options @@ -200,6 +219,12 @@ export const useCreateSecretBatch = ({ queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ environment, workspaceId, directory: secretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId, environment, directory: secretPath }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId, environment, directory: secretPath }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId }) }); }, ...options @@ -236,6 +261,12 @@ export const useUpdateSecretBatch = ({ queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ environment, workspaceId, directory: secretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId, environment, directory: secretPath }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId, environment, directory: secretPath }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId }) }); }, ...options @@ -274,6 +305,12 @@ export const useDeleteSecretBatch = ({ queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ environment, workspaceId, directory: secretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId, environment, directory: secretPath }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId, environment, directory: secretPath }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId }) }); }, ...options @@ -347,6 +384,20 @@ export const useMoveSecrets = ({ directory: sourceSecretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ + workspaceId: projectId, + environment: sourceEnvironment, + directory: sourceSecretPath + }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ + workspaceId: projectId, + environment: sourceEnvironment, + directory: sourceSecretPath + }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId: projectId }) }); diff --git a/frontend/src/hooks/api/workspace/queries.tsx b/frontend/src/hooks/api/workspace/queries.tsx index 2ae2bc463..fa9240101 100644 --- a/frontend/src/hooks/api/workspace/queries.tsx +++ b/frontend/src/hooks/api/workspace/queries.tsx @@ -282,7 +282,8 @@ export const useUpdateProject = () => { newProjectName, newProjectDescription, newSlug, - secretSharing + secretSharing, + showSnapshotsLegacy }) => { const { data } = await apiRequest.patch<{ workspace: Workspace }>( `/api/v1/workspace/${projectID}`, @@ -290,7 +291,8 @@ export const useUpdateProject = () => { name: newProjectName, description: newProjectDescription, slug: newSlug, - secretSharing + secretSharing, + showSnapshotsLegacy } ); return data.workspace; diff --git a/frontend/src/hooks/api/workspace/types.ts b/frontend/src/hooks/api/workspace/types.ts index 481bdcc08..fbaea7742 100644 --- a/frontend/src/hooks/api/workspace/types.ts +++ b/frontend/src/hooks/api/workspace/types.ts @@ -39,6 +39,7 @@ export type Workspace = { roles?: TProjectRole[]; hasDeleteProtection: boolean; secretSharing: boolean; + showSnapshotsLegacy: boolean; }; export type WorkspaceEnv = { @@ -79,6 +80,7 @@ export type UpdateProjectDTO = { newProjectDescription?: string; newSlug?: string; secretSharing?: boolean; + showSnapshotsLegacy?: boolean; }; export type UpdatePitVersionLimitDTO = { projectSlug: string; pitVersionLimit: number }; diff --git a/frontend/src/pages/organization/AccessManagementPage/components/OrgIdentityTab/components/IdentitySection/IdentityKubernetesAuthForm.tsx b/frontend/src/pages/organization/AccessManagementPage/components/OrgIdentityTab/components/IdentitySection/IdentityKubernetesAuthForm.tsx index e43c94a7e..708b44a35 100644 --- a/frontend/src/pages/organization/AccessManagementPage/components/OrgIdentityTab/components/IdentitySection/IdentityKubernetesAuthForm.tsx +++ b/frontend/src/pages/organization/AccessManagementPage/components/OrgIdentityTab/components/IdentitySection/IdentityKubernetesAuthForm.tsx @@ -46,13 +46,13 @@ const schema = z tokenReviewMode: z .nativeEnum(IdentityKubernetesAuthTokenReviewMode) .default(IdentityKubernetesAuthTokenReviewMode.Api), - kubernetesHost: z.string().min(1), + kubernetesHost: z.string().optional().nullable(), tokenReviewerJwt: z.string().optional(), gatewayId: z.string().optional().nullable(), allowedNames: z.string(), allowedNamespaces: z.string(), allowedAudience: z.string(), - caCert: z.string(), + caCert: z.string().optional(), accessTokenTTL: z.string().refine((val) => Number(val) <= 315360000, { message: "Access Token TTL cannot be greater than 315360000" }), @@ -69,6 +69,17 @@ const schema = z .min(1) }) .superRefine((data, ctx) => { + if ( + data.tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Api && + !data.kubernetesHost?.length + ) { + ctx.addIssue({ + path: ["kubernetesHost"], + code: z.ZodIssueCode.custom, + message: "When token review mode is set to API, a Kubernetes host must be provided" + }); + } + if (data.tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Gateway && !data.gatewayId) { ctx.addIssue({ path: ["gatewayId"], @@ -201,7 +212,13 @@ export const IdentityKubernetesAuthForm = ({ if (data) { await updateMutateAsync({ organizationId: orgId, - kubernetesHost, + ...(tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Api + ? { + kubernetesHost: kubernetesHost || "" + } + : { + kubernetesHost: null + }), tokenReviewerJwt: tokenReviewerJwt || null, allowedNames, allowedNamespaces, @@ -219,7 +236,13 @@ export const IdentityKubernetesAuthForm = ({ await addMutateAsync({ organizationId: orgId, identityId, - kubernetesHost: kubernetesHost || "", + ...(tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Api + ? { + kubernetesHost: kubernetesHost || "" + } + : { + kubernetesHost: null + }), tokenReviewerJwt: tokenReviewerJwt || undefined, allowedNames: allowedNames || "", allowedNamespaces: allowedNamespaces || "", @@ -278,23 +301,6 @@ export const IdentityKubernetesAuthForm = ({ Advanced - ( - - - - )} - /> -
+ {tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Api && ( + ( + + + + )} + /> + )} - {tokenReviewMode === "api" && ( + {tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Api && ( )} /> - ( - -